Experiments

.changeset/bright-experiments-flags.md

@@ -0,0 +1,9 @@
+---
+'flags': patch
+---
+
+Add the `experimental_reportOverride` adapter hook for observing values set by
+the Flags SDK override cookie.
+
+This API is not supported for general use yet. Do not use it unless Vercel has
+explicitly enabled it for you.

.changeset/bright-experiments-report.md

@@ -0,0 +1,19 @@
+---
+'@vercel/flags-core': patch
+---
+
+Add APIs for reporting flag exposures and override values.
+
+- The `experimental_reportExposures` client option for supplying an exposure
+  reporter.
+- The `experimental_reportOverride` client method for reporting values set by
+  the Flags SDK override cookie.
+- The `experimental_exposureLogging` option on `evaluate()` and
+  `bulkEvaluate()` for disabling exposure reporting for an individual call.
+- Experiment assignment metadata on `EvaluationResult.experiment`.
+- The `experimental_EvaluationOptions`,
+  `experimental_ExperimentAssignment`, `experimental_Exposure`, and
+  `experimental_ReportExposures` types.
+
+These APIs are not supported for general use yet. Do not use them unless
+Vercel has explicitly enabled them for you.

.changeset/bright-experiments-vercel-adapter.md

@@ -0,0 +1,9 @@
+---
+'@flags-sdk/vercel': patch
+---
+
+Implement the `experimental_reportOverride` adapter hook to forward override
+values to the underlying Vercel Flags client.
+
+This API is not supported for general use yet. Do not use it unless Vercel has
+explicitly enabled it for you.

packages/adapter-vercel/src/index.test.ts

@@ -1,4 +1,8 @@
-import { flagsClient, resetDefaultFlagsClient } from '@vercel/flags-core';
+import {
+  type FlagsClient,
+  flagsClient,
+  resetDefaultFlagsClient,
+} from '@vercel/flags-core';
 import type { Adapter, Origin, ProviderData } from 'flags';
 import { flag } from 'flags/next';
 import { HttpResponse, http } from 'msw';
@@ -104,6 +108,37 @@ describe('createVercelAdapter', () => {
     } satisfies Origin);
   });
 
+  it('forwards override observations to the flags client', async () => {
+    const reportOverride = vi.fn();
+    const fakeClient = {
+      origin: { provider: 'vercel', sdkKey: 'vf_x' },
+      experimental_reportOverride: reportOverride,
+    } as unknown as typeof flagsClient;
+    const adapter = createVercelAdapter(fakeClient)();
+    const entities = { user: { key: 'user_1' } };
+
+    await adapter.experimental_reportOverride?.({
+      key: 'checkout',
+      value: 'treatment',
+      entities,
+    });
+
+    expect(reportOverride).toHaveBeenCalledWith({
+      key: 'checkout',
+      value: 'treatment',
+      entities,
+    });
+  });
+
+  it('does not expose override reporting when the flags client does not support it', () => {
+    const fakeClient: FlagsClient = { ...flagsClient };
+    delete fakeClient.experimental_reportOverride;
+
+    const adapter = createVercelAdapter(fakeClient)();
+
+    expect(adapter.experimental_reportOverride).toBeUndefined();
+  });
+
   it('has correct types', () => {
     const adapter = createVercelAdapter(flagsClient);
     type SampleValue = boolean;

packages/adapter-vercel/src/index.ts

@@ -40,6 +40,7 @@ export function createVercelAdapter(
     adapterId,
     origin: flagsClient.origin,
     config: { reportValue: false },
+    experimental_reportOverride: flagsClient.experimental_reportOverride,
     async decide({ key, entities }) {
       const evaluationResult = await flagsClient.evaluate<unknown, unknown>(
         key,

packages/flags/src/index.test.ts

@@ -27,7 +27,7 @@ describe('exports', () => {
 
   it('exports version', () => {
     expect(version).toBeTypeOf('string');
-    expect(version).toMatch(/^\d+\.\d+\.\d+(-\w+-\d+)?$/);
+    expect(version).toMatch(/^\d+\.\d+\.\d+(-[\w.-]+)?$/);
   });
 });
 

packages/flags/src/next/evaluate.ts

@@ -40,6 +40,27 @@ const evaluationCache = new WeakMap<
   Map</* flagKey */ string, Map</* entitiesKey */ string, any>>
 >();
 
+const adapterInitializationCache = new WeakMap<object, Promise<void>>();
+
+async function ensureAdapterInitialized(
+  adapter: Pick<Adapter<unknown, unknown>, 'initialize'>,
+): Promise<void> {
+  if (!adapter.initialize) return;
+
+  let initialization = adapterInitializationCache.get(adapter);
+  if (!initialization) {
+    initialization = adapter.initialize();
+    adapterInitializationCache.set(adapter, initialization);
+  }
+
+  try {
+    await initialization;
+  } catch (error) {
+    adapterInitializationCache.delete(adapter);
+    throw error;
+  }
+}
+
 function getCachedValuePromise(
   /**
    * supports Headers for App Router and IncomingHttpHeaders for Pages Router
@@ -197,7 +218,10 @@ type FlagInfo<ValueType> = {
   key: string;
   defaultValue?: ValueType;
   config?: { reportValue?: boolean };
-  adapter?: { config?: { reportValue?: boolean } };
+  adapter?: Pick<
+    Adapter<ValueType, any>,
+    'config' | 'initialize' | 'experimental_reportOverride'
+  >;
 };
 
 function hasOverride(
@@ -227,10 +251,18 @@ async function applyResult<ValueType>(args: {
   definition: FlagInfo<ValueType>;
   readonlyHeaders: ReadonlyHeaders;
   entitiesKey: string;
+  entities?: unknown;
   overrides: Record<string, any> | null;
   produce: () => ValueType | PromiseLike<ValueType>;
 }): Promise<ValueType> {
-  const { definition, readonlyHeaders, entitiesKey, overrides, produce } = args;
+  const {
+    definition,
+    readonlyHeaders,
+    entitiesKey,
+    entities,
+    overrides,
+    produce,
+  } = args;
 
   const cachedValue = getCachedValuePromise(
     readonlyHeaders,
@@ -254,6 +286,19 @@ async function applyResult<ValueType>(args: {
     internalReportValue(definition.key, decision, {
       reason: 'override',
     });
+    try {
+      const adapter = definition.adapter;
+      if (adapter?.experimental_reportOverride) {
+        await ensureAdapterInitialized(adapter);
+        await adapter.experimental_reportOverride({
+          key: definition.key,
+          value: decision,
+          entities,
+        });
+      }
+    } catch (error) {
+      console.error('flags: Failed to report flag override', error);
+    }
     return decision;
   }
 
@@ -401,6 +446,7 @@ export function getRun<ValueType, EntitiesType>(
       definition,
       readonlyHeaders,
       entitiesKey,
+      entities,
       overrides,
       produce: () =>
         decide({
@@ -641,6 +687,7 @@ async function evaluateImpl(
                     definition: flagFn,
                     readonlyHeaders,
                     entitiesKey,
+                    entities,
                     overrides,
                     produce: () => {
                       if (bulkError) throw bulkError;

packages/flags/src/next/index.test.ts

@@ -191,7 +191,23 @@ describe('flag on app router', () => {
 
   it('respects overrides', async () => {
     const decide = vi.fn(() => false);
-    const f = flag<boolean>({ key: 'first-flag', decide });
+    const calls: string[] = [];
+    const initialize = vi.fn(async () => {
+      calls.push('initialize');
+    });
+    const reportOverride = vi.fn(async () => {
+      calls.push('reportOverride');
+    });
+    const entities = { user: { id: 'user_1' } };
+    const f = flag<boolean, typeof entities>({
+      key: 'first-flag',
+      identify: () => entities,
+      adapter: {
+        decide,
+        initialize,
+        experimental_reportOverride: reportOverride,
+      },
+    });
 
     // first request using the flag twice
     const headersOfFirstRequest = new Headers();
@@ -207,6 +223,13 @@ describe('flag on app router', () => {
     await expect(f()).resolves.toEqual(true);
     expect(cookieMock).toHaveBeenCalledWith('vercel-flag-overrides');
     expect(decide).not.toHaveBeenCalled();
+    expect(initialize).toHaveBeenCalledOnce();
+    expect(reportOverride).toHaveBeenCalledWith({
+      key: 'first-flag',
+      value: true,
+      entities,
+    });
+    expect(calls).toEqual(['initialize', 'reportOverride']);
   });
 
   it('does not crash when override reporting hook is not a function', async () => {
@@ -879,6 +902,10 @@ describe('evaluate', () => {
     bulkDecide?: Adapter<V, any>['bulkDecide'];
     decide?: Adapter<V, any>['decide'];
     identify?: Adapter<V, any>['identify'];
+    experimental_reportOverride?: Adapter<
+      V,
+      any
+    >['experimental_reportOverride'];
     omitAdapterId?: boolean;
     omitBulkDecide?: boolean;
   }) {
@@ -892,6 +919,7 @@ describe('evaluate', () => {
           throw new Error('decide should not be called in bulk path');
         }),
       identify: opts?.identify,
+      experimental_reportOverride: opts?.experimental_reportOverride,
       ...(opts?.omitBulkDecide ? {} : { bulkDecide: opts?.bulkDecide }),
     });
   }
@@ -1084,7 +1112,13 @@ describe('evaluate', () => {
 
   it('lets overrides win over bulkDecide results', async () => {
     const bulkDecideMock = vi.fn().mockResolvedValue({ a: 'bulk-value' });
-    const adapter = makeBulkAdapter<boolean>({ bulkDecide: bulkDecideMock });
+    const reportOverride = vi.fn();
+    const entities = { user: { id: 'user_1' } };
+    const adapter = makeBulkAdapter<boolean>({
+      bulkDecide: bulkDecideMock,
+      identify: () => entities,
+      experimental_reportOverride: reportOverride,
+    });
 
     const a = flag<boolean>({ key: 'a', adapter: adapter() });
 
@@ -1099,6 +1133,11 @@ describe('evaluate', () => {
 
     await expect(evaluate({ a })).resolves.toEqual({ a: true });
     expect(bulkDecideMock).not.toHaveBeenCalled();
+    expect(reportOverride).toHaveBeenCalledWith({
+      key: 'a',
+      value: true,
+      entities,
+    });
   });
 
   it('omits overridden flags from bulkDecide input', async () => {

packages/flags/src/types.ts

@@ -165,6 +165,17 @@ export interface Adapter<ValueType, EntitiesType> {
    * an `adapterId` are never batched.
    */
   adapterId?: string | symbol;
+  /**
+   * Observe a value supplied by the Flags SDK override cookie.
+   *
+   * @remarks This API is not supported for general use yet. Do not use it
+   * unless Vercel has explicitly enabled it for you.
+   */
+  experimental_reportOverride?: (params: {
+    key: string;
+    value: unknown;
+    entities?: EntitiesType;
+  }) => void | Promise<void>;
   decide: (params: {
     key: string;
     entities?: EntitiesType;

packages/vercel-flags-core/README.md

@@ -24,6 +24,8 @@ const result = await client.evaluate<boolean>('show-new-feature', false, {
 });
 ```
 
+## Evaluation Metrics
+
 To associate evaluation metrics with an environment, pass the
 `metricEnvironment` option:
 

packages/vercel-flags-core/package.json

@@ -75,6 +75,7 @@
   "dependencies": {
     "@vercel/functions": "^3.4.3",
     "@vercel/oidc": "3.5.0",
+    "dequal": "2.0.3",
     "jose": "5.2.1",
     "js-xxhash": "4.0.0"
   },