Experiments

.changeset/bright-experiments-report.md

@@ -0,0 +1,7 @@
+---
+'@vercel/flags-core': minor
+'@flags-sdk/vercel': minor
+'flags': minor
+---
+
+Add randomized experiment enrollment, assignment reasons for every experiment-managed flag outcome, readiness-aware cookie override exposure reporting, and per-evaluation exposure logging controls.

packages/adapter-vercel/src/index.test.ts

@@ -104,6 +104,28 @@ describe('createVercelAdapter', () => {
     } satisfies Origin);
   });
 
+  it('forwards override observations to the flags client', async () => {
+    const reportOverride = vi.fn();
+    const fakeClient = {
+      origin: { provider: 'vercel', sdkKey: 'vf_x' },
+      reportOverride,
+    } as unknown as typeof flagsClient;
+    const adapter = createVercelAdapter(fakeClient)();
+    const entities = { user: { key: 'user_1' } };
+
+    await adapter.reportOverride?.({
+      key: 'checkout',
+      value: 'treatment',
+      entities,
+    });
+
+    expect(reportOverride).toHaveBeenCalledWith(
+      'checkout',
+      'treatment',
+      entities,
+    );
+  });
+
   it('has correct types', () => {
     const adapter = createVercelAdapter(flagsClient);
     type SampleValue = boolean;

packages/adapter-vercel/src/index.ts

@@ -40,6 +40,9 @@ export function createVercelAdapter(
     adapterId,
     origin: flagsClient.origin,
     config: { reportValue: false },
+    async reportOverride({ key, value, entities }) {
+      await flagsClient.reportOverride(key, value, entities);
+    },
     async decide({ key, entities }) {
       const evaluationResult = await flagsClient.evaluate<unknown, unknown>(
         key,

packages/flags/src/index.test.ts

@@ -27,7 +27,7 @@ describe('exports', () => {
 
   it('exports version', () => {
     expect(version).toBeTypeOf('string');
-    expect(version).toMatch(/^\d+\.\d+\.\d+(-\w+-\d+)?$/);
+    expect(version).toMatch(/^\d+\.\d+\.\d+(-[\w.-]+)?$/);
   });
 });
 

packages/flags/src/next/evaluate.ts

@@ -40,6 +40,27 @@ const evaluationCache = new WeakMap<
   Map</* flagKey */ string, Map</* entitiesKey */ string, any>>
 >();
 
+const adapterInitializationCache = new WeakMap<object, Promise<void>>();
+
+async function ensureAdapterInitialized(
+  adapter: Pick<Adapter<unknown, unknown>, 'initialize'>,
+): Promise<void> {
+  if (!adapter.initialize) return;
+
+  let initialization = adapterInitializationCache.get(adapter);
+  if (!initialization) {
+    initialization = adapter.initialize();
+    adapterInitializationCache.set(adapter, initialization);
+  }
+
+  try {
+    await initialization;
+  } catch (error) {
+    adapterInitializationCache.delete(adapter);
+    throw error;
+  }
+}
+
 function getCachedValuePromise(
   /**
    * supports Headers for App Router and IncomingHttpHeaders for Pages Router
@@ -197,7 +218,10 @@ type FlagInfo<ValueType> = {
   key: string;
   defaultValue?: ValueType;
   config?: { reportValue?: boolean };
-  adapter?: { config?: { reportValue?: boolean } };
+  adapter?: Pick<
+    Adapter<ValueType, any>,
+    'config' | 'initialize' | 'reportOverride'
+  >;
 };
 
 function hasOverride(
@@ -227,10 +251,18 @@ async function applyResult<ValueType>(args: {
   definition: FlagInfo<ValueType>;
   readonlyHeaders: ReadonlyHeaders;
   entitiesKey: string;
+  entities?: unknown;
   overrides: Record<string, any> | null;
   produce: () => ValueType | PromiseLike<ValueType>;
 }): Promise<ValueType> {
-  const { definition, readonlyHeaders, entitiesKey, overrides, produce } = args;
+  const {
+    definition,
+    readonlyHeaders,
+    entitiesKey,
+    entities,
+    overrides,
+    produce,
+  } = args;
 
   const cachedValue = getCachedValuePromise(
     readonlyHeaders,
@@ -254,6 +286,19 @@ async function applyResult<ValueType>(args: {
     internalReportValue(definition.key, decision, {
       reason: 'override',
     });
+    try {
+      const adapter = definition.adapter;
+      if (adapter?.reportOverride) {
+        await ensureAdapterInitialized(adapter);
+        await adapter.reportOverride({
+          key: definition.key,
+          value: decision,
+          entities,
+        });
+      }
+    } catch (error) {
+      console.error('flags: Failed to report flag override', error);
+    }
     return decision;
   }
 
@@ -401,6 +446,7 @@ export function getRun<ValueType, EntitiesType>(
       definition,
       readonlyHeaders,
       entitiesKey,
+      entities,
       overrides,
       produce: () =>
         decide({
@@ -641,6 +687,7 @@ async function evaluateImpl(
                     definition: flagFn,
                     readonlyHeaders,
                     entitiesKey,
+                    entities,
                     overrides,
                     produce: () => {
                       if (bulkError) throw bulkError;

packages/flags/src/next/index.test.ts

@@ -191,7 +191,23 @@ describe('flag on app router', () => {
 
   it('respects overrides', async () => {
     const decide = vi.fn(() => false);
-    const f = flag<boolean>({ key: 'first-flag', decide });
+    const calls: string[] = [];
+    const initialize = vi.fn(async () => {
+      calls.push('initialize');
+    });
+    const reportOverride = vi.fn(async () => {
+      calls.push('reportOverride');
+    });
+    const entities = { user: { id: 'user_1' } };
+    const f = flag<boolean, typeof entities>({
+      key: 'first-flag',
+      identify: () => entities,
+      adapter: {
+        decide,
+        initialize,
+        reportOverride,
+      },
+    });
 
     // first request using the flag twice
     const headersOfFirstRequest = new Headers();
@@ -207,6 +223,13 @@ describe('flag on app router', () => {
     await expect(f()).resolves.toEqual(true);
     expect(cookieMock).toHaveBeenCalledWith('vercel-flag-overrides');
     expect(decide).not.toHaveBeenCalled();
+    expect(initialize).toHaveBeenCalledOnce();
+    expect(reportOverride).toHaveBeenCalledWith({
+      key: 'first-flag',
+      value: true,
+      entities,
+    });
+    expect(calls).toEqual(['initialize', 'reportOverride']);
   });
 
   it('does not crash when override reporting hook is not a function', async () => {
@@ -879,6 +902,7 @@ describe('evaluate', () => {
     bulkDecide?: Adapter<V, any>['bulkDecide'];
     decide?: Adapter<V, any>['decide'];
     identify?: Adapter<V, any>['identify'];
+    reportOverride?: Adapter<V, any>['reportOverride'];
     omitAdapterId?: boolean;
     omitBulkDecide?: boolean;
   }) {
@@ -892,6 +916,7 @@ describe('evaluate', () => {
           throw new Error('decide should not be called in bulk path');
         }),
       identify: opts?.identify,
+      reportOverride: opts?.reportOverride,
       ...(opts?.omitBulkDecide ? {} : { bulkDecide: opts?.bulkDecide }),
     });
   }
@@ -1084,7 +1109,13 @@ describe('evaluate', () => {
 
   it('lets overrides win over bulkDecide results', async () => {
     const bulkDecideMock = vi.fn().mockResolvedValue({ a: 'bulk-value' });
-    const adapter = makeBulkAdapter<boolean>({ bulkDecide: bulkDecideMock });
+    const reportOverride = vi.fn();
+    const entities = { user: { id: 'user_1' } };
+    const adapter = makeBulkAdapter<boolean>({
+      bulkDecide: bulkDecideMock,
+      identify: () => entities,
+      reportOverride,
+    });
 
     const a = flag<boolean>({ key: 'a', adapter: adapter() });
 
@@ -1099,6 +1130,11 @@ describe('evaluate', () => {
 
     await expect(evaluate({ a })).resolves.toEqual({ a: true });
     expect(bulkDecideMock).not.toHaveBeenCalled();
+    expect(reportOverride).toHaveBeenCalledWith({
+      key: 'a',
+      value: true,
+      entities,
+    });
   });
 
   it('omits overridden flags from bulkDecide input', async () => {

packages/flags/src/types.ts

@@ -165,6 +165,12 @@ export interface Adapter<ValueType, EntitiesType> {
    * an `adapterId` are never batched.
    */
   adapterId?: string | symbol;
+  /** Observe a value supplied by the Flags SDK override cookie. */
+  reportOverride?: (params: {
+    key: string;
+    value: unknown;
+    entities?: EntitiesType;
+  }) => void | Promise<void>;
   decide: (params: {
     key: string;
     entities?: EntitiesType;

packages/vercel-flags-core/README.md

@@ -24,6 +24,50 @@ const result = await client.evaluate<boolean>('show-new-feature', false, {
 });
 ```
 
+## Experiment exposures
+
+Flags linked to an experiment report exposures automatically, regardless of
+whether the evaluated value came from a fixed variant, target, split, rollout,
+or fallthrough. Provide a custom reporter to send them to your analytics
+system:
+
+```ts
+const client = createClient(process.env.FLAGS!, {
+  reportExposures: async (exposures, entity) => {
+    await analytics.reportExposures(exposures, entity);
+  },
+});
+```
+
+`evaluate()` reports at most one exposure. `bulkEvaluate()` reports all
+experiment exposures in one callback with the single entity object shared by
+the evaluations. The default reporter currently maps exposures to the Vercel
+Web Analytics shape and logs them through a temporary console-backed tracker.
+
+Disable exposure logging for an evaluation when evaluating speculatively or
+prefetching:
+
+```ts
+const result = await client.evaluate(
+  'show-new-feature',
+  false,
+  { user: { key: 'user-123' } },
+  { exposureLogging: false },
+);
+```
+
+The same option is supported by `bulkEvaluate()`:
+
+```ts
+await client.bulkEvaluate(
+  [{ key: 'show-new-feature', defaultValue: false }],
+  { user: { key: 'user-123' } },
+  { exposureLogging: false },
+);
+```
+
+## Evaluation Metrics
+
 To associate evaluation metrics with an environment, pass the
 `metricEnvironment` option: