From 8505d179b6d1e43d3401cd7d6a7b89bf5ae92854 Mon Sep 17 00:00:00 2001 From: Dominik Ferber Date: Sat, 15 Aug 2026 17:13:15 +0300 Subject: [PATCH 01/13] step 1 --- .changeset/bright-experiments-report.md | 5 + packages/vercel-flags-core/README.md | 40 ++++ .../vercel-flags-core/src/black-box.test.ts | 195 ++++++++++++++++++ .../src/create-raw-client.ts | 77 ++++++- .../vercel-flags-core/src/evaluate.test.ts | 91 ++++++++ packages/vercel-flags-core/src/evaluate.ts | 90 +++++--- .../src/exposure-reporting.test.ts | 109 ++++++++++ .../src/exposure-reporting.ts | 98 +++++++++ .../vercel-flags-core/src/index.common.ts | 5 + .../vercel-flags-core/src/index.make.test.ts | 21 ++ packages/vercel-flags-core/src/index.make.ts | 26 ++- packages/vercel-flags-core/src/types.ts | 113 +++++++++- 12 files changed, 831 insertions(+), 39 deletions(-) create mode 100644 .changeset/bright-experiments-report.md create mode 100644 packages/vercel-flags-core/src/exposure-reporting.test.ts create mode 100644 packages/vercel-flags-core/src/exposure-reporting.ts diff --git a/.changeset/bright-experiments-report.md b/.changeset/bright-experiments-report.md new file mode 100644 index 00000000..fc7169fd --- /dev/null +++ b/.changeset/bright-experiments-report.md @@ -0,0 +1,5 @@ +--- +'@vercel/flags-core': minor +--- + +Add experiment outcomes, exposure reporting, and per-evaluation exposure logging controls. diff --git a/packages/vercel-flags-core/README.md b/packages/vercel-flags-core/README.md index b84b0862..89b97f38 100644 --- a/packages/vercel-flags-core/README.md +++ b/packages/vercel-flags-core/README.md @@ -24,6 +24,46 @@ const result = await client.evaluate('show-new-feature', false, { }); ``` +## Experiment exposures + +Experiment-backed flag evaluations report exposures automatically. Provide a +custom reporter to send them to your analytics system: + +```ts +const client = createClient(process.env.FLAGS!, { + reportExposures: async (exposures, entity) => { + await analytics.reportExposures(exposures, entity); + }, +}); +``` + +`evaluate()` reports at most one exposure. `bulkEvaluate()` reports all +experiment exposures in one callback with the single entity object shared by +the evaluations. The default reporter currently maps exposures to the Vercel +Web Analytics shape and logs them through a temporary console-backed tracker. + +Disable exposure logging for an evaluation when evaluating speculatively or +prefetching: + +```ts +const result = await client.evaluate( + 'show-new-feature', + false, + { user: { key: 'user-123' } }, + { exposureLogging: false }, +); +``` + +The same option is supported by `bulkEvaluate()`: + +```ts +await client.bulkEvaluate( + [{ key: 'show-new-feature', defaultValue: false }], + { user: { key: 'user-123' } }, + { exposureLogging: false }, +); +``` + ## OpenFeature An OpenFeature-compatible provider is available at `@vercel/flags-core/openfeature`: diff --git a/packages/vercel-flags-core/src/black-box.test.ts b/packages/vercel-flags-core/src/black-box.test.ts index 281524a7..88805918 100644 --- a/packages/vercel-flags-core/src/black-box.test.ts +++ b/packages/vercel-flags-core/src/black-box.test.ts @@ -3687,6 +3687,201 @@ describe('Controller (black-box)', () => { }); }); + // --------------------------------------------------------------------------- + // Experiment exposure reporting + // --------------------------------------------------------------------------- + describe('experiment exposure reporting', () => { + const definitions: BundledDefinitions['definitions'] = { + flagA: { + environments: { + production: { + fallthrough: { type: 'experiment', experiment: 0 }, + }, + }, + variants: ['control-a', 'treatment-a'], + experiments: [ + { + id: 'exp_a', + base: ['user', 'key'], + weights: [0, 1], + variantIds: ['exp-a-control', 'exp-a-treatment'], + defaultVariant: 0, + seed: 101, + rampId: 'ramp_a', + rampPercentage: 50, + }, + ], + }, + flagB: { + environments: { + production: { + fallthrough: { type: 'experiment', experiment: 0 }, + }, + }, + variants: ['control-b', 'treatment-b'], + experiments: [ + { + id: 'exp_b', + base: ['session', 'key'], + weights: [1, 0], + variantIds: ['exp-b-control', 'exp-b-treatment'], + defaultVariant: 0, + seed: 202, + }, + ], + }, + }; + + const entity = { + user: { key: 'user_123' }, + session: { key: 'session_123' }, + }; + + it('reports one exposure with the exact evaluation entity', async () => { + const reportExposures = vi.fn(); + const client = createClient(sdkKey, { + fetch: fetchMock, + stream: false, + polling: false, + buildStep: true, + datafile: makeBundled({ definitions }), + reportExposures, + }); + + const result = await client.evaluate('flagA', undefined, entity); + + expect(result).toMatchObject({ + value: 'treatment-a', + outcomeType: 'experiment', + experiment: { + id: 'exp_a', + variantId: 'exp-a-treatment', + base: ['user', 'key'], + rampId: 'ramp_a', + rampPercentage: 50, + }, + }); + expect(reportExposures).toHaveBeenCalledOnce(); + expect(reportExposures).toHaveBeenCalledWith( + [ + { + flagKey: 'flagA', + experimentId: 'exp_a', + variantId: 'exp-a-treatment', + base: ['user', 'key'], + rampId: 'ramp_a', + rampPercentage: 50, + }, + ], + entity, + ); + + await client.shutdown(); + }); + + it('can disable exposure logging for a single evaluation', async () => { + const reportExposures = vi.fn(); + const client = createClient(sdkKey, { + fetch: fetchMock, + stream: false, + polling: false, + buildStep: true, + datafile: makeBundled({ definitions }), + reportExposures, + }); + + const result = await client.evaluate('flagA', undefined, entity, { + exposureLogging: false, + }); + + expect(result.experiment?.id).toBe('exp_a'); + expect(reportExposures).not.toHaveBeenCalled(); + await client.shutdown(); + }); + + it('reports all bulk exposures in one callback', async () => { + const reportExposures = vi.fn(); + const client = createClient(sdkKey, { + fetch: fetchMock, + stream: false, + polling: false, + buildStep: true, + datafile: makeBundled({ definitions }), + reportExposures, + }); + + await client.bulkEvaluate([{ key: 'flagA' }, { key: 'flagB' }], entity); + + expect(reportExposures).toHaveBeenCalledOnce(); + expect(reportExposures).toHaveBeenCalledWith( + [ + { + flagKey: 'flagA', + experimentId: 'exp_a', + variantId: 'exp-a-treatment', + base: ['user', 'key'], + rampId: 'ramp_a', + rampPercentage: 50, + }, + { + flagKey: 'flagB', + experimentId: 'exp_b', + variantId: 'exp-b-control', + base: ['session', 'key'], + }, + ], + entity, + ); + + await client.shutdown(); + }); + + it('can disable exposure logging for a bulk evaluation', async () => { + const reportExposures = vi.fn(); + const client = createClient(sdkKey, { + fetch: fetchMock, + stream: false, + polling: false, + buildStep: true, + datafile: makeBundled({ definitions }), + reportExposures, + }); + + const results = await client.bulkEvaluate( + [{ key: 'flagA' }, { key: 'flagB' }], + entity, + { exposureLogging: false }, + ); + + expect(results.flagA?.experiment?.id).toBe('exp_a'); + expect(results.flagB?.experiment?.id).toBe('exp_b'); + expect(reportExposures).not.toHaveBeenCalled(); + await client.shutdown(); + }); + + it('does not fail evaluation when the exposure reporter fails', async () => { + const error = new Error('analytics unavailable'); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const client = createClient(sdkKey, { + fetch: fetchMock, + stream: false, + polling: false, + buildStep: true, + datafile: makeBundled({ definitions }), + reportExposures: () => Promise.reject(error), + }); + + const result = await client.evaluate('flagA', undefined, entity); + + expect(result.value).toBe('treatment-a'); + expect(errorSpy).toHaveBeenCalledWith( + '@vercel/flags-core: Failed to report experiment exposures', + error, + ); + await client.shutdown(); + }); + }); + // --------------------------------------------------------------------------- // Usage tracking // --------------------------------------------------------------------------- diff --git a/packages/vercel-flags-core/src/create-raw-client.ts b/packages/vercel-flags-core/src/create-raw-client.ts index bf4acb06..d9440145 100644 --- a/packages/vercel-flags-core/src/create-raw-client.ts +++ b/packages/vercel-flags-core/src/create-raw-client.ts @@ -10,12 +10,16 @@ import { type ControllerInstance, controllerInstanceMap, } from './controller-fns'; +import { defaultReportExposures } from './exposure-reporting'; import type { BulkEvaluateInput, BundledDefinitions, ControllerInterface, + EvaluationOptions, EvaluationResult, + Exposure, FlagsClient, + ReportExposures, Value, } from './types'; @@ -46,9 +50,11 @@ export function createCreateRawClient(fns: { return function createRawClient>({ controller, origin, + reportExposures, }: { controller: ControllerInterface; origin?: { provider: string; sdkKey?: string }; + reportExposures?: ReportExposures; }): FlagsClient { const id = idCount++; controllerInstanceMap.set(id, { @@ -57,6 +63,43 @@ export function createCreateRawClient(fns: { initPromise: null, }); + const exposureReporter = + reportExposures ?? (defaultReportExposures as ReportExposures); + + async function report( + exposures: readonly Exposure[], + entity: Readonly, + ): Promise { + if (exposures.length === 0) return; + try { + await exposureReporter(exposures, entity); + } catch (error) { + console.error( + '@vercel/flags-core: Failed to report experiment exposures', + error, + ); + } + } + + function getExposure( + flagKey: string, + result: EvaluationResult, + ): Exposure | null { + if (!result.experiment) return null; + return { + flagKey, + experimentId: result.experiment.id, + variantId: result.experiment.variantId, + base: result.experiment.base, + ...(result.experiment.rampId === undefined + ? {} + : { rampId: result.experiment.rampId }), + ...(result.experiment.rampPercentage === undefined + ? {} + : { rampPercentage: result.experiment.rampPercentage }), + }; + } + const api = { origin, initialize: async () => { @@ -99,6 +142,7 @@ export function createCreateRawClient(fns: { flagKey: string, defaultValue?: T, entities?: E, + options?: EvaluationOptions, ): Promise> => { const instance = controllerInstanceMap.get(id); if (!instance?.initialized) { @@ -109,11 +153,25 @@ export function createCreateRawClient(fns: { // chain (last known value → datafile → bundled → defaultValue → throw) } } - return fns.evaluate(id, flagKey, defaultValue, entities); + const entity = entities ?? ({} as E); + const result = await fns.evaluate( + id, + flagKey, + defaultValue, + entity, + ); + if (options?.exposureLogging !== false) { + const exposure = getExposure(flagKey, result); + if (exposure) { + await report([exposure], entity as unknown as Readonly); + } + } + return result; }, bulkEvaluate: async ( flags: BulkEvaluateInput[], entities?: E, + options?: EvaluationOptions, ): Promise>> => { const instance = controllerInstanceMap.get(id); if (!instance?.initialized) { @@ -124,7 +182,22 @@ export function createCreateRawClient(fns: { // chain (last known value → datafile → bundled → defaultValue → throw) } } - return fns.bulkEvaluate(id, flags, entities); + const entity = entities ?? ({} as E); + const results = await fns.bulkEvaluate(id, flags, entity); + if (options?.exposureLogging !== false) { + const exposures: Exposure[] = []; + const seen = new Set(); + for (const flag of flags) { + if (seen.has(flag.key)) continue; + seen.add(flag.key); + const result = results[flag.key]; + if (!result) continue; + const exposure = getExposure(flag.key, result); + if (exposure) exposures.push(exposure); + } + await report(exposures, entity as unknown as Readonly); + } + return results; }, }; return api; diff --git a/packages/vercel-flags-core/src/evaluate.test.ts b/packages/vercel-flags-core/src/evaluate.test.ts index 6ecb9312..eb7b9ad5 100644 --- a/packages/vercel-flags-core/src/evaluate.test.ts +++ b/packages/vercel-flags-core/src/evaluate.test.ts @@ -2700,6 +2700,97 @@ describe('evaluate', () => { }); }); +describe('experiment outcomes', () => { + const definition = { + environments: { + production: { + rules: [ + { + conditions: [[['user', 'country'], Comparator.EQ, 'DE']], + outcome: { type: 'experiment', experiment: 0 }, + }, + ], + fallthrough: 0, + }, + }, + variants: ['control', 'treatment'], + variantIds: ['flag-control', 'flag-treatment'], + experiments: [ + { + id: 'exp_checkout', + base: ['user', 'key'], + weights: [0, 1], + variantIds: ['exp-control', 'exp-treatment'], + defaultVariant: 0, + seed: 123, + rampId: 'ramp_1', + rampPercentage: 25, + }, + ], + } satisfies Packed.FlagDefinition; + + it('evaluates an experiment referenced by a rule', () => { + expect( + evaluate({ + definition, + environment: 'production', + entities: { user: { key: 'user_123', country: 'DE' } }, + }), + ).toEqual({ + value: 'treatment', + variantId: 'flag-treatment', + reason: ResolutionReason.RULE_MATCH, + outcomeType: OutcomeType.EXPERIMENT, + experiment: { + id: 'exp_checkout', + variantId: 'exp-treatment', + base: ['user', 'key'], + rampId: 'ramp_1', + rampPercentage: 25, + }, + }); + }); + + it('uses the experiment default variant when its base is missing', () => { + expect( + evaluate({ + definition, + environment: 'production', + entities: { user: { country: 'DE' } }, + }), + ).toEqual({ + value: 'control', + variantId: 'flag-control', + reason: ResolutionReason.RULE_MATCH, + outcomeType: OutcomeType.EXPERIMENT, + experiment: { + id: 'exp_checkout', + variantId: 'exp-control', + base: ['user', 'key'], + rampId: 'ramp_1', + rampPercentage: 25, + }, + }); + }); + + it('throws for an invalid experiment reference', () => { + expect(() => + evaluate({ + definition: { + environments: { + production: { + fallthrough: { type: 'experiment', experiment: 1 }, + }, + }, + variants: [false], + }, + environment: 'production', + entities: {}, + }), + ).toThrow('@vercel/flags-core: Experiment index 1 not found'); + }); +}); + describe('bulkEvaluate', () => { it('evaluates multiple flags against shared entities, segments, and environment', () => { const activeDef: Packed.FlagDefinition = { diff --git a/packages/vercel-flags-core/src/evaluate.ts b/packages/vercel-flags-core/src/evaluate.ts index 1e51f82c..5bb3ba1c 100644 --- a/packages/vercel-flags-core/src/evaluate.ts +++ b/packages/vercel-flags-core/src/evaluate.ts @@ -3,6 +3,7 @@ import { Comparator, type EvaluationParams, type EvaluationResult, + type ExperimentAssignment, OutcomeType, Packed, ResolutionReason, @@ -40,14 +41,14 @@ function boundaryFor(numerator: number, denominator: number): number { // symbol-keyed props) and serialize cleanly across the RSC boundary; entries // are GC'd with the datafile. Split boundaries are static per outcome, so the // cumulative cut points are computed once and reused across evaluations. -const splitBoundariesCache = new WeakMap(); +const splitBoundariesCache = new WeakMap(); const compiledRegexCache = new WeakMap(); /** * Cumulative hash boundaries for a split, one per variant in index order. * Variant `i` is served for hashes in `[boundaries[i-1], boundaries[i])`. */ -function getSplitBoundaries(outcome: Packed.SplitOutcome): number[] { +function getSplitBoundaries(outcome: { weights: number[] }): number[] { const cached = splitBoundariesCache.get(outcome); if (cached) return cached; const total = sum(outcome.weights); @@ -414,6 +415,31 @@ function getVariant( }; } +type WeightedAssignment = { + base: Packed.EntityAccessor; + weights: number[]; + defaultVariant: Packed.VariantIndex; +}; + +function getWeightedVariantIndex( + params: EvaluationParams, + assignment: WeightedAssignment, + seed: number | undefined, +): Packed.VariantIndex { + const lhs = access(assignment.base, params); + + if (typeof lhs !== 'string') return assignment.defaultVariant; + + const bucket = hashInput(lhs, seed); + const boundaries = getSplitBoundaries(assignment); + for (let index = 0; index < boundaries.length; index++) { + if (bucket < (boundaries[index] as number)) return index; + } + + // Only reached when the weights sum to 0 (every boundary is NaN). + return assignment.defaultVariant; +} + function handleOutcome( params: EvaluationParams, outcome: Packed.Outcome, @@ -421,6 +447,7 @@ function handleOutcome( value: T; outcomeType: OutcomeType; variantId: VariantId | null; + experiment?: ExperimentAssignment; } { if (typeof outcome === 'number') { const variant = getVariant(params.definition, outcome); @@ -431,37 +458,46 @@ function handleOutcome( } switch (outcome.type) { case 'split': { - const lhs = access(outcome.base, params); - const defaultOutcome = getVariant( - params.definition, - outcome.defaultVariant, + const index = getWeightedVariantIndex( + params, + outcome, + params.definition.seed, ); - - // serve the default variant if the lhs is not a string - if (typeof lhs !== 'string') { - return { - ...defaultOutcome, - outcomeType: OutcomeType.SPLIT, - }; + return { + ...getVariant(params.definition, index), + outcomeType: OutcomeType.SPLIT, + }; + } + case 'experiment': { + const experiment = params.definition.experiments?.[outcome.experiment]; + if (!experiment) { + throw new Error( + `@vercel/flags-core: Experiment index ${outcome.experiment} not found`, + ); } - const bucket = hashInput(lhs, params.definition.seed); - const boundaries = getSplitBoundaries(outcome); - - // Return the first variant whose cumulative boundary covers the bucket. - for (let index = 0; index < boundaries.length; index++) { - if (bucket < (boundaries[index] as number)) { - return { - ...getVariant(params.definition, index), - outcomeType: OutcomeType.SPLIT, - }; - } + const index = getWeightedVariantIndex( + params, + experiment, + experiment.seed, + ); + const experimentVariantId = experiment.variantIds[index]; + if (typeof experimentVariantId !== 'string') { + throw new Error( + `@vercel/flags-core: Experiment variant ID not found at index ${index} for experiment "${experiment.id}"`, + ); } - // Only reached when the weights sum to 0 (every boundary is NaN). return { - ...defaultOutcome, - outcomeType: OutcomeType.SPLIT, + ...getVariant(params.definition, index), + outcomeType: OutcomeType.EXPERIMENT, + experiment: { + id: experiment.id, + variantId: experimentVariantId, + base: experiment.base, + rampId: experiment.rampId, + rampPercentage: experiment.rampPercentage, + }, }; } case 'rollout': { diff --git a/packages/vercel-flags-core/src/exposure-reporting.test.ts b/packages/vercel-flags-core/src/exposure-reporting.test.ts new file mode 100644 index 00000000..846c46d3 --- /dev/null +++ b/packages/vercel-flags-core/src/exposure-reporting.test.ts @@ -0,0 +1,109 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { defaultReportExposures } from './exposure-reporting'; + +describe('defaultReportExposures', () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('maps known and custom entity bases to Web Analytics units', () => { + const log = vi.spyOn(console, 'log').mockImplementation(() => {}); + + defaultReportExposures( + [ + { + flagKey: 'checkout', + experimentId: 'exp_user', + variantId: 'variant_a', + base: ['user', 'key'], + rampId: 'ramp_1', + rampPercentage: 50, + }, + { + flagKey: 'pricing', + experimentId: 'exp_team', + variantId: 'variant_b', + base: ['team', 'key'], + }, + { + flagKey: 'visitor', + experimentId: 'exp_visitor', + variantId: 'variant_c', + base: ['visitor', 'id'], + }, + { + flagKey: 'device', + experimentId: 'exp_device', + variantId: 'variant_d', + base: ['device', 'key'], + }, + ], + { + user: { key: 'user_123' }, + team: { key: 'team_123' }, + visitor: { id: 'visitor_123' }, + }, + ); + + expect(log).toHaveBeenNthCalledWith( + 1, + '@vercel/flags-core: trackExposure', + { + experimentId: 'exp_user', + variantId: 'variant_a', + unitKey: 'user', + unitValue: 'user_123', + rampId: 'ramp_1', + rampPercentage: 50, + }, + ); + expect(log).toHaveBeenNthCalledWith( + 2, + '@vercel/flags-core: trackExposure', + { + experimentId: 'exp_team', + variantId: 'variant_b', + unitKey: 'group', + unitValue: 'team_123', + }, + ); + expect(log).toHaveBeenNthCalledWith( + 3, + '@vercel/flags-core: trackExposure', + { + experimentId: 'exp_visitor', + variantId: 'variant_c', + unitKey: 'event_data.visitorId', + unitValue: 'visitor_123', + }, + ); + expect(log).toHaveBeenNthCalledWith( + 4, + '@vercel/flags-core: trackExposure', + { + experimentId: 'exp_device', + variantId: 'variant_d', + unitKey: 'device', + unitValue: 'fake-device-id', + }, + ); + }); + + it('does not track an exposure whose entity value cannot be resolved', () => { + const log = vi.spyOn(console, 'log').mockImplementation(() => {}); + + defaultReportExposures( + [ + { + flagKey: 'checkout', + experimentId: 'exp_user', + variantId: 'variant_a', + base: ['user', 'key'], + }, + ], + {}, + ); + + expect(log).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/vercel-flags-core/src/exposure-reporting.ts b/packages/vercel-flags-core/src/exposure-reporting.ts new file mode 100644 index 00000000..c7259cb9 --- /dev/null +++ b/packages/vercel-flags-core/src/exposure-reporting.ts @@ -0,0 +1,98 @@ +import type { Exposure, Packed, ReportExposures } from './types'; + +type WebAnalyticsExposure = { + experimentId: string; + variantId: string; + unitKey: 'user' | 'session' | 'device' | 'group' | `event_data.${string}`; + unitValue: string; + rampId?: string; + rampPercentage?: number; +}; + +const FAKE_DEVICE_ID = 'fake-device-id'; + +function getProperty( + entity: Readonly>, + path: Packed.EntityAccessor, +): unknown { + return path.reduce((value, key) => { + if (typeof value !== 'object' || value === null || !(key in value)) { + return undefined; + } + return (value as Record)[key]; + }, entity); +} + +function isBase(base: Packed.EntityAccessor, kind: string): boolean { + return base.length === 2 && base[0] === kind && base[1] === 'key'; +} + +function flattenBase(base: Packed.EntityAccessor): string { + return base + .map(String) + .map((part, index) => + index === 0 ? part : part.charAt(0).toUpperCase() + part.slice(1), + ) + .join(''); +} + +function mapExposure( + exposure: Exposure, + entity: Readonly>, +): WebAnalyticsExposure | null { + let unitKey: WebAnalyticsExposure['unitKey']; + let unitValue: unknown; + + if (isBase(exposure.base, 'user')) { + unitKey = 'user'; + unitValue = getProperty(entity, exposure.base); + } else if (isBase(exposure.base, 'session')) { + unitKey = 'session'; + unitValue = getProperty(entity, exposure.base); + } else if (isBase(exposure.base, 'device')) { + unitKey = 'device'; + unitValue = FAKE_DEVICE_ID; + } else if (isBase(exposure.base, 'team')) { + unitKey = 'group'; + unitValue = getProperty(entity, exposure.base); + } else { + const flattenedBase = flattenBase(exposure.base); + if (!flattenedBase) return null; + unitKey = `event_data.${flattenedBase}`; + unitValue = getProperty(entity, exposure.base); + } + + if (typeof unitValue !== 'string') return null; + + return { + experimentId: exposure.experimentId, + variantId: exposure.variantId, + unitKey, + unitValue, + ...(exposure.rampId === undefined ? {} : { rampId: exposure.rampId }), + ...(exposure.rampPercentage === undefined + ? {} + : { rampPercentage: exposure.rampPercentage }), + }; +} + +/** + * Temporary stand-in for the Vercel Web Analytics exposure API. + */ +function trackExposure(exposure: WebAnalyticsExposure): void { + console.log('@vercel/flags-core: trackExposure', exposure); +} + +/** + * Default exposure reporter. It maps Vercel Flags entity paths to the current + * Vercel Web Analytics exposure format and calls a temporary console-backed + * `trackExposure` implementation. + */ +export const defaultReportExposures: ReportExposures< + Record +> = (exposures, entity) => { + for (const exposure of exposures) { + const mapped = mapExposure(exposure, entity); + if (mapped) trackExposure(mapped); + } +}; diff --git a/packages/vercel-flags-core/src/index.common.ts b/packages/vercel-flags-core/src/index.common.ts index a8834192..fbe88036 100644 --- a/packages/vercel-flags-core/src/index.common.ts +++ b/packages/vercel-flags-core/src/index.common.ts @@ -11,16 +11,21 @@ export { FallbackNotFoundError, } from './errors'; export { evaluate } from './evaluate'; +export { defaultReportExposures } from './exposure-reporting'; export type { CreateClientOptions } from './index.make'; export { type BundledDefinitions, type Datafile, type DatafileInput, + type EvaluationOptions, type EvaluationParams, type EvaluationResult, + type ExperimentAssignment, + type Exposure, type FlagsClient, type Packed, type PollingOptions, + type ReportExposures, ResolutionReason as Reason, type StreamOptions, type Value, diff --git a/packages/vercel-flags-core/src/index.make.test.ts b/packages/vercel-flags-core/src/index.make.test.ts index fa139e28..8b7ce176 100644 --- a/packages/vercel-flags-core/src/index.make.test.ts +++ b/packages/vercel-flags-core/src/index.make.test.ts @@ -120,6 +120,27 @@ describe('make', () => { expect(client).toBeDefined(); }); + it('should pass reportExposures to the raw client, not the controller', () => { + const createRawClient = createMockCreateRawClient(); + const { createClient } = make(createRawClient); + const reportExposures = vi.fn(); + + createClient('vf_server_test_key', { + stream: false, + reportExposures, + }); + + expect(Controller).toHaveBeenCalledWith({ + auth: expect.objectContaining({ sdkKey: 'vf_server_test_key' }), + stream: false, + }); + expect(createRawClient).toHaveBeenCalledWith({ + controller: expect.any(Object), + origin: { provider: 'vercel', sdkKey: 'vf_server_test_key' }, + reportExposures, + }); + }); + it('should throw for empty SDK key', () => { const createRawClient = createMockCreateRawClient(); const { createClient } = make(createRawClient); diff --git a/packages/vercel-flags-core/src/index.make.ts b/packages/vercel-flags-core/src/index.make.ts index 19343c94..908a8d2f 100644 --- a/packages/vercel-flags-core/src/index.make.ts +++ b/packages/vercel-flags-core/src/index.make.ts @@ -5,20 +5,26 @@ import { Controller, type ControllerOptions } from './controller'; import { Authentication } from './controller/auth'; import type { createCreateRawClient } from './create-raw-client'; -import type { FlagsClient } from './types'; +import type { FlagsClient, ReportExposures } from './types'; /** * Options for createClient */ -export type CreateClientOptions = Omit; +export type CreateClientOptions> = Omit< + ControllerOptions, + 'auth' +> & { + /** Reports experiment exposures produced by evaluation calls. */ + reportExposures?: ReportExposures; +}; type CreateClient = { >( - options: CreateClientOptions, + options: CreateClientOptions, ): FlagsClient; >( sdkKeyOrConnectionString?: string, - options?: CreateClientOptions, + options?: CreateClientOptions, ): FlagsClient; }; @@ -35,15 +41,15 @@ export function make( // - data source must specify the environment & projectId as sdkKey has that info // - "reuse" functionality relies on the data source having the data for all envs function createClient>( - options: CreateClientOptions, + options: CreateClientOptions, ): FlagsClient; function createClient>( sdkKeyOrConnectionString?: string, - options?: CreateClientOptions, + options?: CreateClientOptions, ): FlagsClient; function createClient>( - sdkKeyOrConnectionStringOrOptions?: string | CreateClientOptions, - options?: CreateClientOptions, + sdkKeyOrConnectionStringOrOptions?: string | CreateClientOptions, + options?: CreateClientOptions, ): FlagsClient { const optionsOnly = typeof sdkKeyOrConnectionStringOrOptions === 'object' && @@ -55,13 +61,15 @@ export function make( ? sdkKeyOrConnectionStringOrOptions : options; + const { reportExposures, ...controllerOptions } = createClientOptions ?? {}; const auth = new Authentication(sdkKeyOrConnectionString); // sdk key contains the environment - const controller = new Controller({ auth, ...createClientOptions }); + const controller = new Controller({ auth, ...controllerOptions }); return createRawClient({ controller, origin: { provider: 'vercel', sdkKey: auth.sdkKey }, + ...(reportExposures ? { reportExposures } : {}), }); } diff --git a/packages/vercel-flags-core/src/types.ts b/packages/vercel-flags-core/src/types.ts index 6e7fbe4d..553974fd 100644 --- a/packages/vercel-flags-core/src/types.ts +++ b/packages/vercel-flags-core/src/types.ts @@ -123,6 +123,51 @@ export type BulkEvaluateInput = { defaultValue?: T; }; +/** Options that control side effects of an evaluation call. */ +export type EvaluationOptions = { + /** + * Whether experiment exposures should be reported for this evaluation. + * @default true + */ + exposureLogging?: boolean; +}; + +/** Information about the experiment assignment that produced a flag value. */ +export type ExperimentAssignment = { + /** Experiment identifier. */ + id: string; + /** Identifier of the selected experiment variant. */ + variantId: string; + /** Entity path on which the experiment assignment is based. */ + base: Packed.EntityAccessor; + /** Identifier of the ramp active for this assignment. */ + rampId?: string; + /** Percentage of eligible units included in the ramp, from 0 through 100. */ + rampPercentage?: number; +}; + +/** An experiment exposure passed to a client's exposure reporter. */ +export type Exposure = { + /** Flag whose evaluation produced the exposure. */ + flagKey: FlagKey; + /** Experiment identifier. */ + experimentId: string; + /** Identifier of the selected experiment variant. */ + variantId: string; + /** Entity path on which the experiment assignment is based. */ + base: Packed.EntityAccessor; + /** Identifier of the ramp active for this assignment. */ + rampId?: string; + /** Percentage of eligible units included in the ramp, from 0 through 100. */ + rampPercentage?: number; +}; + +/** Reports experiment exposures produced by one evaluation call. */ +export type ReportExposures> = ( + exposures: readonly Exposure[], + entity: Readonly, +) => void | Promise; + /** * A client for Vercel Flags */ @@ -143,12 +188,14 @@ export type FlagsClient> = { * @param flagKey * @param defaultValue * @param entities + * @param options Evaluation side-effect options. * @returns */ evaluate: ( flagKey: string, defaultValue?: T, entities?: E, + options?: EvaluationOptions, ) => Promise>; /** * Evaluate multiple feature flags against the same entities in a single call. @@ -160,11 +207,13 @@ export type FlagsClient> = { * * @param flags Array of `{ key, defaultValue? }` entries to evaluate. * @param entities Shared entities used for every flag in the bulk call. + * @param options Evaluation side-effect options. * @returns Object mapping each key to its EvaluationResult. */ bulkEvaluate: ( flags: BulkEvaluateInput[], entities?: E, + options?: EvaluationOptions, ) => Promise>>; /** * Retrieve the latest datafile during startup, and set up subscriptions if needed. @@ -250,6 +299,8 @@ export type EvaluationResult = * The variant we want to report for o11y */ variantId: VariantId | null; + /** Experiment assignment when an experiment outcome produced the value. */ + experiment?: ExperimentAssignment; /** * Indicates why the flag evaluated to a certain value */ @@ -264,6 +315,7 @@ export type EvaluationResult = errorMessage: string; errorCode?: ErrorCode; outcomeType?: never; + experiment?: never; /** * The variant we want to report for o11y */ @@ -307,6 +359,8 @@ export enum OutcomeType { SPLIT = 'split', /** When the outcome type was a progressive rollout */ ROLLOUT = 'rollout', + /** When the outcome type was an experiment assignment */ + EXPERIMENT = 'experiment', } /** @@ -540,8 +594,30 @@ export namespace Original { * Once all slots are exhausted, the rollout is complete (100% rollToVariant). */ slots: { promille: number; durationMs: number }[]; + } + | { + type: 'experiment'; + /** Identifier of the experiment in `FlagDefinition.experiments`. */ + experimentId: string; }; + export type ExperimentDefinition = { + id: string; + /** Based on which entity attribute traffic should be assigned. */ + base: EntityAccessor; + /** Distribution keyed by flag variant ID. */ + weights: Record; + /** Experiment variant ID keyed by flag variant ID. */ + variantIds: Record; + /** Flag variant used when the base attribute does not exist. */ + defaultVariantId: VariantId; + /** Seed used to keep experiment assignment stable and independent. */ + seed: number; + rampId?: string; + /** Percentage from 0 through 100. */ + rampPercentage?: number; + }; + export type SegmentAllOutcome = { type: 'all'; }; @@ -669,6 +745,8 @@ export namespace Original { export type FlagDefinition = { variants: FlagVariant[]; + /** Experiment definitions keyed by experiment ID. */ + experiments?: Record; environments: Record; /** @@ -696,6 +774,7 @@ export namespace Packed { * Idenitifies a variant based on its index in the variants array. */ export type VariantIndex = number; + export type ExperimentIndex = number; export type Data = { /** map of flag keys to definitions */ @@ -764,6 +843,32 @@ export namespace Packed { slots: [number, number][]; }; + /** An outcome which delegates assignment to a flag-level experiment. */ + export type ExperimentOutcome = { + type: 'experiment'; + /** Index into `FlagDefinition.experiments`. */ + experiment: ExperimentIndex; + }; + + export type ExperimentDefinition = { + /** Experiment identifier. */ + id: string; + /** Entity path used for deterministic assignment. */ + base: EntityAccessor; + /** Distribution indexed by the corresponding flag variant. */ + weights: number[]; + /** Experiment variant IDs indexed by the corresponding flag variant. */ + variantIds: (string | null)[]; + /** Flag variant used when the base attribute does not exist. */ + defaultVariant: VariantIndex; + /** Seed used to keep experiment assignment stable and independent. */ + seed: number; + /** Identifier of the ramp active for this experiment. */ + rampId?: string; + /** Percentage of eligible units included in the ramp, from 0 through 100. */ + rampPercentage?: number; + }; + export type SegmentAllOutcome = 1; export type SegmentSplitOutcome = { @@ -782,7 +887,11 @@ export namespace Packed { export type SegmentOutcome = SegmentAllOutcome | SegmentSplitOutcome; - export type Outcome = VariantIndex | SplitOutcome | RolloutOutcome; + export type Outcome = + | VariantIndex + | SplitOutcome + | RolloutOutcome + | ExperimentOutcome; // an array means it's an entity, the string "segment" means a segment export type EntityAccessor = (string | number)[]; @@ -891,6 +1000,8 @@ export namespace Packed { variantIds?: string[]; /** variants, packed down to just their values */ variants: Value[]; + /** Experiment definitions referenced by experiment outcomes. */ + experiments?: ExperimentDefinition[]; /** environments */ environments: Record; /** From 98866ff3dfe979e94704187826e2a3aa533fe77d Mon Sep 17 00:00:00 2001 From: Dominik Ferber Date: Sat, 15 Aug 2026 21:04:06 +0300 Subject: [PATCH 02/13] single experiment --- .../vercel-flags-core/src/black-box.test.ts | 44 +++++++++---------- .../vercel-flags-core/src/evaluate.test.ts | 28 ++++++------ packages/vercel-flags-core/src/evaluate.ts | 6 +-- packages/vercel-flags-core/src/types.ts | 13 ++---- 4 files changed, 39 insertions(+), 52 deletions(-) diff --git a/packages/vercel-flags-core/src/black-box.test.ts b/packages/vercel-flags-core/src/black-box.test.ts index 88805918..e7c22f6e 100644 --- a/packages/vercel-flags-core/src/black-box.test.ts +++ b/packages/vercel-flags-core/src/black-box.test.ts @@ -3695,40 +3695,36 @@ describe('Controller (black-box)', () => { flagA: { environments: { production: { - fallthrough: { type: 'experiment', experiment: 0 }, + fallthrough: { type: 'experiment' }, }, }, variants: ['control-a', 'treatment-a'], - experiments: [ - { - id: 'exp_a', - base: ['user', 'key'], - weights: [0, 1], - variantIds: ['exp-a-control', 'exp-a-treatment'], - defaultVariant: 0, - seed: 101, - rampId: 'ramp_a', - rampPercentage: 50, - }, - ], + experiment: { + id: 'exp_a', + base: ['user', 'key'], + weights: [0, 1], + variantIds: ['exp-a-control', 'exp-a-treatment'], + defaultVariant: 0, + seed: 101, + rampId: 'ramp_a', + rampPercentage: 50, + }, }, flagB: { environments: { production: { - fallthrough: { type: 'experiment', experiment: 0 }, + fallthrough: { type: 'experiment' }, }, }, variants: ['control-b', 'treatment-b'], - experiments: [ - { - id: 'exp_b', - base: ['session', 'key'], - weights: [1, 0], - variantIds: ['exp-b-control', 'exp-b-treatment'], - defaultVariant: 0, - seed: 202, - }, - ], + experiment: { + id: 'exp_b', + base: ['session', 'key'], + weights: [1, 0], + variantIds: ['exp-b-control', 'exp-b-treatment'], + defaultVariant: 0, + seed: 202, + }, }, }; diff --git a/packages/vercel-flags-core/src/evaluate.test.ts b/packages/vercel-flags-core/src/evaluate.test.ts index eb7b9ad5..efa252d1 100644 --- a/packages/vercel-flags-core/src/evaluate.test.ts +++ b/packages/vercel-flags-core/src/evaluate.test.ts @@ -2707,7 +2707,7 @@ describe('experiment outcomes', () => { rules: [ { conditions: [[['user', 'country'], Comparator.EQ, 'DE']], - outcome: { type: 'experiment', experiment: 0 }, + outcome: { type: 'experiment' }, }, ], fallthrough: 0, @@ -2715,18 +2715,16 @@ describe('experiment outcomes', () => { }, variants: ['control', 'treatment'], variantIds: ['flag-control', 'flag-treatment'], - experiments: [ - { - id: 'exp_checkout', - base: ['user', 'key'], - weights: [0, 1], - variantIds: ['exp-control', 'exp-treatment'], - defaultVariant: 0, - seed: 123, - rampId: 'ramp_1', - rampPercentage: 25, - }, - ], + experiment: { + id: 'exp_checkout', + base: ['user', 'key'], + weights: [0, 1], + variantIds: ['exp-control', 'exp-treatment'], + defaultVariant: 0, + seed: 123, + rampId: 'ramp_1', + rampPercentage: 25, + }, } satisfies Packed.FlagDefinition; it('evaluates an experiment referenced by a rule', () => { @@ -2779,7 +2777,7 @@ describe('experiment outcomes', () => { definition: { environments: { production: { - fallthrough: { type: 'experiment', experiment: 1 }, + fallthrough: { type: 'experiment' }, }, }, variants: [false], @@ -2787,7 +2785,7 @@ describe('experiment outcomes', () => { environment: 'production', entities: {}, }), - ).toThrow('@vercel/flags-core: Experiment index 1 not found'); + ).toThrow('@vercel/flags-core: Experiment not found'); }); }); diff --git a/packages/vercel-flags-core/src/evaluate.ts b/packages/vercel-flags-core/src/evaluate.ts index 5bb3ba1c..6d4b17d5 100644 --- a/packages/vercel-flags-core/src/evaluate.ts +++ b/packages/vercel-flags-core/src/evaluate.ts @@ -469,11 +469,9 @@ function handleOutcome( }; } case 'experiment': { - const experiment = params.definition.experiments?.[outcome.experiment]; + const experiment = params.definition.experiment; if (!experiment) { - throw new Error( - `@vercel/flags-core: Experiment index ${outcome.experiment} not found`, - ); + throw new Error('@vercel/flags-core: Experiment not found'); } const index = getWeightedVariantIndex( diff --git a/packages/vercel-flags-core/src/types.ts b/packages/vercel-flags-core/src/types.ts index 553974fd..e937c37d 100644 --- a/packages/vercel-flags-core/src/types.ts +++ b/packages/vercel-flags-core/src/types.ts @@ -597,8 +597,6 @@ export namespace Original { } | { type: 'experiment'; - /** Identifier of the experiment in `FlagDefinition.experiments`. */ - experimentId: string; }; export type ExperimentDefinition = { @@ -745,8 +743,8 @@ export namespace Original { export type FlagDefinition = { variants: FlagVariant[]; - /** Experiment definitions keyed by experiment ID. */ - experiments?: Record; + /** Experiment linked to this flag. */ + experiment?: ExperimentDefinition; environments: Record; /** @@ -774,7 +772,6 @@ export namespace Packed { * Idenitifies a variant based on its index in the variants array. */ export type VariantIndex = number; - export type ExperimentIndex = number; export type Data = { /** map of flag keys to definitions */ @@ -846,8 +843,6 @@ export namespace Packed { /** An outcome which delegates assignment to a flag-level experiment. */ export type ExperimentOutcome = { type: 'experiment'; - /** Index into `FlagDefinition.experiments`. */ - experiment: ExperimentIndex; }; export type ExperimentDefinition = { @@ -1000,8 +995,8 @@ export namespace Packed { variantIds?: string[]; /** variants, packed down to just their values */ variants: Value[]; - /** Experiment definitions referenced by experiment outcomes. */ - experiments?: ExperimentDefinition[]; + /** Experiment linked to this flag. */ + experiment?: ExperimentDefinition; /** environments */ environments: Record; /** From 7905762f55029f4c8a6e7a66b259467284103a1e Mon Sep 17 00:00:00 2001 From: Dominik Ferber Date: Sat, 15 Aug 2026 22:02:40 +0300 Subject: [PATCH 03/13] reuse variants --- packages/vercel-flags-core/src/black-box.test.ts | 12 ++++++------ packages/vercel-flags-core/src/evaluate.test.ts | 5 ++--- packages/vercel-flags-core/src/evaluate.ts | 10 +++++----- packages/vercel-flags-core/src/types.ts | 4 ---- 4 files changed, 13 insertions(+), 18 deletions(-) diff --git a/packages/vercel-flags-core/src/black-box.test.ts b/packages/vercel-flags-core/src/black-box.test.ts index e7c22f6e..9cb8ec80 100644 --- a/packages/vercel-flags-core/src/black-box.test.ts +++ b/packages/vercel-flags-core/src/black-box.test.ts @@ -3699,11 +3699,11 @@ describe('Controller (black-box)', () => { }, }, variants: ['control-a', 'treatment-a'], + variantIds: ['control-a', 'treatment-a'], experiment: { id: 'exp_a', base: ['user', 'key'], weights: [0, 1], - variantIds: ['exp-a-control', 'exp-a-treatment'], defaultVariant: 0, seed: 101, rampId: 'ramp_a', @@ -3717,11 +3717,11 @@ describe('Controller (black-box)', () => { }, }, variants: ['control-b', 'treatment-b'], + variantIds: ['control-b', 'treatment-b'], experiment: { id: 'exp_b', base: ['session', 'key'], weights: [1, 0], - variantIds: ['exp-b-control', 'exp-b-treatment'], defaultVariant: 0, seed: 202, }, @@ -3751,7 +3751,7 @@ describe('Controller (black-box)', () => { outcomeType: 'experiment', experiment: { id: 'exp_a', - variantId: 'exp-a-treatment', + variantId: 'treatment-a', base: ['user', 'key'], rampId: 'ramp_a', rampPercentage: 50, @@ -3763,7 +3763,7 @@ describe('Controller (black-box)', () => { { flagKey: 'flagA', experimentId: 'exp_a', - variantId: 'exp-a-treatment', + variantId: 'treatment-a', base: ['user', 'key'], rampId: 'ramp_a', rampPercentage: 50, @@ -3814,7 +3814,7 @@ describe('Controller (black-box)', () => { { flagKey: 'flagA', experimentId: 'exp_a', - variantId: 'exp-a-treatment', + variantId: 'treatment-a', base: ['user', 'key'], rampId: 'ramp_a', rampPercentage: 50, @@ -3822,7 +3822,7 @@ describe('Controller (black-box)', () => { { flagKey: 'flagB', experimentId: 'exp_b', - variantId: 'exp-b-control', + variantId: 'control-b', base: ['session', 'key'], }, ], diff --git a/packages/vercel-flags-core/src/evaluate.test.ts b/packages/vercel-flags-core/src/evaluate.test.ts index efa252d1..1aa51e4f 100644 --- a/packages/vercel-flags-core/src/evaluate.test.ts +++ b/packages/vercel-flags-core/src/evaluate.test.ts @@ -2719,7 +2719,6 @@ describe('experiment outcomes', () => { id: 'exp_checkout', base: ['user', 'key'], weights: [0, 1], - variantIds: ['exp-control', 'exp-treatment'], defaultVariant: 0, seed: 123, rampId: 'ramp_1', @@ -2741,7 +2740,7 @@ describe('experiment outcomes', () => { outcomeType: OutcomeType.EXPERIMENT, experiment: { id: 'exp_checkout', - variantId: 'exp-treatment', + variantId: 'flag-treatment', base: ['user', 'key'], rampId: 'ramp_1', rampPercentage: 25, @@ -2763,7 +2762,7 @@ describe('experiment outcomes', () => { outcomeType: OutcomeType.EXPERIMENT, experiment: { id: 'exp_checkout', - variantId: 'exp-control', + variantId: 'flag-control', base: ['user', 'key'], rampId: 'ramp_1', rampPercentage: 25, diff --git a/packages/vercel-flags-core/src/evaluate.ts b/packages/vercel-flags-core/src/evaluate.ts index 6d4b17d5..ad6c1f5c 100644 --- a/packages/vercel-flags-core/src/evaluate.ts +++ b/packages/vercel-flags-core/src/evaluate.ts @@ -479,19 +479,19 @@ function handleOutcome( experiment, experiment.seed, ); - const experimentVariantId = experiment.variantIds[index]; - if (typeof experimentVariantId !== 'string') { + const variant = getVariant(params.definition, index); + if (typeof variant.variantId !== 'string') { throw new Error( - `@vercel/flags-core: Experiment variant ID not found at index ${index} for experiment "${experiment.id}"`, + `@vercel/flags-core: Flag variant ID not found at index ${index} for experiment "${experiment.id}"`, ); } return { - ...getVariant(params.definition, index), + ...variant, outcomeType: OutcomeType.EXPERIMENT, experiment: { id: experiment.id, - variantId: experimentVariantId, + variantId: variant.variantId, base: experiment.base, rampId: experiment.rampId, rampPercentage: experiment.rampPercentage, diff --git a/packages/vercel-flags-core/src/types.ts b/packages/vercel-flags-core/src/types.ts index e937c37d..88ab3ecf 100644 --- a/packages/vercel-flags-core/src/types.ts +++ b/packages/vercel-flags-core/src/types.ts @@ -605,8 +605,6 @@ export namespace Original { base: EntityAccessor; /** Distribution keyed by flag variant ID. */ weights: Record; - /** Experiment variant ID keyed by flag variant ID. */ - variantIds: Record; /** Flag variant used when the base attribute does not exist. */ defaultVariantId: VariantId; /** Seed used to keep experiment assignment stable and independent. */ @@ -852,8 +850,6 @@ export namespace Packed { base: EntityAccessor; /** Distribution indexed by the corresponding flag variant. */ weights: number[]; - /** Experiment variant IDs indexed by the corresponding flag variant. */ - variantIds: (string | null)[]; /** Flag variant used when the base attribute does not exist. */ defaultVariant: VariantIndex; /** Seed used to keep experiment assignment stable and independent. */ From 321356cb8b58fc98a41f917d4b5af54882a729c9 Mon Sep 17 00:00:00 2001 From: Dominik Ferber Date: Sun, 16 Aug 2026 15:03:11 +0300 Subject: [PATCH 04/13] single experiment --- packages/vercel-flags-core/src/black-box.test.ts | 4 ++-- packages/vercel-flags-core/src/evaluate.test.ts | 2 +- packages/vercel-flags-core/src/evaluate.ts | 2 +- packages/vercel-flags-core/src/types.ts | 4 ---- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/vercel-flags-core/src/black-box.test.ts b/packages/vercel-flags-core/src/black-box.test.ts index 9cb8ec80..8db782b2 100644 --- a/packages/vercel-flags-core/src/black-box.test.ts +++ b/packages/vercel-flags-core/src/black-box.test.ts @@ -3700,12 +3700,12 @@ describe('Controller (black-box)', () => { }, variants: ['control-a', 'treatment-a'], variantIds: ['control-a', 'treatment-a'], + seed: 101, experiment: { id: 'exp_a', base: ['user', 'key'], weights: [0, 1], defaultVariant: 0, - seed: 101, rampId: 'ramp_a', rampPercentage: 50, }, @@ -3718,12 +3718,12 @@ describe('Controller (black-box)', () => { }, variants: ['control-b', 'treatment-b'], variantIds: ['control-b', 'treatment-b'], + seed: 202, experiment: { id: 'exp_b', base: ['session', 'key'], weights: [1, 0], defaultVariant: 0, - seed: 202, }, }, }; diff --git a/packages/vercel-flags-core/src/evaluate.test.ts b/packages/vercel-flags-core/src/evaluate.test.ts index 1aa51e4f..cfaa82c4 100644 --- a/packages/vercel-flags-core/src/evaluate.test.ts +++ b/packages/vercel-flags-core/src/evaluate.test.ts @@ -2715,12 +2715,12 @@ describe('experiment outcomes', () => { }, variants: ['control', 'treatment'], variantIds: ['flag-control', 'flag-treatment'], + seed: 123, experiment: { id: 'exp_checkout', base: ['user', 'key'], weights: [0, 1], defaultVariant: 0, - seed: 123, rampId: 'ramp_1', rampPercentage: 25, }, diff --git a/packages/vercel-flags-core/src/evaluate.ts b/packages/vercel-flags-core/src/evaluate.ts index ad6c1f5c..051f1796 100644 --- a/packages/vercel-flags-core/src/evaluate.ts +++ b/packages/vercel-flags-core/src/evaluate.ts @@ -477,7 +477,7 @@ function handleOutcome( const index = getWeightedVariantIndex( params, experiment, - experiment.seed, + params.definition.seed, ); const variant = getVariant(params.definition, index); if (typeof variant.variantId !== 'string') { diff --git a/packages/vercel-flags-core/src/types.ts b/packages/vercel-flags-core/src/types.ts index 88ab3ecf..a803e51f 100644 --- a/packages/vercel-flags-core/src/types.ts +++ b/packages/vercel-flags-core/src/types.ts @@ -607,8 +607,6 @@ export namespace Original { weights: Record; /** Flag variant used when the base attribute does not exist. */ defaultVariantId: VariantId; - /** Seed used to keep experiment assignment stable and independent. */ - seed: number; rampId?: string; /** Percentage from 0 through 100. */ rampPercentage?: number; @@ -852,8 +850,6 @@ export namespace Packed { weights: number[]; /** Flag variant used when the base attribute does not exist. */ defaultVariant: VariantIndex; - /** Seed used to keep experiment assignment stable and independent. */ - seed: number; /** Identifier of the ramp active for this experiment. */ rampId?: string; /** Percentage of eligible units included in the ramp, from 0 through 100. */ From 21e62e54c2dbffb2099dc3128ba993906c24ea58 Mon Sep 17 00:00:00 2001 From: Dominik Ferber Date: Sun, 16 Aug 2026 15:55:24 +0300 Subject: [PATCH 05/13] exposures --- .../vercel-flags-core/src/black-box.test.ts | 31 +++++++++++++++++++ .../src/create-raw-client.ts | 2 +- .../vercel-flags-core/src/evaluate.test.ts | 31 +++++++++++++------ packages/vercel-flags-core/src/evaluate.ts | 18 +++++++++++ packages/vercel-flags-core/src/types.ts | 6 ++++ 5 files changed, 78 insertions(+), 10 deletions(-) diff --git a/packages/vercel-flags-core/src/black-box.test.ts b/packages/vercel-flags-core/src/black-box.test.ts index 8db782b2..a2ad43e3 100644 --- a/packages/vercel-flags-core/src/black-box.test.ts +++ b/packages/vercel-flags-core/src/black-box.test.ts @@ -3706,6 +3706,7 @@ describe('Controller (black-box)', () => { base: ['user', 'key'], weights: [0, 1], defaultVariant: 0, + exposureLogging: true, rampId: 'ramp_a', rampPercentage: 50, }, @@ -3724,6 +3725,7 @@ describe('Controller (black-box)', () => { base: ['session', 'key'], weights: [1, 0], defaultVariant: 0, + exposureLogging: true, }, }, }; @@ -3753,6 +3755,7 @@ describe('Controller (black-box)', () => { id: 'exp_a', variantId: 'treatment-a', base: ['user', 'key'], + exposureLogging: true, rampId: 'ramp_a', rampPercentage: 50, }, @@ -3795,6 +3798,34 @@ describe('Controller (black-box)', () => { await client.shutdown(); }); + it('does not report exposures when the experiment lifecycle disables them', async () => { + const reportExposures = vi.fn(); + const client = createClient(sdkKey, { + fetch: fetchMock, + stream: false, + polling: false, + buildStep: true, + datafile: makeBundled({ + definitions: { + flagA: { + ...definitions.flagA!, + experiment: { + ...definitions.flagA!.experiment!, + exposureLogging: false, + }, + }, + }, + }), + reportExposures, + }); + + const result = await client.evaluate('flagA', undefined, entity); + + expect(result.experiment?.exposureLogging).toBe(false); + expect(reportExposures).not.toHaveBeenCalled(); + await client.shutdown(); + }); + it('reports all bulk exposures in one callback', async () => { const reportExposures = vi.fn(); const client = createClient(sdkKey, { diff --git a/packages/vercel-flags-core/src/create-raw-client.ts b/packages/vercel-flags-core/src/create-raw-client.ts index d9440145..5e01a5e6 100644 --- a/packages/vercel-flags-core/src/create-raw-client.ts +++ b/packages/vercel-flags-core/src/create-raw-client.ts @@ -85,7 +85,7 @@ export function createCreateRawClient(fns: { flagKey: string, result: EvaluationResult, ): Exposure | null { - if (!result.experiment) return null; + if (!result.experiment?.exposureLogging) return null; return { flagKey, experimentId: result.experiment.id, diff --git a/packages/vercel-flags-core/src/evaluate.test.ts b/packages/vercel-flags-core/src/evaluate.test.ts index cfaa82c4..8cd4b57d 100644 --- a/packages/vercel-flags-core/src/evaluate.test.ts +++ b/packages/vercel-flags-core/src/evaluate.test.ts @@ -2721,8 +2721,9 @@ describe('experiment outcomes', () => { base: ['user', 'key'], weights: [0, 1], defaultVariant: 0, + exposureLogging: true, rampId: 'ramp_1', - rampPercentage: 25, + rampPercentage: 100, }, } satisfies Packed.FlagDefinition; @@ -2742,8 +2743,9 @@ describe('experiment outcomes', () => { id: 'exp_checkout', variantId: 'flag-treatment', base: ['user', 'key'], + exposureLogging: true, rampId: 'ramp_1', - rampPercentage: 25, + rampPercentage: 100, }, }); }); @@ -2760,13 +2762,24 @@ describe('experiment outcomes', () => { variantId: 'flag-control', reason: ResolutionReason.RULE_MATCH, outcomeType: OutcomeType.EXPERIMENT, - experiment: { - id: 'exp_checkout', - variantId: 'flag-control', - base: ['user', 'key'], - rampId: 'ramp_1', - rampPercentage: 25, - }, + }); + }); + + it('uses control without an assignment outside the experiment ramp', () => { + expect( + evaluate({ + definition: { + ...definition, + experiment: { ...definition.experiment, rampPercentage: 0 }, + }, + environment: 'production', + entities: { user: { key: 'user_123', country: 'DE' } }, + }), + ).toEqual({ + value: 'control', + variantId: 'flag-control', + reason: ResolutionReason.RULE_MATCH, + outcomeType: OutcomeType.EXPERIMENT, }); }); diff --git a/packages/vercel-flags-core/src/evaluate.ts b/packages/vercel-flags-core/src/evaluate.ts index 051f1796..f6a528e8 100644 --- a/packages/vercel-flags-core/src/evaluate.ts +++ b/packages/vercel-flags-core/src/evaluate.ts @@ -474,6 +474,23 @@ function handleOutcome( throw new Error('@vercel/flags-core: Experiment not found'); } + const unitValue = access(experiment.base, params); + if (typeof unitValue !== 'string') { + return { + ...getVariant(params.definition, experiment.defaultVariant), + outcomeType: OutcomeType.EXPERIMENT, + }; + } + + const rampPercentage = experiment.rampPercentage ?? 100; + const rampSeed = ((params.definition.seed ?? 0) ^ 0x9e3779b9) >>> 0; + if (hashInput(unitValue, rampSeed) >= boundaryFor(rampPercentage, 100)) { + return { + ...getVariant(params.definition, experiment.defaultVariant), + outcomeType: OutcomeType.EXPERIMENT, + }; + } + const index = getWeightedVariantIndex( params, experiment, @@ -493,6 +510,7 @@ function handleOutcome( id: experiment.id, variantId: variant.variantId, base: experiment.base, + exposureLogging: experiment.exposureLogging, rampId: experiment.rampId, rampPercentage: experiment.rampPercentage, }, diff --git a/packages/vercel-flags-core/src/types.ts b/packages/vercel-flags-core/src/types.ts index a803e51f..2d3557aa 100644 --- a/packages/vercel-flags-core/src/types.ts +++ b/packages/vercel-flags-core/src/types.ts @@ -140,6 +140,8 @@ export type ExperimentAssignment = { variantId: string; /** Entity path on which the experiment assignment is based. */ base: Packed.EntityAccessor; + /** Whether this assignment can produce an exposure report. */ + exposureLogging: boolean; /** Identifier of the ramp active for this assignment. */ rampId?: string; /** Percentage of eligible units included in the ramp, from 0 through 100. */ @@ -607,6 +609,8 @@ export namespace Original { weights: Record; /** Flag variant used when the base attribute does not exist. */ defaultVariantId: VariantId; + /** Whether evaluations assigned by this experiment report exposures. */ + exposureLogging: boolean; rampId?: string; /** Percentage from 0 through 100. */ rampPercentage?: number; @@ -850,6 +854,8 @@ export namespace Packed { weights: number[]; /** Flag variant used when the base attribute does not exist. */ defaultVariant: VariantIndex; + /** Whether evaluations assigned by this experiment report exposures. */ + exposureLogging: boolean; /** Identifier of the ramp active for this experiment. */ rampId?: string; /** Percentage of eligible units included in the ramp, from 0 through 100. */ From 279b91940e8e128616ce0a045694eb8248aaf48a Mon Sep 17 00:00:00 2001 From: Dominik Ferber Date: Sun, 16 Aug 2026 16:00:17 +0300 Subject: [PATCH 06/13] undo exposureLogging boolean --- .../vercel-flags-core/src/black-box.test.ts | 31 ------------------- .../src/create-raw-client.ts | 2 +- .../vercel-flags-core/src/evaluate.test.ts | 2 -- packages/vercel-flags-core/src/evaluate.ts | 1 - packages/vercel-flags-core/src/types.ts | 6 ---- 5 files changed, 1 insertion(+), 41 deletions(-) diff --git a/packages/vercel-flags-core/src/black-box.test.ts b/packages/vercel-flags-core/src/black-box.test.ts index a2ad43e3..8db782b2 100644 --- a/packages/vercel-flags-core/src/black-box.test.ts +++ b/packages/vercel-flags-core/src/black-box.test.ts @@ -3706,7 +3706,6 @@ describe('Controller (black-box)', () => { base: ['user', 'key'], weights: [0, 1], defaultVariant: 0, - exposureLogging: true, rampId: 'ramp_a', rampPercentage: 50, }, @@ -3725,7 +3724,6 @@ describe('Controller (black-box)', () => { base: ['session', 'key'], weights: [1, 0], defaultVariant: 0, - exposureLogging: true, }, }, }; @@ -3755,7 +3753,6 @@ describe('Controller (black-box)', () => { id: 'exp_a', variantId: 'treatment-a', base: ['user', 'key'], - exposureLogging: true, rampId: 'ramp_a', rampPercentage: 50, }, @@ -3798,34 +3795,6 @@ describe('Controller (black-box)', () => { await client.shutdown(); }); - it('does not report exposures when the experiment lifecycle disables them', async () => { - const reportExposures = vi.fn(); - const client = createClient(sdkKey, { - fetch: fetchMock, - stream: false, - polling: false, - buildStep: true, - datafile: makeBundled({ - definitions: { - flagA: { - ...definitions.flagA!, - experiment: { - ...definitions.flagA!.experiment!, - exposureLogging: false, - }, - }, - }, - }), - reportExposures, - }); - - const result = await client.evaluate('flagA', undefined, entity); - - expect(result.experiment?.exposureLogging).toBe(false); - expect(reportExposures).not.toHaveBeenCalled(); - await client.shutdown(); - }); - it('reports all bulk exposures in one callback', async () => { const reportExposures = vi.fn(); const client = createClient(sdkKey, { diff --git a/packages/vercel-flags-core/src/create-raw-client.ts b/packages/vercel-flags-core/src/create-raw-client.ts index 5e01a5e6..d9440145 100644 --- a/packages/vercel-flags-core/src/create-raw-client.ts +++ b/packages/vercel-flags-core/src/create-raw-client.ts @@ -85,7 +85,7 @@ export function createCreateRawClient(fns: { flagKey: string, result: EvaluationResult, ): Exposure | null { - if (!result.experiment?.exposureLogging) return null; + if (!result.experiment) return null; return { flagKey, experimentId: result.experiment.id, diff --git a/packages/vercel-flags-core/src/evaluate.test.ts b/packages/vercel-flags-core/src/evaluate.test.ts index 8cd4b57d..880edc8f 100644 --- a/packages/vercel-flags-core/src/evaluate.test.ts +++ b/packages/vercel-flags-core/src/evaluate.test.ts @@ -2721,7 +2721,6 @@ describe('experiment outcomes', () => { base: ['user', 'key'], weights: [0, 1], defaultVariant: 0, - exposureLogging: true, rampId: 'ramp_1', rampPercentage: 100, }, @@ -2743,7 +2742,6 @@ describe('experiment outcomes', () => { id: 'exp_checkout', variantId: 'flag-treatment', base: ['user', 'key'], - exposureLogging: true, rampId: 'ramp_1', rampPercentage: 100, }, diff --git a/packages/vercel-flags-core/src/evaluate.ts b/packages/vercel-flags-core/src/evaluate.ts index f6a528e8..0a4ee3cd 100644 --- a/packages/vercel-flags-core/src/evaluate.ts +++ b/packages/vercel-flags-core/src/evaluate.ts @@ -510,7 +510,6 @@ function handleOutcome( id: experiment.id, variantId: variant.variantId, base: experiment.base, - exposureLogging: experiment.exposureLogging, rampId: experiment.rampId, rampPercentage: experiment.rampPercentage, }, diff --git a/packages/vercel-flags-core/src/types.ts b/packages/vercel-flags-core/src/types.ts index 2d3557aa..a803e51f 100644 --- a/packages/vercel-flags-core/src/types.ts +++ b/packages/vercel-flags-core/src/types.ts @@ -140,8 +140,6 @@ export type ExperimentAssignment = { variantId: string; /** Entity path on which the experiment assignment is based. */ base: Packed.EntityAccessor; - /** Whether this assignment can produce an exposure report. */ - exposureLogging: boolean; /** Identifier of the ramp active for this assignment. */ rampId?: string; /** Percentage of eligible units included in the ramp, from 0 through 100. */ @@ -609,8 +607,6 @@ export namespace Original { weights: Record; /** Flag variant used when the base attribute does not exist. */ defaultVariantId: VariantId; - /** Whether evaluations assigned by this experiment report exposures. */ - exposureLogging: boolean; rampId?: string; /** Percentage from 0 through 100. */ rampPercentage?: number; @@ -854,8 +850,6 @@ export namespace Packed { weights: number[]; /** Flag variant used when the base attribute does not exist. */ defaultVariant: VariantIndex; - /** Whether evaluations assigned by this experiment report exposures. */ - exposureLogging: boolean; /** Identifier of the ramp active for this experiment. */ rampId?: string; /** Percentage of eligible units included in the ramp, from 0 through 100. */ From 6f06dfd542fbe7eda17791dc0af49b7a12b311d6 Mon Sep 17 00:00:00 2001 From: Dominik Ferber Date: Thu, 20 Aug 2026 08:19:31 +0300 Subject: [PATCH 07/13] Attach experiment metadata to all evaluated flag outcomes --- .changeset/bright-experiments-report.md | 2 +- packages/vercel-flags-core/README.md | 6 +- .../vercel-flags-core/src/black-box.test.ts | 15 ++-- .../vercel-flags-core/src/evaluate.test.ts | 63 ++++++++-------- packages/vercel-flags-core/src/evaluate.ts | 75 +++++++------------ packages/vercel-flags-core/src/types.ts | 28 ++----- 6 files changed, 74 insertions(+), 115 deletions(-) diff --git a/.changeset/bright-experiments-report.md b/.changeset/bright-experiments-report.md index fc7169fd..bad23bd1 100644 --- a/.changeset/bright-experiments-report.md +++ b/.changeset/bright-experiments-report.md @@ -2,4 +2,4 @@ '@vercel/flags-core': minor --- -Add experiment outcomes, exposure reporting, and per-evaluation exposure logging controls. +Add flag-level experiment exposure reporting and per-evaluation exposure logging controls. diff --git a/packages/vercel-flags-core/README.md b/packages/vercel-flags-core/README.md index 89b97f38..e4bba588 100644 --- a/packages/vercel-flags-core/README.md +++ b/packages/vercel-flags-core/README.md @@ -26,8 +26,10 @@ const result = await client.evaluate('show-new-feature', false, { ## Experiment exposures -Experiment-backed flag evaluations report exposures automatically. Provide a -custom reporter to send them to your analytics system: +Flags linked to an experiment report exposures automatically, regardless of +whether the evaluated value came from a fixed variant, target, split, rollout, +or fallthrough. Provide a custom reporter to send them to your analytics +system: ```ts const client = createClient(process.env.FLAGS!, { diff --git a/packages/vercel-flags-core/src/black-box.test.ts b/packages/vercel-flags-core/src/black-box.test.ts index 8db782b2..a6a43f15 100644 --- a/packages/vercel-flags-core/src/black-box.test.ts +++ b/packages/vercel-flags-core/src/black-box.test.ts @@ -3695,7 +3695,12 @@ describe('Controller (black-box)', () => { flagA: { environments: { production: { - fallthrough: { type: 'experiment' }, + fallthrough: { + type: 'split', + base: ['user', 'key'], + weights: [0, 1], + defaultVariant: 0, + }, }, }, variants: ['control-a', 'treatment-a'], @@ -3704,8 +3709,6 @@ describe('Controller (black-box)', () => { experiment: { id: 'exp_a', base: ['user', 'key'], - weights: [0, 1], - defaultVariant: 0, rampId: 'ramp_a', rampPercentage: 50, }, @@ -3713,7 +3716,7 @@ describe('Controller (black-box)', () => { flagB: { environments: { production: { - fallthrough: { type: 'experiment' }, + fallthrough: 0, }, }, variants: ['control-b', 'treatment-b'], @@ -3722,8 +3725,6 @@ describe('Controller (black-box)', () => { experiment: { id: 'exp_b', base: ['session', 'key'], - weights: [1, 0], - defaultVariant: 0, }, }, }; @@ -3748,7 +3749,7 @@ describe('Controller (black-box)', () => { expect(result).toMatchObject({ value: 'treatment-a', - outcomeType: 'experiment', + outcomeType: 'split', experiment: { id: 'exp_a', variantId: 'treatment-a', diff --git a/packages/vercel-flags-core/src/evaluate.test.ts b/packages/vercel-flags-core/src/evaluate.test.ts index 880edc8f..706fd5e6 100644 --- a/packages/vercel-flags-core/src/evaluate.test.ts +++ b/packages/vercel-flags-core/src/evaluate.test.ts @@ -2700,14 +2700,19 @@ describe('evaluate', () => { }); }); -describe('experiment outcomes', () => { +describe('experiment metadata', () => { const definition = { environments: { production: { rules: [ { conditions: [[['user', 'country'], Comparator.EQ, 'DE']], - outcome: { type: 'experiment' }, + outcome: { + type: 'split', + base: ['user', 'key'], + weights: [0, 1], + defaultVariant: 0, + }, }, ], fallthrough: 0, @@ -2719,14 +2724,12 @@ describe('experiment outcomes', () => { experiment: { id: 'exp_checkout', base: ['user', 'key'], - weights: [0, 1], - defaultVariant: 0, rampId: 'ramp_1', rampPercentage: 100, }, } satisfies Packed.FlagDefinition; - it('evaluates an experiment referenced by a rule', () => { + it('adds experiment metadata to a split outcome', () => { expect( evaluate({ definition, @@ -2737,7 +2740,7 @@ describe('experiment outcomes', () => { value: 'treatment', variantId: 'flag-treatment', reason: ResolutionReason.RULE_MATCH, - outcomeType: OutcomeType.EXPERIMENT, + outcomeType: OutcomeType.SPLIT, experiment: { id: 'exp_checkout', variantId: 'flag-treatment', @@ -2748,7 +2751,7 @@ describe('experiment outcomes', () => { }); }); - it('uses the experiment default variant when its base is missing', () => { + it('adds experiment metadata when a split uses its default variant', () => { expect( evaluate({ definition, @@ -2759,44 +2762,38 @@ describe('experiment outcomes', () => { value: 'control', variantId: 'flag-control', reason: ResolutionReason.RULE_MATCH, - outcomeType: OutcomeType.EXPERIMENT, + outcomeType: OutcomeType.SPLIT, + experiment: { + id: 'exp_checkout', + variantId: 'flag-control', + base: ['user', 'key'], + rampId: 'ramp_1', + rampPercentage: 100, + }, }); }); - it('uses control without an assignment outside the experiment ramp', () => { + it('adds experiment metadata to a non-split outcome', () => { expect( evaluate({ - definition: { - ...definition, - experiment: { ...definition.experiment, rampPercentage: 0 }, - }, + definition, environment: 'production', - entities: { user: { key: 'user_123', country: 'DE' } }, + entities: { user: { key: 'user_123', country: 'US' } }, }), ).toEqual({ value: 'control', variantId: 'flag-control', - reason: ResolutionReason.RULE_MATCH, - outcomeType: OutcomeType.EXPERIMENT, + reason: ResolutionReason.FALLTHROUGH, + outcomeType: OutcomeType.VALUE, + experiment: { + id: 'exp_checkout', + variantId: 'flag-control', + base: ['user', 'key'], + rampId: 'ramp_1', + rampPercentage: 100, + }, }); }); - - it('throws for an invalid experiment reference', () => { - expect(() => - evaluate({ - definition: { - environments: { - production: { - fallthrough: { type: 'experiment' }, - }, - }, - variants: [false], - }, - environment: 'production', - entities: {}, - }), - ).toThrow('@vercel/flags-core: Experiment not found'); - }); }); describe('bulkEvaluate', () => { diff --git a/packages/vercel-flags-core/src/evaluate.ts b/packages/vercel-flags-core/src/evaluate.ts index 0a4ee3cd..23086cd9 100644 --- a/packages/vercel-flags-core/src/evaluate.ts +++ b/packages/vercel-flags-core/src/evaluate.ts @@ -440,14 +440,13 @@ function getWeightedVariantIndex( return assignment.defaultVariant; } -function handleOutcome( +function resolveOutcome( params: EvaluationParams, outcome: Packed.Outcome, ): { value: T; outcomeType: OutcomeType; variantId: VariantId | null; - experiment?: ExperimentAssignment; } { if (typeof outcome === 'number') { const variant = getVariant(params.definition, outcome); @@ -468,53 +467,6 @@ function handleOutcome( outcomeType: OutcomeType.SPLIT, }; } - case 'experiment': { - const experiment = params.definition.experiment; - if (!experiment) { - throw new Error('@vercel/flags-core: Experiment not found'); - } - - const unitValue = access(experiment.base, params); - if (typeof unitValue !== 'string') { - return { - ...getVariant(params.definition, experiment.defaultVariant), - outcomeType: OutcomeType.EXPERIMENT, - }; - } - - const rampPercentage = experiment.rampPercentage ?? 100; - const rampSeed = ((params.definition.seed ?? 0) ^ 0x9e3779b9) >>> 0; - if (hashInput(unitValue, rampSeed) >= boundaryFor(rampPercentage, 100)) { - return { - ...getVariant(params.definition, experiment.defaultVariant), - outcomeType: OutcomeType.EXPERIMENT, - }; - } - - const index = getWeightedVariantIndex( - params, - experiment, - params.definition.seed, - ); - const variant = getVariant(params.definition, index); - if (typeof variant.variantId !== 'string') { - throw new Error( - `@vercel/flags-core: Flag variant ID not found at index ${index} for experiment "${experiment.id}"`, - ); - } - - return { - ...variant, - outcomeType: OutcomeType.EXPERIMENT, - experiment: { - id: experiment.id, - variantId: variant.variantId, - base: experiment.base, - rampId: experiment.rampId, - rampPercentage: experiment.rampPercentage, - }, - }; - } case 'rollout': { const lhs = access(outcome.base, params); const defaultOutcome = getVariant( @@ -608,6 +560,31 @@ function handleOutcome( } } +function handleOutcome( + params: EvaluationParams, + outcome: Packed.Outcome, +): { + value: T; + outcomeType: OutcomeType; + variantId: VariantId | null; + experiment?: ExperimentAssignment; +} { + const result = resolveOutcome(params, outcome); + const experiment = params.definition.experiment; + if (!experiment || result.variantId === null) return result; + + return { + ...result, + experiment: { + id: experiment.id, + variantId: result.variantId, + base: experiment.base, + rampId: experiment.rampId, + rampPercentage: experiment.rampPercentage, + }, + }; +} + /** * Evaluates a single feature flag. * diff --git a/packages/vercel-flags-core/src/types.ts b/packages/vercel-flags-core/src/types.ts index a803e51f..3f9db94e 100644 --- a/packages/vercel-flags-core/src/types.ts +++ b/packages/vercel-flags-core/src/types.ts @@ -132,7 +132,7 @@ export type EvaluationOptions = { exposureLogging?: boolean; }; -/** Information about the experiment assignment that produced a flag value. */ +/** Information about the experiment linked to an evaluated flag value. */ export type ExperimentAssignment = { /** Experiment identifier. */ id: string; @@ -299,7 +299,7 @@ export type EvaluationResult = * The variant we want to report for o11y */ variantId: VariantId | null; - /** Experiment assignment when an experiment outcome produced the value. */ + /** Experiment metadata when the flag is linked to an experiment. */ experiment?: ExperimentAssignment; /** * Indicates why the flag evaluated to a certain value @@ -359,8 +359,6 @@ export enum OutcomeType { SPLIT = 'split', /** When the outcome type was a progressive rollout */ ROLLOUT = 'rollout', - /** When the outcome type was an experiment assignment */ - EXPERIMENT = 'experiment', } /** @@ -594,14 +592,11 @@ export namespace Original { * Once all slots are exhausted, the rollout is complete (100% rollToVariant). */ slots: { promille: number; durationMs: number }[]; - } - | { - type: 'experiment'; }; export type ExperimentDefinition = { id: string; - /** Based on which entity attribute traffic should be assigned. */ + /** Entity attribute used as the experiment unit. */ base: EntityAccessor; /** Distribution keyed by flag variant ID. */ weights: Record; @@ -836,20 +831,11 @@ export namespace Packed { slots: [number, number][]; }; - /** An outcome which delegates assignment to a flag-level experiment. */ - export type ExperimentOutcome = { - type: 'experiment'; - }; - export type ExperimentDefinition = { /** Experiment identifier. */ id: string; - /** Entity path used for deterministic assignment. */ + /** Entity path used as the experiment unit. */ base: EntityAccessor; - /** Distribution indexed by the corresponding flag variant. */ - weights: number[]; - /** Flag variant used when the base attribute does not exist. */ - defaultVariant: VariantIndex; /** Identifier of the ramp active for this experiment. */ rampId?: string; /** Percentage of eligible units included in the ramp, from 0 through 100. */ @@ -874,11 +860,7 @@ export namespace Packed { export type SegmentOutcome = SegmentAllOutcome | SegmentSplitOutcome; - export type Outcome = - | VariantIndex - | SplitOutcome - | RolloutOutcome - | ExperimentOutcome; + export type Outcome = VariantIndex | SplitOutcome | RolloutOutcome; // an array means it's an entity, the string "segment" means a segment export type EntityAccessor = (string | number)[]; From bb14f27f0c5ef45f184f5d1130a3e216e01c3348 Mon Sep 17 00:00:00 2001 From: Dominik Ferber Date: Thu, 20 Aug 2026 08:45:16 +0300 Subject: [PATCH 08/13] version --- packages/vercel-flags-core/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/vercel-flags-core/package.json b/packages/vercel-flags-core/package.json index 616f244d..e07dc928 100644 --- a/packages/vercel-flags-core/package.json +++ b/packages/vercel-flags-core/package.json @@ -1,6 +1,6 @@ { "name": "@vercel/flags-core", - "version": "1.7.1", + "version": "1.7.1-engulf.0", "description": "A server-side client for Vercel Flags", "keywords": [ "vercel", From 3ea6ea935fa100d653f24b1ae4c093eb1a3f710f Mon Sep 17 00:00:00 2001 From: Dominik Ferber Date: Fri, 21 Aug 2026 11:50:29 +0300 Subject: [PATCH 09/13] hybrid and override exposure reporting --- .changeset/bright-experiments-report.md | 4 +- packages/adapter-vercel/src/index.test.ts | 22 +++ packages/adapter-vercel/src/index.ts | 3 + packages/flags/src/next/evaluate.ts | 23 ++- packages/flags/src/next/index.test.ts | 31 +++- packages/flags/src/types.ts | 6 + .../vercel-flags-core/src/black-box.test.ts | 51 ++++++- .../src/create-raw-client.ts | 49 +++++++ .../vercel-flags-core/src/evaluate.test.ts | 136 ++++++++++++++++-- packages/vercel-flags-core/src/evaluate.ts | 96 +++++++++++-- .../src/exposure-reporting.test.ts | 9 ++ .../src/exposure-reporting.ts | 4 +- packages/vercel-flags-core/src/types.ts | 42 +++++- 13 files changed, 442 insertions(+), 34 deletions(-) diff --git a/.changeset/bright-experiments-report.md b/.changeset/bright-experiments-report.md index bad23bd1..83a8a695 100644 --- a/.changeset/bright-experiments-report.md +++ b/.changeset/bright-experiments-report.md @@ -1,5 +1,7 @@ --- '@vercel/flags-core': minor +'@flags-sdk/vercel': minor +'flags': minor --- -Add flag-level experiment exposure reporting and per-evaluation exposure logging controls. +Add randomized experiment enrollment, assignment reasons for every experiment-managed flag outcome, cookie override exposure reporting, and per-evaluation exposure logging controls. diff --git a/packages/adapter-vercel/src/index.test.ts b/packages/adapter-vercel/src/index.test.ts index 9920b7fc..b36f05ba 100644 --- a/packages/adapter-vercel/src/index.test.ts +++ b/packages/adapter-vercel/src/index.test.ts @@ -104,6 +104,28 @@ describe('createVercelAdapter', () => { } satisfies Origin); }); + it('forwards override observations to the flags client', async () => { + const reportOverride = vi.fn(); + const fakeClient = { + origin: { provider: 'vercel', sdkKey: 'vf_x' }, + reportOverride, + } as unknown as typeof flagsClient; + const adapter = createVercelAdapter(fakeClient)(); + const entities = { user: { key: 'user_1' } }; + + await adapter.reportOverride?.({ + key: 'checkout', + value: 'treatment', + entities, + }); + + expect(reportOverride).toHaveBeenCalledWith( + 'checkout', + 'treatment', + entities, + ); + }); + it('has correct types', () => { const adapter = createVercelAdapter(flagsClient); type SampleValue = boolean; diff --git a/packages/adapter-vercel/src/index.ts b/packages/adapter-vercel/src/index.ts index 6c4ef850..41865bf3 100644 --- a/packages/adapter-vercel/src/index.ts +++ b/packages/adapter-vercel/src/index.ts @@ -40,6 +40,9 @@ export function createVercelAdapter( adapterId, origin: flagsClient.origin, config: { reportValue: false }, + async reportOverride({ key, value, entities }) { + await flagsClient.reportOverride(key, value, entities); + }, async decide({ key, entities }) { const evaluationResult = await flagsClient.evaluate( key, diff --git a/packages/flags/src/next/evaluate.ts b/packages/flags/src/next/evaluate.ts index 23ced1a4..226c90b6 100644 --- a/packages/flags/src/next/evaluate.ts +++ b/packages/flags/src/next/evaluate.ts @@ -197,7 +197,7 @@ type FlagInfo = { key: string; defaultValue?: ValueType; config?: { reportValue?: boolean }; - adapter?: { config?: { reportValue?: boolean } }; + adapter?: Pick, 'config' | 'reportOverride'>; }; function hasOverride( @@ -227,10 +227,18 @@ async function applyResult(args: { definition: FlagInfo; readonlyHeaders: ReadonlyHeaders; entitiesKey: string; + entities?: unknown; overrides: Record | null; produce: () => ValueType | PromiseLike; }): Promise { - const { definition, readonlyHeaders, entitiesKey, overrides, produce } = args; + const { + definition, + readonlyHeaders, + entitiesKey, + entities, + overrides, + produce, + } = args; const cachedValue = getCachedValuePromise( readonlyHeaders, @@ -254,6 +262,15 @@ async function applyResult(args: { internalReportValue(definition.key, decision, { reason: 'override', }); + try { + await definition.adapter?.reportOverride?.({ + key: definition.key, + value: decision, + entities, + }); + } catch (error) { + console.error('flags: Failed to report flag override', error); + } return decision; } @@ -401,6 +418,7 @@ export function getRun( definition, readonlyHeaders, entitiesKey, + entities, overrides, produce: () => decide({ @@ -641,6 +659,7 @@ async function evaluateImpl( definition: flagFn, readonlyHeaders, entitiesKey, + entities, overrides, produce: () => { if (bulkError) throw bulkError; diff --git a/packages/flags/src/next/index.test.ts b/packages/flags/src/next/index.test.ts index 120ac5d7..4c9293bb 100644 --- a/packages/flags/src/next/index.test.ts +++ b/packages/flags/src/next/index.test.ts @@ -191,7 +191,16 @@ describe('flag on app router', () => { it('respects overrides', async () => { const decide = vi.fn(() => false); - const f = flag({ key: 'first-flag', decide }); + const reportOverride = vi.fn(); + const entities = { user: { id: 'user_1' } }; + const f = flag({ + key: 'first-flag', + identify: () => entities, + adapter: { + decide, + reportOverride, + }, + }); // first request using the flag twice const headersOfFirstRequest = new Headers(); @@ -207,6 +216,11 @@ describe('flag on app router', () => { await expect(f()).resolves.toEqual(true); expect(cookieMock).toHaveBeenCalledWith('vercel-flag-overrides'); expect(decide).not.toHaveBeenCalled(); + expect(reportOverride).toHaveBeenCalledWith({ + key: 'first-flag', + value: true, + entities, + }); }); it('does not crash when override reporting hook is not a function', async () => { @@ -879,6 +893,7 @@ describe('evaluate', () => { bulkDecide?: Adapter['bulkDecide']; decide?: Adapter['decide']; identify?: Adapter['identify']; + reportOverride?: Adapter['reportOverride']; omitAdapterId?: boolean; omitBulkDecide?: boolean; }) { @@ -892,6 +907,7 @@ describe('evaluate', () => { throw new Error('decide should not be called in bulk path'); }), identify: opts?.identify, + reportOverride: opts?.reportOverride, ...(opts?.omitBulkDecide ? {} : { bulkDecide: opts?.bulkDecide }), }); } @@ -1084,7 +1100,13 @@ describe('evaluate', () => { it('lets overrides win over bulkDecide results', async () => { const bulkDecideMock = vi.fn().mockResolvedValue({ a: 'bulk-value' }); - const adapter = makeBulkAdapter({ bulkDecide: bulkDecideMock }); + const reportOverride = vi.fn(); + const entities = { user: { id: 'user_1' } }; + const adapter = makeBulkAdapter({ + bulkDecide: bulkDecideMock, + identify: () => entities, + reportOverride, + }); const a = flag({ key: 'a', adapter: adapter() }); @@ -1099,6 +1121,11 @@ describe('evaluate', () => { await expect(evaluate({ a })).resolves.toEqual({ a: true }); expect(bulkDecideMock).not.toHaveBeenCalled(); + expect(reportOverride).toHaveBeenCalledWith({ + key: 'a', + value: true, + entities, + }); }); it('omits overridden flags from bulkDecide input', async () => { diff --git a/packages/flags/src/types.ts b/packages/flags/src/types.ts index dc9563d0..2b08f9bb 100644 --- a/packages/flags/src/types.ts +++ b/packages/flags/src/types.ts @@ -165,6 +165,12 @@ export interface Adapter { * an `adapterId` are never batched. */ adapterId?: string | symbol; + /** Observe a value supplied by the Flags SDK override cookie. */ + reportOverride?: (params: { + key: string; + value: unknown; + entities?: EntitiesType; + }) => void | Promise; decide: (params: { key: string; entities?: EntitiesType; diff --git a/packages/vercel-flags-core/src/black-box.test.ts b/packages/vercel-flags-core/src/black-box.test.ts index a6a43f15..57e93eb7 100644 --- a/packages/vercel-flags-core/src/black-box.test.ts +++ b/packages/vercel-flags-core/src/black-box.test.ts @@ -3696,10 +3696,7 @@ describe('Controller (black-box)', () => { environments: { production: { fallthrough: { - type: 'split', - base: ['user', 'key'], - weights: [0, 1], - defaultVariant: 0, + type: 'experiment', }, }, }, @@ -3709,6 +3706,9 @@ describe('Controller (black-box)', () => { experiment: { id: 'exp_a', base: ['user', 'key'], + weights: [0, 1], + defaultVariant: 0, + enrollmentSeed: 101, rampId: 'ramp_a', rampPercentage: 50, }, @@ -3716,7 +3716,7 @@ describe('Controller (black-box)', () => { flagB: { environments: { production: { - fallthrough: 0, + fallthrough: { type: 'experiment' }, }, }, variants: ['control-b', 'treatment-b'], @@ -3725,6 +3725,9 @@ describe('Controller (black-box)', () => { experiment: { id: 'exp_b', base: ['session', 'key'], + weights: [1, 0], + defaultVariant: 0, + enrollmentSeed: 202, }, }, }; @@ -3749,13 +3752,14 @@ describe('Controller (black-box)', () => { expect(result).toMatchObject({ value: 'treatment-a', - outcomeType: 'split', + outcomeType: 'experiment', experiment: { id: 'exp_a', variantId: 'treatment-a', base: ['user', 'key'], rampId: 'ramp_a', rampPercentage: 50, + assignmentReason: 'experiment', }, }); expect(reportExposures).toHaveBeenCalledOnce(); @@ -3768,6 +3772,39 @@ describe('Controller (black-box)', () => { base: ['user', 'key'], rampId: 'ramp_a', rampPercentage: 50, + assignmentReason: 'experiment', + }, + ], + entity, + ); + + await client.shutdown(); + }); + + it('reports cookie overrides without evaluating the flag', async () => { + const reportExposures = vi.fn(); + const client = createClient(sdkKey, { + fetch: fetchMock, + stream: false, + polling: false, + buildStep: true, + datafile: makeBundled({ definitions }), + reportExposures, + }); + + await client.reportOverride('flagA', 'treatment-a', entity); + + expect(reportExposures).toHaveBeenCalledOnce(); + expect(reportExposures).toHaveBeenCalledWith( + [ + { + flagKey: 'flagA', + experimentId: 'exp_a', + variantId: 'treatment-a', + base: ['user', 'key'], + rampId: 'ramp_a', + rampPercentage: 50, + assignmentReason: 'override', }, ], entity, @@ -3819,12 +3856,14 @@ describe('Controller (black-box)', () => { base: ['user', 'key'], rampId: 'ramp_a', rampPercentage: 50, + assignmentReason: 'experiment', }, { flagKey: 'flagB', experimentId: 'exp_b', variantId: 'control-b', base: ['session', 'key'], + assignmentReason: 'experiment', }, ], entity, diff --git a/packages/vercel-flags-core/src/create-raw-client.ts b/packages/vercel-flags-core/src/create-raw-client.ts index d9440145..9058fec4 100644 --- a/packages/vercel-flags-core/src/create-raw-client.ts +++ b/packages/vercel-flags-core/src/create-raw-client.ts @@ -19,6 +19,7 @@ import type { EvaluationResult, Exposure, FlagsClient, + Packed, ReportExposures, Value, } from './types'; @@ -97,6 +98,7 @@ export function createCreateRawClient(fns: { ...(result.experiment.rampPercentage === undefined ? {} : { rampPercentage: result.experiment.rampPercentage }), + assignmentReason: result.experiment.assignmentReason, }; } @@ -199,6 +201,53 @@ export function createCreateRawClient(fns: { } return results; }, + reportOverride: async ( + flagKey: string, + value: T, + entities?: E, + ): Promise => { + try { + const instance = controllerInstanceMap.get(id); + if (!instance?.initialized) await api.initialize(); + const datafile = await fns.getDatafile(id); + const definition = datafile.definitions[ + flagKey + ] as Packed.FlagDefinition; + const experiment = definition?.experiment; + if (!experiment) return; + + const serializedValue = JSON.stringify(value); + const variantIndex = definition.variants.findIndex( + (variant) => + Object.is(variant, value) || + JSON.stringify(variant) === serializedValue, + ); + const variantId = + variantIndex < 0 + ? null + : (definition.variantIds?.[variantIndex] ?? null); + const entity = entities ?? ({} as E); + await report( + [ + { + flagKey, + experimentId: experiment.id, + variantId, + base: experiment.base, + rampId: experiment.rampId, + rampPercentage: experiment.rampPercentage, + assignmentReason: 'override', + }, + ], + entity as unknown as Readonly, + ); + } catch (error) { + console.error( + '@vercel/flags-core: Failed to report experiment override', + error, + ); + } + }, }; return api; }; diff --git a/packages/vercel-flags-core/src/evaluate.test.ts b/packages/vercel-flags-core/src/evaluate.test.ts index 706fd5e6..dc00b1e9 100644 --- a/packages/vercel-flags-core/src/evaluate.test.ts +++ b/packages/vercel-flags-core/src/evaluate.test.ts @@ -2707,12 +2707,7 @@ describe('experiment metadata', () => { rules: [ { conditions: [[['user', 'country'], Comparator.EQ, 'DE']], - outcome: { - type: 'split', - base: ['user', 'key'], - weights: [0, 1], - defaultVariant: 0, - }, + outcome: { type: 'experiment' }, }, ], fallthrough: 0, @@ -2724,12 +2719,15 @@ describe('experiment metadata', () => { experiment: { id: 'exp_checkout', base: ['user', 'key'], + weights: [0, 1], + defaultVariant: 0, + enrollmentSeed: 456, rampId: 'ramp_1', rampPercentage: 100, }, } satisfies Packed.FlagDefinition; - it('adds experiment metadata to a split outcome', () => { + it('randomizes an enrolled experiment outcome', () => { expect( evaluate({ definition, @@ -2740,18 +2738,19 @@ describe('experiment metadata', () => { value: 'treatment', variantId: 'flag-treatment', reason: ResolutionReason.RULE_MATCH, - outcomeType: OutcomeType.SPLIT, + outcomeType: OutcomeType.EXPERIMENT, experiment: { id: 'exp_checkout', variantId: 'flag-treatment', base: ['user', 'key'], rampId: 'ramp_1', rampPercentage: 100, + assignmentReason: 'experiment', }, }); }); - it('adds experiment metadata when a split uses its default variant', () => { + it('marks a missing experiment base as not enrolled', () => { expect( evaluate({ definition, @@ -2762,18 +2761,19 @@ describe('experiment metadata', () => { value: 'control', variantId: 'flag-control', reason: ResolutionReason.RULE_MATCH, - outcomeType: OutcomeType.SPLIT, + outcomeType: OutcomeType.EXPERIMENT, experiment: { id: 'exp_checkout', variantId: 'flag-control', base: ['user', 'key'], rampId: 'ramp_1', rampPercentage: 100, + assignmentReason: 'not-enrolled', }, }); }); - it('adds experiment metadata to a non-split outcome', () => { + it('marks a fixed outcome as a non-randomized variant exposure', () => { expect( evaluate({ definition, @@ -2791,8 +2791,122 @@ describe('experiment metadata', () => { base: ['user', 'key'], rampId: 'ramp_1', rampPercentage: 100, + assignmentReason: 'variant', + }, + }); + }); + + it('marks an ordinary split as a non-experiment split exposure', () => { + expect( + evaluate({ + definition: { + ...definition, + environments: { + production: { + fallthrough: { + type: 'split', + base: ['user', 'key'], + weights: [1, 0], + defaultVariant: 0, + }, + }, + }, + }, + environment: 'production', + entities: { user: { key: 'user_123' } }, + }), + ).toMatchObject({ + value: 'control', + outcomeType: OutcomeType.SPLIT, + experiment: { assignmentReason: 'split' }, + }); + }); + + it('marks direct targets as targeted exposures', () => { + expect( + evaluate({ + definition: { + ...definition, + environments: { + production: { + targets: [{ user: { key: ['user_123'] } }], + fallthrough: { type: 'experiment' }, + }, + }, + }, + environment: 'production', + entities: { user: { key: 'user_123' } }, + }), + ).toMatchObject({ + value: 'control', + experiment: { assignmentReason: 'targeted' }, + }); + }); + + it('preserves enrolled assignments as ramp percentage increases', () => { + const makeExperimentDefinition = ( + rampPercentage: number, + ): Packed.FlagDefinition => ({ + ...definition, + environments: { + production: { fallthrough: { type: 'experiment' } }, + }, + experiment: { + ...definition.experiment, + weights: [1, 1], + rampPercentage, }, }); + const splitDefinition: Packed.FlagDefinition = { + ...definition, + environments: { + production: { + fallthrough: { + type: 'split', + base: definition.experiment.base, + weights: [1, 1], + defaultVariant: 0, + }, + }, + }, + experiment: undefined, + }; + let enrolledAtTwenty = 0; + let newlyEnrolled = 0; + + for (let index = 0; index < 500; index++) { + const entities = { user: { key: `user_${index}` } }; + const atTwenty = evaluate({ + definition: makeExperimentDefinition(20), + environment: 'production', + entities, + }); + const atEighty = evaluate({ + definition: makeExperimentDefinition(80), + environment: 'production', + entities, + }); + + if (atTwenty.experiment?.assignmentReason === 'experiment') { + enrolledAtTwenty++; + expect(atEighty.experiment?.assignmentReason).toBe('experiment'); + expect(atEighty.variantId).toBe(atTwenty.variantId); + } else if (atEighty.experiment?.assignmentReason === 'experiment') { + newlyEnrolled++; + } + + if (atEighty.experiment?.assignmentReason === 'experiment') { + const withoutExperiment = evaluate({ + definition: splitDefinition, + environment: 'production', + entities, + }); + expect(atEighty.variantId).toBe(withoutExperiment.variantId); + } + } + + expect(enrolledAtTwenty).toBeGreaterThan(0); + expect(newlyEnrolled).toBeGreaterThan(0); }); }); diff --git a/packages/vercel-flags-core/src/evaluate.ts b/packages/vercel-flags-core/src/evaluate.ts index 23086cd9..4c8ca4b1 100644 --- a/packages/vercel-flags-core/src/evaluate.ts +++ b/packages/vercel-flags-core/src/evaluate.ts @@ -4,6 +4,7 @@ import { type EvaluationParams, type EvaluationResult, type ExperimentAssignment, + type ExperimentAssignmentReason, OutcomeType, Packed, ResolutionReason, @@ -440,6 +441,40 @@ function getWeightedVariantIndex( return assignment.defaultVariant; } +function experimentAssignment( + experiment: Packed.ExperimentDefinition, + variantId: VariantId | null, + assignmentReason: ExperimentAssignmentReason, +): ExperimentAssignment | undefined { + if (variantId === null) return undefined; + return { + id: experiment.id, + variantId, + base: experiment.base, + rampId: experiment.rampId, + rampPercentage: experiment.rampPercentage, + assignmentReason, + }; +} + +function outcomeAssignmentReason( + outcome: Packed.Outcome, +): ExperimentAssignmentReason { + if (typeof outcome === 'number') return 'variant'; + switch (outcome.type) { + case 'experiment': + return 'experiment'; + case 'split': + return 'split'; + case 'rollout': + return 'rollout'; + default: { + const { type } = outcome; + return exhaustivenessCheck(type); + } + } +} + function resolveOutcome( params: EvaluationParams, outcome: Packed.Outcome, @@ -447,6 +482,7 @@ function resolveOutcome( value: T; outcomeType: OutcomeType; variantId: VariantId | null; + experiment?: ExperimentAssignment; } { if (typeof outcome === 'number') { const variant = getVariant(params.definition, outcome); @@ -467,6 +503,49 @@ function resolveOutcome( outcomeType: OutcomeType.SPLIT, }; } + case 'experiment': { + const experiment = params.definition.experiment; + if (!experiment) { + throw new Error('@vercel/flags-core: Experiment not found'); + } + + const unitValue = access(experiment.base, params); + const defaultVariant = getVariant( + params.definition, + experiment.defaultVariant, + ); + const assignment = ( + variant: typeof defaultVariant, + assignmentReason: ExperimentAssignmentReason, + ) => ({ + ...variant, + outcomeType: OutcomeType.EXPERIMENT, + experiment: experimentAssignment( + experiment, + variant.variantId, + assignmentReason, + ), + }); + + if (typeof unitValue !== 'string') { + return assignment(defaultVariant, 'not-enrolled'); + } + + const rampPercentage = experiment.rampPercentage ?? 100; + const enrolled = + rampPercentage >= 100 || + (rampPercentage > 0 && + hashInput(unitValue, experiment.enrollmentSeed) < + boundaryFor(rampPercentage, 100)); + if (!enrolled) return assignment(defaultVariant, 'not-enrolled'); + + const index = getWeightedVariantIndex( + params, + experiment, + params.definition.seed, + ); + return assignment(getVariant(params.definition, index), 'experiment'); + } case 'rollout': { const lhs = access(outcome.base, params); const defaultOutcome = getVariant( @@ -563,6 +642,7 @@ function resolveOutcome( function handleOutcome( params: EvaluationParams, outcome: Packed.Outcome, + assignmentReason?: ExperimentAssignmentReason, ): { value: T; outcomeType: OutcomeType; @@ -571,17 +651,15 @@ function handleOutcome( } { const result = resolveOutcome(params, outcome); const experiment = params.definition.experiment; - if (!experiment || result.variantId === null) return result; + if (!experiment || result.experiment) return result; return { ...result, - experiment: { - id: experiment.id, - variantId: result.variantId, - base: experiment.base, - rampId: experiment.rampId, - rampPercentage: experiment.rampPercentage, - }, + experiment: experimentAssignment( + experiment, + result.variantId, + assignmentReason ?? outcomeAssignmentReason(outcome), + ), }; } @@ -651,7 +729,7 @@ export function evaluate( ); if (matchedIndex > -1) { - return Object.assign(handleOutcome(params, matchedIndex), { + return Object.assign(handleOutcome(params, matchedIndex, 'targeted'), { reason: ResolutionReason.TARGET_MATCH as const, }) satisfies EvaluationResult; } diff --git a/packages/vercel-flags-core/src/exposure-reporting.test.ts b/packages/vercel-flags-core/src/exposure-reporting.test.ts index 846c46d3..6bf45a04 100644 --- a/packages/vercel-flags-core/src/exposure-reporting.test.ts +++ b/packages/vercel-flags-core/src/exposure-reporting.test.ts @@ -18,24 +18,28 @@ describe('defaultReportExposures', () => { base: ['user', 'key'], rampId: 'ramp_1', rampPercentage: 50, + assignmentReason: 'experiment', }, { flagKey: 'pricing', experimentId: 'exp_team', variantId: 'variant_b', base: ['team', 'key'], + assignmentReason: 'targeted', }, { flagKey: 'visitor', experimentId: 'exp_visitor', variantId: 'variant_c', base: ['visitor', 'id'], + assignmentReason: 'split', }, { flagKey: 'device', experimentId: 'exp_device', variantId: 'variant_d', base: ['device', 'key'], + assignmentReason: 'override', }, ], { @@ -55,6 +59,7 @@ describe('defaultReportExposures', () => { unitValue: 'user_123', rampId: 'ramp_1', rampPercentage: 50, + assignmentReason: 'experiment', }, ); expect(log).toHaveBeenNthCalledWith( @@ -65,6 +70,7 @@ describe('defaultReportExposures', () => { variantId: 'variant_b', unitKey: 'group', unitValue: 'team_123', + assignmentReason: 'targeted', }, ); expect(log).toHaveBeenNthCalledWith( @@ -75,6 +81,7 @@ describe('defaultReportExposures', () => { variantId: 'variant_c', unitKey: 'event_data.visitorId', unitValue: 'visitor_123', + assignmentReason: 'split', }, ); expect(log).toHaveBeenNthCalledWith( @@ -85,6 +92,7 @@ describe('defaultReportExposures', () => { variantId: 'variant_d', unitKey: 'device', unitValue: 'fake-device-id', + assignmentReason: 'override', }, ); }); @@ -99,6 +107,7 @@ describe('defaultReportExposures', () => { experimentId: 'exp_user', variantId: 'variant_a', base: ['user', 'key'], + assignmentReason: 'experiment', }, ], {}, diff --git a/packages/vercel-flags-core/src/exposure-reporting.ts b/packages/vercel-flags-core/src/exposure-reporting.ts index c7259cb9..e12a88c4 100644 --- a/packages/vercel-flags-core/src/exposure-reporting.ts +++ b/packages/vercel-flags-core/src/exposure-reporting.ts @@ -7,6 +7,7 @@ type WebAnalyticsExposure = { unitValue: string; rampId?: string; rampPercentage?: number; + assignmentReason: Exposure['assignmentReason']; }; const FAKE_DEVICE_ID = 'fake-device-id'; @@ -66,13 +67,14 @@ function mapExposure( return { experimentId: exposure.experimentId, - variantId: exposure.variantId, + variantId: exposure.variantId ?? 'override', unitKey, unitValue, ...(exposure.rampId === undefined ? {} : { rampId: exposure.rampId }), ...(exposure.rampPercentage === undefined ? {} : { rampPercentage: exposure.rampPercentage }), + assignmentReason: exposure.assignmentReason, }; } diff --git a/packages/vercel-flags-core/src/types.ts b/packages/vercel-flags-core/src/types.ts index 3f9db94e..a7fa0b00 100644 --- a/packages/vercel-flags-core/src/types.ts +++ b/packages/vercel-flags-core/src/types.ts @@ -132,6 +132,15 @@ export type EvaluationOptions = { exposureLogging?: boolean; }; +export type ExperimentAssignmentReason = + | 'experiment' + | 'not-enrolled' + | 'targeted' + | 'split' + | 'variant' + | 'rollout' + | 'override'; + /** Information about the experiment linked to an evaluated flag value. */ export type ExperimentAssignment = { /** Experiment identifier. */ @@ -144,6 +153,8 @@ export type ExperimentAssignment = { rampId?: string; /** Percentage of eligible units included in the ramp, from 0 through 100. */ rampPercentage?: number; + /** How this evaluation received its value. */ + assignmentReason: ExperimentAssignmentReason; }; /** An experiment exposure passed to a client's exposure reporter. */ @@ -153,13 +164,15 @@ export type Exposure = { /** Experiment identifier. */ experimentId: string; /** Identifier of the selected experiment variant. */ - variantId: string; + variantId: string | null; /** Entity path on which the experiment assignment is based. */ base: Packed.EntityAccessor; /** Identifier of the ramp active for this assignment. */ rampId?: string; /** Percentage of eligible units included in the ramp, from 0 through 100. */ rampPercentage?: number; + /** How this evaluation received its value. */ + assignmentReason: ExperimentAssignmentReason; }; /** Reports experiment exposures produced by one evaluation call. */ @@ -215,6 +228,12 @@ export type FlagsClient> = { entities?: E, options?: EvaluationOptions, ) => Promise>>; + /** Report a Flags SDK override without evaluating the provider value. */ + reportOverride: ( + flagKey: string, + value: T, + entities?: E, + ) => Promise; /** * Retrieve the latest datafile during startup, and set up subscriptions if needed. */ @@ -359,6 +378,8 @@ export enum OutcomeType { SPLIT = 'split', /** When the outcome type was a progressive rollout */ ROLLOUT = 'rollout', + /** When the experiment assignment mechanism produced the value */ + EXPERIMENT = 'experiment', } /** @@ -592,6 +613,9 @@ export namespace Original { * Once all slots are exhausted, the rollout is complete (100% rollToVariant). */ slots: { promille: number; durationMs: number }[]; + } + | { + type: 'experiment'; }; export type ExperimentDefinition = { @@ -602,6 +626,8 @@ export namespace Original { weights: Record; /** Flag variant used when the base attribute does not exist. */ defaultVariantId: VariantId; + /** Stable seed used only for experiment enrollment. */ + enrollmentSeed: number; rampId?: string; /** Percentage from 0 through 100. */ rampPercentage?: number; @@ -836,6 +862,12 @@ export namespace Packed { id: string; /** Entity path used as the experiment unit. */ base: EntityAccessor; + /** Distribution indexed by the corresponding flag variant. */ + weights: number[]; + /** Flag variant used when the experiment base is unavailable. */ + defaultVariant: VariantIndex; + /** Stable seed used only for experiment enrollment. */ + enrollmentSeed: number; /** Identifier of the ramp active for this experiment. */ rampId?: string; /** Percentage of eligible units included in the ramp, from 0 through 100. */ @@ -860,7 +892,13 @@ export namespace Packed { export type SegmentOutcome = SegmentAllOutcome | SegmentSplitOutcome; - export type Outcome = VariantIndex | SplitOutcome | RolloutOutcome; + export type ExperimentOutcome = { type: 'experiment' }; + + export type Outcome = + | VariantIndex + | SplitOutcome + | RolloutOutcome + | ExperimentOutcome; // an array means it's an entity, the string "segment" means a segment export type EntityAccessor = (string | number)[]; From b8d03bfdbb876e55cb3dfa47bc29098cbb380c56 Mon Sep 17 00:00:00 2001 From: Dominik Ferber Date: Fri, 21 Aug 2026 16:11:02 +0300 Subject: [PATCH 10/13] Initialize adapters before reporting flag overrides --- .changeset/bright-experiments-report.md | 2 +- packages/flags/src/next/evaluate.ts | 40 +++++++++++++++++++++---- packages/flags/src/next/index.test.ts | 11 ++++++- 3 files changed, 45 insertions(+), 8 deletions(-) diff --git a/.changeset/bright-experiments-report.md b/.changeset/bright-experiments-report.md index 83a8a695..cf94d494 100644 --- a/.changeset/bright-experiments-report.md +++ b/.changeset/bright-experiments-report.md @@ -4,4 +4,4 @@ 'flags': minor --- -Add randomized experiment enrollment, assignment reasons for every experiment-managed flag outcome, cookie override exposure reporting, and per-evaluation exposure logging controls. +Add randomized experiment enrollment, assignment reasons for every experiment-managed flag outcome, readiness-aware cookie override exposure reporting, and per-evaluation exposure logging controls. diff --git a/packages/flags/src/next/evaluate.ts b/packages/flags/src/next/evaluate.ts index 226c90b6..fbbcfab6 100644 --- a/packages/flags/src/next/evaluate.ts +++ b/packages/flags/src/next/evaluate.ts @@ -40,6 +40,27 @@ const evaluationCache = new WeakMap< Map> >(); +const adapterInitializationCache = new WeakMap>(); + +async function ensureAdapterInitialized( + adapter: Pick, 'initialize'>, +): Promise { + if (!adapter.initialize) return; + + let initialization = adapterInitializationCache.get(adapter); + if (!initialization) { + initialization = adapter.initialize(); + adapterInitializationCache.set(adapter, initialization); + } + + try { + await initialization; + } catch (error) { + adapterInitializationCache.delete(adapter); + throw error; + } +} + function getCachedValuePromise( /** * supports Headers for App Router and IncomingHttpHeaders for Pages Router @@ -197,7 +218,10 @@ type FlagInfo = { key: string; defaultValue?: ValueType; config?: { reportValue?: boolean }; - adapter?: Pick, 'config' | 'reportOverride'>; + adapter?: Pick< + Adapter, + 'config' | 'initialize' | 'reportOverride' + >; }; function hasOverride( @@ -263,11 +287,15 @@ async function applyResult(args: { reason: 'override', }); try { - await definition.adapter?.reportOverride?.({ - key: definition.key, - value: decision, - entities, - }); + const adapter = definition.adapter; + if (adapter?.reportOverride) { + await ensureAdapterInitialized(adapter); + await adapter.reportOverride({ + key: definition.key, + value: decision, + entities, + }); + } } catch (error) { console.error('flags: Failed to report flag override', error); } diff --git a/packages/flags/src/next/index.test.ts b/packages/flags/src/next/index.test.ts index 4c9293bb..1204edeb 100644 --- a/packages/flags/src/next/index.test.ts +++ b/packages/flags/src/next/index.test.ts @@ -191,13 +191,20 @@ describe('flag on app router', () => { it('respects overrides', async () => { const decide = vi.fn(() => false); - const reportOverride = vi.fn(); + const calls: string[] = []; + const initialize = vi.fn(async () => { + calls.push('initialize'); + }); + const reportOverride = vi.fn(async () => { + calls.push('reportOverride'); + }); const entities = { user: { id: 'user_1' } }; const f = flag({ key: 'first-flag', identify: () => entities, adapter: { decide, + initialize, reportOverride, }, }); @@ -216,11 +223,13 @@ describe('flag on app router', () => { await expect(f()).resolves.toEqual(true); expect(cookieMock).toHaveBeenCalledWith('vercel-flag-overrides'); expect(decide).not.toHaveBeenCalled(); + expect(initialize).toHaveBeenCalledOnce(); expect(reportOverride).toHaveBeenCalledWith({ key: 'first-flag', value: true, entities, }); + expect(calls).toEqual(['initialize', 'reportOverride']); }); it('does not crash when override reporting hook is not a function', async () => { From c476b82eccf76c98a52e624ce17da4cb0f2a23a1 Mon Sep 17 00:00:00 2001 From: Dominik Ferber Date: Sat, 22 Aug 2026 22:46:51 +0300 Subject: [PATCH 11/13] Update package versions for experiment replacement --- packages/adapter-vercel/package.json | 2 +- packages/flags/package.json | 2 +- packages/vercel-flags-core/package.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/adapter-vercel/package.json b/packages/adapter-vercel/package.json index 805509bd..905265cb 100644 --- a/packages/adapter-vercel/package.json +++ b/packages/adapter-vercel/package.json @@ -1,6 +1,6 @@ { "name": "@flags-sdk/vercel", - "version": "1.4.6", + "version": "1.4.6-exp-rep.0", "description": "A Flags SDK adapter for Vercel Flags", "keywords": [ "vercel", diff --git a/packages/flags/package.json b/packages/flags/package.json index eb4d212c..d99821eb 100644 --- a/packages/flags/package.json +++ b/packages/flags/package.json @@ -1,6 +1,6 @@ { "name": "flags", - "version": "4.3.0", + "version": "4.3.0-exp-rep.0", "description": "Flags SDK by Vercel - The feature flags toolkit for Next.js and SvelteKit", "keywords": [ "feature flags", diff --git a/packages/vercel-flags-core/package.json b/packages/vercel-flags-core/package.json index e07dc928..9e3f7873 100644 --- a/packages/vercel-flags-core/package.json +++ b/packages/vercel-flags-core/package.json @@ -1,6 +1,6 @@ { "name": "@vercel/flags-core", - "version": "1.7.1-engulf.0", + "version": "1.7.1-exp-rep.0", "description": "A server-side client for Vercel Flags", "keywords": [ "vercel", From de9e9f97c8535671ca84fa97cc750a42b0d74cdc Mon Sep 17 00:00:00 2001 From: Dominik Ferber Date: Mon, 31 Aug 2026 15:17:44 +0300 Subject: [PATCH 12/13] versions --- packages/flags/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/flags/package.json b/packages/flags/package.json index 2875cd5d..280d4b95 100644 --- a/packages/flags/package.json +++ b/packages/flags/package.json @@ -1,6 +1,6 @@ { "name": "flags", - "version": "4.3.0-exp-rep.0", + "version": "4.3.0", "description": "Flags SDK by Vercel - The feature flags toolkit for Next.js and SvelteKit", "keywords": [ "feature flags", From b67dc74f49e4cdedc8f750b70fdc58a0091e2942 Mon Sep 17 00:00:00 2001 From: Dominik Ferber Date: Mon, 31 Aug 2026 15:19:44 +0300 Subject: [PATCH 13/13] fix test --- packages/flags/src/index.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/flags/src/index.test.ts b/packages/flags/src/index.test.ts index eefa3f0f..d26edc3c 100644 --- a/packages/flags/src/index.test.ts +++ b/packages/flags/src/index.test.ts @@ -27,7 +27,7 @@ describe('exports', () => { it('exports version', () => { expect(version).toBeTypeOf('string'); - expect(version).toMatch(/^\d+\.\d+\.\d+(-\w+-\d+)?$/); + expect(version).toMatch(/^\d+\.\d+\.\d+(-[\w.-]+)?$/); }); });