From 90ffb58988f36a3a2ab5474f14c3ab75112397fe Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Thu, 13 Aug 2026 14:37:36 +0000 Subject: [PATCH 1/2] feat: Apply auto instrumentation with eval command --- .../auto-instrumentations/bundler/plugin.ts | 9 +- js/src/cli/auto-instrumentation.test.ts | 164 ++++++++++++++++++ js/src/cli/functions/load-module.ts | 6 + js/src/cli/index.ts | 6 +- 4 files changed, 180 insertions(+), 5 deletions(-) create mode 100644 js/src/cli/auto-instrumentation.test.ts diff --git a/js/src/auto-instrumentations/bundler/plugin.ts b/js/src/auto-instrumentations/bundler/plugin.ts index 28ce3e445..f527a5253 100644 --- a/js/src/auto-instrumentations/bundler/plugin.ts +++ b/js/src/auto-instrumentations/bundler/plugin.ts @@ -4,7 +4,7 @@ import { extname, isAbsolute, join, sep } from "path"; import { readFileSync } from "fs"; import { fileURLToPath } from "url"; import moduleDetailsFromPath from "module-details-from-path"; -import { getDefaultInstrumentationConfigs } from "../configs/all"; +import { getDefaultAutoInstrumentationConfigs } from "../configs/all"; import { applySpecialCasePatch } from "../loader/special-case-patches"; import { getPackageName } from "../loader/get-package-version"; @@ -57,9 +57,10 @@ function getModuleVersion(basedir: string): string | undefined { export const unplugin = createUnplugin((options = {}) => { const browser = options.browser ?? options.useDiagnosticChannelCompatShim ?? false; - const allInstrumentations = getDefaultInstrumentationConfigs({ - additionalInstrumentations: options.instrumentations, - }); + const allInstrumentations = [ + ...getDefaultAutoInstrumentationConfigs(), + ...(options.instrumentations ?? []), + ]; // Create the code transformer instrumentor const instrumentationMatcher = create(allInstrumentations); diff --git a/js/src/cli/auto-instrumentation.test.ts b/js/src/cli/auto-instrumentation.test.ts new file mode 100644 index 000000000..519abb8c2 --- /dev/null +++ b/js/src/cli/auto-instrumentation.test.ts @@ -0,0 +1,164 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; +import { newGlobalTracingChannel } from "../global-instrumentation-hooks"; +import { initializeHandles } from "./index"; +import type { FileHandle } from "./types"; + +vi.mock("./functions/load-module", () => ({ + loadModule: () => ({ + evaluators: {}, + functions: [], + parameters: [], + prompts: [], + reporters: {}, + }), +})); + +const googleGenAIChannel = "orchestrion:@google/genai:models.generateContent"; + +describe("eval auto-instrumentation", () => { + let fixtureDir: string; + let handles: Record = {}; + + beforeEach(async () => { + fixtureDir = await fs.mkdtemp( + path.join(os.tmpdir(), "braintrust-eval-instrumentation-"), + ); + await writeFixturePackages(fixtureDir); + }); + + afterEach(async () => { + vi.unstubAllEnvs(); + await Promise.all(Object.values(handles).map((handle) => handle.destroy())); + handles = {}; + await fs.rm(fixtureDir, { recursive: true, force: true }); + }); + + it.each([ + [ + "directly", + `export { Models } from "@google/genai";`, + (exports: Record) => { + const Models = exports.Models as new () => { + generateContentInternal(input: string): Promise<{ text: string }>; + }; + return new Models().generateContentInternal("payload"); + }, + ], + [ + "through a dependency", + `export { invoke } from "google-genai-client";`, + (exports: Record) => + (exports.invoke as (input: string) => Promise<{ text: string }>)( + "payload", + ), + ], + ])( + "instruments @google/genai when imported %s", + async (_label, source, invoke) => { + const output = await buildEval(fixtureDir, source); + + expect(output).toContain(googleGenAIChannel); + + const lifecycle: string[] = []; + const hook = newGlobalTracingChannel(googleGenAIChannel); + const handlers = { + asyncEnd: () => lifecycle.push("asyncEnd"), + asyncStart: () => lifecycle.push("asyncStart"), + end: () => lifecycle.push("end"), + start: () => lifecycle.push("start"), + }; + hook.subscribe(handlers); + + try { + const loadedModule = { exports: {} as Record }; + Function( + "module", + "exports", + output, + )(loadedModule, loadedModule.exports); + + await expect(invoke(loadedModule.exports)).resolves.toEqual({ + text: "payload", + }); + expect(lifecycle).toEqual(["start", "end", "asyncStart", "asyncEnd"]); + } finally { + hook.unsubscribe(handlers); + } + }, + ); + + it("respects BRAINTRUST_DISABLE_INSTRUMENTATION", async () => { + vi.stubEnv("BRAINTRUST_DISABLE_INSTRUMENTATION", "google-genai"); + + const output = await buildEval( + fixtureDir, + `export { Models } from "@google/genai";`, + ); + + expect(output).not.toContain(googleGenAIChannel); + }); + + async function buildEval(fixtureDir: string, source: string) { + const evalFile = path.join(fixtureDir, "instrumentation.eval.ts"); + await fs.writeFile(evalFile, source); + handles = await initializeHandles({ files: [evalFile], mode: "eval" }); + + const result = await handles[evalFile].rebuild(); + if (result.type !== "success") { + throw result.error; + } + + return result.result.outputFiles?.[0].text ?? ""; + } +}); + +async function writeFixturePackages(fixtureDir: string) { + const googlePackageDir = path.join(fixtureDir, "node_modules/@google/genai"); + const indirectPackageDir = path.join( + fixtureDir, + "node_modules/google-genai-client", + ); + await fs.mkdir(path.join(googlePackageDir, "dist/node"), { + recursive: true, + }); + await fs.mkdir(indirectPackageDir, { recursive: true }); + + await Promise.all([ + fs.writeFile( + path.join(googlePackageDir, "package.json"), + JSON.stringify({ + name: "@google/genai", + version: "1.50.0", + type: "module", + exports: "./dist/node/index.mjs", + }), + ), + fs.writeFile( + path.join(googlePackageDir, "dist/node/index.mjs"), + `export class Models { + async generateContentInternal(input) { + return { text: input }; + } + }`, + ), + fs.writeFile( + path.join(indirectPackageDir, "package.json"), + JSON.stringify({ + name: "google-genai-client", + version: "1.0.0", + type: "module", + exports: "./index.mjs", + }), + ), + fs.writeFile( + path.join(indirectPackageDir, "index.mjs"), + `import { Models } from "@google/genai"; + export function invoke(input) { + return new Models().generateContentInternal(input); + }`, + ), + ]); +} diff --git a/js/src/cli/functions/load-module.ts b/js/src/cli/functions/load-module.ts index 593b45ae7..c39e016c4 100644 --- a/js/src/cli/functions/load-module.ts +++ b/js/src/cli/functions/load-module.ts @@ -33,6 +33,12 @@ export function loadModule({ (globalThis as any)[Symbol.for("braintrust-state")] = state; const __filename = inFile; const __dirname = dirname(__filename); + try { + require("braintrust/apply-auto-instrumentation"); + } catch { + // The bundler transform still covers inlined dependencies when the + // runtime hook is unavailable, such as when running from source. + } new Function("require", "module", "__filename", "__dirname", moduleText)( require, module, diff --git a/js/src/cli/index.ts b/js/src/cli/index.ts index ba36d79e3..0972913e9 100755 --- a/js/src/cli/index.ts +++ b/js/src/cli/index.ts @@ -61,6 +61,7 @@ import { } from "./util/debug-logging"; import { pullCommand } from "./util/pull"; import { runDevServer } from "../../dev/server"; +import { braintrustEsbuildPlugin } from "../auto-instrumentations/bundler/esbuild"; // This requires require // https://stackoverflow.com/questions/50822310/how-to-import-package-json-in-typescript @@ -828,6 +829,7 @@ function buildOpts({ externalPackages?: string[]; }): esbuild.BuildOptions { const plugins = [ + braintrustEsbuildPlugin(), nativeNodeModulesPlugin, createMarkKnownPackagesExternalPlugin(externalPackages), ...(argPlugins || []).map((fn) => fn(fileName)), @@ -1188,4 +1190,6 @@ async function main() { } } -main(); +if (require.main === module) { + void main(); +} From 63b136641026fd427de233ff0c65a1a004208df2 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Thu, 13 Aug 2026 15:03:00 +0000 Subject: [PATCH 2/2] feat: Apply auto instrumentation with eval command --- .../auto-instrumentations/bundler/plugin.ts | 7 +- .../bundler/webpack-loader.ts | 8 +- js/src/auto-instrumentations/configs/all.ts | 5 +- js/src/cli/auto-instrumentation.test.ts | 142 ++++++++++++++++++ js/src/cli/functions/load-module.ts | 12 +- js/src/cli/index.ts | 3 +- .../node/apply-auto-instrumentation-entry.ts | 72 +-------- js/src/node/apply-auto-instrumentation.ts | 71 +++++++++ .../transformation.test.ts | 36 +++++ 9 files changed, 271 insertions(+), 85 deletions(-) create mode 100644 js/src/node/apply-auto-instrumentation.ts diff --git a/js/src/auto-instrumentations/bundler/plugin.ts b/js/src/auto-instrumentations/bundler/plugin.ts index f527a5253..47562db0d 100644 --- a/js/src/auto-instrumentations/bundler/plugin.ts +++ b/js/src/auto-instrumentations/bundler/plugin.ts @@ -57,10 +57,9 @@ function getModuleVersion(basedir: string): string | undefined { export const unplugin = createUnplugin((options = {}) => { const browser = options.browser ?? options.useDiagnosticChannelCompatShim ?? false; - const allInstrumentations = [ - ...getDefaultAutoInstrumentationConfigs(), - ...(options.instrumentations ?? []), - ]; + const allInstrumentations = getDefaultAutoInstrumentationConfigs( + options.instrumentations, + ); // Create the code transformer instrumentor const instrumentationMatcher = create(allInstrumentations); diff --git a/js/src/auto-instrumentations/bundler/webpack-loader.ts b/js/src/auto-instrumentations/bundler/webpack-loader.ts index 75fb1d053..211cd31a5 100644 --- a/js/src/auto-instrumentations/bundler/webpack-loader.ts +++ b/js/src/auto-instrumentations/bundler/webpack-loader.ts @@ -25,7 +25,7 @@ import { create } from "../orchestrion-js"; import { extname, join, sep } from "path"; import { readFileSync } from "fs"; import moduleDetailsFromPath from "module-details-from-path"; -import { getDefaultInstrumentationConfigs } from "../configs/all"; +import { getDefaultAutoInstrumentationConfigs } from "../configs/all"; import { type BundlerPluginOptions } from "./plugin"; import { applySpecialCasePatch } from "../loader/special-case-patches"; import { getPackageName } from "../loader/get-package-version"; @@ -57,9 +57,9 @@ const matcherCache = new Map(); * Get or create a matcher instance, caching by config hash */ function getMatcher(options: BundlerPluginOptions): Matcher { - const allInstrumentations = getDefaultInstrumentationConfigs({ - additionalInstrumentations: options.instrumentations, - }); + const allInstrumentations = getDefaultAutoInstrumentationConfigs( + options.instrumentations, + ); const configHash = JSON.stringify({ allInstrumentations }); if (matcherCache.has(configHash)) { diff --git a/js/src/auto-instrumentations/configs/all.ts b/js/src/auto-instrumentations/configs/all.ts index 9d7f16309..23d58969a 100644 --- a/js/src/auto-instrumentations/configs/all.ts +++ b/js/src/auto-instrumentations/configs/all.ts @@ -160,8 +160,11 @@ export function getDefaultInstrumentationConfigs({ ]; } -export function getDefaultAutoInstrumentationConfigs(): InstrumentationConfig[] { +export function getDefaultAutoInstrumentationConfigs( + additionalInstrumentations?: readonly InstrumentationConfig[], +): InstrumentationConfig[] { return getDefaultInstrumentationConfigs({ + additionalInstrumentations, disabledIntegrationConfig: readDisabledInstrumentationEnvConfig( process.env.BRAINTRUST_DISABLE_INSTRUMENTATION, ).integrations, diff --git a/js/src/cli/auto-instrumentation.test.ts b/js/src/cli/auto-instrumentation.test.ts index 519abb8c2..0b9f41495 100644 --- a/js/src/cli/auto-instrumentation.test.ts +++ b/js/src/cli/auto-instrumentation.test.ts @@ -2,6 +2,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import * as fs from "node:fs/promises"; import * as os from "node:os"; import * as path from "node:path"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; import { newGlobalTracingChannel } from "../global-instrumentation-hooks"; import { initializeHandles } from "./index"; import type { FileHandle } from "./types"; @@ -17,6 +20,17 @@ vi.mock("./functions/load-module", () => ({ })); const googleGenAIChannel = "orchestrion:@google/genai:models.generateContent"; +const anthropicChannel = "orchestrion:@anthropic-ai/sdk:messages.create"; +const execFileAsync = promisify(execFile); +const loadModulePath = fileURLToPath( + new URL("./functions/load-module.ts", import.meta.url), +); +const globalHooksPath = fileURLToPath( + new URL("../global-instrumentation-hooks.ts", import.meta.url), +); +const debugLoggerPath = fileURLToPath( + new URL("../debug-logger.ts", import.meta.url), +); describe("eval auto-instrumentation", () => { let fixtureDir: string; @@ -32,6 +46,18 @@ describe("eval auto-instrumentation", () => { afterEach(async () => { vi.unstubAllEnvs(); await Promise.all(Object.values(handles).map((handle) => handle.destroy())); + await Promise.all( + [ + ...new Set( + Object.values(handles) + .map((handle) => handle.bundleFile) + .filter( + (bundleFile): bundleFile is string => bundleFile !== undefined, + ) + .map((bundleFile) => path.dirname(bundleFile)), + ), + ].map((directory) => fs.rm(directory, { recursive: true, force: true })), + ); handles = {}; await fs.rm(fixtureDir, { recursive: true, force: true }); }); @@ -101,6 +127,95 @@ describe("eval auto-instrumentation", () => { expect(output).not.toContain(googleGenAIChannel); }); + it("instruments the final uploaded bundle", async () => { + await buildEval(fixtureDir, `export { Models } from "@google/genai";`); + + await handles[path.join(fixtureDir, "instrumentation.eval.ts")].bundle(); + const output = await fs.readFile( + handles[path.join(fixtureDir, "instrumentation.eval.ts")].bundleFile!, + "utf8", + ); + + expect(output).toContain(googleGenAIChannel); + }); + + it.each([ + ["instruments", undefined, ["start", "end", "asyncStart", "asyncEnd"]], + ["respects opt-out for", "anthropic", []], + ])("%s external eval dependencies", async (_label, disabled, expected) => { + const runnerPath = path.join(fixtureDir, "external-eval-runner.cjs"); + const evalFile = path.join(fixtureDir, "external.eval.ts"); + await fs.writeFile(evalFile, ""); + await fs.writeFile( + runnerPath, + `require("tsx/cjs"); +require("node:module").register = () => {}; +const { loadModule } = require(${JSON.stringify(loadModulePath)}); +const { newGlobalTracingChannel } = require(${JSON.stringify(globalHooksPath)}); +const lifecycle = []; +const channel = newGlobalTracingChannel(${JSON.stringify(anthropicChannel)}); +const handlers = Object.fromEntries( + ["start", "end", "asyncStart", "asyncEnd"].map((name) => [name, () => lifecycle.push(name)]), +); +channel.subscribe(handlers); +loadModule({ + inFile: ${JSON.stringify(evalFile)}, + moduleText: 'const { Messages } = require("@anthropic-ai/sdk/resources/messages/messages.js"); globalThis.__externalEvalResult = new Messages().create("payload");', +}); +Promise.resolve(globalThis.__externalEvalResult).then((result) => { + channel.unsubscribe(handlers); + process.stdout.write(JSON.stringify({ lifecycle, result })); +});`, + ); + + const env = { ...process.env }; + if (disabled) { + env.BRAINTRUST_DISABLE_INSTRUMENTATION = disabled; + } else { + delete env.BRAINTRUST_DISABLE_INSTRUMENTATION; + } + const { stdout } = await execFileAsync(process.execPath, [runnerPath], { + env, + }); + + expect(JSON.parse(stdout)).toEqual({ + lifecycle: expected, + result: { text: "payload" }, + }); + }); + + it("reports unexpected runtime instrumentation failures", async () => { + const runnerPath = path.join( + fixtureDir, + "failed-instrumentation-runner.cjs", + ); + const evalFile = path.join(fixtureDir, "failed-instrumentation.eval.ts"); + await fs.writeFile(evalFile, ""); + await fs.writeFile( + runnerPath, + `require("tsx/cjs"); +require("node:module").register = () => { throw new Error("setup failed"); }; +require(${JSON.stringify(debugLoggerPath)}).setGlobalDebugLogLevel("warn"); +const { loadModule } = require(${JSON.stringify(loadModulePath)}); +loadModule({ inFile: ${JSON.stringify(evalFile)}, moduleText: "" }); +process.stdout.write("loaded");`, + ); + + const { stderr, stdout } = await execFileAsync( + process.execPath, + [runnerPath], + { + env: { ...process.env, BRAINTRUST_DEBUG_LOG_LEVEL: "warn" }, + }, + ); + + expect(stdout).toBe("loaded"); + expect(stderr).toContain( + "Failed to enable auto-instrumentation for external eval dependencies", + ); + expect(stderr).toContain("setup failed"); + }); + async function buildEval(fixtureDir: string, source: string) { const evalFile = path.join(fixtureDir, "instrumentation.eval.ts"); await fs.writeFile(evalFile, source); @@ -121,10 +236,17 @@ async function writeFixturePackages(fixtureDir: string) { fixtureDir, "node_modules/google-genai-client", ); + const anthropicPackageDir = path.join( + fixtureDir, + "node_modules/@anthropic-ai/sdk", + ); await fs.mkdir(path.join(googlePackageDir, "dist/node"), { recursive: true, }); await fs.mkdir(indirectPackageDir, { recursive: true }); + await fs.mkdir(path.join(anthropicPackageDir, "resources/messages"), { + recursive: true, + }); await Promise.all([ fs.writeFile( @@ -160,5 +282,25 @@ async function writeFixturePackages(fixtureDir: string) { return new Models().generateContentInternal(input); }`, ), + fs.writeFile( + path.join(anthropicPackageDir, "package.json"), + JSON.stringify({ + name: "@anthropic-ai/sdk", + version: "0.60.0", + exports: { + "./resources/messages/messages.js": + "./resources/messages/messages.js", + }, + }), + ), + fs.writeFile( + path.join(anthropicPackageDir, "resources/messages/messages.js"), + `class Messages { + async create(input) { + return { text: input }; + } + } + module.exports = { Messages };`, + ), ]); } diff --git a/js/src/cli/functions/load-module.ts b/js/src/cli/functions/load-module.ts index c39e016c4..1fa7296b0 100644 --- a/js/src/cli/functions/load-module.ts +++ b/js/src/cli/functions/load-module.ts @@ -2,6 +2,8 @@ import nodeModulesPaths from "../jest/nodeModulesPaths"; import path, { dirname } from "node:path"; import { _internalGetGlobalState } from "../../logger"; import { EvaluatorFile } from "../../framework"; +import { applyAutoInstrumentation } from "../../node/apply-auto-instrumentation"; +import { debugLogger } from "../../debug-logger"; function evalWithModuleContext(inFile: string, evalFn: () => T): T { const modulePaths = [...module.paths]; @@ -34,10 +36,12 @@ export function loadModule({ const __filename = inFile; const __dirname = dirname(__filename); try { - require("braintrust/apply-auto-instrumentation"); - } catch { - // The bundler transform still covers inlined dependencies when the - // runtime hook is unavailable, such as when running from source. + applyAutoInstrumentation(); + } catch (error) { + debugLogger.warn( + "Failed to enable auto-instrumentation for external eval dependencies; bundled dependencies remain instrumented:", + error, + ); } new Function("require", "module", "__filename", "__dirname", moduleText)( require, diff --git a/js/src/cli/index.ts b/js/src/cli/index.ts index 0972913e9..eb5004555 100755 --- a/js/src/cli/index.ts +++ b/js/src/cli/index.ts @@ -379,12 +379,11 @@ async function initFile({ fileName: inFile, outFile: bundleFile, tsconfig, - plugins, + plugins: [], externalPackages, }), external: ["fsevents", "chokidar"], write: true, - plugins: [], minify: true, sourcemap: true, }; diff --git a/js/src/node/apply-auto-instrumentation-entry.ts b/js/src/node/apply-auto-instrumentation-entry.ts index 5e9c1010e..cd199bba4 100644 --- a/js/src/node/apply-auto-instrumentation-entry.ts +++ b/js/src/node/apply-auto-instrumentation-entry.ts @@ -1,73 +1,5 @@ -import { register } from "node:module"; -import { pathToFileURL } from "node:url"; -import { getDefaultAutoInstrumentationConfigs } from "../auto-instrumentations/configs/all"; -import { ModulePatch } from "../auto-instrumentations/loader/cjs-patch"; -import { GLOBAL_INSTRUMENTATION_HOOKS_PROTOCOL_VERSION } from "../global-instrumentation-hooks"; +import { applyAutoInstrumentation } from "./apply-auto-instrumentation"; -interface ApplyAutoInstrumentationState { - applied?: boolean; -} - -const stateKey = Symbol.for( - `braintrust.applyAutoInstrumentation.global-hooks.v${GLOBAL_INSTRUMENTATION_HOOKS_PROTOCOL_VERSION}`, -); -const existingState = Object.getOwnPropertyDescriptor( - globalThis, - stateKey, -)?.value; -const state: ApplyAutoInstrumentationState = isApplyAutoInstrumentationState( - existingState, -) - ? existingState - : {}; - -if (state !== existingState) { - Object.defineProperty(globalThis, stateKey, { - configurable: false, - enumerable: false, - value: state, - writable: false, - }); -} - -if (!state.applied) { - const allConfigs = getDefaultAutoInstrumentationConfigs(); - - const currentModuleUrl = getCurrentModuleUrl(); - register("./auto-instrumentations/loader/esm-hook.mjs", { - parentURL: currentModuleUrl, - data: { instrumentations: allConfigs }, - }); - - state.applied = true; - - try { - const patch = new ModulePatch({ instrumentations: allConfigs }); - patch.patch(); - } catch { - // ESM instrumentation is already active; keep user code running if CJS patching fails. - } -} - -function isApplyAutoInstrumentationState( - value: unknown, -): value is ApplyAutoInstrumentationState { - return typeof value === "object" && value !== null; -} - -function getCurrentModuleUrl(): string { - if (typeof __filename !== "undefined") { - return pathToFileURL(__filename).href; - } - - const stack = new Error().stack ?? ""; - const match = - stack.match(/\((file:\/\/[^)]+)\)/) ?? stack.match(/\s(file:\/\/\S+)/); - if (match) { - return match[1].replace(/:\d+:\d+$/, ""); - } - - return pathToFileURL(process.argv[1] ?? process.cwd()).href; -} +applyAutoInstrumentation(); export {}; diff --git a/js/src/node/apply-auto-instrumentation.ts b/js/src/node/apply-auto-instrumentation.ts new file mode 100644 index 000000000..07dad6a61 --- /dev/null +++ b/js/src/node/apply-auto-instrumentation.ts @@ -0,0 +1,71 @@ +import { register } from "node:module"; +import { pathToFileURL } from "node:url"; +import { getDefaultAutoInstrumentationConfigs } from "../auto-instrumentations/configs/all"; +import { ModulePatch } from "../auto-instrumentations/loader/cjs-patch"; +import { debugLogger } from "../debug-logger"; +import { GLOBAL_INSTRUMENTATION_HOOKS_PROTOCOL_VERSION } from "../global-instrumentation-hooks"; + +interface ApplyAutoInstrumentationState { + applied?: boolean; +} + +const stateKey = Symbol.for( + `braintrust.applyAutoInstrumentation.global-hooks.v${GLOBAL_INSTRUMENTATION_HOOKS_PROTOCOL_VERSION}`, +); + +export function applyAutoInstrumentation(): void { + const existingState = Object.getOwnPropertyDescriptor( + globalThis, + stateKey, + )?.value; + const state: ApplyAutoInstrumentationState = + typeof existingState === "object" && existingState !== null + ? existingState + : {}; + + if (state !== existingState) { + Object.defineProperty(globalThis, stateKey, { + configurable: false, + enumerable: false, + value: state, + writable: false, + }); + } + + if (state.applied) { + return; + } + + const allConfigs = getDefaultAutoInstrumentationConfigs(); + const currentModuleUrl = getCurrentModuleUrl(); + + register("./auto-instrumentations/loader/esm-hook.mjs", { + parentURL: currentModuleUrl, + data: { instrumentations: allConfigs }, + }); + state.applied = true; + + try { + new ModulePatch({ instrumentations: allConfigs }).patch(); + } catch (error) { + debugLogger.warn( + "Failed to enable CommonJS auto-instrumentation; ESM instrumentation remains active:", + error, + ); + } +} + +function getCurrentModuleUrl(): string { + if (typeof __filename !== "undefined") { + return pathToFileURL(__filename).href; + } + + const stack = new Error().stack ?? ""; + const match = + stack.match(/\((file:\/\/[^)]+)\)/) ?? stack.match(/\s(file:\/\/\S+)/); + if (match) { + return match[1].replace(/:\d+:\d+$/, ""); + } + + return pathToFileURL(process.argv[1] ?? process.cwd()).href; +} diff --git a/js/tests/auto-instrumentations/transformation.test.ts b/js/tests/auto-instrumentations/transformation.test.ts index 0dd44294c..0a21456cd 100644 --- a/js/tests/auto-instrumentations/transformation.test.ts +++ b/js/tests/auto-instrumentations/transformation.test.ts @@ -866,6 +866,42 @@ describe("Orchestrion Transformation Tests", () => { expectGlobalHookTransform(output); }); + it("should respect instrumentation opt-outs in loader-only mode", async () => { + vi.stubEnv("BRAINTRUST_DISABLE_INSTRUMENTATION", "openai"); + try { + const { errors, output } = await runWebpackWithLoader({ + entry: path.join(fixturesDir, "test-app.js"), + output: { + path: outputDir, + filename: "turbopack-opt-out-bundle.js", + library: { type: "module" }, + }, + experiments: { outputModule: true }, + mode: "development", + resolve: { modules: [nodeModulesDir, "node_modules"] }, + module: { + rules: [ + { + use: [ + { + loader: webpackLoaderPath, + options: { browser: false }, + }, + ], + }, + ], + }, + }); + + expect(errors).toHaveLength(0); + expect(output).not.toContain( + "orchestrion:openai:chat.completions.create", + ); + } finally { + vi.unstubAllEnvs(); + } + }); + it("should use global hooks when browser mode is true (turbopack loader-only mode)", async () => { const { errors, output } = await runWebpackWithLoader({ entry: path.join(fixturesDir, "test-app.js"),