diff --git a/.github/hooks/deval-hook.cjs b/.github/hooks/deval-hook.cjs deleted file mode 100644 index 61e5c26..0000000 --- a/.github/hooks/deval-hook.cjs +++ /dev/null @@ -1,47 +0,0 @@ -const fs = require('node:fs'); -const path = require('node:path'); - -let input = ''; -process.stdin.setEncoding('utf8'); -process.stdin.on('data', (chunk) => { - input += chunk; -}); -process.stdin.on('end', () => { - try { - const event = JSON.parse(input); - if (!event || typeof event !== 'object' || Array.isArray(event)) { - throw new Error('Hook input must be a JSON object.'); - } - - const rawSessionId = typeof event.session_id === 'string' ? event.session_id : 'unknown'; - const sessionId = rawSessionId.replace(/[^a-zA-Z0-9_-]/g, '_'); - const eventFields = [ - 'timestamp', - 'cwd', - 'session_id', - 'hook_event_name', - 'transcript_path', - 'prompt', - 'tool_name', - 'tool_input', - 'tool_use_id', - 'tool_response', - 'stop_hook_active', - ]; - const record = Object.fromEntries( - eventFields - .filter((field) => Object.hasOwn(event, field)) - .map((field) => [field, event[field]]), - ); - const outputDirectory = path.join(process.cwd(), '.deepeval', 'vscode-agent-events'); - fs.mkdirSync(outputDirectory, { recursive: true }); - fs.appendFileSync( - path.join(outputDirectory, `${sessionId}.jsonl`), - `${JSON.stringify(record)}\n`, - { encoding: 'utf8', mode: 0o600 }, - ); - } catch (error) { - console.error(`DeepEval hook could not record the event: ${error.message}`); - process.exitCode = 1; - } -}); diff --git a/.github/hooks/deval.json b/.github/hooks/deval.json deleted file mode 100644 index 0adeab5..0000000 --- a/.github/hooks/deval.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "hooks": { - "UserPromptSubmit": [ - { - "type": "command", - "command": "node .github/hooks/deval-hook.cjs", - "cwd": ".", - "timeout": 10 - } - ], - "PreToolUse": [ - { - "type": "command", - "command": "node .github/hooks/deval-hook.cjs", - "cwd": ".", - "timeout": 10 - } - ], - "PostToolUse": [ - { - "type": "command", - "command": "node .github/hooks/deval-hook.cjs", - "cwd": ".", - "timeout": 10 - } - ], - "Stop": [ - { - "type": "command", - "command": "node .github/hooks/deval-hook.cjs", - "cwd": ".", - "timeout": 10 - } - ] - } -} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 6508d15..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: CI - -on: - push: - branches: [main] - pull_request: - workflow_dispatch: - -jobs: - build: - name: Type-check, lint, and test - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: npm - - - name: Install dependencies - run: npm ci - - - name: Type-check extension and benchmarks - run: npm run check-types - - - name: Lint - run: npm run lint - - - name: Bundle the extension - run: node esbuild.js - - # The reference suites use deterministic metrics only, so they need no - # DEEPEVAL_API_KEY and behave the same here as they do locally. - - name: Run DeepEval benchmarks - run: npm run benchmark - env: - DEEPEVAL_TELEMETRY_OPT_OUT: "YES" - - extension-host-tests: - name: Extension host tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: npm - - - name: Install dependencies - run: npm ci - - - name: Compile tests - run: npm run compile-tests - - - name: Compile extension - run: node esbuild.js - - # xvfb is required to launch VS Code, even for headless extension tests. - - name: Run extension tests - run: xvfb-run -a npm test diff --git a/CHANGELOG.md b/CHANGELOG.md index 71eb2eb..5b8eb34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,22 @@ All notable changes to the `deval` extension are documented here. Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how to structure this file. +## [Unreleased] + +### Added + +- `@deval` is now a real tool-calling agent instead of a chat forwarder. It runs an agentic loop (`src/agent/loop.ts`) against the Chat model with three read-only tools — `deval_readFile`, `deval_listFiles`, `deval_grep` — declared in `package.json` and registered via `vscode.lm`. +- `src/events/event-writer.ts` — writes agent-loop runs as JSONL captures in the format `event-log.ts` reads, so `DeepEval: Evaluate Captured Agent Run` scores the agent's trajectory with no extra wiring. +- `benchmarks/agent-loop-benchmark.test.ts` — deterministic suite driving the loop with a scripted model through `observe()`, asserting with `ContainsAllMetric` and `ToolCorrectnessMetric`. +- Unit tests for the loop core (fake model/tool ports) and the capture writer. + +### Removed + +- The reference agent (`src/agent-harness.ts`, `benchmarks/todo-benchmark.test.ts`) — the `@deval` agent loop is the system under test now. The `AgentRun` contract moved into `src/agent/loop.ts`. +- The Local-harness capture pipeline: `src/capture-hook.ts`, `.github/hooks/`, and the `DeepEval: Install Capture Hook` command. Captures are written automatically by the agent loop. +- The `DeepEval: Scaffold Sample Benchmark` command. +- The GitHub Actions CI workflow; the README documents the local verification commands instead. + ## [0.1.0] First complete starter release. diff --git a/README.md b/README.md index aaa88cb..78e9926 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@ -# DeepEval for VS Code +# deval -A working starter for evaluating coding agents with [DeepEval](https://deepeval.com/docs/getting-started) TypeScript suites, run from VS Code. +A VS Code extension with its own benchmarkable coding agent. Chat with `@deval` and it runs a real tool-calling loop against your Chat model; every run is captured, and DeepEval TypeScript suites score the runs — trajectory and tool use — with deterministic metrics. -It gives you three things: +It gives you one thing: **an agent you can measure**. -1. **A reference agent harness** (`src/agent-harness.ts`) and a benchmark suite that evaluates it — the seam you replace with your own agent. -2. **A capture pipeline** for the VS Code **Local** agent harness (`.github/hooks/`, `.deepeval/vscode-agent-events/*.jsonl`) and a suite that evaluates a captured run. -3. **Commands that run both** and report pass/fail, plus a scaffolder for a new suite. +1. **The `@deval` agent** (`src/agent/`) — a tool-calling loop on `vscode.lm` with read-only workspace tools. No API key needed; the model comes from the Chat model picker. +2. **Automatic capture** — every run is written to `.deepeval/vscode-agent-events/.jsonl`. +3. **Benchmarks** — DeepEval suites that score the loop (scripted model) and score real captured runs (trajectory, tool resolution, policy). -The bundled suites use **deterministic metrics only**, so they run offline with no API key — locally and in CI. +The bundled suites use **deterministic metrics only**, so they run offline with no API key. ## Quick start @@ -23,8 +23,7 @@ Press `F5` to launch the Extension Development Host, then use the Command Palett | Command | What it does | | --- | --- | | `DeepEval: Run Test Suite` | Pick a `*.test.ts` / `*.spec.ts` suite and run it with `deepeval test run`. | -| `DeepEval: Evaluate Captured Local Agent Run` | Pick a suite **and** a captured `.jsonl`, then run the suite with `DEEPEVAL_VSCODE_EVENTS` pointing at that capture. | -| `DeepEval: Scaffold Sample Benchmark` | Write `benchmarks/my-agent.test.ts` — a self-contained starter suite — into the workspace, open it, and offer to install `deepeval` + `vitest` with `deval.packageManager`. | +| `DeepEval: Evaluate Captured Agent Run` | Pick a captured `.jsonl` from `.deepeval/vscode-agent-events/`, then run the captured-run suite with `DEEPEVAL_VSCODE_EVENTS` pointing at that capture. | Both run commands launch a VS Code task with the workspace folder as its working directory, and report the exit code in a notification when it finishes. They also refuse to launch `npx` in a workspace that has no `deepeval` of its own: npx would download an unpatched copy into its own cache, and the suite would still be missing its `vitest`, so the run offers **Install dependencies** instead of failing. @@ -36,14 +35,13 @@ Both run commands launch a VS Code task with the workspace folder as its working | `npm run lint` | ESLint over `src`, `benchmarks`, and the Vitest config. | | `npm run compile` | Type-check, lint, and bundle to `dist/extension.js`. | | `npm run benchmark` | `deepeval test run benchmarks` — both bundled suites. | -| `npm run benchmark:todo` | Only the reference-agent suite. | | `npm run benchmark:captured` | Only the captured-run suite (skips itself if `DEEPEVAL_VSCODE_EVENTS` is unset). | | `npm test` | Extension-host tests via `@vscode/test-electron`. | | `npm run package` | Production bundle for packaging. | -## Replace the agent under test +## The agent under test -`src/agent-harness.ts` is the only file you need to change to evaluate your own agent. It is deliberately dependency-free — it imports neither `vscode` nor `deepeval` — so the same file loads in the extension host and in the Vitest process that DeepEval spawns. +The system under test is the `@deval` agent loop itself (`src/agent/loop.ts`). It is deliberately dependency-free — it imports neither `vscode` nor `deepeval` — so the same loop runs in the extension host (via `src/agent/vscode-adapter.ts`), in unit tests with fake ports, and in the Vitest process that DeepEval spawns. ```ts export interface AgentRun { @@ -53,7 +51,7 @@ export interface AgentRun { } ``` -Return that shape from your implementation and every suite keeps working. `todoAppAgent` is a deterministic reference implementation that emits a single-file todo app. +`runAgentLoop()` takes the model, the tools, and the capture sink as ports and returns that shape, so every suite keeps working whatever drives the loop. To extend the agent, add a schema in `src/agent/tool-schemas.ts` and an implementation in `src/agent/tools.ts`, then declare it in the `languageModelTools` contribution in `package.json`. ## Instrument the agent so metrics can see it @@ -62,11 +60,11 @@ Model-judged and trajectory metrics do not read your return value — they read ```ts import { SpanType, observe, updateCurrentSpan, updateCurrentTrace } from 'deepeval/tracing'; -const observedAgent = observe({ +const observedAgentLoop = observe({ type: SpanType.AGENT, - name: 'my-agent', + name: 'deval-agent-loop', fn: async (input: string) => { - const run = await runAgent(input); + const run = await runAgentLoop(input, ports, { toolSchemas: DEVAL_TOOL_SCHEMAS }); const turn = { input: run.input, output: run.output, toolsCalled: run.toolsCalled }; updateCurrentSpan(turn); // component scope updateCurrentTrace(turn); // turn scope — this is what ToolCorrectnessMetric reads @@ -78,10 +76,10 @@ const observedAgent = observe({ Then assert against a golden: ```ts -const golden = new Golden({ input: 'Build a todo app.', expectedTools: expectedTodoTools() }); +const golden = new Golden({ input: 'What does the deval_readFile tool do?', expectedTools: [...] }); -await expect(golden).toPass([structuralChecks(), new ToolCorrectnessMetric({ threshold: 1 })], { - task: (testCase) => observedAgent(testCase.input), +await expect(golden).toPass([new ContainsAllMetric({ required: [...] }), new ToolCorrectnessMetric({ threshold: 1 })], { + task: (testCase) => observedAgentLoop(testCase.input), }); ``` @@ -94,7 +92,7 @@ await expect(golden).toPass([structuralChecks(), new ToolCorrectnessMetric({ thr - **`ContainsAllMetric`** — required/forbidden substrings; the score is the fraction satisfied. Used for structural checks on generated artifacts. - **`ToolCallResolutionMetric`** — the share of captured tool calls that returned a result, with a documented tolerance. -Extend `BaseMetric`, set `requiredParams`, compute a `score` in `measure`, and let `isSuccessful()` compare it to the threshold. Deterministic metrics need no key, which is what keeps CI green. +Extend `BaseMetric`, set `requiredParams`, compute a `score` in `measure`, and let `isSuccessful()` compare it to the threshold. Deterministic metrics need no key, which is what keeps the suites offline-friendly. For subjective checks, DeepEval's built-in judge metrics (`TaskCompletionMetric`, `GEval`, `ToolCorrectnessMetric` with `availableTools`, …) work as-is — they need `DEEPEVAL_API_KEY`: @@ -102,47 +100,60 @@ For subjective checks, DeepEval's built-in judge metrics (`TaskCompletionMetric` export DEEPEVAL_API_KEY=... ``` -## Capture the VS Code Local harness +## Captures -This extension ships a Preview Local hook at `.github/hooks/deval.json`. With **Local** selected as the session target, a trusted workspace, and `chat.useHooks` enabled, it records prompt, tool-use, tool-result, and stop events to `.deepeval/vscode-agent-events/.jsonl`: +Every `@deval` run is captured automatically — no hooks to install, nothing to enable. The agent loop records prompt, tool-use, tool-result, and stop events to `.deepeval/vscode-agent-events/.jsonl`: ```json -{"timestamp":"...","cwd":"...","session_id":"...","hook_event_name":"PreToolUse","tool_name":"read_file","tool_use_id":"call_...","tool_input":{...}} +{"timestamp":"...","session_id":"...","hook_event_name":"PreToolUse","tool_name":"deval_readFile","tool_use_id":"call_...","tool_input":{...}} ``` -`DeepEval: Evaluate Captured Local Agent Run` sets `DEEPEVAL_VSCODE_EVENTS` to the selected file's absolute path. A suite reads it with `resolveCapturedRunPath()` / `loadCapturedRun()` from `src/events/event-log.ts`, which parses the JSONL, pairs `PreToolUse` with `PostToolUse`, and summarizes the trajectory. +`DeepEval: Evaluate Captured Agent Run` sets `DEEPEVAL_VSCODE_EVENTS` to the selected file's absolute path. A suite reads it with `resolveCapturedRunPath()` / `loadCapturedRun()` from `src/events/event-log.ts`, which parses the JSONL, pairs `PreToolUse` with `PostToolUse`, and summarizes the trajectory. -The hook files are **not** inside the packaged extension: `.vscodeignore` excludes `.github/**`, so only this repository has them, and a workspace that only ever received a scaffolded benchmark has no hook to enable. Run `DeepEval: Install Capture Hook` to write `.github/hooks/deval.json` and `.github/hooks/deval-hook.cjs` into any workspace. The same button is offered by **Evaluate Captured Local Agent Run** when it finds no captures — alongside the exact directory it searched — and by **Scaffold Sample Benchmark** next to **Install dependencies**. Afterwards, enable `chat.useHooks`, confirm discovery with `Chat: Configure Hooks`, and send one turn in a **Local** session. +The captured-run suite evaluates the **trajectory** — prompts, tool names, pairing, completion, destructive-tool policy. A tool call that errored appears as a `PostToolUse` whose response carries the error, so interrupted runs stay evaluable. -The captured-run suite evaluates the **trajectory** — prompts, tool names, pairing, completion, destructive-tool policy — because the documented `Stop` payload does not include the response text. A suite that needs the prose must read `transcript_path`, whose format VS Code documents as unstable across releases. +**Privacy:** captures contain prompts, tool arguments, and tool results, so `.deepeval/` is Git-ignored. Do not commit it. -Two behaviours worth knowing: +## The `@deval` agent -- A tool call the user **denies** appears as a `PreToolUse` with no matching `PostToolUse`. That is normal, which is why `ToolCallResolutionMetric` allows a small unresolved share instead of failing on any dangling call. -- The Local harness documents eight events — `SessionStart`, `UserPromptSubmit`, `PreToolUse`, `PostToolUse`, `Stop`, `SubagentStart`, `SubagentStop`, `PreCompact`. This reader interprets the middle four (`HOOK_EVENT_NAMES`); the rest are written to the capture and ignored, so a run with subagents or compaction still evaluates. Run `Chat: Configure Hooks` to confirm which hook files the harness discovered, and check the **GitHub Copilot Chat Hooks** output channel for hook errors. -- Hooks are Preview and apply only to the VS Code Local harness; Copilot, Claude, and Codex Agent Host sessions have provider-specific hook behaviour. +`@deval` is a real tool-calling agent, not just a chat forwarder. When you send it a prompt, it runs an agentic loop against the model selected in Chat: -**Privacy:** captures contain prompts, tool arguments, tool results, and a `transcript_path` into your VS Code history, so `.deepeval/` is Git-ignored. Do not commit it. +1. The model decides whether it needs information, and calls a tool. +2. The extension executes the tool and feeds the result back. +3. The loop repeats until the model answers, or stops after 10 turns. -## The `@deval` chat participant +v1 ships three **read-only** tools, so the agent needs no confirmation UX: -`@deval` sends the conversation to the model selected in Chat and streams the response. It is an extension-owned chat participant, not a hook into GitHub Copilot's built-in agent mode. +| Tool | What it does | +| --- | --- | +| `deval_readFile` | Reads a workspace file's full text. | +| `deval_listFiles` | Lists a workspace directory, non-recursive. | +| `deval_grep` | Searches file contents for a pattern. | + +Every run is captured to `.deepeval/vscode-agent-events/.jsonl`. That means `DeepEval: Evaluate Captured Agent Run` scores the agent's trajectory with no extra wiring, and `benchmarks/agent-loop-benchmark.test.ts` measures the loop itself with a scripted model. + +The loop core (`src/agent/loop.ts`) is dependency-free: it takes the model, the tools, and the capture sink as ports, so the same code runs in the extension host (via `src/agent/vscode-adapter.ts`), in unit tests with fakes, and in benchmark suites. -The test runner executes **outside** the extension host, so it cannot call the participant's `vscode.lm` request. That is exactly why `src/agent-harness.ts` exists as a plain, importable seam: the suites call the harness, and you point the harness at whatever agent you want to measure. +It is an extension-owned agent, not a hook into GitHub Copilot's built-in agent mode. The model comes from the user's Chat model picker via the Language Model API — no API key needed. ## Repository layout ``` src/ - agent-harness.ts # the agent under test (replace this) - capture-hook.ts # the embedded capture hook, written into any workspace + agent/loop.ts # dependency-free agentic loop: model turns, tool dispatch, capture + agent/tool-schemas.ts # the read-only tools the @deval agent can call (dependency-free) + agent/tools.ts # vscode.lm tool registration + implementations + agent/vscode-adapter.ts # adapts the loop to vscode.lm (messages, tool calls, streaming) deepeval-patch.ts # repairs the deepeval@0.9.22 CLI in any workspace events/event-log.ts # JSONL capture parser + trajectory summarizer - extension.ts # commands, chat participant, task runner, scaffolder + events/event-writer.ts # writes captures in the format event-log.ts reads + agent/participant.ts # the @deval chat participant: wires the loop to vscode.lm + suites.ts # the run commands, suite discovery, capture picker, install flow + extension.ts # thin entry point: registers commands and the participant test/ # extension-host tests (Mocha, via `npm test`) benchmarks/ - todo-benchmark.test.ts # evaluates the reference agent through a trace - captured-run-benchmark.test.ts # evaluates a captured Local-harness run + captured-run-benchmark.test.ts # evaluates a captured @deval run + agent-loop-benchmark.test.ts # evaluates the @deval agent loop with a scripted model metrics/ # custom deterministic metrics scripts/patch-deepeval.mjs # see "Known upstream issue" below ``` @@ -156,7 +167,7 @@ scripts/patch-deepeval.mjs # see "Known upstream issue" below Two things repair it, so this is handled for you: - In **this repository**, `scripts/patch-deepeval.mjs` runs on `postinstall` and renames the shadowing files. `@sentry/node` is also listed as a devDependency so an install with `--ignore-scripts` still loads. -- In **any other workspace** — including one you just scaffolded — the extension renames the same files in-process (see `src/deepeval-patch.ts`) before every run, and again after **Install dependencies**. A packaged extension ships without `scripts/`, which is why this is done in-process rather than by spawning a script. +- In **any other workspace**, the extension renames the same files in-process (see `src/deepeval-patch.ts`) before every run, and again after **Install dependencies**. A packaged extension ships without `scripts/`, which is why this is done in-process rather than by spawning a script. A read-only `node_modules` or a future fixed release is tolerated; the patch is never the reason a run fails. Remove all of this once upstream publishes a fix. @@ -168,14 +179,21 @@ If the stack points at `%LOCALAPPDATA%\npm-cache\_npx\\node_modules\deepev node scripts/patch-deepeval.mjs "%LOCALAPPDATA%\npm-cache\_npx\" ``` -## CI +## Verification -`.github/workflows/ci.yml` type-checks, lints, bundles, runs the DeepEval suites, and runs the extension-host tests under `xvfb`. The deterministic suites mean no secrets are needed. +Run the checks locally — no CI, no secrets needed (the suites are deterministic): + +```sh +npm run check-types # type-check the extension and the benchmarks +npm run lint # ESLint over src, benchmarks, and the Vitest config +npm run benchmark # run the DeepEval suites +npm test # extension-host tests (needs a display; use xvfb-run headless) +``` ## Settings - `deval.deepevalCommand`: executable used to launch the DeepEval runner. Defaults to `npx`. -- `deval.packageManager`: package manager the scaffolder installs with — `npm`, `pnpm`, `yarn`, or `bun`. Defaults to `npm`, and is used only by **Install dependencies** on the scaffold notification. +- `deval.packageManager`: package manager used to install `deepeval` + `vitest` — `npm`, `pnpm`, `yarn`, or `bun`. Defaults to `npm`, and is used only by **Install dependencies** when a run needs it. These are deliberately separate settings. `deval.deepevalCommand` names the *runner* (`npx deepeval test run …`), so it cannot be reused to install: `npx install --save-dev deepeval vitest` makes npx look for an executable called `install` and fail with `could not determine executable to run`. diff --git a/benchmarks/agent-loop-benchmark.test.ts b/benchmarks/agent-loop-benchmark.test.ts new file mode 100644 index 0000000..b6ab5b7 --- /dev/null +++ b/benchmarks/agent-loop-benchmark.test.ts @@ -0,0 +1,101 @@ +/** + * Benchmark: does the `@deval` agent loop use its tools and answer? + * + * The model here is scripted, so this suite is deterministic and needs no + * `DEEPEVAL_API_KEY` — it measures the *loop* (tool dispatch, result + * feedback, capture events), not any particular model. Swap the scripted + * model for a real one to evaluate model behavior instead. + * + * Run it with: + * npm run benchmark + * or, from the Command Palette, `DeepEval: Run Test Suite`. + */ + +import { expect, it } from 'vitest'; +import 'deepeval/vitest'; +import { Golden } from 'deepeval/dataset'; +import { ToolCorrectnessMetric } from 'deepeval/metrics'; +import { SpanType, observe, updateCurrentSpan, updateCurrentTrace } from 'deepeval/tracing'; + +import { + runAgentLoop, + type EventPort, + type ModelPort, + type ModelTurn, +} from '../src/agent/loop'; +import { DEVAL_TOOL_SCHEMAS, READ_FILE_TOOL } from '../src/agent/tool-schemas'; +import { ContainsAllMetric } from './metrics/contains-all-metric'; + +/** + * A model with two turns baked in: first it reads a file, then it answers + * using what the tool returned. + */ +function scriptedModel(): ModelPort { + const turns: ModelTurn[] = [ + { + text: '', + toolCalls: [{ id: 'call_1', name: READ_FILE_TOOL, input: { path: 'src/agent/tool-schemas.ts' } }], + }, + { + text: 'The deval_readFile tool reads the full text contents of a file in the workspace.', + toolCalls: [], + }, + ]; + let index = 0; + return { + send: async () => turns[Math.min(index++, turns.length - 1)], + }; +} + +const silentEvents: EventPort = { + record: () => {}, +}; + +/** + * Run the loop the way the extension does, and publish the turn for metrics. + * + * Both levels are published on purpose: `updateCurrentSpan` fills the agent + * span, and `updateCurrentTrace` fills the turn itself — which is the scope + * `ToolCorrectnessMetric` reads `toolsCalled` from. + */ +const observedAgentLoop = observe({ + type: SpanType.AGENT, + name: 'deval-agent-loop', + fn: async (input: string) => { + const run = await runAgentLoop( + input, + { + model: scriptedModel(), + tools: { + invoke: async (name) => + name === READ_FILE_TOOL + ? 'Reads the full text contents of a file in the workspace.' + : `unknown tool: ${name}`, + }, + events: silentEvents, + }, + { toolSchemas: DEVAL_TOOL_SCHEMAS }, + ); + const turn = { input: run.input, output: run.output, toolsCalled: run.toolsCalled }; + updateCurrentSpan(turn); + updateCurrentTrace(turn); + return run.output; + }, +}); + +it('the agent loop reads a file and answers from it', async () => { + const golden = new Golden({ + input: 'What does the deval_readFile tool do?', + expectedTools: [{ name: READ_FILE_TOOL, inputParameters: { path: 'src/agent/tool-schemas.ts' } }], + }); + + await expect(golden).toPass( + [ + new ContainsAllMetric({ + required: ['deval_readFile', 'full text contents'], + }), + new ToolCorrectnessMetric({ threshold: 1 }), + ], + { task: (testCase) => observedAgentLoop(testCase.input) }, + ); +}); diff --git a/benchmarks/captured-run-benchmark.test.ts b/benchmarks/captured-run-benchmark.test.ts index 412875e..cb67bdc 100644 --- a/benchmarks/captured-run-benchmark.test.ts +++ b/benchmarks/captured-run-benchmark.test.ts @@ -1,7 +1,7 @@ /** - * Benchmark: evaluate a captured VS Code Local-harness run. + * Benchmark: evaluate a captured `@deval` agent run. * - * `DeepEval: Evaluate Captured Local Agent Run` picks a JSONL capture and sets + * `DeepEval: Evaluate Captured Agent Run` picks a JSONL capture and sets * `DEEPEVAL_VSCODE_EVENTS` to its absolute path before starting the suite. This * file reads that variable, turns the trajectory into a deterministic digest, * and judges the digest with DeepEval metrics. diff --git a/benchmarks/todo-benchmark.test.ts b/benchmarks/todo-benchmark.test.ts deleted file mode 100644 index 52f1b3b..0000000 --- a/benchmarks/todo-benchmark.test.ts +++ /dev/null @@ -1,101 +0,0 @@ -/** - * Benchmark: does the reference agent produce a working todo app? - * - * The metrics here are deterministic, so this suite needs no `DEEPEVAL_API_KEY` - * and runs identically in CI. Swap in a model-judged metric (for example - * `TaskCompletionMetric`) to add judgement-based checks. - * - * Run it with: - * npm run benchmark - * or, from the Command Palette, `DeepEval: Run Test Suite`. - */ - -import { expect, it } from 'vitest'; -import 'deepeval/vitest'; -import { Golden } from 'deepeval/dataset'; -import { ToolCorrectnessMetric } from 'deepeval/metrics'; -import { SpanType, observe, updateCurrentSpan, updateCurrentTrace } from 'deepeval/tracing'; - -import { expectedTodoTools, runAgent, TODO_AGENT_NAME } from '../src/agent-harness'; -import { ContainsAllMetric } from './metrics/contains-all-metric'; - -/** - * Wrap the harness in a DeepEval span. - * - * `expect(golden).toPass(..., { task })` judges the trace the task produces, so - * the task must run inside an `observe`d function and publish its I/O. That is - * the whole instrumentation story: one wrapper, no changes to the agent itself. - * - * Both levels are published on purpose: `updateCurrentSpan` fills the agent - * span (so metrics can run per component), and `updateCurrentTrace` fills the - * turn itself — which is the scope `ToolCorrectnessMetric` reads - * `toolsCalled` from. - */ -const observedTodoAgent = observe({ - type: SpanType.AGENT, - name: TODO_AGENT_NAME, - fn: async (input: string) => { - const run = await runAgent(input); - const turn = { input: run.input, output: run.output, toolsCalled: run.toolsCalled }; - updateCurrentSpan(turn); - updateCurrentTrace(turn); - return run.output; - }, -}); - -const todoBenchmarks = [ - 'Build a todo app with add, complete, and delete actions.', - 'Create a simple todo app with a task input, a list, and a done toggle.', - 'Build a todo manager that saves tasks in localStorage and marks items done.', -]; - -/** Fragments every todo app must contain, checked without a model. */ -function structuralChecks(prompt: string): ContainsAllMetric { - return new ContainsAllMetric({ - required: [ - '', - '
', - '', - 'localStorage.getItem(STORAGE_KEY)', - 'localStorage.setItem(STORAGE_KEY', - 'addTask(', - 'completeTask(', - 'removeTask(', - `data-prompt="${prompt}"`, - ], - forbidden: ['TODO:', 'FIXME', 'placeholder('], - }); -} - -it.each(todoBenchmarks)('todo app is complete for: %s', async (prompt) => { - const golden = new Golden({ - input: prompt, - expectedTools: expectedTodoTools(), - }); - - await expect(golden).toPass( - [structuralChecks(prompt), new ToolCorrectnessMetric({ threshold: 1 })], - { task: (testCase) => observedTodoAgent(testCase.input) }, - ); -}); - -it('deleting a task is wired to the rendered list', async () => { - const golden = new Golden({ - input: todoBenchmarks[0], - expectedTools: expectedTodoTools(), - }); - - await expect(golden).toPass( - [ - new ContainsAllMetric({ - required: [ - 'tasks = tasks.filter((task) => task.id !== id)', - "remove.addEventListener('click', () => removeTask(task.id))", - ], - }), - ], - { task: (testCase) => observedTodoAgent(testCase.input) }, - ); -}); - diff --git a/package.json b/package.json index ce81259..a212a91 100644 --- a/package.json +++ b/package.json @@ -37,15 +37,7 @@ }, { "command": "deval.evaluateAgentRun", - "title": "DeepEval: Evaluate Captured Local Agent Run" - }, - { - "command": "deval.scaffoldBenchmark", - "title": "DeepEval: Scaffold Sample Benchmark" - }, - { - "command": "deval.installCaptureHook", - "title": "DeepEval: Install Capture Hook" + "title": "DeepEval: Evaluate Captured Agent Run" } ], "chatParticipants": [ @@ -56,6 +48,57 @@ "description": "A local chat harness for evaluating agent behavior with DeepEval suites." } ], + "languageModelTools": [ + { + "name": "deval_readFile", + "displayName": "Read file", + "modelDescription": "Reads the full text contents of a file in the workspace. Input: { path: workspace-relative file path }.", + "inputSchema": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Workspace-relative path of the file to read, e.g. \"src/extension.ts\"." + } + }, + "required": ["path"] + } + }, + { + "name": "deval_listFiles", + "displayName": "List files", + "modelDescription": "Lists the files and directories directly inside a workspace folder, non-recursive. Input: { directory?: workspace-relative directory, defaults to the workspace root }.", + "inputSchema": { + "type": "object", + "properties": { + "directory": { + "type": "string", + "description": "Workspace-relative directory to list. Defaults to the workspace root." + } + }, + "required": [] + } + }, + { + "name": "deval_grep", + "displayName": "Search file contents", + "modelDescription": "Searches file contents for a text pattern across the workspace and returns matching lines. Input: { pattern: text or regular expression to search for, directory?: workspace-relative directory to search }.", + "inputSchema": { + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "The text or regular expression to search for." + }, + "directory": { + "type": "string", + "description": "Workspace-relative directory to search. Defaults to the workspace root." + } + }, + "required": ["pattern"] + } + } + ], "configuration": { "title": "DeepEval", "properties": { @@ -92,7 +135,6 @@ "check-types": "tsc --noEmit -p tsconfig.json && tsc --noEmit -p tsconfig.benchmarks.json", "lint": "eslint src benchmarks vitest.config.mts", "benchmark": "deepeval test run benchmarks", - "benchmark:todo": "deepeval test run benchmarks/todo-benchmark.test.ts", "benchmark:captured": "deepeval test run benchmarks/captured-run-benchmark.test.ts", "test": "vscode-test" }, diff --git a/src/agent-harness.ts b/src/agent-harness.ts deleted file mode 100644 index d732145..0000000 --- a/src/agent-harness.ts +++ /dev/null @@ -1,161 +0,0 @@ -/** - * The system under test. - * - * This module is deliberately dependency-free: it imports neither `vscode` nor - * `deepeval`, so the same harness can be loaded inside the extension host and - * inside a Vitest process started by `npx deepeval test run`. - * - * Replace `todoAppAgent` with your own implementation to evaluate a real agent. - * Everything else (the benchmark suites, the metrics, the hook pipeline) stays - * the same, because the suites only depend on the `AgentRun` shape below. - */ - -/** A tool invocation an agent made while producing its artifact. */ -export interface AgentToolCall { - name: string; - inputParameters?: Record; - output?: unknown; -} - -/** One completed agent run, in the shape the benchmark suites consume. */ -export interface AgentRun { - /** The prompt the agent was asked to satisfy. */ - input: string; - /** The artifact the agent produced, as text. */ - output: string; - /** The tools the agent used to get there, in call order. */ - toolsCalled: AgentToolCall[]; -} - -/** A swappable agent implementation. */ -export interface AgentHarness { - readonly name: string; - run(input: string): Promise; -} - -export const TODO_AGENT_NAME = 'todo-app-agent'; - -/** Escape a prompt so it can be embedded in markup without breaking it. */ -export function escapeHtml(value: string): string { - return value - .replace(/&/g, '&') - .replace(//g, '>') - .replace(/"/g, '"') - .replace(/'/g, '''); -} - -/** The tools every `todo-app-agent` run is expected to call, in order. */ -export function expectedTodoTools(): AgentToolCall[] { - return [ - { name: 'create_file', inputParameters: { path: 'index.html' } }, - { name: 'read_file', inputParameters: { path: 'index.html' } }, - ]; -} - -/** - * Produce the single-file todo app the reference agent emits. - * - * The output is deterministic for a given prompt, except for the prompt marker - * echoed into `data-prompt`, which keeps runs input-dependent on purpose. - */ -export function createTodoArtifact(prompt: string): AgentRun { - const output = ` - - - - -Todo - - -

Todo

- - - - -
    - - - -`; - - return { - input: prompt, - output, - toolsCalled: expectedTodoTools(), - }; -} - -/** The reference agent. Swap this out to evaluate your own implementation. */ -export const todoAppAgent: AgentHarness = { - name: TODO_AGENT_NAME, - async run(input: string): Promise { - return createTodoArtifact(input); - }, -}; - -/** The harness used when a caller does not name one. */ -export const defaultHarness: AgentHarness = todoAppAgent; - -/** Run the harness under evaluation and return its artifact. */ -export async function runAgent(input: string, harness: AgentHarness = defaultHarness): Promise { - return harness.run(input); -} diff --git a/src/agent/loop.ts b/src/agent/loop.ts new file mode 100644 index 0000000..e9a7698 --- /dev/null +++ b/src/agent/loop.ts @@ -0,0 +1,169 @@ +/** + * The `@deval` agent loop, as dependency-free logic. + * + * This module imports neither `vscode` nor `deepeval`: the model, the tools, + * and the capture sink arrive as ports, so the same loop runs inside the + * extension host (wired to `vscode.lm` by `vscode-adapter.ts`), in unit tests + * with fakes, and in benchmark suites with a scripted model. + * + * The loop speaks the same capture language as the Local-harness hook: every + * turn records `UserPromptSubmit` / `PreToolUse` / `PostToolUse` / `Stop` + * events through the event port, so a run of this loop is a capture that + * `src/events/event-log.ts` already knows how to read and the captured-run + * benchmark already knows how to score. + */ + +import type { AgentEvent } from '../events/event-log'; +import type { DevalToolSchema } from './tool-schemas'; + +/** A tool invocation an agent made while producing its artifact. */ +export interface AgentToolCall { + name: string; + inputParameters?: Record; + output?: unknown; +} + +/** One completed agent run, in the shape the benchmark suites consume. */ +export interface AgentRun { + /** The prompt the agent was asked to satisfy. */ + input: string; + /** The artifact the agent produced, as text. */ + output: string; + /** The tools the agent used to get there, in call order. */ + toolsCalled: AgentToolCall[]; +} + +/** One message in the agent's conversation. */ +export type LoopMessage = + | { role: 'user'; text: string } + | { role: 'assistant'; text: string } + | { role: 'tool-call'; id: string; name: string; input: unknown } + | { role: 'tool-result'; id: string; output: unknown }; + +/** One tool call the model requested this turn. */ +export interface RequestedToolCall { + id: string; + name: string; + input: unknown; +} + +/** What the model returned for a single turn. */ +export interface ModelTurn { + /** Text the model produced this turn, possibly empty. */ + text: string; + /** Tool calls the model requested this turn, in order. */ + toolCalls: RequestedToolCall[]; +} + +/** Sends the conversation to the model. Implemented by `vscode-adapter.ts`. */ +export interface ModelPort { + send(messages: LoopMessage[], tools: DevalToolSchema[]): Promise; +} + +/** Executes a tool by name. Implemented by `vscode-adapter.ts`. */ +export interface ToolPort { + invoke(name: string, input: unknown): Promise; +} + +/** Records capture events. Implemented by `event-writer.ts`. */ +export interface EventPort { + record(event: Omit & { timestamp?: string }): void; +} + +export interface AgentLoopOptions { + /** Tool schemas offered to the model. */ + toolSchemas: DevalToolSchema[]; + /** Turns before the loop gives up. Defaults to `DEFAULT_MAX_TURNS`. */ + maxTurns?: number; + /** System instruction placed before the user prompt. */ + systemPrompt?: string; + /** Called with each text chunk the model streams, for live display. */ + onTextChunk?: (chunk: string) => void; +} + +/** How many model turns the loop attempts before stopping. */ +export const DEFAULT_MAX_TURNS = 10; + +/** True when the loop stopped because it ran out of turns. */ +export const LOOP_CUTOFF_MESSAGE = 'The agent reached the turn limit before finishing.'; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** + * Run one agentic turn loop for `input`. + * + * Returns an `AgentRun` — the same shape the benchmark suites consume — so the + * agent this loop drives is directly measurable. Capture events are recorded + * through the event port; a `Stop` event is recorded only when the model + * finishes on its own, so a run that hits the turn limit reads as incomplete. + */ +export async function runAgentLoop( + input: string, + ports: { model: ModelPort; tools: ToolPort; events: EventPort }, + options: AgentLoopOptions, +): Promise { + const maxTurns = options.maxTurns ?? DEFAULT_MAX_TURNS; + const messages: LoopMessage[] = []; + if (options.systemPrompt) { + messages.push({ role: 'user', text: options.systemPrompt }); + } + messages.push({ role: 'user', text: input }); + ports.events.record({ hook_event_name: 'UserPromptSubmit', prompt: input }); + + const toolsCalled: AgentToolCall[] = []; + const outputParts: string[] = []; + let finished = false; + + for (let turn = 0; turn < maxTurns && !finished; turn += 1) { + const { text, toolCalls } = await ports.model.send(messages, options.toolSchemas); + + if (text) { + outputParts.push(text); + messages.push({ role: 'assistant', text }); + options.onTextChunk?.(text); + } + + if (toolCalls.length === 0) { + finished = true; + break; + } + + for (const call of toolCalls) { + const inputParameters = isRecord(call.input) ? call.input : undefined; + ports.events.record({ + hook_event_name: 'PreToolUse', + tool_name: call.name, + tool_input: call.input, + tool_use_id: call.id, + }); + + let output: unknown; + try { + output = await ports.tools.invoke(call.name, call.input); + } catch (error) { + output = { error: error instanceof Error ? error.message : String(error) }; + } + + ports.events.record({ + hook_event_name: 'PostToolUse', + tool_name: call.name, + tool_input: call.input, + tool_use_id: call.id, + tool_response: output, + }); + toolsCalled.push({ name: call.name, inputParameters, output }); + messages.push({ role: 'tool-call', id: call.id, name: call.name, input: call.input }); + messages.push({ role: 'tool-result', id: call.id, output }); + } + } + + if (finished) { + ports.events.record({ hook_event_name: 'Stop' }); + } else { + outputParts.push(`\n\n${LOOP_CUTOFF_MESSAGE}`); + } + + return { input, output: outputParts.join(''), toolsCalled }; +} diff --git a/src/agent/participant.ts b/src/agent/participant.ts new file mode 100644 index 0000000..e64d6e9 --- /dev/null +++ b/src/agent/participant.ts @@ -0,0 +1,66 @@ +// The `@deval` chat participant: wires the dependency-free agent loop up to +// VS Code's Language Model API and the read-only workspace tools. +import * as vscode from 'vscode'; + +import { createCaptureWriter } from '../events/event-writer'; +import { runAgentLoop, type EventPort } from './loop'; +import { DEVAL_TOOL_SCHEMAS } from './tool-schemas'; +import { registerDevalTools } from './tools'; +import { createVscodeModelPort, createVscodeToolPort } from './vscode-adapter'; + +/** System instruction for the `@deval` agent loop. */ +export const DEVAL_AGENT_SYSTEM_PROMPT = + 'You are Deval, a helpful coding assistant inside VS Code. ' + + 'You have read-only tools to inspect the workspace: read files, list directories, and search file contents. ' + + 'Use them to ground your answers in the actual workspace instead of guessing. ' + + 'Call a tool whenever you need information you do not already have, then answer concisely.'; + +/** + * Run the `@deval` agent: an agentic loop over the chat model with tools. + * + * Every run is captured to `.deepeval/vscode-agent-events/.jsonl` + * in the workspace's own capture format, so `DeepEval: Evaluate Captured + * Agent Run` can score the agent's trajectory without any extra wiring. + */ +export async function registerChatParticipant(context: vscode.ExtensionContext) { + const participant = vscode.chat.createChatParticipant( + 'deval.agent', + async (request, _chatContext, response, token) => { + const folders = vscode.workspace.workspaceFolders; + if (!folders?.length) { + response.markdown('Open a workspace folder to use the deval agent.'); + return; + } + + const writer = createCaptureWriter(folders[0].uri.fsPath); + const events: EventPort = { + record: (event) => writer.append(event), + }; + const model = createVscodeModelPort(request.model, token, (chunk) => + response.markdown(chunk), + ); + const tools = createVscodeToolPort(token, request.toolInvocationToken); + + response.progress('Deval agent is working…'); + try { + const run = await runAgentLoop( + request.prompt, + { model, tools, events }, + { toolSchemas: DEVAL_TOOL_SCHEMAS, systemPrompt: DEVAL_AGENT_SYSTEM_PROMPT }, + ); + if (!run.output.trim()) { + response.markdown('_The agent finished without producing text._'); + } + response.markdown( + `\n\n---\nFinished with ${run.toolsCalled.length} tool call(s). ` + + 'This run was captured for benchmarking.', + ); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + response.markdown(`\n\nThe agent loop failed: ${message}`); + } + }, + ); + + context.subscriptions.push(participant, registerDevalTools()); +} diff --git a/src/agent/tool-schemas.ts b/src/agent/tool-schemas.ts new file mode 100644 index 0000000..4a82207 --- /dev/null +++ b/src/agent/tool-schemas.ts @@ -0,0 +1,81 @@ +/** + * The tools the `@deval` agent can call, as dependency-free schemas. + * + * This module imports neither `vscode` nor `deepeval`, so the schemas can be + * shared between the extension host (tool registration in `tools.ts`), the + * agent loop (`loop.ts`), and the benchmark suites. + * + * v1 is deliberately read-only: no tool here can modify the workspace, so the + * agent needs no confirmation UX. Write and execute tools belong in a v2 behind + * an explicit user confirmation. + */ + +/** A tool the agent loop can offer to the model. */ +export interface DevalToolSchema { + /** Unique tool name, also used for `vscode.lm.registerTool`. */ + name: string; + /** What the tool does, shown to the model. */ + description: string; + /** JSON schema for the tool's input. */ + inputSchema: object; +} + +export const READ_FILE_TOOL = 'deval_readFile'; +export const LIST_FILES_TOOL = 'deval_listFiles'; +export const GREP_TOOL = 'deval_grep'; + +/** Every tool the v1 agent loop offers, in the order they are presented. */ +export const DEVAL_TOOL_SCHEMAS: DevalToolSchema[] = [ + { + name: READ_FILE_TOOL, + description: + 'Reads the full text contents of a file in the workspace. ' + + 'Use this to inspect code, docs, or configuration before answering.', + inputSchema: { + type: 'object', + properties: { + path: { + type: 'string', + description: 'Workspace-relative path of the file to read, e.g. "src/extension.ts".', + }, + }, + required: ['path'], + }, + }, + { + name: LIST_FILES_TOOL, + description: + 'Lists the files and directories directly inside a workspace folder, non-recursive. ' + + 'Use this to discover what exists before reading.', + inputSchema: { + type: 'object', + properties: { + directory: { + type: 'string', + description: 'Workspace-relative directory to list. Defaults to the workspace root.', + }, + }, + required: [], + }, + }, + { + name: GREP_TOOL, + description: + 'Searches file contents for a text pattern across the workspace and returns matching lines. ' + + 'Use this to find where something is defined or used.', + inputSchema: { + type: 'object', + properties: { + pattern: { + type: 'string', + description: 'The text or regular expression to search for.', + }, + directory: { + type: 'string', + description: 'Workspace-relative directory to search. Defaults to the workspace root.', + }, + }, + required: ['pattern'], + }, + }, +]; diff --git a/src/agent/tools.ts b/src/agent/tools.ts new file mode 100644 index 0000000..e7275e1 --- /dev/null +++ b/src/agent/tools.ts @@ -0,0 +1,162 @@ +/** + * The `@deval` agent's tools, implemented for the extension host. + * + * Schemas live in the dependency-free `tool-schemas.ts`; this module registers + * them with `vscode.lm` and implements their bodies. v1 is read-only on + * purpose: no tool here can modify the workspace, so the agent needs no + * confirmation UX. + */ + +import * as vscode from 'vscode'; +import { readdir, readFile } from 'node:fs/promises'; +import { join, relative, resolve, sep } from 'node:path'; + +import { + DEVAL_TOOL_SCHEMAS, + GREP_TOOL, + LIST_FILES_TOOL, + READ_FILE_TOOL, +} from './tool-schemas'; + +/** Largest file the read tool returns before truncating. */ +const MAX_FILE_BYTES = 100_000; + +/** Most grep matches returned for one call. */ +const MAX_GREP_MATCHES = 50; + +/** Directories the grep tool never descends into. */ +const GREP_IGNORED_DIRS = new Set(['node_modules', '.git', 'dist', 'out', '.vscode-test', '.deepeval']); + +function workspaceRoot(): string { + const folder = vscode.workspace.workspaceFolders?.[0]; + if (!folder) { + throw new Error('Open a workspace folder to use the deval agent tools.'); + } + return folder.uri.fsPath; +} + +/** + * Resolve a workspace-relative path, refusing anything that escapes the + * workspace. The model is not trusted with absolute paths or `..`. + */ +function resolveInWorkspace(root: string, relativePath: string): string { + const absolute = resolve(root, relativePath); + if (absolute !== root && !absolute.startsWith(root + sep)) { + throw new Error(`Path escapes the workspace: ${relativePath}`); + } + return absolute; +} + +function asStringRecord(input: unknown): Record { + if (typeof input !== 'object' || input === null) { + return {}; + } + const record: Record = {}; + for (const [key, value] of Object.entries(input)) { + if (typeof value === 'string') { + record[key] = value; + } + } + return record; +} + +async function readFileTool(input: unknown): Promise { + const { path } = asStringRecord(input); + if (!path) { + throw new Error('readFile needs a "path".'); + } + const absolute = resolveInWorkspace(workspaceRoot(), path); + const content = await readFile(absolute, 'utf8'); + return content.length > MAX_FILE_BYTES + ? `${content.slice(0, MAX_FILE_BYTES)}\n…[truncated after ${MAX_FILE_BYTES} bytes]` + : content; +} + +async function listFilesTool(input: unknown): Promise { + const { directory } = asStringRecord(input); + const absolute = resolveInWorkspace(workspaceRoot(), directory || '.'); + const entries = await readdir(absolute, { withFileTypes: true }); + if (!entries.length) { + return '(empty)'; + } + return entries.map((entry) => `${entry.isDirectory() ? 'dir ' : 'file'} ${entry.name}`).join('\n'); +} + +async function grepTool(input: unknown): Promise { + const { pattern, directory } = asStringRecord(input); + if (!pattern) { + throw new Error('grep needs a "pattern".'); + } + const root = workspaceRoot(); + const start = resolveInWorkspace(root, directory || '.'); + let regex: RegExp; + try { + regex = new RegExp(pattern, 'm'); + } catch { + throw new Error(`Invalid regular expression: ${pattern}`); + } + + const matches: string[] = []; + async function walk(dir: string): Promise { + if (matches.length >= MAX_GREP_MATCHES) { + return; + } + const entries = await readdir(dir, { withFileTypes: true }); + for (const entry of entries) { + if (matches.length >= MAX_GREP_MATCHES) { + return; + } + const full = join(dir, entry.name); + if (entry.isDirectory()) { + if (!GREP_IGNORED_DIRS.has(entry.name)) { + await walk(full); + } + } else if (entry.isFile()) { + let content: string; + try { + content = await readFile(full, 'utf8'); + } catch { + continue; + } + if (content.includes('\0')) { + continue; + } + content.split('\n').forEach((line, index) => { + if (matches.length < MAX_GREP_MATCHES && regex.test(line)) { + matches.push(`${relative(root, full)}:${index + 1}: ${line.trim().slice(0, 200)}`); + } + }); + } + } + } + await walk(start); + return matches.length ? matches.join('\n') : '(no matches)'; +} + +function toolResult(text: string): vscode.LanguageModelToolResult { + return new vscode.LanguageModelToolResult([new vscode.LanguageModelTextPart(text)]); +} + +/** + * Register the deval tools with `vscode.lm`. + * + * Each tool must also be declared in the `languageModelTools` contribution + * point in `package.json`; registration here provides the implementation. + */ +export function registerDevalTools(): vscode.Disposable { + const implementations: Record Promise> = { + [READ_FILE_TOOL]: readFileTool, + [LIST_FILES_TOOL]: listFilesTool, + [GREP_TOOL]: grepTool, + }; + + const disposables = DEVAL_TOOL_SCHEMAS.map((schema) => + vscode.lm.registerTool(schema.name, { + prepareInvocation: (options) => ({ + invocationMessage: `deval: ${schema.name}`, + }), + invoke: async (options) => toolResult(await implementations[schema.name](options.input)), + }), + ); + return vscode.Disposable.from(...disposables); +} diff --git a/src/agent/vscode-adapter.ts b/src/agent/vscode-adapter.ts new file mode 100644 index 0000000..028d49b --- /dev/null +++ b/src/agent/vscode-adapter.ts @@ -0,0 +1,113 @@ +/** + * Adapts the dependency-free agent loop (`loop.ts`) to `vscode.lm`. + * + * The loop core speaks `LoopMessage`s; this module translates them to and from + * `vscode.LanguageModelChatMessage`s on every model call, and executes tools + * through `vscode.lm.invokeTool` so the chat UI shows the invocations. + */ + +import * as vscode from 'vscode'; + +import type { + EventPort, + LoopMessage, + ModelPort, + ModelTurn, + RequestedToolCall, + ToolPort, +} from './loop'; +import type { DevalToolSchema } from './tool-schemas'; + +export type { EventPort }; + +function stringifyToolOutput(output: unknown): string { + return typeof output === 'string' ? output : JSON.stringify(output, null, 2); +} + +function toVscodeMessage(message: LoopMessage): vscode.LanguageModelChatMessage { + switch (message.role) { + case 'user': + return vscode.LanguageModelChatMessage.User(message.text); + case 'assistant': + return vscode.LanguageModelChatMessage.Assistant(message.text); + case 'tool-call': { + const input = typeof message.input === 'object' && message.input !== null ? message.input : {}; + return vscode.LanguageModelChatMessage.Assistant([ + new vscode.LanguageModelToolCallPart(message.id, message.name, input), + ]); + } + case 'tool-result': + return vscode.LanguageModelChatMessage.User([ + new vscode.LanguageModelToolResultPart(message.id, [ + new vscode.LanguageModelTextPart(stringifyToolOutput(message.output)), + ]), + ]); + } +} + +/** + * A `ModelPort` backed by a `vscode.LanguageModelChat`. + * + * Text streams to `onTextChunk` as it arrives, so the chat participant can + * display the agent thinking in real time. + */ +export function createVscodeModelPort( + model: vscode.LanguageModelChat, + token: vscode.CancellationToken, + onTextChunk: (chunk: string) => void, +): ModelPort { + return { + async send(messages: LoopMessage[], tools: DevalToolSchema[]): Promise { + const response = await model.sendRequest( + messages.map(toVscodeMessage), + { + justification: 'deval runs its agent loop to answer your request.', + tools: tools.map((tool) => ({ + name: tool.name, + description: tool.description, + inputSchema: tool.inputSchema, + })), + }, + token, + ); + + let text = ''; + const toolCalls: RequestedToolCall[] = []; + for await (const part of response.stream) { + if (part instanceof vscode.LanguageModelTextPart) { + text += part.value; + onTextChunk(part.value); + } else if (part instanceof vscode.LanguageModelToolCallPart) { + toolCalls.push({ id: part.callId, name: part.name, input: part.input }); + } + } + return { text, toolCalls }; + }, + }; +} + +/** + * A `ToolPort` backed by `vscode.lm.invokeTool`. + * + * `toolInvocationToken` comes from the chat request and makes the chat UI + * show each invocation; it is `undefined` outside a chat participant. + */ +export function createVscodeToolPort( + token: vscode.CancellationToken, + toolInvocationToken: vscode.ChatParticipantToolToken | undefined, +): ToolPort { + return { + async invoke(name: string, input: unknown): Promise { + const result = await vscode.lm.invokeTool( + name, + { input: (input ?? {}) as object, toolInvocationToken }, + token, + ); + return result.content + .map((part) => + part instanceof vscode.LanguageModelTextPart ? part.value : JSON.stringify(part), + ) + .join('\n'); + }, + }; +} diff --git a/src/capture-hook.ts b/src/capture-hook.ts deleted file mode 100644 index c153e95..0000000 --- a/src/capture-hook.ts +++ /dev/null @@ -1,144 +0,0 @@ -/** - * The Local-harness capture hook, embedded as text so *any* workspace can get it. - * - * `.vscodeignore` excludes `.github/**`, so the packaged extension does not ship - * the repository's hook files. A workspace scaffolded by `DeepEval: Scaffold - * Sample Benchmark` therefore has nothing for `DeepEval: Evaluate Captured Local - * Agent Run` to find, and telling the user to "enable the workspace hook" points - * them at a file that was never written there. `installCaptureHook` in - * `extension.ts` writes what this module returns instead of copying from the - * extension's own install directory, which is the one place the files cannot be - * read from once packaged. - * - * The copies under this repository's `.github/hooks/` remain the reference - * implementation: `src/test/capture-hook.test.ts` compares both byte for byte, - * so the embedded text cannot drift from what the repository documents. - */ - -/** Events `deval-hook.cjs` records and `deval.json` subscribes the script to. */ -export const CAPTURE_HOOK_EVENTS = [ - 'UserPromptSubmit', - 'PreToolUse', - 'PostToolUse', - 'Stop', -] as const; - -/** Folder holding the hook files, relative to the workspace root. */ -export const CAPTURE_HOOK_DIRECTORY = ['.github', 'hooks'] as const; - -/** The hook registration file the Local harness discovers. */ -export const CAPTURE_HOOK_CONFIG_FILE = 'deval.json'; - -/** The script every hook action runs. */ -export const CAPTURE_HOOK_SCRIPT_FILE = 'deval-hook.cjs'; - -/** One file `DeepEval: Install Capture Hook` writes into the workspace. */ -export interface CaptureHookFile { - /** Workspace-relative path segments, starting at the workspace root. */ - segments: string[]; - /** The exact file content, newlines included. */ - content: string; -} - -/** - * The command every hook action runs. - * - * Built from the path constants so the config cannot name a script the - * installer writes somewhere else, and so the paths stay forward-slashed the - * way the harness expects them in JSON. - */ -export const CAPTURE_HOOK_COMMAND = `node ${[...CAPTURE_HOOK_DIRECTORY, CAPTURE_HOOK_SCRIPT_FILE].join('/')}`; - -/** - * `.github/hooks/deval.json` as text. - * - * Generated from `CAPTURE_HOOK_EVENTS` rather than pasted, so subscribing a new - * event is a one-line change here, and stringified with two-space indentation so - * the result matches the repository's copy byte for byte. - */ -export function getCaptureHookConfigSource(): string { - const hooks = Object.fromEntries( - CAPTURE_HOOK_EVENTS.map((event) => [ - event, - [{ type: 'command', command: CAPTURE_HOOK_COMMAND, cwd: '.', timeout: 10 }], - ]), - ); - return `${JSON.stringify({ hooks }, null, 2)}\n`; -} - -/** - * `.github/hooks/deval-hook.cjs` as text. - * - * The script appends one JSON record per event to - * `/.deepeval/vscode-agent-events/.jsonl`, which is exactly - * where `CAPTURED_RUN_DIRECTORY` points and where `src/events/event-log.ts` - * reads from, so its constant names are asserted against the real directories in - * the tests. - * - * Written as a template literal with the script's own backticks, `${…}`, and - * `\n` escaped; the byte-comparison test is what keeps those escapes honest. - */ -export function getCaptureHookScriptSource(): string { - return `const fs = require('node:fs'); -const path = require('node:path'); - -let input = ''; -process.stdin.setEncoding('utf8'); -process.stdin.on('data', (chunk) => { - input += chunk; -}); -process.stdin.on('end', () => { - try { - const event = JSON.parse(input); - if (!event || typeof event !== 'object' || Array.isArray(event)) { - throw new Error('Hook input must be a JSON object.'); - } - - const rawSessionId = typeof event.session_id === 'string' ? event.session_id : 'unknown'; - const sessionId = rawSessionId.replace(/[^a-zA-Z0-9_-]/g, '_'); - const eventFields = [ - 'timestamp', - 'cwd', - 'session_id', - 'hook_event_name', - 'transcript_path', - 'prompt', - 'tool_name', - 'tool_input', - 'tool_use_id', - 'tool_response', - 'stop_hook_active', - ]; - const record = Object.fromEntries( - eventFields - .filter((field) => Object.hasOwn(event, field)) - .map((field) => [field, event[field]]), - ); - const outputDirectory = path.join(process.cwd(), '.deepeval', 'vscode-agent-events'); - fs.mkdirSync(outputDirectory, { recursive: true }); - fs.appendFileSync( - path.join(outputDirectory, \`\${sessionId}.jsonl\`), - \`\${JSON.stringify(record)}\\n\`, - { encoding: 'utf8', mode: 0o600 }, - ); - } catch (error) { - console.error(\`DeepEval hook could not record the event: \${error.message}\`); - process.exitCode = 1; - } -}); -`; -} - -/** Every file `DeepEval: Install Capture Hook` writes, in write order. */ -export function getCaptureHookFiles(): CaptureHookFile[] { - return [ - { - segments: [...CAPTURE_HOOK_DIRECTORY, CAPTURE_HOOK_CONFIG_FILE], - content: getCaptureHookConfigSource(), - }, - { - segments: [...CAPTURE_HOOK_DIRECTORY, CAPTURE_HOOK_SCRIPT_FILE], - content: getCaptureHookScriptSource(), - }, - ]; -} diff --git a/src/events/event-log.ts b/src/events/event-log.ts index 8a2ce03..7e7e4b8 100644 --- a/src/events/event-log.ts +++ b/src/events/event-log.ts @@ -1,7 +1,8 @@ /** - * Reader for the JSONL files written by `.github/hooks/deval-hook.cjs`. + * Reader for the JSONL files written by the `@deval` agent loop + * (`src/events/event-writer.ts`). * - * Like `agent-harness.ts`, this module is dependency-free so it can be loaded + * Like `src/agent/loop.ts`, this module is dependency-free so it can be loaded * both in the extension host and in a plain Node process started by the * DeepEval test runner. * @@ -278,7 +279,7 @@ export function loadCapturedRun(filePath: string): { parsed: ParsedEventLog; sum * * Returns `undefined` when the variable is unset, which is the normal case when * a suite is launched directly through `npx deepeval test run` rather than - * through the `DeepEval: Evaluate Captured Local Agent Run` command. + * through the `DeepEval: Evaluate Captured Agent Run` command. */ export function resolveCapturedRunPath(env: NodeJS.ProcessEnv = process.env): string | undefined { const value = env[DEEPEVAL_VSCODE_EVENTS]; diff --git a/src/events/event-writer.ts b/src/events/event-writer.ts new file mode 100644 index 0000000..7634264 --- /dev/null +++ b/src/events/event-writer.ts @@ -0,0 +1,68 @@ +/** + * Writer for the JSONL captures `src/events/event-log.ts` reads. + * + * Like `event-log.ts`, this module is dependency-free: it imports neither + * `vscode` nor `deepeval`, so the same writer can be used inside the extension + * host and in a plain Node process. + * + * The hook script (`.github/hooks/deval-hook.cjs`) writes captures for the + * Local harness; this writer produces the same record shape for runs the + * extension itself drives — currently the `@deval` agent loop in + * `src/agent/loop.ts`. Either way, `loadCapturedRun()` and the captured-run + * benchmark consume the file unchanged. + */ + +import { appendFileSync, mkdirSync } from 'node:fs'; +import { randomUUID } from 'node:crypto'; +import { join } from 'node:path'; + +import type { AgentEvent } from './event-log'; + +/** A capture file being written. */ +export interface CaptureWriter { + /** The session this writer records. */ + readonly sessionId: string; + /** Absolute path of the `.jsonl` file being appended to. */ + readonly filePath: string; + /** + * Append one event. + * + * `session_id` and `timestamp` are filled in when absent, so callers only + * pass the hook fields. + */ + append(event: Omit & { timestamp?: string }): void; +} + +/** Make a session id safe to use as a file name. */ +export function sanitizeSessionId(sessionId: string): string { + return sessionId.replace(/[^a-zA-Z0-9_-]/g, '_'); +} + +/** + * Open a capture for writing. + * + * The file is `/.deepeval/vscode-agent-events/.jsonl` + * — the same directory `CAPTURED_RUN_DIRECTORY` points at, so + * `DeepEval: Evaluate Captured Agent Run` offers agent-loop runs without + * any extra wiring. The directory is created on open; each `append` adds one + * JSON object per line, with mode `0o600` like the hook script. + */ +export function createCaptureWriter(workspaceDir: string, sessionId: string = randomUUID()): CaptureWriter { + const safeId = sanitizeSessionId(sessionId); + const directory = join(workspaceDir, '.deepeval', 'vscode-agent-events'); + mkdirSync(directory, { recursive: true }); + const filePath = join(directory, `${safeId}.jsonl`); + + return { + sessionId: safeId, + filePath, + append(event) { + const record: AgentEvent = { + ...event, + timestamp: event.timestamp ?? new Date().toISOString(), + session_id: safeId, + }; + appendFileSync(filePath, `${JSON.stringify(record)}\n`, { encoding: 'utf8', mode: 0o600 }); + }, + }; +} diff --git a/src/extension.ts b/src/extension.ts index de10375..b2089ad 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -1,652 +1,18 @@ // The module 'vscode' contains the VS Code extensibility API // Import the module and reference it with the alias vscode in your code below -import * as vscode from 'vscode'; +import { commands, ExtensionContext } from 'vscode'; -import { - CAPTURE_HOOK_CONFIG_FILE, - CAPTURE_HOOK_DIRECTORY, - getCaptureHookFiles, -} from './capture-hook'; -import { hasDeepEvalInstall, patchDeepEvalInstall } from './deepeval-patch'; -import { DEEPEVAL_VSCODE_EVENTS } from './events/event-log'; +import { registerChatParticipant } from './agent/participant'; +import { runDeepEval } from './suites'; -/** Folder the hook writes captures into, relative to the workspace root. */ -export const CAPTURED_RUN_DIRECTORY = ['.deepeval', 'vscode-agent-events']; +export function activate(context: ExtensionContext) { + const rc = commands.registerCommand; -/** Globs searched for DeepEval TypeScript suites. */ -export const DEEPEVAL_TEST_PATTERNS = [ - '**/*.test.ts', - '**/*.spec.ts', - '**/benchmarks/**/*.test.ts', - '**/src/**/*.test.ts', - '**/test/**/*.test.ts', -]; - -/** Paths that never contain a DeepEval suite worth running from here. */ -export const DEEPEVAL_TEST_EXCLUDES = '**/{node_modules,dist,out,.vscode-test}/**'; - -export function getDeepEvalTestPatterns(): string[] { - return [...DEEPEVAL_TEST_PATTERNS]; -} - -export function mergeUniqueUris(...groups: vscode.Uri[][]): vscode.Uri[] { - const seen = new Map(); - for (const group of groups) { - for (const uri of group) { - seen.set(uri.toString(), uri); - } - } - return [...seen.values()]; -} - -export async function findDeepEvalTestFiles(): Promise { - const discoveryGroups = await Promise.all( - getDeepEvalTestPatterns().map(async (pattern) => { - try { - return await vscode.workspace.findFiles(pattern, DEEPEVAL_TEST_EXCLUDES); - } catch { - return []; - } - }), - ); - - return mergeUniqueUris(...discoveryGroups); -} - -function buildChatMessages(chatContext: vscode.ChatContext, request: vscode.ChatRequest): vscode.LanguageModelChatMessage[] { - const messages: vscode.LanguageModelChatMessage[] = [ - vscode.LanguageModelChatMessage.User( - 'You are Deval, a helpful local agent harness. Answer the user clearly and accurately.', - ), - ]; - - for (const turn of chatContext.history) { - if (turn instanceof vscode.ChatRequestTurn) { - messages.push(vscode.LanguageModelChatMessage.User(turn.prompt)); - } else if (turn instanceof vscode.ChatResponseTurn) { - const text = turn.response - .filter((part): part is vscode.ChatResponseMarkdownPart => part instanceof vscode.ChatResponseMarkdownPart) - .map((part) => part.value.value) - .join(''); - if (text) { - messages.push(vscode.LanguageModelChatMessage.Assistant(text)); - } - } - } - - messages.push(vscode.LanguageModelChatMessage.User(request.prompt)); - return messages; -} - -async function registerChatParticipant(context: vscode.ExtensionContext) { - const participant = vscode.chat.createChatParticipant('deval.agent', async (request, chatContext, response, token) => { - const messages = buildChatMessages(chatContext, request); - const modelResponse = await request.model.sendRequest(messages, {}, token); - for await (const fragment of modelResponse.text) { - response.markdown(fragment); - } - }); - - context.subscriptions.push(participant); -} - -/** - * Ask the user which captured Local-harness run to evaluate. - * - * Returns `undefined` when the caller did not ask for a capture - * (`withCapturedRun` is false) or when the user cancelled the picker. - */ -export async function selectDeepEvalRun( - workspaceFolder: vscode.WorkspaceFolder, - withCapturedRun: boolean, -): Promise { - if (!withCapturedRun) { - return undefined; - } - - const eventsDirectory = vscode.Uri.joinPath(workspaceFolder.uri, ...CAPTURED_RUN_DIRECTORY); - let entries: [string, vscode.FileType][]; - try { - entries = await vscode.workspace.fs.readDirectory(eventsDirectory); - } catch { - await reportMissingCaptures(workspaceFolder, eventsDirectory); - return undefined; - } - - const runFiles = entries - .filter(([name, type]) => type === vscode.FileType.File && name.endsWith('.jsonl')) - .map(([name]) => vscode.Uri.joinPath(eventsDirectory, name)); - if (!runFiles.length) { - await reportMissingCaptures(workspaceFolder, eventsDirectory); - return undefined; - } - - const selectedRun = await vscode.window.showQuickPick( - runFiles.map((uri) => ({ label: uri.path.split('/').pop() ?? uri.fsPath, uri })), - { placeHolder: 'Select a captured Local agent run' }, - ); - if (!selectedRun) { - return undefined; - } - return selectedRun.uri; -} - -/** True when `uri` can be stat'ed, i.e. it already exists. */ -async function fileExists(uri: vscode.Uri): Promise { - try { - await vscode.workspace.fs.stat(uri); - return true; - } catch { - return false; - } -} - -/** The hook registration file inside `folder`. */ -function captureHookConfigUri(folder: vscode.WorkspaceFolder): vscode.Uri { - return vscode.Uri.joinPath(folder.uri, ...CAPTURE_HOOK_DIRECTORY, CAPTURE_HOOK_CONFIG_FILE); -} - -/** - * Explain why no captured run could be offered. - * - * The previous wording told every caller to "enable the workspace hook", which - * was wrong in the common case: the hook ships with this repository, so a - * workspace that only ever received a scaffolded benchmark has never had one. - * Naming the directory that was searched and the hook's actual state turns a - * dead end into a next step. Exported so the tests can assert on both halves. - */ -export function describeMissingCaptures(details: { - workspaceName: string; - eventsDirectory: string; - hookConfig: string; - hookInstalled: boolean; -}): string { - const searched = `No captured Local agent runs found in "${details.workspaceName}". Looked in ${details.eventsDirectory}.`; - if (details.hookInstalled) { - return `${searched} ${details.hookConfig} is installed but has not recorded a run yet: start a chat session with the Local target, send one turn, and run this again.`; - } - return `${searched} This workspace has no capture hook: ${details.hookConfig} does not exist.`; -} - -/** - * Report an empty capture directory and, when the missing hook is the reason, - * offer to install it into this workspace. Resolves once the message is gone. - */ -async function reportMissingCaptures( - workspaceFolder: vscode.WorkspaceFolder, - eventsDirectory: vscode.Uri, -): Promise { - const hookConfig = captureHookConfigUri(workspaceFolder); - const hookInstalled = await fileExists(hookConfig); - const message = describeMissingCaptures({ - workspaceName: workspaceFolder.name, - eventsDirectory: vscode.workspace.asRelativePath(eventsDirectory, false), - hookConfig: vscode.workspace.asRelativePath(hookConfig, false), - hookInstalled, - }); - - if (hookInstalled) { - // There is nothing to install, so the only useful content is the path - // and what to do in the harness; a button here would be noise. - await vscode.window.showWarningMessage(message); - return; - } - - const install = 'Install Capture Hook'; - const choice = await vscode.window.showWarningMessage(message, install); - if (choice === install) { - await installCaptureHook(workspaceFolder); - } -} - -/** - * Run a task and report how it ended. - * - * `vscode.tasks.executeTask` resolves as soon as the process starts, so the - * exit code has to be picked up from `onDidEndTaskProcess`. Without this the - * only feedback for a broken suite is a red terminal the user may never see. - */ -export async function executeTaskWithReporting(task: vscode.Task, label: string): Promise { - return new Promise((resolve) => { - let subscription: vscode.Disposable | undefined; - - const finish = (succeeded: boolean): void => { - subscription?.dispose(); - resolve(succeeded); - }; - - vscode.tasks.executeTask(task).then( - (execution) => { - subscription = vscode.tasks.onDidEndTaskProcess((event) => { - if (event.execution !== execution) { - return; - } - if (event.exitCode === 0) { - vscode.window.showInformationMessage(`${label} passed.`); - finish(true); - } else { - vscode.window.showErrorMessage( - `${label} failed with exit code ${event.exitCode ?? 'unknown'}. See the terminal for details.`, - ); - finish(false); - } - }); - }, - (error: unknown) => { - const message = error instanceof Error ? error.message : String(error); - vscode.window.showErrorMessage(`Could not start ${label}: ${message}`); - finish(false); - }, - ); - }); -} - -/** - * Repair the DeepEval CLI inside `projectRoot` before spawning it. - * - * Handles the `deepeval@0.9.22` shadowing bug that otherwise makes every run - * fail with "Cannot find module '@sentry/node'". Doing it here as well as after - * an install heals workspaces whose deepeval was installed by hand. - */ -function applyDeepEvalPatch(projectRoot: string): void { - const { patched, blocked } = patchDeepEvalInstall(projectRoot); - - if (patched.length) { - console.log(`[deval] Disabled stale deepeval dist file(s): ${patched.join(', ')}.`); - } - if (blocked.length) { - void vscode.window.showWarningMessage( - `The installed deepeval package is broken (${blocked.join(', ')}) and could not be repaired. See "Known upstream issue" in the deval README.`, - ); - } -} - -async function runDeepEval(withCapturedRun = false): Promise { - if (!vscode.workspace.workspaceFolders?.length) { - vscode.window.showWarningMessage('Open a workspace folder to run DeepEval tests.'); - return; - } - - const uniqueTestFiles = await findDeepEvalTestFiles(); - if (!uniqueTestFiles.length) { - vscode.window.showWarningMessage('No TypeScript test files found. Expected *.test.ts or *.spec.ts.'); - return; - } - - const selectedFile = await vscode.window.showQuickPick( - uniqueTestFiles.map((uri) => ({ - label: vscode.workspace.asRelativePath(uri), - uri, - })), - { placeHolder: 'Select a DeepEval TypeScript test suite' }, - ); - if (!selectedFile) { - return; - } - - const workspaceFolder = vscode.workspace.getWorkspaceFolder(selectedFile.uri); - if (!workspaceFolder) { - vscode.window.showErrorMessage('Could not determine the workspace folder for this test suite.'); - return; - } - - const command = vscode.workspace.getConfiguration('deval').get('deepevalCommand', 'npx'); - if (requiresLocalDeepEvalInstall(command, workspaceFolder.uri.fsPath)) { - // npx would download DeepEval into npm's cache: unpinned, outside the - // workspace, and unusable here because the suite also needs Vitest. - const install = 'Install dependencies'; - const choice = await vscode.window.showWarningMessage( - `DeepEval is not installed in "${workspaceFolder.name}", so npx would download a copy that cannot run ${vscode.workspace.asRelativePath(selectedFile.uri)}.`, - { modal: true }, - install, - ); - if (choice !== install || !(await installDeepEvalDependencies(workspaceFolder))) { - return; - } - } - - // The child process cannot repair its own dependency, so patch first. - applyDeepEvalPatch(workspaceFolder.uri.fsPath); - - const eventsFile = await selectDeepEvalRun(workspaceFolder, withCapturedRun); - if (withCapturedRun && !eventsFile) { - return; - } - - const relativeTestPath = vscode.workspace.asRelativePath(selectedFile.uri, false); - const label = `DeepEval: ${selectedFile.label}`; - const task = new vscode.Task( - { type: 'deepeval', testFile: relativeTestPath }, - workspaceFolder, - label, - 'DeepEval', - new vscode.ProcessExecution(command, ['deepeval', 'test', 'run', relativeTestPath], { - cwd: workspaceFolder.uri.fsPath, - env: eventsFile ? { DEEPEVAL_VSCODE_EVENTS: eventsFile.fsPath } : undefined, - }), - ); - task.presentationOptions = { reveal: vscode.TaskRevealKind.Always, panel: vscode.TaskPanelKind.Dedicated }; - await executeTaskWithReporting(task, label); -} - -/** File name used by the `DeepEval: Scaffold Sample Benchmark` command. */ -export const SCAFFOLD_FILE_NAME = 'my-agent.test.ts'; - -/** Package managers the scaffolder knows how to install with. */ -export const PACKAGE_MANAGERS = ['npm', 'pnpm', 'yarn', 'bun'] as const; - -export type PackageManager = (typeof PACKAGE_MANAGERS)[number]; - -/** Arguments that add the benchmark dependencies as devDependencies. */ -const INSTALL_ARGUMENTS: Record = { - npm: ['install', '--save-dev', 'deepeval', 'vitest'], - pnpm: ['add', '--save-dev', 'deepeval', 'vitest'], - yarn: ['add', '--dev', 'deepeval', 'vitest'], - bun: ['add', '--dev', 'deepeval', 'vitest'], -}; - -/** - * Build the command that installs the benchmark dependencies. - * - * Deliberately *not* derived from `deval.deepevalCommand`: that setting names - * the DeepEval runner (`npx`), and `npx install --save-dev deepeval vitest` - * makes npx look for an executable called `install`, which fails with - * "could not determine executable to run". - * - * An unset or unrecognised value falls back to npm rather than erroring, since - * a typo in a setting should not block the scaffolder. - */ -export function getInstallInvocation(packageManager: string | undefined): { - command: string; - args: string[]; -} { - const name = (packageManager ?? '').trim().toLowerCase(); - const resolved = Object.hasOwn(INSTALL_ARGUMENTS, name) ? (name as PackageManager) : 'npm'; - return { command: resolved, args: [...INSTALL_ARGUMENTS[resolved]] }; -} - -/** - * A self-contained starter suite. - * - * Built from an array of lines rather than one template literal so the - * backticks and `${}` inside the generated code need no escaping here. - */ -const SCAFFOLD_SUITE_LINES = [ - '/**', - ' * A DeepEval suite for your own agent.', - ' *', - ' * The metrics below are deterministic, so this file runs without a', - ' * DEEPEVAL_API_KEY. Add a model-judged metric (TaskCompletionMetric, GEval,', - ' * ToolCorrectnessMetric with `availableTools`, ...) once you have one.', - ' */', - '', - "import { expect, it } from 'vitest';", - "import 'deepeval/vitest';", - "import { Golden } from 'deepeval/dataset';", - "import { BaseMetric, checkSingleTurnParams } from 'deepeval/metrics';", - "import { LLMTestCase, SingleTurnParams } from 'deepeval/test-case';", - "import { SpanType, observe, updateCurrentSpan } from 'deepeval/tracing';", - '', - '// 1. The system under test. Point this at your own implementation.', - 'async function runMyAgent(input: string): Promise {', - ' return `handled: ${input}`;', - '}', - '', - '// 2. A deterministic metric. Swap in a model-judged one when you need', - '// subjective checks and have DEEPEVAL_API_KEY set.', - 'class MentionsInputMetric extends BaseMetric {', - ' constructor() {', - ' super(1);', - ' this.requiredParams = [SingleTurnParams.INPUT, SingleTurnParams.ACTUAL_OUTPUT];', - ' }', - '', - ' async measure(testCase: LLMTestCase): Promise {', - ' checkSingleTurnParams(testCase, this.requiredParams, this);', - ' this.score = testCase.actualOutput.includes(testCase.input) ? 1 : 0;', - ' this.reason =', - ' this.score === 1 ? "The output echoes the prompt." : "The output dropped the prompt.";', - ' this.success = this.isSuccessful();', - ' return this.score;', - ' }', - '', - ' get name(): string {', - ' return "Mentions Input";', - ' }', - '}', - '', - '// 3. Instrument the agent. `toPass` judges the trace the task produces, so the', - '// call has to run inside an observed function and publish its I/O.', - 'const observedAgent = observe({', - ' type: SpanType.AGENT,', - ' name: "my-agent",', - ' fn: async (input: string) => {', - ' const output = await runMyAgent(input);', - ' updateCurrentSpan({ input, output, toolsCalled: [{ name: "runMyAgent" }] });', - ' return output;', - ' },', - '});', - '', - 'const cases = ["Summarise the change log.", "Draft a release note."];', - '', - "it.each(cases)('handles: %s', async (prompt) => {", - ' const golden = new Golden({ input: prompt });', - '', - ' await expect(golden).toPass([new MentionsInputMetric()], {', - ' task: (testCase) => observedAgent(testCase.input),', - ' });', - '});', - '', -]; - -/** The generated starter suite, as text. Exported so tests can assert on it. */ -export function getScaffoldSuiteSource(): string { - return SCAFFOLD_SUITE_LINES.join('\n'); -} - -/** - * Write a starter suite into the workspace and open it. - * - * Writing into the *workspace* (rather than into the extension) is the point: - * `npx deepeval test run` is spawned with the workspace folder as its cwd, so - * the suite and the `deepeval` dependency both have to live there. - */ -async function scaffoldBenchmark(): Promise { - const folders = vscode.workspace.workspaceFolders; - if (!folders?.length) { - vscode.window.showWarningMessage('Open a workspace folder to scaffold a benchmark into.'); - return; - } - - let folder = folders[0]; - if (folders.length > 1) { - const picked = await vscode.window.showQuickPick( - folders.map((candidate) => ({ label: candidate.name, folder: candidate })), - { placeHolder: 'Select the workspace folder for the new benchmark' }, - ); - if (!picked) { - return; - } - folder = picked.folder; - } - - const benchmarksDirectory = vscode.Uri.joinPath(folder.uri, 'benchmarks'); - const target = vscode.Uri.joinPath(benchmarksDirectory, SCAFFOLD_FILE_NAME); - - try { - await vscode.workspace.fs.stat(target); - const overwrite = await vscode.window.showWarningMessage( - `${vscode.workspace.asRelativePath(target)} already exists. Overwrite it?`, - { modal: true }, - 'Overwrite', - ); - if (overwrite !== 'Overwrite') { - return; - } - } catch { - // No existing file: carry on and create it. - } - - try { - await vscode.workspace.fs.createDirectory(benchmarksDirectory); - await vscode.workspace.fs.writeFile(target, Buffer.from(getScaffoldSuiteSource(), 'utf8')); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - vscode.window.showErrorMessage(`Could not create the benchmark: ${message}`); - return; - } - - await vscode.window.showTextDocument(target); - const action = await vscode.window.showInformationMessage( - `Created ${vscode.workspace.asRelativePath(target)}. Install DeepEval in this workspace, then run it.`, - 'Install dependencies', - 'Install Capture Hook', - ); - if (action === 'Install dependencies') { - await installDeepEvalDependencies(folder); - } else if (action === 'Install Capture Hook') { - await installCaptureHook(folder); - } -} - -/** - * Install DeepEval and Vitest into `folder`, then repair the known-broken - * upstream CLI. Resolves to true only when the install task succeeded. - */ -async function installDeepEvalDependencies(folder: vscode.WorkspaceFolder): Promise { - const packageManager = vscode.workspace.getConfiguration('deval').get('packageManager', 'npm'); - const { command, args } = getInstallInvocation(packageManager); - const install = new vscode.Task( - { type: 'deepeval', install: true }, - folder, - 'DeepEval: Install dependencies', - 'DeepEval', - new vscode.ProcessExecution(command, args, { cwd: folder.uri.fsPath }), - ); - install.presentationOptions = { reveal: vscode.TaskRevealKind.Always, panel: vscode.TaskPanelKind.Dedicated }; - const succeeded = await executeTaskWithReporting(install, 'DeepEval: Install dependencies'); - if (succeeded) { - // A fresh install ships the broken CLI; repair it before the user runs it. - applyDeepEvalPatch(folder.uri.fsPath); - } - return succeeded; -} - -/** - * Write the capture hook into a workspace so the Local harness records runs. - * - * This command exists because the packaged extension deliberately ships no - * `.github/**` (`.vscodeignore`): a workspace that never received the hook from - * this repository has nothing to enable, which is exactly what - * `DeepEval: Evaluate Captured Local Agent Run` used to tell its users. Files - * whose content already matches are left alone, and a differing file is replaced - * only after an explicit confirmation, since the hook may have been edited to - * record other events. - * - * Resolves to true when both files are in place afterwards. - */ -async function installCaptureHook(picked?: vscode.WorkspaceFolder): Promise { - const folders = vscode.workspace.workspaceFolders; - if (!folders?.length) { - vscode.window.showWarningMessage('Open a workspace folder to install the capture hook into.'); - return false; - } - - let folder = picked ?? folders[0]; - if (!picked && folders.length > 1) { - const selected = await vscode.window.showQuickPick( - folders.map((candidate) => ({ label: candidate.name, candidate })), - { placeHolder: 'Select the workspace folder to install the capture hook into' }, - ); - if (!selected) { - return false; - } - folder = selected.candidate; - } - - for (const file of getCaptureHookFiles()) { - const target = vscode.Uri.joinPath(folder.uri, ...file.segments); - if (await fileExists(target)) { - const existing = Buffer.from(await vscode.workspace.fs.readFile(target)).toString('utf8'); - if (existing === file.content) { - continue; - } - const overwrite = await vscode.window.showWarningMessage( - `${vscode.workspace.asRelativePath(target, false)} exists and differs from DeepEval's capture hook. Overwrite it?`, - { modal: true }, - 'Overwrite', - ); - if (overwrite !== 'Overwrite') { - return false; - } - } - - try { - await vscode.workspace.fs.createDirectory( - vscode.Uri.joinPath(folder.uri, ...file.segments.slice(0, -1)), - ); - await vscode.workspace.fs.writeFile(target, Buffer.from(file.content, 'utf8')); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - vscode.window.showErrorMessage(`Could not install the capture hook: ${message}`); - return false; - } - } - - const hooksDirectory = `${CAPTURE_HOOK_DIRECTORY.join('/')}/`; - const chat = vscode.workspace.getConfiguration('chat'); - const enable = 'Enable chat hooks'; - const base = `Installed ${hooksDirectory} in "${folder.name}".`; - const next = chat.get('useHooks', false) - ? `${base} Select the Local session target, send one chat turn, then run DeepEval: Evaluate Captured Local Agent Run again.` - : `${base} Enable chat hooks, select the Local session target, and send one chat turn, then run DeepEval: Evaluate Captured Local Agent Run again.`; - const choice = chat.get('useHooks', false) - ? await vscode.window.showInformationMessage(next) - : await vscode.window.showInformationMessage(next, enable); - - if (choice === enable) { - try { - await chat.update('useHooks', true, vscode.ConfigurationTarget.Global); - } catch { - try { - await chat.update('useHooks', true, vscode.ConfigurationTarget.Workspace); - } catch { - // The setting's scope is not one this extension may write to; - // leaving the user at the setting is still a next step. - await vscode.commands.executeCommand('workbench.action.openSettings', '@id:chat.useHooks'); - } - } - } - return true; -} - -/** True for every form `npx` is invoked through (npx, npx.cmd, full path, quoted). */ -function isNpxCommand(command: string): boolean { - const executable = command.trim().replace(/^["']|["']$/g, '').split(/[\\/]/).pop() ?? ''; - return executable.replace(/\.(cmd|bat|exe)$/i, '').toLowerCase() === 'npx'; -} - -/** - * True when `command` would make npx *download* DeepEval instead of using a - * workspace install. That copy lands in npm's npx cache — unpinned, outside the - * workspace, and never patched — and the suite still needs the workspace's own - * Vitest, so the run must install dependencies first. Exported for tests. - */ -export function requiresLocalDeepEvalInstall(command: string, projectRoot: string): boolean { - return isNpxCommand(command) && !hasDeepEvalInstall(projectRoot); -} - -function registerCommands(context: vscode.ExtensionContext) { - const rc = vscode.commands.registerCommand; const d1 = rc('deval.runDeepEval', () => runDeepEval()); const d2 = rc('deval.evaluateAgentRun', () => runDeepEval(true)); - const d3 = rc('deval.scaffoldBenchmark', () => scaffoldBenchmark()); - const d4 = rc('deval.installCaptureHook', () => installCaptureHook()); - context.subscriptions.push(d1, d2, d3, d4); -} + context.subscriptions.push(d1, d2); -export function activate(context: vscode.ExtensionContext) { - registerChatParticipant(context); - registerCommands(context); + void registerChatParticipant(context); } export function deactivate() {} diff --git a/src/suites.ts b/src/suites.ts new file mode 100644 index 0000000..bbf02f2 --- /dev/null +++ b/src/suites.ts @@ -0,0 +1,333 @@ +// Everything about running DeepEval TypeScript suites from VS Code: the two +// run commands, suite discovery, the captured-run picker, and the +// dependency-install flow that keeps them working. +import * as vscode from 'vscode'; + +import { hasDeepEvalInstall, patchDeepEvalInstall } from './deepeval-patch'; +import { DEEPEVAL_VSCODE_EVENTS } from './events/event-log'; + +/** Folder the agent loop writes captures into, relative to the workspace root. */ +export const CAPTURED_RUN_DIRECTORY = ['.deepeval', 'vscode-agent-events']; + +/** Globs searched for DeepEval TypeScript suites. */ +export const DEEPEVAL_TEST_PATTERNS = [ + '**/*.test.ts', + '**/*.spec.ts', + '**/benchmarks/**/*.test.ts', + '**/src/**/*.test.ts', + '**/test/**/*.test.ts', +]; + +/** Paths that never contain a DeepEval suite worth running from here. */ +export const DEEPEVAL_TEST_EXCLUDES = '**/{node_modules,dist,out,.vscode-test}/**'; + +export function getDeepEvalTestPatterns(): string[] { + return [...DEEPEVAL_TEST_PATTERNS]; +} + +export function mergeUniqueUris(...groups: vscode.Uri[][]): vscode.Uri[] { + const seen = new Map(); + for (const group of groups) { + for (const uri of group) { + seen.set(uri.toString(), uri); + } + } + return [...seen.values()]; +} + +export async function findDeepEvalTestFiles(): Promise { + const discoveryGroups = await Promise.all( + getDeepEvalTestPatterns().map(async (pattern) => { + try { + return await vscode.workspace.findFiles(pattern, DEEPEVAL_TEST_EXCLUDES); + } catch { + return []; + } + }), + ); + + return mergeUniqueUris(...discoveryGroups); +} + +/** + * Ask the user which captured `@deval` run to evaluate. + * + * Returns `undefined` when the caller did not ask for a capture + * (`withCapturedRun` is false) or when the user cancelled the picker. + */ +export async function selectDeepEvalRun( + workspaceFolder: vscode.WorkspaceFolder, + withCapturedRun: boolean, +): Promise { + if (!withCapturedRun) { + return undefined; + } + + const eventsDirectory = vscode.Uri.joinPath(workspaceFolder.uri, ...CAPTURED_RUN_DIRECTORY); + let entries: [string, vscode.FileType][]; + try { + entries = await vscode.workspace.fs.readDirectory(eventsDirectory); + } catch { + await reportMissingCaptures(workspaceFolder, eventsDirectory); + return undefined; + } + + const runFiles = entries + .filter(([name, type]) => type === vscode.FileType.File && name.endsWith('.jsonl')) + .map(([name]) => vscode.Uri.joinPath(eventsDirectory, name)); + if (!runFiles.length) { + await reportMissingCaptures(workspaceFolder, eventsDirectory); + return undefined; + } + + const selectedRun = await vscode.window.showQuickPick( + runFiles.map((uri) => ({ label: uri.path.split('/').pop() ?? uri.fsPath, uri })), + { placeHolder: 'Select a captured agent run' }, + ); + if (!selectedRun) { + return undefined; + } + return selectedRun.uri; +} + +/** True when `uri` can be stat'ed, i.e. it already exists. */ +async function fileExists(uri: vscode.Uri): Promise { + try { + await vscode.workspace.fs.stat(uri); + return true; + } catch { + return false; + } +} + +/** + * Explain why no captured run could be offered. + * + * Captures are written automatically by the `@deval` agent loop, so an empty + * directory just means the agent has not run yet in this workspace. + */ +export function describeMissingCaptures(details: { + workspaceName: string; + eventsDirectory: string; +}): string { + return ( + `No captured agent runs found in "${details.workspaceName}". ` + + `Looked in ${details.eventsDirectory}. ` + + `Chat with @deval first — every run is captured automatically.` + ); +} + +/** Report an empty capture directory. Resolves once the message is gone. */ +async function reportMissingCaptures( + workspaceFolder: vscode.WorkspaceFolder, + eventsDirectory: vscode.Uri, +): Promise { + await vscode.window.showWarningMessage( + describeMissingCaptures({ + workspaceName: workspaceFolder.name, + eventsDirectory: vscode.workspace.asRelativePath(eventsDirectory, false), + }), + ); +} + +/** + * Run a task and report how it ended. + * + * `vscode.tasks.executeTask` resolves as soon as the process starts, so the + * exit code has to be picked up from `onDidEndTaskProcess`. Without this the + * only feedback for a broken suite is a red terminal the user may never see. + */ +export async function executeTaskWithReporting(task: vscode.Task, label: string): Promise { + return new Promise((resolve) => { + let subscription: vscode.Disposable | undefined; + + const finish = (succeeded: boolean): void => { + subscription?.dispose(); + resolve(succeeded); + }; + + vscode.tasks.executeTask(task).then( + (execution) => { + subscription = vscode.tasks.onDidEndTaskProcess((event) => { + if (event.execution !== execution) { + return; + } + if (event.exitCode === 0) { + vscode.window.showInformationMessage(`${label} passed.`); + finish(true); + } else { + vscode.window.showErrorMessage( + `${label} failed with exit code ${event.exitCode ?? 'unknown'}. See the terminal for details.`, + ); + finish(false); + } + }); + }, + (error: unknown) => { + const message = error instanceof Error ? error.message : String(error); + vscode.window.showErrorMessage(`Could not start ${label}: ${message}`); + finish(false); + }, + ); + }); +} + +/** + * Repair the DeepEval CLI inside `projectRoot` before spawning it. + * + * Handles the `deepeval@0.9.22` shadowing bug that otherwise makes every run + * fail with "Cannot find module '@sentry/node'". Doing it here as well as after + * an install heals workspaces whose deepeval was installed by hand. + */ +function applyDeepEvalPatch(projectRoot: string): void { + const { patched, blocked } = patchDeepEvalInstall(projectRoot); + + if (patched.length) { + console.log(`[deval] Disabled stale deepeval dist file(s): ${patched.join(', ')}.`); + } + if (blocked.length) { + void vscode.window.showWarningMessage( + `The installed deepeval package is broken (${blocked.join(', ')}) and could not be repaired. See "Known upstream issue" in the deval README.`, + ); + } +} + +export async function runDeepEval(withCapturedRun = false): Promise { + if (!vscode.workspace.workspaceFolders?.length) { + vscode.window.showWarningMessage('Open a workspace folder to run DeepEval tests.'); + return; + } + + const uniqueTestFiles = await findDeepEvalTestFiles(); + if (!uniqueTestFiles.length) { + vscode.window.showWarningMessage('No TypeScript test files found. Expected *.test.ts or *.spec.ts.'); + return; + } + + const selectedFile = await vscode.window.showQuickPick( + uniqueTestFiles.map((uri) => ({ + label: vscode.workspace.asRelativePath(uri), + uri, + })), + { placeHolder: 'Select a DeepEval TypeScript test suite' }, + ); + if (!selectedFile) { + return; + } + + const workspaceFolder = vscode.workspace.getWorkspaceFolder(selectedFile.uri); + if (!workspaceFolder) { + vscode.window.showErrorMessage('Could not determine the workspace folder for this test suite.'); + return; + } + + const command = vscode.workspace.getConfiguration('deval').get('deepevalCommand', 'npx'); + if (requiresLocalDeepEvalInstall(command, workspaceFolder.uri.fsPath)) { + // npx would download DeepEval into npm's cache: unpinned, outside the + // workspace, and unusable here because the suite also needs Vitest. + const install = 'Install dependencies'; + const choice = await vscode.window.showWarningMessage( + `DeepEval is not installed in "${workspaceFolder.name}", so npx would download a copy that cannot run ${vscode.workspace.asRelativePath(selectedFile.uri)}.`, + { modal: true }, + install, + ); + if (choice !== install || !(await installDeepEvalDependencies(workspaceFolder))) { + return; + } + } + + // The child process cannot repair its own dependency, so patch first. + applyDeepEvalPatch(workspaceFolder.uri.fsPath); + + const eventsFile = await selectDeepEvalRun(workspaceFolder, withCapturedRun); + if (withCapturedRun && !eventsFile) { + return; + } + + const relativeTestPath = vscode.workspace.asRelativePath(selectedFile.uri, false); + const label = `DeepEval: ${selectedFile.label}`; + const task = new vscode.Task( + { type: 'deepeval', testFile: relativeTestPath }, + workspaceFolder, + label, + 'DeepEval', + new vscode.ProcessExecution(command, ['deepeval', 'test', 'run', relativeTestPath], { + cwd: workspaceFolder.uri.fsPath, + env: eventsFile ? { DEEPEVAL_VSCODE_EVENTS: eventsFile.fsPath } : undefined, + }), + ); + task.presentationOptions = { reveal: vscode.TaskRevealKind.Always, panel: vscode.TaskPanelKind.Dedicated }; + await executeTaskWithReporting(task, label); +} + +/** Package managers the dependency installer knows how to install with. */ +export const PACKAGE_MANAGERS = ['npm', 'pnpm', 'yarn', 'bun'] as const; + +export type PackageManager = (typeof PACKAGE_MANAGERS)[number]; + +/** Arguments that add the benchmark dependencies as devDependencies. */ +const INSTALL_ARGUMENTS: Record = { + npm: ['install', '--save-dev', 'deepeval', 'vitest'], + pnpm: ['add', '--save-dev', 'deepeval', 'vitest'], + yarn: ['add', '--dev', 'deepeval', 'vitest'], + bun: ['add', '--dev', 'deepeval', 'vitest'], +}; + +/** + * Build the command that installs the benchmark dependencies. + * + * Deliberately *not* derived from `deval.deepevalCommand`: that setting names + * the DeepEval runner (`npx`), and `npx install --save-dev deepeval vitest` + * makes npx look for an executable called `install`, which fails with + * "could not determine executable to run". + * + * An unset or unrecognised value falls back to npm rather than erroring, since + * a typo in a setting should not block the installer. + */ +export function getInstallInvocation(packageManager: string | undefined): { + command: string; + args: string[]; +} { + const name = (packageManager ?? '').trim().toLowerCase(); + const resolved = Object.hasOwn(INSTALL_ARGUMENTS, name) ? (name as PackageManager) : 'npm'; + return { command: resolved, args: [...INSTALL_ARGUMENTS[resolved]] }; +} + +/** + * Install DeepEval and Vitest into `folder`, then repair the known-broken + * upstream CLI. Resolves to true only when the install task succeeded. + */ +async function installDeepEvalDependencies(folder: vscode.WorkspaceFolder): Promise { + const packageManager = vscode.workspace.getConfiguration('deval').get('packageManager', 'npm'); + const { command, args } = getInstallInvocation(packageManager); + const install = new vscode.Task( + { type: 'deepeval', install: true }, + folder, + 'DeepEval: Install dependencies', + 'DeepEval', + new vscode.ProcessExecution(command, args, { cwd: folder.uri.fsPath }), + ); + install.presentationOptions = { reveal: vscode.TaskRevealKind.Always, panel: vscode.TaskPanelKind.Dedicated }; + const succeeded = await executeTaskWithReporting(install, 'DeepEval: Install dependencies'); + if (succeeded) { + // A fresh install ships the broken CLI; repair it before the user runs it. + applyDeepEvalPatch(folder.uri.fsPath); + } + return succeeded; +} + +/** True for every form `npx` is invoked through (npx, npx.cmd, full path, quoted). */ +function isNpxCommand(command: string): boolean { + const executable = command.trim().replace(/^["']|["']$/g, '').split(/[\\/]/).pop() ?? ''; + return executable.replace(/\.(cmd|bat|exe)$/i, '').toLowerCase() === 'npx'; +} + +/** + * True when `command` would make npx *download* DeepEval instead of using a + * workspace install. That copy lands in npm's npx cache — unpinned, outside the + * workspace, and never patched — and the suite still needs the workspace's own + * Vitest, so the run must install dependencies first. Exported for tests. + */ +export function requiresLocalDeepEvalInstall(command: string, projectRoot: string): boolean { + return isNpxCommand(command) && !hasDeepEvalInstall(projectRoot); +} diff --git a/src/test/agent-loop.test.ts b/src/test/agent-loop.test.ts new file mode 100644 index 0000000..dc518b4 --- /dev/null +++ b/src/test/agent-loop.test.ts @@ -0,0 +1,154 @@ +import * as assert from 'assert'; + +import { + LOOP_CUTOFF_MESSAGE, + runAgentLoop, + type EventPort, + type LoopMessage, + type ModelPort, + type ModelTurn, + type ToolPort, +} from '../agent/loop'; +import { DEVAL_TOOL_SCHEMAS, READ_FILE_TOOL } from '../agent/tool-schemas'; +import type { AgentEvent } from '../events/event-log'; + +interface FakeWorld { + events: AgentEvent[]; + toolCalls: Array<{ name: string; input: unknown }>; + sentMessages: LoopMessage[][]; + ports: { model: ModelPort; tools: ToolPort; events: EventPort }; +} + +/** Ports driven by a script of model turns, recording everything. */ +function fakeWorld( + turns: ModelTurn[], + toolImpl?: (name: string, input: unknown) => Promise | unknown, +): FakeWorld { + const events: AgentEvent[] = []; + const toolCalls: Array<{ name: string; input: unknown }> = []; + const sentMessages: LoopMessage[][] = []; + let turnIndex = 0; + + return { + events, + toolCalls, + sentMessages, + ports: { + model: { + send: async (messages) => { + sentMessages.push(messages); + return turns[Math.min(turnIndex++, turns.length - 1)]; + }, + }, + tools: { + invoke: async (name, input) => { + toolCalls.push({ name, input }); + return toolImpl ? toolImpl(name, input) : `result-of-${name}`; + }, + }, + events: { + record: (event) => + events.push({ session_id: 'test', timestamp: 't', ...event }), + }, + }, + }; +} + +function hookNames(events: AgentEvent[]): Array { + return events.map((event) => event.hook_event_name); +} + +suite('agent loop', () => { + test('a text-only run records prompt and stop, with no tool calls', async () => { + const world = fakeWorld([{ text: 'hello there', toolCalls: [] }]); + + const run = await runAgentLoop('hi', world.ports, { toolSchemas: DEVAL_TOOL_SCHEMAS }); + + assert.strictEqual(run.input, 'hi'); + assert.strictEqual(run.output, 'hello there'); + assert.deepStrictEqual(run.toolsCalled, []); + assert.deepStrictEqual(hookNames(world.events), ['UserPromptSubmit', 'Stop']); + assert.strictEqual(world.toolCalls.length, 0); + }); + + test('a tool call is executed, paired, and its result fed back', async () => { + const world = fakeWorld( + [ + { + text: '', + toolCalls: [{ id: 'call_1', name: READ_FILE_TOOL, input: { path: 'a.txt' } }], + }, + { text: 'the file says hello', toolCalls: [] }, + ], + () => 'file contents: hello', + ); + + const run = await runAgentLoop('read a.txt', world.ports, { + toolSchemas: DEVAL_TOOL_SCHEMAS, + }); + + assert.deepStrictEqual(hookNames(world.events), [ + 'UserPromptSubmit', + 'PreToolUse', + 'PostToolUse', + 'Stop', + ]); + const pre = world.events[1]; + const post = world.events[2]; + assert.strictEqual(pre.tool_use_id, 'call_1'); + assert.strictEqual(post.tool_use_id, 'call_1'); + assert.strictEqual(pre.tool_name, READ_FILE_TOOL); + assert.strictEqual(post.tool_response, 'file contents: hello'); + + assert.strictEqual(run.toolsCalled.length, 1); + assert.strictEqual(run.toolsCalled[0].name, READ_FILE_TOOL); + assert.deepStrictEqual(run.toolsCalled[0].inputParameters, { path: 'a.txt' }); + assert.strictEqual(run.toolsCalled[0].output, 'file contents: hello'); + assert.strictEqual(run.output, 'the file says hello'); + + // The second model turn saw the tool call and its result. + const secondTurn = world.sentMessages[1]; + assert.ok(secondTurn.some((m) => m.role === 'tool-call' && m.id === 'call_1')); + assert.ok( + secondTurn.some((m) => m.role === 'tool-result' && m.output === 'file contents: hello'), + ); + }); + + test('a failing tool is recorded as an error result and the loop continues', async () => { + const world = fakeWorld( + [ + { + text: '', + toolCalls: [{ id: 'call_1', name: 'nope', input: {} }], + }, + { text: 'done anyway', toolCalls: [] }, + ], + () => { + throw new Error('boom'); + }, + ); + + const run = await runAgentLoop('go', world.ports, { toolSchemas: DEVAL_TOOL_SCHEMAS }); + + const post = world.events.find((e) => e.hook_event_name === 'PostToolUse'); + assert.deepStrictEqual(post?.tool_response, { error: 'boom' }); + assert.deepStrictEqual(hookNames(world.events).at(-1), 'Stop'); + assert.strictEqual(run.output, 'done anyway'); + }); + + test('the loop stops after maxTurns without a Stop event', async () => { + const world = fakeWorld([ + { text: '', toolCalls: [{ id: 'c1', name: READ_FILE_TOOL, input: {} }] }, + ]); + + const run = await runAgentLoop('go', world.ports, { + toolSchemas: DEVAL_TOOL_SCHEMAS, + maxTurns: 2, + }); + + assert.strictEqual(world.sentMessages.length, 2); + assert.ok(run.output.includes(LOOP_CUTOFF_MESSAGE)); + assert.ok(!hookNames(world.events).includes('Stop'), 'a cut-off run reads as incomplete'); + assert.strictEqual(run.toolsCalled.length, 2); + }); +}); diff --git a/src/test/capture-hook.test.ts b/src/test/capture-hook.test.ts deleted file mode 100644 index 5fd9b2d..0000000 --- a/src/test/capture-hook.test.ts +++ /dev/null @@ -1,105 +0,0 @@ -import * as assert from 'assert'; -import { readFileSync } from 'node:fs'; -import { join } from 'node:path'; - -import { - CAPTURE_HOOK_COMMAND, - CAPTURE_HOOK_CONFIG_FILE, - CAPTURE_HOOK_DIRECTORY, - CAPTURE_HOOK_EVENTS, - CAPTURE_HOOK_SCRIPT_FILE, - getCaptureHookConfigSource, - getCaptureHookFiles, - getCaptureHookScriptSource, -} from '../capture-hook'; -import { CAPTURED_RUN_DIRECTORY, describeMissingCaptures } from '../extension'; - -suite('capture hook', () => { - /** The repository's own copy, which is what the embedded text must match. */ - function referenceFile(name: string): string { - return readFileSync(join(__dirname, '..', '..', ...CAPTURE_HOOK_DIRECTORY, name), 'utf8'); - } - - /** The repository stores both files with CRLF; the embedded text uses LF. */ - function normalized(text: string): string { - return text.replace(/\r\n/g, '\n'); - } - - test('ships the two files the Local harness discovers', () => { - const hookRoot = CAPTURE_HOOK_DIRECTORY.join('/'); - - assert.deepStrictEqual( - getCaptureHookFiles().map((file) => file.segments.join('/')), - [`${hookRoot}/${CAPTURE_HOOK_CONFIG_FILE}`, `${hookRoot}/${CAPTURE_HOOK_SCRIPT_FILE}`], - ); - }); - - test('the config subscribes every recorded event to the script', () => { - const source = getCaptureHookConfigSource(); - - assert.ok(source.endsWith('}\n'), 'the config ends with a newline'); - type HookAction = { type: string; command: string; cwd: string; timeout: number }; - const config = JSON.parse(source) as { hooks: Record }; - assert.deepStrictEqual(Object.keys(config.hooks), [...CAPTURE_HOOK_EVENTS]); - for (const event of CAPTURE_HOOK_EVENTS) { - const [action] = config.hooks[event]; - assert.strictEqual(action.type, 'command'); - assert.strictEqual(action.command, CAPTURE_HOOK_COMMAND); - assert.strictEqual(action.command, 'node .github/hooks/deval-hook.cjs'); - assert.strictEqual(action.cwd, '.', 'the script must run at the workspace root'); - } - }); - - test('the script appends to the directory the evaluator reads', () => { - const script = getCaptureHookScriptSource(); - - assert.ok(script.includes('process.cwd()'), 'captures must land relative to the hook cwd'); - for (const segment of CAPTURED_RUN_DIRECTORY) { - assert.ok(script.includes(`'${segment}'`), `the script must write ${segment}`); - } - assert.ok(script.includes('appendFileSync')); - assert.ok(script.includes('hook_event_name'), 'the reader keys off the event name'); - assert.ok(script.includes('deval-hook.cjs') === false, 'the script does not reference itself'); - }); - - test('stays in sync with the copies in .github/hooks', () => { - const embedded = [getCaptureHookConfigSource(), getCaptureHookScriptSource()]; - - for (const [index, file] of getCaptureHookFiles().entries()) { - const name = file.segments[file.segments.length - 1]; - assert.strictEqual( - normalized(embedded[index]), - normalized(referenceFile(name)), - `the embedded ${name} must match .github/hooks/${name}`, - ); - } - }); - - test('the no-capture message names the directory that was searched', () => { - const message = describeMissingCaptures({ - workspaceName: 'mt', - eventsDirectory: '.deepeval/vscode-agent-events', - hookConfig: '.github/hooks/deval.json', - hookInstalled: false, - }); - - assert.ok(message.includes('Looked in .deepeval/vscode-agent-events'), message); - assert.ok(message.includes('This workspace has no capture hook'), message); - assert.ok(message.includes('.github/hooks/deval.json'), message); - // The old wording blamed the user for a hook that was never installed. - assert.ok(!message.includes('Enable the workspace hook'), message); - }); - - test('the no-capture message blames a silent hook rather than a missing one', () => { - const message = describeMissingCaptures({ - workspaceName: 'deval', - eventsDirectory: '.deepeval/vscode-agent-events', - hookConfig: '.github/hooks/deval.json', - hookInstalled: true, - }); - - assert.ok(message.includes('is installed but has not recorded a run yet'), message); - assert.ok(message.includes('Local target'), message); - assert.ok(!message.includes('no capture hook'), message); - }); -}); diff --git a/src/test/event-writer.test.ts b/src/test/event-writer.test.ts new file mode 100644 index 0000000..3f2705d --- /dev/null +++ b/src/test/event-writer.test.ts @@ -0,0 +1,81 @@ +import * as assert from 'assert'; +import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { createCaptureWriter, sanitizeSessionId } from '../events/event-writer'; + +suite('capture writer', () => { + const createdRoots: string[] = []; + + teardown(() => { + for (const root of createdRoots.splice(0)) { + rmSync(root, { recursive: true, force: true }); + } + }); + + function tempWorkspace(): string { + const root = mkdtempSync(join(tmpdir(), 'deval-writer-')); + createdRoots.push(root); + return root; + } + + test('fills in session_id and timestamp', () => { + const writer = createCaptureWriter(tempWorkspace(), 'session-1'); + + writer.append({ hook_event_name: 'UserPromptSubmit', prompt: 'hi' }); + + const lines = readFileSync(writer.filePath, 'utf8').trim().split('\n'); + assert.strictEqual(lines.length, 1); + const record = JSON.parse(lines[0]); + assert.strictEqual(record.session_id, 'session-1'); + assert.strictEqual(record.hook_event_name, 'UserPromptSubmit'); + assert.strictEqual(record.prompt, 'hi'); + assert.ok(typeof record.timestamp === 'string' && record.timestamp.length > 0); + }); + + test('writes into the capture directory the run commands read', () => { + const root = tempWorkspace(); + const writer = createCaptureWriter(root, 'abc'); + + assert.strictEqual( + writer.filePath, + join(root, '.deepeval', 'vscode-agent-events', 'abc.jsonl'), + ); + assert.ok(!existsSync(writer.filePath), 'file appears on first append, not on open'); + writer.append({ hook_event_name: 'Stop' }); + assert.ok(existsSync(writer.filePath)); + }); + + test('appends one JSON object per line', () => { + const writer = createCaptureWriter(tempWorkspace(), 'multi'); + + writer.append({ hook_event_name: 'PreToolUse', tool_name: 'deval_readFile' }); + writer.append({ hook_event_name: 'PostToolUse', tool_name: 'deval_readFile' }); + + const lines = readFileSync(writer.filePath, 'utf8').trim().split('\n'); + assert.strictEqual(lines.length, 2); + for (const line of lines) { + assert.strictEqual(JSON.parse(line).session_id, 'multi'); + } + }); + + test('keeps an explicit timestamp when given', () => { + const writer = createCaptureWriter(tempWorkspace(), 'ts'); + + writer.append({ hook_event_name: 'Stop', timestamp: '2026-01-01T00:00:00.000Z' }); + + const record = JSON.parse(readFileSync(writer.filePath, 'utf8')); + assert.strictEqual(record.timestamp, '2026-01-01T00:00:00.000Z'); + }); + + test('sanitizes hostile session ids for the file name', () => { + assert.strictEqual(sanitizeSessionId('../../etc/passwd'), '______etc_passwd'); + assert.strictEqual(sanitizeSessionId('ok-1_2'), 'ok-1_2'); + + const writer = createCaptureWriter(tempWorkspace(), '../../evil'); + assert.ok(!writer.filePath.includes('..')); + writer.append({ hook_event_name: 'Stop' }); + assert.ok(existsSync(writer.filePath)); + }); +}); diff --git a/src/test/extension.test.ts b/src/test/extension.test.ts index 8ca6817..415c494 100644 --- a/src/test/extension.test.ts +++ b/src/test/extension.test.ts @@ -10,14 +10,12 @@ import { CAPTURED_RUN_DIRECTORY, DEEPEVAL_TEST_EXCLUDES, PACKAGE_MANAGERS, - SCAFFOLD_FILE_NAME, findDeepEvalTestFiles, getDeepEvalTestPatterns, getInstallInvocation, - getScaffoldSuiteSource, mergeUniqueUris, requiresLocalDeepEvalInstall, -} from '../extension'; +} from '../suites'; import { DEEPEVAL_VSCODE_EVENTS } from '../events/event-log'; suite('Extension Test Suite', () => { @@ -125,22 +123,6 @@ suite('Extension Test Suite', () => { } }); - test('the scaffolded suite names the file the command reports', () => { - assert.strictEqual(SCAFFOLD_FILE_NAME, 'my-agent.test.ts'); - }); - - test('the scaffolded suite is a runnable DeepEval suite', () => { - const source = getScaffoldSuiteSource(); - assert.ok(source.includes("from 'vitest'")); - assert.ok(source.includes("deepeval/vitest")); - assert.ok(source.includes('observe({')); - assert.ok(source.includes('updateCurrentSpan(')); - assert.ok(source.includes('.toPass(')); - assert.ok(source.includes('extends BaseMetric')); - // The scaffold must run without any credentials, so it must not read env. - assert.ok(!source.includes('process.env')); - }); - test('discovery returns only real files', async () => { const files = await findDeepEvalTestFiles(); for (const file of files) {