Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions docs/development/testing.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -274,3 +274,25 @@ Inspect the run's individual platform jobs before claiming full success. A
queued, canceled or failed job is missing evidence, even if other jobs passed.
The platform fixtures exercise local paths. Authenticated agent tool execution
has separate acceptance.

## Normal-flow shadow cancellation

The local normal-flow shadow runner forwards its explicit cancellation signal
into the ordinary review operation, detector and verifier. An interrupted run
records `workflow-result.status=interrupted`, retains its started/pending request
observations, and exits with status 130. It cannot report successful ordinary
completion or issue a completed verification receipt. A matched experimental
arm that is canceled does not start its inherited comparison arm.

Run `pnpm run test:bugbot-shadow-verification` for the model-free stage controls:
before detection, during detection, after preparation, during verification,
after a technical provider interruption, and uninterrupted completion. The
controlled programmatic ports exercise the actual runner and application with
pinned temporary Git sources and fail-on-write SCM ports. They are synthetic
controls, not model measurements or qualification evidence. The loader uses the
shipping compiler options so the real progress/cancellation path is exercised.

Use a new output destination for every actual attempt and retain the interrupted
ledger. Old ledgers remain historical observations; replay needs the new driver
pin and its applicable source/runtime admission. This adds no Action setting,
elapsed-time cutoff, token budget or turn limit. Usage remains optional telemetry.
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@
"test": "jest",
"test:watch": "jest --watch",
"test:coverage": "jest --coverage --json --outputFile=coverage/setup-acceptance-jest.json && node scripts/validate-coverage-budgets.cjs && node scripts/validate-setup-assistant-acceptance.cjs coverage/setup-acceptance-jest.json && pnpm run test:extension",
"test:bugbot-shadow-verification": "node --test quality/bugbot/evidence/issue-950/shadow-verification/verification.test.cjs quality/bugbot/evidence/issue-950/normal-flow-shadow/config.test.cjs quality/bugbot/evidence/issue-950/normal-flow-shadow/paths.test.cjs quality/bugbot/evidence/issue-950/ev07-shadow/launch-full-review.test.cjs quality/bugbot/evidence/issue-950/ev07-shadow/review-runtime.test.cjs quality/bugbot/evidence/issue-950/ev07-shadow/score-review.test.cjs quality/bugbot/evidence/issue-950/ev07-shadow/plan.test.cjs quality/bugbot/evidence/issue-950/status-card-publication/publisher.test.cjs",
"test:bugbot-shadow-verification": "node --test quality/bugbot/evidence/issue-950/shadow-verification/verification.test.cjs quality/bugbot/evidence/issue-950/normal-flow-shadow/config.test.cjs quality/bugbot/evidence/issue-950/normal-flow-shadow/paths.test.cjs quality/bugbot/evidence/issue-950/ev07-shadow/launch-full-review.test.cjs quality/bugbot/evidence/issue-950/ev07-shadow/review-runtime.test.cjs quality/bugbot/evidence/issue-950/ev07-shadow/score-review.test.cjs quality/bugbot/evidence/issue-950/ev07-shadow/plan.test.cjs quality/bugbot/evidence/issue-950/status-card-publication/publisher.test.cjs quality/bugbot/evidence/issue-950/normal-flow-shadow/cancellation.test.cjs",
"eval:bugbot": "jest src/tooling/__tests__/bugbot_quality_eval.test.ts src/tooling/__tests__/bugbot_benchmark.test.ts src/tooling/__tests__/bugbot_request_transform_evidence.test.ts --runInBand",
"typecheck": "tsc --noEmit && pnpm run typecheck:extension",
"metrics:architecture": "node scripts/collect-architecture-metrics.cjs",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
'use strict';

const assert = require('node:assert/strict');
const { execFileSync } = require('node:child_process');
const fs = require('node:fs');
const { tmpdir } = require('node:os');
const path = require('node:path');
const { test, after } = require('node:test');
const { run, verifyMatchedCandidate } = require('./run.cjs');
const retained = [];
after(() => {
if (process.env.COPILOT_CANCELLATION_EVIDENCE) fs.writeFileSync(process.env.COPILOT_CANCELLATION_EVIDENCE,
JSON.stringify({ method: 'actual normal-flow runner and ordinary operation; controlled providers',
modelCalls: 0, evaluatedPrWriteAttempts: 0, attempts: retained }, null, 2) + '\n', { flag: 'wx' });
});

for (const stage of ['before-detection', 'during-detection', 'between-preparation-and-verification',
'during-verification', 'after-provider-interruption', 'uninterrupted']) {
test('the actual normal-flow operation preserves explicit cancellation: ' + stage, async t => {
const directory = fs.mkdtempSync(path.join(tmpdir(), 'bugbot-972-cancel-'));
t.after(() => fs.rmSync(directory, { recursive: true, force: true }));
const source = path.join(directory, 'project.git');
fs.mkdirSync(source);
const git = (...args) => execFileSync('git', ['-C', source, ...args], { encoding: 'utf8' }).trim();
git('init', '--quiet', '--template=');
fs.writeFileSync(path.join(source, 'index.ts'), 'export const value = 1;\n');
git('add', '.');
const commit = () => git('-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', 'commit', '--quiet', '-m', 'fixed source');
commit();
const base = git('rev-parse', 'HEAD'), baseTree = git('rev-parse', 'HEAD^{tree}');
fs.writeFileSync(path.join(source, 'index.ts'), 'export const value = 1;\nexport const added = 2;\n');
git('add', '.'); commit();
const head = git('rev-parse', 'HEAD'), tree = git('rev-parse', 'HEAD^{tree}');
const manifest = path.join(directory, 'subjects.json'), output = path.join(directory, 'attempt.jsonl');
fs.writeFileSync(manifest, JSON.stringify({ publication: 'dry-run', effort: 'xhigh', subjects: [{
repository: 'owner/project', number: 1, baseRef: 'master', headRef: 'review', head, base, tree, baseTree,
model: 'gpt-6-luna',
}] }));
const cancellation = new AbortController();
let detectorCalls = 0, verifierCalls = 0;
const pending = [];
const result = await run(directory, 1, output, manifest, {
signal: cancellation.signal,
beforeReview() { if (stage === 'before-detection') cancellation.abort(); },
progress: { observe(observation) {
if (stage === 'between-preparation-and-verification' && observation.stage === 'verifying') cancellation.abort();
} },
findings: { async query(request) {
detectorCalls++;
pending.push({ stage: 'detector', abortedAtEntry: request.signal.aborted });
if (stage === 'during-detection') cancellation.abort();
return { outputLocale: 'en-US', findings: [{ id: 'controlled-candidate', file: 'index.ts', line: 2,
title: 'Controlled candidate', description: 'An owned cancellation fixture hypothesis.',
codeSnippet: 'export const added = 2;', confidence: 0.99, severity: 'medium', category: 'correctness' }],
resolved_findings: [], partition_id: request.prompt.match(/Return partition_id exactly as `([^`]+)`/u)?.[1],
reviewed_head_sha: head };
} },
verifier: { async verify(request) {
verifierCalls++;
const observation = { stage: 'verifier', id: request.candidate.id, abortedAtEntry: request.signal.aborted };
pending.push(observation);
if (stage === 'during-verification' || stage === 'after-provider-interruption') {
cancellation.abort();
observation.abortedAfterInterruption = request.signal.aborted;
return { status: 'technical-failure', reason: stage === 'after-provider-interruption'
? 'provider-turn-failed' : 'investigation-canceled' };
}
return { status: 'verdict', verdict: 'inconclusive', reason: 'Controlled missing evidence.',
missing: 'A test-only source gap.', citations: [], premises: [] };
} },
});
const rows = fs.readFileSync(output, 'utf8').trim().split('\n').map(JSON.parse);
const terminal = rows.find(row => row.event === 'workflow-result');
retained.push({ stage, pending, detectorCalls, verifierCalls, rows });
assert.deepEqual(result.writes, []);
assert.equal(terminal.checkoutUnchanged, true);
if (stage === 'uninterrupted') {
assert.equal(result.status, 'completed');
assert.equal(result.results[0].success, true);
assert.equal(detectorCalls, 1); assert.equal(verifierCalls, 1);
} else {
assert.equal(result.status, 'interrupted');
assert.equal(result.results[0].success, false);
assert.equal(result.results[0].payload.bugbotPublicationBlocked, true);
assert.equal(JSON.stringify(result.results[0].payload).includes('copilot:bugbot-verification'), false);
assert.equal(verifierCalls, ['during-verification', 'after-provider-interruption'].includes(stage) ? 1 : 0);
assert.equal(detectorCalls, stage === 'before-detection' ? 0 : 1);
}
});
}

test('explicit cancellation in the experimental arm prevents starting the inherited arm', async () => {
const controller = new AbortController(), calls = [];
const profile = { provider: 'codex', modelProvider: 'openai', model: 'gpt-6-luna', effort: 'xhigh' };
await verifyMatchedCandidate({ signal: controller.signal, candidate: { id: 'owned' }, configuration: profile },
{ primary: profile, experimental: profile, comparison: profile }, async arm => {
calls.push(arm); controller.abort(); return { status: 'technical-failure', reason: 'investigation-canceled' };
}, () => {});
assert.deepEqual(calls, ['experimental']);
});
31 changes: 19 additions & 12 deletions quality/bugbot/evidence/issue-950/normal-flow-shadow/run.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,7 @@ const path = require('node:path');
const official = require('../shadow-real-prs/official.json');
const shadowPlan = require('../shadow-verification/plan.json');
const root = path.resolve(__dirname, '../../../../..');
const ts = require(path.join(root, 'node_modules/typescript'));
require.extensions['.ts'] = (module, filename) => module._compile(ts.transpileModule(fs.readFileSync(filename, 'utf8'), {
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022, esModuleInterop: true },
}).outputText, filename);
require('../ev07-shadow/load.cjs');

const load = file => require(path.join(root, 'src', file));
const { composeBugbotReview } = load('infrastructure/composition/bugbot_review_composition');
Expand Down Expand Up @@ -84,6 +81,7 @@ async function verifyMatchedCandidate(request, profiles, verifyArm, recordCompar
// The review already runs up to two candidates at a time. Run the arms
// sequentially within each candidate so the same session limit applies.
const settled = async (arm, input) => {
if (input.signal?.aborted) return { status: 'technical-failure', reason: 'investigation-canceled' };
try {
return await verifyArm(arm, input);
} catch {
Expand All @@ -96,7 +94,7 @@ async function verifyMatchedCandidate(request, profiles, verifyArm, recordCompar
return experimental;
}

async function run(lab, number, destination, manifestPath = path.join(__dirname, 'subjects.json')) {
async function run(lab, number, destination, manifestPath = path.join(__dirname, 'subjects.json'), ports = {}) {
lab = path.resolve(lab);
const manifest = fs.readFileSync(manifestPath);
const frozen = JSON.parse(manifest);
Expand All @@ -120,7 +118,7 @@ async function run(lab, number, destination, manifestPath = path.join(__dirname,
const cancel = () => cancellation.abort();
process.once('SIGINT', cancel);
process.once('SIGTERM', cancel);
const signal = cancellation.signal;
const signal = ports.signal ? AbortSignal.any([cancellation.signal, ports.signal]) : cancellation.signal;
const withCancellation = original => original ? AbortSignal.any([original, signal]) : signal;
const writes = [];
const guard = new Proxy({}, { get: (_target, method) => () => {
Expand All @@ -142,6 +140,10 @@ async function run(lab, number, destination, manifestPath = path.join(__dirname,
headRepositoryOwner: owner, headRepositoryName: name, headRef: subject.headRef,
headSha: subject.head, changedFileCount: diff.changes.length };
const scm = {
progress: { async observe(progress, dryRun, locale) {
log({ event: 'operation-progress', progress });
await ports.progress?.observe(progress, dryRun, locale);
} },
context: {
getPullRequest: async () => identity, getSourceBoundPullRequest: async () => identity,
findOpenPullRequestsByExactHead: async () => [],
Expand Down Expand Up @@ -170,7 +172,7 @@ async function run(lab, number, destination, manifestPath = path.join(__dirname,
const ai = new Ai('shadow-campaign', '', false, workflowInputs ? ['build/*', 'specs/CATALOG.md'] : [], false,
'low', 20, [], tasks, undefined, { publicationMode: frozen.publication, effort: reviewEffort });
const operation = projectBugbotReviewOperationContext({
ai, owner, repo: name, issueNumber: subject.number, isPullRequest: true, eventName: 'pull_request',
bugbotReviewSignal: signal, ai, owner, repo: name, issueNumber: subject.number, isPullRequest: true, eventName: 'pull_request',
tokenUser: 'copilot', locale: { issue: 'en-US', pullRequest: 'en-US' },
pullRequest: { number: subject.number, head: subject.headRef, action: 'opened' },
commit: { branch: subject.headRef }, currentConfiguration: { parentBranch: subject.baseRef }, branches: {},
Expand All @@ -186,7 +188,7 @@ async function run(lab, number, destination, manifestPath = path.join(__dirname,
assert.equal(invocation.modelProvider, expected.modelProvider);
assert.equal(invocation.sandbox, 'read-only');
};
const standardDetector = createFindingsQueryPort({ cli: new AgentCliClient({
const standardDetector = ports.findings ?? createFindingsQueryPort({ cli: new AgentCliClient({
prepare(request) {
const plan = planner.prepare(request);
const flags = codexInvocation(plan.argv);
Expand Down Expand Up @@ -258,7 +260,7 @@ async function run(lab, number, destination, manifestPath = path.join(__dirname,
peak = Math.max(peak, active);
log(label(arm, { event: 'verifier-start', id: candidate.id, active, atMs: startedAt }));
try {
return await adapters[arm].verify({ ...request, signal: withCancellation(request.signal) });
return await (ports.verifier ?? adapters[arm]).verify({ ...request, signal: withCancellation(request.signal) });
} finally {
active -= 1;
lastVerifier = sinceStart();
Expand All @@ -272,13 +274,16 @@ async function run(lab, number, destination, manifestPath = path.join(__dirname,
return verifyMatchedCandidate(request, profiles, verifyArm, log);
},
};
await ports.beforeReview?.({ signal });
const results = await composeBugbotReview({ findings: detector, scm, shadowVerification: verifier,
experimentalShadowProfile: profiles.experimental }).invoke(operation);
const wallMs = sinceStart();
log({ event: 'workflow-result', results, writes, checkoutUnchanged: git(checkout, 'status', '--porcelain') === '',
const status = signal.aborted ? 'interrupted' : results.every(result => result.success) ? 'completed' : 'failed';
log({ event: 'workflow-result', status, results, writes, checkoutUnchanged: git(checkout, 'status', '--porcelain') === '',
wallMs, verificationWallMs: firstVerifier === undefined ? 0 : lastVerifier - firstVerifier,
peakVerifierConcurrency: peak });
assert.deepEqual(writes, []);
return { status, results, writes };
} finally {
process.removeListener('SIGINT', cancel);
process.removeListener('SIGTERM', cancel);
Expand All @@ -288,10 +293,12 @@ async function run(lab, number, destination, manifestPath = path.join(__dirname,
}

if (require.main === module) {
run(...process.argv.slice(2)).catch(error => {
run(...process.argv.slice(2)).then(result => {
if (result.status !== 'completed') process.exitCode = result.status === 'interrupted' ? 130 : 1;
}).catch(error => {
console.error(error);
process.exitCode = 1;
});
}

module.exports = { selectVerifierProfiles, verifyMatchedCandidate };
module.exports = { run, selectVerifierProfiles, verifyMatchedCandidate };
Loading