From 2cecce8baf5c5c5ded20b6461d3459e525b961b6 Mon Sep 17 00:00:00 2001 From: viktar-b <60892287+viktar-b@users.noreply.github.com> Date: Fri, 6 Nov 2026 09:03:18 +0000 Subject: [PATCH] test(initializer): qualify figure and reference authoring --- tests/integration/README.md | 8 +- .../installed/_initializer-browser.ts | 28 +- .../installed/_initializer-evidence.ts | 4 +- .../installed/_initializer-guidance.ts | 1 + .../installed/_initializer-processes.ts | 3 +- .../installed/_initializer-references.ts | 375 ++++++++++++++++++ .../installed/initializer-acceptance.ts | 18 +- 7 files changed, 424 insertions(+), 13 deletions(-) create mode 100644 tests/integration/installed/_initializer-references.ts diff --git a/tests/integration/README.md b/tests/integration/README.md index 90a7a84..d8a2533 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -29,8 +29,12 @@ The [initializer acceptance](installed/initializer-acceptance.ts) runs `npm create cs-object my-report` against a temporary loopback npm registry backed by actual archives. It installs the Python wheel into the generated project, starts `npm run dev`, and checks browser edits, the output-only calculation API, -report/PDF evidence and restart. It also replaces the starter with a copy of the -canonical two-panel calculation. Third-party dependencies still need network +report/PDF evidence and restart. It executes the packaged composed-report/figure recipe and follows the shipped +reference-binding recipe with independently established intermediate and output +values. Correct, wrong and unmatched cases, formula/text changes and explicit +rebinding are checked before browser edits, output-only curl and same-run HTML, +PDF and evidence. Automatic results leave visual inspection and engineering +approval separate. It also exercises a copy of the canonical two-panel calculation. Third-party dependencies still need network access. This tests installation without publishing a release. The canonical two-panel calculations and metadata stay in `examples/two-panel`. diff --git a/tests/integration/installed/_initializer-browser.ts b/tests/integration/installed/_initializer-browser.ts index 16734fd..8e2098e 100644 --- a/tests/integration/installed/_initializer-browser.ts +++ b/tests/integration/installed/_initializer-browser.ts @@ -19,6 +19,7 @@ export async function exerciseBrowser(options: { readonly outputs: Readonly>; readonly reportId: string; readonly reportTitle: string; + readonly referenceStatus?: 'passed' | 'not_applicable'; }) { const browser = await chromium.launch({ headless: true }); try { @@ -115,7 +116,10 @@ export async function exerciseBrowser(options: { assert.equal(await page.locator('[data-slot=sheet-overlay]').count(), 0); await page.locator('iframe#report').waitFor({ state: 'visible' }); assert(await page.locator('iframe#report').getAttribute('src')); - assert.match(await page.locator('button#pdf').innerText(), /review pending/i); + assert.match( + await page.locator('button#pdf').innerText(), + /review pending/i, + ); assert.equal(await page.locator('#checks').isVisible(), true); const initialReportText = await page .frameLocator('iframe#report') @@ -152,15 +156,17 @@ export async function exerciseBrowser(options: { assert(reportMatch?.[1]); const runBase = `/api/reports/${options.reportId}/runs/${reportMatch[1]}`; assert.equal(evidencePath, `${runBase}/evidence.json`); - assert.match( - await page.locator('#checks').innerText(), - /Human inspection\s+WIP/i, - ); - if (options.reportId === 'rectangle-area') + const checksText = await page.locator('#checks').innerText(); + assert.match(checksText, /Human inspection\s+WIP/i); + if (options.referenceStatus) assert.match( - await page.locator('#checks').innerText(), - /Independent reference\s+WIP/i, + checksText, + options.referenceStatus === 'passed' + ? /Independent reference\s+Passed/i + : /Independent reference\s+WIP/i, ); + if (options.reportId === 'rectangle-area') + assert.match(checksText, /Independent reference\s+WIP/i); await page.setViewportSize({ width: 950, height: 900 }); assert.equal( await page.evaluate( @@ -230,6 +236,11 @@ export async function exerciseBrowser(options: { evidence.artifacts.pdf?.sha256, hashBytes(readFileSync(options.downloadPath)), ); + if (options.referenceStatus) + assert.equal( + evidence.checks.independentReferenceAgreement?.status, + options.referenceStatus, + ); if (options.reportId === 'rectangle-area') { await page.locator('[name=width]').fill('-0'); await page.getByRole('button', { name: 'API', exact: true }).click(); @@ -270,6 +281,7 @@ export async function exerciseBrowser(options: { ), ); } + return summary; } finally { await browser.close(); } diff --git a/tests/integration/installed/_initializer-evidence.ts b/tests/integration/installed/_initializer-evidence.ts index 645fe1b..7684439 100644 --- a/tests/integration/installed/_initializer-evidence.ts +++ b/tests/integration/installed/_initializer-evidence.ts @@ -125,6 +125,7 @@ export async function artifactSet(options: { readonly unit: string; }[]; readonly referencePath?: string; + readonly referenceStatus?: 'passed' | 'not_applicable'; readonly name: string; readonly reportId: string; }) { @@ -183,7 +184,8 @@ export async function artifactSet(options: { assert.equal(evidence.checks.sourceToDocumentConsistency?.status, 'passed'); assert.equal( evidence.checks.independentReferenceAgreement?.status, - options.referencePath ? 'passed' : 'not_applicable', + options.referenceStatus ?? + (options.referencePath ? 'passed' : 'not_applicable'), ); assert.equal(evidence.checks.documentContent?.status, 'passed'); assert.equal(evidence.checks.rendering?.status, 'passed'); diff --git a/tests/integration/installed/_initializer-guidance.ts b/tests/integration/installed/_initializer-guidance.ts index 7ec4004..3888ea5 100644 --- a/tests/integration/installed/_initializer-guidance.ts +++ b/tests/integration/installed/_initializer-guidance.ts @@ -171,4 +171,5 @@ export async function exerciseGuidance(options: { 'data:image/svg+xml', ]) assert(html.includes(expected), `Missing recipe content: ${expected}`); + return { trial }; } diff --git a/tests/integration/installed/_initializer-processes.ts b/tests/integration/installed/_initializer-processes.ts index 05050d8..1e5b52a 100644 --- a/tests/integration/installed/_initializer-processes.ts +++ b/tests/integration/installed/_initializer-processes.ts @@ -25,6 +25,7 @@ export type CommandRunner = (options: { readonly args: readonly string[]; readonly env?: NodeJS.ProcessEnv; readonly timeoutMs?: number; + readonly expectedStatus?: number; }) => Promise; export function createCommandRunner( @@ -97,7 +98,7 @@ export function createCommandRunner( `${options.command} stdout exceeded ${maximumCapturedCharacters} characters; see ${log}`, ), ); - if (status !== 0) + if (status !== (options.expectedStatus ?? 0)) return rejectRun( new Error( `${options.command} ${options.args.join(' ')} exited ${status}; see ${log}\n${stderr}${stderrTruncated ? '\n[stderr truncated in memory; full output is in the log]' : ''}`, diff --git a/tests/integration/installed/_initializer-references.ts b/tests/integration/installed/_initializer-references.ts new file mode 100644 index 0000000..d5d9e29 --- /dev/null +++ b/tests/integration/installed/_initializer-references.ts @@ -0,0 +1,375 @@ +import assert from 'node:assert/strict'; +import { cpSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { + CommandReportSchema, + ExecutionResponseSchema, + type LegacyReferenceCaseSchema, + ReferenceBindingSchema, + ReferenceFileSchema, +} from '@cs-object/core'; +import { z } from 'zod'; +import { exerciseBrowser } from './_initializer-browser.ts'; +import { + artifactSet, + hashBytes, + OutputsSchema, + writeJson, +} from './_initializer-evidence.ts'; +import { type CommandRunner, startDev } from './_initializer-processes.ts'; + +// Follow the shipped composed report and reference recipes. Expectations are +// elementary geometry established before execution, never observed answers. +const independentCases = [ + { width: 2, area: 6, totalArea: 12 }, + { width: 4, area: 12, totalArea: 24 }, +]; + +export async function exerciseReferenceAuthoring(options: { + readonly project: string; + readonly trial: string; + readonly installedPython: string; + readonly root: string; + readonly environment: NodeJS.ProcessEnv; + readonly run: CommandRunner; +}) { + const { project, trial, installedPython, root, environment, run } = options; + // The generated report registry requires sources below calculations/. Keep + // this recipe separate from the maintained example exercised earlier. + const calculations = join(project, 'calculations/reference_trial'); + cpSync(join(trial, 'calculations'), calculations, { + recursive: true, + filter: (path) => + !path + .split(/[\\/]/) + .some((part) => ['__pycache__', '_cso_bindings'].includes(part)), + }); + await run({ + cwd: project, + command: installedPython, + args: ['-I', '-m', 'cso_python', 'bindings', calculations], + }); + const source = join(calculations, 'composed.cso.py'); + const referencePath = join(trial, 'reference.json'); + const cli = join(project, 'node_modules/@cs-object/cli/dist/cli.js'); + const env = { ...environment, PYTHON: installedPython }; + const receipts: { name: string; ok: boolean; referenceStatus: string }[] = []; + async function verify( + name: string, + width: number, + reference: string | undefined, + expectedStatus = 0, + ) { + const report = CommandReportSchema.parse( + JSON.parse( + await run({ + cwd: project, + command: process.execPath, + args: [ + cli, + 'verify', + source, + '--function', + 'calculate', + '--input', + `width=${width}`, + ...(reference ? ['--reference', reference] : []), + '--format', + 'json', + ], + env, + expectedStatus, + }), + ), + ); + writeJson(join(root, `reference-${name}.json`), report); + receipts.push({ + name, + ok: report.ok, + referenceStatus: report.checks.independentReferenceAgreement.status, + }); + assert.equal(report.checks.sourceToDocumentConsistency.status, 'passed'); + return report; + } + const capture = ExecutionResponseSchema.parse( + JSON.parse( + await run({ + cwd: project, + command: installedPython, + args: [ + '-I', + '-m', + 'cso_python', + 'execute', + source, + '--function', + 'calculate', + ], + }), + ), + ); + assert(capture.ok); + const outputs = capture.execution.authoring?.outputs; + const areaId = z + .string() + .min(1) + .parse( + outputs?.find( + (item) => item.invocationId !== 'root' && item.name === 'area', + )?.symbolId, + ); + const totalId = z + .string() + .min(1) + .parse( + outputs?.find( + (item) => item.invocationId === 'root' && item.name === 'total_area', + )?.symbolId, + ); + assert.deepEqual( + new Set([areaId, totalId]), + new Set( + capture.execution.observations + .filter((item) => item.kind === 'formula') + .map((item) => item.symbolId), + ), + ); + + async function bind(revision: string) { + const cases: z.infer[] = []; + for (const item of independentCases) { + const report = await verify( + `bind-${revision}-${item.width}`, + item.width, + undefined, + ); + const { + entryModuleId, + entrySourceHash, + sourceClosureHash, + function: entryFunction, + resolvedInputs, + resolvedInputKinds, + } = report.provenance; + cases.push({ + id: `pair-3-by-${item.width}`, + revision, + basis: { + method: 'Independent elementary rectangle geometry', + derivation: `Perpendicular sides: 3 m times ${item.width} m gives ${item.area} m^2. Two equal non-overlapping rectangles give ${item.totalArea} m^2.`, + sourceDescription: + 'Elementary geometry stated before executing the packaged recipe.', + }, + binding: ReferenceBindingSchema.parse({ + entryModuleId, + entrySourceHash, + sourceClosureHash, + function: entryFunction, + resolvedInputs, + resolvedInputKinds, + }), + expected: [ + { symbolId: areaId, value: item.area, unit: 'm^2' }, + { symbolId: totalId, value: item.totalArea, unit: 'm^2' }, + ], + }); + } + const reference = ReferenceFileSchema.parse({ + referenceVersion: '1', + cases, + }); + writeJson(referencePath, reference); + writeJson(join(root, `reference-revision-${revision}.json`), reference); + return reference; + } + const original = await bind('1'); + const expectedValues = original.cases.map((item) => item.expected); + for (const item of independentCases) { + const report = await verify( + `matching-${item.width}`, + item.width, + referencePath, + ); + assert.equal(report.ok, true); + assert.equal(report.checks.independentReferenceAgreement.status, 'passed'); + assert.equal( + report.checks.independentReferenceAgreement.counts?.checked, + 2, + ); + } + const wrongPath = join(trial, 'wrong-reference.json'); + writeJson(wrongPath, { + ...original, + cases: original.cases.map((item) => ({ + ...item, + expected: item.expected.map((expected) => ({ + ...expected, + value: expected.value + 1, + })), + })), + }); + const wrong = await verify('wrong-matching', 4, wrongPath, 1); + assert.equal(wrong.ok, false); + assert.equal(wrong.checks.independentReferenceAgreement.status, 'failed'); + const unmatched = await verify('unmatched-inputs', 5, referencePath); + assert.equal(unmatched.ok, true); + assert.equal( + unmatched.checks.independentReferenceAgreement.status, + 'not_applicable', + ); + + const authored = readFileSync(source, 'utf8'); + const edits = [ + { + name: 'formula', + before: '2.0 * single["area"]', + after: 'single["area"] + single["area"]', + }, + { + name: 'text', + before: 'Two equal perpendicular rectangles.', + after: 'Two identical perpendicular rectangles.', + }, + ]; + for (const [index, edit] of edits.entries()) { + const current = readFileSync(source, 'utf8'); + assert(current.includes(edit.before)); + writeFileSync(source, current.replace(edit.before, edit.after)); + const stale = await verify(`stale-${edit.name}`, 4, referencePath); + assert.equal(stale.ok, true); + assert.equal( + stale.checks.independentReferenceAgreement.status, + 'not_applicable', + ); + // Explicit review: the equivalent doubling formula and wording change leave + // the independently derived geometry and expected numbers unchanged. + const rebound = await bind(String(index + 2)); + assert.deepEqual( + rebound.cases.map((item) => item.expected), + expectedValues, + ); + for (const item of independentCases) { + const checked = await verify( + `rebound-${edit.name}-${item.width}`, + item.width, + referencePath, + ); + assert.equal( + checked.checks.independentReferenceAgreement.status, + 'passed', + ); + assert.equal( + checked.checks.independentReferenceAgreement.counts?.checked, + 2, + ); + } + } + assert.notEqual(readFileSync(source, 'utf8'), authored); + + const reportsPath = join(project, 'reports.json'); + const reportsBefore = readFileSync(reportsPath); + writeJson(reportsPath, [ + { + id: 'composed-reference', + title: 'Referenced rectangle pair', + source: 'calculations/reference_trial/composed.cso.py', + function: 'calculate', + reference: 'guidance-trial/reference.json', + }, + ]); + try { + const dev = await startDev({ + project, + name: 'reference-authoring-dev', + environment, + logs: root, + }); + try { + const curl = OutputsSchema.parse( + JSON.parse( + await run({ + cwd: project, + command: 'curl', + args: [ + '--fail-with-body', + '--silent', + '--show-error', + '-H', + 'content-type: application/json', + '--data', + JSON.stringify({ inputs: { length: 3, width: 4 } }), + `${dev.origin}/api/reports/composed-reference/calculate`, + ], + }), + ), + ); + assert.deepEqual(curl, { total_area: 24 }); + const artifacts = []; + for (const item of [ + { width: 4, totalArea: 24, status: 'passed' }, + { width: 5, totalArea: 30, status: 'not_applicable' }, + ] satisfies { + width: number; + totalArea: number; + status: 'passed' | 'not_applicable'; + }[]) { + const name = `reference-width-${item.width}`; + const downloadPath = join(root, `${name}-browser.pdf`); + const browserRun = await exerciseBrowser({ + origin: dev.origin, + reportId: 'composed-reference', + reportTitle: 'Referenced rectangle pair', + width: String(item.width), + initialText: ['Combined area', '12'], + expectedText: [ + 'Combined area', + String(item.totalArea), + 'Rectangle dimensions', + ], + screenshot: join(root, `${name}-browser.png`), + downloadPath, + inputs: { length: 3, width: item.width }, + outputs: { total_area: item.totalArea }, + referenceStatus: item.status, + }); + const artifact = await artifactSet({ + root, + origin: dev.origin, + run: browserRun, + inputs: { length: 3, width: item.width }, + outputs: { total_area: item.totalArea }, + rows: [{ description: 'Combined area', unit: 'm^2' }], + referencePath, + referenceStatus: item.status, + name, + reportId: 'composed-reference', + }); + assert.equal( + hashBytes(readFileSync(downloadPath)), + hashBytes(readFileSync(artifact.pdfPath)), + ); + assert( + readFileSync(artifact.htmlPath, 'utf8').includes( + 'data:image/svg+xml', + ), + ); + artifacts.push(artifact); + } + const evidence = { + receipts, + artifacts, + referencePath, + source, + automatic: 'passed', + visualInspection: 'pending', + engineeringApproval: 'not_supplied', + }; + writeJson(join(root, 'reference-authoring.json'), evidence); + return evidence; + } finally { + await dev.stop(); + } + } finally { + writeFileSync(reportsPath, reportsBefore); + } +} diff --git a/tests/integration/installed/initializer-acceptance.ts b/tests/integration/installed/initializer-acceptance.ts index 9832491..54acbd5 100644 --- a/tests/integration/installed/initializer-acceptance.ts +++ b/tests/integration/installed/initializer-acceptance.ts @@ -28,6 +28,7 @@ import { writeJson, } from './_initializer-evidence.ts'; import { exerciseGuidance } from './_initializer-guidance.ts'; +import { exerciseReferenceAuthoring } from './_initializer-references.ts'; import { createCommandRunner, type DevServer, @@ -219,7 +220,12 @@ try { ).trim(), ); assert(pythonLocation.startsWith(`${join(project, '.venv')}${sep}`)); - await exerciseGuidance({ project, installedPython, environment, run }); + const { trial } = await exerciseGuidance({ + project, + installedPython, + environment, + run, + }); dev = await startDev({ project, @@ -456,6 +462,15 @@ try { await dev.stop(); dev = undefined; + const referenceAuthoring = await exerciseReferenceAuthoring({ + project, + trial, + installedPython, + root, + environment, + run, + }); + const qualification = join(root, 'qualification.json'); writeJson(qualification, { ok: true, @@ -470,6 +485,7 @@ try { sha256: hashBytes(readFileSync(item.archive)), })), pythonLocation, + referenceAuthoring, screenshots: [syntheticScreenshot, twoPanelScreenshot], browserDownloads: [syntheticDownload, twoPanelDownload], reports: [syntheticArtifacts, twoPanelArtifacts].map((item) => ({