diff --git a/README.md b/README.md index c76bafa9e..b45eac484 100644 --- a/README.md +++ b/README.md @@ -53,13 +53,13 @@ You keep notebooks, sections, and pages in a notes folder you choose, as plain f ### Write and draw -The Word-level text, Excel-level table, and code studio foundations are on the beta branch behind flags that are off. The text work so far adds a Format pane with every font and paragraph field, a ruler for indents and tab stops, and a language and a direction for each passage. The table work so far adds a cell format bar, spreadsheet keys, conditional formatting, styled tables in Word and HTML exports, and the logic, lookup, array, and statistics functions. +The Word-level text, Excel-level table, and code studio foundations are on the beta branch behind flags that are off. The text work so far adds a Format pane with every font and paragraph field, a ruler for indents and tab stops, a language and a direction for each passage, and comments and suggested changes. The table work so far adds a cell format bar, spreadsheet keys, conditional formatting, styled tables in Word and HTML exports, and the logic, lookup, array, and statistics functions. The code studio work so far adds Python, JavaScript, and TypeScript runs with tests, error detection, and completion without a model. Sync can merge a page by block once a block type has a merger. - [Typed notes](docs/help/typed-notes.md): text, headings, text styles kept with the notebook, lists, checklists, tables, code, callouts, Markdown, templates, and find and replace. Find by format, special character, wildcard, or regular expression. The Home tab has font and size boxes, split buttons for underline, colors, and lists, and a styles gallery. Typed web and e-mail addresses turn into links. Pasted tables and spreadsheet cells become tables. Copies give other apps plain text. - [Moving between blocks](docs/help/moving-between-blocks.md): keys that move between text, pictures, tables, and code without the mouse. - [Document structure](docs/help/document-structure.md): equation numbers and references, formula sheets, practice mode, problem-set templates, and figures. - [Code cells](docs/help/code-cells.md): short Python or JavaScript programs that run on the page; a cell from a newer version opens read-only instead of being changed. -- Code projects (Beta and Nightly builds): several files and tests in one page block, shown as a small card with Run and Open in Code studio, with .zip export and import; search finds their code. +- Code projects (Beta and Nightly builds): several files and tests in one page block, shown as a small card with Run and Open in Code studio, with .zip export and import; search finds their code. Help for [Python](docs/help/python-projects.md) and [JavaScript and TypeScript](docs/help/javascript-and-typescript-projects.md) projects. - [Ink and palm rejection](docs/help/ink-and-palm-rejection.md): pens, a highlighter, erasers, the lasso, shapes, and gestures, with your palm resting on the screen. - [My handwriting](docs/help/my-handwriting.md): typed text and math written in your own hand, learned from a calibration page. - [Paper and page views](docs/help/paper-and-page-views.md): flow, paginated, and canvas pages, with lined, grid, and dot paper; slides, print, and PDF. diff --git a/app/bundle-budget.json b/app/bundle-budget.json index 95c825874..1064950fc 100644 --- a/app/bundle-budget.json +++ b/app/bundle-budget.json @@ -1,6 +1,6 @@ { "startup": { - "jsKb": 620, + "jsKb": 622, "cssKb": 30 }, "after": [ @@ -16,6 +16,7 @@ "core/audio", "editor/", "features/docstructure/", + "features/review/", "features/ink/", "features/pdf/" ], @@ -71,6 +72,7 @@ "theme/fonts/caladea": 8, "node_modules/mathlive/mathlive.min.mjs": 226, "node_modules/pdfjs-dist/build/pdf.mjs": 131, + "node_modules/sucrase": 46, "core/audio/index.ts": 117, "features/accounts/googleOffice/commands.ts": 96, "features/accounts/lms/commands.ts": 297, diff --git a/app/src-tauri/src/shellqol/conflicts.rs b/app/src-tauri/src/shellqol/conflicts.rs index 37eadb46a..a28968ff6 100644 --- a/app/src-tauri/src/shellqol/conflicts.rs +++ b/app/src-tauri/src/shellqol/conflicts.rs @@ -25,6 +25,7 @@ pub fn call(app: &AppHandle, name: &str, args: &Value) -> IpcResult { Ok(json!({ "mine": handle.text(), "theirs": handle.conflict_text(revision).map_err(core_error)?, + "blocks": handle.conflict_blocks(revision).map_err(core_error)?, })) } "conflict.resolve" => { diff --git a/app/src/editor/diff/formatDiff.test.ts b/app/src/editor/diff/formatDiff.test.ts new file mode 100644 index 000000000..184dec90d --- /dev/null +++ b/app/src/editor/diff/formatDiff.test.ts @@ -0,0 +1,52 @@ +// Formatting-aware compare (text DESIGN.md 9; ADR 0086). The expected notes are worked by hand from the two versions: +// the text is the same, so each difference is a mark that covers some text in one version only, or a `paras` key +// (spec 6.8) with another value. +import { describe, expect, it } from 'vitest'; +import { describeFormatting } from './formatDiff'; +import { diffPages, diffText } from './pageDiff'; +import type { DiffBlock } from './pageDiff'; + +const text = (markdown: string, extra: Record = {}): DiffBlock => ({ + id: 'b', + type: 'text', + order: 'a0', + data: { markdown, ...extra }, +}); + +describe('formatting-aware compare', () => { + it("shows 'Formatting changed: bold added' for two versions that differ only by bold", () => { + const [change] = diffText('Plants use photosynthesis daily.', 'Plants use **photosynthesis** daily.'); + expect(change.status).toBe('changed'); + expect(change.words?.every((part) => part.kind === 'same')).toBe(true); + expect(change.formatting).toEqual([{ kind: 'mark', change: 'added', mark: 'bold', text: 'photosynthesis' }]); + expect(describeFormatting(change.formatting!)).toBe('Formatting changed: bold added to “photosynthesis”'); + }); + + it('names marks removed and changed', () => { + const [change] = diffText( + '*Leaf* and stem', + 'Leaf and stem', + ); + expect(describeFormatting(change.formatting!)).toBe( + 'Formatting changed: italic removed from “Leaf”; font changed on “stem”', + ); + }); + + it('compares paras per key: an alignment change alone makes a changed paragraph', () => { + const ids = ['01m3sbpara000000000000p001']; + const before = text('Centered title', { ids, paras: { [ids[0]]: { align: 'start', before: 6 } } }); + const after = text('Centered title', { ids, paras: { [ids[0]]: { align: 'center', before: 6 } } }); + const [block] = diffPages({ blocks: [before] }, { blocks: [after] }).blocks; + expect(block.status).toBe('changed'); + const paragraph = block.kind === 'text' ? block.paragraphs[0] : null; + expect(paragraph?.formatting).toEqual([{ kind: 'para', key: 'align', before: 'start', after: 'center' }]); + expect(describeFormatting(paragraph!.formatting!)).toBe('Formatting changed: alignment: center'); + }); + + it('keeps equal paragraphs with equal paras the same', () => { + const ids = ['01m3sbpara000000000000p001']; + const one = text('Same', { ids, paras: { [ids[0]]: { align: 'center' } } }); + const [block] = diffPages({ blocks: [one] }, { blocks: [one] }).blocks; + expect(block.status).toBe('same'); + }); +}); diff --git a/app/src/editor/diff/formatDiff.ts b/app/src/editor/diff/formatDiff.ts new file mode 100644 index 000000000..240943af2 --- /dev/null +++ b/app/src/editor/diff/formatDiff.ts @@ -0,0 +1,194 @@ +// Formatting-aware compare (text DESIGN.md 9; ADR 0086). When a paragraph's text is the same in two versions but its +// Markdown or its paragraph formatting differs, the compare view says what changed: "bold added to 'photosynthesis'", +// "alignment: center". Marks compare character by character; `paras` entries compare key by key, so a later +// paragraph merge can combine them per key (RFC 7396). +import type { Mark, Node as PMNode } from '@tiptap/pm/model'; +import { entryFromAttrs } from '../paragraph/attributes'; +import { ELEMENT_TYPES } from '../schema/specs'; +import { t } from '../../strings/t'; +import type { MessageKey } from '../../strings/t'; + +/** A mark that covers some text in one version and not in the other, or with other values. */ +export interface MarkNote { + kind: 'mark'; + change: 'added' | 'removed' | 'changed'; + mark: string; + text: string; +} + +/** A paragraph formatting key with another value, or none, in the newer version. */ +export interface ParaNote { + kind: 'para'; + key: string; + before: unknown; + after: unknown; +} + +export type FormatNote = MarkNote | ParaNote; + +/** One character's marks, by mark name; a block boundary or a leaf node is a character too. */ +type Cell = { char: string; marks: ReadonlyMap }; + +function cells(node: PMNode): Cell[] { + const out: Cell[] = []; + let first = true; + node.descendants((child) => { + if (child.isTextblock && !first) out.push({ char: '\n', marks: new Map() }); + if (child.isTextblock) first = false; + if (!child.isInline) return true; + const marks = new Map(child.marks.map((mark) => [mark.type.name, mark])); + const text = child.isText ? (child.text ?? '') : ''; + for (const char of text) out.push({ char, marks }); + return false; + }); + return out; +} + +function stateOf(before: Mark | undefined, after: Mark | undefined): MarkNote['change'] | null { + if (!before && !after) return null; + if (!before) return 'added'; + if (!after) return 'removed'; + return before.eq(after) ? null : 'changed'; +} + +/** The text's runs that each mark was added to, removed from, or changed on, in text order. */ +export function markNotes(before: PMNode, after: PMNode): MarkNote[] { + const a = cells(before); + const b = cells(after); + if (a.length !== b.length || a.some((cell, i) => cell.char !== b[i].char)) return []; + const names = new Set(); + for (const cell of [...a, ...b]) for (const name of cell.marks.keys()) names.add(name); + const runs: (MarkNote & { at: number })[] = []; + for (const mark of names) { + let open: (MarkNote & { at: number }) | null = null; + for (let i = 0; i <= a.length; i++) { + const change = i < a.length ? stateOf(a[i].marks.get(mark), b[i].marks.get(mark)) : null; + if (open && change === open.change && a[i].char !== '\n') { + open.text += a[i].char; + continue; + } + if (open) runs.push(open); + open = change && a[i].char !== '\n' ? { kind: 'mark', change, mark, text: a[i].char, at: i } : null; + } + } + return runs + .sort((x, y) => x.at - y.at || x.mark.localeCompare(y.mark)) + .map(({ kind, change, mark, text }) => ({ kind, change, mark, text: text.trim() || text })); +} + +/** The `paras` entries of a top-level block's elements, in document order. */ +export function paraEntries(node: PMNode): Record[] { + const out: Record[] = []; + const add = (one: PMNode) => { + if (!ELEMENT_TYPES.has(one.type.name)) return; + const entry = entryFromAttrs(one.attrs); + out.push(entry && typeof entry === 'object' && !Array.isArray(entry) ? (entry as Record) : {}); + }; + add(node); + node.descendants((child) => { + add(child); + return !child.isTextblock; + }); + return out; +} + +const same = (x: unknown, y: unknown) => JSON.stringify(x) === JSON.stringify(y); + +/** The `paras` keys whose values differ between two blocks' elements, key by key. */ +export function paraNotes(before: PMNode, after: PMNode): ParaNote[] { + const a = paraEntries(before); + const b = paraEntries(after); + const notes: ParaNote[] = []; + for (let i = 0; i < Math.max(a.length, b.length); i++) { + const was = a[i] ?? {}; + const now = b[i] ?? {}; + for (const key of [...new Set([...Object.keys(was), ...Object.keys(now)])].sort()) { + if (!same(was[key], now[key])) notes.push({ kind: 'para', key, before: was[key], after: now[key] }); + } + } + return notes; +} + +/** A signature of a block's paragraph formatting, for telling equal blocks apart. */ +export function paraSignature(node: PMNode): string { + const entries = paraEntries(node); + return entries.some((entry) => Object.keys(entry).length > 0) ? JSON.stringify(entries) : ''; +} + +const MARK_LABELS = new Set([ + 'link', + 'comment', + 'insertion', + 'deletion', + 'formatChange', + 'dir', + 'lang', + 'charStyle', + 'bold', + 'weight', + 'italic', + 'strike', + 'underline', + 'highlight', + 'shade', + 'textColor', + 'font', + 'textSize', + 'caps', + 'tracking', + 'scale', + 'shift', + 'kern', + 'otf', + 'effect', + 'box', + 'subscript', + 'superscript', + 'code', +]); + +const PARA_LABELS = new Set([ + 'align', + 'indent', + 'before', + 'after', + 'contextual', + 'line', + 'keepNext', + 'keepLines', + 'breakBefore', + 'widows', + 'tabs', + 'border', + 'shade', + 'dir', + 'lang', + 'outline', + 'dropCap', + 'list', + 'hyphenate', +]); + +const markLabel = (mark: string) => + t(`textReview.marks.${MARK_LABELS.has(mark) ? mark : 'other'}` as MessageKey & `textReview.marks.${string}`); +const paraLabel = (key: string) => + t(`textReview.paras.${PARA_LABELS.has(key) ? key : 'other'}` as MessageKey & `textReview.paras.${string}`); + +const valueText = (value: unknown) => (typeof value === 'string' ? value : JSON.stringify(value)); + +/** One note in words, such as "bold added to “photosynthesis”" or "alignment: center". */ +export function describeNote(note: FormatNote): string { + if (note.kind === 'para') { + if (note.after === undefined) return t('textReview.paraRemoved', { key: paraLabel(note.key) }); + return t('textReview.paraChanged', { key: paraLabel(note.key), value: valueText(note.after) }); + } + const values = { mark: markLabel(note.mark), text: note.text }; + if (note.change === 'added') return t('textReview.markAdded', values); + if (note.change === 'removed') return t('textReview.markRemoved', values); + return t('textReview.markChanged', values); +} + +/** The compare view's line for a paragraph whose formatting changed: "Formatting changed: bold added to …". */ +export function describeFormatting(notes: readonly FormatNote[]): string { + return t('textReview.formattingChanged', { details: notes.map(describeNote).join('; ') }); +} diff --git a/app/src/editor/diff/pageDiff.ts b/app/src/editor/diff/pageDiff.ts index acf536e07..e0eac7f97 100644 Binary files a/app/src/editor/diff/pageDiff.ts and b/app/src/editor/diff/pageDiff.ts differ diff --git a/app/src/features/codestudio/complete/bounds.test.ts b/app/src/features/codestudio/complete/bounds.test.ts new file mode 100644 index 000000000..4f9c1ab24 --- /dev/null +++ b/app/src/features/codestudio/complete/bounds.test.ts @@ -0,0 +1,85 @@ +// Hostile and large inputs (DESIGN 10.2, lane code:L12). Completion runs on the main thread after each key. A pasted +// or opened file must never stall it. Long runs of spaces, very long lines, and very large files each finish within +// a small bound, and the list stays short. +import { pythonLanguage } from '@codemirror/lang-python'; +import { describe, expect, it } from 'vitest'; +import { contextAt } from './ghost'; +import { predict } from './pattern'; +import { MAX_ITEMS, rank } from './rank'; + +/** Milliseconds one call takes. */ +function timed(run: () => unknown): number { + const start = performance.now(); + run(); + return performance.now() - start; +} + +const listAfter = (language: string, text: string) => rank(contextAt(language, text, text.length)); + +describe('hostile lines before a member access', () => { + const spaces = ' '.repeat(5000); + + it('stays fast in Python, JavaScript, and C++', () => { + const cases: Array<[string, string]> = [ + ['python', `self: ${spaces}\nself.`], + ['python', `self: list${spaces}x\nself.`], + ['python', `${`self: ${' '.repeat(200)}x\n`.repeat(200)}self.`], + ['javascript', `let self${spaces}: x${spaces}\nself.`], + ['javascript', `const self:${spaces}x${spaces}\nself.`], + ['cpp', `vector${spaces}${'vector listAfter(language, text))).toBeLessThan(100); + }); + + it('reads a type from a line of ordinary length after a hostile one', () => { + const text = `nums: ${' '.repeat(5000)}\nnums = [1, 2]\nnums.`; + expect(listAfter('python', text).map((item) => item.label)).toContain('append'); + }); +}); + +describe('large files', () => { + const lines: string[] = []; + for (let i = 0; i < 60_000; i++) lines.push(`value_${i} = compute_${i}(value_${i - 1}) + offset`); + const big = lines.join('\n'); + + it('caps the list and answers quickly in a 60,000-line file', () => { + const general = `${big}\nval`; + const items = listAfter('python', general); + expect(items.length).toBeLessThanOrEqual(MAX_ITEMS); + expect(items.length).toBeGreaterThan(0); + expect(timed(() => listAfter('python', `${big}\nva`))).toBeLessThan(150); + expect(timed(() => listAfter('python', `${big}\nunknown.`))).toBeLessThan(150); + }); + + it('caps the scope walk with a tree', () => { + const text = `${big}\ndef last(radius):\n return rad`; + const tree = pythonLanguage.parser.parse(text); + rank(contextAt('python', text, text.length), { tree }); + const longer = `${text}i`; + const fresh = pythonLanguage.parser.parse(longer); + let top = ''; + expect( + timed(() => (top = rank(contextAt('python', longer, longer.length), { tree: fresh })[0]!.label)), + ).toBeLessThan(150); + expect(top).toBe('radius'); + }, 60_000); + + it('reads a bounded window next to a multi-megabyte line', () => { + const dump = `CREATE TABLE big (a INT);\nINSERT INTO big VALUES ${'(1),'.repeat(1_750_000)}(1);\n`; + const select = `${dump}SELECT * FROM `; + const other = `${dump}x = 1\nx`; + // The editor's document text is one flat string; a search flattens the joined test texts the same way. + select.indexOf('\u0001'); + other.indexOf('\u0001'); + expect(timed(() => predict('sql', select, select.length, false))).toBeLessThan(50); + expect(timed(() => predict('python', other, other.length, true))).toBeLessThan(15); + }); + + it('keeps the pattern rules fast on long lines', () => { + const arrows = `def f(x) ${') -> '.repeat(16_000)}`; + expect(timed(() => predict('python', arrows, arrows.length, true))).toBeLessThan(50); + const run = `${'a1 = 1\n'.repeat(3)}${'x'.repeat(100_000)}`; + expect(timed(() => predict('python', run, run.length, true))).toBeLessThan(50); + }); +}); diff --git a/app/src/features/codestudio/complete/ghost.ts b/app/src/features/codestudio/complete/ghost.ts new file mode 100644 index 000000000..1c9e59e4a --- /dev/null +++ b/app/src/features/codestudio/complete/ghost.ts @@ -0,0 +1,31 @@ +// Ghost text without a model (DESIGN 10.2, 5.3.1). Automatic: the pattern predictor at its normal threshold. +// Explicit (Ctrl+Tab with nothing showing, or Alt+\): the predictor at its lower threshold first. Then the list's +// top item, when the caret ends a word and that item leads the next by the fixed margin. Nothing below that. + +import type { CompletionContext, GhostSuggestion, ProjectLanguage } from '../contracts'; +import { predict } from './pattern'; +import { leader } from './rank'; +import type { RankInput } from './rank'; +import { wordBefore } from './tokens'; + +export interface GhostQuery { + language: string; + text: string; + pos: number; + explicit: boolean; +} + +/** The completion context for a caret, as the list would see it after an explicit request. */ +export function contextAt(language: string, text: string, pos: number, file = ''): CompletionContext { + return { language: language as ProjectLanguage, file, text, pos, word: wordBefore(text, pos), explicit: true }; +} + +/** One suggestion for the caret, or null. */ +export function suggest(query: GhostQuery, input: RankInput = {}): GhostSuggestion | null { + const { language, text, pos, explicit } = query; + const predicted = predict(language, text, pos, explicit); + if (predicted) return { from: pos, text: predicted.text, origin: 'pattern', confidence: predicted.confidence }; + if (!explicit) return null; + const rest = leader(contextAt(language, text, pos), input); + return rest ? { from: pos, text: rest, origin: 'pattern', confidence: 0.5 } : null; +} diff --git a/app/src/features/codestudio/complete/index.ts b/app/src/features/codestudio/complete/index.ts new file mode 100644 index 000000000..9ee63cfa1 --- /dev/null +++ b/app/src/features/codestudio/complete/index.ts @@ -0,0 +1,71 @@ +// Completion without a model (DESIGN 10.2, lane code:L12): the ranked list, snippets with fields, pattern ghost text, +// and explicit requests. It works on every build. The studio window (code:L04) and code cells (code:L19) call +// attachCompletion on each editor they create, through loadCompletion in features/codestudio/index.ts. + +import { pickedCompletion } from '@codemirror/autocomplete'; +import { ensureSyntaxTree, syntaxTree } from '@codemirror/language'; +import { StateEffect } from '@codemirror/state'; +import { EditorView } from '@codemirror/view'; +import type { CompletionSource } from '../contracts'; +import type { CodeEditor, GhostSource } from '../editor'; +import { suggest } from './ghost'; +import { rank, rememberPick } from './rank'; +import type { RankInput } from './rank'; +import type { SyntaxTree } from './scope'; + +export { contextAt, suggest } from './ghost'; +export type { GhostQuery } from './ghost'; +export { AUTO_THRESHOLD, EXPLICIT_THRESHOLD, predict } from './pattern'; +export type { Prediction } from './pattern'; +export { forgetPicks, LEAD_MARGIN, leader, matchScore, PRIOR, rank, rememberPick } from './rank'; +export type { RankInput } from './rank'; +export { SNIPPETS } from './snippets'; + +export interface CompletionOptions { + /** The text of the other open files, for the words source. */ + otherTexts?: () => readonly string[]; +} + +/** Waits at most this long for the parser before ranking without the tree. */ +const PARSE_BUDGET_MS = 15; + +/** The editor's syntax tree, when it describes `text`. */ +function treeOf(editor: CodeEditor, text: string): SyntaxTree | null { + const state = editor.view.state; + if (state.doc.length !== text.length) return null; + return ensureSyntaxTree(state, state.doc.length, PARSE_BUDGET_MS) ?? syntaxTree(state); +} + +/** The ranked list as one completion source: the sources merge before scoring, so a name shows once. */ +export function rankedSource(input: (text: string) => RankInput): CompletionSource { + return { id: 'scope', prior: 0, complete: (context) => rank(context, input(context.text)) }; +} + +/** The pattern predictor and explicit requests as the editor's ghost text source. */ +export function patternSource(input: (text: string) => RankInput): GhostSource { + return (request) => suggest(request, input(request.text)); +} + +/** Gives an editor the ranked list and pattern ghost text, and remembers accepted items. Returns an undo. */ +export function attachCompletion(editor: CodeEditor, options: CompletionOptions = {}): () => void { + const input = (text: string): RankInput => ({ tree: treeOf(editor, text), otherTexts: options.otherTexts?.() }); + editor.setCompletionSources([rankedSource(input)]); + editor.setSuggestionSource(patternSource(input)); + let attached = true; + editor.view.dispatch({ + effects: StateEffect.appendConfig.of( + EditorView.updateListener.of((update) => { + if (!attached) return; + for (const tr of update.transactions) { + const picked = tr.annotation(pickedCompletion); + if (picked) rememberPick(picked.label); + } + }), + ), + }); + return () => { + attached = false; + editor.setCompletionSources([]); + editor.setSuggestionSource(null); + }; +} diff --git a/app/src/features/codestudio/complete/members.ts b/app/src/features/codestudio/complete/members.ts new file mode 100644 index 000000000..15d6b2c6d --- /dev/null +++ b/app/src/features/codestudio/complete/members.ts @@ -0,0 +1,218 @@ +// The members source (DESIGN 10.2), after a `.`. A literal, a constructor call, or an annotation shows the +// receiver's type, and its methods are offered. Known modules offer their members. Otherwise the names used after +// the same receiver near the caret are offered. +// +// The type rules read whole lines with string operations, or with patterns whose quantifiers never match the same +// characters side by side, so a hostile line costs linear time. Lines longer than MAX_LINE are skipped, and only +// the WINDOW characters before the caret are read. + +import { family, MODULE_MEMBERS, own, TYPE_MEMBERS } from './tables'; + +/** Lines longer than this are skipped by the type rules. */ +export const MAX_LINE = 500; +/** The members source reads at most this many characters on each side of the caret. */ +export const WINDOW = 200_000; + +const escape = (text: string): string => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + +/** The receiver before the `.` that ends at `dot`: a dotted name such as `os.path`, or a literal's type marker. */ +export function receiverAt(text: string, dot: number): string | null { + if (text[dot] !== '.') return null; + const before = text.slice(Math.max(0, dot - 200), dot); + if (/["'`]$/.test(before)) return '"'; + const name = /[\p{L}_$][\p{L}\p{N}_$]*(?:\.[\p{L}_$][\p{L}\p{N}_$]*)*$/u.exec(before); + return name ? name[0] : null; +} + +/** + * Reads the lines of `text` from the last one up, until `read` returns something other than undefined. + * Lines over MAX_LINE characters and text beyond WINDOW are not read. + */ +function nearestLine(text: string, read: (line: string) => T | undefined): T | null { + const stop = Math.max(0, text.length - WINDOW); + let end = text.length; + while (end >= stop) { + const start = text.lastIndexOf('\n', end - 1) + 1; + if (end - start <= MAX_LINE) { + const found = read(text.slice(start, end)); + if (found !== undefined) return found; + } + if (start === 0) break; + end = start - 1; + } + return null; +} + +function pythonValue(value: string): string | null { + if (/^\[/.test(value) || /^(list|sorted)\(|\.split\(/.test(value)) return 'list'; + if (/^\{\s*\}/.test(value) || /^\{[^}]*:/.test(value) || /^dict\(/.test(value)) return 'dict'; + if (/^\{/.test(value) || /^set\(/.test(value)) return 'set'; + if (/^[rfbu]?["']/i.test(value) || /^(str|input)\(/.test(value)) return 'str'; + return null; +} + +const pythonAnnotation = (annotation: string): string | null => + /^(list|dict|set|str)\b/.exec(annotation.trim().toLowerCase())?.[1] ?? null; + +/** + * The nearest line that binds `name`: an assignment, a bare annotation (`nums: list[int]`), or an annotated + * parameter (`def f(name: str):`). An annotation names the type; otherwise the assigned value shows it. + */ +function pythonType(text: string, name: string): string | null { + const head = new RegExp(`^[ \\t]*${escape(name)}[ \\t]*(?=[:=])`); + const parameter = new RegExp(`[(,][ \\t]*${escape(name)}[ \\t]*:([^,)=]*)`); + return nearestLine(text, (line) => { + const bound = head.exec(line); + if (bound) { + const rest = line.slice(bound[0].length); + if (rest.startsWith('==')) return undefined; + if (rest.startsWith('=')) return pythonValue(rest.slice(1).trim()); + const equals = rest.indexOf('='); + const annotation = rest.slice(1, equals < 0 ? undefined : equals); + return pythonAnnotation(annotation) ?? (equals < 0 ? null : pythonValue(rest.slice(equals + 1).trim())); + } + if (/^[ \t]*(?:async[ \t]+)?def[ \t]/.test(line)) { + const annotated = parameter.exec(line); + if (annotated) return pythonAnnotation(annotated[1] ?? ''); + } + return undefined; + }); +} + +function javascriptValue(annotation: string, value: string): string | null { + if (/\[\]$|^Array\\[\\]]*)`); + return nearestLine(text, (line) => { + const found = declared.exec(line); + if (found) { + const rest = line.slice(found.index + found[0].length).trimStart(); + const equals = rest.indexOf('='); + const annotation = rest.startsWith(':') ? rest.slice(1, equals < 0 ? undefined : equals).trim() : ''; + const value = equals < 0 ? '' : rest.slice(equals + 1).trim(); + return javascriptValue(annotation, value); + } + const annotated = parameter.exec(line); + if (annotated) return javascriptValue(annotated[1] ?? '', ''); + return undefined; + }); +} + +/** The nearest declaration of `name` with a standard container type, read within one line. */ +function cppType(text: string, name: string): string | null { + const pattern = new RegExp( + `\\b(?:std::)?(vector|string|map|unordered_map)\\b(?:[ \\t]*<[^;(){}\\n]*>)?[ \\t]*(?:&[ \\t]*)?${escape(name)}\\b`, + ); + return nearestLine(text, (line) => { + if (!line.includes(name)) return undefined; + const found = pattern.exec(line)?.[1]; + if (!found) return undefined; + return found === 'unordered_map' ? 'map' : found; + }); +} + +/** The type of `name` from the text before `pos` (the end of the text by default), as a TYPE_MEMBERS key suffix. */ +export function typeOf(language: string, text: string, name: string, pos = text.length): string | null { + const lang = family(language); + if (name === '"') return lang === 'python' ? 'str' : lang === 'javascript' ? 'String' : null; + const before = text.slice(Math.max(0, pos - WINDOW), pos); + if (lang === 'python') return pythonType(before, name); + if (lang === 'javascript') return javascriptType(before, name); + if (lang === 'cpp') return cppType(before, name); + return null; +} + +// Words that start a table constraint, not a column, inside `CREATE TABLE (...)`. +const SQL_CONSTRAINTS = new Set(['constraint', 'primary', 'foreign', 'unique', 'check', 'key', 'index']); +/** A CREATE TABLE's column list is read for at most this many characters. */ +const SQL_COLUMNS_SPAN = 20_000; + +const sqlName = (name: string): string => + name + .replace(/["`[\]]/g, '') + .split('.') + .pop()! + .toLowerCase(); + +/** The columns of a `CREATE TABLE` whose `(` is at `open`, in order: the first word of each top-level item. */ +function sqlColumns(text: string, open: number): string[] { + const columns: string[] = []; + let depth = 0; + let item = ''; + const end = Math.min(text.length, open + SQL_COLUMNS_SPAN); + for (let i = open; i < end; i++) { + const char = text[i]!; + if (char === '(') depth++; + else if (char === ')') depth--; + if (depth === 1 && char !== '(' && char !== ',') item += char; + if ((depth === 1 && char === ',') || depth === 0) { + const name = /^\s*["`[]?([\p{L}_][\p{L}\p{N}_]*)/u.exec(item)?.[1]; + if (name && !SQL_CONSTRAINTS.has(name.toLowerCase())) columns.push(name); + item = ''; + } + if (depth === 0) break; + } + return columns; +} + +/** + * The SQL columns after `table.`: the nearest `CREATE TABLE table (...)` around the caret. An alias from + * `FROM table alias` or `JOIN table AS alias` is read first. + */ +function sqlMembers(text: string, receiver: string, pos: number): string[] { + const near = text.slice(Math.max(0, pos - WINDOW), pos + WINDOW); + const offset = Math.max(0, pos - WINDOW); + const want = sqlName(receiver); + let table = want; + for (const match of near.matchAll(/\b(?:FROM|JOIN)\s+([\w.]+)\s+(?:AS\s+)?(\w+)/gi)) { + if (match[2]!.toLowerCase() === want) table = sqlName(match[1]!); + } + let best: { at: number; open: number } | null = null; + for (const match of near.matchAll( + /\bCREATE\s+(?:TEMP(?:ORARY)?\s+)?TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?([\w."`[\]]+)\s*\(/gi, + )) { + if (sqlName(match[1]!) !== table) continue; + const at = offset + match.index; + if (!best || Math.abs(at - pos) < Math.abs(best.at - pos)) best = { at, open: at + match[0].length - 1 }; + } + return best ? sqlColumns(text, best.open) : []; +} + +/** A member name and its rank in its table (0 is the most used), or -1 when it came from the file. */ +export interface Member { + name: string; + rank: number; + size: number; + module: boolean; +} + +/** The members after `receiver.` at `pos`: the type's or module's table, else names used after it near the caret. */ +export function membersOf(language: string, text: string, receiver: string, pos = text.length): Member[] { + const lang = family(language); + const module = own(MODULE_MEMBERS, `${lang}:${receiver}`); + const type = module ? null : typeOf(language, text, receiver, pos); + const table = module ?? (type ? own(TYPE_MEMBERS, `${lang}:${type}`) : undefined); + if (table) return table.map((name, rank) => ({ name, rank, size: table.length, module: Boolean(module) })); + if (receiver === '"') return []; + if (lang === 'sql') { + const columns = sqlMembers(text, receiver, pos); + if (columns.length) return columns.map((name, rank) => ({ name, rank, size: columns.length, module: false })); + } + const used = new Set(); + const near = text.slice(Math.max(0, pos - WINDOW), pos + WINDOW); + for (const match of near.matchAll(new RegExp(`\\b${escape(receiver)}\\.([\\p{L}_$][\\p{L}\\p{N}_$]*)`, 'gu'))) { + if (match[1]) used.add(match[1]); + } + return [...used].map((name) => ({ name, rank: -1, size: 0, module: false })); +} diff --git a/app/src/features/codestudio/complete/pattern.test.ts b/app/src/features/codestudio/complete/pattern.test.ts new file mode 100644 index 000000000..ef91593e0 --- /dev/null +++ b/app/src/features/codestudio/complete/pattern.test.ts @@ -0,0 +1,162 @@ +// Pattern ghost text and explicit requests (DESIGN 10.2, lane code:L12). Expected texts are hand-worked. One is +// the next line of a run with a constant step. Others are the idioms DESIGN 10.2 lists: Python's `__main__` guard, +// the common import aliases, and C's and C++'s first standard header. Explicit requests follow DESIGN 5.3.1. +import { pythonLanguage } from '@codemirror/lang-python'; +import { describe, expect, it } from 'vitest'; +import { suggest } from './ghost'; +import { AUTO_THRESHOLD, EXPLICIT_THRESHOLD, predict } from './pattern'; +import { tokenize } from './tokens'; + +/** The automatic ghost text at the end of `text`. */ +const ghost = (language: string, text: string): string | null => + predict(language, text, text.length, false)?.text ?? null; + +/** What an explicit request inserts at the end of `text`. */ +function explicit(language: string, text: string): string | null { + const tree = language === 'python' ? pythonLanguage.parser.parse(text) : null; + return suggest({ language, text, pos: text.length, explicit: true }, { tree })?.text ?? null; +} + +describe('pattern ghost text', () => { + it('continues a run of lines with a numeric step', () => { + const text = 'a1 = x[1]\na2 = x[2]\na'; + expect(`a${ghost('python', text)}`).toBe('a3 = x[3]'); + expect(ghost('python', 'v10 = 10\nv20 = 20\nv30 = 30\nv')).toBe('40 = 40'); + }); + + it('completes the __main__ guard', () => { + expect(ghost('python', 'if __name__ == "')).toBe('__main__":'); + expect(ghost('python', "if __name__ == '")).toBe("__main__':"); + }); + + it('proposes the usual import aliases', () => { + expect(ghost('python', 'import numpy as ')).toBe('np'); + expect(ghost('python', 'import numpy as')).toBe(' np'); + expect(ghost('python', 'import pandas as ')).toBe('pd'); + expect(ghost('python', 'import matplotlib.pyplot as ')).toBe('plt'); + }); + + it('proposes the first standard header for C and C++', () => { + expect(ghost('cpp', '#include <')).toBe('iostream>'); + expect(ghost('c', '#include <')).toBe('stdio.h>'); + }); + + it('closes main and counted loops', () => { + expect(ghost('c', 'int main(')).toBe('void) {'); + expect(ghost('java', 'public static void main(')).toBe('String[] args) {'); + expect(ghost('cpp', 'int n = 5;\nfor (int i = 0; i < ')).toBe('n; i++) {'); + expect(ghost('cpp', 'for (size_t k = 0; k < ')).toBeNull(); + expect(predict('cpp', 'for (size_t k = 0; k < ', 23, true)?.text).toBe('n; k++) {'); + }); + + it('closes a def line with a colon', () => { + expect(ghost('python', 'def area(radius)')).toBe(':'); + expect(ghost('python', 'def area(radius')).toBeNull(); + }); + + it('reuses an earlier line with the same start', () => { + const text = 'print("total:", total, sep="")\nx = 1\nprint("total:", '; + expect(ghost('python', text)).toBe('total, sep="")'); + }); + + it('keeps nested brackets and strings when the editor already typed the closer', () => { + const text = 'print("total:", foo(total), sep="")\nx = 1\nprint("total:", )'; + expect(predict('python', text, text.length - 1, false)?.text).toBe('foo(total), sep=""'); + const listed = 'rows.append([a[0], b])\nrows.append([])'; + expect(predict('python', listed, listed.length - 2, true)?.text).toBe('a[0], b'); + }); + + it('keeps padded widths and never steps below zero or past exact integers', () => { + expect(ghost('python', 'img01 = load(1)\nimg02 = load(2)\nimg')).toBe('03 = load(3)'); + expect(ghost('python', 'a1 = 1\na0 = 0\na')).toBeNull(); + const huge = 'n = 9007199254740990\nn = 9007199254740991\nn'; + expect(ghost('python', huge)).toBeNull(); + }); + + it('tokenizes digits outside ASCII as single characters, so a line rebuilds exactly', () => { + const line = 'x² = ٣ + ①'; + expect( + tokenize(line) + .map((token) => token.text) + .join(''), + ).toBe(line); + }); + + it('stops before a closer the editor already typed', () => { + const text = 'if __name__ == ""'; + expect(predict('python', text, text.length - 1, false)?.text).toBe('__main__'); + }); + + it('shows nothing mid-line, on a new line, or below the threshold', () => { + expect(predict('python', 'import numpy as x', 15, false)).toBeNull(); + expect(ghost('python', 'def f():\n ')).toBeNull(); + expect(ghost('python', 'total = 0\ntot')).toBeNull(); + expect(EXPLICIT_THRESHOLD).toBeLessThan(AUTO_THRESHOLD); + }); + + it('keeps every suggestion to one line', () => { + for (const text of ['a1 = x[1]\na2 = x[2]\na', 'if __name__ == "', '#include <']) { + expect(ghost(text.startsWith('#') ? 'c' : 'python', text)).not.toContain('\n'); + } + }); +}); + +describe('ghost text guards', () => { + it('closes a signature only when its brackets and strings are balanced', () => { + expect(ghost('python', 'def f(a, b=(1, 2))')).toBe(':'); + expect(ghost('python', 'class A(B())')).toBe(':'); + expect(ghost('python', 'def f(a) -> list[int]')).toBe(':'); + expect(ghost('python', 'def f(a) -> None')).toBe(':'); + expect(ghost('python', 'def f(a=(1, 2)')).toBeNull(); + // The editor already closed the `)` after the caret. + expect(predict('python', 'def f(x=g())', 'def f(x=g()'.length, false)).toBeNull(); + expect(predict('python', 'class A(B())', 'class A(B()'.length, false)).toBeNull(); + expect(ghost('python', 'def f(a, b="x)')).toBeNull(); + expect(ghost('python', 'def f(a=lambda: g()')).toBeNull(); + expect(ghost('python', 'def f(a) # note (x)')).toBeNull(); + expect(ghost('python', 'def f(a):')).toBeNull(); + // The caret may still be typing the annotation; an explicit request still offers the colon. + expect(ghost('python', 'def f(a) -> tup')).toBeNull(); + expect(predict('python', 'def f(a) -> tup', 15, true)?.text).toBe(':'); + }); + + it('does not step hex digits as decimal or step past zero after a minus', () => { + expect(ghost('python', 'v = -2\nv = -1\nv')).toBeNull(); + expect(ghost('python', 'v = -3\nv = -2\nv')).toBe(' = -1'); + expect(ghost('c', 'a = 0x08;\na = 0x09;\na')).toBeNull(); + }); + + it('does not fire idioms inside a string or a comment', () => { + expect(ghost('python', 'x = "if __name__ == "')).toBeNull(); + expect(ghost('cpp', '// for (int i = 0; i < ')).toBeNull(); + expect(ghost('cpp', '// int main(')).toBeNull(); + expect(ghost('sql', 'CREATE TABLE t (a INT);\n-- SELECT * FROM ')).toBeNull(); + expect(ghost('sql', "CREATE TABLE t (a INT);\nSELECT 'x FROM ")).toBeNull(); + expect(ghost('sql', 'CREATE TABLE t (a INT);\nSELECT * FROM ')).toBe('t'); + }); + + it('treats a language named like an object key as unknown', () => { + for (const language of ['constructor', 'toString', '__proto__']) { + expect(() => predict(language, 'x = 1\nx', 7, true)).not.toThrow(); + } + }); +}); + +describe('explicit requests', () => { + it('inserts the leading list item where the automatic predictor shows nothing', () => { + const text = 'total = 0\nfor n in nums:\n tot'; + expect(ghost('python', text)).toBeNull(); + expect(explicit('python', text)).toBe('al'); + }); + + it('returns nothing on a blank line in an empty file', () => { + expect(explicit('python', '')).toBeNull(); + }); + + it('uses the pattern predictor first, at its lower threshold', () => { + // Two complete tokens are below the automatic threshold and above the explicit one. + const text = 'print("a")\nprint('; + expect(ghost('python', text)).toBeNull(); + expect(explicit('python', text)).toBe('"a")'); + }); +}); diff --git a/app/src/features/codestudio/complete/pattern.ts b/app/src/features/codestudio/complete/pattern.ts new file mode 100644 index 000000000..ae95f186f --- /dev/null +++ b/app/src/features/codestudio/complete/pattern.ts @@ -0,0 +1,289 @@ +// The pattern predictor (DESIGN 10.2): one line of ghost text without a model, shown only when confident. +// Rules, in order: the language's idioms, closing a `def` or `class` line, repeated lines with a numeric step, +// and an earlier line with the same start. Block bodies after a new line are left to the model. + +import { family, own } from './tables'; +import { lineAt, tokenize } from './tokens'; +import type { Token } from './tokens'; + +/** Automatic ghost text needs this confidence. */ +export const AUTO_THRESHOLD = 0.6; +/** An explicit request (Ctrl+Tab with nothing showing, or Alt+\) needs only this. */ +export const EXPLICIT_THRESHOLD = 0.35; + +export type PatternRule = 'idiom' | 'signature' | 'step' | 'sameStart'; + +export interface Prediction { + /** The text to insert at the caret: one line, never empty. */ + text: string; + confidence: number; + rule: PatternRule; +} + +interface Line { + text: string; + before: string; + pos: number; +} + +type Idiom = (line: Line, language: string) => { text: string; confidence: number } | null; + +const sure = (text: string | null, confidence = 0.9) => (text ? { text, confidence } : null); + +/** `import numpy as` gives ` np`; with the space already typed, `np`. */ +function alias(module: string, short: string): Idiom { + const pattern = new RegExp(`^\\s*import ${module.replace(/\./g, '\\.')} as( ?)$`); + return ({ before }) => { + const match = pattern.exec(before); + return match ? sure(match[1] ? short : ` ${short}`) : null; + }; +} + +const ends = + (pattern: RegExp, text: string): Idiom => + ({ before }) => + sure(pattern.test(before) ? text : null); + +/** `for (int i = 0; i < ` gives `n; i++) {`, with the bound from a nearby `n`, `.size()`, or `.length`. */ +const countedFor: Idiom = ({ before, text, pos }, language) => { + const match = /^\s*for\s*\(\s*(?:int|let|var|size_t|auto|long)\s+(\w+)\s*=\s*0\s*;\s*\1\s*<\s*$/.exec(before); + if (!match) return null; + const counter = match[1]!; + const nearby = text.slice(Math.max(0, pos - before.length - 2000), pos - before.length); + const sized = language === 'javascript' ? /(\w+)\.length\b/g : /(\w+)\.size\(\)/g; + const lastSized = [...nearby.matchAll(sized)].pop()?.[0]; + const bound = /\bn\b/.test(nearby) ? 'n' : (lastSized ?? 'n'); + const confidence = bound === 'n' && !/\bn\b/.test(nearby) ? 0.5 : 0.85; + return { text: `${bound}; ${counter}++) {`, confidence }; +}; + +/** `SELECT * FROM ` gives the table named last in the PREDICT_WINDOW characters above. Not in a comment. */ +const lastTable: Idiom = ({ before, text, pos }) => { + if (!/\bFROM\s$/i.test(before) || before.includes('--') || before.split("'").length % 2 === 0) return null; + const start = pos - before.length; + const earlier = text.slice(Math.max(0, start - PREDICT_WINDOW), start); + const tables = [...earlier.matchAll(/\b(?:FROM|JOIN|INTO|UPDATE|TABLE)\s+([A-Za-z_]\w*)/gi)]; + return sure(tables.pop()?.[1] ?? null, 0.7); +}; + +// Each idiom starts at the line start, so the same words inside a string or after a comment marker never match. +const IDIOMS: Readonly> = { + python: [ + ends(/^\s*if __name__ == "$/, '__main__":'), + ends(/^\s*if __name__ == '$/, "__main__':"), + alias('numpy', 'np'), + alias('pandas', 'pd'), + alias('matplotlib.pyplot', 'plt'), + ], + c: [ends(/^\s*#\s*include\s*<$/, 'stdio.h>'), ends(/^\s*int main\($/, 'void) {'), countedFor], + cpp: [ends(/^\s*#\s*include\s*<$/, 'iostream>'), ends(/^\s*int main\($/, ') {'), countedFor], + java: [ends(/^\s*public static void main\($/, 'String[] args) {'), countedFor], + javascript: [countedFor], + sql: [lastTable], +}; + +// Return annotations that are whole words: `-> None` gets its `:` at once. +const ANNOTATIONS = new Set(['None', 'int', 'str', 'float', 'bool', 'bytes', 'list', 'dict', 'tuple', 'set', 'object']); + +/** + * A Python `def` or `class` line whose parameters are closed gets its `:`. One pass tracks bracket depth and strings, + * so a nested `(`, a `)` inside a string, or a comment never counts as the close. After the parameters only a return + * annotation may follow. An annotation that ends in a word the user may still be typing (`-> tup`) shows only on + * request. + */ +function signature(line: Line, language: string): { text: string; confidence: number } | null { + if (family(language) !== 'python') return null; + const { before } = line; + const isClass = /^\s*class\s+\w+\s*\(/.test(before); + if (!isClass && !/^\s*(?:async\s+)?def\s+\w+\s*\(/.test(before)) return null; + let depth = 0; + let quote = ''; + let close = -1; + for (let i = 0; i < before.length; i++) { + const char = before[i]!; + if (quote) { + if (char === '\\') i++; + else if (char === quote) quote = ''; + } else if (char === '"' || char === "'") quote = char; + else if (char === '#') return null; + else if ('([{'.includes(char)) depth++; + else if (')]}'.includes(char)) { + depth--; + if (depth < 0) return null; + if (depth === 0 && close < 0) close = i; + } else if (char === ':' && depth === 0 && close >= 0) return null; + } + if (quote || depth !== 0 || close < 0) return null; + const rest = before.slice(close + 1).trim(); + if (!rest) return { text: ':', confidence: 0.9 }; + if (isClass || !rest.startsWith('->') || !rest.slice(2).trim()) return null; + const word = /\w+$/.exec(rest)?.[0]; + return { text: ':', confidence: !word || ANNOTATIONS.has(word) ? 0.9 : 0.5 }; +} + +const indentOf = (line: string): string => /^\s*/.exec(line)?.[0] ?? ''; + +/** The next line of a run whose lines differ only in integers with a constant step, or null. */ +function extrapolate(older: Token[], newer: Token[]): { tokens: string[]; steps: number[] } | null { + if (older.length !== newer.length) return null; + const tokens: string[] = []; + const steps: number[] = []; + for (let i = 0; i < newer.length; i++) { + const a = older[i]!; + const b = newer[i]!; + if (a.text === b.text) { + tokens.push(b.text); + } else if (a.kind === 'number' && b.kind === 'number' && /^\d{1,15}$/.test(a.text) && /^\d{1,15}$/.test(b.text)) { + // Integers of at most 15 digits stay exact. A step below zero would change the line's shape, and a padded + // number such as `02` keeps its width. + // `0x08` reads as `0`, `x`, `08`. Its digits are not decimal, so a base prefix stops the rule. + if (/^[xXbBoO]$/.test(newer[i - 1]?.text ?? '') && newer[i - 2]?.text === '0') return null; + const step = Number(b.text) - Number(a.text); + const value = Number(b.text) + step; + if (value < 0 || String(value).length > 15) return null; + // After a minus sign, zero would read `-0`. + if (value === 0 && newer[i - 1]?.text === '-') return null; + steps.push(step); + tokens.push(b.text.startsWith('0') ? String(value).padStart(b.text.length, '0') : String(value)); + } else if (a.kind === 'space' && b.kind === 'space') { + tokens.push(b.text); + } else { + return null; + } + } + return steps.length ? { tokens, steps } : null; +} + +function step(lines: readonly string[], index: number, before: string): { text: string; confidence: number } | null { + const indent = indentOf(before); + const above: string[] = []; + for (let i = index - 1; i >= 0 && above.length < 3; i--) { + const line = lines[i]!; + // A long line (a data dump or minified code) is never tokenized: it ends the run. + if (line.length > MAX_PREDICT_LINE || !line.trim()) break; + if (indentOf(line) !== indent) break; + above.push(line.trim()); + } + const [newer, older, oldest] = above; + if (!newer || !older) return null; + const next = extrapolate(tokenize(older), tokenize(newer)); + if (!next) return null; + const predicted = indent + next.tokens.join(''); + if (!predicted.startsWith(before) || predicted.length === before.length) return null; + let confidence = 0.75; + if (oldest) { + const earlier = extrapolate(tokenize(oldest), tokenize(older)); + if (earlier && earlier.steps.join() === next.steps.join()) confidence = 0.9; + } + if (!before.trim()) confidence -= 0.3; + return { text: predicted.slice(before.length), confidence }; +} + +/** Complete tokens typed on the line: a word or number still being typed does not count. */ +function completeTokens(prefix: string): number { + const tokens = tokenize(prefix).filter((token) => token.kind !== 'space'); + const last = tokens[tokens.length - 1]; + const typing = last && (last.kind === 'word' || last.kind === 'number') && prefix.endsWith(last.text); + return tokens.length - (typing ? 1 : 0); +} + +function sameStart( + lines: readonly string[], + index: number, + before: string, +): { text: string; confidence: number } | null { + const prefix = before.trimStart(); + const complete = completeTokens(prefix); + if (complete < 1) return null; + const order: number[] = []; + for (let distance = 1; distance < lines.length; distance++) { + if (index - distance >= 0) order.push(index - distance); + if (index + distance < lines.length) order.push(index + distance); + } + let found: string | null = null; + let other = false; + for (const i of order) { + const line = lines[i]!.trim(); + if (line.length <= prefix.length || !line.startsWith(prefix)) continue; + const rest = line.slice(prefix.length); + if (found === null) found = rest; + else if (rest !== found) { + other = true; + break; + } + } + if (found === null) return null; + return { text: found, confidence: Math.min(0.85, 0.35 + 0.12 * complete) - (other ? 0.15 : 0) }; +} + +// Text after the caret that ghost text may sit before: closing brackets and quotes the editor added. +const CLOSERS = /^[)\]}>"'`]*\s*$/; +const QUOTES = '"\'`'; + +/** + * Cuts a prediction before the closer that already follows the caret, so accepting never doubles it. Brackets and + * strings the prediction opens itself are skipped, so `foo(total), sep="")` before `)` keeps `foo(total), sep=""`. + * Inside a string (the closer is a quote) the cut is at the first matching quote. + */ +function fit(text: string, after: string): string { + const closer = after.trim()[0]; + if (!closer) return text; + if (QUOTES.includes(closer)) { + const at = text.indexOf(closer); + return at < 0 ? text : text.slice(0, at); + } + let depth = 0; + let quote = ''; + for (let i = 0; i < text.length; i++) { + const char = text[i]!; + if (quote) { + if (char === '\\') i++; + else if (char === quote) quote = ''; + } else if (QUOTES.includes(char)) quote = char; + else if ('([{'.includes(char)) depth++; + else if (depth > 0 && ')]}'.includes(char)) depth--; + else if (char === closer || ')]}'.includes(char)) return text.slice(0, i); + } + return text; +} + +/** The predictor skips a caret line longer than this, and reads at most PREDICT_WINDOW characters around it. */ +export const MAX_PREDICT_LINE = 500; +const PREDICT_WINDOW = 100_000; + +/** The ghost text for the caret, or null when no rule is confident enough. */ +export function predict(language: string, fullText: string, fullPos: number, explicit: boolean): Prediction | null { + const { before, after } = lineAt(fullText, fullPos); + if (before.length + after.length > MAX_PREDICT_LINE || !CLOSERS.test(after)) return null; + // A window of whole lines around the caret bounds the work in a large file. A very long line at an edge is cut, so + // the window never reaches past twice PREDICT_WINDOW on either side. + // Each edge searches only its own band, so the search never crosses a long line. + const low = Math.max(0, fullPos - 2 * PREDICT_WINDOW); + const from = + fullPos <= PREDICT_WINDOW ? 0 : low + fullText.slice(low, fullPos - PREDICT_WINDOW).lastIndexOf('\n') + 1; + const high = Math.min(fullText.length, fullPos + 2 * PREDICT_WINDOW); + const newline = fullText.slice(0, high).indexOf('\n', fullPos + PREDICT_WINDOW); + const text = fullText.slice(from, newline < 0 ? high : newline); + const pos = fullPos - from; + const threshold = explicit ? EXPLICIT_THRESHOLD : AUTO_THRESHOLD; + const line: Line = { text, before, pos }; + const lang = family(language); + const lines = text.split('\n'); + const index = text.slice(0, pos).split('\n').length - 1; + const rules: Array<[PatternRule, () => { text: string; confidence: number } | null]> = [ + ...(own(IDIOMS, lang) ?? []).map((idiom): [PatternRule, () => { text: string; confidence: number } | null] => [ + 'idiom', + () => idiom(line, lang), + ]), + ['signature', () => signature(line, lang)], + ['step', () => step(lines, index, before)], + ['sameStart', () => sameStart(lines, index, before)], + ]; + for (const [rule, run] of rules) { + const found = run(); + if (!found || found.confidence < threshold) continue; + const ghost = fit(found.text.split('\n')[0] ?? '', after); + if (ghost) return { text: ghost, confidence: found.confidence, rule }; + } + return null; +} diff --git a/app/src/features/codestudio/complete/perf.test.ts b/app/src/features/codestudio/complete/perf.test.ts new file mode 100644 index 000000000..7136cd9d9 --- /dev/null +++ b/app/src/features/codestudio/complete/perf.test.ts @@ -0,0 +1,89 @@ +// Latency budgets (DESIGN 15, gate G-C2): the list under 20 ms and the pattern ghost text under 30 ms in a +// 1,000-line file. Every sample runs cold, as after a keystroke: the text is new and its tree is freshly parsed, so +// no per-text or per-tree cache can answer. The parse is not timed; the editor has the tree before it asks. Best of +// three runs of the median, so a busy test machine does not fail the gate on one slow sample. +import { pythonLanguage } from '@codemirror/lang-python'; +import { describe, expect, it } from 'vitest'; +import { contextAt, suggest } from './ghost'; +import { rank } from './rank'; +import type { SyntaxTree } from './scope'; + +function thousandLines(): string { + const lines: string[] = []; + for (let i = 0; lines.length < 995; i++) { + lines.push(`def step_${i}(values, limit_${i}):`); + lines.push(` total_${i} = 0`); + lines.push(` for item in values:`); + lines.push(` total_${i} += item * ${i}`); + lines.push(` return min(total_${i}, limit_${i})`); + } + lines.push('def last(radius, count):', ' nums = [1, 2, 3]', ' result = radius * count', ' print(res'); + return lines.join('\n'); +} + +const SAMPLES = 15; + +/** The median of cold samples: each one types a different character earlier in the file and reparses. */ +function coldMedianMs(base: string, run: (text: string, tree: SyntaxTree) => void): number { + const times: number[] = []; + for (let i = 0; i < SAMPLES; i++) { + const at = base.indexOf('\n', (base.length * i) / SAMPLES); + const text = `${base.slice(0, at)} # ${String.fromCharCode(97 + i)}${base.slice(at)}`; + const tree = pythonLanguage.parser.parse(text); + const start = performance.now(); + run(text, tree); + times.push(performance.now() - start); + } + times.sort((a, b) => a - b); + return times[SAMPLES >> 1]!; +} + +const best = (base: string, run: (text: string, tree: SyntaxTree) => void): number => + Math.min(coldMedianMs(base, run), coldMedianMs(base, run), coldMedianMs(base, run)); + +describe('completion latency in a 1,000-line file', () => { + const base = thousandLines(); + + it('has 1,000 lines', () => { + expect(base.split('\n').length).toBeGreaterThanOrEqual(999); + }); + + it('computes the list under 20 ms, cold', () => { + const tree = pythonLanguage.parser.parse(base); + expect(rank(contextAt('python', base, base.length), { tree })[0]?.label).toBe('result'); + const ms = best(base, (text, fresh) => rank(contextAt('python', text, text.length), { tree: fresh })); + expect(ms).toBeLessThan(20); + }, 60_000); + + it('keeps the word counts while typing on one line of a 300 KB file', () => { + const lines: string[] = []; + for (let i = 0; lines.join('\n').length < 300_000; i++) { + for (let j = 0; j < 100; j++) lines.push(`value_${i % 8}_${j % 10} = compute(item_${j % 30}, limit) + offset`); + } + const middle = lines.length >> 1; + const head = `${lines.slice(0, middle).join('\n')}\n`; + const tail = `\n${lines.slice(middle).join('\n')}`; + const typed = 'value_12_3'; + const times: number[] = []; + for (let k = 1; k <= typed.length; k++) { + const text = `${head}${typed.slice(0, k)}${tail}`; + // The editor's document text is one flat string; a search flattens the joined test text the same way. + text.indexOf('\u0001'); + const start = performance.now(); + rank(contextAt('python', text, head.length + k)); + times.push(performance.now() - start); + } + // The first keystroke on the line counts the window; every later one reuses those counts. + const later = times.slice(1).sort((a, b) => a - b); + expect(later[later.length >> 1]).toBeLessThan(2); + }); + + it('computes the ghost text under 30 ms, cold', () => { + for (const explicit of [false, true]) { + const ms = best(base, (text, tree) => + suggest({ language: 'python', text, pos: text.length, explicit }, { tree }), + ); + expect(ms).toBeLessThan(30); + } + }, 60_000); +}); diff --git a/app/src/features/codestudio/complete/rank.test.ts b/app/src/features/codestudio/complete/rank.test.ts new file mode 100644 index 000000000..fb2bb7e24 --- /dev/null +++ b/app/src/features/codestudio/complete/rank.test.ts @@ -0,0 +1,153 @@ +// Ranking known answers (DESIGN 10.2, lane code:L12). The expected names are hand-worked: a parameter of the +// enclosing function, the list methods the Python documentation lists, and the C++ and JavaScript members their +// references document. The tree comes from the same Lezer grammars the editor uses. +import { cppLanguage } from '@codemirror/lang-cpp'; +import { javascriptLanguage } from '@codemirror/lang-javascript'; +import { pythonLanguage } from '@codemirror/lang-python'; +import { afterEach, describe, expect, it } from 'vitest'; +import { contextAt } from './ghost'; +import { forgetPicks, LEAD_MARGIN, leader, matchScore, rank, rememberPick } from './rank'; + +const PARSERS = { python: pythonLanguage.parser, javascript: javascriptLanguage.parser, cpp: cppLanguage.parser }; + +/** The ranked labels at the end of `text` (or at `|`), parsed like the editor parses it. */ +function labels(language: keyof typeof PARSERS, source: string): string[] { + const pos = source.includes('|') ? source.indexOf('|') : source.length; + const text = source.replace('|', ''); + const tree = PARSERS[language].parse(text); + return rank(contextAt(language, text, pos), { tree }).map((item) => item.label); +} + +afterEach(() => forgetPicks()); + +describe('the ranked list', () => { + it('puts the parameter first in "def area(radius):\\n return rad"', () => { + expect(labels('python', 'def area(radius):\n return rad')[0]).toBe('radius'); + }); + + it('offers list methods after "nums." with append in the top three', () => { + const top = labels('python', 'nums = [3, 1, 2]\nnums.').slice(0, 3); + expect(top).toContain('append'); + }); + + it('offers string methods after a string literal and math members after "math."', () => { + expect(labels('python', 'name = "ada"\nname.').slice(0, 3)).toContain('join'); + expect(labels('python', 'import math\nmath.sq')[0]).toBe('sqrt'); + }); + + it('knows JavaScript arrays and C++ vectors', () => { + expect(labels('javascript', 'const items = [];\nitems.').slice(0, 3)).toContain('push'); + expect(labels('cpp', 'std::vector values;\nvalues.').slice(0, 3)).toContain('push_back'); + }); + + it('offers nothing after a dot whose receiver is unknown', () => { + expect(labels('python', 'def get():\n return 1\nvalue = get().')).toEqual([]); + expect(labels('python', 'rows = [[1]]\nrows[0].')).toEqual([]); + expect(labels('python', 'x = 3.')).toEqual([]); + expect(labels('javascript', 'const total = 1;\nfetch(url)\n .')).toEqual([]); + // A spread is not a member access. + expect(labels('javascript', 'const items = [];\nconst more = [...ite')).toContain('items'); + }); + + it('reads parameter and bare annotations, and the nearest assignment before the caret', () => { + expect(labels('python', 'def f(name: str):\n return name.').slice(0, 3)).toContain('join'); + expect(labels('python', 'nums: list[int]\nnums.').slice(0, 3)).toContain('append'); + expect(labels('python', 'def g(self, rows: list, k=1):\n rows.').slice(0, 3)).toContain('append'); + expect(labels('python', 'data = "a"\ndata = []\ndata.').slice(0, 3)).toContain('append'); + // An assignment after the caret does not count. + expect(labels('python', 'data = []\ndata.|\ndata = "a"').slice(0, 3)).toContain('append'); + expect(labels('javascript', 'function f(items: string[]) {\n items.').slice(0, 3)).toContain('push'); + expect(labels('javascript', 'const f = (items?: number[]) => items.').slice(0, 3)).toContain('push'); + }); + + it('ranks a nearer scope above a farther one, and a scope name above a keyword', () => { + const text = 'value_outer = 1\ndef f(value_inner):\n return valu'; + expect(labels('python', text).slice(0, 2)).toEqual(['value_inner', 'value_outer']); + expect(labels('python', 'retry = 3\nre')[0]).toBe('retry'); + }); + + it('hides names from another function', () => { + const text = 'def a(secret):\n pass\ndef b(other):\n return sec'; + const found = rank(contextAt('python', text.replace('|', ''), text.length), { + tree: pythonLanguage.parser.parse(text), + }); + expect(found.find((item) => item.label === 'secret')?.source).toBe('words'); + }); + + it('is deterministic: equal scores sort alphabetically', () => { + const first = labels('python', 'b_two = 1\nb_one = 1\nb'); + expect(labels('python', 'b_two = 1\nb_one = 1\nb')).toEqual(first); + const items = rank(contextAt('python', 'x', 1)); + for (let i = 1; i < items.length; i++) { + const [a, b] = [items[i - 1]!, items[i]!]; + expect(a.score > b.score || (a.score === b.score && a.label <= b.label)).toBe(true); + } + }); + + it('includes snippets with fields and keywords from the frequency table', () => { + const items = rank(contextAt('python', 'fo', 2)); + const snippet = items.find((item) => item.snippet && item.label === 'for'); + expect(snippet?.insert).toBe('for ${1:item} in ${2:items}:\n\t${3:pass}'); + expect(items.some((item) => item.kind === 'keyword' && item.label === 'for')).toBe(true); + }); + + it('boosts items accepted earlier in the session', () => { + const text = 'alpha_one = 1\nalpha_two = 2\nalpha'; + const tree = pythonLanguage.parser.parse(text); + const before = rank(contextAt('python', text, text.length), { tree }).map((item) => item.label); + rememberPick(before[1]!); + const after = rank(contextAt('python', text, text.length), { tree }).map((item) => item.label); + expect(after[0]).toBe(before[1]); + }); + + it('matches a prefix above another case above letters in order', () => { + expect(matchScore('radius', 'rad')).toBe(3); + expect(matchScore('Radius', 'rad')).toBe(2); + expect(matchScore('read_all', 'rdl')).toBe(1); + expect(matchScore('radius', 'xyz')).toBeNull(); + }); +}); + +describe('SQL members, snippet notes, and unknown languages', () => { + it('offers SQL columns after "table." from the CREATE TABLE', () => { + const sql = (text: string) => rank(contextAt('sql', text, text.length)).map((item) => item.label); + expect(sql('CREATE TABLE users (id INT, name TEXT);\nSELECT users.')).toEqual(['id', 'name']); + const keyed = 'CREATE TABLE IF NOT EXISTS "users" (\n id INT,\n name VARCHAR(20, 0),\n PRIMARY KEY (id)\n);\n'; + // An alias after FROM names the table; a constraint item is not a column. + const aliased = `${keyed}SELECT u. FROM users u`; + const at = keyed.length + 'SELECT u.'.length; + expect(rank(contextAt('sql', aliased, at)).map((item) => item.label)).toEqual(['id', 'name']); + }); + + it('marks snippets as snippets in the list', () => { + const item = rank(contextAt('python', 'fo', 2)).find((entry) => entry.snippet); + expect(item?.detail).toMatch(/^snippet: /); + }); + + it('treats a language named like an object key as unknown', () => { + for (const language of ['constructor', 'toString', '__proto__']) { + expect(() => rank(contextAt(language, 'value = 1\nva', 12))).not.toThrow(); + } + }); +}); + +describe('the leader for an explicit request', () => { + it('returns the rest of a word whose top item leads by the margin', () => { + const text = 'total = 0\nfor n in nums:\n tot'; + const tree = pythonLanguage.parser.parse(text); + expect(leader(contextAt('python', text, text.length), { tree })).toBe('al'); + }); + + it('returns nothing when two items are close', () => { + const text = 'count_a = 1\ncount_b = 2\ncou'; + const tree = pythonLanguage.parser.parse(text); + const [top, second] = rank(contextAt('python', text, text.length), { tree }); + expect(top!.score - second!.score).toBeLessThan(LEAD_MARGIN); + expect(leader(contextAt('python', text, text.length), { tree })).toBeNull(); + }); + + it('returns nothing inside a word or with no word', () => { + expect(leader(contextAt('python', 'total = 0\ntot', 11))).toBeNull(); + expect(leader(contextAt('python', '', 0))).toBeNull(); + }); +}); diff --git a/app/src/features/codestudio/complete/rank.ts b/app/src/features/codestudio/complete/rank.ts new file mode 100644 index 000000000..3440ac80d --- /dev/null +++ b/app/src/features/codestudio/complete/rank.ts @@ -0,0 +1,252 @@ +// Ranking without a model (DESIGN 10.2). Every source's candidates are merged and scored. The score is +// match + source prior + proximity + frequency + recency. Ties sort alphabetically, so a document and a caret +// always give the same list. + +import { t } from '../../../strings/t'; +import type { CompletionContext, CompletionItem, CompletionSourceId } from '../contracts'; +import { membersOf, receiverAt } from './members'; +import { bindings, namesInScope } from './scope'; +import type { SyntaxTree } from './scope'; +import { SNIPPETS } from './snippets'; +import { BUILTINS, KEYWORDS, own } from './tables'; +import { countWords } from './tokens'; + +/** Each source's prior weight. */ +export const PRIOR: Readonly, number>> = { + scope: 4, + members: 4, + keywords: 2, + snippets: 2, + words: 1, + recency: 0, +}; + +/** How far the top item must lead the second for an explicit request to insert it (DESIGN 10.2). */ +export const LEAD_MARGIN = 1; +/** The frequency bonus of the most used item in a table. */ +const FREQUENCY = 1.5; +/** The recency bonus of the last accepted item. */ +const RECENCY = 2; +const RECENT_KEPT = 10; + +/** Items accepted earlier in this session, the latest last. Shared by every editor. */ +const recent: string[] = []; + +export function rememberPick(label: string): void { + const at = recent.indexOf(label); + if (at >= 0) recent.splice(at, 1); + recent.push(label); + if (recent.length > RECENT_KEPT) recent.shift(); +} + +export function forgetPicks(): void { + recent.length = 0; +} + +function recency(label: string): number { + const at = recent.lastIndexOf(label); + return at < 0 ? 0 : (RECENCY * (at + 1)) / recent.length; +} + +/** + * How well `label` matches the typed word: 3 for a prefix, 2 for a prefix in another case, 1 for the letters in + * order with the same first letter, null for no match. An empty word matches everything at 0. + */ +export function matchScore(label: string, typed: string): number | null { + if (!typed) return 0; + if (label.startsWith(typed)) return 3; + const lower = label.toLowerCase(); + const want = typed.toLowerCase(); + if (lower.startsWith(want)) return 2; + if (lower[0] !== want[0]) return null; + let at = 0; + for (const char of lower) if (char === want[at]) at++; + return at === want.length ? 1 : null; +} + +/** What the ranking reads besides the context: the parsed tree and the other open files. */ +export interface RankInput { + tree?: SyntaxTree | null; + otherTexts?: readonly string[]; +} + +const frequency = (rank: number, size: number): number => (size > 0 ? FREQUENCY * (1 - rank / size) : 0); + +type Candidate = Omit & { extra: number }; + +function details() { + return { + keyword: t('codeStudio.complete.detail.keyword'), + builtin: t('codeStudio.complete.detail.builtin'), + member: t('codeStudio.complete.detail.member'), + module: t('codeStudio.complete.detail.module'), + word: t('codeStudio.complete.detail.word'), + }; +} + +/** The words source reads this many characters of the file around the caret, and as many of the other files. */ +export const WORDS_WINDOW = 200_000; +/** The window's edges snap to blocks of this size, so it stays put while the caret moves within a block. */ +const WORDS_BLOCK = 16_384; +/** The list returns at most this many items. */ +export const MAX_ITEMS = 200; + +const EMPTY = new Map(); + +/** Counts of the file's window outside the caret's line. Typing on one line leaves both texts unchanged. */ +let aroundCache: { head: string; tail: string; counts: Map } | null = null; +/** Counts of the other open files, by the texts last seen. */ +let otherCache: { texts: readonly string[]; counts: Map } | null = null; + +function sameTexts(a: readonly string[], b: readonly string[]): boolean { + return a.length === b.length && a.every((text, i) => text === b[i]); +} + +const snap = (offset: number): number => Math.max(0, Math.floor(offset / WORDS_BLOCK) * WORDS_BLOCK); + +/** + * The file's window split at the caret's line: the counts outside it (kept between keystrokes) and the line itself. + * The head ends at the line start. The tail starts at the line end, and the line start fixes its length. So typing on + * one line leaves both texts unchanged. A line longer than WORDS_WINDOW is cut at block edges. + */ +function fileCounts(text: string, pos: number): { around: Map; line: Map } { + const lineStart = Math.max(text.lastIndexOf('\n', pos - 1) + 1, snap(pos - WORDS_WINDOW / 2)); + const from = snap(lineStart - WORDS_WINDOW / 2); + const newline = text.indexOf('\n', pos); + const lineEnd = Math.min(newline < 0 ? text.length : newline, lineStart + WORDS_WINDOW); + const head = text.slice(from, lineStart); + const tail = text.slice(lineEnd, lineEnd + (from + WORDS_WINDOW - lineStart)); + if (aroundCache?.head !== head || aroundCache.tail !== tail) { + aroundCache = { head, tail, counts: countWords(`${head}\n${tail}`) }; + } + return { around: aroundCache.counts, line: countWords(text.slice(lineStart, lineEnd)) }; +} + +/** Counts of the other open files, up to WORDS_WINDOW characters in all. */ +function otherCounts(texts: readonly string[]): Map { + if (!texts.length) return EMPTY; + if (otherCache && sameTexts(otherCache.texts, texts)) return otherCache.counts; + let left = WORDS_WINDOW; + const kept: string[] = []; + for (const text of texts) { + if (left <= 0) break; + kept.push(text.slice(0, left)); + left -= text.length + 1; + } + otherCache = { texts: [...texts], counts: countWords(kept.join('\n')) }; + return otherCache.counts; +} + +type Details = ReturnType; + +/** Keywords and built-ins from the bundled tables, and the language's snippets. */ +function tableCandidates(language: string, detail: Details): Candidate[] { + const list: Candidate[] = []; + const keywords = own(KEYWORDS, language) ?? []; + keywords.forEach((label, rank) => { + const extra = frequency(rank, keywords.length); + list.push({ label, kind: 'keyword', source: 'keywords', detail: detail.keyword, extra }); + }); + const builtins = own(BUILTINS, language) ?? []; + builtins.forEach((label, rank) => { + const extra = frequency(rank, builtins.length); + list.push({ label, kind: 'function', source: 'keywords', detail: detail.builtin, extra }); + }); + for (const snippet of own(SNIPPETS, language) ?? []) { + // The list shows no icons, so the note itself says that the item is a snippet. + const note = t('codeStudio.complete.detail.snippet', { note: t(`codeStudio.complete.snippets.${snippet.note}`) }); + const { label, template } = snippet; + list.push({ label, kind: 'snippet', source: 'snippets', detail: note, insert: template, snippet: true, extra: 0 }); + } + return list; +} + +/** Identifiers in the file and the other open files, by frequency; the word being typed does not count itself. */ +function wordCandidates(context: CompletionContext, input: RankInput, detail: Details): Candidate[] { + const { text, pos, word } = context; + const { around, line } = fileCounts(text, pos); + const others = otherCounts(input.otherTexts ?? []); + const list: Candidate[] = []; + const add = (label: string, mine: number, sources: ReadonlyArray>) => { + if (label.length < 2 || matchScore(label, word.text) === null) return; + let count = mine - (label === word.text ? 1 : 0); + for (const counts of sources) count += counts.get(label) ?? 0; + if (count < 1) return; + const extra = Math.min(1, Math.log2(1 + count) / 3); + list.push({ label, kind: 'text', source: 'words', detail: detail.word, extra }); + }; + for (const [label, count] of around) add(label, count, [line, others]); + for (const [label, count] of line) if (!around.has(label)) add(label, count, [others]); + for (const [label, count] of others) if (!around.has(label) && !line.has(label)) add(label, count, []); + return list; +} + +/** True when the word follows a `.` that is not part of `..` or `...`: a member access, a number, or a chain. */ +function afterDot(text: string, from: number): boolean { + return text[from - 1] === '.' && text[from - 2] !== '.'; +} + +function candidates(context: CompletionContext, input: RankInput): Candidate[] { + const { language, text, word } = context; + const detail = details(); + const receiver = receiverAt(text, word.from - 1); + // After `get().`, `rows[0].`, `3.`, or a `.` that starts a line, the receiver is unknown: the general list + // would only offer names that cannot follow. + if (!receiver && afterDot(text, word.from)) return []; + if (receiver) { + return membersOf(language, text, receiver, word.from - 1).map((member) => ({ + label: member.name, + kind: member.module ? 'property' : 'method', + source: 'members', + detail: member.module ? detail.module : detail.member, + extra: member.rank < 0 ? -1 : frequency(member.rank, member.size), + })); + } + const scope: Candidate[] = input.tree + ? namesInScope(bindings(input.tree, text, language, context.pos), text, context.pos, word.from).map( + ({ name, proximity }) => ({ + label: name, + kind: 'variable', + source: 'scope', + extra: proximity, + }), + ) + : []; + return [...scope, ...tableCandidates(language, detail), ...wordCandidates(context, input, detail)]; +} + +/** + * The ranked list for a context: deduplicated by label (the best source wins), best first, ties alphabetical, and at + * most MAX_ITEMS long. + */ +export function rank(context: CompletionContext, input: RankInput = {}): CompletionItem[] { + const best = new Map(); + for (const candidate of candidates(context, input)) { + const { extra, ...item } = candidate; + const exact = item.label === context.word.text; + const match = matchScore(item.label, context.word.text); + if (match === null || (exact && !item.snippet)) continue; + const prior = PRIOR[item.source as keyof typeof PRIOR] ?? 0; + const score = match + prior + extra + recency(item.label); + const key = item.snippet ? `snippet:${item.label}` : item.label; + const known = best.get(key); + if (!known || score > known.score) best.set(key, { ...item, score: Math.round(score * 1000) / 1000 }); + } + const sorted = [...best.values()].sort( + (a, b) => b.score - a.score || (a.label < b.label ? -1 : a.label > b.label ? 1 : 0), + ); + return sorted.slice(0, MAX_ITEMS); +} + +/** + * The list's leader for an explicit request. It is the top plain item, when the caret ends a word that the item + * extends, and the item leads the second by LEAD_MARGIN. Returns the text to insert, or null. + */ +export function leader(context: CompletionContext, input: RankInput = {}): string | null { + const { text, pos, word } = context; + if (!word.text || /[\p{L}\p{N}_$]/u.test(text[pos] ?? '')) return null; + const [top, second] = rank(context, input); + if (!top || top.snippet || !top.label.startsWith(word.text)) return null; + if (second && top.score - second.score < LEAD_MARGIN) return null; + return top.label.slice(word.text.length) || null; +} diff --git a/app/src/features/codestudio/complete/scope.ts b/app/src/features/codestudio/complete/scope.ts new file mode 100644 index 000000000..88aae1c33 --- /dev/null +++ b/app/src/features/codestudio/complete/scope.ts @@ -0,0 +1,148 @@ +// The scope source (DESIGN 10.2): identifiers declared in the scopes around the caret, read from the Lezer tree. +// A nearer scope and a nearer line rank higher. The walk skips the insides of scopes away from the caret. + +import type { syntaxTree } from '@codemirror/language'; + +export type SyntaxTree = ReturnType; + +/** One name the tree binds, and the scope it lives in. */ +export interface Binding { + name: string; + from: number; + scopeFrom: number; + scopeTo: number; + /** How many scopes enclose it; 0 is the file. */ + depth: number; + /** A definition (a function, class, or parameter name) rather than a use. */ + declared: boolean; +} + +// Identifier nodes of the lang-* grammars (Python, JavaScript, C, C++, Java, Rust, and Go). +const NAMES = new Set(['VariableName', 'VariableDefinition', 'Identifier', 'Definition', 'DefName', 'BoundIdentifier']); +const PARAMETERS = new Set(['ParamList', 'ParameterList', 'FormalParameters', 'Parameter', 'Parameters']); +// Scopes that own their name: the name belongs to the outer scope. +const DEFINITIONS = new Set([ + 'FunctionDefinition', + 'ClassDefinition', + 'FunctionDeclaration', + 'ClassDeclaration', + 'MethodDeclaration', + 'FunctionItem', +]); +// Python binds names per function and class; blocks do not open scopes. +const PYTHON_SCOPES = new Set(['FunctionDefinition', 'ClassDefinition', 'LambdaExpression']); +const BLOCK_SCOPES = new Set([ + ...DEFINITIONS, + 'ArrowFunction', + 'FunctionExpression', + 'Block', + 'CompoundStatement', + 'ClassBody', +]); + +interface Open { + from: number; + to: number; + definition: boolean; + named: boolean; + /** The scope does not contain the caret, so only its own name can be visible there. */ + away: boolean; +} + +/** The walk reads the nodes within this many characters of the caret, and the scopes that enclose them. */ +export const SCOPE_WINDOW = 60_000; + +const cache = new WeakMap(); + +/** + * The bindings that can be visible at `pos`: those of the scopes around it, and the own names of the scopes away + * from it, whose insides the walk skips. Only nodes within SCOPE_WINDOW of the caret are read, so a large file costs + * no more than its window. Cached per tree and caret. + */ +export function bindings(tree: SyntaxTree, text: string, language: string, pos: number): Binding[] { + const from = Math.max(0, pos - SCOPE_WINDOW); + const to = pos + SCOPE_WINDOW; + const known = cache.get(tree); + if (known?.pos === pos) return known.found; + const scopes = language === 'python' ? PYTHON_SCOPES : BLOCK_SCOPES; + const found: Binding[] = []; + const stack: Open[] = [{ from: 0, to: Number.MAX_SAFE_INTEGER, definition: false, named: true, away: false }]; + let inParameters = 0; + tree.iterate({ + from, + to, + enter(node) { + const name = node.name; + const top = stack[stack.length - 1]!; + // A skipped node gets no leave call, so it must not touch the stack or the parameter count. + if (top.away && top.named) return false; + if (scopes.has(name)) { + const away = pos < node.from || pos > node.to; + const definition = DEFINITIONS.has(name); + if (away && !definition) return false; + stack.push({ from: node.from, to: node.to, definition, named: false, away }); + return; + } + if (PARAMETERS.has(name)) inParameters++; + if (!NAMES.has(name)) return; + // The first name inside a function or class definition is its own name, bound in the outer scope. + const own = top.definition && !top.named && inParameters === 0; + if (top.definition) top.named = true; + const scope = own ? (stack[stack.length - 2] ?? top) : top; + found.push({ + name: text.slice(node.from, node.to), + from: node.from, + scopeFrom: scope.from, + scopeTo: scope.to, + depth: own ? stack.length - 2 : stack.length - 1, + declared: own || inParameters > 0 || name === 'VariableDefinition' || name === 'DefName', + }); + }, + leave(node) { + if (scopes.has(node.name)) stack.pop(); + else if (PARAMETERS.has(node.name)) inParameters--; + }, + }); + cache.set(tree, { pos, found }); + return found; +} + +/** A visible name with its proximity bonus. */ +export interface ScopeName { + name: string; + proximity: number; +} + +function lineOf(lineStarts: readonly number[], pos: number): number { + let low = 0; + let high = lineStarts.length - 1; + while (low < high) { + const mid = (low + high + 1) >> 1; + if (lineStarts[mid]! <= pos) low = mid; + else high = mid - 1; + } + return low; +} + +function starts(text: string): number[] { + const result = [0]; + for (let i = text.indexOf('\n'); i >= 0; i = text.indexOf('\n', i + 1)) result.push(i + 1); + return result; +} + +/** + * The names visible at `pos` (their scope contains it), except the word being typed, each with its best proximity. + * Deeper scopes and nearer lines score higher, and declarations a little more than uses. + */ +export function namesInScope(all: readonly Binding[], text: string, pos: number, wordFrom: number): ScopeName[] { + const lineStarts = starts(text); + const caretLine = lineOf(lineStarts, pos); + const best = new Map(); + for (const binding of all) { + if (binding.from === wordFrom || pos < binding.scopeFrom || pos > binding.scopeTo) continue; + const distance = Math.abs(lineOf(lineStarts, binding.from) - caretLine); + const proximity = Math.min(binding.depth, 3) * 0.5 + 1 / (1 + distance / 20) + (binding.declared ? 0.5 : 0); + if (proximity > (best.get(binding.name) ?? -1)) best.set(binding.name, proximity); + } + return [...best].map(([name, proximity]) => ({ name, proximity })); +} diff --git a/app/src/features/codestudio/complete/snippetFields.test.ts b/app/src/features/codestudio/complete/snippetFields.test.ts new file mode 100644 index 000000000..5bb1f5f51 --- /dev/null +++ b/app/src/features/codestudio/complete/snippetFields.test.ts @@ -0,0 +1,94 @@ +// @vitest-environment jsdom +// Completion in a real editor (DESIGN 10.2, 5.3). The Python for snippet from the list has fields. Enter and +// Shift+Enter move between them, while Tab indents and stays in the field. An explicit request inserts the leading +// list item. Expected texts are hand-worked from the snippet template and PEP 8's 4-space indent. +import { + acceptCompletion, + completionStatus, + currentCompletions, + setSelectedCompletion, + startCompletion, +} from '@codemirror/autocomplete'; +import { afterEach, beforeAll, describe, expect, it } from 'vitest'; +import { registerCodeEditorCommands } from '../editor/commands'; +import { configureTestCommands, mount, press, resetSettings, unmountAll } from '../editor/testEditor'; +import type { CodeEditor } from '../editor'; +import { attachCompletion, forgetPicks } from './index'; + +// The first mount loads the Python language chunk, which can take seconds on a busy machine. +beforeAll(() => { + configureTestCommands(); + registerCodeEditorCommands(); +}); + +afterEach(() => { + unmountAll(); + resetSettings(); + forgetPicks(); +}); + +const until = async (check: () => boolean) => { + for (let i = 0; i < 100 && !check(); i++) await new Promise((resolve) => setTimeout(resolve, 10)); +}; + +const selected = (editor: CodeEditor): string => { + const { from, to } = editor.view.state.selection.main; + return editor.view.state.sliceDoc(from, to); +}; + +async function insertForSnippet(editor: CodeEditor): Promise { + startCompletion(editor.view); + await until(() => completionStatus(editor.view.state) === 'active'); + const options = currentCompletions(editor.view.state); + const index = options.findIndex((option) => option.label === 'for' && option.type === 'snippet'); + expect(index).toBeGreaterThanOrEqual(0); + editor.view.dispatch({ effects: setSelectedCompletion(index) }); + // CodeMirror ignores an accept within its interaction delay (75 ms) after the list opens. + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(acceptCompletion(editor.view)).toBe(true); +} + +describe('snippet fields', () => { + it('moves with Enter and Shift+Enter, and Tab indents without moving', async () => { + const editor = await mount({ language: 'python', text: 'fo' }); + attachCompletion(editor); + await insertForSnippet(editor); + expect(editor.getText()).toBe('for item in items:\n pass'); + expect(selected(editor)).toBe('item'); + + press(editor, 'Enter'); + expect(selected(editor)).toBe('items'); + press(editor, 'Enter', { shift: true }); + expect(selected(editor)).toBe('item'); + + press(editor, 'Tab'); + expect(editor.getText()).toBe(' for item in items:\n pass'); + expect(selected(editor)).toBe('item'); + // Still in the same snippet: Enter goes on to the next field. + press(editor, 'Enter'); + expect(selected(editor)).toBe('items'); + expect(editor.getText()).toBe(' for item in items:\n pass'); + }, 20_000); +}); + +describe('explicit requests in the editor', () => { + it('inserts the leading list item, and nothing on a blank line', async () => { + const editor = await mount({ language: 'python', text: 'total = 0\nfor n in nums:\n tot' }); + attachCompletion(editor); + expect(await editor.requestSuggestion(true)).toBe(true); + expect(editor.getText()).toBe('total = 0\nfor n in nums:\n total'); + + const empty = await mount({ language: 'python', text: '' }); + attachCompletion(empty); + expect(await empty.requestSuggestion(true)).toBe(false); + expect(empty.getText()).toBe(''); + }, 20_000); + + it('shows the import alias as ghost text', async () => { + const editor = await mount({ language: 'python', text: 'import numpy as ' }); + attachCompletion(editor); + expect(await editor.requestSuggestion(false)).toBe(true); + expect(editor.acceptSuggestion('all')).toBe('np'); + expect(editor.getText()).toBe('import numpy as np'); + }, 20_000); +}); diff --git a/app/src/features/codestudio/complete/snippets.ts b/app/src/features/codestudio/complete/snippets.ts new file mode 100644 index 000000000..7288fd070 --- /dev/null +++ b/app/src/features/codestudio/complete/snippets.ts @@ -0,0 +1,67 @@ +// Snippets with fields (DESIGN 10.2). Templates use CodeMirror's `${1:name}` fields; a tab character in a template +// becomes one indent unit of the editor. Enter and Shift+Enter move between fields; Tab indents (DESIGN 5.3). + +import type { codeStudioComplete } from '../../../strings/en/codeStudio/complete'; + +export type SnippetNote = keyof typeof codeStudioComplete.snippets; + +export interface Snippet { + label: string; + note: SnippetNote; + template: string; +} + +const s = (label: string, note: SnippetNote, template: string): Snippet => ({ label, note, template }); + +const C_LOOPS: readonly Snippet[] = [ + s('fori', 'forIndex', 'for (int ${1:i} = 0; ${1:i} < ${2:n}; ${1:i}++) {\n\t${3}\n}'), + s('while', 'whileLoop', 'while (${1:condition}) {\n\t${2}\n}'), + s('if', 'ifBlock', 'if (${1:condition}) {\n\t${2}\n}'), +]; + +const JS: readonly Snippet[] = [ + s('for', 'forEach', 'for (const ${1:item} of ${2:items}) {\n\t${3}\n}'), + s('fori', 'forIndex', 'for (let ${1:i} = 0; ${1:i} < ${2:n}; ${1:i}++) {\n\t${3}\n}'), + s('while', 'whileLoop', 'while (${1:condition}) {\n\t${2}\n}'), + s('function', 'functionDef', 'function ${1:name}(${2:params}) {\n\t${3}\n}'), + s('class', 'classDef', 'class ${1:Name} {\n\tconstructor(${2:params}) {\n\t\t${3}\n\t}\n}'), + s('if', 'ifBlock', 'if (${1:condition}) {\n\t${2}\n}'), + s('try', 'tryBlock', 'try {\n\t${1}\n} catch (${2:error}) {\n\t${3}\n}'), +]; + +/** Snippets per language. */ +export const SNIPPETS: Readonly> = { + python: [ + s('for', 'forEach', 'for ${1:item} in ${2:items}:\n\t${3:pass}'), + s('fori', 'forIndex', 'for ${1:i} in range(${2:n}):\n\t${3:pass}'), + s('while', 'whileLoop', 'while ${1:condition}:\n\t${2:pass}'), + s('def', 'functionDef', 'def ${1:name}(${2:params}):\n\t${3:pass}'), + s('class', 'classDef', 'class ${1:Name}:\n\tdef __init__(self${2}):\n\t\t${3:pass}'), + s('if', 'ifBlock', 'if ${1:condition}:\n\t${2:pass}'), + s('main', 'mainBlock', 'if __name__ == "__main__":\n\t${1:main()}'), + s('try', 'tryBlock', 'try:\n\t${1:pass}\nexcept ${2:Exception} as ${3:error}:\n\t${4:raise}'), + ], + javascript: JS, + typescript: JS, + c: [ + ...C_LOOPS, + s('main', 'mainBlock', 'int main(void) {\n\t${1}\n\treturn 0;\n}'), + s('printf', 'printf', 'printf("${1:%d}\\n", ${2:value});'), + ], + cpp: [ + s('for', 'forEach', 'for (const auto& ${1:item} : ${2:items}) {\n\t${3}\n}'), + ...C_LOOPS, + s('main', 'mainBlock', 'int main() {\n\t${1}\n\treturn 0;\n}'), + s('class', 'classDef', 'class ${1:Name} {\npublic:\n\t${2}\n};'), + s('try', 'tryBlock', 'try {\n\t${1}\n} catch (const std::exception& ${2:error}) {\n\t${3}\n}'), + s('printf', 'printf', 'printf("${1:%d}\\n", ${2:value});'), + ], + java: [ + s('for', 'forEach', 'for (${1:int} ${2:item} : ${3:items}) {\n\t${4}\n}'), + ...C_LOOPS, + s('main', 'mainBlock', 'public static void main(String[] args) {\n\t${1}\n}'), + s('class', 'classDef', 'public class ${1:Name} {\n\t${2}\n}'), + s('try', 'tryBlock', 'try {\n\t${1}\n} catch (${2:Exception} ${3:error}) {\n\t${4}\n}'), + ], + sql: [s('select', 'select', 'SELECT ${1:*} FROM ${2:table} WHERE ${3:condition};')], +}; diff --git a/app/src/features/codestudio/complete/tables.ts b/app/src/features/codestudio/complete/tables.ts new file mode 100644 index 000000000..7cdcaeb22 --- /dev/null +++ b/app/src/features/codestudio/complete/tables.ts @@ -0,0 +1,180 @@ +// Bundled tables for completion without a model (DESIGN 10.2): keywords and built-ins per language, and the members +// of common built-in types and modules. Each list is ordered from the most used to the least used, by our own +// estimate of everyday code; the order is the frequency the ranking reads. The names come from each language's +// reference documentation. + +// prettier-ignore +const JS_KEYWORDS: readonly string[] = [ + 'const', 'return', 'if', 'function', 'let', 'this', 'else', 'for', 'new', 'true', 'false', 'null', 'undefined', + 'of', 'import', 'export', 'await', 'async', 'class', 'while', 'typeof', 'break', 'continue', 'throw', 'try', + 'catch', 'finally', 'switch', 'case', 'default', 'in', 'instanceof', 'var', 'delete', 'do', 'extends', 'super', + 'static', 'yield', 'void', +]; + +/** Keywords per language, most used first. */ +// prettier-ignore +export const KEYWORDS: Readonly> = { + python: [ + 'def', 'return', 'if', 'for', 'in', 'import', 'else', 'elif', 'self', 'None', 'True', 'False', 'and', 'not', + 'from', 'class', 'while', 'or', 'is', 'try', 'except', 'with', 'as', 'break', 'continue', 'pass', 'lambda', + 'raise', 'finally', 'yield', 'global', 'nonlocal', 'assert', 'del', 'async', 'await', + ], + javascript: JS_KEYWORDS, + typescript: [...JS_KEYWORDS, 'interface', 'type', 'enum', 'implements', 'readonly', 'keyof', 'as'], + c: [ + 'int', 'return', 'if', 'for', 'char', 'void', 'else', 'while', 'const', 'struct', 'unsigned', 'double', 'float', + 'long', 'sizeof', 'static', 'break', 'case', 'switch', 'default', 'continue', 'typedef', 'NULL', 'enum', 'short', + 'do', 'goto', 'signed', 'extern', 'union', 'volatile', 'bool', 'true', 'false', + ], + cpp: [ + 'int', 'return', 'if', 'for', 'auto', 'const', 'std', 'void', 'else', 'while', 'char', 'double', 'bool', 'true', + 'false', 'class', 'struct', 'public', 'private', 'nullptr', 'new', 'delete', 'this', 'using', 'namespace', + 'template', 'typename', 'static', 'break', 'continue', 'switch', 'case', 'default', 'float', 'long', 'unsigned', + 'virtual', 'override', 'protected', 'try', 'catch', 'throw', 'enum', 'sizeof', 'constexpr', 'do', + ], + java: [ + 'public', 'return', 'if', 'int', 'new', 'private', 'void', 'static', 'final', 'for', 'else', 'this', 'class', + 'String', 'boolean', 'null', 'true', 'false', 'import', 'while', 'double', 'try', 'catch', 'throw', 'throws', + 'extends', 'implements', 'interface', 'break', 'continue', 'switch', 'case', 'default', 'long', 'char', + 'protected', 'package', 'super', 'instanceof', 'abstract', 'finally', 'do', 'enum', 'var', + ], + sql: [ + 'SELECT', 'FROM', 'WHERE', 'AND', 'JOIN', 'ON', 'AS', 'ORDER', 'BY', 'GROUP', 'INSERT', 'INTO', 'VALUES', + 'UPDATE', 'SET', 'DELETE', 'LEFT', 'INNER', 'NOT', 'NULL', 'IS', 'OR', 'IN', 'LIKE', 'LIMIT', 'DISTINCT', + 'COUNT', 'SUM', 'AVG', 'MIN', 'MAX', 'HAVING', 'CREATE', 'TABLE', 'PRIMARY', 'KEY', 'INTEGER', 'TEXT', 'DESC', + 'ASC', 'BETWEEN', 'CASE', 'WHEN', 'THEN', 'ELSE', 'END', 'UNION', 'EXISTS', 'DROP', 'ALTER', + ], + lua: [ + 'local', 'end', 'function', 'if', 'then', 'return', 'for', 'do', 'else', 'nil', 'in', 'true', 'false', 'and', + 'not', 'or', 'elseif', 'while', 'break', 'repeat', 'until', 'goto', + ], + go: [ + 'func', 'return', 'if', 'err', 'nil', 'for', 'range', 'var', 'string', 'int', 'type', 'struct', 'package', + 'import', 'else', 'true', 'false', 'break', 'continue', 'switch', 'case', 'default', 'go', 'defer', 'map', + 'chan', 'select', 'const', 'interface', + ], + rust: [ + 'let', 'fn', 'mut', 'self', 'if', 'return', 'pub', 'impl', 'use', 'match', 'for', 'in', 'Some', 'None', 'Ok', + 'Err', 'else', 'struct', 'enum', 'mod', 'true', 'false', 'while', 'loop', 'break', 'continue', 'const', 'static', + 'trait', 'where', 'as', 'move', 'ref', 'crate', 'super', 'unsafe', 'async', 'await', 'dyn', 'type', + ], +}; + +// prettier-ignore +const JS_BUILTINS: readonly string[] = [ + 'console', 'Math', 'JSON', 'Array', 'Object', 'String', 'Number', 'Promise', 'Map', 'Set', 'Date', 'Error', + 'parseInt', 'parseFloat', 'setTimeout', 'Boolean', 'RegExp', 'isNaN', 'clearTimeout', 'setInterval', 'Symbol', + 'BigInt', 'globalThis', +]; + +/** Built-in functions and types per language, most used first. */ +// prettier-ignore +export const BUILTINS: Readonly> = { + python: [ + 'print', 'len', 'range', 'int', 'str', 'list', 'input', 'dict', 'float', 'enumerate', 'sum', 'max', 'min', + 'sorted', 'abs', 'open', 'zip', 'set', 'isinstance', 'map', 'round', 'tuple', 'bool', 'type', 'any', 'all', + 'reversed', 'filter', 'super', 'Exception', 'ValueError', 'TypeError', 'KeyError', 'IndexError', 'ord', 'chr', + 'hasattr', 'getattr', 'setattr', 'iter', 'next', 'pow', 'divmod', 'hex', 'bin', 'format', 'repr', 'id', 'hash', + ], + javascript: JS_BUILTINS, + typescript: [...JS_BUILTINS, 'Record', 'Partial', 'Readonly', 'string', 'number', 'boolean'], + c: [ + 'printf', 'scanf', 'malloc', 'free', 'strlen', 'size_t', 'main', 'fprintf', 'memset', 'strcmp', 'strcpy', 'puts', + 'calloc', 'realloc', 'memcpy', 'fgets', 'sprintf', 'exit', 'abs', 'sqrt', 'pow', 'stdout', 'stderr', 'stdin', + ], + cpp: [ + 'cout', 'endl', 'vector', 'string', 'cin', 'size_t', 'main', 'map', 'pair', 'make_pair', 'sort', 'min', 'max', + 'unordered_map', 'set', 'swap', 'push_back', 'unique_ptr', 'make_unique', 'shared_ptr', 'move', 'begin', 'end', + 'abs', 'cerr', 'getline', 'to_string', 'stoi', 'printf', + ], + java: [ + 'System', 'String', 'Integer', 'List', 'ArrayList', 'Math', 'Map', 'HashMap', 'Scanner', 'Object', 'Override', + 'Exception', 'Arrays', 'Double', 'Set', 'HashSet', 'StringBuilder', 'Collections', 'Long', 'Boolean', + ], + lua: [ + 'print', 'pairs', 'ipairs', 'table', 'string', 'tostring', 'tonumber', 'type', 'math', 'require', 'error', + 'pcall', 'select', 'setmetatable', 'assert', 'io', 'os', + ], + go: ['fmt', 'len', 'make', 'append', 'error', 'bool', 'byte', 'cap', 'copy', 'delete', 'panic', 'float64', 'rune'], + rust: [ + 'println', 'String', 'Vec', 'Option', 'Result', 'Box', 'format', 'vec', 'usize', 'i32', 'u8', 'i64', 'f64', + 'bool', 'str', 'HashMap', 'Self', 'clone', 'unwrap', + ], +}; + +/** Members of built-in types, most used first. Keys are `language:type`. */ +// prettier-ignore +export const TYPE_MEMBERS: Readonly> = { + 'python:list': ['append', 'extend', 'pop', 'sort', 'insert', 'index', 'remove', 'reverse', 'count', 'copy', 'clear'], + 'python:str': [ + 'join', 'split', 'strip', 'replace', 'format', 'lower', 'upper', 'startswith', 'endswith', 'find', 'encode', + 'isdigit', 'rstrip', 'lstrip', 'splitlines', 'count', 'title', 'isalpha', 'index', 'capitalize', 'zfill', + 'center', 'ljust', 'rjust', 'isspace', 'isupper', 'islower', 'partition', 'swapcase', + ], + 'python:dict': ['get', 'items', 'keys', 'values', 'update', 'pop', 'setdefault', 'copy', 'clear', 'popitem'], + 'python:set': [ + 'add', 'remove', 'discard', 'union', 'intersection', 'difference', 'update', 'pop', 'issubset', 'issuperset', + 'symmetric_difference', 'clear', 'copy', 'isdisjoint', + ], + 'javascript:Array': [ + 'push', 'length', 'map', 'filter', 'forEach', 'join', 'slice', 'indexOf', 'includes', 'reduce', 'find', 'pop', + 'splice', 'concat', 'sort', 'some', 'every', 'shift', 'unshift', 'reverse', 'findIndex', 'flat', 'flatMap', + 'fill', 'at', 'keys', 'values', 'entries', 'lastIndexOf', + ], + 'javascript:String': [ + 'length', 'split', 'replace', 'trim', 'slice', 'indexOf', 'includes', 'toLowerCase', 'toUpperCase', 'startsWith', + 'endsWith', 'substring', 'charAt', 'padStart', 'match', 'repeat', 'replaceAll', 'charCodeAt', 'padEnd', + 'trimStart', 'trimEnd', 'at', 'concat', 'lastIndexOf', 'search', 'localeCompare', + ], + 'javascript:Map': ['get', 'set', 'has', 'delete', 'size', 'keys', 'values', 'entries', 'forEach', 'clear'], + 'javascript:Set': ['add', 'has', 'delete', 'size', 'values', 'forEach', 'clear', 'keys', 'entries'], + 'javascript:Object': [ + 'keys', 'assign', 'entries', 'values', 'freeze', 'create', 'defineProperty', 'fromEntries', 'getPrototypeOf', + 'is', 'getOwnPropertyNames', + ], + 'javascript:Math': [ + 'floor', 'max', 'min', 'round', 'random', 'abs', 'ceil', 'sqrt', 'pow', 'PI', 'trunc', 'sign', 'log', 'sin', + 'cos', 'atan2', 'hypot', 'exp', 'log2', 'log10', 'tan', 'E', + ], + 'javascript:JSON': ['stringify', 'parse'], + 'javascript:console': ['log', 'error', 'warn', 'info', 'debug', 'table', 'time', 'timeEnd', 'assert', 'group'], + 'cpp:vector': [ + 'push_back', 'size', 'begin', 'end', 'empty', 'back', 'clear', 'emplace_back', 'pop_back', 'front', 'resize', + 'reserve', 'insert', 'erase', 'at', 'data', 'capacity', 'assign', 'swap', + ], + 'cpp:string': [ + 'size', 'length', 'substr', 'find', 'c_str', 'empty', 'push_back', 'append', 'begin', 'end', 'at', 'clear', + 'compare', 'replace', 'insert', 'erase', 'back', 'front', 'pop_back', 'rfind', + ], + 'cpp:map': [ + 'find', 'insert', 'count', 'size', 'begin', 'end', 'erase', 'empty', 'at', 'emplace', 'clear', 'contains', + 'lower_bound', 'upper_bound', + ], +}; + +/** Members of well-known modules, most used first. Keys are `language:module`. */ +// prettier-ignore +export const MODULE_MEMBERS: Readonly> = { + 'python:math': [ + 'sqrt', 'pi', 'floor', 'ceil', 'pow', 'log', 'sin', 'cos', 'exp', 'inf', 'fabs', 'gcd', 'isclose', 'factorial', + 'tan', 'atan2', 'radians', 'degrees', 'e', 'hypot', 'log10', 'log2', 'isnan', 'comb', 'perm', 'prod', 'trunc', + ], + 'python:random': [ + 'randint', 'random', 'choice', 'shuffle', 'seed', 'uniform', 'sample', 'choices', 'randrange', 'gauss', + ], + 'python:os.path': [ + 'join', 'exists', 'basename', 'dirname', 'isfile', 'isdir', 'splitext', 'abspath', 'getsize', 'split', + 'expanduser', 'relpath', 'normpath', + ], + 'python:os': ['path', 'listdir', 'getcwd', 'makedirs', 'environ', 'remove', 'mkdir', 'walk', 'rename', 'sep'], +}; + +/** Which language family a table key uses. TypeScript reads JavaScript's members, C++'s tables serve C++ only. */ +/** A table's own entry for `key`, so a key such as `constructor` never reads the object's prototype. */ +export function own(table: Readonly>, key: string): T | undefined { + return Object.hasOwn(table, key) ? table[key] : undefined; +} + +export function family(language: string): string { + return language === 'typescript' ? 'javascript' : language; +} diff --git a/app/src/features/codestudio/complete/tokens.ts b/app/src/features/codestudio/complete/tokens.ts new file mode 100644 index 000000000..01ebcf548 --- /dev/null +++ b/app/src/features/codestudio/complete/tokens.ts @@ -0,0 +1,57 @@ +// Small text helpers for completion without a model: a line tokenizer that keeps every character, so a line can be +// rebuilt from its tokens, and the word before the caret. + +export type TokenKind = 'word' | 'number' | 'space' | 'punct'; + +export interface Token { + kind: TokenKind; + text: string; +} + +// A run of letters (with `_` and `$`), a run of ASCII digits, white space, or any single other character, which +// includes digits outside ASCII such as `²`. A name such as `a1` splits into `a` and `1`, so a numeric step inside a +// name can be extrapolated. +const TOKEN = /[\p{L}_$]+|\d+(?:\.\d+)?|\s+|[^\p{L}\d_$\s]/gu; + +export function tokenize(line: string): Token[] { + const tokens: Token[] = []; + for (const match of line.matchAll(TOKEN)) { + const text = match[0]; + const kind: TokenKind = /^\d/.test(text) + ? 'number' + : /^\s/.test(text) + ? 'space' + : /^[\p{L}_$]/u.test(text) + ? 'word' + : 'punct'; + tokens.push({ kind, text }); + } + return tokens; +} + +/** Identifier characters, as the completion list's word match reads them. */ +export const WORD_CHAR = /[\p{L}\p{N}_$]/u; + +/** The identifier that ends at `pos`. */ +export function wordBefore(text: string, pos: number): { from: number; to: number; text: string } { + let from = pos; + while (from > 0 && WORD_CHAR.test(text[from - 1] ?? '')) from--; + return { from, to: pos, text: text.slice(from, pos) }; +} + +/** The line around `pos`: its start, the text before the caret, and the text after it. */ +export function lineAt(text: string, pos: number): { start: number; before: string; after: string } { + const start = text.lastIndexOf('\n', pos - 1) + 1; + const endIndex = text.indexOf('\n', pos); + const end = endIndex < 0 ? text.length : endIndex; + return { start, before: text.slice(start, pos), after: text.slice(pos, end) }; +} + +/** Identifiers in a text with their counts. Names inside strings and comments count too; that is fine for words. */ +export function countWords(text: string): Map { + const counts = new Map(); + for (const match of text.matchAll(/[\p{L}_$][\p{L}\p{N}_$]*/gu)) { + counts.set(match[0], (counts.get(match[0]) ?? 0) + 1); + } + return counts; +} diff --git a/app/src/features/codestudio/contracts/run.ts b/app/src/features/codestudio/contracts/run.ts index c193c95d4..c74d6830f 100644 --- a/app/src/features/codestudio/contracts/run.ts +++ b/app/src/features/codestudio/contracts/run.ts @@ -178,6 +178,11 @@ export interface RuntimeAdapter { allowances?: readonly string[]; /** The runtime's version, reported when a run starts, such as `MicroPython 1.27.0`. */ version?: string; + /** + * True when the runtime reads only the prefilled input. The worker then reports a run as not interactive, and the + * console says that interactive input is not available (DESIGN 4.4). + */ + prefilledOnly?: boolean; load(onProgress: (percent: number) => void): Promise; /** Runs one request and sends everything but `started` and `done`; the worker sends those. */ run(request: RunRequest, emit: Emit, context: RunContext): Promise; diff --git a/app/src/features/codestudio/diagnostics/analyze.ts b/app/src/features/codestudio/diagnostics/analyze.ts new file mode 100644 index 000000000..b6fa46b63 --- /dev/null +++ b/app/src/features/codestudio/diagnostics/analyze.ts @@ -0,0 +1,51 @@ +// One analysis of one file: the syntax layer (L-a) and the checker layer (L-b) over a single parse. The diagnostics +// worker runs this; when a page has no workers (tests, or a worker that failed), the page runs it itself. + +import { checkCLike } from './clike'; +import { checkJavaScript } from './javascript'; +import { checkLua } from './lua'; +import { loadParser, type ParserLoader } from './parsers'; +import { checkPython } from './python'; +import { syntaxFindings } from './syntax'; +import { lineAt, lineStarts } from './text'; +import type { EditorLanguage, Finding } from './types'; + +/** The languages that have an OpenNote checker (DESIGN 6.1, layer L-b). */ +export const CHECKER_LANGUAGES: readonly EditorLanguage[] = ['python', 'javascript', 'typescript', 'c', 'cpp', 'lua']; + +export interface AnalyzeRequest { + language: EditorLanguage; + text: string; + layers: readonly ('syntax' | 'checker')[]; +} + +export interface AnalyzeResult { + syntax: Finding[]; + checker: Finding[]; +} + +/** Analyzes one file. The worker passes its own parser loader; the page uses the language packs' parsers. */ +export async function analyze(request: AnalyzeRequest, load: ParserLoader = loadParser): Promise { + const { language, text, layers } = request; + if (language === 'lua') return { syntax: [], checker: layers.includes('checker') ? checkLua(text) : [] }; + const loading = load(language); + if (!loading) return { syntax: [], checker: [] }; + const tree = (await loading).parse(text); + const syntax = syntaxFindings(text, tree, language); + if (!layers.includes('checker') || !CHECKER_LANGUAGES.includes(language)) { + return { syntax: layers.includes('syntax') ? syntax : [], checker: [] }; + } + let checker = + language === 'python' + ? checkPython(text, tree) + : language === 'c' || language === 'cpp' + ? checkCLike(text, tree) + : checkJavaScript(text, tree); + if (syntax.length) { + // Past a syntax error the tree is a guess, so checker marks after the first broken line would mislead. + const starts = lineStarts(text); + const firstBroken = Math.min(...syntax.map((f) => lineAt(starts, f.from))); + checker = checker.filter((f) => lineAt(starts, f.from) < firstBroken); + } + return { syntax: layers.includes('syntax') ? syntax : [], checker }; +} diff --git a/app/src/features/codestudio/diagnostics/bench.ts b/app/src/features/codestudio/diagnostics/bench.ts new file mode 100644 index 000000000..fe2b5081e --- /dev/null +++ b/app/src/features/codestudio/diagnostics/bench.ts @@ -0,0 +1,143 @@ +// The diagnostics benchmark harness (DESIGN 15, gate G-D2): times one checker run, parse included, on a file of a +// given length, as the diagnostics worker does it. Lane L20's nightly table reads the same numbers. + +import { analyze } from './analyze'; +import type { EditorLanguage } from './types'; + +/** One block of ordinary code per language, about 20 lines, repeated to build a large file. */ +const BLOCKS: Partial string>> = { + python: (i) => + [ + `def step_${i}(items, scale=2):`, + ` """Scales the even items."""`, + ` total = 0`, + ` for index, value in enumerate(items):`, + ` if value % 2 == 0 and index > 0:`, + ` total += value * scale`, + ` elif value is None:`, + ` continue`, + ` else:`, + ` total -= math.floor(value / 3)`, + ` result = [x * x for x in items if x]`, + ` try:`, + ` ratio = total / len(result)`, + ` except ZeroDivisionError as error:`, + ` print(error)`, + ` ratio = 0`, + ` return {"total": total, "ratio": ratio, "n": len(result)}`, + ``, + `print(step_${i}([1, 2, 3, 4]))`, + ``, + ].join('\n'), + javascript: (i) => + [ + `function step${i}(items, scale = 2) {`, + ` let total = 0;`, + ` for (const [index, value] of items.entries()) {`, + ` if (value % 2 === 0 && index > 0) {`, + ` total += value * scale;`, + ` } else if (value === null) {`, + ` continue;`, + ` } else {`, + ` total -= Math.floor(value / 3);`, + ` }`, + ` }`, + ` const result = items.filter((x) => x).map((x) => x * x);`, + ` switch (result.length) {`, + ` case 0:`, + ` return { total, ratio: 0 };`, + ` default:`, + ` return { total, ratio: total / result.length };`, + ` }`, + `}`, + `console.log(step${i}([1, 2, 3, 4]));`, + ].join('\n'), + c: (i) => + [ + `static int step${i}(const int *items, int n, int scale) {`, + ` int total = 0;`, + ` for (int index = 0; index < n; index++) {`, + ` int value = items[index];`, + ` if (value % 2 == 0 && index > 0) {`, + ` total += value * scale;`, + ` } else if (value < 0) {`, + ` continue;`, + ` } else {`, + ` total -= value / 3;`, + ` }`, + ` }`, + ` if (n == 0) {`, + ` return 0;`, + ` }`, + ` printf("%d\\n", total / n);`, + ` return total;`, + `}`, + ``, + ``, + ].join('\n'), + lua: (i) => + [ + `local function step${i}(items, scale)`, + ` local total = 0`, + ` for index, value in ipairs(items) do`, + ` if value % 2 == 0 and index > 1 then`, + ` total = total + value * scale`, + ` elseif value < 0 then`, + ` total = total - 1`, + ` else`, + ` total = total - math.floor(value / 3)`, + ` end`, + ` end`, + ` local result = {}`, + ` for _, x in ipairs(items) do`, + ` result[#result + 1] = x * x`, + ` end`, + ` return { total = total, n = #result }`, + `end`, + `print(step${i}({ 1, 2, 3, 4 }, 2).total)`, + ``, + ``, + ].join('\n'), +}; +BLOCKS.typescript = BLOCKS.javascript; +BLOCKS.cpp = BLOCKS.c; + +/** A file of at least `lines` lines in `language`, built from ordinary code. */ +export function sampleFile(language: EditorLanguage, lines: number): string { + const block = BLOCKS[language]; + if (!block) throw new Error(`No sample code for ${language}.`); + const head = + language === 'python' ? 'import math\n' : language === 'c' || language === 'cpp' ? '#include \n' : ''; + let text = head; + for (let i = 0; text.split('\n').length < lines; i += 1) text += `${block(i)}\n`; + return text; +} + +/** The median time of `runs` checker runs, parse included, after one warm-up run. */ +export async function measureChecker(language: EditorLanguage, text: string, runs = 7): Promise { + await analyze({ language, text, layers: ['checker'] }); + const times: number[] = []; + for (let i = 0; i < runs; i += 1) { + const start = performance.now(); + await analyze({ language, text, layers: ['checker'] }); + times.push(performance.now() - start); + } + times.sort((a, b) => a - b); + return times[Math.floor(times.length / 2)]; +} + +/** The best of `attempts` medians per language, for a run that wants one number per language. */ +export async function benchmark( + languages: readonly EditorLanguage[], + lines = 1000, + attempts = 3, +): Promise> { + const result: Record = {}; + for (const language of languages) { + const text = sampleFile(language, lines); + let best = Infinity; + for (let i = 0; i < attempts; i += 1) best = Math.min(best, await measureChecker(language, text)); + result[language] = best; + } + return result; +} diff --git a/app/src/features/codestudio/diagnostics/checker.worker.ts b/app/src/features/codestudio/diagnostics/checker.worker.ts new file mode 100644 index 000000000..2e4db40ab --- /dev/null +++ b/app/src/features/codestudio/diagnostics/checker.worker.ts @@ -0,0 +1,27 @@ +// The diagnostics worker's entry (client.ts starts it). It answers each request with the findings of one file, so +// parsing and checking a large file never holds up typing. + +import { analyze, type AnalyzeRequest, type AnalyzeResult } from './analyze'; +import { loadWorkerParser } from './workerParsers'; + +export interface WorkerRequest extends AnalyzeRequest { + id: number; +} + +export type WorkerReply = { id: number; result: AnalyzeResult } | { id: number; error: string }; + +interface WorkerScope { + onmessage: ((event: MessageEvent) => void) | null; + postMessage(message: WorkerReply): void; +} + +if ('WorkerGlobalScope' in globalThis) { + const scope = globalThis as unknown as WorkerScope; + scope.onmessage = (event) => { + const { id } = event.data; + analyze(event.data, loadWorkerParser).then( + (result) => scope.postMessage({ id, result }), + (error: unknown) => scope.postMessage({ id, error: String(error) }), + ); + }; +} diff --git a/app/src/features/codestudio/diagnostics/client.ts b/app/src/features/codestudio/diagnostics/client.ts new file mode 100644 index 000000000..98cf1549b --- /dev/null +++ b/app/src/features/codestudio/diagnostics/client.ts @@ -0,0 +1,81 @@ +// The page's side of the diagnostics worker. One worker serves every file. Where workers are missing (tests) or the +// worker fails to start, the page analyzes the file itself, so marks still come. + +import type { AnalyzeRequest, AnalyzeResult } from './analyze'; +import type { WorkerReply, WorkerRequest } from './checker.worker'; + +/** The part of a Worker the client uses, so tests can give a stand-in. */ +export interface WorkerLike { + onmessage: ((event: { data: WorkerReply }) => void) | null; + onerror: ((event: unknown) => void) | null; + postMessage(message: WorkerRequest): void; + terminate(): void; +} + +export interface CheckerClient { + analyze(request: AnalyzeRequest): Promise; + dispose(): void; +} + +function spawnDefault(): WorkerLike | null { + if (typeof Worker === 'undefined') return null; + return new Worker(new URL('./checker.worker.ts', import.meta.url), { type: 'module' }) as unknown as WorkerLike; +} + +const inThread = (request: AnalyzeRequest) => import('./analyze').then((m) => m.analyze(request)); + +export function createCheckerClient(spawn: () => WorkerLike | null = spawnDefault): CheckerClient { + let worker: WorkerLike | null | undefined; + let next = 1; + const pending = new Map< + number, + { request: AnalyzeRequest; resolve: (r: AnalyzeResult) => void; reject: (e: Error) => void } + >(); + + const failOver = () => { + worker?.terminate(); + worker = null; + for (const [id, p] of pending) { + pending.delete(id); + inThread(p.request).then(p.resolve, p.reject); + } + }; + + const start = (): WorkerLike | null => { + if (worker !== undefined) return worker; + try { + worker = spawn(); + } catch { + worker = null; + } + if (worker) { + worker.onmessage = ({ data }) => { + const p = pending.get(data.id); + if (!p) return; + pending.delete(data.id); + if ('result' in data) p.resolve(data.result); + else p.reject(new Error(data.error)); + }; + worker.onerror = failOver; + } + return worker; + }; + + return { + analyze(request) { + const w = start(); + if (!w) return inThread(request); + const id = next++; + return new Promise((resolve, reject) => { + pending.set(id, { request, resolve, reject }); + w.postMessage({ id, ...request }); + }); + }, + dispose() { + worker?.terminate(); + worker = null; + for (const p of pending.values()) p.reject(new Error('The diagnostics worker stopped.')); + pending.clear(); + }, + }; +} diff --git a/app/src/features/codestudio/diagnostics/clike.ts b/app/src/features/codestudio/diagnostics/clike.ts new file mode 100644 index 000000000..6b83be5af --- /dev/null +++ b/app/src/features/codestudio/diagnostics/clike.ts @@ -0,0 +1,371 @@ +// The C and C++ scope checker over the Lezer tree (layer L-b, DESIGN 6.1). It finds two things within one file: +// identifiers used as values that nothing in the file declares (clang's "use of undeclared identifier"), and local +// variables that are never used (clang's -Wunused-variable). Without the headers it cannot know every library name, +// so it stays quiet where a name could come from elsewhere: calls, qualified names, members inside classes and +// methods, files that include a project header, and `using namespace` other than std. Clang (layer L-e) covers those. + +import type { SyntaxNode, Tree } from '@lezer/common'; +import type { Finding } from './types'; + +interface Variable { + name: string; + from: number; + to: number; + local: boolean; + /** Only plain values are reported unused: a class object may do its work in its constructor. */ + plain: boolean; + uses: number; +} + +interface Scope { + parent: Scope | null; + names: Map; +} + +/** Names the C and C++ standard headers define as values or macros, which a file may use without declaring. */ +const C_NAMES = new Set( + ( + 'NULL EOF stdin stdout stderr errno true false nullptr RAND_MAX EXIT_SUCCESS EXIT_FAILURE INT_MAX INT_MIN ' + + 'UINT_MAX LONG_MAX LONG_MIN ULONG_MAX LLONG_MAX LLONG_MIN ULLONG_MAX CHAR_BIT CHAR_MAX CHAR_MIN SCHAR_MAX ' + + 'SCHAR_MIN UCHAR_MAX SHRT_MAX SHRT_MIN USHRT_MAX DBL_MAX DBL_MIN DBL_EPSILON FLT_MAX FLT_MIN FLT_EPSILON ' + + 'SIZE_MAX M_PI M_E HUGE_VAL INFINITY NAN BUFSIZ FILENAME_MAX SEEK_SET SEEK_CUR SEEK_END CLOCKS_PER_SEC ' + + '__FILE__ __LINE__ __func__ __DATE__ __TIME__ __cplusplus this' + ).split(' '), +); + +/** Names `using namespace std;` brings in that people use as values. */ +const STD_NAMES = new Set( + ( + 'cout cin cerr clog endl ends flush ws boolalpha noboolalpha fixed scientific hex dec oct left right internal ' + + 'showpoint noshowpoint npos' + ).split(' '), +); + +const PLAIN_TYPES = new Set(['PrimitiveType', 'SizedTypeSpecifier', 'auto']); +const DECLARATORS = new Set([ + 'InitDeclarator', + 'PointerDeclarator', + 'ReferenceDeclarator', + 'ArrayDeclarator', + 'ParenthesizedDeclarator', + 'FunctionDeclarator', + 'StructuredBindingDeclarator', + 'VariadicDeclarator', +]); + +/** A macro-shaped name. The standard and POSIX headers define hundreds (ERANGE, SIG_IGN, UINT32_MAX, O_RDONLY). */ +const MACRO_SHAPE = /^_?[A-Z][A-Z0-9_]+$/; + +function kids(node: SyntaxNode): SyntaxNode[] { + const out: SyntaxNode[] = []; + for (let c = node.firstChild; c; c = c.nextSibling) out.push(c); + return out; +} + +class CChecker { + readonly findings: Finding[] = []; + private readonly variables: Variable[] = []; + private readonly refs: { name: string; from: number; to: number; scope: Scope; quiet: boolean }[] = []; + private names = C_NAMES; + private checkUndeclared = true; + /** True when the file includes a system header, which may define any macro-shaped name. */ + private systemHeader = false; + private member = 0; + /** Function-like macros the file defines: their arguments may be names the macro declares (`REP(i, n)`). */ + private readonly macros = new Set(); + + constructor(private readonly text: string) {} + + private slice(node: SyntaxNode): string { + return this.text.slice(node.from, node.to); + } + + run(tree: Tree): void { + const usings = [...this.text.matchAll(/\busing\s+namespace\s+([\w:]+)/g)].map((m) => m[1]); + if (usings.some((u) => u !== 'std') || /#\s*include\s*"/.test(this.text)) this.checkUndeclared = false; + this.systemHeader = /#\s*include\s* a.from - b.from); + } + + /** Clang (layer L-e), which reads the headers, is the authority on macro names a system header may define. */ + private maybeMacro(name: string): boolean { + return this.systemHeader && MACRO_SHAPE.test(name); + } + + private declare(scope: Scope, node: SyntaxNode, local: boolean, plain: boolean): void { + const v: Variable = { name: this.slice(node), from: node.from, to: node.to, local, plain, uses: 0 }; + if (!scope.names.has(v.name)) scope.names.set(v.name, v); + this.variables.push(v); + } + + private visitAll(nodes: SyntaxNode[], scope: Scope, local: boolean): void { + for (const node of nodes) this.visit(node, scope, local); + } + + private visit(node: SyntaxNode, scope: Scope, local: boolean): void { + switch (node.name) { + case 'Identifier': + this.refs.push({ + name: this.slice(node), + from: node.from, + to: node.to, + scope, + quiet: this.member > 0 || this.isCall(node), + }); + return; + case 'PreprocDirective': + return this.preproc(node, scope); + case 'ScopedIdentifier': + case 'FieldIdentifier': + case 'TypeIdentifier': + case 'NamespaceIdentifier': + case 'StatementIdentifier': + case 'Attribute': + case 'AttributeSpecifier': + case 'NamespaceAliasDefinition': + case 'FieldExpression': + if (node.name === 'FieldExpression' && node.firstChild) this.visit(node.firstChild, scope, local); + return; + case 'Declaration': + case 'FieldDeclaration': + case 'TypeDefinition': + case 'ParameterDeclaration': + case 'VariadicParameterDeclaration': + return this.declaration(node, scope, local); + case 'OptionalParameterDeclaration': + return this.optionalParameter(node, scope); + case 'UsingDeclaration': + return this.usingDeclaration(node, scope); + case 'LambdaCaptureSpecifier': + return this.captures(node, scope); + case 'Macro': + case 'CallExpression': + return this.macroCall(node, scope, local); + case 'Enumerator': { + const id = node.getChild('Identifier'); + if (id) this.declare(scope, id, false, false); + return this.visitAll(kids(node).slice(1), scope, local); + } + case 'FunctionDefinition': + return this.functionDef(node, scope, local); + case 'FieldDeclarationList': + this.member += 1; + this.visitAll(kids(node), { parent: scope, names: new Map() }, false); + this.member -= 1; + return; + case 'ForRangeLoop': + return this.rangeLoop(node, scope); + case 'NamespaceDefinition': { + // `namespace util { ... }` declares `util` in the enclosing scope. + const id = node.getChild('Identifier'); + if (id) this.declare(scope, id, false, false); + const inner: Scope = { parent: scope, names: new Map() }; + return this.visitAll( + kids(node).filter((k) => k !== id && k.from !== id?.from), + inner, + local, + ); + } + case 'CompoundStatement': + case 'ForStatement': + case 'LambdaExpression': + return this.visitAll(kids(node), { parent: scope, names: new Map() }, true); + default: + return this.visitAll(kids(node), scope, local); + } + } + + private isCall(node: SyntaxNode): boolean { + const parent = node.parent; + return ( + (parent?.name === 'CallExpression' || parent?.name === 'TemplateFunction') && + parent.firstChild?.from === node.from + ); + } + + private preproc(node: SyntaxNode, scope: Scope): void { + const parts = kids(node); + if (parts[0] && /^#\s*define$/.test(this.slice(parts[0]))) { + const id = parts.find((p) => p.name === 'Identifier'); + if (id) this.declare(scope, id, false, false); + // `#define REP(i, n) ...`: the parenthesis follows the name directly. + if (id && this.text[id.to] === '(') this.macros.add(this.slice(id)); + } + } + + private lookup(scope: Scope, name: string): Variable | undefined { + for (let s: Scope | null = scope; s; s = s.parent) { + const found = s.names.get(name); + if (found) return found; + } + return undefined; + } + + /** + * A call of a function-like macro the file defines. Its body may declare a name from an argument + * (`#define REP(i, n) for (int i = 0; ...)`), so a bare name argument counts as a use when it resolves and is + * declared in the current scope when it does not. + */ + private macroCall(node: SyntaxNode, scope: Scope, local: boolean): void { + const callee = node.firstChild; + const args = node.getChild('ArgumentList'); + if (!callee || !args || !this.macros.has(this.slice(callee))) return this.visitAll(kids(node), scope, local); + for (const arg of kids(args)) { + if (arg.name !== 'Identifier') { + this.visit(arg, scope, local); + continue; + } + const name = this.slice(arg); + this.refs.push({ name, from: arg.from, to: arg.to, scope, quiet: true }); + if (!this.lookup(scope, name)) this.declare(scope, arg, false, false); + } + } + + /** `int f(int b = 2)` and `template `: the name is declared, the default value is a read. */ + private optionalParameter(node: SyntaxNode, scope: Scope): void { + let declared = false; + for (const p of kids(node)) { + if (!declared && (p.name === 'Identifier' || DECLARATORS.has(p.name))) { + this.declarator(p, scope, false, false); + declared = true; + } else this.visit(p, scope, false); + } + } + + /** `using std::cout;` declares `cout`; `using namespace std;` declares nothing. */ + private usingDeclaration(node: SyntaxNode, scope: Scope): void { + if (kids(node).some((k) => k.name === 'namespace')) return; + let last: SyntaxNode | null = node.lastChild; + while (last && (last.name === ';' || last.name === 'ScopedIdentifier')) { + last = last.name === 'ScopedIdentifier' ? last.lastChild : last.prevSibling; + } + if (last?.name === 'Identifier') this.declare(scope, last, false, false); + } + + /** `[&, extra = 1]` declares `extra` in the lambda; `[x]` and the value of an init capture are reads. */ + private captures(node: SyntaxNode, scope: Scope): void { + for (const c of kids(node)) { + const id = c.name === 'AssignmentExpression' ? c.firstChild : null; + if (id?.name !== 'Identifier') { + this.visit(c, scope, true); + continue; + } + this.declare(scope, id, false, false); + for (const r of kids(c).slice(1)) this.visit(r, scope, true); + } + } + + /** Declares the name a declarator introduces; array sizes and initializers are reads. */ + private declarator(node: SyntaxNode, scope: Scope, local: boolean, plain: boolean): void { + if (node.name === 'Identifier') return this.declare(scope, node, local, plain); + if (node.name === 'FieldIdentifier' || node.name === 'TypeIdentifier') return; + if (!DECLARATORS.has(node.name)) return this.visit(node, scope, local); + const [first, ...rest] = kids(node); + if (!first) return; + if (node.name === 'StructuredBindingDeclarator') { + // `auto [key, value] = ...`: every name in the brackets is declared. + for (const id of node.getChildren('Identifier')) this.declare(scope, id, local, false); + return; + } + const pointer = node.name === 'PointerDeclarator' || node.name === 'ArrayDeclarator'; + if (node.name === 'FunctionDeclarator') { + if (first.name === 'Identifier') this.declare(scope, first, false, false); + // A function pointer, `int (*fp)(int)` or `int (*table[2])(int)`, declares the name inside the parentheses. + else if (DECLARATORS.has(first.name)) this.declarator(first, scope, local, plain); + const prototype: Scope = { parent: scope, names: new Map() }; + for (const p of rest) this.visit(p, prototype, false); + return; + } + const lead = + DECLARATORS.has(first.name) || first.name === 'Identifier' + ? first + : rest.find((r) => DECLARATORS.has(r.name) || r.name === 'Identifier'); + if (lead) this.declarator(lead, scope, local, plain || pointer); + for (const r of [first, ...rest]) if (r !== lead) this.visit(r, scope, local); + } + + /** `for (auto& [k, v] : m)` and `for (int x : v)`: the first declarator after the type is the loop variable. */ + private rangeLoop(node: SyntaxNode, scope: Scope): void { + const inner: Scope = { parent: scope, names: new Map() }; + const parts = kids(node); + const lead = parts.find((p) => p.name === 'Identifier' || DECLARATORS.has(p.name)); + for (const p of parts) { + if (p === lead || p.from === lead?.from) this.declarator(p, inner, true, false); + else this.visit(p, inner, true); + } + } + + private declaration(node: SyntaxNode, scope: Scope, local: boolean): void { + const parts = kids(node); + const attributed = /maybe_unused|__attribute__\s*\(\s*\(\s*unused/.test(this.slice(node)); + const plain = !attributed && parts.some((p) => PLAIN_TYPES.has(p.name)) && node.name === 'Declaration'; + const isLocal = + local && node.name === 'Declaration' && !parts.some((p) => /^(?:static|extern)$/.test(this.slice(p))); + for (const p of parts) { + if (p.name === 'Identifier' || DECLARATORS.has(p.name)) this.declarator(p, scope, isLocal, plain); + else if (p.name !== 'FieldIdentifier') this.visit(p, scope, local); + } + } + + private functionDef(node: SyntaxNode, scope: Scope, local: boolean): void { + const inner: Scope = { parent: scope, names: new Map() }; + const declarator = node.getChild('FunctionDeclarator'); + if (!declarator) { + // The grammar reads `std::array b{};` as a function definition without a declarator: it is a + // variable `b` with a brace initializer. + for (const child of kids(node)) { + if (child.name === 'Identifier') this.declare(scope, child, local, false); + else this.visit(child, scope, local); + } + return; + } + const name = declarator?.firstChild; + const method = this.member > 0 || (name !== null && name !== undefined && name.name !== 'Identifier'); + if (name?.name === 'Identifier') this.declare(scope, name, false, false); + if (method) this.member += 1; + for (const child of kids(node)) { + if (child === declarator || child.from === declarator?.from) { + for (const p of kids(child).slice(1)) this.visit(p, inner, false); + } else if (child.name === 'CompoundStatement') this.visitAll(kids(child), inner, true); + else this.visit(child, scope, false); + } + if (method) this.member -= 1; + } +} + +/** The C or C++ checker's findings for a parsed file. */ +export function checkCLike(text: string, tree: Tree): Finding[] { + const checker = new CChecker(text); + checker.run(tree); + return checker.findings; +} diff --git a/app/src/features/codestudio/diagnostics/engine.test.ts b/app/src/features/codestudio/diagnostics/engine.test.ts new file mode 100644 index 000000000..aa09f2094 --- /dev/null +++ b/app/src/features/codestudio/diagnostics/engine.test.ts @@ -0,0 +1,103 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { RuntimeAdapter } from '../contracts'; +import { createCheckerClient, type WorkerLike } from './client'; +import { engineChecks, fromRuntimeAdapter, registerEngineCheck, runtimeToDiagnostic } from './engine'; +import { createDiagnostics } from './providers'; + +describe('engine checks', () => { + beforeEach(() => vi.useFakeTimers()); + afterEach(() => vi.useRealTimers()); + + it('turns a runtime line and column into the word there, and a bare line into its text', () => { + const text = 'SELECT nmae\n FROM people;\n'; + expect( + runtimeToDiagnostic( + text, + { file: 'q.sql', line: 1, column: 8, message: 'no such column: nmae', kind: 'error' }, + 'sqlite', + ), + ).toEqual({ + file: 'q.sql', + from: 7, + to: 11, + severity: 'error', + code: 'error', + message: 'no such column: nmae', + source: 'sqlite', + }); + const line = runtimeToDiagnostic(text, { file: 'q.sql', line: 2, message: 'm', kind: 'warning' }, 'sqlite'); + expect(text.slice(line.from, line.to)).toBe('FROM people;'); + expect(line.severity).toBe('warning'); + }); + + it('joins checks registered before and after the service starts, and drops removed ones', async () => { + const off = registerEngineCheck({ + id: 'sqlite', + languages: ['sql'], + check: async (file) => [{ file: file.path, line: 1, message: 'near "FORM": syntax error', kind: 'error' }], + }); + const publish = vi.fn(); + const client = createCheckerClient(() => null); + const service = createDiagnostics({ publish, client }); + service.update({ file: 'q.sql', version: 1, language: 'sql', text: 'SELECT 1 FORM t;' }); + await vi.advanceTimersByTimeAsync(300); + expect( + service + .current('q.sql') + .filter((d) => d.source === 'sqlite') + .map((d) => d.message), + ).toEqual(['near "FORM": syntax error']); + + const lua: RuntimeAdapter = { + id: 'lua', + load: async () => {}, + run: () => Promise.reject(new Error('not run here')), + check: async (file) => [{ file: file.path, line: 1, message: "'=' expected near 'x'", kind: 'error' }], + }; + const offLua = registerEngineCheck(fromRuntimeAdapter(lua, ['lua'], 'wasmoon')!); + service.update({ file: 'm.lua', version: 1, language: 'lua', text: 'local = x' }); + await vi.advanceTimersByTimeAsync(300); + expect(service.current('m.lua').map((d) => d.source)).toContain('wasmoon'); + + off(); + offLua(); + expect(engineChecks()).toEqual([]); + expect(service.current('q.sql').filter((d) => d.source === 'sqlite')).toEqual([]); + service.dispose(); + }); + + it('has no check for an adapter without one', () => { + expect( + fromRuntimeAdapter({ id: 'r', load: async () => {}, run: () => Promise.reject(new Error('not run here')) }, [ + 'r', + ]), + ).toBeNull(); + }); +}); + +describe('checker client', () => { + it('sends work to the worker and matches each answer to its request', async () => { + const sent: unknown[] = []; + const worker: WorkerLike = { + onmessage: null, + onerror: null, + postMessage: (m) => sent.push(m), + terminate: vi.fn(), + }; + const client = createCheckerClient(() => worker); + const answer = client.analyze({ language: 'python', text: 'x', layers: ['checker'] }); + expect(sent).toEqual([{ id: 1, language: 'python', text: 'x', layers: ['checker'] }]); + worker.onmessage?.({ data: { id: 1, result: { syntax: [], checker: [] } } }); + await expect(answer).resolves.toEqual({ syntax: [], checker: [] }); + }); + + it('analyzes on the page when the worker fails', async () => { + const worker: WorkerLike = { onmessage: null, onerror: null, postMessage: () => {}, terminate: vi.fn() }; + const client = createCheckerClient(() => worker); + const answer = client.analyze({ language: 'python', text: 'import os\n', layers: ['checker'] }); + worker.onerror?.(new Event('error')); + const result = await answer; + expect(result.checker.map((f) => f.code)).toEqual(['F401']); + expect(worker.terminate).toHaveBeenCalled(); + }); +}); diff --git a/app/src/features/codestudio/diagnostics/engine.ts b/app/src/features/codestudio/diagnostics/engine.ts new file mode 100644 index 000000000..9d93335e5 --- /dev/null +++ b/app/src/features/codestudio/diagnostics/engine.ts @@ -0,0 +1,116 @@ +// Engine checks (layers L-c and L-e, DESIGN 6.1): the runtime lanes register a check that asks the real engine, +// such as SQLite's sqlite3_prepare_v2 against the project schema, wasmoon's load(), Python's compile(), or clang +// -fsyntax-only. A check reports lines, as runtimes do (RuntimeDiagnostic); this turns them into offsets and joins +// them to the scheduler like any other provider. + +import type { Diagnostic, RunFile, RuntimeAdapter, RuntimeDiagnostic, Severity } from '../contracts'; +import { lineRange, lineStarts } from './text'; +import type { DiagnosticProvider, EditorLanguage } from './types'; + +export interface EngineCheck { + /** Also the `source` of its marks, such as `sqlite`, `wasmoon`, `pyodide`, or `clang`. */ + id: string; + /** `engine` (300 ms pause) by default; clang uses `compiler` (800 ms). */ + layer?: 'engine' | 'compiler'; + languages: readonly EditorLanguage[]; + check(file: RunFile, files: RunFile[]): Promise; +} + +type Listener = (check: EngineCheck, added: boolean) => void; + +const checks = new Map(); +const listeners = new Set(); + +/** Registers a runtime's check. A second check with the same id replaces the first. Returns the way to remove it. */ +export function registerEngineCheck(check: EngineCheck): () => void { + const before = checks.get(check.id); + if (before) listeners.forEach((l) => l(before, false)); + checks.set(check.id, check); + listeners.forEach((l) => l(check, true)); + return () => { + if (checks.get(check.id) !== check) return; + checks.delete(check.id); + listeners.forEach((l) => l(check, false)); + }; +} + +/** The checks registered now. */ +export function engineChecks(): EngineCheck[] { + return [...checks.values()]; +} + +/** Hears checks come and go. Returns the way to stop listening. */ +export function onEngineChecks(listener: Listener): () => void { + listeners.add(listener); + return () => listeners.delete(listener); +} + +/** The engine check of a runtime adapter that has one. */ +export function fromRuntimeAdapter( + adapter: RuntimeAdapter, + languages: readonly EditorLanguage[], + id: string = adapter.id, + layer: EngineCheck['layer'] = 'engine', +): EngineCheck | null { + const check = adapter.check?.bind(adapter); + return check ? { id, layer, languages, check } : null; +} + +function severityOf(kind: string): Severity { + if (/warn/i.test(kind)) return 'warning'; + if (/note|info/i.test(kind)) return 'info'; + return 'error'; +} + +/** + * A runtime's line-based error as a mark in `text`. A column marks the word there; no column marks the line from its + * first character that is not a space. + */ +export function runtimeToDiagnostic( + text: string, + item: RuntimeDiagnostic, + source: string, + starts = lineStarts(text), +): Diagnostic { + const { from: lineFrom, to: lineTo } = lineRange(text, starts, Math.max(1, item.line)); + const line = text.slice(lineFrom, lineTo); + let from: number; + let to: number; + if (item.column !== undefined && item.column >= 1) { + from = Math.min(lineTo, lineFrom + item.column - 1); + const word = /^\w+/.exec(line.slice(from - lineFrom)); + to = word ? from + word[0].length : Math.min(lineTo, from + 1); + } else { + from = lineFrom + (line.length - line.trimStart().length); + to = lineTo; + } + return { + file: item.file, + from, + to: Math.max(from, to), + severity: severityOf(item.kind), + code: item.kind, + message: item.message, + source, + }; +} + +/** The scheduler provider for an engine check: it reads the file's version and keeps only this file's marks. */ +export function engineProvider(check: EngineCheck): DiagnosticProvider { + return { + id: check.id, + layer: check.layer ?? 'engine', + languages: check.languages, + async run(request) { + const file: RunFile = { path: request.file, text: request.text }; + const others = (request.files ?? []) + .filter((f) => f.path !== request.file) + .map((f) => ({ path: f.path, text: f.text })); + const items = await check.check(file, [file, ...others]); + const starts = lineStarts(request.text); + return items + .filter((d) => d.file === request.file) + .map((d) => runtimeToDiagnostic(request.text, d, check.id, starts)); + }, + }; +} diff --git a/app/src/features/codestudio/diagnostics/index.ts b/app/src/features/codestudio/diagnostics/index.ts new file mode 100644 index 000000000..afe730f40 --- /dev/null +++ b/app/src/features/codestudio/diagnostics/index.ts @@ -0,0 +1,25 @@ +// Automatic error detection for the code studio (DESIGN 6, ADR 0074), loaded with the studio. The studio creates one +// service, feeds it each file's text and version, and shows what it publishes with the editor's showProblems. +// Runtime lanes register their engine checks here (registerEngineCheck), loaded or not. + +export { analyze, CHECKER_LANGUAGES } from './analyze'; +export type { AnalyzeRequest, AnalyzeResult } from './analyze'; +export { createCheckerClient } from './client'; +export type { CheckerClient, WorkerLike } from './client'; +export { + engineChecks, + engineProvider, + fromRuntimeAdapter, + onEngineChecks, + registerEngineCheck, + runtimeToDiagnostic, +} from './engine'; +export type { EngineCheck } from './engine'; +export { pythonEngineCheck, registerRuntimeChecks } from './runtimeChecks'; +export { GRAMMAR_LANGUAGES, loadParser } from './parsers'; +export { checkerProvider, createDiagnostics, syntaxProvider, toDiagnostic } from './providers'; +export type { DiagnosticsService } from './providers'; +export { createDiagnosticsScheduler } from './scheduler'; +export type { DiagnosticsScheduler, MapPos, SchedulerOptions } from './scheduler'; +export { LAYER_DEBOUNCE_MS, LAYERS } from './types'; +export type { DiagnosticProvider, DiagnosticRequest, Finding, Layer, MessageId } from './types'; diff --git a/app/src/features/codestudio/diagnostics/javascript.test.ts b/app/src/features/codestudio/diagnostics/javascript.test.ts new file mode 100644 index 000000000..a5adb33b1 --- /dev/null +++ b/app/src/features/codestudio/diagnostics/javascript.test.ts @@ -0,0 +1,384 @@ +// ESLint-parity known answers (ADR 0074): each input is an "incorrect code" example from ESLint's documentation of +// that rule, kept with its `/*eslint*/` line, and the expected lines are the ones the rule's documentation describes +// as reported. The "correct code" examples must give nothing for the rule. + +import { parser } from '@lezer/javascript'; +import { describe, expect, it } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { checkJavaScript } from './javascript'; +import { JS_GLOBALS, RUNTIME_GLOBALS } from './jsGlobals'; +import { lineAt, lineStarts } from './text'; + +const ts = parser.configure({ dialect: 'ts' }); + +function lines(source: string, code: string, typescript = false): number[] { + const starts = lineStarts(source); + const tree = (typescript ? ts : parser).parse(source); + return checkJavaScript(source, tree) + .filter((f) => f.code === code) + .map((f) => lineAt(starts, f.from)); +} + +const join = (...rows: string[]) => rows.join('\n') + '\n'; + +describe('JavaScript checker: ESLint examples for names', () => { + it('no-undef', () => { + const source = join('/*eslint no-undef: "error"*/', '', 'const foo = someFunction();', 'const bar = a + 1;'); + expect(lines(source, 'no-undef')).toEqual([3, 4]); + const correct = join( + '/*global someFunction, a*/', + '/*eslint no-undef: "error"*/', + '', + 'const foo = someFunction();', + 'const bar = a + 1;', + ); + expect(lines(correct, 'no-undef')).toEqual([]); + expect( + lines(join('if (typeof UndefinedIdentifier === "undefined") {', ' // do something ...', '}'), 'no-undef'), + ).toEqual([]); + }); + + it('no-unused-vars', () => { + const source = join( + '/*eslint no-unused-vars: "error"*/', + '/*global some_unused_var*/', + '', + '// It checks variables you have defined as global', + 'some_unused_var = 42;', + '', + 'let x;', + '', + '// Write-only variables are not considered as used.', + 'let y = 10;', + 'y = 5;', + '', + '// A read for a modification of itself is not considered as used.', + 'let z = 0;', + 'z = z + 1;', + '', + '// By default, unused arguments cause warnings.', + '(function(foo) {', + ' return 5;', + '})();', + '', + '// Unused recursive functions also cause warnings.', + 'function fact(n) {', + ' if (n < 2) return 1;', + ' return n * fact(n - 1);', + '}', + '', + '// When a function definition destructures an array, unused entries from the array also cause warnings.', + 'function getY([x, y]) {', + ' return y;', + '}', + 'getY(["a", "b"]);', + ); + expect(lines(source, 'no-unused-vars')).toEqual([5, 7, 11, 15, 18, 23, 29]); + const correct = join( + '/*eslint no-unused-vars: "error"*/', + '', + 'const x = 10;', + 'alert(x);', + '', + '// foo is considered used here', + 'myFunc(function foo() {', + ' // ...', + '}.bind(this));', + '', + '(function(foo) {', + ' return foo;', + '})();', + '', + 'var myFunc;', + 'myFunc = setTimeout(function() {', + ' // myFunc is considered used', + ' myFunc();', + '}, 50);', + '', + '// Only the second argument from the destructured array is used.', + 'function getY([, y]) {', + ' return y;', + '}', + 'getY(["a", "b"]);', + ); + expect(lines(correct, 'no-unused-vars')).toEqual([]); + }); +}); + +describe('JavaScript checker: ESLint examples for constants and reachability', () => { + it('no-const-assign', () => { + expect(lines(join('/*eslint no-const-assign: "error"*/', '', 'const a = 0;', 'a = 1;'), 'no-const-assign')).toEqual( + [4], + ); + expect( + lines(join('/*eslint no-const-assign: "error"*/', '', 'const a = 0;', 'a += 1;'), 'no-const-assign'), + ).toEqual([4]); + expect(lines(join('/*eslint no-const-assign: "error"*/', '', 'const a = 0;', '++a;'), 'no-const-assign')).toEqual([ + 4, + ]); + expect( + lines( + join( + '/*eslint no-const-assign: "error"*/', + '', + 'for (const a in [1, 2, 3]) { // `a` is re-defined (not modified) on each loop step.', + ' a = 1;', + '}', + ), + 'no-const-assign', + ), + ).toEqual([4]); + expect( + lines( + join('const a = 0;', 'console.log(a);', 'for (const a in [1, 2, 3]) {', ' console.log(a);', '}'), + 'no-const-assign', + ), + ).toEqual([]); + }); + + it('no-unreachable', () => { + const source = join( + '/*eslint no-unreachable: "error"*/', + '', + 'function foo() {', + ' return true;', + ' console.log("done");', + '}', + '', + 'function bar() {', + ' throw new Error("Oops!");', + ' console.log("done");', + '}', + '', + 'while(value) {', + ' break;', + ' console.log("done");', + '}', + '', + 'throw new Error("Oops!");', + 'console.log("done");', + '', + 'function baz() {', + ' if (Math.random() < 0.5) {', + ' return;', + ' } else {', + ' throw new Error();', + ' }', + ' console.log("done");', + '}', + '', + 'for (;;) {}', + 'console.log("done");', + ); + // The `throw` on line 18 already makes lines 30 and 31 unreachable, so their run is marked from line 30. + expect(lines(source, 'no-unreachable')).toEqual([5, 10, 15, 19, 27, 30]); + expect(lines(join('for (;;) {}', 'console.log("done");'), 'no-unreachable')).toEqual([2]); + expect( + lines( + join( + 'function foo() {', + ' return bar();', + ' function bar() {', + ' return 1;', + ' }', + '}', + 'while (true) {', + ' if (x) break;', + '}', + 'console.log("done");', + ), + 'no-unreachable', + ), + ).toEqual([]); + }); +}); + +describe('JavaScript checker: ESLint examples for comparisons and conditions', () => { + it('eqeqeq', () => { + const source = join( + '/*eslint eqeqeq: "error"*/', + '', + 'if (x == 42) { }', + '', + 'if ("" == text) { }', + '', + 'if (obj.getStuff() != undefined) { }', + ); + expect(lines(source, 'eqeqeq')).toEqual([3, 5, 7]); + expect(lines(join('if (x === 42) { }', 'if (a !== b) { }'), 'eqeqeq')).toEqual([]); + }); + + it('no-cond-assign', () => { + const source = join( + '/*eslint no-cond-assign: "error"*/', + '', + '// Unintentional assignment', + 'let x;', + 'if (x = 0) {', + ' const b = 1;', + '}', + '', + '// Practical example that is similar to an error', + 'const setHeight = function (someNode) {', + ' do {', + ' someNode.height = "100px";', + ' } while (someNode = someNode.parentNode);', + '}', + ); + expect(lines(source, 'no-cond-assign')).toEqual([5, 13]); + const correct = join( + 'let x;', + 'if (x === 0) {', + ' const b = 1;', + '}', + 'const setHeight = function (someNode) {', + ' do {', + ' someNode.height = "100px";', + ' } while ((someNode = someNode.parentNode));', + '}', + ); + expect(lines(correct, 'no-cond-assign')).toEqual([]); + }); +}); + +describe('JavaScript checker: ESLint examples for switch cases', () => { + it('no-fallthrough', () => { + const source = join( + '/*eslint no-fallthrough: "error"*/', + '', + 'switch(foo) {', + ' case 1:', + ' doSomething();', + '', + ' case 2:', + ' doSomething();', + '}', + ); + expect(lines(source, 'no-fallthrough')).toEqual([7]); + const correct = join( + 'switch(foo) {', + ' case 1:', + ' doSomething();', + ' break;', + ' case 2:', + ' doSomething();', + '}', + 'function bar(foo) {', + ' switch(foo) {', + ' case 1:', + ' doSomething();', + ' return;', + ' case 2:', + ' doSomething();', + ' }', + '}', + 'switch(foo) {', + ' case 1:', + ' doSomething();', + ' // falls through', + ' case 2:', + ' doSomething();', + '}', + 'switch(foo) {', + ' case 1:', + ' case 2:', + ' doSomething();', + '}', + ); + expect(lines(correct, 'no-fallthrough')).toEqual([]); + }); +}); + +describe('JavaScript checker: scopes', () => { + it('handles imports, exports, classes, closures, and patterns', () => { + const source = join( + "import d, { a as b, c } from 'm';", + 'export const k = d + b;', + 'export function run({ v }, ...rest) {', + ' const f = (w) => w + v + rest.length;', + ' class C { m(q) { return q + c; } }', + ' try { return f(new C()); } catch (e) { return e; }', + '}', + 'const o = { k };', + 'export { o };', + ); + const found = checkJavaScript(source, parser.parse(source)).filter((f) => f.code.startsWith('no-')); + expect(found).toEqual([]); + }); + + it('skips TypeScript types for no-undef and counts them as uses', () => { + const source = join( + "import type { Shape } from './shape';", + 'interface Box { size: Size; }', + 'function area(s: Shape, b: Box): number { return s.w * b.size; }', + 'console.log(area);', + ); + expect(lines(source, 'no-undef', true)).toEqual([]); + expect(lines(source, 'no-unused-vars', true)).toEqual([]); + }); + + it('knows the test API the app injects (ADR 0075)', () => { + const source = join( + 'function add(a, b) { return a + b; }', + 'let n = 0;', + 'beforeAll(() => {}); afterAll(() => {});', + 'beforeEach(() => { n += 1; }); afterEach(() => {});', + "describe('add', () => {", + " it('adds', () => { expect(add(1, 2)).toBe(3); });", + " test('negates', () => { expect(add(1, -1)).not.toBe(1); });", + '});', + ); + expect(lines(source, 'no-undef')).toEqual([]); + expect(lines(source, 'no-undef', true)).toEqual([]); + }); + + it('knows every global the JavaScript sandbox injects', () => { + // The test reads the sandbox's source as text: the names it sets with setProp(vm.global, ...). + const sandboxSource = readFileSync(new URL('../../codecells/sandbox/javascript.ts', import.meta.url), 'utf8'); + const injected = [...sandboxSource.matchAll(/setProp\(vm\.global,\s*'(\w+)'/g)].map((m) => m[1]); + expect(injected.sort()).toEqual(['console', 'print']); + for (const name of injected) { + expect(RUNTIME_GLOBALS).toContain(name); + expect(JS_GLOBALS.has(name)).toBe(true); + } + expect(lines(join('print("hello");'), 'no-undef')).toEqual([]); + expect(lines(join('print("hello");'), 'no-undef', true)).toEqual([]); + }); + + it('knows every global the studio JavaScript runtime sets', () => { + // The test reads the runtime's sources as text: the names they set with `globalThis.name =`. + const read = (file: string) => readFileSync(new URL(`../runtimes/js/${file}`, import.meta.url), 'utf8'); + const source = read('environment.ts') + read('engine.ts'); + const injected = [...new Set([...source.matchAll(/globalThis\.(\w+)\s*=/g)].map((m) => m[1]))]; + expect(injected.sort()).toEqual([ + 'console', + 'exports', + 'module', + 'print', + 'process', + 'prompt', + 'readline', + 'require', + ]); + for (const name of injected) { + expect(RUNTIME_GLOBALS).toContain(name); + expect(JS_GLOBALS.has(name)).toBe(true); + } + const program = join('const name = readline();', 'process.stdout.write(name);', 'module.exports = { name };'); + expect(lines(program, 'no-undef')).toEqual([]); + expect(lines(program, 'no-undef', true)).toEqual([]); + }); + + // @typescript-eslint/no-unused-vars skips the parameters of a signature without a body; no-undef knows the name a + // `declare function` adds (typescript-eslint documentation). + it('leaves the parameters of overloads and abstract methods alone, and knows declared functions', () => { + const overload = join('function p(a: number): number;', 'function p(a: any) { return a; }', 'p(1);'); + expect(lines(overload, 'no-unused-vars', true)).toEqual([]); + const abstract = join('abstract class A { abstract m(b: number): void; }', 'export { A };'); + expect(lines(abstract, 'no-unused-vars', true)).toEqual([]); + const ambient = join('declare function d(c: string): void;', 'd("x");'); + expect(lines(ambient, 'no-undef', true)).toEqual([]); + expect(lines(ambient, 'no-unused-vars', true)).toEqual([]); + expect(lines(join('class Q { m(s?: number): void {} }', 'new Q().m();'), 'no-unused-vars', true)).toEqual([1]); + }); +}); diff --git a/app/src/features/codestudio/diagnostics/javascript.ts b/app/src/features/codestudio/diagnostics/javascript.ts new file mode 100644 index 000000000..350baca67 --- /dev/null +++ b/app/src/features/codestudio/diagnostics/javascript.ts @@ -0,0 +1,482 @@ +// OpenNote's JavaScript and TypeScript name checker over the Lezer tree (layer L-b, DESIGN 6.1). It uses ESLint's +// rule names and default options as ESLint documents them: +// - no-undef, where `typeof` is allowed; +// - no-unused-vars with vars "all", args "after-used", and caughtErrors "all". A write is not a use, and neither is a +// read that only feeds the variable's own update. A function that only calls itself is unused; +// - no-const-assign. +// `/*global a, b*/` comments declare names, as in ESLint. The shape rules live in jsFlow.ts. + +import type { SyntaxNode, Tree } from '@lezer/common'; +import { checkFlow } from './jsFlow'; +import { JS_GLOBALS } from './jsGlobals'; +import type { Finding } from './types'; + +type Kind = 'var' | 'let' | 'const' | 'function' | 'class' | 'param' | 'catch' | 'import' | 'global' | 'quiet'; + +interface Binding { + name: string; + from: number; + to: number; + kind: Kind; + /** Set when the declaration gives a value, for "assigned" in the message. */ + init: boolean; + /** References inside this range are the binding's own (a function calling itself). */ + self: [number, number] | null; + reads: number; + writes: { from: number; to: number }[]; + exported: boolean; +} + +interface Scope { + parent: Scope | null; + /** Function scopes hold `var` and function names; block scopes hold `let`, `const`, and classes. */ + fn: boolean; + names: Map; +} + +interface Ref { + name: string; + from: number; + to: number; + scope: Scope; + read: boolean; + write: boolean; + /** `x = x + 1` or `x++` as a statement: the read only feeds the variable's own update. */ + selfOnly: boolean; + typeOf: boolean; + type: boolean; +} + +interface Params { + list: Binding[]; +} + +const FUNCTION_VALUES = /^(?:FunctionExpression|ArrowFunction|ClassExpression)$/; + +function kids(node: SyntaxNode): SyntaxNode[] { + const out: SyntaxNode[] = []; + for (let c = node.firstChild; c; c = c.nextSibling) out.push(c); + return out; +} + +class JsChecker { + readonly findings: Finding[] = []; + private readonly refs: Ref[] = []; + private readonly bindings: Binding[] = []; + private readonly params: Params[] = []; + private readonly top: Scope = { parent: null, fn: true, names: new Map() }; + private selfTarget: string | null = null; + + constructor(private readonly text: string) {} + + private slice(node: { from: number; to: number }): string { + return this.text.slice(node.from, node.to); + } + + private fnScope(scope: Scope): Scope { + let s = scope; + while (!s.fn && s.parent) s = s.parent; + return s; + } + + private declare( + scope: Scope, + node: { from: number; to: number }, + kind: Kind, + init: boolean, + self: [number, number] | null = null, + ): Binding { + const name = this.slice(node); + const target = kind === 'var' || kind === 'function' ? (kind === 'var' ? this.fnScope(scope) : scope) : scope; + const existing = target.names.get(name); + if (existing) return existing; + const binding: Binding = { + name, + from: node.from, + to: node.to, + kind, + init, + self, + reads: 0, + writes: [], + exported: false, + }; + target.names.set(name, binding); + this.bindings.push(binding); + return binding; + } + + private ref(scope: Scope, node: SyntaxNode, extra: Partial = {}): void { + const name = this.slice(node); + this.refs.push({ + name, + from: node.from, + to: node.to, + scope, + read: true, + write: false, + selfOnly: this.selfTarget === name, + typeOf: false, + type: false, + ...extra, + }); + } + + run(tree: Tree): void { + this.globalComments(tree); + this.visitAll(kids(tree.topNode), this.top); + this.resolve(); + this.report(); + } + + private globalComments(tree: Tree): void { + for (const comment of tree.topNode.getChildren('BlockComment')) { + const match = /^\/\*\s*(globals?|exported)\s+([\s\S]*?)\*\/$/.exec(this.slice(comment)); + if (!match) continue; + for (const entry of match[2].split(',')) { + const name = entry.split(':')[0].trim(); + if (!/^[\w$]+$/.test(name)) continue; + if (match[1] === 'exported') { + const b = this.top.names.get(name); + if (b) b.exported = true; + continue; + } + const at = comment.from + this.slice(comment).indexOf(name); + this.declare(this.top, { from: at, to: at + name.length }, 'global', false); + } + } + } + + private visitAll(nodes: SyntaxNode[], scope: Scope): void { + for (const node of nodes) this.visit(node, scope); + } + + private visit(node: SyntaxNode, scope: Scope): void { + switch (node.name) { + case 'VariableName': + return this.ref(scope, node); + case 'TypeName': + return this.ref(scope, node, { type: true }); + case 'TypeDefinition': + this.declare(scope, node, 'quiet', true); + return; + case 'VariableDeclaration': + return this.declaration(node, scope); + case 'ForOfSpec': + case 'ForInSpec': + return this.declaration(node, scope); + case 'ForStatement': { + const inner: Scope = { parent: scope, fn: false, names: new Map() }; + return this.visitAll(kids(node), inner); + } + case 'FunctionDeclaration': + case 'FunctionExpression': + case 'ArrowFunction': + case 'MethodDeclaration': + return this.functionNode(node, scope); + case 'ClassDeclaration': + case 'ClassExpression': + return this.classNode(node, scope); + case 'Block': + return this.visitAll(kids(node), { parent: scope, fn: false, names: new Map() }); + case 'CatchClause': + return this.catchClause(node, scope); + case 'ImportDeclaration': + return this.importNode(node, scope); + case 'ExportDeclaration': + return this.exportNode(node, scope); + case 'AssignmentExpression': + return this.assignment(node, scope); + case 'PostfixExpression': + case 'UnaryExpression': + return this.unary(node, scope); + case 'Property': + return this.property(node, scope); + case 'NamespaceDeclaration': + case 'EnumDeclaration': + case 'AmbientDeclaration': + return this.quiet(node, scope); + case 'PropertyName': + case 'PropertyDefinition': + case 'PrivatePropertyDefinition': + case 'Label': + return; + default: + return this.visitAll(kids(node), scope); + } + } + + /** Declarations people do not expect a warning for: namespaces, enums, and `declare`. */ + private quiet(node: SyntaxNode, scope: Scope): void { + for (const child of kids(node)) { + if (child.name === 'VariableDefinition' || child.name === 'TypeDefinition') + this.declare(scope, child, 'quiet', true); + else if (child.name === 'AmbientFunctionDeclaration') { + // `declare function d(c: string): void;` declares `d`; its parameters are types only. + const name = child.getChild('VariableDefinition'); + if (name) this.declare(scope, name, 'quiet', true); + } else if (child.name === 'VariableDeclaration') { + for (const d of child.getChildren('VariableDefinition')) this.declare(scope, d, 'quiet', true); + } else this.visit(child, scope); + } + } + + /** Binds every name in a pattern; defaults and computed keys are reads. */ + private pattern(node: SyntaxNode, scope: Scope, kind: Kind, init: boolean, out: Binding[]): void { + switch (node.name) { + case 'VariableDefinition': + out.push(this.declare(scope, node, kind, init)); + return; + case 'ArrayPattern': + case 'ObjectPattern': + case 'PatternProperty': + for (const child of kids(node)) { + if (child.prevSibling?.name === 'Equals' || child.name === 'TypeAnnotation') this.visit(child, scope); + else if (child.name !== 'PropertyName') this.pattern(child, scope, kind, init, out); + } + return; + default: + if (/^[[\](){}:,=.]|^Spread$|^Equals$/.test(node.name)) return; + this.visit(node, scope); + } + } + + private declaration(node: SyntaxNode, scope: Scope): void { + const parts = kids(node).filter((p) => p.name !== '(' && p.name !== ')'); + const keyword = parts[0]?.name; + const kind: Kind = keyword === 'const' ? 'const' : keyword === 'let' ? 'let' : 'var'; + const loop = node.name !== 'VariableDeclaration'; + if (loop && keyword !== 'var' && keyword !== 'let' && keyword !== 'const') { + // `for (x of list)`: the loop writes an existing name. + if (parts[0]) this.writeTarget(parts[0], scope, false); + return this.visitAll(parts.slice(1), scope); + } + for (let i = 0; i < parts.length; i += 1) { + const part = parts[i]; + if (part.name === 'VariableDefinition' || part.name === 'ArrayPattern' || part.name === 'ObjectPattern') { + let j = i + 1; + while (parts[j]?.name === 'TypeAnnotation' || parts[j]?.name === 'Optional') this.visit(parts[j++], scope); + const value = parts[j]?.name === 'Equals' ? parts[j + 1] : undefined; + const self: [number, number] | null = value && FUNCTION_VALUES.test(value.name) ? [value.from, value.to] : null; + const made: Binding[] = []; + this.pattern(part, scope, kind, !!value || loop, made); + if (self) made.forEach((b) => (b.self = self)); + } else if (part.name !== 'TypeAnnotation' && part.name !== 'Optional') this.visit(part, scope); + } + } + + /** A signature without a body (an overload, an abstract method) declares its parameters without using them. */ + private paramList(list: SyntaxNode, inner: Scope, bodiless = false): void { + const record: Params = { list: [] }; + this.params.push(record); + for (const child of kids(list)) { + if (child.name === 'VariableDefinition') { + const b = this.declare(inner, child, bodiless ? 'quiet' : 'param', true); + if (child.prevSibling?.name === 'Privacy' || child.prevSibling?.name === 'readonly') b.exported = true; + record.list.push(b); + } else if (child.name === 'ArrayPattern' || child.name === 'ObjectPattern') { + const made: Binding[] = []; + this.pattern(child, inner, 'let', true, made); + } else if ( + child.prevSibling?.name === 'Equals' || + child.name === 'TypeAnnotation' || + child.name === 'Decorator' + ) { + this.visit(child, inner); + } + } + } + + private functionNode(node: SyntaxNode, scope: Scope): void { + const inner: Scope = { parent: scope, fn: true, names: new Map() }; + const bodiless = node.name !== 'ArrowFunction' && !node.getChild('Block'); + for (const child of kids(node)) { + if (child.name === 'VariableDefinition' && child.nextSibling?.name !== 'Arrow') { + if (node.name === 'FunctionDeclaration') this.declare(scope, child, 'function', true, [node.from, node.to]); + else this.declare(inner, child, 'quiet', true); + } else if (child.name === 'VariableDefinition') { + this.params.push({ list: [this.declare(inner, child, 'param', true)] }); + } else if (child.name === 'ParamList') this.paramList(child, inner, bodiless); + else if (child.name === 'Block') this.visitAll(kids(child), inner); + else this.visit(child, inner); + } + } + + private classNode(node: SyntaxNode, scope: Scope): void { + const inner: Scope = { parent: scope, fn: false, names: new Map() }; + for (const child of kids(node)) { + if (child.name === 'VariableDefinition') { + if (node.name === 'ClassDeclaration') this.declare(scope, child, 'class', true, [node.from, node.to]); + else this.declare(inner, child, 'quiet', true); + } else this.visit(child, inner); + } + } + + private catchClause(node: SyntaxNode, scope: Scope): void { + const inner: Scope = { parent: scope, fn: false, names: new Map() }; + for (const child of kids(node)) { + if (child.name === 'VariableDefinition' || child.name === 'ArrayPattern' || child.name === 'ObjectPattern') { + this.pattern(child, inner, 'catch', true, []); + } else if (child.name === 'Block') this.visitAll(kids(child), inner); + else this.visit(child, inner); + } + } + + private importNode(node: SyntaxNode, scope: Scope): void { + const typeOnly = node.getChild('type') !== null; + const walk = (n: SyntaxNode) => { + for (const child of kids(n)) { + if (child.name === 'VariableDefinition') this.declare(scope, child, typeOnly ? 'quiet' : 'import', true); + else if (child.name === 'ImportGroup') walk(child); + } + }; + walk(node); + } + + private exportNode(node: SyntaxNode, scope: Scope): void { + const group = node.getChild('ExportGroup'); + if (group) { + if (node.getChild('from')) return; + for (const child of kids(group)) { + if (child.name === 'VariableName' && child.prevSibling?.name !== 'as') this.ref(scope, child); + } + return; + } + const before = this.bindings.length; + this.visitAll(kids(node), scope); + const own = new Set(scope.names.values()); + for (const b of this.bindings.slice(before)) if (own.has(b)) b.exported = true; + } + + private statementLevel(node: SyntaxNode): boolean { + let parent = node.parent; + while (parent?.name === 'ParenthesizedExpression') parent = parent.parent; + return parent?.name === 'ExpressionStatement'; + } + + private writeTarget(node: SyntaxNode, scope: Scope, read: boolean): void { + if (node.name === 'VariableName') return this.ref(scope, node, { write: true, read, selfOnly: read }); + if (node.name === 'ArrayPattern' || node.name === 'ObjectPattern' || node.name === 'PatternProperty') { + for (const child of kids(node)) { + if (child.prevSibling?.name === 'Equals') this.visit(child, scope); + else if (child.name !== 'PropertyName') this.writeTarget(child, scope, false); + } + return; + } + if (node.name === 'ParenthesizedExpression') return kids(node).forEach((c) => this.writeTarget(c, scope, read)); + this.visit(node, scope); + } + + private assignment(node: SyntaxNode, scope: Scope): void { + const [target, op, ...rest] = kids(node); + if (!target) return; + const compound = op?.name === 'UpdateOp'; + const own = this.statementLevel(node) && target.name === 'VariableName'; + this.writeTarget(target, scope, compound); + if (compound && !own) this.refs[this.refs.length - 1].selfOnly = false; + const outer = this.selfTarget; + this.selfTarget = own ? this.slice(target) : null; + this.visitAll(rest, scope); + this.selfTarget = outer; + } + + private unary(node: SyntaxNode, scope: Scope): void { + const parts = kids(node); + const update = parts.find((p) => p.name === 'ArithOp' && /^(?:\+\+|--)$/.test(this.slice(p))); + const operand = parts.find((p) => p.name === 'VariableName'); + if (update && operand) { + this.ref(scope, operand, { write: true, selfOnly: this.statementLevel(node) }); + return; + } + if (parts[0]?.name === 'typeof' && operand && parts.length === 2) { + this.ref(scope, operand, { typeOf: true }); + return; + } + this.visitAll(parts, scope); + } + + private property(node: SyntaxNode, scope: Scope): void { + const parts = kids(node); + if (parts.length === 1 && parts[0].name === 'PropertyDefinition') return this.ref(scope, parts[0]); + if (node.getChild('ParamList')) return this.functionNode(node, scope); + this.visitAll(parts, scope); + } + + private lookup(ref: Ref): Binding | null { + for (let s: Scope | null = ref.scope; s; s = s.parent) { + const b = s.names.get(ref.name); + if (b) return b; + } + return null; + } + + private resolve(): void { + for (const ref of this.refs) { + const b = this.lookup(ref); + if (!b) { + if (!ref.type && !ref.typeOf && !JS_GLOBALS.has(ref.name) && ref.name !== 'arguments') { + this.findings.push({ + from: ref.from, + to: ref.to, + severity: 'error', + code: 'no-undef', + message: 'undefinedVar', + params: { name: ref.name }, + }); + } + continue; + } + if (ref.write) { + b.writes.push({ from: ref.from, to: ref.to }); + if (b.kind === 'const') { + this.findings.push({ + from: ref.from, + to: ref.to, + severity: 'error', + code: 'no-const-assign', + message: 'constAssign', + params: { name: ref.name }, + }); + } + } + const inSelf = b.self !== null && b.self[0] <= ref.from && ref.to <= b.self[1]; + if ((ref.read && !ref.selfOnly && !inSelf) || ref.type) b.reads += 1; + } + } + + private report(): void { + const afterUsed = new Set(); + for (const { list } of this.params) { + const last = list.map((b) => b.reads > 0).lastIndexOf(true); + list.slice(0, last + 1).forEach((b) => afterUsed.add(b)); + } + for (const b of this.bindings) { + if (b.reads > 0 || b.exported || b.kind === 'quiet' || afterUsed.has(b) || b.name.length === 0) continue; + const at = b.writes[b.writes.length - 1] ?? b; + const assigned = b.init || b.writes.length > 0; + this.findings.push({ + from: at.from, + to: at.to, + severity: 'warning', + code: 'no-unused-vars', + message: 'unusedVar', + params: { + name: b.name, + kind: + assigned && b.kind !== 'function' && b.kind !== 'class' && b.kind !== 'param' && b.kind !== 'import' + ? 'assigned' + : 'defined', + }, + }); + } + } +} + +/** The JavaScript or TypeScript checker's findings for a parsed file. */ +export function checkJavaScript(text: string, tree: Tree): Finding[] { + const checker = new JsChecker(text); + checker.run(tree); + return [...checker.findings, ...checkFlow(text, tree)].sort( + (a, b) => a.from - b.from || a.code.localeCompare(b.code), + ); +} diff --git a/app/src/features/codestudio/diagnostics/jsFlow.ts b/app/src/features/codestudio/diagnostics/jsFlow.ts new file mode 100644 index 000000000..e44f234c3 --- /dev/null +++ b/app/src/features/codestudio/diagnostics/jsFlow.ts @@ -0,0 +1,225 @@ +// The JavaScript and TypeScript rules that read the shape of the code rather than its names, with ESLint's rule +// names and default options as ESLint documents them: no-unreachable, eqeqeq ("always"), no-cond-assign +// ("except-parens"), and no-fallthrough (a comment matching /falls?\s?through/i allows it). + +import type { SyntaxNode, Tree } from '@lezer/common'; +import type { Finding } from './types'; + +const SKIP = new Set([';', '{', '}', ':', 'LineComment', 'BlockComment', '⚠']); +const FALLS_THROUGH = /falls?\s?through/i; + +function kids(node: SyntaxNode): SyntaxNode[] { + const out: SyntaxNode[] = []; + for (let c = node.firstChild; c; c = c.nextSibling) out.push(c); + return out; +} + +function statements(node: SyntaxNode): SyntaxNode[] { + return kids(node).filter((c) => !SKIP.has(c.name) && !c.type.isError); +} + +export class FlowRules { + readonly findings: Finding[] = []; + + constructor(private readonly text: string) {} + + run(tree: Tree): void { + const cursor = tree.cursor(); + do { + const node = cursor.node; + switch (node.name) { + case 'Script': + case 'Block': + this.unreachable(statements(node)); + break; + case 'SwitchBody': + this.switchBody(node); + break; + case 'BinaryExpression': + this.eqeqeq(node); + break; + case 'IfStatement': + case 'WhileStatement': + case 'DoStatement': + this.condition(node.getChild('ParenthesizedExpression'), 2); + break; + case 'ForSpec': + this.condition(this.forTest(node), 1); + break; + case 'ConditionalExpression': + this.condition(node.firstChild, 1); + break; + } + } while (cursor.next()); + } + + private add( + from: number, + to: number, + severity: Finding['severity'], + code: string, + message: Finding['message'], + params?: Finding['params'], + ): void { + this.findings.push({ from, to, severity, code, message, params }); + } + + /** True when control never reaches the statement after `node`. */ + terminates(node: SyntaxNode): boolean { + switch (node.name) { + case 'ReturnStatement': + case 'ThrowStatement': + case 'BreakStatement': + case 'ContinueStatement': + return true; + case 'Block': + return statements(node).some((s) => this.terminates(s)); + case 'IfStatement': { + const branches = statements(node).filter((c) => !['if', 'else', 'ParenthesizedExpression'].includes(c.name)); + return node.getChild('else') !== null && branches.length === 2 && branches.every((b) => this.terminates(b)); + } + case 'TryStatement': { + const block = node.getChild('Block'); + const handler = node.getChild('CatchClause')?.getChild('Block'); + const final = node.getChild('FinallyClause')?.getChild('Block'); + if (final && this.terminates(final)) return true; + return !!block && this.terminates(block) && (!handler || this.terminates(handler)); + } + case 'ForStatement': + case 'WhileStatement': + case 'DoStatement': + return this.infinite(node) && !this.breaksOut(node, null); + case 'LabeledStatement': { + const inner = node.lastChild; + const label = node.getChild('Label'); + return !!inner && this.infinite(inner) && !this.breaksOut(inner, label ? this.slice(label) : null); + } + default: + return false; + } + } + + private slice(node: SyntaxNode): string { + return this.text.slice(node.from, node.to); + } + + private forTest(spec: SyntaxNode): SyntaxNode | null { + // ForSpec reads: init ";" test ";" update. The test is what sits between the first and second semicolon. + const parts = kids(spec); + const semis = parts.flatMap((p, i) => + p.name === ';' || (p.name === 'VariableDeclaration' && this.slice(p).endsWith(';')) ? [i] : [], + ); + if (semis.length < 2) return null; + const between = parts.slice(semis[0] + 1, semis[1]).filter((p) => p.name !== ';'); + return between[0] ?? null; + } + + private infinite(loop: SyntaxNode): boolean { + if (loop.name === 'ForStatement') { + const spec = loop.getChild('ForSpec'); + return !!spec && this.forTest(spec) === null; + } + if (loop.name === 'WhileStatement' || loop.name === 'DoStatement') { + const test = loop.getChild('ParenthesizedExpression'); + return !!test && /^\(\s*true\s*\)$/.test(this.slice(test)); + } + return false; + } + + /** True when a `break` inside `loop` leaves it: an unlabeled one outside inner loops and switches, or `break label`. */ + private breaksOut(loop: SyntaxNode, label: string | null): boolean { + const walk = (node: SyntaxNode, nested: boolean): boolean => { + for (const child of kids(node)) { + if (child.name === 'BreakStatement') { + const target = child.getChild('Label'); + if (target ? this.slice(target) === label : !nested) return true; + } + if (/^(?:Function|Arrow|Class|Method)/.test(child.name)) continue; + const inner = nested || /^(?:For|While|Do|Switch)Statement$/.test(child.name); + if (walk(child, inner)) return true; + } + return false; + }; + return walk(loop, false); + } + + private isUnreachableKind(node: SyntaxNode): boolean { + if (node.name === 'FunctionDeclaration' || node.name === 'EmptyStatement') return false; + if (node.name === 'VariableDeclaration' && node.firstChild?.name === 'var') return node.getChild('Equals') !== null; + return !/Declaration$/.test(node.name) || node.name === 'VariableDeclaration' || node.name === 'ClassDeclaration'; + } + + /** no-unreachable: one mark over each run of statements after one that never completes. */ + private unreachable(list: SyntaxNode[]): void { + let dead = false; + const run = { from: -1, to: -1 }; + const flush = () => { + if (run.from >= 0) this.add(run.from, run.to, 'warning', 'no-unreachable', 'unreachable'); + run.from = -1; + }; + for (const node of list) { + if (node.name === 'CaseLabel' || node.name === 'DefaultLabel') { + flush(); + dead = false; + continue; + } + if (dead && this.isUnreachableKind(node)) { + if (run.from < 0) run.from = node.from; + run.to = node.to; + } else if (dead) flush(); + if (!dead && this.terminates(node)) dead = true; + } + flush(); + } + + private switchBody(body: SyntaxNode): void { + const items = kids(body).filter((c) => !['{', '}', '⚠'].includes(c.name)); + this.unreachable(items.filter((c) => c.name !== 'LineComment' && c.name !== 'BlockComment')); + let group: SyntaxNode[] = []; + let seenLabel = false; + for (const item of items) { + if (item.name === 'CaseLabel' || item.name === 'DefaultLabel') { + const code = group.filter((g) => g.name !== 'LineComment' && g.name !== 'BlockComment'); + const last = code[code.length - 1]; + const comment = group[group.length - 1]; + const allowed = + comment && comment !== last && /Comment$/.test(comment.name) && FALLS_THROUGH.test(this.slice(comment)); + if (seenLabel && last && !this.terminates(last) && !allowed) { + this.add(item.from, item.to, 'hint', 'no-fallthrough', 'fallthrough'); + } + seenLabel = true; + group = []; + } else group.push(item); + } + } + + private eqeqeq(node: SyntaxNode): void { + for (const op of kids(node)) { + const word = this.slice(op); + if (op.name === 'CompareOp' && (word === '==' || word === '!=')) { + this.add(op.from, op.to, 'hint', 'eqeqeq', 'eqeqeq', { op: word, better: `${word}=` }); + } + } + } + + /** + * no-cond-assign with "except-parens": an assignment in a test is reported unless it has parentheses of its own. + * `if`, `while`, and `do` tests already sit in parentheses, so they need a second pair; `for` and `?:` need one. + */ + private condition(test: SyntaxNode | null, depth: number): void { + let node = test; + let parens = 0; + while (node?.name === 'ParenthesizedExpression') { + parens += 1; + node = statements(node).find((c) => c.name !== '(' && c.name !== ')') ?? null; + } + if (node?.name === 'AssignmentExpression' && parens < depth) + this.add(node.from, node.to, 'warning', 'no-cond-assign', 'condAssign'); + } +} + +export function checkFlow(text: string, tree: Tree): Finding[] { + const rules = new FlowRules(text); + rules.run(tree); + return rules.findings; +} diff --git a/app/src/features/codestudio/diagnostics/jsGlobals.ts b/app/src/features/codestudio/diagnostics/jsGlobals.ts new file mode 100644 index 000000000..c5a12b9c3 --- /dev/null +++ b/app/src/features/codestudio/diagnostics/jsGlobals.ts @@ -0,0 +1,40 @@ +// Names a JavaScript program may use without declaring them: the value properties, functions, and constructors of +// the global object in the ECMAScript 2024 specification (section 19), plus the host names the app's runtimes give +// (console, timers, text encoding, URLs, and the like). no-undef accepts these. + +const ECMASCRIPT = + 'globalThis Infinity NaN undefined eval isFinite isNaN parseFloat parseInt decodeURI decodeURIComponent ' + + 'encodeURI encodeURIComponent escape unescape AggregateError Array ArrayBuffer BigInt BigInt64Array ' + + 'BigUint64Array Boolean DataView Date Error EvalError FinalizationRegistry Float32Array Float64Array Function ' + + 'Int8Array Int16Array Int32Array Map Number Object Promise Proxy RangeError ReferenceError RegExp Set ' + + 'SharedArrayBuffer String Symbol SyntaxError TypeError Uint8Array Uint8ClampedArray Uint16Array Uint32Array ' + + 'URIError WeakMap WeakRef WeakSet Atomics JSON Math Reflect Intl'; + +const HOST = + 'console setTimeout clearTimeout setInterval clearInterval queueMicrotask structuredClone TextEncoder ' + + 'TextDecoder URL URLSearchParams atob btoa performance crypto prompt alert'; + +/** + * The names the app's JavaScript runtimes set on the global object themselves. The code cell sandbox + * (features/codecells/sandbox/javascript.ts) sets `print` and `console`. The studio runtime (runtimes/js, code:L06) + * also sets `readline`, `prompt`, `process`, and the CommonJS `module`, `exports`, and `require`. Tests read both + * sources to keep this list true. + */ +export const RUNTIME_GLOBALS: readonly string[] = [ + 'console', + 'print', + 'readline', + 'prompt', + 'process', + 'module', + 'exports', + 'require', +]; + +/** The Jest-compatible test API the app injects into JavaScript and TypeScript programs (ADR 0075). */ +const TEST_API = 'describe it test expect beforeEach afterEach beforeAll afterAll'; + +export const JS_GLOBALS: ReadonlySet = new Set([ + ...`${ECMASCRIPT} ${HOST} ${TEST_API}`.split(' '), + ...RUNTIME_GLOBALS, +]); diff --git a/app/src/features/codestudio/diagnostics/lua.ts b/app/src/features/codestudio/diagnostics/lua.ts new file mode 100644 index 000000000..c56b968ec --- /dev/null +++ b/app/src/features/codestudio/diagnostics/lua.ts @@ -0,0 +1,305 @@ +// The Lua scope checker (layer L-b, DESIGN 6.1). Lua has no Lezer grammar in the app (the editor uses a legacy +// mode), so this reads tokens. It reports a read of a name that no `local` declares, that the file never assigns as +// a global, and that the Lua 5.4 standard library does not define, and a local that is never read. The codes are +// luacheck's: W113 accessing an undefined variable, W211 unused variable. + +import type { Finding } from './types'; + +interface Token { + kind: 'name' | 'keyword' | 'string' | 'number' | 'op'; + text: string; + from: number; + to: number; + line: number; +} + +interface Local { + name: string; + from: number; + to: number; + reads: number; + report: boolean; +} + +/** The global names of the Lua 5.4 reference manual, section 6, plus `unpack` and `arg`, which scripts expect. */ +const LUA_GLOBALS = new Set( + ( + '_G _VERSION assert collectgarbage dofile error getmetatable ipairs load loadfile next pairs pcall print ' + + 'rawequal rawget rawlen rawset require select setmetatable tonumber tostring type warn xpcall coroutine debug ' + + 'io math os package string table utf8 unpack arg' + ).split(' '), +); + +const KEYWORDS = new Set( + 'and break do else elseif end false for function goto if in local nil not or repeat return then true until while'.split( + ' ', + ), +); +const OPS = ['...', '..', '==', '~=', '<=', '>=', '//', '::', '<<', '>>']; +/** Token kinds that end an expression: a name right after one starts a new statement, even on the same line. */ +const ENDS_EXPRESSION = new Set(['name', 'string', 'number']); +const ENDS_EXPRESSION_TEXT = /^(?:[)\]}]|true|false|nil|\.\.\.)$/; +/** Keywords that start a statement, so a pending `local` on the same line is in scope after them. */ +const STATEMENT_KEYWORDS = /^(?:return|if|while|for|local|do|repeat|break|goto|end|else|elseif|until)$/; + +function longBracket(text: string, at: number): number { + const m = /^\[(=*)\[/.exec(text.slice(at, at + 64)); + if (!m) return -1; + const close = text.indexOf(`]${m[1]}]`, at + m[0].length); + return close < 0 ? text.length : close + m[1].length + 2; +} + +export function tokenizeLua(text: string): Token[] { + const tokens: Token[] = []; + let line = 1; + let i = 0; + const push = (kind: Token['kind'], from: number, to: number) => { + tokens.push({ kind, text: text.slice(from, to), from, to, line }); + for (let k = from; k < to; k += 1) if (text.charCodeAt(k) === 10) line += 1; + }; + while (i < text.length) { + const ch = text[i]; + if (ch === '\n') { + line += 1; + i += 1; + } else if (/\s/.test(ch)) i += 1; + else if (text.startsWith('--', i)) { + const end = longBracket(text, i + 2); + const stop = end >= 0 ? end : text.indexOf('\n', i) < 0 ? text.length : text.indexOf('\n', i); + for (let k = i; k < stop; k += 1) if (text.charCodeAt(k) === 10) line += 1; + i = stop; + } else if (ch === '"' || ch === "'") { + let j = i + 1; + while (j < text.length && text[j] !== ch && text[j] !== '\n') j += text[j] === '\\' ? 2 : 1; + push('string', i, (i = Math.min(text.length, j + 1))); + } else if (ch === '[' && longBracket(text, i) >= 0) { + push('string', i, (i = longBracket(text, i))); + } else if (/[A-Za-z_]/.test(ch)) { + const m = /^[A-Za-z_]\w*/.exec(text.slice(i, i + 256))![0]; + push(KEYWORDS.has(m) ? 'keyword' : 'name', i, (i += m.length)); + } else if (/\d/.test(ch) || (ch === '.' && /\d/.test(text[i + 1] ?? ''))) { + const m = /^(?:0[xX][\da-fA-F.]+(?:[pP][+-]?\d+)?|[\d.]+(?:[eE][+-]?\d+)?)/.exec(text.slice(i, i + 128))![0]; + push('number', i, (i += m.length)); + } else { + const op = OPS.find((o) => text.startsWith(o, i)) ?? ch; + push('op', i, (i += op.length)); + } + } + return tokens; +} + +class LuaChecker { + readonly findings: Finding[] = []; + private readonly scopes: Map[] = [new Map()]; + private readonly locals: Local[] = []; + private readonly globals = new Set(); + private readonly unresolved: Token[] = []; + /** Locals waiting for the end of their `local` statement, so `local x = x` reads the outer `x`. */ + private pending: { token: Token; report: boolean; scope: Map }[] = []; + private pendingLine = 0; + private braces = 0; + /** Loops whose `do` is still to come: a `for` has its scope already, a `while` gets one at `do`. */ + private readonly loops: ('for' | 'while')[] = []; + private repeatUntil = -1; + + constructor(private readonly tokens: Token[]) {} + + private declare(token: Token, report: boolean, scope = this.scopes[this.scopes.length - 1]): void { + const local: Local = { + name: token.text, + from: token.from, + to: token.to, + reads: 0, + report: report && token.text !== '_', + }; + scope.set(token.text, local); + this.locals.push(local); + } + + private flushPending(): void { + // A value can open a function scope (`local f = function() ... end`), so the names go into the scope the + // `local` statement stood in, never into the scope on top when the statement ends. + for (const p of this.pending) this.declare(p.token, p.report, p.scope); + this.pending = []; + } + + private lookup(name: string): Local | undefined { + for (let s = this.scopes.length - 1; s >= 0; s -= 1) { + const found = this.scopes[s].get(name); + if (found) return found; + } + return undefined; + } + + private statementStart(i: number): boolean { + const prev = this.tokens[i - 1]; + if (!prev) return true; + if (prev.text === ';' || (prev.kind === 'keyword' && /^(?:do|then|else|end|repeat|break|local)$/.test(prev.text))) + return true; + // No expression has a name right after a name, a literal, or a closing bracket. So `x = false\ny = 0` and the + // one-line bodies `function init() state = 1 end` and `do local v = 1 print(v) end` start statements there. + return ENDS_EXPRESSION.has(prev.kind) || ENDS_EXPRESSION_TEXT.test(prev.text); + } + + private startsStatement(i: number): boolean { + const tok = this.tokens[i]; + if (tok.kind === 'keyword') return STATEMENT_KEYWORDS.test(tok.text); + return tok.kind === 'name' && this.statementStart(i); + } + + /** True when the name at `i` starts `a, b = ...` or `a = ...`. */ + private assignmentTarget(i: number): boolean { + if (!this.statementStart(i)) return false; + let j = i + 1; + while (this.tokens[j]?.text === ',' && this.tokens[j + 1]?.kind === 'name') j += 2; + return this.tokens[j]?.text === '='; + } + + private params(i: number): number { + let j = i; + while (j < this.tokens.length && this.tokens[j].text !== '(') j += 1; + this.scopes.push(new Map()); + for (j += 1; j < this.tokens.length && this.tokens[j].text !== ')'; j += 1) { + if (this.tokens[j].kind === 'name') this.declare(this.tokens[j], false); + } + return j; + } + + run(): void { + const t = this.tokens; + for (let i = 0; i < t.length; i += 1) { + const tok = t[i]; + if (this.pending.length && (tok.line > this.pendingLine || tok.text === ';' || this.startsStatement(i))) { + this.flushPending(); + } + if (this.repeatUntil >= 0 && tok.line > this.repeatUntil) { + this.scopes.pop(); + this.repeatUntil = -1; + } + if (tok.text === '{') this.braces += 1; + else if (tok.text === '}') this.braces = Math.max(0, this.braces - 1); + if (tok.kind === 'keyword') { + i = this.keyword(i); + continue; + } + if (tok.kind !== 'name') continue; + const prev = t[i - 1]?.text; + const next = t[i + 1]?.text; + if (prev === '.' || prev === ':' || prev === '::' || prev === 'goto') continue; + if (this.braces > 0 && next === '=' && (prev === '{' || prev === ',' || prev === ';')) continue; + const local = this.lookup(tok.text); + if (this.assignmentTarget(i)) { + if (!local) this.globals.add(tok.text); + continue; + } + if (local) local.reads += 1; + else this.unresolved.push(tok); + } + this.flushPending(); + for (const tok of this.unresolved) { + if (!this.globals.has(tok.text) && !LUA_GLOBALS.has(tok.text)) { + this.findings.push({ + from: tok.from, + to: tok.to, + severity: 'warning', + code: 'W113', + message: 'luaUndefined', + params: { name: tok.text }, + }); + } + } + for (const l of this.locals) { + if (l.report && l.reads === 0) { + this.findings.push({ + from: l.from, + to: l.to, + severity: 'warning', + code: 'W211', + message: 'luaUnused', + params: { name: l.name }, + }); + } + } + this.findings.sort((a, b) => a.from - b.from); + } + + /** Handles a keyword at `i` and returns the index of the last token it used. */ + private keyword(i: number): number { + const t = this.tokens; + switch (t[i].text) { + case 'local': { + if (t[i + 1]?.text === 'function' && t[i + 2]?.kind === 'name') { + this.declare(t[i + 2], true); + return this.params(i + 3); + } + let j = i + 1; + const scope = this.scopes[this.scopes.length - 1]; + while (t[j]?.kind === 'name') { + this.pending.push({ token: t[j], report: true, scope }); + j += 1; + if (t[j]?.text === '<') j += 3; + if (t[j]?.text !== ',') break; + j += 1; + } + this.pendingLine = t[i].line; + return j - 1; + } + case 'function': { + let j = i + 1; + const name = t[j]; + if (name?.kind === 'name') { + const local = this.lookup(name.text); + const plain = t[j + 1]?.text === '('; + if (local) local.reads += plain ? 0 : 1; + else if (plain) this.globals.add(name.text); + else this.unresolved.push(name); + while (t[j + 1]?.text === '.' || t[j + 1]?.text === ':') j += 2; + } + const method = t[j]?.text !== undefined && t[j - 1]?.text === ':'; + const end = this.params(j); + if (method) this.declare({ ...t[j], text: 'self' }, false); + return end; + } + case 'for': { + this.scopes.push(new Map()); + this.loops.push('for'); + let j = i + 1; + while (t[j]?.kind === 'name') { + this.declare(t[j], false); + j += t[j + 1]?.text === ',' ? 2 : 1; + } + return j - 1; + } + case 'while': + this.loops.push('while'); + return i; + case 'do': + if (this.loops.pop() !== 'for') this.scopes.push(new Map()); + return i; + case 'then': + case 'repeat': + this.scopes.push(new Map()); + return i; + case 'elseif': + case 'else': + this.scopes.pop(); + if (t[i].text === 'else') this.scopes.push(new Map()); + return i; + case 'until': + this.repeatUntil = t[i].line; + return i; + case 'end': + if (this.scopes.length > 1) this.scopes.pop(); + return i; + default: + return i; + } + } +} + +/** The Lua checker's findings. */ +export function checkLua(text: string): Finding[] { + const checker = new LuaChecker(tokenizeLua(text)); + checker.run(); + return checker.findings; +} diff --git a/app/src/features/codestudio/diagnostics/parsers.ts b/app/src/features/codestudio/diagnostics/parsers.ts new file mode 100644 index 000000000..e13c01ef4 --- /dev/null +++ b/app/src/features/codestudio/diagnostics/parsers.ts @@ -0,0 +1,38 @@ +// The Lezer parser of each language that has a grammar, loaded on first use. On the page they come through the +// editor's language packs, so the chunks the editor already loaded serve both, and a syntax mark agrees with the +// highlighting. The worker loads the bare grammars instead (workerParsers.ts). + +import type { Parser } from '@lezer/common'; +import type { EditorLanguage } from './types'; + +export type ParserLoader = (language: EditorLanguage) => Promise | null; + +const LOADERS: Partial Promise>> = { + python: () => import('@codemirror/lang-python').then((m) => m.pythonLanguage.parser), + javascript: () => import('@codemirror/lang-javascript').then((m) => m.javascriptLanguage.parser), + typescript: () => import('@codemirror/lang-javascript').then((m) => m.typescriptLanguage.parser), + c: () => import('@codemirror/lang-cpp').then((m) => m.cppLanguage.parser), + cpp: () => import('@codemirror/lang-cpp').then((m) => m.cppLanguage.parser), + java: () => import('@codemirror/lang-java').then((m) => m.javaLanguage.parser), + rust: () => import('@codemirror/lang-rust').then((m) => m.rustLanguage.parser), + go: () => import('@codemirror/lang-go').then((m) => m.goLanguage.parser), + sql: () => import('@codemirror/lang-sql').then((m) => m.SQLite.language.parser), +}; + +/** The languages with a Lezer grammar. */ +export const GRAMMAR_LANGUAGES = Object.keys(LOADERS) as EditorLanguage[]; + +/** A loader that keeps each parser once it has loaded. */ +export function cachedLoader(loaders: Partial Promise>>): ParserLoader { + const cache = new Map>(); + return (language) => { + const load = loaders[language]; + if (!load) return null; + let parser = cache.get(language); + if (!parser) cache.set(language, (parser = load())); + return parser; + }; +} + +/** The language's parser, or null when it has no Lezer grammar (Lua and the legacy modes). */ +export const loadParser: ParserLoader = cachedLoader(LOADERS); diff --git a/app/src/features/codestudio/diagnostics/perf.test.ts b/app/src/features/codestudio/diagnostics/perf.test.ts new file mode 100644 index 000000000..8df3719e4 --- /dev/null +++ b/app/src/features/codestudio/diagnostics/perf.test.ts @@ -0,0 +1,54 @@ +// Latency gate (DESIGN 6.1 and 15): one checker run on a 1,000-line file takes under 50 ms. The test's module runner +// rewrites every import into a property read, which makes the parser several times slower than in the app. So the +// harness is bundled as the app bundles it and timed in a plain Node process, as the diagnostics worker runs it. + +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { rolldown } from 'rolldown'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { analyze } from './analyze'; +import { sampleFile } from './bench'; +import type { EditorLanguage } from './types'; + +const BUDGET_MS = 50; +const LANGUAGES: EditorLanguage[] = ['python', 'javascript', 'typescript', 'c', 'cpp', 'lua']; + +let dir = ''; +let times: Record = {}; + +beforeAll(async () => { + dir = mkdtempSync(join(tmpdir(), 'opennote-diagnostics-bench-')); + const entry = join(dir, 'entry.ts'); + const bench = join(import.meta.dirname, 'bench.ts').replace(/\\/g, '/'); + writeFileSync( + entry, + `import { benchmark } from '${bench}';\nconsole.log(JSON.stringify(await benchmark(${JSON.stringify(LANGUAGES)})));\n`, + ); + const bundle = await rolldown({ input: entry, platform: 'node', cwd: import.meta.dirname }); + await bundle.write({ dir, format: 'esm', entryFileNames: 'bench.mjs' }); + await bundle.close(); + times = JSON.parse(execFileSync(process.execPath, [join(dir, 'bench.mjs')], { encoding: 'utf8' })) as Record< + string, + number + >; +}, 120_000); + +afterAll(() => { + if (dir) rmSync(dir, { recursive: true, force: true }); +}); + +describe('checker latency', () => { + for (const language of LANGUAGES) { + it(`checks a 1,000-line ${language} file in under ${BUDGET_MS} ms`, async () => { + const text = sampleFile(language, 1000); + expect(text.split('\n').length).toBeGreaterThanOrEqual(1000); + // The sample is clean code: a run that only reports noise would not be a fair timing. + const { checker } = await analyze({ language, text, layers: ['checker'] }); + expect(checker.filter((f) => f.severity === 'error')).toEqual([]); + expect(times[language]).toBeGreaterThan(0); + expect(times[language]).toBeLessThan(BUDGET_MS); + }, 60_000); + } +}); diff --git a/app/src/features/codestudio/diagnostics/providers.ts b/app/src/features/codestudio/diagnostics/providers.ts new file mode 100644 index 000000000..b2042095e --- /dev/null +++ b/app/src/features/codestudio/diagnostics/providers.ts @@ -0,0 +1,84 @@ +// The built-in providers and the ready-made diagnostics service: the Lezer syntax layer, OpenNote's checkers (both in +// the diagnostics worker), and every engine check the runtime lanes register, all behind one scheduler. + +import type { Diagnostic } from '../contracts'; +import { t, type MessageKey } from '../../../strings/t'; +import { CHECKER_LANGUAGES } from './analyze'; +import { createCheckerClient, type CheckerClient } from './client'; +import { engineChecks, engineProvider, onEngineChecks } from './engine'; +import { GRAMMAR_LANGUAGES } from './parsers'; +import { createDiagnosticsScheduler, type DiagnosticsScheduler, type SchedulerOptions } from './scheduler'; +import type { DiagnosticProvider, Finding } from './types'; + +/** A checker finding in words, for the editor and the Problems panel. */ +export function toDiagnostic(file: string, source: string, finding: Finding): Diagnostic { + const key = `codeStudio.diagnostics.messages.${finding.message}` as MessageKey; + return { + file, + from: finding.from, + to: finding.to, + severity: finding.severity, + code: finding.code, + message: (t as (k: MessageKey, p?: Record) => string)(key, finding.params), + source, + }; +} + +/** Layer L-a: Lezer syntax errors for every language with a grammar. */ +export function syntaxProvider(client: CheckerClient): DiagnosticProvider { + return { + id: 'lezer', + layer: 'syntax', + languages: GRAMMAR_LANGUAGES, + async run(request) { + const { syntax } = await client.analyze({ language: request.language, text: request.text, layers: ['syntax'] }); + return syntax.map((f) => toDiagnostic(request.file, 'lezer', f)); + }, + }; +} + +/** Layer L-b: OpenNote's checkers for Python, JavaScript, TypeScript, C, C++, and Lua. */ +export function checkerProvider(client: CheckerClient): DiagnosticProvider { + return { + id: 'opennote', + layer: 'checker', + languages: CHECKER_LANGUAGES, + async run(request) { + const { checker } = await client.analyze({ language: request.language, text: request.text, layers: ['checker'] }); + return checker.map((f) => toDiagnostic(request.file, 'opennote', f)); + }, + }; +} + +export interface DiagnosticsService extends DiagnosticsScheduler { + readonly client: CheckerClient; +} + +/** A scheduler with the built-in layers and the registered engine checks, which follows checks added later. */ +export function createDiagnostics( + options: Omit & { client?: CheckerClient }, +): DiagnosticsService { + const client = options.client ?? createCheckerClient(); + const scheduler = createDiagnosticsScheduler({ + ...options, + providers: [syntaxProvider(client), checkerProvider(client)], + }); + const removers = new Map void>(); + const add = (check: Parameters[0]) => + removers.set(check.id, scheduler.register(engineProvider(check))); + engineChecks().forEach(add); + const stop = onEngineChecks((check, added) => { + removers.get(check.id)?.(); + removers.delete(check.id); + if (added) add(check); + }); + return { + ...scheduler, + client, + dispose() { + stop(); + scheduler.dispose(); + client.dispose(); + }, + }; +} diff --git a/app/src/features/codestudio/diagnostics/python.test.ts b/app/src/features/codestudio/diagnostics/python.test.ts new file mode 100644 index 000000000..ecf4ca2a0 --- /dev/null +++ b/app/src/features/codestudio/diagnostics/python.test.ts @@ -0,0 +1,186 @@ +// Ruff-parity known answers (ADR 0074): each input is the example in Ruff's documentation of that rule, and the +// expected line is the line Ruff's example marks. The checker must give the same code on the same line. + +import { parser } from '@lezer/python'; +import { describe, expect, it } from 'vitest'; +import { checkPython } from './python'; +import { lineAt, lineStarts } from './text'; + +function lines(source: string, code: string): number[] { + const starts = lineStarts(source); + return checkPython(source, parser.parse(source)) + .filter((f) => f.code === code) + .map((f) => lineAt(starts, f.from)); +} + +const RUFF_EXAMPLES: { code: string; source: string; lines: number[] }[] = [ + // F401 unused-import: "import numpy as np # unused import". + { + code: 'F401', + source: 'import numpy as np # unused import\n\n\ndef area(radius):\n return 3.14 * radius**2\n', + lines: [1], + }, + // F811 redefined-while-unused: the third line redefines `foo` from line 1. + { + code: 'F811', + source: 'import foo\nimport bar\nimport foo # Redefinition of unused `foo` from line 1\n', + lines: [3], + }, + // F821 undefined-name: `n` is never defined. + { + code: 'F821', + source: 'def double():\n return n * 2 # raises `NameError` if `n` is undefined when `double` is called\n', + lines: [2], + }, + // F841 unused-variable: `y` is assigned and never read. + { code: 'F841', source: 'def foo():\n x = 1\n y = 2\n return x\n', lines: [3] }, + // E711 none-comparison: both comparisons to None. + { code: 'E711', source: 'if arg != None:\n pass\nif None == arg:\n pass\n', lines: [1, 3] }, + // E712 true-false-comparison. + { code: 'E712', source: 'if foo == True:\n ...\n\nif bar == False:\n ...\n', lines: [1, 4] }, + // F632 is-literal. + { code: 'F632', source: 'x = 200\nif x is 200:\n print("It\'s 200!")\n', lines: [2] }, + // W191 tab-indentation. + { code: 'W191', source: 'if True:\n\ta = 1\n', lines: [2] }, + // E101 mixed-spaces-and-tabs: spaces set the file's indent, so the tab line is mixed. + { code: 'E101', source: 'if a == 0:\n a = 1\n\tb = 1\n', lines: [3] }, +]; + +describe('Python checker: Ruff documentation examples', () => { + for (const example of RUFF_EXAMPLES) { + it(`${example.code} on the documented line`, () => { + expect(lines(example.source, example.code)).toEqual(example.lines); + }); + } + + it('gives nothing on the "use instead" versions', () => { + const fixed = [ + 'def area(radius):\n return 3.14 * radius**2\n', + 'import foo\nimport bar\nbar.x(foo)\n', + 'def foo():\n x = 1\n return x\n', + 'arg = 1\nif arg is not None:\n pass\n', + 'foo = 1\nif foo:\n ...\n', + 'x = 200\nif x == 200:\n print("It\'s 200!")\n', + 'if True:\n a = 1\n', + ]; + for (const source of fixed) expect(checkPython(source, parser.parse(source))).toEqual([]); + }); +}); + +const codes = (source: string) => + checkPython(source, parser.parse(source)).map((f) => `${f.code}:${lineAt(lineStarts(source), f.from)}`); + +describe('Python checker: scope rules', () => { + it('lets a function use a module name bound after it', () => { + expect(codes('def f():\n return g()\n\ndef g():\n return 1\n\nf()\n')).toEqual([]); + }); + + it('reports module code that reads a name before it is bound', () => { + expect(codes('print(total)\ntotal = 1\n')).toEqual(['F821:1']); + }); + + it('hides class names from the methods inside the class', () => { + expect(codes('class K:\n size = 1\n def f(self):\n return size\n')).toEqual(['F821:4']); + }); + + it('knows builtins, comprehensions, lambdas, globals, patterns, and except names', () => { + const source = [ + 'import math', + 'def f(items, *args, key=len, **kw):', + ' global counter', + ' counter = 1', + ' squares = [x * x for x in items if x]', + ' g = lambda a, b=2: a + b + math.pi', + ' try:', + ' pass', + ' except ValueError as err:', + ' print(err, squares, g, args, kw, key)', + ' match items:', + ' case [first, *rest]:', + ' return first, rest', + 'print(counter)', + ].join('\n'); + expect(codes(source)).toEqual([]); + }); + + it('reports an unused except name and skips dummy names and locals()', () => { + expect(codes('try:\n pass\nexcept OSError as e:\n pass\n')).toEqual(['F841:3']); + expect(codes('def f():\n _ = 1\n __ = 2\n')).toEqual([]); + expect(codes('def f():\n x = 1\n return locals()\n')).toEqual([]); + }); + + it('counts a name in __all__ as a use of the import', () => { + expect(codes("from os import path\n__all__ = ['path']\n")).toEqual([]); + expect(codes('import sys\nimport os\n__all__ = ["os"]\n__all__ += ["sys"]\n')).toEqual([]); + }); +}); + +describe('Python checker: imports and valid programs', () => { + // pyflakes SubmoduleImportation.redefines: `import a.b` redefines only an import of the same full name. + it('does not take two submodule imports of one package for a redefinition', () => { + expect( + codes('import urllib.request\nimport urllib.parse\nurllib.request.urlopen(urllib.parse.quote("a"))\n'), + ).toEqual([]); + expect(codes('import os\nimport os.path\nprint(os.getcwd(), os.path.sep)\n')).toEqual([]); + expect(codes('import os.path\nimport os.path\nos.path.join("a")\n')).toEqual(['F811:2']); + expect(codes('import os\nimport os\nos.getcwd()\n')).toEqual(['F811:2']); + }); + + // pyflakes and Ruff read `x as x` as an explicit re-export (the PEP 484 convention). + it('leaves an explicit re-export alias alone', () => { + expect(codes('from os import sep as sep\nimport typing as typing\n')).toEqual([]); + expect(codes('import typing as t\n')).toEqual(['F401:1']); + }); + + it('does not report a redefinition in another branch or after a use', () => { + expect(codes('try:\n import json\nexcept ImportError:\n import simplejson as json\njson.dumps(1)\n')).toEqual( + [], + ); + expect(codes('def f():\n pass\nf()\ndef f():\n pass\nf()\n')).toEqual([]); + }); + + // Valid programs that pyflakes and Ruff pass without a finding: the checker must give none either. + it.each([ + ['a with name at module level', 'with open("data.txt") as f:\n data = f.read()\nprint(data)\n'], + ['a with name in a class', 'class A:\n with open("d") as f:\n text = f.read()\n'], + ['a match capture at module level', 'match [1, 2]:\n case [_, name]:\n print(name)\n'], + ['a match capture with a guard', 'match 3:\n case n if n > 2:\n print(n)\n'], + ['an as pattern', 'match [1]:\n case [x] as whole:\n print(x, whole)\n'], + ['a dotted decorator', 'import flask\napp = flask.Flask(__name__)\n@app.route("/")\ndef index():\n pass\n'], + [ + 'a property setter', + 'class A:\n @property\n def x(self):\n return 1\n @x.setter\n def x(self, v):\n pass\n', + ], + [ + 'a dotted call decorator', + 'import functools\ndef deco(fn):\n @functools.wraps(fn)\n def w():\n return fn()\n return w\n', + ], + ['a nested dotted decorator', 'import pytest\n@pytest.mark.parametrize("x", [1])\ndef test_x(x):\n pass\n'], + ['a generator as the only argument', 'words = ["a"]\nprint(sum(len(w) for w in words))\n'], + ['a generator to join', 'xs = [1]\nprint(",".join(str(c) for c in xs))\n'], + ['a generator to any', 'words = ["a"]\nprint(any(w for w in words))\n'], + ['a comprehension in a class body', 'class A:\n xs = [1]\n ys = [x for x in xs]\n'], + ['an import and a use on one line', 'import sys; sys.exit(0)\n'], + ['PEP 695 function type parameters', 'def f[T](x: T) -> T:\n return x\n'], + ['PEP 695 class type parameters', 'class Box[T]:\n def get(self) -> T:\n return self.v\n'], + ['a PEP 695 type alias', 'type Alias = list[int]\nx: Alias = [1]\nprint(x)\n'], + ['a recursive type alias', 'type Tree = list[Tree] | int\n'], + ['a string annotation', 'from typing import List\nx: "List[int]" = []\nprint(x)\n'], + [ + 'a TYPE_CHECKING import', + 'from typing import TYPE_CHECKING\nif TYPE_CHECKING:\n from collections import OrderedDict\n' + + 'def f(d: "OrderedDict[str, int]") -> None:\n print(d)\n', + ], + ])('gives nothing on %s', (_label, source) => { + expect(codes(source)).toEqual([]); + }); + + it('still reports a name read in a class method and a with name read before its block', () => { + expect(codes('print(f)\nwith open("d") as f:\n pass\n')).toEqual(['F821:1']); + expect(codes('class A:\n xs = [1]\n def f(self):\n return [x for x in xs]\n')).toEqual(['F821:4']); + }); + + it('ignores tabs inside a multi-line string', () => { + expect(codes('x = """\n\tkept\n"""\nprint(x)\n')).toEqual([]); + }); +}); diff --git a/app/src/features/codestudio/diagnostics/python.ts b/app/src/features/codestudio/diagnostics/python.ts new file mode 100644 index 000000000..685a038d0 --- /dev/null +++ b/app/src/features/codestudio/diagnostics/python.ts @@ -0,0 +1,685 @@ +// OpenNote's Python checker over the Lezer tree (layer L-b, DESIGN 6.1). Its rule codes are the pyflakes and +// pycodestyle codes as Ruff documents them, so the marks read as people know them. F401, F811, F821, and F841 cover +// imports and names. E711, E712, and F632 cover comparisons. W191 and E101 cover tabs in indentation. +// The scopes follow Python's name resolution. Module code runs in order and function bodies run later. Class bodies +// are not visible to the functions inside them. + +import type { SyntaxNode, Tree } from '@lezer/common'; +import { PYTHON_BUILTINS } from './pythonNames'; +import { lineAt, lineStarts } from './text'; +import type { Finding } from './types'; + +/** An annotation scope holds the type parameters of `def f[T]`, `class C[T]`, and `type A[T]` (PEP 695). */ +type ScopeKind = 'module' | 'class' | 'function' | 'comprehension' | 'annotation'; +type BindKind = 'import' | 'future' | 'def' | 'class' | 'assign' | 'param' | 'loop' | 'except' | 'other'; + +interface Scope { + kind: ScopeKind; + parent: Scope | null; + bindings: Map; + globals: Set; + nonlocals: Set; + star: boolean; + callsLocals: boolean; +} + +interface Binding { + name: string; + /** What the message names: the dotted module for `import a.b`. */ + shown: string; + from: number; + to: number; + /** Where the name becomes visible to code that runs in order. */ + at: number; + kind: BindKind; + /** A lone name target, the only kind F841 reports. */ + simple: boolean; + /** The block that holds the statement, so F811 compares bindings of one block only. */ + block: number; + decorators: string[]; + uses: number; + /** The full module of an import (`a.b` for `import a.b`), and whether it binds a submodule's top name. */ + module?: string; + submodule?: boolean; + /** `import x as x` or `from m import x as x`: an explicit re-export, which F401 leaves alone. */ + reexport?: boolean; +} + +interface Ref { + name: string; + from: number; + to: number; + scope: Scope; + /** The loops around the reference, so a name bound later in the same loop counts. */ + loops: [number, number][]; +} + +/** Ruff's default dummy-variable pattern: names F841 never reports. */ +const DUMMY = /^(?:_+|_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?)$/; +const COMPREHENSIONS = new Set([ + 'ArrayComprehensionExpression', + 'SetComprehensionExpression', + 'DictionaryComprehensionExpression', + 'ComprehensionExpression', +]); +const LITERALS = new Set(['Number', 'String', 'FormatString', 'ContinuedString', 'Bytes']); + +function kids(node: SyntaxNode): SyntaxNode[] { + const out: SyntaxNode[] = []; + for (let c = node.firstChild; c; c = c.nextSibling) out.push(c); + return out; +} + +class PythonChecker { + readonly findings: Finding[] = []; + private readonly refs: Ref[] = []; + private readonly scopes: Scope[] = []; + private readonly exported = new Set(); + private loops: [number, number][] = []; + /** Names read inside string annotations, such as `x: "List[int]"`: they count as uses but are never reported. */ + private readonly softRefs: Ref[] = []; + private annotationDepth = 0; + + constructor(private readonly text: string) {} + + private slice(node: SyntaxNode): string { + return this.text.slice(node.from, node.to); + } + + private scope(kind: ScopeKind, parent: Scope | null): Scope { + const s: Scope = { + kind, + parent, + bindings: new Map(), + globals: new Set(), + nonlocals: new Set(), + star: false, + callsLocals: false, + }; + this.scopes.push(s); + return s; + } + + private bind( + scope: Scope, + node: SyntaxNode, + kind: BindKind, + at: number, + extra: Partial> = {}, + ): void { + const name = this.slice(node); + if (name === '_' && kind !== 'assign') return; + let target = scope; + if (scope.globals.has(name)) target = this.scopes[0]; + else if (scope.nonlocals.has(name)) target = this.enclosingWith(scope, name) ?? scope; + while (target.kind === 'comprehension' && kind === 'other' && target.parent) target = target.parent; + const binding: Binding = { + name, + shown: extra.shown ?? name, + from: node.from, + to: node.to, + at: target === scope ? at : -1, + kind, + simple: extra.simple ?? false, + block: this.block(node), + decorators: extra.decorators ?? [], + uses: 0, + module: extra.module, + submodule: extra.submodule, + reexport: extra.reexport, + }; + const list = target.bindings.get(name); + if (list) list.push(binding); + else target.bindings.set(name, [binding]); + } + + private enclosingWith(scope: Scope, name: string): Scope | null { + for (let s = scope.parent; s; s = s.parent) if (s.kind === 'function' && s.bindings.has(name)) return s; + return null; + } + + /** The start of the statement list that holds `node`. */ + private block(node: SyntaxNode): number { + for (let n: SyntaxNode | null = node.parent; n; n = n.parent) { + if (n.name === 'Body' || n.name === 'Script') return n.from; + } + return 0; + } + + private ref(scope: Scope, node: SyntaxNode): void { + const name = this.slice(node); + if (name === 'locals') scope.callsLocals = true; + this.refs.push({ name, from: node.from, to: node.to, scope, loops: this.loops }); + } + + private statementEnd(node: SyntaxNode): number { + let n = node; + while (n.parent && !['Body', 'Script', 'StatementGroup'].includes(n.parent.name)) n = n.parent; + return n.to; + } + + run(tree: Tree): void { + const module = this.scope('module', null); + this.visitAll(kids(tree.topNode), module); + this.resolve(); + this.report(); + } + + private visitAll(nodes: SyntaxNode[], scope: Scope): void { + for (const node of nodes) this.visit(node, scope); + } + + private visit(node: SyntaxNode, scope: Scope): void { + switch (node.name) { + case 'VariableName': + return this.ref(scope, node); + case 'PropertyName': + return; + case 'ImportStatement': + return this.importStatement(node, scope); + case 'DecoratedStatement': + return this.decorated(node, scope); + case 'FunctionDefinition': + return this.functionDef(node, scope, []); + case 'ClassDefinition': + return this.classDef(node, scope, []); + case 'LambdaExpression': + return this.lambda(node, scope); + case 'AssignStatement': + return this.assign(node, scope); + case 'UpdateStatement': + return this.update(node, scope); + case 'ForStatement': + return this.forStatement(node, scope); + case 'WhileStatement': + return this.loop(node, () => this.visitAll(kids(node), scope)); + case 'TryStatement': + case 'WithStatement': + return this.asBindings(node, scope, node.name === 'TryStatement' ? 'except' : 'other'); + case 'ScopeStatement': + return this.scopeStatement(node, scope); + case 'NamedExpression': + return this.named(node, scope); + case 'ArgList': + return this.argList(node, scope); + case 'CapturePattern': + return this.patternBind(node, scope); + case 'AsPattern': + return this.asBindings(node, scope, 'other'); + case 'KeywordPattern': + return this.visitAll(kids(node).slice(1), scope); + case 'BinaryExpression': + this.comparison(node); + return this.visitAll(kids(node), scope); + case 'TypeDefinition': + return this.typeAlias(node, scope); + case 'TypeDef': + this.annotationDepth += 1; + this.visitAll(kids(node), scope); + this.annotationDepth -= 1; + return; + case 'String': + if (this.annotationDepth) this.stringAnnotation(node, scope); + return; + default: + if (COMPREHENSIONS.has(node.name)) return this.comprehension(node, scope); + return this.visitAll(kids(node), scope); + } + } + + private loop(node: SyntaxNode, body: () => void): void { + const outer = this.loops; + this.loops = [...outer, [node.from, node.to]]; + body(); + this.loops = outer; + } + + private importStatement(node: SyntaxNode, scope: Scope): void { + const parts = kids(node); + const isFrom = parts[0]?.name === 'from'; + const importAt = parts.findIndex((p) => p.name === 'import'); + const module = isFrom ? this.text.slice(parts[0].to, parts[importAt]?.from ?? node.to).trim() : ''; + const at = node.to; + let group: SyntaxNode[] = []; + const flush = () => { + const names = group.filter((p) => p.name === 'VariableName'); + const as = group.find((p) => p.name === 'as'); + if (group.some((p) => p.name === '*')) scope.star = true; + if (names.length) { + const target = as || isFrom ? names[names.length - 1] : names[0]; + const kind: BindKind = module === '__future__' ? 'future' : 'import'; + const shown = this.text.slice(group[0].from, group[group.length - 1].to); + const imported = this.text.slice(group[0].from, as?.from ?? group[group.length - 1].to).trim(); + this.bind(scope, target, kind, at, { + shown, + // pyflakes compares full names: `import a.b` and `import a.c` both bind `a` but neither redefines the other. + module: isFrom ? `${module}.${imported}` : imported, + submodule: !isFrom && !as && imported.includes('.'), + reexport: !!as && imported === this.slice(target), + }); + } + group = []; + }; + for (const part of parts.slice(importAt + 1)) { + if (part.name === ',') flush(); + else if (part.name !== '(' && part.name !== ')') group.push(part); + } + flush(); + } + + private decorated(node: SyntaxNode, scope: Scope): void { + const names: string[] = []; + for (const child of kids(node)) { + if (child.name === 'Decorator') { + // Lezer names every part of `@app.route` a VariableName; only the first part is a read. + for (const part of kids(child)) { + if (part.name !== 'VariableName' || part.prevSibling?.name !== '.') this.visit(part, scope); + } + names.push(this.slice(child).replace(/^@\s*/, '').replace(/\(.*$/s, '').trim()); + } else if (child.name === 'FunctionDefinition') this.functionDef(child, scope, names); + else if (child.name === 'ClassDefinition') this.classDef(child, scope, names); + else this.visit(child, scope); + } + } + + private params(list: SyntaxNode, outer: Scope, inner: Scope): void { + for (const child of kids(list)) { + if (child.name === 'VariableName') { + if (child.prevSibling?.name === 'AssignOp') this.ref(outer, child); + else this.bind(inner, child, 'param', -1); + } else this.visit(child, outer); + } + } + + /** The scope that holds a definition's type parameters, or the enclosing scope when it has none. */ + private typeParams(node: SyntaxNode, scope: Scope): Scope { + const list = node.getChild('TypeParamList'); + if (!list) return scope; + const annotation = this.scope('annotation', scope); + for (const param of list.getChildren('TypeParam')) { + // The first name is the parameter (`T`, `*Ts`, `**P`); a bound or default after it is a read. + const name = param.getChild('VariableName'); + for (const part of kids(param)) { + if (part === name || (name && part.from === name.from)) this.bind(annotation, part, 'param', -1); + else this.visit(part, annotation); + } + } + return annotation; + } + + private functionDef(node: SyntaxNode, scope: Scope, decorators: string[]): void { + const outer = this.typeParams(node, scope); + const inner = this.scope('function', outer); + for (const child of kids(node)) { + if (child.name === 'VariableName') this.bind(scope, child, 'def', this.statementEnd(node), { decorators }); + else if (child.name === 'TypeParamList') continue; + else if (child.name === 'ParamList') this.params(child, outer, inner); + else if (child.name === 'Body') this.visitAll(kids(child), inner); + else this.visit(child, outer); + } + } + + private classDef(node: SyntaxNode, scope: Scope, decorators: string[]): void { + const outer = this.typeParams(node, scope); + const inner = this.scope('class', outer); + for (const child of kids(node)) { + if (child.name === 'VariableName') this.bind(scope, child, 'class', this.statementEnd(node), { decorators }); + else if (child.name === 'TypeParamList') continue; + else if (child.name === 'Body') this.visitAll(kids(child), inner); + else this.visit(child, outer); + } + } + + /** `type Alias[T] = value`: the alias is bound at once, and its value is read lazily, so it may name later names. */ + private typeAlias(node: SyntaxNode, scope: Scope): void { + const name = node.getChild('VariableName'); + if (name) this.bind(scope, name, 'assign', node.from); + const lazy = this.scope('function', this.typeParams(node, scope)); + for (const child of kids(node)) { + if (name && child.from === name.from) continue; + if (child.name !== 'TypeParamList') this.visit(child, lazy); + } + } + + /** The first name of each name path in a string annotation counts as a use. */ + private stringAnnotation(node: SyntaxNode, scope: Scope): void { + const raw = this.slice(node); + const quote = /^[rRuUbB]*("""|'''|"|')/.exec(raw); + if (!quote) return; + const offset = node.from + quote[0].length; + const body = raw.slice(quote[0].length, raw.length - quote[1].length); + for (const m of body.matchAll(/(? { + if (p.name === 'AssignOp') last = i; + }); + if (last < 0) { + // An annotation alone, such as `x: int`. + parts.forEach((p) => + p.name === 'TypeDef' ? this.visit(p, scope) : this.target(p, scope, 'assign', node.to, false), + ); + return; + } + this.visitAll(parts.slice(last + 1), scope); + const targets = parts.slice(0, last).filter((p) => p.name !== 'AssignOp' && p.name !== 'TypeDef'); + const simple = targets.length === 1 && targets[0].name === 'VariableName'; + for (const p of parts.slice(0, last)) { + if (p.name === 'TypeDef') this.visit(p, scope); + else if (p.name !== 'AssignOp') this.target(p, scope, 'assign', node.to, simple); + } + if (scope.kind === 'module' && simple && this.slice(targets[0]) === '__all__') { + this.readAll([parts[last + 1] ?? node]); + } + } + + /** The names in the strings of an `__all__ = [...]` or `__all__ += [...]` value. */ + private readAll(nodes: SyntaxNode[]): void { + for (const n of nodes) for (const m of this.slice(n).matchAll(/['"]([A-Za-z_]\w*)['"]/g)) this.exported.add(m[1]); + } + + private update(node: SyntaxNode, scope: Scope): void { + const parts = kids(node); + this.visitAll(parts.slice(1), scope); + if (parts[0]?.name === 'VariableName') { + this.ref(scope, parts[0]); + this.bind(scope, parts[0], 'other', node.to); + if (scope.kind === 'module' && this.slice(parts[0]) === '__all__') this.readAll(parts.slice(1)); + } else if (parts[0]) this.visit(parts[0], scope); + } + + private forStatement(node: SyntaxNode, scope: Scope): void { + const parts = kids(node); + const inAt = parts.findIndex((p) => p.name === 'in'); + if (inAt < 0) return this.visitAll(parts, scope); + this.visitAll(parts.slice(inAt + 1, inAt + 2), scope); + this.loop(node, () => { + for (const p of parts.slice(1, inAt)) this.target(p, scope, 'loop', p.from, false); + this.visitAll(parts.slice(inAt + 2), scope); + }); + } + + private comprehension(node: SyntaxNode, scope: Scope): void { + const inner = this.scope('comprehension', scope); + let binding = false; + for (const child of kids(node)) { + if (child.name === 'for') binding = true; + else if (child.name === 'in') binding = false; + else if (binding) this.target(child, inner, 'loop', -1, false); + else this.visit(child, inner); + } + } + + /** `except E as e`, `with x as y`, and `case p as q`: the name after `as` is bound. */ + private asBindings(node: SyntaxNode, scope: Scope, kind: BindKind): void { + for (const child of kids(node)) { + if (child.prevSibling?.name === 'as') { + // An except name is visible inside its handler; a `with` or pattern name from the end of its own target. + this.target(child, scope, kind, kind === 'except' ? child.from : child.to, kind === 'except'); + } else this.visit(child, scope); + } + } + + private patternBind(node: SyntaxNode, scope: Scope): void { + const name = node.firstChild; + if (name?.name === 'VariableName') this.bind(scope, name, 'other', node.to); + } + + private scopeStatement(node: SyntaxNode, scope: Scope): void { + const parts = kids(node); + const set = parts[0]?.name === 'global' ? scope.globals : scope.nonlocals; + for (const p of parts) if (p.name === 'VariableName') set.add(this.slice(p)); + } + + private named(node: SyntaxNode, scope: Scope): void { + const parts = kids(node); + this.visitAll(parts.slice(1), scope); + if (parts[0]?.name === 'VariableName') this.bind(scope, parts[0], 'other', node.to); + } + + private argList(node: SyntaxNode, scope: Scope): void { + // A generator as the only argument, `sum(x for x in xs)`, sits directly in the ArgList. + if (node.getChild('for')) return this.comprehension(node, scope); + for (const child of kids(node)) { + if (child.name === 'VariableName' && child.nextSibling?.name === 'AssignOp') continue; + this.visit(child, scope); + } + } + + /** E711, E712, and F632, which read one comparison each. */ + private comparison(node: SyntaxNode): void { + for (const op of kids(node)) { + const word = this.slice(op); + if (op.name === 'CompareOp' && (word === '==' || word === '!=')) { + for (const side of [op.prevSibling, op.nextSibling]) { + if (side?.name === 'None') this.add(side, 'warning', 'E711', 'noneComparison', { op: word }); + if (side?.name === 'Boolean') + this.add(side, 'warning', 'E712', 'boolComparison', { op: word, value: this.slice(side) }); + } + } + if (op.name === 'is') { + const right = op.nextSibling?.name === 'not' ? op.nextSibling.nextSibling : op.nextSibling; + if ([op.prevSibling, right].some((side) => side && this.isLiteral(side))) { + const negated = op.nextSibling?.name === 'not'; + this.add(node, 'warning', 'F632', 'isLiteral', { + op: negated ? 'is not' : 'is', + better: negated ? '!=' : '==', + }); + } + } + } + } + + private isLiteral(node: SyntaxNode): boolean { + if (LITERALS.has(node.name)) return true; + if (node.name === 'UnaryExpression') return node.lastChild?.name === 'Number'; + if (node.name === 'TupleExpression') return kids(node).every((c) => /^[(),]$/.test(c.name) || this.isLiteral(c)); + return false; + } + + private add( + node: { from: number; to: number }, + severity: Finding['severity'], + code: string, + message: Finding['message'], + params?: Finding['params'], + ): void { + this.findings.push({ from: node.from, to: node.to, severity, code, message, params }); + } + + /** True when the reference runs later than the code of `target` around it: a function or lambda lies between. */ + private deferred(from: Scope, target: Scope): boolean { + for (let s: Scope | null = from; s && s !== target; s = s.parent) { + if (s.kind === 'function' || s.kind === 'comprehension') return true; + } + return false; + } + + private onlyComprehensions(from: Scope, target: Scope): boolean { + for (let s: Scope | null = from; s && s !== target; s = s.parent) if (s.kind !== 'comprehension') return false; + return true; + } + + private pick(list: Binding[], ref: Ref, ordered: boolean): Binding | null { + const before = list.filter((b) => b.at <= ref.from); + if (before.length) return before[before.length - 1]; + if (!ordered) return list[list.length - 1]; + const inLoop = list.find((b) => ref.loops.some(([a, z]) => a <= b.from && b.from < z && a <= ref.from)); + return inLoop ?? null; + } + + private lookup(ref: Ref): Binding | 'builtin' | 'star' | null { + const start = ref.scope; + let s: Scope | null = start; + if (start.globals.has(ref.name)) s = this.scopes[0]; + let star = false; + while (s) { + star ||= s.star; + // A class body is hidden from the scopes inside it, except from comprehensions directly in it (as pyflakes). + const skipClass = s.kind === 'class' && s !== start && !this.onlyComprehensions(start, s); + const list = skipClass ? undefined : s.bindings.get(ref.name); + if (list?.length) { + const ordered = (s.kind === 'module' || s.kind === 'class') && !this.deferred(start, s); + const found = this.pick(list, ref, ordered); + if (found) return found; + if (s.kind === 'function') return list[0]; + } + s = s.parent; + } + if (PYTHON_BUILTINS.has(ref.name) || (ref.name === '__class__' && start.kind === 'function')) return 'builtin'; + return star ? 'star' : null; + } + + /** Marks a binding used, and with it the imports of the same name in other blocks (try and except variants). */ + private use(binding: Binding, from: Scope): void { + binding.uses += 1; + for (let s: Scope | null = from; s; s = s.parent) { + const list = s.bindings.get(binding.name); + if (!list?.includes(binding)) continue; + for (const other of list) { + if (other === binding || other.kind !== 'import' || binding.kind !== 'import') continue; + // Imports in other blocks (try and except variants) and other submodules of one package share the name. + if (other.block !== binding.block || this.sharesPackage(other, binding)) other.uses += 1; + } + return; + } + } + + /** `import a.b` next to `import a.c` or `import a`: one package name, two different imports. */ + private sharesPackage(a: Binding, b: Binding): boolean { + return (!!a.submodule || !!b.submodule) && a.module !== b.module; + } + + /** True when `b` redefines the import or definition `prev` of the same name (pyflakes `redefines`). */ + private redefines(b: Binding, prev: Binding): boolean { + if (b.kind === 'import' && prev.kind === 'import' && this.sharesPackage(b, prev)) return false; + return true; + } + + private resolve(): void { + for (const ref of this.refs) { + const found = this.lookup(ref); + if (found === null) this.add(ref, 'error', 'F821', 'undefinedName', { name: ref.name }); + else if (typeof found === 'object') this.use(found, ref.scope); + } + for (const ref of this.softRefs) { + const found = this.lookup(ref); + if (found !== null && typeof found === 'object') this.use(found, ref.scope); + } + for (const name of this.exported) { + for (const b of this.scopes[0].bindings.get(name) ?? []) b.uses += 1; + } + } + + private report(): void { + const starts = lineStarts(this.text); + for (const scope of this.scopes) { + for (const [name, list] of scope.bindings) { + list.forEach((b, i) => { + const prev = list[i - 1]; + if ( + prev && + (b.kind === 'def' || b.kind === 'class' || b.kind === 'import') && + (prev.kind === 'def' || prev.kind === 'class' || prev.kind === 'import') && + prev.uses === 0 && + prev.block === b.block && + this.redefines(b, prev) && + !prev.decorators.some((d) => /(?:^|\.)overload$/.test(d)) + ) { + this.add(b, 'warning', 'F811', 'redefined', { name, line: lineAt(starts, prev.from) }); + prev.uses = -1; + } + }); + for (const b of list) + if (b.kind === 'import' && b.uses === 0 && !b.reexport) + this.add(b, 'warning', 'F401', 'unusedImport', { name: b.shown }); + const used = list.some((b) => b.uses !== 0); + const local = scope.kind === 'function' && !scope.callsLocals; + const candidate = list.find((b) => b.simple && (b.kind === 'except' || (local && b.kind === 'assign'))); + if (!used && candidate && !DUMMY.test(name) && !scope.globals.has(name) && !scope.nonlocals.has(name)) { + this.add(candidate, 'warning', 'F841', 'unusedLocal', { name }); + } + } + } + this.indentation(starts); + this.findings.sort((a, b) => a.from - b.from || a.code.localeCompare(b.code)); + } + + /** W191 and E101, read line by line as pycodestyle does, skipping lines inside multi-line strings. */ + private indentation(starts: readonly number[]): void { + let indentChar = ''; + for (const start of starts) { + const indent = /^[ \t]*/.exec(this.text.slice(start, start + 200))![0]; + const rest = this.text.charAt(start + indent.length); + if (!indent || rest === '\n' || rest === '\r' || rest === '' || this.inString(start)) continue; + indentChar ||= indent[0]; + const tab = indent.indexOf('\t'); + if (tab >= 0) this.add({ from: start, to: start + indent.length }, 'warning', 'W191', 'tabIndent'); + const mixed = [...indent].findIndex((c) => c !== indentChar); + if (mixed >= 0) this.add({ from: start + mixed, to: start + indent.length }, 'warning', 'E101', 'mixedIndent'); + } + } + + private stringRanges: [number, number][] | null = null; + + private inString(offset: number): boolean { + return (this.stringRanges ?? []).some(([a, z]) => a < offset && offset < z); + } + + collectStrings(tree: Tree): void { + const ranges: [number, number][] = []; + tree.iterate({ + enter: (n) => { + if (n.name === 'String' || n.name === 'FormatString') { + if (this.text.slice(n.from, n.to).includes('\n')) ranges.push([n.from, n.to]); + return false; + } + return true; + }, + }); + this.stringRanges = ranges; + } +} + +/** The Python checker's findings for a parsed file. */ +export function checkPython(text: string, tree: Tree): Finding[] { + const checker = new PythonChecker(text); + checker.collectStrings(tree); + checker.run(tree); + return checker.findings; +} diff --git a/app/src/features/codestudio/diagnostics/pythonNames.ts b/app/src/features/codestudio/diagnostics/pythonNames.ts new file mode 100644 index 000000000..efe6b01e3 --- /dev/null +++ b/app/src/features/codestudio/diagnostics/pythonNames.ts @@ -0,0 +1,31 @@ +// Names Python defines without an import: the built-in functions, constants, types, and exceptions that the Python +// 3.12 library reference lists under "Built-in Functions", "Built-in Constants", and "Built-in Exceptions", plus the +// module attributes every module has. + +const FUNCTIONS = + 'abs aiter all anext any ascii bin bool breakpoint bytearray bytes callable chr classmethod compile complex ' + + 'delattr dict dir divmod enumerate eval exec filter float format frozenset getattr globals hasattr hash help hex ' + + 'id input int isinstance issubclass iter len list locals map max memoryview min next object oct open ord pow ' + + 'print property range repr reversed round set setattr slice sorted staticmethod str sum super tuple type vars ' + + 'zip __import__'; + +const CONSTANTS = 'False True None NotImplemented Ellipsis __debug__ quit exit copyright credits license'; + +const EXCEPTIONS = + 'BaseException BaseExceptionGroup GeneratorExit KeyboardInterrupt SystemExit Exception ArithmeticError ' + + 'FloatingPointError OverflowError ZeroDivisionError AssertionError AttributeError BufferError EOFError ' + + 'ExceptionGroup ImportError ModuleNotFoundError LookupError IndexError KeyError MemoryError NameError ' + + 'UnboundLocalError OSError BlockingIOError ChildProcessError ConnectionError BrokenPipeError ' + + 'ConnectionAbortedError ConnectionRefusedError ConnectionResetError FileExistsError FileNotFoundError ' + + 'InterruptedError IsADirectoryError NotADirectoryError PermissionError ProcessLookupError TimeoutError ' + + 'ReferenceError RuntimeError NotImplementedError PythonFinalizationError RecursionError StopAsyncIteration ' + + 'StopIteration SyntaxError IndentationError TabError SystemError TypeError ValueError UnicodeError ' + + 'UnicodeDecodeError UnicodeEncodeError UnicodeTranslateError Warning BytesWarning DeprecationWarning ' + + 'EncodingWarning FutureWarning ImportWarning PendingDeprecationWarning ResourceWarning RuntimeWarning ' + + 'SyntaxWarning UnicodeWarning UserWarning EnvironmentError IOError'; + +const MODULE = '__name__ __file__ __doc__ __package__ __spec__ __loader__ __builtins__ __annotations__ __path__'; + +export const PYTHON_BUILTINS: ReadonlySet = new Set( + [FUNCTIONS, CONSTANTS, EXCEPTIONS, MODULE].join(' ').split(' '), +); diff --git a/app/src/features/codestudio/diagnostics/runtimeChecks.test.ts b/app/src/features/codestudio/diagnostics/runtimeChecks.test.ts new file mode 100644 index 000000000..3a114afb6 --- /dev/null +++ b/app/src/features/codestudio/diagnostics/runtimeChecks.test.ts @@ -0,0 +1,52 @@ +// Python's engine check (layer L-c) with the real Pyodide in Node: compile() marks a syntax error at its line, good +// code gets no mark, and a missing pack gives no marks instead of an error. + +import { fileURLToPath } from 'node:url'; +import { loadPyodide } from 'pyodide'; +import { describe, expect, it, vi } from 'vitest'; +import type { RuntimeAdapter } from '../contracts'; +import { pyodideAdapter, type LoadPyodide } from '../runtimes/python/pyodide'; +import { engineChecks, engineProvider } from './engine'; +import { lineAt, lineStarts } from './text'; +import { pythonEngineCheck, registerRuntimeChecks } from './runtimeChecks'; + +// Pyodide takes seconds to start on a loaded machine. +vi.setConfig({ testTimeout: 180_000 }); + +const PYODIDE_DIR = fileURLToPath(new URL('../../../../../node_modules/pyodide/', import.meta.url)); +const full = pyodideAdapter(PYODIDE_DIR, () => Promise.resolve(loadPyodide as unknown as LoadPyodide)); + +const request = (text: string) => ({ file: 'main.py', version: 1, language: 'python' as const, text }); + +describe('runtime engine checks', () => { + it('marks a Python syntax error at its line through full Python, and good code not at all', async () => { + const provider = engineProvider(pythonEngineCheck(() => Promise.resolve(full))); + expect(provider).toMatchObject({ id: 'pyodide', layer: 'engine', languages: ['python'] }); + const bad = 'x = 1\nif x\n print(x)\n'; + const marks = await provider.run(request(bad)); + expect(marks).toHaveLength(1); + expect(marks[0]).toMatchObject({ file: 'main.py', severity: 'error', source: 'pyodide' }); + expect(marks[0]!.message).toContain("expected ':'"); + expect(lineAt(lineStarts(bad), marks[0]!.from)).toBe(2); + expect(await provider.run(request('x = 1\nif x:\n print(x)\n'))).toEqual([]); + }); + + it('gives no marks when the full Python pack is missing', async () => { + const missing = () => Promise.reject(new Error('pack missing')); + expect(await engineProvider(pythonEngineCheck(missing)).run(request('if x\n'))).toEqual([]); + const failing: RuntimeAdapter = { + id: 'python-full', + load: async () => {}, + run: () => Promise.reject(new Error('not run here')), + check: () => Promise.reject(new Error('runtime stopped')), + }; + expect(await engineProvider(pythonEngineCheck(() => Promise.resolve(failing))).run(request('if x\n'))).toEqual([]); + }); + + it('registers the Python check and removes it again', () => { + const off = registerRuntimeChecks(() => Promise.resolve(full)); + expect(engineChecks().map((check) => [check.id, check.languages])).toEqual([['pyodide', ['python']]]); + off(); + expect(engineChecks()).toEqual([]); + }); +}); diff --git a/app/src/features/codestudio/diagnostics/runtimeChecks.ts b/app/src/features/codestudio/diagnostics/runtimeChecks.ts new file mode 100644 index 000000000..f2368dffd --- /dev/null +++ b/app/src/features/codestudio/diagnostics/runtimeChecks.ts @@ -0,0 +1,35 @@ +// The engine checks of the runtimes in this build (layer L-c, DESIGN 6.1). Python's check is compile() in full +// Python (Pyodide, so CPython's own parser): quick Python (MicroPython) does not know every CPython form, so it would +// mark good code. The runtime loads on the first check, not before. Without the full Python pack the check finds +// nothing, and the Lezer and OpenNote layers still run. The studio calls registerRuntimeChecks once, next to +// createDiagnostics. + +import type { RuntimeAdapter } from '../contracts'; +import { fromRuntimeAdapter, registerEngineCheck, type EngineCheck } from './engine'; + +const loadFullPython = (): Promise => + import('../runtimes/python').then((loaded) => loaded.pythonFullAdapter); + +/** Python's engine check. `load` gives the adapter whose `check` runs compile(); by default full Python. */ +export function pythonEngineCheck(load: () => Promise = loadFullPython): EngineCheck { + return { + id: 'pyodide', + layer: 'engine', + languages: ['python'], + async check(file, files) { + try { + const check = fromRuntimeAdapter(await load(), ['python'], 'pyodide'); + return check ? await check.check(file, files) : []; + } catch { + // A missing pack, or a runtime that failed to start: no engine marks. + return []; + } + }, + }; +} + +/** Registers every runtime's engine check. Returns the way to remove them. */ +export function registerRuntimeChecks(load?: () => Promise): () => void { + const removers = [registerEngineCheck(pythonEngineCheck(load))]; + return () => removers.forEach((remove) => remove()); +} diff --git a/app/src/features/codestudio/diagnostics/scheduler.test.ts b/app/src/features/codestudio/diagnostics/scheduler.test.ts new file mode 100644 index 000000000..10ffb9432 --- /dev/null +++ b/app/src/features/codestudio/diagnostics/scheduler.test.ts @@ -0,0 +1,146 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { Diagnostic } from '../contracts'; +import { createDiagnosticsScheduler } from './scheduler'; +import type { DiagnosticProvider, DiagnosticRequest } from './types'; + +function item(source: string, from: number): Diagnostic { + return { file: 'main.py', from, to: from + 1, severity: 'error', code: 'X', message: 'm', source }; +} + +function doc(version: number, text = 'x'): DiagnosticRequest { + return { file: 'main.py', version, language: 'python', text }; +} + +describe('diagnostics scheduler versions', () => { + beforeEach(() => vi.useFakeTimers()); + afterEach(() => vi.useRealTimers()); + + it('drops results computed for version 3 when the document is at version 4', () => { + const publish = vi.fn(); + const scheduler = createDiagnosticsScheduler({ publish }); + scheduler.update(doc(3)); + scheduler.update(doc(4)); + expect(scheduler.accept({ file: 'main.py', version: 3, source: 'opennote', items: [item('opennote', 0)] })).toBe( + false, + ); + expect(publish).not.toHaveBeenCalled(); + expect(scheduler.current('main.py')).toEqual([]); + expect(scheduler.accept({ file: 'main.py', version: 4, source: 'opennote', items: [item('opennote', 0)] })).toBe( + true, + ); + expect(publish).toHaveBeenCalledWith('main.py', [item('opennote', 0)]); + }); + + it('drops a slow provider answer once the file has moved on', async () => { + let finish: (items: Diagnostic[]) => void = () => {}; + const slow: DiagnosticProvider = { + id: 'slow', + layer: 'checker', + languages: ['python'], + run: () => new Promise((resolve) => (finish = resolve)), + }; + const publish = vi.fn(); + const scheduler = createDiagnosticsScheduler({ publish, providers: [slow] }); + scheduler.update(doc(3)); + await vi.advanceTimersByTimeAsync(300); + scheduler.update(doc(4)); + finish([item('slow', 0)]); + await vi.advanceTimersByTimeAsync(0); + expect(publish).not.toHaveBeenCalled(); + }); +}); + +describe('diagnostics scheduler layers', () => { + beforeEach(() => vi.useFakeTimers()); + afterEach(() => vi.useRealTimers()); + + it('waits each layer its own pause and restarts it on every change', async () => { + const runs: string[] = []; + const provider = (id: string, layer: DiagnosticProvider['layer']): DiagnosticProvider => ({ + id, + layer, + languages: ['python'], + run: (r) => { + runs.push(`${id}@${r.version}`); + return []; + }, + }); + const scheduler = createDiagnosticsScheduler({ + publish: () => {}, + providers: [provider('lezer', 'syntax'), provider('opennote', 'checker'), provider('clang', 'compiler')], + }); + scheduler.update(doc(1)); + await vi.advanceTimersByTimeAsync(49); + expect(runs).toEqual([]); + await vi.advanceTimersByTimeAsync(1); + expect(runs).toEqual(['lezer@1']); + await vi.advanceTimersByTimeAsync(200); + scheduler.update(doc(2)); + await vi.advanceTimersByTimeAsync(299); + expect(runs).toEqual(['lezer@1', 'lezer@2']); + await vi.advanceTimersByTimeAsync(1); + expect(runs).toEqual(['lezer@1', 'lezer@2', 'opennote@2']); + await vi.advanceTimersByTimeAsync(500); + expect(runs).toEqual(['lezer@1', 'lezer@2', 'opennote@2', 'clang@2']); + }); + + it('merges the layers, sorted, and runs only the languages a provider names', async () => { + const publish = vi.fn(); + const scheduler = createDiagnosticsScheduler({ + publish, + debounce: { syntax: 0, checker: 0 }, + providers: [ + { id: 'opennote', layer: 'checker', languages: ['python'], run: () => [item('opennote', 5)] }, + { id: 'lezer', layer: 'syntax', languages: ['python'], run: () => [item('lezer', 2)] }, + { id: 'sqlite', layer: 'engine', languages: ['sql'], run: () => [item('sqlite', 0)] }, + ], + }); + scheduler.update(doc(1)); + await vi.advanceTimersByTimeAsync(0); + expect(scheduler.current('main.py').map((d) => d.source)).toEqual(['lezer', 'opennote']); + }); +}); + +describe('diagnostics scheduler upkeep', () => { + beforeEach(() => vi.useFakeTimers()); + afterEach(() => vi.useRealTimers()); + + it('keeps shown problems on their text through a mapped change until the layers answer', () => { + const scheduler = createDiagnosticsScheduler({ publish: () => {} }); + scheduler.update(doc(1, 'abc')); + scheduler.accept({ file: 'main.py', version: 1, source: 'opennote', items: [item('opennote', 2)] }); + scheduler.update(doc(2, 'XXabc'), (pos) => pos + 2); + expect(scheduler.current('main.py').map((d) => d.from)).toEqual([4]); + scheduler.update(doc(3, 'abc')); + expect(scheduler.current('main.py')).toEqual([]); + }); + + it('reports a failing provider and keeps the others', async () => { + const onError = vi.fn(); + const scheduler = createDiagnosticsScheduler({ + publish: () => {}, + onError, + debounce: { checker: 0 }, + providers: [ + { id: 'bad', layer: 'checker', languages: ['python'], run: () => Promise.reject(new Error('boom')) }, + { id: 'good', layer: 'checker', languages: ['python'], run: () => [item('good', 1)] }, + ], + }); + scheduler.update(doc(1)); + await vi.advanceTimersByTimeAsync(0); + expect(onError).toHaveBeenCalledOnce(); + expect(scheduler.current('main.py').map((d) => d.source)).toEqual(['good']); + }); + + it('removes a provider and its marks, and forgets closed files', () => { + const publish = vi.fn(); + const scheduler = createDiagnosticsScheduler({ publish }); + const off = scheduler.register({ id: 'sqlite', layer: 'engine', languages: ['sql'], run: () => [] }); + scheduler.update(doc(1)); + scheduler.accept({ file: 'main.py', version: 1, source: 'sqlite', items: [item('sqlite', 0)] }); + off(); + expect(publish).toHaveBeenLastCalledWith('main.py', []); + scheduler.close('main.py'); + expect(scheduler.accept({ file: 'main.py', version: 1, source: 'x', items: [] })).toBe(false); + }); +}); diff --git a/app/src/features/codestudio/diagnostics/scheduler.ts b/app/src/features/codestudio/diagnostics/scheduler.ts new file mode 100644 index 000000000..1bedc4cc4 --- /dev/null +++ b/app/src/features/codestudio/diagnostics/scheduler.ts @@ -0,0 +1,138 @@ +// The diagnostics scheduler (DESIGN 6.1, ADR 0074). Each change of a file starts one timer per layer; when a layer's +// pause is over, its providers read that version. Every result carries the version it read, and a result for an +// older version is dropped (F18), so a slow checker can never paint marks on text that has moved. + +import type { Diagnostic, DiagnosticResult } from '../contracts'; +import { LAYER_DEBOUNCE_MS, LAYERS, type DiagnosticProvider, type DiagnosticRequest, type Layer } from './types'; + +/** Maps an offset of the previous version to the new one, as CodeMirror's `ChangeSet.mapPos` does. */ +export type MapPos = (pos: number, assoc: -1 | 1) => number; + +export interface SchedulerOptions { + providers?: readonly DiagnosticProvider[]; + /** Gets a file's problems each time they change: every layer's marks in one sorted list. */ + publish(file: string, items: Diagnostic[]): void; + /** Overrides the pause of some layers, in milliseconds. */ + debounce?: Partial>; + /** Hears a provider that failed. The other layers carry on. */ + onError?(provider: DiagnosticProvider, error: unknown): void; +} + +export interface DiagnosticsScheduler { + /** A new version of a file. `map` keeps the shown problems on their text until the layers answer again. */ + update(request: DiagnosticRequest, map?: MapPos): void; + /** Takes a provider's result. Returns false, and changes nothing, when the file has moved past its version. */ + accept(result: DiagnosticResult): boolean; + register(provider: DiagnosticProvider): () => void; + /** The problems shown for a file now. */ + current(file: string): Diagnostic[]; + /** Forgets a closed file and its timers. */ + close(file: string): void; + dispose(): void; +} + +interface Held { + version: number; + items: Diagnostic[]; +} + +function compare(a: Diagnostic, b: Diagnostic): number { + return a.from - b.from || a.to - b.to || a.source.localeCompare(b.source); +} + +export function createDiagnosticsScheduler(options: SchedulerOptions): DiagnosticsScheduler { + const providers = new Set(options.providers ?? []); + const docs = new Map(); + const results = new Map>(); + const timers = new Map>(); + const pause = { ...LAYER_DEBOUNCE_MS, ...options.debounce }; + + const merged = (file: string): Diagnostic[] => { + const version = docs.get(file)?.version; + const held = [...(results.get(file)?.values() ?? [])].filter((h) => h.version === version); + return held.flatMap((h) => h.items).sort(compare); + }; + + const publish = (file: string) => options.publish(file, merged(file)); + + const accept = (result: DiagnosticResult): boolean => { + const doc = docs.get(result.file); + if (!doc || doc.version !== result.version) return false; + let bySource = results.get(result.file); + if (!bySource) results.set(result.file, (bySource = new Map())); + bySource.set(result.source, { version: result.version, items: result.items }); + publish(result.file); + return true; + }; + + const runLayer = (request: DiagnosticRequest, layer: Layer) => { + for (const provider of providers) { + if (provider.layer !== layer || !provider.languages.includes(request.language)) continue; + Promise.resolve() + .then(() => provider.run(request)) + .then( + (items) => accept({ file: request.file, version: request.version, source: provider.id, items }), + (error: unknown) => options.onError?.(provider, error), + ); + } + }; + + const clearTimers = (file: string) => { + for (const layer of LAYERS) { + const key = `${layer}\u0000${file}`; + const timer = timers.get(key); + if (timer !== undefined) clearTimeout(timer); + timers.delete(key); + } + }; + + const remap = (file: string, version: number, map: MapPos) => { + for (const held of results.get(file)?.values() ?? []) { + held.items = held.items.map((item) => { + const from = map(item.from, 1); + return { ...item, from, to: Math.max(from, map(item.to, -1)) }; + }); + held.version = version; + } + }; + + return { + update(request, map) { + const before = docs.get(request.file); + docs.set(request.file, request); + if (map && before) remap(request.file, request.version, map); + clearTimers(request.file); + const layers = new Set([...providers].filter((p) => p.languages.includes(request.language)).map((p) => p.layer)); + for (const layer of layers) { + timers.set( + `${layer}\u0000${request.file}`, + setTimeout(() => { + timers.delete(`${layer}\u0000${request.file}`); + runLayer(request, layer); + }, pause[layer]), + ); + } + }, + accept, + register(provider) { + providers.add(provider); + return () => { + providers.delete(provider); + for (const [file, bySource] of results) if (bySource.delete(provider.id)) publish(file); + }; + }, + current: merged, + close(file) { + clearTimers(file); + docs.delete(file); + results.delete(file); + }, + dispose() { + for (const timer of timers.values()) clearTimeout(timer); + timers.clear(); + docs.clear(); + results.clear(); + providers.clear(); + }, + }; +} diff --git a/app/src/features/codestudio/diagnostics/scopes.test.ts b/app/src/features/codestudio/diagnostics/scopes.test.ts new file mode 100644 index 000000000..1b024976f --- /dev/null +++ b/app/src/features/codestudio/diagnostics/scopes.test.ts @@ -0,0 +1,294 @@ +// The scope checkers for C, C++, and Lua. The expected answers are worked by hand from each language's scope rules. +// In C11 section 6.2.1 and C++ [basic.scope], a name is visible from its declaration to the end of its block. +// In the Lua 5.4 manual section 3.5, a local is visible from the statement after its declaration to the end of its +// block. + +import { parser as cppParser } from '@lezer/cpp'; +import { describe, expect, it } from 'vitest'; +import { checkCLike } from './clike'; +import { checkLua } from './lua'; +import { lineAt, lineStarts } from './text'; + +const join = (...rows: string[]) => rows.join('\n') + '\n'; + +function c(text: string): string[] { + const starts = lineStarts(text); + return checkCLike(text, cppParser.parse(text)).map((f) => `${f.code}:${lineAt(starts, f.from)}:${f.params?.name}`); +} + +function lua(text: string): string[] { + const starts = lineStarts(text); + return checkLua(text).map((f) => `${f.code}:${lineAt(starts, f.from)}:${f.params?.name}`); +} + +describe('C and C++ scope checker', () => { + it('finds an undeclared identifier and an unused local', () => { + const text = join( + '#include ', + '#define LIMIT 10', + 'int total = 0;', + 'int add(int a, int b) {', + ' int unused = 0;', + ' int sum = a + b + count;', + ' return sum + LIMIT + total;', + '}', + ); + expect(c(text)).toEqual(['unused-variable:5:unused', 'undeclared-identifier:6:count']); + }); + + it('ends a block variable at its closing brace', () => { + const text = join('int f(void) {', ' { int inner = 1; inner++; }', ' return inner;', '}'); + expect(c(text)).toEqual(['undeclared-identifier:3:inner']); + }); + + it('leaves calls, members, enums, loop variables, and std names alone', () => { + const text = join( + '#include ', + 'using namespace std;', + 'enum Color { RED, GREEN };', + 'struct P { int x; };', + 'class K { int m; void set() { m = RED; } };', + 'int main() {', + ' P p; p.x = GREEN;', + ' for (int i = 0; i < 3; i++) cout << helper(i) << endl;', + ' return EXIT_SUCCESS;', + '}', + ); + expect(c(text)).toEqual([]); + }); + + it('stays quiet about undeclared names when a project header may declare them', () => { + expect(c(join('#include "shapes.h"', 'int f(void) { return SIDES; }'))).toEqual([]); + }); + + it('does not report a class object or a marked variable as unused', () => { + const text = join( + '#include ', + 'std::mutex m;', + 'void f() {', + ' std::lock_guard lock(m);', + ' [[maybe_unused]] int spare = 0;', + '}', + ); + expect(c(text)).toEqual([]); + }); + + it('leaves macro names of an included system header to clang', () => { + const text = join( + '#include ', + '#include ', + '#include ', + '#include ', + 'int f(void) {', + ' signal(SIGINT, SIG_IGN);', + ' unsigned long m = UINT32_MAX + INT64_MAX + O_RDONLY;', + ' return errno == ERANGE ? (int)m : 0;', + '}', + ); + expect(c(text)).toEqual([]); + expect(c(join('int f(void) { return ERANGE; }'))).toEqual(['undeclared-identifier:1:ERANGE']); + }); + + it('declares function pointer variables and tables', () => { + const text = join( + 'int add(int a, int b) { return a + b; }', + 'int apply(int (*g)(int, int), int x) { return g(x, x); }', + 'int f(void) {', + ' int (*fp)(int, int) = add;', + ' int (*table[2])(int, int) = { add, add };', + ' return apply(fp, 1) + table[0](1, 2);', + '}', + ); + expect(c(text)).toEqual([]); + }); + + it('declares the names of structured bindings and range loops', () => { + const text = join( + '#include ', + 'int f(std::map m) {', + ' int total = 0;', + ' for (auto& [key, val] : m) total += val + key;', + ' for (int x : {1, 2}) total += x;', + ' auto [lo, hi] = std::pair(1, 2);', + ' return total + lo + hi;', + '}', + ); + expect(c(text)).toEqual([]); + }); +}); + +describe('C and C++ scope checker: declarations', () => { + it('declares default arguments, template parameters, and parameter packs', () => { + expect(c(join('int f(int a, int b = 2) { return a + b; }'))).toEqual([]); + expect(c(join('template T twice(T v) { return v * N; }'))).toEqual([]); + expect(c(join('template int twice() { return N * 2; }'))).toEqual([]); + expect(c(join('template auto sum(Ts... xs) { return (xs + ...); }'))).toEqual([]); + // The default value is a read, not a declaration. + expect(c(join('int f(int b = limit) { return b; }'))).toEqual(['undeclared-identifier:1:limit']); + }); + + it('declares names from using declarations, namespaces, and lambda init captures', () => { + const text = join( + '#include ', + 'using std::cout;', + 'using std::endl;', + 'namespace util { int twice(int x) { return x * 2; } }', + 'int main() {', + ' int y = 1;', + ' auto h = [&, extra = 1](int z) { return y + z + extra; };', + ' cout << util::twice(h(1)) << endl;', + ' return 0;', + '}', + ); + expect(c(text)).toEqual([]); + expect(c(join('namespace util { }', 'int f() { return 0; }'))).toEqual([]); + }); + + it('treats the arguments of a function-like macro of the file as declared', () => { + const cpp = join( + '#include ', + 'using namespace std;', + '#define REP(i, n) for (int i = 0; i < (n); i++)', + 'int a[5];', + 'int main() {', + ' int n = 3;', + ' REP(i, n) cin >> a[i];', + ' return 0;', + '}', + ); + expect(c(cpp)).toEqual([]); + const cText = join( + '#include ', + '#define FOR(i, n) for (int i = 0; i < (n); i++)', + 'int main(void) {', + ' FOR(i, 3) printf("%d", i);', + ' return 0;', + '}', + ); + expect(c(cText)).toEqual([]); + }); + + it('declares a variable the grammar reads as a function definition', () => { + const text = join( + '#include ', + 'int len(std::array a);', + 'int f() {', + ' std::array b{};', + ' return len(b);', + '}', + ); + expect(c(text)).toEqual([]); + }); +}); + +describe('Lua scope checker', () => { + it('finds an undefined global and an unused local', () => { + const text = join( + 'local count = 0', + 'local spare = 1', + 'for i = 1, 10 do', + ' count = count + i', + 'end', + 'print(count, totl)', + ); + expect(lua(text)).toEqual(['W211:2:spare', 'W113:6:totl']); + }); + + it('knows globals the file assigns, functions, methods, and table keys', () => { + const text = join( + 'function greet(name)', + ' return "hi " .. name', + 'end', + 'Point = {}', + 'function Point:len()', + ' return math.sqrt(self.x ^ 2 + self.y ^ 2)', + 'end', + 'local t = { x = 1, y = 2 }', + 'setmetatable(t, { __index = Point })', + 'print(greet("a"), t:len())', + ); + expect(lua(text)).toEqual([]); + }); + + it('ends a local at the end of its block, and reads the outer name in its own initializer', () => { + const text = join('local x = 1', 'do', ' local y = x', ' local x = x + y', ' print(x)', 'end', 'print(y)'); + expect(lua(text)).toEqual(['W113:7:y']); + }); + + it('sees a global assigned after a line that ends in true, false, nil, ..., or end', () => { + expect(lua(join('debug_mode = false', 'counter = 0', 'print(counter, debug_mode)'))).toEqual([]); + const config = join( + 'fullscreen = false', + 'width = 800', + 'title = nil', + 'height = 600', + 'vsync = true', + 'depth = 24', + ); + expect(lua(config + 'print(fullscreen, width, title, height, vsync, depth)\n')).toEqual([]); + expect(lua(join('function f(...) local a = ...', ' b = 1', ' return a, b', 'end'))).toEqual([]); + }); + + it('reads one-line bodies as statements', () => { + expect(lua(join('function init() state = 1 end', 'init()', 'print(state)'))).toEqual([]); + expect(lua(join('for i = 1, 3 do local v = i * 2 print(v) end'))).toEqual([]); + expect(lua(join('local x = 1 return x'))).toEqual([]); + }); +}); + +describe('Lua scope checker: function values', () => { + it('declares a local whose value holds a function in the scope of the local statement', () => { + expect(lua(join('local f = function(x) return x end', 'print(f(1))'))).toEqual([]); + expect(lua(join('local f = function(x)', ' return x', 'end', 'print(f(1))'))).toEqual([]); + expect(lua(join("local ok, err = pcall(function() error('x') end)", 'print(ok, err)'))).toEqual([]); + expect(lua(join('local ok, err = pcall(function()', " error('x')", 'end)', 'print(ok, err)'))).toEqual([]); + const meta = 'local obj = setmetatable({}, { __index = function(_, k) return k end })'; + expect(lua(join(meta, 'print(obj.a)'))).toEqual([]); + expect( + lua( + join( + 'local obj = setmetatable({}, {', + ' __index = function(_, k)', + ' return k', + ' end,', + '})', + 'print(obj.a)', + ), + ), + ).toEqual([]); + expect(lua(join('local co = coroutine.wrap(function() coroutine.yield(1) end)', 'print(co())'))).toEqual([]); + expect(lua(join('local co = coroutine.wrap(function()', ' coroutine.yield(1)', 'end)', 'print(co())'))).toEqual( + [], + ); + expect( + lua( + join( + 'local counter = (function() local n = 0 return function() n = n + 1 return n end end)()', + 'print(counter())', + ), + ), + ).toEqual([]); + expect( + lua( + join( + 'local counter = (function()', + ' local n = 0', + ' return function()', + ' n = n + 1', + ' return n', + ' end', + 'end)()', + 'print(counter())', + ), + ), + ).toEqual([]); + }); + + it('keeps a repeat body visible to its until condition', () => { + expect(lua(join('repeat', ' local done = true', 'until done'))).toEqual([]); + }); + + it('ignores names in strings and comments', () => { + expect(lua(join('-- uses nothing_here', 'print("also_nothing")', '--[[ block', 'gone ]]'))).toEqual([]); + }); +}); diff --git a/app/src/features/codestudio/diagnostics/syntax.test.ts b/app/src/features/codestudio/diagnostics/syntax.test.ts new file mode 100644 index 000000000..2d51fdd24 --- /dev/null +++ b/app/src/features/codestudio/diagnostics/syntax.test.ts @@ -0,0 +1,139 @@ +import { describe, expect, it } from 'vitest'; +import { analyze } from './analyze'; +import { GRAMMAR_LANGUAGES, loadParser } from './parsers'; +import { loadWorkerParser } from './workerParsers'; +import { toDiagnostic } from './providers'; +import { lineAt, lineStarts } from './text'; +import type { EditorLanguage } from './types'; + +async function syntax(language: EditorLanguage, text: string) { + const { syntax: found } = await analyze({ language, text, layers: ['syntax'] }); + const starts = lineStarts(text); + return found.map((f) => ({ line: lineAt(starts, f.from), text: toDiagnostic('f', 'lezer', f).message })); +} + +describe('Lezer syntax errors', () => { + it('has a grammar for every tier-1 language with a Lezer pack', async () => { + expect([...GRAMMAR_LANGUAGES].sort()).toEqual([ + 'c', + 'cpp', + 'go', + 'java', + 'javascript', + 'python', + 'rust', + 'sql', + 'typescript', + ]); + for (const language of GRAMMAR_LANGUAGES) { + expect(typeof (await loadParser(language))?.parse).toBe('function'); + expect(typeof (await loadWorkerParser(language))?.parse).toBe('function'); + } + expect(loadParser('lua')).toBeNull(); + expect(loadWorkerParser('lua')).toBeNull(); + }, 60_000); + + it("says a Python block header lacks its ':'", async () => { + expect(await syntax('python', 'x = 1\nif x > 0\n print(x)\n')).toEqual([ + { line: 2, text: "Expected ':' at the end of the 'if' line." }, + ]); + expect(await syntax('python', 'def area(r)\n return r\n')).toEqual([ + { line: 1, text: "Expected ':' at the end of the 'def' line." }, + ]); + }); + + it("marks the real error line, not a one-line compound statement that has its ':'", async () => { + const lines = async (text: string) => (await syntax('python', text)).map((f) => f.line); + const colonFree = async (text: string) => + (await syntax('python', text)).every((f) => !f.text.startsWith("Expected ':'")); + const cases: [string, number][] = [ + ['def fib(n):\n if n < 2: return n\n return fib(n - 1) +\n', 3], + ['for i in range(3): print(i)\ny = 5 +\n', 2], + ['while True: break\nprint("a" "b" 3)\n', 2], + ['class E(Exception): pass\ntry: x = 1\nexcept E: pass\nz = 1 2\n', 4], + ['match(m)\nx = = 1\n', 2], + ]; + for (const [text, line] of cases) { + expect(await lines(text), text).toEqual([line]); + expect(await colonFree(text), text).toBe(true); + } + }); + + it("says a match or case header lacks its ':' only inside a match statement", async () => { + expect(await syntax('python', 'match x\n case 1:\n pass\n')).toEqual([ + { line: 1, text: "Expected ':' at the end of the 'match' line." }, + ]); + expect(await syntax('python', 'match x:\n case 1\n pass\n')).toEqual([ + { line: 2, text: "Expected ':' at the end of the 'case' line." }, + ]); + }); +}); + +describe('Lezer syntax errors: brackets and tokens', () => { + it('names the line where an unclosed bracket opened', async () => { + expect(await syntax('python', 'total = sum([1, 2, 3)\nprint(total)\n')).toEqual([ + { line: 1, text: "')' does not match '[' opened on line 1." }, + ]); + expect(await syntax('javascript', 'function f() {\n if (x) {\n return 1;\n}\n')).toEqual([ + { line: 1, text: "'{' opened on line 1 is never closed." }, + ]); + expect(await syntax('c', 'int main(void) {\n return 0;\n}\n}\n')).toEqual([ + { line: 4, text: "'}' has no opening bracket." }, + ]); + }); + + it('ignores brackets inside strings and comments', async () => { + expect(await syntax('python', 'print("(") # )\n')).toEqual([]); + expect(await syntax('javascript', 'const s = "{"; // }\n/* ( */\n')).toEqual([]); + }); + + it('ignores brackets inside a regex, rune, char, text block, macro body, or quoted name', async () => { + const valid: [EditorLanguage, string][] = [ + ['javascript', 'const open = /\\(/;\nconst t = s.replace(/[()]/g, "");\nconst c = /[)\\]}]/.test(ch);\n'], + ['typescript', 'const open: RegExp = /\\{/;\n'], + ['go', "package main\nfunc f(c rune) bool {\n\treturn c == '(' || c == ']'\n}\n"], + ['c', "#define OPEN '{'\n#define CLOSE ')'\nint main(void) { return 0; }\n"], + ['cpp', "#define OPEN '['\nint main() { char c = '('; return c; }\n"], + ['java', 'class A {\n String s = """\n a ( b [\n """;\n char c = \'{\';\n}\n'], + ['rust', "fn main() { let c = '('; }\n"], + ['sql', 'SELECT "col(" FROM t;\n'], + ]; + for (const [language, text] of valid) expect(await syntax(language, text), language).toEqual([]); + }, 60_000); + + it('keeps checker marks below a line with a regex that holds a bracket', async () => { + const text = 'const open = /\\(/;\nconsole.log(open, missing_name);\n'; + const { checker } = await analyze({ language: 'javascript', text, layers: ['checker'] }); + expect(checker.map((f) => f.code)).toEqual(['no-undef']); + }); + + it('names the unexpected token, or the early end of the file', async () => { + expect(await syntax('javascript', 'let x = 1 +;\n')).toEqual([ + { line: 1, text: "Syntax error: ';' is not expected here." }, + ]); + expect(await syntax('sql', 'SELECT * FROM t WHERE;\n')).toHaveLength(0); + const end = await syntax('python', 'x = (1 +\n'); + expect(end.map((e) => e.text)).toEqual(["'(' opened on line 1 is never closed."]); + }); + + it('finds nothing in valid code of each grammar', async () => { + const valid: [EditorLanguage, string][] = [ + ['python', 'def f(x):\n return [i for i in range(x)]\n'], + ['javascript', 'const f = (x) => ({ a: [x] });\n'], + ['typescript', 'function f(x: T): T[] { return [x]; }\n'], + ['c', '#include \nint main(void) { printf("%d\\n", 1); return 0; }\n'], + ['cpp', '#include \nint main() { std::vector v{1, 2}; return v.size(); }\n'], + ['java', 'class A { int f() { return 1; } }\n'], + ['rust', 'fn main() { let v = vec![1, 2]; println!("{}", v.len()); }\n'], + ['go', 'package main\nfunc main() { x := []int{1}; _ = x }\n'], + ['sql', 'SELECT name FROM people WHERE age > 3;\n'], + ]; + for (const [language, text] of valid) expect(await syntax(language, text), language).toEqual([]); + }, 60_000); + + it('keeps checker marks only above the first broken line', async () => { + const text = 'import os\nx = 1\nif x\n y = undefined_name\n'; + const { checker } = await analyze({ language: 'python', text, layers: ['checker'] }); + expect(checker.map((f) => f.code)).toEqual(['F401']); + }); +}); diff --git a/app/src/features/codestudio/diagnostics/syntax.ts b/app/src/features/codestudio/diagnostics/syntax.ts new file mode 100644 index 000000000..504a38bec --- /dev/null +++ b/app/src/features/codestudio/diagnostics/syntax.ts @@ -0,0 +1,204 @@ +// Layer L-a: syntax errors from the Lezer tree (DESIGN 6.1). Lezer marks each place it could not parse with an error +// node. This turns those places into messages that say more than "syntax error" where the context allows. It names +// a bracket that is never closed and the line it opened on, a closing bracket with no opener, a Python block header +// without its colon, and the end of the file reached too early. + +import type { SyntaxNode, Tree } from '@lezer/common'; +import { lineAt, lineRange, lineStarts } from './text'; +import type { EditorLanguage, Finding } from './types'; + +/** At most this many syntax errors per file: after the first few, the rest are mostly echoes. */ +export const MAX_SYNTAX_ERRORS = 10; + +const OPAQUE = /String|Comment|Char|Template|Regex|Docstring|Literal$/; +const BRACKETS = new Set(['(', ')', '[', ']', '{', '}']); +const PAIRS: Record = { ')': '(', ']': '[', '}': '{' }; +const PY_HEADER = /^(?:async\s+)?(if|elif|else|while|for|def|class|try|except|finally|with|match|case)\b(?!\s*[=.])/; + +/** + * Ranges the bracket scan skips. Only a bracket token of the tree, or a bracket inside an error node, counts. So every + * other leaf token is skipped: a regex, a rune, a macro body, a text block, or a quoted identifier holds text, not + * code. Strings, comments, and templates are skipped whole, with the parts inside them. + */ +function opaqueRanges(tree: Tree): [number, number][] { + const ranges: [number, number][] = []; + const cursor = tree.cursor(); + for (;;) { + const error = cursor.type.isError; + const literal = + !error && OPAQUE.test(cursor.name) && cursor.name !== 'BooleanLiteral' && cursor.name !== 'NullLiteral'; + if (literal) ranges.push([cursor.from, cursor.to]); + else if (cursor.firstChild()) continue; + else if (!error && cursor.to > cursor.from && !BRACKETS.has(cursor.name)) ranges.push([cursor.from, cursor.to]); + while (!cursor.nextSibling()) if (!cursor.parent()) return ranges; + } +} + +/** Brackets that do not pair up. Strings and comments do not count. */ +export function bracketFindings(text: string, tree: Tree): Finding[] { + const skip = opaqueRanges(tree); + const starts = lineStarts(text); + const stack: { char: string; at: number }[] = []; + const found: Finding[] = []; + let r = 0; + for (let i = 0; i < text.length; i += 1) { + while (r < skip.length && skip[r][1] <= i) r += 1; + if (r < skip.length && skip[r][0] <= i) { + i = skip[r][1] - 1; + continue; + } + const ch = text[i]; + if (ch === '(' || ch === '[' || ch === '{') stack.push({ char: ch, at: i }); + else if (ch in PAIRS) { + const open = stack.pop(); + if (!open) { + found.push({ + from: i, + to: i + 1, + severity: 'error', + code: 'syntax', + message: 'unexpectedClose', + params: { close: ch }, + }); + } else if (open.char !== PAIRS[ch]) { + found.push({ + from: i, + to: i + 1, + severity: 'error', + code: 'syntax', + message: 'mismatched', + params: { close: ch, open: open.char, line: lineAt(starts, open.at) }, + }); + return found; + } + } + } + for (const open of stack.reverse()) { + found.push({ + from: open.at, + to: open.at + 1, + severity: 'error', + code: 'syntax', + message: 'unclosed', + params: { open: open.char, line: lineAt(starts, open.at) }, + }); + } + return found; +} + +/** + * Reads one line of Python code: the code before its comment, and whether it has a `:` outside brackets and strings. + * `if n < 2: return n` has one; `x = d[1:2]` and `f"{a:3}"` do not. A walrus `:=` is not a colon. + */ +function scanPythonLine(line: string): { code: string; colon: boolean } { + let depth = 0; + let quote = ''; + let colon = false; + for (let i = 0; i < line.length; i += 1) { + const ch = line[i]; + if (quote) { + if (ch === '\\') i += 1; + else if (ch === quote) quote = ''; + } else if (ch === '"' || ch === "'") quote = ch; + else if (ch === '#') return { code: line.slice(0, i).trimEnd(), colon }; + else if (ch === '(' || ch === '[' || ch === '{') depth += 1; + else if (ch === ')' || ch === ']' || ch === '}') depth = Math.max(0, depth - 1); + else if (ch === ':' && depth === 0 && line[i + 1] !== '=') colon = true; + } + return { code: line.trimEnd(), colon }; +} + +/** True when Lezer read the line starting at `at` as part of a match statement. */ +function inMatch(tree: Tree, at: number): boolean { + for (let node: SyntaxNode | null = tree.resolveInner(at, 1); node; node = node.parent) { + if (node.name === 'MatchStatement') return true; + } + return false; +} + +/** + * A Python block header that lacks its colon, as the keyword and the line's end. The header is the error's line or + * the line before it. It counts only when it has no `:` of its own and the error sits at the end of its code, or + * after it but before the next token. A `match` or `case` line counts only when Lezer read a match statement. + */ +function missingColon( + text: string, + tree: Tree, + starts: readonly number[], + at: number, +): { keyword: string; at: number } | null { + const line = lineAt(starts, at); + for (const candidate of [line, line - 1]) { + if (candidate < 1) continue; + const { from, to } = lineRange(text, starts, candidate); + const { code, colon } = scanPythonLine(text.slice(from, to)); + const match = PY_HEADER.exec(code.trimStart()); + if (!match || colon || /[([{,\\]$/.test(code)) continue; + const end = from + code.length; + const next = end + (/^\s*/.exec(text.slice(end))?.[0].length ?? 0); + if (at < end || at > next) continue; + const keyword = match[1]; + if ((keyword === 'match' || keyword === 'case') && !inMatch(tree, from + code.length - code.trimStart().length)) { + continue; + } + return { keyword, at: end }; + } + return null; +} + +function describe(text: string, tree: Tree, language: EditorLanguage, starts: readonly number[], at: number): Finding { + if (language === 'python') { + const colon = missingColon(text, tree, starts, at); + if (colon) { + const from = Math.max(0, colon.at - 1); + return { + from, + to: colon.at, + severity: 'error', + code: 'syntax', + message: 'expectedColon', + params: { keyword: colon.keyword }, + }; + } + } + if (at >= text.trimEnd().length) { + const end = text.trimEnd().length; + return { from: Math.max(0, end - 1), to: end, severity: 'error', code: 'syntax', message: 'endOfFile' }; + } + const token = /^(?:\w+|\S)/.exec(text.slice(at).replace(/^[ \t]+/, '')); + if (!token || token[0] === '\n') { + return { from: at, to: Math.min(text.length, at + 1), severity: 'error', code: 'syntax', message: 'syntax' }; + } + const from = at + (text.slice(at).length - text.slice(at).replace(/^[ \t]+/, '').length); + return { + from, + to: from + token[0].length, + severity: 'error', + code: 'syntax', + message: 'syntaxAt', + params: { token: token[0] }, + }; +} + +/** The syntax errors in a parsed file, at most MAX_SYNTAX_ERRORS and at most one per line. */ +export function syntaxFindings(text: string, tree: Tree, language: EditorLanguage): Finding[] { + const brackets = bracketFindings(text, tree); + if (brackets.length) return brackets.slice(0, MAX_SYNTAX_ERRORS); + const starts = lineStarts(text); + const lines = new Set(); + const found: Finding[] = []; + tree.iterate({ + enter(node) { + if (found.length >= MAX_SYNTAX_ERRORS) return false; + if (!node.type.isError) return true; + const finding = describe(text, tree, language, starts, node.from); + const line = lineAt(starts, finding.from); + if (!lines.has(line)) { + lines.add(line); + found.push(finding); + } + return false; + }, + }); + return found; +} diff --git a/app/src/features/codestudio/diagnostics/text.ts b/app/src/features/codestudio/diagnostics/text.ts new file mode 100644 index 000000000..452f4cf15 --- /dev/null +++ b/app/src/features/codestudio/diagnostics/text.ts @@ -0,0 +1,29 @@ +// Line helpers for checkers that report offsets but word their messages in lines. + +/** The offset where each line starts. */ +export function lineStarts(text: string): number[] { + const starts = [0]; + for (let i = 0; i < text.length; i += 1) if (text.charCodeAt(i) === 10) starts.push(i + 1); + return starts; +} + +/** The 1-based line of `offset`. */ +export function lineAt(starts: readonly number[], offset: number): number { + let low = 0; + let high = starts.length - 1; + while (low < high) { + const mid = (low + high + 1) >> 1; + if (starts[mid] <= offset) low = mid; + else high = mid - 1; + } + return low + 1; +} + +/** The offset range of a 1-based line, without its line break. */ +export function lineRange(text: string, starts: readonly number[], line: number): { from: number; to: number } { + const from = starts[Math.max(0, Math.min(line - 1, starts.length - 1))]; + let to = text.indexOf('\n', from); + if (to < 0) to = text.length; + if (to > from && text.charCodeAt(to - 1) === 13) to -= 1; + return { from, to }; +} diff --git a/app/src/features/codestudio/diagnostics/types.ts b/app/src/features/codestudio/diagnostics/types.ts new file mode 100644 index 000000000..7cd4335fe --- /dev/null +++ b/app/src/features/codestudio/diagnostics/types.ts @@ -0,0 +1,84 @@ +// Shapes the diagnostics layers share (DESIGN 6.1, ADR 0074). The checkers run in a worker, so they report a +// message key and its values; the page turns them into text with the app's strings. + +import type { Diagnostic, Severity } from '../contracts'; +import type { EditorLanguage } from '../editor/indent'; + +export type { EditorLanguage }; + +/** + * The layers of DESIGN 6.1, cheapest first. `syntax` is L-a, `checker` is L-b, `engine` is L-c, `tools` is the + * code tools pack (L-d), and `compiler` is clang (L-e). + */ +export const LAYERS = ['syntax', 'checker', 'engine', 'tools', 'compiler'] as const; +export type Layer = (typeof LAYERS)[number]; + +/** How long each layer waits after the last change (DESIGN 6.1). Syntax marks must show within 100 ms. */ +export const LAYER_DEBOUNCE_MS: Readonly> = { + syntax: 50, + checker: 300, + engine: 300, + tools: 500, + compiler: 800, +}; + +/** One file at one version, as a provider reads it. */ +export interface DiagnosticRequest { + /** The project path. */ + file: string; + version: number; + language: EditorLanguage; + text: string; + /** The other files of the project, for engine checks that read more than one file. */ + files?: readonly { path: string; text: string }[]; +} + +/** A provider finds problems in one layer for some languages. */ +export interface DiagnosticProvider { + /** Also the `source` of its results, such as `lezer` or `opennote`. */ + id: string; + layer: Layer; + languages: readonly EditorLanguage[]; + run(request: DiagnosticRequest): Promise | Diagnostic[]; +} + +/** The checker message ids. Each one has a sentence in strings/en/codeStudio/diagnostics.ts. */ +export type MessageId = + | 'syntax' + | 'syntaxAt' + | 'endOfFile' + | 'expectedColon' + | 'unclosed' + | 'unexpectedClose' + | 'mismatched' + | 'unusedImport' + | 'redefined' + | 'undefinedName' + | 'unusedLocal' + | 'noneComparison' + | 'boolComparison' + | 'isLiteral' + | 'tabIndent' + | 'mixedIndent' + | 'undefinedVar' + | 'unusedVar' + | 'constAssign' + | 'unreachable' + | 'eqeqeq' + | 'condAssign' + | 'fallthrough' + | 'undeclared' + | 'unusedVariable' + | 'luaUndefined' + | 'luaUnused'; + +/** What a checker found, before the page words it. */ +export interface Finding { + from: number; + to: number; + severity: Severity; + /** The familiar rule name: a pyflakes or pycodestyle code, an ESLint rule, a clang warning, or a luacheck code. */ + code: string; + message: MessageId; + params?: Record; +} diff --git a/app/src/features/codestudio/diagnostics/workerParsers.ts b/app/src/features/codestudio/diagnostics/workerParsers.ts new file mode 100644 index 000000000..15c1ce7f5 --- /dev/null +++ b/app/src/features/codestudio/diagnostics/workerParsers.ts @@ -0,0 +1,16 @@ +// The diagnostics worker's parsers: the bare Lezer grammars, without the editor's CodeMirror packages, which a worker +// has no use for. They are the same grammars the language packs wrap. SQL's grammar only ships inside its package. + +import { cachedLoader, type ParserLoader } from './parsers'; + +export const loadWorkerParser: ParserLoader = cachedLoader({ + python: () => import('@lezer/python').then((m) => m.parser), + javascript: () => import('@lezer/javascript').then((m) => m.parser), + typescript: () => import('@lezer/javascript').then((m) => m.parser.configure({ dialect: 'ts' })), + c: () => import('@lezer/cpp').then((m) => m.parser), + cpp: () => import('@lezer/cpp').then((m) => m.parser), + java: () => import('@lezer/java').then((m) => m.parser), + rust: () => import('@lezer/rust').then((m) => m.parser), + go: () => import('@lezer/go').then((m) => m.parser), + sql: () => import('@codemirror/lang-sql').then((m) => m.SQLite.language.parser), +}); diff --git a/app/src/features/codestudio/index.ts b/app/src/features/codestudio/index.ts index aeabbfba5..ecc4eaade 100644 --- a/app/src/features/codestudio/index.ts +++ b/app/src/features/codestudio/index.ts @@ -19,3 +19,9 @@ export const openCodeDownloads = () => import('./packs/store').then((module) => export const loadCodeEditor = () => import('./editor'); /** Loads code projects' chunk (features/codestudio/project, code:L02): the page card, the store, and the exchange. */ export const loadCodeProjects = () => import('./project'); + +/** Loads error detection (features/codestudio/diagnostics, code:L10): the scheduler, checkers, and engine checks. */ +export const loadDiagnostics = () => import('./diagnostics'); + +/** Loads completion without a model (features/codestudio/complete, code:L12): attachCompletion for an editor. */ +export const loadCompletion = () => import('./complete'); diff --git a/app/src/features/codestudio/run/adapters.ts b/app/src/features/codestudio/run/adapters.ts index 8ee79496c..f17279b1f 100644 --- a/app/src/features/codestudio/run/adapters.ts +++ b/app/src/features/codestudio/run/adapters.ts @@ -6,7 +6,12 @@ import type { RuntimeId } from '../contracts/run'; import type { AdapterRegistry } from './serve'; -export const ADAPTERS: AdapterRegistry = {}; +export const ADAPTERS: AdapterRegistry = { + 'python-quick': () => import('../runtimes/python').then((loaded) => loaded.pythonQuickAdapter), + 'python-full': () => import('../runtimes/python').then((loaded) => loaded.pythonFullAdapter), + javascript: () => import('../runtimes/js/adapter').then((loaded) => loaded.javascriptAdapter), + typescript: () => import('../runtimes/js/adapter').then((loaded) => loaded.typescriptAdapter), +}; /** * The extra globals each runtime's lockdown keeps (RuntimeAdapter.allowances), in one place so the run tests can diff --git a/app/src/features/codestudio/run/serve.test.ts b/app/src/features/codestudio/run/serve.test.ts index 77dff05e1..f8e858fd2 100644 --- a/app/src/features/codestudio/run/serve.test.ts +++ b/app/src/features/codestudio/run/serve.test.ts @@ -3,6 +3,10 @@ // - Output written before an adapter throws still reaches the page. // - A cut sends one character past the limit, so the page can tell a cut from output at the limit. // Lockdown and the Math.random seeding are stand-ins here: the real ones would change this test's own global. +// +// It also covers two run ends: +// - Output goes to the page as it is written, so a run that the time limit ends keeps what it printed. +// - A runtime that reads only prefilled input makes the run not interactive, so the console says so. import { describe, expect, it, vi } from 'vitest'; import type { RunEnd, RunRequest, RuntimeAdapter, WorkerMessage } from '../contracts/run'; @@ -15,6 +19,11 @@ vi.mock('./random', async (original) => ({ ...(await original()), seedMathRandom: () => undefined, })); +// JSPI as in WebView2, whatever Node has. +vi.mock('./stdin', async (original) => ({ + ...(await original()), + jspiAvailable: () => true, +})); /** A stand-in worker scope: records what the worker sent, and lets the test send as the page would. */ function scope() { @@ -90,3 +99,37 @@ describe('serve', () => { expect(await texts('1234567')).toEqual(['123456']); }); }); + +describe('serve, while the program runs', () => { + it('sends output as it is written, before the run ends', async () => { + const worker = scope(); + let release = () => undefined as void; + const waiting = new Promise((resolve) => (release = resolve)); + const looping = adapter(async (_, emit): Promise => { + emit({ kind: 'out', stream: 'out', text: 'before the loop\n' }); + await waiting; + return { status: 'ok', message: '' }; + }); + serve({ javascript: async () => looping }, worker); + worker.page(request); + await vi.waitFor(() => + expect(worker.sent).toContainEqual({ kind: 'out', stream: 'out', text: 'before the loop\n' }), + ); + expect(worker.sent.some((message) => message.kind === 'done')).toBe(false); + release(); + await ended(worker.sent); + }); + + it('reports a run as not interactive when the runtime reads only prefilled input', async () => { + const started = async (prefilledOnly: boolean) => { + const worker = scope(); + const reader: RuntimeAdapter = { ...adapter(async () => ({ status: 'ok', message: '' })), prefilledOnly }; + serve({ javascript: async () => reader }, worker); + worker.page({ ...request, stdin: { mode: 'interactive', text: '' } }); + const sent = await ended(worker.sent); + return sent.find((message) => message.kind === 'started'); + }; + expect(await started(false)).toMatchObject({ kind: 'started', interactive: true }); + expect(await started(true)).toMatchObject({ kind: 'started', interactive: false }); + }); +}); diff --git a/app/src/features/codestudio/run/serve.ts b/app/src/features/codestudio/run/serve.ts index 7328a0225..2ca25d223 100644 --- a/app/src/features/codestudio/run/serve.ts +++ b/app/src/features/codestudio/run/serve.ts @@ -15,6 +15,12 @@ export type AdapterRegistry = Partial>; /** Output goes to the page in pieces of this many characters. */ const PIECE = 4096; +/** + * Messages sent as soon as the program writes. A runtime that runs synchronously keeps the worker busy until the run + * ends, so output held back would be lost when the time limit or Stop ends the worker. Past this many messages, + * output goes in pieces, so a program that prints in a loop does not flood the page. + */ +const PROMPT_MESSAGES = 1000; /** Error output kept for the diagnostic mapper. */ const ERROR_TEXT = 64 * 1024; @@ -28,6 +34,7 @@ class OutputGate { private pending = ''; private stream: Stream = 'out'; private sent = 0; + private messages = 0; errorText = ''; truncated = false; @@ -49,7 +56,7 @@ class OutputGate { this.pending += text; } while (this.pending.length >= PIECE) this.flush(PIECE); - if (this.truncated) this.flush(); + if (this.truncated || this.messages < PROMPT_MESSAGES) this.flush(); } flush(size = this.pending.length): void { @@ -57,6 +64,7 @@ class OutputGate { const piece = this.pending.slice(0, size); this.pending = this.pending.slice(size); this.sent += piece.length; + this.messages += 1; this.send({ kind: 'out', stream: this.stream, text: piece }); } } @@ -103,7 +111,7 @@ export function serve(registry: AdapterRegistry, scope: WorkerScope = globalThis output.flush(); send(message); }; - const interactive = request.stdin.mode === 'interactive' && jspiAvailable(); + const interactive = request.stdin.mode === 'interactive' && jspiAvailable() && adapter.prefilledOnly !== true; const ask = (prompt: string) => new Promise((resolve) => { answer = resolve; diff --git a/app/src/features/codestudio/runtimes/js/adapter.ts b/app/src/features/codestudio/runtimes/js/adapter.ts new file mode 100644 index 000000000..c357d08d3 --- /dev/null +++ b/app/src/features/codestudio/runtimes/js/adapter.ts @@ -0,0 +1,52 @@ +// The JavaScript and TypeScript runtimes behind the RuntimeAdapter contract (DESIGN 2.1, 4). Both load QuickJS +// (quickjs-emscripten-core and @jitl/quickjs-wasmfile-release-sync, MIT, already shipped for code cells) and Sucrase +// (MIT) in a warm worker, before lockdown. They differ only in their name and version line: a project may mix +// .js and .ts files under either one. + +import { newQuickJSWASMModuleFromVariant, newVariant } from 'quickjs-emscripten-core'; +import releaseSync from '@jitl/quickjs-wasmfile-release-sync'; +import wasmUrl from '@jitl/quickjs-wasmfile-release-sync/wasm?url'; +import type { RuntimeAdapter } from '../../contracts/run'; +import { ALLOWANCES } from '../../run/adapters'; +import { RUN_LIMITS } from '../../run/protocol'; +import { quickjsMapper } from './diagnostics'; +import { cappedHeap, runProgram } from './engine'; +import type { Engine } from './engine'; + +/** The pinned versions in package.json (a unit test keeps them equal). */ +export const VERSIONS = { quickjs: '0.32.0', sucrase: '3.35.1' } as const; + +let engine: Engine | null = null; + +async function loadEngine(onProgress: (percent: number) => void): Promise { + if (engine) return engine; + onProgress(10); + // The worker runs one program, so its heap is capped at the page's memory limit (engine.ts). + const wasmMemory = cappedHeap(RUN_LIMITS.memoryBytes); + const quickjs = await newQuickJSWASMModuleFromVariant(newVariant(releaseSync, { wasmLocation: wasmUrl, wasmMemory })); + onProgress(70); + const { transform } = await import('sucrase'); + engine = { quickjs, transform }; + return engine; +} + +function adapter(id: 'javascript' | 'typescript', version: string): RuntimeAdapter { + return { + id, + version, + allowances: ALLOWANCES[id], + diagnostics: quickjsMapper, + async load(onProgress) { + await loadEngine(onProgress); + }, + async run(request, emit, context) { + return runProgram(await loadEngine(() => undefined), request, emit, context); + }, + }; +} + +export const javascriptAdapter = adapter('javascript', `QuickJS (quickjs-emscripten ${VERSIONS.quickjs})`); +export const typescriptAdapter = adapter( + 'typescript', + `TypeScript through Sucrase ${VERSIONS.sucrase}, QuickJS (quickjs-emscripten ${VERSIONS.quickjs})`, +); diff --git a/app/src/features/codestudio/runtimes/js/diagnostics.ts b/app/src/features/codestudio/runtimes/js/diagnostics.ts new file mode 100644 index 000000000..5bc4dcedd --- /dev/null +++ b/app/src/features/codestudio/runtimes/js/diagnostics.ts @@ -0,0 +1,107 @@ +// QuickJS errors as RuntimeDiagnostic (DESIGN 4.6). The runtime prints an uncaught error as its "Name: message" line +// followed by QuickJS's stack, whose frames read " at f (main.js:3:7)", or " at main.js:3:7" for a syntax +// error. The diagnostic goes on the first frame in a project file, so a frame of the loader or the test API is +// skipped. A syntax error Sucrase finds is printed the same way, with one frame. +// +// A project path may hold spaces and parentheses ("my file.ts", "main (1).js"). So a place is found from its end. +// That is ":line:column", and before it the longest project path that ends there. + +import type { DiagnosticMapper, RunFile, RuntimeDiagnostic } from '../../contracts/run'; + +const HEADER = /^(?[A-Za-z_$][\w$]*): (?.*)$/; +/** A name that starts a new error, not a line of the last one's message such as "Received: 1". */ +const ERROR_NAME = /(?:Error|Exception)$/; +const FRAME_START = /^\s+at /; +const FRAME_END = /:(\d+)(?::(\d+))?\)?\s*$/; +const INLINE_END = /:(\d+):(\d+)\)\s*$/; + +/** How many diagnostics one run may report, as in run/diagnostics.ts. */ +const MAX = 100; + +type Place = { file: string; line: number; column?: number }; + +/** The longest project path that `before` ends with, standing after one of `starts`. */ +export function pathAtEnd(before: string, files: readonly RunFile[], starts: string): RunFile | null { + let best: RunFile | null = null; + for (const file of files) { + if (!before.endsWith(file.path)) continue; + const ahead = before.length - file.path.length - 1; + if (ahead >= 0 && !starts.includes(before[ahead]!)) continue; + if (!best || file.path.length > best.path.length) best = file; + } + return best; +} + +function placeIn(file: RunFile, lineText: string, columnText: string | undefined): Place | null { + const line = Number(lineText); + if (!Number.isInteger(line) || line < 1) return null; + const column = Number(columnText); + return { + file: file.path, + line: Math.min(line, file.text.split('\n').length), + ...(Number.isInteger(column) && column >= 1 ? { column } : {}), + }; +} + +function framePlace(text: string, files: readonly RunFile[]): Place | null { + if (!FRAME_START.test(text)) return null; + const end = FRAME_END.exec(text); + if (!end) return null; + const file = pathAtEnd(text.slice(0, end.index), files, ' ('); + return file ? placeIn(file, end[1]!, end[2]) : null; +} + +/** "... (util.ts:9:5)" at the end of a line: the place, and where its "(" starts. */ +function inlineAt(text: string, files: readonly RunFile[]): { place: Place; index: number } | null { + const end = INLINE_END.exec(text); + if (!end) return null; + const before = text.slice(0, end.index); + const file = pathAtEnd(before, files, '('); + const place = file ? placeIn(file, end[1]!, end[2]) : null; + return place && file ? { place, index: before.length - file.path.length - 1 } : null; +} + +/** The first frame of `stack` in a project file, as a place. */ +export function firstProjectFrame(stack: string, files: readonly RunFile[]): Place | null { + for (const text of stack.split('\n')) { + const place = framePlace(text, files); + if (place) return place; + } + return null; +} + +/** The place a message names at the end of one of its lines, "... (util.ts:9:5)", when that is a project file. */ +export function inlinePlace(message: string, files: readonly RunFile[]): Place | null { + for (const text of message.split('\n')) { + const found = inlineAt(text, files); + if (found) return found.place; + } + return null; +} + +export const quickjsMapper: DiagnosticMapper = { + map(errorText, files) { + const found: RuntimeDiagnostic[] = []; + let header: { kind: string; message: string } | null = null; + for (const text of errorText.split('\n')) { + const head = /^\s/.test(text) ? undefined : HEADER.exec(text)?.groups; + // A header-shaped line inside a message ("Expected: 2") belongs to the error above it. + if (head && (!header || ERROR_NAME.test(head.kind ?? ''))) { + header = { kind: head.kind ?? 'Error', message: head.message ?? '' }; + // A syntax error in an imported file names its place at the end of the message: "... (util.ts:9:5)". + const inline = inlineAt(header.message, files); + if (!inline) continue; + found.push({ ...inline.place, message: header.message.slice(0, inline.index).trimEnd(), kind: header.kind }); + header = null; + if (found.length >= MAX) break; + continue; + } + const place = header ? framePlace(text, files) : null; + if (!header || !place) continue; + found.push({ ...place, message: header.message, kind: header.kind }); + header = null; + if (found.length >= MAX) break; + } + return found; + }, +}; diff --git a/app/src/features/codestudio/runtimes/js/engine.ts b/app/src/features/codestudio/runtimes/js/engine.ts new file mode 100644 index 000000000..3bea2d4f5 --- /dev/null +++ b/app/src/features/codestudio/runtimes/js/engine.ts @@ -0,0 +1,693 @@ +// One run of a JavaScript or TypeScript program in QuickJS (DESIGN 2.1, 4). Each run gets a fresh QuickJS runtime +// with the page's memory, stack, and time limits. Project files are modules: Sucrase turns `import` into `require` +// (transpile.ts), and the loader below serves `require` from the mounted project (modules.ts). The main file runs +// as a script, so its last value is shown after an arrow, as in code cells. With a test harness that asks for +// framework tests, the test API (harness.ts) runs every *.test and *.spec file and reports one result per test. +// +// Limits. QuickJS's own memory limit cannot see the size of a block in this WebAssembly build. So the heap itself +// is capped (`cappedHeap`). The interrupt handler also ends the run when the heap has grown past the run's limit. +// The cap leaves some room above that limit, so the handler trips first and QuickJS can still unwind. +// +// Many small blocks can still fill the heap between two checks. QuickJS then cannot make its out-of-memory error +// and throws null. So any error while the heap is past the limit ends the run as "memory". +// +// The handler sets `limit` when it stops the run at the deadline or at the memory limit. An interrupt inside a +// promise job only rejects a promise, so the run checks `limit` after every batch of jobs. + +import type { QuickJSContext, QuickJSHandle, QuickJSRuntime, QuickJSWASMModule } from 'quickjs-emscripten-core'; +import type { Emit, RunContext, RunEnd, RunFile, RunRequest, Stream } from '../../contracts/run'; +import type { TestResult, TestStatus } from '../../contracts/testing'; +import { PROJECT_LIMITS } from '../../contracts/project'; +import { MESSAGE_LIMITS } from '../../run/protocol'; +import { t } from '../../../../strings/t'; +import { firstProjectFrame, inlinePlace, pathAtEnd } from './diagnostics'; +import { ENVIRONMENT } from './environment'; +import { TEST_API } from './harness'; +import { SCRIPT_FILE, resolveModule } from './modules'; +import { compile } from './transpile'; +import type { Compiled } from './transpile'; +import type { Transform } from './transpile'; + +export interface Engine { + quickjs: QuickJSWASMModule; + transform: Transform; +} + +/** Test files: run in place of `main` when a run asks for framework tests. */ +export const TEST_FILE = /\.(?:test|spec)\.[mc]?[jt]s$/; + +const PAGE_BYTES = 64 * 1024; +/** The heap the QuickJS release build starts with: Emscripten's default INITIAL_MEMORY, 16 MiB. */ +export const BASE_HEAP_BYTES = 16 * 1024 * 1024; + +/** + * Room in the heap above the run's limit, for QuickJS to stop and unwind once the limit has tripped. Emscripten + * grows the heap by a fifth of its size, then by less, down to a twentieth, and stops growing when even that does + * not fit under the cap. Room of an eighth of the limit's heap lets the heap always grow past the limit first. + */ +export function headroomBytes(memoryBytes: number): number { + return Math.max(2 * 1024 * 1024, Math.ceil((BASE_HEAP_BYTES + memoryBytes) / 8)); +} + +/** A WebAssembly heap for QuickJS that cannot grow more than `memoryBytes` and the headroom past its start. */ +export function cappedHeap(memoryBytes: number): WebAssembly.Memory { + return new WebAssembly.Memory({ + initial: BASE_HEAP_BYTES / PAGE_BYTES, + maximum: Math.ceil((BASE_HEAP_BYTES + memoryBytes + headroomBytes(memoryBytes)) / PAGE_BYTES), + }); +} + +/** + * Serves `require` inside QuickJS. `compile(path)` gives a module's function; `resolve(specifier, from)` gives the + * project path or throws. Modules are cached before they run, so an import cycle gets the exports made so far, + * as in Node. Returns the function that makes a `require` for a file. + */ +const LOADER = String.raw`(function (compile, resolve, root) { + 'use strict'; + const cache = Object.create(null); + function folder(path) { + const slash = path.lastIndexOf('/'); + return slash < 0 ? '' : path.slice(0, slash); + } + function requireFrom(from) { + return function require(specifier) { + const path = resolve(String(specifier), from); + if (cache[path]) return cache[path].exports; + const module = { id: path, exports: {} }; + cache[path] = module; + compile(path).call(module.exports, module.exports, requireFrom(path), module, path, folder(path)); + return module.exports; + }; + } + const main = { id: root, exports: {} }; + cache[root] = main; + globalThis.module = main; + globalThis.exports = main.exports; + globalThis.require = requireFrom(root); + return requireFrom; +})`; + +/** The wrapper around a module's text. It starts on the file's first line, so every line keeps its number. */ +const WRAPPER = '(function (exports, require, module, __filename, __dirname) {'; + +/** How long the test API may take to report the tests a stopped run did not finish. */ +const CLOSE_MS = 250; + +const STACK_OVERFLOW = 'InternalError: stack overflow (too much recursion)\n'; +const OUT_OF_MEMORY = /^InternalError: out of memory/; + +interface Thrown { + name?: unknown; + message?: unknown; + stack?: unknown; + /** The thrown value as console.log shows it, for a value that is not an error. */ + shown?: string; +} + +interface Reported { + name: string; + file: string; + status: TestStatus; + timeMs: number; + index?: number; + message?: string; + expected?: string; + actual?: string; + stack?: string; +} + +const cut = (text: string) => text.slice(0, MESSAGE_LIMITS.shortChars); + +/** A value as text, with no way to throw: JSON for a string, as Node shows a thrown string. */ +function safeText(value: unknown): string { + try { + if (typeof value === 'string') return JSON.stringify(value); + if (typeof value === 'bigint') return `${value}n`; + return String(value); + } catch { + return Object.prototype.toString.call(value); + } +} + +/** Runs one request. Never throws for the program's own errors; those end the run as "error". */ +export function runProgram(engine: Engine, request: RunRequest, emit: Emit, context: RunContext): RunEnd { + const runtime = engine.quickjs.newRuntime(); + // QuickJS's own count only backs up the heap cap. It is set above the cap, so the heap check always trips first. + // Set lower, it tripped before the heap passed the limit and threw null. + const { memoryBytes } = request.limits; + runtime.setMemoryLimit(BASE_HEAP_BYTES + memoryBytes + headroomBytes(memoryBytes)); + runtime.setMaxStackSize(request.limits.stackBytes); + const run = new Run(engine.transform, runtime, request, emit, context, engine.quickjs.getWasmMemory()); + runtime.setInterruptHandler(() => run.interrupt()); + try { + return run.start(); + } catch (error) { + // Deep recursion can use up the worker's own stack before QuickJS's limit trips; the browser then throws a + // RangeError out of the interpreter. It is the program's error, as in code cells. + if (error instanceof RangeError) { + emit({ kind: 'out', stream: 'err', text: STACK_OVERFLOW }); + return { status: 'error', message: 'InternalError', exitCode: 1 }; + } + throw error; + } finally { + try { + run.dispose(); + runtime.dispose(); + } catch { + // A runtime stopped at a limit may still hold values; the worker ends right after anyway. + } + } +} + +class Run { + readonly vm: QuickJSContext; + exitCode: number | undefined; + /** The limit that ended the run, set by the interrupt handler or by an out-of-memory error. */ + limit: 'time' | 'memory' | null = null; + /** While the host reports the end of a stopped test run, the interrupt handler lets QuickJS go on until `closeBy`. */ + private closing = false; + private closeBy = 0; + private readonly heapStart: number; + private inspect: QuickJSHandle | null = null; + private requireFrom: QuickJSHandle | null = null; + private results: TestResult[] = []; + private ids = new Map(); + private testsReported = 0; + /** Characters put before line 1 of each compiled file, taken off columns on line 1 in a stack. */ + private shifts = new Map(); + + constructor( + private transform: Transform, + private runtime: QuickJSRuntime, + private request: RunRequest, + private emit: Emit, + private context: RunContext, + private heap: WebAssembly.Memory, + ) { + this.heapStart = heap.buffer.byteLength; + this.vm = runtime.newContext(); + } + + dispose(): void { + this.inspect?.dispose(); + this.requireFrom?.dispose(); + this.vm.dispose(); + } + + /** The interrupt handler: true stops QuickJS. */ + interrupt(): boolean { + if (this.closing) return performance.now() > this.closeBy; + if (this.exitCode !== undefined || this.limit !== null) return true; + if (performance.now() > this.context.deadline) this.limit = 'time'; + else if (this.overMemory()) this.limit = 'memory'; + return this.limit !== null; + } + + /** Whether the heap has grown past the run's limit, from where this run started or from the base heap. */ + private overMemory(): boolean { + const bytes = this.heap.buffer.byteLength; + const limit = this.request.limits.memoryBytes; + return bytes - this.heapStart > limit || bytes > BASE_HEAP_BYTES + limit; + } + + private write(stream: Stream, text: string): void { + if (this.exitCode === undefined && text !== '') this.emit({ kind: 'out', stream, text }); + } + + /** A host function on the VM, disposed after `after`. */ + private withFunction( + name: string, + body: (...args: QuickJSHandle[]) => QuickJSHandle | { error: QuickJSHandle } | undefined, + after: (handle: QuickJSHandle) => T, + ): T { + const handle = this.vm.newFunction(name, body); + try { + return after(handle); + } finally { + handle.dispose(); + } + } + + /** Evaluates a prelude, a function in text, and calls it with `args`. Its result or the error. */ + private callPrelude(source: string, file: string, args: QuickJSHandle[]): { value: QuickJSHandle } | Thrown { + const made = this.vm.evalCode(source, file, { type: 'global' }); + if (made.error) return this.dumpError(made.error); + const called = this.vm.callFunction(made.value, this.vm.undefined, ...args); + made.value.dispose(); + if (called.error) return this.dumpError(called.error); + return { value: called.value }; + } + + /** A thrown value as host data. A value that is not an error also gets its console.log text. */ + private dumpError(handle: QuickJSHandle): Thrown { + const type = this.vm.typeof(handle); + let dumped: unknown; + try { + dumped = this.vm.dump(handle); + } catch { + dumped = undefined; + } + // dump() gives a string for an object JSON cannot hold, such as a cyclic one. + const object = type === 'object' && typeof dumped === 'object' && dumped !== null ? (dumped as Thrown) : null; + const plain = !object || (object.name === undefined && object.stack === undefined); + const shown = + plain && type !== 'string' && this.inspect !== null && this.limit === null && !this.overMemory() + ? this.show(handle) + : ''; + handle.dispose(); + const thrown: Thrown = object + ? { name: object.name, message: object.message, stack: object.stack } + : { message: type === 'object' ? undefined : dumped }; + return shown === '' ? thrown : { ...thrown, shown }; + } + + private setUp(): Thrown | null { + const vm = this.vm; + const stdin = this.context.stdin; + const argv = JSON.stringify(['opennote', this.request.main, ...this.request.args]); + const write = vm.newFunction('write', (stream, text) => { + this.write(vm.getString(stream) === 'err' ? 'err' : 'out', vm.getString(text)); + }); + const readLine = vm.newFunction('readLine', () => { + const line = stdin.line(); + return line === null ? vm.null : vm.newString(line); + }); + const exit = vm.newFunction('exit', (code) => { + this.exitCode = vm.getNumber(code); + throw new Error('exit'); + }); + const random = vm.newFunction('random', () => vm.newNumber(this.context.random())); + const argvText = vm.newString(argv); + const handles = [write, readLine, exit, random, argvText]; + const made = this.callPrelude(ENVIRONMENT, '', handles); + handles.forEach((handle) => handle.dispose()); + if (!('value' in made)) return made; + this.inspect = made.value; + + const loaded = this.withFunction( + 'compile', + (path) => this.compileModule(vm.getString(path)), + (compileFn) => + this.withFunction( + 'resolve', + (specifier, from) => vm.newString(this.resolve(vm.getString(specifier), vm.getString(from))), + (resolveFn) => { + // In a test run no file is the main script, so a test file that is also `main` still loads as a module. + const root = vm.newString(this.request.test?.framework ? '' : this.request.main); + try { + return this.callPrelude(LOADER, '', [compileFn, resolveFn, root]); + } finally { + root.dispose(); + } + }, + ), + ); + if (!('value' in loaded)) return loaded; + this.requireFrom = loaded.value; + return null; + } + + private resolve(specifier: string, from: string): string { + const found = resolveModule(specifier, from, (path) => this.context.files.read(path) !== null); + if ('path' in found) return found.path; + const where = from === '' ? this.request.main : from; + throw new Error( + found.missing === 'package' + ? t('codeStudio.js.modules.missingPackage', { specifier, from: where }) + : t('codeStudio.js.modules.missingFile', { specifier, from: where }), + ); + } + + /** Why a file cannot be compiled, in the words a person reads. */ + private compileMessage(compiled: Exclude): string { + if (compiled.unsupported === 'namespace') return t('codeStudio.js.typescript.namespace'); + if (compiled.unsupported === 'declarators') return t('codeStudio.js.typescript.declarators'); + return compiled.message; + } + + /** A module's function, from its file. Sucrase's syntax errors carry their place in the message. */ + private compileModule(path: string): QuickJSHandle | { error: QuickJSHandle } { + const text = this.context.files.read(path) ?? ''; + let code: string; + let shift = 0; + if (path.endsWith('.json')) { + code = `module.exports = JSON.parse(${JSON.stringify(text)});`; + } else if (SCRIPT_FILE.test(path)) { + const compiled = compile(path, text, this.transform); + if (!('code' in compiled)) { + const message = this.compileMessage(compiled); + throw new SyntaxError(`${message} (${path}:${compiled.line}:${compiled.column})`); + } + code = compiled.code; + shift = compiled.shift; + } else { + throw new Error(t('codeStudio.js.modules.notScript', { path })); + } + this.shifts.set(path, WRAPPER.length + shift); + const made = this.vm.evalCode(`${WRAPPER}${code}\n})`, path, { type: 'global' }); + return made.error ? { error: made.error } : made.value; + } + + /** `stack` with the columns on line 1 of compiled files moved back to the file's own columns. */ + private fileColumns(stack: string): string { + if (this.shifts.size === 0) return stack; + const paths = [...this.shifts.keys()].map((path) => ({ path, text: '' })); + return stack.replace(/^(\s+at .*):1:(\d+)(\)?\s*)$/gm, (whole, before: string, column: string, end: string) => { + const file = pathAtEnd(before, paths, ' ('); + const shift = file ? (this.shifts.get(file.path) ?? 0) : 0; + return shift === 0 ? whole : `${before}:1:${Math.max(1, Number(column) - shift)}${end}`; + }); + } + + start(): RunEnd { + const failed = this.setUp(); + if (failed) return this.failure(failed); + if (this.request.test?.framework) return this.runTests(); + return this.runMain(); + } + + private syntaxError(path: string, compiled: Exclude): RunEnd { + const message = this.compileMessage(compiled); + this.write('err', `SyntaxError: ${message}\n at ${path}:${compiled.line}:${compiled.column}\n`); + return { status: 'error', message: 'SyntaxError', exitCode: 1 }; + } + + private runMain(): RunEnd { + const { main } = this.request; + const text = this.context.files.read(main); + if (!SCRIPT_FILE.test(main) || text === null) { + const message = + text === null + ? t('codeStudio.js.modules.missingMain', { path: main }) + : t('codeStudio.js.modules.notScript', { path: main }); + this.write('err', `Error: ${message}\n`); + return { status: 'error', message: 'Error', exitCode: 1 }; + } + const compiled = compile(main, text, this.transform); + if (!('code' in compiled)) return this.syntaxError(main, compiled); + if (compiled.shift > 0) this.shifts.set(main, compiled.shift); + const result = this.vm.evalCode(compiled.code, main, { type: 'global' }); + if (result.error) return this.failure(this.dumpError(result.error)); + // Promises the program made settle now, inside the same limits. + const jobs = this.runtime.executePendingJobs(); + if (this.limit === null && this.overMemory()) this.limit = 'memory'; + if (this.limit !== null || this.exitCode !== undefined) { + if (jobs.error) jobs.error.dispose(); + result.value.dispose(); + return this.limit !== null ? this.limitEnd() : this.exited(); + } + if (jobs.error) { + result.value.dispose(); + return this.failure(this.dumpError(jobs.error)); + } + const state = this.vm.getPromiseState(result.value); + if (state.type === 'rejected') { + result.value.dispose(); + return this.failure(this.dumpError(state.error)); + } + let shown: string | null = null; + if (state.type === 'fulfilled') { + const value = state.value; + if (this.vm.typeof(value) !== 'undefined') shown = this.show(value); + if (!state.notAPromise) value.dispose(); + } + result.value.dispose(); + if (this.exitCode !== undefined) return this.exited(); + if (this.limit !== null) return this.limitEnd(); + if (shown !== null) this.write('out', `→ ${shown}\n`); + return { status: 'ok', message: '', exitCode: 0 }; + } + + /** A value as console.log shows it; empty when that fails. */ + private show(value: QuickJSHandle): string { + const shown = this.vm.callFunction(this.inspect!, this.vm.undefined, value); + if (shown.error) { + shown.error.dispose(); + return ''; + } + const text = this.vm.getString(shown.value); + shown.value.dispose(); + return text; + } + + private exited(): RunEnd { + const code = this.exitCode ?? 0; + return { status: code === 0 ? 'ok' : 'error', message: '', exitCode: code }; + } + + private limitEnd(): RunEnd { + if (this.limit === 'memory') { + this.write('err', 'InternalError: out of memory\n'); + return { status: 'memory', message: '' }; + } + return { status: 'time', message: '' }; + } + + private failure(thrown: Thrown): RunEnd { + if (this.exitCode !== undefined) return this.exited(); + // A full heap throws whatever it can, often null; that is the memory limit, not the program's error. + if (this.limit === null && this.overMemory()) this.limit = 'memory'; + if (this.limit !== null) return this.limitEnd(); + try { + return this.describe(thrown); + } catch { + // Formatting the error must never end the run as a runtime failure. + this.write('err', 'Uncaught exception\n'); + return { status: 'error', message: 'Error', exitCode: 1 }; + } + } + + private describe(thrown: Thrown): RunEnd { + const name = typeof thrown.name === 'string' ? thrown.name : ''; + const message = + typeof thrown.message === 'string' + ? thrown.message + : thrown.message === undefined + ? '' + : safeText(thrown.message); + if (name === 'InternalError' && OUT_OF_MEMORY.test(`${name}: ${message}`)) { + this.limit = 'memory'; + return this.limitEnd(); + } + if (name === '' && thrown.stack === undefined) { + this.write('err', `Uncaught ${thrown.shown ?? safeText(thrown.message)}\n`); + return { status: 'error', message: 'Error', exitCode: 1 }; + } + const stack = typeof thrown.stack === 'string' ? this.fileColumns(thrown.stack) : ''; + this.write('err', `${name || 'Error'}: ${message}\n${stack}`); + return { status: 'error', message: name || 'Error', exitCode: 1 }; + } + + // Tests ------------------------------------------------------------------------------------------------------------ + + private testFiles(): RunFile[] { + const tests = this.request.files.filter((file) => TEST_FILE.test(file.path)); + if (tests.length > 0) return tests; + return this.request.files.filter((file) => file.path === this.request.main && SCRIPT_FILE.test(file.path)); + } + + /** One result from the test API. Dropped after an exit or a limit: those end the run, not the test. */ + private received(json: string): void { + if (this.exitCode !== undefined || (this.limit !== null && !this.closing)) return; + if (!this.closing && performance.now() > this.context.deadline) { + this.limit = 'time'; + return; + } + let found: Reported; + try { + found = JSON.parse(json) as Reported; + } catch { + return; + } + if (!this.closing && found.status === 'error' && (OUT_OF_MEMORY.test(found.message ?? '') || this.overMemory())) { + this.limit = 'memory'; + return; + } + if (typeof found.index === 'number') this.testsReported = Math.max(this.testsReported, found.index + 1); + this.report(found); + } + + private report(found: Reported): void { + // `ids` keeps every id given and, for a repeated one, the last number added to it, so equal names stay cheap. + const base = `${found.file}::${found.name}`; + let id = base; + let n = this.ids.get(base); + if (n !== undefined) { + do id = `${base} (${++n})`; + while (this.ids.has(id)); + this.ids.set(base, n); + } + if (!this.ids.has(id)) this.ids.set(id, 1); + const location = + found.status === 'fail' || found.status === 'error' || found.status === 'time' + ? (inlinePlace(found.message ?? '', this.request.files) ?? + firstProjectFrame(found.stack ?? '', this.request.files)) + : null; + const result: TestResult = { + id: cut(id), + name: cut(found.name), + status: found.status, + timeMs: Math.max(0, found.timeMs), + ...(found.message === undefined ? {} : { message: cut(found.message) }), + ...(found.expected === undefined ? {} : { expected: cut(found.expected) }), + ...(found.actual === undefined ? {} : { actual: cut(found.actual) }), + ...(location ? { location: { file: location.file, line: location.line } } : {}), + }; + this.results.push(result); + this.emit({ kind: 'test', result }); + const line = + result.status === 'pass' + ? t('codeStudio.js.tests.pass', { name: result.name }) + : result.status === 'skipped' + ? t('codeStudio.js.tests.skip', { name: result.name }) + : t('codeStudio.js.tests.fail', { name: result.name }); + const detail = result.message ? `\n${result.message.replace(/^/gm, ' ')}` : ''; + this.write('out', `${line}${detail}\n`); + } + + private runTests(): RunEnd { + const vm = this.vm; + const unsupported = ['test.only', 'it.only', 'describe.only', 'test.each', 'it.each', 'describe.each']; + const text = vm.newString( + JSON.stringify({ + nested: t('codeStudio.js.tests.nested'), + notSupported: Object.fromEntries( + unsupported.map((name) => [name, t('codeStudio.js.tests.notSupported', { name })]), + ), + }), + ); + const api = this.withFunction( + 'report', + (json) => void this.received(vm.getString(json)), + (reportFn) => + this.withFunction( + 'load', + (path) => { + const from = vm.newString(''); + const required = vm.callFunction(this.requireFrom!, vm.undefined, from); + from.dispose(); + if (required.error) return { error: required.error }; + const specifier = vm.newString(`./${vm.getString(path)}`); + const loaded = vm.callFunction(required.value, vm.undefined, specifier); + specifier.dispose(); + required.value.dispose(); + if (loaded.error) return { error: loaded.error }; + loaded.value.dispose(); + return undefined; + }, + (loadFn) => this.callPrelude(TEST_API, '', [reportFn, loadFn, text]), + ), + ); + text.dispose(); + if (!('value' in api)) return this.failure(api); + try { + return this.runTestsWith(api.value); + } finally { + api.value.dispose(); + } + } + + private callMethod( + object: QuickJSHandle, + name: string, + args: (QuickJSHandle | string | number)[] = [], + ): { value: QuickJSHandle } | Thrown { + const handles = args.map((arg) => + typeof arg === 'string' ? this.vm.newString(arg) : typeof arg === 'number' ? this.vm.newNumber(arg) : arg, + ); + const method = this.vm.getProp(object, name); + const called = this.vm.callFunction(method, object, ...handles); + method.dispose(); + handles.forEach((handle, index) => { + if (handle !== args[index]) handle.dispose(); + }); + if (called.error) return this.dumpError(called.error); + return { value: called.value }; + } + + /** Sends a result for each test from the first one not reported: `first` for it, `rest` for the others. */ + private unfinished(api: QuickJSHandle, first: [TestStatus, string], rest: [TestStatus, string]): void { + this.closing = true; + this.closeBy = performance.now() + CLOSE_MS; + const done = this.callMethod(api, 'unfinished', [this.testsReported, ...first, ...rest]); + if ('value' in done) done.value.dispose(); + } + + private runTestsWith(api: QuickJSHandle): RunEnd { + const vm = this.vm; + for (const file of this.testFiles()) { + const loaded = this.callMethod(api, 'load', [file.path]); + if ('value' in loaded) { + loaded.value.dispose(); + continue; + } + if (this.stoppedBy(loaded)) return this.failure(loaded); + const name = typeof loaded.name === 'string' ? loaded.name : 'Error'; + const message = typeof loaded.message === 'string' ? loaded.message : safeText(loaded.message); + this.failure(loaded); + this.report({ + name: file.path, + file: file.path, + status: 'error', + timeMs: 0, + message: `${t('codeStudio.js.tests.loadFailed')}\n${name}: ${message}`, + ...(typeof loaded.stack === 'string' ? { stack: loaded.stack } : {}), + }); + } + const counted = this.callMethod(api, 'count'); + if (!('value' in counted)) return this.failure(counted); + const total = vm.getNumber(counted.value); + counted.value.dispose(); + if (total === 0 && this.results.length === 0) this.write('out', `${t('codeStudio.js.tests.none')}\n`); + const planned = Math.min(total, PROJECT_LIMITS.tests); + if (total > planned) this.write('err', `${t('codeStudio.js.tests.tooMany', { count: total, max: planned })}\n`); + + const ran = this.callMethod(api, 'run', [planned]); + if (!('value' in ran)) return this.failure(ran); + const jobs = this.runtime.executePendingJobs(); + const state = vm.getPromiseState(ran.value); + ran.value.dispose(); + if (state.type === 'fulfilled' && !state.notAPromise) state.value.dispose(); + const rejected = state.type === 'rejected' ? this.dumpError(state.error) : null; + const jobError = jobs.error ? this.dumpError(jobs.error) : null; + if (this.exitCode !== undefined) return this.exited(); + if (this.limit === null && this.overMemory()) this.limit = 'memory'; + if (this.limit !== null) { + const first: [TestStatus, string] = + this.limit === 'time' ? ['time', t('codeStudio.js.tests.time')] : ['error', t('codeStudio.js.tests.memory')]; + this.unfinished(api, first, ['skipped', t('codeStudio.js.tests.notRunLimit')]); + this.summary(); + return this.limitEnd(); + } + if (jobError) return this.failure(jobError); + if (rejected) return this.failure(rejected); + if (state.type === 'pending') { + const stuck = t('codeStudio.js.tests.stuck'); + this.unfinished(api, ['error', stuck], ['error', t('codeStudio.js.tests.notRun')]); + } + const failed = this.summary(); + return failed > 0 || total > planned + ? { status: 'error', message: '', exitCode: 1 } + : { status: 'ok', message: '', exitCode: 0 }; + } + + /** Writes the summary line; the number of tests that did not pass. */ + private summary(): number { + const count = (status: TestStatus) => this.results.filter((result) => result.status === status).length; + const failed = count('fail') + count('error') + count('time'); + const summary = t('codeStudio.js.tests.summary', { + passed: count('pass'), + failed, + skipped: count('skipped'), + total: this.results.length, + }); + this.write('out', `${summary}\n`); + return failed; + } + + /** Whether an error is a limit or an exit, which ends the whole run rather than one test. */ + private stoppedBy(thrown: Thrown): boolean { + if (this.exitCode !== undefined || this.limit !== null || this.overMemory()) return true; + return thrown.name === 'InternalError' && OUT_OF_MEMORY.test(`InternalError: ${String(thrown.message)}`); + } +} diff --git a/app/src/features/codestudio/runtimes/js/environment.ts b/app/src/features/codestudio/runtimes/js/environment.ts new file mode 100644 index 000000000..b199b6bcb --- /dev/null +++ b/app/src/features/codestudio/runtimes/js/environment.ts @@ -0,0 +1,141 @@ +// What a program sees besides the language: `console`, `print`, standard input, `process`, and a seeded +// `Math.random` (DESIGN 4.4, 4.5). The text below runs inside QuickJS, evaluated as the file "", and is +// called with host functions: write(stream, text), readLine() (a line, or null at the end of input), exit(code), and +// random(). It returns `inspect`, which formats a value the way console.log shows it. +// +// console.log follows Node's documented format: strings as they are, other values as `inspect` shows them +// ("[ 1, 2 ]", "{ a: 1, b: 'x' }", "[Function: f]", "Map(1) { 'a' => 1 }"). log, info, and debug write to standard +// output; warn, error, and trace write to standard error. + +export const ENVIRONMENT = String.raw`(function (write, readLine, exit, random, argv) { + 'use strict'; + const MAX_ITEMS = 100; + + function quote(text) { + return "'" + text.replace(/\\/g, '\\\\').replace(/'/g, "\\'").replace(/\n/g, '\\n') + "'"; + } + + function keyText(key) { + return /^[A-Za-z_$][\w$]*$/.test(key) ? key : quote(key); + } + + function inspect(value, depth, seen) { + depth = depth || 0; + seen = seen || []; + if (typeof value === 'string') return depth === 0 ? value : quote(value); + if (typeof value === 'number') return Object.is(value, -0) ? '-0' : String(value); + if (typeof value === 'bigint') return String(value) + 'n'; + if (typeof value === 'symbol') return value.toString(); + if (typeof value === 'function') { + const kind = /^class\b/.test(Function.prototype.toString.call(value)) ? 'class' : 'Function'; + return '[' + kind + (value.name ? (kind === 'class' ? ' ' : ': ') + value.name : ' (anonymous)') + ']'; + } + if (value === null || typeof value !== 'object') return String(value); + if (seen.indexOf(value) >= 0) return '[Circular]'; + if (value instanceof Error) { + const head = value.name + ': ' + value.message; + return value.stack ? head + '\n' + value.stack.replace(/\n$/, '') : head; + } + if (value instanceof Date) return isNaN(value.getTime()) ? 'Invalid Date' : value.toISOString(); + if (value instanceof RegExp) return String(value); + if (depth > 2) return Array.isArray(value) ? '[Array]' : '[Object]'; + const inner = seen.concat([value]); + const show = function (item) { return inspect(item, depth + 1, inner); }; + let parts; + let open; + let close; + if (Array.isArray(value)) { + parts = value.slice(0, MAX_ITEMS).map(show); + for (let i = 0; i < Math.min(value.length, MAX_ITEMS); i++) if (!(i in value)) parts[i] = ''; + if (value.length > MAX_ITEMS) parts.push('... ' + (value.length - MAX_ITEMS) + ' more items'); + open = '['; + close = ']'; + } else if (value instanceof Map) { + parts = Array.from(value).slice(0, MAX_ITEMS).map(function (pair) { return show(pair[0]) + ' => ' + show(pair[1]); }); + open = 'Map(' + value.size + ') {'; + close = '}'; + } else if (value instanceof Set) { + parts = Array.from(value).slice(0, MAX_ITEMS).map(show); + open = 'Set(' + value.size + ') {'; + close = '}'; + } else { + parts = Object.keys(value).slice(0, MAX_ITEMS).map(function (key) { return keyText(key) + ': ' + show(value[key]); }); + const proto = Object.getPrototypeOf(value); + const name = proto && proto !== Object.prototype && proto.constructor ? proto.constructor.name : ''; + open = (proto === null ? '[Object: null prototype] ' : name ? name + ' ' : '') + '{'; + close = '}'; + } + if (parts.length === 0) return open + close; + const line = open + ' ' + parts.join(', ') + ' ' + close; + if (line.length <= 72 && line.indexOf('\n') < 0) return line; + const pad = ' '.repeat(depth + 1); + return open + '\n' + parts.map(function (part) { return pad + part; }).join(',\n') + '\n' + ' '.repeat(depth) + close; + } + + let indent = ''; + const counts = Object.create(null); + const timers = Object.create(null); + function writer(stream) { + return function () { + const text = Array.prototype.map.call(arguments, function (arg) { return inspect(arg); }).join(' '); + write(stream, (indent ? text.split('\n').map(function (line) { return indent + line; }).join('\n') : text) + '\n'); + }; + } + const log = writer('out'); + const err = writer('err'); + const console = { + log: log, + info: log, + debug: log, + warn: err, + error: err, + trace: function () { + err.apply(null, ['Trace:'].concat(Array.prototype.slice.call(arguments))); + }, + assert: function (condition) { + if (condition) return; + const rest = Array.prototype.slice.call(arguments, 1); + err.apply(null, ['Assertion failed' + (rest.length ? ':' : '')].concat(rest)); + }, + group: function () { + if (arguments.length) log.apply(null, arguments); + indent += ' '; + }, + groupEnd: function () { + indent = indent.slice(2); + }, + count: function (label) { + const key = label === undefined ? 'default' : String(label); + counts[key] = (counts[key] || 0) + 1; + log(key + ': ' + counts[key]); + }, + countReset: function (label) { + counts[label === undefined ? 'default' : String(label)] = 0; + }, + time: function (label) { + timers[label === undefined ? 'default' : String(label)] = Date.now(); + }, + timeEnd: function (label) { + const key = label === undefined ? 'default' : String(label); + if (key in timers) log(key + ': ' + (Date.now() - timers[key]) + 'ms'); + delete timers[key]; + }, + }; + globalThis.console = console; + globalThis.print = log; + globalThis.readline = function () { return readLine(); }; + globalThis.prompt = function (message) { + if (message !== undefined) write('out', String(message)); + return readLine(); + }; + globalThis.process = { + argv: JSON.parse(argv), + env: {}, + platform: 'opennote', + exit: function (code) { exit(code === undefined ? 0 : Number(code) | 0); }, + stdout: { write: function (text) { write('out', String(text)); return true; } }, + stderr: { write: function (text) { write('err', String(text)); return true; } }, + }; + Object.defineProperty(Math, 'random', { value: function () { return random(); }, writable: true, configurable: true }); + return inspect; +})`; diff --git a/app/src/features/codestudio/runtimes/js/harness.ts b/app/src/features/codestudio/runtimes/js/harness.ts new file mode 100644 index 000000000..520d4959e --- /dev/null +++ b/app/src/features/codestudio/runtimes/js/harness.ts @@ -0,0 +1,427 @@ +// The test API inside QuickJS (DESIGN 9, ADR 0075): a Jest-compatible subset written from Jest's documented +// behavior. `describe`, `it`, `test`, `beforeEach`, `afterEach`, and `expect` with `.not` and the matchers DESIGN 9 +// lists. The text below runs inside the sandbox, so it is JavaScript in a string, evaluated as the file "". +// It is called with two host functions: `report(json)` sends one result, and `load(path)` runs one test file. +// +// Each rule below is from Jest's documentation. +// +// toBe compares two values by sameness. So NaN equals NaN, and zero differs from negative zero. +// +// toEqual ignores properties whose value is undefined. It also ignores array sparseness and object types. Arrays of +// different lengths differ, as in Jest's own equality check, so an undefined item is only equal to a hole. +// toStrictEqual counts all of those. +// +// toBeCloseTo(n, digits = 2) passes when |received - n| < 10^-digits / 2. +// +// toThrow(x) passes when the function throws. A string must be in the message. A regular expression must match +// the message. A class must be the error's class or a parent of it. An error must have the same message. +// +// afterEach hooks run from the innermost describe outward. beforeEach hooks run from the outermost inward. +// +// beforeAll runs once before the first test of its describe, and afterAll once after the last one. When a beforeAll +// fails, every test of its describe fails with that error. A test cannot start inside another test. +// +// `.only` and `.each` are not supported: they throw a clear error, so a file that uses them does not pass silently. +// The third argument is the text of those errors, from the strings file. The harness keeps its own JSON.stringify, +// Object.keys, Date.now, and Array.prototype.join, so a program that replaces them cannot break the reports. Each +// test's full name is joined when the test is added, and the results for unfinished tests use no program code. + +export const TEST_API = String.raw`(function (report, load, textJson) { + 'use strict'; + const text = JSON.parse(textJson); + const stringify = JSON.stringify; + const objectKeys = Object.keys; + const ownSymbols = Object.getOwnPropertySymbols; + const consoleError = console.error; + const now = Date.now; + const join = Function.prototype.call.bind(Array.prototype.join); + function suiteIn(parent, titles) { + return { titles, beforeEach: [], afterEach: [], beforeAll: [], afterAll: [], parent, started: false, failure: null }; + } + const root = suiteIn(null, []); + const tests = []; + let current = root; + let skipping = 0; + let file = ''; + let running = false; + let planned = 0; + + function outsideTests() { + if (running) throw new Error(text.nested); + } + function addTest(name, fn, skip) { + outsideTests(); + const titles = current.titles.concat(String(name)); + tests.push({ name: join(titles, ' > '), fn, suite: current, file, skip: skip || skipping > 0, index: tests.length }); + } + function describe(name, fn) { + outsideTests(); + const suite = suiteIn(current, current.titles.concat(String(name))); + const saved = current; + current = suite; + try { + fn(); + } finally { + current = saved; + } + } + describe.skip = function (name, fn) { + skipping++; + try { + describe(name, fn); + } finally { + skipping--; + } + }; + function test(name, fn) { + addTest(name, fn, false); + } + test.skip = function (name, fn) { + addTest(name, fn, true); + }; + test.todo = function (name) { + addTest(name, undefined, true); + }; + function it(name, fn) { + addTest(name, fn, false); + } + it.skip = test.skip; + it.todo = test.todo; + function notSupported(name) { + return function () { throw new Error(text.notSupported[name]); }; + } + test.only = notSupported('test.only'); + test.each = notSupported('test.each'); + it.only = notSupported('it.only'); + it.each = notSupported('it.each'); + describe.only = notSupported('describe.only'); + describe.each = notSupported('describe.each'); + + class AssertionError extends Error {} + AssertionError.prototype.name = 'AssertionError'; + + function format(value, depth) { + depth = depth || 0; + if (typeof value === 'string') return stringify(value); + if (typeof value === 'number') return Object.is(value, -0) ? '-0' : String(value); + if (typeof value === 'bigint') return String(value) + 'n'; + if (typeof value === 'function') return '[Function ' + (value.name || 'anonymous') + ']'; + if (typeof value !== 'object' || value === null) return String(value); + if (depth > 3) return Array.isArray(value) ? '[Array]' : '[Object]'; + if (value instanceof Error) return value.name + ': ' + value.message; + if (value instanceof Date) return 'Date(' + value.toISOString() + ')'; + if (value instanceof RegExp) return String(value); + if (Array.isArray(value)) return '[' + value.map(function (item) { return format(item, depth + 1); }).join(', ') + ']'; + if (value instanceof Map) { + return 'Map {' + Array.from(value).map(function (pair) { + return format(pair[0], depth + 1) + ' => ' + format(pair[1], depth + 1); + }).join(', ') + '}'; + } + if (value instanceof Set) { + return 'Set {' + Array.from(value).map(function (item) { return format(item, depth + 1); }).join(', ') + '}'; + } + const keys = objectKeys(value); + return '{' + keys.map(function (key) { return key + ': ' + format(value[key], depth + 1); }).join(', ') + '}'; + } + + const tag = function (value) { return Object.prototype.toString.call(value); }; + const has = function (object, key) { return Object.prototype.hasOwnProperty.call(object, key); }; + + function keysOf(object, strict) { + const symbols = ownSymbols(object).filter(function (key) { + return Object.prototype.propertyIsEnumerable.call(object, key); + }); + return objectKeys(object).concat(symbols).filter(function (key) { return strict || object[key] !== undefined; }); + } + const isIndex = function (key) { return typeof key === 'string' && String(key >>> 0) === key; }; + + function equals(a, b, strict, seen) { + if (Object.is(a, b)) return true; + if (typeof a !== 'object' || typeof b !== 'object' || a === null || b === null) return false; + if (tag(a) !== tag(b)) return false; + if (strict && Object.getPrototypeOf(a) !== Object.getPrototypeOf(b)) return false; + if (a instanceof Date) return Object.is(a.getTime(), b.getTime()); + if (a instanceof RegExp) return a.source === b.source && a.flags === b.flags; + if (a instanceof Error) return a.message === b.message; + if (a instanceof Number || a instanceof String || a instanceof Boolean) return Object.is(a.valueOf(), b.valueOf()); + for (let i = 0; i < seen.length; i++) if (seen[i][0] === a) return seen[i][1] === b; + seen = seen.concat([[a, b]]); + if (a instanceof Map) { + if (a.size !== b.size) return false; + for (const pair of a) { + if (b.has(pair[0])) { + if (!equals(pair[1], b.get(pair[0]), strict, seen)) return false; + continue; + } + // A key that is an object or an array matches an equal key, as Jest compares Map keys. + let matched = false; + for (const other of b) { + if (equals(pair[0], other[0], strict, seen) && equals(pair[1], other[1], strict, seen)) { + matched = true; + break; + } + } + if (!matched) return false; + } + return true; + } + if (a instanceof Set) { + if (a.size !== b.size) return false; + const rest = Array.from(b); + for (const item of a) { + const index = rest.findIndex(function (other) { return equals(item, other, strict, seen); }); + if (index < 0) return false; + rest.splice(index, 1); + } + return true; + } + if (Array.isArray(a)) { + if (a.length !== b.length) return false; + const length = Math.max(a.length, b.length); + for (let i = 0; i < length; i++) { + if (strict && (i in a) !== (i in b)) return false; + if (!equals(a[i], b[i], strict, seen)) return false; + } + } + const ka = keysOf(a, strict).filter(function (key) { return !Array.isArray(a) || !isIndex(key); }); + const kb = keysOf(b, strict).filter(function (key) { return !Array.isArray(b) || !isIndex(key); }); + if (ka.length !== kb.length) return false; + for (const key of ka) if (!has(b, key) || !equals(a[key], b[key], strict, seen)) return false; + return true; + } + + function thrownBy(fn) { + if (typeof fn !== 'function') throw new TypeError('expect(received).toThrow(): received value must be a function'); + try { + fn(); + } catch (error) { + return { error }; + } + return null; + } + + function messageOf(error) { + return error !== null && typeof error === 'object' && 'message' in error ? String(error.message) : String(error); + } + + const matchers = { + toBe: function (r, e) { return { pass: Object.is(r, e), expected: e }; }, + toEqual: function (r, e) { return { pass: equals(r, e, false, []), expected: e }; }, + toStrictEqual: function (r, e) { return { pass: equals(r, e, true, []), expected: e }; }, + toBeTruthy: function (r) { return { pass: !!r }; }, + toBeFalsy: function (r) { return { pass: !r }; }, + toBeNull: function (r) { return { pass: r === null, expected: null }; }, + toBeUndefined: function (r) { return { pass: r === undefined, expected: undefined }; }, + toBeDefined: function (r) { return { pass: r !== undefined }; }, + toBeNaN: function (r) { return { pass: typeof r === 'number' && r !== r }; }, + toBeInstanceOf: function (r, e) { return { pass: r instanceof e, expected: e }; }, + toContain: function (r, e) { + const pass = typeof r === 'string' ? r.includes(String(e)) : Array.from(r).some(function (item) { return item === e; }); + return { pass, expected: e }; + }, + toHaveLength: function (r, e) { + return { pass: r !== null && r !== undefined && r.length === e, expected: e, actual: r === null || r === undefined ? r : r.length }; + }, + toMatch: function (r, e) { + if (typeof r !== 'string') throw new TypeError('expect(received).toMatch(): received value must be a string'); + return { pass: typeof e === 'string' ? r.includes(e) : e.test(r), expected: e }; + }, + toBeCloseTo: function (r, e, digits) { + const places = digits === undefined ? 2 : digits; + const pass = (r === Infinity && e === Infinity) || (r === -Infinity && e === -Infinity) || + Math.abs(e - r) < Math.pow(10, -places) / 2; + return { pass, expected: e }; + }, + toBeGreaterThan: function (r, e) { return { pass: r > e, expected: e }; }, + toBeGreaterThanOrEqual: function (r, e) { return { pass: r >= e, expected: e }; }, + toBeLessThan: function (r, e) { return { pass: r < e, expected: e }; }, + toBeLessThanOrEqual: function (r, e) { return { pass: r <= e, expected: e }; }, + toThrow: function (r, e) { + const thrown = thrownBy(r); + const actual = thrown ? thrown.error : undefined; + let pass = thrown !== null; + if (pass && e !== undefined) { + const message = messageOf(actual); + if (typeof e === 'string') pass = message.includes(e); + else if (e instanceof RegExp) pass = e.test(message); + else if (typeof e === 'function') pass = actual instanceof e; + else pass = message === messageOf(e); + } + return { pass, expected: e, actual, thrown: thrown !== null }; + }, + }; + matchers.toThrowError = matchers.toThrow; + + function expect(received) { + function bind(negated) { + const bound = {}; + objectKeys(matchers).forEach(function (name) { + bound[name] = function () { + const args = Array.prototype.slice.call(arguments); + const result = matchers[name].apply(null, [received].concat(args)); + if (result.pass !== negated) return; + const shown = 'actual' in result ? result.actual : received; + const shownArgs = args.map(function (arg) { return format(arg); }).join(', '); + const lines = ['expect(received)' + (negated ? '.not.' : '.') + name + '(' + shownArgs + ')', '']; + if ('expected' in result) lines.push('Expected: ' + (negated ? 'not ' : '') + format(result.expected)); + if (name === 'toThrow' || name === 'toThrowError') { + lines.push(result.thrown ? 'Received error: ' + format(shown) : 'Received function did not throw'); + } else { + lines.push('Received: ' + format(shown)); + } + const error = new AssertionError(lines.join('\n')); + error.expected = 'expected' in result ? format(result.expected) : undefined; + error.actual = format(shown); + throw error; + }; + }); + return bound; + } + const matched = bind(false); + matched.not = bind(true); + return matched; + } + + function chain(suite, key) { + const hooks = []; + for (let s = suite; s; s = s.parent) hooks.unshift.apply(hooks, s[key]); + return key === 'afterEach' ? hooks.reverse() : hooks; + } + + function call(fn) { + if (fn.length === 0) return fn(); + return new Promise(function (resolve, reject) { + fn(function (error) { if (error) reject(error); else resolve(); }); + }); + } + + function send(entry, status, started, failure) { + const result = { + name: entry.name, file: entry.file, status, timeMs: now() - started, index: entry.index, + }; + if (failure !== undefined) { + result.message = failure instanceof Error ? failure.name + ': ' + failure.message : 'Thrown: ' + format(failure); + if (failure instanceof AssertionError) { + result.message = failure.message; + if (failure.expected !== undefined) result.expected = failure.expected; + result.actual = failure.actual; + } + if (failure instanceof Error && failure.stack) result.stack = String(failure.stack); + } + report(stringify(result)); + } + + function suitesOf(entry) { + const list = []; + for (let s = entry.suite; s; s = s.parent) list.unshift(s); + return list; + } + + function runs(entry) { + return !entry.skip && typeof entry.fn === 'function'; + } + + function beforeAll(entry) { + let step = Promise.resolve(); + suitesOf(entry).forEach(function (suite) { + if (suite.started) return; + suite.started = true; + suite.beforeAll.forEach(function (hook) { + step = step.then(function () { if (!suite.failure) return call(hook); }).then(undefined, function (error) { + if (!suite.failure) suite.failure = { error }; + }); + }); + }); + return step; + } + + function afterAll(entry, index, last) { + let step = Promise.resolve(); + suitesOf(entry).reverse().forEach(function (suite) { + if (!suite.started || last.get(suite) !== index) return; + suite.afterAll.forEach(function (hook) { + step = step.then(function () { return call(hook); }).then(undefined, function (error) { + consoleError('afterAll: ' + messageOf(error)); + }); + }); + }); + return step; + } + + function runOne(entry) { + const started = now(); + if (!runs(entry)) { + send(entry, 'skipped', started); + return Promise.resolve(); + } + const blocked = suitesOf(entry).find(function (suite) { return suite.failure; }); + if (blocked) { + const error = blocked.failure.error; + send(entry, error instanceof AssertionError ? 'fail' : 'error', started, error); + return Promise.resolve(); + } + running = true; + let failure; + let failed = false; + const note = function (error) { if (!failed) { failed = true; failure = error; } }; + let step = Promise.resolve(); + chain(entry.suite, 'beforeEach').forEach(function (hook) { step = step.then(function () { return call(hook); }); }); + step = step.then(function () { return call(entry.fn); }).then(undefined, note); + chain(entry.suite, 'afterEach').forEach(function (hook) { + step = step.then(function () { return call(hook); }).then(undefined, note); + }); + return step.then(function () { + running = false; + const status = !failed ? 'pass' : failure instanceof AssertionError ? 'fail' : 'error'; + send(entry, status, started, failed ? failure : undefined); + }); + } + + const api = { + describe, + it, + test, + expect, + beforeEach: function (fn) { current.beforeEach.push(fn); }, + afterEach: function (fn) { current.afterEach.push(fn); }, + beforeAll: function (fn) { current.beforeAll.push(fn); }, + afterAll: function (fn) { current.afterAll.push(fn); }, + }; + Object.keys(api).forEach(function (name) { globalThis[name] = api[name]; }); + + return { + load: function (path) { + file = path; + current = root; + load(path); + }, + run: function (limit) { + planned = Math.min(limit, tests.length); + const last = new Map(); + for (let i = 0; i < planned; i++) { + if (runs(tests[i])) suitesOf(tests[i]).forEach(function (suite) { last.set(suite, i); }); + } + let step = Promise.resolve(); + tests.slice(0, planned).forEach(function (entry, index) { + step = step.then(function () { + const before = runs(entry) ? beforeAll(entry) : Promise.resolve(); + return before.then(function () { return runOne(entry); }).then(function () { + return afterAll(entry, index, last); + }); + }); + }); + return step; + }, + unfinished: function (from, firstStatus, firstMessage, restStatus, restMessage) { + for (let i = from; i < planned; i++) { + const first = i === from; + const entry = tests[i]; + report(stringify({ + name: entry.name, file: entry.file, status: first ? firstStatus : restStatus, timeMs: 0, index: entry.index, + message: 'Error: ' + (first ? firstMessage : restMessage), + })); + } + }, + count: function () { return tests.length; }, + }; +})`; diff --git a/app/src/features/codestudio/runtimes/js/js.test.ts b/app/src/features/codestudio/runtimes/js/js.test.ts new file mode 100644 index 000000000..cc1621df5 --- /dev/null +++ b/app/src/features/codestudio/runtimes/js/js.test.ts @@ -0,0 +1,1004 @@ +// The JavaScript and TypeScript runtime in Node, with the real QuickJS and the real Sucrase (DESIGN 2.1, 4, 9). +// - The 20-program suite runs in both languages. Its answers are worked by hand or come from the language specification. +// - Sucrase keeps line numbers, so errors map to the line in the TypeScript file. +// - Modules work across project files, and a missing one gives a clear error. +// - The Jest-compatible test API gives the known answers from Jest's documentation. +// The sandbox escapes run in a real worker in js.test.tsx. + +// checks-disable-file modifiability: one describe block per feature +import { readFileSync } from 'node:fs'; +import { beforeAll, describe, expect, it } from 'vitest'; +import { newQuickJSWASMModuleFromVariant, newVariant } from 'quickjs-emscripten-core'; +import releaseSync from '@jitl/quickjs-wasmfile-release-sync'; +import { transform } from 'sucrase'; +import type { RunFile, RunRequest, RuntimeDiagnostic, WorkerMessage } from '../../contracts/run'; +import type { TestResult } from '../../contracts/testing'; +import { mapDiagnostics } from '../../run/diagnostics'; +import { Mount } from '../../run/mount'; +import { RUN_LIMITS } from '../../run/protocol'; +import { seeded } from '../../run/random'; +import { Stdin } from '../../run/stdin'; +import { VERSIONS } from './adapter'; +import { quickjsMapper } from './diagnostics'; +import { BASE_HEAP_BYTES, cappedHeap, headroomBytes, runProgram } from './engine'; +import type { Engine } from './engine'; +import { resolveModule } from './modules'; +import { compile, usesModules } from './transpile'; + +/** A fresh QuickJS with its heap capped, as the worker makes one for each run. */ +async function newEngine(memoryBytes = RUN_LIMITS.memoryBytes): Promise { + const quickjs = await newQuickJSWASMModuleFromVariant( + newVariant(releaseSync, { wasmMemory: cappedHeap(memoryBytes) }), + ); + return { quickjs, transform }; +} + +let engine: Engine; +beforeAll(async () => { + engine = await newEngine(); +}); + +interface Ran { + status: string; + exitCode?: number; + out: string; + err: string; + tests: TestResult[]; + diagnostics: RuntimeDiagnostic[]; +} + +function run( + files: Record, + more: { + main?: string; + stdin?: string; + args?: string[]; + tests?: boolean; + timeMs?: number; + memoryBytes?: number; + engine?: Engine; + } = {}, +): Ran { + const list: RunFile[] = Object.entries(files).map(([path, text]) => ({ path, text })); + const request: RunRequest = { + kind: 'run', + runtime: 'typescript', + files: list, + main: more.main ?? list[0]!.path, + args: more.args ?? [], + stdin: { mode: 'prefilled', text: more.stdin ?? '' }, + limits: { + ...RUN_LIMITS, + timeMs: more.timeMs ?? RUN_LIMITS.timeMs, + memoryBytes: more.memoryBytes ?? RUN_LIMITS.memoryBytes, + }, + seed: 1, + ...(more.tests ? { test: { cases: [], framework: true } } : {}), + }; + const messages: WorkerMessage[] = []; + const random = seeded(request.seed); + const end = runProgram(more.engine ?? engine, request, (message) => messages.push(message), { + stdin: new Stdin(request.stdin.text, false), + files: new Mount(list), + random: random.random, + randomBytes: random.fill, + deadline: performance.now() + request.limits.timeMs, + }); + const text = (stream: string) => + messages.map((message) => (message.kind === 'out' && message.stream === stream ? message.text : '')).join(''); + const err = text('err'); + return { + status: end.status, + ...(end.exitCode === undefined ? {} : { exitCode: end.exitCode }), + out: text('out'), + err, + tests: messages.flatMap((message) => (message.kind === 'test' ? [message.result] : [])), + diagnostics: mapDiagnostics(quickjsMapper, err, list), + }; +} + +interface Program { + name: string; + js: string; + ts: string; + out: string; + stdin?: string; +} + +// Each answer is worked by hand or follows from the ECMAScript specification (default sort compares strings; +// 0.1 + 0.2 is the double nearest 0.30000000000000004), and console.log's format is Node's documented one. +const PROGRAMS: Program[] = [ + { + name: 'hello', + js: 'console.log("Hello, world!");', + ts: 'const who: string = "world";\nconsole.log(`Hello, ${who}!`);', + out: 'Hello, world!\n', + }, + { + name: 'sum 1..100', + js: 'let s = 0;\nfor (let i = 1; i <= 100; i++) s += i;\nconsole.log(s);', + ts: 'let s: number = 0;\nfor (let i: number = 1; i <= 100; i++) s += i;\nconsole.log(s);', + out: '5050\n', + }, + { + name: 'fib(30)', + js: 'function fib(n) { return n < 2 ? n : fib(n - 1) + fib(n - 2); }\nconsole.log(fib(30));', + ts: 'function fib(n: number): number { return n < 2 ? n : fib(n - 1) + fib(n - 2); }\nconsole.log(fib(30));', + out: '832040\n', + }, + { + name: '20! with BigInt', + js: 'let f = 1n;\nfor (let i = 1n; i <= 20n; i++) f *= i;\nconsole.log(f.toString());', + ts: 'let f: bigint = 1n;\nfor (let i: bigint = 1n; i <= 20n; i++) f *= i;\nconsole.log(f.toString());', + out: '2432902008176640000\n', + }, + { + name: 'primes below 50', + js: 'const p = [];\nfor (let n = 2; n < 50; n++) if (p.every((q) => n % q)) p.push(n);\nconsole.log(p.join(" "));', + ts: [ + 'const p: number[] = [];', + 'for (let n = 2; n < 50; n++) if (p.every((q: number) => n % q)) p.push(n);', + 'console.log(p.join(" "));', + ].join('\n'), + out: '2 3 5 7 11 13 17 19 23 29 31 37 41 43 47\n', + }, + { + name: 'sorted', + js: 'const a = [5, 3, 10, 1];\nconsole.log([...a].sort((x, y) => x - y).join(","), [...a].sort().join(","));', + ts: [ + 'const a: number[] = [5, 3, 10, 1];', + 'console.log([...a].sort((x: number, y: number) => x - y).join(","), [...a].sort().join(","));', + ].join('\n'), + out: '1,3,5,10 1,10,3,5\n', + }, + { + name: 'reverse a string', + js: 'console.log("OpenNote".split("").reverse().join(""));', + ts: 'const word: string = "OpenNote";\nconsole.log(word.split("").reverse().join(""));', + out: 'etoNnepO\n', + }, + { + name: 'Unicode', + js: 'console.log("héllo wörld".toUpperCase(), "😀".length, [..."😀"].length);', + ts: 'const s: string = "héllo wörld";\nconsole.log(s.toUpperCase(), "😀".length, [..."😀"].length);', + out: 'HÉLLO WÖRLD 2 1\n', + }, + { + name: 'formatting numbers', + js: 'console.log((3.14159).toFixed(2), (255).toString(16), (1e21).toString());', + ts: 'const pi: number = 3.14159;\nconsole.log(pi.toFixed(2), (255).toString(16), (1e21).toString());', + out: '3.14 ff 1e+21\n', + }, + { + name: 'gcd(1071, 462)', + js: 'const gcd = (a, b) => (b === 0 ? a : gcd(b, a % b));\nconsole.log(gcd(1071, 462));', + ts: 'const gcd = (a: number, b: number): number => (b === 0 ? a : gcd(b, a % b));\nconsole.log(gcd(1071, 462));', + out: '21\n', + }, + { + name: 'FizzBuzz to 15', + js: [ + 'for (let i = 1; i <= 15; i++) console.log(i % 15 === 0 ? "FizzBuzz" : i % 3 === 0 ? "Fizz" : i % 5 === 0 ? "Buzz" : i);', + ].join('\n'), + ts: [ + 'for (let i: number = 1; i <= 15; i++) console.log(i % 15 === 0 ? "FizzBuzz" : i % 3 === 0 ? "Fizz" : i % 5 === 0 ? "Buzz" : i);', + ].join('\n'), + out: '1\n2\nFizz\n4\nBuzz\nFizz\n7\n8\nFizz\nBuzz\n11\nFizz\n13\n14\nFizzBuzz\n', + }, + { + name: 'word count', + js: 'const c = {};\nfor (const w of "the cat the hat".split(" ")) c[w] = (c[w] || 0) + 1;\nconsole.log(c);', + ts: [ + 'const c: Record = {};', + 'for (const w of "the cat the hat".split(" ")) c[w] = (c[w] ?? 0) + 1;', + 'console.log(c);', + ].join('\n'), + out: '{ the: 2, cat: 1, hat: 1 }\n', + }, + { + name: 'JSON', + js: 'const text = JSON.stringify({ a: [1, 2], b: "x" });\nconsole.log(text, JSON.parse(text).a[1]);', + ts: [ + 'interface D { a: number[]; b: string }', + 'const text = JSON.stringify({ a: [1, 2], b: "x" } as D);', + 'console.log(text, (JSON.parse(text) as D).a[1]);', + ].join('\n'), + out: '{"a":[1,2],"b":"x"} 2\n', + }, + { + name: 'classes', + js: [ + 'class Animal { constructor(name) { this.name = name; } speak() { return `${this.name} makes a sound`; } }', + 'class Dog extends Animal { speak() { return `${this.name} barks`; } }', + 'console.log(new Dog("Rex").speak());', + ].join('\n'), + ts: [ + 'class Animal { constructor(public name: string) {} speak(): string { return `${this.name} makes a sound`; } }', + 'class Dog extends Animal { override speak(): string { return `${this.name} barks`; } }', + 'console.log(new Dog("Rex").speak());', + ].join('\n'), + out: 'Rex barks\n', + }, + { + name: 'closures', + js: [ + 'function counter() { let n = 0; return () => ++n; }', + 'const next = counter();', + 'console.log(next(), next(), next());', + ].join('\n'), + ts: [ + 'function counter(): () => number { let n = 0; return () => ++n; }', + 'const next = counter();', + 'console.log(next(), next(), next());', + ].join('\n'), + out: '1 2 3\n', + }, + { + name: 'generators', + js: [ + 'function* squares() { for (let i = 0; ; i++) yield i * i; }', + 'const out = [];', + 'for (const s of squares()) { if (out.length === 5) break; out.push(s); }', + 'console.log(out);', + ].join('\n'), + ts: [ + 'function* squares(): Generator { for (let i = 0; ; i++) yield i * i; }', + 'const out: number[] = [];', + 'for (const s of squares()) { if (out.length === 5) break; out.push(s); }', + 'console.log(out);', + ].join('\n'), + out: '[ 0, 1, 4, 9, 16 ]\n', + }, + { + name: 'promises', + js: 'async function answer() { return 42; }\nanswer().then((v) => console.log("async", v));\nconsole.log("sync");', + ts: [ + 'async function answer(): Promise { return 42; }', + 'answer().then((v: number) => console.log("async", v));', + 'console.log("sync");', + ].join('\n'), + out: 'sync\nasync 42\n', + }, + { + name: 'destructuring', + js: 'const [first, ...rest] = [1, 2, 3];\nconst { x, y = 5 } = { x: 4 };\nconsole.log(first, rest, x + y);', + ts: [ + 'const [first, ...rest]: number[] = [1, 2, 3];', + 'const { x, y = 5 }: { x: number; y?: number } = { x: 4 };', + 'console.log(first, rest, x + y);', + ].join('\n'), + out: '1 [ 2, 3 ] 9\n', + }, + { + name: 'number edges', + js: 'console.log(2 ** 53 + 1 === 2 ** 53, 0.1 + 0.2, 7 / 0, -7 % 3);', + ts: 'const big: number = 2 ** 53;\nconsole.log(big + 1 === big, 0.1 + 0.2, 7 / 0, -7 % 3);', + out: 'true 0.30000000000000004 Infinity -1\n', + }, + { + name: 'standard input', + js: 'const a = Number(readline());\nconst b = Number(readline());\nconsole.log(a + b, readline());', + ts: [ + 'const a: number = Number(readline());', + 'const b: number = Number(readline());', + 'console.log(a + b, readline());', + ].join('\n'), + stdin: '3\n4\n', + out: '7 null\n', + }, +]; + +describe('the 20-program suite', () => { + it('has 20 programs', () => expect(PROGRAMS).toHaveLength(20)); + + it.each(PROGRAMS)('runs "$name" in JavaScript', (program) => { + const ran = run({ 'main.js': program.js }, { stdin: program.stdin ?? '' }); + expect(ran.err).toBe(''); + expect(ran.out).toBe(program.out); + expect(ran.status).toBe('ok'); + }); + + it.each(PROGRAMS)('runs "$name" in TypeScript', (program) => { + const ran = run({ 'main.ts': program.ts }, { stdin: program.stdin ?? '' }); + expect(ran.err).toBe(''); + expect(ran.out).toBe(program.out); + expect(ran.status).toBe('ok'); + }); +}); + +describe('the run', () => { + it('shows the last value after an arrow, and nothing for undefined', () => { + expect(run({ 'main.js': 'const x = 2;\nx * 21' }).out).toBe('→ 42\n'); + expect(run({ 'main.js': 'console.log(1)' }).out).toBe('1\n'); + expect(run({ 'main.ts': 'const t: string = "ok";\nt' }).out).toBe('→ ok\n'); + // Sucrase's "use strict" is a directive, not the program's last value. + expect(run({ 'main.ts': 'const t: string = "ok";' }).out).toBe(''); + }); + + it('seeds Math.random from the run seed (xorshift32 of 1 is 270369, worked by hand)', () => { + expect(run({ 'main.js': 'console.log(Math.random() * 2 ** 32)' }).out).toBe('270369\n'); + }); + + it('writes console methods to the right streams', () => { + const ran = run({ + 'main.js': + 'console.info("i"); console.debug("d"); console.warn("w"); console.error("e");\nconsole.assert(1 > 2, "no");', + }); + expect(ran.out).toBe('i\nd\n'); + expect(ran.err).toBe('w\ne\nAssertion failed: no\n'); + }); + + it('formats values the way Node documents console.log', () => { + const ran = run({ + 'main.js': + 'console.log([1, "a"], { k: [true, null] }, new Map([["a", 1]]), new Set([2]), function f() {}, undefined);', + }); + expect(ran.out).toBe("[ 1, 'a' ] { k: [ true, null ] } Map(1) { 'a' => 1 } Set(1) { 2 } [Function: f] undefined\n"); + }); + + it('gives arguments, and ends with process.exit', () => { + const ran = run( + { 'main.js': 'console.log(process.argv.slice(2).join("+"));\nprocess.exit(3);\nconsole.log("after");' }, + { + args: ['a', 'b'], + }, + ); + expect(ran.out).toBe('a+b\n'); + expect(ran.exitCode).toBe(3); + expect(ran.status).toBe('error'); + }); + + // QuickJS's own limit cannot see block sizes in this build. Without the heap cap, the program below grew the heap + // to its 2 GB ceiling (1,935 arrays of 800 KB), whatever limit was set. + it('stops at the memory limit within the capped heap, in well under 2 s', async () => { + const limit = 16 * 1024 * 1024; + const small = await newEngine(limit); + const started = performance.now(); + const ran = run( + { 'main.js': 'const a = []; while (true) a.push(new Array(1e5).fill(1));' }, + { engine: small, memoryBytes: limit, timeMs: 60_000 }, + ); + expect(ran.status).toBe('memory'); + expect(performance.now() - started).toBeLessThan(2000); + expect(small.quickjs.getWasmMemory().buffer.byteLength).toBeLessThanOrEqual( + BASE_HEAP_BYTES + limit + headroomBytes(limit), + ); + }); + + it("ends the run when the heap grows past the run's own, smaller limit", async () => { + const fresh = await newEngine(); + const limit = 4 * 1024 * 1024; + const ran = run( + // Arrays that grow by push are the blocks QuickJS's own limit misses: alone, it let this program use 64 MB. + { + 'main.js': 'const a = []; while (true) { const b = []; for (let i = 0; i < 1000; i++) b.push(i); a.push(b); }', + }, + { engine: fresh, memoryBytes: limit, timeMs: 60_000 }, + ); + expect(ran.status).toBe('memory'); + // The heap grows in steps of about a fifth, so it stops a few megabytes past the limit, far below the cap. + expect(fresh.quickjs.getWasmMemory().buffer.byteLength).toBeLessThan(BASE_HEAP_BYTES + 3 * limit); + }); + + // Many small blocks fill the heap between two checks of the interrupt handler. QuickJS then cannot make its own + // out-of-memory error and throws null, which ended these runs as "Uncaught null" before the heap had headroom. + it('ends many small allocations as "memory" under the default limits', async () => { + const programs: Record[] = [ + { 'main.js': "const a = []; for (;;) a.push({ i: a.length, s: 'x' + a.length });" }, + { 'main.js': 'let n = null; for (;;) n = { next: n };' }, + { 'main.js': 'const m = new Map(); for (let i = 0; ; i++) m.set(i, { i });' }, + { + 'main.ts': [ + 'class P { constructor(public x: number, public y: number) {} }', + 'const all: P[] = [];', + 'for (;;) all.push(new P(1, 2));', + ].join('\n'), + }, + ]; + for (const files of programs) { + const ran = run(files, { engine: await newEngine(), timeMs: 60_000 }); + expect([files, ran.status, ran.err]).toEqual([files, 'memory', 'InternalError: out of memory\n']); + } + }); + + it('stops at the time limit, also inside a promise job', () => { + expect(run({ 'main.js': 'while (true) {}' }, { timeMs: 300 }).status).toBe('time'); + const later = 'async function main() { await null; while (true) {} }\nmain();\nconsole.log("started");'; + const ran = run({ 'main.js': later }, { timeMs: 600 }); + expect(ran.status).toBe('time'); + expect(ran.out).toBe('started\n'); + }); + + it('does not count a thrown look-alike of the interrupt as the time limit', () => { + const ran = run({ 'main.js': 'throw { name: "InternalError", message: "interrupted" };' }); + expect(ran.status).toBe('error'); + }); + + it('prints any thrown value, and never fails the run while formatting it', () => { + const cases: [string, string][] = [ + ['throw 10n;', 'Uncaught 10n\n'], + ['Promise.reject(10n)', 'Uncaught 10n\n'], + ['throw Symbol("s");', 'Uncaught Symbol(s)\n'], + ['throw null;', 'Uncaught null\n'], + ['throw undefined;', 'Uncaught undefined\n'], + ['throw "text";', 'Uncaught "text"\n'], + ['const o = { a: 1 }; o.self = o; throw o;', 'Uncaught { a: 1, self: [Circular] }\n'], + ]; + for (const [program, err] of cases) { + const ran = run({ 'main.js': program }); + expect([program, ran.status, ran.err]).toEqual([program, 'error', err]); + } + const proxy = 'throw new Proxy({}, { get() { throw new Error("no"); }, ownKeys() { throw new Error("no"); } });'; + expect(run({ 'main.js': proxy }).status).toBe('error'); + }); + + it('pins that a rejection is silent unless it is the last value (no rejection tracker in quickjs-emscripten)', () => { + const ran = run({ 'main.js': 'async function main() { throw new Error("boom"); }\nmain();\nconsole.log("done");' }); + expect([ran.status, ran.out, ran.err]).toEqual(['ok', 'done\n', '']); + }); + + it('ends with a clear error when the main file is not in the project', () => { + const ran = run({ 'util.js': '1' }, { main: 'main.js' }); + expect(ran.status).toBe('error'); + expect(ran.err).toContain('The project has no file with that name'); + }); + + it('reports an uncaught error with its type, and a rejected last promise', () => { + const thrown = run({ 'main.js': 'null.x' }); + expect(thrown.status).toBe('error'); + expect(thrown.err).toContain('TypeError'); + const rejected = run({ 'main.js': 'async function main() { throw new RangeError("late"); }\nmain();' }); + expect(rejected.status).toBe('error'); + expect(rejected.err).toContain('RangeError: late'); + }); + + it('keeps the version lines equal to the pinned packages', () => { + const root = JSON.parse(readFileSync(new URL('../../../../../../package.json', import.meta.url), 'utf8')) as { + dependencies: Record; + }; + const pins = root.dependencies; + expect(pins['quickjs-emscripten-core']).toBe(VERSIONS.quickjs); + expect(pins.sucrase).toBe(VERSIONS.sucrase); + }); +}); + +describe('Sucrase keeps line numbers', () => { + const header = [ + 'interface Point {', + ' x: number;', + ' y: number;', + '}', + 'type Pair = [number, number];', + 'const p: Point = { x: 1, y: 2 };', + 'const pair: Pair = [p.x, p.y];', + 'console.log(pair.length);', + ].join('\n'); + + it('reports a runtime error on line 9 of a TypeScript file with interfaces above as line 9', () => { + const ran = run({ 'main.ts': `${header}\nthrow new Error("line nine");\n` }); + expect(ran.out).toBe('2\n'); + expect(ran.diagnostics).toEqual([{ file: 'main.ts', line: 9, column: 16, message: 'line nine', kind: 'Error' }]); + }); + + it('reports an error inside a function called later at the function line', () => { + const ran = run({ 'main.ts': `${header}\nfunction f(o: Point | null): number { return o!.x; }\nf(null);\n` }); + expect(ran.diagnostics[0]).toMatchObject({ file: 'main.ts', line: 9, kind: 'TypeError' }); + }); + + it('reports a syntax error Sucrase finds on line 9 as line 9', () => { + const ran = run({ 'main.ts': `${header}\nlet broken: number = ;\n` }); + expect(ran.status).toBe('error'); + expect(ran.diagnostics).toEqual([expect.objectContaining({ file: 'main.ts', line: 9, kind: 'SyntaxError' })]); + }); + + it('keeps every line of the transpiled text in place', () => { + const source = `${header}\nexport const z: number = 1;\nimport { q } from "./q";\n`; + const compiled = compile('main.ts', source, transform); + expect('code' in compiled && compiled.code.split('\n').length).toBe(source.split('\n').length); + }); + + it('leaves a plain script alone, so it keeps sloppy mode', () => { + expect(usesModules('x = 1;\nconsole.log(x);')).toBe(false); + expect(usesModules('a.import = 1; exported = 2;')).toBe(false); + expect(usesModules('import x from "./x";')).toBe(true); + expect(run({ 'main.js': 'undeclared = 5;\nconsole.log(undeclared);' }).out).toBe('5\n'); + // The words import and export in a string or a comment do not make a module. + expect(run({ 'main.js': 'x = 1; console.log("please import data", x);' }).out).toBe('please import data 1\n'); + expect(run({ 'main.js': '// TODO: export later\ny = 2;\nconsole.log(y);' }).out).toBe('2\n'); + }); + + it('keeps columns on line 1 of a transpiled file equal to the plain file', () => { + const column = (files: Record) => run(files).diagnostics[0]?.column; + const plain = column({ 'main.js': 'throw new Error("one");' }); + expect(plain).toBe(16); + expect(column({ 'main.ts': 'throw new Error("one");' })).toBe(plain); + expect(column({ 'main.js': 'throw new Error("one");\nimport "./u.js";', 'u.js': '' })).toBe(plain); + expect(column({ 'main.js': 'require("./u.js");', 'u.js': 'throw new Error("one");' })).toBe(plain); + }); + + it('stops with a clear message at constructs Sucrase 3.35.1 compiles wrong', () => { + const several = run({ 'main.ts': 'const x = 1;\nexport const a = 1, b = 2;\nconsole.log(a, b);' }); + expect(several.status).toBe('error'); + expect(several.err).toContain('Export one name in each statement here'); + expect(several.diagnostics).toEqual([expect.objectContaining({ file: 'main.ts', line: 2, kind: 'SyntaxError' })]); + const namespace = run({ + 'main.ts': '// set up\nnamespace Setup { console.log("configured"); }\nconsole.log("done");', + }); + expect(namespace.status).toBe('error'); + expect(namespace.out).toBe(''); + expect(namespace.err).toContain('A namespace or module block with code in it cannot run here'); + expect(namespace.diagnostics).toEqual([expect.objectContaining({ file: 'main.ts', line: 2 })]); + // Every form Sucrase compiles wrong, also where a global of the second name exists and the wrong code would run. + const forms = [ + 'export const a = 1,\n b = 2;', + 'export let a = 1,b = 2;', + 'export let a, b;', + 'export var a\n, b = 2;', + ]; + for (const form of forms) { + const module = run({ 'main.ts': 'var b = 0;\nimport * as u from "./u";\nconsole.log(u.a, b);', 'u.ts': form }); + expect([form, module.status, module.out]).toEqual([form, 'error', '']); + expect(module.err).toContain('Export one name in each statement here'); + expect(module.diagnostics).toEqual([expect.objectContaining({ file: 'u.ts', line: 1, kind: 'SyntaxError' })]); + } + // A comma expression on a line of its own is not an export, even when it starts with an exported name. + const comma = run({ 'main.ts': 'export let x = 1;\nx, console.log("comma");' }); + expect([comma.status, comma.out]).toEqual(['ok', 'comma\n']); + const imported = run({ 'main.ts': 'import { a } from "./u";\nconsole.log(a);', 'u.ts': 'export let a = 1, b;' }); + expect(imported.diagnostics).toEqual([expect.objectContaining({ file: 'u.ts', line: 1, kind: 'SyntaxError' })]); + // A declared namespace has no code, so Sucrase may drop it; one export per statement is fine. + const fine = [ + 'declare namespace D { const x: number; }', + 'export const a = 1;', + 'export const m: Map = new Map();', + 'console.log(a, m.size);', + ].join('\n'); + expect(run({ 'main.ts': fine }).out).toBe('1 0\n'); + }); +}); + +describe('modules', () => { + it('runs main.ts importing ./util.ts, and folders, .js names for .ts files, and JSON', () => { + const ran = run( + { + 'main.ts': [ + "import { add, PI } from './util';", + "import scale from './lib/scale.js';", + "import * as data from './data.json';", + 'console.log(add(2, 3), PI, scale(4), data.name);', + ].join('\n'), + 'util.ts': 'export const add = (a: number, b: number): number => a + b;\nexport const PI = 3.14;', + 'lib/scale.ts': + "import { add } from '../util.ts';\nexport default function scale(n: number): number { return add(n, n); }", + 'data.json': '{ "name": "demo" }', + }, + { main: 'main.ts' }, + ); + expect(ran.err).toBe(''); + expect(ran.out).toBe('5 3.14 8 demo\n'); + }); + + it('runs a module once, however many files import it', () => { + const ran = run({ + 'main.js': "import './a.js';\nimport './b.js';\nimport { count } from './counter.js';\nconsole.log(count());", + 'a.js': "import { count } from './counter.js';\ncount();", + 'b.js': "import { count } from './counter.js';\ncount();", + 'counter.js': 'console.log("loaded");\nlet n = 0;\nexport function count() { return ++n; }', + }); + expect(ran.out).toBe('loaded\n3\n'); + }); + + it('gives a clear error naming the path of a missing module', () => { + const ran = run({ 'main.ts': "import { x } from './nope';\nconsole.log(x);" }); + expect(ran.status).toBe('error'); + expect(ran.err).toContain("Cannot find module './nope' imported from main.ts."); + const pkg = run({ 'main.js': "import _ from 'lodash';" }); + expect(pkg.err).toContain("Cannot find module 'lodash' imported from main.js. Packages are not available here"); + }); + + it('maps an error in an imported file to that file and line', () => { + const ran = run({ + 'main.ts': "import { boom } from './util';\nboom();", + 'util.ts': 'export function boom(): void {\n const o: any = undefined;\n o.x;\n}', + }); + expect(ran.diagnostics[0]).toMatchObject({ file: 'util.ts', line: 3, kind: 'TypeError' }); + const syntax = run({ + 'main.ts': "import { y } from './bad';\nconsole.log(y);", + 'bad.ts': 'export const y = 1;\nlet z: = 2;', + }); + expect(syntax.diagnostics[0]).toMatchObject({ file: 'bad.ts', line: 2, kind: 'SyntaxError' }); + }); + + it('resolves only paths inside the project', () => { + const exists = (path: string) => ['a/b.ts', 'c.js'].includes(path); + expect(resolveModule('./b', 'a/main.ts', exists)).toEqual({ path: 'a/b.ts' }); + expect(resolveModule('../c', 'a/main.ts', exists)).toEqual({ path: 'c.js' }); + expect(resolveModule('../../c', 'a/main.ts', exists)).toEqual({ missing: 'file' }); + expect(resolveModule('https://example.com/x.js', 'main.js', exists)).toEqual({ missing: 'package' }); + }); +}); + +const tests = (source: string, path = 'math.test.ts') => run({ [path]: source }, { tests: true }); + +describe('TypeScript guards on hostile input', () => { + // The namespace guard once tried every blank line against all the blank lines after it: 80,000 took 7.9 s. + it('compiles a 190,000-line blank TypeScript file in well under a second', () => { + const source = `${'\n'.repeat(190_000)}console.log(1);`; + compile('main.ts', source, transform); + const started = performance.now(); + const compiled = compile('main.ts', source, transform); + expect(performance.now() - started).toBeLessThan(200); + expect('code' in compiled).toBe(true); + }); +}); + +describe('the Jest-compatible test API (known answers from Jest documentation)', () => { + it('passes toBeCloseTo and fails toBe for 0.1 + 0.2 against 0.3', () => { + const ran = tests( + [ + "test('close', () => { expect(0.1 + 0.2).toBeCloseTo(0.3); });", + "test('exact', () => {", + ' expect(0.1 + 0.2).toBe(0.3);', + '});', + ].join('\n'), + ); + expect(ran.tests.map((result) => [result.name, result.status])).toEqual([ + ['close', 'pass'], + ['exact', 'fail'], + ]); + const failed = ran.tests[1]!; + expect(failed.message).toContain('Expected: 0.3'); + expect(failed.message).toContain('Received: 0.30000000000000004'); + expect(failed.location).toEqual({ file: 'math.test.ts', line: 3 }); + expect(ran.status).toBe('error'); + expect(ran.out).toContain('Tests: 1 passed, 1 failed, 0 skipped, 2 total.'); + }); + + // toBeCloseTo(n, digits = 2) passes when |received - n| < 10^-digits / 2: these sit on each side of that bound. + it('puts the toBeCloseTo bound at half a unit of the last digit', () => { + const ran = tests( + [ + "test('0.304', () => expect(0.304).toBeCloseTo(0.3));", + "test('0.306', () => expect(0.306).toBeCloseTo(0.3));", + "test('1.23', () => expect(1.23).toBeCloseTo(1.2, 1));", + "test('1.26', () => expect(1.26).toBeCloseTo(1.2, 1));", + ].join('\n'), + ); + expect(ran.tests.map((result) => [result.name, result.status])).toEqual([ + ['0.304', 'pass'], + ['0.306', 'fail'], + ['1.23', 'pass'], + ['1.26', 'fail'], + ]); + }); + + it('lets toEqual ignore undefined properties and toStrictEqual not', () => { + const ran = tests( + [ + "test('equal', () => expect({ a: 1, b: undefined }).toEqual({ a: 1 }));", + "test('strict', () => expect({ a: 1, b: undefined }).toStrictEqual({ a: 1 }));", + "test('array items', () => expect([2]).toEqual([2, undefined]));", + "test('array length', () => expect([undefined]).toEqual([]));", + "test('sparse', () => expect([, 1]).toEqual([undefined, 1]));", + "test('array items strict', () => expect([2]).toStrictEqual([2, undefined]));", + "test('sparse strict', () => expect([, 1]).toStrictEqual([undefined, 1]));", + 'class LaCroix { constructor(flavor) { this.flavor = flavor; } }', + "test('types', () => expect(new LaCroix('lemon')).toEqual({ flavor: 'lemon' }));", + "test('types strict', () => expect(new LaCroix('lemon')).toStrictEqual({ flavor: 'lemon' }));", + ].join('\n'), + 'equal.test.js', + ); + // Arrays of different lengths differ, as in Jest's equality check; only a hole equals undefined. + expect(ran.tests.map((result) => result.status)).toEqual([ + 'pass', + 'fail', + 'fail', + 'fail', + 'pass', + 'fail', + 'fail', + 'pass', + 'fail', + ]); + }); + + it('matches toThrow against a message substring, a pattern, and a class', () => { + const ran = tests( + [ + "const boom = () => { throw new TypeError('bad thing happened'); };", + "test('substring', () => expect(boom).toThrow('thing'));", + "test('other text', () => expect(boom).toThrow('other'));", + "test('pattern', () => expect(boom).toThrow(/^bad/));", + "test('class', () => expect(boom).toThrow(TypeError));", + "test('no throw', () => expect(() => 1).toThrow());", + "test('not', () => expect(() => 1).not.toThrow());", + ].join('\n'), + ); + expect(ran.tests.map((result) => result.status)).toEqual(['pass', 'fail', 'pass', 'pass', 'fail', 'pass']); + }); + + it('has the other matchers and .not', () => { + const ran = tests( + [ + 'test("all", () => {', + ' expect(1).not.toBe(2);', + ' expect("x").toBeTruthy(); expect(0).toBeFalsy(); expect(null).toBeNull();', + ' expect(undefined).toBeUndefined(); expect(0).toBeDefined();', + ' expect([1, 2]).toContain(2); expect("team").not.toContain("I"); expect([1, 2, 3]).toHaveLength(3);', + ' expect("grey").toMatch(/gr[ae]y/); expect(10).toBeGreaterThan(9); expect(10).toBeGreaterThanOrEqual(10);', + ' expect(1).toBeLessThan(2); expect(2).toBeLessThanOrEqual(2); expect(0.1234).toBeCloseTo(0.12, 1);', + '});', + ].join('\n'), + ); + expect(ran.tests).toEqual([expect.objectContaining({ name: 'all', status: 'pass' })]); + }); + + it('names tests by their describe blocks and runs hooks in order', () => { + const ran = tests( + [ + 'const log = [];', + 'beforeEach(() => log.push("outer before"));', + 'afterEach(() => log.push("outer after"));', + 'describe("math", () => {', + ' beforeEach(() => log.push("inner before"));', + ' afterEach(() => log.push("inner after"));', + ' it("adds", () => { log.push("test"); expect(1 + 1).toBe(2); });', + '});', + 'const order = ["outer before", "inner before", "test", "inner after", "outer after", "outer before"];', + 'test("order", () => expect(log).toEqual(order));', + ].join('\n'), + 'hooks.test.js', + ); + expect(ran.tests.map((result) => [result.id, result.name, result.status])).toEqual([ + ['hooks.test.js::math > adds', 'math > adds', 'pass'], + ['hooks.test.js::order', 'order', 'pass'], + ]); + }); + + it('runs async tests, and says so when one never finishes', () => { + const ran = tests( + [ + 'test("async", async () => { const v = await Promise.resolve(3); expect(v).toBe(3); });', + 'test("never", () => new Promise(() => {}));', + 'test("after", () => {});', + ].join('\n'), + ); + expect(ran.tests.map((result) => result.status)).toEqual(['pass', 'error', 'error']); + expect(ran.tests[1]!.message).toContain('The test did not finish'); + }); + + it('tells an error in a test from a failed expectation, and a test file that does not load', () => { + const ran = run( + { + 'main.ts': 'export const one = 1;', + 'a.test.ts': [ + "import { one } from './main';", + "test('throws', () => { throw new RangeError('no'); });", + "test('uses', () => expect(one).toBe(1));", + ].join('\n'), + 'b.test.ts': "import './missing';", + }, + { tests: true }, + ); + // A file that does not load is reported when it is loaded, before any test runs. + expect(ran.tests.map((result) => [result.name, result.status])).toEqual([ + ['b.test.ts', 'error'], + ['throws', 'error'], + ['uses', 'pass'], + ]); + expect(ran.tests[0]!.message).toContain("Cannot find module './missing'"); + }); +}); + +describe('limits and caps in a test run', () => { + it('ends the run at the time limit inside a test, sync or async, and does not run later tests', () => { + for (const body of ['() => { while (true) {} }', 'async () => { await null; while (true) {} }']) { + const ran = run( + { + 'spin.test.js': ['test("first", () => {});', `test("spin", ${body});`, 'test("later", () => {});'].join('\n'), + }, + { tests: true, timeMs: 500 }, + ); + expect(ran.status).toBe('time'); + expect(ran.tests.map((result) => [result.name, result.status])).toEqual([ + ['first', 'pass'], + ['spin', 'time'], + ['later', 'skipped'], + ]); + expect(ran.tests[1]!.message).toContain('time limit'); + } + }); + + it('ends the run when a test runs out of memory', async () => { + const limit = 16 * 1024 * 1024; + const ran = run( + { + 'hog.test.js': [ + 'test("hog", () => { const a = []; while (true) a.push(new Array(1e5).fill(1)); });', + 'test("later", () => {});', + ].join('\n'), + }, + { tests: true, engine: await newEngine(limit), memoryBytes: limit, timeMs: 60_000 }, + ); + expect(ran.status).toBe('memory'); + expect(ran.tests.map((result) => [result.name, result.status])).toEqual([ + ['hog', 'error'], + ['later', 'skipped'], + ]); + }); + + it('ends the run as "memory" when a test fills the heap with small objects', async () => { + const ran = run( + { + 'small.test.js': [ + 'test("hog", () => { const a = []; for (;;) a.push({ i: a.length }); });', + 'test("later", () => {});', + ].join('\n'), + }, + { tests: true, engine: await newEngine(), timeMs: 60_000 }, + ); + expect(ran.status).toBe('memory'); + expect(ran.tests.map((result) => [result.name, result.status])).toEqual([ + ['hog', 'error'], + ['later', 'skipped'], + ]); + expect(ran.tests[0]!.message).toContain('memory'); + expect(ran.out).not.toContain('PASS'); + }); + + // While the run closes, the test API reports the unfinished tests. It must not call code the program replaced, and + // it has a hard deadline of its own, so the run still ends on time. + it('reports unfinished tests without the built-ins a program replaced', () => { + const spoil = [ + 'const spin = function () { for (;;) {} };', + 'Array.prototype.join = spin; Date.now = spin; globalThis.Error = spin;', + 'Object.defineProperty(Object.getPrototypeOf(RangeError.prototype), "name", { get: spin });', + ].join('\n'); + let started = performance.now(); + const timed = run( + { 'spin.test.js': `${spoil}\ntest("spin", () => { for (;;) {} });\ntest("later", () => {});` }, + { tests: true, timeMs: 300 }, + ); + expect(performance.now() - started).toBeLessThan(3000); + expect(timed.status).toBe('time'); + expect(timed.tests.map((result) => [result.name, result.status])).toEqual([ + ['spin', 'time'], + ['later', 'skipped'], + ]); + started = performance.now(); + const stuck = run( + { 'stuck.test.js': `${spoil}\ntest("stuck", () => new Promise(() => {}));\ntest("later", () => {});` }, + { tests: true, timeMs: 2000 }, + ); + expect(performance.now() - started).toBeLessThan(3000); + expect(stuck.status).toBe('error'); + expect(stuck.tests.map((result) => [result.name, result.status])).toEqual([ + ['stuck', 'error'], + ['later', 'error'], + ]); + }); + + it('runs at most PROJECT_LIMITS.tests tests, says so, and numbers equal names', () => { + const ran = run( + { 'many.test.js': 'for (let i = 0; i < 501; i++) test("same", () => {});' }, + { tests: true, timeMs: 60_000 }, + ); + expect(ran.tests).toHaveLength(500); + expect(ran.err).toContain('This run has 501 tests. Only the first 500 ran.'); + expect(ran.status).toBe('error'); + expect(ran.tests.slice(0, 3).map((result) => result.id)).toEqual([ + 'many.test.js::same', + 'many.test.js::same (2)', + 'many.test.js::same (3)', + ]); + expect(new Set(ran.tests.map((result) => result.id)).size).toBe(500); + }); + + it('stops sending results after process.exit in a test', () => { + const ran = tests('test("a", () => {});\ntest("b", () => { process.exit(0); });\ntest("c", () => {});'); + expect(ran.tests.map((result) => result.name)).toEqual(['a']); + expect(ran.exitCode).toBe(0); + }); +}); + +describe('more of the test API', () => { + it('runs beforeAll and afterAll once around their describe block', () => { + const ran = tests( + [ + 'const log = [];', + 'beforeAll(() => log.push("all"));', + 'describe("d", () => {', + ' beforeAll(() => log.push("d-all"));', + ' afterAll(() => log.push("d-end"));', + ' test("one", () => log.push("one"));', + ' test("two", () => log.push("two"));', + '});', + 'test("check", () => expect(log).toEqual(["all", "d-all", "one", "two", "d-end"]));', + ].join('\n'), + ); + expect(ran.tests.map((result) => result.status)).toEqual(['pass', 'pass', 'pass']); + }); + + it('fails every test of a describe whose beforeAll throws', () => { + const ran = tests( + 'describe("d", () => {\n beforeAll(() => { throw new Error("setup"); });\n test("x", () => {});\n});', + ); + expect(ran.tests.map((result) => [result.status, result.message])).toEqual([['error', 'Error: setup']]); + }); + + it('refuses .only and .each, and a test inside a test, with clear errors', () => { + const only = tests('test.only("x", () => {});'); + expect(only.tests[0]!.message).toContain('test.only is not supported here'); + const each = tests('it.each([1])("x", () => {});'); + expect(each.tests[0]!.message).toContain('it.each is not supported here'); + const nested = tests('test("outer", () => { test("inner", () => {}); });'); + expect(nested.tests.map((result) => result.status)).toEqual(['error']); + expect(nested.tests[0]!.message).toContain('A test cannot start inside another test'); + }); + + it('compares symbol keys, and Map keys by value, in toEqual', () => { + const ran = tests( + [ + 'const s = Symbol("s");', + 'test("symbols", () => expect({ [s]: 1 }).not.toEqual({ [s]: 2 }));', + 'test("map keys", () => expect(new Map([[{ k: 1 }, "v"]])).toEqual(new Map([[{ k: 1 }, "v"]])));', + ].join('\n'), + ); + expect(ran.tests.map((result) => result.status)).toEqual(['pass', 'pass']); + }); + + it('keeps reporting when the program replaces JSON.stringify and Object.keys', () => { + const ran = tests( + 'JSON.stringify = () => "x";\nObject.keys = () => [];\ntest("a", () => expect({ a: 1 }).not.toEqual({ a: 2 }));', + ); + expect(ran.tests.map((result) => result.status)).toEqual(['pass']); + }); +}); + +describe('diagnostics in project paths with spaces and parentheses', () => { + const files = (paths: string[]): RunFile[] => paths.map((path) => ({ path, text: 'a\nb\nc' })); + + it('finds the file from the end of each frame, longest path first', () => { + const list = files(['my file.ts', 'file.ts', 'main (1).js', 'src/my dir/x (2).ts']); + const map = (text: string) => quickjsMapper.map(text, list); + expect(map('Error: a\n at (my file.ts:2:16)\n')).toEqual([ + { file: 'my file.ts', line: 2, column: 16, message: 'a', kind: 'Error' }, + ]); + expect(map('TypeError: b\n at f (main (1).js:3:1)\n')[0]).toMatchObject({ file: 'main (1).js', line: 3 }); + expect(map('Error: c\n at src/my dir/x (2).ts:1:2\n')[0]).toMatchObject({ file: 'src/my dir/x (2).ts' }); + expect(map('SyntaxError: Unexpected token (main (1).js:2:3)\n')[0]).toMatchObject({ + file: 'main (1).js', + line: 2, + message: 'Unexpected token', + }); + }); + + it('keeps the first error line as the header when its message has more lines', () => { + const text = + 'AssertionError: expect(received).toBe(2)\n\nExpected: 2\nReceived: 1\n at (a.test.ts:2:5)\n'; + expect(quickjsMapper.map(text, files(['a.test.ts']))).toEqual([ + { file: 'a.test.ts', line: 2, column: 5, message: 'expect(received).toBe(2)', kind: 'AssertionError' }, + ]); + }); + + it('maps a real run and a failing test in such files', () => { + const ran = run({ 'my file.ts': 'const a = 1;\nthrow new Error("here");' }); + expect(ran.diagnostics).toEqual([expect.objectContaining({ file: 'my file.ts', line: 2 })]); + const tested = run({ 'main (1).test.js': 'test("t", () => {\n expect(1).toBe(2);\n});' }, { tests: true }); + expect(tested.tests[0]!.location).toEqual({ file: 'main (1).test.js', line: 2 }); + }); +}); + +describe('module edges', () => { + it('imports "." and ".." as their folder index, and a module that starts with "#!"', () => { + const ran = run({ + 'main.js': 'const a = require("./lib/a.js");\nconsole.log(a.value, require("./tool.js").name);', + 'lib/a.js': 'module.exports = { value: require(".").base + require("..").top };', + 'lib/index.js': 'module.exports = { base: 1 };', + 'index.js': 'module.exports = { top: 2 };', + 'tool.js': '#!/usr/bin/env node\nexports.name = "tool";', + }); + expect(ran.err).toBe(''); + expect(ran.out).toBe('3 tool\n'); + }); +}); diff --git a/app/src/features/codestudio/runtimes/js/js.test.tsx b/app/src/features/codestudio/runtimes/js/js.test.tsx new file mode 100644 index 000000000..3d2366022 --- /dev/null +++ b/app/src/features/codestudio/runtimes/js/js.test.tsx @@ -0,0 +1,143 @@ +// The JavaScript and TypeScript runtimes in a real browser, in the real run worker (run/worker.ts) after lockdown. +// The sandbox escapes are the same as code cells' JavaScript tests (codecells/sandbox/sandbox.test.tsx): no network, +// timers, modules from outside the project, or worker globals; the time and memory limits; deep recursion; and a +// program that fakes the time limit. Then one TypeScript project end to end: a module, a diagnostic, and tests. + +// checks-disable-file modifiability: one describe block per feature +import { describe, expect, it, vi } from 'vitest'; +import type { RunFile, RunLimits, RuntimeDiagnostic, RuntimeId } from '../../contracts/run'; +import { RunHost, createRunWorker } from '../../run/host'; +import type { RunOrder } from '../../run/host'; +import { RUN_LIMITS } from '../../run/protocol'; + +// Each run starts a worker and a WebAssembly runtime, which a loaded machine slows. +vi.setConfig({ testTimeout: 60_000 }); + +const QUICK = { ...RUN_LIMITS, timeMs: 1500 }; + +async function run(files: RunFile[], more: { runtime?: RuntimeId; limits?: RunLimits; tests?: boolean } = {}) { + const runs = new RunHost({ createWorker: createRunWorker }); + const order: RunOrder = { + runtime: more.runtime ?? 'javascript', + files, + main: files[0]!.path, + args: [], + stdin: { mode: 'prefilled', text: '' }, + limits: more.limits ?? RUN_LIMITS, + seed: 1, + ...(more.tests ? { test: { cases: [], framework: true } } : {}), + }; + let out = ''; + let err = ''; + const tests: string[] = []; + const diagnostics: RuntimeDiagnostic[] = []; + const handle = runs.run(order, { + onOutput: (stream, text) => (stream === 'out' ? (out += text) : (err += text)), + onTest: (result) => tests.push(`${result.name}: ${result.status}`), + onDiagnostics: (items) => diagnostics.push(...items), + }); + const done = await handle.done; + runs.dispose(); + return { ...done, out, err, tests, diagnostics }; +} + +const js = (source: string, limits?: RunLimits) => run([{ path: 'main.js', text: source }], limits ? { limits } : {}); + +describe('the sandbox', () => { + it('gives JavaScript programs no network, timers, modules, or worker globals', async () => { + const ran = await js( + [ + 'const names = ["fetch", "XMLHttpRequest", "WebSocket", "importScripts", "indexedDB", "caches", "navigator",', + ' "postMessage", "setTimeout", "Worker", "self", "location", "document", "window", "__TAURI_INTERNALS__"];', + 'console.log(names.filter((name) => typeof globalThis[name] !== "undefined").join(",") || "none");', + 'console.log(typeof Function("return this")().fetch);', + 'import("https://example.com/x.js").then(() => console.log("loaded"), () => console.log("no import"));', + ].join('\n'), + ); + expect(ran.status).toBe('ok'); + expect(ran.out).toBe('none\nundefined\nno import\n'); + }); + + // Without the capped heap, the memory program grew toward the 2 GB WebAssembly ceiling and met the time limit first. + it('stops a program at the time limit and at the memory limit', async () => { + expect((await js('while (true) {}', QUICK)).status).toBe('time'); + const hog = await js('const a = []; while (true) a.push(new Array(1e5).fill(1));', QUICK); + expect(hog.status).toBe('memory'); + // A smaller limit of the run's own: arrays grown by push, which QuickJS's own count misses. + const pushes = 'const a = []; while (true) { const b = []; for (let i = 0; i < 1000; i++) b.push(i); a.push(b); }'; + expect((await js(pushes, { ...RUN_LIMITS, memoryBytes: 8 * 1024 * 1024 })).status).toBe('memory'); + }); + + // Small blocks fill the heap between two checks; QuickJS then threw null and the run ended "Uncaught null". + it('stops many small allocations at the default memory limit', async () => { + const programs = [ + "const a = []; for (;;) a.push({ i: a.length, s: 'x' + a.length });", + 'let n = null; for (;;) n = { next: n };', + 'const m = new Map(); for (let i = 0; ; i++) m.set(i, { i });', + ]; + for (const program of programs) { + const ran = await js(program); + expect([program, ran.status, ran.err]).toEqual([program, 'memory', 'InternalError: out of memory\n']); + } + }); + + it('ends at the time limit when the loop runs inside a promise job', async () => { + const ran = await js( + 'async function main() { await null; while (true) {} }\nmain();\nconsole.log("started");', + QUICK, + ); + expect(ran.status).toBe('time'); + }); + + it('reports deep recursion as an error in the program', async () => { + const ran = await js('function f() { f(); }\nf();'); + expect(ran.status).toBe('error'); + expect(ran.err).toMatch(/stack overflow/i); + }); + + it('does not let a program fake the time limit', async () => { + const ran = await js('throw { name: "InternalError", message: "interrupted" };'); + expect(ran.status).toBe('error'); + }); +}); + +describe('a TypeScript project', () => { + it('runs a module, reports its version, and maps an error to its line', async () => { + const ran = await run( + [ + { + path: 'main.ts', + text: "import { half } from './util';\nconst n: number = half(10);\nconsole.log(n);\nhalf(-1);", + }, + { + path: 'util.ts', + text: 'export function half(n: number): number {\n if (n < 0) throw new RangeError("negative");\n return n / 2;\n}', + }, + ], + { runtime: 'typescript' }, + ); + expect(ran.out).toBe('5\n'); + expect(ran.status).toBe('error'); + expect(ran.version).toMatch(/^TypeScript through Sucrase 3\.35\.1, QuickJS/); + expect(ran.diagnostics).toEqual([expect.objectContaining({ file: 'util.ts', line: 2, kind: 'RangeError' })]); + }); + + it('runs its tests and sends one result per test', async () => { + const ran = await run( + [ + { path: 'main.ts', text: 'export const sum = (a: number, b: number): number => a + b;' }, + { + path: 'main.test.ts', + text: [ + "import { sum } from './main';", + "test('adds', () => expect(sum(1, 2)).toBe(3));", + "test('wrong', () => expect(sum(1, 2)).toBe(4));", + ].join('\n'), + }, + ], + { runtime: 'typescript', tests: true }, + ); + expect(ran.tests).toEqual(['adds: pass', 'wrong: fail']); + expect(ran.out).toContain('Tests: 1 passed, 1 failed, 0 skipped, 2 total.'); + }); +}); diff --git a/app/src/features/codestudio/runtimes/js/modules.ts b/app/src/features/codestudio/runtimes/js/modules.ts new file mode 100644 index 000000000..ad3a7b3d5 --- /dev/null +++ b/app/src/features/codestudio/runtimes/js/modules.ts @@ -0,0 +1,52 @@ +// Finding the project file an `import` or `require` names (DESIGN 4.3). Only project files can be imported: a path that +// starts with "./" or "../", relative to the importing file. There are no npm packages and no network. + +import { isProjectPath } from '../../run/mount'; + +/** The file kinds a program can run or import. TypeScript's are transpiled by Sucrase first (transpile.ts). */ +export const SCRIPT_FILE = /\.(?:[mc]?js|[mc]?ts)$/; +export const TYPESCRIPT_FILE = /\.[mc]?ts$/; + +/** Tried in order after the specifier as written. Node tries endings this way, and so does TypeScript's bundler mode. */ +const ENDINGS = ['', '.ts', '.js', '.mts', '.mjs', '.cts', '.cjs', '.json', '/index.ts', '/index.js']; + +/** The folder of a project path: "" for a file at the top. */ +export function folderOf(path: string): string { + const slash = path.lastIndexOf('/'); + return slash < 0 ? '' : path.slice(0, slash); +} + +/** Joins a relative specifier to a folder, resolving "." and ".."; null when it leaves the project. */ +export function joinPath(folder: string, specifier: string): string | null { + const parts = folder === '' ? [] : folder.split('/'); + for (const part of specifier.split('/')) { + if (part === '' || part === '.') continue; + if (part === '..') { + if (parts.length === 0) return null; + parts.pop(); + } else { + parts.push(part); + } + } + const joined = parts.join('/'); + return isProjectPath(joined) ? joined : null; +} + +export type Resolved = { path: string } | { missing: 'file' | 'package' }; + +/** + * The project file `specifier` names when imported from `from`. A specifier that does not start with "./" or "../" + * is a package, which the sandbox does not have. TypeScript's own rule is kept too: "./util.js" finds util.ts. + */ +export function resolveModule(written: string, from: string, exists: (path: string) => boolean): Resolved { + // "." and ".." name a folder, as in Node: its index file. + const specifier = written === '.' || written === '..' ? `${written}/index` : written; + if (!specifier.startsWith('./') && !specifier.startsWith('../')) return { missing: 'package' }; + const joined = joinPath(folderOf(from), specifier); + if (joined === null) return { missing: 'file' }; + const typed = joined.replace(/\.([mc]?)js$/, '.$1ts'); + for (const candidate of [...ENDINGS.map((ending) => joined + ending), typed]) { + if (exists(candidate)) return { path: candidate }; + } + return { missing: 'file' }; +} diff --git a/app/src/features/codestudio/runtimes/js/transpile.ts b/app/src/features/codestudio/runtimes/js/transpile.ts new file mode 100644 index 000000000..d8905eb04 --- /dev/null +++ b/app/src/features/codestudio/runtimes/js/transpile.ts @@ -0,0 +1,112 @@ +// Turns a project file into the script text QuickJS runs (DESIGN 2.1). Sucrase (MIT; THIRD_PARTY_NOTICES.md) strips +// TypeScript's types and turns `import` and `export` into `require` and `exports`, which the module loader serves. +// Sucrase keeps every line where it was, so a line in a QuickJS error is the same line in the file. +// +// Sucrase puts its own code ("use strict" and helpers) in front of line 1. The file goes in with an empty line on +// top, so that code comes out alone on line 1; it is then joined to the file's first line again. Its length is the +// `shift` of every column on line 1, which the engine takes off again in error places. + +import { TYPESCRIPT_FILE } from './modules'; + +/** Sucrase's `transform`, passed in so this file loads Sucrase only when the adapter does. */ +export type Transform = ( + code: string, + options: { transforms: ('typescript' | 'imports')[]; filePath: string; disableESTransforms: boolean }, +) => { code: string }; + +/** The text to run and how many characters were put before line 1; or where the file cannot be compiled. */ +export type Compiled = + | { code: string; shift: number } + | { line: number; column: number; message: string; unsupported?: 'namespace' | 'declarators' }; + +/** Ends Sucrase's line 1, so a program with no expression of its own has no last value to show. */ +const END_PREFIX = 'void 0;'; +/** `import` or `export` as a statement or as `import(...)`, not as part of a longer name or a property. */ +const MODULE_SYNTAX = /(?:^|[^.\w$])(?:import|export)\b/; +/** + * A TypeScript namespace or module block that is not `declare`d. Sucrase 3.35.1 removes such a block without a word, + * so its code would never run. A match starts only at one of `;{}` or a line break. The spaces after that start are + * not line breaks, so each character is tried once, and long runs of blank lines stay linear. + */ +const NAMESPACE = /(^|[;{}\n])[^\S\n]*(?:export\s+)?(?:namespace|module)\s+[A-Za-z_$][\w$.]*\s*\{/g; +/** + * Sucrase 3.35.1 turns `export const a = 1, b = 2;` into `const a = 1; exports.a = a, b = 2;`, and `export let a, b;` + * into `exports.a, b;`, which are wrong. The comma may stand after any spaces or line breaks. A match starts only at + * one of `;{}` or a line start. The spaces after that start are not line breaks, so each start is tried once. + */ +const SEVERAL_DECLARATORS = /(?:^|[;{}])[^\S\n]*exports\.([A-Za-z_$][\w$]*)(?:\s*=\s*\1)?\s*,/gm; +const EXPORT = /\bexport\b/; + +/** Whether a JavaScript file might need the imports transform: a quick test before the exact one in `compile`. */ +export function usesModules(text: string): boolean { + return MODULE_SYNTAX.test(text); +} + +/** Gives the line of each index in `text`, for indexes that only grow: each line break is counted once. */ +function lineCounter(text: string): (index: number) => number { + let line = 1; + let counted = 0; + return (index) => { + for (let at = text.indexOf('\n', counted); at >= 0 && at < index; at = text.indexOf('\n', at + 1)) line++; + counted = Math.max(counted, index); + return line; + }; +} + +/** The line of the first export statement with several declarators, which Sucrase compiles wrong. */ +function severalDeclarators(text: string, out: string): number | null { + const lineOf = lineCounter(out); + const textLines = text.split('\n'); + for (const found of out.matchAll(SEVERAL_DECLARATORS)) { + const line = lineOf(found.index ?? 0); + // Sucrase keeps lines, so the statement's own line still says `export`; a plain `x, y;` there does not. + if (EXPORT.test(textLines[line - 1] ?? '')) return line; + } + return null; +} + +/** The first `namespace` block Sucrase dropped: in the file, but no longer on its line in the output. */ +function droppedNamespace(text: string, out: string): number | null { + const outLines = out.split('\n'); + const lineOf = lineCounter(text); + for (const found of text.matchAll(NAMESPACE)) { + const line = lineOf((found.index ?? 0) + (found[1]?.length ?? 0)); + if (!/\b(?:namespace|module)\b/.test(outLines[line - 1] ?? '')) return line; + } + return null; +} + +/** + * The text QuickJS runs for a project file, or where it cannot be compiled. TypeScript always goes through the + * imports transform too, since TypeScript files are modules. A JavaScript file goes through it only when that + * changes the text, so a plain script keeps sloppy mode, as code cells do. + */ +export function compile(path: string, source: string, transform: Transform): Compiled { + // A "#!" line is a comment of the same length, so it may stand inside the module wrapper too. + const text = source.startsWith('#!') ? `//${source.slice(2)}` : source; + const typescript = TYPESCRIPT_FILE.test(path); + if (!typescript && !usesModules(text)) return { code: text, shift: 0 }; + let out: string; + try { + const transforms: ('typescript' | 'imports')[] = typescript ? ['typescript', 'imports'] : ['imports']; + out = transform(`\n${text}`, { transforms, filePath: path, disableESTransforms: true }).code; + } catch (error) { + const raw = String((error as Error)?.message ?? error); + // Sucrase's message reads "Error transforming main.ts: Unexpected token (9:5)", with the column from 1. The + // empty line on top moves every line down by one. + const place = /\((\d+):(\d+)\)\s*$/.exec(raw); + const message = raw.replace(/^Error transforming [^:]*: /, '').replace(/\s*\(\d+:\d+\)\s*$/, ''); + return { line: place ? Math.max(1, Number(place[1]) - 1) : 1, column: place ? Number(place[2]) : 1, message }; + } + const newline = out.indexOf('\n'); + const prefix = newline < 0 ? out : out.slice(0, newline); + const body = newline < 0 ? '' : out.slice(newline + 1); + if (!typescript && body === text) return { code: text, shift: 0 }; + if (typescript) { + const line = droppedNamespace(text, body); + if (line !== null) return { line, column: 1, message: '', unsupported: 'namespace' }; + } + const several = severalDeclarators(text, body); + if (several !== null) return { line: several, column: 1, message: '', unsupported: 'declarators' }; + return { code: `${prefix}${END_PREFIX}${body}`, shift: prefix.length + END_PREFIX.length }; +} diff --git a/app/src/features/codestudio/runtimes/python/fixtures/reach.ts b/app/src/features/codestudio/runtimes/python/fixtures/reach.ts new file mode 100644 index 000000000..c5977874f --- /dev/null +++ b/app/src/features/codestudio/runtimes/python/fixtures/reach.ts @@ -0,0 +1,123 @@ +// A program that tries each route past full Python's cut-off it knows. Then it looks for a JavaScript value, or a +// way to make one, in all Python sees. It runs without lockdown in python.test.ts and in the worker in python.test.tsx. +// +// The routes: the import hook's table, dropping the refusing finder, the original `_imp.create_builtin` from the +// wrapper's closure, and Pyodide's classes. The walk covers each object the garbage collector tracks, what it refers +// to, and its attributes. A JavaScript value found there is walked in turn to a depth of four: its properties, its +// prototypes, and theirs. + +const lines = (...text: string[]) => text.join('\n') + '\n'; + +export const REACH = lines( + 'import sys, gc, _imp', + 'reached = []', + 'def is_js(value):', + ' try:', + ' return any(c.__name__ == "JsProxy" for c in type(value).__mro__)', + ' except Exception:', + ' return False', + 'def makes_js(value):', + ' owner = getattr(value, "__self__", None)', + ' return getattr(owner, "__name__", None) == "_pyodide_core" and callable(value)', + 'held = [f for f in sys.meta_path if getattr(f, "jsproxies", None)]', + 'print("import hook:", len(held))', + 'for f in held:', + ' reached.extend(f.jsproxies.values())', + 'first = sys.meta_path.pop(0)', + 'try:', + ' import js', + ' reached.append(js)', + ' print("import js: imported")', + 'except ImportError:', + ' print("import js: refused")', + 'sys.meta_path.insert(0, first)', + 'makers = []', + 'create = _imp.create_builtin', + 'while getattr(create, "__closure__", None):', + ' create = create.__closure__[0].cell_contents', + 'try:', + ' from importlib.machinery import BuiltinImporter', + ' core = create(BuiltinImporter.find_spec("_pyodide_core"))', + ' makers.extend(name for name in ("to_js", "create_proxy", "create_once_callable") if hasattr(core, name))', + 'except Exception:', + ' pass', + 'print("closure makers:", makers)', + 'def classes(base, out, ids):', + ' for sub in type.__subclasses__(base):', + ' if id(sub) not in ids:', + ' ids.add(id(sub))', + ' out.append(sub)', + ' classes(sub, out, ids)', + ' return out', + 'made = []', + 'for cls in classes(object, [], set()):', + ' if not cls.__name__.startswith("Js"):', + ' continue', + ' for how in ("call", "new", "dunder"):', + ' try:', + ' if how == "call":', + ' value = cls("A", "b")', + ' elif how == "new":', + ' value = cls.new("A", "b")', + ' else:', + ' value = cls.__new__(cls, "A", "b")', + ' if is_js(value):', + ' made.append(cls.__name__ + "." + how)', + ' reached.append(value)', + ' except BaseException:', + ' pass', + 'print("classes make:", made)', + 'found = []', + 'for item in gc.get_objects():', + ' values = [item] + list(gc.get_referents(item))', + ' attrs = getattr(item, "__dict__", None)', + ' if isinstance(attrs, dict):', + ' values += list(attrs.values())', + ' found.extend(v for v in values if is_js(v) or makes_js(v))', + 'reached.extend(v for v in found if is_js(v))', + 'makers.extend(getattr(v, "__name__", "?") for v in found if makes_js(v))', + 'print("makers:", sorted(set(makers)))', + 'names = {"fetch", "eval", "postMessage", "setTimeout", "setInterval", "clearTimeout", "importScripts"}', + 'names |= {"XMLHttpRequest", "WebSocket", "queueMicrotask", "MessageChannel", "Worker", "runPython", "FS"}', + 'powers = set()', + 'seen = set()', + 'def walk(value, depth):', + ' if depth > 4 or id(value) in seen:', + ' return', + ' seen.add(id(value))', + ' try:', + ' keys = dir(value)', + ' except Exception:', + ' return', + ' powers.update(k for k in keys if k in names)', + ' for key in keys + ["constructor", "__proto__", "prototype"]:', + ' try:', + ' child = getattr(value, key)', + ' except Exception:', + ' continue', + ' if is_js(child):', + ' walk(child, depth + 1)', + 'ran = False', + 'for value in reached:', + ' walk(value, 0)', + ' try:', + ' value.constructor.constructor("return 1")()', + ' ran = True', + ' except Exception:', + ' pass', + 'print("reached:", len(reached))', + 'print("powers:", sorted(powers))', + 'print("code from text:", "ran" if ran else "refused")', +); + +/** What REACH prints when the cut-off holds. */ +export const REACH_HELD = lines( + 'import hook: 0', + 'import js: refused', + 'closure makers: []', + 'classes make: []', + 'makers: []', + 'reached: 0', + 'powers: []', + 'code from text: refused', +); diff --git a/app/src/features/codestudio/runtimes/python/fixtures/worker.ts b/app/src/features/codestudio/runtimes/python/fixtures/worker.ts new file mode 100644 index 000000000..cee72fc16 --- /dev/null +++ b/app/src/features/codestudio/runtimes/python/fixtures/worker.ts @@ -0,0 +1,9 @@ +// The run worker of python.test.tsx for full Python. It uses the real run protocol (serve.ts) and Pyodide adapter. +// Pyodide's files come from the test server, since only the app serves the pack scheme. +import lockUrl from 'pyodide/pyodide-lock.json?url'; +import { serve } from '../../../run/serve'; +import { pyodideAdapter } from '../pyodide'; + +const indexURL = new URL('./', new URL(lockUrl, location.href)).href; + +serve({ 'python-full': () => Promise.resolve(pyodideAdapter(indexURL)) }); diff --git a/app/src/features/codestudio/runtimes/python/heap.ts b/app/src/features/codestudio/runtimes/python/heap.ts new file mode 100644 index 000000000..905173619 --- /dev/null +++ b/app/src/features/codestudio/runtimes/python/heap.ts @@ -0,0 +1,190 @@ +// The memory limit for both Python runtimes. Each runtime's heap is a WebAssembly memory that Emscripten grows when +// an allocation does not fit. Emscripten takes a refused `grow` as an allocation that failed, so Python raises +// MemoryError and the run ends as "memory". Neither runtime exports its memory object to the page, so `grow` is +// wrapped once on the prototype, and each heap says which memory is its own. + +/** Bytes in one WebAssembly page. */ +const PAGE = 65_536; + +interface Capped { + owns(memory: WebAssembly.Memory): boolean; + cap: number; +} + +const capped: Capped[] = []; + +function wrapGrow(): void { + const proto = WebAssembly.Memory.prototype as WebAssembly.Memory & { onCapped?: true }; + if (proto.onCapped) return; + const grow = proto.grow; + Object.defineProperty(proto, 'grow', { + configurable: true, + writable: true, + value: function (this: WebAssembly.Memory, delta: number): number { + const heap = capped.find((item) => item.owns(this)); + if (heap && this.buffer.byteLength + delta * PAGE > heap.cap) { + throw new RangeError('The run has no more memory.'); + } + return grow.call(this, delta); + }, + }); + Object.defineProperty(proto, 'onCapped', { value: true }); +} + +/** + * Caps how far one heap may grow. `owns` tells the heap's memory from others, and `size` is its size in bytes now. + * Returns the setter of the cap, in bytes past the size at this call, which is the size after the runtime loaded. + */ +export function capHeap(owns: (memory: WebAssembly.Memory) => boolean, size: () => number): (bytes: number) => void { + wrapGrow(); + const entry: Capped = { owns, cap: Infinity }; + capped.push(entry); + const base = size(); + return (bytes) => { + entry.cap = base + bytes; + }; +} + +/** A node of Emscripten's file system, as far as the file cap reads it. MEMFS files keep their size in usedBytes. */ +interface FileNode { + mode: number; + usedBytes?: number; + contents?: unknown; + mounted?: { root: FileNode } | null; +} + +/** The part of Emscripten's FS object the file cap uses. */ +export interface CappedFS { + root: FileNode; + isFile(mode: number): boolean; + isDir(mode: number): boolean; + lookupPath(path: string): { node: FileNode }; + writeFile(path: string, data: string): void; + unlink(path: string): void; + ErrnoError: new (errno: number) => Error; +} + +type StreamWrite = ( + stream: { node: FileNode }, + buffer: unknown, + offset: number, + length: number, + position: number, + own?: boolean, +) => number; +type SetAttr = (node: FileNode, attr: { size?: number }) => void; + +/** Emscripten's errno for "No space left on device"; Python raises OSError with errno.ENOSPC. */ +const ENOSPC = 51; + +/** The bytes in every MEMFS file under the root, mounts included. */ +function fileBytes(fs: CappedFS): number { + let total = 0; + const seen = new Set(); + const visit = (node: FileNode) => { + if (seen.has(node)) return; + seen.add(node); + if (node.mounted) visit(node.mounted.root); + if (fs.isFile(node.mode)) total += node.usedBytes ?? 0; + else if (fs.isDir(node.mode) && node.contents && typeof node.contents === 'object') { + for (const child of Object.values(node.contents as Record)) visit(child); + } + }; + visit(fs.root); + return total; +} + +const fileCaps = new WeakMap void>(); + +/** + * Caps the file data a run may add to the in-memory file system, which lives outside the WebAssembly heap. Each write + * and size change of an MEMFS file is checked. One that takes the files more than `bytes` past their size at this call + * fails with ENOSPC, so Python raises OSError. `null` lifts the cap. The file at `spare` may use `reserve` bytes more: + * it is the run's own state file, written after the program. + */ +export function capFiles( + fs: CappedFS, + scratch: string, + spare: string, + reserve: number, +): (bytes: number | null) => void { + const known = fileCaps.get(fs); + if (known) return known; + let cap = Infinity; + let used = 0; + let spareNode: FileNode | null = null; + const grow = (node: FileNode, size: number) => { + const growth = size - (node.usedBytes ?? 0); + if (growth <= 0 || cap === Infinity) return; + const room = cap + (node === spareNode ? reserve : 0); + // `used` misses files removed since the last count, so it is counted again before a write is refused. + if (used + growth > room) used = fileBytes(fs); + if (used + growth > room) throw new fs.ErrnoError(ENOSPC); + used += growth; + }; + // MEMFS files share one table of stream functions and one of node functions; a file made here finds them. + const probe = `${scratch}/.cap`; + fs.writeFile(probe, ''); + const sample = fs.lookupPath(probe).node as FileNode & { + stream_ops: { write: StreamWrite }; + node_ops: { setattr: SetAttr }; + }; + fs.unlink(probe); + const streamOps = sample.stream_ops; + const nodeOps = sample.node_ops; + const write = streamOps.write; + streamOps.write = function (this: unknown, ...args: Parameters) { + const [stream, , , length, position] = args; + grow(stream.node, Math.max(stream.node.usedBytes ?? 0, position + length)); + return write.apply(this, args); + }; + const setattr = nodeOps.setattr; + nodeOps.setattr = function (node, attr) { + if (typeof attr.size === 'number') grow(node, attr.size); + setattr.call(this, node, attr); + }; + const limit = (bytes: number | null) => { + used = fileBytes(fs); + cap = bytes === null ? Infinity : used + bytes; + if (bytes === null) return; + try { + fs.writeFile(spare, ''); + spareNode = fs.lookupPath(spare).node; + } catch { + spareNode = null; + } + }; + fileCaps.set(fs, limit); + return limit; +} + +type Instantiate = (...args: unknown[]) => Promise; + +/** + * Runs `load` and returns the memory of the WebAssembly instance it makes that exports `marker`. It watches + * WebAssembly.instantiate and instantiateStreaming while `load` runs, and puts both back after. + */ +export async function loadWithMemory( + marker: string, + load: () => Promise, +): Promise<{ value: T; memory: WebAssembly.Memory | null }> { + const wasm = WebAssembly as unknown as Record<'instantiate' | 'instantiateStreaming', Instantiate | undefined>; + const originals = { instantiate: wasm.instantiate, instantiateStreaming: wasm.instantiateStreaming }; + let memory: WebAssembly.Memory | null = null; + const watch = (original: Instantiate | undefined) => + original && + (async (...args: unknown[]) => { + const made = (await original.apply(WebAssembly, args)) as { instance?: WebAssembly.Instance; exports?: unknown }; + const exports = (made.instance ?? (made as WebAssembly.Instance)).exports; + if (exports && marker in exports && exports.memory instanceof WebAssembly.Memory) memory = exports.memory; + return made; + }); + wasm.instantiate = watch(originals.instantiate); + wasm.instantiateStreaming = watch(originals.instantiateStreaming); + try { + return { value: await load(), memory }; + } finally { + wasm.instantiate = originals.instantiate; + wasm.instantiateStreaming = originals.instantiateStreaming; + } +} diff --git a/app/src/features/codestudio/runtimes/python/index.ts b/app/src/features/codestudio/runtimes/python/index.ts new file mode 100644 index 000000000..4cdbaaf8b --- /dev/null +++ b/app/src/features/codestudio/runtimes/python/index.ts @@ -0,0 +1,8 @@ +// The two Python runtimes as the run worker loads them (run/adapters.ts): quick (MicroPython, bundled) and full +// (Pyodide, from its pack). +import wasmUrl from '@micropython/micropython-webassembly-pyscript/micropython.wasm?url'; +import { microPythonAdapter } from './micropython'; +import { pyodideAdapter } from './pyodide'; + +export const pythonQuickAdapter = microPythonAdapter(() => Promise.resolve(wasmUrl)); +export const pythonFullAdapter = pyodideAdapter(); diff --git a/app/src/features/codestudio/runtimes/python/micropython.ts b/app/src/features/codestudio/runtimes/python/micropython.ts new file mode 100644 index 000000000..d32326fd4 --- /dev/null +++ b/app/src/features/codestudio/runtimes/python/micropython.ts @@ -0,0 +1,112 @@ +// Python (quick): MicroPython's WebAssembly port (MIT, bundled, already shipped for code cells) behind the +// RuntimeAdapter contract (DESIGN 2.1). It loads in the warm worker before lockdown. Its heap may grow by the run's +// memory limit and no further (heap.ts), so a program that asks for more gets MemoryError. Input is the prefilled +// text only: the port runs synchronously, so it cannot wait for the page (DESIGN 4.4). + +import { loadMicroPython } from '@micropython/micropython-webassembly-pyscript'; +import type { RuntimeAdapter } from '../../contracts/run'; +import { ALLOWANCES } from '../../run/adapters'; +import { codeStudioPython } from '../../../../strings/en/codeStudio/python'; +import { capHeap, loadWithMemory } from './heap'; +import { checkPython, runPython } from './session'; +import type { EmscriptenFS, PythonEngine } from './session'; +import { pythonMapper } from './traceback'; + +/** The pinned version in package.json (a unit test keeps them equal). */ +export const MICROPYTHON_VERSION = '1.27.0'; + +/** The runtime's tables of the JavaScript values Python holds: by number, and the number of each value. */ +interface RuntimeTables { + proxy_js_ref?: unknown[]; + proxy_js_ref_map?: Map; +} + +/** + * Leaves Python no JavaScript value to use, as code cells do (codecells/sandbox/python.ts). Python reaches one only + * through the runtime's table, where entry 0 is the global object (the js module) and every value it is given is + * added. Entry 0 and every object or function added become one empty object, and the map from values to numbers + * keeps nothing, so `js` and `jsffi` give nothing to call, read, or grow. + */ +export function cutOffJavaScript(tables: RuntimeTables): void { + const table = tables.proxy_js_ref; + if (!Array.isArray(table) || !(tables.proxy_js_ref_map instanceof Map)) { + throw new Error(codeStudioPython.startFailed); + } + const inert: unknown = Object.freeze(Object.create(null)); + const isValue = (value: unknown) => (typeof value === 'object' && value !== null) || typeof value === 'function'; + table[0] = inert; + tables.proxy_js_ref = new Proxy(table, { + set(target, key, value: unknown) { + return Reflect.set(target, key, typeof key === 'string' && /^\d+$/.test(key) && isValue(value) ? inert : value); + }, + }); + tables.proxy_js_ref_map = new Map(); + tables.proxy_js_ref_map.set = function (this: Map) { + return this; + }; +} + +/** + * The heap MicroPython starts with. When it is full, MicroPython adds more heap, so the run's memory limit is the cap + * on that growth (heap.ts), not this size. + */ +const START_HEAP = 1024 * 1024; + +/** Loads MicroPython with its output going to the engine's sink. `url` is the wasm file. */ +export async function loadMicroPythonEngine(url: string): Promise { + const engine: PythonEngine = { + fs: null as unknown as EmscriptenFS, + exec: () => undefined, + sink: () => undefined, + }; + const { value: python, memory } = await loadWithMemory('mp_js_init', () => + loadMicroPython({ + url, + heapsize: START_HEAP, + linebuffer: false, + stdout: (bytes: Uint8Array) => engine.sink('out', bytes), + stderr: (bytes: Uint8Array) => engine.sink('err', bytes), + }), + ); + // Without its memory there is no limit, so the runtime does not start. + if (!memory) throw new Error(codeStudioPython.startFailed); + // The limit counts from the top of what the runtime uses, not from the end of its memory. The memory starts with + // free room, and the program would get it on top of the limit. A small block marks that top. + const module = (python as unknown as { _module: { _malloc(size: number): number; _free(at: number): void } })._module; + const top = module._malloc(1); + module._free(top); + engine.limitMemory = capHeap( + (other) => other === memory, + () => top, + ); + cutOffJavaScript(globalThis as unknown as RuntimeTables); + engine.fs = (python as unknown as { FS: EmscriptenFS }).FS; + engine.exec = (source) => { + python.runPython(source); + }; + return engine; +} + +/** The quick Python adapter. `wasmUrl` is where the worker finds micropython.wasm. */ +export function microPythonAdapter(wasmUrl: () => Promise): RuntimeAdapter { + let engine: Promise | null = null; + const loaded = () => (engine ??= wasmUrl().then((url) => loadMicroPythonEngine(url))); + return { + id: 'python-quick', + version: `MicroPython ${MICROPYTHON_VERSION}`, + allowances: ALLOWANCES['python-quick'], + // The port runs synchronously, so it cannot wait for the page (DESIGN 4.4). + prefilledOnly: true, + diagnostics: pythonMapper, + async load(onProgress) { + onProgress(10); + await loaded(); + }, + async run(request, emit, context) { + return runPython(await loaded(), request, emit, context); + }, + async check(file, files) { + return checkPython(await loaded(), file, files); + }, + }; +} diff --git a/app/src/features/codestudio/runtimes/python/prelude.ts b/app/src/features/codestudio/runtimes/python/prelude.ts new file mode 100644 index 000000000..4e7071501 --- /dev/null +++ b/app/src/features/codestudio/runtimes/python/prelude.ts @@ -0,0 +1,303 @@ +// The Python side of a run, the same text for MicroPython and CPython (Pyodide). The page writes the run's settings +// to RUN_FILE, runs RUN_SOURCE at the top level, and reads how the run ended from STATE_FILE. +// +// The program runs as `python main.py` would run it. Its `__name__` is "__main__". sys.argv holds the main path and +// the arguments. sys.path[0] is the main file's folder, and the working folder is the project's root. Standard +// input is the prefilled text; input() at its end raises EOFError and the run notes that the program wanted more. A +// traceback leaves out this file's own frames and shows project paths without the mount's root, so it reads like +// CPython's or MicroPython's own. + +/** Where the project is mounted inside the runtime's in-memory file system. */ +export const HOME = '/project'; +/** The run's own files, outside the project. */ +export const SCRATCH = '/opennote'; +export const RUN_FILE = `${SCRATCH}/run.json`; +export const STATE_FILE = `${SCRATCH}/state.json`; +export const CHECK_FILE = `${SCRATCH}/check.json`; +export const CHECKED_FILE = `${SCRATCH}/checked.json`; + +/** How the Python side says the run ended. `kind` is the exception's type name. */ +export interface PythonState { + status: 'ok' | 'error'; + exit: number; + kind: string; + wantedInput: boolean; + /** True when the program failed before it ran: compile() itself gave up, as on a parser stack overflow. */ + compiling?: boolean; +} + +/** What the compile() check found: nothing, or one error with the place CPython gives (MicroPython gives text). */ +export interface PythonChecked { + ok: boolean; + kind?: string; + text?: string; + /** The error's own last line, for an error that has no frame to map. */ + message?: string; + line?: number | null; + column?: number | null; +} + +// Shared helpers: frames of this file are "" (Pyodide's runPython) or "" (MicroPython's). +// MicroPython's `import sys` gives the stand-in a run puts in sys.modules, so the real module is kept once. +const HELPERS = + String.raw` +try: + _ON_SYS +except NameError: + import sys as _ON_SYS +_on_sys = _ON_SYS + +_ON_WRAP = ('', '', '') + +def _on_flush(): + for s in (_on_sys.stdout, _on_sys.stderr): + try: + s.flush() + except Exception: + pass + +def _on_format(e, home): + real = _on_sys + if real.implementation.name == 'micropython': + io, = _on_import('io') + buf = io.StringIO() + real.print_exception(e, buf) + lines = [l for l in buf.getvalue().split('\n') if not any(('File "' + w + '"') in l for w in _ON_WRAP)] + text = '\n'.join(lines) + else: + traceback, = _on_import('traceback') + tb = e.__traceback__ + while tb is not None and tb.tb_frame.f_code.co_filename in _ON_WRAP: + tb = tb.tb_next + text = ''.join(traceback.format_exception(type(e), e, tb)) + return text.replace('File "' + home + '/', 'File "') + +def _on_import(*names): + # Imports the runtime's own modules, never a project file of the same name. MicroPython lets a file override most + # of its built-in modules, and the working folder and sys.path point at the project during and after a run, so + # the project's folders leave the path for the import, and a project module of that name is forgotten. + real = _on_sys + home = '` + + HOME + + String.raw`' + kept = list(real.path) + real.path.clear() + real.path.extend([p for p in kept if p not in ('', '.') and p != home and not p.startswith(home + '/')]) + try: + found = [] + for name in names: + mod = real.modules.get(name) + f = getattr(mod, '__file__', None) + if isinstance(f, str) and f.startswith(home + '/'): + del real.modules[name] + found.append(__import__(name)) + return found + finally: + real.path.clear() + real.path.extend(kept) + +def _on_read(path): + f = open(path, 'rb') + try: + return f.read().decode('utf-8') + finally: + f.close() + +def _on_write(path, text): + f = open(path, 'w') + try: + f.write(text) + finally: + f.close() +`; + +/** Runs one program. Safe to run again in the same runtime: it forgets the last run's project modules first. */ +export const RUN_SOURCE = + HELPERS + + String.raw` +def _on_run(): + json, os, builtins, random = _on_import('json', 'os', 'builtins', 'random') + real = _on_sys + micro = real.implementation.name == 'micropython' + cfg = json.loads(_on_read('` + + RUN_FILE + + String.raw`')) + home = cfg['home'] + main = cfg['main'] + state = {'status': 'ok', 'exit': 0, 'kind': '', 'wantedInput': False} + + # A project module from an earlier run in this runtime is not reused. + for name in list(real.modules): + mod = real.modules[name] + f = getattr(mod, '__file__', None) + if isinstance(f, str) and f.startswith(home + '/'): + del real.modules[name] + if not micro: + importlib, linecache = _on_import('importlib', 'linecache') + importlib.invalidate_caches() + linecache.clearcache() + + class Stdin: + def __init__(self, text): + self.text = text + self.at = 0 + def read(self, size=-1): + end = len(self.text) if size is None or size < 0 else min(len(self.text), self.at + size) + piece = self.text[self.at:end] + self.at = end + return piece + def readline(self, size=-1): + i = self.text.find('\n', self.at) + end = len(self.text) if i < 0 else i + 1 + if size is not None and size >= 0: + end = min(end, self.at + size) + piece = self.text[self.at:end] + self.at = end + return piece + def readlines(self, hint=-1): + return list(self) + def __iter__(self): + return self + def __next__(self): + line = self.readline() + if line == '': + raise StopIteration + return line + def isatty(self): + return False + def readable(self): + return True + def close(self): + pass + def __enter__(self): + return self + def __exit__(self, *args): + pass + + stdin = Stdin(cfg['stdin']) + out = real.stdout + + def input(prompt=''): + if prompt != '': + out.write(str(prompt)) + line = stdin.readline() + if line == '': + state['wantedInput'] = True + raise EOFError('EOF when reading a line') + return line[:-1] if line.endswith('\n') else line + + builtins.input = input + argv = [main] + list(cfg['args']) + folder = home + '/' + main.rsplit('/', 1)[0] if '/' in main else home + for entry in [p for p in real.path if p == home or p.startswith(home + '/')]: + real.path.remove(entry) + real.path.insert(0, folder) + # The runner loaded these; a fresh "python main.py" would not have, so the program's own file of that name wins. + for name in ('random', 'json'): + try: + os.stat(folder + '/' + name + '.py') + real.modules.pop(name, None) + except OSError: + pass + if micro: + real.argv.clear() + real.argv.extend(argv) + class Sys: + def __getattr__(self, name): + return getattr(real, name) + shown = Sys() + shown.stdin = stdin + real.modules['sys'] = shown + else: + real.argv = argv + real.stdin = stdin + try: + real.stdout.reconfigure(line_buffering=True) + except Exception: + pass + os.chdir(home) + # A failed seed leaves the module's own seed; the run goes on. + try: + random.seed(cfg['seed']) + except Exception: + pass + + path = home + '/' + main + scope = {'__name__': '__main__', '__file__': path, '__builtins__': builtins} + if not micro: + types, = _on_import('types') + module = types.ModuleType('__main__') + module.__file__ = path + scope = module.__dict__ + real.modules['__main__'] = module + state['compiling'] = True + try: + text = _on_read(path) + if text.startswith('\ufeff'): + text = text[1:] + code = compile(text, main, 'exec') + state['compiling'] = False + exec(code, scope) + except SystemExit as e: + code = e.args[0] if len(e.args) > 0 else None + # A bool is an int in CPython but not in MicroPython: test it first. A code past 32 bits is clamped, so the + # page gets a number it accepts and a code that is not 0 stays not 0. + if code is None: + state['exit'] = 0 + elif code is True or code is False or isinstance(code, int): + state['exit'] = max(-2147483648, min(2147483647, int(code))) + else: + _on_flush() + real.stderr.write(str(code) + '\n') + state['exit'] = 1 + if state['exit'] != 0: + state['status'] = 'error' + state['kind'] = 'SystemExit' + except BaseException as e: + _on_flush() + real.stderr.write(_on_format(e, home)) + state['status'] = 'error' + state['exit'] = 1 + state['kind'] = type(e).__name__ + _on_flush() + _on_write('` + + STATE_FILE + + String.raw`', json.dumps(state)) + +_on_run() +`; + +/** Compiles one file without running it (DESIGN 6.1, layer L-c) and writes what it found to CHECKED_FILE. */ +export const CHECK_SOURCE = + HELPERS + + String.raw` +def _on_check(): + json, = _on_import('json') + cfg = json.loads(_on_read('` + + CHECK_FILE + + String.raw`')) + found = {'ok': True} + text = cfg['text'] + # A byte order mark is not part of the program: a run strips it, so the check does too. + if text.startswith('\ufeff'): + text = text[1:] + try: + compile(text, cfg['path'], 'exec') + except Exception as e: + # MemoryError and RecursionError: the parser's own stack ran out, as it will in the run. MicroPython has no + # RecursionError, so it is matched by name. + if isinstance(e, (SyntaxError, MemoryError)) or type(e).__name__ == 'RecursionError': + found = { + 'ok': False, + 'kind': type(e).__name__, + 'text': _on_format(e, cfg['home']), + 'message': type(e).__name__ + ': ' + str(getattr(e, 'msg', None) or e), + 'line': getattr(e, 'lineno', None), + 'column': getattr(e, 'offset', None), + } + _on_write('` + + CHECKED_FILE + + String.raw`', json.dumps(found)) + +_on_check() +`; diff --git a/app/src/features/codestudio/runtimes/python/pyodide.ts b/app/src/features/codestudio/runtimes/python/pyodide.ts new file mode 100644 index 000000000..22f573c32 --- /dev/null +++ b/app/src/features/codestudio/runtimes/python/pyodide.ts @@ -0,0 +1,301 @@ +// Python (full): Pyodide 314.0.7 (CPython 3.14.2, MPL-2.0) behind the RuntimeAdapter contract (DESIGN 2.1). It is +// a pack (ADR 0070): the worker loads it from the opennote-pack scheme while it is warm, before lockdown, since +// lockdown takes fetch away. The project lives in Pyodide's MEMFS, as for MicroPython. +// +// The boundary is lockdown. It empties the global object and refuses code from text, so a JavaScript value Python +// still reaches has nothing to call. +// +// The cut-off (CUT_OFF_SOURCE) is a second layer. It gives plain errors and takes away what Pyodide's Python side +// still holds: +// - An `import` of `js`, `pyodide_js`, `pyodide.ffi`, `pyodide.http`, or the rest of `pyodide` and `_pyodide` meets +// ModuleNotFoundError. +// - Every JavaScript proxy Python holds is destroyed and dropped. The Pyodide API object is one of them. +// - The functions and classes that make new JavaScript values (`to_js`, `create_proxy`, `JsException`) are gone or +// refuse. +// +// Nothing can be hidden from Python's own introspection, so the cut-off is not a wall. A program that gets past it may +// still make a JavaScript value; lockdown makes that value harmless. python.test.tsx walks all a program reaches to +// check it. The memory limit caps the WebAssembly heap and the files. A JavaScript value is not in either, so the time +// limit, which ends the worker, is what bounds it. + +import type { RuntimeAdapter, Stream } from '../../contracts/run'; +import { packFileUrl } from '../../packs/client'; +import { ALLOWANCES } from '../../run/adapters'; +import { codeStudioPython } from '../../../../strings/en/codeStudio/python'; +import { capHeap } from './heap'; +import { checkPython, runPython } from './session'; +import type { EmscriptenFS, PythonEngine } from './session'; +import { pythonMapper } from './traceback'; + +/** The pinned versions: the pack in the catalog (catalog.rs) and the dev dependency (unit tests keep them equal). */ +export const PYODIDE_VERSION = '314.0.7'; +export const CPYTHON_VERSION = '3.14.2'; + +/** The part of Pyodide's API the engine uses. `_module` is Emscripten's module, read for the heap's buffer. */ +interface PyodideWriter { + write(buffer: Uint8Array): number; +} +export interface PyodideLike { + FS: unknown; + version: string; + runPython(source: string): unknown; + setStdout(options: PyodideWriter): void; + setStderr(options: PyodideWriter): void; + setStdin(options: { error: boolean }): void; + _module?: { HEAP8?: Int8Array }; +} +export type LoadPyodide = (options: { indexURL: string; fullStdLib?: boolean }) => Promise; + +/** + * Gives asyncio a plain event loop. Pyodide's own loop, and its version of asyncio.run, use the page's timers and + * import `pyodide_js`. Lockdown and the cut-off take both away. This loop has no JavaScript and no timers: when it + * waits, it sleeps. Nothing can wake it early, since a run has no threads, sockets, or signals. + */ +const EVENT_LOOP_SOURCE = String.raw` +def _on_event_loop(): + import asyncio, time, warnings + from asyncio import base_events, events + + class Selector: + def select(self, timeout=None): + if timeout is not None and timeout > 0: + time.sleep(timeout) + return [] + def close(self): + pass + + class PlainLoop(base_events.BaseEventLoop): + def __init__(self): + super().__init__() + self._selector = Selector() + def _process_events(self, event_list): + pass + def _write_to_self(self): + pass + + base = getattr(events, '_BaseDefaultEventLoopPolicy', None) or events.BaseDefaultEventLoopPolicy + class PlainPolicy(base): + _loop_factory = PlainLoop + + PlainLoop.__module__ = PlainPolicy.__module__ = Selector.__module__ = 'asyncio' + # Pyodide wraps asyncio.run and time.sleep for its own loop; the wrappers keep the originals. + asyncio.run = getattr(asyncio.run, '__wrapped__', asyncio.run) + time.sleep = getattr(time.sleep, '__wrapped__', time.sleep) + # Pyodide marks its own loop as always running, which asyncio.run refuses. + events._set_running_loop(None) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + asyncio.set_event_loop_policy(PlainPolicy()) + +_on_event_loop() +del _on_event_loop +`; + +/** + * Refuses the program every module that reaches JavaScript or the network, and forgets the ones already loaded. An + * `import` meets ModuleNotFoundError, and Pyodide's import hook for the js modules leaves the import path. + * + * Then `_on_forget` drops every JavaScript proxy the garbage collector can see from what holds it, and destroys it. + * The built-in functions of `_pyodide_core` go too: from every module, and from the copy Python keeps to make the + * module again. So the module `_imp.create_builtin` makes has none of them. Pyodide's classes that make a JavaScript + * value from Python arguments refuse to. Pyodide's runtime keeps its own references in C, so it still works. + * + * This is not the boundary: lockdown is (see the top of this file). + */ +const CUT_OFF_SOURCE = String.raw` +def _on_cut_off(): + import sys, _imp + roots = ('js', 'pyodide_js', 'pyodide', '_pyodide', '_pyodide_core', 'micropip', 'pyodide_http') + for name in list(sys.modules): + if name.split('.')[0] in roots: + del sys.modules[name] + for finder in list(sys.meta_path): + held = getattr(finder, 'jsproxies', None) + if held is not None: + held.clear() + sys.meta_path.remove(finder) + class Refused: + def find_spec(self, name, path=None, target=None): + if name.split('.')[0] in roots: + raise ModuleNotFoundError("No module named '" + name + "'", name=name) + return None + sys.meta_path.insert(0, Refused()) + create = _imp.create_builtin + def create_builtin(spec): + if getattr(spec, 'name', '').split('.')[0] in roots: + raise ModuleNotFoundError("No module named '" + spec.name + "'", name=spec.name) + return create(spec) + _imp.create_builtin = create_builtin + +_on_cut_off() +del _on_cut_off + +def _on_forget(): + import gc + def is_js(value): + try: + return any(c.__name__ == 'JsProxy' for c in type(value).__mro__) + except Exception: + return False + def makes_js(value): + owner = getattr(value, '__self__', None) + native = type(value).__name__ == 'builtin_function_or_method' + return native and getattr(owner, '__name__', None) == '_pyodide_core' + def held(value): + return is_js(value) or makes_js(value) + gc.collect() + found = [] + for item in gc.get_objects(): + if held(item): + found.append(item) + found.extend(value for value in gc.get_referents(item) if held(value)) + done = set() + for value in found: + if id(value) in done: + continue + done.add(id(value)) + for holder in gc.get_referrers(value): + if holder is found: + continue + if isinstance(holder, dict): + for key in [key for key, item in holder.items() if item is value]: + del holder[key] + elif isinstance(holder, list): + holder[:] = [item for item in holder if item is not value] + elif type(holder).__name__ == 'cell': + holder.cell_contents = None + elif isinstance(getattr(holder, '__dict__', None), dict): + for key in [key for key, item in vars(holder).items() if item is value]: + try: + delattr(holder, key) + except Exception: + pass + if is_js(value): + try: + value.destroy() + except Exception: + pass + def refuse(*args, **kwargs): + raise TypeError('JavaScript objects are not available here.') + def classes(base, out, ids): + for sub in type.__subclasses__(base): + if id(sub) not in ids: + ids.add(id(sub)) + out.append(sub) + classes(sub, out, ids) + return out + for cls in classes(object, [], set()): + if not cls.__name__.startswith('Js') or not str(cls.__module__).startswith(('pyodide', '_pyodide')): + continue + for name in ('new', '_new_exc'): + if name in vars(cls): + try: + delattr(cls, name) + except Exception: + pass + if '__new__' in vars(cls): + try: + cls.__new__ = staticmethod(refuse) + except Exception: + pass + found.clear() + gc.collect() + +_on_forget() +del _on_forget +`; + +/** Loads Pyodide from `indexURL` and makes it an engine. Output goes to the engine's sink. */ +export async function loadPyodideEngine(load: LoadPyodide, indexURL: string): Promise { + const pyodide = await load({ indexURL, fullStdLib: false }); + const engine: PythonEngine = { + fs: pyodide.FS as EmscriptenFS, + exec: (source) => { + pyodide.runPython(source); + }, + sink: () => undefined, + limitMemory: capHeap( + (memory) => memory.buffer === pyodide._module?.HEAP8?.buffer, + () => pyodide._module?.HEAP8?.buffer.byteLength ?? 0, + ), + }; + const writer = (stream: Stream): PyodideWriter => ({ + write(buffer) { + engine.sink(stream, buffer.slice()); + return buffer.length; + }, + }); + pyodide.setStdout(writer('out')); + pyodide.setStderr(writer('err')); + // Input comes from the prelude's stand-in for sys.stdin; the real one ends at once. + pyodide.setStdin({ error: true }); + return engine; +} + +/** Cuts the program off from JavaScript. The worker calls it after lockdown, before the program runs. */ +export function cutOffPyodide(engine: PythonEngine): void { + engine.exec(EVENT_LOOP_SOURCE); + engine.exec(CUT_OFF_SOURCE); +} + +/** + * Pyodide's own code reads the global `Function` for `Function.prototype.apply` and `instanceof Function`, and + * lockdown removes that global, since calling it makes code from text. This stand-in has the real prototype, so those + * two still work, and it refuses to be called. + */ +export function restoreFunctionShape(scope: object = globalThis): void { + if ('Function' in scope) return; + const stand = function Function(): never { + throw new EvalError('Code from text is not allowed here.'); + }; + Object.defineProperty(stand, 'prototype', { value: Object.getPrototypeOf(stand) as object, writable: false }); + Object.defineProperty(scope, 'Function', { value: stand, configurable: true, writable: false }); +} + +/** Imports Pyodide's loader from the pack, or says the pack is missing. */ +async function loadFromPack(indexURL: string): Promise { + try { + const loaded = (await import(/* @vite-ignore */ `${indexURL}pyodide.mjs`)) as { loadPyodide: LoadPyodide }; + return loaded.loadPyodide; + } catch { + throw new Error(codeStudioPython.fullMissing); + } +} + +/** + * The full Python adapter. `indexURL` is the folder of Pyodide's files, ending in "/": the pack scheme in the app, + * a test server in tests. `load` imports Pyodide's loader; by default from `indexURL`. + */ +export function pyodideAdapter( + indexURL: string = packFileUrl('pyodide/'), + load: () => Promise = () => loadFromPack(indexURL), +): RuntimeAdapter { + let engine: Promise | null = null; + let cut = false; + const loaded = () => (engine ??= load().then((loader) => loadPyodideEngine(loader, indexURL))); + return { + id: 'python-full', + pack: 'pyodide', + version: `Python ${CPYTHON_VERSION} (Pyodide ${PYODIDE_VERSION})`, + allowances: ALLOWANCES['python-full'], + // Input is the prefilled text until Pyodide's input waits for the page through JSPI (DESIGN 4.4, R2). + prefilledOnly: true, + diagnostics: pythonMapper, + async load(onProgress) { + onProgress(5); + await loaded(); + }, + async run(request, emit, context) { + const ready = await loaded(); + if (!cut) { + restoreFunctionShape(); + cutOffPyodide(ready); + cut = true; + } + return runPython(ready, request, emit, context); + }, + async check(file, files) { + return checkPython(await loaded(), file, files); + }, + }; +} diff --git a/app/src/features/codestudio/runtimes/python/python.test.ts b/app/src/features/codestudio/runtimes/python/python.test.ts new file mode 100644 index 000000000..d9558b221 --- /dev/null +++ b/app/src/features/codestudio/runtimes/python/python.test.ts @@ -0,0 +1,751 @@ +// Both Python runtimes in Node, with the real MicroPython and the real Pyodide (DESIGN 2.1, 4, 6.1, 7.1). +// - The 20-program suite gives byte-exact output in both. Each answer is worked by hand. +// - CPython 3.12+ messages: "Did you mean" and "Did you forget to import" (the Python 3.12 release notes). +// - A traceback maps to the innermost project file, and the compile() check finds syntax errors without running. +// - Pyodide's ways to JavaScript are refused here too; the sandbox escapes after lockdown run in python.test.tsx. + +// checks-disable-file modifiability: one describe block per feature +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { beforeAll, describe, expect, it, vi } from 'vitest'; +import { loadPyodide } from 'pyodide'; +import type { RunFile, RunRequest, RuntimeDiagnostic, WorkerMessage } from '../../contracts/run'; +import { mapDiagnostics } from '../../run/diagnostics'; +import { Mount } from '../../run/mount'; +import { RUN_LIMITS } from '../../run/protocol'; +import { seeded } from '../../run/random'; +import { Stdin } from '../../run/stdin'; +import { MICROPYTHON_VERSION, loadMicroPythonEngine } from './micropython'; +import { CPYTHON_VERSION, PYODIDE_VERSION, cutOffPyodide, loadPyodideEngine } from './pyodide'; +import type { LoadPyodide } from './pyodide'; +import { checkPython, runPython } from './session'; +import type { PythonEngine } from './session'; +import { pythonMapper } from './traceback'; +import { REACH, REACH_HELD } from './fixtures/reach'; + +// Pyodide takes seconds to start on a loaded machine, and fib(30) takes a while in MicroPython. +vi.setConfig({ testTimeout: 120_000, hookTimeout: 180_000 }); + +const ROOT = new URL('../../../../../../', import.meta.url); +const MICROPYTHON_WASM = fileURLToPath( + new URL('node_modules/@micropython/micropython-webassembly-pyscript/micropython.wasm', ROOT), +); +const PYODIDE_DIR = fileURLToPath(new URL('node_modules/pyodide/', ROOT)); + +type Kind = 'quick' | 'full'; + +let full: PythonEngine; +beforeAll(async () => { + full = await loadPyodideEngine(loadPyodide as unknown as LoadPyodide, PYODIDE_DIR); + cutOffPyodide(full); +}); + +/** A fresh MicroPython for each run, as each run gets a fresh worker; Pyodide is reused (the prelude resets it). */ +const engineFor = async (kind: Kind) => (kind === 'full' ? full : await loadMicroPythonEngine(MICROPYTHON_WASM)); + +interface Ran { + status: string; + exitCode?: number; + message: string; + out: string; + err: string; + diagnostics: RuntimeDiagnostic[]; + written: RunFile[]; + inputEnded: boolean; +} + +async function run( + kind: Kind, + files: Record, + more: { main?: string; stdin?: string; args?: string[]; seed?: number; limits?: RunRequest['limits'] } = {}, +): Promise { + const list: RunFile[] = Object.entries(files).map(([path, text]) => ({ path, text })); + const request: RunRequest = { + kind: 'run', + runtime: kind === 'full' ? 'python-full' : 'python-quick', + files: list, + main: more.main ?? list[0]!.path, + args: more.args ?? [], + stdin: { mode: 'prefilled', text: more.stdin ?? '' }, + limits: more.limits ?? RUN_LIMITS, + seed: more.seed ?? 1, + }; + const messages: WorkerMessage[] = []; + const random = seeded(request.seed); + const stdin = new Stdin(request.stdin.text, false); + const mount = new Mount(list); + const end = runPython(await engineFor(kind), request, (message) => messages.push(message), { + stdin, + files: mount, + random: random.random, + randomBytes: random.fill, + deadline: performance.now() + request.limits.timeMs, + }); + const text = (stream: string) => + messages.map((message) => (message.kind === 'out' && message.stream === stream ? message.text : '')).join(''); + const err = text('err'); + return { + status: end.status, + ...(end.exitCode === undefined ? {} : { exitCode: end.exitCode }), + message: end.message, + out: text('out'), + err, + diagnostics: mapDiagnostics(pythonMapper, err, list), + written: mount.written(), + inputEnded: stdin.askedPastEnd, + }; +} + +interface Program { + name: string; + files: Record; + out: string; + stdin?: string; + args?: string[]; + written?: RunFile[]; +} + +const lines = (...text: string[]) => text.join('\n') + '\n'; +const lastLine = (text: string) => text.trimEnd().split('\n').at(-1); + +// Every answer below is worked by hand: sums and products by arithmetic, 0.1 + 0.2 as the double nearest +// 0.30000000000000004 (shortest repr), floor division and modulo rounding toward negative infinity. +const PROGRAMS: Program[] = [ + { name: 'hello', files: { 'main.py': 'print("Hello, world!")\n' }, out: 'Hello, world!\n' }, + { name: 'sum 1..100', files: { 'main.py': 'print(sum(range(1, 101)))\n' }, out: '5050\n' }, + { + name: 'fib(30)', + files: { 'main.py': lines('def fib(n):', ' return n if n < 2 else fib(n - 1) + fib(n - 2)', 'print(fib(30))') }, + out: '832040\n', + }, + { + name: 'sorted', + files: { + 'main.py': lines( + 'print(sorted([5, 2, 9, 1, 5, 6]))', + 'print(sorted(["pear", "fig", "apple"], key=len, reverse=True))', + 'print(sorted("banana"))', + ), + }, + out: lines('[1, 2, 5, 5, 6, 9]', "['apple', 'pear', 'fig']", "['a', 'a', 'a', 'b', 'n', 'n']"), + }, + { + name: 'f-strings', + files: { + 'main.py': lines( + 'name, pi, n = "Ada", 3.14159, 7', + 'print(f"{name}: {pi:.3f} [{n:>4}] {n:03d} {255:x} {name!r}")', + 'print(f"{n * 6} {pi:.1f}% {-n:+d}")', + ), + }, + out: lines("Ada: 3.142 [ 7] 007 ff 'Ada'", '42 3.1% -7'), + }, + { + name: 'Unicode', + files: { + 'main.py': lines( + 's = "héllo wörld ✓"', + 'print(s, len(s), s[1], s[-1], s.split()[1])', + 'print(ord("é"), chr(0x2713), len("🐍"))', + ), + }, + out: lines('héllo wörld ✓ 13 é ✓ wörld', '233 ✓ 1'), + }, + { + name: 'big integers', + files: { 'main.py': lines('f = 1', 'for i in range(1, 21):', ' f *= i', 'print(f)', 'print(2 ** 100)') }, + out: lines('2432902008176640000', '1267650600228229401496703205376'), + }, + { + name: 'primes below 50', + files: { + 'main.py': lines( + 'primes = [n for n in range(2, 50) if all(n % d for d in range(2, n))]', + 'print(" ".join(str(p) for p in primes))', + ), + }, + out: '2 3 5 7 11 13 17 19 23 29 31 37 41 43 47\n', + }, + { + name: 'comprehensions', + files: { + 'main.py': lines('print([x * x for x in range(10) if x % 2 == 0])', 'print({k: len(k) for k in ["a", "bb"]})'), + }, + out: lines('[0, 4, 16, 36, 64]', "{'a': 1, 'bb': 2}"), + }, + { + name: 'word count', + files: { + 'main.py': lines( + 'counts = {}', + 'for word in "the cat and the hat".split():', + ' counts[word] = counts.get(word, 0) + 1', + 'print(sorted(counts.items()))', + ), + }, + out: "[('and', 1), ('cat', 1), ('hat', 1), ('the', 2)]\n", + }, + { + name: 'string methods', + files: { + 'main.py': lines( + 'parts = " a,b,,c ".strip().split(",")', + 'print(parts, "-".join(parts), "Abc".lower(), "abc".upper(), "a1".isdigit(), "Hi".startswith("H"))', + ), + }, + out: "['a', 'b', '', 'c'] a-b--c abc ABC False True\n", + }, + { + name: 'classes', + files: { + 'main.py': lines( + 'class Point:', + ' def __init__(self, x, y):', + ' self.x, self.y = x, y', + ' def __add__(self, other):', + ' return Point(self.x + other.x, self.y + other.y)', + ' def __repr__(self):', + ' return "Point(%d, %d)" % (self.x, self.y)', + 'print(Point(1, 2) + Point(3, 4))', + ), + }, + out: 'Point(4, 6)\n', + }, + { + name: 'exceptions', + files: { + 'main.py': lines( + 'try:', + ' int("x")', + 'except ValueError:', + ' print("bad number")', + 'finally:', + ' print("done")', + ), + }, + out: lines('bad number', 'done'), + }, + { + name: 'generators', + files: { + 'main.py': lines( + 'def evens(n):', + ' for i in range(n):', + ' if i % 2 == 0:', + ' yield i', + 'print(list(evens(9)), sum(evens(100)))', + ), + }, + out: '[0, 2, 4, 6, 8] 2450\n', + }, + { + name: 'standard input', + files: { 'main.py': lines('a = int(input())', 'b = int(input("b? "))', 'print(a + b)') }, + stdin: '3\n4\n', + out: 'b? 7\n', + }, + { + name: 'arguments', + files: { 'main.py': lines('import sys', 'print(sys.argv)') }, + args: ['a', 'b c'], + out: "['main.py', 'a', 'b c']\n", + }, + { + name: 'modules across files', + files: { + 'main.py': lines('from util import double', 'print(double(21))'), + 'util.py': 'def double(x):\n return 2 * x\n', + }, + out: '42\n', + }, + { + name: 'floor division and modulo', + files: { 'main.py': 'print(-7 // 2, -7 % 2, divmod(17, 5), 7 / 2, 2 ** -1)\n' }, + out: '-4 1 (3, 2) 3.5 0.5\n', + }, + { + name: 'floats', + files: { 'main.py': 'print(0.1 + 0.2, 1 / 3, 1e16, 1e-05, float("inf"), round(7.5))\n' }, + out: '0.30000000000000004 0.3333333333333333 1e+16 1e-05 inf 8\n', + }, + { + name: 'writes a file', + files: { + 'main.py': lines( + 'with open("out.txt", "w") as f:', + ' f.write("done\\n")', + 'print(open("out.txt").read(), end="")', + ), + }, + out: 'done\n', + written: [{ path: 'out.txt', text: 'done\n' }], + }, +]; + +describe.each(['quick', 'full'] as const)('the 20-program suite in Python (%s)', (kind) => { + it.each(PROGRAMS)('$name', async (program) => { + const ran = await run(kind, program.files, { + ...(program.stdin === undefined ? {} : { stdin: program.stdin }), + ...(program.args === undefined ? {} : { args: program.args }), + }); + expect(ran.err).toBe(''); + expect(ran.out).toBe(program.out); + expect(ran.status).toBe('ok'); + expect(ran.exitCode).toBe(0); + expect(ran.written).toEqual(program.written ?? []); + }); + + it('has twenty programs', () => { + expect(PROGRAMS).toHaveLength(20); + }); +}); + +describe('CPython known answers (Python 3.12 and later, full Python)', () => { + // The messages are CPython's, as the Python 3.12 release notes show them ("Improved Error Messages"). + it('suggests print for pritn', async () => { + const ran = await run('full', { 'main.py': 'pritn(1)\n' }); + expect(ran.status).toBe('error'); + expect(ran.message).toBe('NameError'); + expect(lastLine(ran.err)).toBe("NameError: name 'pritn' is not defined. Did you mean: 'print'?"); + expect(ran.err.startsWith('Traceback (most recent call last):\n File "main.py", line 1, in \n')).toBe( + true, + ); + }); + + it('asks whether math was imported', async () => { + const ran = await run('full', { 'main.py': 'print(math.sqrt(4))\n' }); + const message = "NameError: name 'math' is not defined. Did you forget to import 'math'?"; + expect(lastLine(ran.err)).toBe(message); + expect(ran.diagnostics).toEqual([{ file: 'main.py', line: 1, message, kind: 'NameError' }]); + }); + + it('reports the version it pins', async () => { + const ran = await run('full', { 'main.py': 'import sys\nprint(sys.version.split()[0])\n' }); + expect(ran.out).toBe(`${CPYTHON_VERSION}\n`); + }); +}); + +describe.each(['quick', 'full'] as const)('errors and diagnostics in Python (%s)', (kind) => { + const project = { + 'main.py': lines('import util', 'print("start")', 'util.f()'), + 'util.py': lines('def f():', ' x = 1', ' return y'), + }; + + it('maps an error on line 3 of util.py, imported from main.py, to util.py:3', async () => { + const ran = await run(kind, project); + expect(ran.out).toBe('start\n'); + expect(ran.status).toBe('error'); + expect(ran.exitCode).toBe(1); + expect(ran.diagnostics).toHaveLength(1); + expect(ran.diagnostics[0]).toMatchObject({ file: 'util.py', line: 3, kind: 'NameError' }); + // The traceback shows project paths, never the mount's folder, and none of the runner's own frames. + expect(ran.err).toContain('File "util.py", line 3, in f'); + expect(ran.err).toContain('File "main.py", line 3, in '); + expect(ran.err).not.toMatch(/\/project||/); + }); + + it('runs a main file in a folder with imports beside it', async () => { + const ran = await run(kind, { 'src/app.py': 'import helper\nprint(helper.N)\n', 'src/helper.py': 'N = 5\n' }); + expect(ran.out).toBe('5\n'); + }); + + it('reports a syntax error in the main file and runs nothing', async () => { + const ran = await run(kind, { 'main.py': 'print(1)\nif x\n pass\n' }); + expect(ran.status).toBe('error'); + expect(ran.out).toBe(''); + expect(ran.message).toBe('SyntaxError'); + expect(ran.diagnostics[0]).toMatchObject({ file: 'main.py', line: 2, kind: 'SyntaxError' }); + if (kind === 'full') expect(lastLine(ran.err)).toBe("SyntaxError: expected ':'"); + }); + + it('gives exit codes: sys.exit(), sys.exit(3), and sys.exit("bye")', async () => { + const ok = await run(kind, { 'main.py': 'import sys\nprint("a")\nsys.exit()\nprint("b")\n' }); + expect([ok.status, ok.exitCode, ok.out, ok.err]).toEqual(['ok', 0, 'a\n', '']); + const three = await run(kind, { 'main.py': 'import sys\nsys.exit(3)\n' }); + expect([three.status, three.exitCode, three.message, three.err]).toEqual(['error', 3, 'SystemExit', '']); + const bye = await run(kind, { 'main.py': 'import sys\nsys.exit("bye")\n' }); + expect([bye.status, bye.exitCode, bye.err]).toEqual(['error', 1, 'bye\n']); + }); + + // CPython's sys.exit: a bool is an int, so False is 0 and True is 1, and neither prints anything. + it('gives exit codes for a bool and for a number past 32 bits', async () => { + const no = await run(kind, { 'main.py': 'import sys\nsys.exit(False)\n' }); + expect([no.status, no.exitCode, no.err]).toEqual(['ok', 0, '']); + const yes = await run(kind, { 'main.py': 'import sys\nsys.exit(True)\n' }); + expect([yes.status, yes.exitCode, yes.err]).toEqual(['error', 1, '']); + const not = await run(kind, { 'main.py': 'import sys\nok = True\nsys.exit(not ok)\n' }); + expect([not.status, not.exitCode, not.err]).toEqual(['ok', 0, '']); + // 2**40 does not fit the page's 32-bit exit code: it is clamped to 2**31 - 1, and the files are kept. + const big = await run(kind, { 'main.py': 'import sys\nopen("o.txt", "w").write("x")\nsys.exit(2**40)\n' }); + expect([big.status, big.exitCode, big.err]).toEqual(['error', 2 ** 31 - 1, '']); + expect(big.written).toEqual([{ path: 'o.txt', text: 'x' }]); + const minus = await run(kind, { 'main.py': 'import sys\nsys.exit(-1)\n' }); + expect([minus.status, minus.exitCode, minus.err]).toEqual(['error', -1, '']); + }); + + it('treats a byte order mark the same way in a run, a check, and the written files', async () => { + const text = '\ufeffprint("hi")\n'; + const ran = await run(kind, { 'main.py': text }); + expect([ran.status, ran.out, ran.err]).toEqual(['ok', 'hi\n', '']); + expect(ran.written).toEqual([]); + const file = { path: 'main.py', text }; + expect(checkPython(await engineFor(kind), file, [file])).toEqual([]); + }); + + it('checks a file with compile() without running it', async () => { + const engine = await engineFor(kind); + const bad = { path: 'main.py', text: 'print("ran")\nif x\n pass\n' }; + const found = checkPython(engine, bad, [bad]); + expect(found).toHaveLength(1); + expect(found[0]).toMatchObject({ file: 'main.py', line: 2, kind: 'SyntaxError' }); + // CPython gives the column too: the colon is missing after "if x", at column 5. + if (kind === 'full') expect(found[0]).toMatchObject({ column: 5, message: "SyntaxError: expected ':'" }); + const good = { path: 'util.py', text: 'def f():\n return y\n' }; + expect(checkPython(engine, good, [bad, good])).toEqual([]); + }); +}); + +describe('the check and the run agree on sources compile() gives up on (full Python)', () => { + it('reports a NUL character, which has no frame to map, as a diagnostic', async () => { + const file = { path: 'main.py', text: 'print(1)\nx = "a\0b"\n' }; + const found = checkPython(full, file, [file]); + expect(found).toHaveLength(1); + expect(found[0]).toMatchObject({ file: 'main.py', kind: 'SyntaxError' }); + expect(found[0]!.message).toMatch(/null/); + const ran = await run('full', { 'main.py': file.text }); + expect([ran.status, ran.out]).toEqual(['error', '']); + }); + + it('reports a parser stack overflow in the check, and the run ends as "error", not "memory"', async () => { + // CPython's parser gives up on deep nesting with MemoryError ("Parser stack overflowed"). + const text = 'x = ' + '-'.repeat(200_000) + '1\n'; + const file = { path: 'main.py', text }; + const found = checkPython(full, file, [file]); + expect(found).toHaveLength(1); + expect(found[0]).toMatchObject({ file: 'main.py', line: 1 }); + expect(found[0]!.message).toMatch(/^(MemoryError|RecursionError|SyntaxError)/); + const ran = await run('full', { 'main.py': text }); + expect(ran.status).toBe('error'); + expect((await run('full', { 'main.py': 'print(2)\n' })).out).toBe('2\n'); + }); +}); + +describe('a reused full Python', () => { + it('forgets the last run project modules', async () => { + await run('full', { 'main.py': 'import util\nprint(util.A)\n', 'util.py': 'A = 1\n' }); + const second = await run('full', { 'main.py': 'import util\nprint(util.A)\n', 'util.py': 'A = 2\n' }); + expect(second.out).toBe('2\n'); + }); + + it('turns an exit code the page would refuse into 1', async () => { + // The program replaces json.dumps, which the runner uses for its state: the state's exit becomes JSON true. + // It puts the real one back on that call, since this Pyodide is reused by the next test. + const forged = lines( + 'import json', + 'real = json.dumps', + 'def forged(state):', + ' json.dumps = real', + ` return '{"status": "ok", "exit": true, "kind": "", "wantedInput": false}'`, + 'json.dumps = forged', + ); + const ran = await run('full', { 'main.py': forged }); + expect([ran.status, ran.exitCode]).toEqual(['error', 1]); + const back = await run('full', { 'main.py': 'import json\nprint(json.dumps(1))\n' }); + expect(back.out).toBe('1\n'); + }); +}); + +describe('full Python asyncio and memory', () => { + // asyncio.gather returns the results in the order of its arguments; the shorter sleep finishes first. + it('runs asyncio.run with sleep and gather after the cut-off', async () => { + const program = lines( + 'import asyncio', + 'async def job(name, delay):', + ' await asyncio.sleep(delay)', + ' print(name, "done")', + ' return name', + 'async def main():', + ' return await asyncio.gather(job("a", 0.05), job("b", 0.01))', + 'print(asyncio.run(main()))', + ); + const ran = await run('full', { 'main.py': program }); + expect([ran.status, ran.err]).toEqual(['ok', '']); + expect(ran.out).toBe(lines('b done', 'a done', "['a', 'b']")); + }); + + it('ends with "memory" when the heap would grow past the memory limit', async () => { + // Eight blocks of 16 MB need 128 MB; the limit lets the heap grow by 32 MB. + const program = 'blocks = []\nfor _ in range(8):\n blocks.append(bytearray(16 * 1024 * 1024))\nprint("all")\n'; + const limits = { ...RUN_LIMITS, memoryBytes: 32 * 1024 * 1024 }; + const ran = await run('full', { 'main.py': program }, { limits }); + expect([ran.status, ran.out]).toEqual(['memory', '']); + // The heap is still usable afterward. + expect((await run('full', { 'main.py': 'print(sum(range(10)))\n' })).out).toBe('45\n'); + }); +}); + +describe.each(['quick', 'full'] as const)('input, arguments, and seeds in Python (%s)', (kind) => { + it('raises EOFError past the end of the input, and the run notes it', async () => { + const ran = await run(kind, { 'main.py': 'print(input())\nprint(input())\n' }, { stdin: 'one\n' }); + expect(ran.out).toBe('one\n'); + expect(lastLine(ran.err)).toBe('EOFError: EOF when reading a line'); + expect(ran.inputEnded).toBe(true); + }); + + it('reads sys.stdin to its end without the note', async () => { + const program = lines( + 'import sys', + 'for line in sys.stdin:', + ' print(line.strip().upper())', + 'print(sys.stdin.read() == "")', + ); + const ran = await run(kind, { 'main.py': program }, { stdin: 'a\r\nb' }); + expect(ran.out).toBe('A\nB\nTrue\n'); + expect(ran.inputEnded).toBe(false); + }); + + it('gives the same random numbers for the same seed', async () => { + const program = 'import random\nprint([random.randint(1, 1000) for _ in range(5)])\n'; + const first = await run(kind, { 'main.py': program }, { seed: 7 }); + const again = await run(kind, { 'main.py': program }, { seed: 7 }); + const other = await run(kind, { 'main.py': program }, { seed: 8 }); + expect(first.out).toBe(again.out); + expect(first.out).not.toBe(other.out); + }); +}); + +describe('the sandbox of full Python, without lockdown (python.test.tsx repeats it after lockdown)', () => { + it('refuses the js module, pyodide.ffi, and pyodide.http, and has no host files', async () => { + const program = lines( + 'import os', + 'for name in ["js", "pyodide.ffi", "pyodide.http", "pyodide_js", "_pyodide_core"]:', + ' try:', + ' __import__(name)', + ' print(name, "imported")', + ' except ModuleNotFoundError:', + ' print(name, "refused")', + 'try:', + ' open("/etc/passwd")', + 'except OSError as e:', + ' print("passwd", type(e).__name__)', + 'print(sorted(os.listdir(".")))', + ); + const ran = await run('full', { 'main.py': program, 'data.txt': 'x' }); + expect(ran.err).toBe(''); + expect(ran.out).toBe( + lines( + 'js refused', + 'pyodide.ffi refused', + 'pyodide.http refused', + 'pyodide_js refused', + '_pyodide_core refused', + 'passwd FileNotFoundError', + "['data.txt', 'main.py']", + ), + ); + }); + + it('leaves the program no JavaScript value and no way to make one, by any route REACH knows', async () => { + const ran = await run('full', { 'main.py': REACH }); + expect([ran.status, ran.err]).toEqual(['ok', '']); + expect(ran.out).toBe(REACH_HELD); + }); +}); + +// A program that writes 1 MiB blocks to a file until the write fails. File data lives outside the heap, so only the +// file cap stops it; without the cap a program wrote 400 MiB under a 64 MiB limit. +const FILL = lines( + 'n = 0', + 'chunk = "x" * (1024 * 1024)', + 'try:', + ' with open("big.txt", "w") as f:', + ' for _ in range(64):', + ' f.write(chunk)', + ' f.flush()', + ' n += 1', + 'except OSError as e:', + ' print("refused", e.errno)', + 'print(n)', +); + +describe.each(['quick', 'full'] as const)('the file cap in Python (%s)', (kind) => { + const MiB = 1024 * 1024; + + it('fails a write with ENOSPC once the files would pass the memory limit', async () => { + const ran = await run(kind, { 'main.py': FILL }, { limits: { ...RUN_LIMITS, memoryBytes: 8 * MiB } }); + expect([ran.status, ran.err]).toEqual(['ok', '']); + // Emscripten's ENOSPC is 51. + expect(ran.out).toMatch(/^refused 51\n[0-8]\n$/); + const big = ran.written.find((file) => file.path === 'big.txt'); + expect(big?.text.length ?? 0).toBeLessThanOrEqual(8 * MiB); + }); + + it('counts files the program removed as free again', async () => { + const program = lines( + 'import os', + 'chunk = "x" * (1024 * 1024)', + 'for _ in range(6):', + ' with open("t.txt", "w") as f:', + ' for _ in range(4):', + ' f.write(chunk)', + ' os.remove("t.txt")', + 'print("done")', + ); + const ran = await run(kind, { 'main.py': program }, { limits: { ...RUN_LIMITS, memoryBytes: 8 * MiB } }); + expect([ran.status, ran.out, ran.err]).toEqual(['ok', 'done\n', '']); + }); +}); + +describe('links in the project folder (full Python)', () => { + it('keeps the status and the written files when links are left behind, and the next run works', async () => { + const program = lines( + 'import os', + 'os.symlink("/", "root")', + 'os.symlink("me", "me")', + 'os.symlink("missing", "dangling")', + 'os.mkdir("sub")', + 'os.symlink("..", "sub/up")', + 'open("o.txt", "w").write("kept")', + 'print("end")', + ); + const ran = await run('full', { 'main.py': program }); + expect([ran.status, ran.message, ran.out, ran.err]).toEqual(['ok', '', 'end\n', '']); + expect(ran.written).toEqual([{ path: 'o.txt', text: 'kept' }]); + const next = await run('full', { 'main.py': 'import os\nprint(sorted(os.listdir(".")))\n' }); + expect([next.status, next.out]).toEqual(['ok', "['main.py']\n"]); + }); +}); + +// A program that takes 1 MiB blocks until MemoryError. With the limit at L MiB it gets fewer than L + 2 blocks: the +// starting heap and the runtime's own data come on top of the limit. With a limit 4 times as high it gets more. +const BLOCKS = lines( + 'blocks = []', + 'try:', + ' for _ in range(1000):', + ' blocks.append(bytearray(1024 * 1024))', + 'except MemoryError:', + ' pass', + 'n = len(blocks)', + 'blocks = None', + 'print(n)', +); + +describe('the memory limit in quick Python', () => { + const MiB = 1024 * 1024; + const blocks = async (limitMiB: number) => { + const began = performance.now(); + const ran = await run('quick', { 'main.py': BLOCKS }, { limits: { ...RUN_LIMITS, memoryBytes: limitMiB * MiB } }); + expect([ran.status, ran.err]).toEqual(['ok', '']); + // A heap that grew without a cap took about 2 GB and many seconds here. + expect(performance.now() - began).toBeLessThan(10_000); + return Number(ran.out); + }; + + it('stops the heap near the limit the request gives', async () => { + const low = await blocks(8); + const high = await blocks(32); + expect(low).toBeGreaterThan(0); + expect(low).toBeLessThan(8 + 2); + expect(high).toBeGreaterThan(low); + expect(high).toBeLessThan(32 + 2); + }); + + it('ends with "memory" when the program does not catch MemoryError', async () => { + const program = 'a = []\nwhile True:\n a.append(bytearray(1024 * 1024))\n'; + const ran = await run('quick', { 'main.py': program }, { limits: { ...RUN_LIMITS, memoryBytes: 8 * MiB } }); + expect([ran.status, ran.out]).toEqual(['memory', '']); + }); +}); + +// A project file named like a module the runner uses must not replace the runner's module. The program's own +// imports still find the project file first, as in CPython, since sys.path[0] is the main file's folder. +const SHADOWS: Record = { + 'random.py': 'print("project random")\n', + 'io.py': 'print("project io")\n', + 'json.py': 'print("project json")\n', + 'os.py': 'print("project os")\n', +}; + +describe.each(['quick', 'full'] as const)('project files named like standard modules (%s)', (kind) => { + it('runs main.py, not an unused random.py, io.py, json.py, or os.py', async () => { + const ran = await run(kind, { 'main.py': 'print("main")\n', ...SHADOWS }); + expect([ran.status, ran.out, ran.err]).toEqual(['ok', 'main\n', '']); + }); + + it('keeps the traceback of an error next to an io.py', async () => { + const ran = await run(kind, { 'main.py': 'print("main")\nraise ValueError("bad")\n', ...SHADOWS }); + expect([ran.status, ran.out, ran.message]).toEqual(['error', 'main\n', 'ValueError']); + expect(ran.err).toContain('Traceback (most recent call last):\n'); + expect(ran.err).toContain('File "main.py", line 2'); + expect(lastLine(ran.err)).toBe('ValueError: bad'); + }); + + it('runs a main file named random.py', async () => { + const program = lines('import sys', 'print(sys.argv[0])', 'print("main random")'); + const ran = await run(kind, { 'random.py': program, 'util.py': 'A = 1\n' }); + expect([ran.status, ran.out, ran.err]).toEqual(['ok', 'random.py\nmain random\n', '']); + }); + + it('lets the program import its own random.py', async () => { + const ran = await run(kind, { 'main.py': 'import random\nprint(random.NAME)\n', 'random.py': 'NAME = "mine"\n' }); + expect([ran.status, ran.out, ran.err]).toEqual(['ok', 'mine\n', '']); + }); +}); + +describe('versions', () => { + const pkg = JSON.parse(readFileSync(new URL('package.json', ROOT), 'utf8')) as { + dependencies: Record; + devDependencies: Record; + }; + + it('pins the versions in package.json and the pack catalog', () => { + expect(pkg.dependencies['@micropython/micropython-webassembly-pyscript']).toBe(MICROPYTHON_VERSION); + expect(pkg.devDependencies['pyodide']).toBe(PYODIDE_VERSION); + const catalog = readFileSync(new URL('app/src-tauri/src/packs/catalog.rs', ROOT), 'utf8'); + expect(catalog).toContain(`pub const PYODIDE_VERSION: &str = "${PYODIDE_VERSION}";`); + }); + + it('runs the MicroPython it names', async () => { + const ran = await run('quick', { 'main.py': 'import sys\nprint(sys.implementation.version[:3])\n' }); + expect(ran.out).toBe(`(${MICROPYTHON_VERSION.split('.').join(', ')})\n`); + }); +}); + +describe('the traceback mapper', () => { + const files: RunFile[] = [ + { path: 'main.py', text: 'a\nb\nc\n' }, + { path: 'pkg/util.py', text: 'a\nb\nc\nd\n' }, + ]; + + it('takes the innermost project frame of the last traceback, past the runtime files', () => { + const text = lines( + 'some output', + 'Traceback (most recent call last):', + ' File "main.py", line 2, in ', + ' b', + ' File "/project/pkg/util.py", line 4, in g', + ' d', + ' File "/lib/python314.zip/json/__init__.py", line 9, in loads', + 'ValueError: bad', + ); + expect(pythonMapper.map(text, files)).toEqual([ + { file: 'pkg/util.py', line: 4, message: 'ValueError: bad', kind: 'ValueError' }, + ]); + }); + + it('reads a syntax error without a header and an exception without a message', () => { + const syntax = lines(' File "main.py", line 9', ' x', ' ^', 'SyntaxError: invalid syntax'); + expect(pythonMapper.map(syntax, files)).toEqual([ + { file: 'main.py', line: 4, message: 'SyntaxError: invalid syntax', kind: 'SyntaxError' }, + ]); + const bare = lines( + 'Traceback (most recent call last):', + ' File "main.py", line 1, in ', + 'KeyboardInterrupt', + ); + expect(pythonMapper.map(bare, files)).toEqual([ + { file: 'main.py', line: 1, message: 'KeyboardInterrupt', kind: 'KeyboardInterrupt' }, + ]); + expect(pythonMapper.map('NameError: no frames here\n', files)).toEqual([]); + }); +}); diff --git a/app/src/features/codestudio/runtimes/python/python.test.tsx b/app/src/features/codestudio/runtimes/python/python.test.tsx new file mode 100644 index 000000000..d2dcc32f0 --- /dev/null +++ b/app/src/features/codestudio/runtimes/python/python.test.tsx @@ -0,0 +1,300 @@ +// Both Python runtimes in a real browser, in a real run worker after lockdown (DESIGN 4.8). +// - Quick Python runs through the app's own run worker (run/worker.ts and run/adapters.ts). +// - Full Python runs through fixtures/worker.ts, with the same protocol and adapter. It loads Pyodide from the test +// server, since the pack scheme exists only in the app. +// +// Sandbox escapes: the js module has no fetch, and pyodide.http.pyfetch fails. Files outside the project and the +// runtime's own library are absent. The project mount is the only folder with the program's files. Lockdown is the +// boundary: a JavaScript object that full Python still reaches past the cut-off has no fetch, eval, postMessage, or +// timers, and code from text is refused. +// +// Run ends: output printed before an endless loop reaches the page, and an exit code past 32 bits ends the run at +// once. Asyncio runs without the page's timers. Interactive input says it is not available. + +// checks-disable-file modifiability: one describe block per feature +import { describe, expect, it, vi } from 'vitest'; +import type { RunFile, RuntimeDiagnostic, RuntimeId } from '../../contracts/run'; +import { RunHost, createRunWorker, fromWorker } from '../../run/host'; +import type { RunOrder } from '../../run/host'; +import { RUN_LIMITS } from '../../run/protocol'; +import { REACH, REACH_HELD } from './fixtures/reach'; + +// Loading Pyodide in a worker takes seconds, more on a loaded machine; the test server also transforms its loader. +vi.setConfig({ testTimeout: 180_000 }); + +const LIMITS = { ...RUN_LIMITS, loadMs: 120_000 }; + +const createWorker = (runtime: RuntimeId) => + runtime === 'python-full' + ? fromWorker(new Worker(new URL('./fixtures/worker.ts', import.meta.url), { type: 'module', name: 'test-python' })) + : createRunWorker(runtime); + +interface More { + stdin?: string; + args?: string[]; + timeMs?: number; + interactive?: boolean; +} + +async function run(runtime: RuntimeId, files: RunFile[], more: More = {}) { + const limits = { ...LIMITS, ...(more.timeMs ? { timeMs: more.timeMs } : {}) }; + const runs = new RunHost({ createWorker, limits }); + const order: RunOrder = { + runtime, + files, + main: files[0]!.path, + args: more.args ?? [], + stdin: { mode: more.interactive ? 'interactive' : 'prefilled', text: more.stdin ?? '' }, + limits, + seed: 1, + }; + let out = ''; + let err = ''; + const diagnostics: RuntimeDiagnostic[] = []; + const handle = runs.run(order, { + onOutput: (stream, text) => (stream === 'out' ? (out += text) : (err += text)), + onDiagnostics: (items) => diagnostics.push(...items), + }); + const done = await handle.done; + runs.dispose(); + return { ...done, out, err, diagnostics }; +} + +const lines = (...text: string[]) => text.join('\n') + '\n'; + +const ESCAPES = lines( + 'import os', + 'for name in ["js", "pyodide.ffi", "pyodide.http", "pyodide_js", "_pyodide_core"]:', + ' try:', + ' __import__(name)', + ' print(name, "imported")', + ' except ImportError:', + ' print(name, "refused")', + 'try:', + ' import pyodide.http', + ' pyodide.http.pyfetch("https://example.com/")', + ' print("pyfetch ran")', + 'except Exception as e:', + ' print("pyfetch", type(e).__name__)', + 'for path in ["/etc/passwd", "../outside.txt", "/project/../etc/hosts"]:', + ' try:', + ' open(path).read()', + ' print(path, "read")', + ' except OSError as e:', + ' print(path, type(e).__name__)', + 'print(sorted(os.listdir(".")))', +); + +// After REACH (fixtures/reach.ts), the network from Python's side. +const NETWORK = lines( + 'import socket', + 'try:', + ' socket.create_connection(("example.com", 80), timeout=1)', + ' print("socket: connected")', + 'except OSError:', + ' print("socket: refused")', + 'from urllib.request import urlopen', + 'try:', + ' urlopen("http://example.com/", timeout=1)', + ' print("urlopen: read")', + 'except Exception:', + ' print("urlopen: refused")', +); + +describe('full Python in the run worker', () => { + it('cannot reach fetch, pyfetch, or files outside the project', async () => { + const ran = await run('python-full', [ + { path: 'main.py', text: ESCAPES }, + { path: 'data.txt', text: 'x' }, + ]); + expect([ran.status, ran.message, ran.err]).toEqual(['ok', '', '']); + expect(ran.out).toBe( + lines( + 'js refused', + 'pyodide.ffi refused', + 'pyodide.http refused', + 'pyodide_js refused', + '_pyodide_core refused', + 'pyfetch ModuleNotFoundError', + '/etc/passwd FileNotFoundError', + '../outside.txt FileNotFoundError', + '/project/../etc/hosts FileNotFoundError', + "['data.txt', 'main.py']", + ), + ); + }); + + it('lockdown is the boundary: the program reaches no JavaScript value, nor any way to make one', async () => { + const ran = await run('python-full', [{ path: 'main.py', text: REACH + NETWORK }]); + expect([ran.status, ran.err]).toEqual(['ok', '']); + expect(ran.out).toBe(REACH_HELD + lines('socket: refused', 'urlopen: refused')); + }); + + it('runs no code from text through ctypes, and the run says so in its own words', async () => { + const program = lines( + 'import ctypes', + 'try:', + ' print("eval:", ctypes.CDLL(None).emscripten_run_script_int(b"6*7"), flush=True)', + 'except Exception as e:', + ' print("eval:", type(e).__name__, flush=True)', + ); + const ran = await run('python-full', [{ path: 'main.py', text: program }]); + expect(ran.out).not.toContain('eval: 42'); + // Whatever ends the run, its message is the app's text or a Python error name, never the runtime's raw text. + expect(ran.message).not.toMatch(/console|not defined|Program terminated/); + }); + + it('gives os._exit its exit code', async () => { + const three = await run('python-full', [ + { path: 'main.py', text: 'import os\nprint("a", flush=True)\nos._exit(3)\n' }, + ]); + expect([three.status, three.exitCode, three.out]).toEqual(['error', 3, 'a\n']); + const zero = await run('python-full', [{ path: 'main.py', text: 'import os\nos._exit(0)\n' }]); + expect([zero.status, zero.exitCode, zero.message]).toEqual(['ok', 0, '']); + }); + + it('runs asyncio with sleep and gather after lockdown', async () => { + const program = lines( + 'import asyncio', + 'async def job(name, delay):', + ' await asyncio.sleep(delay)', + ' print(name, "done")', + ' return name', + 'async def main():', + ' return await asyncio.gather(job("a", 0.05), job("b", 0.01))', + 'print(asyncio.run(main()))', + ); + const ran = await run('python-full', [{ path: 'main.py', text: program }]); + expect([ran.status, ran.err]).toEqual(['ok', '']); + expect(ran.out).toBe(lines('b done', 'a done', "['a', 'b']")); + }); + + it('runs a project end to end: a module, input, arguments, a diagnostic, the version, a written file', async () => { + const ran = await run( + 'python-full', + [ + { + path: 'main.py', + text: lines( + 'import sys, util', + 'print(sys.argv[1:], util.twice(input()))', + 'open("out.txt", "w").write("ok")', + 'util.boom()', + ), + }, + { path: 'util.py', text: lines('def twice(s):', ' return s * 2', 'def boom():', ' return 1 / 0') }, + ], + { stdin: 'ab\n', args: ['x'] }, + ); + expect(ran.out).toBe("['x'] abab\n"); + expect(ran.status).toBe('error'); + expect(ran.message).toBe('ZeroDivisionError'); + expect(ran.version).toBe('Python 3.14.2 (Pyodide 314.0.7)'); + expect(ran.written).toEqual([{ path: 'out.txt', text: 'ok' }]); + expect(ran.diagnostics).toEqual([ + { file: 'util.py', line: 4, message: 'ZeroDivisionError: division by zero', kind: 'ZeroDivisionError' }, + ]); + }); +}); + +describe.each(['python-quick', 'python-full'] as const)('how a run of %s ends', (runtime) => { + it('shows what the program printed before an endless loop, and ends with "time"', async () => { + const program = lines('print("before the loop")', 'while True:', ' pass'); + const ran = await run(runtime, [{ path: 'main.py', text: program }], { timeMs: 3_000 }); + expect([ran.status, ran.out]).toEqual(['time', 'before the loop\n']); + }); + + it('ends at once on an exit code past 32 bits, and keeps the written files', async () => { + const program = lines('import sys', 'open("o.txt", "w").write("x")', 'sys.exit(2**40)'); + const ran = await run(runtime, [{ path: 'main.py', text: program }]); + expect([ran.status, ran.exitCode, ran.err]).toEqual(['error', 2 ** 31 - 1, '']); + expect(ran.written).toEqual([{ path: 'o.txt', text: 'x' }]); + }); + + it('says interactive input is not available, and reads the input box', async () => { + const program = 'print(input())\n'; + const ran = await run(runtime, [{ path: 'main.py', text: program }], { stdin: 'hi\n', interactive: true }); + expect([ran.status, ran.out]).toEqual(['ok', 'hi\n']); + expect(ran.notes).toContain('interactive-unavailable'); + }); + + // The limit is 64 MiB: 1 MiB blocks stop with MemoryError before 66, within the time limit. + it('stops a program at the memory limit', async () => { + const program = lines( + 'blocks = []', + 'try:', + ' for _ in range(300):', + ' blocks.append(bytearray(1024 * 1024))', + 'except MemoryError:', + ' pass', + 'n = len(blocks)', + 'blocks = None', + 'print(n)', + ); + const ran = await run(runtime, [{ path: 'main.py', text: program }]); + expect([ran.status, ran.err]).toEqual(['ok', '']); + expect(Number(ran.out)).toBeGreaterThan(0); + expect(Number(ran.out)).toBeLessThan(66); + }); + + it('stops a file that grows past the memory limit with ENOSPC', async () => { + const program = lines( + 'n = 0', + 'chunk = "x" * (1024 * 1024)', + 'try:', + ' with open("big.txt", "w") as f:', + ' for _ in range(400):', + ' f.write(chunk)', + ' f.flush()', + ' n += 1', + 'except OSError as e:', + ' print("refused", e.errno)', + 'print(n)', + ); + const ran = await run(runtime, [{ path: 'main.py', text: program }]); + expect([ran.status, ran.err]).toEqual(['ok', '']); + const [refused, n] = ran.out.split('\n'); + expect(refused).toBe('refused 51'); + expect(Number(n)).toBeLessThanOrEqual(64); + }); + + it('runs main.py next to project files named random.py, io.py, and json.py', async () => { + const files = [ + { path: 'main.py', text: 'print("main")\nraise ValueError("bad")\n' }, + { path: 'random.py', text: 'print("project random")\n' }, + { path: 'io.py', text: 'print("project io")\n' }, + { path: 'json.py', text: 'print("project json")\n' }, + ]; + const ran = await run(runtime, files); + expect([ran.status, ran.out]).toEqual(['error', 'main\n']); + expect(ran.err).toContain('ValueError: bad'); + }); +}); + +describe('quick Python in the app run worker', () => { + it('runs a project end to end and cannot reach JavaScript', async () => { + const ran = await run( + 'python-quick', + [ + { + path: 'main.py', + text: lines( + 'import sys, util', + 'print(sys.argv[1:], util.twice(input()))', + 'import js', + 'print(getattr(js, "fetch", None))', + 'util.boom()', + ), + }, + { path: 'util.py', text: lines('def twice(s):', ' return s * 2', 'def boom():', ' return 1 // 0') }, + ], + { stdin: 'ab\n', args: ['x'] }, + ); + expect(ran.out).toBe("['x'] abab\nNone\n"); + expect(ran.status).toBe('error'); + expect(ran.version).toBe('MicroPython 1.27.0'); + expect(ran.diagnostics).toEqual([ + { file: 'util.py', line: 4, message: 'ZeroDivisionError: divide by zero', kind: 'ZeroDivisionError' }, + ]); + }); +}); diff --git a/app/src/features/codestudio/runtimes/python/session.ts b/app/src/features/codestudio/runtimes/python/session.ts new file mode 100644 index 000000000..c93a5525b --- /dev/null +++ b/app/src/features/codestudio/runtimes/python/session.ts @@ -0,0 +1,269 @@ +// One run or one check inside a loaded Python runtime (DESIGN 4.3 to 4.6). MicroPython and Pyodide share it. +// The project goes into the runtime's in-memory file system (Emscripten's MEMFS), the program runs through +// prelude.ts, and the files it wrote or changed go back to the run's overlay. Neither runtime can reach the page: +// MicroPython's JavaScript table is cut off at load, and Pyodide's js modules are refused before the program runs. + +import type { Emit, RunContext, RunEnd, RunFile, RunRequest, RuntimeDiagnostic, Stream } from '../../contracts/run'; +import { mapDiagnostics } from '../../run/diagnostics'; +import { isProjectPath } from '../../run/mount'; +import { capFiles } from './heap'; +import type { CappedFS } from './heap'; +import { CHECKED_FILE, CHECK_FILE, CHECK_SOURCE, HOME, RUN_FILE, RUN_SOURCE, SCRATCH, STATE_FILE } from './prelude'; +import type { PythonChecked, PythonState } from './prelude'; +import { pythonMapper } from './traceback'; +import { codeStudioPython } from '../../../../strings/en/codeStudio/python'; + +/** Room the run's state file has past the file cap, so a program that fills the cap still ends with its state. */ +const STATE_RESERVE = 1024 * 1024; + +/** The part of Emscripten's FS object both runtimes expose. */ +export interface EmscriptenFS extends CappedFS { + mkdir(path: string): void; + writeFile(path: string, data: string | Uint8Array): void; + readFile(path: string): Uint8Array; + readdir(path: string): string[]; + stat(path: string): { mode: number }; + lstat(path: string): { mode: number }; + isLink(mode: number): boolean; + unlink(path: string): void; + rmdir(path: string): void; + analyzePath(path: string): { exists: boolean }; + chdir(path: string): void; +} + +/** A loaded runtime. `exec` runs Python at the top level and throws when the runtime itself fails. */ +export interface PythonEngine { + fs: EmscriptenFS; + exec(source: string): void; + /** Where the runtime's output bytes go; set for each run. */ + sink: (stream: Stream, bytes: Uint8Array) => void; + /** Lets the program's heap grow by at most `bytes` past its size after load. Absent: the heap is fixed at load. */ + limitMemory?(bytes: number): void; +} + +/** Files read back after a run, at most. The overlay has its own, smaller limit. */ +const MAX_WALK = 2_000; + +function exists(fs: EmscriptenFS, path: string): boolean { + return fs.analyzePath(path).exists; +} + +function makeFolders(fs: EmscriptenFS, path: string): void { + let at = ''; + for (const part of path.split('/').filter(Boolean)) { + at += `/${part}`; + if (!exists(fs, at)) fs.mkdir(at); + } +} + +/** The mode of `path` itself: a link is not followed. Null when nothing is there. */ +function modeOf(fs: EmscriptenFS, path: string): number | null { + try { + return fs.lstat(path).mode; + } catch { + return null; + } +} + +/** Removes `path` and what it holds. A link is removed, never followed, so a link to "/" or to itself is safe. */ +function removeTree(fs: EmscriptenFS, path: string): void { + const mode = modeOf(fs, path); + if (mode === null) return; + if (!fs.isDir(mode)) return fs.unlink(path); + for (const name of fs.readdir(path)) if (name !== '.' && name !== '..') removeTree(fs, `${path}/${name}`); + fs.rmdir(path); +} + +/** Every file under `root`, as project paths. Links are skipped: the overlay holds files and folders only. */ +function walk(fs: EmscriptenFS, root: string): string[] { + const found: string[] = []; + const visit = (folder: string, prefix: string) => { + for (const name of fs.readdir(folder)) { + if (name === '.' || name === '..' || found.length >= MAX_WALK) continue; + const full = `${folder}/${name}`; + const mode = modeOf(fs, full); + if (mode === null || fs.isLink(mode)) continue; + if (fs.isDir(mode)) visit(full, `${prefix}${name}/`); + else if (fs.isFile(mode)) found.push(`${prefix}${name}`); + } + }; + visit(root, ''); + return found; +} + +function readText(fs: EmscriptenFS, path: string): string | null { + try { + // A byte order mark is kept, so a project file that starts with one reads back as it was mounted. + return new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(fs.readFile(path)); + } catch { + return null; + } +} + +/** Puts the project under HOME, replacing what an earlier run left there. */ +function mountProject(fs: EmscriptenFS, files: readonly RunFile[]): void { + // The last run left the working folder inside the project, and a folder in use cannot be removed. + fs.chdir('/'); + removeTree(fs, HOME); + makeFolders(fs, HOME); + makeFolders(fs, SCRATCH); + for (const file of files) { + if (!isProjectPath(file.path)) continue; + const slash = file.path.lastIndexOf('/'); + if (slash > 0) makeFolders(fs, `${HOME}/${file.path.slice(0, slash)}`); + fs.writeFile(`${HOME}/${file.path}`, file.text); + } +} + +/** Sends the files the program added or changed to the run's overlay. A file that is not UTF-8 text is skipped. */ +function collectWrites(fs: EmscriptenFS, files: readonly RunFile[], context: RunContext): void { + const before = new Map(files.map((file) => [file.path, file.text])); + let paths: string[]; + try { + paths = walk(fs, HOME); + } catch { + // A folder the program left in a state the walk cannot read: the run keeps its own status. + return; + } + for (const path of paths) { + const text = readText(fs, `${HOME}/${path}`); + if (text === null || before.get(path) === text) continue; + context.files.write(path, text); + } +} + +function readJson(fs: EmscriptenFS, path: string): T | null { + if (!exists(fs, path)) return null; + const text = readText(fs, path); + fs.unlink(path); + try { + return text === null ? null : (JSON.parse(text) as T); + } catch { + return null; + } +} + +/** UTF-8 output, decoded across byte boundaries, one decoder per stream. */ +function outputTo(emit: Emit): { write(stream: Stream, bytes: Uint8Array): void; end(): void } { + const decoders = { out: new TextDecoder(), err: new TextDecoder() }; + return { + write(stream, bytes) { + const text = decoders[stream].decode(bytes, { stream: true }); + if (text !== '') emit({ kind: 'out', stream, text }); + }, + end() { + for (const stream of ['out', 'err'] as const) { + const text = decoders[stream].decode(); + if (text !== '') emit({ kind: 'out', stream, text }); + } + }, + }; +} + +/** How a run that the runtime itself ended (not the program) is reported. */ +function runtimeFailure(error: unknown): RunEnd { + const text = String((error as Error)?.message ?? error); + // `os._exit(n)` ends the runtime with Emscripten's ExitStatus, "Program terminated with exit(n)", which carries the + // program's exit code. + const exit = error as { name?: unknown; status?: unknown } | null; + const status = + exit?.name === 'ExitStatus' ? exit.status : Number(/^Program terminated with exit\((-?\d+)\)/.exec(text)?.[1]); + if (typeof status === 'number' && Number.isInteger(status)) { + const code = Math.abs(status) > 2 ** 31 ? 1 : status; + return { status: code === 0 ? 'ok' : 'error', exitCode: code, message: code === 0 ? '' : 'SystemExit' }; + } + if (/memory|MemoryError|Cannot enlarge memory/i.test(text)) return { status: 'memory', message: 'MemoryError' }; + // The runtime's own text (a JavaScript error, say) means nothing to the reader of the console. + return { status: 'error', exitCode: 1, message: codeStudioPython.runtimeStopped }; +} + +/** Runs one request in `engine`. */ +export function runPython(engine: PythonEngine, request: RunRequest, emit: Emit, context: RunContext): RunEnd { + const { fs } = engine; + mountProject(fs, request.files); + const settings = { + home: HOME, + main: request.main, + args: request.args, + seed: request.seed, + stdin: request.stdin.text.replace(/\r\n?/g, '\n'), + }; + fs.writeFile(RUN_FILE, JSON.stringify(settings)); + engine.limitMemory?.(request.limits.memoryBytes); + // File data lives outside the heap, so the files the program writes have a cap of the same size. + const limitFiles = capFiles(fs, SCRATCH, STATE_FILE, STATE_RESERVE); + limitFiles(request.limits.memoryBytes); + const output = outputTo(emit); + engine.sink = output.write; + let failure: RunEnd | null = null; + try { + engine.exec(RUN_SOURCE); + } catch (error) { + failure = runtimeFailure(error); + } finally { + limitFiles(null); + output.end(); + engine.sink = () => undefined; + } + const state = readJson(fs, STATE_FILE); + if (exists(fs, RUN_FILE)) fs.unlink(RUN_FILE); + collectWrites(fs, request.files, context); + if (state?.wantedInput) { + // The program read past the end of the input: the run's Stdin records it for the console's note. + context.stdin.read(); + context.stdin.line(); + } + if (failure) return failure; + if (!state) return { status: 'error', exitCode: 1, message: '' }; + // A MemoryError from compile() is the parser's stack, not the run's memory: the program is too deeply nested. + if (state.kind === 'MemoryError' && !state.compiling) + return { status: 'memory', exitCode: 1, message: 'MemoryError' }; + return readState(state); +} + +/** + * How the state the Python side wrote ends the run. The page refuses an exit code that is not an integer within + * 2^31, and a refused `done` would leave the run waiting for its time limit, so such a code becomes 1. + */ +function readState(state: PythonState): RunEnd { + const message = typeof state.kind === 'string' ? state.kind : ''; + const exit: unknown = state.exit; + if (typeof exit !== 'number' || !Number.isInteger(exit) || Math.abs(exit) > 2 ** 31) { + return { status: 'error', exitCode: 1, message }; + } + return { status: state.status === 'ok' && exit === 0 ? 'ok' : 'error', exitCode: exit, message }; +} + +/** The diagnostic for a check error the traceback mapper finds no frame for. */ +function frameless(path: string, found: PythonChecked): RuntimeDiagnostic | undefined { + const message = found.message ?? found.text?.trimEnd().split('\n').at(-1); + if (!message) return undefined; + return { file: path, line: 1, message, kind: found.kind ?? 'SyntaxError' }; +} + +/** Compiles one file without running it: the syntax error it has, if any, as diagnostics (DESIGN 6.1, L-c). */ +export function checkPython(engine: PythonEngine, file: RunFile, files: readonly RunFile[]): RuntimeDiagnostic[] { + const { fs } = engine; + makeFolders(fs, SCRATCH); + fs.writeFile(CHECK_FILE, JSON.stringify({ home: HOME, path: file.path, text: file.text })); + try { + engine.exec(CHECK_SOURCE); + } catch { + return []; + } finally { + if (exists(fs, CHECK_FILE)) fs.unlink(CHECK_FILE); + } + const found = readJson(fs, CHECKED_FILE); + if (!found || found.ok || !found.text) return []; + const known = files.some((item) => item.path === file.path) ? files : [...files, file]; + const items = mapDiagnostics(pythonMapper, found.text, known).filter((item) => item.file === file.path); + const lineCount = file.text.split('\n').length; + // An error with no frame in the file (a NUL character, a parser stack that ran out) still gets a diagnostic: at its + // own line, or at line 1. + const item = items[items.length - 1] ?? frameless(file.path, found); + if (!item) return []; + // CPython gives the exact place; MicroPython gives the line only, which the traceback already holds. + const line = typeof found.line === 'number' && found.line >= 1 ? Math.min(found.line, lineCount) : item.line; + const column = typeof found.column === 'number' && found.column >= 1 ? found.column : undefined; + return [{ ...item, line, ...(column === undefined ? {} : { column }) }]; +} diff --git a/app/src/features/codestudio/runtimes/python/traceback.ts b/app/src/features/codestudio/runtimes/python/traceback.ts new file mode 100644 index 000000000..d569db32e --- /dev/null +++ b/app/src/features/codestudio/runtimes/python/traceback.ts @@ -0,0 +1,58 @@ +// Python tracebacks as RuntimeDiagnostic (DESIGN 4.6). CPython and MicroPython print the same frame lines, +// ` File "util.py", line 3, in f`, then the exception's own line, `NameError: name 'y' is not defined`. A syntax +// error has one frame and no "Traceback" header. The error is placed at the innermost frame in a project file, so an +// error inside util.py, called from main.py, marks util.py. + +import type { DiagnosticMapper, RuntimeDiagnostic } from '../../contracts/run'; +import { normalPath } from '../../run/mount'; +import { HOME } from './prelude'; + +const FRAME = /^\s+File "(?[^"]+)", line (?\d+)/; +/** The exception's line: its type (possibly dotted), then an optional message. */ +const EXCEPTION = /^(?[A-Za-z_][\w.]*)(?::\s?(?.*))?$/; + +/** A traceback's path as a project path, or null for the runtime's own files. */ +function projectPath(file: string): string | null { + const local = file.startsWith(`${HOME}/`) ? file.slice(HOME.length + 1) : file; + return normalPath(local); +} + +/** The frames of the block that ends at line `end`: the indented lines above it, up to its header. */ +function framesAbove(lines: readonly string[], end: number): { file: string; line: number }[] { + const frames: { file: string; line: number }[] = []; + for (let at = end - 1; at >= 0; at--) { + const text = lines[at]!; + if (text === '' || !/^\s/.test(text)) break; + const match = FRAME.exec(text); + if (match?.groups) frames.unshift({ file: match.groups.file!, line: Number(match.groups.line) }); + } + return frames; +} + +export const pythonMapper: DiagnosticMapper = { + map(errorText, files) { + const counts = new Map(files.map((file) => [file.path, file.text.split('\n').length])); + const lines = errorText.replace(/\r\n?/g, '\n').split('\n'); + for (let at = lines.length - 1; at > 0; at--) { + const match = EXCEPTION.exec(lines[at]!); + if (!match?.groups) continue; + const frames = framesAbove(lines, at); + if (frames.length === 0) continue; + for (let index = frames.length - 1; index >= 0; index--) { + const frame = frames[index]!; + const file = projectPath(frame.file); + const last = file === null ? undefined : counts.get(file); + if (file === null || last === undefined || frame.line < 1) continue; + const found: RuntimeDiagnostic = { + file, + line: Math.min(frame.line, last), + message: lines[at]!.trim(), + kind: match.groups.kind!, + }; + return [found]; + } + return []; + } + return []; + }, +}; diff --git a/app/src/features/page/history/CompareView.tsx b/app/src/features/page/history/CompareView.tsx index e60c2ef83..436e39b08 100644 --- a/app/src/features/page/history/CompareView.tsx +++ b/app/src/features/page/history/CompareView.tsx @@ -5,6 +5,7 @@ import { useEffect, useRef, useState } from 'react'; import type { BlockChange, PageDiff, ParagraphChange } from '../../../editor/diff/pageDiff'; import type { WordPart } from '../../../editor/diff/myers'; +import { describeFormatting } from '../../../editor/diff/formatDiff'; import { parseTextBlock } from '../../../editor/markdown'; import { renderStatic } from '../../../editor/schema/dom'; import { t } from '../../../strings/t'; @@ -88,6 +89,7 @@ function Paragraph({ change, onRestore }: { change: ParagraphChange; onRestore: return (
{body} + {change.formatting &&

{describeFormatting(change.formatting)}

} {onRestore && ( + + ); +} + +let unregister: (() => void) | undefined; + +afterEach(async () => { + unregister?.(); + unregister = undefined; + for (let i = 0; i < 3 && document.querySelector('[role="dialog"]'); i += 1) await pressChord('Escape'); + resetFakeShell(); +}); + +async function open(blocks: unknown) { + const calls: Array<[string, Record]> = []; + setFakeShell((name, args) => { + calls.push([name, args]); + if (name === 'conflict.text') return { ...TEXTS, blocks }; + if (name === 'conflict.resolve') return { ok: true }; + return null; + }); + await renderApp(); + const closed = openConflictDialog(PAGE, FIRST, 1); + const dialog = await screen.findByRole('dialog', { name: 'Two versions of this page' }); + return { calls, closed, dialog }; +} + +describe('the conflict view registry', () => { + it('shows the registered view for its block type and resolves through it', async () => { + unregister = registerConflictView('test.view', TestView); + const { calls, closed, dialog } = await open([block('b1', 'test.view')]); + const view = await within(dialog).findByRole('region', { name: 'Test compare b1' }); + expect(within(view).getByText('Here 1, on Laptop 2')).toBeTruthy(); + // The whole-page text stays one click away, so a title or tag difference is never hidden. + const details = within(dialog).getByText('Both whole pages as text').closest('details'); + expect(details?.open).toBe(false); + await userEvent.click(within(dialog).getByText('Both whole pages as text')); + expect(details?.open).toBe(true); + expect(within(dialog).getByText(TEXTS.mine)).toBeTruthy(); + await userEvent.click(within(view).getByRole('button', { name: 'Take theirs' })); + await closed; + expect(calls.find(([name]) => name === 'conflict.resolve')?.[1]).toEqual({ + pageId: PAGE, + revision: FIRST.revision, + choice: 'keepTheirs', + }); + }); + + it('keeps the text view for block types without a view', async () => { + unregister = registerConflictView('test.view', TestView); + const { dialog } = await open([block('b1', 'test.view'), block('b2', 'text')]); + expect(await within(dialog).findByRole('region', { name: 'Test compare b1' })).toBeTruthy(); + expect(within(dialog).queryByRole('region', { name: 'Test compare b2' })).toBeNull(); + expect(within(dialog).getByText(TEXTS.mine)).toBeTruthy(); + expect(within(dialog).getByText(TEXTS.theirs)).toBeTruthy(); + expect(within(dialog).queryByText('Both whole pages as text')).toBeNull(); + }); + + it('shows only the text view when no type registered a view', async () => { + const { dialog } = await open([block('b1', 'test.view')]); + expect(await within(dialog).findByText(TEXTS.mine)).toBeTruthy(); + expect(within(dialog).queryByRole('region', { name: /^Test compare/ })).toBeNull(); + }); +}); diff --git a/app/src/features/qol/conflict/registry.ts b/app/src/features/qol/conflict/registry.ts new file mode 100644 index 000000000..69a272795 --- /dev/null +++ b/app/src/features/qol/conflict/registry.ts @@ -0,0 +1,58 @@ +// The conflict view registry (ADR 0089). A block type registers its own compare and resolve view, and the +// conflict dialog shows it for each changed block of that type. Every other block keeps the side-by-side text. +// Tables (table merge on sync) and code projects (the code project conflict view) register theirs. + +import type { ComponentType } from 'react'; + +/** How a conflict is resolved: this version, the other one, or both as separate pages. */ +export type ConflictChoice = 'keepMine' | 'keepTheirs' | 'keepBoth'; + +/** One block that differs between the two versions, as the core writes it in `page.json`. */ +export interface ConflictBlock { + readonly id: string; + readonly type: string; + /** The block on this device, or `null` if this version has no such block. */ + readonly mine: unknown; + /** The block in the other version, or `null` if that version has no such block. */ + readonly theirs: unknown; +} + +export interface ConflictViewProps { + readonly block: ConflictBlock; + /** The other device's label, or a generic name. */ + readonly otherDevice: string; + /** Resolves the whole conflict, as the dialog's own buttons do. */ + resolve(choice: ConflictChoice): void; +} + +export type ConflictView = ComponentType; + +const views = new Map(); + +/** Registers the compare and resolve view for a block type. Returns a function that removes it again. */ +export function registerConflictView(type: string, view: ConflictView): () => void { + views.set(type, view); + return () => { + if (views.get(type) === view) views.delete(type); + }; +} + +/** The view registered for a block type, if any. */ +export function conflictViewFor(type: string): ConflictView | undefined { + return views.get(type); +} + +/** Splits changed blocks into those with a registered view and the rest, which the text view shows. */ +export function splitByView(blocks: readonly ConflictBlock[] | undefined): { + readonly registered: ReadonlyArray<{ block: ConflictBlock; View: ConflictView }>; + readonly others: readonly ConflictBlock[]; +} { + const registered: Array<{ block: ConflictBlock; View: ConflictView }> = []; + const others: ConflictBlock[] = []; + for (const block of blocks ?? []) { + const View = views.get(block.type); + if (View) registered.push({ block, View }); + else others.push(block); + } + return { registered, others }; +} diff --git a/app/src/features/qol/index.ts b/app/src/features/qol/index.ts index c550b6b8f..6cba7d5ca 100644 --- a/app/src/features/qol/index.ts +++ b/app/src/features/qol/index.ts @@ -9,6 +9,8 @@ export { showDialog } from './dialogHost'; export { pagesClient } from './pagesApi'; export { readPrefs, writePrefs } from './prefs'; export { openInNewTab } from './tabs'; +export { registerConflictView } from './conflict/registry'; +export type { ConflictBlock, ConflictChoice, ConflictViewProps } from './conflict/registry'; /** The notice about a page edited in another app loads when first needed. */ export const loadExternalNotice = () => import('./ExternalNotice'); /** A page in a window of its own; it loads only in that window. */ diff --git a/app/src/features/review/CommentsPane.tsx b/app/src/features/review/CommentsPane.tsx new file mode 100644 index 000000000..4fe547afc --- /dev/null +++ b/app/src/features/review/CommentsPane.tsx @@ -0,0 +1,199 @@ +// The comments pane (ADR 0086): the page's comments in reading order, each with the text it covers, its author and +// time, its text, and its replies. A comment can be answered, resolved, reopened, deleted, or followed to its text. +// The comment at the caret is marked. Comments are notes on their ranges for screen readers (text DESIGN.md 12). +import { useEffect, useId, useMemo, useReducer, useState } from 'react'; +import type { KeyboardEvent } from 'react'; +import { XIcon } from '@phosphor-icons/react/dist/csr/X'; +import { escapeParagraphText, parseTextBlock } from '../../editor/markdown'; +import { currentLocale } from '../../state/language'; +import { t } from '../../strings/t'; +import { Button, IconButton, TextField } from '../../ui'; +import { targetEditor } from '../page'; +import { deleteComment, goToComment, setResolved } from './actions'; +import { commentsAt } from './comments'; +import type { PageComment, PageReview } from './page'; +import { MAX_REPLIES, fitsRecord, stamp } from './records'; +import type { CommentJson } from './records'; +import { authorName, reviewPrefs, setReviewPrefs } from './state'; +import styles from './review.module.css'; + +/** A record's inline Markdown as plain text, hard breaks as new lines. */ +function plain(markdown: string): string { + const doc = parseTextBlock(markdown); + return doc.textBetween(0, doc.content.size, '\n', (leaf) => (leaf.type.name === 'hardBreak' ? '\n' : '')); +} + +/** Plain text typed in the pane as inline Markdown. */ +const markdownOf = (text: string) => escapeParagraphText(text.trim()); + +function when(time: string): string { + const date = new Date(time); + return Number.isNaN(date.getTime()) + ? time + : date.toLocaleString(currentLocale(), { dateStyle: 'medium', timeStyle: 'short' }); +} + +function Editor(props: { label: string; action: string; autoFocus?: boolean; onSubmit(text: string): void }) { + const [text, setText] = useState(''); + const tooLong = !fitsRecord(markdownOf(text)); + const submit = () => { + if (!text.trim() || tooLong) return; + props.onSubmit(text); + setText(''); + }; + const onKeyDown = (event: KeyboardEvent) => { + if (event.key === 'Enter' && (event.ctrlKey || event.metaKey)) { + event.preventDefault(); + submit(); + } + }; + const helpId = useId(); + return ( +
+