diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f0c3c1798..43a62df8c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -114,6 +114,27 @@ jobs: path: ./coverage-report.md include-hidden-files: true + # Spec conformance (see spec/README.md). Non-blocking until the corpus in + # spec/DISAGREEMENTS.md is empty; then drop continue-on-error. + spec-conformance: + runs-on: blacksmith-2vcpu-ubuntu-2204 + timeout-minutes: 15 + continue-on-error: true + strategy: + matrix: + node-version: [22.x] + steps: + - uses: actions/checkout@v3 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + cache: "yarn" + - name: Install dependencies + run: yarn install --frozen-lockfile + - name: Run spec conformance tests + run: yarn workspace @dbml/parse test:conformance + comment-coverage: runs-on: blacksmith-2vcpu-ubuntu-2204 timeout-minutes: 5 diff --git a/.gitignore b/.gitignore index b44693f4c..7c546a82a 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,6 @@ coverage/ __benchmarks__/output/ .gitnexus + +# spec conformance: parsers generated from spec/antlr at test time +packages/dbml-parse/__tests__/conformance/generated/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5a42dccc0..64b60e236 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,8 +6,8 @@ The following is a set of guidelines for contributing to DBML and its packages, ## Not sure where to start ? -- [PEG.js](https://pegjs.org/): DBML uses PEG.js library to parse different database languages and structure them as plain Javascript object. -- Check out the `parse` folder in `@dbml/core` source code to learn more about how to structure and import different grammars from multiple PEG.js files. +- The DBML parser lives in [`packages/dbml-parse`](./packages/dbml-parse). It is a hand-written lexer and recursive-descent parser (`src/core/lexer`, `src/core/parser`) followed by per-element validation in `src/core/local_modules` and binding and interpretation in `src/core/global_modules`. The formal grammar and the conformance tests that keep it honest live in [`spec/`](./spec). +- Importing other database languages (PostgreSQL, MySQL, MSSQL, ...) is done in `@dbml/core` under `packages/dbml-core/src/parse`, using ANTLR grammars. The [PEG.js](https://pegjs.org/) grammars under `parse/deprecated` are no longer maintained and are kept for reference only. - When you're ready to jump to the source code, look for issues tagged with [help wanted](https://github.com/holistics/dbml/labels/help%20wanted) label to get list of current issues you can make a contribution to or create a new issue for suggesting enhancement. - Check out DBML [website](https://www.dbml-lang.org/home/) for more documentation on DBML syntax. diff --git a/packages/dbml-parse/__tests__/conformance/antlr.setup.ts b/packages/dbml-parse/__tests__/conformance/antlr.setup.ts new file mode 100644 index 000000000..27ad14b32 --- /dev/null +++ b/packages/dbml-parse/__tests__/conformance/antlr.setup.ts @@ -0,0 +1,7 @@ +// vitest global setup: compile spec/antlr/*.g4 once before the conformance +// test files are collected. +import { generateAntlrParser } from './antlr'; + +export default function setup (): void { + generateAntlrParser(); +} diff --git a/packages/dbml-parse/__tests__/conformance/antlr.ts b/packages/dbml-parse/__tests__/conformance/antlr.ts new file mode 100644 index 000000000..4e84f850d --- /dev/null +++ b/packages/dbml-parse/__tests__/conformance/antlr.ts @@ -0,0 +1,300 @@ +// ANTLR notation of the Layer 1 spec (spec/antlr/*.g4). +// +// The grammars are compiled with antlr-ng (a Node port of the ANTLR tool, no +// Java needed) for the JavaScript target, which runs on the same `antlr4` +// runtime @dbml/core uses. Generation happens in the vitest global setup +// (antlr.setup.ts) into ./generated/antlr, which is not committed. +// +// ANTLR produces a parse tree shaped like the grammar rules, so this file also +// converts that tree into the same { kind, children } spec nodes the peggy +// grammar builds directly. The element reinterpretation of a function +// application (parser/utils.ts: convertFuncAppToElem) is applied here. +import { execFileSync } from 'node:child_process'; +import { existsSync, mkdirSync, statSync } from 'node:fs'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import antlr4 from 'antlr4'; +import { + type SpecNode, + type SpecParser, + type SpecToken, + SPEC_DIR, + convertFuncAppToElem, + node, + tok, +} from './harness'; + +export const ANTLR_GRAMMAR_DIR = path.join(SPEC_DIR, 'antlr'); +export const ANTLR_OUTPUT_DIR = path.resolve(__dirname, 'generated/antlr'); + +const GRAMMARS = ['DbmlLexer.g4', 'DbmlParser.g4']; + +function isStale (): boolean { + const outputs = ['DbmlLexer.js', 'DbmlParser.js'].map((f) => path.join(ANTLR_OUTPUT_DIR, f)); + if (!outputs.every(existsSync)) return true; + const newestGrammar = Math.max(...GRAMMARS.map((g) => statSync(path.join(ANTLR_GRAMMAR_DIR, g)).mtimeMs)); + const oldestOutput = Math.min(...outputs.map((o) => statSync(o).mtimeMs)); + return newestGrammar > oldestOutput; +} + +// antlr-ng only exposes an ESM entry point, so `require.resolve` cannot find +// it; locate its CLI by walking up the node_modules chain instead. +function findAntlrNgRunner (): string { + for (let dir = __dirname; ; dir = path.dirname(dir)) { + const candidate = path.join(dir, 'node_modules/antlr-ng/dist/cli/runner.js'); + if (existsSync(candidate)) return candidate; + if (path.dirname(dir) === dir) throw new Error('antlr-ng is not installed'); + } +} + +// Compile spec/antlr/*.g4 to JavaScript with antlr-ng. Idempotent: skipped +// when the generated files are newer than the grammars. +export function generateAntlrParser (): void { + if (!isStale()) return; + mkdirSync(ANTLR_OUTPUT_DIR, { recursive: true }); + const runner = findAntlrNgRunner(); + const run = (grammar: string) => execFileSync(process.execPath, [ + runner, + '-Dlanguage=JavaScript', + '-o', + ANTLR_OUTPUT_DIR, + '--lib', + ANTLR_OUTPUT_DIR, + '--generate-listener', + 'false', + path.join(ANTLR_GRAMMAR_DIR, grammar), + ], { stdio: 'pipe' }); + // The parser needs the lexer's .tokens file, so generate in order. + GRAMMARS.forEach(run); +} + +// The bundled runtime keeps the error strategies under `antlr4.error`, which +// its type definitions do not declare. +const runtime = antlr4 as unknown as typeof antlr4 & { error: { BailErrorStrategy: new () => unknown } }; + +// Minimal view of the antlr4 JavaScript runtime's tree types. +interface Terminal { + symbol: { type: number; text: string }; +} +interface RuleNode { + ruleIndex: number; + children?: (RuleNode | Terminal)[] | null; + getText (): string; +} +type Tree = RuleNode | Terminal; + +function isTerminal (t: Tree): t is Terminal { + return 'symbol' in t; +} + +// Map ANTLR token types onto the reference SyntaxTokenKind names. +const TOKEN_KINDS: Record = { + IDENTIFIER: '', + USE: '', + REUSE: '', + FROM: '', + AS: '', + METADATA: '', + QUOTED_VARIABLE: '', + STRING: '', + NUMBER: '', + COLOR: '', + FUNCTION_EXPRESSION: '', + WILDCARD: '', + LPAREN: '', + RPAREN: '', + LBRACKET: '', + RBRACKET: '', + LBRACE: '', + RBRACE: '', + COMMA: '', + COLON: '', + EOF: '', +}; + +class TreeConverter { + constructor (private ruleNames: string[], private symbolicNames: (string | null)[]) {} + + token (t: Terminal): SpecToken { + const name = t.symbol.type === antlr4.Token.EOF ? 'EOF' : this.symbolicNames[t.symbol.type] ?? ''; + return tok(TOKEN_KINDS[name] ?? '', t.symbol.text); + } + + kids (ctx: RuleNode): Tree[] { + return ctx.children ?? []; + } + + // Convert a child that is either a token or a rule wrapping a single token. + leafToken (t: Tree): SpecToken { + if (isTerminal(t)) return this.token(t); + return this.leafToken(this.kids(t)[0]); + } + + // Fold `head (op operand)*` into left-associative infix nodes. + foldInfix (ctx: RuleNode): SpecNode { + const kids = this.kids(ctx); + let left = this.convert(kids[0]) as SpecNode; + for (let i = 1; i < kids.length; i += 2) { + left = node('', left, this.leafToken(kids[i]), this.convert(kids[i + 1])); + } + return left; + } + + // Fold postfix steps (callStep, chainStep, indexStep, memberStep) onto a head. + foldSteps (head: SpecNode, steps: Tree[]): SpecNode { + return steps.reduce((left, step) => { + const rule = this.ruleNames[(step as RuleNode).ruleIndex]; + const inner = this.kids(step as RuleNode); + switch (rule) { + case 'chainStep': return this.foldSteps(left, inner); + case 'callStep': return node('', left, this.convert(inner[0])); + case 'indexStep': return node('', left, this.convert(inner[0])); + case 'memberStep': return node('', left, this.leafToken(inner[0]), this.convert(inner[1])); + default: throw new Error(`unexpected postfix step ${rule}`); + } + }, head); + } + + convertAll (trees: Tree[]): (SpecNode | SpecToken)[] { + return trees.map((t) => this.convert(t)); + } + + convert (tree: Tree): SpecNode | SpecToken { + if (isTerminal(tree)) return this.token(tree); + const ctx = tree; + const kids = this.kids(ctx); + const rule = this.ruleNames[ctx.ruleIndex]; + switch (rule) { + case 'program': + return node('', this.convertAll(kids)); + case 'useDeclaration': + return node('', this.convertAll(kids)); + case 'useSpecifierList': + return node('', this.convertAll(kids)); + case 'useSpecifier': + return node('', this.convertAll(kids)); + case 'elementDeclaration': { + // METADATA identifier elementTail | (IDENTIFIER | FROM | AS) elementTail + const tail = kids[kids.length - 1] as RuleNode; + const head = kids.slice(0, -1).map((k) => this.leafToken(k)); + return node('', head, this.convertAll(this.kids(tail))); + } + case 'elementBody': + // COLON simpleBody | blockExpression: splice into the parent + return node('', this.convertAll(kids)); + case 'fieldDeclaration': + return node('', this.convertAll(kids)); + case 'blockExpression': + return node('', this.convertAll(kids)); + case 'expression': { + const [callee, ...args] = kids.map((k) => this.convert(k) as SpecNode); + return convertFuncAppToElem(callee, args) ?? node('', callee, args); + } + case 'commaExpression': { + // normalExpression (COMMA commaRest)? | COMMA commaRest + if (kids.length === 1) return this.convert(kids[0]); + const first = isTerminal(kids[0]) ? node('') : this.convert(kids[0]); + const rest = kids.filter((k) => !isTerminal(k) || k.symbol.type !== antlr4.Token.EOF).slice(isTerminal(kids[0]) ? 0 : 1); + return node('', first, this.convertAll(rest)); + } + case 'commaRest': { + // empty | COMMA commaRest | normalExpression (COMMA commaRest)? + if (kids.length === 0) return node('', node('')); + if (isTerminal(kids[0])) return node('', node(''), this.convertAll(kids)); + return node('', this.convertAll(kids)); + } + case 'assignmentExpression': + case 'equalityExpression': + case 'comparisonExpression': + case 'additiveExpression': + case 'multiplicativeExpression': + return this.foldInfix(ctx); + case 'postfixExpression': + case 'memberChain': + return this.foldSteps(this.convert(kids[0]) as SpecNode, kids.slice(1)); + case 'unaryExpression': + if (kids.length === 2) return node('', this.leafToken(kids[0]), this.convert(kids[1])); + return this.convert(kids[0]); + case 'primaryExpression': { + const inner = kids[0] as RuleNode; + const kind = this.ruleNames[inner.ruleIndex] === 'literal' ? '' : ''; + return node('', node(kind, this.leafToken(inner))); + } + case 'functionExpression': + return node('', this.leafToken(kids[0])); + case 'wildcard': + return node('', this.leafToken(kids[0])); + case 'tupleExpression': { + const converted = this.convertAll(kids); + const kind = kids.length === 3 ? '' : ''; + return node(kind, converted); + } + case 'listExpression': + return node('', this.convertAll(kids)); + case 'attribute': + return node('', this.convertAll(kids)); + case 'attributeName': { + const inner = kids[0]; + if (!isTerminal(inner) && this.ruleNames[inner.ruleIndex] === 'identifierStream') return this.convert(inner); + const kind = !isTerminal(inner) && this.ruleNames[inner.ruleIndex] === 'literal' ? '' : ''; + return node('', node(kind, this.leafToken(inner))); + } + case 'identifierStream': + return node('', kids.map((k) => this.leafToken(k))); + case 'identifier': + case 'prefixOp': + case 'additiveOp': + case 'comparisonOp': + return this.leafToken(kids[0]); + default: + // statement, useName, elementTail, simpleBody, bodyItem, normalExpression, + // operand, attributeValue, literal, variable: transparent wrappers + if (kids.length !== 1) throw new Error(`unexpected shape for rule ${rule}`); + return this.convert(kids[0]); + } + } +} + +// `` nodes are transparent lists produced by the converter for +// sub-rules whose children belong to the parent node; flatten them. +function flattenSplices (n: SpecNode | SpecToken): (SpecNode | SpecToken)[] { + if ('token' in n) return [n]; + const children = n.children.flatMap(flattenSplices); + if (n.kind === '') return children; + return [{ kind: n.kind, children }]; +} + +export async function loadAntlrParser (): Promise { + generateAntlrParser(); + const importGenerated = (file: string) => import(/* @vite-ignore */ pathToFileURL(path.join(ANTLR_OUTPUT_DIR, file)).href); + const DbmlLexer = (await importGenerated('DbmlLexer.js')).default; + const DbmlParser = (await importGenerated('DbmlParser.js')).default; + const converter = new TreeConverter(DbmlParser.ruleNames, DbmlParser.symbolicNames); + + return { + name: 'antlr', + parse (source: string): SpecNode { + const errors: string[] = []; + const listener = { + syntaxError: (_r: unknown, _s: unknown, line: number, column: number, msg: string) => { + errors.push(`${line}:${column} ${msg}`); + }, + reportAmbiguity: () => {}, + reportAttemptingFullContext: () => {}, + reportContextSensitivity: () => {}, + }; + // Decode to code points so that \p{L} sees supplementary-plane letters. + const lexer = new DbmlLexer(new antlr4.CharStream(source, true)); + lexer.removeErrorListeners(); + lexer.addErrorListener(listener); + const parser = new DbmlParser(new antlr4.CommonTokenStream(lexer)); + parser.removeErrorListeners(); + parser.addErrorListener(listener); + parser._errHandler = new runtime.error.BailErrorStrategy(); + const tree = parser.program(); + if (errors.length > 0) throw new Error(errors.join('; ')); + const [program] = flattenSplices(converter.convert(tree)); + return program as SpecNode; + }, + }; +} diff --git a/packages/dbml-parse/__tests__/conformance/harness.ts b/packages/dbml-parse/__tests__/conformance/harness.ts new file mode 100644 index 000000000..95fa0fcc1 --- /dev/null +++ b/packages/dbml-parse/__tests__/conformance/harness.ts @@ -0,0 +1,213 @@ +// Shared machinery for the Layer 1 conformance tests. +// +// The reference parser (`@dbml/parse` lexer + parser from src/) runs over the +// same source text as each spec parser: +// - `peggy`: generated at test time from spec/dbml-syntax.peggy +// - `antlr`: generated at test time from spec/antlr/*.g4 (see antlr.ts) +// +// They are compared on two things: +// 1. acceptance: the reference "accepts" when lexer and parser report no error; +// the spec parser accepts when peggy does not throw. +// 2. tree shape: both trees are flattened to the same S-expression of node +// kinds and token kinds, in source order, mirroring getMemberChain(). +import { readFileSync } from 'node:fs'; +import path from 'node:path'; +import peggy from 'peggy'; +import { getMemberChain } from '@/core/parser/utils'; +import { SyntaxNode } from '@/core/types/nodes'; +import { SyntaxToken } from '@/core/types/tokens'; +import { parse } from '@tests/utils'; + +export const SPEC_DIR = path.resolve(__dirname, '../../../../spec'); +export const GRAMMAR_PATH = path.join(SPEC_DIR, 'dbml-syntax.peggy'); + +export interface SpecToken { + token: string; + value: string; +} + +export interface SpecNode { + kind: string; + children: (SpecNode | SpecToken)[]; +} + +export interface SpecParser { + // Short notation name used in test titles: 'peggy' or 'antlr'. + name: string; + parse (input: string): SpecNode; +} + +export function tok (kind: string, value: string): SpecToken { + return { token: kind, value }; +} + +type Child = SpecNode | SpecToken | Child[] | null | undefined; + +function flattenChildren (children: Child[]): (SpecNode | SpecToken)[] { + return children.flatMap((c) => { + if (c === null || c === undefined) return []; + if (Array.isArray(c)) return flattenChildren(c); + return [c]; + }); +} + +export function node (kind: string, ...children: Child[]): SpecNode { + return { + kind, + children: flattenChildren(children), + }; +} + +function isIdentifierPrimary (n: SpecNode | SpecToken): n is SpecNode { + if ('token' in n || n.kind !== '') return false; + const variable = n.children[0]; + if ('token' in variable || variable.kind !== '') return false; + const token = variable.children[0]; + return 'token' in token && token.token === ''; +} + +function identifierToken (n: SpecNode): SpecToken { + return (n.children[0] as SpecNode).children[0] as SpecToken; +} + +// parser/utils.ts: convertFuncAppToElem() +// A function application of the form [ [as ]] [] +// is reinterpreted as a nested element declaration. The peggy grammar carries +// its own copy of this rewrite in its actions; the ANTLR converter uses this one. +export function convertFuncAppToElem (callee: SpecNode, args: SpecNode[]): SpecNode | null { + let type = callee; + let rest = args; + if (type.kind === '') { + rest = [type.children[1] as SpecNode, ...rest]; + type = type.children[0] as SpecNode; + } + if (!isIdentifierPrimary(type) || rest.length === 0) return null; + const typeToken = identifierToken(type); + rest = [...rest]; + const body = rest.pop()!; + if (body.kind !== '') return null; + const attributeList = rest.length > 0 && rest[rest.length - 1].kind === '' ? rest.pop() : null; + if (rest.length === 3) { + if (!isIdentifierPrimary(rest[1]) || identifierToken(rest[1]).value.toLowerCase() !== 'as') return null; + return node('', typeToken, rest[0], identifierToken(rest[1]), rest[2], attributeList, body); + } + if (rest.length === 1) return node('', typeToken, rest[0], attributeList, body); + if (rest.length === 0) return node('', typeToken, attributeList, body); + return null; +} + +export type Verdict = 'accept' | 'reject'; + +export interface Outcome { + verdict: Verdict; + // Normalised tree, only when the verdict is `accept`. + tree?: string; + // Human-readable reason for a `reject`. + detail?: string; +} + +export interface Comparison { + reference: Outcome; + spec: Outcome; + agreeOnVerdict: boolean; + // Undefined when at least one side rejected. + agreeOnTree?: boolean; +} + +export function loadPeggyParser (): SpecParser { + const grammar = readFileSync(GRAMMAR_PATH, 'utf-8'); + const generated = peggy.generate(grammar, { + output: 'parser', + grammarSource: GRAMMAR_PATH, + }) as unknown as { parse (input: string): SpecNode }; + return { + name: 'peggy', + parse: (input) => generated.parse(input), + }; +} + +// Both notations of the spec. Kept in a separate module so that the ANTLR +// toolchain is only loaded when needed. +export async function loadSpecParsers (): Promise { + const { loadAntlrParser } = await import('./antlr'); + return [loadPeggyParser(), await loadAntlrParser()]; +} + +function stripAngles (kind: string): string { + return kind.replace(/^<|>$/g, ''); +} + +export function normalizeReference (node: SyntaxNode): string { + const children = getMemberChain(node).map((child) => ( + child instanceof SyntaxToken + ? `#${stripAngles(child.kind)}` + : normalizeReference(child as SyntaxNode) + )); + return `(${[stripAngles(node.kind), ...children].join(' ')})`; +} + +export function normalizeSpec (node: SpecNode): string { + const children = node.children.map((child) => ( + 'token' in child + ? `#${stripAngles(child.token)}` + : normalizeSpec(child) + )); + return `(${[stripAngles(node.kind), ...children].join(' ')})`; +} + +export function runReference (source: string): Outcome { + const report = parse(source); + const errors = report.getErrors(); + if (errors.length > 0) { + return { + verdict: 'reject', + detail: errors.map((e) => `${e.code} ${e.diagnostic} @${e.start}`).join('; '), + }; + } + return { + verdict: 'accept', + tree: normalizeReference(report.getValue().ast), + }; +} + +export function runSpec (parser: SpecParser, source: string): Outcome { + try { + return { + verdict: 'accept', + tree: normalizeSpec(parser.parse(source)), + }; + } catch (e) { + return { + verdict: 'reject', + detail: e instanceof Error ? e.message.split('\n')[0] : String(e), + }; + } +} + +export function compare (parser: SpecParser, source: string): Comparison { + const reference = runReference(source); + const spec = runSpec(parser, source); + const agreeOnVerdict = reference.verdict === spec.verdict; + const agreeOnTree = reference.verdict === 'accept' && spec.verdict === 'accept' + ? reference.tree === spec.tree + : undefined; + return { + reference, + spec, + agreeOnVerdict, + agreeOnTree, + }; +} + +// Pretty-print a comparison for assertion messages. +export function describeComparison (source: string, c: Comparison): string { + const lines = [ + `reference: ${c.reference.verdict}${c.reference.detail ? ` (${c.reference.detail})` : ''}`, + `spec: ${c.spec.verdict}${c.spec.detail ? ` (${c.spec.detail})` : ''}`, + ]; + if (c.agreeOnTree === false) { + lines.push(`reference tree: ${c.reference.tree}`, `spec tree: ${c.spec.tree}`); + } + lines.push('source:', source); + return lines.join('\n'); +} diff --git a/packages/dbml-parse/__tests__/conformance/properties.test.ts b/packages/dbml-parse/__tests__/conformance/properties.test.ts new file mode 100644 index 000000000..ab5cf1c80 --- /dev/null +++ b/packages/dbml-parse/__tests__/conformance/properties.test.ts @@ -0,0 +1,115 @@ +// Layer 1 conformance: property-based agreement between the spec grammar and +// the reference parser over the generators in __tests__/utils/arbitraries. +// +// Every generated source is parsed by both sides; they must agree on +// accept/reject and, when both accept, on the shape of the syntax tree. +import { describe, expect, it } from 'vitest'; +import * as fc from 'fast-check'; +import { + anyRefArbitrary, + binaryGarbageArbitrary, + crlfSchemaArbitrary, + dbmlSchemaArbitrary, + enumArbitrary, + malformedEnumArbitrary, + malformedRefArbitrary, + malformedTableArbitrary, + mismatchedBracketsArbitrary, + partialInjectionArbitrary, + projectArbitrary, + standaloneNoteArbitrary, + tableArbitrary, + tableGroupArbitrary, + tablePartialArbitrary, + tokenStreamArbitrary, + truncatedInputArbitrary, + unclosedBracketArbitrary, + unclosedStringArbitrary, +} from '../utils/arbitraries'; +import { + type SpecParser, + compare, + describeComparison, + loadSpecParsers, +} from './harness'; + +const SCHEMA_RUNS = 500; +const ELEMENT_RUNS = 100; +const MALFORMED_RUNS = 100; + +function expectAgreement (parser: SpecParser, source: string): void { + const result = compare(parser, source); + expect(result.agreeOnVerdict, describeComparison(source, result)).toBe(true); + if (result.agreeOnTree !== undefined) { + expect(result.agreeOnTree, describeComparison(source, result)).toBe(true); + } +} + +const parsers = await loadSpecParsers(); + +describe.each(parsers)('[conformance] properties ($name)', (parser) => { + it('agrees on whole schemas', { + timeout: 300000, + }, () => { + fc.assert( + fc.property(dbmlSchemaArbitrary, (source: string) => expectAgreement(parser, source)), + { + numRuns: SCHEMA_RUNS, + }, + ); + }); + + it('agrees on schemas with CRLF line endings', () => { + fc.assert( + fc.property(crlfSchemaArbitrary, (source: string) => expectAgreement(parser, source)), + { + numRuns: ELEMENT_RUNS, + }, + ); + }); + + const elementArbitraries: [string, fc.Arbitrary][] = [ + ['tables', tableArbitrary], + ['enums', enumArbitrary], + ['refs', anyRefArbitrary], + ['table groups', tableGroupArbitrary], + ['table partials', tablePartialArbitrary], + ['partial injections', partialInjectionArbitrary], + ['projects', projectArbitrary], + ['standalone notes', standaloneNoteArbitrary], + ]; + + elementArbitraries.forEach(([name, arbitrary]) => { + it(`agrees on ${name}`, () => { + fc.assert( + fc.property(arbitrary, (source: string) => expectAgreement(parser, source)), + { + numRuns: ELEMENT_RUNS, + }, + ); + }); + }); + + const malformedArbitraries: [string, fc.Arbitrary][] = [ + ['malformed tables', malformedTableArbitrary], + ['malformed enums', malformedEnumArbitrary], + ['malformed refs', malformedRefArbitrary], + ['unclosed brackets', unclosedBracketArbitrary], + ['mismatched brackets', mismatchedBracketsArbitrary], + ['unclosed strings', unclosedStringArbitrary], + ['truncated input', truncatedInputArbitrary], + ['binary garbage', binaryGarbageArbitrary], + ['raw token streams', tokenStreamArbitrary], + ]; + + malformedArbitraries.forEach(([name, arbitrary]) => { + it(`agrees on ${name}`, () => { + fc.assert( + fc.property(arbitrary, (source: string) => expectAgreement(parser, source)), + { + numRuns: MALFORMED_RUNS, + }, + ); + }); + }); +}); diff --git a/packages/dbml-parse/__tests__/conformance/syntax.test.ts b/packages/dbml-parse/__tests__/conformance/syntax.test.ts new file mode 100644 index 000000000..259561e78 --- /dev/null +++ b/packages/dbml-parse/__tests__/conformance/syntax.test.ts @@ -0,0 +1,163 @@ +// Layer 1 conformance: the spec grammar and the reference parser must agree on +// every snapshot input in __tests__/snapshots/**/input/*.dbml. +// +// Disagreements are not silently tolerated. Each one is pinned below with the +// outcome of both sides and documented in spec/DISAGREEMENTS.md. If either +// side changes behaviour, the pinned expectation fails and the entry has to be +// revisited. +import { readdirSync, readFileSync } from 'node:fs'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { + type Verdict, + compare, + describeComparison, + loadSpecParsers, +} from './harness'; + +interface KnownDisagreement { + reference: Verdict; + spec: Verdict; + // Entry in spec/DISAGREEMENTS.md + see: string; +} + +// Snapshot inputs that disagree, keyed by `/`. +const KNOWN_DISAGREEMENTS: Record = { + 'parser/last_invalid_number': { + reference: 'reject', + spec: 'accept', + see: 'D1', + }, +}; + +// Inline inputs for the disagreements that the snapshot corpus does not reach. +// `treesAgree: false` pins a case where both sides accept but build different +// trees. +const PINNED_DISAGREEMENTS: (KnownDisagreement & { source: string; treesAgree?: boolean })[] = [ + { + see: 'D1', + source: 'Note: 12.', + reference: 'reject', + spec: 'accept', + }, + { + see: 'D2', + source: 'Note: 1a', + reference: 'accept', + spec: 'accept', + treesAgree: false, + }, + { + see: 'D3', + source: 'Note: 1.a', + reference: 'accept', + spec: 'reject', + }, + { + see: 'D4', + source: "Note: '\\uzzzz'", + reference: 'accept', + spec: 'reject', + }, + { + see: 'D5', + source: 'Table \u{2000B} { id int }', + reference: 'reject', + spec: 'accept', + }, +]; + +// Behaviour that both sides share and that spec/DISAGREEMENTS.md calls out as +// surprising. Pinned so that the description stays truthful. +const AGREED_QUIRKS: [string, string][] = [ + ['operator continues previous line', 'Table t {\n (a, b) [pk]\n -2()\n}'], + ['index across a line break', 'Table t {\n a int\n[note: 1]\n}'], + ['trailing space defeats index across a line break', 'Table t {\n a int \n[note: 1]\n}'], + ['semicolon is never valid', 'Table t { id int; }'], + ['comment alone does not separate arguments', 'Table t { a/**/int }'], + ['comment with space separates arguments', 'Table t { a /**/int }'], + ['carriage return does not separate arguments', 'Table t { a\rint }'], + ['non-hex colour literal', 'Table t [headercolor: #zzz] { id int }'], + ['bare colour literal', 'Table t [headercolor: #] { id int }'], + ['use never falls back to an element', 'use x {}'], + ['simple body swallows the line', "Note: 'x' Table y {}"], + ['trailing-dot number mid-file', 'Note: 12.\n'], + ['digit-leading identifier mid-file', 'Note: 1a\n'], + ['BMP letters in identifiers', 'Table \u65E5\u672C { id int }'], +]; + +const SNAPSHOT_ROOT = path.resolve(__dirname, '../snapshots'); + +function listInputs (): { key: string; file: string }[] { + return readdirSync(SNAPSHOT_ROOT, { withFileTypes: true }) + .filter((d) => d.isDirectory()) + .flatMap((dir) => { + const inputDir = path.join(SNAPSHOT_ROOT, dir.name, 'input'); + return readdirSync(inputDir) + .filter((f) => f.endsWith('.in.dbml')) + .map((f) => ({ + key: `${dir.name}/${f.replace(/\.in\.dbml$/, '')}`, + file: path.join(inputDir, f), + })); + }) + .sort((a, b) => a.key.localeCompare(b.key)); +} + +const parsers = await loadSpecParsers(); + +describe.each(parsers)('[conformance] syntax ($name)', (parser) => { + const inputs = listInputs(); + + it('covers every snapshot input', () => { + expect(inputs.length).toBeGreaterThan(0); + }); + + it('pins only disagreements that still have an input file', () => { + const keys = new Set(inputs.map((i) => i.key)); + Object.keys(KNOWN_DISAGREEMENTS).forEach((key) => expect(keys.has(key), key).toBe(true)); + }); + + inputs.forEach(({ key, file }) => { + it(key, () => { + const source = readFileSync(file, 'utf-8'); + const result = compare(parser, source); + const known = KNOWN_DISAGREEMENTS[key]; + + if (known) { + expect(result.reference.verdict, `reference verdict changed; see DISAGREEMENTS.md ${known.see}\n${describeComparison(source, result)}`).toBe(known.reference); + expect(result.spec.verdict, `spec verdict changed; see DISAGREEMENTS.md ${known.see}\n${describeComparison(source, result)}`).toBe(known.spec); + return; + } + + expect(result.agreeOnVerdict, describeComparison(source, result)).toBe(true); + if (result.agreeOnTree !== undefined) { + expect(result.agreeOnTree, describeComparison(source, result)).toBe(true); + } + }); + }); +}); + +describe.each(parsers)('[conformance] recorded disagreements ($name)', (parser) => { + PINNED_DISAGREEMENTS.forEach((pinned) => { + it(`${pinned.see}: ${JSON.stringify(pinned.source)}`, () => { + const result = compare(parser, pinned.source); + const message = `behaviour changed; revisit DISAGREEMENTS.md ${pinned.see}\n${describeComparison(pinned.source, result)}`; + expect(result.reference.verdict, message).toBe(pinned.reference); + expect(result.spec.verdict, message).toBe(pinned.spec); + if (pinned.treesAgree !== undefined) { + expect(result.agreeOnTree, message).toBe(pinned.treesAgree); + } + }); + }); + + AGREED_QUIRKS.forEach(([name, source]) => { + it(`agreed: ${name}`, () => { + const result = compare(parser, source); + expect(result.agreeOnVerdict, describeComparison(source, result)).toBe(true); + if (result.agreeOnTree !== undefined) { + expect(result.agreeOnTree, describeComparison(source, result)).toBe(true); + } + }); + }); +}); diff --git a/packages/dbml-parse/eslint.config.ts b/packages/dbml-parse/eslint.config.ts index 3e802863b..35849c899 100644 --- a/packages/dbml-parse/eslint.config.ts +++ b/packages/dbml-parse/eslint.config.ts @@ -21,6 +21,8 @@ export default defineConfig( 'dist-profile/*', 'vite.config.ts', 'vite.profile.config.ts', + 'vitest.conformance.config.ts', + '__tests__/conformance/generated/*', 'eslint.config.ts', ], }, diff --git a/packages/dbml-parse/package.json b/packages/dbml-parse/package.json index 1543bc892..bd42ac340 100644 --- a/packages/dbml-parse/package.json +++ b/packages/dbml-parse/package.json @@ -36,11 +36,15 @@ "lint:fix": "eslint --fix .", "bench": "npx tsx __benchmarks__/compiler.benchmark.ts", "profile:build": "vite build --config vite.profile.config.mts && cp -r __benchmarks__/input dist-profile/input", - "profile": "yarn profile:build && node --cpu-prof --cpu-prof-dir __benchmarks__/output/ dist-profile/__benchmarks__/compiler.profile.mjs" + "profile": "yarn profile:build && node --cpu-prof --cpu-prof-dir __benchmarks__/output/ dist-profile/__benchmarks__/compiler.profile.mjs", + "test:conformance": "vitest run --config vitest.conformance.config.ts" }, "devDependencies": { "@types/luxon": "^3.7.1", + "antlr-ng": "^1.0.10", + "antlr4": "^4.13.2", "monaco-editor-core": "^0.44.0", + "peggy": "^5.1.0", "vite-plugin-no-bundle": "^4.0.0" }, "dependencies": { diff --git a/packages/dbml-parse/vite.config.ts b/packages/dbml-parse/vite.config.ts index 7f4c2c896..0c3875d08 100644 --- a/packages/dbml-parse/vite.config.ts +++ b/packages/dbml-parse/vite.config.ts @@ -3,6 +3,7 @@ import path from 'path'; import { defineConfig } from 'vite'; import dts from 'vite-plugin-dts'; +import { configDefaults } from 'vitest/config'; export default defineConfig({ plugins: [ @@ -33,6 +34,8 @@ export default defineConfig({ }, test: { globals: true, + // Spec conformance tests run separately via vitest.conformance.config.ts + exclude: [...configDefaults.exclude, '__tests__/conformance/**'], coverage: { provider: 'v8', reporter: ['json-summary', 'text'], diff --git a/packages/dbml-parse/vitest.conformance.config.ts b/packages/dbml-parse/vitest.conformance.config.ts new file mode 100644 index 000000000..b17586d8e --- /dev/null +++ b/packages/dbml-parse/vitest.conformance.config.ts @@ -0,0 +1,23 @@ +/// + +// Runs only the spec conformance tests (see spec/README.md). They are kept out +// of the default `vitest run` so that they can be non-blocking in CI while the +// spec is being brought into agreement with the parser. +import path from 'path'; +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + resolve: { + alias: { + '@': path.resolve(__dirname, 'src/'), + '@tests': path.resolve(__dirname, '__tests__/'), + }, + }, + test: { + globals: true, + include: ['__tests__/conformance/**/*.test.ts'], + // Compiles spec/antlr/*.g4 into __tests__/conformance/generated (not committed). + globalSetup: ['__tests__/conformance/antlr.setup.ts'], + testTimeout: 120000, + }, +}); diff --git a/spec/DISAGREEMENTS.md b/spec/DISAGREEMENTS.md new file mode 100644 index 000000000..4b9bd9cbc --- /dev/null +++ b/spec/DISAGREEMENTS.md @@ -0,0 +1,58 @@ +# Disagreements between the spec and the reference parser + +Every case where the spec grammar and `@dbml/parse` disagree, with a verdict. The spec exists in two notations (`spec/dbml-syntax.peggy` and `spec/antlr/*.g4`); both are checked and both produce the same outcomes below. Entries are pinned by `packages/dbml-parse/__tests__/conformance/syntax.test.ts` for each notation, so a change on any side fails a test and forces this file to be updated. + +Verdicts: **spec bug** (fix the spec), **parser bug** (fix the parser; entry stays until then), **intentional leniency** (parser accepts more on purpose; the spec documents the strict form). + +## Summary + +| ID | Input | Reference | Spec | Verdict | +| --- | --- | --- | --- | --- | +| D1 | `Note: 12.` with no trailing newline | reject (`Invalid number`) | accept | parser bug | +| D2 | `Note: 1a` with no trailing newline | accept, `1a` is a number literal | accept, `1a` is an identifier | parser bug | +| D3 | `Note: 1.a` with no trailing newline | accept, `1.a` is a number literal | reject | parser bug | +| D4 | `'\uzzzz'` (four non-hex alphanumerics after `\u`) | accept | reject | parser bug | +| D5 | Identifier containing a letter outside the Basic Multilingual Plane, e.g. `Table 𠀋 {}` | reject (`Unexpected token`) | accept | parser bug | + +The snapshot corpus exercises D1 only (`snapshots/parser/last_invalid_number`). D2 to D5 are pinned with inline inputs in the same test file. + +## D1 to D3: digit-leading token at end of input + +`Lexer.numericLiteralOrIdentifier()` has two end-of-input special cases that disagree with its behaviour everywhere else in the file: + +- When the text ends right after a trailing dot (`12.`), the scanning loop exits at end of input with `nDots > 0` and reports `Invalid number`. The same token followed by a space, newline or bracket is a valid number, and the property generators in `__tests__/utils/arbitraries/tokens.ts` produce `\d+\.` deliberately. +- When the character after the digits is the last character of the file (`1a`, `1.a`), the branch `this.current.offset === this.text.length - 1` consumes it and emits a number literal, even though it is a letter. Anywhere else `1a` is an identifier and `1.a` is an invalid number. + +The spec follows the mid-file behaviour: `digits . digits?` is a number, `digits letters` is an identifier, `digits . letters` is an error. + +Status: open. Fix belongs in `lexer.ts`; the `last_invalid_number` snapshot output will change when it lands. + +## D4: `\u` escape accepts non-hex characters + +`Lexer.escapedString()` reads four characters after `\u` and only checks `isAlphaNumeric`, then calls `parseInt(hex, 16)` which yields `NaN` for `zzzz` and silently produces a U+0000 character. The error message in the same function says the form must be `\uHHHH` with hexadecimal `H`, which is what the spec requires. + +Status: open. The generators only produce hex escapes, so no existing test depends on the lenient form. + +## D5: letters outside the Basic Multilingual Plane + +The lexer walks UTF-16 code units (`this.text[offset]`) and tests each unit against `/\p{L}|\p{M}|_/u`. A supplementary-plane letter such as `𠀋` (U+2000B) is two surrogate code units, neither of which matches, so the lexer reports two `Unexpected token` errors. `isAlphaOrUnderscore()` clearly intends to accept every Unicode letter, so the spec is written over code points and accepts these identifiers. + +Status: open. The `unicode_identifiers` snapshot contains only BMP characters and is unaffected. + +## Agreed, but worth knowing + +These are not disagreements. Both parsers behave the same way, and the behaviour is surprising enough that a spec reader should be warned. The grammar reproduces each one deliberately. + +- A line that starts with an infix operator continues the expression on the previous line. In a table body, `(a, b) [pk]` followed by a line `-2()` parses as the single expression `[pk] - 2()`. +- A `[` at the start of a line with no space before the preceding line break is array indexing of the previous expression (`int` newline `[note: 1]` is `int[note: 1]`). A trailing space before the line break turns it into a new item. +- A semicolon is a token but is never valid anywhere; any `;` is a syntax error. +- Comments do not separate function-application arguments. `a/**/int` is an error (missing space); `a /**/int` is fine. +- A carriage return is dropped by the lexer and is neither whitespace nor a line break. `aint` is an error (missing space). +- Colour literals are `#` followed by any alphanumerics at this layer; `#zzz` and a bare `#` are syntactically valid. Hex validation happens in Layer 2. +- A statement starting with the identifier `use` or `reuse` is always parsed as a use declaration; `use x {}` is an error, not an element named `x`. +- A top-level simple body swallows the rest of its line. `Note: 'x' Table y {}` is one `Note` element whose body is the function application `'x' Table y {}`. + +## Harness limitations + +- The peggy-generated parser overflows the JavaScript stack at roughly 1000 nested parentheses; the reference parser copes with about twice that. `extremeNestingArbitrary` is therefore excluded from the property test. This is a limitation of the executable spec, not a statement about the language. +- The ANTLR parser reports the position of the first error only, because it runs with a bail-out error strategy; the reference reports every error it can recover from. Only accept/reject and the tree are compared, so this does not affect conformance. diff --git a/spec/README.md b/spec/README.md new file mode 100644 index 000000000..fd408b329 --- /dev/null +++ b/spec/README.md @@ -0,0 +1,103 @@ +# DBML formal specification + +This directory holds the formal specification of DBML. It is organised in the same three layers as the reference implementation in [`packages/dbml-parse`](../packages/dbml-parse), so that every statement in the spec can be checked mechanically against the code. + +| Layer | What it specifies | Spec artefact | Reference implementation | +| --- | --- | --- | --- | +| 1. Generic syntax | Tokens and the element-agnostic syntax tree (`SyntaxNodeKind`) | [`dbml-syntax.peggy`](./dbml-syntax.peggy), with an ANTLR rendering in [`antlr/`](./antlr) | `src/core/lexer/lexer.ts`, `src/core/parser/parser.ts` | +| 2. Element semantics | Per-element rules: name shape, body form, allowed settings and sub-elements | `elements/.schema.json` (future PRs) | `src/core/local_modules//validate.ts` | +| 3. Interpreted model | The `Database` JSON model produced by `@dbml/core` | `output/database.schema.json` (future PR) | `packages/dbml-core/src/model_structure/*.ts` | + +## The authority rule + +The reference parser is the authority on what DBML *is* today. The spec is the authority on what DBML *should be*. Whenever the two disagree, the disagreement is recorded in [`DISAGREEMENTS.md`](./DISAGREEMENTS.md) with one of three verdicts: + +- **spec bug**: the spec is wrong and must be changed. +- **parser bug**: the parser is wrong; the spec stands and the entry stays until the parser is fixed. +- **intentional leniency**: the parser accepts more than the spec on purpose (for example, to give better error recovery). The spec documents the strict form. + +Conformance tests in `packages/dbml-parse/__tests__/conformance/` run both the reference parser and a parser generated from the spec over the same inputs and fail on any disagreement that is not recorded with a verdict. "Accept" means the reference lexer and parser report no error, and the spec parser parses without throwing. When both accept, the trees must also match: both are flattened to the same list of node kinds and token kinds in source order. The tests are currently non-blocking in CI; they become blocking once the corpus is clean. + +Nothing under `packages/dbml-parse/src` changes because of the spec. The spec restates behaviour; it does not alter it. + +## Layer 1 in one page + +DBML source is parsed in two passes. A lexer turns UTF-16 code units into tokens and attaches whitespace and comments to neighbouring tokens as *trivia*. A recursive-descent parser then builds a small, element-agnostic tree. Element keywords such as `Table` or `Ref` are ordinary identifiers at this layer; nothing in Layer 1 knows what a table is. + +### Tokens + +- **Whitespace and comments** are trivia: space, tab, newline, `// ...` to end of line, and `/* ... */`. A carriage return is dropped entirely. Trivia after a token on the same line, up to and including the first newline, is that token's *trailing trivia*; trivia at the start of a line is the next token's *leading trivia*. The parser consults trivia in exactly three places, described below. +- **Identifiers** start with a Unicode letter, combining mark, or underscore and continue with those or digits. A token that starts with a digit is an identifier when it contains letters or underscores and is not a well-formed number (`12_abc`, `3a`). +- **Numbers** are `digits`, `digits . digits?`, optionally followed by an exponent `e[+-]?digits`. A number followed directly by a letter is an identifier (`1e5x`); a number with two dots or with a dot followed by letters is a lexical error (`1.2.3`, `1.a`). +- **Strings** are single-quoted `'...'` on one line or triple-quoted `'''...'''` across lines. Backslash escapes are processed, including `\uHHHH` and backslash-newline for line continuation. A newline inside a single-quoted string is an error. +- **Quoted variables** `"..."` are identifiers with arbitrary content; they follow the single-quoted string rules for escapes and newlines. +- **Function expressions** `` `...` `` are raw: no escapes, newlines allowed, ends at the first backtick. +- **Colour literals** are `#` followed by alphanumerics. Layer 1 does not check that the digits are hexadecimal. +- **Wildcard** is `*`. +- **Punctuation**: `( ) [ ] { } , :`. A semicolon is also a token, but no rule accepts it, so `;` is always an error. +- **Operators**, scanned greedily as a prefix-closed set: `. / % + - = == != ! ~ & | ?` and the relationship family `< <= > >= <> <- -> -? ?- ?-? ? ?> ?? <>? ?<> ?<>?`. + +Anything else is a lexical error. + +### Program structure + +A program is a sequence of top-level statements with no separators required between them. + +- **Use declaration**: `use` or `reuse` (case-insensitive), then either `*` or a braced list of newline-separated specifiers ` [as ]`, then `from` and a string literal. +- **Element declaration**: ` [] [] [as ] [] (: | )`. `` is an identifier; `` is present only when `` is `metadata`. `` and `` are normal expressions. A simple body is a function application and must not itself resolve to an element declaration. + +### Blocks + +A block `{ ... }` contains body items. An item that starts with ` :` is a *field declaration*: an element declaration with a simple body (`Note: 'text'`). Any other item is a *function application*: a callee expression followed by zero or more argument expressions on the same line, each separated by at least one space or tab. When a function application has the shape ` [ [as ]] [] `, it is reinterpreted as a nested element declaration. Otherwise it stays a function application; `id integer [pk]` is the callee `id` applied to `integer` and `[pk]`. + +A function application stops at a newline, at end of input, or before `}`, `]`, `)`, `,`, or `:`. + +### Expressions + +Each callee and argument is a *comma expression*: one or more normal expressions separated by commas, where an empty slot (`a, , c` or a trailing comma) yields an empty node. A comma continues the list only on the same line and not before a bracket or colon. + +A *normal expression* is a Pratt-style operator expression over these operands: + +- primary: a literal (number, string, colour) or a variable (identifier, quoted variable) +- function expression, wildcard +- list `[ attribute, ... ]`, block `{ ... }`, tuple `( expr, ... )`; a tuple with exactly one element is a *group* + +Binding powers, tightest first: member access `.` (its right operand is a bare operand, never a prefix expression), prefix operators (`+ - ! ~` and every relationship operator), call `( ... )`, multiplicative `/ %`, additive `+ - -? ?- ?-?`, comparison (all relationship operators, including `->` and `<-`), equality `== !=`, assignment `=`. All infix operators are left-associative. Newlines are permitted around infix operators. + +The three trivia-sensitive rules: + +1. `[` directly after an operand is array indexing (`int[]`) only when no space or tab separates them on that line; otherwise it starts a new argument (`int [pk]`). +2. `(` directly after an operand is a call unless it begins a new line. Inside an unclosed `(` or `[` the newline rule is suspended, so `(f\n(1))` is a call. +3. Function-application arguments must be separated by at least one space or tab; `1 **2` is an error. + +### Attribute lists + +`[ attr, attr ]` where each attribute is ` [: ]`. A name is a stream of identifiers (`primary key`) or a single literal or quoted variable. A value is an identifier stream when it starts with two identifiers (`no action`), otherwise a normal expression (`> users.id`, `'text'`, `` `now()` ``). Empty names and trailing commas are errors. + +### Syntax node kinds + +The grammar produces exactly the `SyntaxNodeKind` set from `src/core/types/nodes.ts`: program, element-declaration, use-declaration, use-specifier, use-specifier-list, attribute, identifier-stream, literal, variable, primary-expression, prefix-expression, infix-expression, postfix-expression, function-expression, function-application, block-expression, list-expression, tuple-expression, group-expression, call-expression, comma-expression, array, wildcard, and the empty node. Postfix-expression is reachable only through error recovery today and has no grammar rule. + +## Notations + +The Layer 1 grammar exists in two notations that are kept in agreement by running both through the same conformance tests. This is deliberate: which notation the spec should standardise on is an open question, and the corpus gives an objective way to compare them. + +| | peggy ([`dbml-syntax.peggy`](./dbml-syntax.peggy)) | ANTLR 4 ([`antlr/DbmlLexer.g4`](./antlr/DbmlLexer.g4), [`antlr/DbmlParser.g4`](./antlr/DbmlParser.g4)) | +| --- | --- | --- | +| Model | Scannerless PEG; ordered choice mirrors the reference's recursive descent | Separate lexer and parser; ALL(*) prediction with predicates where the reference commits on one token | +| Trivia | Threaded explicitly through rules (`GapAny`, `GapInline`, `GapNewline`) | Hidden channel, inspected with `getHiddenTokensToLeft()`, close to the reference lexer's trivia model | +| Embedded code | About 116 lines of JavaScript in the initializer plus one-line tree-building actions on most rules; 15 semantic predicates | 71-line members block (50 lines of JavaScript), 17 predicates, 6 actions; no tree-building code | +| Tree shaping | In the grammar | In the test harness (`__tests__/conformance/antlr.ts`, about 170 lines), because ANTLR yields a rule-shaped parse tree | +| Toolchain | `peggy` (dev dependency), compiled in memory at test time | `antlr-ng` (Node port of the ANTLR tool, no Java) generating JavaScript for the `antlr4` runtime that `@dbml/core` already uses; generated in vitest global setup, not committed | +| Unicode | Code points via `[\p{L}]u` classes | Code points via `CharStream(source, true)` | +| Corpus result | 121 of 122 snapshot inputs agree, all property tests pass | Identical | + +Both notations reproduce the same recorded disagreements (see [`DISAGREEMENTS.md`](./DISAGREEMENTS.md)); the pinned cases run against each. + +## Running the conformance tests + +```bash +yarn workspace @dbml/parse test:conformance +``` + +Both grammars are compiled at test time. The peggy parser is built in memory; the ANTLR parser is generated into `packages/dbml-parse/__tests__/conformance/generated/`, which is ignored by git. No generated parser is committed. diff --git a/spec/antlr/DbmlLexer.g4 b/spec/antlr/DbmlLexer.g4 new file mode 100644 index 000000000..82657b266 --- /dev/null +++ b/spec/antlr/DbmlLexer.g4 @@ -0,0 +1,159 @@ +// DBML formal specification, Layer 1: generic syntax. ANTLR notation, lexer. +// +// This is the same specification as ../dbml-syntax.peggy expressed as a +// two-stage ANTLR 4 grammar (DbmlLexer.g4 + DbmlParser.g4), so that the two +// notations can be compared on the same conformance corpus. It restates +// packages/dbml-parse/src/core/lexer/lexer.ts. +// +// Trivia (whitespace and comments) goes to the hidden channel; the parser +// inspects it with getHiddenTokensToLeft(), which is the ANTLR equivalent of +// the reference lexer's trailing/leading trivia (lexer.ts: gatherTrivia()). +// The character stream must be opened with code-point decoding so that +// letters outside the Basic Multilingual Plane match \p{L}. + +lexer grammar DbmlLexer; + +// --------------------------------------------------------------------------- +// Trivia (lexer.ts: scanTokens(), gatherTrivia()) +// --------------------------------------------------------------------------- +// A carriage return is discarded outright: it is neither whitespace nor a line +// break. A single-line comment does not include its terminating newline. + +NEWLINE: '\n' -> channel(HIDDEN); +SPACES: [ \t]+ -> channel(HIDDEN); +CR: '\r' -> skip; +LINE_COMMENT: '//' ~[\n]* -> channel(HIDDEN); +BLOCK_COMMENT: '/*' .*? '*/' -> channel(HIDDEN); + +// lexer.ts: multilineComment() reports UNEXPECTED_EOF. Without this rule the +// lexer would fall back to SLASH WILDCARD and the parser could accept the file. +UNTERMINATED_BLOCK_COMMENT: '/*' (~'*' | '*'+ ~[*/])* '*'* EOF; + +// --------------------------------------------------------------------------- +// Keywords (utils/tokens.ts: isUseKeyword() etc.) +// --------------------------------------------------------------------------- +// Case-insensitive, and ordinary identifiers everywhere the parser does not +// look for them: the parser's `identifier` rule includes all of them. They must +// precede IDENTIFIER so that they win the longest-match tie. + +USE: [uU] [sS] [eE]; +REUSE: [rR] [eE] [uU] [sS] [eE]; +FROM: [fF] [rR] [oO] [mM]; +AS: [aA] [sS]; +METADATA: [mM] [eE] [tT] [aA] [dD] [aA] [tT] [aA]; + +// --------------------------------------------------------------------------- +// Punctuation +// --------------------------------------------------------------------------- + +LPAREN: '('; +RPAREN: ')'; +LBRACKET: '['; +RBRACKET: ']'; +LBRACE: '{'; +RBRACE: '}'; +COMMA: ','; +COLON: ':'; +// Lexed by the reference but accepted nowhere; any ';' is a syntax error. +SEMICOLON: ';'; +WILDCARD: '*'; + +// --------------------------------------------------------------------------- +// Operators (lexer.ts: operator()) +// --------------------------------------------------------------------------- +// The set is prefix-closed, so ANTLR's longest match reproduces the reference +// lexer's greedy scan. One token per operator lets the parser classify them +// without predicates. + +Q_LTGT_Q: '?<>?'; +Q_LTGT: '?<>'; +Q_LT_Q: '??'; +Q_GT: '?>'; +Q_MINUS_Q: '?-?'; +Q_MINUS: '?-'; +QUESTION: '?'; +LTGT_Q: '<>?'; +LTGT: '<>'; +LTE: '<='; +LT_Q: '='; +GT_Q: '>?'; +GT: '>'; +ARROW_RIGHT: '->'; +MINUS_Q: '-?'; +MINUS: '-'; +EQ: '=='; +ASSIGN: '='; +NEQ: '!='; +BANG: '!'; +PLUS: '+'; +SLASH: '/'; +PERCENT: '%'; +DOT: '.'; +AMP: '&'; +PIPE: '|'; +TILDE: '~'; + +// --------------------------------------------------------------------------- +// Literals and identifiers +// --------------------------------------------------------------------------- + +// lexer.ts: numericLiteralOrIdentifier(). Longest match decides between +// NUMBER, INVALID_NUMBER and IDENTIFIER; on a tie the first rule wins, so the +// order below matters: 1e5 is a NUMBER, 1e5x is an IDENTIFIER, 1.5x is an +// INVALID_NUMBER. +NUMBER + : DIGITS EXPONENT + | DIGITS '.' DIGITS? EXPONENT? + | DIGITS + ; + +// A dot part followed by letters, or a second dot, is a lexical error in the +// reference (UNKNOWN_TOKEN "Invalid number"). The parser has no rule for this +// token, so its presence rejects the file. +INVALID_NUMBER + : DIGITS '.' DIGITS? IDENT_START ALNUM_DOT* + | DIGITS '.' DIGITS? '.' ALNUM_DOT* + ; + +// lexer.ts: identifier(). A token that starts with a digit is an identifier +// when it contains a letter or underscore and is not a well-formed number. +IDENTIFIER + : IDENT_START ALNUM* + | DIGITS IDENT_START ALNUM* + ; + +// lexer.ts: singleLineStringLiteral(), multilineStringLiteral() +STRING + : '\'\'\'' (ESCAPE | ~'\\')*? '\'\'\'' + | '\'' (ESCAPE | ~['\\\n])* '\'' + ; + +// lexer.ts: quotedVariable() +QUOTED_VARIABLE: '"' (ESCAPE | ~["\\\n])* '"'; + +// lexer.ts: functionExpression(); raw text, no escapes, may span lines. +FUNCTION_EXPRESSION: '`' ~'`'* '`'; + +// lexer.ts: colorLiteral(); hex validation is a Layer 2 rule. +COLOR: '#' ALNUM*; + +// lexer.ts: escapedString(). A backslash always starts an escape. Any escaped +// character is accepted; \uHHHH needs four hexadecimal digits; a backslash +// before a line break continues the line. +fragment ESCAPE + : '\\u' HEX HEX HEX HEX + | '\\\r\n' + | '\\' ~'u' + ; + +fragment HEX: [0-9a-fA-F]; +fragment DIGITS: [0-9]+; +fragment EXPONENT: [eE] [+-]? DIGITS; +fragment IDENT_START: [\p{L}\p{M}_]; +fragment ALNUM: [\p{L}\p{M}_0-9]; +fragment ALNUM_DOT: [\p{L}\p{M}_0-9.]; diff --git a/spec/antlr/DbmlParser.g4 b/spec/antlr/DbmlParser.g4 new file mode 100644 index 000000000..8a3e5217a --- /dev/null +++ b/spec/antlr/DbmlParser.g4 @@ -0,0 +1,335 @@ +// DBML formal specification, Layer 1: generic syntax. ANTLR notation, parser. +// +// Restates packages/dbml-parse/src/core/parser/parser.ts. Rule comments name +// the reference method they correspond to. The parser is generated for the +// JavaScript target (the runtime @dbml/core already uses); the members block +// is therefore JavaScript and is emitted into the generated constructor. +// +// Three groups of semantic predicates carry what the notation cannot: +// 1. trivia-sensitive rules (index vs new argument, call vs new line, argument +// spacing, line-separated use specifiers), read from the hidden channel; +// 2. one-token commitments the reference makes with a single lookahead, so +// that ALL(*) does not find a second parse the reference never tries; +// 3. the bracket-depth context that suspends the call/new-line rule. +// One embedded action rejects a simple body that resolves to an element +// declaration (parser.ts: UNEXPECTED_ELEMENT_DECLARATION). + +parser grammar DbmlParser; + +options { tokenVocab = DbmlLexer; } + +@parser::members { + // parser/contextStack.ts: number of unclosed '(' and '['. Inside them a '(' + // that starts a new line still continues a call expression. + this.depth = 0; + + // Hidden-channel tokens between the previous token and the next one: the + // reference lexer's trailing/leading trivia (lexer.ts: gatherTrivia()). + this.gapAhead = () => { + const next = this._input.LT(1); + return this._input.getHiddenTokensToLeft(next.tokenIndex, antlr4.Token.HIDDEN_CHANNEL) ?? []; + }; + // lexer/utils.ts: hasTrailingNewLines(), isAtStartOfLine() + this.newlineBefore = () => this.gapAhead().some((t) => t.type === DbmlParser.NEWLINE); + // lexer/utils.ts: hasTrailingSpaces(): a space or tab before the first line break + this.spaceBefore = () => { + for (const t of this.gapAhead()) { + if (t.type === DbmlParser.NEWLINE) return false; + if (t.type === DbmlParser.SPACES) return true; + } + return false; + }; + + this.ahead = (k) => this._input.LT(k).type; + this.isIdentifierToken = (type) => [ + DbmlParser.IDENTIFIER, DbmlParser.USE, DbmlParser.REUSE, DbmlParser.FROM, DbmlParser.AS, DbmlParser.METADATA, + ].includes(type); + + // parser.ts: elementDeclaration(): a name or alias is read unless ':' '{' or '[' follows + this.bodyStartAhead = () => [DbmlParser.COLON, DbmlParser.LBRACE, DbmlParser.LBRACKET].includes(this.ahead(1)); + // parser.ts: useSpecifier(): a name must start with an identifier or quoted variable + this.nameStartAhead = () => this.isIdentifierToken(this.ahead(1)) || this.ahead(1) === DbmlParser.QUOTED_VARIABLE; + // parser.ts: canBeField() + this.fieldAhead = () => this.isIdentifierToken(this.ahead(1)) && this.ahead(2) === DbmlParser.COLON; + // parser.ts: attributeValue(): two leading identifiers make an identifier stream + this.twoIdentifiersAhead = () => this.isIdentifierToken(this.ahead(1)) && this.isIdentifierToken(this.ahead(2)); + // parser.ts: expression(): arguments sit on the same line, after a space, and never start with a comma + this.argAhead = () => this.spaceBefore() && !this.newlineBefore() && this.ahead(1) !== DbmlParser.COMMA; + // parser.ts: shouldStopFunctionApplication() + this.functionApplicationStops = () => this.ahead(1) === antlr4.Token.EOF || this.newlineBefore() + || [DbmlParser.RBRACE, DbmlParser.RBRACKET, DbmlParser.RPAREN, DbmlParser.COMMA, DbmlParser.COLON].includes(this.ahead(1)); + // parser.ts: shouldStopCommaExpression() + this.commaStops = () => this.ahead(1) === antlr4.Token.EOF || this.newlineBefore() + || [DbmlParser.LBRACE, DbmlParser.RBRACE, DbmlParser.LBRACKET, DbmlParser.RBRACKET, DbmlParser.LPAREN, DbmlParser.RPAREN, DbmlParser.COLON].includes(this.ahead(1)); + + // parser/utils.ts: convertFuncAppToElem(), shape test only. A function + // application of the form [ [as ]] [] + // is an element declaration and may not be a simple body. + this.unwrap = (ctx) => { + while (ctx && ctx.children && ctx.children.length === 1 && ctx.children[0].children) ctx = ctx.children[0]; + return ctx; + }; + this.isElementShaped = (exprCtx) => { + const parts = exprCtx.commaExpression(); + let callee = this.unwrap(parts[0]); + let args = parts.slice(1); + if (callee instanceof DbmlParser.PostfixExpressionContext) { + if (callee.callStep().length !== 1 || callee.chainStep().length !== 0) return false; + args = [callee.callStep(0).tupleExpression(), ...args]; + callee = this.unwrap(callee.unaryExpression()); + } + if (!(callee instanceof DbmlParser.IdentifierContext) || args.length === 0) return false; + const rest = args.slice(); + if (!(this.unwrap(rest.pop()) instanceof DbmlParser.BlockExpressionContext)) return false; + if (rest.length > 0 && this.unwrap(rest[rest.length - 1]) instanceof DbmlParser.ListExpressionContext) rest.pop(); + if (rest.length === 3) { + const as = this.unwrap(rest[1]); + return as instanceof DbmlParser.IdentifierContext && as.getText().toLowerCase() === 'as'; + } + return rest.length <= 1; + }; +} + +// --------------------------------------------------------------------------- +// Program +// --------------------------------------------------------------------------- + +// parser.ts: program() +program: statement* EOF; + +// A statement starting with `use` or `reuse` is always a use declaration; +// elementDeclaration does not accept those tokens as a type. +statement + : useDeclaration + | elementDeclaration + ; + +// --------------------------------------------------------------------------- +// Use declarations +// --------------------------------------------------------------------------- + +// parser.ts: useDeclaration() +// use * from '' +// use { \n ... } from '' +useDeclaration: (USE | REUSE) (wildcard | useSpecifierList) FROM STRING; + +// parser.ts: useSpecifierList(); specifiers are separated by line breaks. +useSpecifierList: LBRACE useSpecifier? ({this.newlineBefore()}? useSpecifier)* RBRACE; + +// parser.ts: useSpecifier() +// [as ] +useSpecifier: identifier useName (AS useName)?; + +// The name must start with an identifier or quoted variable; it is then read +// as a full normal expression. +useName: {this.nameStartAhead()}? normalExpression; + +// --------------------------------------------------------------------------- +// Element declarations +// --------------------------------------------------------------------------- + +// parser.ts: elementDeclaration() +// [] [] [as ] [] (: | { }) +// is present exactly when is `metadata`. +elementDeclaration + : METADATA identifier elementTail + | (IDENTIFIER | FROM | AS) elementTail + ; + +// The reference decides on one token whether a name or alias is present; the +// predicates keep ALL(*) from choosing the other reading. +elementTail + : ({!this.bodyStartAhead()}? normalExpression | {this.bodyStartAhead()}?) + (AS {!this.bodyStartAhead()}? normalExpression)? + listExpression? + elementBody + ; + +elementBody + : COLON simpleBody + | blockExpression + ; + +// parser.ts: UNEXPECTED_ELEMENT_DECLARATION / INVALID_ELEMENT_IN_SIMPLE_BODY +simpleBody + : expression + { if (this.isElementShaped(localctx.expression())) this.notifyErrorListeners('a simple body must not be an element declaration'); } + ; + +// --------------------------------------------------------------------------- +// Blocks +// --------------------------------------------------------------------------- + +// parser.ts: blockExpression() +blockExpression: LBRACE bodyItem* RBRACE; + +// parser.ts: canBeField() decides between fieldDeclaration() and expression(). +bodyItem + : {this.fieldAhead()}? fieldDeclaration + | {!this.fieldAhead()}? expression + ; + +// parser.ts: fieldDeclaration() +// : +fieldDeclaration: identifier COLON simpleBody; + +// --------------------------------------------------------------------------- +// Function application +// --------------------------------------------------------------------------- + +// parser.ts: expression() +// A callee followed by arguments on the same line, each preceded by at least +// one space or tab. The application must end at a line break, end of input, or +// before } ] ) , : (parser.ts: MISSING_SPACES otherwise). +expression + : commaExpression ({this.argAhead()}? commaExpression)* {this.functionApplicationStops()}? + ; + +// parser.ts: commaExpression() +// CSV-like list without delimiters. Empty slots are recorded by the tree +// converter as nodes. +commaExpression + : normalExpression (COMMA commaRest)? + | COMMA commaRest + ; + +// After a comma: a line break, end of input, bracket or colon ends the list +// with an empty slot; another comma is an empty slot; otherwise a normal +// expression, which continues only if a comma follows on the same line. +// parser.ts: shouldStopCommaExpression() +commaRest + : {this.commaStops()}? + | {!this.commaStops()}? COMMA commaRest + | {!this.commaStops()}? normalExpression ({!this.newlineBefore()}? COMMA commaRest)? + ; + +// --------------------------------------------------------------------------- +// Normal expressions (parser.ts: expression_bp(), Pratt precedence climbing) +// --------------------------------------------------------------------------- +// Binding powers from parser.ts, tightest first: +// . 16 right operand is a bare operand +// prefix operators 15 +// call ( ... ) 14 +// / % 11 +// + - -? ?- ?-? 9 +// relationship ops 7 < <= > >= <> and the ? variants, -> <- +// == != 4 +// = 2 +// All infix operators are left-associative. Trivia, including line breaks, +// may surround infix operators. + +normalExpression: assignmentExpression; + +assignmentExpression: equalityExpression (ASSIGN equalityExpression)*; +equalityExpression: comparisonExpression ((EQ | NEQ) comparisonExpression)*; +comparisonExpression: additiveExpression (comparisonOp additiveExpression)*; +additiveExpression: multiplicativeExpression (additiveOp multiplicativeExpression)*; +multiplicativeExpression: postfixExpression ((SLASH | PERCENT) postfixExpression)*; + +// Calls applied to a unary expression; after a call, indexing and member +// access may continue the chain. +postfixExpression: unaryExpression (callStep chainStep*)*; + +// parser.ts: leftExpression_bp() +// A prefix operator binds tighter than a call: -2() is (-2)(). +unaryExpression + : prefixOp unaryExpression + | memberChain + ; + +// Member access and indexing on a bare operand. The right side of `.` is a +// bare operand (parser.ts: extractOperand()), never a prefix expression. +memberChain: operand chainStep*; + +chainStep + : indexStep + | memberStep + ; + +memberStep: DOT operand; + +// A '(' continues a call unless it starts a new line; inside an unclosed '(' +// or '[' the line-break rule is suspended. +callStep: {this.depth > 0 || !this.newlineBefore()}? tupleExpression; + +// A '[' is array indexing only when no space or tab separates it from the +// operand on the same line; otherwise it starts a new argument. +indexStep: {!this.spaceBefore()}? listExpression; + +// parser.ts: extractOperand() +operand + : wildcard + | primaryExpression + | functionExpression + | listExpression + | blockExpression + | tupleExpression + ; + +// parser.ts: primaryExpression() +primaryExpression + : literal + | variable + ; + +literal: STRING | NUMBER | COLOR; +variable: identifier | QUOTED_VARIABLE; +functionExpression: FUNCTION_EXPRESSION; +wildcard: WILDCARD; + +// parser.ts: tupleExpression(); one element without a comma is a group. +tupleExpression + : LPAREN {this.depth += 1;} (normalExpression (COMMA normalExpression)*)? RPAREN {this.depth -= 1;} + ; + +// --------------------------------------------------------------------------- +// Attribute lists +// --------------------------------------------------------------------------- + +// parser.ts: listExpression() +// [ , , ... ] +listExpression + : LBRACKET {this.depth += 1;} (attribute (COMMA attribute)*)? RBRACKET {this.depth -= 1;} + ; + +// parser.ts: attribute() +// [: ] +attribute: attributeName (COLON attributeValue)?; + +// parser.ts: attributeName() +// A stream of identifiers (primary key), or a single literal or quoted variable. +attributeName + : identifierStream + | literal + | QUOTED_VARIABLE + ; + +identifierStream: identifier+; + +// parser.ts: attributeValue() +attributeValue + : {this.twoIdentifiersAhead()}? identifierStream + | {!this.twoIdentifiersAhead()}? normalExpression + ; + +// --------------------------------------------------------------------------- +// Operators and identifiers +// --------------------------------------------------------------------------- + +// parser.ts: prefixBpMap +prefixOp + : PLUS | MINUS | LT | GT | LTGT | MINUS_Q | Q_MINUS | Q_MINUS_Q | Q_GT | GT_Q | Q_LT | LT_Q + | Q_GT_Q | Q_LT_Q | Q_LTGT | LTGT_Q | Q_LTGT_Q | ARROW_RIGHT | ARROW_LEFT | BANG | TILDE + ; + +// parser.ts: infixBpMap, Prec.Additive +additiveOp: PLUS | MINUS | MINUS_Q | Q_MINUS | Q_MINUS_Q; + +// parser.ts: infixBpMap, Prec.Comparison +comparisonOp + : LT | LTE | GT | GTE | LTGT | Q_GT | GT_Q | Q_LT | LT_Q | Q_GT_Q | Q_LT_Q | Q_LTGT | LTGT_Q | Q_LTGT_Q + | ARROW_RIGHT | ARROW_LEFT + ; + +// Keywords are ordinary identifiers wherever the parser is not looking for them. +identifier: IDENTIFIER | USE | REUSE | FROM | AS | METADATA; diff --git a/spec/dbml-syntax.peggy b/spec/dbml-syntax.peggy new file mode 100644 index 000000000..dba5c94c5 --- /dev/null +++ b/spec/dbml-syntax.peggy @@ -0,0 +1,551 @@ +// DBML formal specification, Layer 1: generic syntax. +// +// This grammar restates the lexer and parser in packages/dbml-parse +// (src/core/lexer/lexer.ts, src/core/parser/parser.ts). It accepts exactly +// the inputs the reference parser accepts without reporting an error, and it +// produces the same tree of SyntaxNodeKind values (src/core/types/nodes.ts). +// Element keywords such as Table or Ref are ordinary identifiers here. +// +// Conventions +// - Rules never consume trailing trivia. Trivia is consumed at the start of +// the construct that needs it, or inspected with lookahead. +// - Node shape: { kind, children }, children in source order, tokens as +// { token, value }. This mirrors getMemberChain() in src/core/parser/utils.ts. +// - Comments of the form "parser.ts: foo()" name the reference method a rule +// corresponds to. +// +// Grammar is written for peggy >= 5 (unicode character classes need the u flag). + +{{ + const PREFIX_OPS = new Set([ + '+', '-', '<', '>', '<>', '-?', '?-', '?-?', '?>', '>?', '?<', '?', '?', '<>?', '?<>?', '->', '<-', '!', '~', + ]); + const ADDITIVE_OPS = new Set(['+', '-', '-?', '?-', '?-?']); + const COMPARISON_OPS = new Set([ + '<', '<=', '>', '>=', '<>', '?>', '>?', '?<', '?', '?', + '<>?', '?<>?', '->', '<-', + ]); + const MULTIPLICATIVE_OPS = new Set(['/', '%']); + const EQUALITY_OPS = new Set(['==', '!=']); + + function tok (kind, value) { + return { token: kind, value }; + } + + function node (kind, ...children) { + return { + kind, + children: children.flat(Infinity).filter((c) => c !== null && c !== undefined), + }; + } + + const empty = () => node(''); + + // Trivia helpers. A gap is an array of trivia units: ' ', '\t', '\r', '\n', + // or the full text of a comment. A multi-line comment is a single unit, so a + // newline inside a comment is not a line break for the parser. + + // lexer/utils.ts: hasTrailingNewLines() + function hasNewline (gap) { + return gap.includes('\n'); + } + + // lexer/utils.ts: hasTrailingSpaces(), restricted to the trivia before the + // first line break (that is what the lexer attaches as trailing trivia). + function hasTrailingSpaces (gap) { + for (const unit of gap) { + if (unit === '\n') return false; + if (unit === ' ' || unit === '\t') return true; + } + return false; + } + + function isIdentifierPrimary (n) { + return n && n.kind === '' + && n.children[0].kind === '' + && n.children[0].children[0].token === ''; + } + + function identifierValue (n) { + return n.children[0].children[0].value; + } + + function isKeyword (identifierToken, word) { + return identifierToken.token === '' + && identifierToken.value.toLowerCase() === word; + } + + function foldInfix (head, tail) { + return tail.reduce((left, [op, right]) => node('', left, op, right), head); + } + + function foldPostfix (head, tail) { + return tail.reduce((left, step) => { + switch (step.step) { + case 'call': return node('', left, step.arg); + case 'index': return node('', left, step.arg); + default: return node('', left, step.op, step.arg); + } + }, head); + } + + function interleave (items, separators) { + const out = []; + items.forEach((item, i) => { + out.push(item); + if (i < separators.length) out.push(separators[i]); + }); + return out; + } + + // parser/utils.ts: convertFuncAppToElem() + // A function application whose shape is + // [ [as ]] [] + // is reinterpreted as a nested element declaration. + function convertFuncAppToElem (callee, args) { + let type = callee; + let rest = args; + if (type.kind === '') { + rest = [type.children[1], ...rest]; + type = type.children[0]; + } + if (!isIdentifierPrimary(type) || rest.length === 0) return null; + const typeToken = type.children[0].children[0]; + rest = [...rest]; + const body = rest.pop(); + if (body.kind !== '') return null; + const attributeList = rest.length > 0 && rest[rest.length - 1].kind === '' + ? rest.pop() + : null; + if (rest.length === 3) { + if (!isIdentifierPrimary(rest[1]) || identifierValue(rest[1]).toLowerCase() !== 'as') return null; + const asToken = rest[1].children[0].children[0]; + return node('', typeToken, rest[0], asToken, rest[2], attributeList, body); + } + if (rest.length === 1) { + return node('', typeToken, rest[0], attributeList, body); + } + if (rest.length === 0) { + return node('', typeToken, attributeList, body); + } + return null; + } +}} + +{ + // parser/contextStack.ts: number of unclosed '(' and '['. Inside them a '(' + // that starts a new line still continues a call expression. + let depth = 0; +} + +// --------------------------------------------------------------------------- +// Program +// --------------------------------------------------------------------------- + +// parser.ts: program() +Program + = body:Statement* GapAny EOF + { return node('', body, tok('', '')); } + +// A statement starting with the identifier `use` or `reuse` is always a use +// declaration; it never falls back to an element declaration. +Statement + = GapAny &UseKeyword d:UseDeclaration { return d; } + / GapAny !UseKeyword d:ElementDeclaration { return d; } + +// --------------------------------------------------------------------------- +// Use declarations +// --------------------------------------------------------------------------- + +// parser.ts: useDeclaration() +// use * from '' +// use { \n ... } from '' +UseDeclaration + = kw:UseKeyword GapAny specifiers:(WildcardNode / UseSpecifierList) + GapAny from:FromKeyword GapAny path:StringLiteral + { return node('', kw, specifiers, from, path); } + +// parser.ts: useSpecifierList(); specifiers are separated by line breaks. +UseSpecifierList + = open:LBrace first:(GapAny s:UseSpecifier { return s; })? + rest:(GapNewline s:UseSpecifier { return s; })* + GapAny close:RBrace + { return node('', open, first, rest, close); } + +// parser.ts: useSpecifier() +// [as ] +UseSpecifier + = kind:IdentifierToken GapAny name:UseName + alias:(GapAny as:AsKeyword GapAny a:UseName { return [as, a]; })? + { return node('', kind, name, alias); } + +// The name must start with an identifier or quoted variable; it is then read +// as a full normal expression. +UseName + = &(IdentifierToken / QuotedVariable) e:NormalExpression { return e; } + +// --------------------------------------------------------------------------- +// Element declarations +// --------------------------------------------------------------------------- + +// parser.ts: elementDeclaration() +// [] [] [as ] [] (: | { }) +// is present exactly when is `metadata`. +ElementDeclaration + = type:IdentifierToken + targetKind:( + &{ return isKeyword(type, 'metadata'); } GapAny t:IdentifierToken { return t; } + / !{ return isKeyword(type, 'metadata'); } { return null; } + ) + name:(&(GapAny !BodyStart) GapAny n:NormalExpression { return n; })? + alias:(GapAny as:AsKeyword GapAny !BodyStart a:NormalExpression { return [as, a]; })? + attributeList:(GapAny l:ListExpression { return l; })? + GapAny body:ElementBody + { return node('', type, targetKind, name, alias, attributeList, body); } + +BodyStart = ":" / "{" / "[" + +ElementBody + = colon:Colon GapAny body:SimpleBody { return [colon, body]; } + / BlockExpression + +// The simple body is a function application that must not itself resolve to +// an element declaration (parser.ts: UNEXPECTED_ELEMENT_DECLARATION). +SimpleBody + = e:Expression &{ return e.kind !== ''; } { return e; } + +// --------------------------------------------------------------------------- +// Blocks +// --------------------------------------------------------------------------- + +// parser.ts: blockExpression() +BlockExpression + = open:LBrace body:BodyItem* GapAny close:RBrace + { return node('', open, body, close); } + +// parser.ts: canBeField() decides between fieldDeclaration() and expression(). +BodyItem + = GapAny &FieldStart d:FieldDeclaration { return d; } + / GapAny !FieldStart e:Expression { return e; } + +FieldStart = IdentifierToken GapAny ":" + +// parser.ts: fieldDeclaration() +// : +FieldDeclaration + = type:IdentifierToken GapAny colon:Colon GapAny body:SimpleBody + { return node('', type, colon, body); } + +// --------------------------------------------------------------------------- +// Function application +// --------------------------------------------------------------------------- + +// parser.ts: expression() +// A callee followed by arguments on the same line, each preceded by at least +// one space or tab. The application ends at a line break, end of input, or +// before } ] ) , : -- anything else there is an error (MISSING_SPACES). +Expression + = callee:CommaExpression + args:(gap:GapInline &{ return hasTrailingSpaces(gap); } !"," a:CommaExpression { return a; })* + &FunctionApplicationStop + { return convertFuncAppToElem(callee, args) ?? node('', callee, args); } + +// parser.ts: shouldStopFunctionApplication() +FunctionApplicationStop = GapInline ("\n" / EOF / [}\]):,]) + +// parser.ts: commaExpression() +// CSV-like list without delimiters. Empty slots produce nodes. +CommaExpression + = first:NormalExpression tail:CommaTail? + { return tail ? node('', first, tail) : first; } + / &"," tail:CommaTail + { return node('', empty(), tail); } + +CommaTail + = GapAny comma:Comma rest:CommaRest { return [comma, rest]; } + +// After a comma: a line break, end of input, bracket or colon ends the list +// with an empty slot; another comma is an empty slot; otherwise a normal +// expression, which continues only if a comma follows on the same line. +// parser.ts: shouldStopCommaExpression() +CommaRest + = &CommaStop { return [empty()]; } + / GapInline comma:Comma rest:CommaRest { return [empty(), comma, rest]; } + / GapInline e:NormalExpression more:(GapInline comma:Comma rest:CommaRest { return [comma, rest]; })? + { return [e, more]; } + +CommaStop = GapInline ("\n" / EOF / [{}\[\]():]) + +// --------------------------------------------------------------------------- +// Normal expressions (parser.ts: expression_bp(), Pratt precedence climbing) +// --------------------------------------------------------------------------- +// Binding powers from parser.ts, tightest first: +// . 16 right operand is a bare operand +// prefix operators 15 +// call ( ... ) 14 +// / % 11 +// + - -? ?- ?-? 9 +// relationship ops 7 < <= > >= <> and the ? variants, -> <- +// == != 4 +// = 2 +// All infix operators are left-associative. Trivia, including line breaks, +// may surround infix operators. + +NormalExpression = AssignmentExpression + +AssignmentExpression + = head:EqualityExpression tail:(GapAny op:AssignOp GapAny rhs:EqualityExpression { return [op, rhs]; })* + { return foldInfix(head, tail); } + +EqualityExpression + = head:ComparisonExpression tail:(GapAny op:EqualityOp GapAny rhs:ComparisonExpression { return [op, rhs]; })* + { return foldInfix(head, tail); } + +ComparisonExpression + = head:AdditiveExpression tail:(GapAny op:ComparisonOp GapAny rhs:AdditiveExpression { return [op, rhs]; })* + { return foldInfix(head, tail); } + +AdditiveExpression + = head:MultiplicativeExpression tail:(GapAny op:AdditiveOp GapAny rhs:MultiplicativeExpression { return [op, rhs]; })* + { return foldInfix(head, tail); } + +MultiplicativeExpression + = head:PostfixExpression tail:(GapAny op:MultiplicativeOp GapAny rhs:PostfixExpression { return [op, rhs]; })* + { return foldInfix(head, tail); } + +// Calls, indexing and member access applied to a unary expression. +PostfixExpression + = head:UnaryExpression tail:(CallStep / IndexStep / MemberStep)* + { return foldPostfix(head, tail); } + +// parser.ts: leftExpression_bp() +// A prefix operator binds tighter than a call: -2() is (-2)(). +UnaryExpression + = op:PrefixOp GapAny e:UnaryExpression { return node('', op, e); } + / MemberChain + +// Member access and indexing on a bare operand. The right side of `.` is a +// bare operand (parser.ts: extractOperand()), never a prefix expression. +MemberChain + = head:Operand tail:(IndexStep / MemberStep)* + { return foldPostfix(head, tail); } + +MemberStep + = GapAny op:MemberOp GapAny rhs:Operand { return { step: 'member', op, arg: rhs }; } + +// A '(' continues a call unless it starts a new line; inside an unclosed '(' +// or '[' the line-break rule is suspended. +CallStep + = gap:GapAny &"(" &{ return depth > 0 || !hasNewline(gap); } arg:TupleExpression + { return { step: 'call', arg }; } + +// A '[' is array indexing only when no space or tab separates it from the +// operand on the same line; otherwise it starts a new argument. +IndexStep + = gap:GapAny &"[" &{ return !hasTrailingSpaces(gap); } arg:ListExpression + { return { step: 'index', arg }; } + +// parser.ts: extractOperand() +Operand + = WildcardNode + / PrimaryExpression + / FunctionExpression + / ListExpression + / BlockExpression + / TupleExpression + +// parser.ts: primaryExpression() +PrimaryExpression + = t:(StringLiteral / NumericLiteral / ColorLiteral) + { return node('', node('', t)); } + / t:(QuotedVariable / IdentifierToken) + { return node('', node('', t)); } + +FunctionExpression + = t:FunctionExpressionToken { return node('', t); } + +WildcardNode + = t:Wildcard { return node('', t); } + +// parser.ts: tupleExpression(); one element without a comma is a group. +TupleExpression + = open:TupleOpen GapAny elements:TupleElements? GapAny close:TupleClose + { + if (elements && elements.items.length === 1) { + return node('', open, elements.items[0], close); + } + return node('', open, elements ? interleave(elements.items, elements.commas) : [], close); + } + +TupleElements + = head:NormalExpression tail:(GapAny c:Comma GapAny e:NormalExpression { return [c, e]; })* + { return { items: [head, ...tail.map((t) => t[1])], commas: tail.map((t) => t[0]) }; } + +// --------------------------------------------------------------------------- +// Attribute lists +// --------------------------------------------------------------------------- + +// parser.ts: listExpression() +// [ , , ... ] +ListExpression + = open:ListOpen GapAny attributes:ListAttributes? GapAny close:ListClose + { return node('', open, attributes ? interleave(attributes.items, attributes.commas) : [], close); } + +ListAttributes + = head:Attribute tail:(GapAny c:Comma GapAny a:Attribute { return [c, a]; })* + { return { items: [head, ...tail.map((t) => t[1])], commas: tail.map((t) => t[0]) }; } + +// parser.ts: attribute() +// [: ] +Attribute + = name:AttributeName value:(GapAny c:Colon GapAny v:AttributeValue { return [c, v]; })? + { return node('', name, value); } + +// parser.ts: attributeName() +// A stream of identifiers (primary key), or a single literal or quoted variable. +AttributeName + = IdentifierStream + / t:(StringLiteral / NumericLiteral / ColorLiteral) + { return node('', node('', t)); } + / t:QuotedVariable + { return node('', node('', t)); } + +IdentifierStream + = head:IdentifierToken tail:(GapAny t:IdentifierToken { return t; })* + { return node('', head, tail); } + +// parser.ts: attributeValue() +// Two leading identifiers make the value an identifier stream (no action); +// anything else is a normal expression. +AttributeValue + = &(IdentifierToken GapAny IdentifierToken) s:IdentifierStream { return s; } + / NormalExpression + +// --------------------------------------------------------------------------- +// Operators +// --------------------------------------------------------------------------- + +// lexer.ts: operator(). Scanned greedily; the set is prefix-closed so longest +// match reproduces the lexer. +OpToken + = $("?<>?" / "?<>" / "??" / "?>" / "?-?" / "?-" / "?" + / "<>?" / "<>" / "<=" / "=" / ">?" / ">" + / "->" / "-?" / "-" + / "==" / "=" / "!=" / "!" + / "+" / "/" ![/*] / "%" / "." / "&" / "|" / "~") + +PrefixOp = op:OpToken &{ return PREFIX_OPS.has(op); } { return tok('', op); } +MemberOp = op:OpToken &{ return op === '.'; } { return tok('', op); } +MultiplicativeOp = op:OpToken &{ return MULTIPLICATIVE_OPS.has(op); } { return tok('', op); } +AdditiveOp = op:OpToken &{ return ADDITIVE_OPS.has(op); } { return tok('', op); } +ComparisonOp = op:OpToken &{ return COMPARISON_OPS.has(op); } { return tok('', op); } +EqualityOp = op:OpToken &{ return EQUALITY_OPS.has(op); } { return tok('', op); } +AssignOp = op:OpToken &{ return op === '='; } { return tok('', op); } + +// --------------------------------------------------------------------------- +// Keywords (case-insensitive identifiers) +// --------------------------------------------------------------------------- + +UseKeyword = t:IdentifierToken &{ return isKeyword(t, 'use') || isKeyword(t, 'reuse'); } { return t; } +FromKeyword = t:IdentifierToken &{ return isKeyword(t, 'from'); } { return t; } +AsKeyword = t:IdentifierToken &{ return isKeyword(t, 'as'); } { return t; } + +// --------------------------------------------------------------------------- +// Punctuation +// --------------------------------------------------------------------------- + +// text() is used here because peggy balances braces inside action code +// without understanding JavaScript string literals. +LBrace = "{" { return tok('', text()); } +RBrace = "}" { return tok('', text()); } +Comma = "," { return tok('', ','); } +Colon = ":" { return tok('', ':'); } +Wildcard = "*" { return tok('', '*'); } + +TupleOpen = "(" { depth += 1; return tok('', '('); } +TupleClose = ")" { depth -= 1; return tok('', ')'); } +ListOpen = "[" { depth += 1; return tok('', '['); } +ListClose = "]" { depth -= 1; return tok('', ']'); } + +// --------------------------------------------------------------------------- +// Tokens (lexer.ts) +// --------------------------------------------------------------------------- + +// lexer.ts: identifier(), numericLiteralOrIdentifier() +IdentifierToken + = t:$(IdentifierStart AlnumChar*) { return tok('', t); } + / DigitIdentifier + +IdentifierStart = [\p{L}\p{M}_]u +AlnumChar = [\p{L}\p{M}_0-9]u +Digits = $[0-9]+ + +// A token starting with a digit that is not a well-formed number is an +// identifier: 12_abc, 3a, 1e5x. It never contains a dot. +DigitIdentifier + = !NumericLiteral t:$([0-9] !"." AlnumChar*) { return tok('', t); } + +// lexer.ts: numericLiteralOrIdentifier() +// 123 1.5 5. 1e5 1.5e-3 +// A number followed by a letter is not a number (1e5x is an identifier); a +// number with two dots or a dot followed by letters is a lexical error. +NumericLiteral + = t:$(Digits Exponent) !AlnumChar { return tok('', t); } + / t:$(Digits "." Digits? Exponent) !AlnumChar { return tok('', t); } + / t:$(Digits "." Digits?) !(AlnumChar / ".") { return tok('', t); } + / t:$(Digits) !(AlnumChar / ".") { return tok('', t); } + +Exponent = [eE] [+-]? Digits + +// lexer.ts: singleLineStringLiteral(), multilineStringLiteral() +StringLiteral + = t:$("'''" (Escape / !("'''" / "\\") .)* "'''") { return tok('', t); } + / t:$("'" (Escape / !("'" / "\n" / "\\") .)* "'") { return tok('', t); } + +// lexer.ts: quotedVariable() +QuotedVariable + = t:$('"' (Escape / !('"' / "\n" / "\\") .)* '"') { return tok('', t); } + +// lexer.ts: escapedString(). A backslash always starts an escape. Any escaped +// character is accepted; \uHHHH needs four hexadecimal digits; a backslash +// before a line break continues the line. +Escape + = "\\u" Hex Hex Hex Hex + / "\\\r\n" + / "\\" !"u" . + +Hex = [0-9a-fA-F] + +// lexer.ts: functionExpression(); raw text, no escapes, may span lines. +FunctionExpressionToken + = t:$("`" (!"`" .)* "`") { return tok('', t); } + +// lexer.ts: colorLiteral(); '#' followed by alphanumerics. Hex validation is a +// Layer 2 rule (utils/validate.ts: isValidHexColor). +ColorLiteral + = t:$("#" AlnumChar*) { return tok('', t); } + +// --------------------------------------------------------------------------- +// Trivia (lexer.ts: gatherTrivia()) +// --------------------------------------------------------------------------- +// A carriage return is discarded by the lexer, so it is neither whitespace nor +// a line break. A single-line comment does not include its terminating +// newline. A multi-line comment is one unit even when it spans lines. + +GapAny = (InlineTrivia / Newline)* +GapInline = InlineTrivia* +GapNewline = GapInline Newline GapAny + +InlineTrivia + = [ \t\r] + / Comment + +Newline = "\n" + +Comment + = $("//" (!"\n" .)*) + / $("/*" (!"*/" .)* "*/") + +EOF = !. diff --git a/yarn.lock b/yarn.lock index 0508fc5bb..86a30b6e5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1837,6 +1837,13 @@ node-addon-api "^3.2.1" node-gyp-build "^4.3.0" +"@peggyjs/from-mem@3.1.3": + version "3.1.3" + resolved "https://registry.yarnpkg.com/@peggyjs/from-mem/-/from-mem-3.1.3.tgz#2956c3c68ec2e08e55178049a327009ee61541b8" + integrity sha512-LLlgtfXIaeYXoOYovOI0spLM8ZXaqkAlmcRRrLzHJzLMqkU6Sw0R4KMoCoHx1PjaP815pSCBlS+BN6aD8t1Jgg== + dependencies: + semver "7.7.4" + "@phosphor-icons/vue@^2.2.0": version "2.2.1" resolved "https://registry.npmjs.org/@phosphor-icons/vue/-/vue-2.2.1.tgz" @@ -3560,11 +3567,31 @@ ansi-styles@^6.1.0: resolved "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz" integrity sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug== -antlr4@^4.13.1: +antlr-ng@^1.0.10: + version "1.0.10" + resolved "https://registry.yarnpkg.com/antlr-ng/-/antlr-ng-1.0.10.tgz#fdca4f793bc2ff71dc16607f11a4286a8b895ed4" + integrity sha512-fw3NdsQP3dabuZrDhKAMewrBsY5KSAcMrvhWBVDmHYegv5D51pypzCYK1PpjaRVKcVeP/5xKfqJY31TvXACOdA== + dependencies: + antlr4ng "3.0.16" + commander "13.1.0" + stringtemplate4ts "1.0.9" + unicode-properties "1.4.1" + +antlr4@^4.13.1, antlr4@^4.13.2: version "4.13.2" resolved "https://registry.npmjs.org/antlr4/-/antlr4-4.13.2.tgz" integrity sha512-QiVbZhyy4xAZ17UPEuG3YTOt8ZaoeOR1CvEAqrEsDBsOqINslaB147i9xqljZqoyf5S+EUlGStaj+t22LT9MOg== +antlr4ng@3.0.15: + version "3.0.15" + resolved "https://registry.yarnpkg.com/antlr4ng/-/antlr4ng-3.0.15.tgz#8ed2ac92e4b604974775dbac3eeaf6d9ef1965aa" + integrity sha512-VELFqTfcpGI2bj6ScMWuxM3FI6HOsojrgmnw3cCbUtsQ1DNOq32wJsjOt7vLvfIniyyuE1DIYegGcuFmn+jgyw== + +antlr4ng@3.0.16: + version "3.0.16" + resolved "https://registry.yarnpkg.com/antlr4ng/-/antlr4ng-3.0.16.tgz#2e9db4fde07293a4d48c6537a90f9c1a917cbaca" + integrity sha512-DQuJkC7kX3xunfF4K2KsWTSvoxxslv+FQp/WHQZTJSsH2Ec3QfFmrxC3Nky2ok9yglXn6nHM4zUaVDxcN5f6kA== + any-promise@^1.0.0: version "1.3.0" resolved "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz" @@ -4160,11 +4187,21 @@ combined-stream@^1.0.8: dependencies: delayed-stream "~1.0.0" +commander@13.1.0: + version "13.1.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-13.1.0.tgz#776167db68c78f38dcce1f9b8d7b8b9a488abf46" + integrity sha512-/rFeCpNJQbhSZjGVwO9RFV3xPqbnERS8MmIQzCtD/zl6gpJuV/bMLuN92oG3F7d8oDEHHRrujSXNUr8fpjntKw== + commander@^11.0.0: version "11.1.0" resolved "https://registry.npmjs.org/commander/-/commander-11.1.0.tgz" integrity sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ== +commander@^14.0.3: + version "14.0.3" + resolved "https://registry.yarnpkg.com/commander/-/commander-14.0.3.tgz#425d79b48f9af82fcd9e4fc1ea8af6c5ec07bbc2" + integrity sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw== + commander@^2.20.0, commander@~2.20.3: version "2.20.3" resolved "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz" @@ -4939,6 +4976,11 @@ fast-levenshtein@^2.0.6: resolved "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz" integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw== +fast-printf@1.6.10: + version "1.6.10" + resolved "https://registry.yarnpkg.com/fast-printf/-/fast-printf-1.6.10.tgz#c44ad871726152159d7a903a5af0d65cf3d75875" + integrity sha512-GwTgG9O4FVIdShhbVF3JxOgSBY2+ePGsu2V/UONgoCPzF9VY6ZdBMKsHKCYQHZwNk3qNouUolRDsgVxcVA5G1w== + fast-xml-builder@^1.1.4: version "1.2.0" resolved "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.2.0.tgz" @@ -5541,7 +5583,7 @@ hasown@^2.0.4: dependencies: function-bind "^1.1.2" -he@^1.2.0: +he@1.2.0, he@^1.2.0: version "1.2.0" resolved "https://registry.npmjs.org/he/-/he-1.2.0.tgz" integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw== @@ -6689,6 +6731,11 @@ lru-cache@^8.0.0: resolved "https://registry.npmjs.org/lru-cache/-/lru-cache-8.0.5.tgz" integrity sha512-MhWWlVnuab1RG5/zMRRcVGXZLCXrZTgfwMikgzCegsPnG62yDQo5JnqKkrK4jO5iKqDAZGItAqN5CtKBCBWRUA== +luxon@3.5.0: + version "3.5.0" + resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.5.0.tgz#6b6f65c5cd1d61d1fd19dbf07ee87a50bf4b8e20" + integrity sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ== + luxon@^3.4.4, luxon@^3.7.2: version "3.7.2" resolved "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz" @@ -7770,6 +7817,11 @@ pacote@^15.2.0: ssri "^10.0.0" tar "^6.1.11" +pako@^0.2.5: + version "0.2.9" + resolved "https://registry.yarnpkg.com/pako/-/pako-0.2.9.tgz#f3f7522f4ef782348da8161bad9ecfd51bf83a75" + integrity sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA== + parent-module@^1.0.0: version "1.0.1" resolved "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz" @@ -7904,6 +7956,15 @@ pathe@^2.0.1, pathe@^2.0.3: resolved "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz" integrity sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w== +peggy@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/peggy/-/peggy-5.1.0.tgz#8e2a681128cf776648417f60394a40b4a712f503" + integrity sha512-IEo5aYRZ2kXH4Qby06cjtL114PZnwLoTiA41vUmg2vPZgANn+c87m5BUurhuDr5/cu758ZlpgsAfBVx+hhO5+w== + dependencies: + "@peggyjs/from-mem" "3.1.3" + commander "^14.0.3" + source-map-generator "2.0.6" + pegjs-require-import@^0.0.2: version "0.0.2" resolved "https://registry.npmjs.org/pegjs-require-import/-/pegjs-require-import-0.0.2.tgz" @@ -8549,6 +8610,11 @@ semver@7.5.3: dependencies: lru-cache "^6.0.0" +semver@7.7.4: + version "7.7.4" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.7.4.tgz#28464e36060e991fa7a11d0279d2d3f3b57a7e8a" + integrity sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA== + semver@^6.0.0: version "6.3.1" resolved "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz" @@ -8724,6 +8790,11 @@ sort-keys@^2.0.0: dependencies: is-plain-obj "^1.0.0" +source-map-generator@2.0.6: + version "2.0.6" + resolved "https://registry.yarnpkg.com/source-map-generator/-/source-map-generator-2.0.6.tgz#284776dff4bd5efdefead67aaed0c0f85f5a42ca" + integrity sha512-IlassDs1Ve8nV6uyQZXF9kdkJpVKnMte2JZQXu13M0A5zwc+vu6+LNHfmxsHBMDtoZE21RHiKI0/xvpecZRCNg== + source-map-js@^1.2.1: version "1.2.1" resolved "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz" @@ -8907,6 +8978,16 @@ string_decoder@~1.1.1: dependencies: safe-buffer "~5.1.0" +stringtemplate4ts@1.0.9: + version "1.0.9" + resolved "https://registry.yarnpkg.com/stringtemplate4ts/-/stringtemplate4ts-1.0.9.tgz#b48c10f10218a6729b32de188d879439959f9b53" + integrity sha512-KYZm2bJlSjynG5Y+L46fkaKBQG6mhV6hb2RBA8dpx3/Vj6G4u7gwXNKYvaN9+QD5sj68/1srtSNDvqEso7MwsQ== + dependencies: + antlr4ng "3.0.15" + fast-printf "1.6.10" + he "1.2.0" + luxon "3.5.0" + "strip-ansi-cjs@npm:strip-ansi@^6.0.1": version "6.0.1" resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz" @@ -9136,6 +9217,11 @@ through@2, "through@>=2.2.7 <3", through@^2.3.4, through@^2.3.6: resolved "https://registry.npmjs.org/through/-/through-2.3.8.tgz" integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU= +tiny-inflate@^1.0.0: + version "1.0.3" + resolved "https://registry.yarnpkg.com/tiny-inflate/-/tiny-inflate-1.0.3.tgz#122715494913a1805166aaf7c93467933eea26c4" + integrity sha512-pkY1fj1cKHb2seWDy0B16HeWyczlJA9/WW3u3c4z/NiWDsO3DOU5D7nhTLE9CF0yXv/QZFY7sEJmj24dK+Rrqw== + tinybench@^2.9.0: version "2.9.0" resolved "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz" @@ -9360,6 +9446,22 @@ undici-types@~7.8.0: resolved "https://registry.npmjs.org/undici-types/-/undici-types-7.8.0.tgz" integrity sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw== +unicode-properties@1.4.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/unicode-properties/-/unicode-properties-1.4.1.tgz#96a9cffb7e619a0dc7368c28da27e05fc8f9be5f" + integrity sha512-CLjCCLQ6UuMxWnbIylkisbRj31qxHPAurvena/0iwSVbQ2G1VY5/HjV0IRabOEbDHlzZlRdCrD4NhB0JtU40Pg== + dependencies: + base64-js "^1.3.0" + unicode-trie "^2.0.0" + +unicode-trie@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/unicode-trie/-/unicode-trie-2.0.0.tgz#8fd8845696e2e14a8b67d78fa9e0dd2cad62fec8" + integrity sha512-x7bc76x0bm4prf1VLg79uhAzKw8DVboClSN5VxJuQ+LKDOVEW9CdH+VY7SP+vX7xCYQqzzgQpFqz15zeLvAtZQ== + dependencies: + pako "^0.2.5" + tiny-inflate "^1.0.0" + unique-filename@^1.1.1: version "1.1.1" resolved "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.1.tgz"