From 174eb7844dc19441d2da9df0808674210bf5ac45 Mon Sep 17 00:00:00 2001 From: Anton Wiklund Date: Sun, 8 Nov 2026 05:28:21 +0100 Subject: [PATCH] fix(solid): bound how deep a Turtle text may nest The read limits held a Turtle text to its bytes, its triples and the characters of their terms. The parser makes the triple that links a nested collection, blank node, annotation or triple term to the level above it only once that level closes, and keeps a context for every level it is in. A text that only opened levels made no triple to count, so only its size bounded the memory the parse held, and a release or a foreign instance's document within the byte limit could use up the tab's memory. The scan that already looks through every Turtle text before the parser reads it, for its base and prefix declarations and long IRIs, now also counts the levels it opens and closes, past strings, IRIs and comments, and refuses a text nested more than 64 levels deep with a ReadLimitError, with limits or with none. Every Turtle read goes through it, on either client: pod documents, releases, profiles, access controls and deck files. The app's own documents and the library's releases nest a few levels. Refs: GHSA-qv47-r346-839c --- docs/boundaries.md | 2 +- docs/data-model.md | 9 ++- docs/deck-library.md | 13 ++- docs/testing.md | 2 +- packages/solid-memo-client/src/resources.ts | 8 +- packages/solid/src/client/interface.ts | 4 +- packages/solid/src/client/linearDataset.ts | 3 +- packages/solid/src/client/readLimits.test.ts | 46 +++++++++++ packages/solid/src/client/readLimits.ts | 79 +++++++++++++++---- packages/solid/src/client/resources.ts | 4 +- packages/solid/src/solidDeckArchive.ts | 5 +- packages/solid/src/solidReleasePublisher.ts | 8 +- .../solid/src/testing/contract/resources.ts | 42 +++++++++- 13 files changed, 182 insertions(+), 43 deletions(-) diff --git a/docs/boundaries.md b/docs/boundaries.md index 77c95ebce..10d8dd748 100644 --- a/docs/boundaries.md +++ b/docs/boundaries.md @@ -129,7 +129,7 @@ library could take their place there alone | [interface.ts](../packages/solid/src/client/interface.ts) | the types the other modules take and give, `HttpError`, the sign-in client's interface and the grant's type; it imports no library | | [rdf.ts](../packages/solid/src/client/rdf.ts) | the data model: datasets and their subjects, read and built, many subjects set or removed in one pass | | [linearDataset.ts](../packages/solid/src/client/linearDataset.ts) | a document read into a dataset in one pass, where the library takes time in the square of its size, and the Turtle parser it and `parseQuads` read with, on `n3`, for `resources.ts` and `rdf.ts` alone | -| [readLimits.ts](../packages/solid/src/client/readLimits.ts) | the limits a read of a release, or of an instance's documents outside the user's own pod, takes in, the bounds every read keeps on a Turtle text's base and prefix declarations, on the long IRIs the parser resolves and on the length of each term, the fetch that holds a body to them, the tally that holds its triples to them, and the parts a parser is given a text in, to be stopped between them; it imports no library | +| [readLimits.ts](../packages/solid/src/client/readLimits.ts) | the limits a read of a release, or of an instance's documents outside the user's own pod, takes in, the bounds every read keeps on a Turtle text's base and prefix declarations, on the long IRIs the parser resolves, on how deep it nests and on the length of each term, the fetch that holds a body to them, the tally that holds its triples to them, and the parts a parser is given a text in, to be stopped between them; it imports no library | | [linkHeader.ts](../packages/solid/src/client/linkHeader.ts) | the bound on the `Link` and `WAC-Allow` headers of a response, dropped past it before any parser reads them; it imports no library | | [resources.ts](../packages/solid/src/client/resources.ts) | reading and writing pods, conditional reads and writes among them, and the Turtle and JSON-LD parsers | | [sparqlUpdate.ts](../packages/solid/src/client/sparqlUpdate.ts) | the SPARQL Update bodies of the app's PATCH requests; it imports no library | diff --git a/docs/data-model.md b/docs/data-model.md index dfebe567e..9e09fb5ab 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -1657,9 +1657,12 @@ An import reads the file as the pod is read, with the same mappers: So is one with an IRI longer than 512 characters, in a triple or a prefix, that's relative or may hold a `.` or `..` segment, as written or escaped. The parser removes those segments as it resolves - the IRI, in time growing with the square of its length. Its parse stops - there, as it stops at the first triple past the limits. A - declaration quoted in a card's text doesn't count. So is a file with + the IRI, in time growing with the square of its length. So is one + nested more than 64 levels deep in collections, blank nodes, + annotations or triple terms, which the parser holds memory for + without making a triple to count. Its parse stops there, as it stops + at the first triple past the limits. A declaration or a bracket + quoted in a card's text doesn't count. So is a file with a term longer than 16,383 characters, as every document the app reads is, and a JSON-LD file with a string or key that long, which is refused before it's parsed. A diff --git a/docs/deck-library.md b/docs/deck-library.md index f60228774..3282f3bf6 100644 --- a/docs/deck-library.md +++ b/docs/deck-library.md @@ -1016,8 +1016,13 @@ flowchart LR parser walks the whole base for each relative IRI, base and prefix it resolves, and a declaration makes no triple to count. The search reads the text as the parser does, past strings, IRIs and comments, - so a declaration quoted in a card's text doesn't count. With the - bases and prefixes bounded and the parse stopped at the first limit, + so a declaration or a bracket quoted in a card's text doesn't count. + It's refused, too, where it nests more than 64 levels deep in + collections, blank nodes, annotations or triple terms (`(`, `[`, + `{|`, `<<`). The parser holds memory for each level it's in, and + makes the triple that links a level to the one above only once the + level closes, so levels that never close make no triple to count. + With the bases and prefixes bounded and the parse stopped at the first limit, the time to read or refuse a release grows with its size, not faster. Each case refuses the release (`releaseTooLarge`), however the release is read. The same limits apply to a creator's catalog read for the release's @@ -1033,8 +1038,8 @@ flowchart LR (`releaseUnreadable`), however it's read: the parser reads none, as for a deck file, and stops at the first one. The library's index, which comes with the site and grows with the library, and the - documents in the user's storages aren't limited in size. Their bases - and prefixes are, as those of every document the app reads into a + documents in the user's storages aren't limited in size. Their bases, + prefixes and nesting are, as those of every document the app reads into a dataset, a folder's listing too; the document's own address counts toward a base's length. An instance's documents outside the user's storages, in an instance diff --git a/docs/testing.md b/docs/testing.md index 3df5bab10..35f28a79d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1273,7 +1273,7 @@ Dependency inversion gives every layer a seam for its fakes: | Layer | Seam | Technique | |---|---|---| -| Domain | none needed | Pure functions; inputs (including `now: Date`) passed as parameters. Plain assertions. What runs over a whole deck or log is timed at two sizes in its own process's CPU time: eight times the input must take under four times as long as an eighth of it done eight times (a linear pass gives about one, work growing with the input about eight). The growth tests (`growth` in `src/testing/growth.ts`, `growthAsync` for work that awaits, `growthAfterSetup` for work that changes what it works on, set up afresh before each run and not timed) time `queryCards` over 5,000 cards, `lapseIndex`, `chainIndexOf` and `statisticsOf` over years of answers, `instanceGraphOf` over an instance's cards and answers and over a document thousands of folders deep, a course's outline (`courseOutlineFromRecords`) over its chapters and steps, the learner's progress in it (`courseProgress`) over a chapter's steps, the statistics of many courses (`getCourseStatistics`) over as many of their answers, the release check's course rules (`courseProblems`) and its rules on text written in Markdown (`markdownProblems`) over many cards, steps and chapters and their text formats and over one distractor that many cards name in many languages, a draft's outline (`draftOutline`) over the chapters and steps, a draft's cards with their wrong options, as the Studio's outline and its step, chapter and question pages show them (`draftCardOf`), over many cards, many wrong options of one card and one wrong option that many cards name in many languages, and the changes to a card's many wrong options (`distractorChanges`), the deck health's cards that say the same (`duplicateCardsOf`) over many cards alike, which of many decks with long release links are copies of a library of as many decks (`libraryCopyTest`) and which of the library's decks they copy (`copiedTest`), a JSON-LD copy (`jsonLdOf` in `solid`) over many values of one property, a document the guest's pod on the device stores and changes (`createLocalPod` in `solid`) over many lines past the bound on a term, alike but for their end, the search of a Turtle text for its base and prefix declarations and its long IRIs (`writeTurtle` in `solid`, and the refusal of a long IRI with dot segments in the client modules' contract tests), the shape check's report on a pod document and its check of a write (`createShaclShapeValidator` in `solid`, over an engine that answers at once) over a catalog's many links that aren't members and many dates it can't read, the SHACL engine's check of a graph and of each of its subjects (`createEngine` in `shacl`) over many subjects linking to a node with many types, the notes on a release's making (`makingNotesOf`) over many in one language, a draft's listing (`draftLibraryDeck`) and its release diff (`releaseDiff` and `simulateLearnerUpgrade`) over many sources, the draft as a release (`draftLibraryContent`) and a learner's upgrade to it over one wrong option that many cards name in many languages, a merge of drafts (`mergedDraft`) over many statements of one subject, the blank nodes a removal takes along (`withOnlyTheirBlankNodes` in `solid`) over many of one subject and many subjects naming one, many decks taken out of a catalog whose deck group lists many links, one of them no URL the app can write (`withoutDecks` in `solid`), the agent nodes of many authors that share a slug and a hash, alone and beside as many nodes other authors hold (`agentUrlsOf`), authors whose agent URLs are longer than any term, from a long subject or long names, at each place that keeps them (`TextMap`, `agentUrlsOf`, `deckAgents`, `withProvenance`, `deckReport`, `otherAuthorsOf`, a draft's listing, and `toDecks`, `deckSubjects` and `toLibraryDeckContent` in `solid`) and refused before anything is keyed by them (`deckToDraft`, and `withDeck`, a draft's entries, moved past the bound too, and a release assembled at a longer address, in `solid`), a catalog's decks (`toDecks` in `solid`) and a deck file's refusal (`readDeckFile`) over many decks and agents, a document's cards (`toCards` in `solid`) and a release's, read and refused (`toLibraryDeckContent` in `solid`), over many cards that name one distractor in many languages, a catalog written over many licenses it no longer states (`withCatalog` in `solid`), a deck's cards added from a release, or refused for a distractor many of them name, removed and brought to this app's format, one card's many distractors added and changed, a deck removed from a cards document another app wrote in, many decks of a catalog changed and removed at once, catalog entries and review states brought to this app's format, review states removed, a day's answers removed, many month summaries dropped from the digest and as many receipts noted in it at once as it holds, and many review states' snapshots dropped, decks removed and one deck's unreadable dates dropped by the instance check's repair, many registrations of an instance renamed and removed in a type index, and the storages found in a profile that names a large extended profile document, against a pod in memory (`importDeck`, `applyCardChanges`, `removeDeck`, `saveDecks`, `removeDecks`, `upgradeCards`, `upgradeDecks`, `upgradeReviewStates`, `applyReviewChanges`, `removeDay`, `updateDigest`, `applyRepairs`, `renameInstanceRegistrations`, `removeInstanceRegistrations` and `discoverStorages` in `solid`), the receipts a visit notes in the digest it keeps (`createDigestStore`) over a digest of as many, the repairs planned from a check of many subjects (`planRepair`), a subject's many values set or removed in one call (`setIriAll`, `setStringAll` and `removeLiteralAll`, in the client modules' contract tests), the Studio's graph layout and routes over a deck's cards, its card History tab and the learner's card page's history over many answers that name one of a card's many wrong options, and the Studio's cut of a label (`drawnLabel`), which must not grow with the label at all, over one millions of characters long. | +| Domain | none needed | Pure functions; inputs (including `now: Date`) passed as parameters. Plain assertions. What runs over a whole deck or log is timed at two sizes in its own process's CPU time: eight times the input must take under four times as long as an eighth of it done eight times (a linear pass gives about one, work growing with the input about eight). The growth tests (`growth` in `src/testing/growth.ts`, `growthAsync` for work that awaits, `growthAfterSetup` for work that changes what it works on, set up afresh before each run and not timed) time `queryCards` over 5,000 cards, `lapseIndex`, `chainIndexOf` and `statisticsOf` over years of answers, `instanceGraphOf` over an instance's cards and answers and over a document thousands of folders deep, a course's outline (`courseOutlineFromRecords`) over its chapters and steps, the learner's progress in it (`courseProgress`) over a chapter's steps, the statistics of many courses (`getCourseStatistics`) over as many of their answers, the release check's course rules (`courseProblems`) and its rules on text written in Markdown (`markdownProblems`) over many cards, steps and chapters and their text formats and over one distractor that many cards name in many languages, a draft's outline (`draftOutline`) over the chapters and steps, a draft's cards with their wrong options, as the Studio's outline and its step, chapter and question pages show them (`draftCardOf`), over many cards, many wrong options of one card and one wrong option that many cards name in many languages, and the changes to a card's many wrong options (`distractorChanges`), the deck health's cards that say the same (`duplicateCardsOf`) over many cards alike, which of many decks with long release links are copies of a library of as many decks (`libraryCopyTest`) and which of the library's decks they copy (`copiedTest`), a JSON-LD copy (`jsonLdOf` in `solid`) over many values of one property, a document the guest's pod on the device stores and changes (`createLocalPod` in `solid`) over many lines past the bound on a term, alike but for their end, the search of a Turtle text for its base and prefix declarations, its long IRIs and its nesting (`writeTurtle` in `solid`, and the refusal of a long IRI with dot segments in the client modules' contract tests), the shape check's report on a pod document and its check of a write (`createShaclShapeValidator` in `solid`, over an engine that answers at once) over a catalog's many links that aren't members and many dates it can't read, the SHACL engine's check of a graph and of each of its subjects (`createEngine` in `shacl`) over many subjects linking to a node with many types, the notes on a release's making (`makingNotesOf`) over many in one language, a draft's listing (`draftLibraryDeck`) and its release diff (`releaseDiff` and `simulateLearnerUpgrade`) over many sources, the draft as a release (`draftLibraryContent`) and a learner's upgrade to it over one wrong option that many cards name in many languages, a merge of drafts (`mergedDraft`) over many statements of one subject, the blank nodes a removal takes along (`withOnlyTheirBlankNodes` in `solid`) over many of one subject and many subjects naming one, many decks taken out of a catalog whose deck group lists many links, one of them no URL the app can write (`withoutDecks` in `solid`), the agent nodes of many authors that share a slug and a hash, alone and beside as many nodes other authors hold (`agentUrlsOf`), authors whose agent URLs are longer than any term, from a long subject or long names, at each place that keeps them (`TextMap`, `agentUrlsOf`, `deckAgents`, `withProvenance`, `deckReport`, `otherAuthorsOf`, a draft's listing, and `toDecks`, `deckSubjects` and `toLibraryDeckContent` in `solid`) and refused before anything is keyed by them (`deckToDraft`, and `withDeck`, a draft's entries, moved past the bound too, and a release assembled at a longer address, in `solid`), a catalog's decks (`toDecks` in `solid`) and a deck file's refusal (`readDeckFile`) over many decks and agents, a document's cards (`toCards` in `solid`) and a release's, read and refused (`toLibraryDeckContent` in `solid`), over many cards that name one distractor in many languages, a catalog written over many licenses it no longer states (`withCatalog` in `solid`), a deck's cards added from a release, or refused for a distractor many of them name, removed and brought to this app's format, one card's many distractors added and changed, a deck removed from a cards document another app wrote in, many decks of a catalog changed and removed at once, catalog entries and review states brought to this app's format, review states removed, a day's answers removed, many month summaries dropped from the digest and as many receipts noted in it at once as it holds, and many review states' snapshots dropped, decks removed and one deck's unreadable dates dropped by the instance check's repair, many registrations of an instance renamed and removed in a type index, and the storages found in a profile that names a large extended profile document, against a pod in memory (`importDeck`, `applyCardChanges`, `removeDeck`, `saveDecks`, `removeDecks`, `upgradeCards`, `upgradeDecks`, `upgradeReviewStates`, `applyReviewChanges`, `removeDay`, `updateDigest`, `applyRepairs`, `renameInstanceRegistrations`, `removeInstanceRegistrations` and `discoverStorages` in `solid`), the receipts a visit notes in the digest it keeps (`createDigestStore`) over a digest of as many, the repairs planned from a check of many subjects (`planRepair`), a subject's many values set or removed in one call (`setIriAll`, `setStringAll` and `removeLiteralAll`, in the client modules' contract tests), the Studio's graph layout and routes over a deck's cards, its card History tab and the learner's card page's history over many answers that name one of a card's many wrong options, and the Studio's cut of a label (`drawnLabel`), which must not grow with the label at all, over one millions of characters long. | | Application | ports | Fake ports are injected (`vi.fn` per method): the fixtures and fake ports in [fakePorts.ts](../packages/application/src/testing/fakePorts.ts), and `NO_PORTS` ([noPorts.ts](../packages/application/src/testing/noPorts.ts)), which other packages' tests spread too, for the ports a test doesn't use. Each slice is tested in its own `.test.ts`, the stores the slices share in theirs. No module mocking. | | UI | `UseCases` prop | Screens render with a fake `UseCases`: `makeUseCasesFake` ([useCasesFake.ts](../packages/ui/src/test/useCasesFake.ts)) makes every slice's fake ([test/fakes/](../packages/ui/src/test/fakes/)) in one object, each calling the others through it. A screen that takes a slice (the trial's) can be given that slice's fake alone. Assertions use testing-library queries. Query-dependent components get a fresh `QueryClient` (retries off). | | Infrastructure mappers | none needed | Pure `Dataset`/`Subject` → domain functions, fed in-memory datasets built with `datasetFrom`/`buildThing` ([datasets.ts](../packages/solid/src/testing/datasets.ts)). | diff --git a/packages/solid-memo-client/src/resources.ts b/packages/solid-memo-client/src/resources.ts index 58cc6669b..f8f7b7698 100644 --- a/packages/solid-memo-client/src/resources.ts +++ b/packages/solid-memo-client/src/resources.ts @@ -167,7 +167,7 @@ function resourceOf(response: Response): Resource { * drops what was kept and rejects once, with a ReadLimitError (the * triples counted into `spent` too, when the read shares its limits * with others: limitedFetch), as - * `refuse` does for a text whose declarations or IRIs are past what a + * `refuse` does for a text whose declarations, IRIs or nesting are past what a * read takes in, with limits or with none (writeTurtle); nothing * more is kept, and `refused` says to stop the parser. A quad that * names an IRI kept for subjects not saved yet is tallied and not kept @@ -202,8 +202,8 @@ function collector( /** * The quads of a Turtle text, its relative IRIs resolved against - * `baseIRI`, under `limits` (collector), refused at a base, prefix or - * IRI past what any read takes in (writeTurtle). The text is written to + * `baseIRI`, under `limits` (collector), refused at a base, prefix, + * IRI or level of nesting past what any read takes in (writeTurtle). The text is written to * the parser in parts, so that it reads no further than the part it is * in. A text with an RDF 1.2 triple term does not parse: n3 is told to * read none (TURTLE_OPTIONS), and the parse fails at the first triple @@ -694,7 +694,7 @@ const MEDIA_TYPES = { turtle: TURTLE, jsonld: "application/ld+json" } as const; * against `baseUrl`, parsed as a pod's documents are: in every graph the * text names. Rejects with the parser's error on a text that does not * parse, and with a ReadLimitError at the first quad past `limits`, or - * on a Turtle text at its first base, prefix or IRI past what a read + * on a Turtle text at its first base, prefix, IRI or level of nesting past what a read * takes in, with limits or with none (writeTurtle). */ export async function parseQuads( diff --git a/packages/solid/src/client/interface.ts b/packages/solid/src/client/interface.ts index ffd4f9dac..8e08f6d77 100644 --- a/packages/solid/src/client/interface.ts +++ b/packages/solid/src/client/interface.ts @@ -167,8 +167,8 @@ export class ReadLimitError extends Error { ) { super( limits.bytes === Number.POSITIVE_INFINITY - ? `The document at [${url}] has more or longer base or prefix declarations, longer IRIs to resolve, or a longer term than a read takes in.` - : `The document at [${url}] is larger than a read takes in: ${limits.bytes} bytes, ${limits.quads} triples, ${limits.characters} characters of terms, ${limits.subjects === undefined ? "" : `${limits.subjects} subjects, `}a few base and prefix declarations, short IRIs to resolve and short terms.`, + ? `The document at [${url}] has more or longer base or prefix declarations, longer IRIs to resolve, deeper nesting, or a longer term than a read takes in.` + : `The document at [${url}] is larger than a read takes in: ${limits.bytes} bytes, ${limits.quads} triples, ${limits.characters} characters of terms, ${limits.subjects === undefined ? "" : `${limits.subjects} subjects, `}a few base and prefix declarations, short IRIs to resolve, a few levels of nesting and short terms.`, ); this.name = "ReadLimitError"; } diff --git a/packages/solid/src/client/linearDataset.ts b/packages/solid/src/client/linearDataset.ts index d7a2e548f..f565ca922 100644 --- a/packages/solid/src/client/linearDataset.ts +++ b/packages/solid/src/client/linearDataset.ts @@ -134,7 +134,8 @@ interface Collected { * The Turtle parser (turtleParser), whose quads go to `collected` * instead of to solid-client; one past the limits of the read of `url` (quadTally) * fails the parse instead, once, and drops what was kept, as does a - * base, prefix or IRI past what any read takes in (writeTurtle). One + * base, prefix or IRI past what any read takes in, or nesting deeper + * than any read takes in (writeTurtle). One * that names an IRI under LOCAL_NODE is tallied and not kept. The * text is written to the parser in parts, so that it reads no further * than the part it is in. diff --git a/packages/solid/src/client/readLimits.test.ts b/packages/solid/src/client/readLimits.test.ts index 0d1138f93..d1af81bd5 100644 --- a/packages/solid/src/client/readLimits.test.ts +++ b/packages/solid/src/client/readLimits.test.ts @@ -443,6 +443,49 @@ describe("writeTurtle", () => { expect(ended).toBe(false); }); + it("lets a text nest 64 levels, by collections, blank nodes, annotations or triple terms, and no more", () => { + for (const [opening, closing] of [ + ["(", ")"], + ["[

", "]"], + ["{|

", "|}"], + ["<< ", ">>"], + ]) { + const nested = (depth: number) => + ` ${opening!.repeat(depth)}${closing!.repeat(depth)} .\n`; + expect(refuses(nested(64)), opening).toBe(false); + expect(refuses(nested(65)), opening).toBe(true); + // Levels closed are left: one after another, they don't add up. + expect(refuses(nested(64).repeat(100)), opening).toBe(false); + } + // A level closed that was never opened makes no room for another. + expect(refuses(`${")".repeat(100)}${"(".repeat(65)}`)).toBe(true); + }); + + it("counts no level in a string, an IRI or a comment, as the parser does not", () => { + const quoted = [ + '"(((((((("', + "'[[[[[[[['", + '"""{{{{\n<<<<"""', + "", + "", + ]; + const cards = quoted.map((text) => ` ${text} .\n`).join(""); + expect(refuses(cards.repeat(20))).toBe(false); + expect(refuses("# ((((((((\n .\n".repeat(20))).toBe(false); + // What the parser reads as no IRI, nor a closing `>>`, it stops at. + expect(refuses(" > .\n".repeat(100))).toBe(false); + }); + + it("refuses a text at the part where it nests past the bound, and writes none of it", () => { + for (const opening of ["(", "[

"]) { + const text = ` ${opening.repeat(1_000_000)}`; + const { wrote, written, ended } = writing(text, () => true); + expect(wrote).toBe(false); + expect(written).toEqual([]); + expect(ended).toBe(false); + } + }); + it("refuses a long IRI with dot segments, and looks through short ones, in time that grows with the text, not its square", () => { // Eight times the text must take less than four times as long as an eighth of it done eight times over (testing.md). for (const text of [ @@ -480,6 +523,9 @@ describe("writeTurtle", () => { "'''\\'", // IRIs with no end, each up to the next. "> |} ] ) ", + " > ", ]) { const ratio = growth( (count) => { diff --git a/packages/solid/src/client/readLimits.ts b/packages/solid/src/client/readLimits.ts index be2a0fd77..bde72714c 100644 --- a/packages/solid/src/client/readLimits.ts +++ b/packages/solid/src/client/readLimits.ts @@ -32,8 +32,8 @@ export const RELEASE_LIMITS: ReadLimits = { * instance anyone may make can be added by its address or registered by * another app, a catalog may name a deck's documents on any server, and * the app keeps every triple of what it reads. The user's storages are - * read with no limits but on their declarations (writeTurtle) and on - * the length of each term (MAX_TERM_CHARACTERS). + * read with no limits but on their declarations and nesting + * (writeTurtle) and on the length of each term (MAX_TERM_CHARACTERS). */ export const FOREIGN_LIMITS: ReadLimits = RELEASE_LIMITS; @@ -97,10 +97,10 @@ export const MAX_PROFILE_DOCUMENTS = 16; /** * What a read with no limits is refused under, by the bounds every read - * keeps: a Turtle text's base and prefix declarations and its long IRIs - * that the parser resolves (writeTurtle), and the length of each term - * (MAX_TERM_CHARACTERS). Its body, its triples and the characters of - * all their terms are not bounded. + * keeps: a Turtle text's base and prefix declarations, its long IRIs + * that the parser resolves and how deep it nests (writeTurtle), and the + * length of each term (MAX_TERM_CHARACTERS). Its body, its triples and + * the characters of all their terms are not bounded. */ export const NO_SIZE_LIMITS: ReadLimits = { bytes: Number.POSITIVE_INFINITY, @@ -348,6 +348,18 @@ const MAX_PREFIX_DECLARATIONS = 4096; */ export const MAX_BASE_CHARACTERS = 512; +/** + * How deep a Turtle text may nest its collections (`(`), blank nodes + * (`[`), annotations (`{|`) and triple terms (`<<`), read under limits + * or with none. The parser keeps a context for each level it is in, and + * makes the triple that links a level to the one above it only once the + * level ends, so a text that only opens levels makes no triple for + * quadTally to count, and holds memory in the measure of its length. + * The app's own documents, and every release in the library, nest a few + * levels; the bound leaves a wide margin over that. + */ +const MAX_TURTLE_DEPTH = 64; + const SPACE = 0x20; const TAB = 0x09; const LF = 0x0a; @@ -369,9 +381,26 @@ const ANY_DOT_SEGMENT = /(?:^|\/)\.\.?(?:$|[/?#])/; /** * Where the search of a Turtle text stops to look: a string, an IRI, an - * escape, a comment, or what may start a directive. + * escape, a comment, a bracket, or what may start a directive. */ -const NEXT = /["'<\\#@bBpP]/g; +const NEXT = /["'<>\\#@bBpP()[\]{}]/g; + +/** How each bracket of one character, by its code, changes how deep a Turtle text nests: one that opens a level, or one that closes it. */ +const NESTING = new Map([ + [0x28, 1], // ( + [0x5b, 1], // [ + [0x7b, 1], // { + [0x29, -1], // ) + [0x5d, -1], // ] + [0x7d, -1], // } +]); + +/** How the bracket of two characters at `at` changes how deep a Turtle text nests: `<<` opens a level, `>>` closes one, and 0 where there is none. */ +function doubleBracketAt(text: string, at: number): number { + const code = text.charCodeAt(at); + if ((code !== LESS_THAN && code !== GREATER_THAN) || text.charCodeAt(at + 1) !== code) return 0; + return code === LESS_THAN ? 1 : -1; +} /** * Where an IRI the parser reads stops: at its `>`, or at what no IRI @@ -467,10 +496,11 @@ function stringEnd(text: string, at: number): number { * prefixes, a base whose path is not plain (plainBase), a base of more * than MAX_BASE_CHARACTERS, or an IRI longer than that which is * relative or may hold a dot segment (longIriPast), in a triple or a - * declaration. A document needs one base, a few prefixes and short - * relative IRIs, and the parser takes time in the square of a chain of - * relative bases, and of an IRI with dot segments, wherever the text is - * read from. Each call looks on from where the last stopped, so a text + * declaration, or nesting deeper than MAX_TURTLE_DEPTH. A document + * needs one base, a few prefixes, short relative IRIs and a few levels, + * and the parser takes time in the square of a chain of relative bases, + * and of an IRI with dot segments, and holds memory for every level it + * is in, wherever the text is read from. Each call looks on from where the last stopped, so a text * is looked through once, as far as it is written (writeTurtle), in * time linear in it. * @@ -483,7 +513,9 @@ function stringEnd(text: string, at: number): number { * differ, the search errs toward a refusal, or the parser stops at an * error before it. The longest base the parser can reach is then the * longest of the URL and the absolute IRIs, with every relative IRI - * added on. + * added on. A level opens at `(`, `[`, `{` and `<<`, and closes at `)`, + * `]`, `}` and `>>`; one closed that was never opened counts for none, + * so the depth found is never less than the parser's. */ function declarationsPast(text: string, url: string): (end: number) => boolean { const lineEnd = /[\n\r]/g; @@ -525,6 +557,14 @@ function declarationsPast(text: string, url: string): (end: number) => boolean { // The document's own address is its first base: a Turtle text at a longer one is refused, with limits or with none. let refused = url.length > MAX_BASE_CHARACTERS; + let depth = 0; + /** Where the search goes on, at `next`, a level opened (1) or closed (-1) before it. */ + const nest = (step: number, next: number): number => { + depth = Math.max(0, depth + step); + refused = depth > MAX_TURTLE_DEPTH; + return next; + }; + /** Where the search goes on after what starts at `at`, having checked a directive there. */ const lookAt = (at: number): number => { const code = text.charCodeAt(at); @@ -533,6 +573,10 @@ function declarationsPast(text: string, url: string): (end: number) => boolean { return lineEnd.exec(text)?.index ?? text.length; } if (code === BACKSLASH) return at + 2; + const single = NESTING.get(code); + if (single !== undefined) return nest(single, at + 1); + const double = doubleBracketAt(text, at); + if (double !== 0) return nest(double, at + 2); if (code === LESS_THAN) { const end = iriEnd(text, at); if (end < 0) return at + 1; @@ -574,10 +618,11 @@ export interface TextStream { * is written to it in parts (textChunks), then ended, while `goOn` says * to. A parser given the whole text goes on through all of it after the * tally is past its limits; here it reads no further than the part it - * is in. Before each part, the base and prefix declarations and the - * IRIs that start in it, or before, are checked (declarationsPast), - * with limits or with none: false when they are past what a read takes - * in, and the rest is not written; true otherwise. + * is in. Before each part, the base and prefix declarations, the IRIs + * and the levels that start in it, or before, are checked + * (declarationsPast), with limits or with none: false when they are + * past what a read takes in, and the rest is not written; true + * otherwise. */ export function writeTurtle( text: string, diff --git a/packages/solid/src/client/resources.ts b/packages/solid/src/client/resources.ts index 8699d8b55..1a25fd9c2 100644 --- a/packages/solid/src/client/resources.ts +++ b/packages/solid/src/client/resources.ts @@ -430,8 +430,8 @@ const MEDIA_TYPES = { turtle: "text/turtle", jsonld: "application/ld+json" } as * parse, as a Turtle text with a triple term does not (turtleParser) nor * JSON-LD with RDF-star, and with a ReadLimitError at the first quad past `limits` * (quadTally; the text's own size is its caller's to check: it holds the - * text already), or on a Turtle text at its first base, prefix or IRI - * past what a read takes in, with limits or with none (writeTurtle); the + * text already), or on a Turtle text at its first base, prefix, IRI or + * level of nesting past what a read takes in, with limits or with none (writeTurtle); the * parser reads no further. A quad that names an IRI the library keeps * for subjects not saved yet is left out (LOCAL_NODE). JSON-LD is read * with no fetch at all: a context named by its address is refused diff --git a/packages/solid/src/solidDeckArchive.ts b/packages/solid/src/solidDeckArchive.ts index f802c17a0..55f34d6ea 100644 --- a/packages/solid/src/solidDeckArchive.ts +++ b/packages/solid/src/solidDeckArchive.ts @@ -249,7 +249,8 @@ const DECK_FILE_LIMITS: ReadLimits = { * is larger than `limits` allow (its characters, which are no more than * its bytes, then its triples and their terms: quadTally); no limits for * a text from the user's own pod, which is then unreadable when its base - * or prefix declarations are past what any read takes in. A deck file is triples: statements in + * or prefix declarations are past what any read takes in, or it nests + * too deep (writeTurtle). A deck file is triples: statements in * a named graph (JSON-LD can say them, Turtle cannot) are not the file's * and are left out (GHSA-m4hj-7c28-fq4h). */ @@ -266,7 +267,7 @@ export function parseDeckFile( return parseQuads(checked, format, baseUrl, limits ?? undefined).then( (parsed) => parsed.filter((quad) => quad.graph.termType === "DefaultGraph"), (error: unknown) => { - // With no limits, a text is refused only for its declarations and IRIs (writeTurtle), not its size. + // With no limits, a text is refused only for its declarations, IRIs and nesting (writeTurtle), not its size. if (error instanceof ReadLimitError && limits !== null) throw tooLarge(); throw new AppError("deckFileUnreadable", { reason: String(error) }); }, diff --git a/packages/solid/src/solidReleasePublisher.ts b/packages/solid/src/solidReleasePublisher.ts index 05bbaa3d2..bd1392edd 100644 --- a/packages/solid/src/solidReleasePublisher.ts +++ b/packages/solid/src/solidReleasePublisher.ts @@ -55,8 +55,8 @@ import { RDF, SM } from "./vocab"; * Its access control, which the grant reads before writing it, is read * under ACCESS_CONTROL_LIMITS for the same reason, and the release * itself, should the grant read it, under RELEASE_LIMITS: their bytes, - * their triples, their characters and their base and prefix - * declarations, and an access control's subjects, to + * their triples, their characters, their base and prefix declarations + * and how deep they nest, and an access control's subjects, to * MAX_ACCESS_CONTROL_SUBJECTS (accessControlFetch). */ export function createSolidReleasePublisher({ @@ -205,8 +205,8 @@ function statementsOf(quads: readonly Quad[], url: string): string { * anyone's, and the grant's library parses what it reads with no * limits, so every answer is read under limits: its body on its bytes, * and the body of a read that succeeds is parsed first under them, on - * its triples, their characters and its base and prefix declarations, - * then handed on as it was. A read of the release itself, or of a + * its triples, their characters, its base and prefix declarations and + * how deep it nests, then handed on as it was. A read of the release itself, or of a * folder above it, which a grant may make when its server answers no * HEAD, is held to RELEASE_LIMITS, as any release is; every other read, * of an access control, to ACCESS_CONTROL_LIMITS, far smaller, and to diff --git a/packages/solid/src/testing/contract/resources.ts b/packages/solid/src/testing/contract/resources.ts index 21398283b..9f2557399 100644 --- a/packages/solid/src/testing/contract/resources.ts +++ b/packages/solid/src/testing/contract/resources.ts @@ -317,7 +317,7 @@ export function resourcesContract({ rdf, resources }: DataClient): void { url: URL_, limits: NO_SIZE_LIMITS, message: expect.stringContaining( - "declarations, longer IRIs to resolve, or a longer term than a read takes in", + "declarations, longer IRIs to resolve, deeper nesting, or a longer term than a read takes in", ), }); for (const read of [ @@ -469,6 +469,43 @@ export function resourcesContract({ rdf, resources }: DataClient): void { } }); + it("refuses a document nested deeper than 64 levels, with limits or with none, and reads one nested 64 deep", async () => { + // Levels that are opened and never closed make no triple for the limits to count. + for (const opening of ["(", `[ <${EX}p> `]) { + const past = `<${EX}s> <${EX}p> ${opening.repeat(1_000_000)}`; + const fetch = limitedFetch(turtleAt(past), RELEASE_LIMITS); + for (const read of [ + () => getSolidDatasetLinear(URL_, { fetch }), + () => readDocument(URL_, { fetch }), + () => parseQuads(past, "turtle", URL_, RELEASE_LIMITS), + ]) { + await expect(read()).rejects.toMatchObject({ + name: "ReadLimitError", + url: URL_, + limits: RELEASE_LIMITS, + }); + } + // A text from the user's own pod, read with no limits, is held to as few levels. + for (const read of [ + () => parseQuads(past, "turtle", URL_), + () => getSolidDatasetLinear(URL_, { fetch: turtleAt(past) }), + () => getSolidDataset(URL_, { fetch: turtleAt(past) }), + ]) { + await expect(read()).rejects.toMatchObject({ name: "ReadLimitError", url: URL_ }); + } + } + // A list of lists, each with a blank node in it. + const nested = `${`( [ <${EX}p> `.repeat(32)}"v"${" ] )".repeat(32)}`; + const within = `<${EX}s> <${EX}p> ${nested} .\n`; + for (const limits of [undefined, RELEASE_LIMITS]) { + const quads = await parseQuads(within, "turtle", URL_, limits); + expect(quads.filter(({ object }) => object.value === "v")).toHaveLength(1); + } + expect( + getThing(await getSolidDatasetLinear(URL_, { fetch: turtleAt(within) }), `${EX}s`), + ).not.toBeNull(); + }); + it("refuses a long IRI with dot segments in time that grows with it, not its square", async () => { const dotted = (count: number) => `<${"x/".repeat(count)}${"y/../".repeat(count)}> <${EX}p> "v" .\n`; @@ -575,7 +612,8 @@ export function resourcesContract({ rdf, resources }: DataClient): void { const annotated = `<#a> <${EX}p> <#o> {| <${EX}q> <#r> |} .`; for (const head of [ `<#a> <${EX}p> ${nested(1)} .`, - `<#a> <${EX}p> ${nested(100)} .`, + // Deeper, the text is refused at its nesting (MAX_TURTLE_DEPTH) before the parser reads it. + `<#a> <${EX}p> ${nested(30)} .`, annotated, ]) { const text = `${head}\n${rest}`;