diff --git a/docs/markdown.md b/docs/markdown.md index 5ea53e86..2902cc54 100644 --- a/docs/markdown.md +++ b/docs/markdown.md @@ -98,7 +98,8 @@ package's own tree, folded to what the app shows: Every entry point for data is bounded, and a text past a limit is plain, its source still readable, unless the limit says otherwise. -[Repository Markdown](#repository-markdown) lifts only `MAX_CHARS`. +[Repository Markdown](#repository-markdown) lifts only `MAX_CHARS` +and `MAX_LOOKAHEAD`. The limits count lines as the parser does: a line ends at a line feed, a carriage return, or both, and only spaces and tabs make it blank. @@ -130,6 +131,34 @@ blank. uncounted. That is at most one for each quote it counts, so a text it lets through opens at most about twice `MAX_QUOTES`, still a bounded cost. The parser is quadratic in them too. +- **`MAX_LISTS`, 200 lists a text opens.** A list's next item opens + none, and neither do the lines its items hold. A line indented less + than an item's content leaves the item. An item with another marker + (`-`, `+`, `*`, `.` or `)`) starts another list, and so do a + thematic break and a change in the quotes a line starts with. An + item can start with at most one blank line, so an item whose marker + line is empty ends at a blank line right after it; only an item with + the same marker goes on with its list then. An empty item, or an + ordered one that starts other than at 1, can't interrupt a + paragraph, so it opens no list while one is open. A paragraph counts + as open only from a line of plain words after a blank line or at the + start, under the same `>`s. A marker counts wherever it is, in + fenced code, indented code or an HTML block too, and a lazy line + leaves the list, so the count errs high: a fence holding a long diff + can pass the limit. The parser is quadratic in lists, such as `+ a`, + a blank line and `b`, repeated. +- **`MAX_LOOKAHEAD`, 500,000 characters read ahead for the end of a + code span, inline HTML or a link's title.** An opener that never + closes is read to the end of its paragraph. Each opener counts the + characters from it to its closer, or to the next blank line when + none comes, and the counts add up over the text. A backtick run + closes at the next run of its length, ``, ``, + ``, and ``. A link's title + closes at its quote or `)`, and a link with no title counts to the + end of its destination. Each backtick run counts, a closer too, so + the count errs high, but closed spans count about once over. The + parser is quadratic in openers that never close. No text in `decks/` + comes near it. - **`MAX_UNDERLINES`, 200 lines that could underline a heading** (`===`, `---`, with up to three spaces before and white space after). The parser is quadratic in setext headings too (`a\n=\n…` at 20,000 @@ -277,15 +306,17 @@ input at build time only, never for data: - **Headings keep their level** (`{ type: "heading", depth, line }`) and their line, for a message about one. Other blocks are as `parseMarkdown` gives them. -- **`MAX_CHARS` is lifted, and only it.** `studio.md` is about 130 KB, - `data-model.md` about 70 KB. The docs come nowhere near the other +- **`MAX_CHARS` is lifted, and with it `MAX_LOOKAHEAD`, which grows + with a page's length.** `studio.md` is about 130 KB, `data-model.md` + about 70 KB. The docs come nowhere near the other [limits](#limits). - **What data would fold, it refuses.** Raw HTML, a picture, a link reference definition, what nests past `MAX_DEPTH`, a table past its caps, and a text past `MAX_LINE_NESTING`, `MAX_NESTED_LINES`, - `MAX_LAZY_LINES`, `MAX_QUOTES`, `MAX_DELIMITERS` or `MAX_UNDERLINES` - each throw a `DocumentError`, naming the problem's first line when - known. A page shows as written, or its build fails and says where. + `MAX_LAZY_LINES`, `MAX_QUOTES`, `MAX_LISTS`, `MAX_DELIMITERS` or + `MAX_UNDERLINES` each throw a `DocumentError`, naming the problem's + first line when known. A page shows as written, or its build fails + and says where. ## Diagrams diff --git a/packages/markdown/src/parse.test.ts b/packages/markdown/src/parse.test.ts index 15ba752a..50dff7fc 100644 --- a/packages/markdown/src/parse.test.ts +++ b/packages/markdown/src/parse.test.ts @@ -9,6 +9,8 @@ import { MAX_DEPTH, MAX_LAZY_LINES, MAX_LINE_NESTING, + MAX_LISTS, + MAX_LOOKAHEAD, MAX_NESTED_LINES, MAX_QUOTES, MAX_TABLE_CELLS, @@ -365,6 +367,212 @@ describe("parseMarkdown", () => { ).not.toBeNull(); }); + it("reads nothing past MAX_LOOKAHEAD characters from openers that never close to the next blank line", () => { + // k openers, one every u characters from o, in one paragraph: u k (k + 1) / 2 - k o characters ahead. + const ahead = (u: number, o: number) => (k: number) => (u * k * (k + 1)) / 2 - k * o; + for (const [unit, cost] of [ + [" ahead(3, 1)(k) + 3 * k - 1], + ] as const) { + let fit = 0; + while (cost(fit + 1) <= MAX_LOOKAHEAD) fit++; + expect(parseMarkdown(unit.repeat(fit)), JSON.stringify(unit)).not.toBeNull(); + expect(parseMarkdown(unit.repeat(fit + 1)), JSON.stringify(unit)).toBeNull(); + } + const fit = Math.floor((Math.sqrt(1 + (8 * MAX_LOOKAHEAD) / 3) - 1) / 2); + // Each block's openers count to its end, and every block counts. + expect(parseMarkdown(`${" { + // A text whose openers read `count` characters ahead: k ` { + let k = Math.floor((Math.sqrt(1 + (8 * count) / 3) - 1) / 2); + let rest = count - (3 * k * (k + 1)) / 2; + if (rest === 1) rest += 3 * k--; + const twos = [0, 2, 1][rest % 3]!; + return [ + "(twos).fill("((rest - 2 * twos) / 3).fill("", 7], + ["", 2], + [" b -->", 14], + ["", 9], + ["\n", + ]) + for (const unit of ["+ a\n\nb\n", "> - a\n>\n> b\n"]) { + const markdown = (count: number) => head + unit.repeat(count); + expect(parseMarkdown(markdown(MAX_LISTS - 1)), JSON.stringify(head + unit)).not.toBeNull(); + expect(parseMarkdown(markdown(MAX_LISTS + 1)), JSON.stringify(head + unit)).toBeNull(); + } + }); + + it("counts lists after an indented fence, whatever the item above it", () => { + const unit = "+ a\n\nb\n"; + // A thematic break, a quote or a lazy line in an item: the fence after it is still the item's. + for (const head of ["- a\n ***\n", "- a\n > b\n", "- a\nb\n"]) { + const markdown = (count: number) => `${head} \`\`\`\nx\n\n${unit.repeat(count)}`; + expect(parseMarkdown(markdown(MAX_LISTS - 1)), JSON.stringify(head)).not.toBeNull(); + expect(parseMarkdown(markdown(MAX_LISTS)), JSON.stringify(head)).toBeNull(); + expect(parseMarkdown(markdown(MAX_LISTS * 2)), JSON.stringify(head)).toBeNull(); + } + }); + + it("counts no list for a marker that cannot interrupt a paragraph, and every other", () => { + const lists = (head: string, unit: string, count: number) => + parseMarkdown(head + unit.repeat(count)); + // An empty item, or an ordered one that starts other than at 1, is a line of the paragraph. + for (const unit of ["2. c\nb\n", "2) c\n", "01. c\n", "+\nb\n", "* \n", "1.\n", "10. c\n b\n"]) + expect(lists("a\n", unit, MAX_LISTS * 2), JSON.stringify(unit)).not.toBeNull(); + expect(lists("a\n\nb\n", "2. c\n", MAX_LISTS * 2)).not.toBeNull(); + // Where a paragraph is not open, or not surely, each marker that opens a list counts. + for (const [unit, per] of [ + ["a\n- c\n\n", 1], + ["a\n1. c\n\n", 1], + ["a\n\n2. c\n\n", 1], + ["# a\n2. c\n\n", 1], + ["a\n-\n2. c\n\n- d\n\n", 2], + ["> a\nb\n2) c\n- d\n", 2], + ["- ```\n a\n2. c\n", 2], + ["\n2. c\n\n", 1], + ] as const) { + expect(lists("", unit, MAX_LISTS / per), JSON.stringify(unit)).not.toBeNull(); + expect(() => parseDocument(unit.repeat(MAX_LISTS / per + 1)), JSON.stringify(unit)).toThrow( + `more than MAX_LISTS (${MAX_LISTS}) lists`, + ); + } + }); + it("reads nothing past MAX_DELIMITERS asterisks and underscores", () => { expect(parseMarkdown("*_".repeat(MAX_DELIMITERS / 2))).not.toBeNull(); expect(parseMarkdown(`${"*_".repeat(MAX_DELIMITERS / 2)}*`)).toBeNull(); @@ -375,7 +583,7 @@ describe("parseMarkdown", () => { parseMarkdown("a\n=\n".repeat(MAX_UNDERLINES / 2) + "b\n---\n".repeat(MAX_UNDERLINES / 2)), ).not.toBeNull(); expect(parseMarkdown(`${"a\n=\n".repeat(MAX_UNDERLINES)} --- `)).toBeNull(); - expect(parseMarkdown("a\n- b\n".repeat(MAX_UNDERLINES + 1))).not.toBeNull(); + expect(parseMarkdown("a\n- b\n- c\n".repeat(MAX_UNDERLINES / 2 + 1))).not.toBeNull(); }); it("reads nothing above MAX_CHARS", () => { @@ -439,6 +647,12 @@ describe("parseDocument", () => { ]); }); + it("reads a text past MAX_LOOKAHEAD, which grows with a page as MAX_CHARS does", () => { + const page = `a${" { const page = "A line of a long page, [linked](https://example.org/).\n\n".repeat(2_500); expect(page.length).toBeGreaterThan(130_000); @@ -522,6 +736,10 @@ describe("parseDocument", () => { what: `more than MAX_LAZY_LINES (${MAX_LAZY_LINES}) lazy continuation lines in a row`, line: MAX_LAZY_LINES + 2, }); + expect(refusal("+ a\n\nb\n".repeat(MAX_LISTS + 1))).toMatchObject({ + what: `more than MAX_LISTS (${MAX_LISTS}) lists`, + line: MAX_LISTS * 3 + 1, + }); }); }); @@ -744,6 +962,24 @@ describe("pathological input", () => { ["backtick runs", fill("`a``")], ["entities", fill("&aaaaaaaa; ")], ["unclosed HTML comments", fill("`, ``, `` and ``. A link reads to its title's closer, or past its destination when + * it has no title (see linkRead). Counted high, as each run counts, + * closers too, and an opener in code or a closed span counts as well. + */ +function readFrom(text: string, at: number, opener: string): number { + if (opener === "](") return linkRead(text, at + 2) - at; + if (opener === "` first comes from `from`, or the paragraph + * ends, whichever is first. The parser closes a comment at `-->` only, + * as CommonMark does, not at the `--!>` a browser also takes, so a count + * that stopped there would count low. Found as a `--` before a `>`. + */ +function commentEnd(text: string, from: number): number { + let end = closerAt(text, from, "--"); + while (text.startsWith("--", end) && text[end + 2] !== ">") end = closerAt(text, end + 1, "--"); + return end; +} + +/** White space that may come between a link's parts: spaces, tabs and at most one line break. */ +const LINK_SPACE = /[ \t]*(?:(?:\r\n?|\n)[ \t]*)?/y; + +/** Where white space that may come between a link's parts ends, from `from`. */ +function spaceAfter(text: string, from: number): number { + LINK_SPACE.lastIndex = from; + LINK_SPACE.exec(text); + return LINK_SPACE.lastIndex; +} + +/** + * Where the parser stops reading a link from after its `](`: at its + * title's closer, or the paragraph's end when none comes; past its + * destination when no title follows. A title, in `"`, `'` or `(`, comes + * after the destination and white space. A destination in `<` and `>` + * ends at its first unescaped `>`, and has no title when a line break or + * an unescaped `<` comes first. Any other ends at white space, or at a + * `)` outside the parentheses it opens, which ends the link. Counted + * high: a destination of too many parentheses, or of a control + * character, holds no title to the parser. + */ +function linkRead(text: string, from: number): number { + let end = spaceAfter(text, from); + if (text[end] === "<") { + end = closerAt(text, end + 1, `${UNESCAPED}[<>\\r\\n]`); + if (text[end] !== ">") return end; + end++; + } else { + let depth = 0; + for (; end < text.length && !/[ \t\r\n]/.test(text[end]!); end++) { + const character = text[end]; + if (character === "\\" && /[()\\]/.test(text[end + 1] ?? "")) end++; + else if (character === "(") depth++; + else if (character === ")" && depth-- === 0) return end; + } + } + const title = spaceAfter(text, end); + const closer = { '"': '"', "'": "'", "(": String.raw`\)` }[text[title] ?? ""]; + if (title === end || closer === undefined) return title; + return closerAt(text, title + 1, UNESCAPED + closer); +} + +/** The block quotes' `>`s a line starts with: a change in how many leaves every list. */ +const QUOTE_PREFIX = /^(?: {0,3}>[ \t]?)*/; +/** A list item's marker, after its indentation, and the white space after it. */ +const LIST_MARKER = /^([ \t]*)([-+*]|\d{1,9}[.)])([ \t]+|$)/; +/** + * A line of a paragraph's plain words: one that starts with a letter or + * digit and holds no `|`, `<` or `>`, so it is no table row, HTML or a + * line that ends an HTML block. + */ +const PLAIN = /^[ \t]*[\p{L}\p{N}][^|<>]*$/u; + +/** + * Where a text opens more than MAX_LISTS lists, if it does. The lists + * open at a line are kept as their marker and the column their items' + * content starts at. A marker goes on with a list when it is of the same + * kind and not indented into that list's item; a line indented less than + * an item's content leaves it; a change in the quotes a line starts + * with, a blank line out of a quote and a thematic break leave every + * list. An item may start with at most one blank line, so a blank line + * right after an item whose marker line has no content ends that item: + * only a marker of its list's kind goes on with the list then. A marker + * that would open a list but cannot interrupt a paragraph (an empty + * item, or an ordered one that starts other than at 1) opens none while + * a paragraph is open, and is a line of it. Counted high: a lazy line + * leaves too, a marker in fenced code, indented code or an HTML block + * opens one, and a paragraph counts as open only where it surely is + * (see paragraphAfter). + */ +function listsPast(lines: string[]): { what: string; line: number } | null { + let lists = 0; + let prefix = ""; + // Each open list's last item ends (`ended`) at a blank line right after its empty marker line. + let open: OpenList[] = []; + let empty: { ended: boolean } | null = null; // the item opened empty on the line above + let paragraph: Paragraph = "none"; + for (const [index, line] of lines.entries()) { + const quotes = QUOTE_PREFIX.exec(line)![0]; + if (quotes.split(">").length !== prefix.split(">").length) open = []; + // Under other `>`s, or the same ones written otherwise, the paragraph may have ended. + if (quotes !== prefix) paragraph = "quiet"; + prefix = quotes; + const rest = line.slice(quotes.length); + const above = empty; + empty = null; + if (/^[ \t]*$/.test(rest)) { + if (above !== null) above.ended = true; + paragraph = "none"; + continue; + } + if (THEMATIC_BREAK.test(rest)) { + open = []; + paragraph = "quiet"; + continue; + } + const marker = LIST_MARKER.exec(rest); + const indent = indentOf(rest); + const place = marker === null ? null : placeItem(open, indent, marker); + if (marker === null || (paragraph === "open" && !place!.goesOn && !interrupts(marker, rest))) { + while (open.length > 0 && (open.at(-1)!.ended || open.at(-1)!.content > indent)) open.pop(); + paragraph = paragraphAfter(paragraph, rest, marker !== null); + continue; + } + open.length = place!.keep; + const item = enterItem(open, place!.goesOn, rest, marker); + if (!place!.goesOn && ++lists > MAX_LISTS) + return { what: `more than MAX_LISTS (${MAX_LISTS}) lists`, line: index + 1 }; + if (marker[0].length === rest.length) empty = item; + paragraph = "quiet"; + } return null; } +/** + * Whether a paragraph is open under a line's `>`s, as listsPast keeps + * it: `open` where one surely is, `none` after a blank line or at the + * start, where the next plain line starts one, and `quiet` anywhere + * else, till a blank line. Only after a blank line is no paragraph + * left for that line to go on with lazily: where it does, under fewer + * `>`s or less indented than an item, the parser lets any marker on the + * lines after open a list. + */ +type Paragraph = "open" | "quiet" | "none"; + +/** + * Whether a paragraph is open after a line that is no list item, or a + * marker that cannot interrupt it (`skipped`). A plain line starts one + * only unindented, as an indented line may be code in an item that the + * next line leaves, and goes on with an open one at any indentation; a + * dash alone underlines a heading; any other line may end it. + */ +function paragraphAfter(paragraph: Paragraph, rest: string, skipped: boolean): Paragraph { + if (skipped) return /^ {0,3}-[ \t]*$/.test(rest) ? "quiet" : paragraph; + if (!PLAIN.test(rest)) return "quiet"; + if (paragraph === "open" || (paragraph === "none" && indentOf(rest) === 0)) return "open"; + return paragraph; +} + +/** Whether a marker may start a list where it interrupts a paragraph: with content, and if ordered, from 1. */ +function interrupts(marker: RegExpExecArray, rest: string): boolean { + return ( + marker[0].length < rest.length && (/^[-+*]$/.test(marker[2]!) || /^1[.)]$/.test(marker[2]!)) + ); +} + +/** A list open at a line, as listsPast keeps it. */ +type OpenList = { kind: string; content: number; ended: boolean }; + +/** + * Where an item at `indent` with `marker` goes among the lists `open`: + * how many of them it keeps, and whether it goes on with the last of + * those or opens a list in it. + */ +function placeItem( + open: readonly OpenList[], + indent: number, + marker: RegExpExecArray, +): { keep: number; goesOn: boolean } { + // A bullet list's kind is its bullet, an ordered list's its `.` or `)`. + const kind = marker[2]!.at(-1)!; + for (let keep = open.length; ; keep--) { + const list = open[keep - 1]; + if (list === undefined || (!list.ended && indent >= list.content)) + return { keep, goesOn: false }; + if (list.kind === kind && indent >= (open[keep - 2]?.content ?? 0)) + return { keep, goesOn: true }; + } +} + +/** + * Puts the item a line's `marker` starts into the lists `open`, which + * placeItem has cut to the ones it keeps: on with the last, or into a + * list it opens. The list it is in. + */ +function enterItem( + open: OpenList[], + goesOn: boolean, + rest: string, + marker: RegExpExecArray, +): OpenList { + const end = marker[1]!.length + marker[2]!.length; + const after = columnAfter(rest, 0, end, 0); + const space = columnAfter(rest, end, marker[0].length, after) - after; + const blank = marker[0].length === rest.length; + // The content starts after one to four columns of white space, or one past the marker. + const content = after + (!blank && space >= 1 && space <= 4 ? space : 1); + if (goesOn) { + const list = open.at(-1)!; + list.content = content; + list.ended = false; + return list; + } + const list = { kind: marker[2]!.at(-1)!, content, ended: false }; + open.push(list); + return list; +} + /** * The destination of each link reference definition, by its normalized * label, the first one winning, as CommonMark has it. Definitions may