diff --git a/lib/html-to-markdown.js b/lib/html-to-markdown.js new file mode 100644 index 000000000..0e12d4e1b --- /dev/null +++ b/lib/html-to-markdown.js @@ -0,0 +1,144 @@ +// Converts the HTML Asciidoctor renders for a page body into Markdown. +// +// Asciidoctor lays out admonitions and callout lists as tables, so without the +// rules below they would come out as Markdown tables with an empty icon column. +const domino = require('@mixmark-io/domino') +const TurndownService = require('turndown') +const { tables } = require('@joplin/turndown-plugin-gfm') + +const ADMONITION_LABELS = { + note: 'Note', + tip: 'Tip', + important: 'Important', + warning: 'Warning', + caution: 'Caution', +} + +function hasClass (node, name) { + return node.nodeType === 1 && (node.getAttribute('class') || '').split(/\s+/).includes(name) +} + +// domino, the DOM turndown runs on in Node, returns lists that are not iterable. +function elements (node, tag) { + return Array.from(node.getElementsByTagName(tag)) +} + +function quote (text) { + return text + .trim() + .split('\n') + .map((line) => (line ? `> ${line}` : '>')) + .join('\n') +} + +function fence (text, info = '') { + const longestRun = Math.max(0, ...(text.match(/`+/g) || []).map((run) => run.length)) + const marker = '`'.repeat(Math.max(3, longestRun + 1)) + return `\n\n${marker}${info}\n${text.replace(/\n$/, '')}\n${marker}\n\n` +} + +// Rewrites the DOM before turndown sees it. Links and images become absolute, +// because llms-full.txt has no directory to resolve relative ones from, and a +// link to a page in markdownUrls goes to its Markdown version instead. The +// Mermaid source becomes a pre, the only element whose whitespace turndown +// keeps; its
s in node labels arrive as elements and are turned back into text. +function prepare (html, pageUrl, markdownUrls) { + const document = domino.createDocument(`${html}`) + const base = /^https?:/.test(pageUrl || '') ? pageUrl : undefined + const absolute = (url) => (!base || /^[a-z][a-z0-9+.-]*:/i.test(url) ? url : new URL(url, base).href) + for (const [tag, attribute] of [['a', 'href'], ['img', 'src']]) { + for (const node of elements(document.body, tag)) { + const value = node.getAttribute(attribute) + if (value) node.setAttribute(attribute, absolute(value)) + } + } + for (const node of elements(document.body, 'a')) { + const [url, fragment] = (node.getAttribute('href') || '').split('#') + const markdownUrl = markdownUrls.get(url) + if (markdownUrl) node.setAttribute('href', fragment === undefined ? markdownUrl : `${markdownUrl}#${fragment}`) + } + for (const node of elements(document.body, 'div').filter((div) => hasClass(div, 'mermaid'))) { + for (const br of elements(node, 'br')) br.parentNode.replaceChild(document.createTextNode('
'), br) + const pre = document.createElement('pre') + pre.setAttribute('class', 'mermaid') + pre.textContent = node.textContent + node.parentNode.replaceChild(pre, node) + } + return document.body +} + +function createConverter (markdownUrls = new Map()) { + const turndown = new TurndownService({ + headingStyle: 'atx', + codeBlockStyle: 'fenced', + bulletListMarker: '-', + emDelimiter: '_', + }) + turndown.use(tables) + + // Section anchors are empty links that only carry a CSS glyph. A rule, not + // remove(), because turndown consults remove() only after its own link rule. + turndown.addRule('sectionAnchor', { + filter: (node) => node.nodeName === 'A' && hasClass(node, 'anchor'), + replacement: () => '', + }) + + // Asciidoctor marks the language with data-lang; the class turndown reads by + // default says "language-none" on untagged blocks. Literal blocks are a bare pre. + turndown.addRule('codeBlock', { + filter: (node) => node.nodeName === 'PRE', + replacement: (_content, node) => { + if (hasClass(node, 'mermaid')) return fence(node.textContent.trim(), 'mermaid') + const code = node.firstChild && node.firstChild.nodeName === 'CODE' ? node.firstChild : node + return fence(code.textContent, code.getAttribute('data-lang') || '') + }, + }) + + // A tab set renders as a list of labels followed by the panels, so each panel + // carries its own label or the alternatives read as consecutive steps. + turndown.addRule('tabs', { + filter: (node) => node.nodeName === 'DIV' && hasClass(node, 'tabs'), + replacement: (_content, node) => { + const labels = new Map(elements(node, 'li').map((li) => [li.getAttribute('id'), li.textContent.trim()])) + const panels = elements(node, 'div').filter( + (div) => hasClass(div, 'tabpanel') && div.parentNode.parentNode === node + ) + const sections = panels.map((panel) => { + const label = labels.get(panel.getAttribute('aria-labelledby')) || '' + return `**Tab: ${label}**\n\n${turndown.turndown(panel.innerHTML)}` + }) + return `\n\n${sections.join('\n\n')}\n\n` + }, + }) + + turndown.addRule('admonition', { + filter: (node) => node.nodeName === 'DIV' && hasClass(node, 'admonitionblock'), + replacement: (_content, node) => { + const type = Object.keys(ADMONITION_LABELS).find((name) => hasClass(node, name)) + const cell = elements(node, 'td').find((td) => hasClass(td, 'content')) + const body = cell ? turndown.turndown(cell.innerHTML) : '' + return `\n\n${quote(`**${ADMONITION_LABELS[type] || 'Note'}:** ${body}`)}\n\n` + }, + }) + + turndown.addRule('calloutList', { + filter: (node) => node.nodeName === 'DIV' && hasClass(node, 'colist'), + replacement: (_content, node) => { + const rows = elements(node, 'tr').map((row) => { + const [marker, text] = elements(row, 'td') + return `(${marker.textContent.trim()}) ${turndown.turndown(text.innerHTML).replace(/\n+/g, ' ')}` + }) + return `\n\n${rows.join('\n')}\n\n` + }, + }) + + turndown.addRule('blockTitle', { + filter: (node) => node.nodeName === 'DIV' && hasClass(node, 'title'), + replacement: (content) => `\n\n**${content.trim()}**\n\n`, + }) + + // Paragraphs inside list items leave whitespace-only lines behind. + return (html, pageUrl) => turndown.turndown(prepare(html, pageUrl, markdownUrls)).replace(/[ \t]+$/gm, '') +} + +module.exports = { createConverter } diff --git a/lib/llms-txt.js b/lib/llms-txt.js index 274db7e4e..4ed65c809 100644 --- a/lib/llms-txt.js +++ b/lib/llms-txt.js @@ -1,21 +1,41 @@ // Generates /llms.txt following https://llmstxt.org/ so that AI agents can -// discover the documentation without scraping HTML. +// discover the documentation without scraping HTML, plus a Markdown copy of +// every indexed page next to its HTML (index.md beside index.html, as the spec +// proposes) and /llms-full.txt, all of those pages in one file. llms-full.txt +// is a widespread convention rather than part of the spec. // // For every component, only the version that 'stable' points to is indexed // (the latest release, or nightly when building without release branches). -// Each entry carries the published page URL, the page description and a link +// Each entry carries the Markdown page URL, the page description and a link // to the AsciiDoc source on GitHub, so agents can both read a page and find // the file to change. // +// A page with the attribute `:page-llms-ignore:` is left out of all of it. +// // Useful links: // Extensions: https://docs.antora.org/antora/latest/extend/extensions/ // Types of events: https://docs.antora.org/antora/latest/extend/generator-events-reference/ +const { createConverter } = require('./html-to-markdown') // Pages on how Stackable builds SDP go under "Optional", the llms.txt convention // for links an agent can skip when context is short (https://llmstxt.org/). +// They are left out of llms-full.txt for the same reason. const OPTIONAL_MODULES = new Set(['contributor', 'internal']) module.exports.register = function () { + const markdown = new Map() + + // Page contents are the bare article HTML here; once pages are composed they + // carry the whole site layout, navigation included. + this.once('documentsConverted', ({ playbook, contentCatalog }) => { + const siteUrl = (playbook.site.url || '').replace(/\/$/, '') + const pages = indexedPages(contentCatalog) + const convert = createConverter(new Map(pages.map((page) => [`${siteUrl}${page.pub.url}`, markdownUrl(page, siteUrl)]))) + for (const page of pages) { + markdown.set(page, convert(page.contents.toString(), `${siteUrl}${page.pub.url}`)) + } + }) + this.once('beforePublish', ({ playbook, contentCatalog, siteCatalog }) => { const siteUrl = (playbook.site.url || '').replace(/\/$/, '') const lines = [ @@ -28,24 +48,24 @@ module.exports.register = function () { 'Machine-readable release, component and CRD data is available from the', 'Stackable Hub: https://hub.stackable.tech/llms.txt', '', - 'Every entry below links the published page and, in parentheses, its AsciiDoc', + 'Every entry below links the page as Markdown and, in parentheses, its AsciiDoc', 'source on GitHub. To suggest a change to a page, edit that source file.', + `All pages outside "Optional" are also available in one file: ${siteUrl}/llms-full.txt`, ] const optional = [] + const full = [] for (const component of contentCatalog.getComponents()) { - const version = component.latest - const pages = contentCatalog - .findBy({ component: component.name, version: version.version, family: 'page' }) - .filter((page) => page.out) - .sort((a, b) => a.pub.url.localeCompare(b.pub.url)) const byModule = new Map() - for (const page of pages) { + for (const page of indexedPages(contentCatalog, component)) { + const body = markdown.get(page) + siteCatalog.addFile({ contents: Buffer.from(markdownPage(page, body), 'utf8'), out: { path: markdownPath(page) } }) const module = page.src.module if (OPTIONAL_MODULES.has(module)) { optional.push(page) continue } + full.push(markdownPage(page, body, `${siteUrl}${page.pub.url}`)) if (!byModule.has(module)) byModule.set(module, []) byModule.get(module).push(page) } @@ -60,21 +80,72 @@ module.exports.register = function () { for (const page of optional) lines.push(entryFor(page, siteUrl)) } - siteCatalog.addFile({ - contents: Buffer.from(lines.join('\n') + '\n', 'utf8'), - out: { path: 'llms.txt' }, - }) + siteCatalog.addFile({ contents: Buffer.from(lines.join('\n') + '\n', 'utf8'), out: { path: 'llms.txt' } }) + siteCatalog.addFile({ contents: Buffer.from(full.join('\n\n'), 'utf8'), out: { path: 'llms-full.txt' } }) }) } -function entryFor (page, siteUrl) { - const title = decodeEntities( +// In the order of the site navigation, which is the reading order the authors +// chose; pages outside it follow, sorted by URL. The navigation only exists from +// navigationBuilt on, so earlier callers get URL order. +function indexedPages (contentCatalog, component) { + const components = component ? [component] : contentCatalog.getComponents() + return components.flatMap((it) => { + const rank = navigationRank(it.latest.navigation || []) + const position = (page) => (rank.has(page.pub.url) ? rank.get(page.pub.url) : Infinity) + return contentCatalog + .findBy({ component: it.name, version: it.latest.version, family: 'page' }) + .filter((page) => page.out && !('page-llms-ignore' in page.asciidoc.attributes)) + .sort((a, b) => position(a) - position(b) || a.pub.url.localeCompare(b.pub.url)) + }) +} + +function navigationRank (navigation) { + const rank = new Map() + const visit = (items) => { + for (const item of items) { + const url = item.urlType === 'internal' && item.url.split('#')[0] + if (url && !rank.has(url)) rank.set(url, rank.size) + visit(item.items || []) + } + } + visit(navigation) + return rank +} + +function markdownPath (page) { + return page.out.path.replace(/\.html$/, '.md') +} + +function markdownUrl (page, siteUrl) { + return `${siteUrl}/${markdownPath(page)}` +} + +function titleOf (page) { + return decodeEntities( (page.asciidoc && page.asciidoc.doctitle ? page.asciidoc.doctitle : page.src.stem).replace(/<[^>]+>/g, '') ) +} + +function descriptionOf (page) { const description = page.asciidoc && page.asciidoc.attributes['description'] + return description ? decodeEntities(description.replace(/\s+/g, ' ').trim()) : undefined +} + +function markdownPage (page, body, htmlUrl) { + const head = [`# ${titleOf(page)}`] + if (htmlUrl) head.push(`URL: ${htmlUrl}`) + const description = descriptionOf(page) + if (description) head.push(`> ${description}`) + return `${head.join('\n\n')}\n\n${body}\n` +} + +function entryFor (page, siteUrl) { + const url = markdownUrl(page, siteUrl) + const description = descriptionOf(page) const source = sourceUrl(page.src) - let entry = `- [${title}](${siteUrl}${page.pub.url})` - if (description) entry += `: ${decodeEntities(description.replace(/\s+/g, ' ').trim())}` + let entry = `- [${titleOf(page)}](${url})` + if (description) entry += `: ${description}` if (source) entry += ` (source: ${source})` return entry } diff --git a/package-lock.json b/package-lock.json index e605f2feb..d4081e1ed 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,10 @@ ], "dependencies": { "@asciidoctor/tabs": "^1.0.0-beta.6", - "@sntke/antora-mermaid-extension": "0.0.8" + "@joplin/turndown-plugin-gfm": "1.0.68", + "@mixmark-io/domino": "2.2.0", + "@sntke/antora-mermaid-extension": "0.0.8", + "turndown": "7.2.4" }, "devDependencies": { "@antora/cli": "^3.1.7", @@ -880,6 +883,12 @@ "dev": true, "license": "ISC" }, + "node_modules/@joplin/turndown-plugin-gfm": { + "version": "1.0.68", + "resolved": "https://registry.npmjs.org/@joplin/turndown-plugin-gfm/-/turndown-plugin-gfm-1.0.68.tgz", + "integrity": "sha512-m8DfAQNC/V7g0j5H6Jv60WhekBBjodD+zTPGrU+g2m+ux/z8ZX07KQIl6kaAmbKwfvsQhC04op52g63I78sVgg==", + "license": "MIT" + }, "node_modules/@keyv/serialize": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.1.1.tgz", @@ -887,6 +896,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@mixmark-io/domino": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz", + "integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==", + "license": "BSD-2-Clause" + }, "node_modules/@napi-rs/wasm-runtime": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.6.tgz", @@ -7362,6 +7377,19 @@ "license": "0BSD", "optional": true }, + "node_modules/turndown": { + "version": "7.2.4", + "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.2.4.tgz", + "integrity": "sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ==", + "license": "MIT", + "dependencies": { + "@mixmark-io/domino": "^2.2.0" + }, + "engines": { + "node": ">=18", + "npm": ">=9" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", diff --git a/package.json b/package.json index 0d9a8ac22..da4ccdf04 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,10 @@ }, "dependencies": { "@asciidoctor/tabs": "^1.0.0-beta.6", - "@sntke/antora-mermaid-extension": "0.0.8" + "@joplin/turndown-plugin-gfm": "1.0.68", + "@mixmark-io/domino": "2.2.0", + "@sntke/antora-mermaid-extension": "0.0.8", + "turndown": "7.2.4" }, "workspaces": [ "ui"