diff --git a/lib/html-to-markdown.js b/lib/html-to-markdown.js
new file mode 100644
index 000000000..0e12d4e1b
--- /dev/null
+++ b/lib/html-to-markdown.js
@@ -0,0 +1,144 @@
+// Converts the HTML Asciidoctor renders for a page body into Markdown.
+//
+// Asciidoctor lays out admonitions and callout lists as tables, so without the
+// rules below they would come out as Markdown tables with an empty icon column.
+const domino = require('@mixmark-io/domino')
+const TurndownService = require('turndown')
+const { tables } = require('@joplin/turndown-plugin-gfm')
+
+const ADMONITION_LABELS = {
+ note: 'Note',
+ tip: 'Tip',
+ important: 'Important',
+ warning: 'Warning',
+ caution: 'Caution',
+}
+
+function hasClass (node, name) {
+ return node.nodeType === 1 && (node.getAttribute('class') || '').split(/\s+/).includes(name)
+}
+
+// domino, the DOM turndown runs on in Node, returns lists that are not iterable.
+function elements (node, tag) {
+ return Array.from(node.getElementsByTagName(tag))
+}
+
+function quote (text) {
+ return text
+ .trim()
+ .split('\n')
+ .map((line) => (line ? `> ${line}` : '>'))
+ .join('\n')
+}
+
+function fence (text, info = '') {
+ const longestRun = Math.max(0, ...(text.match(/`+/g) || []).map((run) => run.length))
+ const marker = '`'.repeat(Math.max(3, longestRun + 1))
+ return `\n\n${marker}${info}\n${text.replace(/\n$/, '')}\n${marker}\n\n`
+}
+
+// Rewrites the DOM before turndown sees it. Links and images become absolute,
+// because llms-full.txt has no directory to resolve relative ones from, and a
+// link to a page in markdownUrls goes to its Markdown version instead. The
+// Mermaid source becomes a pre, the only element whose whitespace turndown
+// keeps; its
s in node labels arrive as elements and are turned back into text.
+function prepare (html, pageUrl, markdownUrls) {
+ const document = domino.createDocument(`
${html}`)
+ const base = /^https?:/.test(pageUrl || '') ? pageUrl : undefined
+ const absolute = (url) => (!base || /^[a-z][a-z0-9+.-]*:/i.test(url) ? url : new URL(url, base).href)
+ for (const [tag, attribute] of [['a', 'href'], ['img', 'src']]) {
+ for (const node of elements(document.body, tag)) {
+ const value = node.getAttribute(attribute)
+ if (value) node.setAttribute(attribute, absolute(value))
+ }
+ }
+ for (const node of elements(document.body, 'a')) {
+ const [url, fragment] = (node.getAttribute('href') || '').split('#')
+ const markdownUrl = markdownUrls.get(url)
+ if (markdownUrl) node.setAttribute('href', fragment === undefined ? markdownUrl : `${markdownUrl}#${fragment}`)
+ }
+ for (const node of elements(document.body, 'div').filter((div) => hasClass(div, 'mermaid'))) {
+ for (const br of elements(node, 'br')) br.parentNode.replaceChild(document.createTextNode('
'), br)
+ const pre = document.createElement('pre')
+ pre.setAttribute('class', 'mermaid')
+ pre.textContent = node.textContent
+ node.parentNode.replaceChild(pre, node)
+ }
+ return document.body
+}
+
+function createConverter (markdownUrls = new Map()) {
+ const turndown = new TurndownService({
+ headingStyle: 'atx',
+ codeBlockStyle: 'fenced',
+ bulletListMarker: '-',
+ emDelimiter: '_',
+ })
+ turndown.use(tables)
+
+ // Section anchors are empty links that only carry a CSS glyph. A rule, not
+ // remove(), because turndown consults remove() only after its own link rule.
+ turndown.addRule('sectionAnchor', {
+ filter: (node) => node.nodeName === 'A' && hasClass(node, 'anchor'),
+ replacement: () => '',
+ })
+
+ // Asciidoctor marks the language with data-lang; the class turndown reads by
+ // default says "language-none" on untagged blocks. Literal blocks are a bare pre.
+ turndown.addRule('codeBlock', {
+ filter: (node) => node.nodeName === 'PRE',
+ replacement: (_content, node) => {
+ if (hasClass(node, 'mermaid')) return fence(node.textContent.trim(), 'mermaid')
+ const code = node.firstChild && node.firstChild.nodeName === 'CODE' ? node.firstChild : node
+ return fence(code.textContent, code.getAttribute('data-lang') || '')
+ },
+ })
+
+ // A tab set renders as a list of labels followed by the panels, so each panel
+ // carries its own label or the alternatives read as consecutive steps.
+ turndown.addRule('tabs', {
+ filter: (node) => node.nodeName === 'DIV' && hasClass(node, 'tabs'),
+ replacement: (_content, node) => {
+ const labels = new Map(elements(node, 'li').map((li) => [li.getAttribute('id'), li.textContent.trim()]))
+ const panels = elements(node, 'div').filter(
+ (div) => hasClass(div, 'tabpanel') && div.parentNode.parentNode === node
+ )
+ const sections = panels.map((panel) => {
+ const label = labels.get(panel.getAttribute('aria-labelledby')) || ''
+ return `**Tab: ${label}**\n\n${turndown.turndown(panel.innerHTML)}`
+ })
+ return `\n\n${sections.join('\n\n')}\n\n`
+ },
+ })
+
+ turndown.addRule('admonition', {
+ filter: (node) => node.nodeName === 'DIV' && hasClass(node, 'admonitionblock'),
+ replacement: (_content, node) => {
+ const type = Object.keys(ADMONITION_LABELS).find((name) => hasClass(node, name))
+ const cell = elements(node, 'td').find((td) => hasClass(td, 'content'))
+ const body = cell ? turndown.turndown(cell.innerHTML) : ''
+ return `\n\n${quote(`**${ADMONITION_LABELS[type] || 'Note'}:** ${body}`)}\n\n`
+ },
+ })
+
+ turndown.addRule('calloutList', {
+ filter: (node) => node.nodeName === 'DIV' && hasClass(node, 'colist'),
+ replacement: (_content, node) => {
+ const rows = elements(node, 'tr').map((row) => {
+ const [marker, text] = elements(row, 'td')
+ return `(${marker.textContent.trim()}) ${turndown.turndown(text.innerHTML).replace(/\n+/g, ' ')}`
+ })
+ return `\n\n${rows.join('\n')}\n\n`
+ },
+ })
+
+ turndown.addRule('blockTitle', {
+ filter: (node) => node.nodeName === 'DIV' && hasClass(node, 'title'),
+ replacement: (content) => `\n\n**${content.trim()}**\n\n`,
+ })
+
+ // Paragraphs inside list items leave whitespace-only lines behind.
+ return (html, pageUrl) => turndown.turndown(prepare(html, pageUrl, markdownUrls)).replace(/[ \t]+$/gm, '')
+}
+
+module.exports = { createConverter }
diff --git a/lib/llms-txt.js b/lib/llms-txt.js
index 274db7e4e..4ed65c809 100644
--- a/lib/llms-txt.js
+++ b/lib/llms-txt.js
@@ -1,21 +1,41 @@
// Generates /llms.txt following https://llmstxt.org/ so that AI agents can
-// discover the documentation without scraping HTML.
+// discover the documentation without scraping HTML, plus a Markdown copy of
+// every indexed page next to its HTML (index.md beside index.html, as the spec
+// proposes) and /llms-full.txt, all of those pages in one file. llms-full.txt
+// is a widespread convention rather than part of the spec.
//
// For every component, only the version that 'stable' points to is indexed
// (the latest release, or nightly when building without release branches).
-// Each entry carries the published page URL, the page description and a link
+// Each entry carries the Markdown page URL, the page description and a link
// to the AsciiDoc source on GitHub, so agents can both read a page and find
// the file to change.
//
+// A page with the attribute `:page-llms-ignore:` is left out of all of it.
+//
// Useful links:
// Extensions: https://docs.antora.org/antora/latest/extend/extensions/
// Types of events: https://docs.antora.org/antora/latest/extend/generator-events-reference/
+const { createConverter } = require('./html-to-markdown')
// Pages on how Stackable builds SDP go under "Optional", the llms.txt convention
// for links an agent can skip when context is short (https://llmstxt.org/).
+// They are left out of llms-full.txt for the same reason.
const OPTIONAL_MODULES = new Set(['contributor', 'internal'])
module.exports.register = function () {
+ const markdown = new Map()
+
+ // Page contents are the bare article HTML here; once pages are composed they
+ // carry the whole site layout, navigation included.
+ this.once('documentsConverted', ({ playbook, contentCatalog }) => {
+ const siteUrl = (playbook.site.url || '').replace(/\/$/, '')
+ const pages = indexedPages(contentCatalog)
+ const convert = createConverter(new Map(pages.map((page) => [`${siteUrl}${page.pub.url}`, markdownUrl(page, siteUrl)])))
+ for (const page of pages) {
+ markdown.set(page, convert(page.contents.toString(), `${siteUrl}${page.pub.url}`))
+ }
+ })
+
this.once('beforePublish', ({ playbook, contentCatalog, siteCatalog }) => {
const siteUrl = (playbook.site.url || '').replace(/\/$/, '')
const lines = [
@@ -28,24 +48,24 @@ module.exports.register = function () {
'Machine-readable release, component and CRD data is available from the',
'Stackable Hub: https://hub.stackable.tech/llms.txt',
'',
- 'Every entry below links the published page and, in parentheses, its AsciiDoc',
+ 'Every entry below links the page as Markdown and, in parentheses, its AsciiDoc',
'source on GitHub. To suggest a change to a page, edit that source file.',
+ `All pages outside "Optional" are also available in one file: ${siteUrl}/llms-full.txt`,
]
const optional = []
+ const full = []
for (const component of contentCatalog.getComponents()) {
- const version = component.latest
- const pages = contentCatalog
- .findBy({ component: component.name, version: version.version, family: 'page' })
- .filter((page) => page.out)
- .sort((a, b) => a.pub.url.localeCompare(b.pub.url))
const byModule = new Map()
- for (const page of pages) {
+ for (const page of indexedPages(contentCatalog, component)) {
+ const body = markdown.get(page)
+ siteCatalog.addFile({ contents: Buffer.from(markdownPage(page, body), 'utf8'), out: { path: markdownPath(page) } })
const module = page.src.module
if (OPTIONAL_MODULES.has(module)) {
optional.push(page)
continue
}
+ full.push(markdownPage(page, body, `${siteUrl}${page.pub.url}`))
if (!byModule.has(module)) byModule.set(module, [])
byModule.get(module).push(page)
}
@@ -60,21 +80,72 @@ module.exports.register = function () {
for (const page of optional) lines.push(entryFor(page, siteUrl))
}
- siteCatalog.addFile({
- contents: Buffer.from(lines.join('\n') + '\n', 'utf8'),
- out: { path: 'llms.txt' },
- })
+ siteCatalog.addFile({ contents: Buffer.from(lines.join('\n') + '\n', 'utf8'), out: { path: 'llms.txt' } })
+ siteCatalog.addFile({ contents: Buffer.from(full.join('\n\n'), 'utf8'), out: { path: 'llms-full.txt' } })
})
}
-function entryFor (page, siteUrl) {
- const title = decodeEntities(
+// In the order of the site navigation, which is the reading order the authors
+// chose; pages outside it follow, sorted by URL. The navigation only exists from
+// navigationBuilt on, so earlier callers get URL order.
+function indexedPages (contentCatalog, component) {
+ const components = component ? [component] : contentCatalog.getComponents()
+ return components.flatMap((it) => {
+ const rank = navigationRank(it.latest.navigation || [])
+ const position = (page) => (rank.has(page.pub.url) ? rank.get(page.pub.url) : Infinity)
+ return contentCatalog
+ .findBy({ component: it.name, version: it.latest.version, family: 'page' })
+ .filter((page) => page.out && !('page-llms-ignore' in page.asciidoc.attributes))
+ .sort((a, b) => position(a) - position(b) || a.pub.url.localeCompare(b.pub.url))
+ })
+}
+
+function navigationRank (navigation) {
+ const rank = new Map()
+ const visit = (items) => {
+ for (const item of items) {
+ const url = item.urlType === 'internal' && item.url.split('#')[0]
+ if (url && !rank.has(url)) rank.set(url, rank.size)
+ visit(item.items || [])
+ }
+ }
+ visit(navigation)
+ return rank
+}
+
+function markdownPath (page) {
+ return page.out.path.replace(/\.html$/, '.md')
+}
+
+function markdownUrl (page, siteUrl) {
+ return `${siteUrl}/${markdownPath(page)}`
+}
+
+function titleOf (page) {
+ return decodeEntities(
(page.asciidoc && page.asciidoc.doctitle ? page.asciidoc.doctitle : page.src.stem).replace(/<[^>]+>/g, '')
)
+}
+
+function descriptionOf (page) {
const description = page.asciidoc && page.asciidoc.attributes['description']
+ return description ? decodeEntities(description.replace(/\s+/g, ' ').trim()) : undefined
+}
+
+function markdownPage (page, body, htmlUrl) {
+ const head = [`# ${titleOf(page)}`]
+ if (htmlUrl) head.push(`URL: ${htmlUrl}`)
+ const description = descriptionOf(page)
+ if (description) head.push(`> ${description}`)
+ return `${head.join('\n\n')}\n\n${body}\n`
+}
+
+function entryFor (page, siteUrl) {
+ const url = markdownUrl(page, siteUrl)
+ const description = descriptionOf(page)
const source = sourceUrl(page.src)
- let entry = `- [${title}](${siteUrl}${page.pub.url})`
- if (description) entry += `: ${decodeEntities(description.replace(/\s+/g, ' ').trim())}`
+ let entry = `- [${titleOf(page)}](${url})`
+ if (description) entry += `: ${description}`
if (source) entry += ` (source: ${source})`
return entry
}
diff --git a/package-lock.json b/package-lock.json
index e605f2feb..d4081e1ed 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -9,7 +9,10 @@
],
"dependencies": {
"@asciidoctor/tabs": "^1.0.0-beta.6",
- "@sntke/antora-mermaid-extension": "0.0.8"
+ "@joplin/turndown-plugin-gfm": "1.0.68",
+ "@mixmark-io/domino": "2.2.0",
+ "@sntke/antora-mermaid-extension": "0.0.8",
+ "turndown": "7.2.4"
},
"devDependencies": {
"@antora/cli": "^3.1.7",
@@ -880,6 +883,12 @@
"dev": true,
"license": "ISC"
},
+ "node_modules/@joplin/turndown-plugin-gfm": {
+ "version": "1.0.68",
+ "resolved": "https://registry.npmjs.org/@joplin/turndown-plugin-gfm/-/turndown-plugin-gfm-1.0.68.tgz",
+ "integrity": "sha512-m8DfAQNC/V7g0j5H6Jv60WhekBBjodD+zTPGrU+g2m+ux/z8ZX07KQIl6kaAmbKwfvsQhC04op52g63I78sVgg==",
+ "license": "MIT"
+ },
"node_modules/@keyv/serialize": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.1.1.tgz",
@@ -887,6 +896,12 @@
"dev": true,
"license": "MIT"
},
+ "node_modules/@mixmark-io/domino": {
+ "version": "2.2.0",
+ "resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz",
+ "integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==",
+ "license": "BSD-2-Clause"
+ },
"node_modules/@napi-rs/wasm-runtime": {
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.6.tgz",
@@ -7362,6 +7377,19 @@
"license": "0BSD",
"optional": true
},
+ "node_modules/turndown": {
+ "version": "7.2.4",
+ "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.2.4.tgz",
+ "integrity": "sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ==",
+ "license": "MIT",
+ "dependencies": {
+ "@mixmark-io/domino": "^2.2.0"
+ },
+ "engines": {
+ "node": ">=18",
+ "npm": ">=9"
+ }
+ },
"node_modules/type-check": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",
diff --git a/package.json b/package.json
index 0d9a8ac22..da4ccdf04 100644
--- a/package.json
+++ b/package.json
@@ -6,7 +6,10 @@
},
"dependencies": {
"@asciidoctor/tabs": "^1.0.0-beta.6",
- "@sntke/antora-mermaid-extension": "0.0.8"
+ "@joplin/turndown-plugin-gfm": "1.0.68",
+ "@mixmark-io/domino": "2.2.0",
+ "@sntke/antora-mermaid-extension": "0.0.8",
+ "turndown": "7.2.4"
},
"workspaces": [
"ui"