feat: Publish Markdown versions of the docs pages and an llms-full.txt

lib/html-to-markdown.js

@@ -0,0 +1,144 @@
+// Converts the HTML Asciidoctor renders for a page body into Markdown.
+//
+// Asciidoctor lays out admonitions and callout lists as tables, so without the
+// rules below they would come out as Markdown tables with an empty icon column.
+const domino = require('@mixmark-io/domino')
+const TurndownService = require('turndown')
+const { tables } = require('@joplin/turndown-plugin-gfm')
+
+const ADMONITION_LABELS = {
+  note: 'Note',
+  tip: 'Tip',
+  important: 'Important',
+  warning: 'Warning',
+  caution: 'Caution',
+}
+
+function hasClass (node, name) {
+  return node.nodeType === 1 && (node.getAttribute('class') || '').split(/\s+/).includes(name)
+}
+
+// domino, the DOM turndown runs on in Node, returns lists that are not iterable.
+function elements (node, tag) {
+  return Array.from(node.getElementsByTagName(tag))
+}
+
+function quote (text) {
+  return text
+    .trim()
+    .split('\n')
+    .map((line) => (line ? `> ${line}` : '>'))
+    .join('\n')
+}
+
+function fence (text, info = '') {
+  const longestRun = Math.max(0, ...(text.match(/`+/g) || []).map((run) => run.length))
+  const marker = '`'.repeat(Math.max(3, longestRun + 1))
+  return `\n\n${marker}${info}\n${text.replace(/\n$/, '')}\n${marker}\n\n`
+}
+
+// Rewrites the DOM before turndown sees it. Links and images become absolute,
+// because llms-full.txt has no directory to resolve relative ones from, and a
+// link to a page in markdownUrls goes to its Markdown version instead. The
+// Mermaid source becomes a pre, the only element whose whitespace turndown
+// keeps; its <br>s in node labels arrive as elements and are turned back into text.
+function prepare (html, pageUrl, markdownUrls) {
+  const document = domino.createDocument(`<!doctype html><html><body>${html}</body></html>`)
+  const base = /^https?:/.test(pageUrl || '') ? pageUrl : undefined
+  const absolute = (url) => (!base || /^[a-z][a-z0-9+.-]*:/i.test(url) ? url : new URL(url, base).href)
+  for (const [tag, attribute] of [['a', 'href'], ['img', 'src']]) {
+    for (const node of elements(document.body, tag)) {
+      const value = node.getAttribute(attribute)
+      if (value) node.setAttribute(attribute, absolute(value))
+    }
+  }
+  for (const node of elements(document.body, 'a')) {
+    const [url, fragment] = (node.getAttribute('href') || '').split('#')
+    const markdownUrl = markdownUrls.get(url)
+    if (markdownUrl) node.setAttribute('href', fragment === undefined ? markdownUrl : `${markdownUrl}#${fragment}`)
+  }
+  for (const node of elements(document.body, 'div').filter((div) => hasClass(div, 'mermaid'))) {
+    for (const br of elements(node, 'br')) br.parentNode.replaceChild(document.createTextNode('<br>'), br)
+    const pre = document.createElement('pre')
+    pre.setAttribute('class', 'mermaid')
+    pre.textContent = node.textContent
+    node.parentNode.replaceChild(pre, node)
+  }
+  return document.body
+}
+
+function createConverter (markdownUrls = new Map()) {
+  const turndown = new TurndownService({
+    headingStyle: 'atx',
+    codeBlockStyle: 'fenced',
+    bulletListMarker: '-',
+    emDelimiter: '_',
+  })
+  turndown.use(tables)
+
+  // Section anchors are empty links that only carry a CSS glyph. A rule, not
+  // remove(), because turndown consults remove() only after its own link rule.
+  turndown.addRule('sectionAnchor', {
+    filter: (node) => node.nodeName === 'A' && hasClass(node, 'anchor'),
+    replacement: () => '',
+  })
+
+  // Asciidoctor marks the language with data-lang; the class turndown reads by
+  // default says "language-none" on untagged blocks. Literal blocks are a bare pre.
+  turndown.addRule('codeBlock', {
+    filter: (node) => node.nodeName === 'PRE',
+    replacement: (_content, node) => {
+      if (hasClass(node, 'mermaid')) return fence(node.textContent.trim(), 'mermaid')
+      const code = node.firstChild && node.firstChild.nodeName === 'CODE' ? node.firstChild : node
+      return fence(code.textContent, code.getAttribute('data-lang') || '')
+    },
+  })
+
+  // A tab set renders as a list of labels followed by the panels, so each panel
+  // carries its own label or the alternatives read as consecutive steps.
+  turndown.addRule('tabs', {
+    filter: (node) => node.nodeName === 'DIV' && hasClass(node, 'tabs'),
+    replacement: (_content, node) => {
+      const labels = new Map(elements(node, 'li').map((li) => [li.getAttribute('id'), li.textContent.trim()]))
+      const panels = elements(node, 'div').filter(
+        (div) => hasClass(div, 'tabpanel') && div.parentNode.parentNode === node
+      )
+      const sections = panels.map((panel) => {
+        const label = labels.get(panel.getAttribute('aria-labelledby')) || ''
+        return `**Tab: ${label}**\n\n${turndown.turndown(panel.innerHTML)}`
+      })
+      return `\n\n${sections.join('\n\n')}\n\n`
+    },
+  })
+
+  turndown.addRule('admonition', {
+    filter: (node) => node.nodeName === 'DIV' && hasClass(node, 'admonitionblock'),
+    replacement: (_content, node) => {
+      const type = Object.keys(ADMONITION_LABELS).find((name) => hasClass(node, name))
+      const cell = elements(node, 'td').find((td) => hasClass(td, 'content'))
+      const body = cell ? turndown.turndown(cell.innerHTML) : ''
+      return `\n\n${quote(`**${ADMONITION_LABELS[type] || 'Note'}:** ${body}`)}\n\n`
+    },
+  })
+
+  turndown.addRule('calloutList', {
+    filter: (node) => node.nodeName === 'DIV' && hasClass(node, 'colist'),
+    replacement: (_content, node) => {
+      const rows = elements(node, 'tr').map((row) => {
+        const [marker, text] = elements(row, 'td')
+        return `(${marker.textContent.trim()}) ${turndown.turndown(text.innerHTML).replace(/\n+/g, ' ')}`
+      })
+      return `\n\n${rows.join('\n')}\n\n`
+    },
+  })
+
+  turndown.addRule('blockTitle', {
+    filter: (node) => node.nodeName === 'DIV' && hasClass(node, 'title'),
+    replacement: (content) => `\n\n**${content.trim()}**\n\n`,
+  })
+
+  // Paragraphs inside list items leave whitespace-only lines behind.
+  return (html, pageUrl) => turndown.turndown(prepare(html, pageUrl, markdownUrls)).replace(/[ \t]+$/gm, '')
+}
+
+module.exports = { createConverter }

lib/llms-txt.js

@@ -1,21 +1,41 @@
 // Generates /llms.txt following https://llmstxt.org/ so that AI agents can
-// discover the documentation without scraping HTML.
+// discover the documentation without scraping HTML, plus a Markdown copy of
+// every indexed page next to its HTML (index.md beside index.html, as the spec
+// proposes) and /llms-full.txt, all of those pages in one file. llms-full.txt
+// is a widespread convention rather than part of the spec.
 //
 // For every component, only the version that 'stable' points to is indexed
 // (the latest release, or nightly when building without release branches).
-// Each entry carries the published page URL, the page description and a link
+// Each entry carries the Markdown page URL, the page description and a link
 // to the AsciiDoc source on GitHub, so agents can both read a page and find
 // the file to change.
 //
+// A page with the attribute `:page-llms-ignore:` is left out of all of it.
+//
 // Useful links:
 // Extensions: https://docs.antora.org/antora/latest/extend/extensions/
 // Types of events: https://docs.antora.org/antora/latest/extend/generator-events-reference/
+const { createConverter } = require('./html-to-markdown')
 
 // Pages on how Stackable builds SDP go under "Optional", the llms.txt convention
 // for links an agent can skip when context is short (https://llmstxt.org/).
+// They are left out of llms-full.txt for the same reason.
 const OPTIONAL_MODULES = new Set(['contributor', 'internal'])
 
 module.exports.register = function () {
+  const markdown = new Map()
+
+  // Page contents are the bare article HTML here; once pages are composed they
+  // carry the whole site layout, navigation included.
+  this.once('documentsConverted', ({ playbook, contentCatalog }) => {
+    const siteUrl = (playbook.site.url || '').replace(/\/$/, '')
+    const pages = indexedPages(contentCatalog)
+    const convert = createConverter(new Map(pages.map((page) => [`${siteUrl}${page.pub.url}`, markdownUrl(page, siteUrl)])))
+    for (const page of pages) {
+      markdown.set(page, convert(page.contents.toString(), `${siteUrl}${page.pub.url}`))
+    }
+  })
+
   this.once('beforePublish', ({ playbook, contentCatalog, siteCatalog }) => {
     const siteUrl = (playbook.site.url || '').replace(/\/$/, '')
     const lines = [
@@ -28,24 +48,24 @@ module.exports.register = function () {
       'Machine-readable release, component and CRD data is available from the',
       'Stackable Hub: https://hub.stackable.tech/llms.txt',
       '',
-      'Every entry below links the published page and, in parentheses, its AsciiDoc',
+      'Every entry below links the page as Markdown and, in parentheses, its AsciiDoc',
       'source on GitHub. To suggest a change to a page, edit that source file.',
+      `All pages outside "Optional" are also available in one file: ${siteUrl}/llms-full.txt`,
     ]
     const optional = []
+    const full = []
 
     for (const component of contentCatalog.getComponents()) {
-      const version = component.latest
-      const pages = contentCatalog
-        .findBy({ component: component.name, version: version.version, family: 'page' })
-        .filter((page) => page.out)
-        .sort((a, b) => a.pub.url.localeCompare(b.pub.url))
       const byModule = new Map()
-      for (const page of pages) {
+      for (const page of indexedPages(contentCatalog, component)) {
+        const body = markdown.get(page)
+        siteCatalog.addFile({ contents: Buffer.from(markdownPage(page, body), 'utf8'), out: { path: markdownPath(page) } })
         const module = page.src.module
         if (OPTIONAL_MODULES.has(module)) {
           optional.push(page)
           continue
         }
+        full.push(markdownPage(page, body, `${siteUrl}${page.pub.url}`))
         if (!byModule.has(module)) byModule.set(module, [])
         byModule.get(module).push(page)
       }
@@ -60,21 +80,72 @@ module.exports.register = function () {
       for (const page of optional) lines.push(entryFor(page, siteUrl))
     }
 
-    siteCatalog.addFile({
-      contents: Buffer.from(lines.join('\n') + '\n', 'utf8'),
-      out: { path: 'llms.txt' },
-    })
+    siteCatalog.addFile({ contents: Buffer.from(lines.join('\n') + '\n', 'utf8'), out: { path: 'llms.txt' } })
+    siteCatalog.addFile({ contents: Buffer.from(full.join('\n\n'), 'utf8'), out: { path: 'llms-full.txt' } })
   })
 }
 
-function entryFor (page, siteUrl) {
-  const title = decodeEntities(
+// In the order of the site navigation, which is the reading order the authors
+// chose; pages outside it follow, sorted by URL. The navigation only exists from
+// navigationBuilt on, so earlier callers get URL order.
+function indexedPages (contentCatalog, component) {
+  const components = component ? [component] : contentCatalog.getComponents()
+  return components.flatMap((it) => {
+    const rank = navigationRank(it.latest.navigation || [])
+    const position = (page) => (rank.has(page.pub.url) ? rank.get(page.pub.url) : Infinity)
+    return contentCatalog
+      .findBy({ component: it.name, version: it.latest.version, family: 'page' })
+      .filter((page) => page.out && !('page-llms-ignore' in page.asciidoc.attributes))
+      .sort((a, b) => position(a) - position(b) || a.pub.url.localeCompare(b.pub.url))
+  })
+}
+
+function navigationRank (navigation) {
+  const rank = new Map()
+  const visit = (items) => {
+    for (const item of items) {
+      const url = item.urlType === 'internal' && item.url.split('#')[0]
+      if (url && !rank.has(url)) rank.set(url, rank.size)
+      visit(item.items || [])
+    }
+  }
+  visit(navigation)
+  return rank
+}
+
+function markdownPath (page) {
+  return page.out.path.replace(/\.html$/, '.md')
+}
+
+function markdownUrl (page, siteUrl) {
+  return `${siteUrl}/${markdownPath(page)}`
+}
+
+function titleOf (page) {
+  return decodeEntities(
     (page.asciidoc && page.asciidoc.doctitle ? page.asciidoc.doctitle : page.src.stem).replace(/<[^>]+>/g, '')
   )
+}
+
+function descriptionOf (page) {
   const description = page.asciidoc && page.asciidoc.attributes['description']
+  return description ? decodeEntities(description.replace(/\s+/g, ' ').trim()) : undefined
+}
+
+function markdownPage (page, body, htmlUrl) {
+  const head = [`# ${titleOf(page)}`]
+  if (htmlUrl) head.push(`URL: ${htmlUrl}`)
+  const description = descriptionOf(page)
+  if (description) head.push(`> ${description}`)
+  return `${head.join('\n\n')}\n\n${body}\n`
+}
+
+function entryFor (page, siteUrl) {
+  const url = markdownUrl(page, siteUrl)
+  const description = descriptionOf(page)
   const source = sourceUrl(page.src)
-  let entry = `- [${title}](${siteUrl}${page.pub.url})`
-  if (description) entry += `: ${decodeEntities(description.replace(/\s+/g, ' ').trim())}`
+  let entry = `- [${titleOf(page)}](${url})`
+  if (description) entry += `: ${description}`
   if (source) entry += ` (source: ${source})`
   return entry
 }

package.json

@@ -6,7 +6,10 @@
   },
   "dependencies": {
     "@asciidoctor/tabs": "^1.0.0-beta.6",
-    "@sntke/antora-mermaid-extension": "0.0.8"
+    "@joplin/turndown-plugin-gfm": "1.0.68",
+    "@mixmark-io/domino": "2.2.0",
+    "@sntke/antora-mermaid-extension": "0.0.8",
+    "turndown": "7.2.4"
   },
   "workspaces": [
     "ui"