fix: render math as native MathML at scrape time

Gemfile

@@ -35,6 +35,7 @@ end
 group :docs do
   gem 'image_optim_pack', platforms: :ruby
   gem 'image_optim'
+  gem 'katex', '~> 0.11', require: false
   gem 'kramdown'
   gem 'kramdown-parser-gfm'
   gem 'redcarpet'

Gemfile.lock

@@ -134,6 +134,8 @@ GEM
       rdoc (>= 4.0.0)
       reline (>= 0.4.2)
     json (2.19.9)
+    katex (0.11.0)
+      execjs (~> 2.8)
     kramdown (2.5.2)
       rexml (>= 3.4.4)
     kramdown-parser-gfm (1.1.0)
@@ -315,6 +317,7 @@ DEPENDENCIES
   html-pipeline (~> 2.14)
   image_optim
   image_optim_pack
+  katex (~> 0.11)
   kramdown
   kramdown-parser-gfm
   minitest
@@ -396,6 +399,7 @@ CHECKSUMS
   io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc
   irb (1.18.0) sha256=de9454a0703a54704b9811a5ef31a60c86949fbf4013fcf244fabc7c775248e3
   json (2.19.9) sha256=9b9025b7cdddafa38d316eca0b2358488e42d417045c1b90d216a9fefe46b79a
+  katex (0.11.0) sha256=6f9fa9a13e2ba616a0f8d783cc9b30979f0eefb0f5f1e04e66fe7dd8eb682e78
   kramdown (2.5.2) sha256=1ba542204c66b6f9111ff00dcc26075b95b220b07f2905d8261740c82f7f02fa
   kramdown-parser-gfm (1.1.0) sha256=fb39745516427d2988543bf01fc4cf0ab1149476382393e0e9c48592f6581729
   logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203

README.md

@@ -114,6 +114,7 @@ Modifications made to each document include:
 * replacing all internal (scraped) URLs with their unqualified and relative counterpart
 * adding content, such as a title and link to the original document
 * ensuring correct syntax highlighting using [Prism](http://prismjs.com/)
+* converting math markup to native MathML using [KaTeX](https://katex.org/)
 
 These modifications are applied via a set of filters using the [HTML::Pipeline](https://github.com/jch/html-pipeline) library. Each scraper includes filters specific to itself, one of which is tasked with figuring out the pages' metadata.
 

assets/stylesheets/global/_base.scss

@@ -149,6 +149,9 @@ table {
   max-width: 100%;
 }
 
+// No overflow rule here: a scroll container on a <math> element leaves it blank in Chrome.
+math[display="block"] { margin: 1.5em 0; }
+
 caption {
   font-weight: var(--boldFontWeight);
   padding: 0 .7em .3em;

docs/filter-reference.md

@@ -74,6 +74,7 @@ The `call` method must return either `doc` or `html`, depending on the type of f
 ## Core filters
 
 * [`ContainerFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/container.rb) — changes the root node of the document (remove everything outside)
+* [`MathFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/math.rb) — converts Sphinx/MathJax math markup (`\(...\)` and `\[...\]`) to native MathML using [KaTeX](https://katex.org/), which requires a JavaScript runtime such as Node.js; not in the default stack, add `math` to a scraper's `html_filters` to use it
 * [`CleanHtmlFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/clean_html.rb) — removes HTML comments, `<script>`, `<style>`, etc.
 * [`NormalizeUrlsFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/normalize_urls.rb) — replaces all URLs with their fully qualified counterpart
 * [`InternalUrlsFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/internal_urls.rb) — detects internal URLs (the ones to scrape) and replaces them with their unqualified, relative counterpart

lib/docs/core/math_renderer.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require 'execjs'
+require 'katex'
+
+module Docs
+  module MathRenderer
+    class Error < StandardError; end
+
+    class RuntimeUnavailable < SetupError
+      def initialize
+        super "Rendering math requires a JavaScript runtime (e.g. Node.js) to be installed. " \
+              "See https://github.com/rails/execjs#readme for the list of supported runtimes."
+      end
+    end
+
+    # Options passed to katex.renderToString (https://katex.org/docs/options).
+    OPTIONS = {
+      output: 'mathml',
+      throwOnError: true,
+      strict: false, # don't warn about non-strict LaTeX like Unicode text
+      trust: false
+    }.freeze
+
+    BATCH_JS = <<~JS
+      function renderMathBatch(items, options) {
+        return items.map(function(item) {
+          try {
+            return { html: katex.renderToString(item.tex, Object.assign({ displayMode: item.display }, options)) };
+          } catch (error) {
+            return { error: String(error.message || error) };
+          }
+        });
+      }
+    JS
+
+    MATH_RGX = /<math\b.*<\/math>/m
+
+    # MathJax tolerates bare underscores in text mode (e.g. \text{log_loss}),
+    # which KaTeX rejects, so escape them before rendering.
+    TEXT_COMMAND_RGX = /\\text(?:rm|bf|it|sf|tt|normal)?\{[^{}]*\}/
+    BARE_UNDERSCORE_RGX = /(?<!\\)_/
+
+    class << self
+      # Renders a single expression to a <math> element.
+      def to_mathml(tex, display: false)
+        result = render_all([[tex, display]]).first
+        raise result[:error] if result[:error]
+        result[:mathml]
+      end
+
+      # Renders many expressions in a single JavaScript call.
+      def render_all(items)
+        return [] if items.empty?
+
+        payload = items.map { |tex, display| { tex: normalize(tex.to_s), display: display ? true : false } }
+        context.call('renderMathBatch', payload, OPTIONS).map do |result|
+          if result['error']
+            { error: Error.new(result['error']) }
+          else
+            { mathml: result['html'][MATH_RGX] }
+          end
+        end
+      rescue ExecJS::RuntimeUnavailable
+        raise RuntimeUnavailable
+      end
+
+      def render(tex, display: false)
+        to_mathml(tex, display: display)
+      rescue Error
+        fallback(tex, display: display)
+      end
+
+      def normalize(tex)
+        tex.gsub(TEXT_COMMAND_RGX) { |text| text.gsub(BARE_UNDERSCORE_RGX, '\\_') }
+      end
+
+      def fallback(tex, display: false)
+        tag = display ? 'pre' : 'code'
+        %(<#{tag} class="_math-fallback">#{CGI.escapeHTML(tex)}</#{tag}>)
+      end
+
+      def context
+        @context ||= ExecJS.compile(File.read(Katex.katex_js_path) + BATCH_JS)
+      rescue ExecJS::RuntimeUnavailable
+        raise RuntimeUnavailable
+      end
+
+      def reset!
+        @context = nil
+      end
+    end
+  end
+end

lib/docs/filters/core/clean_text.rb

@@ -2,7 +2,7 @@
 
 module Docs
   class CleanTextFilter < Filter
-    EMPTY_NODES_RGX = /<(?!td|th|iframe|mspace|rect|path|ellipse|line|polyline)(\w+)[^>]*>[[:space:]]*<\/\1>/
+    EMPTY_NODES_RGX = /<(?!td|th|iframe|mspace|mtext|mtd|mrow|mphantom|rect|path|ellipse|line|polyline)(\w+)[^>]*>[[:space:]]*<\/\1>/
 
     def call
       return html if context[:clean_text] == false

lib/docs/filters/core/math.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Docs
+  # Replaces Sphinx/MathJax math markup with native MathML, rendered at
+  # scrape time by MathRenderer (KaTeX):
+  # <span class="math">\(...\)</span> and <div class="math">\[...\]</div>
+  class MathFilter < Filter
+    TEX_DELIMITERS_RGX = /\\\((.*?)\\\)|\\\[(.*?)\\\]/m
+
+    def call
+      items = css('span.math', 'div.math').filter_map do |node|
+        node.css('.eqno').remove
+        content = node.content
+        next unless content =~ TEX_DELIMITERS_RGX
+        [node, split(content)]
+      end
+      render(items)
+      doc
+    end
+
+    private
+
+    # Splits text into plain strings and [tex, display] pairs.
+    def split(content)
+      pieces = []
+      last = 0
+      content.scan(TEX_DELIMITERS_RGX) do
+        match = Regexp.last_match
+        pieces << content[last...match.begin(0)] unless match.begin(0) == last
+        pieces << [match[1] || match[2], !match[1]]
+        last = match.end(0)
+      end
+      pieces << content[last..] if last < content.length
+      pieces
+    end
+
+    def render(items)
+      expressions = items.flat_map { |_, pieces| pieces.grep(Array) }
+      return if expressions.empty?
+
+      rendered = MathRenderer.render_all(expressions)
+
+      items.each do |node, pieces|
+        html = pieces.map do |piece|
+          next CGI.escapeHTML(piece) if piece.is_a?(String)
+          tex, display = piece
+          rendered.shift[:mathml] || MathRenderer.fallback(tex, display: display)
+        end.join
+
+        fragment = doc.document.fragment(html)
+        fragment.at_css('math')['id'] = node['id'] if node['id'] && fragment.at_css('math')
+        node.replace(fragment)
+      end
+    end
+  end
+end

lib/docs/scrapers/numpy.rb

@@ -8,7 +8,7 @@ class Numpy < FileScraper
       code: 'https://github.com/numpy/numpy'
     }
 
-    html_filters.push 'numpy/entries', 'numpy/clean_html', 'sphinx/clean_html'
+    html_filters.push 'math', 'numpy/entries', 'numpy/clean_html', 'sphinx/clean_html'
 
     # .main contains more than the page's content alone, but we need something
     # that includes the navigation bar as well in order to guess the type of

lib/docs/scrapers/polars.rb

@@ -10,7 +10,7 @@ class Polars < UrlScraper
       code: 'https://github.com/pola-rs/polars'
     }
 
-    html_filters.push 'polars/entries', 'sphinx/clean_html', 'polars/clean_html'
+    html_filters.push 'math', 'polars/entries', 'sphinx/clean_html', 'polars/clean_html'
 
     # pydata-sphinx-theme keeps the page content in the article body.
     options[:container] = 'article.bd-article'

lib/docs/scrapers/scikit_image.rb

@@ -12,7 +12,7 @@ class ScikitImage < UrlScraper
       code: 'https://github.com/scikit-image/scikit-image'
     }
 
-    html_filters.push 'scikit_image/entries', 'sphinx/clean_html'
+    html_filters.push 'math', 'scikit_image/entries', 'sphinx/clean_html'
 
     options[:container] = 'main article'
     options[:skip] = %w(api_changes.html)

lib/docs/scrapers/scikit_learn.rb

@@ -13,7 +13,7 @@ class ScikitLearn < UrlScraper
       code: 'https://github.com/scikit-learn/scikit-learn'
     }
 
-    html_filters.push 'scikit_learn/entries', 'scikit_learn/clean_html', 'sphinx/clean_html', 'title'
+    html_filters.push 'math', 'scikit_learn/entries', 'scikit_learn/clean_html', 'sphinx/clean_html', 'title'
 
     options[:skip] = %w(modules/generated/sklearn.experimental.enable_iterative_imputer.html
                         modules/generated/sklearn.experimental.enable_hist_gradient_boosting.html)

lib/docs/scrapers/statsmodels.rb

@@ -9,7 +9,7 @@ class Statsmodels < UrlScraper
       code: 'https://github.com/statsmodels/statsmodels/'
     }
 
-    html_filters.push 'statsmodels/entries', 'statsmodels/clean_html', 'sphinx/clean_html'
+    html_filters.push 'math', 'statsmodels/entries', 'statsmodels/clean_html', 'sphinx/clean_html'
 
     options[:skip] = %w(about.html search.html genindex.html)
     options[:skip_patterns] = [/\Arelease/, /\Adev/, /\A_modules/, /\Adatasets/]

test/lib/docs/core/math_renderer_test.rb

@@ -0,0 +1,74 @@
+require_relative '../../../test_helper'
+require_relative '../../../../lib/docs'
+
+class DocsMathRendererTest < Minitest::Spec
+  let(:renderer) { Docs::MathRenderer }
+
+  describe ".to_mathml" do
+    it "returns a <math> element" do
+      output = renderer.to_mathml('x^2')
+      assert_match %r{\A<math [^>]*>.*</math>\z}m, output
+      refute_includes output, 'class="katex"'
+    end
+
+    it "keeps the TeX source as an annotation" do
+      assert_includes renderer.to_mathml('x \in \mathbb{R}'),
+                      '<annotation encoding="application/x-tex">x \in \mathbb{R}</annotation>'
+    end
+
+    it "renders inline math by default" do
+      refute_includes renderer.to_mathml('x^2'), 'display="block"'
+    end
+
+    it "renders display math when display is true" do
+      assert_includes renderer.to_mathml('x^2', display: true), '<math xmlns="http://www.w3.org/1998/Math/MathML" display="block">'
+    end
+
+    it "accepts bare underscores in text mode, like MathJax does" do
+      output = renderer.to_mathml('\text{log_loss}(y) + n_{\text{nonzero\_coefs}}')
+      assert_includes output, '<mtext>log_loss</mtext>'
+      assert_includes output, '<mtext>nonzero_coefs</mtext>'
+    end
+
+    it "raises an Error when the TeX can't be parsed" do
+      error = assert_raises(Docs::MathRenderer::Error) { renderer.to_mathml('\frac{a') }
+      assert_includes error.message, 'KaTeX parse error'
+    end
+
+    it "raises a SetupError when no JavaScript runtime is available" do
+      renderer.reset!
+      stub(ExecJS).compile { raise ExecJS::RuntimeUnavailable }
+      assert_raises(Docs::SetupError) { renderer.to_mathml('x') }
+      renderer.reset!
+    end
+  end
+
+  describe ".render_all" do
+    it "renders many expressions at once, in order" do
+      results = renderer.render_all([['a', false], ['\frac{', true], ['b', true]])
+      assert_equal 3, results.length
+      assert_includes results[0][:mathml], '<mi>a</mi>'
+      assert_kind_of Docs::MathRenderer::Error, results[1][:error]
+      assert_includes results[2][:mathml], 'display="block"'
+      assert_includes results[2][:mathml], '<mi>b</mi>'
+    end
+
+    it "returns an empty array when given nothing" do
+      assert_equal [], renderer.render_all([])
+    end
+  end
+
+  describe ".render" do
+    it "returns MathML for valid TeX" do
+      assert_match %r{\A<math}, renderer.render('x')
+    end
+
+    it "falls back to the escaped TeX source for invalid inline TeX" do
+      assert_equal '<code class="_math-fallback">\frac{a &lt; b</code>', renderer.render('\frac{a < b')
+    end
+
+    it "falls back to a <pre> for invalid display TeX" do
+      assert_equal '<pre class="_math-fallback">\frac{a</pre>', renderer.render('\frac{a', display: true)
+    end
+  end
+end

test/lib/docs/filters/core/clean_text_test.rb

@@ -15,6 +15,11 @@ class CleanTextFilterTest < Minitest::Spec
     assert_equal @body, filter_output
   end
 
+  it "doesn't remove empty MathML nodes" do
+    @body = "<mspace width=\"1em\"></mspace><mtext> </mtext><mtd></mtd><mrow></mrow>"
+    assert_equal @body, filter_output
+  end
+
   it "strips leading and trailing whitespace" do
     @body = "\n\r Test \r\n"
     assert_equal 'Test', filter_output