diff --git a/Gemfile b/Gemfile index e867adff10..349eab2449 100644 --- a/Gemfile +++ b/Gemfile @@ -35,6 +35,7 @@ end group :docs do gem 'image_optim_pack', platforms: :ruby gem 'image_optim' + gem 'katex', '~> 0.11', require: false gem 'kramdown' gem 'kramdown-parser-gfm' gem 'redcarpet' diff --git a/Gemfile.lock b/Gemfile.lock index 41448d4c62..365f1c2f07 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -134,6 +134,8 @@ GEM rdoc (>= 4.0.0) reline (>= 0.4.2) json (2.19.9) + katex (0.11.0) + execjs (~> 2.8) kramdown (2.5.2) rexml (>= 3.4.4) kramdown-parser-gfm (1.1.0) @@ -315,6 +317,7 @@ DEPENDENCIES html-pipeline (~> 2.14) image_optim image_optim_pack + katex (~> 0.11) kramdown kramdown-parser-gfm minitest @@ -396,6 +399,7 @@ CHECKSUMS io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc irb (1.18.0) sha256=de9454a0703a54704b9811a5ef31a60c86949fbf4013fcf244fabc7c775248e3 json (2.19.9) sha256=9b9025b7cdddafa38d316eca0b2358488e42d417045c1b90d216a9fefe46b79a + katex (0.11.0) sha256=6f9fa9a13e2ba616a0f8d783cc9b30979f0eefb0f5f1e04e66fe7dd8eb682e78 kramdown (2.5.2) sha256=1ba542204c66b6f9111ff00dcc26075b95b220b07f2905d8261740c82f7f02fa kramdown-parser-gfm (1.1.0) sha256=fb39745516427d2988543bf01fc4cf0ab1149476382393e0e9c48592f6581729 logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203 diff --git a/README.md b/README.md index fe1b1ee25a..d9df1f2d1d 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,7 @@ Modifications made to each document include: * replacing all internal (scraped) URLs with their unqualified and relative counterpart * adding content, such as a title and link to the original document * ensuring correct syntax highlighting using [Prism](http://prismjs.com/) +* converting math markup to native MathML using [KaTeX](https://katex.org/) These modifications are applied via a set of filters using the [HTML::Pipeline](https://github.com/jch/html-pipeline) library. Each scraper includes filters specific to itself, one of which is tasked with figuring out the pages' metadata. diff --git a/assets/stylesheets/global/_base.scss b/assets/stylesheets/global/_base.scss index 640ca929a5..527d631ff9 100644 --- a/assets/stylesheets/global/_base.scss +++ b/assets/stylesheets/global/_base.scss @@ -149,6 +149,9 @@ table { max-width: 100%; } +// No overflow rule here: a scroll container on a element leaves it blank in Chrome. +math[display="block"] { margin: 1.5em 0; } + caption { font-weight: var(--boldFontWeight); padding: 0 .7em .3em; diff --git a/docs/filter-reference.md b/docs/filter-reference.md index e3f8cb39f9..423d9d480b 100644 --- a/docs/filter-reference.md +++ b/docs/filter-reference.md @@ -74,6 +74,7 @@ The `call` method must return either `doc` or `html`, depending on the type of f ## Core filters * [`ContainerFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/container.rb) — changes the root node of the document (remove everything outside) +* [`MathFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/math.rb) — converts Sphinx/MathJax math markup (`\(...\)` and `\[...\]`) to native MathML using [KaTeX](https://katex.org/), which requires a JavaScript runtime such as Node.js; not in the default stack, add `math` to a scraper's `html_filters` to use it * [`CleanHtmlFilter`](https://github.com/freeCodeCamp/devdocs/blob/main/lib/docs/filters/core/clean_html.rb) — removes HTML comments, `\(...\) and
\[...\]
+ class MathFilter < Filter + TEX_DELIMITERS_RGX = /\\\((.*?)\\\)|\\\[(.*?)\\\]/m + + def call + items = css('span.math', 'div.math').filter_map do |node| + node.css('.eqno').remove + content = node.content + next unless content =~ TEX_DELIMITERS_RGX + [node, split(content)] + end + render(items) + doc + end + + private + + # Splits text into plain strings and [tex, display] pairs. + def split(content) + pieces = [] + last = 0 + content.scan(TEX_DELIMITERS_RGX) do + match = Regexp.last_match + pieces << content[last...match.begin(0)] unless match.begin(0) == last + pieces << [match[1] || match[2], !match[1]] + last = match.end(0) + end + pieces << content[last..] if last < content.length + pieces + end + + def render(items) + expressions = items.flat_map { |_, pieces| pieces.grep(Array) } + return if expressions.empty? + + rendered = MathRenderer.render_all(expressions) + + items.each do |node, pieces| + html = pieces.map do |piece| + next CGI.escapeHTML(piece) if piece.is_a?(String) + tex, display = piece + rendered.shift[:mathml] || MathRenderer.fallback(tex, display: display) + end.join + + fragment = doc.document.fragment(html) + fragment.at_css('math')['id'] = node['id'] if node['id'] && fragment.at_css('math') + node.replace(fragment) + end + end + end +end diff --git a/lib/docs/scrapers/numpy.rb b/lib/docs/scrapers/numpy.rb index e07757af84..febad36591 100644 --- a/lib/docs/scrapers/numpy.rb +++ b/lib/docs/scrapers/numpy.rb @@ -8,7 +8,7 @@ class Numpy < FileScraper code: 'https://github.com/numpy/numpy' } - html_filters.push 'numpy/entries', 'numpy/clean_html', 'sphinx/clean_html' + html_filters.push 'math', 'numpy/entries', 'numpy/clean_html', 'sphinx/clean_html' # .main contains more than the page's content alone, but we need something # that includes the navigation bar as well in order to guess the type of diff --git a/lib/docs/scrapers/polars.rb b/lib/docs/scrapers/polars.rb index 1e561e8887..1cbba46f3f 100644 --- a/lib/docs/scrapers/polars.rb +++ b/lib/docs/scrapers/polars.rb @@ -10,7 +10,7 @@ class Polars < UrlScraper code: 'https://github.com/pola-rs/polars' } - html_filters.push 'polars/entries', 'sphinx/clean_html', 'polars/clean_html' + html_filters.push 'math', 'polars/entries', 'sphinx/clean_html', 'polars/clean_html' # pydata-sphinx-theme keeps the page content in the article body. options[:container] = 'article.bd-article' diff --git a/lib/docs/scrapers/scikit_image.rb b/lib/docs/scrapers/scikit_image.rb index ebb7adcfaf..0c23828043 100644 --- a/lib/docs/scrapers/scikit_image.rb +++ b/lib/docs/scrapers/scikit_image.rb @@ -12,7 +12,7 @@ class ScikitImage < UrlScraper code: 'https://github.com/scikit-image/scikit-image' } - html_filters.push 'scikit_image/entries', 'sphinx/clean_html' + html_filters.push 'math', 'scikit_image/entries', 'sphinx/clean_html' options[:container] = 'main article' options[:skip] = %w(api_changes.html) diff --git a/lib/docs/scrapers/scikit_learn.rb b/lib/docs/scrapers/scikit_learn.rb index 902614e824..44969a2737 100644 --- a/lib/docs/scrapers/scikit_learn.rb +++ b/lib/docs/scrapers/scikit_learn.rb @@ -13,7 +13,7 @@ class ScikitLearn < UrlScraper code: 'https://github.com/scikit-learn/scikit-learn' } - html_filters.push 'scikit_learn/entries', 'scikit_learn/clean_html', 'sphinx/clean_html', 'title' + html_filters.push 'math', 'scikit_learn/entries', 'scikit_learn/clean_html', 'sphinx/clean_html', 'title' options[:skip] = %w(modules/generated/sklearn.experimental.enable_iterative_imputer.html modules/generated/sklearn.experimental.enable_hist_gradient_boosting.html) diff --git a/lib/docs/scrapers/statsmodels.rb b/lib/docs/scrapers/statsmodels.rb index 74dcaf9cf4..906c89096a 100644 --- a/lib/docs/scrapers/statsmodels.rb +++ b/lib/docs/scrapers/statsmodels.rb @@ -9,7 +9,7 @@ class Statsmodels < UrlScraper code: 'https://github.com/statsmodels/statsmodels/' } - html_filters.push 'statsmodels/entries', 'statsmodels/clean_html', 'sphinx/clean_html' + html_filters.push 'math', 'statsmodels/entries', 'statsmodels/clean_html', 'sphinx/clean_html' options[:skip] = %w(about.html search.html genindex.html) options[:skip_patterns] = [/\Arelease/, /\Adev/, /\A_modules/, /\Adatasets/] diff --git a/test/lib/docs/core/math_renderer_test.rb b/test/lib/docs/core/math_renderer_test.rb new file mode 100644 index 0000000000..60f3fbdb69 --- /dev/null +++ b/test/lib/docs/core/math_renderer_test.rb @@ -0,0 +1,74 @@ +require_relative '../../../test_helper' +require_relative '../../../../lib/docs' + +class DocsMathRendererTest < Minitest::Spec + let(:renderer) { Docs::MathRenderer } + + describe ".to_mathml" do + it "returns a element" do + output = renderer.to_mathml('x^2') + assert_match %r{\A]*>.*\z}m, output + refute_includes output, 'class="katex"' + end + + it "keeps the TeX source as an annotation" do + assert_includes renderer.to_mathml('x \in \mathbb{R}'), + 'x \in \mathbb{R}' + end + + it "renders inline math by default" do + refute_includes renderer.to_mathml('x^2'), 'display="block"' + end + + it "renders display math when display is true" do + assert_includes renderer.to_mathml('x^2', display: true), '' + end + + it "accepts bare underscores in text mode, like MathJax does" do + output = renderer.to_mathml('\text{log_loss}(y) + n_{\text{nonzero\_coefs}}') + assert_includes output, 'log_loss' + assert_includes output, 'nonzero_coefs' + end + + it "raises an Error when the TeX can't be parsed" do + error = assert_raises(Docs::MathRenderer::Error) { renderer.to_mathml('\frac{a') } + assert_includes error.message, 'KaTeX parse error' + end + + it "raises a SetupError when no JavaScript runtime is available" do + renderer.reset! + stub(ExecJS).compile { raise ExecJS::RuntimeUnavailable } + assert_raises(Docs::SetupError) { renderer.to_mathml('x') } + renderer.reset! + end + end + + describe ".render_all" do + it "renders many expressions at once, in order" do + results = renderer.render_all([['a', false], ['\frac{', true], ['b', true]]) + assert_equal 3, results.length + assert_includes results[0][:mathml], 'a' + assert_kind_of Docs::MathRenderer::Error, results[1][:error] + assert_includes results[2][:mathml], 'display="block"' + assert_includes results[2][:mathml], 'b' + end + + it "returns an empty array when given nothing" do + assert_equal [], renderer.render_all([]) + end + end + + describe ".render" do + it "returns MathML for valid TeX" do + assert_match %r{\A\frac{a < b', renderer.render('\frac{a < b') + end + + it "falls back to a 
 for invalid display TeX" do
+      assert_equal '
\frac{a
', renderer.render('\frac{a', display: true)
+    end
+  end
+end
diff --git a/test/lib/docs/filters/core/clean_text_test.rb b/test/lib/docs/filters/core/clean_text_test.rb
index 4126b88686..630b754e4e 100644
--- a/test/lib/docs/filters/core/clean_text_test.rb
+++ b/test/lib/docs/filters/core/clean_text_test.rb
@@ -15,6 +15,11 @@ class CleanTextFilterTest < Minitest::Spec
     assert_equal @body, filter_output
   end
 
+  it "doesn't remove empty MathML nodes" do
+    @body = " "
+    assert_equal @body, filter_output
+  end
+
   it "strips leading and trailing whitespace" do
     @body = "\n\r Test \r\n"
     assert_equal 'Test', filter_output
diff --git a/test/lib/docs/filters/core/math_test.rb b/test/lib/docs/filters/core/math_test.rb
new file mode 100644
index 0000000000..d5b1f6d086
--- /dev/null
+++ b/test/lib/docs/filters/core/math_test.rb
@@ -0,0 +1,69 @@
+require_relative '../../../../test_helper'
+require_relative '../../../../../lib/docs'
+
+class MathFilterTest < Minitest::Spec
+  include FilterTestHelper
+  self.filter_class = Docs::MathFilter
+  self.filter_type = 'html'
+
+  def math_nodes
+    filter_output.css('math')
+  end
+
+  def tex_of(node)
+    node.at_css('annotation[encoding="application/x-tex"]').content
+  end
+
+  it "does nothing when there's no math" do
+    @body = '
Costs $5
no delimiters
'
+    assert_equal @body, filter_output_string
+  end
+
+  it "leaves existing  elements untouched" do
+    @body = '
x
'
+    assert_equal @body, filter_output_string
+  end
+
+  context "Sphinx markup" do
+    it "replaces  with inline MathML" do
+      @body = '
Let \(x \in \mathbb{R}^d\) be
'
+      assert_equal 1, math_nodes.length
+      assert_equal 'x \in \mathbb{R}^d', tex_of(math_nodes.first)
+      assert_nil math_nodes.first['display']
+      assert_empty filter_output.css('.math')
+      assert_equal 'Let ', filter_output.at_css('math').previous.content
+    end
+
+    it "replaces 
 with display MathML" do
+      @body = %(
\n\\[\\begin{split}a &= b \\\\ &= c\\end{split}\\]
)
+      assert_equal 1, math_nodes.length
+      assert_equal 'block', math_nodes.first['display']
+      assert_equal '\begin{split}a &= b \\\\ &= c\end{split}', tex_of(math_nodes.first)
+      assert_equal [filter_output.at_css('math')], filter_output.element_children.to_a
+    end
+
+    it "renders every expression in the node" do
+      @body = '
\[a\]\[b\]
'
+      assert_equal %w(a b), math_nodes.map { |node| tex_of(node) }
+    end
+
+    it "keeps the surrounding text" do
+      @body = 'see \(a\) and \(b\)'
+      assert_match %r{\Asee  and \z}, filter_output_string
+    end
+
+    it "keeps the equation id and drops the equation number" do
+      @body = '
(1)\[a\]
'
+      assert_equal 'equation-foo', math_nodes.first['id']
+      assert_empty filter_output.css('.eqno, .headerlink')
+      assert_equal 1, filter_output.children.length
+    end
+
+    it "shows the TeX source when it can't be rendered" do
+      @body = '
\(\frac{a\) 
\[\frac{b\]
'
+      assert_empty math_nodes
+      assert_equal '\frac{a', filter_output.at_css('code').to_s
+      assert_equal '
\frac{b
', filter_output.at_css('pre').to_s
+    end
+  end
+end