diff --git a/lib/docs/filters/statsmodels/clean_html.rb b/lib/docs/filters/statsmodels/clean_html.rb index 79e3424194..a3c07e8980 100644 --- a/lib/docs/filters/statsmodels/clean_html.rb +++ b/lib/docs/filters/statsmodels/clean_html.rb @@ -2,16 +2,36 @@ module Docs class Statsmodels class CleanHtmlFilter < Filter def call - @doc = at_css('.body', '#notebook-container') + # The container keeps the whole main column so that EntriesFilter can + # read the breadcrumbs; only the article is documentation. + @doc = at_css('article.bd-article') || doc + + css('.bd-sidebar-secondary', '.bd-toc', '.prev-next-area', '.prev-next-footer', '.bd-footer', 'form').remove + + # "[source]" links point to the module sources, which aren't scraped. + css('a:has(.viewcode-link)').remove if root_page? at_css('h1').content = 'Statsmodels' - at_css('#basic-documentation').remove - at_css('#table-of-contents').remove - at_css('#indices-and-tables').remove + css('#index').remove end - css('div.cell', 'div.inner_cell', 'div.text_cell_render', 'div.input_area').each do |node| + # Parameter lists render as "name" followed by its type in a classifier + # span; separate them before Sphinx::CleanHtmlFilter flattens the
blocks.
+ css('.nbinput .prompt', '.nboutput .prompt').remove
+ css('.output_area div.highlight').each do |node|
+ pre = node.at_css('pre')
+ pre.content = pre.content
+ node.replace(pre)
+ end
+ css('.nbinput', '.nboutput', '.output_area', 'div.cell', 'div.inner_cell', 'div.text_cell_render').each do |node|
node.before(node.children).remove
end
diff --git a/lib/docs/filters/statsmodels/entries.rb b/lib/docs/filters/statsmodels/entries.rb
index e0d0f42991..bf4d155ef2 100644
--- a/lib/docs/filters/statsmodels/entries.rb
+++ b/lib/docs/filters/statsmodels/entries.rb
@@ -1,15 +1,30 @@
module Docs
class Statsmodels
class EntriesFilter < Docs::EntriesFilter
+ # A couple of statsmodels.tools pages are only linked from the
+ # introductory chapters.
+ TYPE_ALIASES = {
+ 'Getting started' => 'Tools',
+ 'About statsmodels' => 'Tools'
+ }.freeze
+
def get_name
if subpath.start_with?('generated')
- name = at_css('dt').content.strip
- name.sub! %r{\(.*}, '()' # Remove method arguments
+ node = at_css('dt')
+ node.css('.headerlink').remove
+ name = node.content.strip
+ name.sub! %r{\(.*}m, '()' # Remove method arguments
name.remove! %r{[\=\[].*} # Remove "[source]"
- name.remove! %r{\A(class(method)?) }
+ name.remove! %r{\A(class(method)?|exception|property|static) }
name.remove! %r{\Astatsmodels\.}
else
- name = at_css('h1', 'h2').content.strip
+ node = at_css('h1', 'h2')
+ node.css('.headerlink').remove
+ # Chapter titles end with the module name, e.g. "Statistics stats"
+ if (code = node.at_css('code')) && node.content.strip.end_with?(code.content.strip)
+ code.remove
+ end
+ name = node.content.strip
name.prepend 'Manual: ' if type == 'Manual'
name.prepend 'Example: ' if type == 'Examples'
end
@@ -19,15 +34,28 @@ def get_name
def get_type
if subpath.start_with?('generated')
- # '> text()' doesn't include children's text in type naming
- at_xpath('//div[@class="related"]//li[not(@class="right")][7]/a/text()').content
-
+ # The breadcrumbs read "User Guide > [> ] >