Update Statsmodels documentation (0.15.0)

lib/docs/filters/statsmodels/clean_html.rb

@@ -2,16 +2,36 @@ module Docs
   class Statsmodels
     class CleanHtmlFilter < Filter
       def call
-        @doc = at_css('.body', '#notebook-container')
+        # The container keeps the whole main column so that EntriesFilter can
+        # read the breadcrumbs; only the article is documentation.
+        @doc = at_css('article.bd-article') || doc
+
+        css('.bd-sidebar-secondary', '.bd-toc', '.prev-next-area', '.prev-next-footer', '.bd-footer', 'form').remove
+
+        # "[source]" links point to the module sources, which aren't scraped.
+        css('a:has(.viewcode-link)').remove
 
         if root_page?
           at_css('h1').content = 'Statsmodels'
-          at_css('#basic-documentation').remove
-          at_css('#table-of-contents').remove
-          at_css('#indices-and-tables').remove
+          css('#index').remove
         end
 
-        css('div.cell', 'div.inner_cell', 'div.text_cell_render', 'div.input_area').each do |node|
+        # Parameter lists render as "name" followed by its type in a classifier
+        # span; separate them before Sphinx::CleanHtmlFilter flattens the <dt>.
+        css('dt > .classifier').each do |node|
+          node.before(': ')
+        end
+
+        # Notebook pages (nbsphinx): drop the "In [1]:" prompts, keep the input
+        # cells for Sphinx::CleanHtmlFilter (they're "highlight-ipython3"
+        # blocks) and turn the outputs into plain <pre> blocks.
+        css('.nbinput .prompt', '.nboutput .prompt').remove
+        css('.output_area div.highlight').each do |node|
+          pre = node.at_css('pre')
+          pre.content = pre.content
+          node.replace(pre)
+        end
+        css('.nbinput', '.nboutput', '.output_area', 'div.cell', 'div.inner_cell', 'div.text_cell_render').each do |node|
           node.before(node.children).remove
         end
 

lib/docs/filters/statsmodels/entries.rb

@@ -1,15 +1,30 @@
 module Docs
   class Statsmodels
     class EntriesFilter < Docs::EntriesFilter
+      # A couple of statsmodels.tools pages are only linked from the
+      # introductory chapters.
+      TYPE_ALIASES = {
+        'Getting started' => 'Tools',
+        'About statsmodels' => 'Tools'
+      }.freeze
+
       def get_name
         if subpath.start_with?('generated')
-          name = at_css('dt').content.strip
-          name.sub! %r{\(.*}, '()' # Remove method arguments
+          node = at_css('dt')
+          node.css('.headerlink').remove
+          name = node.content.strip
+          name.sub! %r{\(.*}m, '()' # Remove method arguments
           name.remove! %r{[\=\[].*} # Remove "[source]"
-          name.remove! %r{\A(class(method)?) }
+          name.remove! %r{\A(class(method)?|exception|property|static) }
           name.remove! %r{\Astatsmodels\.}
         else
-          name = at_css('h1', 'h2').content.strip
+          node = at_css('h1', 'h2')
+          node.css('.headerlink').remove
+          # Chapter titles end with the module name, e.g. "Statistics <code>stats</code>"
+          if (code = node.at_css('code')) && node.content.strip.end_with?(code.content.strip)
+            code.remove
+          end
+          name = node.content.strip
           name.prepend 'Manual: ' if type == 'Manual'
           name.prepend 'Example: ' if type == 'Examples'
         end
@@ -19,15 +34,28 @@ def get_name
 
       def get_type
         if subpath.start_with?('generated')
-          # '> text()' doesn't include children's text in type naming
-          at_xpath('//div[@class="related"]//li[not(@class="right")][7]/a/text()').content
-
+          # The breadcrumbs read "User Guide > <chapter> [> <class>] > <object>",
+          # where the chapter link reads e.g. "Statistics <code>stats</code>".
+          crumbs = css('.bd-breadcrumbs .breadcrumb-item:not(.breadcrumb-home):not(.active) a').map do |node|
+            node.css('code').remove
+            node.content.strip
+          end
+          crumbs.delete('User Guide')
+          TYPE_ALIASES.fetch(crumbs.first, crumbs.first) || api_type
         elsif subpath.start_with?('examples')
           'Examples'
         else
           'Manual'
         end
       end
+
+      private
+
+      # Pages that aren't reachable from a user guide chapter, such as the
+      # statsmodels.api aliases, are typed after their module.
+      def api_type
+        subpath[%r{\Agenerated/statsmodels\.([\w\.]+?)\.\w+\.html\z}, 1] || 'statsmodels'
+      end
     end
   end
 end

lib/docs/scrapers/statsmodels.rb

@@ -1,21 +1,25 @@
 module Docs
   class Statsmodels < UrlScraper
     self.type = 'sphinx'
-    self.release = '0.9.0'
-    self.base_url = 'http://www.statsmodels.org/stable/'
+    self.release = '0.15.0'
+    self.base_url = 'https://www.statsmodels.org/stable/'
     self.root_path = 'index.html'
     self.links = {
-      home: 'http://www.statsmodels.org/',
+      home: 'https://www.statsmodels.org/',
       code: 'https://github.com/statsmodels/statsmodels/'
     }
 
     html_filters.push 'math', 'statsmodels/entries', 'statsmodels/clean_html', 'sphinx/clean_html'
 
-    options[:skip] = %w(about.html search.html genindex.html)
+    # pydata-sphinx-theme: keep the whole main column so that the breadcrumbs,
+    # which sit outside the article, are available to EntriesFilter.
+    options[:container] = 'main.bd-main'
+
+    options[:skip] = %w(about.html search.html genindex.html py-modindex.html)
     options[:skip_patterns] = [/\Arelease/, /\Adev/, /\A_modules/, /\Adatasets/]
 
     options[:attribution] = <<-HTML
-      &copy; 2009&ndash;2012 Statsmodels Developers<br>
+      &copy; 2009&ndash;2018 Statsmodels Developers<br>
       &copy; 2006&ndash;2008 Scipy Developers<br>
       &copy; 2006 Jonathan E. Taylor<br>
       Licensed under the 3-clause BSD License.