diff --git a/lib/docs/filters/statsmodels/clean_html.rb b/lib/docs/filters/statsmodels/clean_html.rb index 79e3424194..a3c07e8980 100644 --- a/lib/docs/filters/statsmodels/clean_html.rb +++ b/lib/docs/filters/statsmodels/clean_html.rb @@ -2,16 +2,36 @@ module Docs class Statsmodels class CleanHtmlFilter < Filter def call - @doc = at_css('.body', '#notebook-container') + # The container keeps the whole main column so that EntriesFilter can + # read the breadcrumbs; only the article is documentation. + @doc = at_css('article.bd-article') || doc + + css('.bd-sidebar-secondary', '.bd-toc', '.prev-next-area', '.prev-next-footer', '.bd-footer', 'form').remove + + # "[source]" links point to the module sources, which aren't scraped. + css('a:has(.viewcode-link)').remove if root_page? at_css('h1').content = 'Statsmodels' - at_css('#basic-documentation').remove - at_css('#table-of-contents').remove - at_css('#indices-and-tables').remove + css('#index').remove end - css('div.cell', 'div.inner_cell', 'div.text_cell_render', 'div.input_area').each do |node| + # Parameter lists render as "name" followed by its type in a classifier + # span; separate them before Sphinx::CleanHtmlFilter flattens the
. + css('dt > .classifier').each do |node| + node.before(': ') + end + + # Notebook pages (nbsphinx): drop the "In [1]:" prompts, keep the input + # cells for Sphinx::CleanHtmlFilter (they're "highlight-ipython3" + # blocks) and turn the outputs into plain
 blocks.
+        css('.nbinput .prompt', '.nboutput .prompt').remove
+        css('.output_area div.highlight').each do |node|
+          pre = node.at_css('pre')
+          pre.content = pre.content
+          node.replace(pre)
+        end
+        css('.nbinput', '.nboutput', '.output_area', 'div.cell', 'div.inner_cell', 'div.text_cell_render').each do |node|
           node.before(node.children).remove
         end
 
diff --git a/lib/docs/filters/statsmodels/entries.rb b/lib/docs/filters/statsmodels/entries.rb
index e0d0f42991..bf4d155ef2 100644
--- a/lib/docs/filters/statsmodels/entries.rb
+++ b/lib/docs/filters/statsmodels/entries.rb
@@ -1,15 +1,30 @@
 module Docs
   class Statsmodels
     class EntriesFilter < Docs::EntriesFilter
+      # A couple of statsmodels.tools pages are only linked from the
+      # introductory chapters.
+      TYPE_ALIASES = {
+        'Getting started' => 'Tools',
+        'About statsmodels' => 'Tools'
+      }.freeze
+
       def get_name
         if subpath.start_with?('generated')
-          name = at_css('dt').content.strip
-          name.sub! %r{\(.*}, '()' # Remove method arguments
+          node = at_css('dt')
+          node.css('.headerlink').remove
+          name = node.content.strip
+          name.sub! %r{\(.*}m, '()' # Remove method arguments
           name.remove! %r{[\=\[].*} # Remove "[source]"
-          name.remove! %r{\A(class(method)?) }
+          name.remove! %r{\A(class(method)?|exception|property|static) }
           name.remove! %r{\Astatsmodels\.}
         else
-          name = at_css('h1', 'h2').content.strip
+          node = at_css('h1', 'h2')
+          node.css('.headerlink').remove
+          # Chapter titles end with the module name, e.g. "Statistics stats"
+          if (code = node.at_css('code')) && node.content.strip.end_with?(code.content.strip)
+            code.remove
+          end
+          name = node.content.strip
           name.prepend 'Manual: ' if type == 'Manual'
           name.prepend 'Example: ' if type == 'Examples'
         end
@@ -19,15 +34,28 @@ def get_name
 
       def get_type
         if subpath.start_with?('generated')
-          # '> text()' doesn't include children's text in type naming
-          at_xpath('//div[@class="related"]//li[not(@class="right")][7]/a/text()').content
-
+          # The breadcrumbs read "User Guide >  [> ] > ",
+          # where the chapter link reads e.g. "Statistics stats".
+          crumbs = css('.bd-breadcrumbs .breadcrumb-item:not(.breadcrumb-home):not(.active) a').map do |node|
+            node.css('code').remove
+            node.content.strip
+          end
+          crumbs.delete('User Guide')
+          TYPE_ALIASES.fetch(crumbs.first, crumbs.first) || api_type
         elsif subpath.start_with?('examples')
           'Examples'
         else
           'Manual'
         end
       end
+
+      private
+
+      # Pages that aren't reachable from a user guide chapter, such as the
+      # statsmodels.api aliases, are typed after their module.
+      def api_type
+        subpath[%r{\Agenerated/statsmodels\.([\w\.]+?)\.\w+\.html\z}, 1] || 'statsmodels'
+      end
     end
   end
 end
diff --git a/lib/docs/scrapers/statsmodels.rb b/lib/docs/scrapers/statsmodels.rb
index 906c89096a..a057f9c5e4 100644
--- a/lib/docs/scrapers/statsmodels.rb
+++ b/lib/docs/scrapers/statsmodels.rb
@@ -1,21 +1,25 @@
 module Docs
   class Statsmodels < UrlScraper
     self.type = 'sphinx'
-    self.release = '0.9.0'
-    self.base_url = 'http://www.statsmodels.org/stable/'
+    self.release = '0.15.0'
+    self.base_url = 'https://www.statsmodels.org/stable/'
     self.root_path = 'index.html'
     self.links = {
-      home: 'http://www.statsmodels.org/',
+      home: 'https://www.statsmodels.org/',
       code: 'https://github.com/statsmodels/statsmodels/'
     }
 
     html_filters.push 'math', 'statsmodels/entries', 'statsmodels/clean_html', 'sphinx/clean_html'
 
-    options[:skip] = %w(about.html search.html genindex.html)
+    # pydata-sphinx-theme: keep the whole main column so that the breadcrumbs,
+    # which sit outside the article, are available to EntriesFilter.
+    options[:container] = 'main.bd-main'
+
+    options[:skip] = %w(about.html search.html genindex.html py-modindex.html)
     options[:skip_patterns] = [/\Arelease/, /\Adev/, /\A_modules/, /\Adatasets/]
 
     options[:attribution] = <<-HTML
-      © 2009–2012 Statsmodels Developers
+ © 2009–2018 Statsmodels Developers
© 2006–2008 Scipy Developers
© 2006 Jonathan E. Taylor
Licensed under the 3-clause BSD License.