Finish Pygame scraper

Author: Thibaut

assets/images/docs-2.png

@@ -1,5 +1,8 @@
 [
   [
+    "2018-07-29",
+    "New documentation: <a href=\"/pygame/\">Pygame</a>"
+  ], [
     "2018-07-08",
     "New documentations: <a href=\"/leaflet/\">Leaflet</a>, <a href=\"/terraform/\">Terraform</a> and <a href=\"/koa/\">Koa</a>"
   ], [

assets/images/docs-2@2x.png

@@ -545,6 +545,11 @@ credits = [
     '1996-2018 The PostgreSQL Global Development Group<br>&copy; 1994 The Regents of the University of California',
     'PostgreSQL',
     'https://www.postgresql.org/about/licence/'
+  ], [
+    'Pygame',
+    'Pygame Developpers',
+    'LGPLv2.1',
+    'https://raw.githubusercontent.com/pygame/pygame/master/LICENSE'
   ], [
     'Python',
     '2001-2018 Python Software Foundation<br>Python is a trademark of the Python Software Foundation.',

assets/javascripts/news.json

@@ -173,3 +173,4 @@
 ._icon-babel:before         { background-position: -2rem -3rem; @extend %doc-icon-2; }
 ._icon-leaflet:before       { background-position: -3rem -3rem; @extend %doc-icon-2; }
 ._icon-terraform:before     { background-position: -4rem -3rem; @extend %doc-icon-2; }
+._icon-pygame:before        { background-position: -5rem -3rem; @extend %doc-icon-2; }

assets/javascripts/templates/pages/about_tmpl.coffee

@@ -2,35 +2,29 @@ module Docs
   class Pygame
     class CleanHtmlFilter < Filter
       def call
-
         @doc = at_css '.body'
 
         if root_page?
           # remove unneeded stuff
           at_css('.modindex-jumpbox').remove
-          css('[role="navigation"],.pcap, .cap, .footer').each do |node|
-            node.remove
-          end
+          css('[role="navigation"],.pcap, .cap, .footer').remove
           # table -> list
           list = at_css('table')
           list.replace(list.children)
           list.name = 'ul'
           css('tr').each do |row|
             row.name = 'li'
-            row['class'] = ''
+            row.remove_attribute('class')
           end
+          at_css('h1').content = 'Pygame'
           return doc
         end
 
         # remove unwanted stuff
         # .headerlink => ¶ after links
         # .toc => table of content
         # .tooltip-content => tooltips after links to functions
-        if toremove = css('table.toc.docutils, .headerlink, .tooltip-content')
-          toremove.each do |node|
-            node.remove
-          end
-        end
+        css('table.toc.docutils, .headerlink, .tooltip-content').remove
 
         # Remove wrapper .section
         section = at_css('.section')
@@ -43,7 +37,7 @@ def call
           pre = node.at_css('pre')
           node.replace(pre)
           # gets rid of the already existing syntax highlighting
-          pre.inner_html = pre.inner_text
+          pre.content = pre.content
           pre['class'] = 'language-python'
           pre['data-language'] = "python"
         end
@@ -97,6 +91,10 @@ def call
           end
         end
 
+        css('> dl', '> dl > dd', 'h1 code').each do |node|
+          node.before(node.children).remove
+        end
+
         doc
       end
     end

assets/stylesheets/global/_icons.scss

@@ -2,33 +2,27 @@ module Docs
   class Pygame
     class EntriesFilter < Docs::EntriesFilter
       def get_name
-        return 'pygame'
+        at_css('h1').content.remove('pygame.')
       end
 
       def get_type
         at_css('h1').content
       end
 
-      def include_default_entry?
-        return false
-      end
-
       def additional_entries
-        return ['pygame'] if root_page?
+        return [] if root_page?
+
+        css('h1, h2, h3').each_with_object [] do |node, entries|
+          name = node['id'] || node['data-name']
 
-        entries = []
-        css('h1,h2,h3').each do |node|
-          parentclass = node.parent['class']
-          name = node['id']
-          if not name
-            name = node['data-name']
-          elsif parentclass.include?('function') or parentclass.include?('method')
-            name += '()'
+          if node.parent['class'].include?('function') or node.parent['class'].include?('method')
+            name << '()'
           end
-          name = name.sub('pygame.', '')
-          entries << [name, node['id'], nil]
+
+          name.remove!('pygame.')
+
+          entries << [name, node['id']] unless name == self.name
         end
-        entries
       end
     end
   end

lib/docs/filters/pygame/clean_html.rb

@@ -1,15 +1,8 @@
 module Docs
   class Pygame < UrlScraper
-
     self.type = 'simple'
-
-    self.release = 'v1.9.4.dev0'
-
-    self.initial_paths = ['py-modindex.html']
-    self.base_url = 'https://www.pygame.org/docs/'
+    self.release = '1.9.4'
     self.root_path = 'py-modindex.html'
-    self.initial_paths = []
-
     self.links = {
       home: 'https://www.pygame.org/',
       code: 'https://github.com/pygame/pygame'
@@ -20,7 +13,8 @@ class Pygame < UrlScraper
     options[:only_patterns] = [/ref\//]
 
     options[:attribution] = <<-HTML
-         &copy; Pygame Developpers.
+      &copy; Pygame Developpers.<br>
+      Licensed under the GNU LGPL License version 2.1.
     HTML
   end
 end