Finish Dart scraper

Author: Thibaut

assets/images/docs-2.png

@@ -1,5 +1,8 @@
 [
   [
+    "2018-08-06",
+    "New documentation: <a href=\"/dart/\">Dart</a>"
+  ], [
     "2018-07-29",
     "New documentations: <a href=\"/bash/\">Bash</a>, <a href=\"/graphite/\">Graphite</a> and <a href=\"/pygame/\">Pygame</a>"
   ], [

assets/images/docs-2@2x.png

@@ -176,3 +176,4 @@
 ._icon-terraform:before     { background-position: -4rem -3rem; @extend %doc-icon-2; }
 ._icon-pygame:before        { background-position: -5rem -3rem; @extend %doc-icon-2; }
 ._icon-bash:before          { background-position: -6rem -3rem; @extend %doc-icon-2; }
+._icon-dart:before          { background-position: -7rem -3rem; @extend %doc-icon-2; }

assets/javascripts/news.json

@@ -2,11 +2,11 @@
   @extend %simple;
 
   dl:not(.dl-horizontal) dt, .multi-line-signature {
-    @extend %note, %note-blue;
-    padding: 1px 0.5rem 2px 0.5rem;
+    @extend %block-label;
 
     .features {
       float: right;
+      color: $textColorLight;
     }
   }
 }

assets/stylesheets/global/_icons.scss

@@ -4,7 +4,7 @@ class CleanHtmlFilter < Filter
       def call
         # Move the title into the main content node in the v1 docs
         title = at_css('h1.title')
-        unless title.nil?
+        if title
           name = title.children.last.content.strip
           kind = title.at_css('.kind').content
           at_css('.main-content').prepend_child("<h1>#{name} #{kind}</h1>")
@@ -39,11 +39,26 @@ def call
           header.add_child node unless header.nil?
         end
 
+        css('section').each do |node|
+          if node['id'] && node.first_element_child
+            node.first_element_child['id'] ||= node['id']
+          end
+
+          node.before(node.children).remove
+        end
+
+        css('span').each do |node|
+          node.before(node.children).remove
+        end
+
         # Make code blocks detectable by Prism
         css('pre').each do |node|
           node['data-language'] = 'dart'
+          node.content = node.content
         end
 
+        css('.properties', '.property', '.callables', '.callable').remove_attr('class')
+
         doc
       end
     end

assets/stylesheets/pages/_dart.scss

@@ -2,8 +2,15 @@ module Docs
   class Dart
     class EntriesFilter < Docs::EntriesFilter
       def get_name
-        title = get_title
-        kind = get_kind
+        title = at_css('h1.title')
+        if title # v1
+          name = title.element_children.last.content.strip
+          kind = title.at_css('.kind').content
+        else # v2
+          title = at_css('.main-content > h1')
+          name = title.content[/(.*)( )/, 1].split(' top-level')[0]
+          kind = title.content[/(.*)( )(.+)/, 3]
+        end
 
         breadcrumbs = at_css('.breadcrumbs').css('li:not(.self-crumb) > a')
         first_part = ''
@@ -15,6 +22,7 @@ def get_name
         end
 
         separator = ''
+
         unless first_part.empty?
           if kind.include?('class')
             separator = ':'
@@ -23,36 +31,12 @@ def get_name
           end
         end
 
-        first_part + separator + title
+        "#{first_part}#{separator}#{name}"
       end
 
       def get_type
         at_css('.breadcrumbs > li:nth-child(2)').content.split(' ')[0]
       end
-
-      def get_title
-        title = at_css('h1.title')
-
-        if not title.nil?
-          # v1
-          title.children.last.content.strip
-        else
-          # v2
-          at_css('.main-content > h1').content[/(.*)( )/, 1].split(' top-level')[0]
-        end
-      end
-
-      def get_kind
-        title = at_css('h1.title')
-
-        if not title.nil?
-          # v1
-          title.at_css('.kind').content
-        else
-          # v2
-          at_css('.main-content > h1').content[/(.*)( )(.+)/, 3]
-        end
-      end
     end
   end
 end

lib/docs/filters/dart/clean_html.rb

@@ -11,24 +11,27 @@ class Dart < FileScraper
 
     options[:fix_urls] = ->(url) do
       # localhost/dart-web_audio/..dart-io/dart-io-library.html > localhost/dart-io/dart-io-library.html
-      url.sub(/(([^\/]+)\/\.\.)/, '')
+      url.remove!(/[^\/]+\/\.\./)
+      url
     end
 
     options[:attribution] = <<-HTML
-      &copy; 2012, the Dart project authors<br>
+      &copy; 2012 the Dart project authors<br>
       Licensed under the Creative Commons Attribution-ShareAlike License v4.0.
     HTML
 
     # Download the documentation from https://www.dartlang.org/tools/sdk/archive
 
-    version '1' do
-      self.release = '1.24.3'
-      self.dir = '/home/jasper/Documents/dart-docs-1.24.3'
+    version '2' do
+      self.release = '2.0.0'
+      self.dir = '/Users/Thibaut/DevDocs/Docs/Dart2'
+      self.base_url = "https://api.dartlang.org/stable/#{release}/"
     end
 
-    version '2' do
-      self.release = '2.0.0-dev.68.0'
-      self.dir = '/home/jasper/Documents/dart-docs-2.0.0-dev.68.0'
+    version '1' do
+      self.release = '1.24.3'
+      self.dir = '/Users/Thibaut/DevDocs/Docs/Dart1'
+      self.base_url = "https://api.dartlang.org/stable/#{release}/"
     end
   end
 end