Add better logging and get_latest_version implementations for 10 scrapers

Author: jmerle

lib/docs/core/scraper.rb

@@ -132,7 +132,7 @@ def options
       end
     end
 
-    def get_latest_version
+    def get_latest_version(&block)
       raise NotImplementedError
     end
 
@@ -231,6 +231,38 @@ def additional_options
       {}
     end
 
+    # Utility methods for get_latest_version
+
+    def fetch(url, &block)
+      Request.run(url) do |response|
+        if response.success?
+          block.call response.body
+        else
+          block.call nil
+        end
+      end
+    end
+
+    def fetch_doc(url, &block)
+      fetch(url) do |body|
+        parser = Parser.new(body)
+        block.call parser.html
+      end
+    end
+
+    def fetch_json(url, &block)
+      fetch(url) do |body|
+        json = JSON.parse(body)
+        block.call json
+      end
+    end
+
+    def get_npm_version(package, &block)
+      fetch_json("https://registry.npmjs.com/#{package}") do |json|
+        block.call json['dist-tags']['latest']
+      end
+    end
+
     module FixInternalUrlsBehavior
       def self.included(base)
         base.extend ClassMethods

lib/docs/scrapers/angular.rb

@@ -155,6 +155,10 @@ def handle_response(response)
       end
     end
 
+    def get_latest_version(&block)
+      get_npm_version('@angular/core', &block)
+    end
+
     private
 
     def parse(response)

lib/docs/scrapers/angularjs.rb

@@ -69,5 +69,9 @@ class Angularjs < UrlScraper
       self.release = '1.2.32'
       self.base_url = "https://code.angularjs.org/#{release}/docs/partials/"
     end
+
+    def get_latest_version(&block)
+      get_npm_version('angular', &block)
+    end
   end
 end

lib/docs/scrapers/ansible.rb

@@ -87,5 +87,11 @@ class Ansible < UrlScraper
         quickstart.html
         list_of_all_modules.html)
     end
+
+    def get_latest_version(&block)
+      fetch_doc('https://docs.ansible.com/ansible/latest/index.html') do |doc|
+        block.call doc.at_css('.DocSiteProduct-CurrentVersion').content.strip
+      end
+    end
   end
 end

lib/docs/scrapers/apache.rb

@@ -33,5 +33,11 @@ class Apache < UrlScraper
       &copy; 2018 The Apache Software Foundation<br>
       Licensed under the Apache License, Version 2.0.
     HTML
+
+    def get_latest_version(&block)
+      fetch_doc('http://httpd.apache.org/docs/') do |doc|
+        block.call doc.at_css('#apcontents > ul a')['href'][0...-1]
+      end
+    end
   end
 end

lib/docs/scrapers/apache_pig.rb

@@ -43,5 +43,11 @@ class ApachePig < UrlScraper
       self.base_url = "https://pig.apache.org/docs/r#{release}/"
     end
 
+    def get_latest_version(&block)
+      fetch_doc('https://pig.apache.org/') do |doc|
+        item = doc.at_css('div[id="menu_1.2"] > .menuitem:last-child')
+        block.call item.content.strip.sub(/Release /, '')
+      end
+    end
   end
 end

lib/docs/scrapers/async.rb

@@ -17,5 +17,12 @@ class Async < UrlScraper
       &copy; 2010&ndash;2018 Caolan McMahon<br>
       Licensed under the MIT License.
     HTML
+
+    def get_latest_version(&block)
+      fetch_doc('https://caolan.github.io/async/') do |doc|
+        version = doc.at_css('#version-dropdown > a').content.strip[1..-1]
+        block.call version
+      end
+    end
   end
 end

lib/docs/scrapers/babel.rb

@@ -22,5 +22,11 @@ class Babel < UrlScraper
     stub '' do
       '<div></div>'
     end
+
+    def get_latest_version(&block)
+      fetch_doc('https://babeljs.io/docs/en/') do |doc|
+        block.call doc.at_css('a[href="/versions"] > h3').content
+      end
+    end
   end
 end

lib/docs/scrapers/backbone.rb

@@ -20,5 +20,12 @@ class Backbone < UrlScraper
       &copy; 2010&ndash;2016 Jeremy Ashkenas, DocumentCloud<br>
       Licensed under the MIT License.
     HTML
+
+    def get_latest_version(&block)
+      fetch_doc('https://backbonejs.org/') do |doc|
+        version = doc.at_css('.version').content
+        block.call version[1...-1]
+      end
+    end
   end
 end

lib/docs/scrapers/bash.rb

@@ -17,5 +17,12 @@ class Bash < UrlScraper
       Copyright &copy; 2000, 2001, 2002, 2007, 2008 Free Software Foundation, Inc.<br>
       Licensed under the GNU Free Documentation License.
     HTML
+
+    def get_latest_version(&block)
+      fetch('https://www.gnu.org/software/bash/manual/html_node/index.html') do |body|
+        version = body.scan(/, Version ([0-9.]+)/)[0][0]
+        block.call version[0...-1]
+      end
+    end
   end
 end