Finish get_latest_version for 81 scrapers and add uploading functionality

Author: jmerle

Gemfile

@@ -40,6 +40,7 @@ group :docs do
   gem 'unix_utils', require: false
   gem 'tty-pager', require: false
   gem 'net-sftp', '>= 2.1.3.rc2', require: false
+  gem 'terminal-table', require: false
 end
 
 group :test do

Gemfile.lock

@@ -101,6 +101,8 @@ GEM
       unicode-display_width (~> 1.4.0)
       unicode_utils (~> 1.4.0)
     strings-ansi (0.1.0)
+    terminal-table (1.8.0)
+      unicode-display_width (~> 1.1, >= 1.1.1)
     thin (1.7.2)
       daemons (~> 1.0, >= 1.0.9)
       eventmachine (~> 1.0, >= 1.0.4)
@@ -153,6 +155,7 @@ DEPENDENCIES
   sinatra-contrib
   sprockets
   sprockets-helpers
+  terminal-table
   thin
   thor
   tty-pager

lib/docs/core/scraper.rb

@@ -132,7 +132,7 @@ def options
       end
     end
 
-    def get_latest_version(&block)
+    def get_latest_version(options, &block)
       raise NotImplementedError
     end
 
@@ -147,15 +147,15 @@ def get_latest_version(&block)
     # 1 -> 2 = outdated
     # 1.1 -> 1.2 = outdated
     # 1.1.1 -> 1.1.2 = not outdated
-    def is_outdated(current_version, latest_version)
-      current_parts = current_version.split(/\./).map(&:to_i)
+    def is_outdated(scraper_version, latest_version)
+      scraper_parts = scraper_version.split(/\./).map(&:to_i)
       latest_parts = latest_version.split(/\./).map(&:to_i)
 
       # Only check the first two parts, the third part is for patch updates
       [0, 1].each do |i|
-        break if i >= current_parts.length or i >= latest_parts.length
-        return true if latest_parts[i] > current_parts[i]
-        return false if latest_parts[i] < current_parts[i]
+        break if i >= scraper_parts.length or i >= latest_parts.length
+        return true if latest_parts[i] > scraper_parts[i]
+        return false if latest_parts[i] < scraper_parts[i]
       end
 
       false
@@ -231,38 +231,62 @@ def additional_options
       {}
     end
 
+    #
     # Utility methods for get_latest_version
+    #
 
-    def fetch(url, &block)
-      Request.run(url) do |response|
+    def fetch(url, options, &block)
+      headers = {}
+
+      if options.key?(:github_token) and url.start_with?('https://api.github.com/')
+        headers['Authorization'] = "token #{options[:github_token]}"
+      end
+
+      options[:logger].debug("Fetching #{url}")
+
+      Request.run(url, { headers: headers }) do |response|
         if response.success?
           block.call response.body
         else
+          options[:logger].error("Couldn't fetch #{url} (response code #{response.code})")
           block.call nil
         end
       end
     end
 
-    def fetch_doc(url, &block)
-      fetch(url) do |body|
-        parser = Parser.new(body)
-        block.call parser.html
+    def fetch_doc(url, options, &block)
+      fetch(url, options) do |body|
+        block.call Nokogiri::HTML.parse body, nil, 'UTF-8'
       end
     end
 
-    def fetch_json(url, &block)
-      fetch(url) do |body|
+    def fetch_json(url, options, &block)
+      fetch(url, options) do |body|
         json = JSON.parse(body)
         block.call json
       end
     end
 
-    def get_npm_version(package, &block)
-      fetch_json("https://registry.npmjs.com/#{package}") do |json|
+    def get_npm_version(package, options, &block)
+      fetch_json("https://registry.npmjs.com/#{package}", options) do |json|
         block.call json['dist-tags']['latest']
       end
     end
 
+    def get_latest_github_release(owner, repo, options, &block)
+      fetch_json("https://api.github.com/repos/#{owner}/#{repo}/releases/latest", options, &block)
+    end
+
+    def get_github_tags(owner, repo, options, &block)
+      fetch_json("https://api.github.com/repos/#{owner}/#{repo}/tags", options, &block)
+    end
+
+    def get_github_file_contents(owner, repo, path, options, &block)
+      fetch_json("https://api.github.com/repos/#{owner}/#{repo}/contents/#{path}", options) do |json|
+        block.call(Base64.decode64(json['content']))
+      end
+    end
+
     module FixInternalUrlsBehavior
       def self.included(base)
         base.extend ClassMethods

lib/docs/scrapers/angular.rb

@@ -155,8 +155,8 @@ def handle_response(response)
       end
     end
 
-    def get_latest_version(&block)
-      get_npm_version('@angular/core', &block)
+    def get_latest_version(options, &block)
+      get_npm_version('@angular/core', options, &block)
     end
 
     private

lib/docs/scrapers/angularjs.rb

@@ -70,8 +70,8 @@ class Angularjs < UrlScraper
       self.base_url = "https://code.angularjs.org/#{release}/docs/partials/"
     end
 
-    def get_latest_version(&block)
-      get_npm_version('angular', &block)
+    def get_latest_version(options, &block)
+      get_npm_version('angular', options, &block)
     end
   end
 end

lib/docs/scrapers/ansible.rb

@@ -88,8 +88,8 @@ class Ansible < UrlScraper
         list_of_all_modules.html)
     end
 
-    def get_latest_version(&block)
-      fetch_doc('https://docs.ansible.com/ansible/latest/index.html') do |doc|
+    def get_latest_version(options, &block)
+      fetch_doc('https://docs.ansible.com/ansible/latest/index.html', options) do |doc|
         block.call doc.at_css('.DocSiteProduct-CurrentVersion').content.strip
       end
     end

lib/docs/scrapers/apache.rb

@@ -34,8 +34,8 @@ class Apache < UrlScraper
       Licensed under the Apache License, Version 2.0.
     HTML
 
-    def get_latest_version(&block)
-      fetch_doc('http://httpd.apache.org/docs/') do |doc|
+    def get_latest_version(options, &block)
+      fetch_doc('http://httpd.apache.org/docs/', options) do |doc|
         block.call doc.at_css('#apcontents > ul a')['href'][0...-1]
       end
     end

lib/docs/scrapers/apache_pig.rb

@@ -43,8 +43,8 @@ class ApachePig < UrlScraper
       self.base_url = "https://pig.apache.org/docs/r#{release}/"
     end
 
-    def get_latest_version(&block)
-      fetch_doc('https://pig.apache.org/') do |doc|
+    def get_latest_version(options, &block)
+      fetch_doc('https://pig.apache.org/', options) do |doc|
         item = doc.at_css('div[id="menu_1.2"] > .menuitem:last-child')
         block.call item.content.strip.sub(/Release /, '')
       end

lib/docs/scrapers/async.rb

@@ -18,8 +18,8 @@ class Async < UrlScraper
       Licensed under the MIT License.
     HTML
 
-    def get_latest_version(&block)
-      fetch_doc('https://caolan.github.io/async/') do |doc|
+    def get_latest_version(options, &block)
+      fetch_doc('https://caolan.github.io/async/', options) do |doc|
         version = doc.at_css('#version-dropdown > a').content.strip[1..-1]
         block.call version
       end

lib/docs/scrapers/babel.rb

@@ -23,8 +23,8 @@ class Babel < UrlScraper
       '<div></div>'
     end
 
-    def get_latest_version(&block)
-      fetch_doc('https://babeljs.io/docs/en/') do |doc|
+    def get_latest_version(options, &block)
+      fetch_doc('https://babeljs.io/docs/en/', options) do |doc|
         block.call doc.at_css('a[href="/versions"] > h3').content
       end
     end