Merge ComponentsGuide.Fetch.Get into ComponentsGuide.Fetch

Author: royalicing

lib/components_guide/fetch.ex

@@ -1,11 +1,177 @@
 defmodule ComponentsGuide.Fetch do
-  alias __MODULE__.{Get, Request}
+  alias ComponentsGuide.Fetch.{Request, Response}
 
+  @timeout 5000
+
+  @doc ~S"""
+  Fetches the given URL, following redirects (if any).
+  """
   def get!(url_string) when is_binary(url_string) do
-    Get.get_following_redirects!(url_string)
+    get_following_redirects!(url_string)
+  end
+
+  defmodule Timings do
+    defstruct [:duration, :start]
+
+    def start() do
+      %__MODULE__{
+        start: System.monotonic_time()
+      }
+    end
+
+    def finish(timings = %__MODULE__{start: start}) do
+      duration = System.monotonic_time() - start
+      put_in(timings.duration, duration)
+    end
+
+    def start_with_telemetry(event_name, metadata \\ %{}) do
+      t = start()
+
+      :telemetry.execute(
+        event_name,
+        %{start: t.start},
+        metadata
+      )
+
+      t
+    end
+
+    def finish_with_telemetry(t = %__MODULE__{}, event_name, metadata \\ %{}) do
+      t = finish(t)
+
+      :telemetry.execute(
+        event_name,
+        %{duration: t.duration},
+        metadata
+      )
+
+      t
+    end
+  end
+
+  def get_following_redirects!(url_string) when is_binary(url_string) do
+    response =
+      case Request.new(url_string) do
+        {:ok, request} ->
+          load!(request)
+
+        {:error, reason} ->
+          Response.failed(url_string, reason)
+      end
+
+    case response do
+      %Response{status: status, headers: headers} = resp when status >= 300 and status < 400 ->
+        case Enum.find(headers, fn {key, _} -> key == "location" end) do
+          # No redirect!
+          nil ->
+            resp
+
+          {_, location} ->
+            IO.puts("Following #{status} redirect to #{location}")
+
+            case Request.new(location) do
+              {:ok, request} ->
+                # TODO: use existing conn if host is the same.
+                load!(request)
+
+              {:error, reason} ->
+                Response.failed(url_string, reason)
+            end
+        end
+
+      other ->
+        other
+    end
+  end
+
+  def load!(req = %Request{uri: %URI{host: host, port: 443}}) do
+    t = Timings.start_with_telemetry([:fetch, :load!, :start], %{req: req})
+
+    {:ok, conn} = Mint.HTTP.connect(:https, host, 443, mode: :passive, protocols: [:http1])
+    {conn, response} = do_request(conn, req)
+    Mint.HTTP.close(conn)
+
+    t =
+      Timings.finish_with_telemetry(t, [:fetch, :load!, :done], %{
+        req: req
+      })
+
+    response = Response.add_timings(response, t)
+
+    IO.puts(
+      "Loaded #{req.url_string} in #{System.convert_time_unit(t.duration, :native, :millisecond)}ms. #{inspect(response.done?)}"
+    )
+
+    response
+  end
+
+  def load_many_example(n \\ 2) do
+    load_many!(
+      "components.guide",
+      Enum.map(0..n, fn _ -> Request.new!("https://components.guide/") end)
+    )
+  end
+
+  def load_many!(host, reqs) when is_binary(host) and is_list(reqs) do
+    t = Timings.start_with_telemetry([:fetch, :load_many!, :start], %{host: host})
+
+    {:ok, conn} = Mint.HTTP.connect(:https, host, 443, mode: :passive, protocols: [:http1])
+
+    {conn, results} =
+      Enum.reduce(reqs, {conn, []}, fn
+        %Request{uri: %URI{host: ^host, port: 443}} = req, {conn, results} ->
+          t = Timings.start_with_telemetry([:fetch, :load_many!, :request, :start], %{req: req})
+
+          {conn, response} = do_request(conn, req)
+
+          t =
+            Timings.finish_with_telemetry(t, [:fetch, :load_many!, :request, :done], %{
+              req: req
+            })
+
+          response = Response.add_timings(response, t)
+
+          {conn, [response | results]}
+      end)
+
+    Mint.HTTP.close(conn)
+    results = Enum.reverse(results)
+
+    Timings.finish_with_telemetry(t, [:fetch, :load_many!, :done], %{host: host})
+    results
   end
 
-  def load!(%Request{} = request) do
-    Get.load!(request)
+  defp recv_all(result = %Response{done?: true}, conn, _request_ref), do: {conn, result}
+
+  defp recv_all(result, conn, request_ref) do
+    case Mint.HTTP.recv(conn, 0, @timeout) do
+      {:ok, conn, responses} ->
+        Response.add_responses(result, responses, request_ref)
+        |> recv_all(conn, request_ref)
+
+      {:error, conn, error, _responses} ->
+        {conn, Response.add_error(result, error)}
+    end
+  end
+
+  defp do_request(
+         conn,
+         %Request{
+           method: method,
+           uri: %URI{path: path},
+           headers: headers,
+           body: body,
+           url_string: url_string
+         }
+       ) do
+    result = Response.new(url_string)
+
+    case Mint.HTTP.request(conn, method, path || "/", headers, body) do
+      {:error, conn, reason} ->
+        {conn, Response.add_error(result, reason)}
+
+      {:ok, conn, request_ref} ->
+        recv_all(result, conn, request_ref)
+    end
   end
 end

lib/components_guide/fetch/get.ex

@@ -1,6 +1,39 @@
 defmodule ComponentsGuide.FetchTest do
   use ExUnit.Case, async: true
 
-  doctest ComponentsGuide.Fetch
-  # defdelegate test(message, a, contents), to: ComponentsGuide.Fetch.Test
+  # doctest ComponentsGuide.Fetch
+
+  test "can fetch news.ycombinator.com" do
+    response = ComponentsGuide.Fetch.get!("https://news.ycombinator.com/")
+
+    assert %ComponentsGuide.Fetch.Response{
+             done?: true,
+             url: "https://news.ycombinator.com/",
+             status: 200,
+             headers: _,
+             body: _
+           } = response
+  end
+
+  test "can load HEAD for news.ycombinator.com" do
+    response =
+      ComponentsGuide.Fetch.load!(
+        ComponentsGuide.Fetch.Request.new!("https://news.ycombinator.com/", method: "HEAD")
+      )
+
+    assert %ComponentsGuide.Fetch.Response{
+             done?: true,
+             url: "https://news.ycombinator.com/",
+             status: _,
+             headers: _,
+             body: _
+           } = response
+
+    content_type =
+      Enum.find_value(response.headers, fn
+        {"content-type", value} -> value
+        _ -> nil
+      end)
+    assert "text/html" == content_type
+  end
 end