Shopify.Pagination module

lib/shopify/enumerable.ex

@@ -0,0 +1,44 @@
+defmodule Shopify.Enumerable do
+  @moduledoc false
+
+  alias Shopify.{Error, Pagination, Response}
+
+  @enforce_keys [:data, :func, :middleware, :params, :session]
+  defstruct @enforce_keys
+
+  defimpl Enumerable do
+    # See https://hexdocs.pm/elixir/Enumerable.html
+
+    def count(_), do: {:error, __MODULE__}
+    def member?(_, _), do: {:error, __MODULE__}
+    def slice(_), do: {:error, __MODULE__}
+
+    def reduce(%Shopify.Enumerable{}, {:halt, acc}, _fun), do: {:halted, acc}
+
+    def reduce(%Shopify.Enumerable{} = e, {:suspend, acc}, fun) do
+      {:suspended, acc, &reduce(e, &1, fun)}
+    end
+
+    def reduce(%Shopify.Enumerable{data: [], params: nil}, {:cont, acc}, _fun) do
+      {:done, acc}
+    end
+
+    def reduce(%Shopify.Enumerable{data: []} = e, {:cont, acc}, fun) do
+      case e.middleware.(fn -> e.func.(e.session, e.params) end) do
+        {:ok, %Response{data: data} = resp} ->
+          reduce(
+            %{e | data: data, params: Pagination.next_page_params(resp)},
+            {:cont, acc},
+            fun
+          )
+
+        {:error, %Error{} = error} ->
+          raise error
+      end
+    end
+
+    def reduce(%Shopify.Enumerable{data: [h | t]} = e, {:cont, acc}, fun) do
+      reduce(%{e | data: t}, fun.(h, acc), fun)
+    end
+  end
+end

lib/shopify/pagination.ex

@@ -0,0 +1,112 @@
+defmodule Shopify.Pagination do
+  @moduledoc """
+  Functions for performing cursor-based pagination. API versions prior to `"2019-10"` do not
+  support cursor-based pagination on all endpoints. See [Shopify's
+  documentation](https://help.shopify.com/en/api/guides/paginated-rest-results) for details.
+  """
+
+  alias Shopify.Response
+
+  @doc """
+  Returns an enumerable for iterating over Shopify items. In particular, this enumerable can be
+  used with any of the functions in the `Enum` or `Stream` modules. The enumerable is lazy; no
+  requests are made until iteration begins. It will automatically load additional pages of data as
+  needed.
+
+  If an error occurs while fetching a page of data then `Shopify.Error` will be raised.
+
+  ## Arguments
+
+    * `session` - A `Shopify.Session` struct. See `Shopify.session/0`.
+    * `func` - Function with an arity of 2 that accepts a `Shopify.Session` record and a map of
+      query params. For example, this value could be `&Shopify.Product.all/2` to iterate over
+      products or `&Shopify.Order.all/2` to iterate over orders.
+    * `params` - Map of query params to be passed to `func` when fetching the first page. By
+      default, most Shopify endpoints for listing items will return 50 items per page. If you plan
+      on iterating over a large number of items then it will be more efficient to request the
+      maximum number of items per page by specifying `%{limit: 250}`.
+
+  ## Options
+
+    * `:middleware` - An optional middleware function whose primary purpose is to allow an
+      application to throttle requests to avoid going over Shopify's rate limit. Must be a
+      function with an arity of 1 that will be passed another function. The middleware must call
+      that function and return the value returned by that function, which will be `{:ok,
+      %Shopify.Response{}}` or `{:error, %Shopify.Error{}}`. Defaults to `& &1.()`.
+
+  ## Example
+
+      Shopify.session() |> Shopify.Pagination.enumerable(&Shopify.Product.all/2)
+      |> Stream.each(fn product -> IO.puts(product.id) end)
+      |> Stream.run()
+
+  """
+  def enumerable(session, func, params \\ %{}, opts \\ [])
+
+  def enumerable(session, func, params, opts) when is_list(params) do
+    enumerable(session, func, Enum.into(params, %{}), opts)
+  end
+
+  def enumerable(%Shopify.Session{} = session, func, %{} = params, opts)
+      when is_function(func) and is_list(opts) do
+    %Shopify.Enumerable{
+      data: [],
+      func: func,
+      middleware: Keyword.get(opts, :middleware, & &1.()),
+      params: params,
+      session: session
+    }
+  end
+
+  @doc """
+  Returns a map containing the query params needed to fetch the next page of results when passed a
+  `Shopify.Response` struct. It will return `nil` if there are no additional pages. This map will
+  contain `"limit"` and `"page_info"` keys as [documented by
+  Shopify](https://help.shopify.com/en/api/guides/paginated-rest-results).
+
+  If any sorting or filtering params were passed with the API call for the first page then these
+  params do not need to be passed when fetching subsequent pages. This information is encoded into
+  the `page_info` value.
+
+  ## Example
+
+      {:ok, resp} = Shopify.session() |> Shopify.Product.all()
+      case Shopify.Pagination.next_page_params(resp) do
+         nil ->
+           # no additional pages
+           nil
+         page_params ->
+           # fetch the next page
+           {:ok, resp} = Shopify.session() |> Shopify.Product.all(page_params)
+           # and so on...
+       end
+  """
+  def next_page_params(resp), do: page_params(resp, "next")
+
+  defp page_params(%Response{headers: headers}, rel) do
+    Enum.find_value(headers, fn {k, v} ->
+      with "link" <- String.downcase(k),
+           url when is_binary(url) <- parse_link_header(v)[rel],
+           %URI{query: query} = URI.parse(url) do
+        URI.decode_query(query)
+      else
+        _ -> nil
+      end
+    end)
+  end
+
+  # Returns map like %{"next" => <url>, "previous" => <url>}. Logic inspired by
+  # https://github.com/Shopify/shopify_api/blob/master/lib/shopify_api/pagination_link_headers.rb
+  defp parse_link_header(header) do
+    header
+    |> String.split(",")
+    |> Enum.map(fn link ->
+      [url_part, rel_part] = String.split(link, "; ")
+      [rel] = Regex.run(~R{rel="(.*)"}, rel_part, capture: :all_but_first)
+      [url] = Regex.run(~R{<(.*)>}, url_part, capture: :all_but_first)
+
+      {rel, url}
+    end)
+    |> Enum.into(%{})
+  end
+end

test/pagination_test.exs

@@ -0,0 +1,94 @@
+defmodule Shopify.PaginationTest do
+  use ExUnit.Case, async: true
+
+  alias Shopify.{Error, Pagination, Response, Session}
+
+  @next_link "<https://x.myshopify.com/admin/api/2019-10/products.json?limit=10&page_info=eyJsYXN0X2lkIjoxNTQwOTYwNTUwOTgyLCJsYXN0X3ZhbHVlIjoiRW5lcmd5IiwiZGlyZWN0aW9uIjoibmV4dCJ9>; rel=\"next\""
+  @next_prev_link "<https://x.myshopify.com/admin/api/2019-10/products.json?limit=10&page_info=eyJkaXJlY3Rpb24iOiJwcmV2IiwibGFzdF9pZCI6MTkzOTI2Nzc4MDY3OCwibGFzdF92YWx1ZSI6IkVuZXJneSBTYW1wbGVyIn0>; rel=\"previous\", <https://x.myshopify.com/admin/api/2019-10/products.json?limit=10&page_info=eyJkaXJlY3Rpb24iOiJuZXh0IiwibGFzdF9pZCI6MTc1MTIwMjM5ODI3OCwibGFzdF92YWx1ZSI6IkhhdCAtIFRoZXJlIGlzIG5vIG1hZ2ljIHBpbGwifQ>; rel=\"next\""
+
+  defp next_link(limit, page_info) do
+    {"Link", "<https://x.com/path?limit=#{limit}&page_info=#{page_info}>; rel=\"next\""}
+  end
+
+  defp mock_list(%Session{}, p) when p == %{} do
+    {:ok, %Response{data: ["a1", "a2"], headers: [next_link("10", "p_a2")]}}
+  end
+
+  defp mock_list(%Session{}, p) when p == %{"limit" => "10", "page_info" => "p_a2"} do
+    {:ok, %Response{data: ["a3", "a4"], headers: []}}
+  end
+
+  defp mock_list(%Session{}, p) when p == %{broken: "end"} do
+    {:ok, %Response{data: ["b1", "b2", "b3"], headers: [next_link("10", "p_b2")]}}
+  end
+
+  defp mock_list(%Session{}, p) when p == %{"limit" => "10", "page_info" => "p_b2"} do
+    {:error, %Error{reason: :timeout, source: :httpoison}}
+  end
+
+  defp mock_list(%Session{}, p) when p == %{key: "foo"} do
+    {:ok, %Response{data: ["f1", "f2", "f3"], headers: [next_link("20", "p_f2")]}}
+  end
+
+  defp mock_list(%Session{}, p) when p == %{"limit" => "20", "page_info" => "p_f2"} do
+    {:ok, %Response{data: ["f4"], headers: []}}
+  end
+
+  test "enumerable with Enum.into/2" do
+    assert ["a1", "a2", "a3", "a4"] ==
+             Shopify.session()
+             |> Pagination.enumerable(&mock_list/2)
+             |> Enum.into([])
+
+    assert ["f1", "f2", "f3", "f4"] ==
+             Shopify.session()
+             |> Pagination.enumerable(&mock_list/2, key: "foo")
+             |> Enum.into([])
+
+    exception =
+      assert_raise(Error, fn ->
+        Shopify.session() |> Pagination.enumerable(&mock_list/2, broken: "end") |> Enum.into([])
+      end)
+
+    assert exception == %Error{reason: :timeout, source: :httpoison}
+  end
+
+  test "enumerable with Enum.take/2 (coverage for :halt)" do
+    assert ["b1", "b2"] ==
+             Shopify.session()
+             |> Pagination.enumerable(&mock_list/2, broken: "end")
+             |> Enum.take(2)
+  end
+
+  test "enumerable with Enum.zip/2 (coverage for :suspend)" do
+    assert [{"a1", "f1"}, {"a2", "f2"}, {"a3", "f3"}, {"a4", "f4"}] ==
+             Enum.zip(
+               Shopify.session() |> Pagination.enumerable(&mock_list/2),
+               Shopify.session() |> Pagination.enumerable(&mock_list/2, key: "foo")
+             )
+  end
+
+  test "next_page_params" do
+    assert nil == Pagination.next_page_params(%Response{headers: []})
+
+    assert %{
+             "limit" => "10",
+             "page_info" =>
+               "eyJsYXN0X2lkIjoxNTQwOTYwNTUwOTgyLCJsYXN0X3ZhbHVlIjoiRW5lcmd5IiwiZGlyZWN0aW9uIjoibmV4dCJ9"
+           } ==
+             %Response{headers: [{"Link", @next_link}]}
+             |> Pagination.next_page_params()
+
+    assert %{
+             "limit" => "10",
+             "page_info" =>
+               "eyJkaXJlY3Rpb24iOiJuZXh0IiwibGFzdF9pZCI6MTc1MTIwMjM5ODI3OCwibGFzdF92YWx1ZSI6IkhhdCAtIFRoZXJlIGlzIG5vIG1hZ2ljIHBpbGwifQ"
+           } ==
+             %Response{headers: [{"Link", @next_prev_link}]}
+             |> Pagination.next_page_params()
+
+    assert_raise MatchError, fn ->
+      Pagination.next_page_params(%Response{headers: [{"Link", "nonsense"}]})
+    end
+  end
+end