add market history cache

CHANGELOG.md

@@ -1,4 +1,6 @@
 ## Current
+### Features
+- [#2182](https://github.com/poanetwork/blockscout/pull/2182) - add market history cache
 
 ### Features
 - [#2109](https://github.com/poanetwork/blockscout/pull/2109) - use bigger updates instead of `Multi` transactions in BlocksTransactionsMismatch

apps/block_scout_web/lib/block_scout_web/controllers/chain/market_history_chart_controller.ex

@@ -8,18 +8,20 @@ defmodule BlockScoutWeb.Chain.MarketHistoryChartController do
     with true <- ajax?(conn) do
       exchange_rate = Market.get_exchange_rate(Explorer.coin()) || Token.null()
 
+      recent_market_history = Market.fetch_recent_history()
+
       market_history_data =
-        30
-        |> Market.fetch_recent_history()
-        |> case do
-          [today | the_rest] -> [%{today | closing_price: exchange_rate.usd_value} | the_rest]
-          data -> data
+        case recent_market_history do
+          [today | the_rest] ->
+            encode_market_history_data([%{today | closing_price: exchange_rate.usd_value} | the_rest])
+
+          data ->
+            encode_market_history_data(data)
         end
-        |> encode_market_history_data()
 
       json(conn, %{
         history_data: market_history_data,
-        supply_data: available_supply(Chain.supply_for_days(30), exchange_rate)
+        supply_data: available_supply(Chain.supply_for_days(), exchange_rate)
       })
     else
       _ -> unprocessable_entity(conn)

apps/block_scout_web/lib/block_scout_web/notifier.ex

@@ -37,7 +37,7 @@ defmodule BlockScoutWeb.Notifier do
     exchange_rate = Market.get_exchange_rate(Explorer.coin()) || Token.null()
 
     market_history_data =
-      case Market.fetch_recent_history(30) do
+      case Market.fetch_recent_history() do
         [today | the_rest] -> [%{today | closing_price: exchange_rate.usd_value} | the_rest]
         data -> data
       end

apps/block_scout_web/test/block_scout_web/channels/exchange_rate_channel_test.exs

@@ -43,6 +43,8 @@ defmodule BlockScoutWeb.ExchangeRateChannelTest do
   describe "new_rate" do
     test "subscribed user is notified", %{token: token} do
       ExchangeRates.handle_info({nil, {:ok, [token]}}, %{})
+      Supervisor.terminate_child(Explorer.Supervisor, {ConCache, Explorer.Market.MarketHistoryCache.cache_name()})
+      Supervisor.restart_child(Explorer.Supervisor, {ConCache, Explorer.Market.MarketHistoryCache.cache_name()})
 
       topic = "exchange_rate:new_rate"
       @endpoint.subscribe(topic)
@@ -61,6 +63,8 @@ defmodule BlockScoutWeb.ExchangeRateChannelTest do
 
     test "subscribed user is notified with market history", %{token: token} do
       ExchangeRates.handle_info({nil, {:ok, [token]}}, %{})
+      Supervisor.terminate_child(Explorer.Supervisor, {ConCache, Explorer.Market.MarketHistoryCache.cache_name()})
+      Supervisor.restart_child(Explorer.Supervisor, {ConCache, Explorer.Market.MarketHistoryCache.cache_name()})
 
       today = Date.utc_today()
 
@@ -76,6 +80,8 @@ defmodule BlockScoutWeb.ExchangeRateChannelTest do
 
       Market.bulk_insert_history(records)
 
+      Market.fetch_recent_history()
+
       topic = "exchange_rate:new_rate"
       @endpoint.subscribe(topic)
 

apps/explorer/lib/explorer/application.ex

@@ -7,6 +7,7 @@ defmodule Explorer.Application do
 
   alias Explorer.Admin
   alias Explorer.Chain.{BlockCountCache, BlockNumberCache, BlocksCache, NetVersionCache, TransactionCountCache}
+  alias Explorer.Market.MarketHistoryCache
   alias Explorer.Repo.PrometheusLogger
 
   @impl Application
@@ -32,7 +33,8 @@ defmodule Explorer.Application do
       {TransactionCountCache, [[], []]},
       {BlockCountCache, []},
       con_cache_child_spec(BlocksCache.cache_name()),
-      con_cache_child_spec(NetVersionCache.cache_name())
+      con_cache_child_spec(NetVersionCache.cache_name()),
+      con_cache_child_spec(MarketHistoryCache.cache_name())
     ]
 
     children = base_children ++ configurable_children()

apps/explorer/lib/explorer/chain.ex

@@ -52,6 +52,7 @@ defmodule Explorer.Chain do
   alias Explorer.Chain.Block.{EmissionReward, Reward}
   alias Explorer.Chain.Import.Runner
   alias Explorer.Counters.AddressesWithBalanceCounter
+  alias Explorer.Market.MarketHistoryCache
   alias Explorer.{PagingOptions, Repo}
 
   alias Dataloader.Ecto, as: DataloaderEcto
@@ -2587,7 +2588,7 @@ defmodule Explorer.Chain do
   @doc """
   Calls supply_for_days from the configured supply_module
   """
-  def supply_for_days(days_count), do: supply_module().supply_for_days(days_count)
+  def supply_for_days, do: supply_module().supply_for_days(MarketHistoryCache.recent_days_count())
 
   @doc """
   Streams a lists token contract addresses that haven't been cataloged.

apps/explorer/lib/explorer/market/market.ex

@@ -3,12 +3,10 @@ defmodule Explorer.Market do
   Context for data related to the cryptocurrency market.
   """
 
-  import Ecto.Query
-
   alias Explorer.Chain.Address.CurrentTokenBalance
   alias Explorer.Chain.Hash
   alias Explorer.ExchangeRates.Token
-  alias Explorer.Market.MarketHistory
+  alias Explorer.Market.{MarketHistory, MarketHistoryCache}
   alias Explorer.{ExchangeRates, KnownTokens, Repo}
 
   @doc """
@@ -35,18 +33,9 @@ defmodule Explorer.Market do
 
   Today's date is include as part of the day count
   """
-  @spec fetch_recent_history(non_neg_integer()) :: [MarketHistory.t()]
-  def fetch_recent_history(days) when days >= 1 do
-    day_diff = days * -1
-
-    query =
-      from(
-        mh in MarketHistory,
-        where: mh.date > date_add(^Date.utc_today(), ^day_diff, "day"),
-        order_by: [desc: mh.date]
-      )
-
-    Repo.all(query)
+  @spec fetch_recent_history() :: [MarketHistory.t()]
+  def fetch_recent_history do
+    MarketHistoryCache.fetch()
   end
 
   @doc false

apps/explorer/lib/explorer/market/market_history_cache.ex

@@ -0,0 +1,79 @@
+defmodule Explorer.Market.MarketHistoryCache do
+  @moduledoc """
+  Caches recent market history.
+  """
+
+  import Ecto.Query, only: [from: 2]
+
+  alias Explorer.Market.MarketHistory
+  alias Explorer.Repo
+
+  @cache_name :market_history
+  @last_update_key :last_update
+  @history_key :history
+  # 6 hours
+  @cache_period 1_000 * 60 * 60 * 6
+  @recent_days 30
+
+  def fetch do
+    if cache_expired?() do
+      update_cache()
+    else
+      fetch_from_cache(@history_key)
+    end
+  end
+
+  def cache_name, do: @cache_name
+
+  def data_key, do: @history_key
+
+  def updated_at_key, do: @last_update_key
+
+  def recent_days_count, do: @recent_days
+
+  defp cache_expired? do
+    updated_at = fetch_from_cache(@last_update_key)
+
+    cond do
+      is_nil(updated_at) -> true
+      current_time() - updated_at > @cache_period -> true
+      true -> false
+    end
+  end
+
+  defp update_cache do
+    new_data = fetch_from_db()
+
+    put_into_cache(@last_update_key, current_time())
+    put_into_cache(@history_key, new_data)
+
+    new_data
+  end
+
+  defp fetch_from_db do
+    day_diff = @recent_days * -1
+
+    query =
+      from(
+        mh in MarketHistory,
+        where: mh.date > date_add(^Date.utc_today(), ^day_diff, "day"),
+        order_by: [desc: mh.date]
+      )
+
+    Repo.all(query)
+  end
+
+  defp fetch_from_cache(key) do
+    ConCache.get(@cache_name, key)
+  end
+
+  defp put_into_cache(key, value) do
+    ConCache.put(@cache_name, key, value)
+  end
+
+  defp current_time do
+    utc_now = DateTime.utc_now()
+
+    DateTime.to_unix(utc_now, :millisecond)
+  end
+end

apps/explorer/test/explorer/market/market_history_cache_test.exs

@@ -0,0 +1,90 @@
+defmodule Explorer.Market.MarketHistoryCacheTest do
+  use Explorer.DataCase
+
+  alias Explorer.Market
+  alias Explorer.Market.MarketHistoryCache
+
+  setup do
+    Supervisor.terminate_child(Explorer.Supervisor, {ConCache, MarketHistoryCache.cache_name()})
+    Supervisor.restart_child(Explorer.Supervisor, {ConCache, MarketHistoryCache.cache_name()})
+
+    on_exit(fn ->
+      Supervisor.terminate_child(Explorer.Supervisor, {ConCache, Explorer.Chain.BlocksCache.cache_name()})
+      Supervisor.restart_child(Explorer.Supervisor, {ConCache, Explorer.Chain.BlocksCache.cache_name()})
+    end)
+
+    :ok
+  end
+
+  describe "fetch/1" do
+    test "caches data on the first call" do
+      today = Date.utc_today()
+
+      records =
+        for i <- 0..29 do
+          %{
+            date: Timex.shift(today, days: i * -1),
+            closing_price: Decimal.new(1),
+            opening_price: Decimal.new(1)
+          }
+        end
+
+      Market.bulk_insert_history(records)
+
+      refute fetch_data()
+
+      assert Enum.count(MarketHistoryCache.fetch()) == 30
+
+      assert fetch_data() == records
+    end
+
+    test "updates cache if cache is stale" do
+      today = Date.utc_today()
+
+      stale_records =
+        for i <- 0..29 do
+          %{
+            date: Timex.shift(today, days: i * -1),
+            closing_price: Decimal.new(1),
+            opening_price: Decimal.new(1)
+          }
+        end
+
+      Market.bulk_insert_history(stale_records)
+
+      MarketHistoryCache.fetch()
+
+      stale_updated_at = fetch_updated_at()
+
+      assert fetch_data() == stale_records
+
+      ConCache.put(MarketHistoryCache.cache_name(), MarketHistoryCache.updated_at_key(), 1)
+
+      fetch_data()
+
+      assert stale_updated_at != fetch_updated_at()
+    end
+  end
+
+  defp fetch_updated_at do
+    ConCache.get(MarketHistoryCache.cache_name(), MarketHistoryCache.updated_at_key())
+  end
+
+  defp fetch_data do
+    MarketHistoryCache.cache_name()
+    |> ConCache.get(MarketHistoryCache.data_key())
+    |> case do
+      nil ->
+        nil
+
+      records ->
+        Enum.map(records, fn record ->
+          %{
+            date: record.date,
+            closing_price: record.closing_price,
+            opening_price: record.opening_price
+          }
+        end)
+    end
+  end
+end

apps/explorer/test/explorer/market/market_test.exs

@@ -1,15 +1,27 @@
 defmodule Explorer.MarketTest do
-  use Explorer.DataCase
+  use Explorer.DataCase, async: false
 
   alias Explorer.Market
   alias Explorer.Market.MarketHistory
   alias Explorer.Repo
 
+  setup do
+    Supervisor.terminate_child(Explorer.Supervisor, {ConCache, Explorer.Chain.BlocksCache.cache_name()})
+    Supervisor.restart_child(Explorer.Supervisor, {ConCache, Explorer.Chain.BlocksCache.cache_name()})
+
+    on_exit(fn ->
+      Supervisor.terminate_child(Explorer.Supervisor, {ConCache, Explorer.Chain.BlocksCache.cache_name()})
+      Supervisor.restart_child(Explorer.Supervisor, {ConCache, Explorer.Chain.BlocksCache.cache_name()})
+    end)
+
+    :ok
+  end
+
   test "fetch_recent_history/1" do
     today = Date.utc_today()
 
     records =
-      for i <- 0..5 do
+      for i <- 0..29 do
         %{
           date: Timex.shift(today, days: i * -1),
           closing_price: Decimal.new(1),
@@ -19,16 +31,9 @@ defmodule Explorer.MarketTest do
 
     Market.bulk_insert_history(records)
 
-    history = Market.fetch_recent_history(1)
-    assert length(history) == 1
+    history = Market.fetch_recent_history()
+    assert length(history) == 30
     assert Enum.at(history, 0).date == Enum.at(records, 0).date
-
-    more_history = Market.fetch_recent_history(5)
-    assert length(more_history) == 5
-
-    for {history_record, index} <- Enum.with_index(more_history) do
-      assert history_record.date == Enum.at(records, index).date
-    end
   end
 
   describe "bulk_insert_history/1" do

apps/explorer/test/support/data_case.ex

@@ -40,8 +40,8 @@ defmodule Explorer.DataCase do
     end
 
     Explorer.Chain.BlockNumberCache.setup()
-    Supervisor.terminate_child(Explorer.Supervisor, {ConCache, :blocks})
-    Supervisor.restart_child(Explorer.Supervisor, {ConCache, :blocks})
+    Supervisor.terminate_child(Explorer.Supervisor, {ConCache, Explorer.Chain.BlocksCache.cache_name()})
+    Supervisor.restart_child(Explorer.Supervisor, {ConCache, Explorer.Chain.BlocksCache.cache_name()})
 
     :ok
   end