Separate liveness and readiness checks

CHANGELOG.md

@@ -20,6 +20,7 @@ All notable changes to this project will be documented in this file.
 - Details modal search inputs are now case-insensitive.
 - Improved report performance in cases where site has a lot of unique pathnames
 - Plausible script now uses `fetch` with keepalive flag as default over `XMLHttpRequest`. This will ensure more reliable tracking. Reminder to use `compat` script variant if tracking Internet Explorer is required.
+- The old `/api/health` healtcheck is soft-deprecated in favour of separate `/api/system/live` and `/api/system/ready` checks
 
 ### Fixed
 

lib/plausible_web/controllers/api/external_controller.ex

@@ -47,54 +47,6 @@ defmodule PlausibleWeb.Api.ExternalController do
     send_resp(conn, 200, "")
   end
 
-  def health(conn, _params) do
-    postgres_health =
-      case Ecto.Adapters.SQL.query(Plausible.Repo, "SELECT 1", []) do
-        {:ok, _} -> "ok"
-        e -> "error: #{inspect(e)}"
-      end
-
-    clickhouse_health =
-      case Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, "SELECT 1", []) do
-        {:ok, _} -> "ok"
-        e -> "error: #{inspect(e)}"
-      end
-
-    cache_health =
-      if postgres_health == "ok" and Plausible.Site.Cache.ready?() and
-           Plausible.Shield.IPRuleCache.ready?() do
-        "ok"
-      end
-
-    status =
-      case {postgres_health, clickhouse_health, cache_health} do
-        {"ok", "ok", "ok"} -> 200
-        _ -> 500
-      end
-
-    put_status(conn, status)
-    |> json(%{
-      postgres: postgres_health,
-      clickhouse: clickhouse_health,
-      sites_cache: cache_health
-    })
-  end
-
-  def info(conn, _params) do
-    build =
-      :plausible
-      |> Application.get_env(:runtime_metadata)
-      |> Keyword.take([:version, :commit, :created, :tags])
-      |> Map.new()
-
-    geo_database = Plausible.Geo.database_type() || "(not configured)"
-
-    json(conn, %{
-      geo_database: geo_database,
-      build: build
-    })
-  end
-
   defp find_first_invalid_changeset(dropped) do
     Enum.find_value(dropped, nil, fn dropped_event ->
       case dropped_event.drop_reason do

lib/plausible_web/controllers/api/system_controller.ex

@@ -0,0 +1,75 @@
+defmodule PlausibleWeb.Api.SystemController do
+  use PlausibleWeb, :controller
+  require Logger
+
+  def info(conn, _params) do
+    build =
+      :plausible
+      |> Application.get_env(:runtime_metadata)
+      |> Keyword.take([:version, :commit, :created, :tags])
+      |> Map.new()
+
+    geo_database = Plausible.Geo.database_type() || "(not configured)"
+
+    json(conn, %{
+      geo_database: geo_database,
+      build: build
+    })
+  end
+
+  def liveness(conn, _params) do
+    json(conn, %{ok: true})
+  end
+
+  @task_timeout 15_000
+  def readiness(conn, _params) do
+    postgres_health_task =
+      Task.async(fn ->
+        Ecto.Adapters.SQL.query(Plausible.Repo, "SELECT 1", [])
+      end)
+
+    clickhouse_health_task =
+      Task.async(fn ->
+        Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, "SELECT 1", [])
+      end)
+
+    postgres_health =
+      case Task.await(postgres_health_task, @task_timeout) do
+        {:ok, _} ->
+          "ok"
+
+        e ->
+          Logger.error("Postgres health check failure: #{inspect(e)}")
+          "error"
+      end
+
+    clickhouse_health =
+      case Task.await(clickhouse_health_task, @task_timeout) do
+        {:ok, _} ->
+          "ok"
+
+        e ->
+          Logger.error("Clickhouse health check failure: #{inspect(e)}")
+          "error"
+      end
+
+    cache_health =
+      if postgres_health == "ok" and Plausible.Site.Cache.ready?() and
+           Plausible.Shield.IPRuleCache.ready?() do
+        "ok"
+      end
+
+    status =
+      case {postgres_health, clickhouse_health, cache_health} do
+        {"ok", "ok", "ok"} -> 200
+        _ -> 500
+      end
+
+    put_status(conn, status)
+    |> json(%{
+      postgres: postgres_health,
+      clickhouse: clickhouse_health,
+      sites_cache: cache_health
+    })
+  end
+end

lib/plausible_web/router.ex

@@ -294,8 +294,14 @@ defmodule PlausibleWeb.Router do
 
       post "/event", Api.ExternalController, :event
       get "/error", Api.ExternalController, :error
-      get "/health", Api.ExternalController, :health
-      get "/system", Api.ExternalController, :info
+      # Remove this once all external checks are migration to new /system/health/* checks
+      get "/health", Api.SystemController, :readiness
+    end
+
+    scope "/system" do
+      get "/", Api.SystemController, :info
+      get "/health/live", Api.SystemController, :liveness
+      get "/health/ready", Api.SystemController, :readiness
     end
 
     scope [] do