WIP

apps/transport/lib/transport_web/plugs/router.ex

@@ -2,7 +2,7 @@ defmodule TransportWeb.Plugs.Router do
   use Plug.Router
 
   plug(TransportWeb.Plugs.HealthCheck, at: "/health-check")
-  plug(TransportWeb.Plugs.Halt, if: {Transport.Application, :worker_only?}, message: "UP (WORKER-ONLY)")
+  plug(TransportWeb.Plugs.WorkerHealthcheck, if: {Transport.Application, :worker_only?})
 
   plug(:match)
   plug(:dispatch)

apps/transport/lib/transport_web/plugs/worker_healthcheck.ex

@@ -0,0 +1,64 @@
+defmodule TransportWeb.Plugs.WorkerHealthcheck do
+  @moduledoc """
+  """
+  import Plug.Conn
+
+  @app_start_waiting_delay {20, :minute}
+  @oban_max_delay_since_last_attempt {60, :minute}
+
+  def init(options), do: options
+
+  def call(conn, opts) do
+    {mod, fun} = opts[:if]
+
+    if apply(mod, fun, []) do
+      status_code = if healthy_state?(), do: 200, else: 503
+
+      conn
+      |> put_resp_content_type("text/plain")
+      |> send_resp(status_code, """
+      UP (WORKER-ONLY)
+      App start time: #{app_start_datetime()}
+      App started recently?: #{app_started_recently?()}
+      Oban last attempt: #{oban_last_attempted_at()}
+      Healthy state?: #{healthy_state?()}
+      """)
+      |> halt()
+    else
+      conn
+    end
+  end
+
+  def healthy_state? do
+    oban_last_attempt = oban_last_attempted_at()
+    {delay, unit} = @oban_max_delay_since_last_attempt
+    oban_attempted_jobs_recently = DateTime.before?(oban_last_attempt, DateTime.add(oban_last_attempt, delay, unit))
+    app_started_recently?() or oban_attempted_jobs_recently
+  end
+
+  def app_started_recently? do
+    start_datetime = app_start_datetime()
+    {delay, unit} = @app_start_waiting_delay
+    DateTime.before?(start_datetime, DateTime.add(start_datetime, delay, unit))
+  end
+
+  def app_start_datetime do
+    Transport.Cache.fetch(app_start_datetime_cache_key_name(), fn -> DateTime.utc_now() end, expire: nil)
+  end
+
+  def app_start_datetime_cache_key_name, do: "#{__MODULE__}::app_start_datetime"
+
+  def oban_last_attempted_at do
+    %Postgrex.Result{rows: [[delay]]} =
+      DB.Repo.query!("""
+      SELECT MAX(attempted_at)
+      FROM oban_jobs
+      WHERE state = 'completed'
+      """)
+
+    case delay do
+      nil -> DateTime.new!(~D[1970-01-01], ~T[00:00:00.000], "Etc/UTC")
+      %NaiveDateTime{} = nt -> DateTime.from_naive!(nt, "Etc/UTC")
+    end
+  end
+end