StatsHandler : ajout de statistiques quotidiennes pour flux GBFS (#4315)

apps/transport/lib/transport/stats_handler.ex

@@ -90,6 +90,74 @@ defmodule Transport.StatsHandler do
       nb_siri_lite: count_dataset_with_format("SIRI Lite"),
       count_geo_data_lines: count_geo_data_lines()
     }
+    |> Map.merge(gbfs_stats())
+  end
+
+  def gbfs_stats do
+    today = Date.utc_today()
+
+    # Latest metadata for each GBFS resource for today
+    rows =
+      DB.ResourceMetadata.base_query()
+      |> join(:inner, [metadata: m], r in DB.Resource, on: r.id == m.resource_id, as: :resource)
+      |> where([resource: r], r.format == "gbfs")
+      |> where([metadata: m], fragment("?::date", m.inserted_at) == ^today and fragment("? \\? 'stats'", m.metadata))
+      |> select([metadata: m], last_value(m.metadata) |> over(partition_by: m.resource_id, order_by: m.resource_id))
+      |> distinct(true)
+      |> DB.Repo.all()
+
+    if Enum.empty?(rows) do
+      %{}
+    else
+      [&gbfs_versions_stats/1, &gbfs_vehicle_types_stats/1, &gbfs_feed_types_stats/1, &gbfs_counters_stats/1]
+      |> Enum.reduce(%{}, fn method, acc -> Map.merge(acc, method.(rows)) end)
+    end
+  end
+
+  @doc """
+  iex> gbfs_counters_stats([%{"stats" => %{"nb_stations" => 40, "nb_vehicles" => 5, "version" => 1}}, %{"stats" => %{"nb_stations" => 2, "nb_vehicles" => 3, "version" => 1}}])
+  %{gbfs_nb_stations_sum: 42, gbfs_nb_vehicles_sum: 8}
+  """
+  def gbfs_counters_stats(rows) do
+    rows
+    # Keep only keys starting with `nb_`
+    |> Enum.map(&Map.filter(&1["stats"], fn {k, _} -> String.starts_with?(k, "nb_") end))
+    # Keep a single map, summing all values for each key
+    |> Enum.reduce(&Map.merge(&1, &2, fn _, v1, v2 -> v1 + v2 end))
+    |> Map.new(fn {k, v} -> {String.to_atom("gbfs_#{k}_sum"), v} end)
+  end
+
+  @doc """
+  iex> gbfs_versions_stats([%{"versions" => ["3.0", "2.2"]}, %{"versions" => ["3.0", "1.0"]}])
+  %{"gbfs_v1.0_count": 1, "gbfs_v2.2_count": 1, "gbfs_v3.0_count": 2}
+  """
+  def gbfs_versions_stats(rows) do
+    rows
+    |> Enum.flat_map(& &1["versions"])
+    |> Enum.frequencies()
+    |> Map.new(fn {k, v} -> {String.to_atom("gbfs_v#{k}_count"), v} end)
+  end
+
+  @doc """
+  iex> gbfs_vehicle_types_stats([%{"vehicle_types" => ["bicycle", "scooter"]}, %{"vehicle_types" => ["bicycle"]}])
+  %{gbfs_vehicle_type_bicycle_count: 2, gbfs_vehicle_type_scooter_count: 1}
+  """
+  def gbfs_vehicle_types_stats(rows) do
+    rows
+    |> Enum.flat_map(& &1["vehicle_types"])
+    |> Enum.frequencies()
+    |> Map.new(fn {k, v} -> {String.to_atom("gbfs_vehicle_type_#{k}_count"), v} end)
+  end
+
+  @doc """
+  iex> gbfs_feed_types_stats([%{"types" =>["free_floating", "stations"]}, %{"types" =>["stations"]}])
+  %{gbfs_feed_type_free_floating_count: 1, gbfs_feed_type_stations_count: 2}
+  """
+  def gbfs_feed_types_stats(rows) do
+    rows
+    |> Enum.flat_map(& &1["types"])
+    |> Enum.frequencies()
+    |> Map.new(fn {k, v} -> {String.to_atom("gbfs_feed_type_#{k}_count"), v} end)
   end
 
   @doc """

apps/transport/test/transport/stats_handler_test.exs

@@ -4,6 +4,8 @@ defmodule Transport.StatsHandlerTest do
   import Ecto.Query
   import Transport.StatsHandler
 
+  doctest Transport.StatsHandler, import: true
+
   setup do
     :ok = Ecto.Adapters.SQL.Sandbox.checkout(DB.Repo)
     insert_bnlc_dataset()
@@ -81,12 +83,21 @@ defmodule Transport.StatsHandlerTest do
   end
 
   test "store_stats" do
-    resource = insert(:resource, format: "gtfs-rt")
-    insert(:resource_metadata, features: ["vehicle_positions"], resource_id: resource.id)
-    insert(:resource_metadata, features: ["trip_updates"], resource_id: resource.id)
-
+    gtfs_rt_resource = insert(:resource, format: "gtfs-rt")
+    insert(:resource_metadata, features: ["vehicle_positions"], resource_id: gtfs_rt_resource.id)
+    insert(:resource_metadata, features: ["trip_updates"], resource_id: gtfs_rt_resource.id)
     insert(:resource_metadata, features: ["vehicle_positions"], resource: insert(:resource, format: "gtfs-rt"))
 
+    insert(:resource_metadata,
+      metadata: %{
+        versions: ["3.0"],
+        types: ["free_floating"],
+        vehicle_types: ["scooter"],
+        stats: %{nb_vehicles: 10, version: 1}
+      },
+      resource_id: insert(:resource, format: "gbfs").id
+    )
+
     stats = compute_stats()
     store_stats()
     assert DB.Repo.aggregate(DB.StatsHistory, :count, :id) >= Enum.count(stats)
@@ -108,6 +119,8 @@ defmodule Transport.StatsHandlerTest do
 
     expected = Decimal.new("2")
     assert %{value: ^expected} = DB.Repo.get_by!(DB.StatsHistory, metric: "gtfs_rt_types::vehicle_positions")
+    expected = Decimal.new("1")
+    assert %{value: ^expected} = DB.Repo.get_by!(DB.StatsHistory, metric: "gbfs_v3.0_count")
   end
 
   test "count dataset per format" do
@@ -177,4 +190,65 @@ defmodule Transport.StatsHandlerTest do
     insert_imported_irve_geo_data(Transport.ConsolidatedDataset.dataset(:irve).id)
     assert 2 == count_geo_data_lines(:irve)
   end
+
+  describe "gbfs_stats" do
+    test "no rows for today" do
+      insert(:resource_metadata,
+        metadata: %{"stats" => 42},
+        resource_id: insert(:resource, format: "gbfs").id,
+        inserted_at: DateTime.add(DateTime.utc_now(), -1, :day)
+      )
+
+      assert %{} == gbfs_stats()
+    end
+
+    test "it works" do
+      gbfs_resource = insert(:resource, format: "gbfs")
+      other_gbfs_resource = insert(:resource, format: "gbfs")
+      gtfs_resource = insert(:resource, format: "GTFS")
+
+      now = DateTime.utc_now()
+
+      # Should be ignored: GTFS
+      insert(:resource_metadata, metadata: %{foo: 42}, resource_id: gtfs_resource.id)
+      # Should be ignored: not the most recent one for `gbfs_resource`
+      insert(:resource_metadata,
+        metadata: %{stats: %{foo: 42}},
+        resource_id: gbfs_resource.id,
+        inserted_at: DateTime.add(now, -5, :second)
+      )
+
+      # Relevant metadata
+      insert(:resource_metadata,
+        metadata: %{
+          versions: ["3.0", "2.3"],
+          types: ["stations"],
+          vehicle_types: ["bicycle", "scooter"],
+          stats: %{nb_vehicles: 20, nb_stations: 3, version: 1}
+        },
+        resource_id: gbfs_resource.id
+      )
+
+      insert(:resource_metadata,
+        metadata: %{
+          versions: ["3.0"],
+          types: ["free_floating"],
+          vehicle_types: ["scooter"],
+          stats: %{nb_vehicles: 10, version: 1}
+        },
+        resource_id: other_gbfs_resource.id
+      )
+
+      assert %{
+               gbfs_feed_type_free_floating_count: 1,
+               gbfs_feed_type_stations_count: 1,
+               gbfs_nb_stations_sum: 3,
+               gbfs_nb_vehicles_sum: 30,
+               "gbfs_v2.3_count": 1,
+               "gbfs_v3.0_count": 2,
+               gbfs_vehicle_type_bicycle_count: 1,
+               gbfs_vehicle_type_scooter_count: 2
+             } == gbfs_stats()
+    end
+  end
 end