Validation NeTEx automatique (#4204)

* NeTEx: fixe l’affichage d’un timeout

* Validation NeTEx automatique

Le validateur NeTEx accepte des ResourceHistory pour se conformer au job de
validation.

Ceci active de fait la validation automatique de resources NETEx.

* Pluriel des résumés de validation

Fixes #3025.

* Fix merge

---------

Co-authored-by: Thibaut Barrère <thibaut.barrere@gmail.com>

apps/transport/lib/jobs/on_demand_validation_job.ex

@@ -76,7 +76,7 @@ defmodule Transport.Jobs.OnDemandValidationJob do
     validator = NeTEx.validator_name()
 
     case NeTEx.validate(url, []) do
-      {:error, msg} ->
+      {:error, %{message: msg}} ->
         %{oban_args: %{"state" => "error", "error_reason" => msg}, validator: validator}
 
       {:ok, %{"validations" => validation, "metadata" => metadata}} ->

apps/transport/lib/transport_web/controllers/resource_controller.ex

@@ -12,7 +12,8 @@ defmodule TransportWeb.ResourceController do
                         Transport.Validators.GTFSRT,
                         Transport.Validators.GBFSValidator,
                         Transport.Validators.TableSchema,
-                        Transport.Validators.EXJSONSchema
+                        Transport.Validators.EXJSONSchema,
+                        Transport.Validators.NeTEx
                       ])
 
   def details(conn, %{"id" => id} = params) do
@@ -31,10 +32,10 @@ defmodule TransportWeb.ResourceController do
       |> assign(:multi_validation, latest_validation(resource))
       |> put_resource_flash(resource.dataset.is_active)
 
-    if Resource.gtfs?(resource) do
-      render_gtfs_details(conn, params, resource)
-    else
-      conn |> assign(:resource, resource) |> render("details.html")
+    cond do
+      Resource.gtfs?(resource) -> render_gtfs_details(conn, params, resource)
+      Resource.netex?(resource) -> render_netex_details(conn, params, resource)
+      true -> render_details(conn, resource)
     end
   end
 
@@ -126,40 +127,70 @@ defmodule TransportWeb.ResourceController do
     DB.MultiValidation.resource_latest_validation(resource_id, validator)
   end
 
-  defp render_gtfs_details(conn, params, resource) do
-    config = make_pagination_config(params)
+  def render_details(conn, resource) do
+    conn |> assign(:resource, resource) |> render("details.html")
+  end
 
-    validation = resource |> latest_validation()
+  defp render_gtfs_details(conn, params, resource) do
+    validator = Transport.Validators.GTFSTransport
 
-    {validation_summary, severities_count, metadata, modes, issues} =
-      case validation do
-        %{result: validation_result, metadata: %DB.ResourceMetadata{metadata: metadata, modes: modes}} ->
-          {Transport.Validators.GTFSTransport.summary(validation_result),
-           Transport.Validators.GTFSTransport.count_by_severity(validation_result), metadata, modes,
-           Transport.Validators.GTFSTransport.get_issues(validation_result, params)}
+    validation = latest_validation(resource)
 
-        nil ->
-          {nil, nil, nil, [], []}
-      end
+    validation_details = {_, _, _, _, issues} = build_validation_details(params, resource, validator)
 
     issue_type =
       case params["issue_type"] do
-        nil -> Transport.Validators.GTFSTransport.issue_type(issues)
+        nil -> validator.issue_type(issues)
         issue_type -> issue_type
       end
 
+    conn
+    |> assign_base_resource_details(params, resource, validation_details, validator)
+    |> assign(:data_vis, encoded_data_vis(issue_type, validation))
+    |> render("gtfs_details.html")
+  end
+
+  defp render_netex_details(conn, params, resource) do
+    validator = Transport.Validators.NeTEx
+
+    validation_details = build_validation_details(params, resource, validator)
+
+    conn
+    |> assign_base_resource_details(params, resource, validation_details, validator)
+    |> assign(:data_vis, nil)
+    |> render("netex_details.html")
+  end
+
+  defp build_validation_details(params, resource, validator) do
+    case latest_validation(resource) do
+      %{result: validation_result, metadata: metadata = %DB.ResourceMetadata{}} ->
+        summary = validator.summary(validation_result)
+        stats = validator.count_by_severity(validation_result)
+        issues = validator.get_issues(validation_result, params)
+
+        {summary, stats, metadata.metadata, metadata.modes, issues}
+
+      nil ->
+        {nil, nil, nil, [], []}
+    end
+  end
+
+  defp assign_base_resource_details(conn, params, resource, validation_details, validator) do
+    config = make_pagination_config(params)
+
+    {validation_summary, severities_count, metadata, modes, issues} = validation_details
+
     conn
     |> assign(:related_files, Resource.get_related_files(resource))
     |> assign(:resource, resource)
     |> assign(:other_resources, Resource.other_resources(resource))
     |> assign(:issues, Scrivener.paginate(issues, config))
-    |> assign(:data_vis, encoded_data_vis(issue_type, validation))
     |> assign(:validation_summary, validation_summary)
     |> assign(:severities_count, severities_count)
-    |> assign(:validation, validation)
+    |> assign(:validation, latest_validation(resource))
     |> assign(:metadata, metadata)
     |> assign(:modes, modes)
-    |> render("gtfs_details.html")
+    |> assign(:validator, validator)
   end
 
   def encoded_data_vis(_, nil), do: nil

apps/transport/lib/transport_web/templates/resource/gtfs_details.html.heex

@@ -10,7 +10,7 @@ locale = get_session(@conn, :locale) %>
       </h2>
       <div class="panel">
         <%= render("_resource_description.html", conn: @conn, resource: @resource, resource_history: @resource_history) %>
-        <%= render("_resources_details.html", conn: @conn, metadata: @metadata, modes: @modes) %>
+        <%= render("_resources_details_gtfs.html", conn: @conn, metadata: @metadata, modes: @modes) %>
         <%= if !is_nil(associated_geojson) or !is_nil(associated_netex) do %>
           <div id="other-formats">
             <%= dgettext("validations", "This resource is also available in the following formats:") %>
@@ -98,7 +98,7 @@ locale = get_session(@conn, :locale) %>
                 conn: @conn,
                 data_vis: @data_vis,
                 token: nil,
-                validator: Transport.Validators.GTFSTransport
+                validator: @validator
               ) %>
             </nav>
             <div class="main-pane">
@@ -114,14 +114,16 @@ locale = get_session(@conn, :locale) %>
             <%= raw(
               dgettext(
                 "validations",
-                ~s(Validation carried out using the <a href="%{gtfs_link}">current GTFS file</a> the %{date} using the <a href="%{validator_url}" target="_blank">PAN GTFS validator</a>.),
-                gtfs_link: Map.fetch!(@validation.resource_history.payload, "permanent_url"),
+                ~s(Validation carried out using the <a href="%{link}">current %{format} file</a> the %{date} using the <a href="%{validator_url}" target="_blank">%{validator_name}</a>.),
+                link: Map.fetch!(@validation.resource_history.payload, "permanent_url"),
+                format: "GTFS",
                 date:
                   DateTimeDisplay.format_datetime_to_paris(
                     @validation.validation_timestamp,
                     locale
                   ),
-                validator_url: gtfs_validator_url()
+                validator_url: gtfs_validator_url(),
+                validator_name: dgettext("validations", "PAN GTFS validator")
               )
             ) %>
           </p>

apps/transport/lib/transport_web/templates/resource/netex_details.html.heex

@@ -0,0 +1,90 @@
+<section>
+  <div class="grey-background">
+    <div class="container">
+      <h2 class="mt-48">
+        <%= dgettext("validations", "Resource details") %>
+      </h2>
+      <div class="panel">
+        <%= render("_resource_description.html", conn: @conn, resource: @resource, resource_history: @resource_history) %>
+      </div>
+
+      <%= unless Enum.empty?(@resource.resources_related) do %>
+        <%= render("_related_resources.html", resource: @resource, conn: @conn) %>
+      <% end %>
+
+      <h2 id="download-availability"><%= dgettext("page-dataset-details", "Download availability") %></h2>
+      <%= render("_download_availability.html", uptime_per_day: @uptime_per_day, conn: @conn) %>
+
+      <h2 id="validation-report" class="mt-48"><%= dgettext("validations", "Validation report") %></h2>
+      <div class="panel" id="issues">
+        <%= if is_nil(@validation_summary) do %>
+          <%= dgettext("validations", "No validation available") %>
+        <% end %>
+        <%= unless is_nil(@validation_summary) do %>
+          <p class="notification warning">
+            <strong><%= dgettext("validations", "NeTEx validation is in beta.") %></strong> <br />
+            <%= dgettext("validations", "Warnings are work in progress. Only XSD errors are considered for now.") %>
+          </p>
+          <%= unless is_nil(@metadata) or @metadata == %{} do %>
+            <%= render("_resources_details_netex.html", conn: @conn, metadata: @metadata) %>
+          <% end %>
+          <%= if @issues.total_entries == 0 do %>
+            <%= dgettext("validations", "No validation error") %>.
+          <% else %>
+            <nav class="issues-list validation" role="navigation">
+              <%= render("_validation_summary.html",
+                validation_summary: @validation_summary,
+                severities_count: @severities_count,
+                issues: @issues,
+                conn: @conn,
+                data_vis: nil,
+                token: nil,
+                validator: @validator
+              ) %>
+            </nav>
+            <div class="main-pane">
+              <%= pagination_links(@conn, @issues, [@resource.id],
+                issue_type: Transport.Validators.NeTEx.issue_type(@issues.entries),
+                path: &resource_path/4,
+                action: :details
+              ) %>
+              <%= render(netex_template(@issues), issues: @issues || [], conn: @conn) %>
+            </div>
+          <% end %>
+          <p>
+            <%= raw(
+              dgettext(
+                "validations",
+                ~s(Validation carried out using the <a href="%{link}">current %{format} file</a> the %{date} using the <a href="%{validator_url}" target="_blank">%{validator_name}</a>.),
+                link: Map.fetch!(@validation.resource_history.payload, "permanent_url"),
+                format: "NeTEx",
+                date:
+                  DateTimeDisplay.format_datetime_to_paris(
+                    @validation.validation_timestamp,
+                    get_session(@conn, :locale)
+                  ),
+                validator_url: netex_validator_url(),
+                validator_name: dgettext("validations", "enRoute Chouette Valid")
+              )
+            ) %>
+          </p>
+        <% end %>
+      </div>
+      <%= if length(@other_resources) > 0 do %>
+        <h2><%= TransportWeb.Gettext.dgettext("validations", "Other resources") %></h2>
+        <div class="panel">
+          <ul>
+            <%= for resource <- @other_resources do %>
+              <li>
+                <%= link(resource.title,
+                  to: resource_path(@conn, :details, resource.id)
+                ) %>
+              </li>
+            <% end %>
+          </ul>
+        </div>
+      <% end %>
+    </div>
+  </div>
+</section>
+<script src={static_path(@conn, "/js/utils.js")} />

apps/transport/lib/transport_web/templates/validation/show_gtfs.html.heex

@@ -16,7 +16,7 @@
     <div class="container">
       <%= unless is_nil(@metadata) do %>
         <div class="panel validation-metadata">
-          <%= render("_resources_details.html", metadata: @metadata, conn: @conn, modes: @modes) %>
+          <%= render("_resources_details_gtfs.html", metadata: @metadata, conn: @conn, modes: @modes) %>
         </div>
       <% end %>
 

apps/transport/lib/transport_web/views/resource_view.ex

@@ -126,6 +126,7 @@ defmodule TransportWeb.ResourceView do
   def gbfs_validator_url, do: "https://github.com/MobilityData/gbfs-validator"
   def gtfs_rt_validator_url, do: "https://github.com/MobilityData/gtfs-realtime-validator"
   def gtfs_validator_url, do: "https://github.com/etalab/transport-validator"
+  def netex_validator_url, do: "https://documenter.getpostman.com/view/9950294/2sA3e2gVEE"
 
   def gtfs_rt_validator_rule_url(error_id) when is_binary(error_id) do
     gtfs_rt_validator_rule_url(%{"error_id" => error_id})

apps/transport/lib/validators/netex_validator.ex

@@ -19,10 +19,9 @@ defmodule Transport.Validators.NeTEx do
   def validator_name, do: "enroute-chouette-netex-validator"
 
   @impl Transport.Validators.Validator
-  def validate_and_save(%DB.Resource{format: "NeTEx", id: resource_id}) do
-    Logger.info("Validating NeTEx #{resource_id} with enRoute Chouette Valid")
+  def validate_and_save(%DB.ResourceHistory{} = resource_history) do
+    Logger.info("Validating NeTEx #{resource_history.id} with enRoute Chouette Valid")
 
-    resource_history = DB.ResourceHistory.latest_resource_history(resource_id)
     with_resource_file(resource_history, &validate_resource_history(resource_history, &1))
   end
 
@@ -54,6 +53,7 @@ defmodule Transport.Validators.NeTEx do
   end
 
   @type validate_options :: [{:graceful_retry, boolean()}]
+  @type error_details :: %{:message => String.t(), optional(:retries) => integer()}
 
   @doc """
   Validate the resource from the given URL.
@@ -62,7 +62,7 @@ defmodule Transport.Validators.NeTEx do
   - graceful_retry is a flag to skip the polling interval. Useful for testing
     purposes mostly. Defaults to false.
   """
-  @spec validate(binary(), validate_options()) :: {:ok, map()} | {:error, binary()}
+  @spec validate(binary(), validate_options()) :: {:ok, map()} | {:error, error_details()}
   def validate(url, opts \\ []) do
     with_url(url, fn filepath ->
       case validate_with_enroute(filepath, opts) do

apps/transport/priv/gettext/en/LC_MESSAGES/validations.po

@@ -275,7 +275,7 @@ msgid "Upload in progress"
 msgstr ""
 
 #, elixir-autogen, elixir-format, fuzzy
-msgid "Validation carried out using the <a href=\"%{gtfs_link}\">current GTFS file</a> the %{date} using the <a href=\"%{validator_url}\" target=\"_blank\">PAN GTFS validator</a>."
+msgid "Validation carried out using the <a href=\"%{link}\">current %{format} file</a> the %{date} using the <a href=\"%{validator_url}\" target=\"_blank\">%{validator_name}</a>."
 msgstr ""
 
 #, elixir-autogen, elixir-format
@@ -429,3 +429,11 @@ msgstr ""
 #, elixir-autogen, elixir-format
 msgid "less than 1 second"
 msgstr ""
+
+#, elixir-autogen, elixir-format
+msgid "PAN GTFS validator"
+msgstr ""
+
+#, elixir-autogen, elixir-format
+msgid "enRoute Chouette Valid"
+msgstr "enRoute Chouette Valid validator"

apps/transport/priv/gettext/fr/LC_MESSAGES/validations.po

@@ -275,8 +275,8 @@ msgid "Upload in progress"
 msgstr "Envoi en cours"
 
 #, elixir-autogen, elixir-format
-msgid "Validation carried out using the <a href=\"%{gtfs_link}\">current GTFS file</a> the %{date} using the <a href=\"%{validator_url}\" target=\"_blank\">PAN GTFS validator</a>."
-msgstr "Validation effectuée en utilisant <a href=\"%{gtfs_link}\">le fichier GTFS en vigueur</a> le %{date} avec <a href=\"%{validator_url}\" target=\"_blank\">le validateur GTFS du <abbr title=\"Point d'Accès National\">PAN</abbr></a>."
+msgid "Validation carried out using the <a href=\"%{link}\">current %{format} file</a> the %{date} using the <a href=\"%{validator_url}\" target=\"_blank\">%{validator_name}</a>."
+msgstr "Validation effectuée en utilisant <a href=\"%{link}\">le fichier %{format} en vigueur</a> le %{date} avec <a href=\"%{validator_url}\" target=\"_blank\">%{validator_name}</a>."
 
 #, elixir-autogen, elixir-format
 msgid "from"
@@ -429,3 +429,11 @@ msgstr "Les avertissements sont temporaires. Seules les erreurs XSD sont prises
 #, elixir-autogen, elixir-format
 msgid "less than 1 second"
 msgstr "moins d’une seconde"
+
+#, elixir-autogen, elixir-format
+msgid "PAN GTFS validator"
+msgstr "le validateur GTFS du <abbr title=\"Point d’Accès National\">PAN</abbr>"
+
+#, elixir-autogen, elixir-format
+msgid "enRoute Chouette Valid"
+msgstr "le validateur NeTEx d’enRoute"

apps/transport/priv/gettext/validations.pot

@@ -273,7 +273,7 @@ msgid "Upload in progress"
 msgstr ""
 
 #, elixir-autogen, elixir-format
-msgid "Validation carried out using the <a href=\"%{gtfs_link}\">current GTFS file</a> the %{date} using the <a href=\"%{validator_url}\" target=\"_blank\">PAN GTFS validator</a>."
+msgid "Validation carried out using the <a href=\"%{link}\">current %{format} file</a> the %{date} using the <a href=\"%{validator_url}\" target=\"_blank\">%{validator_name}</a>."
 msgstr ""
 
 #, elixir-autogen, elixir-format
@@ -427,3 +427,11 @@ msgstr ""
 #, elixir-autogen, elixir-format
 msgid "less than 1 second"
 msgstr ""
+
+#, elixir-autogen, elixir-format
+msgid "PAN GTFS validator"
+msgstr ""
+
+#, elixir-autogen, elixir-format
+msgid "enRoute Chouette Valid"
+msgstr ""