From 8b4f026cac10cbdbb27ae180a7966c97ed0ca5af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Menou?= Date: Mon, 4 Nov 2024 11:13:07 +0100 Subject: [PATCH] Validation NeTEx automatique (#4204) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * NeTEx: fixe l’affichage d’un timeout * Validation NeTEx automatique Le validateur NeTEx accepte des ResourceHistory pour se conformer au job de validation. Ceci active de fait la validation automatique de resources NETEx. * Pluriel des résumés de validation Fixes #3025. * Fix merge --------- Co-authored-by: Thibaut Barrère --- .../lib/jobs/on_demand_validation_job.ex | 2 +- .../controllers/resource_controller.ex | 73 +++++-- ...heex => _resources_details_gtfs.html.heex} | 0 .../templates/resource/gtfs_details.html.heex | 12 +- .../resource/netex_details.html.heex | 90 ++++++++ .../templates/validation/show_gtfs.html.heex | 2 +- .../lib/transport_web/views/resource_view.ex | 1 + .../lib/validators/netex_validator.ex | 8 +- .../gettext/en/LC_MESSAGES/validations.po | 10 +- .../gettext/fr/LC_MESSAGES/validations.po | 12 +- apps/transport/priv/gettext/validations.pot | 10 +- .../jobs/resource_history_job_test.exs | 21 +- .../resource_history_validation_job_test.exs | 199 +++++++++--------- .../validators/netex_validator_test.exs | 10 +- .../controllers/resource_controller_test.exs | 48 +++++ 15 files changed, 356 insertions(+), 142 deletions(-) rename apps/transport/lib/transport_web/templates/resource/{_resources_details.html.heex => _resources_details_gtfs.html.heex} (100%) create mode 100644 apps/transport/lib/transport_web/templates/resource/netex_details.html.heex diff --git a/apps/transport/lib/jobs/on_demand_validation_job.ex b/apps/transport/lib/jobs/on_demand_validation_job.ex index b657c1d449..d70fdde3bb 100644 --- a/apps/transport/lib/jobs/on_demand_validation_job.ex +++ b/apps/transport/lib/jobs/on_demand_validation_job.ex @@ -76,7 +76,7 @@ defmodule Transport.Jobs.OnDemandValidationJob do validator = NeTEx.validator_name() case NeTEx.validate(url, []) do - {:error, msg} -> + {:error, %{message: msg}} -> %{oban_args: %{"state" => "error", "error_reason" => msg}, validator: validator} {:ok, %{"validations" => validation, "metadata" => metadata}} -> diff --git a/apps/transport/lib/transport_web/controllers/resource_controller.ex b/apps/transport/lib/transport_web/controllers/resource_controller.ex index f9b92eeea3..de5fc93cc5 100644 --- a/apps/transport/lib/transport_web/controllers/resource_controller.ex +++ b/apps/transport/lib/transport_web/controllers/resource_controller.ex @@ -12,7 +12,8 @@ defmodule TransportWeb.ResourceController do Transport.Validators.GTFSRT, Transport.Validators.GBFSValidator, Transport.Validators.TableSchema, - Transport.Validators.EXJSONSchema + Transport.Validators.EXJSONSchema, + Transport.Validators.NeTEx ]) def details(conn, %{"id" => id} = params) do @@ -31,10 +32,10 @@ defmodule TransportWeb.ResourceController do |> assign(:multi_validation, latest_validation(resource)) |> put_resource_flash(resource.dataset.is_active) - if Resource.gtfs?(resource) do - render_gtfs_details(conn, params, resource) - else - conn |> assign(:resource, resource) |> render("details.html") + cond do + Resource.gtfs?(resource) -> render_gtfs_details(conn, params, resource) + Resource.netex?(resource) -> render_netex_details(conn, params, resource) + true -> render_details(conn, resource) end end @@ -126,40 +127,70 @@ defmodule TransportWeb.ResourceController do DB.MultiValidation.resource_latest_validation(resource_id, validator) end - defp render_gtfs_details(conn, params, resource) do - config = make_pagination_config(params) + def render_details(conn, resource) do + conn |> assign(:resource, resource) |> render("details.html") + end - validation = resource |> latest_validation() + defp render_gtfs_details(conn, params, resource) do + validator = Transport.Validators.GTFSTransport - {validation_summary, severities_count, metadata, modes, issues} = - case validation do - %{result: validation_result, metadata: %DB.ResourceMetadata{metadata: metadata, modes: modes}} -> - {Transport.Validators.GTFSTransport.summary(validation_result), - Transport.Validators.GTFSTransport.count_by_severity(validation_result), metadata, modes, - Transport.Validators.GTFSTransport.get_issues(validation_result, params)} + validation = latest_validation(resource) - nil -> - {nil, nil, nil, [], []} - end + validation_details = {_, _, _, _, issues} = build_validation_details(params, resource, validator) issue_type = case params["issue_type"] do - nil -> Transport.Validators.GTFSTransport.issue_type(issues) + nil -> validator.issue_type(issues) issue_type -> issue_type end + conn + |> assign_base_resource_details(params, resource, validation_details, validator) + |> assign(:data_vis, encoded_data_vis(issue_type, validation)) + |> render("gtfs_details.html") + end + + defp render_netex_details(conn, params, resource) do + validator = Transport.Validators.NeTEx + + validation_details = build_validation_details(params, resource, validator) + + conn + |> assign_base_resource_details(params, resource, validation_details, validator) + |> assign(:data_vis, nil) + |> render("netex_details.html") + end + + defp build_validation_details(params, resource, validator) do + case latest_validation(resource) do + %{result: validation_result, metadata: metadata = %DB.ResourceMetadata{}} -> + summary = validator.summary(validation_result) + stats = validator.count_by_severity(validation_result) + issues = validator.get_issues(validation_result, params) + + {summary, stats, metadata.metadata, metadata.modes, issues} + + nil -> + {nil, nil, nil, [], []} + end + end + + defp assign_base_resource_details(conn, params, resource, validation_details, validator) do + config = make_pagination_config(params) + + {validation_summary, severities_count, metadata, modes, issues} = validation_details + conn |> assign(:related_files, Resource.get_related_files(resource)) |> assign(:resource, resource) |> assign(:other_resources, Resource.other_resources(resource)) |> assign(:issues, Scrivener.paginate(issues, config)) - |> assign(:data_vis, encoded_data_vis(issue_type, validation)) |> assign(:validation_summary, validation_summary) |> assign(:severities_count, severities_count) - |> assign(:validation, validation) + |> assign(:validation, latest_validation(resource)) |> assign(:metadata, metadata) |> assign(:modes, modes) - |> render("gtfs_details.html") + |> assign(:validator, validator) end def encoded_data_vis(_, nil), do: nil diff --git a/apps/transport/lib/transport_web/templates/resource/_resources_details.html.heex b/apps/transport/lib/transport_web/templates/resource/_resources_details_gtfs.html.heex similarity index 100% rename from apps/transport/lib/transport_web/templates/resource/_resources_details.html.heex rename to apps/transport/lib/transport_web/templates/resource/_resources_details_gtfs.html.heex diff --git a/apps/transport/lib/transport_web/templates/resource/gtfs_details.html.heex b/apps/transport/lib/transport_web/templates/resource/gtfs_details.html.heex index 4d8c9c88aa..a9f9c96d3c 100644 --- a/apps/transport/lib/transport_web/templates/resource/gtfs_details.html.heex +++ b/apps/transport/lib/transport_web/templates/resource/gtfs_details.html.heex @@ -10,7 +10,7 @@ locale = get_session(@conn, :locale) %>
<%= render("_resource_description.html", conn: @conn, resource: @resource, resource_history: @resource_history) %> - <%= render("_resources_details.html", conn: @conn, metadata: @metadata, modes: @modes) %> + <%= render("_resources_details_gtfs.html", conn: @conn, metadata: @metadata, modes: @modes) %> <%= if !is_nil(associated_geojson) or !is_nil(associated_netex) do %>
<%= dgettext("validations", "This resource is also available in the following formats:") %> @@ -98,7 +98,7 @@ locale = get_session(@conn, :locale) %> conn: @conn, data_vis: @data_vis, token: nil, - validator: Transport.Validators.GTFSTransport + validator: @validator ) %>
@@ -114,14 +114,16 @@ locale = get_session(@conn, :locale) %> <%= raw( dgettext( "validations", - ~s(Validation carried out using the current GTFS file the %{date} using the PAN GTFS validator.), - gtfs_link: Map.fetch!(@validation.resource_history.payload, "permanent_url"), + ~s(Validation carried out using the current %{format} file the %{date} using the %{validator_name}.), + link: Map.fetch!(@validation.resource_history.payload, "permanent_url"), + format: "GTFS", date: DateTimeDisplay.format_datetime_to_paris( @validation.validation_timestamp, locale ), - validator_url: gtfs_validator_url() + validator_url: gtfs_validator_url(), + validator_name: dgettext("validations", "PAN GTFS validator") ) ) %>

diff --git a/apps/transport/lib/transport_web/templates/resource/netex_details.html.heex b/apps/transport/lib/transport_web/templates/resource/netex_details.html.heex new file mode 100644 index 0000000000..1b37a4d05e --- /dev/null +++ b/apps/transport/lib/transport_web/templates/resource/netex_details.html.heex @@ -0,0 +1,90 @@ +
+
+
+

+ <%= dgettext("validations", "Resource details") %> +

+
+ <%= render("_resource_description.html", conn: @conn, resource: @resource, resource_history: @resource_history) %> +
+ + <%= unless Enum.empty?(@resource.resources_related) do %> + <%= render("_related_resources.html", resource: @resource, conn: @conn) %> + <% end %> + +

<%= dgettext("page-dataset-details", "Download availability") %>

+ <%= render("_download_availability.html", uptime_per_day: @uptime_per_day, conn: @conn) %> + +

<%= dgettext("validations", "Validation report") %>

+
+ <%= if is_nil(@validation_summary) do %> + <%= dgettext("validations", "No validation available") %> + <% end %> + <%= unless is_nil(@validation_summary) do %> +

+ <%= dgettext("validations", "NeTEx validation is in beta.") %>
+ <%= dgettext("validations", "Warnings are work in progress. Only XSD errors are considered for now.") %> +

+ <%= unless is_nil(@metadata) or @metadata == %{} do %> + <%= render("_resources_details_netex.html", conn: @conn, metadata: @metadata) %> + <% end %> + <%= if @issues.total_entries == 0 do %> + <%= dgettext("validations", "No validation error") %>. + <% else %> + +
+ <%= pagination_links(@conn, @issues, [@resource.id], + issue_type: Transport.Validators.NeTEx.issue_type(@issues.entries), + path: &resource_path/4, + action: :details + ) %> + <%= render(netex_template(@issues), issues: @issues || [], conn: @conn) %> +
+ <% end %> +

+ <%= raw( + dgettext( + "validations", + ~s(Validation carried out using the current %{format} file the %{date} using the %{validator_name}.), + link: Map.fetch!(@validation.resource_history.payload, "permanent_url"), + format: "NeTEx", + date: + DateTimeDisplay.format_datetime_to_paris( + @validation.validation_timestamp, + get_session(@conn, :locale) + ), + validator_url: netex_validator_url(), + validator_name: dgettext("validations", "enRoute Chouette Valid") + ) + ) %> +

+ <% end %> +
+ <%= if length(@other_resources) > 0 do %> +

<%= TransportWeb.Gettext.dgettext("validations", "Other resources") %>

+
+
    + <%= for resource <- @other_resources do %> +
  • + <%= link(resource.title, + to: resource_path(@conn, :details, resource.id) + ) %> +
    • + <% end %> +
+
+ <% end %> +
+
+
+