Validateur NeTEx : validation "à la demande" (#4158)

* Extract helpers for enRoute Chouette Valid * On demand NeTEx validation with enRoute <https://transport.data.gouv.fr/validation?type=netex> * Validation NeTEx : meilleur affichage des erreurs * Affichage des metadata de la validation NeTEx * Validateur NeTEx : meilleur affichage de la durée * NeTEx errors grouped by nature * OnDemand: validateur NeTEx désormais disponible * Meilleur message * Adjust metadata display * Display warning about errors levels * Simplify NeTEx issue template dispatcher * Meilleur message pour les validations rapides * Please the linter * No inline stylesheet for CSP implementation * Fix misusage of heex templates * Fix typo in i18n * i18n: better French version * Formatage de durée : utilisons Cldr.Calendar * Please dialyzer * Please dialyzer * Dialyzer: keep track of the exclusions * Better NeTEx validation introduction * Simplify file generation for on demand validation
etalab · Sep 17, 2024 · 2fc9d35 · 2fc9d35
1 parent 958de1d
commit 2fc9d35
Show file tree

Hide file tree

Showing 36 changed files with 569 additions and 137 deletions.
diff --git a/.dialyzer_ignore.exs b/.dialyzer_ignore.exs
@@ -8,5 +8,10 @@
   # See https://github.com/danielberkompas/cloak_ecto/issues/55
   {"lib/db/contact.ex", :unknown_type, 0},
   {"lib/db/user_feedback.ex", :unknown_type, 0},
-  {"lib/db/notification.ex", :unknown_type, 0}
+  {"lib/db/notification.ex", :unknown_type, 0},
+
+  # Workaround for "Overloaded contract for Transport.Cldr.Calendar.localize/3
+  # has overlapping domains; such contracts are currently unsupported and are
+  # simply ignored."
+  ~r/lib\/cldr.ex/
 ]
diff --git a/apps/shared/lib/cldr.ex b/apps/shared/lib/cldr.ex
@@ -3,5 +3,5 @@ defmodule Transport.Cldr do
   Declares a backend for Cldr as required.
   https://hexdocs.pm/ex_cldr_numbers/readme.html#introduction-and-getting-started
   """
-  use Cldr, locales: ["en", "fr"], providers: [Cldr.Number], default_locale: "fr"
+  use Cldr, locales: ["en", "fr"], providers: [Cldr.Number, Cldr.Calendar, Cldr.Unit, Cldr.List], default_locale: "fr"
 end
diff --git a/apps/shared/lib/date_time_display.ex b/apps/shared/lib/date_time_display.ex
@@ -104,6 +104,66 @@ defmodule Shared.DateTimeDisplay do
 
   def format_datetime_to_paris(nil, _, _), do: ""
 
+  @doc """
+  Formats a duration in seconds to display, according to a locale.
+
+  Supported locales: "fr" and "en".
+
+  iex> format_duration(1, :en)
+  "1 second"
+  iex> format_duration(1, Transport.Cldr.Locale.new!("en"))
+  "1 second"
+  iex> format_duration(1, "en")
+  "1 second"
+  iex> format_duration(3, "en")
+  "3 seconds"
+  iex> format_duration(60, "en")
+  "1 minute"
+  iex> format_duration(61, "en")
+  "1 minute and 1 second"
+  iex> format_duration(65, "en")
+  "1 minute and 5 seconds"
+  iex> format_duration(120, "en")
+  "2 minutes"
+  iex> format_duration(125, "en")
+  "2 minutes and 5 seconds"
+  iex> format_duration(3601, "en")
+  "1 hour and 1 second"
+  iex> format_duration(3661, "en")
+  "1 hour, 1 minute, and 1 second"
+
+  iex> format_duration(1, :fr)
+  "1 seconde"
+  iex> format_duration(1, Transport.Cldr.Locale.new!("fr"))
+  "1 seconde"
+  iex> format_duration(1, "fr")
+  "1 seconde"
+  iex> format_duration(3, "fr")
+  "3 secondes"
+  iex> format_duration(60, "fr")
+  "1 minute"
+  iex> format_duration(61, "fr")
+  "1 minute et 1 seconde"
+  iex> format_duration(65, "fr")
+  "1 minute et 5 secondes"
+  iex> format_duration(120, "fr")
+  "2 minutes"
+  iex> format_duration(125, "fr")
+  "2 minutes et 5 secondes"
+  iex> format_duration(3601, "fr")
+  "1 heure et 1 seconde"
+  iex> format_duration(3661, "fr")
+  "1 heure, 1 minute et 1 seconde"
+  """
+  @spec format_duration(pos_integer(), atom() | Cldr.LanguageTag.t()) :: binary()
+  def format_duration(duration_in_seconds, locale) do
+    locale = Cldr.Locale.new!(locale, Transport.Cldr)
+
+    duration_in_seconds
+    |> Cldr.Calendar.Duration.new_from_seconds()
+    |> Cldr.Calendar.Duration.to_string!(locale: locale)
+  end
+
   @spec convert_to_paris_time(DateTime.t() | NaiveDateTime.t()) :: DateTime.t()
   defp convert_to_paris_time(%DateTime{} = dt) do
     case Timex.Timezone.convert(dt, "Europe/Paris") do

diff --git a/apps/shared/mix.exs b/apps/shared/mix.exs
@@ -57,6 +57,9 @@ defmodule Shared.MixProject do
       # be required no matter what.
       {:jason, ">= 0.0.0"},
       {:ex_cldr_numbers, "~> 2.0"},
+      {:ex_cldr_calendars, "~> 1.26"},
+      {:ex_cldr_lists, "~> 2.11"},
+      {:ex_cldr_units, "~> 3.17"},
       {:cachex, "~> 3.5"},
       {:ex_json_schema, "~> 0.10"},
       # added because of `TransportWeb.Plugs.AppSignalFilter`

diff --git a/apps/transport/client/stylesheets/components/_validation.scss b/apps/transport/client/stylesheets/components/_validation.scss
@@ -123,3 +123,32 @@ details > code {
     display: inline;
   }
 }
+
+table.netex_generic_issue tr.message td {
+  vertical-align: top;
+}
+
+table.netex_generic_issue th:nth-child(1) {
+  width: 60%;
+}
+
+table.netex_generic_issue tr.debug:hover {
+  background: revert;
+}
+
+table.netex_generic_issue tr.debug td {
+  border-top: none;
+  padding-top: 0;
+}
+
+table.netex_generic_issue tr.debug td summary {
+  cursor: pointer;
+}
+
+table.netex_generic_issue tr.debug td pre {
+  margin-block: 0;
+}
+
+table.netex_generic_issue tr.debug td code {
+  width: 100%;
+}
diff --git a/apps/transport/lib/jobs/on_demand_validation_job.ex b/apps/transport/lib/jobs/on_demand_validation_job.ex
@@ -15,6 +15,7 @@ defmodule Transport.Jobs.OnDemandValidationJob do
   alias Transport.DataVisualization
   alias Transport.Validators.GTFSRT
   alias Transport.Validators.GTFSTransport
+  alias Transport.Validators.NeTEx
   @download_timeout_ms 10_000
 
   @impl Oban.Worker
@@ -71,6 +72,28 @@ defmodule Transport.Jobs.OnDemandValidationJob do
     end
   end
 
+  defp perform_validation(%{"type" => "netex", "permanent_url" => url}) do
+    validator = NeTEx.validator_name()
+
+    case NeTEx.validate(url, []) do
+      {:error, msg} ->
+        %{oban_args: %{"state" => "error", "error_reason" => msg}, validator: validator}
+
+      {:ok, %{"validations" => validation, "metadata" => metadata}} ->
+        %{
+          result: validation,
+          metadata: metadata,
+          data_vis: nil,
+          validator: validator,
+          validated_data_name: url,
+          max_error: NeTEx.get_max_severity_error(validation),
+          oban_args: %{
+            "state" => "completed"
+          }
+        }
+    end
+  end
+
   defp perform_validation(%{
          "type" => "tableschema",
          "permanent_url" => url,

diff --git a/apps/transport/lib/transport_web/controllers/resource_controller.ex b/apps/transport/lib/transport_web/controllers/resource_controller.ex
@@ -4,7 +4,7 @@ defmodule TransportWeb.ResourceController do
   alias Transport.DataVisualization
   import Ecto.Query
 
-  import TransportWeb.ResourceView, only: [issue_type: 1, latest_validations_nb_days: 0]
+  import TransportWeb.ResourceView, only: [latest_validations_nb_days: 0]
   import TransportWeb.DatasetView, only: [availability_number_days: 0]
 
   @enabled_validators MapSet.new([
@@ -144,7 +144,7 @@ defmodule TransportWeb.ResourceController do
 
     issue_type =
       case params["issue_type"] do
-        nil -> issue_type(issues)
+        nil -> Transport.Validators.GTFSTransport.issue_type(issues)
         issue_type -> issue_type
       end
 

diff --git a/apps/transport/lib/transport_web/controllers/validation_controller.ex b/apps/transport/lib/transport_web/controllers/validation_controller.ex
@@ -2,7 +2,6 @@ defmodule TransportWeb.ValidationController do
   use TransportWeb, :controller
   alias DB.{MultiValidation, Repo}
   alias Transport.DataVisualization
-  import TransportWeb.ResourceView, only: [issue_type: 1]
   import Ecto.Query
 
   def validate(%Plug.Conn{} = conn, %{"upload" => %{"url" => url, "type" => "gbfs"} = params}) do
@@ -98,44 +97,57 @@ defmodule TransportWeb.ValidationController do
         unauthorized(conn)
 
       %MultiValidation{oban_args: %{"state" => "completed", "type" => "gtfs"}} = validation ->
-        current_issues = Transport.Validators.GTFSTransport.get_issues(validation.result, params)
+        validator = Transport.Validators.GTFSTransport
+        current_issues = validator.get_issues(validation.result, params)
 
         issue_type =
           case params["issue_type"] do
-            nil -> issue_type(current_issues)
+            nil -> validator.issue_type(current_issues)
             issue_type -> issue_type
           end
 
         conn
-        |> assign(:validation_id, params["id"])
-        |> assign(:other_resources, [])
-        |> assign(:issues, Scrivener.paginate(current_issues, make_pagination_config(params)))
-        |> assign(
-          :validation_summary,
-          Transport.Validators.GTFSTransport.summary(validation.result)
-        )
-        |> assign(
-          :severities_count,
-          Transport.Validators.GTFSTransport.count_by_severity(validation.result)
-        )
+        |> assign_base_validation_details(validator, validation, params, current_issues)
         |> assign(:metadata, validation.metadata.metadata)
         |> assign(:modes, validation.metadata.modes)
         |> assign(:data_vis, data_vis(validation, issue_type))
-        |> assign(:token, token)
-        |> render("show.html")
+        |> render("show_gtfs.html")
+
+      %MultiValidation{oban_args: %{"state" => "completed", "type" => "netex"}} = validation ->
+        validator = Transport.Validators.NeTEx
+        current_issues = validator.get_issues(validation.result, params)
+
+        conn
+        |> assign_base_validation_details(validator, validation, params, current_issues)
+        |> assign(:metadata, validation.metadata.metadata)
+        |> assign(:modes, [])
+        |> assign(:data_vis, nil)
+        |> render("show_netex.html")
 
       # Handles waiting for validation to complete, errors and
       # validation for schemas
       _ ->
         live_render(conn, TransportWeb.Live.OnDemandValidationLive,
           session: %{
             "validation_id" => params["id"],
+            "issue_type" => params["issue_type"],
             "current_url" => validation_path(conn, :show, params["id"], token: token)
           }
         )
     end
   end
 
+  defp assign_base_validation_details(conn, validator, validation, params, current_issues) do
+    conn
+    |> assign(:validator, validator)
+    |> assign(:validation_id, params["id"])
+    |> assign(:other_resources, [])
+    |> assign(:issues, Scrivener.paginate(current_issues, make_pagination_config(params)))
+    |> assign(:validation_summary, validator.summary(validation.result))
+    |> assign(:severities_count, validator.count_by_severity(validation.result))
+    |> assign(:token, params["token"])
+  end
+
   defp data_vis(%MultiValidation{} = validation, issue_type) do
     data_vis = validation.data_vis[issue_type]
     has_features = DataVisualization.has_features(data_vis["geojson"])
@@ -148,9 +160,10 @@ defmodule TransportWeb.ValidationController do
   end
 
   defp filepath(type) do
-    cond do
-      type == "tableschema" -> Ecto.UUID.generate() <> ".csv"
-      type in ["jsonschema", "gtfs"] -> Ecto.UUID.generate()
+    if type == "tableschema" do
+      Ecto.UUID.generate() <> ".csv"
+    else
+      Ecto.UUID.generate()
     end
   end
 
@@ -215,6 +228,7 @@ defmodule TransportWeb.ValidationController do
     args =
       case type do
         "gtfs" -> %{"type" => "gtfs"}
+        "netex" -> %{"type" => "netex"}
         schema_name -> %{"schema_name" => schema_name, "type" => schema_type(schema_name)}
       end
 

diff --git a/apps/transport/lib/transport_web/live/on_demand_validation_live.ex b/apps/transport/lib/transport_web/live/on_demand_validation_live.ex
@@ -35,7 +35,7 @@ defmodule TransportWeb.Live.OnDemandValidationLive do
       schedule_next_update_data()
     end
 
-    if gtfs_validation_completed?(socket) do
+    if gtfs_or_netex_validation_completed?(socket) do
       redirect(socket, to: socket_value(socket, :current_url))
     else
       socket
@@ -57,10 +57,10 @@ defmodule TransportWeb.Live.OnDemandValidationLive do
     end
   end
 
-  defp gtfs_validation_completed?(socket) do
+  defp gtfs_or_netex_validation_completed?(socket) do
     case socket_value(socket, :validation) do
       %DB.MultiValidation{oban_args: oban_args} ->
-        oban_args["type"] == "gtfs" and oban_args["state"] == "completed"
+        oban_args["type"] in ["gtfs", "netex"] and oban_args["state"] == "completed"
 
       _ ->
         false

diff --git a/apps/transport/lib/transport_web/live/on_demand_validation_select_live.ex b/apps/transport/lib/transport_web/live/on_demand_validation_select_live.ex
@@ -53,7 +53,6 @@ defmodule TransportWeb.Live.OnDemandValidationSelectLive do
 
   def determine_input_type(type) when type in ["gbfs"], do: "link"
   def determine_input_type(type) when type in ["gtfs-rt"], do: "gtfs-rt"
-  def determine_input_type(type) when type in ["netex"], do: nil
   def determine_input_type(_), do: "file"
 
   def handle_event("form_changed", %{"upload" => params, "_target" => target}, socket) do

diff --git a/apps/transport/lib/transport_web/live/on_demand_validation_select_live.html.heex b/apps/transport/lib/transport_web/live/on_demand_validation_select_live.html.heex
@@ -38,28 +38,13 @@
             type: "url"
           ) %>
         <% end %>
-        <%= unless @input_type == "file" or @type == "netex" do %>
+        <%= unless @input_type == "file" do %>
           <%= submit(dgettext("validations", "Validate"), nodiv: true) %>
         <% end %>
       </.form>
       <p :if={@trigger_submit} class="small">
         <%= TransportWeb.Gettext.dgettext("validations", "Upload in progress") %>
       </p>
-      <div :if={@type == "netex"} class="container section section-grey" id="netex-explanations">
-        <p>
-          <%= raw(
-            TransportWeb.Gettext.dgettext("validations", "netex-siri-validator", link: "https://greenlight.itxpt.eu")
-          ) %>
-        </p>
-
-        <p>
-          <%= raw(
-            TransportWeb.Gettext.dgettext("validations", "doc-eu-formats",
-              link: "https://doc.transport.data.gouv.fr/documentation/normes-europeennes"
-            )
-          ) %>
-        </p>
-      </div>
     </div>
   </div>
 </section>

diff --git a/apps/transport/lib/transport_web/templates/resource/_netex_generic_issue.html.heex b/apps/transport/lib/transport_web/templates/resource/_netex_generic_issue.html.heex
@@ -0,0 +1,27 @@
+<table class="table netex_generic_issue">
+  <tr>
+    <th><%= dgettext("validations-explanations", "Message") %></th>
+    <th><%= dgettext("validations-explanations", "Location") %></th>
+  </tr>
+
+  <%= for issue <- @issues do %>
+    <tr class="message">
+      <td><%= issue["message"] %></td>
+      <td>
+        <%= if is_nil(issue["resource"]) or is_nil(issue["resource"]["filename"]) or is_nil(issue["resource"]["line"]) do %>
+          <%= dgettext("validations-explanations", "Unknown location") %>
+        <% else %>
+          <%= issue["resource"]["filename"] %>:<%= issue["resource"]["line"] %>
+        <% end %>
+      </td>
+    </tr>
+    <tr class="debug">
+      <td colspan="2">
+        <details>
+          <summary><%= dgettext("validations-explanations", "Details for debugging purposes") %></summary>
+          <pre><code><%= to_string(Jason.encode!(issue, pretty: true)) %></code></pre>
+        </details>
+      </td>
+    </tr>
+  <% end %>
+</table>
diff --git a/apps/transport/lib/transport_web/templates/resource/_resources_details.html.heex b/apps/transport/lib/transport_web/templates/resource/_resources_details.html.heex
@@ -27,16 +27,16 @@ stats = @metadata["stats"] %>
           </div>
         </li>
       <% else %>
-        <li if={networks != []}>
+        <li :if={networks != []}>
           <div>
             <div class="networks-list">
-              <%= dngettext("validations", "network", "networks", length(@metadata["networks"])) %> :
-              <strong><%= Enum.join(@metadata["networks"], ", ") %></strong>
+              <%= dngettext("validations", "network", "networks", length(networks)) %> :
+              <strong><%= Enum.join(networks, ", ") %></strong>
             </div>
           </div>
         </li>
       <% end %>
-      <li if={length(@modes) > 0}>
+      <li :if={length(@modes) > 0}>
         <%= dngettext("validations", "transport mode", "transport modes", length(@modes)) %> :
         <strong><%= Enum.join(@modes, ", ") %></strong>
       </li>