From 2fc9d35c61ca7c9c562f6bf12ba1baf87f49950d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Menou?= Date: Tue, 17 Sep 2024 17:23:51 +0200 Subject: [PATCH] =?UTF-8?q?Validateur=20NeTEx=20:=20validation=20"=C3=A0?= =?UTF-8?q?=20la=20demande"=20(#4158)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Extract helpers for enRoute Chouette Valid * On demand NeTEx validation with enRoute * Validation NeTEx : meilleur affichage des erreurs * Affichage des metadata de la validation NeTEx * Validateur NeTEx : meilleur affichage de la durée * NeTEx errors grouped by nature * OnDemand: validateur NeTEx désormais disponible * Meilleur message * Adjust metadata display * Display warning about errors levels * Simplify NeTEx issue template dispatcher * Meilleur message pour les validations rapides * Please the linter * No inline stylesheet for CSP implementation * Fix misusage of heex templates * Fix typo in i18n * i18n: better French version * Formatage de durée : utilisons Cldr.Calendar * Please dialyzer * Please dialyzer * Dialyzer: keep track of the exclusions * Better NeTEx validation introduction * Simplify file generation for on demand validation --- .dialyzer_ignore.exs | 7 +- apps/shared/lib/cldr.ex | 2 +- apps/shared/lib/date_time_display.ex | 60 +++++++++++++++ apps/shared/mix.exs | 3 + .../stylesheets/components/_validation.scss | 29 +++++++ .../lib/jobs/on_demand_validation_job.ex | 23 ++++++ .../controllers/resource_controller.ex | 4 +- .../controllers/validation_controller.ex | 52 ++++++++----- .../live/on_demand_validation_live.ex | 6 +- .../live/on_demand_validation_select_live.ex | 1 - ...on_demand_validation_select_live.html.heex | 17 +---- .../resource/_netex_generic_issue.html.heex | 27 +++++++ .../resource/_resources_details.html.heex | 8 +- .../_resources_details_netex.html.heex | 10 +++ .../resource/_validation_summary.html.heex | 4 +- .../templates/resource/gtfs_details.html.heex | 2 +- .../{show.html.heex => show_gtfs.html.heex} | 9 ++- .../templates/validation/show_netex.html.heex | 70 +++++++++++++++++ .../lib/transport_web/views/resource_view.ex | 14 ++-- .../transport_web/views/validation_view.ex | 2 +- .../validators/gtfs_transport_validator.ex | 4 + .../lib/validators/netex_validator.ex | 23 +++++- .../gettext/en/LC_MESSAGES/netex-validator.po | 6 +- .../LC_MESSAGES/validations-explanations.po | 16 ++++ .../gettext/en/LC_MESSAGES/validations.po | 37 ++++++--- .../gettext/fr/LC_MESSAGES/netex-validator.po | 8 +- .../LC_MESSAGES/validations-explanations.po | 16 ++++ .../gettext/fr/LC_MESSAGES/validations.po | 35 ++++++--- .../priv/gettext/netex-validator.pot | 6 +- .../priv/gettext/validations-explanations.pot | 16 ++++ apps/transport/priv/gettext/validations.pot | 34 ++++++--- .../enroute_chouette_valid_client_helpers.ex | 32 ++++++++ .../jobs/on_demand_validation_job_test.exs | 76 ++++++++++++++++++- .../validators/netex_validator_test.exs | 27 +------ .../validation_controller_test.exs | 17 ++--- mix.lock | 3 + 36 files changed, 569 insertions(+), 137 deletions(-) create mode 100644 apps/transport/lib/transport_web/templates/resource/_netex_generic_issue.html.heex create mode 100644 apps/transport/lib/transport_web/templates/resource/_resources_details_netex.html.heex rename apps/transport/lib/transport_web/templates/validation/{show.html.heex => show_gtfs.html.heex} (90%) create mode 100644 apps/transport/lib/transport_web/templates/validation/show_netex.html.heex create mode 100644 apps/transport/test/support/enroute_chouette_valid_client_helpers.ex diff --git a/.dialyzer_ignore.exs b/.dialyzer_ignore.exs index fbd5600ea4..ca22a3ecb4 100644 --- a/.dialyzer_ignore.exs +++ b/.dialyzer_ignore.exs @@ -8,5 +8,10 @@ # See https://github.com/danielberkompas/cloak_ecto/issues/55 {"lib/db/contact.ex", :unknown_type, 0}, {"lib/db/user_feedback.ex", :unknown_type, 0}, - {"lib/db/notification.ex", :unknown_type, 0} + {"lib/db/notification.ex", :unknown_type, 0}, + + # Workaround for "Overloaded contract for Transport.Cldr.Calendar.localize/3 + # has overlapping domains; such contracts are currently unsupported and are + # simply ignored." + ~r/lib\/cldr.ex/ ] diff --git a/apps/shared/lib/cldr.ex b/apps/shared/lib/cldr.ex index 2c37c71d09..c0ac98f300 100644 --- a/apps/shared/lib/cldr.ex +++ b/apps/shared/lib/cldr.ex @@ -3,5 +3,5 @@ defmodule Transport.Cldr do Declares a backend for Cldr as required. https://hexdocs.pm/ex_cldr_numbers/readme.html#introduction-and-getting-started """ - use Cldr, locales: ["en", "fr"], providers: [Cldr.Number], default_locale: "fr" + use Cldr, locales: ["en", "fr"], providers: [Cldr.Number, Cldr.Calendar, Cldr.Unit, Cldr.List], default_locale: "fr" end diff --git a/apps/shared/lib/date_time_display.ex b/apps/shared/lib/date_time_display.ex index 88d5dcedc6..2524f2a543 100644 --- a/apps/shared/lib/date_time_display.ex +++ b/apps/shared/lib/date_time_display.ex @@ -104,6 +104,66 @@ defmodule Shared.DateTimeDisplay do def format_datetime_to_paris(nil, _, _), do: "" + @doc """ + Formats a duration in seconds to display, according to a locale. + + Supported locales: "fr" and "en". + + iex> format_duration(1, :en) + "1 second" + iex> format_duration(1, Transport.Cldr.Locale.new!("en")) + "1 second" + iex> format_duration(1, "en") + "1 second" + iex> format_duration(3, "en") + "3 seconds" + iex> format_duration(60, "en") + "1 minute" + iex> format_duration(61, "en") + "1 minute and 1 second" + iex> format_duration(65, "en") + "1 minute and 5 seconds" + iex> format_duration(120, "en") + "2 minutes" + iex> format_duration(125, "en") + "2 minutes and 5 seconds" + iex> format_duration(3601, "en") + "1 hour and 1 second" + iex> format_duration(3661, "en") + "1 hour, 1 minute, and 1 second" + + iex> format_duration(1, :fr) + "1 seconde" + iex> format_duration(1, Transport.Cldr.Locale.new!("fr")) + "1 seconde" + iex> format_duration(1, "fr") + "1 seconde" + iex> format_duration(3, "fr") + "3 secondes" + iex> format_duration(60, "fr") + "1 minute" + iex> format_duration(61, "fr") + "1 minute et 1 seconde" + iex> format_duration(65, "fr") + "1 minute et 5 secondes" + iex> format_duration(120, "fr") + "2 minutes" + iex> format_duration(125, "fr") + "2 minutes et 5 secondes" + iex> format_duration(3601, "fr") + "1 heure et 1 seconde" + iex> format_duration(3661, "fr") + "1 heure, 1 minute et 1 seconde" + """ + @spec format_duration(pos_integer(), atom() | Cldr.LanguageTag.t()) :: binary() + def format_duration(duration_in_seconds, locale) do + locale = Cldr.Locale.new!(locale, Transport.Cldr) + + duration_in_seconds + |> Cldr.Calendar.Duration.new_from_seconds() + |> Cldr.Calendar.Duration.to_string!(locale: locale) + end + @spec convert_to_paris_time(DateTime.t() | NaiveDateTime.t()) :: DateTime.t() defp convert_to_paris_time(%DateTime{} = dt) do case Timex.Timezone.convert(dt, "Europe/Paris") do diff --git a/apps/shared/mix.exs b/apps/shared/mix.exs index 6c6b785d9a..36e5797b6a 100644 --- a/apps/shared/mix.exs +++ b/apps/shared/mix.exs @@ -57,6 +57,9 @@ defmodule Shared.MixProject do # be required no matter what. {:jason, ">= 0.0.0"}, {:ex_cldr_numbers, "~> 2.0"}, + {:ex_cldr_calendars, "~> 1.26"}, + {:ex_cldr_lists, "~> 2.11"}, + {:ex_cldr_units, "~> 3.17"}, {:cachex, "~> 3.5"}, {:ex_json_schema, "~> 0.10"}, # added because of `TransportWeb.Plugs.AppSignalFilter` diff --git a/apps/transport/client/stylesheets/components/_validation.scss b/apps/transport/client/stylesheets/components/_validation.scss index a88979a24d..04654c3899 100644 --- a/apps/transport/client/stylesheets/components/_validation.scss +++ b/apps/transport/client/stylesheets/components/_validation.scss @@ -123,3 +123,32 @@ details > code { display: inline; } } + +table.netex_generic_issue tr.message td { + vertical-align: top; +} + +table.netex_generic_issue th:nth-child(1) { + width: 60%; +} + +table.netex_generic_issue tr.debug:hover { + background: revert; +} + +table.netex_generic_issue tr.debug td { + border-top: none; + padding-top: 0; +} + +table.netex_generic_issue tr.debug td summary { + cursor: pointer; +} + +table.netex_generic_issue tr.debug td pre { + margin-block: 0; +} + +table.netex_generic_issue tr.debug td code { + width: 100%; +} diff --git a/apps/transport/lib/jobs/on_demand_validation_job.ex b/apps/transport/lib/jobs/on_demand_validation_job.ex index a1f977a0a1..b657c1d449 100644 --- a/apps/transport/lib/jobs/on_demand_validation_job.ex +++ b/apps/transport/lib/jobs/on_demand_validation_job.ex @@ -15,6 +15,7 @@ defmodule Transport.Jobs.OnDemandValidationJob do alias Transport.DataVisualization alias Transport.Validators.GTFSRT alias Transport.Validators.GTFSTransport + alias Transport.Validators.NeTEx @download_timeout_ms 10_000 @impl Oban.Worker @@ -71,6 +72,28 @@ defmodule Transport.Jobs.OnDemandValidationJob do end end + defp perform_validation(%{"type" => "netex", "permanent_url" => url}) do + validator = NeTEx.validator_name() + + case NeTEx.validate(url, []) do + {:error, msg} -> + %{oban_args: %{"state" => "error", "error_reason" => msg}, validator: validator} + + {:ok, %{"validations" => validation, "metadata" => metadata}} -> + %{ + result: validation, + metadata: metadata, + data_vis: nil, + validator: validator, + validated_data_name: url, + max_error: NeTEx.get_max_severity_error(validation), + oban_args: %{ + "state" => "completed" + } + } + end + end + defp perform_validation(%{ "type" => "tableschema", "permanent_url" => url, diff --git a/apps/transport/lib/transport_web/controllers/resource_controller.ex b/apps/transport/lib/transport_web/controllers/resource_controller.ex index 331a8f734a..f9b92eeea3 100644 --- a/apps/transport/lib/transport_web/controllers/resource_controller.ex +++ b/apps/transport/lib/transport_web/controllers/resource_controller.ex @@ -4,7 +4,7 @@ defmodule TransportWeb.ResourceController do alias Transport.DataVisualization import Ecto.Query - import TransportWeb.ResourceView, only: [issue_type: 1, latest_validations_nb_days: 0] + import TransportWeb.ResourceView, only: [latest_validations_nb_days: 0] import TransportWeb.DatasetView, only: [availability_number_days: 0] @enabled_validators MapSet.new([ @@ -144,7 +144,7 @@ defmodule TransportWeb.ResourceController do issue_type = case params["issue_type"] do - nil -> issue_type(issues) + nil -> Transport.Validators.GTFSTransport.issue_type(issues) issue_type -> issue_type end diff --git a/apps/transport/lib/transport_web/controllers/validation_controller.ex b/apps/transport/lib/transport_web/controllers/validation_controller.ex index 633d350962..bf836afe49 100644 --- a/apps/transport/lib/transport_web/controllers/validation_controller.ex +++ b/apps/transport/lib/transport_web/controllers/validation_controller.ex @@ -2,7 +2,6 @@ defmodule TransportWeb.ValidationController do use TransportWeb, :controller alias DB.{MultiValidation, Repo} alias Transport.DataVisualization - import TransportWeb.ResourceView, only: [issue_type: 1] import Ecto.Query def validate(%Plug.Conn{} = conn, %{"upload" => %{"url" => url, "type" => "gbfs"} = params}) do @@ -98,31 +97,32 @@ defmodule TransportWeb.ValidationController do unauthorized(conn) %MultiValidation{oban_args: %{"state" => "completed", "type" => "gtfs"}} = validation -> - current_issues = Transport.Validators.GTFSTransport.get_issues(validation.result, params) + validator = Transport.Validators.GTFSTransport + current_issues = validator.get_issues(validation.result, params) issue_type = case params["issue_type"] do - nil -> issue_type(current_issues) + nil -> validator.issue_type(current_issues) issue_type -> issue_type end conn - |> assign(:validation_id, params["id"]) - |> assign(:other_resources, []) - |> assign(:issues, Scrivener.paginate(current_issues, make_pagination_config(params))) - |> assign( - :validation_summary, - Transport.Validators.GTFSTransport.summary(validation.result) - ) - |> assign( - :severities_count, - Transport.Validators.GTFSTransport.count_by_severity(validation.result) - ) + |> assign_base_validation_details(validator, validation, params, current_issues) |> assign(:metadata, validation.metadata.metadata) |> assign(:modes, validation.metadata.modes) |> assign(:data_vis, data_vis(validation, issue_type)) - |> assign(:token, token) - |> render("show.html") + |> render("show_gtfs.html") + + %MultiValidation{oban_args: %{"state" => "completed", "type" => "netex"}} = validation -> + validator = Transport.Validators.NeTEx + current_issues = validator.get_issues(validation.result, params) + + conn + |> assign_base_validation_details(validator, validation, params, current_issues) + |> assign(:metadata, validation.metadata.metadata) + |> assign(:modes, []) + |> assign(:data_vis, nil) + |> render("show_netex.html") # Handles waiting for validation to complete, errors and # validation for schemas @@ -130,12 +130,24 @@ defmodule TransportWeb.ValidationController do live_render(conn, TransportWeb.Live.OnDemandValidationLive, session: %{ "validation_id" => params["id"], + "issue_type" => params["issue_type"], "current_url" => validation_path(conn, :show, params["id"], token: token) } ) end end + defp assign_base_validation_details(conn, validator, validation, params, current_issues) do + conn + |> assign(:validator, validator) + |> assign(:validation_id, params["id"]) + |> assign(:other_resources, []) + |> assign(:issues, Scrivener.paginate(current_issues, make_pagination_config(params))) + |> assign(:validation_summary, validator.summary(validation.result)) + |> assign(:severities_count, validator.count_by_severity(validation.result)) + |> assign(:token, params["token"]) + end + defp data_vis(%MultiValidation{} = validation, issue_type) do data_vis = validation.data_vis[issue_type] has_features = DataVisualization.has_features(data_vis["geojson"]) @@ -148,9 +160,10 @@ defmodule TransportWeb.ValidationController do end defp filepath(type) do - cond do - type == "tableschema" -> Ecto.UUID.generate() <> ".csv" - type in ["jsonschema", "gtfs"] -> Ecto.UUID.generate() + if type == "tableschema" do + Ecto.UUID.generate() <> ".csv" + else + Ecto.UUID.generate() end end @@ -215,6 +228,7 @@ defmodule TransportWeb.ValidationController do args = case type do "gtfs" -> %{"type" => "gtfs"} + "netex" -> %{"type" => "netex"} schema_name -> %{"schema_name" => schema_name, "type" => schema_type(schema_name)} end diff --git a/apps/transport/lib/transport_web/live/on_demand_validation_live.ex b/apps/transport/lib/transport_web/live/on_demand_validation_live.ex index 9b1f2b70c2..9cae1b91b3 100644 --- a/apps/transport/lib/transport_web/live/on_demand_validation_live.ex +++ b/apps/transport/lib/transport_web/live/on_demand_validation_live.ex @@ -35,7 +35,7 @@ defmodule TransportWeb.Live.OnDemandValidationLive do schedule_next_update_data() end - if gtfs_validation_completed?(socket) do + if gtfs_or_netex_validation_completed?(socket) do redirect(socket, to: socket_value(socket, :current_url)) else socket @@ -57,10 +57,10 @@ defmodule TransportWeb.Live.OnDemandValidationLive do end end - defp gtfs_validation_completed?(socket) do + defp gtfs_or_netex_validation_completed?(socket) do case socket_value(socket, :validation) do %DB.MultiValidation{oban_args: oban_args} -> - oban_args["type"] == "gtfs" and oban_args["state"] == "completed" + oban_args["type"] in ["gtfs", "netex"] and oban_args["state"] == "completed" _ -> false diff --git a/apps/transport/lib/transport_web/live/on_demand_validation_select_live.ex b/apps/transport/lib/transport_web/live/on_demand_validation_select_live.ex index 09ced2116b..e594e46bc4 100644 --- a/apps/transport/lib/transport_web/live/on_demand_validation_select_live.ex +++ b/apps/transport/lib/transport_web/live/on_demand_validation_select_live.ex @@ -53,7 +53,6 @@ defmodule TransportWeb.Live.OnDemandValidationSelectLive do def determine_input_type(type) when type in ["gbfs"], do: "link" def determine_input_type(type) when type in ["gtfs-rt"], do: "gtfs-rt" - def determine_input_type(type) when type in ["netex"], do: nil def determine_input_type(_), do: "file" def handle_event("form_changed", %{"upload" => params, "_target" => target}, socket) do diff --git a/apps/transport/lib/transport_web/live/on_demand_validation_select_live.html.heex b/apps/transport/lib/transport_web/live/on_demand_validation_select_live.html.heex index 6655eeda03..9cff00c6bb 100644 --- a/apps/transport/lib/transport_web/live/on_demand_validation_select_live.html.heex +++ b/apps/transport/lib/transport_web/live/on_demand_validation_select_live.html.heex @@ -38,28 +38,13 @@ type: "url" ) %> <% end %> - <%= unless @input_type == "file" or @type == "netex" do %> + <%= unless @input_type == "file" do %> <%= submit(dgettext("validations", "Validate"), nodiv: true) %> <% end %>

<%= TransportWeb.Gettext.dgettext("validations", "Upload in progress") %>

-
-

- <%= raw( - TransportWeb.Gettext.dgettext("validations", "netex-siri-validator", link: "https://greenlight.itxpt.eu") - ) %> -

- -

- <%= raw( - TransportWeb.Gettext.dgettext("validations", "doc-eu-formats", - link: "https://doc.transport.data.gouv.fr/documentation/normes-europeennes" - ) - ) %> -

-
diff --git a/apps/transport/lib/transport_web/templates/resource/_netex_generic_issue.html.heex b/apps/transport/lib/transport_web/templates/resource/_netex_generic_issue.html.heex new file mode 100644 index 0000000000..865cccc4b6 --- /dev/null +++ b/apps/transport/lib/transport_web/templates/resource/_netex_generic_issue.html.heex @@ -0,0 +1,27 @@ + + + + + + + <%= for issue <- @issues do %> + + + + + + + + <% end %> +
<%= dgettext("validations-explanations", "Message") %><%= dgettext("validations-explanations", "Location") %>
<%= issue["message"] %> + <%= if is_nil(issue["resource"]) or is_nil(issue["resource"]["filename"]) or is_nil(issue["resource"]["line"]) do %> + <%= dgettext("validations-explanations", "Unknown location") %> + <% else %> + <%= issue["resource"]["filename"] %>:<%= issue["resource"]["line"] %> + <% end %> +
+
+ <%= dgettext("validations-explanations", "Details for debugging purposes") %> + 
<%= to_string(Jason.encode!(issue, pretty: true)) %>
+
+
diff --git a/apps/transport/lib/transport_web/templates/resource/_resources_details.html.heex b/apps/transport/lib/transport_web/templates/resource/_resources_details.html.heex index e09fbfe35d..d505eae14f 100644 --- a/apps/transport/lib/transport_web/templates/resource/_resources_details.html.heex +++ b/apps/transport/lib/transport_web/templates/resource/_resources_details.html.heex @@ -27,16 +27,16 @@ stats = @metadata["stats"] %> <% else %> -
  • +
  • - <%= dngettext("validations", "network", "networks", length(@metadata["networks"])) %> : - <%= Enum.join(@metadata["networks"], ", ") %> + <%= dngettext("validations", "network", "networks", length(networks)) %> : + <%= Enum.join(networks, ", ") %>
    • <% end %> -
  • 0}> +
  • 0}> <%= dngettext("validations", "transport mode", "transport modes", length(@modes)) %> : <%= Enum.join(@modes, ", ") %>
    • diff --git a/apps/transport/lib/transport_web/templates/resource/_resources_details_netex.html.heex b/apps/transport/lib/transport_web/templates/resource/_resources_details_netex.html.heex new file mode 100644 index 0000000000..e0e40c96c8 --- /dev/null +++ b/apps/transport/lib/transport_web/templates/resource/_resources_details_netex.html.heex @@ -0,0 +1,10 @@ +<% locale = get_session(@conn, :locale) %> +<% duration = + if @metadata["elapsed_seconds"] < 1 do + dgettext("validations", "less than 1 second") + else + format_duration(@metadata["elapsed_seconds"], locale) + end %> +
    + <%= dgettext("validations", "Elapsed time: %{duration}.", duration: duration) %> +
    diff --git a/apps/transport/lib/transport_web/templates/resource/_validation_summary.html.heex b/apps/transport/lib/transport_web/templates/resource/_validation_summary.html.heex index b46707a082..cea0917ff0 100644 --- a/apps/transport/lib/transport_web/templates/resource/_validation_summary.html.heex +++ b/apps/transport/lib/transport_web/templates/resource/_validation_summary.html.heex @@ -3,7 +3,7 @@ <%= for {severity, issues} <- @validation_summary do %> <%= if Map.get(@severities_count, severity, 0) > 0 do %>
    -

    <%= @severities_count[severity] %> <%= severity(severity).text %>

    +

    <%= @severities_count[severity] %> <%= @validator.severity(severity).text %>

      <%= for {key, issue} <- issues do %>
    • @@ -12,7 +12,7 @@ "#{issue.title} (#{issue.count})", to: "#{current_url(@conn, %{"issue_type" => key, "token" => @token} |> Map.reject(fn {_, v} -> is_nil(v) end))}#issues", - class: if(key == issue_type(@issues.entries), do: "active") + class: if(key == @validator.issue_type(@issues.entries), do: "active") ) %> <% end %>
      • diff --git a/apps/transport/lib/transport_web/templates/resource/gtfs_details.html.heex b/apps/transport/lib/transport_web/templates/resource/gtfs_details.html.heex index b7099eaf3a..c49e46183f 100644 --- a/apps/transport/lib/transport_web/templates/resource/gtfs_details.html.heex +++ b/apps/transport/lib/transport_web/templates/resource/gtfs_details.html.heex @@ -108,7 +108,7 @@ associated_netex = get_associated_netex(@related_files) %>
      <%= pagination_links(@conn, @issues, [@resource.id], - issue_type: issue_type(@issues.entries), + issue_type: Transport.Validators.GTFSTransport.issue_type(@issues.entries), path: &resource_path/4, action: :details ) %> diff --git a/apps/transport/lib/transport_web/templates/validation/show.html.heex b/apps/transport/lib/transport_web/templates/validation/show_gtfs.html.heex similarity index 90% rename from apps/transport/lib/transport_web/templates/validation/show.html.heex rename to apps/transport/lib/transport_web/templates/validation/show_gtfs.html.heex index d3ad86ac72..cffc5b678d 100644 --- a/apps/transport/lib/transport_web/templates/validation/show.html.heex +++ b/apps/transport/lib/transport_web/templates/validation/show_gtfs.html.heex @@ -3,7 +3,7 @@

      <%= dgettext("validations", "GTFS review report") %>

      - <%= dgettext("validations", "explanations") %> + <%= dgettext("validations", "explanations-gtfs") %>

      <%= dgettext("validations", "This report can be shared with") %> @@ -27,7 +27,8 @@ conn: @conn, issues: @issues, data_vis: @data_vis, - token: @token + token: @token, + validator: @validator ) %> <% end %> @@ -35,7 +36,7 @@

      <%= if has_errors?(@validation_summary) do %> <%= pagination_links(@conn, @issues, [@validation_id], - issue_type: issue_type(@issues.entries), + issue_type: @validator.issue_type(@issues.entries), token: @token, path: &validation_path/4, action: :show @@ -43,7 +44,7 @@ <%= render(gtfs_template(@issues), issues: @issues || [], conn: @conn) %>
      <%= pagination_links(@conn, @issues, [@validation_id], - issue_type: issue_type(@issues.entries), + issue_type: @validator.issue_type(@issues.entries), token: @token, path: &validation_path/4, action: :show diff --git a/apps/transport/lib/transport_web/templates/validation/show_netex.html.heex b/apps/transport/lib/transport_web/templates/validation/show_netex.html.heex new file mode 100644 index 0000000000..3862e57783 --- /dev/null +++ b/apps/transport/lib/transport_web/templates/validation/show_netex.html.heex @@ -0,0 +1,70 @@ +
      +
      +
      +

      <%= dgettext("validations", "NeTEx review report") %>

      +

      + <%= dgettext("validations", "explanations-netex") %> +

      +

      + <%= dgettext("validations", "This report can be shared with") %> + <%= link(dgettext("validations", "this permanent link"), to: current_url(@conn)) %>. +

      +
      + + <%= unless is_nil(@metadata) or @metadata == %{} do %> + <%= render("_resources_details_netex.html", metadata: @metadata, conn: @conn) %> + <% end %> +
      + +
      +
      +

      + <%= dgettext("validations", "NeTEx validation is in beta.") %>
      + <%= dgettext("validations", "Warnings are work in progress. Only XSD errors are considered for now.") %> +

      + + <%= if has_errors?(@validation_summary) do %> + <%= render("_validation_summary.html", + validation_summary: @validation_summary, + severities_count: @severities_count, + conn: @conn, + issues: @issues, + data_vis: @data_vis, + token: @token, + validator: @validator + ) %> + <% end %> + +
      +
      + <%= if has_errors?(@validation_summary) do %> + <%= pagination_links(@conn, @issues, [@validation_id], + issue_type: @validator.issue_type(@issues.entries), + token: @token, + path: &validation_path/4, + action: :show + ) %> + <%= render(netex_template(@issues), issues: @issues || [], conn: @conn) %> +
      + <%= pagination_links(@conn, @issues, [@validation_id], + issue_type: @validator.issue_type(@issues.entries), + token: @token, + path: &validation_path/4, + action: :show + ) %> +
      + <% else %> +

      <%= dgettext("validations", "Nice work, there are no issues!") %>

      + <% end %> +
      +
      +
      +
      +
      + +
      + <%= live_render(@conn, TransportWeb.Live.FeedbackLive, session: %{"feature" => "on_demand_validation"}) %> +
      + + diff --git a/apps/transport/lib/transport_web/views/resource_view.ex b/apps/transport/lib/transport_web/views/resource_view.ex index 5d1514884e..99d77602b9 100644 --- a/apps/transport/lib/transport_web/views/resource_view.ex +++ b/apps/transport/lib/transport_web/views/resource_view.ex @@ -2,14 +2,13 @@ defmodule TransportWeb.ResourceView do use TransportWeb, :view use Phoenix.Component import TransportWeb.PaginationHelpers - import Transport.Validators.GTFSTransport import Phoenix.Controller, only: [current_url: 2] import TransportWeb.DatasetView, only: [documentation_url: 1, errors_count: 1, warnings_count: 1, multi_validation_performed?: 1, description: 1] import DB.ResourceUnavailability, only: [floor_float: 2] - import Shared.DateTimeDisplay, only: [format_datetime_to_paris: 2] + import Shared.DateTimeDisplay, only: [format_datetime_to_paris: 2, format_duration: 2] import Shared.Validation.TableSchemaValidator, only: [validata_web_url: 1] import Transport.GBFSUtils, only: [gbfs_validation_link: 1] import Transport.Shared.Schemas.Wrapper, only: [schema_type: 1] @@ -20,9 +19,6 @@ defmodule TransportWeb.ResourceView do for %{"id" => id, "name" => name} <- related_objects, do: content_tag(:li, "#{name} (#{id})") end - def issue_type([]), do: nil - def issue_type([h | _]), do: h["issue_type"] - def gtfs_template(issues) do template = Map.get( @@ -46,13 +42,19 @@ defmodule TransportWeb.ResourceView do "SubFolder" => "_subfolder_issue.html", "NegativeStopDuration" => "_negative_stop_duration_issue.html" }, - issue_type(issues.entries), + Transport.Validators.GTFSTransport.issue_type(issues.entries), "_generic_issue.html" ) "_gtfs#{template}" end + def netex_template(_issues) do + # For now only 1 template has been designed. More to come when the validator + # has matured. + "_netex_generic_issue.html" + end + def has_associated_files(%{} = resources_related_files, resource_id) do # Don't keep records looking like `%{79088 => %{GeoJSON: nil, NeTEx: nil}}` resource_ids = diff --git a/apps/transport/lib/transport_web/views/validation_view.ex b/apps/transport/lib/transport_web/views/validation_view.ex index 1ac865d3f3..4bf7958bd2 100644 --- a/apps/transport/lib/transport_web/views/validation_view.ex +++ b/apps/transport/lib/transport_web/views/validation_view.ex @@ -1,7 +1,7 @@ defmodule TransportWeb.ValidationView do use TransportWeb, :view import Phoenix.Controller, only: [current_url: 1] - import TransportWeb.ResourceView, only: [issue_type: 1, gtfs_template: 1] + import TransportWeb.ResourceView, only: [gtfs_template: 1, netex_template: 1] import TransportWeb.PaginationHelpers def render("_" <> _ = partial, assigns) do diff --git a/apps/transport/lib/validators/gtfs_transport_validator.ex b/apps/transport/lib/validators/gtfs_transport_validator.ex index 421d6e4914..66f37f7933 100644 --- a/apps/transport/lib/validators/gtfs_transport_validator.ex +++ b/apps/transport/lib/validators/gtfs_transport_validator.ex @@ -77,6 +77,10 @@ defmodule Transport.Validators.GTFSTransport do @spec severity(binary()) :: %{level: integer(), text: binary()} def severity(key), do: severities_map()[key] + @spec issue_type(list()) :: nil | binary() + def issue_type([]), do: nil + def issue_type([h | _]), do: h["issue_type"] + @doc """ Get issues from validation results. For a specific issue type if specified, or the most severe. diff --git a/apps/transport/lib/validators/netex_validator.ex b/apps/transport/lib/validators/netex_validator.ex index 9e9d05daf1..cc7bb0fa9d 100644 --- a/apps/transport/lib/validators/netex_validator.ex +++ b/apps/transport/lib/validators/netex_validator.ex @@ -11,6 +11,8 @@ defmodule Transport.Validators.NeTEx do @max_retries 100 + @unknown_code "unknown-code" + @behaviour Transport.Validators.Validator @impl Transport.Validators.Validator @@ -257,7 +259,7 @@ defmodule Transport.Validators.NeTEx do def index_messages(messages), do: Enum.group_by(messages, &get_code/1) defp get_code(%{"code" => code}), do: code - defp get_code(%{}), do: "unknown-code" + defp get_code(%{}), do: @unknown_code # This will change with an actual versioning of the validator def validator_version, do: "saas-production" @@ -282,14 +284,23 @@ defmodule Transport.Validators.NeTEx do {code, %{ count: length(errors), - criticity: Map.get(hd(errors), "criticity"), - title: Map.get(issues_short_translation(), code, code) + criticity: errors |> hd() |> Map.get("criticity"), + title: issues_short_translation_per_code(code) }} end) |> Enum.group_by(fn {_, details} -> details.criticity end) |> Enum.sort_by(fn {criticity, _} -> severity(criticity).level end) end + @spec issues_short_translation_per_code(binary()) :: binary() + def issues_short_translation_per_code(code) do + if String.starts_with?(code, "xsd-") do + dgettext("netex-validator", "XSD validation") + else + Map.get(issues_short_translation(), code, code) + end + end + @spec issues_short_translation() :: %{binary() => binary()} def issues_short_translation, do: %{ @@ -304,9 +315,13 @@ defmodule Transport.Validators.NeTEx do "uic-operating-period" => dgettext("netex-validator", "UIC operating period"), "valid-day-bits" => dgettext("netex-validator", "Valid day bits"), "version-any" => dgettext("netex-validator", "Version any"), - "unknown-code" => dgettext("netex-validator", "Unspecified error") + @unknown_code => dgettext("netex-validator", "Unspecified error") } + @spec issue_type(list()) :: nil | binary() + def issue_type([]), do: nil + def issue_type([h | _]), do: h["code"] || @unknown_code + @doc """ Get issues from validation results. For a specific issue type if specified, or the most severe. diff --git a/apps/transport/priv/gettext/en/LC_MESSAGES/netex-validator.po b/apps/transport/priv/gettext/en/LC_MESSAGES/netex-validator.po index 9d2dc078e6..a6e706cf3b 100644 --- a/apps/transport/priv/gettext/en/LC_MESSAGES/netex-validator.po +++ b/apps/transport/priv/gettext/en/LC_MESSAGES/netex-validator.po @@ -55,6 +55,10 @@ msgstr "" msgid "Version any" msgstr "" +#, elixir-autogen, elixir-format +msgid "Unspecified error" +msgstr "" + #, elixir-autogen, elixir-format msgid "errors" msgstr "" @@ -68,5 +72,5 @@ msgid "warnings" msgstr "" #, elixir-autogen, elixir-format -msgid "Unspecified error" +msgid "XSD validation" msgstr "" diff --git a/apps/transport/priv/gettext/en/LC_MESSAGES/validations-explanations.po b/apps/transport/priv/gettext/en/LC_MESSAGES/validations-explanations.po index f2a23dd45a..094412efc1 100644 --- a/apps/transport/priv/gettext/en/LC_MESSAGES/validations-explanations.po +++ b/apps/transport/priv/gettext/en/LC_MESSAGES/validations-explanations.po @@ -130,3 +130,19 @@ msgstr ".txt files are in subfolder (%{folder}). Files must reside at the root l #, elixir-autogen, elixir-format msgid "NegativeStopDuration" msgstr "The departure time at a stop is earlier than its arrival time." + +#, elixir-autogen, elixir-format +msgid "Location" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Message" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Unknown location" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Details for debugging purposes" +msgstr "" diff --git a/apps/transport/priv/gettext/en/LC_MESSAGES/validations.po b/apps/transport/priv/gettext/en/LC_MESSAGES/validations.po index 48d61d1b08..7855b78a86 100644 --- a/apps/transport/priv/gettext/en/LC_MESSAGES/validations.po +++ b/apps/transport/priv/gettext/en/LC_MESSAGES/validations.po @@ -75,10 +75,15 @@ msgid "to" msgstr "" #, elixir-autogen, elixir-format -msgid "explanations" -msgstr "This is the automated evalation of the GTFS datafile. " +msgid "explanations-gtfs" +msgstr "This is the automated evaluation of the GTFS datafile. " "This validation only considers the datastructure and does not give any information about completeness nor exactitude of the timetables. " +#, elixir-autogen, elixir-format +msgid "explanations-netex" +msgstr "This is the automated evaluation of the NeTEx datafile. " +"This validation only considers the datastructure and does not give any information about completeness nor exactitude of the data. " + #, elixir-autogen, elixir-format msgid "Show anyway" msgstr "" @@ -277,14 +282,6 @@ msgstr "" msgid "from" msgstr "" -#, elixir-autogen, elixir-format -msgid "doc-eu-formats" -msgstr "To access information related to European standards, you can browse our documentation." - -#, elixir-autogen, elixir-format -msgid "netex-siri-validator" -msgstr "The European program DATA4PT is in charge of deploying the NeTEx and SIRI standards. They publish a NeTEx validator available online." - #, elixir-autogen, elixir-format msgid "%{count} times (%{percentage} % of validations)" msgstr "" @@ -412,3 +409,23 @@ msgstr "" #, elixir-autogen, elixir-format msgid "The GTFS file %{gtfs_original_file_name_2} has differences with the GTFS file %{gtfs_original_file_name_1}, as summarized below:" msgstr "" + +#, elixir-autogen, elixir-format, fuzzy +msgid "NeTEx review report" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Elapsed time: %{duration}." +msgstr "" + +#, elixir-autogen, elixir-format +msgid "NeTEx validation is in beta." +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Warnings are work in progress. Only XSD errors are considered for now." +msgstr "" + +#, elixir-autogen, elixir-format +msgid "less than 1 second" +msgstr "" diff --git a/apps/transport/priv/gettext/fr/LC_MESSAGES/netex-validator.po b/apps/transport/priv/gettext/fr/LC_MESSAGES/netex-validator.po index 7b30b0cebd..abcf16b526 100644 --- a/apps/transport/priv/gettext/fr/LC_MESSAGES/netex-validator.po +++ b/apps/transport/priv/gettext/fr/LC_MESSAGES/netex-validator.po @@ -55,6 +55,10 @@ msgstr "" msgid "Version any" msgstr "" +#, elixir-autogen, elixir-format +msgid "Unspecified error" +msgstr "Erreur inconnue" + #, elixir-autogen, elixir-format msgid "errors" msgstr "erreurs" @@ -68,5 +72,5 @@ msgid "warnings" msgstr "avertissements" #, elixir-autogen, elixir-format -msgid "Unspecified error" -msgstr "" +msgid "XSD validation" +msgstr "Validation XSD" diff --git a/apps/transport/priv/gettext/fr/LC_MESSAGES/validations-explanations.po b/apps/transport/priv/gettext/fr/LC_MESSAGES/validations-explanations.po index cc18b6db11..55adbefc56 100644 --- a/apps/transport/priv/gettext/fr/LC_MESSAGES/validations-explanations.po +++ b/apps/transport/priv/gettext/fr/LC_MESSAGES/validations-explanations.po @@ -130,3 +130,19 @@ msgstr "Les fichiers .txt sont dans un sous-dossier (%{folder}). Les fichiers do #, elixir-autogen, elixir-format msgid "NegativeStopDuration" msgstr "L'heure de départ d'un arrêt (departure_time) est antérieure à son heure d'arrivée (arrival_time)." + +#, elixir-autogen, elixir-format +msgid "Location" +msgstr "Emplacement" + +#, elixir-autogen, elixir-format +msgid "Message" +msgstr "Message" + +#, elixir-autogen, elixir-format +msgid "Unknown location" +msgstr "Emplacement inconnu" + +#, elixir-autogen, elixir-format +msgid "Details for debugging purposes" +msgstr "Détails à fin de débogage" diff --git a/apps/transport/priv/gettext/fr/LC_MESSAGES/validations.po b/apps/transport/priv/gettext/fr/LC_MESSAGES/validations.po index 7d5294a072..133abd1a25 100644 --- a/apps/transport/priv/gettext/fr/LC_MESSAGES/validations.po +++ b/apps/transport/priv/gettext/fr/LC_MESSAGES/validations.po @@ -75,10 +75,15 @@ msgid "to" msgstr "au" #, elixir-autogen, elixir-format -msgid "explanations" +msgid "explanations-gtfs" msgstr "Voici le bilan automatisé sur la qualité du fichier GTFS. " "Cette validation ne concerne que la structure des données et pas la complétude ni l’exactitude des horaires. " +#, elixir-autogen, elixir-format +msgid "explanations-netex" +msgstr "Voici le bilan automatisé sur la qualité du fichier NeTEx. " +"Cette validation ne concerne que la structure des données et pas la complétude ni l’exactitude des données. " + #, elixir-autogen, elixir-format msgid "Show anyway" msgstr "Afficher quand même" @@ -277,14 +282,6 @@ msgstr "Validation effectuée en utilisant le fichier G msgid "from" msgstr "du" -#, elixir-autogen, elixir-format -msgid "doc-eu-formats" -msgstr "Pour accéder aux ressources autour des normes européennes, vous pouvez consulter notre documentation." - -#, elixir-autogen, elixir-format -msgid "netex-siri-validator" -msgstr "Le projet européen DATA4PT assure aujourd'hui l'accompagnement au déploiement des normes européennes NeTEx et SIRI. Dans ce cadre, il est mis à disposition un outil de validation NeTEx accessible en ligne." - #, elixir-autogen, elixir-format msgid "%{count} times (%{percentage} % of validations)" msgstr "%{count} fois (%{percentage} % des validations)" @@ -412,3 +409,23 @@ msgstr "nombre d'arrêts :" #, elixir-autogen, elixir-format msgid "The GTFS file %{gtfs_original_file_name_2} has differences with the GTFS file %{gtfs_original_file_name_1}, as summarized below:" msgstr "Le fichier GTFS %{gtfs_original_file_name_2} comporte les différences ci-dessous par rapport au fichier GTFS %{gtfs_original_file_name_1} :" + +#, elixir-autogen, elixir-format, fuzzy +msgid "NeTEx review report" +msgstr "Rapport de l’analyse de fichier NeTEx" + +#, elixir-autogen, elixir-format +msgid "Elapsed time: %{duration}." +msgstr "Temps passé : %{duration}." + +#, elixir-autogen, elixir-format +msgid "NeTEx validation is in beta." +msgstr "La validation NeTEx est en beta." + +#, elixir-autogen, elixir-format +msgid "Warnings are work in progress. Only XSD errors are considered for now." +msgstr "Les avertissements sont temporaires. Seules les erreurs XSD sont prises en considération pour l’instant." + +#, elixir-autogen, elixir-format +msgid "less than 1 second" +msgstr "moins d’une seconde" diff --git a/apps/transport/priv/gettext/netex-validator.pot b/apps/transport/priv/gettext/netex-validator.pot index 82d778082e..669ae75503 100644 --- a/apps/transport/priv/gettext/netex-validator.pot +++ b/apps/transport/priv/gettext/netex-validator.pot @@ -55,6 +55,10 @@ msgstr "" msgid "Version any" msgstr "" +#, elixir-autogen, elixir-format +msgid "Unspecified error" +msgstr "" + #, elixir-autogen, elixir-format msgid "errors" msgstr "" @@ -68,5 +72,5 @@ msgid "warnings" msgstr "" #, elixir-autogen, elixir-format -msgid "Unspecified error" +msgid "XSD validation" msgstr "" diff --git a/apps/transport/priv/gettext/validations-explanations.pot b/apps/transport/priv/gettext/validations-explanations.pot index 0a91a13aa9..e1e89c104f 100644 --- a/apps/transport/priv/gettext/validations-explanations.pot +++ b/apps/transport/priv/gettext/validations-explanations.pot @@ -129,3 +129,19 @@ msgstr "" #, elixir-autogen, elixir-format msgid "NegativeStopDuration" msgstr "" + +#, elixir-autogen, elixir-format +msgid "Location" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Message" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Unknown location" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Details for debugging purposes" +msgstr "" diff --git a/apps/transport/priv/gettext/validations.pot b/apps/transport/priv/gettext/validations.pot index f663383109..15362901e9 100644 --- a/apps/transport/priv/gettext/validations.pot +++ b/apps/transport/priv/gettext/validations.pot @@ -75,7 +75,11 @@ msgid "to" msgstr "" #, elixir-autogen, elixir-format -msgid "explanations" +msgid "explanations-gtfs" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "explanations-netex" msgstr "" #, elixir-autogen, elixir-format @@ -276,14 +280,6 @@ msgstr "" msgid "from" msgstr "" -#, elixir-autogen, elixir-format -msgid "doc-eu-formats" -msgstr "" - -#, elixir-autogen, elixir-format -msgid "netex-siri-validator" -msgstr "" - #, elixir-autogen, elixir-format msgid "%{count} times (%{percentage} % of validations)" msgstr "" @@ -411,3 +407,23 @@ msgstr "" #, elixir-autogen, elixir-format msgid "The GTFS file %{gtfs_original_file_name_2} has differences with the GTFS file %{gtfs_original_file_name_1}, as summarized below:" msgstr "" + +#, elixir-autogen, elixir-format +msgid "NeTEx review report" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Elapsed time: %{duration}." +msgstr "" + +#, elixir-autogen, elixir-format +msgid "NeTEx validation is in beta." +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Warnings are work in progress. Only XSD errors are considered for now." +msgstr "" + +#, elixir-autogen, elixir-format +msgid "less than 1 second" +msgstr "" diff --git a/apps/transport/test/support/enroute_chouette_valid_client_helpers.ex b/apps/transport/test/support/enroute_chouette_valid_client_helpers.ex new file mode 100644 index 0000000000..43112901ac --- /dev/null +++ b/apps/transport/test/support/enroute_chouette_valid_client_helpers.ex @@ -0,0 +1,32 @@ +defmodule Transport.Test.EnRouteChouetteValidClientHelpers do + @moduledoc """ + This module defines helpers to setup a mock enRoute Chouette Valid client. + """ + import Mox + + def expect_create_validation do + validation_id = Ecto.UUID.generate() + expect(Transport.EnRouteChouetteValidClient.Mock, :create_a_validation, fn _ -> validation_id end) + validation_id + end + + def expect_pending_validation(validation_id) do + expect(Transport.EnRouteChouetteValidClient.Mock, :get_a_validation, fn ^validation_id -> :pending end) + end + + def expect_successful_validation(validation_id, elapsed) do + expect(Transport.EnRouteChouetteValidClient.Mock, :get_a_validation, fn ^validation_id -> + {:successful, "http://localhost:9999/chouette-valid/#{validation_id}", elapsed} + end) + end + + def expect_failed_validation(validation_id, elapsed) do + expect(Transport.EnRouteChouetteValidClient.Mock, :get_a_validation, fn ^validation_id -> {:failed, elapsed} end) + end + + def expect_get_messages(validation_id, result) do + expect(Transport.EnRouteChouetteValidClient.Mock, :get_messages, fn ^validation_id -> + {"http://localhost:9999/chouette-valid/#{validation_id}/messages", result} + end) + end +end diff --git a/apps/transport/test/transport/jobs/on_demand_validation_job_test.exs b/apps/transport/test/transport/jobs/on_demand_validation_job_test.exs index 7cfd976fe0..2ed35b6add 100644 --- a/apps/transport/test/transport/jobs/on_demand_validation_job_test.exs +++ b/apps/transport/test/transport/jobs/on_demand_validation_job_test.exs @@ -3,6 +3,7 @@ defmodule Transport.Test.Transport.Jobs.OnDemandValidationJobTest do use Oban.Testing, repo: DB.Repo import DB.Factory import Mox + import Transport.Test.EnRouteChouetteValidClientHelpers import Transport.Test.S3TestUtils alias Transport.Jobs.OnDemandValidationJob alias Transport.Validators.GTFSRT @@ -389,14 +390,68 @@ defmodule Transport.Test.Transport.Jobs.OnDemandValidationJobTest do refute File.exists?(gtfs_rt_path) refute File.exists?(OnDemandValidationJob.gtfs_rt_result_path(gtfs_rt_path)) end + + test "with a NeTEx" do + url = mk_raw_netex_resource() + validation = create_validation(%{"type" => "netex"}, url) + + errors = [ + %{ + "code" => "xsd-1871", + "criticity" => "error", + "message" => + "Element '{http://www.netex.org.uk/netex}OppositeDIrectionRef': This element is not expected. Expected is ( {http://www.netex.org.uk/netex}OppositeDirectionRef )." + }, + %{ + "code" => "uic-operating-period", + "message" => "Resource 23504000009 hasn't expected class but Netex::OperatingPeriod", + "criticity" => "error" + }, + %{ + "code" => "valid-day-bits", + "message" => "Mandatory attribute valid_day_bits not found", + "criticity" => "error" + }, + %{ + "code" => "frame-arret-resources", + "message" => "Tag frame_id doesn't match ''", + "criticity" => "warning" + } + ] + + expect_netex_with_errors(errors) + + s3_mocks_delete_object(Transport.S3.bucket_name(:on_demand_validation), @filename) + + assert :ok == run_job(validation) + + assert %{ + validation_timestamp: date, + result: result, + max_error: "error", + oban_args: %{"state" => "completed", "type" => "netex"}, + metadata: %{}, + data_vis: nil + } = validation |> DB.Repo.reload() |> DB.Repo.preload(:metadata) + + assert %{"xsd-1871" => a1, "uic-operating-period" => a2, "valid-day-bits" => a3, "frame-arret-resources" => a4} = + result + + assert length(a1) == 1 + assert length(a2) == 1 + assert length(a3) == 1 + assert length(a4) == 1 + + assert DateTime.diff(date, DateTime.utc_now()) <= 1 + end end - defp create_validation(details) do + defp create_validation(details, url \\ @url) do details = if details["type"] == "gtfs-rt" do details else - Map.merge(details, %{"filename" => @filename, "permanent_url" => @url}) + Map.merge(details, %{"filename" => @filename, "permanent_url" => url}) end oban_args = Map.merge(%{"state" => "waiting"}, details) @@ -404,10 +459,27 @@ defmodule Transport.Test.Transport.Jobs.OnDemandValidationJobTest do insert(:multi_validation, oban_args: oban_args) end + def expect_netex_with_errors(messages) do + validation_id = expect_create_validation() + expect_failed_validation(validation_id, 10) + + expect_get_messages(validation_id, messages) + end + defp run_job(%DB.MultiValidation{} = validation) do payload = Map.merge(%{"id" => validation.id}, validation.oban_args) perform_job(OnDemandValidationJob, payload) end defp validator_path, do: Path.join(Application.fetch_env!(:transport, :transport_tools_folder), @validator_filename) + + defp mk_raw_netex_resource do + resource_url = "http://localhost:9999/netex-#{Ecto.UUID.generate()}.zip" + + expect(Transport.Req.Mock, :get!, 1, fn ^resource_url, [{:compressed, false}, {:into, _}] -> + {:ok, %Req.Response{status: 200, body: %{"data" => "some_zip_file"}}} + end) + + resource_url + end end diff --git a/apps/transport/test/transport/validators/netex_validator_test.exs b/apps/transport/test/transport/validators/netex_validator_test.exs index 70714a1755..1f0e579deb 100644 --- a/apps/transport/test/transport/validators/netex_validator_test.exs +++ b/apps/transport/test/transport/validators/netex_validator_test.exs @@ -3,6 +3,7 @@ defmodule Transport.Validators.NeTExTest do import DB.Factory import Mox import ExUnit.CaptureLog + import Transport.Test.EnRouteChouetteValidClientHelpers alias Transport.Validators.NeTEx @@ -230,32 +231,6 @@ defmodule Transport.Validators.NeTExTest do end end - defp expect_create_validation do - validation_id = Ecto.UUID.generate() - expect(Transport.EnRouteChouetteValidClient.Mock, :create_a_validation, fn _ -> validation_id end) - validation_id - end - - defp expect_pending_validation(validation_id) do - expect(Transport.EnRouteChouetteValidClient.Mock, :get_a_validation, fn ^validation_id -> :pending end) - end - - defp expect_successful_validation(validation_id, elapsed) do - expect(Transport.EnRouteChouetteValidClient.Mock, :get_a_validation, fn ^validation_id -> - {:successful, "http://localhost:9999/chouette-valid/#{validation_id}", elapsed} - end) - end - - defp expect_failed_validation(validation_id, elapsed) do - expect(Transport.EnRouteChouetteValidClient.Mock, :get_a_validation, fn ^validation_id -> {:failed, elapsed} end) - end - - defp expect_get_messages(validation_id, result) do - expect(Transport.EnRouteChouetteValidClient.Mock, :get_messages, fn ^validation_id -> - {"http://localhost:9999/chouette-valid/#{validation_id}/messages", result} - end) - end - defp mk_netex_resource do dataset = insert(:dataset) diff --git a/apps/transport/test/transport_web/controllers/validation_controller_test.exs b/apps/transport/test/transport_web/controllers/validation_controller_test.exs index 7f699cfb06..7d5a6df586 100644 --- a/apps/transport/test/transport_web/controllers/validation_controller_test.exs +++ b/apps/transport/test/transport_web/controllers/validation_controller_test.exs @@ -39,21 +39,14 @@ defmodule TransportWeb.ValidationControllerTest do refute view |> has_element?("input[name='upload[url]']") assert view |> has_element?("input[name='upload[file]']") - render_change(view, "form_changed", %{"upload" => %{"type" => "gtfs-rt"}, "_target" => ["upload", "type"]}) - assert view |> has_element?("input[name='upload[url]']") - assert view |> has_element?("input[name='upload[feed_url]']") - end - - test "no inputs but explanations when selecting NeTEx", %{conn: conn} do - Transport.Shared.Schemas.Mock |> expect(:transport_schemas, 2, fn -> %{} end) - {:ok, view, _html} = conn |> get(live_path(conn, OnDemandValidationSelectLive)) |> live() - render_change(view, "form_changed", %{"upload" => %{"type" => "netex"}, "_target" => ["upload", "type"]}) assert_patched(view, live_path(conn, OnDemandValidationSelectLive, type: "netex")) refute view |> has_element?("input[name='upload[url]']") - refute view |> has_element?("input[name='upload[file]']") - refute view |> has_element?("input[type='submit']") - assert view |> has_element?("#netex-explanations") + assert view |> has_element?("input[name='upload[file]']") + + render_change(view, "form_changed", %{"upload" => %{"type" => "gtfs-rt"}, "_target" => ["upload", "type"]}) + assert view |> has_element?("input[name='upload[url]']") + assert view |> has_element?("input[name='upload[feed_url]']") end test "takes into account query params", %{conn: conn} do diff --git a/mix.lock b/mix.lock index ced627d17f..ee64dd7f20 100644 --- a/mix.lock +++ b/mix.lock @@ -39,8 +39,11 @@ "ex_aws": {:hex, :ex_aws, "2.5.4", "86c5bb870a49e0ab6f5aa5dd58cf505f09d2624ebe17530db3c1b61c88a673af", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e82bd0091bb9a5bb190139599f922ff3fc7aebcca4374d65c99c4e23aa6d1625"}, "ex_aws_s3": {:hex, :ex_aws_s3, "2.5.3", "422468e5c3e1a4da5298e66c3468b465cfd354b842e512cb1f6fbbe4e2f5bdaf", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "4f09dd372cc386550e484808c5ac5027766c8d0cd8271ccc578b82ee6ef4f3b8"}, "ex_cldr": {:hex, :ex_cldr, "2.40.0", "624717778dbf0a8cd307f1576eabbd44470c16190172abf293fed24150440a5a", [:mix], [{:cldr_utils, "~> 2.28", [hex: :cldr_utils, repo: "hexpm", optional: false]}, {:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:gettext, "~> 0.19", [hex: :gettext, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: true]}], "hexpm", "113394b6dd23aaf7912da583aab103d9cf082b9821bc4a6e287543a895af7cb4"}, + "ex_cldr_calendars": {:hex, :ex_cldr_calendars, "1.26.2", "5f15d1673f46fd1d7b0c770bf65b40ce060465328637d24212ec9e79310eca87", [:mix], [{:calendar_interval, "~> 0.2", [hex: :calendar_interval, repo: "hexpm", optional: true]}, {:ex_cldr_lists, "~> 2.10", [hex: :ex_cldr_lists, repo: "hexpm", optional: true]}, {:ex_cldr_numbers, "~> 2.31", [hex: :ex_cldr_numbers, repo: "hexpm", optional: false]}, {:ex_cldr_units, "~> 3.16", [hex: :ex_cldr_units, repo: "hexpm", optional: true]}, {:ex_doc, "~> 0.21", [hex: :ex_doc, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "b689847f3fbbd145954a9205e19b1e4850a79c2a27cdae1c7912b9b262a8ef35"}, "ex_cldr_currencies": {:hex, :ex_cldr_currencies, "2.16.2", "670d96cc4fb18cfebd82488ed687742683be2d0725d66ec051578d4b13539aa8", [:mix], [{:ex_cldr, "~> 2.38", [hex: :ex_cldr, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "2ccfac2838f4df8c8e5424dbc68eb2f3ac9eeb45e10365050901f7ac7a914ce1"}, + "ex_cldr_lists": {:hex, :ex_cldr_lists, "2.11.1", "ad18f861d7c5ca82aac6d173469c6a2339645c96790172ab0aa255b64fb7303b", [:mix], [{:ex_cldr_numbers, "~> 2.25", [hex: :ex_cldr_numbers, repo: "hexpm", optional: false]}, {:ex_doc, "~> 0.18", [hex: :ex_doc, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "00161c04510ccb3f18b19a6b8562e50c21f1e9c15b8ff4c934bea5aad0b4ade2"}, "ex_cldr_numbers": {:hex, :ex_cldr_numbers, "2.33.2", "c5587a8d84214d9cc42e7827e4c3bed2aa9e52505a55b10540020725954ded2c", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:digital_token, "~> 0.3 or ~> 1.0", [hex: :digital_token, repo: "hexpm", optional: false]}, {:ex_cldr, "~> 2.38", [hex: :ex_cldr, repo: "hexpm", optional: false]}, {:ex_cldr_currencies, "~> 2.16", [hex: :ex_cldr_currencies, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "49f1dbaddc1ad6e3f496a97fa425d25b3ae89e8178ce0416d9909deaf2e5ad80"}, + "ex_cldr_units": {:hex, :ex_cldr_units, "3.17.2", "b0483d5c61c6c8649aafdcafc7372dd71a7a30f52dd4c9b072576467bf721454", [:mix], [{:cldr_utils, "~> 2.25", [hex: :cldr_utils, repo: "hexpm", optional: false]}, {:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ex_cldr_lists, "~> 2.10", [hex: :ex_cldr_lists, repo: "hexpm", optional: false]}, {:ex_cldr_numbers, "~> 2.33.0", [hex: :ex_cldr_numbers, repo: "hexpm", optional: false]}, {:ex_doc, "~> 0.18", [hex: :ex_doc, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "457d76c6e3b548bd7aba3c7b5d157213be2842d1162c2283abf81d9e2f1e1fc7"}, "ex_json_schema": {:hex, :ex_json_schema, "0.10.2", "7c4b8c1481fdeb1741e2ce66223976edfb9bccebc8014f6aec35d4efe964fb71", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm", "37f43be60f8407659d4d0155a7e45e7f406dab1f827051d3d35858a709baf6a6"}, "ex_machina": {:hex, :ex_machina, "2.8.0", "a0e847b5712065055ec3255840e2c78ef9366634d62390839d4880483be38abe", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm", "79fe1a9c64c0c1c1fab6c4fa5d871682cb90de5885320c187d117004627a7729"}, "ex_phone_number": {:hex, :ex_phone_number, "0.4.4", "8e994abe583496a3308cf56af013dca9b47a0424b0a9940af41cb0d66b848dd3", [:mix], [{:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: false]}], "hexpm", "a59875692ec57b3392959a7740f3e9a5cb08da88bcaee4efd480c770f5bb0f2c"},