From 616c46a318ceab2a62d93fbb5451b5da432b9e8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Menou?= Date: Thu, 25 Jul 2024 15:24:00 +0200 Subject: [PATCH 1/3] NeTEx validation with enRoute's Chouette --- .../lib/validators/netex_validator.ex | 311 ++++++++++++++++++ .../lib/validators/validator_selection.ex | 1 + .../gettext/en/LC_MESSAGES/netex-validator.po | 68 ++++ .../gettext/fr/LC_MESSAGES/netex-validator.po | 68 ++++ .../priv/gettext/netex-validator.pot | 68 ++++ .../validators/netex_validator_test.exs | 216 ++++++++++++ .../validators/validator_selection_test.exs | 6 + config/config.exs | 3 +- config/test.exs | 3 +- 9 files changed, 742 insertions(+), 2 deletions(-) create mode 100644 apps/transport/lib/validators/netex_validator.ex create mode 100644 apps/transport/priv/gettext/en/LC_MESSAGES/netex-validator.po create mode 100644 apps/transport/priv/gettext/fr/LC_MESSAGES/netex-validator.po create mode 100644 apps/transport/priv/gettext/netex-validator.pot create mode 100644 apps/transport/test/transport/validators/netex_validator_test.exs diff --git a/apps/transport/lib/validators/netex_validator.ex b/apps/transport/lib/validators/netex_validator.ex new file mode 100644 index 0000000000..a9365243fb --- /dev/null +++ b/apps/transport/lib/validators/netex_validator.ex @@ -0,0 +1,311 @@ +defmodule Transport.Validators.NeTEx do + import TransportWeb.Gettext, only: [dgettext: 2] + + @moduledoc """ + Validator for NeTEx files calling enRoute Chouette Valid API. + """ + require Logger + @behaviour Transport.Validators.Validator + + @impl Transport.Validators.Validator + def validator_name, do: "enroute-chouette-validator" + + @impl Transport.Validators.Validator + def validate_and_save(%DB.Resource{format: "NeTEx", id: resource_id}) do + Logger.info("Validating #{resource_id} with enRoute Chouette Valid") + + resource_history = DB.ResourceHistory.latest_resource_history(resource_id) + with_resource_file(resource_history, &validate_resource_history(resource_history, &1)) + end + + def validate_resource_history(resource_history, filepath) do + case validate_with_enroute(filepath) do + {:ok, result_url} -> + insert_validation_results(resource_history.id, result_url) + :ok + + {:error, {result_url, errors}} -> + insert_validation_results(resource_history.id, result_url, errors) + :ok + + {:error, :unexpected_validation_status} -> + Logger.error("Invalid API call to enRoute Chouette Valid") + {:error, "enRoute Chouette Valid: Unexpected validation status"} + end + end + + @type validate_options :: [{:graceful_retry, boolean()}] + + @doc """ + Validate the resource from the given URL. + """ + @spec validate(binary(), validate_options()) :: {:ok, map()} | {:error, binary()} + def validate(url, opts \\ []) do + with_url(url, fn filepath -> + case validate_with_enroute(filepath, opts) do + {:ok, result_url} -> + # result_url in metadata? + Logger.info("Result URL: #{result_url}") + {:ok, %{"validations" => index_messages([]), "metadata" => %{}}} + + {:error, {result_url, errors}} -> + Logger.info("Result URL: #{result_url}") + # result_url in metadata? + {:ok, %{"validations" => index_messages(errors), "metadata" => %{}}} + + {:error, :unexpected_validation_status} -> + Logger.error("Invalid API call to enRoute Chouette Valid") + {:error, "enRoute Chouette Valid: Unexpected validation status"} + end + end) + end + + @spec with_resource_file(ResourceHistory.t(), (Path.t() -> any())) :: any() + def with_resource_file(resource_history, closure) do + %DB.ResourceHistory{payload: %{"permanent_url" => permanent_url}} = resource_history + filepath = tmp_path(resource_history) + + with_temp_file(permanent_url, filepath, closure) + end + + @spec with_url(binary(), (Path.t() -> any())) :: any() + def with_url(url, closure) do + with_temp_file(url, tmp_path(url), closure) + end + + @spec with_temp_file(binary(), Path.t(), (Path.t() -> any())) :: any() + def with_temp_file(url, filepath, closure) do + http_client().get!(url, compressed: false, into: File.stream!(filepath)) + closure.(filepath) + after + File.rm(filepath) + end + + defp http_client, do: Transport.Req.impl() + + defp tmp_path(%DB.ResourceHistory{id: resource_history_id}) do + System.tmp_dir!() |> Path.join("enroute_validation_netex_#{resource_history_id}") + end + + defp tmp_path(_other) do + System.tmp_dir!() |> Path.join("enroute_validation_netex_#{Ecto.UUID.generate()}") + end + + def insert_validation_results(resource_history_id, result_url, errors \\ []) do + result = index_messages(errors) + + %DB.MultiValidation{ + validation_timestamp: DateTime.utc_now(), + validator: validator_name(), + result: result, + resource_history_id: resource_history_id, + validator_version: validator_version(), + command: result_url, + max_error: get_max_severity_error(result) + } + |> DB.Repo.insert!() + end + + @doc """ + Returns the maximum issue severity found + + iex> validation_result = %{"uic-operating-period" => [%{"criticity" => "error"}], "valid-day-bits" => [%{"criticity" => "error"}], "frame-arret-resources" => [%{"criticity" => "error"}]} + iex> get_max_severity_error(validation_result) + "error" + + iex> get_max_severity_error(%{}) + "NoError" + """ + @spec get_max_severity_error(map()) :: binary() | nil + def get_max_severity_error(validation_result) do + {severity, _} = validation_result |> count_max_severity() + severity + end + + @no_error "NoError" + + @doc """ + Returns the maximum severity, with the issues count + + iex> validation_result = %{"uic-operating-period" => [%{"criticity" => "error"}], "valid-day-bits" => [%{"criticity" => "error"}], "frame-arret-resources" => [%{"criticity" => "error"}]} + iex> count_max_severity(validation_result) + {"error", 3} + iex> count_max_severity(%{}) + {"NoError", 0} + """ + @spec count_max_severity(map()) :: {binary(), integer()} + def count_max_severity(validation_result) when validation_result == %{} do + {@no_error, 0} + end + + def count_max_severity(%{} = validation_result) do + validation_result + |> count_by_severity() + |> Enum.min_by(fn {severity, _count} -> severity |> severities() |> Map.get(:level) end) + end + + @spec severities_map() :: map() + def severities_map, + do: %{ + "error" => %{level: 1, text: dgettext("netex-validator", "errors")}, + "warning" => %{level: 2, text: dgettext("netex-validator", "warnings")}, + "information" => %{level: 3, text: dgettext("netex-validator", "informations")} + } + + @spec severities(binary()) :: %{level: integer(), text: binary()} + def severities(key), do: severities_map()[key] + + @doc """ + Returns the number of issues by severity level + + iex> validation_result = %{"uic-operating-period" => [%{"criticity" => "warning"}], "valid-day-bits" => [%{"criticity" => "error"}], "frame-arret-resources" => [%{"criticity" => "error"}]} + iex> count_by_severity(validation_result) + %{"warning" => 1, "error" => 2} + + iex> count_by_severity(%{}) + %{} + """ + @spec count_by_severity(map()) :: map() + def count_by_severity(%{} = validation_result) do + validation_result + |> Enum.flat_map(fn {_, v} -> v end) + |> Enum.reduce(%{}, fn v, acc -> Map.update(acc, v["criticity"], 1, &(&1 + 1)) end) + end + + def count_by_severity(_), do: %{} + + defp validate_with_enroute(filepath, opts \\ []) do + client().create_a_validation(filepath) |> fetch_validation_results(0, opts) + end + + defp fetch_validation_results(validation_id, retries, opts) do + case client().get_a_validation(validation_id) do + :pending -> + if Keyword.get(opts, :graceful_retry, true) do + retries |> poll_interval() |> :timer.sleep() + end + + fetch_validation_results(validation_id, retries + 1, opts) + + {:successful, url} -> + {:ok, url} + + value when value in [:warning, :failed] -> + {:error, client().get_messages(validation_id)} + + :unexpected_validation_status -> + {:error, :unexpected_validation_status} + end + end + + @doc """ + iex> index_messages([]) + %{} + + iex> index_messages([%{"code"=>"a", "id"=> 1}, %{"code"=>"a", "id"=> 2}, %{"code"=>"b", "id"=> 3}]) + %{"a"=>[%{"code"=>"a", "id"=> 1}, %{"code"=>"a", "id"=> 2}], "b"=>[%{"code"=>"b", "id"=> 3}]} + """ + def index_messages(messages) do + messages |> Enum.group_by(fn %{"code" => code} -> code end) + end + + @doc """ + Poll interval to play nice with the tier. + + iex> 0..9 |> Enum.map(&poll_interval(&1)) + [10000, 10000, 10000, 10000, 10000, 10000, 20000, 20000, 20000, 20000] + """ + def poll_interval(nb_tries) when nb_tries < 6, do: 10_000 + def poll_interval(_), do: 20_000 + + # This will change with an actual versioning of the validator + def validator_version, do: "saas-production" + + @doc """ + iex> validation_result = %{"uic-operating-period" => [%{"code" => "uic-operating-period", "message" => "Resource 23504000009 hasn't expected class but Netex::OperatingPeriod", "criticity" => "error"}], "valid-day-bits" => [%{"code" => "valid-day-bits", "message" => "Mandatory attribute valid_day_bits not found", "criticity" => "error"}], "frame-arret-resources" => [%{"code" => "frame-arret-resources", "message" => "Tag frame_id doesn't match ''", "criticity" => "warning"}]} + iex> summary(validation_result) + [ + {"error", [ + {"uic-operating-period", %{count: 1, criticity: "error", title: "UIC operating period"}}, + {"valid-day-bits", %{count: 1, criticity: "error", title: "Valid day bits"}} + ]}, + {"warning", [{"frame-arret-resources", %{count: 1, criticity: "warning", title: "Frame arret resources"}}]} + ] + iex> summary(%{}) + [] + """ + @spec summary(map()) :: list() + def summary(%{} = validation_result) do + validation_result + |> Enum.map(fn {code, errors} -> + {code, + %{ + count: length(errors), + criticity: Map.get(hd(errors), "criticity"), + title: Map.get(issues_short_translation(), code, code) + }} + end) + |> Enum.group_by(fn {_, details} -> details.criticity end) + |> Enum.sort_by(fn {criticity, _} -> severities(criticity).level end) + end + + @spec issues_short_translation() :: %{binary() => binary()} + def issues_short_translation, + do: %{ + "composite-frame-ligne-mandatory" => dgettext("netex-validator", "Composite frame ligne mandatory"), + "frame-arret-resources" => dgettext("netex-validator", "Frame arret resources"), + "frame-calendrier-resources" => dgettext("netex-validator", "Frame calendrier resources"), + "frame-horaire-resources" => dgettext("netex-validator", "Frame horaire resources"), + "frame-ligne-resources" => dgettext("netex-validator", "Frame ligne resources"), + "frame-reseau-resources" => dgettext("netex-validator", "Frame reseau resources"), + "latitude-mandatory" => dgettext("netex-validator", "Latitude mandatory"), + "longitude-mandatory" => dgettext("netex-validator", "Longitude mandatory"), + "uic-operating-period" => dgettext("netex-validator", "UIC operating period"), + "valid-day-bits" => dgettext("netex-validator", "Valid day bits"), + "version-any" => dgettext("netex-validator", "Version any") + } + + @doc """ + Get issues from validation results. For a specific issue type if specified, or the most severe. + + iex> validation_result = %{"uic-operating-period" => [%{"code" => "uic-operating-period", "message" => "Resource 23504000009 hasn't expected class but Netex::OperatingPeriod", "criticity" => "error"}], "valid-day-bits" => [%{"code" => "valid-day-bits", "message" => "Mandatory attribute valid_day_bits not found", "criticity" => "error"}], "frame-arret-resources" => [%{"code" => "frame-arret-resources", "message" => "Tag frame_id doesn't match ''", "criticity" => "warning"}]} + iex> get_issues(validation_result, %{"issue_type" => "uic-operating-period"}) + [%{"code" => "uic-operating-period", "message" => "Resource 23504000009 hasn't expected class but Netex::OperatingPeriod", "criticity" => "error"}] + iex> get_issues(validation_result, %{"issue_type" => "broken-file"}) + [] + iex> get_issues(validation_result, nil) + [%{"code" => "uic-operating-period", "message" => "Resource 23504000009 hasn't expected class but Netex::OperatingPeriod", "criticity" => "error"}] + iex> get_issues(%{}, nil) + [] + iex> get_issues([], nil) + [] + """ + def get_issues(%{} = validation_result, %{"issue_type" => issue_type}) do + Map.get(validation_result, issue_type, []) |> order_issues_by_location() + end + + def get_issues(%{} = validation_result, _) do + validation_result + |> Map.values() + |> Enum.sort_by(fn [%{"criticity" => severity} | _] -> severities(severity).level end) + |> List.first([]) + |> order_issues_by_location() + end + + def get_issues(_, _), do: [] + + def order_issues_by_location(issues) do + issues + |> Enum.sort_by(fn issue -> + message = Map.get(issue, "message", "") + resource = Map.get(issue, "resource", %{}) + filename = Map.get(resource, "filename", "") + line = Map.get(resource, "line", "") + {filename, line, message} + end) + end + + defp client do + Transport.EnRouteChouetteValidClient.Wrapper.impl() + end +end diff --git a/apps/transport/lib/validators/validator_selection.ex b/apps/transport/lib/validators/validator_selection.ex index 3478275e49..a5ffd7fc6b 100644 --- a/apps/transport/lib/validators/validator_selection.ex +++ b/apps/transport/lib/validators/validator_selection.ex @@ -33,6 +33,7 @@ defmodule Transport.ValidatorsSelection.Impl do def validators(%{format: "GTFS"}), do: [Validators.GTFSTransport] def validators(%{format: "gtfs-rt"}), do: [Validators.GTFSRT] def validators(%{format: "gbfs"}), do: [Validators.GBFSValidator] + def validators(%{format: "NeTEx"}), do: [Validators.NeTEx] def validators(%{schema_name: schema_name}) when not is_nil(schema_name) do cond do diff --git a/apps/transport/priv/gettext/en/LC_MESSAGES/netex-validator.po b/apps/transport/priv/gettext/en/LC_MESSAGES/netex-validator.po new file mode 100644 index 0000000000..8fefa80452 --- /dev/null +++ b/apps/transport/priv/gettext/en/LC_MESSAGES/netex-validator.po @@ -0,0 +1,68 @@ +## "msgid"s in this file come from POT (.pot) files. +### +### Do not add, change, or remove "msgid"s manually here as +### they're tied to the ones in the corresponding POT file +### (with the same domain). +### +### Use "mix gettext.extract --merge" or "mix gettext.merge" +### to merge POT files into PO files. +msgid "" +msgstr "" +"Language: en\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +#, elixir-autogen, elixir-format +msgid "Composite frame ligne mandatory" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Frame arret resources" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Frame calendrier resources" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Frame horaire resources" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Frame ligne resources" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Frame reseau resources" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Latitude mandatory" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Longitude mandatory" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "UIC operating period" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Valid day bits" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Version any" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "errors" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "informations" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "warnings" +msgstr "" diff --git a/apps/transport/priv/gettext/fr/LC_MESSAGES/netex-validator.po b/apps/transport/priv/gettext/fr/LC_MESSAGES/netex-validator.po new file mode 100644 index 0000000000..1886f4e5d8 --- /dev/null +++ b/apps/transport/priv/gettext/fr/LC_MESSAGES/netex-validator.po @@ -0,0 +1,68 @@ +## "msgid"s in this file come from POT (.pot) files. +### +### Do not add, change, or remove "msgid"s manually here as +### they're tied to the ones in the corresponding POT file +### (with the same domain). +### +### Use "mix gettext.extract --merge" or "mix gettext.merge" +### to merge POT files into PO files. +msgid "" +msgstr "" +"Language: fr\n" +"Plural-Forms: nplurals=2; plural=(n>1);\n" + +#, elixir-autogen, elixir-format +msgid "Composite frame ligne mandatory" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Frame arret resources" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Frame calendrier resources" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Frame horaire resources" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Frame ligne resources" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Frame reseau resources" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Latitude mandatory" +msgstr "Latitude requise" + +#, elixir-autogen, elixir-format +msgid "Longitude mandatory" +msgstr "Longitude requise" + +#, elixir-autogen, elixir-format +msgid "UIC operating period" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Valid day bits" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Version any" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "errors" +msgstr "erreurs" + +#, elixir-autogen, elixir-format +msgid "informations" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "warnings" +msgstr "avertissements" diff --git a/apps/transport/priv/gettext/netex-validator.pot b/apps/transport/priv/gettext/netex-validator.pot new file mode 100644 index 0000000000..2566320ef1 --- /dev/null +++ b/apps/transport/priv/gettext/netex-validator.pot @@ -0,0 +1,68 @@ +## This file is a PO Template file. +## +## "msgid"s here are often extracted from source code. +## Add new messages manually only if they're dynamic +## messages that can't be statically extracted. +## +## Run "mix gettext.extract" to bring this file up to +## date. Leave "msgstr"s empty as changing them here has no +## effect: edit them in PO (.po) files instead. +# +msgid "" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Composite frame ligne mandatory" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Frame arret resources" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Frame calendrier resources" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Frame horaire resources" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Frame ligne resources" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Frame reseau resources" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Latitude mandatory" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Longitude mandatory" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "UIC operating period" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Valid day bits" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "Version any" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "errors" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "informations" +msgstr "" + +#, elixir-autogen, elixir-format +msgid "warnings" +msgstr "" diff --git a/apps/transport/test/transport/validators/netex_validator_test.exs b/apps/transport/test/transport/validators/netex_validator_test.exs new file mode 100644 index 0000000000..8acdb48783 --- /dev/null +++ b/apps/transport/test/transport/validators/netex_validator_test.exs @@ -0,0 +1,216 @@ +defmodule Transport.Validators.NeTExTest do + use ExUnit.Case, async: true + import DB.Factory + import Mox + require Logger + + alias Transport.Validators.NeTEx + + doctest Transport.Validators.NeTEx, import: true + + setup do + Ecto.Adapters.SQL.Sandbox.checkout(DB.Repo) + end + + setup :verify_on_exit! + + @sample_error_messages [ + %{ + "code" => "uic-operating-period", + "message" => "Resource 23504000009 hasn't expected class but Netex::OperatingPeriod", + "criticity" => "error" + }, + %{ + "code" => "valid-day-bits", + "message" => "Mandatory attribute valid_day_bits not found", + "criticity" => "error" + }, + %{ + "code" => "frame-arret-resources", + "message" => "Tag frame_id doesn't match ''", + "criticity" => "warning" + } + ] + + @sample_error_message Enum.take(@sample_error_messages, 1) + + describe "existing resource" do + test "valid NeTEx" do + {resource, resource_history} = mk_netex_resource() + + validation_id = expect_create_validation() + expect_successful_validation(validation_id) + + assert :ok == NeTEx.validate_and_save(resource) + + multi_validation = DB.MultiValidation |> DB.Repo.get_by(resource_history_id: resource_history.id) + + assert multi_validation.command == "http://localhost:9999/chouette-valid/#{validation_id}" + assert multi_validation.validator == "enroute-chouette-validator" + assert multi_validation.validator_version == "saas-production" + assert multi_validation.result == %{} + end + + test "invalid NeTEx" do + {resource, resource_history} = mk_netex_resource() + + validation_id = expect_create_validation() + expect_failed_validation(validation_id) + + expect_get_messages(validation_id, @sample_error_messages) + + assert :ok == NeTEx.validate_and_save(resource) + + multi_validation = DB.MultiValidation |> DB.Repo.get_by(resource_history_id: resource_history.id) + + assert multi_validation.command == "http://localhost:9999/chouette-valid/#{validation_id}/messages" + assert multi_validation.validator == "enroute-chouette-validator" + assert multi_validation.validator_version == "saas-production" + + assert multi_validation.result == %{ + "uic-operating-period" => [ + %{ + "code" => "uic-operating-period", + "message" => "Resource 23504000009 hasn't expected class but Netex::OperatingPeriod", + "criticity" => "error" + } + ], + "valid-day-bits" => [ + %{ + "code" => "valid-day-bits", + "message" => "Mandatory attribute valid_day_bits not found", + "criticity" => "error" + } + ], + "frame-arret-resources" => [ + %{ + "code" => "frame-arret-resources", + "message" => "Tag frame_id doesn't match ''", + "criticity" => "warning" + } + ] + } + end + end + + describe "raw URL" do + test "valid NeTEx" do + resource_url = mk_raw_netex_resource() + + validation_id = expect_create_validation() + expect_successful_validation(validation_id) + + assert {:ok, %{"validations" => %{}, "metadata" => %{}}} == + NeTEx.validate(resource_url) + end + + test "invalid NeTEx" do + resource_url = mk_raw_netex_resource() + + validation_id = expect_create_validation() + expect_failed_validation(validation_id) + + expect_get_messages(validation_id, @sample_error_messages) + + validation_result = %{ + "uic-operating-period" => [ + %{ + "code" => "uic-operating-period", + "message" => "Resource 23504000009 hasn't expected class but Netex::OperatingPeriod", + "criticity" => "error" + } + ], + "valid-day-bits" => [ + %{ + "code" => "valid-day-bits", + "message" => "Mandatory attribute valid_day_bits not found", + "criticity" => "error" + } + ], + "frame-arret-resources" => [ + %{ + "code" => "frame-arret-resources", + "message" => "Tag frame_id doesn't match ''", + "criticity" => "warning" + } + ] + } + + assert {:ok, %{"validations" => validation_result, "metadata" => %{}}} == NeTEx.validate(resource_url) + end + + test "retries" do + resource_url = mk_raw_netex_resource() + + validation_id = expect_create_validation() + expect_pending_validation(validation_id) + expect_pending_validation(validation_id) + expect_pending_validation(validation_id) + expect_failed_validation(validation_id) + + expect_get_messages(validation_id, @sample_error_message) + + validation_result = %{ + "uic-operating-period" => [ + %{ + "code" => "uic-operating-period", + "message" => "Resource 23504000009 hasn't expected class but Netex::OperatingPeriod", + "criticity" => "error" + } + ] + } + + # Let's disable graceful retry as we are mocking the API, otherwise the + # test would take almost a minute. + assert {:ok, %{"validations" => validation_result, "metadata" => %{}}} == + NeTEx.validate(resource_url, graceful_retry: false) + end + end + + defp expect_create_validation do + validation_id = Ecto.UUID.generate() + expect(Transport.EnRouteChouetteValidClient.Mock, :create_a_validation, fn _ -> validation_id end) + validation_id + end + + defp expect_pending_validation(validation_id) do + expect(Transport.EnRouteChouetteValidClient.Mock, :get_a_validation, fn ^validation_id -> :pending end) + end + + defp expect_successful_validation(validation_id) do + expect(Transport.EnRouteChouetteValidClient.Mock, :get_a_validation, fn ^validation_id -> + {:successful, "http://localhost:9999/chouette-valid/#{validation_id}"} + end) + end + + defp expect_failed_validation(validation_id) do + expect(Transport.EnRouteChouetteValidClient.Mock, :get_a_validation, fn ^validation_id -> :failed end) + end + + defp expect_get_messages(validation_id, result) do + expect(Transport.EnRouteChouetteValidClient.Mock, :get_messages, fn ^validation_id -> + {"http://localhost:9999/chouette-valid/#{validation_id}/messages", result} + end) + end + + defp mk_netex_resource do + dataset = insert(:dataset) + + resource = insert(:resource, dataset_id: dataset.id, format: "NeTEx") + + resource_history = + insert(:resource_history, resource_id: resource.id, payload: %{"permanent_url" => mk_raw_netex_resource()}) + + {resource, resource_history} + end + + defp mk_raw_netex_resource do + resource_url = "http://localhost:9999/netex-#{Ecto.UUID.generate()}.zip" + + expect(Transport.Req.Mock, :get!, 1, fn ^resource_url, [{:compressed, false}, {:into, _}] -> + {:ok, %Req.Response{status: 200, body: %{"data" => "some_zip_file"}}} + end) + + resource_url + end +end diff --git a/apps/transport/test/transport/validators/validator_selection_test.exs b/apps/transport/test/transport/validators/validator_selection_test.exs index 473e0668cd..bf4b518a5d 100644 --- a/apps/transport/test/transport/validators/validator_selection_test.exs +++ b/apps/transport/test/transport/validators/validator_selection_test.exs @@ -19,6 +19,12 @@ defmodule Transport.ValidatorsSelectionTest do assert ValidatorsSelection.validators(resource_history) == [ Transport.Validators.GTFSTransport ] + + resource_history = insert(:resource_history, payload: %{"format" => "NeTEx"}) + + assert ValidatorsSelection.validators(resource_history) == [ + Transport.Validators.NeTEx + ] end test "for a ResourceHistory with a schema" do diff --git a/config/config.exs b/config/config.exs index e055d11c3e..9f22179142 100644 --- a/config/config.exs +++ b/config/config.exs @@ -106,7 +106,8 @@ config :transport, validator_selection: Transport.ValidatorsSelection.Impl, data_visualization: Transport.DataVisualization.Impl, workflow_notifier: Transport.Jobs.Workflow.ObanNotifier, - enroute_validator_client: Transport.EnRouteChouetteValidClient + enroute_validator_client: Transport.EnRouteChouetteValidClient, + netex_validator: Transport.Validators.NeTEx # Datagouv IDs for national databases created automatically. # These are IDs used in staging, demo.data.gouv.fr diff --git a/config/test.exs b/config/test.exs index e25bd1a9d2..eddc3a5085 100644 --- a/config/test.exs +++ b/config/test.exs @@ -54,7 +54,8 @@ config :transport, export_secret_key: "fake_export_secret_key", enroute_token: "fake_enroute_token", enroute_validation_token: "fake_enroute_token", - enroute_validator_client: Transport.EnRouteChouetteValidClient.Mock + enroute_validator_client: Transport.EnRouteChouetteValidClient.Mock, + netex_validator: Transport.Validators.NeTEx.Mock config :ex_aws, cellar_organisation_id: "fake-cellar_organisation_id" From 32188a67086b9cdb29113bebe690df04aec44be7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Menou?= Date: Thu, 22 Aug 2024 18:23:56 +0200 Subject: [PATCH 2/3] Please Dialyzer --- apps/transport/lib/validators/netex_validator.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/transport/lib/validators/netex_validator.ex b/apps/transport/lib/validators/netex_validator.ex index a9365243fb..48556cd58b 100644 --- a/apps/transport/lib/validators/netex_validator.ex +++ b/apps/transport/lib/validators/netex_validator.ex @@ -60,7 +60,7 @@ defmodule Transport.Validators.NeTEx do end) end - @spec with_resource_file(ResourceHistory.t(), (Path.t() -> any())) :: any() + @spec with_resource_file(DB.ResourceHistory.t(), (Path.t() -> any())) :: any() def with_resource_file(resource_history, closure) do %DB.ResourceHistory{payload: %{"permanent_url" => permanent_url}} = resource_history filepath = tmp_path(resource_history) From cb4adb6028b480b7ee7d68b80e7f30bace8c142a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Menou?= Date: Thu, 29 Aug 2024 15:53:43 +0200 Subject: [PATCH 3/3] Revue de code --- .../lib/validators/netex_validator.ex | 63 +++++++++++-------- .../validators/netex_validator_test.exs | 4 +- 2 files changed, 38 insertions(+), 29 deletions(-) diff --git a/apps/transport/lib/validators/netex_validator.ex b/apps/transport/lib/validators/netex_validator.ex index 48556cd58b..0ea0ce9a12 100644 --- a/apps/transport/lib/validators/netex_validator.ex +++ b/apps/transport/lib/validators/netex_validator.ex @@ -1,18 +1,22 @@ defmodule Transport.Validators.NeTEx do - import TransportWeb.Gettext, only: [dgettext: 2] - @moduledoc """ - Validator for NeTEx files calling enRoute Chouette Valid API. + Validator for NeTEx files calling enRoute Chouette Valid API. This is blocking + (by polling the tier API) and can take quite some time upon completion. """ + + import TransportWeb.Gettext, only: [dgettext: 2] require Logger + + @no_error "NoError" + @behaviour Transport.Validators.Validator @impl Transport.Validators.Validator - def validator_name, do: "enroute-chouette-validator" + def validator_name, do: "enroute-chouette-netex-validator" @impl Transport.Validators.Validator def validate_and_save(%DB.Resource{format: "NeTEx", id: resource_id}) do - Logger.info("Validating #{resource_id} with enRoute Chouette Valid") + Logger.info("Validating NeTEx #{resource_id} with enRoute Chouette Valid") resource_history = DB.ResourceHistory.latest_resource_history(resource_id) with_resource_file(resource_history, &validate_resource_history(resource_history, &1)) @@ -38,6 +42,10 @@ defmodule Transport.Validators.NeTEx do @doc """ Validate the resource from the given URL. + + Options can be passed to tweak the behaviour: + - graceful_retry is a flag to skip the polling interval. Useful for testing + purposes mostly. Defaults to false. """ @spec validate(binary(), validate_options()) :: {:ok, map()} | {:error, binary()} def validate(url, opts \\ []) do @@ -65,16 +73,16 @@ defmodule Transport.Validators.NeTEx do %DB.ResourceHistory{payload: %{"permanent_url" => permanent_url}} = resource_history filepath = tmp_path(resource_history) - with_temp_file(permanent_url, filepath, closure) + with_tmp_file(permanent_url, filepath, closure) end @spec with_url(binary(), (Path.t() -> any())) :: any() def with_url(url, closure) do - with_temp_file(url, tmp_path(url), closure) + with_tmp_file(url, tmp_path(url), closure) end - @spec with_temp_file(binary(), Path.t(), (Path.t() -> any())) :: any() - def with_temp_file(url, filepath, closure) do + @spec with_tmp_file(binary(), Path.t(), (Path.t() -> any())) :: any() + def with_tmp_file(url, filepath, closure) do http_client().get!(url, compressed: false, into: File.stream!(filepath)) closure.(filepath) after @@ -122,14 +130,15 @@ defmodule Transport.Validators.NeTEx do severity end - @no_error "NoError" - @doc """ Returns the maximum severity, with the issues count - iex> validation_result = %{"uic-operating-period" => [%{"criticity" => "error"}], "valid-day-bits" => [%{"criticity" => "error"}], "frame-arret-resources" => [%{"criticity" => "error"}]} + iex> validation_result = %{"uic-operating-period" => [%{"criticity" => "error"}], "valid-day-bits" => [%{"criticity" => "error"}], "frame-arret-resources" => [%{"criticity" => "warning"}]} + iex> count_max_severity(validation_result) + {"error", 2} + iex> validation_result = %{"frame-arret-resources" => [%{"criticity" => "warning"}]} iex> count_max_severity(validation_result) - {"error", 3} + {"warning", 1} iex> count_max_severity(%{}) {"NoError", 0} """ @@ -141,7 +150,7 @@ defmodule Transport.Validators.NeTEx do def count_max_severity(%{} = validation_result) do validation_result |> count_by_severity() - |> Enum.min_by(fn {severity, _count} -> severity |> severities() |> Map.get(:level) end) + |> Enum.min_by(fn {severity, _count} -> severity |> severity() |> Map.get(:level) end) end @spec severities_map() :: map() @@ -152,8 +161,8 @@ defmodule Transport.Validators.NeTEx do "information" => %{level: 3, text: dgettext("netex-validator", "informations")} } - @spec severities(binary()) :: %{level: integer(), text: binary()} - def severities(key), do: severities_map()[key] + @spec severity(binary()) :: %{level: integer(), text: binary()} + def severity(key), do: severities_map()[key] @doc """ Returns the number of issues by severity level @@ -198,6 +207,15 @@ defmodule Transport.Validators.NeTEx do end end + @doc """ + Poll interval to play nice with the tier. + + iex> 0..9 |> Enum.map(&poll_interval(&1)) + [10000, 10000, 10000, 10000, 10000, 10000, 20000, 20000, 20000, 20000] + """ + def poll_interval(nb_tries) when nb_tries < 6, do: 10_000 + def poll_interval(_), do: 20_000 + @doc """ iex> index_messages([]) %{} @@ -209,15 +227,6 @@ defmodule Transport.Validators.NeTEx do messages |> Enum.group_by(fn %{"code" => code} -> code end) end - @doc """ - Poll interval to play nice with the tier. - - iex> 0..9 |> Enum.map(&poll_interval(&1)) - [10000, 10000, 10000, 10000, 10000, 10000, 20000, 20000, 20000, 20000] - """ - def poll_interval(nb_tries) when nb_tries < 6, do: 10_000 - def poll_interval(_), do: 20_000 - # This will change with an actual versioning of the validator def validator_version, do: "saas-production" @@ -246,7 +255,7 @@ defmodule Transport.Validators.NeTEx do }} end) |> Enum.group_by(fn {_, details} -> details.criticity end) - |> Enum.sort_by(fn {criticity, _} -> severities(criticity).level end) + |> Enum.sort_by(fn {criticity, _} -> severity(criticity).level end) end @spec issues_short_translation() :: %{binary() => binary()} @@ -287,7 +296,7 @@ defmodule Transport.Validators.NeTEx do def get_issues(%{} = validation_result, _) do validation_result |> Map.values() - |> Enum.sort_by(fn [%{"criticity" => severity} | _] -> severities(severity).level end) + |> Enum.sort_by(fn [%{"criticity" => severity} | _] -> severity(severity).level end) |> List.first([]) |> order_issues_by_location() end diff --git a/apps/transport/test/transport/validators/netex_validator_test.exs b/apps/transport/test/transport/validators/netex_validator_test.exs index 8acdb48783..64c773d972 100644 --- a/apps/transport/test/transport/validators/netex_validator_test.exs +++ b/apps/transport/test/transport/validators/netex_validator_test.exs @@ -46,7 +46,7 @@ defmodule Transport.Validators.NeTExTest do multi_validation = DB.MultiValidation |> DB.Repo.get_by(resource_history_id: resource_history.id) assert multi_validation.command == "http://localhost:9999/chouette-valid/#{validation_id}" - assert multi_validation.validator == "enroute-chouette-validator" + assert multi_validation.validator == "enroute-chouette-netex-validator" assert multi_validation.validator_version == "saas-production" assert multi_validation.result == %{} end @@ -64,7 +64,7 @@ defmodule Transport.Validators.NeTExTest do multi_validation = DB.MultiValidation |> DB.Repo.get_by(resource_history_id: resource_history.id) assert multi_validation.command == "http://localhost:9999/chouette-valid/#{validation_id}/messages" - assert multi_validation.validator == "enroute-chouette-validator" + assert multi_validation.validator == "enroute-chouette-netex-validator" assert multi_validation.validator_version == "saas-production" assert multi_validation.result == %{