Télécharger les détails des ressources historisées (#4194)

* Télécharger les détails des ressources historisées

* PR comments

apps/transport/lib/transport_web/controllers/dataset_controller.ex

@@ -307,6 +307,67 @@ defmodule TransportWeb.DatasetController do
     |> Enum.reject(fn t -> is_nil(t.msg) end)
   end
 
+  def resources_history_csv(%Plug.Conn{} = conn, %{"dataset_id" => dataset_id}) do
+    filename = "historisation-dataset-#{dataset_id}-#{Date.utc_today() |> Date.to_iso8601()}.csv"
+    # We define CSV columns explicitly because ordering matters for humans
+    columns = [
+      "resource_history_id",
+      "resource_id",
+      "permanent_url",
+      "payload",
+      "validation_validator",
+      "validation_result",
+      "metadata",
+      "inserted_at"
+    ]
+
+    content =
+      Transport.History.Fetcher.history_resources(%DB.Dataset{id: String.to_integer(dataset_id)})
+      |> Enum.map(fn row -> build_history_csv_row(columns, row) end)
+      |> CSV.encode(headers: columns)
+      |> Enum.to_list()
+      |> to_string()
+
+    conn
+    |> put_resp_content_type("text/csv")
+    |> put_resp_header("content-disposition", ~s|attachment; filename="#{filename}"|)
+    |> send_resp(200, content)
+  end
+
+  defp build_history_csv_row(
+         columns,
+         %DB.ResourceHistory{id: rh_id, resource_id: resource_id, payload: payload, inserted_at: inserted_at} = rh
+       ) do
+    {validation, metadata} = validation_and_metadata(rh)
+
+    row =
+      %{
+        "resource_history_id" => rh_id,
+        "resource_id" => resource_id,
+        "permanent_url" => Map.fetch!(payload, "permanent_url"),
+        "payload" => Jason.encode!(payload),
+        "validation_validator" => Map.get(validation, :validator),
+        "validation_result" => Map.get(validation, :result) |> Jason.encode!(),
+        "metadata" => Jason.encode!(metadata),
+        "inserted_at" => inserted_at
+      }
+
+    # Make sure CSV columns match what we're building
+    if MapSet.new(columns) == MapSet.new(Map.keys(row)) do
+      row
+    else
+      raise "Unexpected columns: #{Map.keys(row)} != #{inspect(columns)}"
+    end
+  end
+
+  defp validation_and_metadata(%DB.ResourceHistory{
+         validations: [%{metadata: %DB.ResourceMetadata{metadata: metadata}} = validation]
+       }) do
+    {validation, metadata}
+  end
+
+  defp validation_and_metadata(_), do: {%{}, nil}
+
   defp add_current_type(results, type) do
     case Enum.any?(results, &(&1.type == type)) do
       true -> results

apps/transport/lib/transport_web/router.ex

@@ -144,6 +144,7 @@ defmodule TransportWeb.Router do
     scope "/datasets" do
       get("/", DatasetController, :index)
       get("/:slug/", DatasetController, :details)
+      get("/:dataset_id/resources_history_csv", DatasetController, :resources_history_csv)
       get("/aom/:aom", DatasetController, :by_aom)
       get("/region/:region", DatasetController, :by_region)
       get("/commune/:insee_commune", DatasetController, :by_commune_insee)

apps/transport/lib/transport_web/templates/dataset/_dataset_resources_history.html.heex

@@ -56,16 +56,25 @@
 
       <%= if Enum.count(@history_resources) == max_nb_history_resources() do %>
         <p class="small">
-          <%= raw(
-            dgettext(
-              "page-dataset-details",
-              ~s|Displaying the last %{nb} backed up resources. <a href="#mail_form">Contact us</a> if you want to access previous data.|,
-              nb: max_nb_history_resources()
-            )
+          <%= dgettext("page-dataset-details", "Displaying the last %{nb} backed up resources.",
+            nb: max_nb_history_resources()
           ) %>
         </p>
       <% end %>
     </div>
+    <div class="pt-12">
+      <a
+        class="button-outline small secondary"
+        href={dataset_path(@conn, :resources_history_csv, @dataset_id)}
+        data-tracking-category="dataset_details"
+        data-tracking-action="download_resources_history_csv"
+      >
+        <i class="icon icon--download" aria-hidden="true"></i><%= dgettext(
+          "page-dataset-details",
+          "Download history details"
+        ) %>
+      </a>
+    </div>
   </div>
 </section>
 

apps/transport/lib/transport_web/templates/dataset/details.html.heex

@@ -213,7 +213,12 @@
         </div>
       </section>
     <% end %>
-    <%= render("_dataset_resources_history.html", history_resources: @history_resources, locale: locale, conn: @conn) %>
+    <%= render("_dataset_resources_history.html",
+      history_resources: @history_resources,
+      locale: locale,
+      conn: @conn,
+      dataset_id: @dataset.id
+    ) %>
     <%= unless is_nil(@other_datasets) or @other_datasets == [] do %>
       <section class="pt-48" id="dataset-other-datasets">
         <h2><%= dgettext("page-dataset-details", "Other datasets of %{name}", name: @territory) %></h2>

apps/transport/priv/gettext/en/LC_MESSAGES/page-dataset-details.po

@@ -495,7 +495,11 @@ msgid "The <a href=\"%{link}\" target=\"_blank\">timestamp field</a> appears to
 msgstr ""
 
 #, elixir-autogen, elixir-format
-msgid "Displaying the last %{nb} backed up resources. <a href=\"#mail_form\">Contact us</a> if you want to access previous data."
+msgid "Displaying the last %{nb} backed up resources."
+msgstr ""
+
+#, elixir-autogen, elixir-format
+msgid "Download history details"
 msgstr ""
 
 #, elixir-autogen, elixir-format

apps/transport/priv/gettext/fr/LC_MESSAGES/page-dataset-details.po

@@ -495,8 +495,12 @@ msgid "The <a href=\"%{link}\" target=\"_blank\">timestamp field</a> appears to
 msgstr "Le <a href=\"%{link}\">champ timestamp</a> contient une valeur ancienne par rapport à la date courante : l'écart est de %{seconds} secondes. Essayez de mettre à jour le flux toutes les 30 secondes au plus."
 
 #, elixir-autogen, elixir-format
-msgid "Displaying the last %{nb} backed up resources. <a href=\"#mail_form\">Contact us</a> if you want to access previous data."
-msgstr "Affiche les %{nb} dernières ressources historisées. <a href=\"#mail_form\">Contactez-nous</a> si vous souhaitez accéder aux données passées."
+msgid "Displaying the last %{nb} backed up resources."
+msgstr "Affiche les %{nb} dernières ressources historisées."
+
+#, elixir-autogen, elixir-format
+msgid "Download history details"
+msgstr "Télécharger les ressources historisées"
 
 #, elixir-autogen, elixir-format
 msgid "Data published by"

apps/transport/priv/gettext/page-dataset-details.pot

@@ -495,7 +495,11 @@ msgid "The <a href=\"%{link}\" target=\"_blank\">timestamp field</a> appears to
 msgstr ""
 
 #, elixir-autogen, elixir-format
-msgid "Displaying the last %{nb} backed up resources. <a href=\"#mail_form\">Contact us</a> if you want to access previous data."
+msgid "Displaying the last %{nb} backed up resources."
+msgstr ""
+
+#, elixir-autogen, elixir-format
+msgid "Download history details"
 msgstr ""
 
 #, elixir-autogen, elixir-format

apps/transport/test/transport_web/controllers/dataset_controller_test.exs

@@ -812,6 +812,95 @@ defmodule TransportWeb.DatasetControllerTest do
     assert title == "Autocars longue distance"
   end
 
+  test "resources_history_csv", %{conn: conn} do
+    # Using the real implementation to test end-to-end
+    Mox.stub_with(Transport.History.Fetcher.Mock, Transport.History.Fetcher.Database)
+
+    dataset = insert(:dataset)
+    resource = insert(:resource, dataset: dataset)
+    other_resource = insert(:resource, dataset: dataset)
+    # another resource, no history for this one
+    insert(:resource, dataset: dataset, format: "gtfs-rt")
+
+    rh1 =
+      insert(:resource_history,
+        resource_id: resource.id,
+        payload: %{"foo" => "bar", "permanent_url" => "https://example.com/1"}
+      )
+
+    mv =
+      insert(:multi_validation,
+        resource_history_id: rh1.id,
+        validator: "validator_name",
+        result: %{"validation_details" => 42}
+      )
+
+    insert(:resource_metadata, multi_validation_id: mv.id, metadata: %{"metadata" => 1337})
+
+    # resource_id is nil, but dataset_id is filled in the payload
+    # no resource_metadata/multi_validation associated
+    rh2 =
+      insert(:resource_history,
+        resource_id: nil,
+        payload: %{"dataset_id" => dataset.id, "bar" => "baz", "permanent_url" => "https://example.com/2"}
+      )
+
+    # another resource for this dataset
+    # no resource_metadata/multi_validation associated
+    rh3 =
+      insert(:resource_history,
+        resource_id: other_resource.id,
+        payload: %{"dataset_id" => dataset.id, "permanent_url" => "https://example.com/3"}
+      )
+
+    response = conn |> get(dataset_path(conn, :resources_history_csv, dataset.id))
+    content = response(response, 200)
+
+    # Check CSV header
+    assert content |> String.split("\r\n") |> hd() ==
+             "resource_history_id,resource_id,permanent_url,payload,validation_validator,validation_result,metadata,inserted_at"
+
+    # Check CSV content
+    assert [content] |> CSV.decode!(headers: true) |> Enum.to_list() == [
+             %{
+               "inserted_at" => to_string(rh3.inserted_at),
+               "metadata" => "null",
+               "payload" => Jason.encode!(rh3.payload),
+               "permanent_url" => "https://example.com/3",
+               "resource_history_id" => to_string(rh3.id),
+               "resource_id" => to_string(rh3.resource_id),
+               "validation_result" => "null",
+               "validation_validator" => ""
+             },
+             %{
+               "inserted_at" => to_string(rh2.inserted_at),
+               "metadata" => "null",
+               "payload" => Jason.encode!(rh2.payload),
+               "permanent_url" => "https://example.com/2",
+               "resource_history_id" => to_string(rh2.id),
+               "resource_id" => to_string(rh2.resource_id),
+               "validation_result" => "null",
+               "validation_validator" => ""
+             },
+             %{
+               "inserted_at" => to_string(rh1.inserted_at),
+               "metadata" => ~s|{"metadata":1337}|,
+               "payload" => Jason.encode!(rh1.payload),
+               "permanent_url" => "https://example.com/1",
+               "resource_history_id" => to_string(rh1.id),
+               "resource_id" => to_string(rh1.resource_id),
+               "validation_result" => ~s|{"validation_details":42}|,
+               "validation_validator" => "validator_name"
+             }
+           ]
+
+    assert response_content_type(response, :csv) == "text/csv; charset=utf-8"
+
+    assert Plug.Conn.get_resp_header(response, "content-disposition") == [
+             ~s(attachment; filename="historisation-dataset-#{dataset.id}-#{Date.utc_today() |> Date.to_iso8601()}.csv")
+           ]
+  end
+
   defp dataset_page_title(content) do
     content
     |> Floki.parse_document!()