Resources : filesize porté par l‘historique

apps/transport/lib/db/resource.ex

@@ -40,7 +40,6 @@ defmodule DB.Resource do
     # (this is done for OpenDataSoft)
     field(:datagouv_id, :string)
 
-    field(:filesize, :integer)
     # Can be `remote` or `file`. `file` are for files uploaded and hosted
     # on data.gouv.fr
     field(:filetype, :string)
@@ -109,7 +108,6 @@ defmodule DB.Resource do
         :community_resource_publisher,
         :original_resource_url,
         :description,
-        :filesize,
         :filetype,
         :type,
         :display_position

apps/transport/lib/transport_web/api/controllers/datasets_controller.ex

@@ -251,6 +251,16 @@ defmodule TransportWeb.API.DatasetController do
 
   defp get_metadata(_), do: nil
 
+  defp get_payload(%Resource{format: "GTFS", resource_history: resource_history}) do
+    resource_history
+    |> Enum.at(0)
+    |> Map.get(:payload)
+  rescue
+    _ -> nil
+  end
+
+  defp get_payload(_), do: nil
+
   @spec transform_resource(Resource.t()) :: map()
   defp transform_resource(resource) do
     metadata = get_metadata(resource)
@@ -261,6 +271,8 @@ defmodule TransportWeb.API.DatasetController do
         _ -> nil
       end
 
+    payload = get_payload(resource)
+
     %{
       "page_url" => TransportWeb.Router.Helpers.resource_url(TransportWeb.Endpoint, :details, resource.id),
       "id" => resource.id,
@@ -277,7 +289,7 @@ defmodule TransportWeb.API.DatasetController do
       "community_resource_publisher" => resource.community_resource_publisher,
       "metadata" => metadata_content,
       "original_resource_url" => resource.original_resource_url,
-      "filesize" => resource.filesize,
+      "filesize" => payload && Map.get(payload, "filesize"),
       "modes" => metadata && Map.get(metadata, :modes),
       "features" => metadata && Map.get(metadata, :features),
       "schema_name" => resource.schema_name,

apps/transport/test/transport_web/controllers/api/datasets_controller_test.exs

@@ -52,8 +52,7 @@ defmodule TransportWeb.API.DatasetControllerTest do
         latest_url: "https://static.data.gouv.fr/foo",
         datagouv_id: "1",
         type: "main",
-        format: "GTFS",
-        filesize: 42
+        format: "GTFS"
       )
 
     resource_2 =
@@ -63,8 +62,7 @@ defmodule TransportWeb.API.DatasetControllerTest do
         latest_url: "https://static.data.gouv.fr/foo2",
         datagouv_id: "2",
         type: "main",
-        format: "GTFS",
-        filesize: 43
+        format: "GTFS"
       )
 
     gbfs_resource =
@@ -82,7 +80,7 @@ defmodule TransportWeb.API.DatasetControllerTest do
     insert(:resource_metadata,
       multi_validation:
         insert(:multi_validation,
-          resource_history: insert(:resource_history, resource_id: resource_1.id),
+          resource_history: insert(:resource_history, resource_id: resource_1.id, payload: %{"filesize" => 42}),
           validator: Transport.Validators.GTFSTransport.validator_name()
         ),
       modes: ["bus"],
@@ -93,7 +91,7 @@ defmodule TransportWeb.API.DatasetControllerTest do
     insert(:resource_metadata,
       multi_validation:
         insert(:multi_validation,
-          resource_history: insert(:resource_history, resource_id: resource_2.id),
+          resource_history: insert(:resource_history, resource_id: resource_2.id, payload: %{"filesize" => 43}),
           validator: Transport.Validators.GTFSTransport.validator_name()
         ),
       modes: ["skate"],
@@ -264,18 +262,18 @@ defmodule TransportWeb.API.DatasetControllerTest do
         datagouv_id: "datagouv",
         slug: "slug-1",
         resources: [
-          %DB.Resource{
-            last_import: DateTime.utc_now(),
-            last_update: last_update_gtfs = DateTime.utc_now() |> DateTime.add(-2, :hour),
-            url: "https://link.to/file.zip",
-            latest_url: "https://static.data.gouv.fr/foo",
-            datagouv_id: "1",
-            type: "main",
-            format: "GTFS",
-            filesize: 42,
-            title: "The title"
-          },
-          %DB.Resource{
+          resource1 =
+            insert(:resource,
+              last_import: DateTime.utc_now(),
+              last_update: last_update_gtfs = DateTime.utc_now() |> DateTime.add(-2, :hour),
+              url: "https://link.to/file.zip",
+              latest_url: "https://static.data.gouv.fr/foo",
+              datagouv_id: "1",
+              type: "main",
+              format: "GTFS",
+              title: "The title"
+            ),
+          insert(:resource,
             last_import: DateTime.utc_now(),
             last_update: last_update_geojson = DateTime.utc_now() |> DateTime.add(-1, :hour),
             url: "http://link.to/file.zip?foo=bar",
@@ -285,13 +283,21 @@ defmodule TransportWeb.API.DatasetControllerTest do
             format: "geojson",
             schema_name: "etalab/schema-zfe",
             title: "The other title"
-          }
+          )
         ],
         created_at: ~U[2021-12-23 13:30:40.000000Z],
         last_update: DateTime.utc_now(),
         aom: %DB.AOM{id: 4242, nom: "Angers Métropole", siren: "siren"}
       )
 
+    insert(:resource_metadata,
+      multi_validation:
+        insert(:multi_validation,
+          resource_history: insert(:resource_history, resource_id: resource1.id, payload: %{"filesize" => 42}),
+          validator: Transport.Validators.GTFSTransport.validator_name()
+        )
+    )
+
     setup_empty_history_resources()
 
     path = Helpers.dataset_path(conn, :by_id, dataset.datagouv_id)
@@ -325,7 +331,9 @@ defmodule TransportWeb.API.DatasetControllerTest do
                  "updated" => last_update_gtfs |> DateTime.to_iso8601(),
                  "url" => "https://static.data.gouv.fr/foo",
                  "conversions" => %{},
-                 "title" => "The title"
+                 "title" => "The title",
+                 "features" => [],
+                 "modes" => []
                },
                %{
                  "is_available" => true,
@@ -372,8 +380,7 @@ defmodule TransportWeb.API.DatasetControllerTest do
         latest_url: "https://static.data.gouv.fr/foo",
         datagouv_id: "1",
         type: "main",
-        format: "GTFS",
-        filesize: 42
+        format: "GTFS"
       )
 
     gbfs_resource =
@@ -388,7 +395,7 @@ defmodule TransportWeb.API.DatasetControllerTest do
     resource_history =
       insert(:resource_history,
         resource_id: resource.id,
-        payload: %{"uuid" => uuid1 = Ecto.UUID.generate()},
+        payload: %{"uuid" => uuid1 = Ecto.UUID.generate(), "filesize" => 42},
         last_up_to_date_at: last_up_to_date_at = DateTime.utc_now()
       )