Merge pull request #114 from Urban-Analytics-Technology-Platform/us_w…

…ith_class

Move US to the class based implementation (#99)

pyproject.toml

@@ -54,6 +54,7 @@ dependencies = [
   "xlrd >=2.0.1", # For reading Excel files
   "iso639-lang >=2.2.3", # For checking ISO639-3 language codes
   "aiohttp >=3.9.5", # Async HTTP
+  "more-itertools >= 10.3.0", # For batched iterator
 ]
 
 

python/popgetter/__init__.py

@@ -17,11 +17,15 @@
     AzureGeoIOManager,
     AzureMetadataIOManager,
     AzureMetricsIOManager,
+    AzureMetricsMetadataIOManager,
+    AzureMetricsPartitionedIOManager,
 )
 from popgetter.io_managers.local import (
     LocalGeoIOManager,
     LocalMetadataIOManager,
     LocalMetricsIOManager,
+    LocalMetricsMetadataIOManager,
+    LocalMetricsPartitionedIOManager,
 )
 from popgetter.utils import StagingDirResource
 
@@ -81,12 +85,16 @@ def resources_by_env():
             "geometry_io_manager": AzureGeoIOManager(),
             "metrics_io_manager": AzureMetricsIOManager(),
             "azure_general_io_manager": AzureGeneralIOManager(".bin"),
+            "metrics_partitioned_io_manager": AzureMetricsPartitionedIOManager(),
+            "metrics_metadata_io_manager": AzureMetricsMetadataIOManager(),
         }
         if PROD
         else {
             "metadata_io_manager": LocalMetadataIOManager(),
             "geometry_io_manager": LocalGeoIOManager(),
             "metrics_io_manager": LocalMetricsIOManager(),
+            "metrics_partitioned_io_manager": LocalMetricsPartitionedIOManager(),
+            "metrics_metadata_io_manager": LocalMetricsMetadataIOManager(),
         }
     )
 
@@ -106,6 +114,9 @@ def resources_by_env():
         cloud_outputs.metadata_sensor,
         cloud_outputs.geometry_sensor,
         cloud_outputs.metrics_sensor,
+        cloud_outputs.metrics_metadata_sensor,
+        # TODO: commented out until implemented for partitioned assets
+        # cloud_outputs.metrics_partitioned_sensor,
     ],
     resources=resources,
     jobs=jobs,

python/popgetter/assets/__init__.py

@@ -1,9 +1,9 @@
 from __future__ import annotations
 
-from . import bel, gb_eaw, gb_nir, gb_sct, uk, us
+from . import bel, gb_eaw, gb_nir, gb_sct, uk, usa
 
 countries = [
-    (mod, mod.__name__.split(".")[-1]) for mod in [bel, gb_nir, uk, us, gb_eaw, gb_sct]
+    (mod, mod.__name__.split(".")[-1]) for mod in [bel, gb_nir, uk, usa, gb_eaw, gb_sct]
 ]
 
 __all__ = ["countries"]