Merge pull request #1160 from uktrade/KLS-656-update-ingest-date

KLS-656 add ingest date command call

activitystream/views.py

@@ -4,10 +4,10 @@
 from django.conf import settings
 from django.core.cache import cache
 from django.db.models import Q
-from drf_spectacular.utils import extend_schema, OpenApiParameter, OpenApiExample
 from django.utils.crypto import constant_time_compare
-from drf_spectacular.types import OpenApiTypes
 from django.utils.decorators import decorator_from_middleware
+from drf_spectacular.types import OpenApiTypes
+from drf_spectacular.utils import OpenApiExample, OpenApiParameter, extend_schema
 from field_history.models import FieldHistory
 from mohawk import Receiver
 from mohawk.exc import HawkFail

buyer/views.py

@@ -1,5 +1,6 @@
 from django.conf import settings
 from rest_framework.generics import CreateAPIView
+
 from buyer import serializers
 from core.views import CSVDumpAPIView
 

company/views.py

@@ -1,19 +1,19 @@
 import abc
+
 from directory_constants import user_roles
 from django.conf import settings
 from django.db.models import BooleanField, Case, Count, Q, Value, When
 from django.http import Http404, JsonResponse
 from django.views.decorators.csrf import csrf_exempt
+from drf_spectacular.types import OpenApiTypes
+from drf_spectacular.utils import OpenApiExample, OpenApiParameter, OpenApiResponse, extend_schema, inline_serializer
 from rest_framework import generics, status, views, viewsets
 from rest_framework.permissions import IsAuthenticated
 from rest_framework.renderers import JSONRenderer
 from rest_framework.response import Response
-
-from drf_spectacular.utils import extend_schema, OpenApiParameter, OpenApiResponse, inline_serializer, OpenApiExample
-from drf_spectacular.types import OpenApiTypes
+from rest_framework.serializers import CharField, IntegerField, JSONField
 
 from company import documents, filters, gecko, helpers, models, pagination, permissions, serializers
-from rest_framework.serializers import IntegerField, CharField, JSONField
 from core import authentication
 from core.permissions import IsAuthenticatedSSO
 from core.views import CSVDumpAPIView

conf/urls.py

@@ -3,9 +3,8 @@
 from django.conf import settings
 from django.conf.urls import include
 from django.contrib import admin
-from django.urls import re_path, reverse_lazy, path
+from django.urls import path, re_path, reverse_lazy
 from django.views.generic import RedirectView
-
 from drf_spectacular.views import SpectacularAPIView, SpectacularRedocView, SpectacularSwaggerView
 
 import activitystream.views

dataservices/management/commands/import_market_guides_data.py

@@ -33,5 +33,6 @@ def handle(self, *args, **options):
             if MarketGuidesDataIngestionCommand().should_ingestion_run(v['view_name'], v['table_name']):
                 self.stdout.write(self.style.NOTICE(f'Running {k}'))
                 call_command(k, **options)
+                call_command('import_metadata_source_data', table=v['table_name'])
 
         self.stdout.write(self.style.SUCCESS('All done, bye!'))

dataservices/management/commands/import_metadata_source_data.py

@@ -26,6 +26,14 @@ class Command(BaseCommand):
             x.r <= 1;
     '''
 
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--table',
+            action='store',
+            help='Specify a table to update',
+            default=None,
+        )
+
     def handle(self, *args, **options):
         table_names_view_names = {
             'trade__uk_goods_nsa': ['TopFiveGoodsExportsByCountryView'],
@@ -35,6 +43,8 @@ def handle(self, *args, **options):
         }
         records = pd.read_sql(sa.text(self.sql), self.engine)
 
+        table_names_view_names = self.filter_tables(options, table_names_view_names)
+
         for _idx, row in records.iterrows():
             if row.table_name in table_names_view_names.keys():
                 for view_name in table_names_view_names[row.table_name]:
@@ -49,3 +59,12 @@ def handle(self, *args, **options):
                     instance.save()
 
         self.stdout.write(self.style.SUCCESS('All done, bye!'))
+
+    def filter_tables(self, options, table_names_view_names):
+        if options['table'] is not None:
+            filtered = {}
+            for t, v in table_names_view_names.items():
+                if t == options['table']:
+                    filtered.update({t: v})
+            return filtered
+        return table_names_view_names

dataservices/management/commands/tests/test_import_data.py

@@ -15,6 +15,7 @@
 from conf import settings
 from dataservices import models
 from dataservices.management.commands.helpers import MarketGuidesDataIngestionCommand
+from dataservices.management.commands.import_metadata_source_data import Command
 
 
 @pytest.mark.django_db
@@ -404,6 +405,23 @@ def test_import_metadata_source_data(read_sql_mock, metadata_last_release_raw_da
     assert len(models.Metadata.objects.all()) == 4
 
 
+def test_import_metadata_source_data_filter_tables():
+    table_names_view_names = {
+        't1': ['v1'],
+        't2': ['v2'],
+    }
+    cmd = Command()
+    options_none = {'table': None}
+    skipped_none = cmd.filter_tables(options_none, table_names_view_names)
+    assert skipped_none == table_names_view_names
+    options_skip = {'table': 't9'}
+    skipped = cmd.filter_tables(options_skip, table_names_view_names)
+    assert len(skipped) == 0
+    options_add = {'table': 't1'}
+    added = cmd.filter_tables(options_add, table_names_view_names)
+    assert len(added) == 1
+
+
 @pytest.mark.django_db
 @mock.patch('dataservices.management.commands.helpers.MarketGuidesDataIngestionCommand.should_ingestion_run')
 @mock.patch('dataservices.management.commands.import_market_guides_data.call_command')
@@ -417,11 +435,11 @@ def test_import_market_guides_data(mock_call_command, mock_should_run):
 
     management.call_command('import_market_guides_data', '--write')
 
-    assert mock_call_command.call_count == 3
+    assert mock_call_command.call_count == 6
 
-    for idx, command in enumerate(command_list):
-        assert command in str(mock_call_command.call_args_list[idx])
-        assert 'write=True' in str(mock_call_command.call_args_list[idx])
+    for command in command_list:
+        assert command in str(mock_call_command.call_args_list)
+        assert 'write=True' in str(mock_call_command.call_args_list)
 
 
 @pytest.mark.django_db
@@ -437,11 +455,11 @@ def test_import_market_guides_data_dry_run(mock_call_command, mock_should_run):
 
     management.call_command('import_market_guides_data')
 
-    assert mock_call_command.call_count == 3
+    assert mock_call_command.call_count == 6
 
-    for idx, command in enumerate(command_list):
-        assert command in str(mock_call_command.call_args_list[idx])
-        assert 'write=False' in str(mock_call_command.call_args_list[idx])
+    for command in command_list:
+        assert command in str(mock_call_command.call_args_list)
+        assert 'write=False' in str(mock_call_command.call_args_list)
 
 
 @pytest.fixture()

dataservices/views.py

@@ -2,15 +2,12 @@
 
 from django.apps import apps
 from django.shortcuts import get_object_or_404
+from drf_spectacular.types import OpenApiTypes
+from drf_spectacular.utils import OpenApiExample, OpenApiParameter, extend_schema, inline_serializer
 from rest_framework import generics, status
 from rest_framework.response import Response
-
-from drf_spectacular.utils import extend_schema, OpenApiParameter, OpenApiExample, inline_serializer
-from drf_spectacular.types import OpenApiTypes
-
 from rest_framework.serializers import CharField
 
-
 from dataservices import filters, helpers, models, renderers, serializers
 from dataservices.core import client_api
 from dataservices.helpers import (

enrolment/views.py

@@ -2,12 +2,11 @@
 from django.core import signing
 from django.db import transaction
 from django.shortcuts import Http404, get_object_or_404
+from drf_spectacular.utils import OpenApiResponse, extend_schema
 from rest_framework import generics, status
 from rest_framework.response import Response
 from rest_framework.views import APIView
 
-from drf_spectacular.utils import extend_schema, OpenApiResponse
-
 from company.models import Company
 from company.serializers import CompanyUserSerializer
 from company.signals import send_company_registration_letter

exportplan/views.py

@@ -1,29 +1,27 @@
 import importlib
 
+from drf_spectacular.types import OpenApiTypes
+from drf_spectacular.utils import (
+    OpenApiExample,
+    OpenApiParameter,
+    OpenApiResponse,
+    PolymorphicProxySerializer,
+    extend_schema,
+)
 from rest_framework import generics
 
 from core.permissions import IsAuthenticatedSSO
 from exportplan import models, serializers
 from exportplan.models import CompanyExportPlan
 
-from drf_spectacular.utils import (
-    extend_schema,
-    OpenApiResponse,
-    OpenApiParameter,
-    OpenApiExample,
-    PolymorphicProxySerializer,
-)
-from drf_spectacular.types import OpenApiTypes
-
 from .permissions import IsExportPlanOwner
-
 from .serializers import (
     BusinessRisksSerializer,
     BusinessTripsSerializer,
+    CompanyObjectivesSerializer,
     FundingCreditOptionsSerializer,
-    TargetMarketDocumentsSerializer,
     RouteToMarketsSerializer,
-    CompanyObjectivesSerializer,
+    TargetMarketDocumentsSerializer,
 )
 
 

personalisation/views.py

@@ -1,6 +1,7 @@
 import logging
 
 from django.db.models import Count
+from drf_spectacular.utils import OpenApiParameter, OpenApiResponse, extend_schema, inline_serializer
 from requests.exceptions import HTTPError
 from rest_framework import generics, status
 from rest_framework.response import Response
@@ -10,9 +11,6 @@
 from core.permissions import IsAuthenticatedSSO
 from personalisation import helpers, models, serializers
 
-from drf_spectacular.utils import extend_schema, OpenApiResponse, OpenApiParameter, inline_serializer
-
-
 logger = logging.getLogger(__name__)
 
 

testapi/views.py

@@ -3,6 +3,7 @@
 from django.db.models import Q
 from django.http import Http404
 from django.shortcuts import get_list_or_404
+from drf_spectacular.utils import OpenApiResponse, extend_schema
 from rest_framework.generics import (
     CreateAPIView,
     DestroyAPIView,
@@ -12,7 +13,6 @@
     get_object_or_404,
 )
 from rest_framework.response import Response
-from drf_spectacular.utils import extend_schema, OpenApiResponse
 
 from buyer.models import Buyer
 from buyer.serializers import BuyerSerializer