feat: add GA4 server-side analytics

This adds GA4 server-side analytics using the GA4 measurement protocol, similar
to the UA measurement protocol already in place.

Both will be run at the same time for now as GA4 is tested - and there may well
be more changes to the GA4 server-side analytics in the next week or so.

Note that server-side is fairly important because the primary function of the
public-data-api is the API (which includes data download), which do not have
web pages in front of them to run client-side analytics.

Co-authored-by: Ian Leggett <[email protected]>
Co-authored-by: Michal Charemza <[email protected]>

README.md

@@ -56,9 +56,10 @@ DOCS_GITHUB_REPO_URL=https://github.com/uktrade/public-data-api \
 | READ_AND_WRITE_AWS_ACCESS_KEY_ID     | The AWS access key ID that has write permissions on the S3 bucket (for the csv-generating worker) |
 | READ_AND_WRITE_AWS_SECRET_ACCESS_KEY | The secret part of the read+write AWS access key |
 | ENVIRONMENT           | The current environment where the application is running<hr>`develop` |
-| GA_ENDPOINT           | The endpoint to send analytics info to |
-| GA_TRACKING_ID        | The unique identifier for the google analytics property |
-| SENTRY_DSN            | The DSN of the Sentry server to report exceptions to |
+| GA_ENDPOINT (deprecated)          | The endpoint to send analytics info to |
+| GA_TRACKING_ID (deprecated)       | The unique identifier for the google analytics property |
+| GA4_API_SECRET        | The API secret for Google Analytics 4 (GA4) |
+| GA4_MEASUREMENT_ID    | The measurement ID for Google Analytics 4 (GA4) |
 
 Environment variables used for serving API documentation.
 

app.py

@@ -78,6 +78,7 @@ def proxy_app(
         endpoint_url,
         region_name,
         ga_tracking_id,
+        ga_measurement_id,
 ):
     parsed_endpoint = urllib.parse.urlsplit(endpoint_url)
     PoolClass = (
@@ -125,8 +126,8 @@ def stop():
     def track_analytics(handler):
         """Decorator to send analytics data to google in the background."""
 
-        def _send(requester_ip, request_url, request_headers):
-            logger.info('Sending to Google Analytics %s...', request_url)
+        def _send_to_ua(requester_ip, request_url, request_headers):
+            logger.info('Sending to Google Analytics UA %s...', request_url)
             requests.post(
                 os.environ.get(
                     'GA_ENDPOINT', 'https://www.google-analytics.com/collect'
@@ -145,10 +146,36 @@ def _send(requester_ip, request_url, request_headers):
                 },
             )
 
+        def _send_to_ga4(request_url, request_headers):
+            logger.info('Sending to Google Analytics GA4 %s...', request_url)
+            requests.post(
+                'https://www.google-analytics.com/mp/collect',
+                params={
+                    'api_secret': os.environ.get('GA4_API_SECRET'),
+                    'measurement_id': ga_measurement_id,
+                },
+                json={
+                    'client_id': str(uuid.uuid4()),
+                    'events': [{
+                        'name': 'page_view',
+                        'params': {
+                            'session_id': str(uuid.uuid4()),
+                            'engagement_time_msec': '100',
+                            'page_location': request_url,
+                            'page_title': 'Data API',
+                            'user_agent': request_headers.get('user-agent', ''),
+                            'referrer': request_headers.get('referer', ''),
+                        }
+                    }],
+                },
+            )
+
         @wraps(handler)
         def send(*args, **kwargs):
             if ga_tracking_id:
-                gevent.spawn(_send, request.remote_addr, request.url, request.headers)
+                gevent.spawn(_send_to_ua, request.remote_addr, request.url, request.headers)
+            if ga_measurement_id:
+                gevent.spawn(_send_to_ga4, request.url, request.headers)
             return handler(*args, **kwargs)
 
         return send
@@ -1038,6 +1065,7 @@ def main():
         os.environ['AWS_S3_ENDPOINT'],
         os.environ['AWS_S3_REGION'],
         os.environ.get('GA_TRACKING_ID'),
+        os.environ.get('GA_MEASUREMENT_ID'),
     )
 
     if os.environ.get('SENTRY_DSN'):

test_app.py

@@ -1785,7 +1785,7 @@ def test_logs_asim_format():
     web_output_logs = web_output.decode().split('\n')
     assert len(web_output_logs) >= 1
     web_api_call_log = [json.loads(log) for log in web_output_logs if url in log]
-    assert len(web_api_call_log) == 2
+    assert len(web_api_call_log) == 3
     assert 'EventMessage' in web_api_call_log[0]
     assert b'Shut down gracefully' in web_output