properly handling post request and sanitize request values before logging

cdci_data_analysis/flask_app/app.py

@@ -223,6 +223,16 @@ def remove_nested_keys(D, keys):
     return D
 
 
+def sanitize_dict_before_log(dict_to_sanitize):
+    sensitive_keys = ['token']  # Add any other sensitive keys here
+    sanitized_values = {}
+    for key, value in dict_to_sanitize.items():
+        if key not in sensitive_keys:
+            value = str(value).replace('\n', '').replace('\r', '')
+            sanitized_values[key] = value
+    return sanitized_values
+
+
 def common_exception_payload():
     payload = {}
 
@@ -441,9 +451,11 @@ def run_analysis():
     request_summary = log_run_query_request()
 
     try:
+        sanitized_request_values = sanitize_dict_before_log(request.values)
+
         logger.info('\033[32m===> dataserver_call_back\033[0m')
         logger.info('\033[33m raw request values: %s \033[0m',
-                    dict(request.values))
+                    dict(sanitized_request_values))
 
         query_id = hashlib.sha224(str(request.values).encode()).hexdigest()[:8]
 
@@ -522,9 +534,11 @@ def resolve_job_url():
 
 @app.route('/call_back', methods=['POST', 'GET'])
 def dataserver_call_back():
+    sanitized_request_values = sanitize_dict_before_log(request.values)
+
     logger.info('\033[32m===========================> dataserver_call_back\033[0m')
 
-    logger.info('\033[33m raw request values: %s \033[0m', dict(request.values))
+    logger.info('\033[33m raw request values: %s \033[0m', dict(sanitized_request_values))
 
     query_id = hashlib.sha224(str(request.values).encode()).hexdigest()[:8]
 
@@ -930,13 +944,17 @@ def get_data_product_list_by_source_name():
     return output_list
 
 
-
 @app.route('/post_astro_entity_to_gallery', methods=['POST'])
 def post_astro_entity_to_gallery():
-    logger.info("request.args: %s ", request.args)
+    par_dic = request.values.to_dict()
+    sanitized_par_dic = sanitize_dict_before_log(par_dic)
+
+    logger.info("request.values: %s ", sanitized_par_dic)
     logger.info("request.files: %s ", request.files)
 
-    token = request.args.get('token', None)
+    token = par_dic.get('token', None)
+    par_dic.pop('token')
+
     app_config = app.config.get('conf')
     secret_key = app_config.secret_key
 
@@ -948,9 +966,6 @@ def post_astro_entity_to_gallery():
         return make_response(output, output_code)
     decoded_token = output
 
-    par_dic = request.values.to_dict()
-    par_dic.pop('token')
-
     output_post = drupal_helper.post_content_to_gallery(decoded_token=decoded_token,
                                                         content_type="astrophysical_entity",
                                                         disp_conf=app_config,
@@ -962,10 +977,15 @@ def post_astro_entity_to_gallery():
 
 @app.route('/post_observation_to_gallery', methods=['POST'])
 def post_observation_to_gallery():
-    logger.info("request.args: %s ", request.args)
+    par_dic = request.values.to_dict()
+    sanitized_par_dic = sanitize_dict_before_log(par_dic)
+
+    token = par_dic.get('token', None)
+    par_dic.pop('token')
+
+    logger.info("request.values: %s ", sanitized_par_dic)
     logger.info("request.files: %s ", request.files)
 
-    token = request.args.get('token', None)
     app_config = app.config.get('conf')
     secret_key = app_config.secret_key
 
@@ -977,9 +997,6 @@ def post_observation_to_gallery():
         return make_response(output, output_code)
     decoded_token = output
 
-    par_dic = request.values.to_dict()
-    par_dic.pop('token')
-
     output_post = drupal_helper.post_content_to_gallery(decoded_token=decoded_token,
                                                         content_type="observation",
                                                         disp_conf=app_config,
@@ -991,10 +1008,15 @@ def post_observation_to_gallery():
 
 @app.route('/post_product_to_gallery', methods=['POST'])
 def post_product_to_gallery():
-    logger.info("request.args: %s ", request.args)
+    par_dic = request.values.to_dict()
+    sanitized_par_dic = sanitize_dict_before_log(par_dic)
+
+    logger.info("request.values: %s ", sanitized_par_dic)
     logger.info("request.files: %s ", request.files)
 
-    token = request.args.get('token', None)
+    token = par_dic.get('token', None)
+    par_dic.pop('token')
+
     app_config = app.config.get('conf')
     secret_key = app_config.secret_key
 
@@ -1006,9 +1028,6 @@ def post_product_to_gallery():
         return make_response(output, output_code)
     decoded_token = output
 
-    par_dic = request.values.to_dict()
-    par_dic.pop('token')
-
     output_post = drupal_helper.post_content_to_gallery(decoded_token=decoded_token,
                                                         disp_conf=app_config,
                                                         files=request.files,
@@ -1019,10 +1038,15 @@ def post_product_to_gallery():
 
 @app.route('/delete_product_to_gallery', methods=['POST'])
 def delete_product_to_gallery():
-    logger.info("request.args: %s ", request.args)
+    par_dic = request.values.to_dict()
+    sanitized_par_dic = sanitize_dict_before_log(par_dic)
+
+    logger.info("request.values: %s ", sanitized_par_dic)
     logger.info("request.files: %s ", request.files)
 
-    token = request.args.get('token', None)
+    token = par_dic.get('token', None)
+    par_dic.pop('token')
+
     app_config = app.config.get('conf')
     secret_key = app_config.secret_key
 
@@ -1034,9 +1058,6 @@ def delete_product_to_gallery():
         return make_response(output, output_code)
     decoded_token = output
 
-    par_dic = request.values.to_dict()
-    par_dic.pop('token')
-
     output_post = drupal_helper.delete_content_gallery(decoded_token=decoded_token,
                                                        disp_conf=app_config,
                                                        files=request.files,
@@ -1047,10 +1068,15 @@ def delete_product_to_gallery():
 
 @app.route('/post_revolution_processing_log_to_gallery', methods=['POST'])
 def post_revolution_processing_log_to_gallery():
-    logger.info("request.args: %s ", request.args)
+    par_dic = request.values.to_dict()
+    sanitized_par_dic = sanitize_dict_before_log(par_dic)
+
+    logger.info("request.values: %s ", sanitized_par_dic)
     logger.info("request.files: %s ", request.files)
 
-    token = request.args.get('token', None)
+    token = par_dic.get('token', None)
+    par_dic.pop('token')
+
     app_config = app.config.get('conf')
     secret_key = app_config.secret_key
 
@@ -1062,9 +1088,6 @@ def post_revolution_processing_log_to_gallery():
         return make_response(output, output_code)
     decoded_token = output
 
-    par_dic = request.values.to_dict()
-    par_dic.pop('token')
-
     output_post = drupal_helper.post_content_to_gallery(decoded_token=decoded_token,
                                                         disp_conf=app_config,
                                                         files=request.files,

tests/test_server_basic.py

@@ -2800,7 +2800,7 @@ def test_product_gallery_data_product_with_period_of_observation(dispatcher_live
         params['T2'] = now.strftime('%Y-%m-%dT%H:%M:%S')
 
     c = requests.post(os.path.join(server, "post_product_to_gallery"),
-                      params={**params},
+                      data=params,
                       files=file_obj
                       )
 
@@ -3122,7 +3122,7 @@ def test_product_gallery_get_data_products_list_with_conditions(dispatcher_live_
         }
 
         c = requests.post(os.path.join(server, "post_astro_entity_to_gallery"),
-                          params={**source_params},
+                          data=source_params,
                           )
 
         assert c.status_code == 200
@@ -3141,7 +3141,7 @@ def test_product_gallery_get_data_products_list_with_conditions(dispatcher_live_
             'T2': '2022-08-23T05:29:11'
         }
         c = requests.post(os.path.join(server, "post_product_to_gallery"),
-                          params={**product_params}
+                          data=product_params
                           )
 
         assert c.status_code == 200
@@ -3270,7 +3270,7 @@ def test_product_gallery_get_data_products_list_for_given_source(dispatcher_live
         }
 
         c = requests.post(os.path.join(server, "post_astro_entity_to_gallery"),
-                          params={**source_params},
+                          data=source_params,
                           )
 
         assert c.status_code == 200
@@ -3286,7 +3286,7 @@ def test_product_gallery_get_data_products_list_for_given_source(dispatcher_live
             'insert_new_source': True
         }
         c = requests.post(os.path.join(server, "post_product_to_gallery"),
-                          params={**product_params}
+                          data=product_params
                           )
 
         assert c.status_code == 200
@@ -3434,7 +3434,7 @@ def test_product_gallery_get_period_of_observation_attachments(dispatcher_live_f
 
 
         c = requests.post(os.path.join(server, "post_observation_to_gallery"),
-                          params={**params},
+                          data=params,
                           files=file_obj
                           )
 
@@ -3522,7 +3522,7 @@ def test_product_gallery_post_period_of_observation(dispatcher_live_fixture_with
         params['T2'] = now.strftime('%Y-%m-%dT%H:%M:%S')
 
     c = requests.post(os.path.join(server, "post_observation_to_gallery"),
-                      params={**params},
+                      data=params,
                       files=file_obj
                       )
 
@@ -3621,7 +3621,7 @@ def test_revolution_processing_log_gallery_post(dispatcher_live_fixture_with_gal
     }
 
     c = requests.post(os.path.join(server, "post_revolution_processing_log_to_gallery"),
-                      params={**params},
+                      data=params,
                       )
 
     assert c.status_code == 200
@@ -3740,7 +3740,7 @@ def test_product_gallery_post(dispatcher_live_fixture_with_gallery, dispatcher_t
                 'fits_file_1': open('data/dummy_prods/query_catalog.fits', 'rb')}
 
     c = requests.post(os.path.join(server, "post_product_to_gallery"),
-                      params={**params},
+                      data=params,
                       files=file_obj
                       )
 
@@ -3852,7 +3852,7 @@ def test_post_data_product_with_multiple_sources(dispatcher_live_fixture_with_ga
         'insert_new_source': insert_new_source
     }
     c = requests.post(os.path.join(server, "post_product_to_gallery"),
-                      params={**params}
+                      data=params
                       )
 
     assert c.status_code == 200
@@ -3982,7 +3982,7 @@ def test_product_gallery_update(dispatcher_live_fixture_with_gallery, dispatcher
                 'fits_file_1': open('data/dummy_prods/query_catalog.fits', 'rb')}
 
     c = requests.post(os.path.join(server, "post_product_to_gallery"),
-                      params={**params},
+                      data=params,
                       files=file_obj
                       )
 
@@ -4028,7 +4028,7 @@ def test_product_gallery_update(dispatcher_live_fixture_with_gallery, dispatcher
                 'fits_file_0': open('data/dummy_prods/isgri_query_lc.fits', 'rb')}
 
     c = requests.post(os.path.join(server, "post_product_to_gallery"),
-                      params={**params},
+                      data=params,
                       files=file_obj
                       )
     assert c.status_code == 200
@@ -4084,7 +4084,7 @@ def test_product_gallery_delete(dispatcher_live_fixture_with_gallery, dispatcher
                   token=encoded_token)
 
     c = requests.post(os.path.join(server, "post_product_to_gallery"),
-                      params={**params},
+                      data=params,
                       )
 
     assert c.status_code == 200
@@ -4096,31 +4096,37 @@ def test_product_gallery_delete(dispatcher_live_fixture_with_gallery, dispatcher
     assert 'field_product_id' in drupal_res_obj
     assert drupal_res_obj['field_product_id'][0]['value'] == product_id
 
-    params = {
-        'product_id': product_id,
+    params_products_list = {
+        'product_id_value': product_id,
         'content_type': 'data_product',
         'token': encoded_token
     }
 
     c = requests.get(os.path.join(server, "get_data_product_list_with_conditions"),
-                     params=params
+                     params=params_products_list
                      )
 
     assert c.status_code == 200
     drupal_res_obj = c.json()
     assert len(drupal_res_obj) == 1
     assert drupal_res_obj[0]['nid'] == str(nid_creation)
 
+    params = {
+        'product_id': product_id,
+        'content_type': 'data_product',
+        'token': encoded_token
+    }
+
     c = requests.post(os.path.join(server, "delete_product_to_gallery"),
-                      params={**params},
+                      data=params,
                       )
     assert c.status_code == 200
 
     drupal_res_obj = c.json()
     assert drupal_res_obj == {}
 
     c = requests.get(os.path.join(server, "get_data_product_list_with_conditions"),
-                     params=params
+                     params=params_products_list
                      )
 
     assert c.status_code == 200
@@ -4155,7 +4161,7 @@ def test_product_gallery_error_message(dispatcher_live_fixture_with_gallery):
     }
 
     c = requests.post(os.path.join(server, "post_product_to_gallery"),
-                      params={**params},
+                      data=params,
                       )
 
     assert c.status_code == 500