adql query parser function

cdci_data_analysis/analysis/ivoa_helper.py

@@ -0,0 +1,93 @@
+import os.path
+
+from queryparser.adql import ADQLQueryTranslator
+from queryparser.exceptions import QuerySyntaxError
+
+from psycopg2 import connect, DatabaseError
+
+from ..flask_app.sentry import sentry
+from ..app_logging import app_logging
+
+logger = app_logging.getLogger('ivoa_helper')
+
+
+def parse_adql_query(query):
+    try:
+        adt = ADQLQueryTranslator(query)
+
+        output_obj = dict(
+            mysql_query=None,
+            psql_query=adt.to_postgresql()
+        )
+
+    except QuerySyntaxError as qe:
+        logger.error(f'Error parsing ADQL query: {qe}')
+        output_obj = dict(
+            mysql_query=None,
+            psql_query=None
+        )
+    return output_obj
+
+
+def run_ivoa_query(query, **kwargs):
+    parsed_query_obj = parse_adql_query(query)
+
+    # TODO use a specific dedicated table and schema to refer to the product_gallery DB ?
+    # tables = parsed_query_obj.get('tables', [])
+    # if len(tables) == 1 and tables[0] == 'product_gallery':
+    logger.info('Performing query on the product_gallery')
+    vo_psql_pg_host = kwargs.get('vo_psql_pg_host', None)
+    vo_psql_pg_user = kwargs.get('vo_psql_pg_user', None)
+    vo_psql_pg_password = kwargs.get('vo_psql_pg_password', None)
+    vo_psql_pg_db = kwargs.get('vo_psql_pg_db', None)
+    product_gallery_url = kwargs.get('product_gallery_url', None)
+    result_list = run_ivoa_query_from_product_gallery(parsed_query_obj,
+                                                      vo_psql_pg_host=vo_psql_pg_host,
+                                                      vo_psql_pg_user=vo_psql_pg_user,
+                                                      vo_psql_pg_password=vo_psql_pg_password,
+                                                      vo_psql_pg_db=vo_psql_pg_db,
+                                                      product_gallery_url=product_gallery_url)
+    return result_list
+
+
+def run_ivoa_query_from_product_gallery(parsed_query_obj,
+                                        vo_psql_pg_host,
+                                        vo_psql_pg_user,
+                                        vo_psql_pg_password,
+                                        vo_psql_pg_db,
+                                        product_gallery_url=None
+                                        ):
+    result_list = []
+
+    try:
+        with connect(
+            host=vo_psql_pg_host,
+            database=vo_psql_pg_db,
+            user=vo_psql_pg_user,
+            password=vo_psql_pg_password
+        ) as connection:
+            db_query = parsed_query_obj.get('psql_query')
+            with connection.cursor() as cursor:
+                cursor.execute(db_query)
+                for row in cursor:
+                    list_row = list(row)
+                    if product_gallery_url is not None:
+                        for index, value in enumerate(list_row):
+                            description = cursor.description[index]
+                            if description.name in {'path', 'path_alias'} and value is not None and isinstance(value, str):
+                                if value.startswith('/'):
+                                    value = value[1:]
+                                list_row[index] = os.path.join(product_gallery_url, value)
+                    result_list.append(list_row)
+
+    except (Exception, DatabaseError) as e:
+        sentry.capture_message(f"Error when querying to the Postgresql server: {str(e)}")
+        logger.error(f"Error when querying to the Postgresql server: {str(e)}")
+
+    finally:
+        if connection is not None:
+            cursor.close()
+            connection.close()
+            logger.info('Database connection closed')
+
+    return result_list

cdci_data_analysis/config_dir/conf_env.yml.example

@@ -122,4 +122,15 @@ dispatcher:
         # url for the conversion of a given time, in UTC format, to the correspondent REVNUM
         converttime_revnum_service_url: COVERTTIME_REVNUM_SERVICE_URL
 
-
+    # virtual observatory related configurations (eg mysql credentials)
+    vo_options:
+        # mysql credentials
+        vo_mysql_pg_host: MYSQL_PG_HOST
+        vo_mysql_pg_user: MYSQL_PG_USER
+        vo_mysql_pg_password: MYSQL_PG_PASSWORD
+        vo_mysql_pg_db: MYSQL_PG_DB
+        # postgresql credentials
+        vo_psql_pg_host: PSQL_PG_HOST
+        vo_psql_pg_user: PSQL_PG_USER
+        vo_psql_pg_password: PSQL_PG_PASSWORD
+        vo_psql_pg_db: PSQL_PG_DB

cdci_data_analysis/configurer.py

@@ -266,6 +266,14 @@ def __init__(self, cfg_dict, origin=None):
                                      disp_dict.get('renku_options', {}).get('renku_gitlab_repository_url', None),
                                      disp_dict.get('renku_options', {}).get('renku_base_project_url', None),
                                      disp_dict.get('renku_options', {}).get('ssh_key_path', None),
+                                     disp_dict.get('vo_options', {}).get('vo_mysql_pg_host', None),
+                                     disp_dict.get('vo_options', {}).get('vo_mysql_pg_user', None),
+                                     disp_dict.get('vo_options', {}).get('vo_mysql_pg_password', None),
+                                     disp_dict.get('vo_options', {}).get('vo_mysql_pg_db', None),
+                                     disp_dict.get('vo_options', {}).get('vo_psql_pg_host', None),
+                                     disp_dict.get('vo_options', {}).get('vo_psql_pg_user', None),
+                                     disp_dict.get('vo_options', {}).get('vo_psql_pg_password', None),
+                                     disp_dict.get('vo_options', {}).get('vo_psql_pg_db', None)
                                      )
 
         # not used?
@@ -344,6 +352,14 @@ def set_conf_dispatcher(self,
                             renku_gitlab_repository_url,
                             renku_base_project_url,
                             renku_gitlab_ssh_key_path,
+                            vo_mysql_pg_host,
+                            vo_mysql_pg_user,
+                            vo_mysql_pg_password,
+                            vo_mysql_pg_db,
+                            vo_psql_pg_host,
+                            vo_psql_pg_user,
+                            vo_psql_pg_password,
+                            vo_psql_pg_db
                             ):
         # Generic to dispatcher
         #print(dispatcher_url, dispatcher_port)
@@ -395,6 +411,14 @@ def set_conf_dispatcher(self,
         self.renku_gitlab_repository_url = renku_gitlab_repository_url
         self.renku_gitlab_ssh_key_path = renku_gitlab_ssh_key_path
         self.renku_base_project_url = renku_base_project_url
+        self.vo_mysql_pg_host = vo_mysql_pg_host
+        self.vo_mysql_pg_user = vo_mysql_pg_user
+        self.vo_mysql_pg_password = vo_mysql_pg_password
+        self.vo_mysql_pg_db = vo_mysql_pg_db
+        self.vo_psql_pg_host = vo_psql_pg_host
+        self.vo_psql_pg_user = vo_psql_pg_user
+        self.vo_psql_pg_password = vo_psql_pg_password
+        self.vo_psql_pg_db = vo_psql_pg_db
 
     def get_data_serve_conf(self, instr_name):
         if instr_name in self.data_server_conf_dict.keys():

cdci_data_analysis/flask_app/app.py

@@ -29,7 +29,7 @@
 import time as _time
 from urllib.parse import urlencode, urlparse
 
-from cdci_data_analysis.analysis import drupal_helper, tokenHelper, renku_helper, email_helper, matrix_helper
+from cdci_data_analysis.analysis import drupal_helper, tokenHelper, renku_helper, email_helper, matrix_helper, ivoa_helper
 from .logstash import logstash_message
 from .schemas import QueryOutJSON, dispatcher_strict_validate
 from marshmallow.exceptions import ValidationError
@@ -430,6 +430,41 @@
                                            "Our team is notified and is working on it.")
 
 
+@app.route('/run_adql_query')
+def run_adql_query():
+    par_dic = request.values.to_dict()
+    sanitized_request_values = sanitize_dict_before_log(par_dic)
+    logger.info('\033[32m===========================> run_adql_query\033[0m')
+
+    logger.info('\033[33m raw request values: %s \033[0m', dict(sanitized_request_values))
+
+    token = par_dic.get('token', None)
+    app_config = app.config.get('conf')
+    secret_key = app_config.secret_key
+
+    output, output_code = tokenHelper.validate_token_from_request(token=token, secret_key=secret_key,
+                                                                  required_roles=['ivoa_user'],
+                                                                  action="run an ADQL query")
+
+    if output_code is not None:
+        return make_response(output, output_code)
+
+    adql_query = par_dic.get('adql_query', None)
+    vo_psql_pg_host = app_config.vo_psql_pg_host
+    vo_psql_pg_user = app_config.vo_psql_pg_user
+    vo_psql_pg_password = app_config.vo_psql_pg_password
+    vo_psql_pg_db = app_config.vo_psql_pg_db
+    product_gallery_url = app_config.product_gallery_url
+
+    result_query = ivoa_helper.run_ivoa_query(adql_query,
+                                              vo_psql_pg_host=vo_psql_pg_host,
+                                              vo_psql_pg_user=vo_psql_pg_user,
+                                              vo_psql_pg_password=vo_psql_pg_password,
+                                              vo_psql_pg_db=vo_psql_pg_db,
+                                              product_gallery_url=product_gallery_url)
+
+    return jsonify(result_query)
+
 
 @app.route('/run_analysis', methods=['POST', 'GET'])
 def run_analysis():

cdci_data_analysis/pytest_fixtures.py

@@ -611,6 +611,27 @@
     yield fn
 
 
+@pytest.fixture
+def dispatcher_test_conf_with_vo_options_fn(dispatcher_test_conf_fn):
+    fn = "test-dispatcher-conf-with-vo-options.yaml"
+
+    with open(fn, "w") as f:
+        with open(dispatcher_test_conf_fn) as f_default:
+            f.write(f_default.read())
+
+        f.write('\n    vo_options:'
+                '\n         vo_mysql_pg_host: "localhost"'
+                '\n         vo_mysql_pg_user: "user"'
+                '\n         vo_mysql_pg_password: "password"'
+                '\n         vo_mysql_pg_db: "database"'
+                '\n         vo_psql_pg_host: "localhost"'
+                '\n         vo_psql_pg_user: "user"'
+                '\n         vo_psql_pg_password: "password"'
+                '\n         vo_psql_pg_db: "database"')
+
+    yield fn
+
+
 @pytest.fixture
 def dispatcher_test_conf_with_matrix_options_fn(dispatcher_test_conf_fn):
     fn = "test-dispatcher-conf-with-matrix-options.yaml"
@@ -708,10 +729,16 @@
     yield yaml.load(open(dispatcher_test_conf_with_gallery_fn), Loader=yaml.SafeLoader)['dispatcher']
 
 
+@pytest.fixture
+def dispatcher_test_conf_with_vo_options(dispatcher_test_conf_with_vo_options_fn):
+    yield yaml.load(open(dispatcher_test_conf_with_vo_options_fn), Loader=yaml.SafeLoader)['dispatcher']
+
+
 @pytest.fixture
 def dispatcher_test_conf_with_matrix_options(dispatcher_test_conf_with_matrix_options_fn):
     yield yaml.load(open(dispatcher_test_conf_with_matrix_options_fn), Loader=yaml.SafeLoader)['dispatcher']
 
+
 @pytest.fixture
 def dispatcher_test_conf_with_gallery_no_resolver(dispatcher_test_conf_with_gallery_no_resolver_fn):
     yield yaml.load(open(dispatcher_test_conf_with_gallery_no_resolver_fn), Loader=yaml.SafeLoader)['dispatcher']

requirements.txt

@@ -4,6 +4,7 @@ pyyaml
 simplejson
 flask==2.0.3
 astropy>=5.0.1
+pytest-xdist<=3.5.0
 pylogstash_context>=0.1.19
 gunicorn
 decorator
@@ -29,6 +30,7 @@ GitPython
 nbformat
 sentry-sdk
 pytest-sentry
+queryparser-python3
 -e git+https://github.com/oda-hub/oda_api.git#egg=oda_api
 
 MarkupSafe==2.0.1

setup.py

@@ -45,11 +45,13 @@
     "black>=22.10.0",
     "bs4",
     "GitPython",
+    "queryparser-python3",
     "nbformat",
     "giturlparse",
     "sentry-sdk",
     "validators==0.28.3",
-    "jsonschema<=4.17.3"
+    "jsonschema<=4.17.3",
+    'psycopg2'
 ]
 
 test_req = [

tests/conftest.py

@@ -18,9 +18,11 @@
             gunicorn_dispatcher_long_living_fixture_with_matrix_options,
             dispatcher_test_conf,
             dispatcher_test_conf_with_gallery,
+            dispatcher_test_conf_with_vo_options,
             dispatcher_test_conf_with_gallery_no_resolver,
             dispatcher_test_conf_empty_sentry_fn,
             dispatcher_test_conf_with_gallery_fn,
+            dispatcher_test_conf_with_vo_options_fn,
             dispatcher_test_conf_with_gallery_no_resolver_fn,
             dispatcher_live_fixture_with_external_products_url,
             dispatcher_live_fixture_with_default_route_products_url,

tests/test_server_basic.py

@@ -2407,6 +2407,7 @@ def test_example_config(dispatcher_test_conf):
     example_config = yaml.load(open(example_config_fn), Loader=yaml.SafeLoader)['dispatcher']
     example_config.pop('product_gallery_options', None)
     example_config.pop('matrix_options', None)
+    example_config.pop('vo_options', None)
 
     mapper = lambda x, y: ".".join(map(str, x))
     example_config_keys = flatten_nested_structure(example_config, mapper)
@@ -2428,6 +2429,7 @@ def test_example_config_with_gallery(dispatcher_test_conf_with_gallery):
 
     example_config = yaml.load(open(example_config_fn), Loader=yaml.SafeLoader)['dispatcher']
     example_config.pop('matrix_options', None)
+    example_config.pop('vo_options', None)
 
     mapper = lambda x, y: ".".join(map(str, x))
     example_config_keys = flatten_nested_structure(example_config, mapper)
@@ -2449,6 +2451,7 @@ def test_example_config_with_matrix_options(dispatcher_test_conf_with_matrix_opt
     with open(example_config_fn) as example_config_fn_f:
         example_config = yaml.load(example_config_fn_f, Loader=yaml.SafeLoader)['dispatcher']
     example_config.pop('product_gallery_options', None)
+    example_config.pop('vo_options', None)
 
     mapper = lambda x, y: ".".join(map(str, x))
     example_config_keys = flatten_nested_structure(example_config, mapper)
@@ -2460,6 +2463,28 @@ def test_example_config_with_matrix_options(dispatcher_test_conf_with_matrix_opt
     assert set(example_config_keys) == set(test_config_keys)
 
 
+def test_example_config_with_vo_options(dispatcher_test_conf_with_vo_options):
+    import cdci_data_analysis.config_dir
+
+    example_config_fn = os.path.join(
+        os.path.dirname(cdci_data_analysis.__file__),
+        "config_dir/conf_env.yml.example"
+    )
+    with open(example_config_fn) as example_config_fn_f:
+        example_config = yaml.load(example_config_fn_f, Loader=yaml.SafeLoader)['dispatcher']
+    example_config.pop('product_gallery_options', None)
+    example_config.pop('matrix_options', None)
+
+    mapper = lambda x, y: ".".join(map(str, x))
+    example_config_keys = flatten_nested_structure(example_config, mapper)
+    test_config_keys = flatten_nested_structure(dispatcher_test_conf_with_vo_options, mapper)
+
+    print("\n\n\nexample_config_keys", example_config_keys)
+    print("\n\n\ntest_config_keys", test_config_keys)
+
+    assert set(example_config_keys) == set(test_config_keys)
+
+
 def test_image(dispatcher_live_fixture):
     server = dispatcher_live_fixture