diff --git a/configs/config-dev.json b/configs/config-dev.json
index ec0cf009..0104994f 100644
--- a/configs/config-dev.json
+++ b/configs/config-dev.json
@@ -5,13 +5,19 @@
"host": "server",
"modules": [
{
- "id": "iudx.resource.server.database.DatabaseVerticle",
+ "id": "iudx.resource.server.database.archives.DatabaseVerticle",
"verticleInstances": 2,
"databaseIP": "",
"databasePort": 123,
"dbUser": "",
"dbPassword": "",
- "timeLimit": "test,2020-10-22T00:00:00Z,30"
+ "resourceServerId":"",
+ "timeLimit": "",
+ "redisHost": "",
+ "redisPort": 1234,
+ "redisUser": "",
+ "redisPassword": "",
+ "attributeList": ""
},
{
"id": "iudx.resource.server.authenticator.AuthenticationVerticle",
@@ -82,6 +88,14 @@
"catServerHost": "",
"catServerPort": ""
+ },
+ {
+ "id": "iudx.resource.server.database.latest.LatestVerticle",
+ "redisHost": "",
+ "redisPort": 1234,
+ "redisUser": "",
+ "redisPassword": "",
+ "attributeList": {"itms-info": "some-attr"}
}
]
}
diff --git a/configs/config-test.json b/configs/config-test.json
index 3acdcc65..9b20a4b9 100644
--- a/configs/config-test.json
+++ b/configs/config-test.json
@@ -5,26 +5,24 @@
"host": "server",
"modules": [
{
- "id": "iudx.resource.server.database.DatabaseVerticle",
+ "id": "iudx.resource.server.database.archives.DatabaseVerticle",
"verticleInstances": 2,
"databaseIP": "",
- "databasePort": 123,
+ "databasePort": 12345,
"dbUser": "",
"dbPassword": "",
"resourceServerId":"",
- "timeLimit": "test,2020-10-22T00:00:00Z,30",
+ "timeLimit": "",
"testIdOpen":"",
"testIdSecure":"",
- "temporalStartDate":"2020-10-18T20:45:00+05:30",
- "temporalEndDate":"2020-10-19T20:45:00+05:30"
-
-
+ "temporalStartDate":"",
+ "temporalEndDate":""
},
{
"id": "iudx.resource.server.authenticator.AuthenticationVerticle",
"verticleInstances": 2,
- "keystore": "configs/keystore.jks",
+ "keystore": "",
"keystorePassword": "",
"authServerHost": "",
"testAuthToken": "",
@@ -32,8 +30,8 @@
"testResourceID": "",
"catServerHost": "",
"catServerPort": "",
- "serverMode": "production",
- "resourceServerId":"[[datakaveri.org/27e503da0bdda6efae3a52b3ef423c1f9005657a/rs.iudx.org.in]]"
+ "serverMode": "",
+ "resourceServerId":""
},
{
"id": "iudx.resource.server.databroker.DataBrokerVerticle",
@@ -44,18 +42,18 @@
"dataBrokerUserName": "",
"dataBrokerPassword": "",
"dataBrokerManagementPort": "",
- "connectionTimeout": "6000",
- "requestedHeartbeat": "60",
- "handshakeTimeout": "6000",
- "requestedChannelMax": "5",
- "networkRecoveryInterval": "500",
- "automaticRecoveryEnabled": "true",
+ "connectionTimeout": "",
+ "requestedHeartbeat": "",
+ "handshakeTimeout": "",
+ "requestedChannelMax": "",
+ "networkRecoveryInterval": "",
+ "automaticRecoveryEnabled": "",
"callbackDatabaseIP": "",
"callbackDatabasePort": "",
"callbackDatabaseName": "",
"callbackDatabaseUserName": "",
"callbackDatabasePassword": "",
- "callbackpoolSize": "25",
+ "callbackpoolSize": "",
"testResourceGroup":"",
"testResourceServer":"",
@@ -71,24 +69,24 @@
"dataBrokerUserName": "",
"dataBrokerPassword": "",
"dataBrokerManagementPort": "",
- "connectionTimeout": "6000",
- "requestedHeartbeat": "60",
- "handshakeTimeout": "6000",
- "requestedChannelMax": "5",
- "networkRecoveryInterval": "500",
- "automaticRecoveryEnabled": "true",
+ "connectionTimeout": "",
+ "requestedHeartbeat": "",
+ "handshakeTimeout": "",
+ "requestedChannelMax": "",
+ "networkRecoveryInterval": "",
+ "automaticRecoveryEnabled": "",
"callbackDatabaseIP": "",
"callbackDatabasePort": "",
"callbackDatabaseName": "",
"callbackDatabaseUserName": "",
"callbackDatabasePassword": "",
- "callbackpoolSize": "25"
+ "callbackpoolSize": ""
},
{
"id": "iudx.resource.server.apiserver.ApiServerVerticle",
"ssl": true,
"production": true,
- "keystore": "configs/keystore.jks",
+ "keystore": "",
"keystorePassword": "",
"rsAdmin": "",
"verticleInstances": 2,
@@ -101,11 +99,20 @@
"lineCoords":"",
"temporalTime":"",
"temporalEndTime":"",
- "qparamGreaterThan":"speed>30",
- "qparamLessThan":"speed<500",
- "qparamGreaterEquals":"speed>=30",
- "qparamLessEquals":"speed<=50"
+ "qparamGreaterThan":"",
+ "qparamLessThan":"",
+ "qparamGreaterEquals":"",
+ "qparamLessEquals":""
+ },
+ {
+ "id": "iudx.resource.server.database.latest.LatestVerticle",
+ "verticleInstances": 2,
+ "redisHost": "",
+ "redisPort": 12345,
+ "redisUser": "",
+ "redisPassword": "",
+ "attributeList": {"key":"value"}
}
]
-}
+}
\ No newline at end of file
diff --git a/docs/rs_overview.png b/docs/rs_overview.png
index 6a18043e..d8f8b809 100644
Binary files a/docs/rs_overview.png and b/docs/rs_overview.png differ
diff --git a/pom.xml b/pom.xml
index 1c5b908e..0f39bc67 100644
--- a/pom.xml
+++ b/pom.xml
@@ -158,6 +158,18 @@
jts2geojson0.14.1
+
+
+
+
+
+
+
+ com.redislabs
+ jrejson
+ 1.3.0
+
+
@@ -303,6 +315,9 @@
1111
+
+ io.vertx.codegen.CodeGenProcessor
+
diff --git a/src/main/java/iudx/resource/server/apiserver/ApiServerVerticle.java b/src/main/java/iudx/resource/server/apiserver/ApiServerVerticle.java
index 3c078a53..89890088 100644
--- a/src/main/java/iudx/resource/server/apiserver/ApiServerVerticle.java
+++ b/src/main/java/iudx/resource/server/apiserver/ApiServerVerticle.java
@@ -47,7 +47,8 @@
import iudx.resource.server.apiserver.validation.ValidationFailureHandler;
import iudx.resource.server.apiserver.validation.HTTPRequestValidatiorsHandlersFactory;
import iudx.resource.server.authenticator.AuthenticationService;
-import iudx.resource.server.database.DatabaseService;
+import iudx.resource.server.database.archives.DatabaseService;
+import iudx.resource.server.database.latest.LatestDataService;
import iudx.resource.server.databroker.DataBrokerService;
@@ -78,6 +79,7 @@ public class ApiServerVerticle extends AbstractVerticle {
private static final String DATABASE_SERVICE_ADDRESS = "iudx.rs.database.service";
private static final String AUTH_SERVICE_ADDRESS = "iudx.rs.authentication.service";
private static final String BROKER_SERVICE_ADDRESS = "iudx.rs.broker.service";
+ private static final String LATEST_SEARCH_ADDRESS = "iudx.rs.latest.service";
private HttpServer server;
private Router router;
@@ -93,6 +95,8 @@ public class ApiServerVerticle extends AbstractVerticle {
private DataBrokerService databroker;
private AuthenticationService authenticator;
private Validator validator;
+
+ private LatestDataService latestDataService;
/**
* This method is used to start the Verticle. It deploys a verticle in a cluster, reads the
@@ -276,6 +280,7 @@ public void start() throws Exception {
databroker = DataBrokerService.createProxy(vertx, BROKER_SERVICE_ADDRESS);
+ latestDataService = LatestDataService.createProxy(vertx, LATEST_SEARCH_ADDRESS);
managementApi = new ManagementApiImpl();
subsService = new SubscriptionService();
@@ -319,7 +324,7 @@ private void handleLatestEntitiesQuery(RoutingContext routingContext) {
filtersFuture.onComplete(filtersHandler -> {
if (filtersHandler.succeeded()) {
json.put("applicableFilters", filtersHandler.result());
- executeSearchQuery(json, response);
+ executeLatestSearchQuery(json, response);
} else {
LOGGER.error("catalogue item/group doesn't have filters.");
handleResponse(response, ResponseType.BadRequestData,
@@ -475,6 +480,19 @@ private void executeSearchQuery(JsonObject json, HttpServerResponse response) {
});
}
+ private void executeLatestSearchQuery(JsonObject json, HttpServerResponse response) {
+ latestDataService.getLatestData(json, handler -> {
+ if (handler.succeeded()) {
+ LOGGER.info("Latest data search succeeded");
+ handleSuccessResponse(response, ResponseType.Ok.getCode(),handler.result().toString());
+ } else {
+ LOGGER.error("Fail: Search Fail");
+ processBackendResponse(response, handler.cause().getMessage());
+ }
+ });
+ }
+
+
/**
* This method is used to handler all temporal NGSI-LD queries for endpoint
* /ngsi-ld/v1/temporal/**.
diff --git a/src/main/java/iudx/resource/server/apiserver/subscription/CallbackSubscription.java b/src/main/java/iudx/resource/server/apiserver/subscription/CallbackSubscription.java
index 6ac1f906..f6612ae6 100644
--- a/src/main/java/iudx/resource/server/apiserver/subscription/CallbackSubscription.java
+++ b/src/main/java/iudx/resource/server/apiserver/subscription/CallbackSubscription.java
@@ -5,7 +5,7 @@
import io.vertx.core.json.JsonObject;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import iudx.resource.server.database.DatabaseService;
+import iudx.resource.server.database.archives.DatabaseService;
import iudx.resource.server.databroker.DataBrokerService;
/**
diff --git a/src/main/java/iudx/resource/server/apiserver/subscription/StreamingSubscription.java b/src/main/java/iudx/resource/server/apiserver/subscription/StreamingSubscription.java
index 411294f9..951465e1 100644
--- a/src/main/java/iudx/resource/server/apiserver/subscription/StreamingSubscription.java
+++ b/src/main/java/iudx/resource/server/apiserver/subscription/StreamingSubscription.java
@@ -6,7 +6,7 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import iudx.resource.server.apiserver.util.Constants;
-import iudx.resource.server.database.DatabaseService;
+import iudx.resource.server.database.archives.DatabaseService;
import iudx.resource.server.databroker.DataBrokerService;
/**
diff --git a/src/main/java/iudx/resource/server/apiserver/subscription/SubscriptionService.java b/src/main/java/iudx/resource/server/apiserver/subscription/SubscriptionService.java
index d2c15b6b..b5767773 100644
--- a/src/main/java/iudx/resource/server/apiserver/subscription/SubscriptionService.java
+++ b/src/main/java/iudx/resource/server/apiserver/subscription/SubscriptionService.java
@@ -10,7 +10,7 @@
import org.apache.logging.log4j.Logger;
import iudx.resource.server.apiserver.response.ResponseType;
import iudx.resource.server.apiserver.util.Constants;
-import iudx.resource.server.database.DatabaseService;
+import iudx.resource.server.database.archives.DatabaseService;
import iudx.resource.server.databroker.DataBrokerService;
/**
diff --git a/src/main/java/iudx/resource/server/database/Constants.java b/src/main/java/iudx/resource/server/database/Constants.java
deleted file mode 100644
index 596c2668..00000000
--- a/src/main/java/iudx/resource/server/database/Constants.java
+++ /dev/null
@@ -1,131 +0,0 @@
-package iudx.resource.server.database;
-
-public class Constants {
- /* General Purpose */
- static final String SEARCH_TYPE = "searchType";
- static final String ID = "id";
- static final String RESOURCE_ID_KEY = "id";
- static final String PROD_INSTANCE = "production";
- static final String TEST_INSTANCE = "test";
- /* Database */
- static final String GEO_KEY = "location";
- static final String GEO_CIRCLE = "circle";
- static final String GEO_BBOX = "envelope";
- static final String COORDINATES_KEY = "coordinates";
- static final String GEO_RELATION_KEY = "relation";
- static final String TYPE_KEY = "type";
- static final String GEO_SHAPE_KEY = "geo_shape";
- static final String GEO_RADIUS = "radius";
- static final String SHAPE_KEY = "shape";
- static final String QUERY_KEY = "query";
- static final String FILTER_KEY = "filter";
- static final String BOOL_KEY = "bool";
- static final String VARANASI_TEST_SEARCH_INDEX = "varanasi/_search";
- static final String VARANASI_TEST_COUNT_INDEX = "varanasi/_count";
- static final String LATEST_RESOURCE_INDEX = "latest/_mget";
- static final String SOURCE_FILTER_KEY = "_source";
- static final String RANGE_KEY = "range";
- static final String TERM_KEY = "term";
- static final String TERMS_KEY = "terms";
- static final String FILTER_PATH = "filter_path";
- static final String FILTER_PATH_VAL = "took,hits.hits._source";
- static final String FILTER_PATH_VAL_LATEST = "docs._source";
- static final String SIZE_KEY = "size";
- static final String GREATER_THAN = "gt";
- static final String LESS_THAN = "lt";
- static final String GREATER_THAN_EQ = "gte";
- static final String LESS_THAN_EQ = "lte";
- static final String MUST_NOT = "must_not";
- static final String REQUEST_GET = "GET";
- static final String HITS = "hits";
- static final String SEARCH_KEY = "search";
- static final String ERROR = "Error";
- static final String COUNT = "count";
- static final String DOC_ID = "_id";
- static final String DOCS_KEY = "docs";
- static final String SEARCH_REQ_PARAM = "/_search";
- static final String COUNT_REQ_PARAM = "/_count";
- static final String TIME_FIELD_DB = "observationDateTime";
- /* Request Params */
- /* Temporal */
- static final String REQ_TIMEREL = "timerel";
- static final String TIME_KEY = "time";
- static final String END_TIME = "endtime";
- static final String DURING = "during";
- static final String AFTER = "after";
- static final String BEFORE = "before";
- static final String TEQUALS = "tequals";
- static final String TIME_LIMIT = "timeLimit";
- /* Geo-Spatial */
- static final String LAT = "lat";
- static final String LON = "lon";
- static final String GEOMETRY = "geometry";
- static final String GEOREL = "georel";
- static final String WITHIN = "within";
- static final String POLYGON = "polygon";
- static final String LINESTRING = "linestring";
- static final String GEO_PROPERTY = "geoproperty";
- static final String BBOX = "bbox";
- /* Response Filter */
- static final String RESPONSE_ATTRS = "attrs";
- /* Attribute */
- static final String ATTRIBUTE_QUERY_KEY = "attr-query";
- static final String ATTRIBUTE_KEY = "attribute";
- static final String OPERATOR = "operator";
- static final String VALUE = "value";
- static final String VALUE_LOWER = "valueLower";
- static final String VALUE_UPPER = "valueUpper";
- static final String GREATER_THAN_OP = ">";
- static final String LESS_THAN_OP = "<";
- static final String GREATER_THAN_EQ_OP = ">=";
- static final String LESS_THAN_EQ_OP = "<=";
- static final String EQUAL_OP = "==";
- static final String NOT_EQUAL_OP = "!=";
- static final String BETWEEN_OP = "<==>";
- /* Errors */
- static final String INVALID_OPERATOR = "Invalid operator";
- static final String INVALID_SEARCH = "Invalid search request";
- static final String INVALID_DATE = "Invalid date format";
- static final String MISSING_ATTRIBUTE_FIELDS = "Missing attribute query fields";
- static final String MISSING_TEMPORAL_FIELDS = "Missing/Invalid temporal parameters";
- static final String MISSING_RESPONSE_FILTER_FIELDS = "Missing/Invalid responseFilter parameters";
- static final String MISSING_GEO_FIELDS = "Missing/Invalid geo parameters";
- static final String COORDINATE_MISMATCH = "Coordinate mismatch (Polygon)";
- static final String COUNT_UNSUPPORTED = "Count is not supported with filtering";
- static final String EMPTY_RESPONSE = "Empty response";
- static final String DB_ERROR = "DB request has failed";
- static final String DB_ERROR_2XX = "Status code is not 2xx";
- static final String ID_NOT_FOUND = "No id found";
- static final String EMPTY_RESOURCE_ID = "resource-id is empty";
- static final String SEARCHTYPE_NOT_FOUND = "No searchType found";
- static final String BAD_PARAMETERS = "Bad parameters";
- static final String ERROR_TYPE = "type";
- static final String SUCCESS = "Success";
- static final String FAILED = "Failed";
- static final String TITLE = "title";
- static final String RESULTS = "results";
- static final String DETAIL = "detail";
- static final String ROOT_CAUSE = "root_cause";
- static final String REASON = "reason";
- static final String MALFORMED_ID = "Malformed Id ";
- static final String STATUS = "status";
- static final String INDEX_NOT_FOUND = "index_not_found_exception";
- static final String INVALID_RESOURCE_ID = "Invalid resource id";
- /* Search Regex */
- static final String GEOSEARCH_REGEX = "(.*)geoSearch(.*)";
- static final String RESPONSE_FILTER_REGEX = "(.*)responseFilter(.*)";
- public static final String ATTRIBUTE_SEARCH_REGEX = "(.*)attributeSearch(.*)";
- public static final String TEMPORAL_SEARCH_REGEX = "(.*)temporalSearch(.*)";
- static final String LATEST_SEARCH = "latestSearch";
- /* Query templates */
- static final String GEO_SHAPE_QUERY =
- "{ \"geo_shape\": { \"$4\": { \"shape\": { \"type\": \"$1\", \"coordinates\": $2 },"
- + " \"relation\": \"$3\" } } }";
- static final String TIME_QUERY = "{\"range\":{\"observationDateTime\":{\"$1\":\"$2\"}}}";
- static final String TERM_QUERY = "{\"term\":{\"$1\":\"$2\"}}";
- static final String TERMS_QUERY = "{\"terms\":{\"$1\":$2}}";
- static final String RANGE_QUERY = "{\"range\":{\"$1\":{\"$2\":$3}}}";
- static final String RANGE_QUERY_BW = "{\"range\":{\"$1\":{\"$2\":$3,\"$4\":$5}}}";
- public static final String MUST_NOT_QUERY = "{\"must_not\":[$1]}";
-
-}
diff --git a/src/main/java/iudx/resource/server/database/archives/Constants.java b/src/main/java/iudx/resource/server/database/archives/Constants.java
new file mode 100644
index 00000000..1b7d6b90
--- /dev/null
+++ b/src/main/java/iudx/resource/server/database/archives/Constants.java
@@ -0,0 +1,146 @@
+package iudx.resource.server.database.archives;
+
+public class Constants {
+ /* General Purpose */
+ public static final String SEARCH_TYPE = "searchType";
+ public static final String ID = "id";
+ public static final String RESOURCE_ID_KEY = "id";
+ public static final String PROD_INSTANCE = "production";
+ public static final String TEST_INSTANCE = "test";
+ /* Database */
+ public static final String GEO_KEY = "location";
+ public static final String GEO_CIRCLE = "circle";
+ public static final String GEO_BBOX = "envelope";
+ public static final String COORDINATES_KEY = "coordinates";
+ public static final String GEO_RELATION_KEY = "relation";
+ public static final String TYPE_KEY = "type";
+ public static final String GEO_SHAPE_KEY = "geo_shape";
+ public static final String GEO_RADIUS = "radius";
+ public static final String SHAPE_KEY = "shape";
+ public static final String QUERY_KEY = "query";
+ public static final String FILTER_KEY = "filter";
+ public static final String BOOL_KEY = "bool";
+ public static final String VARANASI_TEST_SEARCH_INDEX = "varanasi/_search";
+ public static final String VARANASI_TEST_COUNT_INDEX = "varanasi/_count";
+ public static final String LATEST_RESOURCE_INDEX = "latest/_mget";
+ public static final String SOURCE_FILTER_KEY = "_source";
+ public static final String RANGE_KEY = "range";
+ public static final String TERM_KEY = "term";
+ public static final String TERMS_KEY = "terms";
+ public static final String FILTER_PATH = "filter_path";
+ public static final String FILTER_PATH_VAL = "took,hits.hits._source";
+ public static final String FILTER_PATH_VAL_LATEST = "docs._source";
+ public static final String SIZE_KEY = "size";
+ public static final String GREATER_THAN = "gt";
+ public static final String LESS_THAN = "lt";
+ public static final String GREATER_THAN_EQ = "gte";
+ public static final String LESS_THAN_EQ = "lte";
+ public static final String MUST_NOT = "must_not";
+ public static final String REQUEST_GET = "GET";
+ public static final String HITS = "hits";
+ public static final String SEARCH_KEY = "search";
+ public static final String ERROR = "Error";
+ public static final String COUNT = "count";
+ public static final String DOC_ID = "_id";
+ public static final String DOCS_KEY = "docs";
+ public static final String SEARCH_REQ_PARAM = "/_search";
+ public static final String COUNT_REQ_PARAM = "/_count";
+ public static final String TIME_FIELD_DB = "observationDateTime";
+ /* Request Params */
+ /* Temporal */
+ public static final String REQ_TIMEREL = "timerel";
+ public static final String TIME_KEY = "time";
+ public static final String END_TIME = "endtime";
+ public static final String DURING = "during";
+ public static final String AFTER = "after";
+ public static final String BEFORE = "before";
+ public static final String TEQUALS = "tequals";
+ public static final String TIME_LIMIT = "timeLimit";
+ /* Geo-Spatial */
+ public static final String LAT = "lat";
+ public static final String LON = "lon";
+ public static final String GEOMETRY = "geometry";
+ public static final String GEOREL = "georel";
+ public static final String WITHIN = "within";
+ public static final String POLYGON = "polygon";
+ public static final String LINESTRING = "linestring";
+ public static final String GEO_PROPERTY = "geoproperty";
+ public static final String BBOX = "bbox";
+ /* Response Filter */
+ public static final String RESPONSE_ATTRS = "attrs";
+ /* Attribute */
+ public static final String ATTRIBUTE_QUERY_KEY = "attr-query";
+ public static final String ATTRIBUTE_KEY = "attribute";
+ public static final String OPERATOR = "operator";
+ public static final String VALUE = "value";
+ public static final String VALUE_LOWER = "valueLower";
+ public static final String VALUE_UPPER = "valueUpper";
+ public static final String GREATER_THAN_OP = ">";
+ public static final String LESS_THAN_OP = "<";
+ public static final String GREATER_THAN_EQ_OP = ">=";
+ public static final String LESS_THAN_EQ_OP = "<=";
+ public static final String EQUAL_OP = "==";
+ public static final String NOT_EQUAL_OP = "!=";
+ public static final String BETWEEN_OP = "<==>";
+ /* Errors */
+ public static final String INVALID_OPERATOR = "Invalid operator";
+ public static final String INVALID_SEARCH = "Invalid search request";
+ public static final String INVALID_DATE = "Invalid date format";
+ public static final String MISSING_ATTRIBUTE_FIELDS = "Missing attribute query fields";
+ public static final String MISSING_TEMPORAL_FIELDS = "Missing/Invalid temporal parameters";
+ public static final String MISSING_RESPONSE_FILTER_FIELDS = "Missing/Invalid responseFilter parameters";
+ public static final String MISSING_GEO_FIELDS = "Missing/Invalid geo parameters";
+ public static final String COORDINATE_MISMATCH = "Coordinate mismatch (Polygon)";
+ public static final String COUNT_UNSUPPORTED = "Count is not supported with filtering";
+ public static final String EMPTY_RESPONSE = "Empty response";
+ public static final String DB_ERROR = "DB request has failed";
+ public static final String DB_ERROR_2XX = "Status code is not 2xx";
+ public static final String ID_NOT_FOUND = "No id found";
+ public static final String EMPTY_RESOURCE_ID = "resource-id is empty";
+ public static final String SEARCHTYPE_NOT_FOUND = "No searchType found";
+ public static final String BAD_PARAMETERS = "Bad parameters";
+ public static final String ERROR_TYPE = "type";
+ public static final String SUCCESS = "Success";
+ public static final String FAILED = "Failed";
+ public static final String TITLE = "title";
+ public static final String RESULTS = "results";
+ public static final String DETAIL = "detail";
+ public static final String ROOT_CAUSE = "root_cause";
+ public static final String REASON = "reason";
+ public static final String MALFORMED_ID = "Malformed Id ";
+ public static final String STATUS = "status";
+ public static final String INDEX_NOT_FOUND = "index_not_found_exception";
+ public static final String INVALID_RESOURCE_ID = "Invalid resource id";
+ /* Search Regex */
+ public static final String GEOSEARCH_REGEX = "(.*)geoSearch(.*)";
+ public static final String RESPONSE_FILTER_REGEX = "(.*)responseFilter(.*)";
+ public static final String ATTRIBUTE_SEARCH_REGEX = "(.*)attributeSearch(.*)";
+ public static final String TEMPORAL_SEARCH_REGEX = "(.*)temporalSearch(.*)";
+ public static final String LATEST_SEARCH = "latestSearch";
+ /* Query templates */
+ public static final String GEO_SHAPE_QUERY =
+ "{ \"geo_shape\": { \"$4\": { \"shape\": { \"type\": \"$1\", \"coordinates\": $2 },"
+ + " \"relation\": \"$3\" } } }";
+ public static final String TIME_QUERY = "{\"range\":{\"observationDateTime\":{\"$1\":\"$2\"}}}";
+ public static final String TERM_QUERY = "{\"term\":{\"$1\":\"$2\"}}";
+ public static final String TERMS_QUERY = "{\"terms\":{\"$1\":$2}}";
+ public static final String RANGE_QUERY = "{\"range\":{\"$1\":{\"$2\":$3}}}";
+ public static final String RANGE_QUERY_BW = "{\"range\":{\"$1\":{\"$2\":$3,\"$4\":$5}}}";
+ public static final String MUST_NOT_QUERY = "{\"must_not\":[$1]}";
+ /* Latest Data Params */
+ public static final String OPTIONS_NOT_FOUND = "options not found";
+ public static final String OPTIONS = "options";
+ public static final String EMPTY_OPTIONS = "options is empty";
+ public static final String LATEST_DATA_SERVICE_ADDRESS = "iudx.rs.latest.service";
+ public static final String ATTRIBUTE_LIST = "attributeList";
+ public static final String DEFAULT_ATTRIBUTE = "_d";
+ public static final String KEY = "key";
+ public static final String PATH_PARAM = "pathParam";
+ public static final String GROUP = "group";
+ public static final String INVALID_LATEST_QUERY = "invalid latest params";
+ public static final String ATTRIBUTE_LIST_NOT_FOUND = "key [attributeList] not found";
+ public static final String REDIS_ERROR = "Redis Error!";
+ public static final String INVALID_OPTIONS = "invalid options for latest";
+ // needs modification depending on the actual error returned from Redis
+ public static final String ID_NOT_PRESENT = "Not found";
+}
diff --git a/src/main/java/iudx/resource/server/database/DatabaseService.java b/src/main/java/iudx/resource/server/database/archives/DatabaseService.java
similarity index 97%
rename from src/main/java/iudx/resource/server/database/DatabaseService.java
rename to src/main/java/iudx/resource/server/database/archives/DatabaseService.java
index e60a40c7..eb6afb29 100644
--- a/src/main/java/iudx/resource/server/database/DatabaseService.java
+++ b/src/main/java/iudx/resource/server/database/archives/DatabaseService.java
@@ -1,4 +1,4 @@
-package iudx.resource.server.database;
+package iudx.resource.server.database.archives;
import io.vertx.codegen.annotations.Fluent;
import io.vertx.codegen.annotations.GenIgnore;
diff --git a/src/main/java/iudx/resource/server/database/DatabaseServiceImpl.java b/src/main/java/iudx/resource/server/database/archives/DatabaseServiceImpl.java
similarity index 97%
rename from src/main/java/iudx/resource/server/database/DatabaseServiceImpl.java
rename to src/main/java/iudx/resource/server/database/archives/DatabaseServiceImpl.java
index a588ff55..b3ce7855 100644
--- a/src/main/java/iudx/resource/server/database/DatabaseServiceImpl.java
+++ b/src/main/java/iudx/resource/server/database/archives/DatabaseServiceImpl.java
@@ -1,22 +1,22 @@
-package iudx.resource.server.database;
+package iudx.resource.server.database.archives;
import io.vertx.core.AsyncResult;
import io.vertx.core.Future;
import io.vertx.core.Handler;
import io.vertx.core.json.JsonObject;
+import static iudx.resource.server.database.archives.Constants.*;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import static iudx.resource.server.database.Constants.*;
/**
* The Database Service Implementation.
*
Database Service Implementation
*
* The Database Service implementation in the IUDX Resource Server implements the definitions of the
- * {@link iudx.resource.server.database.DatabaseService}.
+ * {@link iudx.resource.server.database.archives.DatabaseService}.
*
* The Database Verticle implementation in the the IUDX Resource Server exposes the
- * {@link iudx.resource.server.database.DatabaseService} over the Vert.x Event Bus.
+ * {@link iudx.resource.server.database.archives.DatabaseService} over the Vert.x Event Bus.
*
*
* @version 1.0
diff --git a/src/main/java/iudx/resource/server/database/ElasticClient.java b/src/main/java/iudx/resource/server/database/archives/ElasticClient.java
similarity index 96%
rename from src/main/java/iudx/resource/server/database/ElasticClient.java
rename to src/main/java/iudx/resource/server/database/archives/ElasticClient.java
index 74f7618d..507ab923 100755
--- a/src/main/java/iudx/resource/server/database/ElasticClient.java
+++ b/src/main/java/iudx/resource/server/database/archives/ElasticClient.java
@@ -1,4 +1,4 @@
-package iudx.resource.server.database;
+package iudx.resource.server.database.archives;
import io.vertx.core.AsyncResult;
import io.vertx.core.Future;
@@ -6,6 +6,7 @@
import io.vertx.core.json.DecodeException;
import io.vertx.core.json.JsonArray;
import io.vertx.core.json.JsonObject;
+import static iudx.resource.server.database.archives.Constants.*;
import java.io.IOException;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
@@ -20,8 +21,6 @@
import org.elasticsearch.client.ResponseListener;
import org.elasticsearch.client.RestClient;
-import static iudx.resource.server.database.Constants.*;
-
public class ElasticClient {
private final RestClient client;
diff --git a/src/main/java/iudx/resource/server/database/QueryDecoder.java b/src/main/java/iudx/resource/server/database/archives/QueryDecoder.java
similarity index 81%
rename from src/main/java/iudx/resource/server/database/QueryDecoder.java
rename to src/main/java/iudx/resource/server/database/archives/QueryDecoder.java
index 455bec9c..5192a830 100755
--- a/src/main/java/iudx/resource/server/database/QueryDecoder.java
+++ b/src/main/java/iudx/resource/server/database/archives/QueryDecoder.java
@@ -1,15 +1,14 @@
-package iudx.resource.server.database;
+package iudx.resource.server.database.archives;
import io.vertx.core.json.JsonArray;
import io.vertx.core.json.JsonObject;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import static iudx.resource.server.database.archives.Constants.*;
import java.time.ZonedDateTime;
import java.time.format.DateTimeParseException;
import org.apache.commons.codec.digest.DigestUtils;
-import static iudx.resource.server.database.Constants.*;
-
public class QueryDecoder {
private static final Logger LOGGER = LogManager.getLogger(QueryDecoder.class);
@@ -30,12 +29,6 @@ public JsonObject queryDecoder(JsonObject request) {
JsonArray filterQuery = new JsonArray();
String queryGeoShape = null;
- // Time Object (for limiting query based on time parameters) instantiated to
- // null;
- JsonObject timeObject = null;
- String timeLimit = request.getString(TIME_LIMIT).split(",")[1];
- int numDays = Integer.valueOf(request.getString(TIME_LIMIT).split(",")[2]);
-
JsonObject boolObject = new JsonObject().put(BOOL_KEY, new JsonObject());
filterQuery
.add(new JsonObject(TERMS_QUERY.replace("$1", RESOURCE_ID_KEY).replace("$2", id.encode())));
@@ -47,24 +40,72 @@ public JsonObject queryDecoder(JsonObject request) {
/* Latest Search */
if (LATEST_SEARCH.equalsIgnoreCase(searchType)) {
- JsonArray sourceFilter = null;
- if (request.containsKey(RESPONSE_ATTRS)) {
- sourceFilter = request.getJsonArray(RESPONSE_ATTRS);
+// JsonArray sourceFilter = null;
+// if (request.containsKey(RESPONSE_ATTRS)) {
+// sourceFilter = request.getJsonArray(RESPONSE_ATTRS);
+// }
+// JsonObject latestQuery = new JsonObject();
+// JsonArray docs = new JsonArray();
+// for (Object o : id) {
+// String resourceString = (String) o;
+// String sha1String = DigestUtils.sha1Hex(resourceString);
+// JsonObject json = new JsonObject().put(DOC_ID, sha1String);
+// if (sourceFilter != null) {
+// json.put(SOURCE_FILTER_KEY, sourceFilter);
+// }
+// docs.add(json);
+// }
+// return latestQuery.put(DOCS_KEY, docs);
+
+ // Redis Latest
+
+ LOGGER.debug("******In LatestSearch Redis");
+ String resourceId = id.getString(0);
+ String options = request.getString(OPTIONS);
+ String resourceGroup = resourceId.split("/")[3];
+ resourceGroup = resourceGroup.replace("-","_");
+ if (!request.containsKey(ATTRIBUTE_LIST)) {
+ return new JsonObject().put(ERROR, ATTRIBUTE_LIST_NOT_FOUND);
}
- JsonObject latestQuery = new JsonObject();
- JsonArray docs = new JsonArray();
- for (Object o : id) {
- String resourceString = (String) o;
- String sha1String = DigestUtils.sha1Hex(resourceString);
- JsonObject json = new JsonObject().put(DOC_ID, sha1String);
- if (sourceFilter != null) {
- json.put(SOURCE_FILTER_KEY, sourceFilter);
- }
- docs.add(json);
- }
- return latestQuery.put(DOCS_KEY, docs);
+ JsonObject attributeList = request.getJsonObject(ATTRIBUTE_LIST);
+
+ // SHA1 generator
+ String sha1String = DigestUtils.sha1Hex(resourceId);
+ LOGGER.debug("Generated SHA1: "+sha1String);
+ // read attributeList from request
+ String pathParam;
+ //id query
+
+ // if (ID.equalsIgnoreCase(options)) {
+// if (!attributeList.containsKey(resourceGroup))
+// // aqm/flood type sensor
+// pathParam = resourceGroup.concat("._").concat(sha1String).concat(DEFAULT_ATTRIBUTE);
+// else
+// // itms type sensor
+// pathParam = resourceGroup.concat("._").concat(sha1String).concat(attributeList.getString(resourceGroup));
+ pathParam = resourceGroup.concat("._").concat(sha1String).concat(DEFAULT_ATTRIBUTE);
+ LOGGER.debug("PathParam: "+pathParam);
+ return new JsonObject().put(KEY, resourceGroup).put(PATH_PARAM, pathParam);
+ //}
+ // group query
+
+// else if (GROUP.equalsIgnoreCase(options)) {
+// pathParam = ".".concat(resourceGroup);
+// LOGGER.debug("PathParam: "+pathParam);
+// return new JsonObject().put(KEY, resourceGroup).put(PATH_PARAM, pathParam);
+// }
+// else {
+// // failed query
+// return new JsonObject().put(ERROR, INVALID_LATEST_QUERY);
+// }
}
+ // Time Object (for limiting query based on time parameters) instantiated to
+ // null;
+ JsonObject timeObject = null;
+ String timeLimit = request.getString(TIME_LIMIT).split(",")[1];
+ int numDays = Integer.valueOf(request.getString(TIME_LIMIT).split(",")[2]);
+
/* Geo-Spatial Search */
if (searchType.matches(GEOSEARCH_REGEX)) {
diff --git a/src/main/java/iudx/resource/server/database/ResponseBuilder.java b/src/main/java/iudx/resource/server/database/archives/ResponseBuilder.java
similarity index 79%
rename from src/main/java/iudx/resource/server/database/ResponseBuilder.java
rename to src/main/java/iudx/resource/server/database/archives/ResponseBuilder.java
index d74f8505..a23136b3 100644
--- a/src/main/java/iudx/resource/server/database/ResponseBuilder.java
+++ b/src/main/java/iudx/resource/server/database/archives/ResponseBuilder.java
@@ -1,10 +1,9 @@
-package iudx.resource.server.database;
+package iudx.resource.server.database.archives;
+import static iudx.resource.server.database.archives.Constants.*;
import io.vertx.core.json.JsonArray;
import io.vertx.core.json.JsonObject;
-import static iudx.resource.server.database.Constants.*;
-
public class ResponseBuilder {
private String status;
@@ -12,12 +11,12 @@ public class ResponseBuilder {
/** Initialise the object with Success or Failure. */
- ResponseBuilder(String status) {
+ public ResponseBuilder(String status) {
this.status = status;
response = new JsonObject();
}
- ResponseBuilder setTypeAndTitle(int statusCode) {
+ public ResponseBuilder setTypeAndTitle(int statusCode) {
response.put(ERROR_TYPE, statusCode);
if (SUCCESS.equalsIgnoreCase(status)) {
response.put(TITLE, SUCCESS);
@@ -29,14 +28,14 @@ ResponseBuilder setTypeAndTitle(int statusCode) {
/** Successful Database Request with responses > 0. */
- ResponseBuilder setMessage(JsonArray results) {
+ public ResponseBuilder setMessage(JsonArray results) {
response.put(RESULTS, results);
return this;
}
/** Overloaded methods for Error messages. */
- ResponseBuilder setMessage(String error) {
+ public ResponseBuilder setMessage(String error) {
response.put(DETAIL, error);
return this;
}
@@ -59,7 +58,7 @@ ResponseBuilder setCount(int count) {
return this;
}
- JsonObject getResponse() {
+ public JsonObject getResponse() {
return response;
}
}
diff --git a/src/main/java/iudx/resource/server/database/archives/package-info.java b/src/main/java/iudx/resource/server/database/archives/package-info.java
new file mode 100644
index 00000000..403bbade
--- /dev/null
+++ b/src/main/java/iudx/resource/server/database/archives/package-info.java
@@ -0,0 +1,6 @@
+@ModuleGen(groupPackage = "iudx.resource.server.database.archives",
+ name = "iudx-resource-server-database-archives")
+
+package iudx.resource.server.database.archives;
+
+import io.vertx.codegen.annotations.ModuleGen;
diff --git a/src/main/java/iudx/resource/server/database/latest/LatestDataService.java b/src/main/java/iudx/resource/server/database/latest/LatestDataService.java
new file mode 100644
index 00000000..98fcb1d2
--- /dev/null
+++ b/src/main/java/iudx/resource/server/database/latest/LatestDataService.java
@@ -0,0 +1,56 @@
+package iudx.resource.server.database.latest;
+
+import io.vertx.codegen.annotations.Fluent;
+import io.vertx.codegen.annotations.GenIgnore;
+import io.vertx.codegen.annotations.ProxyGen;
+import io.vertx.codegen.annotations.VertxGen;
+import io.vertx.core.AsyncResult;
+import io.vertx.core.Handler;
+import io.vertx.core.Vertx;
+import io.vertx.core.json.JsonObject;
+
+/**
+ * The Latest Data Service.
+ *
Latest Data Service
+ *
+ * The Latest Data Service in the IUDX Resource Server retrieves the latest data belonging to an ID.
+ *
+ *
+ * @see io.vertx.codegen.annotations.ProxyGen
+ * @see io.vertx.codegen.annotations.VertxGen
+ * @version 1.0
+ * @since 2020-05-31
+ */
+
+@VertxGen
+@ProxyGen
+public interface LatestDataService {
+
+ /**
+ * The getLatestData retrieves the latest data.
+ *
+ * @param request which is a JsonObject
+ * @param handler which is a Request Handler
+ * @return LatestDataService which is a Service
+ */
+
+ @Fluent
+ LatestDataService getLatestData(JsonObject request, Handler> handler);
+
+ @GenIgnore
+ static LatestDataService create(RedisClient client, JsonObject attributeList) {
+ return new LatestDataServiceImpl(client, attributeList);
+ }
+
+ /**
+ * The createProxy helps the code generation blocks to generate proxy code.
+ * @param vertx which is the vertx instance
+ * @param address which is the proxy address
+ * @return LatestDataServiceVertxEBProxy which is a service proxy
+ */
+
+ @GenIgnore
+ static LatestDataService createProxy(Vertx vertx, String address) {
+ return new LatestDataServiceVertxEBProxy(vertx, address);
+ }
+}
diff --git a/src/main/java/iudx/resource/server/database/latest/LatestDataServiceImpl.java b/src/main/java/iudx/resource/server/database/latest/LatestDataServiceImpl.java
new file mode 100644
index 00000000..26ac4727
--- /dev/null
+++ b/src/main/java/iudx/resource/server/database/latest/LatestDataServiceImpl.java
@@ -0,0 +1,132 @@
+package iudx.resource.server.database.latest;
+
+import static iudx.resource.server.database.archives.Constants.ATTRIBUTE_LIST;
+import static iudx.resource.server.database.archives.Constants.EMPTY_RESOURCE_ID;
+import static iudx.resource.server.database.archives.Constants.ERROR;
+import static iudx.resource.server.database.archives.Constants.FAILED;
+import static iudx.resource.server.database.archives.Constants.ID;
+import static iudx.resource.server.database.archives.Constants.ID_NOT_FOUND;
+import static iudx.resource.server.database.archives.Constants.KEY;
+import static iudx.resource.server.database.archives.Constants.PATH_PARAM;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import io.vertx.core.AsyncResult;
+import io.vertx.core.Future;
+import io.vertx.core.Handler;
+import io.vertx.core.json.JsonObject;
+import iudx.resource.server.database.archives.DatabaseServiceImpl;
+import iudx.resource.server.database.archives.QueryDecoder;
+import iudx.resource.server.database.archives.ResponseBuilder;
+
+/**
+ * The LatestData Service Implementation.
+ *
LatestData Service Implementation
+ *
+ * The LatestData Service implementation in the IUDX Resource Server implements the definitions of the
+ * {@link iudx.resource.server.database.latest.LatestDataService}.
+ *
+ *
+ * @version 1.0
+ * @since 2021-03-26
+ */
+
+public class LatestDataServiceImpl implements LatestDataService{
+
+ RedisClient redisClient;
+ private ResponseBuilder responseBuilder;
+ JsonObject attributeList;
+ private static final Logger LOGGER = LogManager.getLogger(DatabaseServiceImpl.class);
+ // private RedisAPI redisAPI;
+ private QueryDecoder decoder = new QueryDecoder();
+ private JsonObject query;
+
+ public LatestDataServiceImpl(RedisClient client, JsonObject attributeList) {
+ this.redisClient = client;
+ this.attributeList = attributeList;
+ }
+
+ /**
+ * Performs a Latest search query using the Redis JReJSON client.
+ *
+ * @param request Json object received from the ApiServerVerticle
+ * @param handler Handler to return redis response in case of success and appropriate error
+ * message in case of failure
+ */
+
+ @Override
+ public LatestDataService getLatestData(JsonObject request, Handler> handler) {
+
+ request.put(ATTRIBUTE_LIST, attributeList);
+
+ // Exceptions
+ if (!request.containsKey(ID)) {
+ LOGGER.debug("Info: " + ID_NOT_FOUND);
+ responseBuilder = new ResponseBuilder(FAILED).setTypeAndTitle(400).setMessage(ID_NOT_FOUND);
+ handler.handle(Future.failedFuture(responseBuilder.getResponse().toString()));
+ return null;
+ }
+
+ if (request.getJsonArray(ID).isEmpty()) {
+ LOGGER.debug("Info: " + EMPTY_RESOURCE_ID);
+ responseBuilder = new ResponseBuilder(FAILED).setTypeAndTitle(400)
+ .setMessage(EMPTY_RESOURCE_ID);
+ handler.handle(Future.failedFuture(responseBuilder.getResponse().toString()));
+ return null;
+ }
+
+// if (!request.containsKey(OPTIONS)) {
+// LOGGER.debug("Info: " + OPTIONS_NOT_FOUND);
+// responseBuilder = new ResponseBuilder(FAILED).setTypeAndTitle(400).setMessage(OPTIONS_NOT_FOUND);
+// handler.handle(Future.failedFuture(responseBuilder.getResponse().toString()));
+// return null;
+// }
+//
+// if (request.getString(OPTIONS).isEmpty()) {
+// LOGGER.debug("Info: " + EMPTY_OPTIONS);
+// responseBuilder = new ResponseBuilder(FAILED).setTypeAndTitle(400)
+// .setMessage(EMPTY_OPTIONS);
+// handler.handle(Future.failedFuture(responseBuilder.getResponse().toString()));
+// return null;
+// }
+
+ // options has to be equal to group or id
+
+// if (!GROUP.equalsIgnoreCase(request.getString(OPTIONS)) && !ID.equalsIgnoreCase(request.getString(OPTIONS))) {
+// LOGGER.debug("Info: " + EMPTY_OPTIONS);
+// responseBuilder = new ResponseBuilder(FAILED).setTypeAndTitle(400)
+// .setMessage(INVALID_OPTIONS);
+// handler.handle(Future.failedFuture(responseBuilder.getResponse().toString()));
+// return null;
+// }
+
+ /** Reusing QueryDecoder
+ * TODO: can use the QueryDecoder module to throw another request concurrently
+ * to ES to ensure better data consistency between cache and DB
+ * query = {key: rg, pathParam: pathParam} for Redis command.
+ */
+
+ LOGGER.debug(" "+request);
+ query = decoder.queryDecoder(request);
+ if (query.containsKey(ERROR)) {
+ LOGGER.error("Fail: Query returned with an error: " + query.getString(ERROR));
+ responseBuilder =
+ new ResponseBuilder(FAILED).setTypeAndTitle(400)
+ .setMessage(query.getString(ERROR));
+ handler.handle(Future.failedFuture(responseBuilder.getResponse().toString()));
+ return null;
+ }
+
+ LOGGER.debug("Info: Query constructed: " + query.toString());
+ redisClient.searchAsync(query.getString(KEY), query.getString(PATH_PARAM), searchRes -> {
+ if (searchRes.succeeded()) {
+ LOGGER.debug("Success: Successful Redis request");
+ handler.handle(Future.succeededFuture(searchRes.result()));
+ } else {
+ LOGGER.error("Fail: Redis Cache Request;" + searchRes.cause().getMessage());
+ handler.handle(Future.failedFuture(searchRes.cause().getMessage()));
+ }
+ });
+
+ return null;
+ }
+}
diff --git a/src/main/java/iudx/resource/server/database/latest/LatestVerticle.java b/src/main/java/iudx/resource/server/database/latest/LatestVerticle.java
new file mode 100644
index 00000000..8d38ad9a
--- /dev/null
+++ b/src/main/java/iudx/resource/server/database/latest/LatestVerticle.java
@@ -0,0 +1,76 @@
+package iudx.resource.server.database.latest;
+
+import io.vertx.core.AbstractVerticle;
+import io.vertx.core.eventbus.MessageConsumer;
+import io.vertx.core.json.JsonObject;
+import io.vertx.serviceproxy.ServiceBinder;
+import iudx.resource.server.database.archives.Constants;
+
+public class LatestVerticle extends AbstractVerticle {
+
+
+ /**
+ * The Latest Verticle.
+ *
Latest Verticle
+ *
+ * The Database Verticle implementation in the the IUDX Resource Server exposes the
+ * {@link iudx.resource.server.database.archives.DatabaseService} over the Vert.x Event Bus.
+ *
+ *
+ * @version 1.0
+ * @since 2020-05-31
+ */
+ private LatestDataService latestData;
+ private RedisClient redisClient;
+ private String redisHost;
+ private String redisUser;
+ private String password;
+ private JsonObject attributeList;
+ private int port;
+ private ServiceBinder binder;
+ private MessageConsumer consumer;
+ private String connectionString;
+
+
+ /**
+ * This method is used to start the Verticle. It deploys a verticle in a cluster, registers the
+ * service with the Event bus against an address, publishes the service with the service discovery
+ * interface.
+ *
+ * @throws Exception which is a start up exception.
+ */
+
+ @Override
+ public void start() throws Exception {
+
+ /** config to read the Redis credentials
+ * IP
+ * redisUser
+ * password
+ * port
+ * */
+ redisHost = config().getString("redisHost");
+ port = config().getInteger("redisPort");
+ redisUser = config().getString("redisUser");
+ password = config().getString("redisPassword");
+ attributeList = config().getJsonObject("attributeList");
+ //connectionString = "redis://:".concat(redisUser).concat(":").concat(password).concat("@").concat(redisHost)
+ // .concat(":").concat(String.valueOf(port));
+ // connectionString = "redis://:@https://database.iudx.io:28734/1";
+ // System.out.println("RedisConnectionString: " + connectionString);
+ // redisClient = new RedisClient(vertx, connectionString);
+ redisClient = new RedisClient(vertx, redisHost, port);
+ binder = new ServiceBinder(vertx);
+ latestData = new LatestDataServiceImpl(redisClient, attributeList);
+
+ consumer =
+ binder.setAddress(Constants.LATEST_DATA_SERVICE_ADDRESS)
+ .register(LatestDataService.class, latestData);
+ }
+
+ @Override
+ public void stop() {
+ binder.unregister(consumer);
+ }
+}
+
diff --git a/src/main/java/iudx/resource/server/database/latest/RedisClient.java b/src/main/java/iudx/resource/server/database/latest/RedisClient.java
new file mode 100644
index 00000000..820755f8
--- /dev/null
+++ b/src/main/java/iudx/resource/server/database/latest/RedisClient.java
@@ -0,0 +1,242 @@
+package iudx.resource.server.database.latest;
+
+import static iudx.resource.server.database.archives.Constants.*;
+import java.util.Map;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import com.redislabs.modules.rejson.JReJSON;
+import com.redislabs.modules.rejson.Path;
+import io.vertx.core.AsyncResult;
+import io.vertx.core.Future;
+import io.vertx.core.Handler;
+import io.vertx.core.Promise;
+import io.vertx.core.Vertx;
+import io.vertx.core.json.JsonArray;
+import io.vertx.core.json.JsonObject;
+import io.vertx.rxjava.redis.client.RedisAPI;
+import iudx.resource.server.database.archives.ResponseBuilder;
+
+public class RedisClient {
+ // private Redis redisClient;
+ private ResponseBuilder responseBuilder;
+ private static final Logger LOGGER = LogManager.getLogger(RedisClient.class);
+ private RedisAPI redis;
+ private Vertx vertx;
+ private JReJSON client;
+
+// public RedisClient(Vertx vertx, String connectionString){
+// this.vertx = vertx;
+// //redisClient = Redis.createClient(vertx, connectionString);
+// //redis = RedisAPI.api(redisClient);
+
+// }
+
+ /**
+ * RedisClient - Redis JReJSON Client Low level wrapper.
+ *
+ * @param vertx Vertx Instance
+ * @param ip IP of the ElasticDB
+ * @param port Port of the ElasticDB
+ */
+
+ public RedisClient(Vertx vertx, String ip, int port) {
+ this.vertx = vertx;
+ this.client = new JReJSON(ip, port);
+ }
+
+ /**
+ * searchAsync - Wrapper around Redis async search requests.
+ *
+ * @param key Redis Key
+ * @param pathParam Path Parameter for Redis Nested JSON object
+ * @param searchHandler JsonObject result {@link AsyncResult}
+ */
+
+ public RedisClient searchAsync(String key, String pathParam, Handler> searchHandler) {
+ // using get command
+ JsonArray response = new JsonArray();
+ get(key, pathParam).onComplete(resultRedis -> {
+ if (resultRedis.succeeded()) {
+ LOGGER.debug("Key found!");
+ JsonObject fromRedis = resultRedis.result();
+ LOGGER.debug("Result from Redis: " + fromRedis);
+ response.add(fromRedis);
+ responseBuilder = new ResponseBuilder(SUCCESS).setTypeAndTitle(200).setMessage(response);
+ searchHandler.handle(Future.succeededFuture(responseBuilder.getResponse()));
+ }
+ else {
+ LOGGER.error("Redis Error: " + resultRedis.cause());
+ resultRedis.cause().printStackTrace();
+ responseBuilder = new ResponseBuilder(FAILED).setTypeAndTitle(204)
+ .setMessage(resultRedis.cause().getLocalizedMessage());
+ searchHandler.handle(Future.failedFuture(responseBuilder.getResponse().toString()));
+ }
+ });
+
+ // using Vertx get command
+ // cannot be used with ReJSON since JSON.GET command is not supported
+
+// redis.get(".".concat(key).concat(".").concat(pathParam), responseAsyncResult -> {
+// if (responseAsyncResult.succeeded()){
+// LOGGER.debug("Successful request " + responseAsyncResult.result());
+// Response redisResponse = responseAsyncResult.result();
+// Object[] resources = redisResponse.stream().toArray();
+// for (Object r: resources) {
+// response.add((JsonObject)r);
+// }
+// // two for loops
+//// for (Object rR : redisResponse.toArray(new Response[0]) )
+// responseBuilder = new ResponseBuilder(SUCCESS).setTypeAndTitle(200).setMessage(response);
+// searchHandler.handle(Future.succeededFuture(responseBuilder.getResponse()));
+//
+// } else {
+// LOGGER.error("Redis Error: " + responseAsyncResult.toString());
+// responseBuilder = new ResponseBuilder(FAILED).setTypeAndTitle(500)
+// .setMessage(REDIS_ERROR);
+// searchHandler.handle(Future.failedFuture(responseBuilder.getResponse().toString()));
+// }
+//
+//
+//
+// // id
+//
+// if (!".".equalsIgnoreCase(pathParam)) {
+// get(key, pathParam).onComplete(resultRedis -> {
+// if (resultRedis.succeeded()) {
+// LOGGER.debug("Key found!");
+// JsonObject fromRedis = resultRedis.result();
+// response.add(fromRedis);
+// responseBuilder = new ResponseBuilder(SUCCESS).setTypeAndTitle(200).setMessage(response);
+// searchHandler.handle(Future.succeededFuture(responseBuilder.getResponse()));
+// }
+// });
+// }
+//
+// // group
+//
+// else {
+// get
+//
+// }
+//
+//
+// });
+
+
+ // using Native send command
+ // Overriding send command to add JSON.GET
+
+// redis.send(new Command() {
+// @Override
+// public byte[] getBytes() {
+// return new byte[0];
+// }
+//
+// @Override
+// public int getArity() {
+// return 0;
+// }
+//
+// @Override
+// public boolean isMultiKey() {
+// return false;
+// }
+//
+// @Override
+// public int getFirstKey() {
+// return 0;
+// }
+//
+// @Override
+// public int getLastKey() {
+// return 0;
+// }
+//
+// @Override
+// public int getInterval() {
+// return 0;
+// }
+//
+// @Override
+// public boolean isKeyless() {
+// return false;
+// }
+//
+// @Override
+// public boolean isReadOnly() {
+// return false;
+// }
+//
+// @Override
+// public boolean isMovable() {
+// return false;
+// }
+//
+// @Override
+// public boolean isVoid() {
+// return false;
+// }
+// }, key.concat(" .").concat(pathParam)).handle(redisResponseHandler);
+ return this;
+ }
+ /**
+ * get - makes sync Redis call asynchronous
+ *
+ * @param key Redis Key
+ * returns Future Object with (JSON) result from Redis
+ */
+
+ public Future get(String key) {
+ return get(key, Path.ROOT_PATH.toString());
+ }
+
+ /**
+ * get - makes sync Redis call asynchronous
+ * overridden method to include path parameter
+ * @param key Redis Key
+ * @param path Redis Path parameter
+ * returns Future Object with (JSON) result from Redis
+ */
+
+ public Future get(String key, String path) {
+ Promise promise = Promise.promise();
+ vertx.executeBlocking(getFromRedisHandler -> {
+ JsonObject json = getFromRedis(key, path);
+ if (json == null) {
+ getFromRedisHandler.fail(ID_NOT_PRESENT);
+ } else {
+ getFromRedisHandler.complete(json);
+ }
+ }, resultHandler -> {
+ if (resultHandler.succeeded()) {
+ promise.complete((JsonObject) resultHandler.result());
+ } else {
+ promise.fail(resultHandler.cause());
+ }
+ });
+ return promise.future();
+ }
+
+ /**
+ * getFromRedis - wrapper around Redis JReJSON client get command
+ * @param key Redis Key
+ * @param path Redis Path parameter
+ * returns (JsonObject) response from Redis
+ */
+
+ private JsonObject getFromRedis(String key, String path) {
+ Map result = null;
+ try {
+ result = client.get(key, Map.class, new Path(path));
+ if (result != null) {
+ JsonObject res = new JsonObject(result);
+ return res;
+ } else {
+ return null;
+ }
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+}
diff --git a/src/main/java/iudx/resource/server/database/latest/package-info.java b/src/main/java/iudx/resource/server/database/latest/package-info.java
new file mode 100644
index 00000000..00d5723f
--- /dev/null
+++ b/src/main/java/iudx/resource/server/database/latest/package-info.java
@@ -0,0 +1,6 @@
+@ModuleGen(groupPackage = "iudx.resource.server.database.latest",
+ name = "iudx-resource-server-database-latest")
+
+package iudx.resource.server.database.latest;
+
+import io.vertx.codegen.annotations.ModuleGen;
diff --git a/src/main/java/iudx/resource/server/database/package-info.java b/src/main/java/iudx/resource/server/database/package-info.java
deleted file mode 100644
index ce43e264..00000000
--- a/src/main/java/iudx/resource/server/database/package-info.java
+++ /dev/null
@@ -1,5 +0,0 @@
-@ModuleGen(groupPackage = "iudx.resource.server.database",
- name = "iudx-resource-server-database-service")
-package iudx.resource.server.database;
-
-import io.vertx.codegen.annotations.ModuleGen;
diff --git a/src/test/java/iudx/resource/server/database/DatabaseServiceTest.java b/src/test/java/iudx/resource/server/database/DatabaseServiceTest.java
index bf27d200..51a551d3 100644
--- a/src/test/java/iudx/resource/server/database/DatabaseServiceTest.java
+++ b/src/test/java/iudx/resource/server/database/DatabaseServiceTest.java
@@ -8,6 +8,9 @@
import io.vertx.core.logging.Logger;
import io.vertx.junit5.VertxTestContext;
import iudx.resource.server.configuration.Configuration;
+import iudx.resource.server.database.archives.DatabaseService;
+import iudx.resource.server.database.archives.DatabaseServiceImpl;
+import iudx.resource.server.database.archives.ElasticClient;
import java.io.FileInputStream;
import java.io.InputStream;
import java.text.ParseException;
@@ -20,6 +23,7 @@
import java.util.Set;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -844,12 +848,10 @@ void searchAttributeNe(VertxTestContext testContext) {
@Test
@DisplayName("Testing Latest Search")
+ @Disabled
void latestSearch(VertxTestContext testContext) {
JsonObject request = new JsonObject().put("id", new JsonArray()
- .add("iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86/rs.iudx.io/pune-env-flood/FWR013")
- .add("iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86/rs.iudx.io/pune-env-flood/FWR020")
- .add("iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86/rs.iudx"
- + ".io/pune-env-flood/FWR064"))
+ .add("datakaveri.org/04a15c9960ffda227e9546f3f46e629e1fe4132b/rs.iudx.io/pune-env-flood/FWR018"))
.put("searchType", "latestSearch");
JsonArray id = request.getJsonArray("id");
JsonArray idFromResponse = new JsonArray();
@@ -865,12 +867,10 @@ void latestSearch(VertxTestContext testContext) {
@Test
@DisplayName("Testing Latest Search with Response Filter")
+ @Disabled
void latestSearchFiltered(VertxTestContext testContext) {
JsonObject request = new JsonObject().put("id", new JsonArray()
- .add("iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86/rs.iudx.io/pune-env-flood/FWR013")
- .add("iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86/rs.iudx.io/pune-env-flood/FWR020")
- .add("iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86/rs.iudx"
- + ".io/pune-env-flood/FWR064"))
+ .add("datakaveri.org/04a15c9960ffda227e9546f3f46e629e1fe4132b/rs.iudx.io/pune-env-flood/FWR018"))
.put("searchType", "latestSearch")
.put("attrs", new JsonArray().add("id")
.add("observationDateTime"));
diff --git a/src/test/java/iudx/resource/server/database/LatestServiceTest.java b/src/test/java/iudx/resource/server/database/LatestServiceTest.java
new file mode 100644
index 00000000..fad66937
--- /dev/null
+++ b/src/test/java/iudx/resource/server/database/LatestServiceTest.java
@@ -0,0 +1,291 @@
+package iudx.resource.server.database;
+
+import io.vertx.core.json.JsonArray;
+import io.vertx.core.json.JsonObject;
+import io.vertx.core.logging.Logger;
+import io.vertx.core.logging.LoggerFactory;
+import io.vertx.junit5.VertxExtension;
+import io.vertx.junit5.VertxTestContext;
+import io.vertx.reactivex.core.Vertx;
+import iudx.resource.server.configuration.Configuration;
+import iudx.resource.server.database.latest.LatestDataService;
+import iudx.resource.server.database.latest.LatestDataServiceImpl;
+import iudx.resource.server.database.latest.RedisClient;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+@ExtendWith({VertxExtension.class})
+public class LatestServiceTest {
+ private static Logger logger = LoggerFactory.getLogger(LatestDataService.class);
+ private static LatestDataService latestService;
+ private static Vertx vertxObj;
+ private static RedisClient client;
+ private static String redisHost, user, password;
+ private static int port;
+ private static Configuration config;
+ private static JsonObject attributeList;
+ // private static String connectionString;
+
+ /* TODO Need to update params to use contants */
+ @BeforeAll
+ @DisplayName("Deploying Latest Test Verticle")
+ static void startVertx(Vertx vertx, VertxTestContext testContext) {
+ vertxObj = vertx;
+ config = new Configuration();
+ JsonObject redisConfig = config.configLoader(5, vertx);
+
+ /* Read the configuration and set the rabbitMQ server properties. */
+ /* Configuration setup */
+ redisHost = redisConfig.getString("redisHost");
+ port = redisConfig.getInteger("redisPort");
+ user = redisConfig.getString("redisUser");
+ password = redisConfig.getString("redisPassword");
+ attributeList = redisConfig.getJsonObject("attributeList");
+ //connectionString = "redis://:".concat(user).concat(":").concat(password).concat("@").concat(redisHost)
+ // .concat(":").concat(String.valueOf(port));
+ // connectionString = "redis://:@https://database.iudx.io:28734/1";
+
+ // logger.debug("Redis ConnectionString: " + connectionString);
+ logger.debug("Host: "+ redisHost + "Port: " + port);
+ client = new RedisClient(io.vertx.core.Vertx.vertx(), redisHost, port);
+ latestService = new LatestDataServiceImpl(client, attributeList);
+ testContext.completeNow();
+ }
+
+ @AfterEach
+ public void finish(VertxTestContext testContext) {
+ logger.info("Finishing the tests....");
+ vertxObj.close(testContext.succeeding(response -> testContext.completeNow()));
+ }
+
+ // Functional Testing
+
+ /**
+ * resource-id query
+ * resource-group aqm
+ * rg flood
+ * rg itms
+ * */
+
+ @Test
+ @DisplayName("Testing Latest Data at resource level- flood")
+ void searchLatestResourceflood(VertxTestContext testContext) {
+ String id = "datakaveri.org/04a15c9960ffda227e9546f3f46e629e1fe4132b/" +
+ "rs.iudx.io/pune-env-flood/FWR018";
+ JsonObject request =
+ new JsonObject()
+ .put("id",
+ new JsonArray().add(id))
+ //.put("options", "id")
+ .put("searchType","latestSearch");
+
+ latestService.getLatestData(request, handler -> {
+ if(handler.succeeded()) {
+ logger.debug("Got the data!");
+ assertEquals(id, handler.result().getJsonArray("results").getJsonObject(0)
+ .getString("id"));
+ testContext.completeNow();
+ } else {
+ testContext.failNow(handler.cause());
+ }
+ });
+ }
+
+ @Test
+ @DisplayName("Testing Latest Data at resource level- itms")
+ void searchLatestResourceItms(VertxTestContext testContext) {
+ String id = "iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86/rs.iudx.io/surat-itms-realtime-information" +
+ "/surat-itms-live-eta";
+ JsonObject request =
+ new JsonObject()
+ .put("id",
+ new JsonArray().add(id))
+ //.put("options", "id")
+ .put("searchType","latestSearch");
+
+ latestService.getLatestData(request, handler -> {
+ if(handler.succeeded()) {
+ logger.debug("Got the data!");
+ assertEquals(id, handler.result().getJsonArray("results").getJsonObject(0)
+ .getString("id"));
+ testContext.completeNow();
+ } else {
+ testContext.failNow(handler.cause());
+ }
+ });
+ }
+
+ /**
+ * Group level testing
+ * --> aqm
+ * --> flood
+ * --> itms
+ */
+
+// @Test
+// @DisplayName("Testing Latest Data at group level- aqm")
+// void searchLatestGroupAQM(VertxTestContext testContext) {
+// String id = "varanasismartcity.gov.in/62d1f729edd3d2a1a090cb1c6c89356296963d55" +
+// "/rs.iudx.io/varanasi-env-aqm";
+// JsonObject request =
+// new JsonObject()
+// .put("id",
+// new JsonArray().add(id))
+// .put("options", "group").put("searchType","latestSearch");
+//
+// latestService.getLatestData(request, result -> {
+// if (result.succeeded()) {
+// logger.debug("Got the data!");
+// assertEquals(id, result.result().getJsonArray("results").getJsonObject(0)
+// .getString("id"));
+// testContext.completeNow();
+// } else {
+// logger.error("Error: " + result.cause());
+// testContext.failNow(result.cause());
+// }
+// });
+// }
+
+// @Test
+// @DisplayName("Testing Latest Data at group level- flood")
+// void searchLatestGroupFlood(VertxTestContext testContext) {
+// String id = "datakaveri.org/04a15c9960ffda227e9546f3f46e629e1fe4132b/" +
+// "rs.iudx.io/pune-env-flood";
+// JsonObject request =
+// new JsonObject()
+// .put("id",
+// new JsonArray().add(id))
+// .put("options", "group").put("searchType","latestSearch");
+//
+// latestService.getLatestData(request, result -> {
+// if (result.succeeded()) {
+// assertEquals(id, result.result().getJsonArray("results").getJsonObject(0)
+// .getString("id"));
+// testContext.completeNow();
+// } else {
+// logger.error("Error: " + result.cause());
+// testContext.failNow(result.cause());
+// }
+// });
+// }
+
+// @Test
+// @DisplayName("Testing Latest Data at group level- itms")
+// void searchLatestGroupITMS(VertxTestContext testContext) {
+// String id = "suratmunicipal.org/6db486cb4f720e8585ba1f45a931c63c25dbbbda/" +
+// "rs.iudx.io/surat-itms-realtime-info/surat-itms-live-eta";
+// JsonObject request =
+// new JsonObject()
+// .put("id",
+// new JsonArray().add(id))
+// .put("options", "group").put("searchType","latestSearch");
+//
+// latestService.getLatestData(request, result -> {
+// if (result.succeeded()) {
+// assertEquals(id, result.result().getJsonArray("results").getJsonObject(0)
+// .getString("id"));
+// testContext.completeNow();
+// } else {
+// logger.error("Error: " + result.cause());
+// testContext.failNow(result.cause());
+// }
+// });
+// }
+
+ // Exception Testing
+
+ /**
+ * id not present in JSONObject from Verticle
+ * options not present in JSONObject
+ * id is empty
+ * options is empty
+ * options not equal to group/id
+ * id not present in the Redis
+ * **/
+
+ @Test
+ @DisplayName("Testing Basic Exceptions (No id key)")
+ void searchWithNoId(VertxTestContext testContext) {
+ JsonObject request = new JsonObject().put("options", "id");
+
+ latestService.getLatestData(request, testContext.failing(response -> testContext.verify(() -> {
+ assertEquals("No id found", new JsonObject(response.getMessage()).getString(
+ "detail"));
+ testContext.completeNow();
+ })));
+ }
+
+// @Test
+// @DisplayName("Testing Basic Exceptions (No options key)")
+// void searchWithNoResourceId(VertxTestContext testContext) {
+// JsonObject request = new JsonObject().put("id", new JsonArray().add(""));
+//
+// latestService.getLatestData(request, testContext.failing(response -> testContext.verify(() -> {
+// assertEquals("options not found", new JsonObject(response.getMessage()).getString(
+// "detail"));
+// testContext.completeNow();
+// })));
+// }
+
+ @Test
+ @DisplayName("Testing Basic Exceptions (id is empty)")
+ void searchEmptyId(VertxTestContext testContext) {
+ JsonObject request =
+ new JsonObject().put("id", new JsonArray()).put("options", "id");
+
+ latestService.getLatestData(request, testContext.failing(response -> testContext.verify(() -> {
+ assertEquals("resource-id is empty", new JsonObject(response.getMessage()).getString(
+ "detail"));
+ testContext.completeNow();
+ })));
+ }
+
+// @Test
+// @DisplayName("Testing Basic Exceptions (options is empty)")
+// void searchEmptyOptions(VertxTestContext testContext) {
+// JsonObject request =
+// new JsonObject().put("id", new JsonArray().add("")).put("options", "");
+//
+// latestService.getLatestData(request, testContext.failing(response -> testContext.verify(() -> {
+// assertEquals("options is empty", new JsonObject(response.getMessage()).getString(
+// "detail"));
+// testContext.completeNow();
+// })));
+// }
+
+// @Test
+// @DisplayName("Testing Basic Exceptions (invalid options parameters)")
+// void searchInvalidOptions(VertxTestContext testContext) {
+// JsonObject request =
+// new JsonObject().put("id", new JsonArray().add("")).put("options", "invalid!");
+//
+// latestService.getLatestData(request, testContext.failing(response -> testContext.verify(() -> {
+// assertEquals("invalid options for latest", new JsonObject(response.getMessage()).getString(
+// "detail"));
+// testContext.completeNow();
+// })));
+// }
+
+ @Test
+ @DisplayName("Testing Basic Exceptions (id not present in Database)")
+ void searchIdNotInRedis(VertxTestContext testContext) {
+ String id = "iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86" +
+ "/rs.iudx.io/surat-itms-realtime-information/surat-itms-live";
+ JsonObject request =
+ new JsonObject().put("id", new JsonArray().add(id))
+ .put("options", "id")
+ .put("searchType", "latestSearch");
+
+ latestService.getLatestData(request, testContext.failing(response -> testContext.verify(() -> {
+ assertEquals("Not found", new JsonObject(response.getMessage())
+ .getString("detail"));
+ testContext.completeNow();
+ })));
+ }
+
+}
\ No newline at end of file