Skip to content

Commit

Permalink
Merge pull request #288 from pranavrd/fix/mlayer-dataset-api
Browse files Browse the repository at this point in the history 
hotfix: new loggers for debugging
  • Loading branch information
@pranavrd
pranavrd authored Oct 11, 2023
2 parents c162877 + ba67ab6 commit 2c22419
Show file tree
Hide file tree
Showing 3 changed files with 107 additions and 58 deletions.
47 changes: 40 additions & 7 deletions src/main/java/iudx/catalogue/server/database/DatabaseServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1779,6 +1779,7 @@ public DatabaseService getMlayerAllDatasets(
JsonObject resourceGroupList = ar.result().resultAt(1);
JsonObject resourceAndPolicyCount = ar.result().resultAt(2);
JsonArray resourceGroupArray = new JsonArray();
LOGGER.debug("getMlayerDatasets resourceGroupList iteration started");
for (int i = 0; i < resourceGroupList.getInteger("resourceGroupCount"); i++) {
JsonObject record =
resourceGroupList.getJsonArray("resourceGroup").getJsonObject(i);
Expand Down Expand Up @@ -1808,12 +1809,14 @@ public DatabaseService getMlayerAllDatasets(
record.remove(TYPE);
resourceGroupArray.add(record);
}
LOGGER.debug("getMlayerDatasets resourceGroupList interation succeeded");
RespBuilder respBuilder =
new RespBuilder()
.withType(TYPE_SUCCESS)
.withTitle(SUCCESS)
.withTotalHits(resourceGroupList.getInteger("resourceGroupCount"))
.withResult(resourceGroupArray);
LOGGER.debug("getMlayerDatasets succeeded");
handler.handle(Future.succeededFuture(respBuilder.getJsonResponse()));

} else {
Expand All @@ -1832,9 +1835,12 @@ private void gettingResourceAccessPolicyCount(Promise<JsonObject> resourceCountR
docIndex,
resourceCountRes -> {
if (resourceCountRes.succeeded()) {
try {
LOGGER.debug("resourceAP started");
JsonObject resourceItemCount = new JsonObject();
JsonObject resourceAccessPolicy = new JsonObject();
JsonArray resultsArray = resourceCountRes.result().getJsonArray(RESULTS);
LOGGER.debug("resourceAP for each resultsArray started");
resultsArray.forEach(record -> {
JsonObject recordObject = (JsonObject) record;
String resourceGroup = recordObject.getString(KEY);
Expand All @@ -1859,11 +1865,17 @@ private void gettingResourceAccessPolicyCount(Promise<JsonObject> resourceCountR
resourceAccessPolicy.put(resourceGroup, accessPolicy);
});

JsonObject results = new JsonObject()
.put("resourceItemCount", resourceItemCount)
.put("resourceAccessPolicy", resourceAccessPolicy);
resourceCountResult.complete(results);
LOGGER.debug("resourceAP for each resultsArray succeeded");

JsonObject results = new JsonObject()
.put("resourceItemCount", resourceItemCount)
.put("resourceAccessPolicy", resourceAccessPolicy);
LOGGER.debug("resourceAP Succeeded : {}", results.containsKey("resourceItemCount"));
resourceCountResult.complete(results);
} catch (Exception e) {
LOGGER.error("resourceAP unexpectedly failed : {}", e.getMessage());
resourceCountResult.fail(e.getMessage());
}
} else {
LOGGER.error("Fail: query fail;" + resourceCountRes.cause());
resourceCountResult.handle(Future.failedFuture(internalErrorResp));
Expand All @@ -1878,15 +1890,23 @@ private void allMlayerInstance(Promise<JsonObject> instanceResult) {
mlayerInstanceIndex,
instanceRes -> {
if (instanceRes.succeeded()) {
try {
LOGGER.debug("getInstance started");
int instanceSize = instanceRes.result().getJsonArray(RESULTS).size();
JsonObject instanceIcon = new JsonObject();
LOGGER.debug("getInstance for each instance started");
for (int i = 0; i < instanceSize; i++) {
JsonObject instanceObject =
instanceRes.result().getJsonArray(RESULTS).getJsonObject(i);
instanceIcon.put(
instanceObject.getString("name").toLowerCase(), instanceObject.getString("icon"));
}
LOGGER.debug("getInstance succeeded");
instanceResult.complete(instanceIcon);
} catch (Exception e) {
LOGGER.error("getInstance enexpectedly failed : {}", e.getMessage());
instanceResult.fail(e.getMessage());
}
} else {

LOGGER.error("Fail: query fail;" + instanceRes.cause());
Expand All @@ -1903,9 +1923,11 @@ private void gettingAllDatasets(String query, Promise<JsonObject> datasetResult)
docIndex,
resultHandler -> {
if (resultHandler.succeeded()) {
LOGGER.debug("Success: Successful DB Request");
try {
LOGGER.debug("getRGs started");
int size = resultHandler.result().getJsonArray(RESULTS).size();
if (size == 0) {
LOGGER.debug("getRGs is zero");
datasetResult.handle(
Future.failedFuture(
new RespBuilder()
Expand All @@ -1918,6 +1940,7 @@ private void gettingAllDatasets(String query, Promise<JsonObject> datasetResult)
JsonObject rsUrl = new JsonObject();
JsonObject providerDescription = new JsonObject();
JsonObject cosUrl = new JsonObject();
LOGGER.debug("getRGs for each provider type result started");
for (int i = 0; i < size; i++) {
JsonObject record = resultHandler.result().getJsonArray(RESULTS).getJsonObject(i);
if (record.getJsonArray(TYPE).getString(0).equals(ITEM_TYPE_PROVIDER)) {
Expand All @@ -1931,8 +1954,10 @@ private void gettingAllDatasets(String query, Promise<JsonObject> datasetResult)
cosUrl.put(record.getString(ID), record.getString("cosURL"));
}
}
LOGGER.debug("getRGs for each provider type result succeeded");
int resourceGroupHits = 0;
JsonArray resourceGroup = new JsonArray();
LOGGER.debug("getRGs for each resource group result started");
for (int i = 0; i < size; i++) {
JsonObject record = resultHandler.result().getJsonArray(RESULTS).getJsonObject(i);
if (record.getJsonArray(TYPE).getString(0).equals(ITEM_TYPE_RESOURCE_GROUP)) {
Expand All @@ -1949,11 +1974,17 @@ private void gettingAllDatasets(String query, Promise<JsonObject> datasetResult)
resourceGroup.add(record);
}
}
LOGGER.debug("getRGs for each resource group result succeeded");
JsonObject resourceGroupResult =
new JsonObject()
.put("resourceGroupCount", resourceGroupHits)
.put("resourceGroup", resourceGroup);
LOGGER.debug("getRGs succeeded");
datasetResult.complete(resourceGroupResult);
} catch (Exception e) {
LOGGER.error("getRGs unexpectedly failed : {}", e.getMessage());
datasetResult.fail(e.getMessage());
}
} else {
LOGGER.error("Fail: failed DB request");
datasetResult.handle(Future.failedFuture(internalErrorResp));
Expand Down Expand Up @@ -2044,7 +2075,8 @@ public DatabaseService getMlayerDataset(
resultHandler.result().remove(TOTAL_HITS);
handler.handle(Future.succeededFuture(resultHandler.result()));
} else {
LOGGER.error("Fail: failed DB request");
LOGGER.error("Fail: failed DB request inner");
LOGGER.error(resultHandler.cause());
handler.handle(Future.failedFuture(internalErrorResp));
}
});
Expand All @@ -2054,7 +2086,8 @@ public DatabaseService getMlayerDataset(
handler.handle(Future.succeededFuture(resultHandler.result()));
}
} else {
LOGGER.error("Fail: failed DB request");
LOGGER.error("Fail: failed DB request outer");
LOGGER.error(resultHandler.cause());
handler.handle(Future.failedFuture(internalErrorResp));
}
});
Expand Down
111 changes: 62 additions & 49 deletions src/main/java/iudx/catalogue/server/database/ElasticClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,57 +155,64 @@ public void onSuccess(Response response) {
provider
.put(ID, record.getString(ID))
.put(DESCRIPTION_ATTR, record.getString(DESCRIPTION_ATTR));
dataset.put("resourceServerRegURL", record.getString("resourceServerRegURL"));
dataset.put(PROVIDER, provider);
}
if (type.equals("iudx:Resource")) {
record.remove("type");
record.put("resourceId", record.getString("id"));
record.remove("id");
resource.add(record);
}
if (type.equals("iudx:ResourceGroup")) {
String schema =
record.getString("@context")
+ record
.getJsonArray(TYPE)
.getString(1)
.substring(5, record.getJsonArray(TYPE)
.getString(1).length());
record.put("schema", schema);
record.remove("@context");
dataset
dataset.put("resourceServerRegURL", record.getString("resourceServerRegURL"));
dataset.put(PROVIDER, provider);
}
if (type.equals("iudx:Resource")) {
record.remove("type");
record.put("resourceId", record.getString("id"));
record.remove("id");
resource.add(record);
}
if (type.equals("iudx:ResourceGroup")) {
dataset
.put(ID, record.getString(ID))
.put(DESCRIPTION_ATTR, record.getString(DESCRIPTION_ATTR))
.put(DESCRIPTION_ATTR, record.getString(DESCRIPTION_ATTR));
if (record.getJsonArray(TYPE).size() > 1) {
String schema =
record.getString("@context")
+ record
.getJsonArray(TYPE)
.getString(1)
.substring(5, record.getJsonArray(TYPE).getString(1).length());
record.put("schema", schema);
record.remove("@context");
dataset
.put("schema", schema);
if (record.containsKey(LABEL)) {
dataset.put(LABEL, record.getString(LABEL));
}
if (record.containsKey(ACCESS_POLICY)) {
dataset.put(ACCESS_POLICY, record.getString(ACCESS_POLICY));
}
if (record.containsKey(INSTANCE)) {
dataset.put(INSTANCE, record.getString(INSTANCE));
}
if (record.containsKey(DATA_SAMPLE)) {
dataset.put(DATA_SAMPLE, record.getJsonObject(DATA_SAMPLE));
}
if (record.containsKey("dataSampleFile")) {
dataset.put("dataSampleFile", record.getJsonArray("dataSampleFile"));
}
if (record.containsKey("dataQualityFile")) {
dataset.put("dataQualityFile", record.getJsonArray("dataQualityFile"));
}
if (record.containsKey(DATA_DESCRIPTOR)) {
dataset.put(DATA_DESCRIPTOR, record.getJsonObject(DATA_DESCRIPTOR));
}
if (record.containsKey("resourceType")) {
dataset.put("resourceType", record.getString("resourceType"));
}
if (record.containsKey("location")) {
dataset.put("location", record.getJsonObject("location"));
}
}
} else {
record.put("schema", "");
record.remove("@context");
dataset
.put("schema", "");
}
if (record.containsKey(LABEL)) {
dataset.put(LABEL, record.getString(LABEL));
}
if (record.containsKey(ACCESS_POLICY)) {
dataset.put(ACCESS_POLICY, record.getString(ACCESS_POLICY));
}
if (record.containsKey(INSTANCE)) {
dataset.put(INSTANCE, record.getString(INSTANCE));
}
if (record.containsKey(DATA_SAMPLE)) {
dataset.put(DATA_SAMPLE, record.getJsonObject(DATA_SAMPLE));
}
if (record.containsKey("dataSampleFile")) {
dataset.put("dataSampleFile", record.getJsonArray("dataSampleFile"));
}
if (record.containsKey("dataQualityFile")) {
dataset.put("dataQualityFile", record.getJsonArray("dataQualityFile"));
}
if (record.containsKey(DATA_DESCRIPTOR)) {
dataset.put(DATA_DESCRIPTOR, record.getJsonObject(DATA_DESCRIPTOR));
}
if (record.containsKey("resourceType")) {
dataset.put("resourceType", record.getString("resourceType"));
}
if (record.containsKey("location")) {
dataset.put("location", record.getJsonObject("location"));
}
}

if (type.equals("iudx:COS")) {
dataset.put("cosURL", record.getString("cosURL"));
Expand Down Expand Up @@ -247,6 +254,8 @@ public ElasticClient searchAsync(String query, String index,

Request queryRequest = new Request(REQUEST_GET, index + "/_search" + FILTER_PATH);
queryRequest.setJsonEntity(query);
LOGGER.debug("searchAsync called");
LOGGER.debug(query);
LOGGER.debug(queryRequest);
Future<JsonObject> future = searchAsync(queryRequest, SOURCE_ONLY);
future.onComplete(resultHandler);
Expand Down Expand Up @@ -284,6 +293,10 @@ public ElasticClient resourceAggregationAsync(
+ "/_search"
+ FILTER_PATH_AGGREGATION);
queryRequest.setJsonEntity(query);

LOGGER.debug("resourceAggregationAsync called");
LOGGER.debug(query);
LOGGER.debug(queryRequest);
Future<JsonObject> future = searchAsync(queryRequest, RESOURCE_AGGREGATION_ONLY);
future.onComplete(resultHandler);
return this;
Expand Down
7 changes: 5 additions & 2 deletions src/main/java/iudx/catalogue/server/mlayer/MlayerServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,7 @@ public MlayerService getMlayerGeoQuery(
@Override
public MlayerService getMlayerAllDatasets(Handler<AsyncResult<JsonObject>> handler) {
String query = GET_MLAYER_ALL_DATASETS;
LOGGER.debug("databse get mlayer all datasets called");
databaseService.getMlayerAllDatasets(
query, getMlayerAllDatasets -> {
if (getMlayerAllDatasets.succeeded()) {
Expand Down Expand Up @@ -275,7 +276,8 @@ public MlayerService getMlayerDataset(
if (requestData.containsKey(TAGS) && !requestData.getJsonArray(TAGS).isEmpty()) {
query =
query.concat(
",{\"terms\":{\"tags.keyword\":$1}}".replace("$1", requestData.getString(TAGS)));
",{\"terms\":{\"tags.keyword\":$1}}"
.replace("$1", requestData.getJsonArray(TAGS).toString()));
}
if (requestData.containsKey(INSTANCE) && !requestData.getString(INSTANCE).isBlank()) {
query =
Expand All @@ -287,9 +289,10 @@ public MlayerService getMlayerDataset(
query =
query.concat(
",{\"terms\":{\"provider.keyword\":$1}}"
.replace("$1", requestData.getString(PROVIDERS)));
.replace("$1", requestData.getJsonArray(PROVIDERS).toString()));
}
query = query.concat(GET_ALL_DATASETS_BY_FIELD_SOURCE);
LOGGER.debug("databse get mlayer all datasets called");
databaseService.getMlayerAllDatasets(
query,
getAllDatasetsHandler -> {
Expand Down

0 comments on commit 2c22419

Please sign in to comment.