Skip to content

Commit

Permalink
work for multi taxonomy support
Browse files Browse the repository at this point in the history
  • Loading branch information
djtfmartin committed Sep 3, 2024
1 parent 3426504 commit 4ce9157
Show file tree
Hide file tree
Showing 6 changed files with 145 additions and 47 deletions.
44 changes: 20 additions & 24 deletions event-ws/src/main/java/org/gbif/event/search/es/EventSearchEs.java
Original file line number Diff line number Diff line change
Expand Up @@ -13,25 +13,9 @@
*/
package org.gbif.event.search.es;

import org.gbif.api.model.checklistbank.NameUsageMatch;
import org.gbif.api.model.common.paging.PageableBase;
import org.gbif.api.model.common.paging.PagingRequest;
import org.gbif.api.model.common.paging.PagingResponse;
import org.gbif.api.model.common.search.SearchResponse;
import org.gbif.api.model.event.Event;
import org.gbif.api.model.event.Lineage;
import org.gbif.api.model.occurrence.Occurrence;
import org.gbif.api.model.occurrence.search.OccurrenceSearchParameter;
import org.gbif.api.model.occurrence.search.OccurrenceSearchRequest;
import org.gbif.api.service.checklistbank.NameUsageMatchingService;
import org.gbif.api.service.common.SearchService;
import org.gbif.occurrence.search.SearchException;
import org.gbif.occurrence.search.es.EsResponseParser;
import org.gbif.occurrence.search.es.EsSearchRequestBuilder;
import org.gbif.occurrence.search.es.OccurrenceBaseEsFieldMapper;
import org.gbif.occurrence.search.es.SearchHitConverter;
import org.gbif.occurrence.search.es.SearchHitOccurrenceConverter;
import static org.gbif.occurrence.search.es.EsQueryUtils.HEADERS;

import com.google.common.base.Preconditions;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
Expand All @@ -42,25 +26,37 @@
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;

import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;

import org.gbif.api.model.checklistbank.NameUsageMatch;
import org.gbif.api.model.common.paging.PageableBase;
import org.gbif.api.model.common.paging.PagingRequest;
import org.gbif.api.model.common.paging.PagingResponse;
import org.gbif.api.model.common.search.SearchResponse;
import org.gbif.api.model.event.Event;
import org.gbif.api.model.event.Lineage;
import org.gbif.api.model.occurrence.Occurrence;
import org.gbif.api.model.occurrence.search.OccurrenceSearchParameter;
import org.gbif.api.model.occurrence.search.OccurrenceSearchRequest;
import org.gbif.api.service.checklistbank.NameUsageMatchingService;
import org.gbif.api.service.common.SearchService;
import org.gbif.occurrence.search.SearchException;
import org.gbif.occurrence.search.es.EsResponseParser;
import org.gbif.occurrence.search.es.EsSearchRequestBuilder;
import org.gbif.occurrence.search.es.OccurrenceBaseEsFieldMapper;
import org.gbif.occurrence.search.es.SearchHitConverter;
import org.gbif.occurrence.search.es.SearchHitOccurrenceConverter;
import org.gbif.vocabulary.client.ConceptClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import com.google.common.base.Preconditions;

import static org.gbif.occurrence.search.es.EsQueryUtils.HEADERS;

@Component
public class EventSearchEs implements SearchService<Event, OccurrenceSearchParameter, OccurrenceSearchRequest> {

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package org.gbif.occurrence.search.configuration;

import lombok.Data;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;

import java.util.List;

@Getter
@Setter
@Configuration
@ConfigurationProperties(prefix = "name-services")
public class NameUsageMatchServiceConfiguration {

List<NameUsageMatchServiceConfig> services;

@Data
public static class NameUsageMatchServiceConfig {
private String name;
private String datasetKey;
private Ws ws;
private String prefix;

@Data
public static class Ws {
private Api api;

@Data
public static class Api {
private String wsUrl;
}
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
package org.gbif.occurrence.search.configuration;

import org.gbif.kvs.species.NameUsageMatchRequest;
import org.gbif.rest.client.species.NameUsageMatchResponse;
import org.gbif.rest.client.species.NameUsageMatchingService;
import org.gbif.ws.client.ClientBuilder;
import org.gbif.ws.json.JacksonJsonObjectMapperProvider;
import org.springframework.stereotype.Service;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Optional;

/**
* Name usage match service triage.
* This class is responsible for routing requests to the appropriate name usage matching service.
*/
@Service
public class NameUsageMatchServiceTriage {

Map<String, NameUsageMatchingService> serviceByPrefix = new LinkedHashMap<>();
Map<String, NameUsageMatchingService> serviceByChecklistKey = new LinkedHashMap<>();

public NameUsageMatchServiceTriage(NameUsageMatchServiceConfiguration nameUsageMatchServiceConfiguration) {

for (NameUsageMatchServiceConfiguration.NameUsageMatchServiceConfig config : nameUsageMatchServiceConfiguration.getServices()) {
NameUsageMatchingService service = new ClientBuilder()
.withUrl(config.getWs().getApi().getWsUrl())
.withObjectMapper(JacksonJsonObjectMapperProvider.getObjectMapperWithBuilderSupport())
.withFormEncoder()
.build(NameUsageMatchingService.class);
serviceByPrefix.put(config.getPrefix(), service);
serviceByChecklistKey.put(config.getDatasetKey(), service);
}
}

/**
* Match a name usage against a service identified by its prefix.
* @param checklistKey the checklist key
* @param matchRequest the match request
* @return the match response
* @throws IllegalArgumentException if no service is found for the checklistKey
*/
public NameUsageMatchResponse match(String checklistKey, NameUsageMatchRequest matchRequest) {
if (checklistKey == null) {
return match(matchRequest);
}
return Optional.ofNullable(serviceByChecklistKey.get(checklistKey))
.map(service -> service.match(matchRequest))
.orElseThrow(() -> new IllegalArgumentException("No service for checklist key " + checklistKey));
}

/**
* Match a name usage against the default service
* @param matchRequest the match request
* @return the match response
* @throws IllegalArgumentException if no service is found for the checklistKey
*/
NameUsageMatchResponse match(NameUsageMatchRequest matchRequest) {
Optional<NameUsageMatchingService> n = serviceByChecklistKey.values().stream().findFirst();
if (n.isPresent()){
return n.get().match(matchRequest);
} else {
throw new IllegalArgumentException("No configured service");
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,6 @@
package org.gbif.occurrence.search.configuration;

import org.gbif.occurrence.search.es.EsConfig;
import org.gbif.rest.client.species.NameUsageMatchingService;
import org.gbif.ws.client.ClientBuilder;
import org.gbif.ws.json.JacksonJsonObjectMapperProvider;

import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
Expand All @@ -29,10 +25,11 @@
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.sniff.SniffOnFailureListener;
import org.elasticsearch.client.sniff.Sniffer;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;


/** Occurrence search configuration. */
public class OccurrenceSearchConfiguration {

Expand Down Expand Up @@ -94,13 +91,4 @@ public RestHighLevelClient provideEsClient(EsConfig esConfig) {

return highLevelClient;
}

@Bean
public NameUsageMatchingService nameUsageMatchingService(@Value("${checklistbank.match.ws.url}") String apiUrl) {
return new ClientBuilder()
.withUrl(apiUrl)
.withObjectMapper(JacksonJsonObjectMapperProvider.getObjectMapperWithBuilderSupport())
.withFormEncoder()
.build(NameUsageMatchingService.class);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
import org.gbif.api.model.occurrence.search.OccurrenceSearchParameter;
import org.gbif.api.model.occurrence.search.OccurrenceSearchRequest;
import org.gbif.kvs.species.NameUsageMatchRequest;
import org.gbif.occurrence.search.configuration.NameUsageMatchServiceTriage;
import org.gbif.rest.client.species.NameUsageMatchResponse;
import org.gbif.rest.client.species.NameUsageMatchingService;
import org.gbif.vocabulary.client.ConceptClient;
Expand All @@ -65,7 +66,7 @@ public class OccurrenceSearchEsImpl implements OccurrenceSearchService, Occurren

private static final Logger LOG = LoggerFactory.getLogger(OccurrenceSearchEsImpl.class);

private final NameUsageMatchingService nameUsageMatchingService;
private final NameUsageMatchServiceTriage nameUsageMatchServiceTriage;
private final RestHighLevelClient esClient;
private final String esIndex;
private final int maxLimit;
Expand All @@ -79,7 +80,7 @@ public class OccurrenceSearchEsImpl implements OccurrenceSearchService, Occurren
@Autowired
public OccurrenceSearchEsImpl(
RestHighLevelClient esClient,
NameUsageMatchingService nameUsageMatchingService,
NameUsageMatchServiceTriage nameUsageMatchServiceTriage,
@Value("${occurrence.search.max.offset}") int maxOffset,
@Value("${occurrence.search.max.limit}") int maxLimit,
@Value("${occurrence.search.es.index}") String esIndex,
Expand All @@ -92,7 +93,7 @@ public OccurrenceSearchEsImpl(
this.esIndex = esIndex;
// create ES client
this.esClient = esClient;
this.nameUsageMatchingService = nameUsageMatchingService;
this.nameUsageMatchServiceTriage = nameUsageMatchServiceTriage;
this.esFieldMapper = esFieldMapper;
this.esFulltextSuggestBuilder = EsFulltextSuggestBuilder.builder().occurrenceBaseEsFieldMapper(esFieldMapper).build();
this.esSearchRequestBuilder = new EsSearchRequestBuilder(esFieldMapper, conceptClient);
Expand Down Expand Up @@ -332,16 +333,25 @@ private boolean hasReplaceableScientificNames(OccurrenceSearchRequest request) {
boolean hasValidReplaces = true;
if (request.getParameters().containsKey(OccurrenceSearchParameter.SCIENTIFIC_NAME)) {
hasValidReplaces = false;
Collection<String> values = request.getParameters().get(OccurrenceSearchParameter.SCIENTIFIC_NAME);
for (String value : values) {
NameUsageMatchResponse nameUsageMatch = nameUsageMatchingService.match(NameUsageMatchRequest.builder()
.withScientificName(value)

Collection<String> scientificNames = request.getParameters().get(OccurrenceSearchParameter.SCIENTIFIC_NAME);

Collection<String> checklistKeys = request.getParameters().get(OccurrenceSearchParameter.CHECKLIST_KEY);
String checklistKey = null;
if (checklistKeys != null && !checklistKeys.isEmpty()) {
checklistKey = checklistKeys.iterator().next();
}

for (String scientificName : scientificNames) {

NameUsageMatchResponse nameUsageMatch = nameUsageMatchServiceTriage.match(checklistKey, NameUsageMatchRequest.builder()
.withScientificName(scientificName)
.withStrict(false)
.withVerbose(false)
.build());
if (nameUsageMatch.getDiagnostics().getMatchType() == NameUsageMatchResponse.MatchType.EXACT && Objects.nonNull(nameUsageMatch.getUsage())) {
hasValidReplaces = true;
values.remove(value);
scientificNames.remove(scientificName);
if (nameUsageMatch.getAcceptedUsage() != null) {
request.addParameter(OccurrenceSearchParameter.TAXON_KEY, nameUsageMatch.getAcceptedUsage().getKey());
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;

import org.gbif.occurrence.search.configuration.NameUsageMatchServiceConfiguration;
import org.gbif.vocabulary.client.ConceptClient;
import org.gbif.ws.client.ClientBuilder;
import org.gbif.ws.json.JacksonJsonObjectMapperProvider;
Expand Down Expand Up @@ -42,7 +43,7 @@
import org.springframework.context.annotation.FilterType;

@SpringBootApplication(exclude = {RabbitAutoConfiguration.class})
@EnableConfigurationProperties
@EnableConfigurationProperties(NameUsageMatchServiceConfiguration.class)
@ComponentScan(
basePackages = {
"org.gbif.ws.server.interceptor",
Expand All @@ -53,6 +54,7 @@
"org.gbif.ws.remoteauth",
"org.gbif.ws.security",
"org.gbif.occurrence.search",
"org.gbif.occurrence.search.configuration",
"org.gbif.occurrence.ws",
"org.gbif.occurrence.download.service",
"org.gbif.occurrence.persistence",
Expand Down

0 comments on commit 4ce9157

Please sign in to comment.