Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

solr functionalty was moved to sparate location #2196

Merged
merged 4 commits into from
Feb 16, 2023
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/main/java/org/ohdsi/webapi/info/InfoService.java
Original file line number Diff line number Diff line change
@@ -24,6 +24,7 @@
import javax.ws.rs.core.MediaType;

import org.apache.commons.lang3.StringUtils;
import org.ohdsi.webapi.info.ConfigurationInfo;
import org.springframework.boot.info.BuildProperties;
import org.springframework.stereotype.Controller;

Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
package org.ohdsi.webapi.security;

import org.ohdsi.webapi.Constants;
import org.ohdsi.webapi.info.ConfigurationInfo;
import org.ohdsi.webapi.shiro.management.AtlasRegularSecurity;
import org.ohdsi.webapi.shiro.management.Security;
import org.ohdsi.webapi.info.ConfigurationInfo;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

Original file line number Diff line number Diff line change
@@ -665,7 +665,8 @@ public Collection<Concept> executeSearch(@PathParam("sourceKey") String sourceKe
try {
Source source = getSourceRepository().findBySourceKey(sourceKey);
VocabularyInfo vocabularyInfo = getInfo(sourceKey);
SearchProviderConfig searchConfig = new SearchProviderConfig(source, vocabularyInfo);
String versionKey = vocabularyInfo.version.replace(' ', '_');
SearchProviderConfig searchConfig = new SearchProviderConfig(source.getSourceKey(), versionKey);
concepts = vocabSearchService.getSearchProvider(searchConfig).executeSearch(searchConfig, query, rows);
} catch (Exception ex) {
log.error("An error occurred during the vocabulary search", ex);
Original file line number Diff line number Diff line change
@@ -1,25 +1,39 @@
package org.ohdsi.webapi.vocabulary;

import java.util.Collection;
import java.util.Objects;

import org.ohdsi.webapi.service.VocabularyService;
import org.ohdsi.webapi.source.Source;
import org.ohdsi.webapi.source.SourceRepository;
import org.ohdsi.webapi.util.PreparedStatementRenderer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

@Component
public class DatabaseSearchProvider implements SearchProvider {
@Autowired
private SourceRepository sourceRepository;

private final static int VOCABULARY_PRIORITY = Integer.MAX_VALUE;

@Autowired
VocabularyService vocabService;

@Override
public boolean supports(VocabularySearchProviderType type) {
return Objects.equals(type, VocabularySearchProviderType.DATABASE);
public boolean supports(String vocabularyVersionKey) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So, it appears that the original implementation provided an API to 'ask' the instance if it supported a particular Type. Ie: if you ask the SOLRSearchProvider if it supports VocabularySearchProviderType.DATABASE, it would return false, and if you asked the DatabaseProvider if it supports VocabularySearchProviderType.SOLR, it would return false. Now, supports seems to work off of a vocabularyVersionKey. This seems like a big divergance between the semantics of the supports() function. If we want something specific for vocabulary version support, shouldn't we have a function that indicates that we're asking something related to the vocabularyVersion?

return true;
}


@Override
public int getPriority() {
return VOCABULARY_PRIORITY;
}

@Override
public Collection<Concept> executeSearch(SearchProviderConfig config, String query, String rows) throws Exception {
PreparedStatementRenderer psr = vocabService.prepareExecuteSearchWithQuery(query, config.getSource());
return vocabService.getSourceJdbcTemplate(config.getSource()).query(psr.getSql(), psr.getSetter(), vocabService.getRowMapper());
Source source = sourceRepository.findBySourceKey(config.getSourceKey());

PreparedStatementRenderer psr = vocabService.prepareExecuteSearchWithQuery(query, source);
return vocabService.getSourceJdbcTemplate(source).query(psr.getSql(), psr.getSetter(), vocabService.getRowMapper());
}
}
8 changes: 6 additions & 2 deletions src/main/java/org/ohdsi/webapi/vocabulary/SearchProvider.java
Original file line number Diff line number Diff line change
@@ -1,8 +1,12 @@
package org.ohdsi.webapi.vocabulary;

import org.ohdsi.webapi.vocabulary.Concept;
import org.ohdsi.webapi.vocabulary.SearchProviderConfig;

import java.util.Collection;

public interface SearchProvider {
public abstract boolean supports(VocabularySearchProviderType type);
public abstract Collection<Concept> executeSearch(SearchProviderConfig config, String query, String rows) throws Exception;
boolean supports(String vocabularyVersionKey);
int getPriority();
Collection<Concept> executeSearch(SearchProviderConfig config, String query, String rows) throws Exception;
}
Original file line number Diff line number Diff line change
@@ -1,23 +1,19 @@
package org.ohdsi.webapi.vocabulary;

import org.ohdsi.webapi.source.Source;

public class SearchProviderConfig {
protected Source source;
protected VocabularyInfo vocabularyInfo;
protected String versionKey;
private String sourceKey;
private String versionKey;

public SearchProviderConfig(Source source, VocabularyInfo vocabularyInfo) {
this.source = source;
this.vocabularyInfo = vocabularyInfo;
this.versionKey = vocabularyInfo.version.replace(' ', '_');
public SearchProviderConfig(String sourceKey, String versionKey) {
this.sourceKey = sourceKey;
this.versionKey = versionKey;
}

public String getVersionKey() {
return versionKey;
}

public Source getSource() {
return source;
public String getSourceKey() {
return sourceKey;
}
}
Original file line number Diff line number Diff line change
@@ -1,57 +1,30 @@
package org.ohdsi.webapi.vocabulary;

import java.util.HashSet;
import java.util.List;
import javax.annotation.PostConstruct;
import org.ohdsi.webapi.service.VocabularyService;
import java.util.Arrays;
import java.util.Comparator;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

@Service
public class VocabularySearchServiceImpl implements VocabularySearchService {
protected final Logger log = LoggerFactory.getLogger(getClass());
private static HashSet<String> availableVocabularyFullTextIndices = new HashSet<>();
private final List<SearchProvider> searchProviderList;

private static final String NO_PROVIDER_ERROR = "There is no vocabulary search provider which for sourceKey: %s";

@Autowired
VocabularyService vocabService;

@Autowired
SolrSearchClient solrSearchClient;

@PostConstruct
protected void init() {
// Get the SOLR cores list if enabled
if (solrSearchClient.enabled()) {
try {
availableVocabularyFullTextIndices = solrSearchClient.getCores();
} catch (Exception ex) {
log.error("SOLR Core Initialization Error: WebAPI was unable to obtain the list of available cores.", ex);
}
}
}

public VocabularySearchServiceImpl(List<SearchProvider> searchProviderList) {
this.searchProviderList = searchProviderList;

private final SearchProvider[] searchProviders;

public VocabularySearchServiceImpl(SearchProvider[] searchProviders) {
this.searchProviders = searchProviders;
}

@Override
public SearchProvider getSearchProvider(SearchProviderConfig config) {
VocabularySearchProviderType type = VocabularySearchProviderType.DATABASE;
if (availableVocabularyFullTextIndices.contains(config.getVersionKey())) {
type = VocabularySearchProviderType.SOLR;
}
return selectSearchProvider(type, config);
}

private SearchProvider selectSearchProvider(VocabularySearchProviderType type, SearchProviderConfig config) {
return searchProviderList.stream()
.filter(p -> p.supports(type))
return Arrays.stream(searchProviders)
.sorted(Comparator.comparingInt(SearchProvider::getPriority))
.filter(p -> p.supports(config.getVersionKey()))
.findFirst()
.orElseThrow(() -> new RuntimeException(String.format(NO_PROVIDER_ERROR, config.getSource().getSourceKey())));
.orElseThrow(() -> new RuntimeException(String.format(NO_PROVIDER_ERROR, config.getSourceKey())));
}
}
Original file line number Diff line number Diff line change
@@ -1,22 +1,22 @@
package org.ohdsi.webapi.vocabulary;
package org.ohdsi.webapi.vocabulary.solr;

import java.util.List;
import java.util.stream.Collectors;
import org.ohdsi.webapi.info.ConfigurationInfo;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

@Component
public class VocabularyConfigurationInfo extends ConfigurationInfo {
import java.util.ArrayList;
import java.util.List;

@Component
public class SolrConfigurationInfo extends ConfigurationInfo {
private static final String KEY = "vocabulary";

@Autowired
public VocabularyConfigurationInfo(SolrSearchClient solrSearchClient) {
properties.put("solrEnabled", solrSearchClient.enabled());
public SolrConfigurationInfo(SolrSearchClient solrSearchClient) {
properties.put("solrEnabled", true);
if (solrSearchClient.enabled()) {
try {
List<String> cores = solrSearchClient.getCores().stream().collect(Collectors.toList());
List<String> cores = new ArrayList<>(solrSearchClient.getCores());
properties.put("cores", cores);
} catch (Exception e) {
properties.put("cores", "unable to retrieve from endpoint.");
@@ -26,7 +26,6 @@ public VocabularyConfigurationInfo(SolrSearchClient solrSearchClient) {

@Override
public String getKey() {

return KEY;
}
}
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
package org.ohdsi.webapi.vocabulary;
package org.ohdsi.webapi.vocabulary.solr;

import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
Original file line number Diff line number Diff line change
@@ -1,40 +1,60 @@
package org.ohdsi.webapi.vocabulary;
package org.ohdsi.webapi.vocabulary.solr;

import java.io.IOException;
import java.util.Date;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Objects;
import org.apache.commons.beanutils.ConvertUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.springframework.stereotype.Component;

import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.BaseHttpSolrClient.RemoteSolrException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.ohdsi.webapi.vocabulary.Concept;
import org.ohdsi.webapi.vocabulary.SearchProviderConfig;
import org.ohdsi.webapi.vocabulary.SearchProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashSet;

@Component
public class SolrSearchProvider implements SearchProvider {
protected final Logger log = LoggerFactory.getLogger(getClass());


private static final int SOLR_PRIORITY = 1000;
private static HashSet<String> solrCores = new HashSet<>();

@Autowired
SolrSearchClient solrSearchClient;


@PostConstruct
protected void init() {
try {
solrCores = solrSearchClient.getCores();
} catch (Exception ex) {
log.error("SOLR Core Initialization Error: WebAPI was unable to obtain the list of available cores.", ex);
}
}

@Override
public boolean supports(VocabularySearchProviderType type) {
return Objects.equals(type, VocabularySearchProviderType.SOLR);
public boolean supports(String vocabularyVersionKey) {
return solrCores.contains(vocabularyVersionKey);
}


@Override
public int getPriority() {
return SOLR_PRIORITY;
}

@Override
public Collection<Concept> executeSearch(SearchProviderConfig config, String query, String rows) throws IOException, SolrServerException {
ArrayList<Concept> concepts = new ArrayList<>();
@@ -45,7 +65,7 @@ public Collection<Concept> executeSearch(SearchProviderConfig config, String que
QueryResponse response;
q.setStart(0);
q.setRows(Integer.parseInt(rows));
Boolean solrSearchError = false;
boolean solrSearchError = false;
try {
q.setQuery(solrSearchClient.formatSearchQuery(query));
response = client.query(q);
@@ -56,7 +76,7 @@ public Collection<Concept> executeSearch(SearchProviderConfig config, String que
log.error("SOLR Search Query: \"" + query + "\" failed with message: " + rse.getMessage());
solrSearchError = true;
}

// If we did not receive results from issuing the initial wildcard
// query OR there was an exception usually due to a maxBooleanClause
// violation from doing a wildcard search on a very common term, then
@@ -66,7 +86,7 @@ public Collection<Concept> executeSearch(SearchProviderConfig config, String que
response = client.query(q);
results = response.getResults();
}

for (int i = 0; i < results.size(); ++i) {
SolrDocument d = results.get(i);
Concept c = new Concept();
@@ -81,23 +101,23 @@ public Collection<Concept> executeSearch(SearchProviderConfig config, String que
c.validStartDate = convertObjectToDate(d.getFieldValue("valid_start_date"));
c.validEndDate = convertObjectToDate(d.getFieldValue("valid_end_date"));
concepts.add(c);
}
}

return concepts;
}

protected String convertObjectToString(Object obj) {
return convertObjectToString(obj, null);
}

protected String convertObjectToString(Object obj, String defaultValue) {
String returnVal = ConvertUtils.convert(obj);
if (defaultValue != null && returnVal == null) {
returnVal = defaultValue;
}
return returnVal;
}

protected Long convertObjectToLong(Object obj) {
return NumberUtils.createLong(ConvertUtils.convert(obj));
}