updates for solr based search

pom.xml

@@ -44,7 +44,10 @@
     <cdm.version>5</cdm.version>
     <!-- Person properties -->
     <person.viewDates>false</person.viewDates>
-
+    <!-- Full Text Search With SOLR Settings -->
+    <solr.endpoint></solr.endpoint>
+    <solr.query.prefix>{!complexphrase inOrder=true}</solr.query.prefix>
+    <solr.version>8.3.1</solr.version>
     <!-- Heracles properties -->
     <heracles.smallcellcount>5</heracles.smallcellcount>
 
@@ -540,7 +543,7 @@
     <dependency>
       <groupId>org.apache.httpcomponents</groupId>
       <artifactId>httpclient</artifactId>
-      <version>4.3.6</version>
+      <version>4.5.10</version>
     </dependency>
     <dependency>
       <groupId>commons-httpclient</groupId>
@@ -788,6 +791,12 @@
       <version>0.13.1</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+        <groupId>org.apache.solr</groupId>
+        <artifactId>solr-solrj</artifactId>
+        <version>8.3.1</version>
+      <type>jar</type>
+    </dependency>
   </dependencies>
   <profiles>
     <profile>

src/main/java/org/ohdsi/webapi/WebApi.java

@@ -12,14 +12,15 @@
 
 import javax.annotation.PostConstruct;
 import java.util.TimeZone;
+import org.springframework.boot.autoconfigure.solr.SolrAutoConfiguration;
 
 
 /**
  * Launch as java application or deploy as WAR (@link {@link WebApplication}
  * will source this file).
  */
 @EnableScheduling
-@SpringBootApplication(exclude={HibernateJpaAutoConfiguration.class, ErrorMvcAutoConfiguration.class})
+@SpringBootApplication(exclude={HibernateJpaAutoConfiguration.class, ErrorMvcAutoConfiguration.class, SolrAutoConfiguration.class})
 @EnableAspectJAutoProxy(proxyTargetClass = true)
 public class WebApi extends SpringBootServletInitializer {
 

src/main/java/org/ohdsi/webapi/service/VocabularyService.java

@@ -10,11 +10,13 @@
 import java.util.stream.Collectors;
 
 import javax.ws.rs.Consumes;
+import javax.ws.rs.DefaultValue;
 import javax.ws.rs.GET;
 import javax.ws.rs.NotFoundException;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
+import javax.ws.rs.QueryParam;
 import javax.ws.rs.Produces;
 import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.MediaType;
@@ -44,31 +46,34 @@
 import org.ohdsi.webapi.vocabulary.Domain;
 import org.ohdsi.webapi.vocabulary.RelatedConcept;
 import org.ohdsi.webapi.vocabulary.RelatedConceptSearch;
+import org.ohdsi.webapi.vocabulary.SearchProviderConfig;
 import org.ohdsi.webapi.vocabulary.Vocabulary;
 import org.ohdsi.webapi.vocabulary.VocabularyInfo;
+import org.ohdsi.webapi.vocabulary.VocabularySearchService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.dao.EmptyResultDataAccessException;
 import org.springframework.jdbc.core.RowCallbackHandler;
 import org.springframework.jdbc.core.RowMapper;
 import org.springframework.stereotype.Component;
 
-/**
- * @author fdefalco
- */
 @Path("vocabulary/")
 @Component
 public class VocabularyService extends AbstractDaoService {
 
   private static Hashtable<String, VocabularyInfo> vocabularyInfoCache = null;
+  public static final String DEFAULT_SEARCH_ROWS = "20000";
 
   @Autowired
   private SourceService sourceService;
+
+  @Autowired
+  private VocabularySearchService vocabSearchService;
 
   @Value("${datasource.driverClassName}")
   private String driver;
-  
-  private final RowMapper<Concept> rowMapper = new RowMapper<Concept>() {
+
+  public final RowMapper<Concept> rowMapper = new RowMapper<Concept>() {
     @Override
     public Concept mapRow(final ResultSet resultSet, final int arg1) throws SQLException {
       final Concept concept = new Concept();
@@ -368,8 +373,6 @@ public Collection<Concept> executeMappedLookup(String sourceKey, ConceptSetExpre
   @Produces(MediaType.APPLICATION_JSON)
   @Consumes(MediaType.APPLICATION_JSON)
   public Collection<Concept> executeSearch(@PathParam("sourceKey") String sourceKey, ConceptSearch search) {
-    Tracker.trackActivity(ActivityType.Search, search.query);
-
     Source source = getSourceRepository().findBySourceKey(sourceKey);
 
     PreparedStatementRenderer psr = prepareExecuteSearch(search, source);
@@ -457,22 +460,51 @@ public Collection<Concept> executeSearch(ConceptSearch search) {
 
     return executeSearch(defaultSourceKey, search);    
   }
-
   /**
+   * @param sourceKey
    * @param query
    * @return
    */
   @Path("{sourceKey}/search/{query}")
   @GET
   @Produces(MediaType.APPLICATION_JSON)
   public Collection<Concept> executeSearch(@PathParam("sourceKey") String sourceKey, @PathParam("query") String query) {
-    Tracker.trackActivity(ActivityType.Search, query);
-    Source source = getSourceRepository().findBySourceKey(sourceKey);
-    PreparedStatementRenderer psr = prepareExecuteSearchWithQuery(query, source);
-    return getSourceJdbcTemplate(source).query(psr.getSql(), psr.getSetter(), this.rowMapper);
+    return this.executeSearch(sourceKey, query, null);
+  }
+
+  /**
+   * @param sourceKey
+   * @param query
+   * @param rows
+   * @return
+   */
+  @Path("{sourceKey}/search")
+  @GET
+  @Produces(MediaType.APPLICATION_JSON)
+  public Collection<Concept> executeSearch(@PathParam("sourceKey") String sourceKey, @QueryParam("query") String query, @DefaultValue(DEFAULT_SEARCH_ROWS) @QueryParam("rows") String rows) {
+    // Verify that the rows parameter contains an integer and is > 0
+    try {
+        Integer r = Integer.parseInt(rows);
+        if (r <= 0) {
+            throw new NumberFormatException("The rows parameter must be greater than 0");
+        }
+    } catch (NumberFormatException nfe) {
+        throw nfe;
+    }
+
+    Collection<Concept> concepts = new ArrayList<>();
+    try {
+        Source source = getSourceRepository().findBySourceKey(sourceKey);
+        VocabularyInfo vocabularyInfo = getInfo(sourceKey);
+        SearchProviderConfig searchConfig = new SearchProviderConfig(source, vocabularyInfo);
+        concepts = vocabSearchService.getSearchProvider(searchConfig).executeSearch(searchConfig, query, rows);
+    } catch (Exception ex) {
+        log.error("An error occurred during the vocabulary search", ex);
+    }
+    return concepts;
   }
 
-  protected PreparedStatementRenderer prepareExecuteSearchWithQuery(String query, Source source) {
+  public PreparedStatementRenderer prepareExecuteSearchWithQuery(String query, Source source) {
 
     String resourcePath = "/resources/vocabulary/sql/search.sql";
     String tqValue = source.getTableQualifier(SourceDaimon.DaimonType.Vocabulary);

src/main/java/org/ohdsi/webapi/vocabulary/DatabaseSearchProvider.java

@@ -0,0 +1,26 @@
+package org.ohdsi.webapi.vocabulary;
+
+import java.util.Collection;
+import java.util.Objects;
+import org.ohdsi.circe.vocabulary.Concept;
+import org.ohdsi.webapi.service.VocabularyService;
+import org.ohdsi.webapi.util.PreparedStatementRenderer;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class DatabaseSearchProvider implements SearchProvider {
+    @Autowired
+    VocabularyService vocabService;
+
+    @Override
+    public boolean supports(VocabularySearchProviderType type) {
+        return Objects.equals(type, VocabularySearchProviderType.DATABASE);
+    }
+
+    @Override
+    public Collection<Concept> executeSearch(SearchProviderConfig config, String query, String rows) throws Exception {
+        PreparedStatementRenderer psr = vocabService.prepareExecuteSearchWithQuery(query, config.getSource());
+        return vocabService.getSourceJdbcTemplate(config.getSource()).query(psr.getSql(), psr.getSetter(), vocabService.rowMapper);
+    }
+}

src/main/java/org/ohdsi/webapi/vocabulary/SearchProvider.java

@@ -0,0 +1,9 @@
+package org.ohdsi.webapi.vocabulary;
+
+import java.util.Collection;
+import org.ohdsi.circe.vocabulary.Concept;
+
+public interface SearchProvider {
+    public abstract boolean supports(VocabularySearchProviderType type);
+    public abstract Collection<Concept> executeSearch(SearchProviderConfig config, String query, String rows) throws Exception;
+}

src/main/java/org/ohdsi/webapi/vocabulary/SearchProviderConfig.java

@@ -0,0 +1,23 @@
+package org.ohdsi.webapi.vocabulary;
+
+import org.ohdsi.webapi.source.Source;
+
+public class SearchProviderConfig {
+    protected Source source;
+    protected VocabularyInfo vocabularyInfo;
+    protected String versionKey;
+
+    public SearchProviderConfig(Source source, VocabularyInfo vocabularyInfo) {
+        this.source = source;
+        this.vocabularyInfo = vocabularyInfo;
+        this.versionKey = vocabularyInfo.version.replace(' ', '_');
+    }
+
+    public String getVersionKey() {
+        return versionKey;
+    }
+
+    public Source getSource() {
+        return source;
+    }
+}

src/main/java/org/ohdsi/webapi/vocabulary/SolrSearchClient.java

@@ -0,0 +1,79 @@
+package org.ohdsi.webapi.vocabulary;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.regex.Pattern;
+import org.apache.commons.lang.StringUtils;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.request.CoreAdminRequest;
+import org.apache.solr.client.solrj.response.CoreAdminResponse;
+import org.apache.solr.client.solrj.util.ClientUtils;
+import org.apache.solr.common.params.CoreAdminParams;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class SolrSearchClient {
+    @Value("${solr.endpoint}")
+    private String solrEndpoint;
+
+    @Value("${solr.query.prefix}")
+    private String solrQueryPrefix;
+
+    public static final List<String> SOLR_ESCAPE_CHARACTERS = Arrays.asList("(", ")", "{", "}", "[", "]", "^", "\"", ":");
+
+    public boolean enabled() {
+        return !StringUtils.isEmpty(solrEndpoint);
+    }
+
+    public SolrClient getSolrClient(String coreName) {
+        return new HttpSolrClient.Builder(solrEndpoint + "/" + coreName).build();
+    }
+
+    public HashSet<String> getCores() throws Exception {
+        HashSet<String> returnVal = new HashSet<>();
+        SolrClient client = this.getSolrClient("");
+        CoreAdminRequest request = new CoreAdminRequest();
+        request.setAction(CoreAdminParams.CoreAdminAction.STATUS);
+        CoreAdminResponse cores;
+
+        try {
+            cores = request.process(client);
+            for (int i = 0; i < cores.getCoreStatus().size(); i++) {
+                returnVal.add(cores.getCoreStatus().getName(i));
+            }
+        } catch (Exception ex) {
+          throw ex;
+        }
+        return returnVal;
+    }
+
+    public String formatSearchQuery(String query) {
+        return formatSearchQuery(query, true);
+    }
+
+    public String formatSearchQuery(String query, Boolean useWildcardSearch) {
+        String returnVal;
+        if (useWildcardSearch) {
+            returnVal = solrQueryPrefix + "query:\"*" + ClientUtils.escapeQueryChars(query) + "*\"";
+        } else {
+            returnVal = "query:" + escapeNonWildcardQuery(query);
+        }
+        System.out.println(returnVal);
+        return returnVal;
+    }
+
+    // This escape function is used when building the non wildcard
+    // query since the ClientUtils.escapeQueryChars will replace 
+    // add an extra "\" to spaces which can change the query results.
+    // So, here we escape a subset of the special characters for
+    // this edge case
+    public String escapeNonWildcardQuery(String query) {
+        for (String item : SOLR_ESCAPE_CHARACTERS) {
+            query = query.replace(item, "\\" + item);
+        }
+        return query;
+    }
+}