Wiring up CLI options.

opensearch-search-quality-evaluation-framework/run.sh

@@ -3,5 +3,8 @@
 # Create a click model.
 java -jar ./target/search-evaluation-framework-1.0.0-SNAPSHOT-jar-with-dependencies.jar -c coec
 
+# Create a query set using sampling.
+#java -jar ./target/search-evaluation-framework-1.0.0-SNAPSHOT-jar-with-dependencies.jar -s sampling.json
+
 # Run a query set.
 #java -jar ./target/search-evaluation-framework-1.0.0-SNAPSHOT-jar-with-dependencies.jar -r queryset.json

opensearch-search-quality-evaluation-framework/sampling.json

@@ -0,0 +1,7 @@
+{
+  "sampler": "all",
+  "name": "",
+  "description": "",
+  "sampling": "",
+  "querySetSize": 500
+}

opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/App.java

@@ -9,6 +9,9 @@
 package org.opensearch.eval;
 
 import com.google.gson.Gson;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
 import org.apache.commons.cli.DefaultParser;
@@ -22,6 +25,10 @@
 import org.opensearch.eval.judgments.clickmodel.coec.CoecClickModelParameters;
 import org.opensearch.eval.runners.OpenSearchQuerySetRunner;
 import org.opensearch.eval.runners.RunQuerySetParameters;
+import org.opensearch.eval.samplers.AllQueriesQuerySampler;
+import org.opensearch.eval.samplers.AllQueriesQuerySamplerParameters;
+import org.opensearch.eval.samplers.ProbabilityProportionalToSizeParameters;
+import org.opensearch.eval.samplers.ProbabilityProportionalToSizeQuerySampler;
 
 import java.io.File;
 import java.nio.charset.StandardCharsets;
@@ -39,6 +46,7 @@ public static void main(String[] args) throws Exception {
 
         final Options options = new Options();
         options.addOption("c", true, "create a click model");
+        options.addOption("s", true, "create a query set using sampling");
         options.addOption("r", true, "run a query set");
 
         final CommandLineParser parser = new DefaultParser();
@@ -82,6 +90,51 @@ public static void main(String[] args) throws Exception {
                 System.err.println("The query set run parameters file does not exist.");
             }
 
+        } else if (cmd.hasOption("s")) {
+
+            final String samplerOptionsFile = cmd.getOptionValue("s");
+            final File file = new File(samplerOptionsFile);
+
+            if(file.exists()) {
+
+                final String jsonString = Files.readString(file.toPath(), StandardCharsets.UTF_8);
+                final JsonElement jsonElement = JsonParser.parseString(jsonString);
+                final JsonObject jsonObject = jsonElement.getAsJsonObject();
+                final String samplerType = jsonObject.get("sampler").getAsString();
+
+                if("all".equalsIgnoreCase(samplerType)) {
+
+                    final AllQueriesQuerySamplerParameters parameters = gson.fromJson(jsonString, AllQueriesQuerySamplerParameters.class);
+
+                    final AllQueriesQuerySampler sampler = new AllQueriesQuerySampler(searchEngine, parameters);
+                    final String querySetId = sampler.sample();
+
+                    System.out.println("Query set created: " + querySetId);
+
+                } else if("pptss".equalsIgnoreCase(samplerType)) {
+
+                    final ProbabilityProportionalToSizeParameters parameters = gson.fromJson(jsonString, ProbabilityProportionalToSizeParameters.class);
+
+                    final ProbabilityProportionalToSizeQuerySampler sampler = new ProbabilityProportionalToSizeQuerySampler(searchEngine, parameters);
+                    final String querySetId = sampler.sample();
+
+                    System.out.println("Query set created: " + querySetId);
+
+                } else {
+
+                    System.err.println("Invalid sampler.");
+
+                }
+
+            } else {
+                System.err.println("The query set run parameters file does not exist.");
+            }
+
+
+        } else {
+
+            System.err.println("Invalid options.");
+
         }
 
     }
@@ -125,37 +178,7 @@ public static void main(String[] args) throws Exception {
 //                    } catch(Exception ex) {
 //                        return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, "{\"error\": \"" + ex.getMessage() + "\"}"));
 //                    }
-//
-//
-//                // Create a query set by using PPTSS sampling.
-//                } else if (ProbabilityProportionalToSizeAbstractQuerySampler.NAME.equalsIgnoreCase(sampling)) {
-//
-//                    LOGGER.info("Creating query set using PPTSS");
-//
-//                    final ProbabilityProportionalToSizeParameters parameters = new ProbabilityProportionalToSizeParameters(name, description, sampling, querySetSize);
-//                    final ProbabilityProportionalToSizeAbstractQuerySampler sampler = new ProbabilityProportionalToSizeAbstractQuerySampler(client, parameters);
-//
-//                    try {
-//
-//                        // Sample and index the queries.
-//                        final String querySetId = sampler.sample();
-//
-//                        return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "{\"query_set\": \"" + querySetId + "\"}"));
-//
-//                    } catch(Exception ex) {
-//                        return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, "{\"error\": \"" + ex.getMessage() + "\"}"));
-//                    }
-//
-//                } else {
-//                    // An Invalid sampling method was provided in the request.
-//                    return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "{\"error\": \"Invalid sampling method: " + sampling + "\"}"));
-//                }
-//
-//            } else {
-//                // Invalid HTTP method for this endpoint.
-//                return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.METHOD_NOT_ALLOWED, "{\"error\": \"" + request.method() + " is not allowed.\"}"));
-//            }
-//
+
 
 //        // Handle the on-demand creation of implicit judgments.
 //        } else if(IMPLICIT_JUDGMENTS_URL.equalsIgnoreCase(request.path())) {

opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/AbstractSamplerParameters.java

@@ -10,12 +10,14 @@
 
 public class AbstractSamplerParameters {
 
+    private final String sampler;
     private final String name;
     private final String description;
     private final String sampling;
     private final int querySetSize;
 
-    public AbstractSamplerParameters(final String name, final String description, final String sampling, final int querySetSize) {
+    public AbstractSamplerParameters(final String sampler, final String name, final String description, final String sampling, final int querySetSize) {
+        this.sampler = sampler;
         this.name = name;
         this.description = description;
         this.sampling = sampling;
@@ -38,4 +40,8 @@ public int getQuerySetSize() {
         return querySetSize;
     }
 
+    public String getSampler() {
+        return sampler;
+    }
+
 }

opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/AllQueriesQuerySamplerParameters.java

@@ -11,7 +11,7 @@
 public class AllQueriesQuerySamplerParameters extends AbstractSamplerParameters {
 
     public AllQueriesQuerySamplerParameters(final String name, final String description, final String sampling, final int querySetSize) {
-        super(name, description, sampling, querySetSize);
+        super("all", name, description, sampling, querySetSize);
     }
 
 }

opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/ProbabilityProportionalToSizeParameters.java

@@ -11,7 +11,7 @@
 public class ProbabilityProportionalToSizeParameters extends AbstractSamplerParameters {
 
     public ProbabilityProportionalToSizeParameters(final String name, final String description, final String sampling, final int querySetSize) {
-        super(name, description, sampling, querySetSize);
+        super("pptss", name, description, sampling, querySetSize);
     }
 
 }

opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/ProbabilityProportionalToSizeAbstractQuerySampler.java → opensearch-search-quality-evaluation-framework/src/main/java/org/opensearch/eval/samplers/ProbabilityProportionalToSizeQuerySampler.java

@@ -25,11 +25,11 @@
  * See https://opensourceconnections.com/blog/2022/10/13/how-to-succeed-with-explicit-relevance-evaluation-using-probability-proportional-to-size-sampling/
  * for more information on PPTSS.
  */
-public class ProbabilityProportionalToSizeAbstractQuerySampler extends AbstractQuerySampler {
+public class ProbabilityProportionalToSizeQuerySampler extends AbstractQuerySampler {
 
     public static final String NAME = "pptss";
 
-    private static final Logger LOGGER = LogManager.getLogger(ProbabilityProportionalToSizeAbstractQuerySampler.class);
+    private static final Logger LOGGER = LogManager.getLogger(ProbabilityProportionalToSizeQuerySampler.class);
 
     private final SearchEngine searchEngine;
     private final ProbabilityProportionalToSizeParameters parameters;
@@ -39,7 +39,7 @@ public class ProbabilityProportionalToSizeAbstractQuerySampler extends AbstractQ
      * @param searchEngine The OpenSearch {@link SearchEngine engine}.
      * @param parameters The {@link ProbabilityProportionalToSizeParameters parameters} for the sampling.
      */
-    public ProbabilityProportionalToSizeAbstractQuerySampler(final SearchEngine searchEngine, final ProbabilityProportionalToSizeParameters parameters) {
+    public ProbabilityProportionalToSizeQuerySampler(final SearchEngine searchEngine, final ProbabilityProportionalToSizeParameters parameters) {
         this.searchEngine = searchEngine;
         this.parameters = parameters;
     }