Add end to end expression tests with Presto sideccar

prestodb · Sep 11, 2024 · ee395c9 · ee395c9
1 parent af06a50
commit ee395c9
Show file tree

Hide file tree

Showing 7 changed files with 218 additions and 109 deletions.
diff --git a/pom.xml b/pom.xml
@@ -698,6 +698,13 @@
                 <version>${project.version}</version>
             </dependency>
 
+            <dependency>
+                <groupId>com.facebook.presto</groupId>
+                <artifactId>presto-tests</artifactId>
+                <version>${project.version}</version>
+                <type>test-jar</type>
+            </dependency>
+
             <dependency>
                 <groupId>com.facebook.presto</groupId>
                 <artifactId>presto-benchmark</artifactId>
@@ -899,6 +906,13 @@
                 <version>${project.version}</version>
             </dependency>
 
+            <dependency>
+                <groupId>com.facebook.presto</groupId>
+                <artifactId>presto-native-execution</artifactId>
+                <version>${project.version}</version>
+                <type>test-jar</type>
+            </dependency>
+
             <dependency>
                 <groupId>com.facebook.hive</groupId>
                 <artifactId>hive-dwrf</artifactId>

diff --git a/...xecution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java b/...xecution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java
@@ -26,6 +26,7 @@
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.net.ServerSocket;
 import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -420,6 +421,11 @@ public static NativeQueryRunnerParameters getNativeQueryRunnerParameters()
     }
 
     public static Optional<BiFunction<Integer, URI, Process>> getExternalWorkerLauncher(String catalogName, String prestoServerPath, int cacheMaxSize, Optional<String> remoteFunctionServerUds, Boolean failOnNestedLoopJoin)
+    {
+        return getExternalWorkerLauncher(catalogName, prestoServerPath, OptionalInt.empty(), cacheMaxSize, remoteFunctionServerUds, failOnNestedLoopJoin);
+    }
+
+    public static Optional<BiFunction<Integer, URI, Process>> getExternalWorkerLauncher(String catalogName, String prestoServerPath, OptionalInt port, int cacheMaxSize, Optional<String> remoteFunctionServerUds, Boolean failOnNestedLoopJoin)
     {
         return
                 Optional.of((workerIndex, discoveryUri) -> {
@@ -428,13 +434,13 @@ public static Optional<BiFunction<Integer, URI, Process>> getExternalWorkerLaunc
                         Files.createDirectories(dir);
                         Path tempDirectoryPath = Files.createTempDirectory(dir, "worker");
                         log.info("Temp directory for Worker #%d: %s", workerIndex, tempDirectoryPath.toString());
-                        int port = 1234 + workerIndex;
 
                         // Write config file
                         String configProperties = format("discovery.uri=%s%n" +
                                 "presto.version=testversion%n" +
                                 "system-memory-gb=4%n" +
-                                "http-server.http.port=%d", discoveryUri, port);
+                                "native-sidecar=true%n" +
+                                "http-server.http.port=%d", discoveryUri, port.orElse(1234 + workerIndex));
 
                         if (remoteFunctionServerUds.isPresent()) {
                             String jsonSignaturesPath = Resources.getResource(REMOTE_FUNCTION_JSON_SIGNATURES).getFile();
@@ -518,4 +524,12 @@ public static void setupJsonFunctionNamespaceManager(QueryRunner queryRunner, St
                         "function-implementation-type", "CPP",
                         "json-based-function-manager.path-to-function-definition", jsonDefinitionPath));
     }
+
+    public static int findRandomPortForWorker()
+            throws IOException
+    {
+        try (ServerSocket socket = new ServerSocket(0)) {
+            return socket.getLocalPort();
+        }
+    }
 }
diff --git a/presto-native-plugin/pom.xml b/presto-native-plugin/pom.xml
@@ -58,6 +58,16 @@
             <artifactId>http-client</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>com.facebook.airlift</groupId>
+            <artifactId>log</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.facebook.airlift</groupId>
+            <artifactId>log-manager</artifactId>
+        </dependency>
+
         <!-- Presto SPI -->
         <dependency>
             <groupId>com.facebook.presto</groupId>
@@ -119,5 +129,62 @@
             <artifactId>testing</artifactId>
             <scope>test</scope>
         </dependency>
+
+        <dependency>
+            <groupId>com.facebook.presto</groupId>
+            <artifactId>presto-tests</artifactId>
+            <scope>test</scope>
+            <type>test-jar</type>
+        </dependency>
+
+        <dependency>
+            <groupId>com.facebook.presto</groupId>
+            <artifactId>presto-tests</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.facebook.presto</groupId>
+            <artifactId>presto-native-execution</artifactId>
+            <scope>test</scope>
+            <type>test-jar</type>
+        </dependency>
+
+        <dependency>
+            <groupId>com.facebook.presto</groupId>
+            <artifactId>presto-main</artifactId>
+            <scope>test</scope>
+            <type>test-jar</type>
+        </dependency>
+
+        <dependency>
+            <groupId>com.facebook.presto</groupId>
+            <artifactId>presto-main</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.facebook.presto</groupId>
+            <artifactId>presto-tpcds</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.facebook.airlift</groupId>
+            <artifactId>jaxrs</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.facebook.presto</groupId>
+            <artifactId>presto-client</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.facebook.airlift</groupId>
+            <artifactId>jaxrs-testing</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 </project>
diff --git a/...in/src/test/java/com/facebook/presto/session/sql/expressions/NativePluginQueryRunner.java b/...in/src/test/java/com/facebook/presto/session/sql/expressions/NativePluginQueryRunner.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.session.sql.expressions;
+
+import com.facebook.airlift.log.Logger;
+import com.facebook.airlift.log.Logging;
+import com.facebook.presto.nativeworker.NativeQueryRunnerUtils;
+import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils;
+import com.facebook.presto.testing.QueryRunner;
+import com.facebook.presto.tests.DistributedQueryRunner;
+import com.google.common.collect.ImmutableMap;
+
+public class NativePluginQueryRunner
+{
+    private NativePluginQueryRunner() {}
+
+    public static void main(String[] args)
+            throws Exception
+    {
+        // You need to add "--user user" to your CLI for your queries to work.
+        Logging.initialize();
+
+        // Create tables before launching distributed runner.
+        QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner(false);
+        NativeQueryRunnerUtils.createAllTables(javaQueryRunner);
+        javaQueryRunner.close();
+
+        // Launch distributed runner.
+        DistributedQueryRunner queryRunner = (DistributedQueryRunner) PrestoNativeQueryRunnerUtils.createQueryRunner(false);
+        queryRunner.getExpressionManager().addExpressionOptimizerFactory(new NativeExpressionOptimizerFactory(ClassLoader.getSystemClassLoader()));
+        queryRunner.getExpressionManager().loadExpressions(ImmutableMap.<String, String>builder().put("expression-manager-factory.name", "native").build());
+        Thread.sleep(10);
+        Logger log = Logger.get(DistributedQueryRunner.class);
+        log.info("======== SERVER STARTED ========");
+        log.info("\n====\n%s\n====", queryRunner.getCoordinator().getBaseUrl());
+    }
+}
diff --git a/...ns/TestDelegatingExpressionOptimizer.java → ...ns/TestDelegatingExpressionOptimizer.java b/...ns/TestDelegatingExpressionOptimizer.java → ...ns/TestDelegatingExpressionOptimizer.java
@@ -11,8 +11,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.facebook.presto.tests.expressions;
+package com.facebook.presto.session.sql.expressions;
 
+import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils;
 import com.facebook.presto.spi.relation.CallExpression;
 import com.facebook.presto.spi.relation.ConstantExpression;
 import com.facebook.presto.spi.relation.ExpressionOptimizer;
@@ -21,17 +22,22 @@
 import com.facebook.presto.sql.planner.Symbol;
 import com.facebook.presto.sql.relational.DelegatingRowExpressionOptimizer;
 import com.facebook.presto.sql.relational.FunctionResolution;
+import com.facebook.presto.tests.expressions.TestExpressions;
 import com.google.common.collect.ImmutableList;
-import org.intellij.lang.annotations.Language;
+import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
+import java.net.URI;
 import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.function.BiFunction;
 
 import static com.facebook.presto.SessionTestUtils.TEST_SESSION;
 import static com.facebook.presto.common.type.BooleanType.BOOLEAN;
 import static com.facebook.presto.common.type.VarcharType.VARCHAR;
-import static com.facebook.presto.operator.scalar.ApplyFunction.APPLY_FUNCTION;
+import static com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils.findRandomPortForWorker;
+import static com.facebook.presto.session.sql.expressions.TestNativeExpressionOptimization.getExpressionOptimizer;
 import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level;
 import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.EVALUATED;
 import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED;
@@ -47,14 +53,30 @@ public class TestDelegatingExpressionOptimizer
 {
     public static final FunctionResolution RESOLUTION = new FunctionResolution(METADATA.getFunctionAndTypeManager().getFunctionAndTypeResolver());
     private ExpressionOptimizer expressionOptimizer;
+    private Process sidecar;
 
     @BeforeClass
     public void setup()
+            throws Exception
     {
-        METADATA.getFunctionAndTypeManager().registerBuiltInFunctions(ImmutableList.of(APPLY_FUNCTION));
-        setupJsonFunctionNamespaceManager(METADATA.getFunctionAndTypeManager());
+        int port = findRandomPortForWorker();
+        URI sidecarUri = URI.create("http://127.0.0.1:" + port);
+        Optional<BiFunction<Integer, URI, Process>> launcher = PrestoNativeQueryRunnerUtils.getExternalWorkerLauncher(
+                "hive",
+                "/Users/tdcmeehan/git/presto/presto-native-execution/_build/release/presto_cpp/main/presto_server",
+                OptionalInt.of(port),
+                0,
+                Optional.empty(),
+                false);
+        sidecar = launcher.get().apply(0, URI.create("http://test.invalid/"));
+
+        expressionOptimizer = new DelegatingRowExpressionOptimizer(METADATA, () -> getExpressionOptimizer(METADATA, HANDLE_RESOLVER, sidecarUri));
+    }
 
-        expressionOptimizer = new DelegatingRowExpressionOptimizer(METADATA, () -> TestNativeExpressions.getExpressionOptimizer(METADATA, HANDLE_RESOLVER));
+    @AfterClass
+    public void tearDown()
+    {
+        sidecar.destroyForcibly();
     }
 
     @Test
@@ -118,7 +140,7 @@ protected Object evaluate(String expression, boolean deterministic)
     }
 
     @Override
-    protected Object optimize(@Language("SQL") String expression)
+    protected Object optimize(String expression)
     {
         assertRoundTrip(expression);
         RowExpression parsedExpression = sqlToRowExpression(expression);
@@ -154,15 +176,15 @@ public Object unwrap(Object result)
     }
 
     @Override
-    protected void assertOptimizedEquals(@Language("SQL") String actual, @Language("SQL") String expected)
+    protected void assertOptimizedEquals(String actual, String expected)
     {
         Object optimizedActual = optimize(actual);
         Object optimizedExpected = optimize(expected);
         assertRowExpressionEvaluationEquals(optimizedActual, optimizedExpected);
     }
 
     @Override
-    protected void assertOptimizedMatches(@Language("SQL") String actual, @Language("SQL") String expected)
+    protected void assertOptimizedMatches(String actual, String expected)
     {
         Object actualOptimized = optimize(actual);
         Object expectedOptimized = optimize(expected);
@@ -172,7 +194,7 @@ protected void assertOptimizedMatches(@Language("SQL") String actual, @Language(
     }
 
     @Override
-    protected void assertDoNotOptimize(@Language("SQL") String expression, Level optimizationLevel)
+    protected void assertDoNotOptimize(String expression, Level optimizationLevel)
     {
         assertRoundTrip(expression);
         RowExpression rowExpression = sqlToRowExpression(expression);