diff --git a/pom.xml b/pom.xml index 02e457baef7e2..34ae62eee83da 100644 --- a/pom.xml +++ b/pom.xml @@ -698,6 +698,13 @@ ${project.version} + + com.facebook.presto + presto-tests + ${project.version} + test-jar + + com.facebook.presto presto-benchmark @@ -899,6 +906,13 @@ ${project.version} + + com.facebook.presto + presto-native-execution + ${project.version} + test-jar + + com.facebook.hive hive-dwrf diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java index 69bb2433b3f18..c245f17b1b394 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.net.ServerSocket; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; @@ -420,6 +421,11 @@ public static NativeQueryRunnerParameters getNativeQueryRunnerParameters() } public static Optional> getExternalWorkerLauncher(String catalogName, String prestoServerPath, int cacheMaxSize, Optional remoteFunctionServerUds, Boolean failOnNestedLoopJoin) + { + return getExternalWorkerLauncher(catalogName, prestoServerPath, OptionalInt.empty(), cacheMaxSize, remoteFunctionServerUds, failOnNestedLoopJoin); + } + + public static Optional> getExternalWorkerLauncher(String catalogName, String prestoServerPath, OptionalInt port, int cacheMaxSize, Optional remoteFunctionServerUds, Boolean failOnNestedLoopJoin) { return Optional.of((workerIndex, discoveryUri) -> { @@ -428,13 +434,13 @@ public static Optional> getExternalWorkerLaunc Files.createDirectories(dir); Path tempDirectoryPath = Files.createTempDirectory(dir, "worker"); log.info("Temp directory for Worker #%d: %s", workerIndex, tempDirectoryPath.toString()); - int port = 1234 + workerIndex; // Write config file String configProperties = format("discovery.uri=%s%n" + "presto.version=testversion%n" + "system-memory-gb=4%n" + - "http-server.http.port=%d", discoveryUri, port); + "native-sidecar=true%n" + + "http-server.http.port=%d", discoveryUri, port.orElse(1234 + workerIndex)); if (remoteFunctionServerUds.isPresent()) { String jsonSignaturesPath = Resources.getResource(REMOTE_FUNCTION_JSON_SIGNATURES).getFile(); @@ -518,4 +524,12 @@ public static void setupJsonFunctionNamespaceManager(QueryRunner queryRunner, St "function-implementation-type", "CPP", "json-based-function-manager.path-to-function-definition", jsonDefinitionPath)); } + + public static int findRandomPortForWorker() + throws IOException + { + try (ServerSocket socket = new ServerSocket(0)) { + return socket.getLocalPort(); + } + } } diff --git a/presto-native-plugin/pom.xml b/presto-native-plugin/pom.xml index eeba938f82aa6..557155fae67e8 100644 --- a/presto-native-plugin/pom.xml +++ b/presto-native-plugin/pom.xml @@ -58,6 +58,16 @@ http-client + + com.facebook.airlift + log + + + + com.facebook.airlift + log-manager + + com.facebook.presto @@ -119,5 +129,62 @@ testing test + + + com.facebook.presto + presto-tests + test + test-jar + + + + com.facebook.presto + presto-tests + test + + + + com.facebook.presto + presto-native-execution + test + test-jar + + + + com.facebook.presto + presto-main + test + test-jar + + + + com.facebook.presto + presto-main + test + + + + com.facebook.presto + presto-tpcds + test + + + + com.facebook.airlift + jaxrs + test + + + + com.facebook.presto + presto-client + test + + + + com.facebook.airlift + jaxrs-testing + test + diff --git a/presto-native-plugin/src/test/java/com/facebook/presto/session/sql/expressions/NativePluginQueryRunner.java b/presto-native-plugin/src/test/java/com/facebook/presto/session/sql/expressions/NativePluginQueryRunner.java new file mode 100644 index 0000000000000..af55642b2acf7 --- /dev/null +++ b/presto-native-plugin/src/test/java/com/facebook/presto/session/sql/expressions/NativePluginQueryRunner.java @@ -0,0 +1,48 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.session.sql.expressions; + +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.log.Logging; +import com.facebook.presto.nativeworker.NativeQueryRunnerUtils; +import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils; +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tests.DistributedQueryRunner; +import com.google.common.collect.ImmutableMap; + +public class NativePluginQueryRunner +{ + private NativePluginQueryRunner() {} + + public static void main(String[] args) + throws Exception + { + // You need to add "--user user" to your CLI for your queries to work. + Logging.initialize(); + + // Create tables before launching distributed runner. + QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner(false); + NativeQueryRunnerUtils.createAllTables(javaQueryRunner); + javaQueryRunner.close(); + + // Launch distributed runner. + DistributedQueryRunner queryRunner = (DistributedQueryRunner) PrestoNativeQueryRunnerUtils.createQueryRunner(false); + queryRunner.getExpressionManager().addExpressionOptimizerFactory(new NativeExpressionOptimizerFactory(ClassLoader.getSystemClassLoader())); + queryRunner.getExpressionManager().loadExpressions(ImmutableMap.builder().put("expression-manager-factory.name", "native").build()); + Thread.sleep(10); + Logger log = Logger.get(DistributedQueryRunner.class); + log.info("======== SERVER STARTED ========"); + log.info("\n====\n%s\n====", queryRunner.getCoordinator().getBaseUrl()); + } +} diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/expressions/TestDelegatingExpressionOptimizer.java b/presto-native-plugin/src/test/java/com/facebook/presto/session/sql/expressions/TestDelegatingExpressionOptimizer.java similarity index 79% rename from presto-tests/src/test/java/com/facebook/presto/tests/expressions/TestDelegatingExpressionOptimizer.java rename to presto-native-plugin/src/test/java/com/facebook/presto/session/sql/expressions/TestDelegatingExpressionOptimizer.java index 57f28fa483645..bc71240832fa1 100644 --- a/presto-tests/src/test/java/com/facebook/presto/tests/expressions/TestDelegatingExpressionOptimizer.java +++ b/presto-native-plugin/src/test/java/com/facebook/presto/session/sql/expressions/TestDelegatingExpressionOptimizer.java @@ -11,8 +11,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.tests.expressions; +package com.facebook.presto.session.sql.expressions; +import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; import com.facebook.presto.spi.relation.ExpressionOptimizer; @@ -21,17 +22,22 @@ import com.facebook.presto.sql.planner.Symbol; import com.facebook.presto.sql.relational.DelegatingRowExpressionOptimizer; import com.facebook.presto.sql.relational.FunctionResolution; +import com.facebook.presto.tests.expressions.TestExpressions; import com.google.common.collect.ImmutableList; -import org.intellij.lang.annotations.Language; +import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import java.net.URI; import java.util.Optional; +import java.util.OptionalInt; +import java.util.function.BiFunction; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; import static com.facebook.presto.common.type.VarcharType.VARCHAR; -import static com.facebook.presto.operator.scalar.ApplyFunction.APPLY_FUNCTION; +import static com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils.findRandomPortForWorker; +import static com.facebook.presto.session.sql.expressions.TestNativeExpressionOptimization.getExpressionOptimizer; import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level; import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.EVALUATED; import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; @@ -47,14 +53,30 @@ public class TestDelegatingExpressionOptimizer { public static final FunctionResolution RESOLUTION = new FunctionResolution(METADATA.getFunctionAndTypeManager().getFunctionAndTypeResolver()); private ExpressionOptimizer expressionOptimizer; + private Process sidecar; @BeforeClass public void setup() + throws Exception { - METADATA.getFunctionAndTypeManager().registerBuiltInFunctions(ImmutableList.of(APPLY_FUNCTION)); - setupJsonFunctionNamespaceManager(METADATA.getFunctionAndTypeManager()); + int port = findRandomPortForWorker(); + URI sidecarUri = URI.create("http://127.0.0.1:" + port); + Optional> launcher = PrestoNativeQueryRunnerUtils.getExternalWorkerLauncher( + "hive", + "/Users/tdcmeehan/git/presto/presto-native-execution/_build/release/presto_cpp/main/presto_server", + OptionalInt.of(port), + 0, + Optional.empty(), + false); + sidecar = launcher.get().apply(0, URI.create("http://test.invalid/")); + + expressionOptimizer = new DelegatingRowExpressionOptimizer(METADATA, () -> getExpressionOptimizer(METADATA, HANDLE_RESOLVER, sidecarUri)); + } - expressionOptimizer = new DelegatingRowExpressionOptimizer(METADATA, () -> TestNativeExpressions.getExpressionOptimizer(METADATA, HANDLE_RESOLVER)); + @AfterClass + public void tearDown() + { + sidecar.destroyForcibly(); } @Test @@ -118,7 +140,7 @@ protected Object evaluate(String expression, boolean deterministic) } @Override - protected Object optimize(@Language("SQL") String expression) + protected Object optimize(String expression) { assertRoundTrip(expression); RowExpression parsedExpression = sqlToRowExpression(expression); @@ -154,7 +176,7 @@ public Object unwrap(Object result) } @Override - protected void assertOptimizedEquals(@Language("SQL") String actual, @Language("SQL") String expected) + protected void assertOptimizedEquals(String actual, String expected) { Object optimizedActual = optimize(actual); Object optimizedExpected = optimize(expected); @@ -162,7 +184,7 @@ protected void assertOptimizedEquals(@Language("SQL") String actual, @Language(" } @Override - protected void assertOptimizedMatches(@Language("SQL") String actual, @Language("SQL") String expected) + protected void assertOptimizedMatches(String actual, String expected) { Object actualOptimized = optimize(actual); Object expectedOptimized = optimize(expected); @@ -172,7 +194,7 @@ protected void assertOptimizedMatches(@Language("SQL") String actual, @Language( } @Override - protected void assertDoNotOptimize(@Language("SQL") String expression, Level optimizationLevel) + protected void assertDoNotOptimize(String expression, Level optimizationLevel) { assertRoundTrip(expression); RowExpression rowExpression = sqlToRowExpression(expression); diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/expressions/TestNativeExpressions.java b/presto-native-plugin/src/test/java/com/facebook/presto/session/sql/expressions/TestNativeExpressionOptimization.java similarity index 59% rename from presto-tests/src/test/java/com/facebook/presto/tests/expressions/TestNativeExpressions.java rename to presto-native-plugin/src/test/java/com/facebook/presto/session/sql/expressions/TestNativeExpressionOptimization.java index 66e786923fe68..b4062c4e7d5e2 100644 --- a/presto-tests/src/test/java/com/facebook/presto/tests/expressions/TestNativeExpressions.java +++ b/presto-native-plugin/src/test/java/com/facebook/presto/session/sql/expressions/TestNativeExpressionOptimization.java @@ -11,14 +11,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.tests.expressions; +package com.facebook.presto.session.sql.expressions; import com.facebook.airlift.bootstrap.Bootstrap; -import com.facebook.airlift.http.client.HttpClient; -import com.facebook.airlift.http.client.testing.TestingHttpClient; import com.facebook.airlift.jaxrs.JsonMapper; -import com.facebook.airlift.jaxrs.testing.JaxrsTestingHttpProcessor; import com.facebook.airlift.json.JsonModule; +import com.facebook.presto.SessionTestUtils; import com.facebook.presto.block.BlockJsonSerde; import com.facebook.presto.client.NodeVersion; import com.facebook.presto.common.block.Block; @@ -35,19 +33,16 @@ import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.MetadataManager; +import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils; import com.facebook.presto.nodeManager.PluginNodeManager; -import com.facebook.presto.session.sql.expressions.ForSidecarInfo; -import com.facebook.presto.session.sql.expressions.NativeExpressionOptimizerProvider; -import com.facebook.presto.session.sql.expressions.NativeExpressionsModule; import com.facebook.presto.spi.ConnectorId; -import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.RowExpressionSerde; import com.facebook.presto.spi.relation.ExpressionOptimizer; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.sql.TestingRowExpressionTranslator; import com.facebook.presto.sql.expressions.JsonCodecRowExpressionSerde; -import com.facebook.presto.sql.planner.RowExpressionInterpreter; +import com.facebook.presto.sql.planner.LiteralEncoder; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.type.TypeDeserializer; import com.google.common.collect.ImmutableList; @@ -57,47 +52,53 @@ import com.google.inject.Scopes; import org.testng.annotations.Test; -import javax.ws.rs.Consumes; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.UriBuilder; - import java.net.URI; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.function.BiFunction; import static com.facebook.airlift.json.JsonBinder.jsonBinder; import static com.facebook.airlift.json.JsonCodecBinder.jsonCodecBinder; -import static com.facebook.presto.SessionTestUtils.TEST_SESSION; +import static com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils.findRandomPortForWorker; import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; -import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.SERIALIZABLE; -import static com.facebook.presto.sql.planner.LiteralEncoder.toRowExpression; -import static com.facebook.presto.testing.TestingSession.testSessionBuilder; -import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.inject.multibindings.Multibinder.newSetBinder; import static java.util.Objects.requireNonNull; import static org.testng.Assert.assertEquals; -public class TestNativeExpressions +public class TestNativeExpressionOptimization { - public static final URI SIDECAR_URI = URI.create("http://127.0.0.1:1122"); + public static final String SIDECAR_URI = "http://127.0.0.1:"; private static final Metadata METADATA = MetadataManager.createTestMetadataManager(); private static final TestingRowExpressionTranslator TRANSLATOR = new TestingRowExpressionTranslator(METADATA); @Test public void testLoadPlugin() + throws Exception { - ExpressionOptimizer interpreterService = getExpressionOptimizer(METADATA, null); - - // Test the native row expression interpreter service with some simple expressions - RowExpression simpleAddition = compileExpression("1+1"); - RowExpression unnecessaryCoalesce = compileExpression("coalesce(1, 2)"); - - // Assert simple optimizations are performed - assertEquals(interpreterService.optimize(simpleAddition, OPTIMIZED, TEST_SESSION.toConnectorSession()), toRowExpression(2L, simpleAddition.getType())); - assertEquals(interpreterService.optimize(unnecessaryCoalesce, OPTIMIZED, TEST_SESSION.toConnectorSession()), toRowExpression(1L, unnecessaryCoalesce.getType())); + int port = findRandomPortForWorker(); + URI sidecarUri = URI.create(SIDECAR_URI + port); + Optional> launcher = PrestoNativeQueryRunnerUtils.getExternalWorkerLauncher( + "hive", + "/Users/tdcmeehan/git/presto/presto-native-execution/_build/release/presto_cpp/main/presto_server", + OptionalInt.of(port), + 0, + Optional.empty(), + false); + Process process = launcher.get().apply(0, URI.create("http://test.invalid/")); + + try { + ExpressionOptimizer interpreterService = getExpressionOptimizer(METADATA, null, sidecarUri); + + // Test the native row expression interpreter service with some simple expressions + RowExpression simpleAddition = compileExpression("1+1"); + RowExpression unnecessaryCoalesce = compileExpression("coalesce(1, 2)"); + + // Assert simple optimizations are performed + assertEquals(interpreterService.optimize(simpleAddition, OPTIMIZED, SessionTestUtils.TEST_SESSION.toConnectorSession()), LiteralEncoder.toRowExpression(2L, simpleAddition.getType())); + assertEquals(interpreterService.optimize(unnecessaryCoalesce, OPTIMIZED, SessionTestUtils.TEST_SESSION.toConnectorSession()), LiteralEncoder.toRowExpression(1L, unnecessaryCoalesce.getType())); + } + finally { + process.destroyForcibly(); + } } private static RowExpression compileExpression(String expression) @@ -105,28 +106,16 @@ private static RowExpression compileExpression(String expression) return TRANSLATOR.translate(expression, ImmutableMap.of()); } - protected static ExpressionOptimizer getExpressionOptimizer(Metadata metadata, HandleResolver handleResolver) + protected static ExpressionOptimizer getExpressionOptimizer(Metadata metadata, HandleResolver handleResolver, URI sidecarUri) { // Set up dependencies in main for this module - InMemoryNodeManager nodeManager = getNodeManagerWithSidecar(SIDECAR_URI); + InMemoryNodeManager nodeManager = getNodeManagerWithSidecar(sidecarUri); Injector prestoMainInjector = getPrestoMainInjector(metadata, handleResolver); - JsonMapper jsonMapper = prestoMainInjector.getInstance(JsonMapper.class); RowExpressionSerde rowExpressionSerde = prestoMainInjector.getInstance(RowExpressionSerde.class); FunctionAndTypeManager functionMetadataManager = prestoMainInjector.getInstance(FunctionAndTypeManager.class); - // Set up the mock HTTP endpoint that delegates to the Java based row expression interpreter - TestingExpressionOptimizerResource resource = new TestingExpressionOptimizerResource( - metadata.getFunctionAndTypeManager(), - testSessionBuilder().build().toConnectorSession(), - SERIALIZABLE); - JaxrsTestingHttpProcessor jaxrsTestingHttpProcessor = new JaxrsTestingHttpProcessor( - UriBuilder.fromUri(SIDECAR_URI).path("/").build(), - resource, - jsonMapper); - TestingHttpClient testingHttpClient = new TestingHttpClient(jaxrsTestingHttpProcessor); - // Create the native row expression interpreter service - return createExpressionOptimizer(nodeManager, rowExpressionSerde, testingHttpClient, functionMetadataManager); + return createExpressionOptimizer(nodeManager, rowExpressionSerde, functionMetadataManager); } private static InMemoryNodeManager getNodeManagerWithSidecar(URI sidecarUri) @@ -136,16 +125,14 @@ private static InMemoryNodeManager getNodeManagerWithSidecar(URI sidecarUri) return nodeManager; } - private static ExpressionOptimizer createExpressionOptimizer(InternalNodeManager internalNodeManager, RowExpressionSerde rowExpressionSerde, HttpClient httpClient, FunctionAndTypeManager functionMetadataManager) + private static ExpressionOptimizer createExpressionOptimizer(InternalNodeManager internalNodeManager, RowExpressionSerde rowExpressionSerde, FunctionAndTypeManager functionMetadataManager) { requireNonNull(internalNodeManager, "inMemoryNodeManager is null"); NodeManager nodeManager = new PluginNodeManager(internalNodeManager); FunctionResolution functionResolution = new FunctionResolution(functionMetadataManager.getFunctionAndTypeResolver()); Bootstrap app = new Bootstrap( - // Specially use a testing HTTP client instead of a real one - binder -> binder.bind(HttpClient.class).annotatedWith(ForSidecarInfo.class).toInstance(httpClient), - // Otherwise use the exact same module as the native row expression interpreter service + new NativeExpressionsCommunicationModule(), new NativeExpressionsModule(nodeManager, rowExpressionSerde, functionMetadataManager, functionResolution)); Injector injector = app @@ -192,36 +179,4 @@ private static Injector getPrestoMainInjector(Metadata metadata, HandleResolver .initialize(); return injector; } - - @Path("/v1/expressions") - public static class TestingExpressionOptimizerResource - { - private final FunctionAndTypeManager functionAndTypeManager; - private final ConnectorSession connectorSession; - private final ExpressionOptimizer.Level level; - - public TestingExpressionOptimizerResource(FunctionAndTypeManager functionAndTypeManager, ConnectorSession connectorSession, ExpressionOptimizer.Level level) - { - this.functionAndTypeManager = requireNonNull(functionAndTypeManager, "functionAndTypeManager is null"); - this.connectorSession = requireNonNull(connectorSession, "connectorSession is null"); - this.level = requireNonNull(level, "level is null"); - } - - @POST - @Consumes(MediaType.APPLICATION_JSON) - @Produces(MediaType.APPLICATION_JSON) - public List post(List rowExpressions) - { - Map input = rowExpressions.stream().collect(toImmutableMap(i -> i, i -> i)); - Map optimizedExpressions = new HashMap<>(); - input.forEach((key, value) -> optimizedExpressions.put( - key, - new RowExpressionInterpreter(key, functionAndTypeManager, connectorSession, level).optimize())); - ImmutableList.Builder builder = ImmutableList.builder(); - for (RowExpression inputExpression : rowExpressions) { - builder.add(toRowExpression(optimizedExpressions.get(inputExpression), inputExpression.getType())); - } - return builder.build(); - } - } } diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index a2d03b17b6cfb..6be55b940be62 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -244,11 +244,6 @@ javax.servlet-api - - com.facebook.airlift - jaxrs - - com.facebook.airlift @@ -282,12 +277,6 @@ test - - com.facebook.presto - presto-native-plugin - test - - com.facebook.airlift jaxrs-testing