diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 78c1bb6..c9835c7 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -13,4 +13,4 @@ jobs: with: java-version: 1.8 - name: Build with Maven - run: mvn clean install + run: mvn -B clean install diff --git a/.gitignore b/.gitignore index e3c6527..5db5505 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *.class *.war *.jar +dependency-reduced-pom.xml # python *.pyc @@ -26,4 +27,4 @@ target/ release/ RESULTS/ CSV_EXPORT/ -.DS_Store \ No newline at end of file +.DS_Store diff --git a/README.md b/README.md index 0c6c489..c54986f 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,41 @@ limitations under the License. ![logo](https://phoenix.apache.org/images/phoenix-logo-small.png) -[Apache Phoenix](http://phoenix.apache.org/) enables OLTP and operational analytics in Hadoop for low latency applications. Visit the Apache Phoenix website [here](http://phoenix.apache.org/). This is the repo for the Query Server. +[Apache Phoenix](http://phoenix.apache.org/) enables OLTP and operational analytics in Hadoop for low latency applications. Visit the Apache Phoenix website [here](http://phoenix.apache.org/). This is the repo for the Phoenix Query Server (PQS). -Copyright ©2019 [Apache Software Foundation](http://www.apache.org/). All Rights Reserved. +Copyright ©2020 [Apache Software Foundation](http://www.apache.org/). All Rights Reserved. + +## Introduction + +The Phoenix Query Server is an JDBC over HTTP abstraction. The Phoenix Query Server proxies the standard +Phoenix JDBC driver and provides a backwards-compatible wire protocol to invoke that JDBC driver. This is +all done via the Apache Avatica project (sub-project of Apache Calcite). + +The reference client implementation for PQS is a "thin" JDBC driver which can communicate with PQS. There +are drivers in other languages which exist in varying levels of maturity including Python, Golang, and .NET. + +## Building + +This repository will build a tarball which is capable of running the Phoenix Query Server. + +By default, this tarball does not contain a Phoenix client jar as it is meant to be agnostic +of Phoenix version (one PQS release can be used against any Phoenix version). Today, PQS builds against +the Phoenix 4.15.0-HBase-1.4 release. + +``` +$ mvn package +``` + +### Bundling a Phoenix Client + +To build a release of PQS which packages a specific version of Phoenix, enable the `package-phoenix-client` profile +and specify properties to indicate a specific Phoenix version. + +By default, PQS will package the same version of Phoenix used for build/test. This version is controlled by the system +property `phoenix.version` system property. Depending on the version of Phoenix, you may also be required to +use the `phoenix.hbase.classifier` system property to identify the correct version of Phoenix built against +the version of HBase of your choosing. + +``` +$ mvn package -Dpackage.phoenix.client -Dphoenix.version=5.1.0-SNAPSHOT -Dphoenix.hbase.classifier=hbase-2.2 +``` diff --git a/assembly/pom.xml b/assembly/pom.xml index d381fde..ab3fa26 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -52,7 +52,6 @@ - target maven-assembly-plugin @@ -65,7 +64,7 @@ - cluster.xml + src/assembly/cluster.xml ${project.parent.artifactId}-${project.version} posix @@ -74,7 +73,43 @@ + + maven-dependency-plugin + + + + prepare-client-repo + + prepare-package + + copy-dependencies + + + phoenix-client,queryserver-client + ${project.build.directory}/maven-repo + true + true + + + + - + + + package-phoenix-client + + + package.phoenix.client + + + + + org.apache.phoenix + phoenix-client + ${phoenix.hbase.classifier} + + + + diff --git a/assembly/cluster.xml b/assembly/src/assembly/cluster.xml similarity index 57% rename from assembly/cluster.xml rename to assembly/src/assembly/cluster.xml index fe91478..59f7bee 100644 --- a/assembly/cluster.xml +++ b/assembly/src/assembly/cluster.xml @@ -33,22 +33,21 @@ ${project.basedir}/../queryserver/target ${project.parent.artifactId}-${project.parent.version}/queryserver/target - *.jar + phoenix*.jar + queryserver*.jar ${project.basedir}/../queryserver-client/target ${project.parent.artifactId}-${project.parent.version}/queryserver-client/target - *.jar + phoenix*.jar + queryserver*.jar - ${project.basedir}/../phoenix-client/target - ${project.parent.artifactId}-${project.parent.version}/phoenix-client/target - - *.jar - + ${project.build.directory}/maven-repo + ${project.parent.artifactId}-${project.parent.version}/maven @@ -59,5 +58,31 @@ sqlline:sqlline:jar:jar-with-dependencies + + false + ${project.parent.artifactId}-${project.parent.version}/ + + org.apache.phoenix:phoenix-client + + + phoenix-${artifact.version}${dashClassifier}-client.${artifact.extension} + + + false + ${project.parent.artifactId}-${project.parent.version}/queryserver/target + + org.apache.phoenix:queryserver + + phoenix-${project.parent.version}-queryserver.${artifact.extension} + + + + false + ${project.parent.artifactId}-${project.parent.version}/queryserver-client/target + + org.apache.phoenix:queryserver-client + + phoenix-${project.parent.version}-thin-client.${artifact.extension} + diff --git a/load-balancer/pom.xml b/load-balancer/pom.xml index c279d15..c0e01dc 100644 --- a/load-balancer/pom.xml +++ b/load-balancer/pom.xml @@ -116,10 +116,6 @@ org.apache.curator curator-framework - - org.apache.calcite.avatica - avatica - org.slf4j slf4j-api diff --git a/pom.xml b/pom.xml index 86ca1b5..56d7d48 100644 --- a/pom.xml +++ b/pom.xml @@ -51,6 +51,7 @@ queryserver + queryserver-it queryserver-client load-balancer assembly @@ -67,7 +68,7 @@ ${project.basedir} - 4.14.2-HBase-1.4 + 4.15.0-HBase-1.4 1.4.10 @@ -372,6 +373,12 @@ + + org.apache.hbase + hbase-server + ${hbase.version} + test + @@ -422,6 +429,11 @@ + + org.eclipse.jetty + jetty-util + ${jetty.version} + org.apache.curator curator-client @@ -480,6 +492,14 @@ 0.8.1 + + + org.apache.phoenix + queryserver + ${project.version} + tests + + org.apache.hbase @@ -494,12 +514,6 @@ - - org.apache.hbase - hbase-server - ${hbase.version} - test - org.apache.hbase hbase-server @@ -531,17 +545,10 @@ - - org.eclipse.jetty - jetty-util - ${jetty.version} - test - org.eclipse.jetty jetty-security ${jetty.version} - test org.eclipse.jetty @@ -587,5 +594,28 @@ - + + + package-phoenix-client + + + package.phoenix.client + + + + + + + + + + org.apache.phoenix + phoenix-client + ${phoenix.version} + ${phoenix.hbase.classifier} + + + + + diff --git a/queryserver-client/pom.xml b/queryserver-client/pom.xml index f0ac6ea..c07ceaa 100644 --- a/queryserver-client/pom.xml +++ b/queryserver-client/pom.xml @@ -74,8 +74,7 @@ shade - phoenix-${project.version}-thin-client - + false diff --git a/queryserver-it/pom.xml b/queryserver-it/pom.xml new file mode 100644 index 0000000..fac2c54 --- /dev/null +++ b/queryserver-it/pom.xml @@ -0,0 +1,217 @@ + + + + + + 4.0.0 + + + org.apache.phoenix + phoenix-queryserver + 1.0.0-SNAPSHOT + + + queryserver-it + Query Server Integration Tests + Integration tests for the Query Server + + + ${project.basedir}/.. + org.apache.phoenix.shaded + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + prepare-package + + test-jar + + + + + true + + + + maven-source-plugin + + + attach-test-sources + prepare-package + + test-jar-no-fork + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + + org.apache.maven.plugins + maven-dependency-plugin + + + + + org.slf4j:slf4j-api + + + + org.apache.hbase:hbase-testing-util + + + org.apache.hbase:hbase-it + + + org.apache.hadoop:hadoop-hdfs:test-jar + + + org.apache.hadoop:hadoop-common:test-jar + + + + + + + + + + org.apache.phoenix + queryserver + + + + + org.apache.phoenix + queryserver-client + test + + + org.apache.phoenix + phoenix-core + test + + + org.apache.phoenix + phoenix-core + test + tests + + + + commons-io + commons-io + test + + + junit + junit + test + + + + org.apache.hbase + hbase-common + test + + + org.apache.hbase + hbase-it + test-jar + test + + + org.apache.hbase + hbase-client + test + + + org.apache.hbase + hbase-server + test + + + org.apache.hbase + hbase-server + test-jar + test + + + org.apache.hbase + hbase-testing-util + test + + + org.apache.hadoop + hadoop-auth + test + + + org.apache.hadoop + hadoop-minikdc + test + + + org.apache.hadoop + hadoop-hdfs + test + + + org.apache.hadoop + hadoop-hdfs + test-jar + test + + + org.apache.hadoop + hadoop-common + test + + + org.apache.hadoop + hadoop-common + test-jar + test + + + com.google.guava + guava + test + + + diff --git a/queryserver-it/src/it/bin/test_phoenixdb.py b/queryserver-it/src/it/bin/test_phoenixdb.py new file mode 100644 index 0000000..0d5d0c6 --- /dev/null +++ b/queryserver-it/src/it/bin/test_phoenixdb.py @@ -0,0 +1,39 @@ +############################################################################ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +############################################################################ + +import phoenixdb +import phoenixdb.cursor +import sys + + +if __name__ == '__main__': + pqs_port = sys.argv[1] + database_url = 'http://localhost:' + str(pqs_port) + '/' + + print("CREATING PQS CONNECTION") + conn = phoenixdb.connect(database_url, autocommit=True, auth="SPNEGO") + cursor = conn.cursor() + + cursor.execute("CREATE TABLE users (id INTEGER PRIMARY KEY, username VARCHAR)") + cursor.execute("UPSERT INTO users VALUES (?, ?)", (1, 'admin')) + cursor.execute("UPSERT INTO users VALUES (?, ?)", (2, 'user')) + cursor.execute("SELECT * FROM users") + print("RESULTS") + print(cursor.fetchall()) diff --git a/queryserver-it/src/it/bin/test_phoenixdb.sh b/queryserver-it/src/it/bin/test_phoenixdb.sh new file mode 100755 index 0000000..7309dbe --- /dev/null +++ b/queryserver-it/src/it/bin/test_phoenixdb.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# +############################################################################ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +############################################################################ + +set -u +set -x +set -e + +function cleanup { + # Capture last command status + RCODE=$? + set +e + set +u + kdestroy + rm -rf $PY_ENV_PATH + exit $RCODE +} + +trap cleanup EXIT + +echo "LAUNCHING SCRIPT" + +LOCAL_PY=$1 +PRINC=$2 +KEYTAB_LOC=$3 +KRB5_CFG_FILE=$4 +PQS_PORT=$5 +PYTHON_SCRIPT=$6 + +PY_ENV_PATH=$( mktemp -d ) + +virtualenv $PY_ENV_PATH + +pushd ${PY_ENV_PATH}/bin + +# conda activate does stuff with unbound variables :( +set +u +. activate "" + +popd + +set -u +echo "INSTALLING COMPONENTS" +pip install -e file:///${LOCAL_PY}/requests-kerberos +pip install -e file:///${LOCAL_PY}/phoenixdb + +export KRB5_CONFIG=$KRB5_CFG_FILE +cat $KRB5_CONFIG +export KRB5_TRACE=/dev/stdout + +echo "RUNNING KINIT" +kinit -kt $KEYTAB_LOC $PRINC +klist + +unset http_proxy +unset https_proxy + +echo "Working Directory is ${PWD}" + +echo "RUN PYTHON TEST on port $PQS_PORT" +python $PYTHON_SCRIPT $PQS_PORT diff --git a/queryserver/src/it/java/org/apache/phoenix/end2end/HttpParamImpersonationQueryServerIT.java b/queryserver-it/src/it/java/org/apache/phoenix/end2end/HttpParamImpersonationQueryServerIT.java similarity index 100% rename from queryserver/src/it/java/org/apache/phoenix/end2end/HttpParamImpersonationQueryServerIT.java rename to queryserver-it/src/it/java/org/apache/phoenix/end2end/HttpParamImpersonationQueryServerIT.java diff --git a/queryserver/src/it/java/org/apache/phoenix/end2end/QueryServerBasicsIT.java b/queryserver-it/src/it/java/org/apache/phoenix/end2end/QueryServerBasicsIT.java similarity index 100% rename from queryserver/src/it/java/org/apache/phoenix/end2end/QueryServerBasicsIT.java rename to queryserver-it/src/it/java/org/apache/phoenix/end2end/QueryServerBasicsIT.java diff --git a/queryserver/src/it/java/org/apache/phoenix/end2end/QueryServerEnvironment.java b/queryserver-it/src/it/java/org/apache/phoenix/end2end/QueryServerEnvironment.java similarity index 100% rename from queryserver/src/it/java/org/apache/phoenix/end2end/QueryServerEnvironment.java rename to queryserver-it/src/it/java/org/apache/phoenix/end2end/QueryServerEnvironment.java diff --git a/queryserver/src/it/java/org/apache/phoenix/end2end/QueryServerTestUtil.java b/queryserver-it/src/it/java/org/apache/phoenix/end2end/QueryServerTestUtil.java similarity index 100% rename from queryserver/src/it/java/org/apache/phoenix/end2end/QueryServerTestUtil.java rename to queryserver-it/src/it/java/org/apache/phoenix/end2end/QueryServerTestUtil.java diff --git a/queryserver/src/it/java/org/apache/phoenix/end2end/QueryServerThread.java b/queryserver-it/src/it/java/org/apache/phoenix/end2end/QueryServerThread.java similarity index 100% rename from queryserver/src/it/java/org/apache/phoenix/end2end/QueryServerThread.java rename to queryserver-it/src/it/java/org/apache/phoenix/end2end/QueryServerThread.java diff --git a/queryserver/src/it/java/org/apache/phoenix/end2end/SecureQueryServerIT.java b/queryserver-it/src/it/java/org/apache/phoenix/end2end/SecureQueryServerIT.java similarity index 100% rename from queryserver/src/it/java/org/apache/phoenix/end2end/SecureQueryServerIT.java rename to queryserver-it/src/it/java/org/apache/phoenix/end2end/SecureQueryServerIT.java diff --git a/queryserver/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java b/queryserver-it/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java similarity index 99% rename from queryserver/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java rename to queryserver-it/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java index 32c5478..a9a5d8f 100644 --- a/queryserver/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java +++ b/queryserver-it/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java @@ -60,6 +60,7 @@ import org.junit.AfterClass; import org.junit.Assume; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.slf4j.Logger; @@ -73,6 +74,7 @@ * files in phoenix-queryserver/src/it/bin. */ @Category(NeedsOwnMiniClusterTest.class) +@Ignore("Failing since QueryServer moved to its own repository") public class SecureQueryServerPhoenixDBIT { private static enum Kdc { MIT, diff --git a/queryserver/src/it/java/org/apache/phoenix/end2end/ServerCustomizersIT.java b/queryserver-it/src/it/java/org/apache/phoenix/end2end/ServerCustomizersIT.java similarity index 56% rename from queryserver/src/it/java/org/apache/phoenix/end2end/ServerCustomizersIT.java rename to queryserver-it/src/it/java/org/apache/phoenix/end2end/ServerCustomizersIT.java index a941749..befe740 100644 --- a/queryserver/src/it/java/org/apache/phoenix/end2end/ServerCustomizersIT.java +++ b/queryserver-it/src/it/java/org/apache/phoenix/end2end/ServerCustomizersIT.java @@ -19,26 +19,15 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.Statement; -import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; -import org.apache.calcite.avatica.server.AvaticaServerConfiguration; -import org.apache.calcite.avatica.server.ServerCustomizer; import org.apache.hadoop.conf.Configuration; -import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.queryserver.QueryServerProperties; import org.apache.phoenix.queryserver.server.ServerCustomizersFactory; +import org.apache.phoenix.queryserver.server.customizers.BasicAuthenticationServerCustomizer; +import org.apache.phoenix.queryserver.server.customizers.BasicAuthenticationServerCustomizer.BasicAuthServerCustomizerFactory; import org.apache.phoenix.util.InstanceResolver; -import org.eclipse.jetty.security.ConstraintMapping; -import org.eclipse.jetty.security.ConstraintSecurityHandler; -import org.eclipse.jetty.security.HashLoginService; -import org.eclipse.jetty.security.UserStore; -import org.eclipse.jetty.security.authentication.BasicAuthenticator; -import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.util.security.Constraint; -import org.eclipse.jetty.util.security.Credential; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -50,9 +39,7 @@ public class ServerCustomizersIT extends BaseHBaseManagedTimeIT { private static final Logger LOG = LoggerFactory.getLogger(ServerCustomizersIT.class); - private static final String USER_AUTHORIZED = "user3"; private static final String USER_NOT_AUTHORIZED = "user1"; - private static final String USER_PW = "s3cr3t"; private static QueryServerTestUtil PQS_UTIL; @@ -62,18 +49,11 @@ public class ServerCustomizersIT extends BaseHBaseManagedTimeIT { @BeforeClass public static synchronized void setup() throws Exception { Configuration conf = getTestClusterConfig(); - conf.set(QueryServerProperties.QUERY_SERVER_CUSTOMIZERS_ENABLED, "true"); PQS_UTIL = new QueryServerTestUtil(conf); PQS_UTIL.startLocalHBaseCluster(ServerCustomizersIT.class); // Register a test jetty server customizer InstanceResolver.clearSingletons(); - InstanceResolver.getSingleton(ServerCustomizersFactory.class, new ServerCustomizersFactory() { - @Override - public List> createServerCustomizers(Configuration conf, - AvaticaServerConfiguration avaticaServerConfiguration) { - return Collections.>singletonList(new TestServerCustomizer()); - } - }); + InstanceResolver.getSingleton(ServerCustomizersFactory.class, new BasicAuthServerCustomizerFactory()); PQS_UTIL.startQueryServer(); } @@ -90,7 +70,7 @@ public static synchronized void teardown() throws Exception { @Test public void testUserAuthorized() throws Exception { try (Connection conn = DriverManager.getConnection(PQS_UTIL.getUrl( - getBasicAuthParams(USER_AUTHORIZED))); + getBasicAuthParams(BasicAuthenticationServerCustomizer.USER_AUTHORIZED))); Statement stmt = conn.createStatement()) { Assert.assertFalse("user3 should have access", stmt.execute( "create table "+ServerCustomizersIT.class.getSimpleName()+" (pk integer not null primary key)")); @@ -113,42 +93,7 @@ private Map getBasicAuthParams(String user) { Map params = new HashMap<>(); params.put("authentication", "BASIC"); params.put("avatica_user", user); - params.put("avatica_password", USER_PW); + params.put("avatica_password", BasicAuthenticationServerCustomizer.USER_PW); return params; } - - /** - * Contrived customizer that enables BASIC auth for a single user - */ - public static class TestServerCustomizer implements ServerCustomizer { - @Override - public void customize(Server server) { - LOG.debug("Customizing server to allow requests for {}", USER_AUTHORIZED); - - UserStore store = new UserStore(); - store.addUser(USER_AUTHORIZED, Credential.getCredential(USER_PW), new String[] {"users"}); - HashLoginService login = new HashLoginService(); - login.setName("users"); - login.setUserStore(store); - - Constraint constraint = new Constraint(); - constraint.setName(Constraint.__BASIC_AUTH); - constraint.setRoles(new String[]{"users"}); - constraint.setAuthenticate(true); - - ConstraintMapping cm = new ConstraintMapping(); - cm.setConstraint(constraint); - cm.setPathSpec("/*"); - - ConstraintSecurityHandler security = new ConstraintSecurityHandler(); - security.setAuthenticator(new BasicAuthenticator()); - security.setRealmName("users"); - security.addConstraintMapping(cm); - security.setLoginService(login); - - // chain the PQS handler to security - security.setHandler(server.getHandlers()[0]); - server.setHandler(security); - } - } } diff --git a/queryserver/src/it/java/org/apache/phoenix/end2end/TlsUtil.java b/queryserver-it/src/it/java/org/apache/phoenix/end2end/TlsUtil.java similarity index 100% rename from queryserver/src/it/java/org/apache/phoenix/end2end/TlsUtil.java rename to queryserver-it/src/it/java/org/apache/phoenix/end2end/TlsUtil.java diff --git a/queryserver/src/it/resources/log4j.properties b/queryserver-it/src/it/resources/log4j.properties similarity index 100% rename from queryserver/src/it/resources/log4j.properties rename to queryserver-it/src/it/resources/log4j.properties diff --git a/queryserver/pom.xml b/queryserver/pom.xml index 15ddb0e..1d08f7d 100644 --- a/queryserver/pom.xml +++ b/queryserver/pom.xml @@ -43,7 +43,6 @@ - org.apache.maven.plugins maven-jar-plugin @@ -53,9 +52,6 @@ - - true - maven-source-plugin @@ -70,11 +66,6 @@ - org.apache.maven.plugins - maven-failsafe-plugin - - - org.apache.maven.plugins maven-dependency-plugin @@ -82,24 +73,10 @@ org.slf4j:slf4j-api - - - org.apache.hbase:hbase-testing-util - - - org.apache.hbase:hbase-it - - - org.apache.hadoop:hadoop-hdfs:test-jar - - - org.apache.hadoop:hadoop-common:test-jar - - org.apache.maven.plugins maven-shade-plugin @@ -109,7 +86,6 @@ shade - phoenix-${project.version}-queryserver false true false @@ -171,10 +147,12 @@ org.apache.hbase hbase-common - - - org.apache.hbase - hbase-client + + + org.mortbay.jetty + * + + org.apache.zookeeper @@ -183,10 +161,12 @@ org.apache.hadoop hadoop-common - - - org.apache.hadoop - hadoop-auth + + + org.mortbay.jetty + * + + org.apache.calcite.avatica @@ -229,6 +209,10 @@ com.google.guava guava + + org.eclipse.jetty + jetty-util + @@ -241,26 +225,9 @@ phoenix-core test - - org.apache.phoenix - phoenix-core - test - tests - - - org.eclipse.jetty - jetty-util - test - org.eclipse.jetty jetty-security - test - - - commons-io - commons-io - test junit @@ -272,49 +239,5 @@ mockito-core test - - org.apache.hbase - hbase-it - test-jar - test - - - org.apache.hbase - hbase-server - test - - - org.apache.hbase - hbase-server - test-jar - test - - - org.apache.hbase - hbase-testing-util - test - - - org.apache.hadoop - hadoop-minikdc - test - - - org.apache.hadoop - hadoop-hdfs - test - - - org.apache.hadoop - hadoop-hdfs - test-jar - test - - - org.apache.hadoop - hadoop-common - test-jar - test - diff --git a/queryserver/src/main/java/org/apache/phoenix/queryserver/QueryServerOptions.java b/queryserver/src/main/java/org/apache/phoenix/queryserver/QueryServerOptions.java index b8b42cb..4c7db87 100644 --- a/queryserver/src/main/java/org/apache/phoenix/queryserver/QueryServerOptions.java +++ b/queryserver/src/main/java/org/apache/phoenix/queryserver/QueryServerOptions.java @@ -37,7 +37,6 @@ public class QueryServerOptions { public static final boolean DEFAULT_QUERY_SERVER_CUSTOM_AUTH_ENABLED = false; public static final String DEFAULT_QUERY_SERVER_REMOTEUSEREXTRACTOR_PARAM = "doAs"; public static final boolean DEFAULT_QUERY_SERVER_DISABLE_KERBEROS_LOGIN = false; - public static final boolean DEFAULT_QUERY_SERVER_CUSTOMIZERS_ENABLED = false; public static final boolean DEFAULT_QUERY_SERVER_TLS_ENABLED = false; //We default to empty *store password @@ -61,6 +60,11 @@ public class QueryServerOptions { public static final String DEFAULT_PHOENIX_QUERY_SERVER_ZK_ACL_USERNAME = "phoenix"; public static final String DEFAULT_PHOENIX_QUERY_SERVER_ZK_ACL_PASSWORD = "phoenix"; + // Maven repo defaults + public static final boolean DEFAULT_CLIENT_JARS_ENABLED = false; + public static final String DEFAULT_CLIENT_JARS_REPO = ""; + public static final String DEFAULT_CLIENT_JARS_CONTEXT = "/maven"; + // Common defaults public static final String DEFAULT_EXTRA_JDBC_ARGUMENTS = ""; diff --git a/queryserver/src/main/java/org/apache/phoenix/queryserver/QueryServerProperties.java b/queryserver/src/main/java/org/apache/phoenix/queryserver/QueryServerProperties.java index f550cbe..562ca13 100644 --- a/queryserver/src/main/java/org/apache/phoenix/queryserver/QueryServerProperties.java +++ b/queryserver/src/main/java/org/apache/phoenix/queryserver/QueryServerProperties.java @@ -50,8 +50,6 @@ public class QueryServerProperties { "phoenix.queryserver.spnego.auth.disabled"; public static final String QUERY_SERVER_WITH_REMOTEUSEREXTRACTOR_ATTRIB = "phoenix.queryserver.withRemoteUserExtractor"; - public static final String QUERY_SERVER_CUSTOMIZERS_ENABLED = - "phoenix.queryserver.customizers.enabled"; public static final String QUERY_SERVER_CUSTOM_AUTH_ENABLED = "phoenix.queryserver.custom.auth.enabled"; public static final String QUERY_SERVER_REMOTEUSEREXTRACTOR_PARAM = @@ -86,4 +84,7 @@ public class QueryServerProperties { public static final String ZOOKEEPER_PORT_ATTRIB = "hbase.zookeeper.property.clientPort"; public static final String EXTRA_JDBC_ARGUMENTS_ATTRIB = "phoenix.jdbc.extra.arguments"; + public static final String CLIENT_JARS_ENABLED_ATTRIB = "phoenix.queryserver.client.jars.enabled"; + public static final String CLIENT_JARS_REPO_ATTRIB = "phoenix.queryserver.client.jars.repo"; + public static final String CLIENT_JARS_CONTEXT_ATTRIB = "phoenix.queryserver.client.jars.context"; } diff --git a/queryserver/src/main/java/org/apache/phoenix/queryserver/server/QueryServer.java b/queryserver/src/main/java/org/apache/phoenix/queryserver/server/QueryServer.java index 6893dd0..1b29415 100644 --- a/queryserver/src/main/java/org/apache/phoenix/queryserver/server/QueryServer.java +++ b/queryserver/src/main/java/org/apache/phoenix/queryserver/server/QueryServer.java @@ -475,9 +475,11 @@ public void setRemoteUserExtractorIfNecessary(HttpServer.Builder builder, Config @VisibleForTesting public void enableServerCustomizersIfNecessary(HttpServer.Builder builder, Configuration conf, AvaticaServerConfiguration avaticaServerConfiguration) { - if (conf.getBoolean(QueryServerProperties.QUERY_SERVER_CUSTOMIZERS_ENABLED, - QueryServerOptions.DEFAULT_QUERY_SERVER_CUSTOMIZERS_ENABLED)) { - builder.withServerCustomizers(createServerCustomizers(conf, avaticaServerConfiguration), Server.class); + // Always try to enable the "provided" ServerCustomizers. The expectation is that the Factory implementation + // will have toggles for each provided customizer, rather than a global toggle to enable customizers. + List> customizers = createServerCustomizers(conf, avaticaServerConfiguration); + if (customizers != null && !customizers.isEmpty()) { + builder.withServerCustomizers(customizers, Server.class); } } diff --git a/queryserver/src/main/java/org/apache/phoenix/queryserver/server/ServerCustomizersFactory.java b/queryserver/src/main/java/org/apache/phoenix/queryserver/server/ServerCustomizersFactory.java index 942660a..346d3e4 100644 --- a/queryserver/src/main/java/org/apache/phoenix/queryserver/server/ServerCustomizersFactory.java +++ b/queryserver/src/main/java/org/apache/phoenix/queryserver/server/ServerCustomizersFactory.java @@ -17,13 +17,20 @@ */ package org.apache.phoenix.queryserver.server; +import java.io.File; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import org.apache.calcite.avatica.server.AvaticaServerConfiguration; import org.apache.calcite.avatica.server.ServerCustomizer; import org.apache.hadoop.conf.Configuration; +import org.apache.phoenix.queryserver.QueryServerOptions; +import org.apache.phoenix.queryserver.QueryServerProperties; +import org.apache.phoenix.queryserver.server.customizers.HostedClientJarsServerCustomizer; import org.eclipse.jetty.server.Server; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Creates customizers for the underlying Avatica HTTP server. @@ -42,11 +49,29 @@ public interface ServerCustomizersFactory { * Factory that creates an empty list of customizers. */ class ServerCustomizersFactoryImpl implements ServerCustomizersFactory { - private static final List> EMPTY_LIST = Collections.emptyList(); + private static final Logger LOG = LoggerFactory.getLogger(ServerCustomizersFactoryImpl.class); @Override public List> createServerCustomizers(Configuration conf, AvaticaServerConfiguration avaticaServerConfiguration) { - return EMPTY_LIST; + List> customizers = new ArrayList<>(); + if (conf.getBoolean(QueryServerProperties.CLIENT_JARS_ENABLED_ATTRIB, QueryServerOptions.DEFAULT_CLIENT_JARS_ENABLED)) { + String repoLocation = conf.get(QueryServerProperties.CLIENT_JARS_REPO_ATTRIB, + QueryServerOptions.DEFAULT_CLIENT_JARS_REPO); + if (repoLocation != null && !repoLocation.isEmpty()) { + File repo = new File(repoLocation); + if (!repo.isDirectory()) { + throw new IllegalArgumentException("Provided maven repository is not a directory. " + repo); + } + String contextPath = conf.get(QueryServerProperties.CLIENT_JARS_CONTEXT_ATTRIB, + QueryServerOptions.DEFAULT_CLIENT_JARS_CONTEXT); + LOG.info("Creating ServerCustomizer to host client jars from {} at HTTP endpoint {}", repo, contextPath); + HostedClientJarsServerCustomizer customizer = new HostedClientJarsServerCustomizer(repo, contextPath); + customizers.add(customizer); + } else { + LOG.warn("Empty value provided for {}, ignoring", QueryServerProperties.CLIENT_JARS_REPO_ATTRIB); + } + } + return Collections.unmodifiableList(customizers); } } } diff --git a/queryserver/src/main/java/org/apache/phoenix/queryserver/server/customizers/BasicAuthenticationServerCustomizer.java b/queryserver/src/main/java/org/apache/phoenix/queryserver/server/customizers/BasicAuthenticationServerCustomizer.java new file mode 100644 index 0000000..34bebc9 --- /dev/null +++ b/queryserver/src/main/java/org/apache/phoenix/queryserver/server/customizers/BasicAuthenticationServerCustomizer.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.queryserver.server.customizers; + +import java.util.Collections; +import java.util.List; + +import org.apache.calcite.avatica.server.AvaticaServerConfiguration; +import org.apache.calcite.avatica.server.ServerCustomizer; +import org.apache.hadoop.conf.Configuration; +import org.apache.phoenix.queryserver.server.ServerCustomizersFactory; +import org.eclipse.jetty.security.ConstraintMapping; +import org.eclipse.jetty.security.ConstraintSecurityHandler; +import org.eclipse.jetty.security.HashLoginService; +import org.eclipse.jetty.security.UserStore; +import org.eclipse.jetty.security.authentication.BasicAuthenticator; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.util.security.Constraint; +import org.eclipse.jetty.util.security.Credential; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Avatica ServerCustomizer which performs HTTP Basic authentication against a static user database. + * + * For testing ONLY. + */ +public class BasicAuthenticationServerCustomizer implements ServerCustomizer { + private static final Logger LOG = LoggerFactory.getLogger(BasicAuthenticationServerCustomizer.class); + + public static final String USER_AUTHORIZED = "user3"; + public static final String USER_PW = "s3cr3t"; + + public static class BasicAuthServerCustomizerFactory implements ServerCustomizersFactory { + @Override + public List> createServerCustomizers( + Configuration conf, AvaticaServerConfiguration avaticaServerConfiguration) { + return Collections.>singletonList(new BasicAuthenticationServerCustomizer()); + } + } + + @Override + public void customize(Server server) { + LOG.debug("Customizing server to allow requests for {}", USER_AUTHORIZED); + + UserStore store = new UserStore(); + store.addUser(USER_AUTHORIZED, Credential.getCredential(USER_PW), new String[] {"users"}); + HashLoginService login = new HashLoginService(); + login.setName("users"); + login.setUserStore(store); + + Constraint constraint = new Constraint(); + constraint.setName(Constraint.__BASIC_AUTH); + constraint.setRoles(new String[]{"users"}); + constraint.setAuthenticate(true); + + ConstraintMapping cm = new ConstraintMapping(); + cm.setConstraint(constraint); + cm.setPathSpec("/*"); + + ConstraintSecurityHandler security = new ConstraintSecurityHandler(); + security.setAuthenticator(new BasicAuthenticator()); + security.setRealmName("users"); + security.addConstraintMapping(cm); + security.setLoginService(login); + + // chain the PQS handler to security + security.setHandler(server.getHandlers()[0]); + server.setHandler(security); + } +} diff --git a/queryserver/src/main/java/org/apache/phoenix/queryserver/server/customizers/HostedClientJarsServerCustomizer.java b/queryserver/src/main/java/org/apache/phoenix/queryserver/server/customizers/HostedClientJarsServerCustomizer.java new file mode 100644 index 0000000..8112196 --- /dev/null +++ b/queryserver/src/main/java/org/apache/phoenix/queryserver/server/customizers/HostedClientJarsServerCustomizer.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.queryserver.server.customizers; + +import java.io.File; +import java.util.Arrays; + +import org.apache.calcite.avatica.server.ServerCustomizer; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.handler.ContextHandler; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.server.handler.ResourceHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Hosts a Maven repository from local filesystem over HTTP from within PQS. + */ +public class HostedClientJarsServerCustomizer implements ServerCustomizer { + private static final Logger LOG = LoggerFactory.getLogger(HostedClientJarsServerCustomizer.class); + + private final File repoRoot; + private final String contextPath; + + /** + * @param localMavenRepoRoot The path to the Phoenix-built maven repository on the local filesystem + * @param contextPath The HTTP path which the repository will be hosted at + */ + public HostedClientJarsServerCustomizer(File localMavenRepoRoot, String contextPath) { + this.repoRoot = localMavenRepoRoot; + this.contextPath = contextPath; + } + + @Override + public void customize(Server server) { + Handler[] handlers = server.getHandlers(); + if (handlers.length != 1) { + LOG.warn("Observed handlers on server {}", Arrays.toString(handlers)); + throw new IllegalStateException("Expected to find one handler"); + } + HandlerList list = (HandlerList) handlers[0]; + + ContextHandler ctx = new ContextHandler(contextPath); + ResourceHandler resource = new ResourceHandler(); + resource.setDirAllowed(true); + resource.setDirectoriesListed(false); + resource.setResourceBase(repoRoot.getAbsolutePath()); + ctx.setHandler(resource); + + Handler[] realHandlers = list.getChildHandlers(); + + Handler[] newHandlers = new Handler[realHandlers.length + 1]; + newHandlers[0] = ctx; + System.arraycopy(realHandlers, 0, newHandlers, 1, realHandlers.length); + server.setHandler(new HandlerList(newHandlers)); + } +} diff --git a/queryserver/src/test/java/org/apache/phoenix/queryserver/server/ServerCustomizersTest.java b/queryserver/src/test/java/org/apache/phoenix/queryserver/server/ServerCustomizersTest.java index 46e57d9..2d25cad 100644 --- a/queryserver/src/test/java/org/apache/phoenix/queryserver/server/ServerCustomizersTest.java +++ b/queryserver/src/test/java/org/apache/phoenix/queryserver/server/ServerCustomizersTest.java @@ -73,21 +73,8 @@ public List> createServerCustomizers(Configuration conf } }); Configuration conf = new Configuration(false); - conf.set(QueryServerProperties.QUERY_SERVER_CUSTOMIZERS_ENABLED, "true"); QueryServer queryServer = new QueryServer(); List> actual = queryServer.createServerCustomizers(conf, avaticaServerConfiguration); Assert.assertEquals("Customizers are different", expected, actual); } - - @Test - @SuppressWarnings("unchecked") - public void testEnableCustomizers() { - AvaticaServerConfiguration avaticaServerConfiguration = null; - HttpServer.Builder builder = mock(HttpServer.Builder.class); - Configuration conf = new Configuration(false); - conf.set(QueryServerProperties.QUERY_SERVER_CUSTOMIZERS_ENABLED, "true"); - QueryServer queryServer = new QueryServer(); - queryServer.enableServerCustomizersIfNecessary(builder, conf, avaticaServerConfiguration); - verify(builder).withServerCustomizers(anyList(), any(Class.class)); - } -} \ No newline at end of file +} diff --git a/queryserver/src/test/java/org/apache/phoenix/queryserver/server/customizers/HostedClientJarsServerCustomizerTest.java b/queryserver/src/test/java/org/apache/phoenix/queryserver/server/customizers/HostedClientJarsServerCustomizerTest.java new file mode 100644 index 0000000..2988a6c --- /dev/null +++ b/queryserver/src/test/java/org/apache/phoenix/queryserver/server/customizers/HostedClientJarsServerCustomizerTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.queryserver.server.customizers; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; + +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.handler.ContextHandler; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.server.handler.ResourceHandler; +import org.junit.Test; +import org.mockito.Mockito; + +public class HostedClientJarsServerCustomizerTest { + + @Test + public void testHandlerIsPrefixed() { + final Handler handler1 = Mockito.mock(Handler.class); + final Handler handler2 = Mockito.mock(Handler.class); + + Server svr = new Server(); + svr.setHandler(new HandlerList(handler1, handler2)); + + File f = new File("/for-test"); + String context = "/my-context"; + HostedClientJarsServerCustomizer customizer = new HostedClientJarsServerCustomizer(f, context); + customizer.customize(svr); + + assertEquals(1, svr.getHandlers().length); + Handler actualHandler = svr.getHandler(); + assertTrue("Handler was " + actualHandler.getClass(), actualHandler instanceof HandlerList); + + HandlerList actualHandlerList = (HandlerList) actualHandler; + assertEquals(3, actualHandlerList.getHandlers().length); + assertEquals(handler1, actualHandlerList.getHandlers()[1]); + assertEquals(handler2, actualHandlerList.getHandlers()[2]); + + Handler injectedHandler = actualHandlerList.getHandlers()[0]; + assertTrue("Handler was " + injectedHandler.getClass(), injectedHandler instanceof ContextHandler); + ContextHandler ctx = (ContextHandler) injectedHandler; + assertTrue("Handler was " + ctx.getHandler().getClass(), ctx.getHandler() instanceof ResourceHandler); + assertEquals(context, ctx.getContextPath()); + ResourceHandler res = (ResourceHandler) ctx.getHandler(); + // Jetty puts in a proper URI for the file we give it + assertEquals("file://" + f.getAbsolutePath(), res.getResourceBase()); + } + +}