Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support kerberos authentication for Kudu connector #13968

Closed
wants to merge 3 commits into from
Closed

Support kerberos authentication for Kudu connector #13968

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2daf51e
Support kerberos authentication for Kudu connector
liyubin117 Jan 15, 2020
b376b27
add Kudu Kerberos auth utils
liyubin117 Jan 15, 2020
aebdd27
Modify the session behavior
liyubin117 Jan 15, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 80 additions & 0 deletions presto-kudu/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,86 @@
<version>${kudu.version}</version>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
liyubin117 marked this conversation as resolved.
Show resolved Hide resolved
<version>2.6.0</version>
<exclusions>
<exclusion>
<artifactId>jaxb-impl</artifactId>
<groupId>com.sun.xml.bind</groupId>
</exclusion>
<exclusion>
<artifactId>commons-beanutils-core</artifactId>
<groupId>commons-beanutils</groupId>
</exclusion>
<exclusion>
<artifactId>commons-codec</artifactId>
<groupId>commons-codec</groupId>
</exclusion>
<exclusion>
<artifactId>commons-lang</artifactId>
<groupId>commons-lang</groupId>
</exclusion>
<exclusion>
<artifactId>commons-logging</artifactId>
<groupId>commons-logging</groupId>
</exclusion>
<exclusion>
<artifactId>netty</artifactId>
<groupId>io.netty</groupId>
</exclusion>
<exclusion>
<artifactId>servlet-api</artifactId>
<groupId>javax.servlet</groupId>
</exclusion>
<exclusion>
<artifactId>httpclient</artifactId>
<groupId>org.apache.httpcomponents</groupId>
</exclusion>
<exclusion>
<artifactId>httpcore</artifactId>
<groupId>org.apache.httpcomponents</groupId>
</exclusion>
<exclusion>
<artifactId>jackson-core-asl</artifactId>
<groupId>org.codehaus.jackson</groupId>
</exclusion>
<exclusion>
<artifactId>jackson-mapper-asl</artifactId>
<groupId>org.codehaus.jackson</groupId>
</exclusion>
<exclusion>
<artifactId>jdk.tools</artifactId>
<groupId>jdk.tools</groupId>
</exclusion>
<exclusion>
<artifactId>jasper-compiler</artifactId>
<groupId>tomcat</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
</exclusion>
<exclusion>
<artifactId>jersey-server</artifactId>
<groupId>com.sun.jersey</groupId>
</exclusion>
<exclusion>
<artifactId>jersey-core</artifactId>
<groupId>com.sun.jersey</groupId>
</exclusion>
<exclusion>
<artifactId>asm</artifactId>
<groupId>asm</groupId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>com.facebook.airlift</groupId>
<artifactId>bootstrap</artifactId>
Expand Down
52 changes: 52 additions & 0 deletions presto-kudu/src/main/java/com/facebook/presto/kudu/KuduClientConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ public class KuduClientConfig
private boolean disableStatistics;
private boolean schemaEmulationEnabled;
private String schemaEmulationPrefix = "presto::";
private boolean kerberosAuthEnabled;
private String kerberosPrincipal;
private String kerberosKeytab;
private boolean kerberosAuthDebugEnabled;

@NotNull
@Size(min = 1)
Expand Down Expand Up @@ -138,4 +142,52 @@ public KuduClientConfig setSchemaEmulationEnabled(boolean enabled)
this.schemaEmulationEnabled = enabled;
return this;
}

public boolean isKerberosAuthEnabled()
{
return kerberosAuthEnabled;
}

@Config("kudu.kerberos-auth.enabled")
public KuduClientConfig setKerberosAuthEnabled(boolean enabled)
{
this.kerberosAuthEnabled = enabled;
return this;
}

public String getKerberosPrincipal()
{
return kerberosPrincipal;
}

@Config("kudu.kerberos-auth.principal")
public KuduClientConfig setKerberosPrincipal(String principal)
{
this.kerberosPrincipal = principal;
return this;
}

public String getKerberosKeytab()
{
return kerberosKeytab;
}

@Config("kudu.kerberos-auth.keytab")
public KuduClientConfig setKerberosKeytab(String keytab)
{
this.kerberosKeytab = keytab;
return this;
}

public boolean isKerberosAuthDebugEnabled()
{
return kerberosAuthDebugEnabled;
}

@Config("kudu.kerberos-auth.debug.enabled")
public KuduClientConfig setKerberosAuthDebugEnabled(boolean enabled)
{
this.kerberosAuthDebugEnabled = enabled;
return this;
}
}
19 changes: 19 additions & 0 deletions presto-kudu/src/main/java/com/facebook/presto/kudu/KuduClientSession.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@
import java.util.stream.Collectors;
import java.util.stream.IntStream;

import static com.facebook.presto.kudu.KuduUtil.reTryKerberos;
import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR;
import static com.facebook.presto.spi.StandardErrorCode.QUERY_REJECTED;
import static com.google.common.collect.ImmutableList.toImmutableList;
Expand All @@ -85,6 +86,7 @@ public KuduClientSession(KuduConnectorId connectorId, KuduClient client, SchemaE

public List<String> listSchemaNames()
{
reTryKerberos();
return schemaEmulation.listSchemaNames(client);
}

Expand All @@ -105,6 +107,7 @@ private List<String> internalListTables(String prefix)

public List<SchemaTableName> listTables(Optional<String> optSchemaName)
{
reTryKerberos();
if (optSchemaName.isPresent()) {
return listTablesSingleSchema(optSchemaName.get());
}
Expand All @@ -130,18 +133,21 @@ private List<SchemaTableName> listTablesSingleSchema(String schemaName)

public Schema getTableSchema(KuduTableHandle tableHandle)
{
reTryKerberos();
KuduTable table = tableHandle.getTable(this);
return table.getSchema();
}

public Map<String, Object> getTableProperties(KuduTableHandle tableHandle)
{
reTryKerberos();
KuduTable table = tableHandle.getTable(this);
return KuduTableProperties.toMap(table);
}

public List<KuduSplit> buildKuduSplits(KuduTableLayoutHandle layoutHandle)
{
reTryKerberos();
KuduTableHandle tableHandle = layoutHandle.getTableHandle();
KuduTable table = tableHandle.getTable(this);
final int primaryKeyColumnCount = table.getSchema().getPrimaryKeyColumnCount();
Expand Down Expand Up @@ -185,6 +191,7 @@ public List<KuduSplit> buildKuduSplits(KuduTableLayoutHandle layoutHandle)

public KuduScanner createScanner(KuduSplit kuduSplit)
{
reTryKerberos();
try {
return KuduScanToken.deserializeIntoScanner(kuduSplit.getSerializedScanToken(), client);
}
Expand All @@ -195,6 +202,7 @@ public KuduScanner createScanner(KuduSplit kuduSplit)

public KuduTable openTable(SchemaTableName schemaTableName)
{
reTryKerberos();
String rawName = schemaEmulation.toRawName(schemaTableName);
try {
return client.openTable(rawName);
Expand All @@ -210,21 +218,25 @@ public KuduTable openTable(SchemaTableName schemaTableName)

public KuduSession newSession()
{
reTryKerberos();
return client.newSession();
}

public void createSchema(String schemaName)
{
reTryKerberos();
schemaEmulation.createSchema(client, schemaName);
}

public void dropSchema(String schemaName)
{
reTryKerberos();
schemaEmulation.dropSchema(client, schemaName);
}

public void dropTable(SchemaTableName schemaTableName)
{
reTryKerberos();
try {
String rawName = schemaEmulation.toRawName(schemaTableName);
client.deleteTable(rawName);
Expand All @@ -236,6 +248,7 @@ public void dropTable(SchemaTableName schemaTableName)

public void renameTable(SchemaTableName schemaTableName, SchemaTableName newSchemaTableName)
{
reTryKerberos();
try {
String rawName = schemaEmulation.toRawName(schemaTableName);
String newRawName = schemaEmulation.toRawName(newSchemaTableName);
Expand All @@ -250,6 +263,7 @@ public void renameTable(SchemaTableName schemaTableName, SchemaTableName newSche

public KuduTable createTable(ConnectorTableMetadata tableMetadata, boolean ignoreExisting)
{
reTryKerberos();
try {
String rawName = schemaEmulation.toRawName(tableMetadata.getTable());
if (ignoreExisting) {
Expand All @@ -276,6 +290,7 @@ public KuduTable createTable(ConnectorTableMetadata tableMetadata, boolean ignor

public void addColumn(SchemaTableName schemaTableName, ColumnMetadata column)
{
reTryKerberos();
try {
String rawName = schemaEmulation.toRawName(schemaTableName);
AlterTableOptions alterOptions = new AlterTableOptions();
Expand All @@ -290,6 +305,7 @@ public void addColumn(SchemaTableName schemaTableName, ColumnMetadata column)

public void dropColumn(SchemaTableName schemaTableName, String name)
{
reTryKerberos();
try {
String rawName = schemaEmulation.toRawName(schemaTableName);
AlterTableOptions alterOptions = new AlterTableOptions();
Expand All @@ -303,6 +319,7 @@ public void dropColumn(SchemaTableName schemaTableName, String name)

public void renameColumn(SchemaTableName schemaTableName, String oldName, String newName)
{
reTryKerberos();
try {
String rawName = schemaEmulation.toRawName(schemaTableName);
AlterTableOptions alterOptions = new AlterTableOptions();
Expand All @@ -316,11 +333,13 @@ public void renameColumn(SchemaTableName schemaTableName, String oldName, String

public void addRangePartition(SchemaTableName schemaTableName, RangePartition rangePartition)
{
reTryKerberos();
changeRangePartition(schemaTableName, rangePartition, RangePartitionChange.ADD);
}

public void dropRangePartition(SchemaTableName schemaTableName, RangePartition rangePartition)
{
reTryKerberos();
changeRangePartition(schemaTableName, rangePartition, RangePartitionChange.DROP);
}

Expand Down
14 changes: 7 additions & 7 deletions presto-kudu/src/main/java/com/facebook/presto/kudu/KuduModule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,14 +91,14 @@ KuduClientSession createKuduClientSession(
{
requireNonNull(config, "config is null");

KuduClient.KuduClientBuilder builder = new KuduClient.KuduClientBuilder(config.getMasterAddresses());
builder.defaultAdminOperationTimeoutMs(config.getDefaultAdminOperationTimeout().toMillis());
builder.defaultOperationTimeoutMs(config.getDefaultOperationTimeout().toMillis());
builder.defaultSocketReadTimeoutMs(config.getDefaultSocketReadTimeout().toMillis());
if (config.isDisableStatistics()) {
builder.disableStatistics();
KuduClient client;
if (!config.isKerberosAuthEnabled()) {
client = KuduUtil.getKuduClient(config);
}
else {
KuduUtil.initKerberosENV(config.getKerberosPrincipal(), config.getKerberosKeytab(), config.isKerberosAuthDebugEnabled());
client = KuduUtil.getKuduKerberosClient(config);
}
KuduClient client = builder.build();

SchemaEmulation strategy;
if (config.isSchemaEmulationEnabled()) {
Expand Down