Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make AbstractIT more independent. #12

Open
wants to merge 10 commits into
base: main
Choose a base branch
from
+280 −201
Open

Make AbstractIT more independent. #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
c7bb9c0
Make AbstractIT more independent.
yuqi1129 Sep 23, 2024
dddb2ef
Fix error.
yuqi1129 Sep 29, 2024
081cdf3
Merge branch 'main' of github.com:apache/gravitino into issue_4951
yuqi1129 Sep 29, 2024
1ab6643
Fix error.
yuqi1129 Sep 29, 2024
97d2bf3
Fix test error.
yuqi1129 Sep 29, 2024
be2ea87
Fix test error.
yuqi1129 Sep 30, 2024
e3aa6a2
fix
yuqi1129 Sep 30, 2024
5ab7af5
fix
yuqi1129 Sep 30, 2024
8a95f32
Resolved comments
yuqi1129 Oct 11, 2024
2948215
x
yuqi1129 Oct 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 8 additions & 5 deletions ...test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,12 @@
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@Tag("gravitino-docker-test")
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
public class RangerHiveE2EIT extends AbstractIT {
private static final Logger LOG = LoggerFactory.getLogger(RangerHiveE2EIT.class);

Expand Down Expand Up @@ -99,15 +101,15 @@ public class RangerHiveE2EIT extends AbstractIT {
private static String RANGER_ADMIN_URL = null;

@BeforeAll
public static void startIntegrationTest() throws Exception {
public void startIntegrationTest() throws Exception {
// Enable Gravitino Authorization mode
Map<String, String> configs = Maps.newHashMap();
configs.put(Configs.ENABLE_AUTHORIZATION.getKey(), String.valueOf(true));
configs.put(Configs.SERVICE_ADMINS.getKey(), RangerITEnv.HADOOP_USER_NAME);
configs.put(Configs.AUTHENTICATORS.getKey(), AuthenticatorType.SIMPLE.name().toLowerCase());
configs.put("SimpleAuthUserName", TEST_USER_NAME);
registerCustomConfigs(configs);
AbstractIT.startIntegrationTest();
super.startIntegrationTest();

RangerITEnv.setup();
RangerITEnv.startHiveRangerContainer();
Expand Down Expand Up @@ -180,7 +182,8 @@ private static void generateRangerSparkSecurityXML() throws IOException {
}

@AfterAll
public static void stop() throws IOException {
public void stop() throws IOException {
client = null;
if (client != null) {
Arrays.stream(catalog.asSchemas().listSchemas())
.filter(schema -> !schema.equals("default"))
Expand All @@ -204,7 +207,7 @@ public static void stop() throws IOException {
LOG.error("Failed to close CloseableGroup", e);
}

AbstractIT.client = null;
client = null;
}

@Test
Expand Down Expand Up @@ -267,7 +270,7 @@ void testAllowUseSchemaPrivilege() throws InterruptedException {
1, rows2.stream().filter(row -> row.getString(0).equals(schemaName)).count());
}

private static void createMetalake() {
private void createMetalake() {
GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes();
Assertions.assertEquals(0, gravitinoMetalakes.length);

Expand Down
12 changes: 7 additions & 5 deletions ...p/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,12 @@
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@Tag("gravitino-docker-test")
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
public class HadoopCatalogIT extends AbstractIT {
private static final Logger LOG = LoggerFactory.getLogger(HadoopCatalogIT.class);
private static final ContainerSuite containerSuite = ContainerSuite.getInstance();
Expand All @@ -74,7 +76,7 @@ public class HadoopCatalogIT extends AbstractIT {
private static String defaultBaseLocation;

@BeforeAll
public static void setup() throws IOException {
public void setup() throws IOException {
containerSuite.startHiveContainer();
Configuration conf = new Configuration();
conf.set("fs.defaultFS", defaultBaseLocation());
Expand All @@ -86,7 +88,7 @@ public static void setup() throws IOException {
}

@AfterAll
public static void stop() throws IOException {
public void stop() throws IOException {
Catalog catalog = metalake.loadCatalog(catalogName);
catalog.asSchemas().dropSchema(schemaName, true);
metalake.dropCatalog(catalogName);
Expand All @@ -102,7 +104,7 @@ public static void stop() throws IOException {
}
}

private static void createMetalake() {
private void createMetalake() {
GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes();
Assertions.assertEquals(0, gravitinoMetalakes.length);

Expand All @@ -114,14 +116,14 @@ private static void createMetalake() {
metalake = loadMetalake;
}

private static void createCatalog() {
private void createCatalog() {
metalake.createCatalog(
catalogName, Catalog.Type.FILESET, provider, "comment", ImmutableMap.of());

catalog = metalake.loadCatalog(catalogName);
}

private static void createSchema() {
private void createSchema() {
Map<String, String> properties = Maps.newHashMap();
properties.put("key1", "val1");
properties.put("key2", "val2");
Expand Down
20 changes: 10 additions & 10 deletions ...java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,12 +54,14 @@
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testcontainers.utility.MountableFile;
import sun.security.krb5.KrbException;

@Tag("gravitino-docker-test")
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
public class HadoopUserAuthenticationIT extends AbstractIT {
private static final Logger LOG = LoggerFactory.getLogger(HadoopUserAuthenticationIT.class);

Expand Down Expand Up @@ -104,7 +106,7 @@ public class HadoopUserAuthenticationIT extends AbstractIT {
private static final String TABLE_NAME = "test_table";

@BeforeAll
public static void startIntegrationTest() throws Exception {
public void startIntegrationTest() throws Exception {
containerSuite.startKerberosHiveContainer();
kerberosHiveContainer = containerSuite.getKerberosHiveContainer();

Expand All @@ -122,7 +124,7 @@ public static void startIntegrationTest() throws Exception {
addKerberosConfig();

// Start Gravitino server
AbstractIT.startIntegrationTest();
super.startIntegrationTest();
}

@AfterAll
Expand Down Expand Up @@ -222,14 +224,12 @@ private static void createKeyTableForSchemaAndFileset() throws IOException {
.copyFileFromContainer(HADOOP_FILESET_KEYTAB, TMP_DIR + HADOOP_FILESET_KEYTAB);
}

private static void addKerberosConfig() {
AbstractIT.customConfigs.put(Configs.AUTHENTICATORS.getKey(), "kerberos");
AbstractIT.customConfigs.put(
"gravitino.authenticator.kerberos.principal", GRAVITINO_SERVER_PRINCIPAL);
AbstractIT.customConfigs.put(
"gravitino.authenticator.kerberos.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB);
AbstractIT.customConfigs.put(SDK_KERBEROS_KEYTAB_KEY, TMP_DIR + GRAVITINO_CLIENT_KEYTAB);
AbstractIT.customConfigs.put(SDK_KERBEROS_PRINCIPAL_KEY, GRAVITINO_CLIENT_PRINCIPAL);
private void addKerberosConfig() {
customConfigs.put(Configs.AUTHENTICATORS.getKey(), "kerberos");
customConfigs.put("gravitino.authenticator.kerberos.principal", GRAVITINO_SERVER_PRINCIPAL);
customConfigs.put("gravitino.authenticator.kerberos.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB);
customConfigs.put(SDK_KERBEROS_KEYTAB_KEY, TMP_DIR + GRAVITINO_CLIENT_KEYTAB);
customConfigs.put(SDK_KERBEROS_PRINCIPAL_KEY, GRAVITINO_CLIENT_PRINCIPAL);
}

@Test
Expand Down
6 changes: 4 additions & 2 deletions .../java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserImpersonationIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,11 +62,13 @@
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.api.condition.EnabledIf;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@Tag("gravitino-docker-test")
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
public class HadoopUserImpersonationIT extends AbstractIT {
private static final Logger LOG = LoggerFactory.getLogger(HadoopCatalogIT.class);

Expand Down Expand Up @@ -111,7 +113,7 @@ private static void refreshKerberosConfig() {
}

@BeforeAll
public static void setup() throws Exception {
public void setup() throws Exception {
if (!isEmbedded()) {
return;
}
Expand Down Expand Up @@ -254,7 +256,7 @@ void testListFileSystem() throws Exception {
});
}

private static void createMetalake() {
private void createMetalake() {
GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes();
Assertions.assertEquals(0, gravitinoMetalakes.length);

Expand Down
2 changes: 1 addition & 1 deletion ...-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,7 @@ public void stop() throws IOException {
LOG.error("Failed to close CloseableGroup", e);
}

AbstractIT.client = null;
client = null;
}

@AfterEach
Expand Down
24 changes: 12 additions & 12 deletions ...est/java/org/apache/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,12 @@
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@Tag("gravitino-docker-test")
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
public class HiveUserAuthenticationIT extends AbstractIT {
private static final Logger LOG = LoggerFactory.getLogger(HiveUserAuthenticationIT.class);

Expand Down Expand Up @@ -98,7 +100,7 @@ public class HiveUserAuthenticationIT extends AbstractIT {
private static final String HIVE_COL_NAME3 = "col3";

@BeforeAll
public static void startIntegrationTest() throws Exception {
public void startIntegrationTest() throws Exception {
containerSuite.startKerberosHiveContainer();
kerberosHiveContainer = containerSuite.getKerberosHiveContainer();

Expand All @@ -119,11 +121,11 @@ public static void startIntegrationTest() throws Exception {
addKerberosConfig();

// Start Gravitino server
AbstractIT.startIntegrationTest();
super.startIntegrationTest();
}

@AfterAll
public static void stop() {
public void stop() {
// Reset the UGI
UserGroupInformation.reset();

Expand All @@ -132,7 +134,7 @@ public static void stop() {
System.clearProperty("java.security.krb5.conf");
System.clearProperty("sun.security.krb5.debug");

AbstractIT.client = null;
client = null;
}

private static void prepareKerberosConfig() throws Exception {
Expand Down Expand Up @@ -188,14 +190,12 @@ private static void refreshKerberosConfig() {
}
}

private static void addKerberosConfig() {
AbstractIT.customConfigs.put(Configs.AUTHENTICATORS.getKey(), "kerberos");
AbstractIT.customConfigs.put(
"gravitino.authenticator.kerberos.principal", GRAVITINO_SERVER_PRINCIPAL);
AbstractIT.customConfigs.put(
"gravitino.authenticator.kerberos.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB);
AbstractIT.customConfigs.put(SDK_KERBEROS_KEYTAB_KEY, TMP_DIR + GRAVITINO_CLIENT_KEYTAB);
AbstractIT.customConfigs.put(SDK_KERBEROS_PRINCIPAL_KEY, GRAVITINO_CLIENT_PRINCIPAL);
private void addKerberosConfig() {
customConfigs.put(Configs.AUTHENTICATORS.getKey(), "kerberos");
customConfigs.put("gravitino.authenticator.kerberos.principal", GRAVITINO_SERVER_PRINCIPAL);
customConfigs.put("gravitino.authenticator.kerberos.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB);
customConfigs.put(SDK_KERBEROS_KEYTAB_KEY, TMP_DIR + GRAVITINO_CLIENT_KEYTAB);
customConfigs.put(SDK_KERBEROS_PRINCIPAL_KEY, GRAVITINO_CLIENT_PRINCIPAL);
}

@Test
Expand Down
22 changes: 12 additions & 10 deletions .../src/test/java/org/apache/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,10 @@
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;

@Tag("gravitino-docker-test")
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
public class ProxyCatalogHiveIT extends AbstractIT {

public static final String METALAKE_NAME =
Expand All @@ -88,10 +90,10 @@ public class ProxyCatalogHiveIT extends AbstractIT {
private static GravitinoAdminClient anotherClientWithNotExistingName;
private static Catalog anotherCatalog;
private static Catalog anotherCatalogWithUsername;
private static Catalog anotherCatatlogWithNotExistingName;
private static Catalog anotherCatalogWithNotExistingName;

@BeforeAll
public static void startIntegrationTest() throws Exception {
public void startIntegrationTest() throws Exception {
originHadoopUser = System.getenv(HADOOP_USER_NAME);
setEnv(HADOOP_USER_NAME, null);

Expand All @@ -100,7 +102,7 @@ public static void startIntegrationTest() throws Exception {
Map<String, String> configs = Maps.newHashMap();
configs.put(Configs.AUTHENTICATORS.getKey(), AuthenticatorType.SIMPLE.name().toLowerCase());
registerCustomConfigs(configs);
AbstractIT.startIntegrationTest();
super.startIntegrationTest();
containerSuite.startHiveContainer();
HIVE_METASTORE_URIS =
String.format(
Expand Down Expand Up @@ -137,13 +139,13 @@ public static void startIntegrationTest() throws Exception {
}

@AfterAll
public static void stop() {
public void stop() {
setEnv(HADOOP_USER_NAME, originHadoopUser);
anotherClient.close();
anotherClientWithUsername.close();
anotherClientWithNotExistingName.close();

AbstractIT.client = null;
client = null;
}

@Test
Expand Down Expand Up @@ -195,7 +197,7 @@ public void testOperateSchema() throws Exception {
Assertions.assertThrows(
RuntimeException.class,
() ->
anotherCatatlogWithNotExistingName
anotherCatalogWithNotExistingName
.asSchemas()
.createSchema("new_schema", comment, properties));
Assertions.assertTrue(e.getMessage().contains("AccessControlException Permission denied"));
Expand Down Expand Up @@ -256,7 +258,7 @@ public void testOperateTable() throws Exception {
Assertions.assertThrows(
RuntimeException.class,
() -> {
anotherCatatlogWithNotExistingName
anotherCatalogWithNotExistingName
.asTableCatalog()
.createTable(
anotherIdentWithNotExisting,
Expand Down Expand Up @@ -370,7 +372,7 @@ public void testOperatePartition() throws Exception {
Assertions.assertThrows(
RuntimeException.class,
() ->
anotherCatatlogWithNotExistingName
anotherCatalogWithNotExistingName
.asTableCatalog()
.loadTable(nameIdentifier)
.supportPartitions()
Expand All @@ -385,7 +387,7 @@ private Column[] createColumns() {
return new Column[] {col1, col2, col3};
}

private static void createMetalake() {
private void createMetalake() {
GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes();
Assertions.assertEquals(0, gravitinoMetalakes.length);

Expand Down Expand Up @@ -421,7 +423,7 @@ private static void loadCatalogWithAnotherClient() {
anotherCatalogWithUsername =
anotherClientWithUsername.loadMetalake(METALAKE_NAME).loadCatalog(CATALOG_NAME);

anotherCatatlogWithNotExistingName =
anotherCatalogWithNotExistingName =
anotherClientWithNotExistingName.loadMetalake(METALAKE_NAME).loadCatalog(CATALOG_NAME);
}

Expand Down
8 changes: 4 additions & 4 deletions ...ris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ public void startup() throws IOException {
public void stop() {
clearTableAndSchema();
metalake.dropCatalog(catalogName);
AbstractIT.client.dropMetalake(metalakeName);
client.dropMetalake(metalakeName);
}

@AfterEach
Expand All @@ -141,12 +141,12 @@ private void clearTableAndSchema() {
}

private void createMetalake() {
GravitinoMetalake[] gravitinoMetaLakes = AbstractIT.client.listMetalakes();
GravitinoMetalake[] gravitinoMetaLakes = client.listMetalakes();
assertEquals(0, gravitinoMetaLakes.length);

GravitinoMetalake createdMetalake =
AbstractIT.client.createMetalake(metalakeName, "comment", Collections.emptyMap());
GravitinoMetalake loadMetalake = AbstractIT.client.loadMetalake(metalakeName);
client.createMetalake(metalakeName, "comment", Collections.emptyMap());
GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName);
assertEquals(createdMetalake, loadMetalake);

metalake = loadMetalake;
Expand Down
Loading
Loading