Merge branch 'main' into guava-cve

.github/workflows/auto-cherry-pick.yml

@@ -7,59 +7,62 @@ on:
     types: ["closed"]
 
 jobs:
-  cherry_pick_branch_0_1:
+  cherry_pick_branch_0_3:
     runs-on: ubuntu-latest
-    name: Cherry pick into branch_0.1
-    if: ${{ contains(github.event.pull_request.labels.*.name, 'branch-0.1') && github.event.pull_request.merged == true }}
+    name: Cherry pick into branch_0.3
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'branch-0.3') && github.event.pull_request.merged == true }}
     steps:
       - name: Checkout
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Cherry pick into branch-0.1
+      - name: Cherry pick into branch-0.3
         uses: carloscastrojumo/[email protected]
         with:
           token: ${{ secrets.BOT_TOKEN }}
-          branch: branch-0.1
+          branch: branch-0.3
           labels: |
             cherry-pick
           reviewers: |
             jerryshao
-  cherry_pick_branch_0_2:
+
+  cherry_pick_branch_0_4:
     runs-on: ubuntu-latest
-    name: Cherry pick into branch_0.2
-    if: ${{ contains(github.event.pull_request.labels.*.name, 'branch-0.2') && github.event.pull_request.merged == true }}
+    name: Cherry pick into branch_0.4
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'branch-0.4') && github.event.pull_request.merged == true }}
     steps:
       - name: Checkout
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Cherry pick into branch-0.2
+      - name: Cherry pick into branch-0.4
         uses: carloscastrojumo/[email protected]
         with:
           token: ${{ secrets.BOT_TOKEN }}
-          branch: branch-0.2
+          branch: branch-0.4
           labels: |
             cherry-pick
           reviewers: |
             jerryshao
-  cherry_pick_branch_0_3:
+
+  cherry_pick_branch_0_5:
     runs-on: ubuntu-latest
-    name: Cherry pick into branch_0.3
-    if: ${{ contains(github.event.pull_request.labels.*.name, 'branch-0.3') && github.event.pull_request.merged == true }}
+    name: Cherry pick into branch_0.5
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'branch-0.5') && github.event.pull_request.merged == true }}
     steps:
       - name: Checkout
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Cherry pick into branch-0.3
+      - name: Cherry pick into branch-0.5
         uses: carloscastrojumo/[email protected]
         with:
           token: ${{ secrets.BOT_TOKEN }}
-          branch: branch-0.3
+          branch: branch-0.5
           labels: |
             cherry-pick
           reviewers: |
             jerryshao
+
 env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

README.md

@@ -12,6 +12,7 @@
 [![Release](https://img.shields.io/github/v/release/datastrato/gravitino)](https://github.com/datastrato/gravitino/releases)
 [![Open Issues](https://img.shields.io/github/issues-raw/datastrato/gravitino)](https://github.com/datastrato/gravitino/issues)
 [![Last Committed](https://img.shields.io/github/last-commit/datastrato/gravitino)](https://github.com/datastrato/gravitino/commits/main/)
+[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/8358/badge)](https://www.bestpractices.dev/projects/8358)
 
 ## Introduction
 

catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTableOperations.java

@@ -18,6 +18,7 @@
 import com.google.common.base.Preconditions;
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
@@ -57,7 +58,31 @@ public String[] listPartitionNames() {
 
   @Override
   public Partition[] listPartitions() {
-    throw new UnsupportedOperationException();
+    List<String> partitionNames;
+    List<org.apache.hadoop.hive.metastore.api.Partition> partitions;
+    try {
+      partitionNames =
+          table
+              .clientPool()
+              .run(c -> c.listPartitionNames(table.schemaName(), table.name(), (short) -1));
+      partitions =
+          table
+              .clientPool()
+              .run(c -> c.getPartitionsByNames(table.schemaName(), table.name(), partitionNames));
+    } catch (TException | InterruptedException e) {
+      throw new RuntimeException(e);
+    }
+
+    // should never happen
+    Preconditions.checkArgument(
+        partitionNames.size() == partitions.size(),
+        "oops?! partition names and partitions size are not equal: %s vs %s",
+        partitionNames.size(),
+        partitions.size());
+
+    return IntStream.range(0, partitionNames.size())
+        .mapToObj(i -> fromHivePartition(partitionNames.get(i), partitions.get(i)))
+        .toArray(Partition[]::new);
   }
 
   @Override

catalogs/catalog-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveTableOperations.java

@@ -89,6 +89,14 @@ public void testListPartitionNames() {
             && Arrays.asList(partitionNames).contains(existingPartition.name()));
   }
 
+  @Test
+  public void testListPartitions() {
+    Partition[] partitions = hiveTable.supportPartitions().listPartitions();
+    // there maybe other partitions in the list, so we only check the added partition
+    Assertions.assertTrue(
+        partitions.length > 0 && Arrays.asList(partitions).contains(existingPartition));
+  }
+
   @Test
   public void testGetPartition() {
     Partition partition = hiveTable.supportPartitions().getPartition(existingPartition.name());

clients/client-java/src/main/java/com/datastrato/gravitino/client/RelationalTable.java

@@ -27,6 +27,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import java.net.URLEncoder;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.Map;
 import javax.annotation.Nullable;
 import lombok.SneakyThrows;
@@ -124,7 +125,16 @@ public String getPartitionRequestPath() {
 
   @Override
   public Partition[] listPartitions() {
-    throw new UnsupportedOperationException();
+    Map<String, String> params = new HashMap<>();
+    params.put("details", "true");
+    PartitionListResponse resp =
+        restClient.get(
+            getPartitionRequestPath(),
+            params,
+            PartitionListResponse.class,
+            Collections.emptyMap(),
+            ErrorHandlers.partitionErrorHandler());
+    return resp.getPartitions();
   }
 
   @Override

clients/client-java/src/test/java/com/datastrato/gravitino/client/TestRelationalTable.java

@@ -148,6 +148,46 @@ public void testListPartitionNames() throws JsonProcessingException {
     Assertions.assertEquals("table does not support partition operations", exception.getMessage());
   }
 
+  @Test
+  public void testListPartitions() throws JsonProcessingException {
+    String partitionName = "p1";
+    RangePartitionDTO partition =
+        RangePartitionDTO.builder()
+            .withName(partitionName)
+            .withLower(
+                new LiteralDTO.Builder()
+                    .withDataType(Types.IntegerType.get())
+                    .withValue("1")
+                    .build())
+            .withUpper(
+                new LiteralDTO.Builder()
+                    .withDataType(Types.IntegerType.get())
+                    .withValue("10")
+                    .build())
+            .build();
+    String partitionPath =
+        withSlash(((RelationalTable) partitionedTable).getPartitionRequestPath());
+    PartitionListResponse resp = new PartitionListResponse(new PartitionDTO[] {partition});
+
+    buildMockResource(Method.GET, partitionPath, null, resp, SC_OK);
+
+    Partition[] partitions = partitionedTable.supportPartitions().listPartitions();
+    Assertions.assertEquals(1, partitions.length);
+    Assertions.assertTrue(partitions[0] instanceof RangePartition);
+    Assertions.assertEquals(partition, partitions[0]);
+
+    // test throws exception
+    ErrorResponse errorResp =
+        ErrorResponse.unsupportedOperation("table does not support partition operations");
+    buildMockResource(Method.GET, partitionPath, null, errorResp, SC_NOT_IMPLEMENTED);
+
+    UnsupportedOperationException exception =
+        Assertions.assertThrows(
+            UnsupportedOperationException.class,
+            () -> partitionedTable.supportPartitions().listPartitions());
+    Assertions.assertEquals("table does not support partition operations", exception.getMessage());
+  }
+
   @Test
   public void testGetPartition() throws JsonProcessingException {
     String partitionName = "p1";

common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java

@@ -316,6 +316,13 @@ public static IndexDTO[] toDTOs(Index[] indexes) {
     return Arrays.stream(indexes).map(DTOConverters::toDTO).toArray(IndexDTO[]::new);
   }
 
+  public static PartitionDTO[] toDTOs(Partition[] partitions) {
+    if (ArrayUtils.isEmpty(partitions)) {
+      return new PartitionDTO[0];
+    }
+    return Arrays.stream(partitions).map(DTOConverters::toDTO).toArray(PartitionDTO[]::new);
+  }
+
   public static Distribution fromDTO(DistributionDTO distributionDTO) {
     if (DistributionDTO.NONE.equals(distributionDTO) || null == distributionDTO) {
       return Distributions.NONE;

docs/apache-hive-catalog.md

@@ -38,7 +38,7 @@ The Hive catalog supports creating, updating, and deleting databases and tables
 
 ### Catalog operations
 
-Refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#catalogs-operations) for more details.
+Refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#catalog-operations) for more details.
 
 ## Schema
 
@@ -57,7 +57,7 @@ The following table lists predefined schema properties for the Hive database. Ad
 
 ### Schema operations
 
-see [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#schemas-operations).
+see [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#schema-operations).
 
 ## Table
 
@@ -138,7 +138,7 @@ Hive automatically adds and manages some reserved properties. Users aren't allow
 
 ### Table operations
 
-Refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino#tables-operations) for more details.
+Refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#table-operations) for more details.
 
 #### Alter operations
 

docs/jdbc-mysql-catalog.md

@@ -44,7 +44,7 @@ You must download the corresponding JDBC driver to the `catalogs/jdbc-mysql/libs
 
 ### Catalog operations
 
-Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#catalogs-operations) for more details.
+Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#catalog-operations) for more details.
 
 ## Schema
 
@@ -61,7 +61,7 @@ Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravit
 
 ### Schema operations
 
-Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#schemas-operations) for more details.
+Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#schema-operations) for more details.
 
 ## Table
 
@@ -102,7 +102,7 @@ MySQL doesn't support Gravitino `Boolean` `Fixed` `Struct` `List` `Map` `Timesta
 
 ### Table operations
 
-Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino#tables-operations) for more details.
+Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#table-operations) for more details.
 
 #### Alter table operations
 

docs/jdbc-postgresql-catalog.md

@@ -50,7 +50,7 @@ In PostgreSQL, the database corresponds to the Gravitino catalog, and the schema
 
 ### Catalog operations
 
-Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#catalogs-operations) for more details.
+Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#catalog-operations) for more details.
 
 ## Schema
 
@@ -67,7 +67,7 @@ Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravit
 
 ### Schema operations
 
-Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#schemas-operations) for more details.
+Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#schema-operations) for more details.
 
 ## Table
 
@@ -110,7 +110,7 @@ PostgreSQL doesn't support Gravitino `Fixed` `Struct` `List` `Map` `IntervalDay`
 
 ### Table operations
 
-Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino#tables-operations) for more details.
+Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#table-operations) for more details.
 
 #### Alter table operations
 

docs/lakehouse-iceberg-catalog.md

@@ -58,7 +58,7 @@ You must download the corresponding JDBC driver to the `catalogs/lakehouse-icebe
 
 ### Catalog operations
 
-Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#catalogs-operations) for more details.
+Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#catalog-operations) for more details.
 
 ## Schema 
 
@@ -72,7 +72,7 @@ You could put properties except `comment`.
 
 ### Schema operations
 
-Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#schemas-operations) for more details.
+Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#schema-operations) for more details.
 
 ## Table 
 
@@ -234,7 +234,7 @@ The Gravitino server doesn't allow passing the following reserved fields.
 
 ### Table operations
 
-Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#tables-operations) for more details.
+Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#table-operations) for more details.
 
 #### Alter table operations
 

docs/trino-connector/installation.md

@@ -113,7 +113,7 @@ system
 
 You can see the `gravitino` catalog in the result set. This signifies the successful installation of the Gravitino connector.
 
-Assuming you have created a catalog named `test.jdbc-mysql` in the Gravitino server, or please refer to [Create a Catalog](../manage-metadata-using-gravitino#create-a-catalog). Then you can use the Trino CLI to connect to the Trino container and run a query like this.
+Assuming you have created a catalog named `test.jdbc-mysql` in the Gravitino server, or please refer to [Create a Catalog](../manage-metadata-using-gravitino.md#create-a-catalog). Then you can use the Trino CLI to connect to the Trino container and run a query like this.
 
 ```text
 docker exec -it trino trino