From 1d660fa19f0d82c2b6ec2ea9590881e513274c25 Mon Sep 17 00:00:00 2001 From: Teja Vemparala <40125841+vteja11@users.noreply.github.com> Date: Thu, 12 Oct 2023 10:19:13 -0700 Subject: [PATCH] feat: Add BigLakeConfiguration Property in StandardTableDefinition.java (#2916) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added BigLakeConfiguration in standard table defenition * feat: Add BigLakeConfiguration Property in StandardTableDefinition.java * feat: Update copyright dates to 2023 * feat: Removed bigstore and gs:/ from docs * feat: Renamed Biglake to BigLake * feat: Adding difference entry to resolve clirr error * feat: Refactored formatting to comply with project rules * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- .../clirr-ignored-differences.xml | 5 + .../cloud/bigquery/BigLakeConfiguration.java | 120 ++++++++++++++++++ .../bigquery/StandardTableDefinition.java | 21 +++ .../bigquery/BigLakeConfigurationTest.java | 80 ++++++++++++ .../bigquery/StandardTableDefinitionTest.java | 8 ++ 5 files changed, 234 insertions(+) create mode 100644 google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigLakeConfiguration.java create mode 100644 google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigLakeConfigurationTest.java diff --git a/google-cloud-bigquery/clirr-ignored-differences.xml b/google-cloud-bigquery/clirr-ignored-differences.xml index eaeeded13..b2d5aef0f 100644 --- a/google-cloud-bigquery/clirr-ignored-differences.xml +++ b/google-cloud-bigquery/clirr-ignored-differences.xml @@ -109,4 +109,9 @@ com/google/cloud/bigquery/DatasetInfo* *setExternalDatasetReference(*) + + 7013 + com/google/cloud/bigquery/StandardTableDefinition* + *BigLakeConfiguration(*) + \ No newline at end of file diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigLakeConfiguration.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigLakeConfiguration.java new file mode 100644 index 000000000..76b3f55d9 --- /dev/null +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigLakeConfiguration.java @@ -0,0 +1,120 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import com.google.auto.value.AutoValue; +import java.io.Serializable; + +@AutoValue +public abstract class BigLakeConfiguration implements Serializable { + + private static final long serialVersionUID = -5951589238459622025L; + + /** + * Credential reference for accessing external storage system. Normalized as + * project_id.location_id.connection_id. + * + * @return value or {@code null} for none + */ + public abstract String getConnectionId(); + + /** + * Open source file format that the table data is stored in. Currently only PARQUET is supported. + * + * @return value or {@code null} for none + */ + public abstract String getFileFormat(); + + /** + * Fully qualified location prefix of the external folder where data is stored. Starts with + * "gs://" ends with "/". Does not contain "*". + * + * @return value or {@code null} for none + */ + public abstract String getStorageUri(); + + /** + * Open source file format that the table data is stored in. Currently only PARQUET is supported. + * + * @return value or {@code null} for none + */ + public abstract String getTableFormat(); + + public static Builder newBuilder() { + return new AutoValue_BigLakeConfiguration.Builder(); + } + + public abstract Builder toBuilder(); + + @AutoValue.Builder + public abstract static class Builder { + /** + * [Required] Required and immutable. Credential reference for accessing external storage + * system. Normalized as project_id.location_id.connection_id. + * + * @param connectionId connectionId or {@code null} for none + */ + public abstract Builder setConnectionId(String connectionId); + + /** + * [Required] Required and immutable. Open source file format that the table data is stored in. + * Currently only PARQUET is supported. + * + * @param fileFormat fileFormat or {@code null} for none + */ + public abstract Builder setFileFormat(String fileFormat); + + /** + * [Required] Required and immutable. Fully qualified location prefix of the external folder + * where data is stored. Starts with "gs://" and ends with "/". Does not contain "*". + * + * @param storageUri storageUri or {@code null} for none + */ + public abstract Builder setStorageUri(String storageUri); + + /** + * [Required] Required and immutable. Open source file format that the table data is stored in. + * Currently only PARQUET is supported. + * + * @param tableFormat tableFormat or {@code null} for none + */ + public abstract Builder setTableFormat(String tableFormat); + + public abstract BigLakeConfiguration build(); + } + + com.google.api.services.bigquery.model.BigLakeConfiguration toPb() { + com.google.api.services.bigquery.model.BigLakeConfiguration biglakeConfiguration = + new com.google.api.services.bigquery.model.BigLakeConfiguration(); + biglakeConfiguration.setConnectionId(getConnectionId()); + biglakeConfiguration.setFileFormat(getFileFormat()); + biglakeConfiguration.setStorageUri(getStorageUri()); + biglakeConfiguration.setTableFormat(getTableFormat()); + + return biglakeConfiguration; + } + + static BigLakeConfiguration fromPb( + com.google.api.services.bigquery.model.BigLakeConfiguration biglakeConfigurationPb) { + return newBuilder() + .setConnectionId(biglakeConfigurationPb.getConnectionId()) + .setFileFormat(biglakeConfigurationPb.getFileFormat()) + .setStorageUri(biglakeConfigurationPb.getStorageUri()) + .setTableFormat(biglakeConfigurationPb.getTableFormat()) + .build(); + } +} diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardTableDefinition.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardTableDefinition.java index f4459d734..0319a6a4d 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardTableDefinition.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardTableDefinition.java @@ -177,6 +177,12 @@ public abstract static class Builder public abstract Builder setTableConstraints(TableConstraints tableConstraints); + /** + * Set the configuration of a BigLake managed table. If not set, the table is not a BigLake + * managed table. + */ + public abstract Builder setBigLakeConfiguration(BigLakeConfiguration biglakeConfiguration); + /** Creates a {@code StandardTableDefinition} object. */ public abstract StandardTableDefinition build(); } @@ -300,6 +306,13 @@ public abstract static class Builder @Nullable public abstract TableConstraints getTableConstraints(); + /** + * [Optional] Specifies the configuration of a BigLake managed table. The value may be {@code + * null}. + */ + @Nullable + public abstract BigLakeConfiguration getBigLakeConfiguration(); + /** Returns a builder for a BigQuery standard table definition. */ public static Builder newBuilder() { return new AutoValue_StandardTableDefinition.Builder().setType(Type.TABLE); @@ -348,6 +361,9 @@ Table toPb() { if (getTableConstraints() != null) { tablePb.setTableConstraints(getTableConstraints().toPb()); } + if (getBigLakeConfiguration() != null) { + tablePb.setBiglakeConfiguration(getBigLakeConfiguration().toPb()); + } return tablePb; } @@ -409,6 +425,11 @@ static StandardTableDefinition fromPb(Table tablePb) { if (tablePb.getTableConstraints() != null) { builder.setTableConstraints(TableConstraints.fromPb(tablePb.getTableConstraints())); } + if (tablePb.getBiglakeConfiguration() != null) { + builder.setBigLakeConfiguration( + BigLakeConfiguration.fromPb(tablePb.getBiglakeConfiguration())); + } + return builder.setNumBytes(tablePb.getNumBytes()).setLocation(tablePb.getLocation()).build(); } } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigLakeConfigurationTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigLakeConfigurationTest.java new file mode 100644 index 000000000..afb2b5b10 --- /dev/null +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigLakeConfigurationTest.java @@ -0,0 +1,80 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +public class BigLakeConfigurationTest { + + private static final String STORAGE_URI = "gs://storage-uri"; + private static final String FILE_FORMAT = "PARQUET"; + private static final String TABLE_FORMAT = "ICEBERG"; + private static final String CONNECTION_ID = "us.test-connection"; + + private static final BigLakeConfiguration BIG_LAKE_CONFIGURATION = + BigLakeConfiguration.newBuilder() + .setStorageUri(STORAGE_URI) + .setFileFormat(FILE_FORMAT) + .setTableFormat(TABLE_FORMAT) + .setConnectionId(CONNECTION_ID) + .build(); + private static final com.google.api.services.bigquery.model.BigLakeConfiguration + BIG_LAKE_CONFIGURATION_PB = + new com.google.api.services.bigquery.model.BigLakeConfiguration() + .setStorageUri(STORAGE_URI) + .setFileFormat(FILE_FORMAT) + .setTableFormat(TABLE_FORMAT) + .setConnectionId(CONNECTION_ID); + + @Test + public void testToBuilder() { + assertEquals(STORAGE_URI, BIG_LAKE_CONFIGURATION.getStorageUri()); + assertEquals(FILE_FORMAT, BIG_LAKE_CONFIGURATION.getFileFormat()); + assertEquals(TABLE_FORMAT, BIG_LAKE_CONFIGURATION.getTableFormat()); + assertEquals(CONNECTION_ID, BIG_LAKE_CONFIGURATION.getConnectionId()); + } + + @Test + public void testToPb() { + assertBigLakeConfiguration(BIG_LAKE_CONFIGURATION_PB, BIG_LAKE_CONFIGURATION.toPb()); + } + + @Test + public void testFromPb() { + assertBigLakeConfiguration( + BIG_LAKE_CONFIGURATION, BigLakeConfiguration.fromPb(BIG_LAKE_CONFIGURATION_PB)); + } + + private static void assertBigLakeConfiguration( + BigLakeConfiguration expected, BigLakeConfiguration actual) { + assertEquals(expected.getConnectionId(), actual.getConnectionId()); + assertEquals(expected.getTableFormat(), actual.getTableFormat()); + assertEquals(expected.getStorageUri(), actual.getStorageUri()); + assertEquals(expected.getFileFormat(), actual.getFileFormat()); + } + + private static void assertBigLakeConfiguration( + com.google.api.services.bigquery.model.BigLakeConfiguration expected, + com.google.api.services.bigquery.model.BigLakeConfiguration actual) { + assertEquals(expected.getConnectionId(), actual.getConnectionId()); + assertEquals(expected.getTableFormat(), actual.getTableFormat()); + assertEquals(expected.getStorageUri(), actual.getStorageUri()); + assertEquals(expected.getFileFormat(), actual.getFileFormat()); + } +} diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/StandardTableDefinitionTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/StandardTableDefinitionTest.java index df0e5d79b..8fbe3cefe 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/StandardTableDefinitionTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/StandardTableDefinitionTest.java @@ -65,6 +65,13 @@ public class StandardTableDefinitionTest { TimePartitioning.of(TimePartitioning.Type.DAY, 42); private static final Clustering CLUSTERING = Clustering.newBuilder().setFields(ImmutableList.of("Foo", "Bar")).build(); + private static final BigLakeConfiguration BIG_LAKE_CONFIGURATION = + BigLakeConfiguration.newBuilder() + .setConnectionId("us.connection-test") + .setTableFormat("ICEBERG") + .setFileFormat("PARQUET") + .setStorageUri("gs://java-bigquery-test/standard-table-def") + .build(); private static final StandardTableDefinition TABLE_DEFINITION = StandardTableDefinition.newBuilder() .setLocation(LOCATION) @@ -82,6 +89,7 @@ public class StandardTableDefinitionTest { .setSchema(TABLE_SCHEMA) .setTimePartitioning(TIME_PARTITIONING) .setClustering(CLUSTERING) + .setBigLakeConfiguration(BIG_LAKE_CONFIGURATION) .build(); @Test