From 7729a629912ab1d0d6852172c1776f83f645119d Mon Sep 17 00:00:00 2001 From: tplevko Date: Thu, 20 Oct 2022 15:35:01 +0200 Subject: [PATCH] Add google bigquery system-x --- .../services/google/cloud/bigquery/pom.xml | 32 ++++++ .../bigquery/service/GoogleBigQuery.java | 73 +++++++++++++ .../validation/BigQueryValidation.java | 100 ++++++++++++++++++ system-x/services/google/cloud/pom.xml | 1 + 4 files changed, 206 insertions(+) create mode 100644 system-x/services/google/cloud/bigquery/pom.xml create mode 100644 system-x/services/google/cloud/bigquery/src/main/java/software/tnb/google/cloud/bigquery/service/GoogleBigQuery.java create mode 100644 system-x/services/google/cloud/bigquery/src/main/java/software/tnb/google/cloud/bigquery/validation/BigQueryValidation.java diff --git a/system-x/services/google/cloud/bigquery/pom.xml b/system-x/services/google/cloud/bigquery/pom.xml new file mode 100644 index 000000000..793d1017d --- /dev/null +++ b/system-x/services/google/cloud/bigquery/pom.xml @@ -0,0 +1,32 @@ + + + + system-x-google-cloud + software.tnb + 1.0-SNAPSHOT + + 4.0.0 + + system-x-google-bigquery + 1.0-SNAPSHOT + TNB :: System-X :: Services :: Google :: Cloud :: BigQuery + + + 2.17.1 + + + + + com.google.cloud + google-cloud-bigquery + ${google-cloud-bigquery.version} + + + software.tnb + system-x-google-cloud-common + 1.0-SNAPSHOT + + + diff --git a/system-x/services/google/cloud/bigquery/src/main/java/software/tnb/google/cloud/bigquery/service/GoogleBigQuery.java b/system-x/services/google/cloud/bigquery/src/main/java/software/tnb/google/cloud/bigquery/service/GoogleBigQuery.java new file mode 100644 index 000000000..7b530798b --- /dev/null +++ b/system-x/services/google/cloud/bigquery/src/main/java/software/tnb/google/cloud/bigquery/service/GoogleBigQuery.java @@ -0,0 +1,73 @@ +package software.tnb.google.cloud.bigquery.service; + +import software.tnb.common.account.AccountFactory; +import software.tnb.common.service.Service; +import software.tnb.google.cloud.bigquery.validation.BigQueryValidation; +import software.tnb.google.cloud.common.account.GoogleCloudAccount; + +import org.junit.jupiter.api.extension.ExtensionContext; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.api.gax.core.CredentialsProvider; +import com.google.api.gax.core.FixedCredentialsProvider; +import com.google.auth.oauth2.GoogleCredentials; +import com.google.auto.service.AutoService; +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryOptions; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Base64; + +@AutoService(GoogleBigQuery.class) +public class GoogleBigQuery implements Service { + + private static final Logger LOG = LoggerFactory.getLogger(GoogleBigQuery.class); + + private GoogleCloudAccount account; + private BigQueryValidation validation; + + private BigQuery client; + + public GoogleCloudAccount account() { + if (account == null) { + account = AccountFactory.create(GoogleCloudAccount.class); + } + return account; + } + + protected BigQuery client() throws IOException { + if (client == null) { + LOG.debug("Creating new Google BigQuery client"); + try { + client = + BigQueryOptions.newBuilder().setCredentials(credentialsProvider().getCredentials()).build().getService(); + } catch (Exception e) { + throw new RuntimeException("Unable to create new Google Storage client", e); + } + } + return client; + } + + private CredentialsProvider credentialsProvider() throws IOException { + InputStream serviceAccountKey = new ByteArrayInputStream(Base64.getDecoder().decode(account().serviceAccountKey())); + return FixedCredentialsProvider.create(GoogleCredentials.fromStream(serviceAccountKey)); + } + + public BigQueryValidation validation() { + return validation; + } + + @Override + public void afterAll(ExtensionContext extensionContext) { + } + + @Override + public void beforeAll(ExtensionContext extensionContext) throws Exception { + LOG.debug("Creating Google BigQuery validation"); + validation = new BigQueryValidation(client(), account().projectId()); + } +} diff --git a/system-x/services/google/cloud/bigquery/src/main/java/software/tnb/google/cloud/bigquery/validation/BigQueryValidation.java b/system-x/services/google/cloud/bigquery/src/main/java/software/tnb/google/cloud/bigquery/validation/BigQueryValidation.java new file mode 100644 index 000000000..7c2f11595 --- /dev/null +++ b/system-x/services/google/cloud/bigquery/src/main/java/software/tnb/google/cloud/bigquery/validation/BigQueryValidation.java @@ -0,0 +1,100 @@ +package software.tnb.google.cloud.bigquery.validation; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.Dataset; +import com.google.cloud.bigquery.DatasetId; +import com.google.cloud.bigquery.DatasetInfo; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.FieldList; +import com.google.cloud.bigquery.QueryJobConfiguration; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.StandardSQLTypeName; +import com.google.cloud.bigquery.StandardTableDefinition; +import com.google.cloud.bigquery.TableDefinition; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableInfo; + +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; + +public class BigQueryValidation { + + private static final Logger LOG = LoggerFactory.getLogger(BigQueryValidation.class); + private BigQuery bigQuery; + private String projectId; + + public BigQueryValidation(BigQuery bigQuery, String projectId) { + this.bigQuery = bigQuery; + this.projectId = projectId; + } + + public void createTableWithSampleSchema(String dataSetId, String tableId) { + createTable(dataSetId, tableId, createSampleSchema()); + } + + public void createTable(String dataSetId, String tableId, Schema schema) { + TableId id = TableId.of(projectId, dataSetId, tableId); + TableDefinition.Builder builder = StandardTableDefinition.newBuilder().setSchema(schema); + TableInfo tableInfo = TableInfo.of(id, builder.build()); + bigQuery.create(tableInfo); + } + + public void deleteTable(String datasetName, String tableName) { + LOG.debug("Deleting BQ table " + tableName); + bigQuery.delete(TableId.of(projectId, datasetName, tableName)); + } + + public void createDataset(String datasetName) { + DatasetInfo datasetInfo = DatasetInfo.newBuilder(datasetName).build(); + + Dataset newDataset = bigQuery.create(datasetInfo); + String newDatasetName = newDataset.getDatasetId().getDataset(); + } + + public void deleteDataset(String datasetName) { + DatasetId datasetId = DatasetId.of(projectId, datasetName); + bigQuery.delete(datasetId, BigQuery.DatasetDeleteOption.deleteContents()); + } + + public long tableRowsCount(String datasetName, String tableName) { + String query = + "SELECT * FROM `" + projectId + "." + datasetName + "." + tableName + "`"; + try { + return bigQuery.query(QueryJobConfiguration.of(query)).getTotalRows(); + } catch (InterruptedException e) { + throw new RuntimeException("Unable to query the BQ table", e); + } + } + + public boolean tableContainsRow(String datasetName, String tableName, Map row) { + String query = "SELECT * FROM `" + projectId + "." + datasetName + "." + tableName + "` WHERE " + + row.entrySet().stream() + .map(e -> e.getKey() + " = '" + e.getValue() + "'") + .collect(Collectors.joining(" AND ")); + LOG.debug("Query: {}", query); + QueryJobConfiguration queryJobConfiguration = QueryJobConfiguration.of(query); + try { + return bigQuery.query(queryJobConfiguration).getTotalRows() == 1; + } catch (InterruptedException e) { + throw new RuntimeException("Unable to query the BQ table", e); + } + } + + public Schema createSchema(Map schema) { + FieldList fields = FieldList.of( + schema.entrySet().stream().map(entry -> Field.of(entry.getKey(), entry.getValue()) + ).collect(Collectors.toList())); + return Schema.of(fields); + } + + public Schema createSampleSchema() { + Map schema = new HashMap<>(); + schema.put("id", StandardSQLTypeName.STRING); + schema.put("field", StandardSQLTypeName.STRING); + return createSchema(schema); + } +} diff --git a/system-x/services/google/cloud/pom.xml b/system-x/services/google/cloud/pom.xml index 948156ea3..0ac111729 100644 --- a/system-x/services/google/cloud/pom.xml +++ b/system-x/services/google/cloud/pom.xml @@ -14,6 +14,7 @@ pom + bigquery common functions pubsub