Add google bigquery system-x

system-x/services/google/cloud/bigquery/pom.xml

@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>system-x-google-cloud</artifactId>
+        <groupId>software.tnb</groupId>
+        <version>1.0-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>system-x-google-bigquery</artifactId>
+    <version>1.0-SNAPSHOT</version>
+    <name>TNB :: System-X :: Services :: Google :: Cloud :: BigQuery</name>
+
+    <properties>
+        <google-cloud-bigquery.version>2.17.1</google-cloud-bigquery.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.google.cloud</groupId>
+            <artifactId>google-cloud-bigquery</artifactId>
+            <version>${google-cloud-bigquery.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>software.tnb</groupId>
+            <artifactId>system-x-google-cloud-common</artifactId>
+            <version>1.0-SNAPSHOT</version>
+        </dependency>
+    </dependencies>
+</project>

system-x/services/google/cloud/bigquery/src/main/java/software/tnb/google/cloud/bigquery/service/GoogleBigQuery.java

@@ -0,0 +1,73 @@
+package software.tnb.google.cloud.bigquery.service;
+
+import software.tnb.common.account.AccountFactory;
+import software.tnb.common.service.Service;
+import software.tnb.google.cloud.bigquery.validation.BigQueryValidation;
+import software.tnb.google.cloud.common.account.GoogleCloudAccount;
+
+import org.junit.jupiter.api.extension.ExtensionContext;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.api.gax.core.CredentialsProvider;
+import com.google.api.gax.core.FixedCredentialsProvider;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.auto.service.AutoService;
+import com.google.cloud.bigquery.BigQuery;
+import com.google.cloud.bigquery.BigQueryOptions;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Base64;
+
+@AutoService(GoogleBigQuery.class)
+public class GoogleBigQuery implements Service {
+
+    private static final Logger LOG = LoggerFactory.getLogger(GoogleBigQuery.class);
+
+    private GoogleCloudAccount account;
+    private BigQueryValidation validation;
+
+    private BigQuery client;
+
+    public GoogleCloudAccount account() {
+        if (account == null) {
+            account = AccountFactory.create(GoogleCloudAccount.class);
+        }
+        return account;
+    }
+
+    protected BigQuery client() throws IOException {
+        if (client == null) {
+            LOG.debug("Creating new Google BigQuery client");
+            try {
+                client =
+                    BigQueryOptions.newBuilder().setCredentials(credentialsProvider().getCredentials()).build().getService();
+            } catch (Exception e) {
+                throw new RuntimeException("Unable to create new Google Storage client", e);
+            }
+        }
+        return client;
+    }
+
+    private CredentialsProvider credentialsProvider() throws IOException {
+        InputStream serviceAccountKey = new ByteArrayInputStream(Base64.getDecoder().decode(account().serviceAccountKey()));
+        return FixedCredentialsProvider.create(GoogleCredentials.fromStream(serviceAccountKey));
+    }
+
+    public BigQueryValidation validation() {
+        return validation;
+    }
+
+    @Override
+    public void afterAll(ExtensionContext extensionContext) {
+    }
+
+    @Override
+    public void beforeAll(ExtensionContext extensionContext) throws Exception {
+        LOG.debug("Creating Google BigQuery validation");
+        validation = new BigQueryValidation(client(), account().projectId());
+    }
+}

system-x/services/google/cloud/bigquery/src/main/java/software/tnb/google/cloud/bigquery/validation/BigQueryValidation.java

@@ -0,0 +1,100 @@
+package software.tnb.google.cloud.bigquery.validation;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.cloud.bigquery.BigQuery;
+import com.google.cloud.bigquery.Dataset;
+import com.google.cloud.bigquery.DatasetId;
+import com.google.cloud.bigquery.DatasetInfo;
+import com.google.cloud.bigquery.Field;
+import com.google.cloud.bigquery.FieldList;
+import com.google.cloud.bigquery.QueryJobConfiguration;
+import com.google.cloud.bigquery.Schema;
+import com.google.cloud.bigquery.StandardSQLTypeName;
+import com.google.cloud.bigquery.StandardTableDefinition;
+import com.google.cloud.bigquery.TableDefinition;
+import com.google.cloud.bigquery.TableId;
+import com.google.cloud.bigquery.TableInfo;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class BigQueryValidation {
+
+    private static final Logger LOG = LoggerFactory.getLogger(BigQueryValidation.class);
+    private BigQuery bigQuery;
+    private String projectId;
+
+    public BigQueryValidation(BigQuery bigQuery, String projectId) {
+        this.bigQuery = bigQuery;
+        this.projectId = projectId;
+    }
+
+    public void createTableWithSampleSchema(String dataSetId, String tableId) {
+        createTable(dataSetId, tableId, createSampleSchema());
+    }
+
+    public void createTable(String dataSetId, String tableId, Schema schema) {
+        TableId id = TableId.of(projectId, dataSetId, tableId);
+        TableDefinition.Builder builder = StandardTableDefinition.newBuilder().setSchema(schema);
+        TableInfo tableInfo = TableInfo.of(id, builder.build());
+        bigQuery.create(tableInfo);
+    }
+
+    public void deleteTable(String datasetName, String tableName) {
+        LOG.debug("Deleting BQ table " + tableName);
+        bigQuery.delete(TableId.of(projectId, datasetName, tableName));
+    }
+
+    public void createDataset(String datasetName) {
+        DatasetInfo datasetInfo = DatasetInfo.newBuilder(datasetName).build();
+
+        Dataset newDataset = bigQuery.create(datasetInfo);
+        String newDatasetName = newDataset.getDatasetId().getDataset();
+    }
+
+    public void deleteDataset(String datasetName) {
+        DatasetId datasetId = DatasetId.of(projectId, datasetName);
+        bigQuery.delete(datasetId, BigQuery.DatasetDeleteOption.deleteContents());
+    }
+
+    public long tableRowsCount(String datasetName, String tableName) {
+        String query =
+            "SELECT * FROM `" + projectId + "." + datasetName + "." + tableName + "`";
+        try {
+            return bigQuery.query(QueryJobConfiguration.of(query)).getTotalRows();
+        } catch (InterruptedException e) {
+            throw new RuntimeException("Unable to query the BQ table", e);
+        }
+    }
+
+    public boolean tableContainsRow(String datasetName, String tableName, Map<String, String> row) {
+        String query = "SELECT * FROM `" + projectId + "." + datasetName + "." + tableName + "` WHERE "
+            + row.entrySet().stream()
+            .map(e -> e.getKey() + " = '" + e.getValue() + "'")
+            .collect(Collectors.joining(" AND "));
+        LOG.debug("Query: {}", query);
+        QueryJobConfiguration queryJobConfiguration = QueryJobConfiguration.of(query);
+        try {
+            return bigQuery.query(queryJobConfiguration).getTotalRows() == 1;
+        } catch (InterruptedException e) {
+            throw new RuntimeException("Unable to query the BQ table", e);
+        }
+    }
+
+    public Schema createSchema(Map<String, StandardSQLTypeName> schema) {
+        FieldList fields = FieldList.of(
+            schema.entrySet().stream().map(entry -> Field.of(entry.getKey(), entry.getValue())
+            ).collect(Collectors.toList()));
+        return Schema.of(fields);
+    }
+
+    public Schema createSampleSchema() {
+        Map<String, StandardSQLTypeName> schema = new HashMap<>();
+        schema.put("id", StandardSQLTypeName.STRING);
+        schema.put("field", StandardSQLTypeName.STRING);
+        return createSchema(schema);
+    }
+}

system-x/services/google/cloud/pom.xml

@@ -14,6 +14,7 @@
     <packaging>pom</packaging>
 
     <modules>
+        <module>bigquery</module>
         <module>common</module>
         <module>functions</module>
         <module>pubsub</module>