From 86f1a30d1168a914af29aaecccf823314391633b Mon Sep 17 00:00:00 2001 From: "syed.javed" Date: Fri, 4 Nov 2022 21:34:17 -0700 Subject: [PATCH 1/4] Add AWS Postgres Iam Auth jar to GMS --- build.gradle | 1 + metadata-service/war/build.gradle | 1 + 2 files changed, 2 insertions(+) diff --git a/build.gradle b/build.gradle index d96da3f8e3f05f..aec41b9d4359a5 100644 --- a/build.gradle +++ b/build.gradle @@ -53,6 +53,7 @@ project.ext.externalDependency = [ 'awsGlueSchemaRegistrySerde': 'software.amazon.glue:schema-registry-serde:1.1.10', 'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:1.1.1', 'awsSecretsManagerJdbc': 'com.amazonaws.secretsmanager:aws-secretsmanager-jdbc:1.0.8', + 'awsPostgresIamAuth': 'software.amazon.jdbc:aws-advanced-jdbc-wrapper:1.0.0', 'cacheApi' : 'javax.cache:cache-api:1.1.0', 'commonsCli': 'commons-cli:commons-cli:1.5.0', 'commonsIo': 'commons-io:commons-io:2.4', diff --git a/metadata-service/war/build.gradle b/metadata-service/war/build.gradle index 3a3fdcb66e6116..840bf39b67ba4a 100644 --- a/metadata-service/war/build.gradle +++ b/metadata-service/war/build.gradle @@ -25,6 +25,7 @@ dependencies { runtime spec.product.pegasus.restliSpringBridge implementation externalDependency.awsMskIamAuth + implementation externalDependency.awsPostgresIamAuth } configurations { From 046dfc917befc05dc39adf6c022e67436f7ca201 Mon Sep 17 00:00:00 2001 From: "syed.javed" Date: Tue, 29 Nov 2022 15:37:18 -0800 Subject: [PATCH 2/4] Added IAM support for Postgres --- build.gradle | 1 + docker/datahub-gms/env/docker.postgres.env | 3 +++ metadata-service/factories/build.gradle | 3 ++- .../factory/common/LocalEbeanServerConfigFactory.java | 11 +++++++++++ .../factories/src/main/resources/application.yml | 1 + metadata-service/war/build.gradle | 1 - 6 files changed, 18 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index ce7eac9c86d5cf..726cc7a9f5c4fc 100644 --- a/build.gradle +++ b/build.gradle @@ -54,6 +54,7 @@ project.ext.externalDependency = [ 'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:1.1.1', 'awsSecretsManagerJdbc': 'com.amazonaws.secretsmanager:aws-secretsmanager-jdbc:1.0.8', 'awsPostgresIamAuth': 'software.amazon.jdbc:aws-advanced-jdbc-wrapper:1.0.0', + 'awsRds':'software.amazon.awssdk:rds:2.18.24', 'cacheApi' : 'javax.cache:cache-api:1.1.0', 'commonsCli': 'commons-cli:commons-cli:1.5.0', 'commonsIo': 'commons-io:commons-io:2.4', diff --git a/docker/datahub-gms/env/docker.postgres.env b/docker/datahub-gms/env/docker.postgres.env index 46f3ed1288e20d..3d05dfb1d059a3 100644 --- a/docker/datahub-gms/env/docker.postgres.env +++ b/docker/datahub-gms/env/docker.postgres.env @@ -3,6 +3,9 @@ EBEAN_DATASOURCE_PASSWORD=datahub EBEAN_DATASOURCE_HOST=postgres:5432 EBEAN_DATASOURCE_URL=jdbc:postgresql://postgres:5432/datahub EBEAN_DATASOURCE_DRIVER=org.postgresql.Driver +# Uncomment EBEAN_POSTGRES_USE_AWS_IAM_AUTH below to add support for IAM authentication for Postgres. +# Password is not required when accessing Postgres using IAM auth. It can be replaced by dummy password +# EBEAN_POSTGRES_USE_AWS_IAM_AUTH=true KAFKA_BOOTSTRAP_SERVER=broker:29092 KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 ELASTICSEARCH_HOST=elasticsearch diff --git a/metadata-service/factories/build.gradle b/metadata-service/factories/build.gradle index 6ad9fc6b89b169..0b555ad44cb029 100644 --- a/metadata-service/factories/build.gradle +++ b/metadata-service/factories/build.gradle @@ -25,7 +25,8 @@ dependencies { compile externalDependency.springKafka compile externalDependency.springWeb compile project(':metadata-service:auth-ranger-impl') - + implementation externalDependency.awsPostgresIamAuth + implementation externalDependency.awsRds annotationProcessor externalDependency.lombok compile spec.product.pegasus.restliSpringBridge diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/LocalEbeanServerConfigFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/LocalEbeanServerConfigFactory.java index fa2e420e7e16ac..4505fced0e8fe5 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/LocalEbeanServerConfigFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/LocalEbeanServerConfigFactory.java @@ -6,6 +6,8 @@ import io.ebean.datasource.DataSourceConfig; import io.ebean.datasource.DataSourcePoolListener; import java.sql.Connection; +import java.util.HashMap; +import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; @@ -51,6 +53,9 @@ public class LocalEbeanServerConfigFactory { @Value("${ebean.autoCreateDdl:false}") private Boolean ebeanAutoCreate; + @Value("${ebean.postgresUseIamAuth:false}") + private Boolean postgresUseIamAuth; + private DataSourcePoolListener getListenerToTrackCounts(String metricName) { final String counterName = "ebeans_connection_pool_size_" + metricName; return new DataSourcePoolListener() { @@ -79,6 +84,12 @@ private DataSourceConfig buildDataSourceConfig(String dataSourceUrl, String data dataSourceConfig.setLeakTimeMinutes(ebeanLeakTimeMinutes); dataSourceConfig.setWaitTimeoutMillis(ebeanWaitTimeoutMillis); dataSourceConfig.setListener(getListenerToTrackCounts(dataSourceType)); + // Adding IAM auth access for AWS Postgres + if (postgresUseIamAuth) { + Map custom = new HashMap<>(); + custom.put("wrapperPlugins","iam"); + dataSourceConfig.setCustomProperties(custom); + } return dataSourceConfig; } diff --git a/metadata-service/factories/src/main/resources/application.yml b/metadata-service/factories/src/main/resources/application.yml index a54a16d59d408a..c3a5bd974d7f82 100644 --- a/metadata-service/factories/src/main/resources/application.yml +++ b/metadata-service/factories/src/main/resources/application.yml @@ -110,6 +110,7 @@ ebean: leakTimeMinutes: ${EBEAN_LEAK_TIME_MINUTES:15} waitTimeoutMillis: ${EBEAN_WAIT_TIMEOUT_MILLIS:1000} autoCreateDdl: ${EBEAN_AUTOCREATE:false} + postgresUseIamAuth: ${EBEAN_POSTGRES_USE_AWS_IAM_AUTH:true} # Only required if entityService.impl is cassandra cassandra: diff --git a/metadata-service/war/build.gradle b/metadata-service/war/build.gradle index 840bf39b67ba4a..3a3fdcb66e6116 100644 --- a/metadata-service/war/build.gradle +++ b/metadata-service/war/build.gradle @@ -25,7 +25,6 @@ dependencies { runtime spec.product.pegasus.restliSpringBridge implementation externalDependency.awsMskIamAuth - implementation externalDependency.awsPostgresIamAuth } configurations { From c581e577228a2f6c85f3e74f69392aa0f48e658c Mon Sep 17 00:00:00 2001 From: "syed.javed" Date: Tue, 29 Nov 2022 15:46:05 -0800 Subject: [PATCH 3/4] Setting IAM support for Postgres as false by default --- metadata-service/factories/src/main/resources/application.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-service/factories/src/main/resources/application.yml b/metadata-service/factories/src/main/resources/application.yml index c3a5bd974d7f82..56bb210ae76b21 100644 --- a/metadata-service/factories/src/main/resources/application.yml +++ b/metadata-service/factories/src/main/resources/application.yml @@ -110,7 +110,7 @@ ebean: leakTimeMinutes: ${EBEAN_LEAK_TIME_MINUTES:15} waitTimeoutMillis: ${EBEAN_WAIT_TIMEOUT_MILLIS:1000} autoCreateDdl: ${EBEAN_AUTOCREATE:false} - postgresUseIamAuth: ${EBEAN_POSTGRES_USE_AWS_IAM_AUTH:true} + postgresUseIamAuth: ${EBEAN_POSTGRES_USE_AWS_IAM_AUTH:false} # Only required if entityService.impl is cassandra cassandra: From 05966d7d6345d3286a98e577277c7aa7eaf1a2d7 Mon Sep 17 00:00:00 2001 From: "syed.javed" Date: Wed, 30 Nov 2022 12:52:32 -0800 Subject: [PATCH 4/4] format fix --- .../gms/factory/common/LocalEbeanServerConfigFactory.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/LocalEbeanServerConfigFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/LocalEbeanServerConfigFactory.java index 4505fced0e8fe5..66d917b444e01c 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/LocalEbeanServerConfigFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/LocalEbeanServerConfigFactory.java @@ -86,8 +86,8 @@ private DataSourceConfig buildDataSourceConfig(String dataSourceUrl, String data dataSourceConfig.setListener(getListenerToTrackCounts(dataSourceType)); // Adding IAM auth access for AWS Postgres if (postgresUseIamAuth) { - Map custom = new HashMap<>(); - custom.put("wrapperPlugins","iam"); + Map custom = new HashMap<>(); + custom.put("wrapperPlugins", "iam"); dataSourceConfig.setCustomProperties(custom); } return dataSourceConfig;