From ad6667a8fa658cbc8725e17436473fb7ab7357df Mon Sep 17 00:00:00 2001 From: lcaggio Date: Sun, 19 Feb 2023 22:37:32 +0100 Subject: [PATCH 01/13] First commit --- .../data-platform-foundations/01-dropoff.tf | 7 ++-- .../data-platform-foundations/02-load.tf | 7 ++-- .../03-orchestration.tf | 7 ++-- .../04-transformation.tf | 7 ++-- .../05-datawarehouse.tf | 21 +++++++----- .../data-platform-foundations/06-common.tf | 7 ++-- .../data-platform-foundations/07-exposure.tf | 7 ++-- .../data-platform-foundations/variables.tf | 34 +++++++++++++++++++ 8 files changed, 70 insertions(+), 27 deletions(-) diff --git a/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf b/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf index 177f940a86..f1dc492e24 100644 --- a/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf +++ b/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf @@ -23,9 +23,10 @@ locals { module "drop-project" { source = "../../../modules/project" parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "drp${local.project_suffix}" + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.drop : "${var.project_config.project_ids.drop}${local.project_suffix}" group_iam = { (local.groups.data-engineers) = [ "roles/bigquery.dataEditor", diff --git a/blueprints/data-solutions/data-platform-foundations/02-load.tf b/blueprints/data-solutions/data-platform-foundations/02-load.tf index 74cb9f8b0c..9c025e69de 100644 --- a/blueprints/data-solutions/data-platform-foundations/02-load.tf +++ b/blueprints/data-solutions/data-platform-foundations/02-load.tf @@ -36,9 +36,10 @@ locals { module "load-project" { source = "../../../modules/project" parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "lod${local.project_suffix}" + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.load : "${var.project_config.project_ids.load}${local.project_suffix}" group_iam = { (local.groups.data-engineers) = [ "roles/compute.viewer", diff --git a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf index a202afdd05..c7f59578ed 100644 --- a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf +++ b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf @@ -35,9 +35,10 @@ locals { module "orch-project" { source = "../../../modules/project" parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "orc${local.project_suffix}" + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.orc : "${var.project_config.project_ids.orc}${local.project_suffix}" group_iam = { (local.groups.data-engineers) = [ "roles/bigquery.dataEditor", diff --git a/blueprints/data-solutions/data-platform-foundations/04-transformation.tf b/blueprints/data-solutions/data-platform-foundations/04-transformation.tf index 3d3a818c57..c2b9b1e21a 100644 --- a/blueprints/data-solutions/data-platform-foundations/04-transformation.tf +++ b/blueprints/data-solutions/data-platform-foundations/04-transformation.tf @@ -30,9 +30,10 @@ locals { module "transf-project" { source = "../../../modules/project" parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "trf${local.project_suffix}" + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.trf : "${var.project_config.project_ids.trf}${local.project_suffix}" group_iam = { (local.groups.data-engineers) = [ "roles/bigquery.jobUser", diff --git a/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf b/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf index 0db5ce4404..6bc1a28eb9 100644 --- a/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf +++ b/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf @@ -83,9 +83,10 @@ locals { module "dwh-lnd-project" { source = "../../../modules/project" parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "dwh-lnd${local.project_suffix}" + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-lnd : "${var.project_config.project_ids.dwh-lnd}${local.project_suffix}" group_iam = local.dwh_group_iam iam = local.dwh_lnd_iam services = local.dwh_services @@ -98,9 +99,10 @@ module "dwh-lnd-project" { module "dwh-cur-project" { source = "../../../modules/project" parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "dwh-cur${local.project_suffix}" + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-cur : "${var.project_config.project_ids.dwh-cur}${local.project_suffix}" group_iam = local.dwh_group_iam iam = local.dwh_iam services = local.dwh_services @@ -113,9 +115,10 @@ module "dwh-cur-project" { module "dwh-conf-project" { source = "../../../modules/project" parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "dwh-conf${local.project_suffix}" + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-conf : "${var.project_config.project_ids.dwh-conf}${local.project_suffix}" group_iam = local.dwh_group_iam iam = local.dwh_iam services = local.dwh_services diff --git a/blueprints/data-solutions/data-platform-foundations/06-common.tf b/blueprints/data-solutions/data-platform-foundations/06-common.tf index 80451500c2..64df392ba4 100644 --- a/blueprints/data-solutions/data-platform-foundations/06-common.tf +++ b/blueprints/data-solutions/data-platform-foundations/06-common.tf @@ -17,9 +17,10 @@ module "common-project" { source = "../../../modules/project" parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "cmn${local.project_suffix}" + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.common : "${var.project_config.project_ids.common}${local.project_suffix}" group_iam = { (local.groups.data-analysts) = [ "roles/datacatalog.viewer", diff --git a/blueprints/data-solutions/data-platform-foundations/07-exposure.tf b/blueprints/data-solutions/data-platform-foundations/07-exposure.tf index 030be0b832..4a8071c026 100644 --- a/blueprints/data-solutions/data-platform-foundations/07-exposure.tf +++ b/blueprints/data-solutions/data-platform-foundations/07-exposure.tf @@ -17,7 +17,8 @@ module "exp-project" { source = "../../../modules/project" parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "exp${local.project_suffix}" + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.exp : "${var.project_config.project_ids.exp}${local.project_suffix}" } diff --git a/blueprints/data-solutions/data-platform-foundations/variables.tf b/blueprints/data-solutions/data-platform-foundations/variables.tf index 6c25406a2f..de033f7262 100644 --- a/blueprints/data-solutions/data-platform-foundations/variables.tf +++ b/blueprints/data-solutions/data-platform-foundations/variables.tf @@ -177,6 +177,40 @@ variable "prefix" { } } +variable "project_config" { + description = "Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format." + type = object({ + billing_account_id = optional(string, null) + project_ids = optional(object({ + drop = string + load = string + orc = string + trf = string + dwh-lnd = string + dwh-cur = string + dwh-conf = string + common = string + exp = string + }), { + drop = "drp" + load = "lod" + orc = "orc" + trf = "trf" + dwh-lnd = "dwh-lnd" + dwh-cur = "dwh-cur" + dwh-conf = "dwh-conf" + common = "cmn" + exp = "exp" + } + ) + }) + default = {} + validation { + condition = var.project_config.billing_account_id != null || var.project_config.project_ids != null + error_message = "At least one attribute should be set." + } +} + variable "project_services" { description = "List of core services enabled on all projects." type = list(string) From c8f25512eb5094cb9bcd73a28200a8e2cfe346ea Mon Sep 17 00:00:00 2001 From: lcaggio Date: Mon, 20 Feb 2023 00:39:37 +0100 Subject: [PATCH 02/13] Fix variabler to support existing projects. --- .../data-platform-foundations/01-dropoff.tf | 34 +++++----- .../data-platform-foundations/02-load.tf | 43 +++++++------ .../03-orchestration.tf | 63 ++++++++++--------- .../04-transformation.tf | 51 ++++++++------- .../05-datawarehouse.tf | 29 +++++---- .../data-platform-foundations/06-common.tf | 23 ++++--- .../data-platform-foundations/07-exposure.tf | 2 +- .../data-platform-foundations/README.md | 4 ++ .../data-platform-foundations/variables.tf | 2 +- 9 files changed, 136 insertions(+), 115 deletions(-) diff --git a/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf b/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf index f1dc492e24..4c4264d329 100644 --- a/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf +++ b/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf @@ -15,37 +15,37 @@ # tfdoc:file:description drop off project and resources. locals { - drop_orch_service_accounts = [ - module.load-sa-df-0.iam_email, module.orch-sa-cmp-0.iam_email - ] -} - -module "drop-project" { - source = "../../../modules/project" - parent = var.folder_id - billing_account = var.project_config.billing_account_id - project_create = var.project_config.billing_account_id != null - prefix = var.project_config.billing_account_id == null ? null : var.prefix - name = var.project_config.billing_account_id == null ? var.project_config.project_ids.drop : "${var.project_config.project_ids.drop}${local.project_suffix}" - group_iam = { + group_iam_drp = { (local.groups.data-engineers) = [ "roles/bigquery.dataEditor", "roles/pubsub.editor", "roles/storage.admin", ] } - iam = { + iam_drp = { "roles/bigquery.dataEditor" = [module.drop-sa-bq-0.iam_email] "roles/bigquery.user" = [module.load-sa-df-0.iam_email] "roles/pubsub.publisher" = [module.drop-sa-ps-0.iam_email] - "roles/pubsub.subscriber" = concat( - local.drop_orch_service_accounts, [module.load-sa-df-0.iam_email] - ) + "roles/pubsub.subscriber" = [ + module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email + ] "roles/storage.objectAdmin" = [module.load-sa-df-0.iam_email] "roles/storage.objectCreator" = [module.drop-sa-cs-0.iam_email] "roles/storage.objectViewer" = [module.orch-sa-cmp-0.iam_email] "roles/storage.admin" = [module.load-sa-df-0.iam_email] } +} + +module "drop-project" { + source = "../../../modules/project" + parent = var.project_config.parent + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.drop : "${var.project_config.project_ids.drop}${local.project_suffix}" + # group_iam = local.group_iam_drp + iam = var.project_config.billing_account_id != null ? local.iam_drp : null + iam_additive = var.project_config.billing_account_id == null ? local.iam_drp : null services = concat(var.project_services, [ "bigquery.googleapis.com", "bigqueryreservation.googleapis.com", diff --git a/blueprints/data-solutions/data-platform-foundations/02-load.tf b/blueprints/data-solutions/data-platform-foundations/02-load.tf index 9c025e69de..b547f050cf 100644 --- a/blueprints/data-solutions/data-platform-foundations/02-load.tf +++ b/blueprints/data-solutions/data-platform-foundations/02-load.tf @@ -15,6 +15,22 @@ # tfdoc:file:description Load project and VPC. locals { + group_iam_load = { + (local.groups.data-engineers) = [ + "roles/compute.viewer", + "roles/dataflow.admin", + "roles/dataflow.developer", + "roles/viewer", + ] + } + iam_load = { + "roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email] + "roles/dataflow.admin" = [ + module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email + ] + "roles/dataflow.worker" = [module.load-sa-df-0.iam_email] + "roles/storage.objectAdmin" = local.load_service_accounts + } load_service_accounts = [ "serviceAccount:${module.load-project.service_accounts.robots.dataflow}", module.load-sa-df-0.iam_email @@ -35,27 +51,14 @@ locals { module "load-project" { source = "../../../modules/project" - parent = var.folder_id + parent = var.project_config.parent billing_account = var.project_config.billing_account_id project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix name = var.project_config.billing_account_id == null ? var.project_config.project_ids.load : "${var.project_config.project_ids.load}${local.project_suffix}" - group_iam = { - (local.groups.data-engineers) = [ - "roles/compute.viewer", - "roles/dataflow.admin", - "roles/dataflow.developer", - "roles/viewer", - ] - } - iam = { - "roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email] - "roles/dataflow.admin" = [ - module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email - ] - "roles/dataflow.worker" = [module.load-sa-df-0.iam_email] - "roles/storage.objectAdmin" = local.load_service_accounts - } + # group_iam = local.group_iam_load + iam = var.project_config.billing_account_id != null ? local.iam_load : null + iam_additive = var.project_config.billing_account_id == null ? local.iam_load : null services = concat(var.project_services, [ "bigquery.googleapis.com", "bigqueryreservation.googleapis.com", @@ -108,11 +111,11 @@ module "load-vpc" { source = "../../../modules/net-vpc" count = local.use_shared_vpc ? 0 : 1 project_id = module.load-project.project_id - name = "${var.prefix}-default" + name = "${var.prefix}-lod" subnets = [ { ip_cidr_range = "10.10.0.0/24" - name = "default" + name = "${var.prefix}-lod" region = var.region } ] @@ -132,7 +135,7 @@ module "load-nat" { source = "../../../modules/net-cloudnat" count = local.use_shared_vpc ? 0 : 1 project_id = module.load-project.project_id - name = "${var.prefix}-default" + name = "${var.prefix}-lod" region = var.region router_network = module.load-vpc.0.name } diff --git a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf index c7f59578ed..f720fc7f33 100644 --- a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf +++ b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf @@ -15,31 +15,7 @@ # tfdoc:file:description Orchestration project and VPC. locals { - orch_subnet = ( - local.use_shared_vpc - ? var.network_config.subnet_self_links.orchestration - : values(module.orch-vpc.0.subnet_self_links)[0] - ) - orch_vpc = ( - local.use_shared_vpc - ? var.network_config.network_self_link - : module.orch-vpc.0.self_link - ) - - # Note: This formatting is needed for output purposes since the fabric artifact registry - # module doesn't yet expose the docker usage path of a registry folder in the needed format. - orch_docker_path = format("%s-docker.pkg.dev/%s/%s", - var.region, module.orch-project.project_id, module.orch-artifact-reg.name) -} - -module "orch-project" { - source = "../../../modules/project" - parent = var.folder_id - billing_account = var.project_config.billing_account_id - project_create = var.project_config.billing_account_id != null - prefix = var.project_config.billing_account_id == null ? null : var.prefix - name = var.project_config.billing_account_id == null ? var.project_config.project_ids.orc : "${var.project_config.project_ids.orc}${local.project_suffix}" - group_iam = { + group_iam_orch = { (local.groups.data-engineers) = [ "roles/bigquery.dataEditor", "roles/bigquery.jobUser", @@ -54,7 +30,7 @@ module "orch-project" { "roles/serviceusage.serviceUsageConsumer", ] } - iam = { + iam_orch = { "roles/bigquery.dataEditor" = [ module.load-sa-df-0.iam_email, module.transf-sa-df-0.iam_email, @@ -85,7 +61,34 @@ module "orch-project" { ] "roles/storage.objectViewer" = [module.load-sa-df-0.iam_email] } - oslogin = false + orch_subnet = ( + local.use_shared_vpc + ? var.network_config.subnet_self_links.orchestration + : values(module.orch-vpc.0.subnet_self_links)[0] + ) + orch_vpc = ( + local.use_shared_vpc + ? var.network_config.network_self_link + : module.orch-vpc.0.self_link + ) + + # Note: This formatting is needed for output purposes since the fabric artifact registry + # module doesn't yet expose the docker usage path of a registry folder in the needed format. + orch_docker_path = format("%s-docker.pkg.dev/%s/%s", + var.region, module.orch-project.project_id, module.orch-artifact-reg.name) +} + +module "orch-project" { + source = "../../../modules/project" + parent = var.project_config.parent + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.orc : "${var.project_config.project_ids.orc}${local.project_suffix}" + # group_iam = local.group_iam_orch + iam = var.project_config.billing_account_id != null ? local.iam_orch : null + iam_additive = var.project_config.billing_account_id == null ? local.iam_orch : null + oslogin = false services = concat(var.project_services, [ "artifactregistry.googleapis.com", "bigquery.googleapis.com", @@ -133,11 +136,11 @@ module "orch-vpc" { source = "../../../modules/net-vpc" count = local.use_shared_vpc ? 0 : 1 project_id = module.orch-project.project_id - name = "${var.prefix}-default" + name = "${var.prefix}-orch" subnets = [ { ip_cidr_range = "10.10.0.0/24" - name = "default" + name = "${var.prefix}-orch" region = var.region secondary_ip_ranges = { pods = "10.10.8.0/22" @@ -161,7 +164,7 @@ module "orch-nat" { count = local.use_shared_vpc ? 0 : 1 source = "../../../modules/net-cloudnat" project_id = module.orch-project.project_id - name = "${var.prefix}-default" + name = "${var.prefix}-orch" region = var.region router_network = module.orch-vpc.0.name } diff --git a/blueprints/data-solutions/data-platform-foundations/04-transformation.tf b/blueprints/data-solutions/data-platform-foundations/04-transformation.tf index c2b9b1e21a..63d3f399aa 100644 --- a/blueprints/data-solutions/data-platform-foundations/04-transformation.tf +++ b/blueprints/data-solutions/data-platform-foundations/04-transformation.tf @@ -15,32 +15,13 @@ # tfdoc:file:description Trasformation project and VPC. locals { - transf_subnet = ( - local.use_shared_vpc - ? var.network_config.subnet_self_links.orchestration - : values(module.transf-vpc.0.subnet_self_links)[0] - ) - transf_vpc = ( - local.use_shared_vpc - ? var.network_config.network_self_link - : module.transf-vpc.0.self_link - ) -} - -module "transf-project" { - source = "../../../modules/project" - parent = var.folder_id - billing_account = var.project_config.billing_account_id - project_create = var.project_config.billing_account_id != null - prefix = var.project_config.billing_account_id == null ? null : var.prefix - name = var.project_config.billing_account_id == null ? var.project_config.project_ids.trf : "${var.project_config.project_ids.trf}${local.project_suffix}" - group_iam = { + group_iam_trf = { (local.groups.data-engineers) = [ "roles/bigquery.jobUser", "roles/dataflow.admin", ] } - iam = { + iam_trf = { "roles/bigquery.jobUser" = [ module.transf-sa-bq-0.iam_email, ] @@ -55,6 +36,28 @@ module "transf-project" { "serviceAccount:${module.transf-project.service_accounts.robots.dataflow}" ] } + transf_subnet = ( + local.use_shared_vpc + ? var.network_config.subnet_self_links.orchestration + : values(module.transf-vpc.0.subnet_self_links)[0] + ) + transf_vpc = ( + local.use_shared_vpc + ? var.network_config.network_self_link + : module.transf-vpc.0.self_link + ) +} + +module "transf-project" { + source = "../../../modules/project" + parent = var.project_config.parent + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.trf : "${var.project_config.project_ids.trf}${local.project_suffix}" + # group_iam = local.group_iam_trf + iam = var.project_config.billing_account_id != null ? local.iam_orch : null + iam_additive = var.project_config.billing_account_id == null ? local.iam_orch : null services = concat(var.project_services, [ "bigquery.googleapis.com", "bigqueryreservation.googleapis.com", @@ -132,11 +135,11 @@ module "transf-vpc" { source = "../../../modules/net-vpc" count = local.use_shared_vpc ? 0 : 1 project_id = module.transf-project.project_id - name = "${var.prefix}-default" + name = "${var.prefix}-trf" subnets = [ { ip_cidr_range = "10.10.0.0/24" - name = "default" + name = "${var.prefix}-trf" region = var.region } ] @@ -156,7 +159,7 @@ module "transf-nat" { source = "../../../modules/net-cloudnat" count = local.use_shared_vpc ? 0 : 1 project_id = module.transf-project.project_id - name = "${var.prefix}-default" + name = "${var.prefix}-trf" region = var.region router_network = module.transf-vpc.0.name } diff --git a/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf b/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf index 6bc1a28eb9..d22cf0aa52 100644 --- a/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf +++ b/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf @@ -82,14 +82,15 @@ locals { module "dwh-lnd-project" { source = "../../../modules/project" - parent = var.folder_id + parent = var.project_config.parent billing_account = var.project_config.billing_account_id project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-lnd : "${var.project_config.project_ids.dwh-lnd}${local.project_suffix}" - group_iam = local.dwh_group_iam - iam = local.dwh_lnd_iam - services = local.dwh_services + # group_iam = local.dwh_group_iam + iam = var.project_config.billing_account_id != null ? local.dwh_lnd_iam : {} + iam_additive = var.project_config.billing_account_id == null ? local.dwh_lnd_iam : {} + services = local.dwh_services service_encryption_key_ids = { bq = [try(local.service_encryption_keys.bq, null)] storage = [try(local.service_encryption_keys.storage, null)] @@ -98,14 +99,15 @@ module "dwh-lnd-project" { module "dwh-cur-project" { source = "../../../modules/project" - parent = var.folder_id + parent = var.project_config.parent billing_account = var.project_config.billing_account_id project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-cur : "${var.project_config.project_ids.dwh-cur}${local.project_suffix}" - group_iam = local.dwh_group_iam - iam = local.dwh_iam - services = local.dwh_services + # group_iam = local.dwh_group_iam + iam = var.project_config.billing_account_id != null ? local.dwh_iam : {} + iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : {} + services = local.dwh_services service_encryption_key_ids = { bq = [try(local.service_encryption_keys.bq, null)] storage = [try(local.service_encryption_keys.storage, null)] @@ -114,14 +116,15 @@ module "dwh-cur-project" { module "dwh-conf-project" { source = "../../../modules/project" - parent = var.folder_id + parent = var.project_config.parent billing_account = var.project_config.billing_account_id project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-conf : "${var.project_config.project_ids.dwh-conf}${local.project_suffix}" - group_iam = local.dwh_group_iam - iam = local.dwh_iam - services = local.dwh_services + # group_iam = local.dwh_group_iam + iam = var.project_config.billing_account_id != null ? local.dwh_iam : null + iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : null + services = local.dwh_services service_encryption_key_ids = { bq = [try(local.service_encryption_keys.bq, null)] storage = [try(local.service_encryption_keys.storage, null)] @@ -141,7 +144,7 @@ module "dwh-lnd-bq-0" { module "dwh-cur-bq-0" { source = "../../../modules/bigquery-dataset" project_id = module.dwh-cur-project.project_id - id = "${replace(var.prefix, "-", "_")}_dwh_lnd_bq_0" + id = "${replace(var.prefix, "-", "_")}_dwh_cur_bq_0" location = var.location encryption_key = try(local.service_encryption_keys.bq, null) } diff --git a/blueprints/data-solutions/data-platform-foundations/06-common.tf b/blueprints/data-solutions/data-platform-foundations/06-common.tf index 64df392ba4..059d6b5eb7 100644 --- a/blueprints/data-solutions/data-platform-foundations/06-common.tf +++ b/blueprints/data-solutions/data-platform-foundations/06-common.tf @@ -14,14 +14,8 @@ # tfdoc:file:description common project. -module "common-project" { - source = "../../../modules/project" - parent = var.folder_id - billing_account = var.project_config.billing_account_id - project_create = var.project_config.billing_account_id != null - prefix = var.project_config.billing_account_id == null ? null : var.prefix - name = var.project_config.billing_account_id == null ? var.project_config.project_ids.common : "${var.project_config.project_ids.common}${local.project_suffix}" - group_iam = { +locals { + group_iam_common = { (local.groups.data-analysts) = [ "roles/datacatalog.viewer", ] @@ -35,7 +29,7 @@ module "common-project" { "roles/datacatalog.admin" ] } - iam = { + iam_common = { "roles/dlp.user" = [ module.load-sa-df-0.iam_email, module.transf-sa-df-0.iam_email @@ -52,6 +46,17 @@ module "common-project" { # local.groups_iam.data-analysts ] } +} +module "common-project" { + source = "../../../modules/project" + parent = var.project_config.parent + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.common : "${var.project_config.project_ids.common}${local.project_suffix}" + # group_iam = local.group_iam_common + iam = var.project_config.billing_account_id != null ? local.iam_common : null + iam_additive = var.project_config.billing_account_id == null ? local.iam_common : null services = concat(var.project_services, [ "datacatalog.googleapis.com", "dlp.googleapis.com", diff --git a/blueprints/data-solutions/data-platform-foundations/07-exposure.tf b/blueprints/data-solutions/data-platform-foundations/07-exposure.tf index 4a8071c026..ea8fca0947 100644 --- a/blueprints/data-solutions/data-platform-foundations/07-exposure.tf +++ b/blueprints/data-solutions/data-platform-foundations/07-exposure.tf @@ -16,7 +16,7 @@ module "exp-project" { source = "../../../modules/project" - parent = var.folder_id + parent = var.project_config.parent billing_account = var.project_config.billing_account_id project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix diff --git a/blueprints/data-solutions/data-platform-foundations/README.md b/blueprints/data-solutions/data-platform-foundations/README.md index 08b24b2116..f6aaac71f6 100644 --- a/blueprints/data-solutions/data-platform-foundations/README.md +++ b/blueprints/data-solutions/data-platform-foundations/README.md @@ -233,6 +233,10 @@ To create Cloud Key Management keys in the Data Platform you can uncomment the C To handle multiple groups of `data-analysts` accessing the same Data Warehouse layer projects but only to the dataset belonging to a specific group, you may want to assign roles at BigQuery dataset level instead of at project-level. To do this, you need to remove IAM binging at project-level for the `data-analysts` group and give roles at BigQuery dataset level using the `iam` variable on `bigquery-dataset` modules. +### Project Configuration + +The solution can be deployed creating projects on a given parent (organization or folder) or on existing projects. Configure variable `project_config` accordingly. + ## Demo pipeline The application layer is out of scope of this script. As a demo purpuse only, several Cloud Composer DAGs are provided. Demos will import data from the `drop off` area to the `Data Warehouse Confidential` dataset suing different features. diff --git a/blueprints/data-solutions/data-platform-foundations/variables.tf b/blueprints/data-solutions/data-platform-foundations/variables.tf index de033f7262..ca5f754fc8 100644 --- a/blueprints/data-solutions/data-platform-foundations/variables.tf +++ b/blueprints/data-solutions/data-platform-foundations/variables.tf @@ -181,6 +181,7 @@ variable "project_config" { description = "Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format." type = object({ billing_account_id = optional(string, null) + parent = string project_ids = optional(object({ drop = string load = string @@ -204,7 +205,6 @@ variable "project_config" { } ) }) - default = {} validation { condition = var.project_config.billing_account_id != null || var.project_config.project_ids != null error_message = "At least one attribute should be set." From db6a4f9ac7f2c5bb510eb72b4fd287e21167257f Mon Sep 17 00:00:00 2001 From: lcaggio Date: Mon, 20 Feb 2023 00:45:40 +0100 Subject: [PATCH 03/13] Remove variables --- .../data-solutions/data-platform-foundations/README.md | 6 ++++-- .../data-platform-foundations/variables.tf | 10 ---------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/blueprints/data-solutions/data-platform-foundations/README.md b/blueprints/data-solutions/data-platform-foundations/README.md index f6aaac71f6..e1bb5f5ec8 100644 --- a/blueprints/data-solutions/data-platform-foundations/README.md +++ b/blueprints/data-solutions/data-platform-foundations/README.md @@ -213,9 +213,11 @@ While this blueprint can be used as a standalone deployment, it can also be call ```hcl module "data-platform" { source = "./fabric/blueprints/data-solutions/data-platform-foundations" - billing_account_id = var.billing_account_id - folder_id = var.folder_id organization_domain = "example.com" + project_config = { + billing_account_id = var.billing_account_id + parent = "folders/12345678" + } prefix = "myprefix" } diff --git a/blueprints/data-solutions/data-platform-foundations/variables.tf b/blueprints/data-solutions/data-platform-foundations/variables.tf index ca5f754fc8..4ec2fd7e1f 100644 --- a/blueprints/data-solutions/data-platform-foundations/variables.tf +++ b/blueprints/data-solutions/data-platform-foundations/variables.tf @@ -14,11 +14,6 @@ # tfdoc:file:description Terraform Variables. -variable "billing_account_id" { - description = "Billing account id." - type = string -} - variable "composer_config" { description = "Cloud Composer config." type = object({ @@ -119,11 +114,6 @@ variable "data_force_destroy" { default = false } -variable "folder_id" { - description = "Folder to be used for the networking resources in folders/nnnn format." - type = string -} - variable "groups" { description = "User groups." type = map(string) From f4c1fa6c20f28e0f071d81d8c1e6762fc7c64f8d Mon Sep 17 00:00:00 2001 From: lcaggio Date: Mon, 20 Feb 2023 00:56:32 +0100 Subject: [PATCH 04/13] Fix tests. --- .../data-platform-foundations/README.md | 4 ++-- fast/stages/3-data-platform/dev/main.tf | 10 ++++++---- .../data_platform_foundations/fixture/main.tf | 8 +++++--- .../data_platform_foundations/test_plan.py | 2 +- tests/fast/stages/s3_data_platform/common.tfvars | 12 +++++++----- 5 files changed, 21 insertions(+), 15 deletions(-) diff --git a/blueprints/data-solutions/data-platform-foundations/README.md b/blueprints/data-solutions/data-platform-foundations/README.md index e1bb5f5ec8..d004cf9440 100644 --- a/blueprints/data-solutions/data-platform-foundations/README.md +++ b/blueprints/data-solutions/data-platform-foundations/README.md @@ -215,13 +215,13 @@ module "data-platform" { source = "./fabric/blueprints/data-solutions/data-platform-foundations" organization_domain = "example.com" project_config = { - billing_account_id = var.billing_account_id + billing_account_id = "123456-123456-123456" parent = "folders/12345678" } prefix = "myprefix" } -# tftest modules=43 resources=297 +# tftest modules=43 resources=264 ``` ## Customizations diff --git a/fast/stages/3-data-platform/dev/main.tf b/fast/stages/3-data-platform/dev/main.tf index 53d901d1be..c600a7586f 100644 --- a/fast/stages/3-data-platform/dev/main.tf +++ b/fast/stages/3-data-platform/dev/main.tf @@ -18,13 +18,15 @@ module "data-platform" { source = "../../../../blueprints/data-solutions/data-platform-foundations" - billing_account_id = var.billing_account.id composer_config = var.composer_config data_force_destroy = var.data_force_destroy data_catalog_tags = var.data_catalog_tags - folder_id = var.folder_ids.data-platform-dev - groups = var.groups - location = var.location + project_config = { + billing_account_id = var.billing_account.id + parent = var.folder_ids.data-platform-dev + } + groups = var.groups + location = var.location network_config = { host_project = var.host_project_ids.dev-spoke-0 network_self_link = var.vpc_self_links.dev-spoke-0 diff --git a/tests/blueprints/data_solutions/data_platform_foundations/fixture/main.tf b/tests/blueprints/data_solutions/data_platform_foundations/fixture/main.tf index 52317d6f5f..5acb29e835 100644 --- a/tests/blueprints/data_solutions/data_platform_foundations/fixture/main.tf +++ b/tests/blueprints/data_solutions/data_platform_foundations/fixture/main.tf @@ -17,7 +17,9 @@ module "test" { source = "../../../../../blueprints/data-solutions/data-platform-foundations/" organization_domain = "example.com" - billing_account_id = "123456-123456-123456" - folder_id = "folders/12345678" - prefix = "prefix" + project_config = { + billing_account_id = "123456-123456-123456" + parent = "folders/12345678" + } + prefix = "prefix" } diff --git a/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py b/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py index 785f470537..f3ed2ba008 100644 --- a/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py +++ b/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py @@ -23,4 +23,4 @@ def test_resources(e2e_plan_runner): modules, resources = e2e_plan_runner(FIXTURES_DIR) assert len(modules) == 42 - assert len(resources) == 296 + assert len(resources) == 264 diff --git a/tests/fast/stages/s3_data_platform/common.tfvars b/tests/fast/stages/s3_data_platform/common.tfvars index 2ec41d37ad..97d8bebc3e 100644 --- a/tests/fast/stages/s3_data_platform/common.tfvars +++ b/tests/fast/stages/s3_data_platform/common.tfvars @@ -1,11 +1,13 @@ automation = { outputs_bucket = "test" } -billing_account = { - id = "012345-67890A-BCDEF0", -} -folder_ids = { - data-platform-dev = "folders/12345678" +project_config = { + billing_account = { + id = "012345-67890A-BCDEF0", + }, + parent = { + data-platform-dev = "folders/12345678" + } } host_project_ids = { dev-spoke-0 = "fast-dev-net-spoke-0" From c523dce3a6890566743a3c9a0c017470d28abd0c Mon Sep 17 00:00:00 2001 From: lcaggio Date: Mon, 20 Feb 2023 01:00:57 +0100 Subject: [PATCH 05/13] Fix --- blueprints/data-solutions/data-platform-foundations/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/data-solutions/data-platform-foundations/README.md b/blueprints/data-solutions/data-platform-foundations/README.md index d004cf9440..98c84ee475 100644 --- a/blueprints/data-solutions/data-platform-foundations/README.md +++ b/blueprints/data-solutions/data-platform-foundations/README.md @@ -221,7 +221,7 @@ module "data-platform" { prefix = "myprefix" } -# tftest modules=43 resources=264 +# tftest modules=43 resources=265 ``` ## Customizations From eda9597f0b01bd58635988f0c23b42761b3c5f54 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Mon, 20 Feb 2023 01:03:38 +0100 Subject: [PATCH 06/13] Fix linting --- .../data-platform-foundations/README.md | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/blueprints/data-solutions/data-platform-foundations/README.md b/blueprints/data-solutions/data-platform-foundations/README.md index 98c84ee475..027c62990c 100644 --- a/blueprints/data-solutions/data-platform-foundations/README.md +++ b/blueprints/data-solutions/data-platform-foundations/README.md @@ -250,20 +250,19 @@ You can find examples in the `[demo](./demo)` folder. | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [billing_account_id](variables.tf#L17) | Billing account id. | string | ✓ | | -| [folder_id](variables.tf#L122) | Folder to be used for the networking resources in folders/nnnn format. | string | ✓ | | -| [organization_domain](variables.tf#L166) | Organization domain. | string | ✓ | | -| [prefix](variables.tf#L171) | Prefix used for resource names. | string | ✓ | | -| [composer_config](variables.tf#L22) | Cloud Composer config. | object({…}) | | {…} | -| [data_catalog_tags](variables.tf#L105) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | -| [data_force_destroy](variables.tf#L116) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | bool | | false | -| [groups](variables.tf#L127) | User groups. | map(string) | | {…} | -| [location](variables.tf#L137) | Location used for multi-regional resources. | string | | "eu" | -| [network_config](variables.tf#L143) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | -| [project_services](variables.tf#L180) | List of core services enabled on all projects. | list(string) | | […] | -| [project_suffix](variables.tf#L191) | Suffix used only for project ids. | string | | null | -| [region](variables.tf#L197) | Region used for regional resources. | string | | "europe-west1" | -| [service_encryption_keys](variables.tf#L203) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | +| [organization_domain](variables.tf#L156) | Organization domain. | string | ✓ | | +| [prefix](variables.tf#L161) | Prefix used for resource names. | string | ✓ | | +| [project_config](variables.tf#L170) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | | +| [composer_config](variables.tf#L17) | Cloud Composer config. | object({…}) | | {…} | +| [data_catalog_tags](variables.tf#L100) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | +| [data_force_destroy](variables.tf#L111) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | bool | | false | +| [groups](variables.tf#L117) | User groups. | map(string) | | {…} | +| [location](variables.tf#L127) | Location used for multi-regional resources. | string | | "eu" | +| [network_config](variables.tf#L133) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | +| [project_services](variables.tf#L204) | List of core services enabled on all projects. | list(string) | | […] | +| [project_suffix](variables.tf#L215) | Suffix used only for project ids. | string | | null | +| [region](variables.tf#L221) | Region used for regional resources. | string | | "europe-west1" | +| [service_encryption_keys](variables.tf#L227) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | ## Outputs From 63a81a9b9b1e50f10df9882bfdd052985e9067ef Mon Sep 17 00:00:00 2001 From: lcaggio Date: Mon, 20 Feb 2023 01:12:19 +0100 Subject: [PATCH 07/13] Fix Fast test --- fast/stages/3-data-platform/dev/variables.tf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/fast/stages/3-data-platform/dev/variables.tf b/fast/stages/3-data-platform/dev/variables.tf index 74a5dbe11d..d0aad16f61 100644 --- a/fast/stages/3-data-platform/dev/variables.tf +++ b/fast/stages/3-data-platform/dev/variables.tf @@ -169,6 +169,14 @@ variable "prefix" { type = string } +variable "project_config" { + description = "Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format." + type = object({ + billing_account_id = string + parent = string + }) +} + variable "project_services" { description = "List of core services enabled on all projects." type = list(string) From 970b8ff2557417038db447d0cd4e6a9faa48305b Mon Sep 17 00:00:00 2001 From: lcaggio Date: Mon, 20 Feb 2023 01:16:22 +0100 Subject: [PATCH 08/13] Fix DP Fast variables. --- fast/stages/3-data-platform/dev/README.md | 11 +++++----- fast/stages/3-data-platform/dev/variables.tf | 21 -------------------- 2 files changed, 6 insertions(+), 26 deletions(-) diff --git a/fast/stages/3-data-platform/dev/README.md b/fast/stages/3-data-platform/dev/README.md index 48d09eafc0..518e4516ad 100644 --- a/fast/stages/3-data-platform/dev/README.md +++ b/fast/stages/3-data-platform/dev/README.md @@ -190,6 +190,7 @@ You can find examples in the `[demo](../../../../blueprints/data-solutions/data- | [host_project_ids](variables.tf#L120) | Shared VPC project ids. | object({…}) | ✓ | | 2-networking | | [organization](variables.tf#L150) | Organization details. | object({…}) | ✓ | | 00-globals | | [prefix](variables.tf#L166) | Unique prefix used for resource names. Not used for projects if 'project_create' is null. | string | ✓ | | 00-globals | +| [project_config](variables.tf#L172) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | | | | [composer_config](variables.tf#L38) | Cloud Composer configuration options. | object({…}) | | {…} | | | [data_catalog_tags](variables.tf#L85) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | | | [data_force_destroy](variables.tf#L96) | Flag to set 'force_destroy' on data services like BigQery or Cloud Storage. | bool | | false | | @@ -197,11 +198,11 @@ You can find examples in the `[demo](../../../../blueprints/data-solutions/data- | [location](variables.tf#L128) | Location used for multi-regional resources. | string | | "eu" | | | [network_config_composer](variables.tf#L134) | Network configurations to use for Composer. | object({…}) | | {…} | | | [outputs_location](variables.tf#L160) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | -| [project_services](variables.tf#L172) | List of core services enabled on all projects. | list(string) | | […] | | -| [region](variables.tf#L183) | Region used for regional resources. | string | | "europe-west1" | | -| [service_encryption_keys](variables.tf#L189) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | | -| [subnet_self_links](variables.tf#L201) | Shared VPC subnet self links. | object({…}) | | null | 2-networking | -| [vpc_self_links](variables.tf#L210) | Shared VPC self links. | object({…}) | | null | 2-networking | +| [project_services](variables.tf#L180) | List of core services enabled on all projects. | list(string) | | […] | | +| [region](variables.tf#L191) | Region used for regional resources. | string | | "europe-west1" | | +| [service_encryption_keys](variables.tf#L197) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | | +| [subnet_self_links](variables.tf#L209) | Shared VPC subnet self links. | object({…}) | | null | 2-networking | +| [vpc_self_links](variables.tf#L218) | Shared VPC self links. | object({…}) | | null | 2-networking | ## Outputs diff --git a/fast/stages/3-data-platform/dev/variables.tf b/fast/stages/3-data-platform/dev/variables.tf index d0aad16f61..392e2dc9e8 100644 --- a/fast/stages/3-data-platform/dev/variables.tf +++ b/fast/stages/3-data-platform/dev/variables.tf @@ -22,19 +22,6 @@ variable "automation" { }) } -variable "billing_account" { - # tfdoc:variable:source 0-bootstrap - description = "Billing account id. If billing account is not part of the same org set `is_org_level` to false." - type = object({ - id = string - is_org_level = optional(bool, true) - }) - validation { - condition = var.billing_account.is_org_level != null - error_message = "Invalid `null` value for `billing_account.is_org_level`." - } -} - variable "composer_config" { description = "Cloud Composer configuration options." type = object({ @@ -99,14 +86,6 @@ variable "data_force_destroy" { default = false } -variable "folder_ids" { - # tfdoc:variable:source 1-resman - description = "Folder to be used for the networking resources in folders/nnnn format." - type = object({ - data-platform-dev = string - }) -} - variable "groups" { description = "Groups." type = map(string) From 2564c9b06a4b51616e9d3351cfd60628d39e2c78 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Mon, 20 Feb 2023 01:17:08 +0100 Subject: [PATCH 09/13] Fix README --- fast/stages/3-data-platform/dev/README.md | 34 +++++++++++------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/fast/stages/3-data-platform/dev/README.md b/fast/stages/3-data-platform/dev/README.md index 518e4516ad..f29903106e 100644 --- a/fast/stages/3-data-platform/dev/README.md +++ b/fast/stages/3-data-platform/dev/README.md @@ -185,24 +185,22 @@ You can find examples in the `[demo](../../../../blueprints/data-solutions/data- | name | description | type | required | default | producer | |---|---|:---:|:---:|:---:|:---:| | [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 0-bootstrap | -| [billing_account](variables.tf#L25) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | object({…}) | ✓ | | 0-bootstrap | -| [folder_ids](variables.tf#L102) | Folder to be used for the networking resources in folders/nnnn format. | object({…}) | ✓ | | 1-resman | -| [host_project_ids](variables.tf#L120) | Shared VPC project ids. | object({…}) | ✓ | | 2-networking | -| [organization](variables.tf#L150) | Organization details. | object({…}) | ✓ | | 00-globals | -| [prefix](variables.tf#L166) | Unique prefix used for resource names. Not used for projects if 'project_create' is null. | string | ✓ | | 00-globals | -| [project_config](variables.tf#L172) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | | | -| [composer_config](variables.tf#L38) | Cloud Composer configuration options. | object({…}) | | {…} | | -| [data_catalog_tags](variables.tf#L85) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | | -| [data_force_destroy](variables.tf#L96) | Flag to set 'force_destroy' on data services like BigQery or Cloud Storage. | bool | | false | | -| [groups](variables.tf#L110) | Groups. | map(string) | | {…} | | -| [location](variables.tf#L128) | Location used for multi-regional resources. | string | | "eu" | | -| [network_config_composer](variables.tf#L134) | Network configurations to use for Composer. | object({…}) | | {…} | | -| [outputs_location](variables.tf#L160) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | -| [project_services](variables.tf#L180) | List of core services enabled on all projects. | list(string) | | […] | | -| [region](variables.tf#L191) | Region used for regional resources. | string | | "europe-west1" | | -| [service_encryption_keys](variables.tf#L197) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | | -| [subnet_self_links](variables.tf#L209) | Shared VPC subnet self links. | object({…}) | | null | 2-networking | -| [vpc_self_links](variables.tf#L218) | Shared VPC self links. | object({…}) | | null | 2-networking | +| [host_project_ids](variables.tf#L99) | Shared VPC project ids. | object({…}) | ✓ | | 2-networking | +| [organization](variables.tf#L129) | Organization details. | object({…}) | ✓ | | 00-globals | +| [prefix](variables.tf#L145) | Unique prefix used for resource names. Not used for projects if 'project_create' is null. | string | ✓ | | 00-globals | +| [project_config](variables.tf#L151) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | | | +| [composer_config](variables.tf#L25) | Cloud Composer configuration options. | object({…}) | | {…} | | +| [data_catalog_tags](variables.tf#L72) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | | +| [data_force_destroy](variables.tf#L83) | Flag to set 'force_destroy' on data services like BigQery or Cloud Storage. | bool | | false | | +| [groups](variables.tf#L89) | Groups. | map(string) | | {…} | | +| [location](variables.tf#L107) | Location used for multi-regional resources. | string | | "eu" | | +| [network_config_composer](variables.tf#L113) | Network configurations to use for Composer. | object({…}) | | {…} | | +| [outputs_location](variables.tf#L139) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | +| [project_services](variables.tf#L159) | List of core services enabled on all projects. | list(string) | | […] | | +| [region](variables.tf#L170) | Region used for regional resources. | string | | "europe-west1" | | +| [service_encryption_keys](variables.tf#L176) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | | +| [subnet_self_links](variables.tf#L188) | Shared VPC subnet self links. | object({…}) | | null | 2-networking | +| [vpc_self_links](variables.tf#L197) | Shared VPC self links. | object({…}) | | null | 2-networking | ## Outputs From 2108b4650deb5814cede7daed89a1548bb1c3117 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Wed, 22 Feb 2023 01:36:01 +0100 Subject: [PATCH 10/13] Fix Tests, rely on iam additive. --- .../data-platform-foundations/01-dropoff.tf | 27 ++++---- .../data-platform-foundations/02-load.tf | 29 ++++---- .../03-orchestration.tf | 41 ++++------- .../04-transformation.tf | 19 ++--- .../05-datawarehouse.tf | 69 ++++++++----------- .../data-platform-foundations/06-common.tf | 29 +++----- .../data-platform-foundations/README.md | 10 +-- .../demo/datapipeline_dc_tags.py | 8 +-- .../demo/delete_table.py | 4 +- fast/stages/3-data-platform/dev/variables.tf | 29 +++++--- .../data_platform_foundations/test_plan.py | 2 +- 11 files changed, 120 insertions(+), 147 deletions(-) diff --git a/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf b/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf index 4c4264d329..46e9a1309a 100644 --- a/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf +++ b/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf @@ -15,24 +15,22 @@ # tfdoc:file:description drop off project and resources. locals { - group_iam_drp = { - (local.groups.data-engineers) = [ - "roles/bigquery.dataEditor", - "roles/pubsub.editor", - "roles/storage.admin", - ] - } iam_drp = { - "roles/bigquery.dataEditor" = [module.drop-sa-bq-0.iam_email] - "roles/bigquery.user" = [module.load-sa-df-0.iam_email] - "roles/pubsub.publisher" = [module.drop-sa-ps-0.iam_email] + "roles/bigquery.dataEditor" = [ + module.drop-sa-bq-0.iam_email, local.groups_iam.data-engineers + ] + "roles/bigquery.user" = [ + module.load-sa-df-0.iam_email, local.groups_iam.data-engineers + ] + "roles/pubsub.publisher" = [module.drop-sa-ps-0.iam_email] "roles/pubsub.subscriber" = [ module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email ] - "roles/storage.objectAdmin" = [module.load-sa-df-0.iam_email] "roles/storage.objectCreator" = [module.drop-sa-cs-0.iam_email] "roles/storage.objectViewer" = [module.orch-sa-cmp-0.iam_email] - "roles/storage.admin" = [module.load-sa-df-0.iam_email] + "roles/storage.objectAdmin" = [ + module.load-sa-df-0.iam_email, module.load-sa-df-0.iam_email + ] } } @@ -43,9 +41,8 @@ module "drop-project" { project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix name = var.project_config.billing_account_id == null ? var.project_config.project_ids.drop : "${var.project_config.project_ids.drop}${local.project_suffix}" - # group_iam = local.group_iam_drp - iam = var.project_config.billing_account_id != null ? local.iam_drp : null - iam_additive = var.project_config.billing_account_id == null ? local.iam_drp : null + iam = var.project_config.billing_account_id != null ? local.iam_drp : null + iam_additive = var.project_config.billing_account_id == null ? local.iam_drp : null services = concat(var.project_services, [ "bigquery.googleapis.com", "bigqueryreservation.googleapis.com", diff --git a/blueprints/data-solutions/data-platform-foundations/02-load.tf b/blueprints/data-solutions/data-platform-foundations/02-load.tf index b547f050cf..9702fce1ec 100644 --- a/blueprints/data-solutions/data-platform-foundations/02-load.tf +++ b/blueprints/data-solutions/data-platform-foundations/02-load.tf @@ -15,18 +15,15 @@ # tfdoc:file:description Load project and VPC. locals { - group_iam_load = { - (local.groups.data-engineers) = [ - "roles/compute.viewer", - "roles/dataflow.admin", - "roles/dataflow.developer", - "roles/viewer", - ] - } iam_load = { "roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email] "roles/dataflow.admin" = [ - module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email + module.orch-sa-cmp-0.iam_email, + module.load-sa-df-0.iam_email, + local.groups_iam.data-engineers + ] + "roles/dataflow.developer" = [ + local.groups_iam.data-engineers ] "roles/dataflow.worker" = [module.load-sa-df-0.iam_email] "roles/storage.objectAdmin" = local.load_service_accounts @@ -56,9 +53,8 @@ module "load-project" { project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix name = var.project_config.billing_account_id == null ? var.project_config.project_ids.load : "${var.project_config.project_ids.load}${local.project_suffix}" - # group_iam = local.group_iam_load - iam = var.project_config.billing_account_id != null ? local.iam_load : null - iam_additive = var.project_config.billing_account_id == null ? local.iam_load : null + iam = var.project_config.billing_account_id != null ? local.iam_load : null + iam_additive = var.project_config.billing_account_id == null ? local.iam_load : null services = concat(var.project_services, [ "bigquery.googleapis.com", "bigqueryreservation.googleapis.com", @@ -90,8 +86,13 @@ module "load-sa-df-0" { name = "load-df-0" display_name = "Data platform Dataflow load service account" iam = { - "roles/iam.serviceAccountTokenCreator" = [local.groups_iam.data-engineers] - "roles/iam.serviceAccountUser" = [module.orch-sa-cmp-0.iam_email] + "roles/iam.serviceAccountTokenCreator" = [ + local.groups_iam.data-engineers, + module.orch-sa-cmp-0.iam_email + ], + "roles/iam.serviceAccountUser" = [ + module.orch-sa-cmp-0.iam_email + ] } } diff --git a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf index f720fc7f33..fc0eda12e6 100644 --- a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf +++ b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf @@ -15,29 +15,22 @@ # tfdoc:file:description Orchestration project and VPC. locals { - group_iam_orch = { - (local.groups.data-engineers) = [ - "roles/bigquery.dataEditor", - "roles/bigquery.jobUser", - "roles/cloudbuild.builds.editor", - "roles/composer.admin", - "roles/composer.environmentAndStorageObjectAdmin", - "roles/iap.httpsResourceAccessor", - "roles/iam.serviceAccountUser", - "roles/storage.objectAdmin", - "roles/storage.admin", - "roles/artifactregistry.admin", - "roles/serviceusage.serviceUsageConsumer", - ] - } iam_orch = { + "roles/artifactregistry.admin" = [local.groups_iam.data-engineers] + "roles/artifactregistry.reader" = [module.load-sa-df-0.iam_email] "roles/bigquery.dataEditor" = [ module.load-sa-df-0.iam_email, module.transf-sa-df-0.iam_email, + local.groups_iam.data-engineers ] "roles/bigquery.jobUser" = [ module.orch-sa-cmp-0.iam_email, + local.groups_iam.data-engineers ] + "roles/cloudbuild.builds.editor" = [local.groups_iam.data-engineers] + "roles/cloudbuild.serviceAgent" = [module.orch-sa-df-build.iam_email] + "roles/composer.admin" = [local.groups_iam.data-engineers] + "roles/composer.environmentAndStorageObjectAdmin" = [local.groups_iam.data-engineers] "roles/composer.ServiceAgentV2Ext" = [ "serviceAccount:${module.orch-project.service_accounts.robots.composer}" ] @@ -45,19 +38,16 @@ locals { module.orch-sa-cmp-0.iam_email ] "roles/iam.serviceAccountUser" = [ - module.orch-sa-cmp-0.iam_email + module.orch-sa-cmp-0.iam_email, local.groups_iam.data-engineers ] + "roles/iap.httpsResourceAccessor" = [local.groups_iam.data-engineers] + "roles/serviceusage.serviceUsageConsumer" = [local.groups_iam.data-engineers] "roles/storage.objectAdmin" = [ module.orch-sa-cmp-0.iam_email, module.orch-sa-df-build.iam_email, "serviceAccount:${module.orch-project.service_accounts.robots.composer}", "serviceAccount:${module.orch-project.service_accounts.robots.cloudbuild}", - ] - "roles/artifactregistry.reader" = [ - module.load-sa-df-0.iam_email, - ] - "roles/cloudbuild.serviceAgent" = [ - module.orch-sa-df-build.iam_email, + local.groups_iam.data-engineers ] "roles/storage.objectViewer" = [module.load-sa-df-0.iam_email] } @@ -85,10 +75,9 @@ module "orch-project" { project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix name = var.project_config.billing_account_id == null ? var.project_config.project_ids.orc : "${var.project_config.project_ids.orc}${local.project_suffix}" - # group_iam = local.group_iam_orch - iam = var.project_config.billing_account_id != null ? local.iam_orch : null - iam_additive = var.project_config.billing_account_id == null ? local.iam_orch : null - oslogin = false + iam = var.project_config.billing_account_id != null ? local.iam_orch : null + iam_additive = var.project_config.billing_account_id == null ? local.iam_orch : null + oslogin = false services = concat(var.project_services, [ "artifactregistry.googleapis.com", "bigquery.googleapis.com", diff --git a/blueprints/data-solutions/data-platform-foundations/04-transformation.tf b/blueprints/data-solutions/data-platform-foundations/04-transformation.tf index 63d3f399aa..394adedf8a 100644 --- a/blueprints/data-solutions/data-platform-foundations/04-transformation.tf +++ b/blueprints/data-solutions/data-platform-foundations/04-transformation.tf @@ -15,22 +15,14 @@ # tfdoc:file:description Trasformation project and VPC. locals { - group_iam_trf = { - (local.groups.data-engineers) = [ - "roles/bigquery.jobUser", - "roles/dataflow.admin", - ] - } iam_trf = { "roles/bigquery.jobUser" = [ - module.transf-sa-bq-0.iam_email, + module.transf-sa-bq-0.iam_email, local.groups_iam.data-engineers ] "roles/dataflow.admin" = [ - module.orch-sa-cmp-0.iam_email, - ] - "roles/dataflow.worker" = [ - module.transf-sa-df-0.iam_email + module.orch-sa-cmp-0.iam_email, local.groups_iam.data-engineers ] + "roles/dataflow.worker" = [module.transf-sa-df-0.iam_email] "roles/storage.objectAdmin" = [ module.transf-sa-df-0.iam_email, "serviceAccount:${module.transf-project.service_accounts.robots.dataflow}" @@ -55,9 +47,8 @@ module "transf-project" { project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix name = var.project_config.billing_account_id == null ? var.project_config.project_ids.trf : "${var.project_config.project_ids.trf}${local.project_suffix}" - # group_iam = local.group_iam_trf - iam = var.project_config.billing_account_id != null ? local.iam_orch : null - iam_additive = var.project_config.billing_account_id == null ? local.iam_orch : null + iam = var.project_config.billing_account_id != null ? local.iam_trf : null + iam_additive = var.project_config.billing_account_id == null ? local.iam_trf : null services = concat(var.project_services, [ "bigquery.googleapis.com", "bigqueryreservation.googleapis.com", diff --git a/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf b/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf index d22cf0aa52..67c43daefe 100644 --- a/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf +++ b/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf @@ -15,54 +15,48 @@ # tfdoc:file:description Data Warehouse projects. locals { - dwh_group_iam = { - (local.groups.data-engineers) = [ - "roles/bigquery.dataEditor", - "roles/storage.admin", - ], - (local.groups.data-analysts) = [ - "roles/bigquery.dataViewer", - "roles/bigquery.jobUser", - "roles/bigquery.metadataViewer", - "roles/bigquery.user", - "roles/datacatalog.viewer", - "roles/datacatalog.tagTemplateViewer", - "roles/storage.objectViewer", - ] - } dwh_lnd_iam = { "roles/bigquery.dataOwner" = [ module.load-sa-df-0.iam_email, + ] + "roles/bigquery.dataViewer" = [ module.transf-sa-df-0.iam_email, module.transf-sa-bq-0.iam_email, + local.groups_iam.data-engineers ] "roles/bigquery.jobUser" = [ - module.load-sa-df-0.iam_email, - ] - "roles/datacatalog.categoryAdmin" = [ - module.transf-sa-bq-0.iam_email - ] - "roles/storage.objectCreator" = [ - module.load-sa-df-0.iam_email, + module.load-sa-df-0.iam_email, local.groups_iam.data-engineers ] + "roles/datacatalog.categoryAdmin" = [module.transf-sa-bq-0.iam_email] + "roles/datacatalog.tagTemplateViewer" = [local.groups_iam.data-engineers] + "roles/datacatalog.viewer" = [local.groups_iam.data-engineers] + "roles/storage.objectCreator" = [module.load-sa-df-0.iam_email] + "roles/storage.objectViewer" = [local.groups_iam.data-engineers] } dwh_iam = { "roles/bigquery.dataOwner" = [ module.transf-sa-df-0.iam_email, module.transf-sa-bq-0.iam_email, ] + "roles/bigquery.dataViewer" = [ + local.groups_iam.data-analysts, + local.groups_iam.data-engineers + ] "roles/bigquery.jobUser" = [ module.transf-sa-bq-0.iam_email, + local.groups_iam.data-analysts, + local.groups_iam.data-engineers ] - "roles/datacatalog.categoryAdmin" = [ - module.load-sa-df-0.iam_email + "roles/datacatalog.tagTemplateViewer" = [ + local.groups_iam.data-analysts, local.groups_iam.data-engineers ] - "roles/storage.objectCreator" = [ - module.transf-sa-df-0.iam_email, + "roles/datacatalog.viewer" = [ + local.groups_iam.data-analysts, local.groups_iam.data-engineers ] "roles/storage.objectViewer" = [ - module.transf-sa-df-0.iam_email, + local.groups_iam.data-analysts, local.groups_iam.data-engineers ] + "roles/storage.objectAdmin" = [module.transf-sa-df-0.iam_email] } dwh_services = concat(var.project_services, [ "bigquery.googleapis.com", @@ -87,10 +81,9 @@ module "dwh-lnd-project" { project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-lnd : "${var.project_config.project_ids.dwh-lnd}${local.project_suffix}" - # group_iam = local.dwh_group_iam - iam = var.project_config.billing_account_id != null ? local.dwh_lnd_iam : {} - iam_additive = var.project_config.billing_account_id == null ? local.dwh_lnd_iam : {} - services = local.dwh_services + iam = var.project_config.billing_account_id != null ? local.dwh_lnd_iam : {} + iam_additive = var.project_config.billing_account_id == null ? local.dwh_lnd_iam : {} + services = local.dwh_services service_encryption_key_ids = { bq = [try(local.service_encryption_keys.bq, null)] storage = [try(local.service_encryption_keys.storage, null)] @@ -104,10 +97,9 @@ module "dwh-cur-project" { project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-cur : "${var.project_config.project_ids.dwh-cur}${local.project_suffix}" - # group_iam = local.dwh_group_iam - iam = var.project_config.billing_account_id != null ? local.dwh_iam : {} - iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : {} - services = local.dwh_services + iam = var.project_config.billing_account_id != null ? local.dwh_iam : {} + iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : {} + services = local.dwh_services service_encryption_key_ids = { bq = [try(local.service_encryption_keys.bq, null)] storage = [try(local.service_encryption_keys.storage, null)] @@ -121,10 +113,9 @@ module "dwh-conf-project" { project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-conf : "${var.project_config.project_ids.dwh-conf}${local.project_suffix}" - # group_iam = local.dwh_group_iam - iam = var.project_config.billing_account_id != null ? local.dwh_iam : null - iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : null - services = local.dwh_services + iam = var.project_config.billing_account_id != null ? local.dwh_iam : null + iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : null + services = local.dwh_services service_encryption_key_ids = { bq = [try(local.service_encryption_keys.bq, null)] storage = [try(local.service_encryption_keys.storage, null)] diff --git a/blueprints/data-solutions/data-platform-foundations/06-common.tf b/blueprints/data-solutions/data-platform-foundations/06-common.tf index 059d6b5eb7..5a84ee777e 100644 --- a/blueprints/data-solutions/data-platform-foundations/06-common.tf +++ b/blueprints/data-solutions/data-platform-foundations/06-common.tf @@ -15,29 +15,21 @@ # tfdoc:file:description common project. locals { - group_iam_common = { - (local.groups.data-analysts) = [ - "roles/datacatalog.viewer", - ] - (local.groups.data-engineers) = [ - "roles/dlp.reader", - "roles/dlp.user", - "roles/dlp.estimatesAdmin", - ] - (local.groups.data-security) = [ - "roles/dlp.admin", - "roles/datacatalog.admin" - ] - } iam_common = { + "roles/dlp.admin" = [local.groups_iam.data-security] + "roles/dlp.estimatesAdmin" = [local.groups_iam.data-engineers] + "roles/dlp.reader" = [local.groups_iam.data-engineers] "roles/dlp.user" = [ module.load-sa-df-0.iam_email, - module.transf-sa-df-0.iam_email + module.transf-sa-df-0.iam_email, + local.groups_iam.data-engineers ] + "roles/datacatalog.admin" = [local.groups_iam.data-security] "roles/datacatalog.viewer" = [ module.load-sa-df-0.iam_email, module.transf-sa-df-0.iam_email, - module.transf-sa-bq-0.iam_email + module.transf-sa-bq-0.iam_email, + local.groups_iam.data-analysts ] "roles/datacatalog.categoryFineGrainedReader" = [ module.transf-sa-df-0.iam_email, @@ -54,9 +46,8 @@ module "common-project" { project_create = var.project_config.billing_account_id != null prefix = var.project_config.billing_account_id == null ? null : var.prefix name = var.project_config.billing_account_id == null ? var.project_config.project_ids.common : "${var.project_config.project_ids.common}${local.project_suffix}" - # group_iam = local.group_iam_common - iam = var.project_config.billing_account_id != null ? local.iam_common : null - iam_additive = var.project_config.billing_account_id == null ? local.iam_common : null + iam = var.project_config.billing_account_id != null ? local.iam_common : null + iam_additive = var.project_config.billing_account_id == null ? local.iam_common : null services = concat(var.project_services, [ "datacatalog.googleapis.com", "dlp.googleapis.com", diff --git a/blueprints/data-solutions/data-platform-foundations/README.md b/blueprints/data-solutions/data-platform-foundations/README.md index 027c62990c..d48ae6cce6 100644 --- a/blueprints/data-solutions/data-platform-foundations/README.md +++ b/blueprints/data-solutions/data-platform-foundations/README.md @@ -215,13 +215,13 @@ module "data-platform" { source = "./fabric/blueprints/data-solutions/data-platform-foundations" organization_domain = "example.com" project_config = { - billing_account_id = "123456-123456-123456" - parent = "folders/12345678" - } - prefix = "myprefix" + billing_account_id = "123456-123456-123456" + parent = "folders/12345678" + } + prefix = "myprefix" } -# tftest modules=43 resources=265 +# tftest modules=43 resources=278 ``` ## Customizations diff --git a/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py b/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py index 4b15eaaba5..86b8e5bbe6 100644 --- a/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py +++ b/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py @@ -123,7 +123,7 @@ task_id="upsert_table_customers", project_id=DWH_LAND_PRJ, dataset_id=DWH_LAND_BQ_DATASET, - impersonation_chain=[TRF_SA_DF], + impersonation_chain=[LOD_SA_DF], table_resource={ "tableReference": {"tableId": "customers"}, }, @@ -133,7 +133,7 @@ task_id="upsert_table_purchases", project_id=DWH_LAND_PRJ, dataset_id=DWH_LAND_BQ_DATASET, - impersonation_chain=[TRF_SA_BQ], + impersonation_chain=[LOD_SA_DF], table_resource={ "tableReference": {"tableId": "purchases"} }, @@ -167,7 +167,7 @@ project_id=DWH_LAND_PRJ, dataset_id=DWH_LAND_BQ_DATASET, table_id="customers", - impersonation_chain=[TRF_SA_BQ], + impersonation_chain=[LOD_SA_DF], include_policy_tags=True, schema_fields_updates=[ { "mode": "REQUIRED", "name": "id", "type": "INTEGER", "description": "ID" }, @@ -182,7 +182,7 @@ project_id=DWH_LAND_PRJ, dataset_id=DWH_LAND_BQ_DATASET, table_id="purchases", - impersonation_chain=[TRF_SA_BQ], + impersonation_chain=[LOD_SA_DF], include_policy_tags=True, schema_fields_updates=[ { "mode": "REQUIRED", "name": "id", "type": "INTEGER", "description": "ID" }, diff --git a/blueprints/data-solutions/data-platform-foundations/demo/delete_table.py b/blueprints/data-solutions/data-platform-foundations/demo/delete_table.py index dc0c954b14..bade038871 100644 --- a/blueprints/data-solutions/data-platform-foundations/demo/delete_table.py +++ b/blueprints/data-solutions/data-platform-foundations/demo/delete_table.py @@ -122,13 +122,13 @@ delete_table_customers = BigQueryDeleteTableOperator( task_id="delete_table_customers", deletion_dataset_table=DWH_LAND_PRJ+"."+DWH_LAND_BQ_DATASET+".customers", - impersonation_chain=[TRF_SA_DF] + impersonation_chain=[LOD_SA_DF] ) delete_table_purchases = BigQueryDeleteTableOperator( task_id="delete_table_purchases", deletion_dataset_table=DWH_LAND_PRJ+"."+DWH_LAND_BQ_DATASET+".purchases", - impersonation_chain=[TRF_SA_DF] + impersonation_chain=[LOD_SA_DF] ) delete_table_customer_purchase_curated = BigQueryDeleteTableOperator( diff --git a/fast/stages/3-data-platform/dev/variables.tf b/fast/stages/3-data-platform/dev/variables.tf index 392e2dc9e8..74a5dbe11d 100644 --- a/fast/stages/3-data-platform/dev/variables.tf +++ b/fast/stages/3-data-platform/dev/variables.tf @@ -22,6 +22,19 @@ variable "automation" { }) } +variable "billing_account" { + # tfdoc:variable:source 0-bootstrap + description = "Billing account id. If billing account is not part of the same org set `is_org_level` to false." + type = object({ + id = string + is_org_level = optional(bool, true) + }) + validation { + condition = var.billing_account.is_org_level != null + error_message = "Invalid `null` value for `billing_account.is_org_level`." + } +} + variable "composer_config" { description = "Cloud Composer configuration options." type = object({ @@ -86,6 +99,14 @@ variable "data_force_destroy" { default = false } +variable "folder_ids" { + # tfdoc:variable:source 1-resman + description = "Folder to be used for the networking resources in folders/nnnn format." + type = object({ + data-platform-dev = string + }) +} + variable "groups" { description = "Groups." type = map(string) @@ -148,14 +169,6 @@ variable "prefix" { type = string } -variable "project_config" { - description = "Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format." - type = object({ - billing_account_id = string - parent = string - }) -} - variable "project_services" { description = "List of core services enabled on all projects." type = list(string) diff --git a/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py b/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py index f3ed2ba008..630944f26d 100644 --- a/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py +++ b/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py @@ -23,4 +23,4 @@ def test_resources(e2e_plan_runner): modules, resources = e2e_plan_runner(FIXTURES_DIR) assert len(modules) == 42 - assert len(resources) == 264 + assert len(resources) == 277 From ac75cbe71ac2ae57162319d16cc557e468581126 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Wed, 22 Feb 2023 01:38:44 +0100 Subject: [PATCH 11/13] Fix lint. --- fast/stages/3-data-platform/dev/README.md | 33 ++++++++++++----------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/fast/stages/3-data-platform/dev/README.md b/fast/stages/3-data-platform/dev/README.md index f29903106e..48d09eafc0 100644 --- a/fast/stages/3-data-platform/dev/README.md +++ b/fast/stages/3-data-platform/dev/README.md @@ -185,22 +185,23 @@ You can find examples in the `[demo](../../../../blueprints/data-solutions/data- | name | description | type | required | default | producer | |---|---|:---:|:---:|:---:|:---:| | [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 0-bootstrap | -| [host_project_ids](variables.tf#L99) | Shared VPC project ids. | object({…}) | ✓ | | 2-networking | -| [organization](variables.tf#L129) | Organization details. | object({…}) | ✓ | | 00-globals | -| [prefix](variables.tf#L145) | Unique prefix used for resource names. Not used for projects if 'project_create' is null. | string | ✓ | | 00-globals | -| [project_config](variables.tf#L151) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | | | -| [composer_config](variables.tf#L25) | Cloud Composer configuration options. | object({…}) | | {…} | | -| [data_catalog_tags](variables.tf#L72) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | | -| [data_force_destroy](variables.tf#L83) | Flag to set 'force_destroy' on data services like BigQery or Cloud Storage. | bool | | false | | -| [groups](variables.tf#L89) | Groups. | map(string) | | {…} | | -| [location](variables.tf#L107) | Location used for multi-regional resources. | string | | "eu" | | -| [network_config_composer](variables.tf#L113) | Network configurations to use for Composer. | object({…}) | | {…} | | -| [outputs_location](variables.tf#L139) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | -| [project_services](variables.tf#L159) | List of core services enabled on all projects. | list(string) | | […] | | -| [region](variables.tf#L170) | Region used for regional resources. | string | | "europe-west1" | | -| [service_encryption_keys](variables.tf#L176) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | | -| [subnet_self_links](variables.tf#L188) | Shared VPC subnet self links. | object({…}) | | null | 2-networking | -| [vpc_self_links](variables.tf#L197) | Shared VPC self links. | object({…}) | | null | 2-networking | +| [billing_account](variables.tf#L25) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | object({…}) | ✓ | | 0-bootstrap | +| [folder_ids](variables.tf#L102) | Folder to be used for the networking resources in folders/nnnn format. | object({…}) | ✓ | | 1-resman | +| [host_project_ids](variables.tf#L120) | Shared VPC project ids. | object({…}) | ✓ | | 2-networking | +| [organization](variables.tf#L150) | Organization details. | object({…}) | ✓ | | 00-globals | +| [prefix](variables.tf#L166) | Unique prefix used for resource names. Not used for projects if 'project_create' is null. | string | ✓ | | 00-globals | +| [composer_config](variables.tf#L38) | Cloud Composer configuration options. | object({…}) | | {…} | | +| [data_catalog_tags](variables.tf#L85) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | | +| [data_force_destroy](variables.tf#L96) | Flag to set 'force_destroy' on data services like BigQery or Cloud Storage. | bool | | false | | +| [groups](variables.tf#L110) | Groups. | map(string) | | {…} | | +| [location](variables.tf#L128) | Location used for multi-regional resources. | string | | "eu" | | +| [network_config_composer](variables.tf#L134) | Network configurations to use for Composer. | object({…}) | | {…} | | +| [outputs_location](variables.tf#L160) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | +| [project_services](variables.tf#L172) | List of core services enabled on all projects. | list(string) | | […] | | +| [region](variables.tf#L183) | Region used for regional resources. | string | | "europe-west1" | | +| [service_encryption_keys](variables.tf#L189) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | | +| [subnet_self_links](variables.tf#L201) | Shared VPC subnet self links. | object({…}) | | null | 2-networking | +| [vpc_self_links](variables.tf#L210) | Shared VPC self links. | object({…}) | | null | 2-networking | ## Outputs From e39be7b01d18f19e509cd82acf5fd5c026a031b2 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Wed, 22 Feb 2023 13:02:29 +0100 Subject: [PATCH 12/13] Fix --- tests/fast/stages/s3_data_platform/common.tfvars | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/fast/stages/s3_data_platform/common.tfvars b/tests/fast/stages/s3_data_platform/common.tfvars index 97d8bebc3e..2ec41d37ad 100644 --- a/tests/fast/stages/s3_data_platform/common.tfvars +++ b/tests/fast/stages/s3_data_platform/common.tfvars @@ -1,13 +1,11 @@ automation = { outputs_bucket = "test" } -project_config = { - billing_account = { - id = "012345-67890A-BCDEF0", - }, - parent = { - data-platform-dev = "folders/12345678" - } +billing_account = { + id = "012345-67890A-BCDEF0", +} +folder_ids = { + data-platform-dev = "folders/12345678" } host_project_ids = { dev-spoke-0 = "fast-dev-net-spoke-0" From b279c083a0bfdd3abb2b90b717609d32ece9caca Mon Sep 17 00:00:00 2001 From: lcaggio Date: Thu, 23 Feb 2023 11:54:16 +0100 Subject: [PATCH 13/13] Fix README and IAM files --- .../data-platform-foundations/IAM.md | 29 +++++++++---------- .../data-platform-foundations/README.md | 6 +++- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/blueprints/data-solutions/data-platform-foundations/IAM.md b/blueprints/data-solutions/data-platform-foundations/IAM.md index dd898bd750..b982f8c4ee 100644 --- a/blueprints/data-solutions/data-platform-foundations/IAM.md +++ b/blueprints/data-solutions/data-platform-foundations/IAM.md @@ -17,51 +17,48 @@ Legend: + additive, • conditional. | members | roles | |---|---| -|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/pubsub.editor](https://cloud.google.com/iam/docs/understanding-roles#pubsub.editor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) | +|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user) | |drp-bq-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) | |drp-cs-0
serviceAccount|[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) | |drp-ps-0
serviceAccount|[roles/pubsub.publisher](https://cloud.google.com/iam/docs/understanding-roles#pubsub.publisher) | -|load-df-0
serviceAccount|[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | +|load-df-0
serviceAccount|[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | |orc-cmp-0
serviceAccount|[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | ## Project dwh-conf | members | roles | |---|---| -|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | -|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) | +|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | +|gcp-data-engineers
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | |SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +| -|load-df-0
serviceAccount|[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) | |trf-bq-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) | -|trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | +|trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | ## Project dwh-cur | members | roles | |---|---| -|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | -|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) | +|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | +|gcp-data-engineers
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | |SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +| -|load-df-0
serviceAccount|[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) | |trf-bq-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) | -|trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | +|trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | ## Project dwh-lnd | members | roles | |---|---| -|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | -|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) | +|gcp-data-engineers
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | |SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +| |load-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) | -|trf-bq-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) | -|trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) | +|trf-bq-0
serviceAccount|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) | +|trf-df-0
serviceAccount|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) | ## Project lod | members | roles | |---|---| -|gcp-data-engineers
group|[roles/compute.viewer](https://cloud.google.com/iam/docs/understanding-roles#compute.viewer)
[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin)
[roles/dataflow.developer](https://cloud.google.com/iam/docs/understanding-roles#dataflow.developer)
[roles/viewer](https://cloud.google.com/iam/docs/understanding-roles#viewer) | +|gcp-data-engineers
group|[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin)
[roles/dataflow.developer](https://cloud.google.com/iam/docs/understanding-roles#dataflow.developer) | |SERVICE_IDENTITY_dataflow-service-producer-prod
serviceAccount|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | |SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +| |load-df-0
serviceAccount|[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin)
[roles/dataflow.worker](https://cloud.google.com/iam/docs/understanding-roles#dataflow.worker)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | @@ -71,7 +68,7 @@ Legend: + additive, • conditional. | members | roles | |---|---| -|gcp-data-engineers
group|[roles/artifactregistry.admin](https://cloud.google.com/iam/docs/understanding-roles#artifactregistry.admin)
[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/cloudbuild.builds.editor](https://cloud.google.com/iam/docs/understanding-roles#cloudbuild.builds.editor)
[roles/composer.admin](https://cloud.google.com/iam/docs/understanding-roles#composer.admin)
[roles/composer.environmentAndStorageObjectAdmin](https://cloud.google.com/iam/docs/understanding-roles#composer.environmentAndStorageObjectAdmin)
[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser)
[roles/iap.httpsResourceAccessor](https://cloud.google.com/iam/docs/understanding-roles#iap.httpsResourceAccessor)
[roles/serviceusage.serviceUsageConsumer](https://cloud.google.com/iam/docs/understanding-roles#serviceusage.serviceUsageConsumer)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | +|gcp-data-engineers
group|[roles/artifactregistry.admin](https://cloud.google.com/iam/docs/understanding-roles#artifactregistry.admin)
[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/cloudbuild.builds.editor](https://cloud.google.com/iam/docs/understanding-roles#cloudbuild.builds.editor)
[roles/composer.admin](https://cloud.google.com/iam/docs/understanding-roles#composer.admin)
[roles/composer.environmentAndStorageObjectAdmin](https://cloud.google.com/iam/docs/understanding-roles#composer.environmentAndStorageObjectAdmin)
[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser)
[roles/iap.httpsResourceAccessor](https://cloud.google.com/iam/docs/understanding-roles#iap.httpsResourceAccessor)
[roles/serviceusage.serviceUsageConsumer](https://cloud.google.com/iam/docs/understanding-roles#serviceusage.serviceUsageConsumer)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | |SERVICE_IDENTITY_cloudcomposer-accounts
serviceAccount|[roles/composer.ServiceAgentV2Ext](https://cloud.google.com/iam/docs/understanding-roles#composer.ServiceAgentV2Ext)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | |SERVICE_IDENTITY_gcp-sa-cloudbuild
serviceAccount|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | |SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +| diff --git a/blueprints/data-solutions/data-platform-foundations/README.md b/blueprints/data-solutions/data-platform-foundations/README.md index d48ae6cce6..ad08721692 100644 --- a/blueprints/data-solutions/data-platform-foundations/README.md +++ b/blueprints/data-solutions/data-platform-foundations/README.md @@ -237,7 +237,11 @@ To do this, you need to remove IAM binging at project-level for the `data-analys ### Project Configuration -The solution can be deployed creating projects on a given parent (organization or folder) or on existing projects. Configure variable `project_config` accordingly. +The solution can be deployed by creating projects on a given parent (organization or folder) or on existing projects. Configure variable `project_config` accordingly. + +When you rely on existing projects, the blueprint is designed to rely on different projects configuring IAM binding with an additive approach. For discovery or experimentation purposes, you may also configure `project_config.project_ids` to point different projects to one project with the granularity you need. For example, deploy resources from the 'load' project with resources in the 'transformation' project. + +Once you have identified the required project granularity for your use case, we suggest adapting the terraform script accordingly and relying on authoritative IAM binding. ## Demo pipeline