Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DataPlatform: Support project creation #1165

Merged
merged 14 commits into from
Feb 23, 2023
Merged

DataPlatform: Support project creation #1165

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
ad6667a
First commit
lcaggio Feb 19, 2023
c8f2551
Fix variabler to support existing projects.
lcaggio Feb 19, 2023
db6a4f9
Remove variables
lcaggio Feb 19, 2023
f4c1fa6
Fix tests.
lcaggio Feb 19, 2023
c523dce
Fix
lcaggio Feb 20, 2023
eda9597
Fix linting
lcaggio Feb 20, 2023
63a81a9
Fix Fast test
lcaggio Feb 20, 2023
970b8ff
Fix DP Fast variables.
lcaggio Feb 20, 2023
2564c9b
Fix README
lcaggio Feb 20, 2023
2108b46
Fix Tests, rely on iam additive.
lcaggio Feb 22, 2023
ac75cbe
Fix lint.
lcaggio Feb 22, 2023
e39be7b
Fix
lcaggio Feb 22, 2023
b279c08
Fix README and IAM files
lcaggio Feb 23, 2023
47855cb
Merge branch 'master' into lcaggio/dp-projectcreate
lcaggio Feb 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 24 additions & 26 deletions blueprints/data-solutions/data-platform-foundations/01-dropoff.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,36 +15,34 @@
# tfdoc:file:description drop off project and resources.

locals {
drop_orch_service_accounts = [
module.load-sa-df-0.iam_email, module.orch-sa-cmp-0.iam_email
]
}

module "drop-project" {
source = "../../../modules/project"
parent = var.folder_id
billing_account = var.billing_account_id
prefix = var.prefix
name = "drp${local.project_suffix}"
group_iam = {
(local.groups.data-engineers) = [
"roles/bigquery.dataEditor",
"roles/pubsub.editor",
"roles/storage.admin",
iam_drp = {
"roles/bigquery.dataEditor" = [
module.drop-sa-bq-0.iam_email, local.groups_iam.data-engineers
]
"roles/bigquery.user" = [
module.load-sa-df-0.iam_email, local.groups_iam.data-engineers
]
"roles/pubsub.publisher" = [module.drop-sa-ps-0.iam_email]
"roles/pubsub.subscriber" = [
module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email
]
}
iam = {
"roles/bigquery.dataEditor" = [module.drop-sa-bq-0.iam_email]
"roles/bigquery.user" = [module.load-sa-df-0.iam_email]
"roles/pubsub.publisher" = [module.drop-sa-ps-0.iam_email]
"roles/pubsub.subscriber" = concat(
local.drop_orch_service_accounts, [module.load-sa-df-0.iam_email]
)
"roles/storage.objectAdmin" = [module.load-sa-df-0.iam_email]
"roles/storage.objectCreator" = [module.drop-sa-cs-0.iam_email]
"roles/storage.objectViewer" = [module.orch-sa-cmp-0.iam_email]
"roles/storage.admin" = [module.load-sa-df-0.iam_email]
"roles/storage.objectAdmin" = [
module.load-sa-df-0.iam_email, module.load-sa-df-0.iam_email
]
}
}

module "drop-project" {
source = "../../../modules/project"
parent = var.project_config.parent
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.drop : "${var.project_config.project_ids.drop}${local.project_suffix}"
iam = var.project_config.billing_account_id != null ? local.iam_drp : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_drp : null
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
Expand Down
55 changes: 30 additions & 25 deletions blueprints/data-solutions/data-platform-foundations/02-load.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,19 @@
# tfdoc:file:description Load project and VPC.

locals {
iam_load = {
"roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email]
"roles/dataflow.admin" = [
module.orch-sa-cmp-0.iam_email,
module.load-sa-df-0.iam_email,
local.groups_iam.data-engineers
]
"roles/dataflow.developer" = [
local.groups_iam.data-engineers
]
"roles/dataflow.worker" = [module.load-sa-df-0.iam_email]
"roles/storage.objectAdmin" = local.load_service_accounts
}
load_service_accounts = [
"serviceAccount:${module.load-project.service_accounts.robots.dataflow}",
module.load-sa-df-0.iam_email
Expand All @@ -35,26 +48,13 @@ locals {

module "load-project" {
source = "../../../modules/project"
parent = var.folder_id
billing_account = var.billing_account_id
prefix = var.prefix
name = "lod${local.project_suffix}"
group_iam = {
(local.groups.data-engineers) = [
"roles/compute.viewer",
"roles/dataflow.admin",
"roles/dataflow.developer",
"roles/viewer",
]
}
iam = {
"roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email]
"roles/dataflow.admin" = [
module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email
]
"roles/dataflow.worker" = [module.load-sa-df-0.iam_email]
"roles/storage.objectAdmin" = local.load_service_accounts
}
parent = var.project_config.parent
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.load : "${var.project_config.project_ids.load}${local.project_suffix}"
iam = var.project_config.billing_account_id != null ? local.iam_load : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_load : null
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
Expand Down Expand Up @@ -86,8 +86,13 @@ module "load-sa-df-0" {
name = "load-df-0"
display_name = "Data platform Dataflow load service account"
iam = {
"roles/iam.serviceAccountTokenCreator" = [local.groups_iam.data-engineers]
"roles/iam.serviceAccountUser" = [module.orch-sa-cmp-0.iam_email]
"roles/iam.serviceAccountTokenCreator" = [
local.groups_iam.data-engineers,
module.orch-sa-cmp-0.iam_email
],
"roles/iam.serviceAccountUser" = [
module.orch-sa-cmp-0.iam_email
]
}
}

Expand All @@ -107,11 +112,11 @@ module "load-vpc" {
source = "../../../modules/net-vpc"
count = local.use_shared_vpc ? 0 : 1
project_id = module.load-project.project_id
name = "${var.prefix}-default"
name = "${var.prefix}-lod"
subnets = [
{
ip_cidr_range = "10.10.0.0/24"
name = "default"
name = "${var.prefix}-lod"
region = var.region
}
]
Expand All @@ -131,7 +136,7 @@ module "load-nat" {
source = "../../../modules/net-cloudnat"
count = local.use_shared_vpc ? 0 : 1
project_id = module.load-project.project_id
name = "${var.prefix}-default"
name = "${var.prefix}-lod"
region = var.region
router_network = module.load-vpc.0.name
}
93 changes: 43 additions & 50 deletions blueprints/data-solutions/data-platform-foundations/03-orchestration.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,76 +15,69 @@
# tfdoc:file:description Orchestration project and VPC.

locals {
orch_subnet = (
local.use_shared_vpc
? var.network_config.subnet_self_links.orchestration
: values(module.orch-vpc.0.subnet_self_links)[0]
)
orch_vpc = (
local.use_shared_vpc
? var.network_config.network_self_link
: module.orch-vpc.0.self_link
)

# Note: This formatting is needed for output purposes since the fabric artifact registry
# module doesn't yet expose the docker usage path of a registry folder in the needed format.
orch_docker_path = format("%s-docker.pkg.dev/%s/%s",
var.region, module.orch-project.project_id, module.orch-artifact-reg.name)
}

module "orch-project" {
source = "../../../modules/project"
parent = var.folder_id
billing_account = var.billing_account_id
prefix = var.prefix
name = "orc${local.project_suffix}"
group_iam = {
(local.groups.data-engineers) = [
"roles/bigquery.dataEditor",
"roles/bigquery.jobUser",
"roles/cloudbuild.builds.editor",
"roles/composer.admin",
"roles/composer.environmentAndStorageObjectAdmin",
"roles/iap.httpsResourceAccessor",
"roles/iam.serviceAccountUser",
"roles/storage.objectAdmin",
"roles/storage.admin",
"roles/artifactregistry.admin",
"roles/serviceusage.serviceUsageConsumer",
]
}
iam = {
iam_orch = {
"roles/artifactregistry.admin" = [local.groups_iam.data-engineers]
"roles/artifactregistry.reader" = [module.load-sa-df-0.iam_email]
"roles/bigquery.dataEditor" = [
module.load-sa-df-0.iam_email,
module.transf-sa-df-0.iam_email,
local.groups_iam.data-engineers
]
"roles/bigquery.jobUser" = [
module.orch-sa-cmp-0.iam_email,
local.groups_iam.data-engineers
]
"roles/cloudbuild.builds.editor" = [local.groups_iam.data-engineers]
"roles/cloudbuild.serviceAgent" = [module.orch-sa-df-build.iam_email]
"roles/composer.admin" = [local.groups_iam.data-engineers]
"roles/composer.environmentAndStorageObjectAdmin" = [local.groups_iam.data-engineers]
"roles/composer.ServiceAgentV2Ext" = [
"serviceAccount:${module.orch-project.service_accounts.robots.composer}"
]
"roles/composer.worker" = [
module.orch-sa-cmp-0.iam_email
]
"roles/iam.serviceAccountUser" = [
module.orch-sa-cmp-0.iam_email
module.orch-sa-cmp-0.iam_email, local.groups_iam.data-engineers
]
"roles/iap.httpsResourceAccessor" = [local.groups_iam.data-engineers]
"roles/serviceusage.serviceUsageConsumer" = [local.groups_iam.data-engineers]
"roles/storage.objectAdmin" = [
module.orch-sa-cmp-0.iam_email,
module.orch-sa-df-build.iam_email,
"serviceAccount:${module.orch-project.service_accounts.robots.composer}",
"serviceAccount:${module.orch-project.service_accounts.robots.cloudbuild}",
]
"roles/artifactregistry.reader" = [
module.load-sa-df-0.iam_email,
]
"roles/cloudbuild.serviceAgent" = [
module.orch-sa-df-build.iam_email,
local.groups_iam.data-engineers
]
"roles/storage.objectViewer" = [module.load-sa-df-0.iam_email]
}
oslogin = false
orch_subnet = (
local.use_shared_vpc
? var.network_config.subnet_self_links.orchestration
: values(module.orch-vpc.0.subnet_self_links)[0]
)
orch_vpc = (
local.use_shared_vpc
? var.network_config.network_self_link
: module.orch-vpc.0.self_link
)

# Note: This formatting is needed for output purposes since the fabric artifact registry
# module doesn't yet expose the docker usage path of a registry folder in the needed format.
orch_docker_path = format("%s-docker.pkg.dev/%s/%s",
var.region, module.orch-project.project_id, module.orch-artifact-reg.name)
}

module "orch-project" {
source = "../../../modules/project"
parent = var.project_config.parent
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.orc : "${var.project_config.project_ids.orc}${local.project_suffix}"
iam = var.project_config.billing_account_id != null ? local.iam_orch : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_orch : null
oslogin = false
services = concat(var.project_services, [
"artifactregistry.googleapis.com",
"bigquery.googleapis.com",
Expand Down Expand Up @@ -132,11 +125,11 @@ module "orch-vpc" {
source = "../../../modules/net-vpc"
count = local.use_shared_vpc ? 0 : 1
project_id = module.orch-project.project_id
name = "${var.prefix}-default"
name = "${var.prefix}-orch"
subnets = [
{
ip_cidr_range = "10.10.0.0/24"
name = "default"
name = "${var.prefix}-orch"
region = var.region
secondary_ip_ranges = {
pods = "10.10.8.0/22"
Expand All @@ -160,7 +153,7 @@ module "orch-nat" {
count = local.use_shared_vpc ? 0 : 1
source = "../../../modules/net-cloudnat"
project_id = module.orch-project.project_id
name = "${var.prefix}-default"
name = "${var.prefix}-orch"
region = var.region
router_network = module.orch-vpc.0.name
}
Expand Down
51 changes: 23 additions & 28 deletions blueprints/data-solutions/data-platform-foundations/04-transformation.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,19 @@
# tfdoc:file:description Trasformation project and VPC.

locals {
iam_trf = {
"roles/bigquery.jobUser" = [
module.transf-sa-bq-0.iam_email, local.groups_iam.data-engineers
]
"roles/dataflow.admin" = [
module.orch-sa-cmp-0.iam_email, local.groups_iam.data-engineers
]
"roles/dataflow.worker" = [module.transf-sa-df-0.iam_email]
"roles/storage.objectAdmin" = [
module.transf-sa-df-0.iam_email,
"serviceAccount:${module.transf-project.service_accounts.robots.dataflow}"
]
}
transf_subnet = (
local.use_shared_vpc
? var.network_config.subnet_self_links.orchestration
Expand All @@ -29,31 +42,13 @@ locals {

module "transf-project" {
source = "../../../modules/project"
parent = var.folder_id
billing_account = var.billing_account_id
prefix = var.prefix
name = "trf${local.project_suffix}"
group_iam = {
(local.groups.data-engineers) = [
"roles/bigquery.jobUser",
"roles/dataflow.admin",
]
}
iam = {
"roles/bigquery.jobUser" = [
module.transf-sa-bq-0.iam_email,
]
"roles/dataflow.admin" = [
module.orch-sa-cmp-0.iam_email,
]
"roles/dataflow.worker" = [
module.transf-sa-df-0.iam_email
]
"roles/storage.objectAdmin" = [
module.transf-sa-df-0.iam_email,
"serviceAccount:${module.transf-project.service_accounts.robots.dataflow}"
]
}
parent = var.project_config.parent
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.trf : "${var.project_config.project_ids.trf}${local.project_suffix}"
iam = var.project_config.billing_account_id != null ? local.iam_trf : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_trf : null
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
Expand Down Expand Up @@ -131,11 +126,11 @@ module "transf-vpc" {
source = "../../../modules/net-vpc"
count = local.use_shared_vpc ? 0 : 1
project_id = module.transf-project.project_id
name = "${var.prefix}-default"
name = "${var.prefix}-trf"
subnets = [
{
ip_cidr_range = "10.10.0.0/24"
name = "default"
name = "${var.prefix}-trf"
region = var.region
}
]
Expand All @@ -155,7 +150,7 @@ module "transf-nat" {
source = "../../../modules/net-cloudnat"
count = local.use_shared_vpc ? 0 : 1
project_id = module.transf-project.project_id
name = "${var.prefix}-default"
name = "${var.prefix}-trf"
region = var.region
router_network = module.transf-vpc.0.name
}
Loading