From 1fe57bf6c88264fd2d28fc60442f759719f96436 Mon Sep 17 00:00:00 2001
From: Justin Brooks <jsbroks@gmail.com>
Date: Wed, 14 Feb 2024 13:43:40 -0500
Subject: [PATCH 1/8] msk branch

---
 modules/msk/main.tf      | 47 ++++++++++++++++++++++++++++++++++++++++
 modules/msk/variables.tf |  3 +++
 2 files changed, 50 insertions(+)
 create mode 100644 modules/msk/main.tf
 create mode 100644 modules/msk/variables.tf

diff --git a/modules/msk/main.tf b/modules/msk/main.tf
new file mode 100644
index 00000000..5610b4a2
--- /dev/null
+++ b/modules/msk/main.tf
@@ -0,0 +1,47 @@
+resource "aws_security_group" "msk_brokers_sg" {
+  name        = "msk-brokers-sg"
+  vpc_id      = data.aws_vpc.existing_vpc.id
+  description = "Security group for MSK brokers"
+
+  # Restrict inbound traffic to only necessary ports from your VPC CIDR
+  ingress {
+    from_port   = 2181 # Zookeeper
+    to_port     = 2181
+    protocol    = "tcp"
+    cidr_blocks = [data.aws_vpc.existing_vpc.cidr_block] 
+  }
+
+  # Add more ingress rules as needed for monitoring, etc.
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = {
+    Name = "msk-brokers-sg"
+  }
+}
+
+resource "aws_msk_cluster" "default" {
+  cluster_name                   = "${var.namespace}"
+  kafka_version          = "3.4.0" # Choose your desired Kafka version
+  number_of_broker_nodes = 3
+
+  broker_node_group_info {
+    instance_type    = "kafka.m5.large"  # Adjust instance type as needed
+    client_subnets   = data.aws_subnets.private_subnets.ids
+    security_groups  = [aws_security_group.msk_brokers_sg.id]
+    # ebs_volume_size  = 50 # In GB
+  }
+
+  encryption_info {
+    encryption_in_transit {
+      client_broker = "TLS" 
+    }
+  }
+
+  depends_on = [aws_security_group.msk_brokers_sg] 
+}
\ No newline at end of file
diff --git a/modules/msk/variables.tf b/modules/msk/variables.tf
new file mode 100644
index 00000000..6857747f
--- /dev/null
+++ b/modules/msk/variables.tf
@@ -0,0 +1,3 @@
+variable "namespace" {
+  type = string
+}
\ No newline at end of file

From 9fe42fa9aa2376a9bdbd50ed74658ad91c669a86 Mon Sep 17 00:00:00 2001
From: Justin Brooks <jsbroks@gmail.com>
Date: Wed, 14 Feb 2024 14:03:44 -0500
Subject: [PATCH 2/8] setup auth

---
 examples/public-dns-external/custom.tf | 66 ++++++++++++++++++++++++++
 1 file changed, 66 insertions(+)
 create mode 100644 examples/public-dns-external/custom.tf

diff --git a/examples/public-dns-external/custom.tf b/examples/public-dns-external/custom.tf
new file mode 100644
index 00000000..bfec6786
--- /dev/null
+++ b/examples/public-dns-external/custom.tf
@@ -0,0 +1,66 @@
+locals {
+  infra_outputs           = data.terraform_remote_state.infra.outputs
+  gcp_credentials         = local.infra_outputs.deployments_credentials
+  aws_deployment_role_arn = local.infra_outputs.deployments_aws_role_arn
+  region = "us-west-1"
+}
+
+provider "aws" {
+  region     = local.region
+  access_key = module.aws_credentials.access_key
+  secret_key = module.aws_credentials.secret_key
+  token      = module.aws_credentials.token
+
+  default_tags {
+    tags = {
+      Owner          = "Deployer"
+      Namespace      = var.namespace
+    }
+  }
+}
+
+# Login using the deployment service account.
+provider "google" {
+  project     = "wandb-production"
+  region      = "us-central1"
+  zone        = "us-central1-c"
+  credentials = local.gcp_credentials
+}
+
+provider "kubernetes" {
+  host                   = data.aws_eks_cluster.app_cluster.endpoint
+  cluster_ca_certificate = base64decode(data.aws_eks_cluster.app_cluster.certificate_authority[0].data)
+  token                  = data.aws_eks_cluster_auth.app_cluster.token
+
+  # To ensure the Kubernetes provider is receiving valid credentials, an
+  # exec-based plugin can be used to fetch a new token before initializing the
+  # provider.
+  exec {
+    api_version = "client.authentication.k8s.io/v1"
+    args = [
+      "eks", "get-token",
+      "--cluster-name", module.wandb_infra.cluster_id,
+      "--region", local.region,
+      "--role-arn", local.aws_deployment_role_arn
+    ]
+    command = "aws"
+  }
+}
+
+provider "helm" {
+  kubernetes {
+    host                   = data.aws_eks_cluster.app_cluster.endpoint
+    cluster_ca_certificate = base64decode(data.aws_eks_cluster.app_cluster.certificate_authority[0].data)
+    token                  = data.aws_eks_cluster_auth.app_cluster.token
+  }
+}
+
+# Create AWS credentials from GCP account
+module "aws_credentials" {
+  source  = "wandb/assume-aws-role/google"
+  version = "1.1.0"
+
+  duration_seconds = 43200 # 12 hours
+  role_arn         = local.aws_deployment_role_arn
+  session_name     = "TerraformDeployment"
+}

From 01695950a135b3815cd3d49112e57528f0bc5348 Mon Sep 17 00:00:00 2001
From: Justin Brooks <jsbroks@gmail.com>
Date: Wed, 14 Feb 2024 14:04:36 -0500
Subject: [PATCH 3/8] add terraform state

---
 examples/public-dns-external/custom.tf | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/examples/public-dns-external/custom.tf b/examples/public-dns-external/custom.tf
index bfec6786..23bbd55f 100644
--- a/examples/public-dns-external/custom.tf
+++ b/examples/public-dns-external/custom.tf
@@ -5,6 +5,14 @@ locals {
   region = "us-west-1"
 }
 
+data "terraform_remote_state" "infra" {
+  backend = "remote"
+  config = {
+    organization = "weights-and-biases"
+    workspaces   = { name = "deployer-global" }
+  }
+}
+
 provider "aws" {
   region     = local.region
   access_key = module.aws_credentials.access_key

From 3eeb4055b17dfc4658d1089d2c29b75ff8d18bc4 Mon Sep 17 00:00:00 2001
From: Justin Brooks <jsbroks@gmail.com>
Date: Wed, 14 Feb 2024 14:06:41 -0500
Subject: [PATCH 4/8] clean up providers

---
 examples/public-dns-external/custom.tf | 28 --------------------------
 examples/public-dns-external/main.tf   | 13 ------------
 2 files changed, 41 deletions(-)

diff --git a/examples/public-dns-external/custom.tf b/examples/public-dns-external/custom.tf
index 23bbd55f..29bca04a 100644
--- a/examples/public-dns-external/custom.tf
+++ b/examples/public-dns-external/custom.tf
@@ -35,34 +35,6 @@ provider "google" {
   credentials = local.gcp_credentials
 }
 
-provider "kubernetes" {
-  host                   = data.aws_eks_cluster.app_cluster.endpoint
-  cluster_ca_certificate = base64decode(data.aws_eks_cluster.app_cluster.certificate_authority[0].data)
-  token                  = data.aws_eks_cluster_auth.app_cluster.token
-
-  # To ensure the Kubernetes provider is receiving valid credentials, an
-  # exec-based plugin can be used to fetch a new token before initializing the
-  # provider.
-  exec {
-    api_version = "client.authentication.k8s.io/v1"
-    args = [
-      "eks", "get-token",
-      "--cluster-name", module.wandb_infra.cluster_id,
-      "--region", local.region,
-      "--role-arn", local.aws_deployment_role_arn
-    ]
-    command = "aws"
-  }
-}
-
-provider "helm" {
-  kubernetes {
-    host                   = data.aws_eks_cluster.app_cluster.endpoint
-    cluster_ca_certificate = base64decode(data.aws_eks_cluster.app_cluster.certificate_authority[0].data)
-    token                  = data.aws_eks_cluster_auth.app_cluster.token
-  }
-}
-
 # Create AWS credentials from GCP account
 module "aws_credentials" {
   source  = "wandb/assume-aws-role/google"
diff --git a/examples/public-dns-external/main.tf b/examples/public-dns-external/main.tf
index ab506ada..2d39dc53 100644
--- a/examples/public-dns-external/main.tf
+++ b/examples/public-dns-external/main.tf
@@ -1,16 +1,3 @@
-provider "aws" {
-  region = "us-west-2"
-
-  default_tags {
-    tags = {
-      GithubRepo = "terraform-aws-wandb"
-      GithubOrg  = "wandb"
-      Enviroment = "Example"
-      Example    = "PublicDnsExternal"
-    }
-  }
-}
-
 module "wandb_infra" {
   source = "../../"
 

From 92be67ed046deb294612c6d95963d5a431356abf Mon Sep 17 00:00:00 2001
From: Justin Brooks <jsbroks@gmail.com>
Date: Wed, 14 Feb 2024 15:34:45 -0500
Subject: [PATCH 5/8] change provider

---
 examples/public-dns-external/custom.tf | 46 --------------------------
 examples/public-dns-external/main.tf   | 13 ++++++++
 2 files changed, 13 insertions(+), 46 deletions(-)

diff --git a/examples/public-dns-external/custom.tf b/examples/public-dns-external/custom.tf
index 29bca04a..e69de29b 100644
--- a/examples/public-dns-external/custom.tf
+++ b/examples/public-dns-external/custom.tf
@@ -1,46 +0,0 @@
-locals {
-  infra_outputs           = data.terraform_remote_state.infra.outputs
-  gcp_credentials         = local.infra_outputs.deployments_credentials
-  aws_deployment_role_arn = local.infra_outputs.deployments_aws_role_arn
-  region = "us-west-1"
-}
-
-data "terraform_remote_state" "infra" {
-  backend = "remote"
-  config = {
-    organization = "weights-and-biases"
-    workspaces   = { name = "deployer-global" }
-  }
-}
-
-provider "aws" {
-  region     = local.region
-  access_key = module.aws_credentials.access_key
-  secret_key = module.aws_credentials.secret_key
-  token      = module.aws_credentials.token
-
-  default_tags {
-    tags = {
-      Owner          = "Deployer"
-      Namespace      = var.namespace
-    }
-  }
-}
-
-# Login using the deployment service account.
-provider "google" {
-  project     = "wandb-production"
-  region      = "us-central1"
-  zone        = "us-central1-c"
-  credentials = local.gcp_credentials
-}
-
-# Create AWS credentials from GCP account
-module "aws_credentials" {
-  source  = "wandb/assume-aws-role/google"
-  version = "1.1.0"
-
-  duration_seconds = 43200 # 12 hours
-  role_arn         = local.aws_deployment_role_arn
-  session_name     = "TerraformDeployment"
-}
diff --git a/examples/public-dns-external/main.tf b/examples/public-dns-external/main.tf
index 2d39dc53..ab506ada 100644
--- a/examples/public-dns-external/main.tf
+++ b/examples/public-dns-external/main.tf
@@ -1,3 +1,16 @@
+provider "aws" {
+  region = "us-west-2"
+
+  default_tags {
+    tags = {
+      GithubRepo = "terraform-aws-wandb"
+      GithubOrg  = "wandb"
+      Enviroment = "Example"
+      Example    = "PublicDnsExternal"
+    }
+  }
+}
+
 module "wandb_infra" {
   source = "../../"
 

From c3fd8e92b2000ed808c439f25df145ec3acf9cff Mon Sep 17 00:00:00 2001
From: Justin Brooks <jsbroks@gmail.com>
Date: Wed, 14 Feb 2024 15:41:26 -0500
Subject: [PATCH 6/8] use old auth

---
 examples/public-dns-external/custom.tf | 47 ++++++++++++++++++++++++++
 examples/public-dns-external/main.tf   | 24 ++++++-------
 2 files changed, 59 insertions(+), 12 deletions(-)

diff --git a/examples/public-dns-external/custom.tf b/examples/public-dns-external/custom.tf
index e69de29b..63b1f221 100644
--- a/examples/public-dns-external/custom.tf
+++ b/examples/public-dns-external/custom.tf
@@ -0,0 +1,47 @@
+locals {
+  infra_outputs           = data.terraform_remote_state.infra.outputs
+  gcp_credentials         = local.infra_outputs.deployments_credentials
+  aws_deployment_role_arn = local.infra_outputs.deployments_aws_role_arn
+  region = "us-west-1"
+}
+
+
+data "terraform_remote_state" "infra" {
+  backend = "remote"
+  config = {
+    organization = "weights-and-biases"
+    workspaces   = { name = "deployer-global" }
+  }
+}
+
+provider "aws" {
+  region     = local.region
+  access_key = module.aws_credentials.access_key
+  secret_key = module.aws_credentials.secret_key
+  token      = module.aws_credentials.token
+
+  default_tags {
+    tags = {
+      Owner          = "Deployer"
+      Namespace      = var.namespace
+    }
+  }
+}
+
+# Login using the deployment service account.
+provider "google" {
+  project     = "wandb-production"
+  region      = "us-central1"
+  zone        = "us-central1-c"
+  credentials = local.gcp_credentials
+}
+
+# Create AWS credentials from GCP account
+module "aws_credentials" {
+  source  = "wandb/assume-aws-role/google"
+  version = "1.1.0"
+
+  duration_seconds = 43200 # 12 hours
+  role_arn         = local.aws_deployment_role_arn
+  session_name     = "TerraformDeployment"
+}
diff --git a/examples/public-dns-external/main.tf b/examples/public-dns-external/main.tf
index ab506ada..bc4dbc93 100644
--- a/examples/public-dns-external/main.tf
+++ b/examples/public-dns-external/main.tf
@@ -1,15 +1,15 @@
-provider "aws" {
-  region = "us-west-2"
-
-  default_tags {
-    tags = {
-      GithubRepo = "terraform-aws-wandb"
-      GithubOrg  = "wandb"
-      Enviroment = "Example"
-      Example    = "PublicDnsExternal"
-    }
-  }
-}
+# provider "aws" {
+#   region = "us-west-2"
+
+#   default_tags {
+#     tags = {
+#       GithubRepo = "terraform-aws-wandb"
+#       GithubOrg  = "wandb"
+#       Enviroment = "Example"
+#       Example    = "PublicDnsExternal"
+#     }
+#   }
+# }
 
 module "wandb_infra" {
   source = "../../"

From 1a0c59eda7b39e95282c1080e12c32d9a40216ed Mon Sep 17 00:00:00 2001
From: Justin Brooks <jsbroks@gmail.com>
Date: Wed, 14 Feb 2024 19:56:19 -0500
Subject: [PATCH 7/8] configure msk

---
 examples/public-dns-external/custom.tf | 49 +++-------------------
 examples/public-dns-external/main.tf   | 24 +++++------
 main.tf                                |  9 ++++
 modules/msk/main.tf                    | 57 +++++++++++++++-----------
 modules/msk/variables.tf               |  8 ++++
 5 files changed, 67 insertions(+), 80 deletions(-)

diff --git a/examples/public-dns-external/custom.tf b/examples/public-dns-external/custom.tf
index 63b1f221..e128d9d5 100644
--- a/examples/public-dns-external/custom.tf
+++ b/examples/public-dns-external/custom.tf
@@ -1,47 +1,8 @@
-locals {
-  infra_outputs           = data.terraform_remote_state.infra.outputs
-  gcp_credentials         = local.infra_outputs.deployments_credentials
-  aws_deployment_role_arn = local.infra_outputs.deployments_aws_role_arn
-  region = "us-west-1"
-}
-
-
-data "terraform_remote_state" "infra" {
-  backend = "remote"
-  config = {
+terraform {
+  cloud {
     organization = "weights-and-biases"
-    workspaces   = { name = "deployer-global" }
-  }
-}
-
-provider "aws" {
-  region     = local.region
-  access_key = module.aws_credentials.access_key
-  secret_key = module.aws_credentials.secret_key
-  token      = module.aws_credentials.token
-
-  default_tags {
-    tags = {
-      Owner          = "Deployer"
-      Namespace      = var.namespace
+    workspaces {
+      name = "apple-replica-msk"
     }
   }
-}
-
-# Login using the deployment service account.
-provider "google" {
-  project     = "wandb-production"
-  region      = "us-central1"
-  zone        = "us-central1-c"
-  credentials = local.gcp_credentials
-}
-
-# Create AWS credentials from GCP account
-module "aws_credentials" {
-  source  = "wandb/assume-aws-role/google"
-  version = "1.1.0"
-
-  duration_seconds = 43200 # 12 hours
-  role_arn         = local.aws_deployment_role_arn
-  session_name     = "TerraformDeployment"
-}
+}
\ No newline at end of file
diff --git a/examples/public-dns-external/main.tf b/examples/public-dns-external/main.tf
index bc4dbc93..ab506ada 100644
--- a/examples/public-dns-external/main.tf
+++ b/examples/public-dns-external/main.tf
@@ -1,15 +1,15 @@
-# provider "aws" {
-#   region = "us-west-2"
-
-#   default_tags {
-#     tags = {
-#       GithubRepo = "terraform-aws-wandb"
-#       GithubOrg  = "wandb"
-#       Enviroment = "Example"
-#       Example    = "PublicDnsExternal"
-#     }
-#   }
-# }
+provider "aws" {
+  region = "us-west-2"
+
+  default_tags {
+    tags = {
+      GithubRepo = "terraform-aws-wandb"
+      GithubOrg  = "wandb"
+      Enviroment = "Example"
+      Example    = "PublicDnsExternal"
+    }
+  }
+}
 
 module "wandb_infra" {
   source = "../../"
diff --git a/main.tf b/main.tf
index 14bafe24..271af945 100644
--- a/main.tf
+++ b/main.tf
@@ -44,6 +44,7 @@ module "networking" {
   elasticache_subnet_cidrs  = var.network_elasticache_subnet_cidrs
 }
 
+
 locals {
   network_id             = var.create_vpc ? module.networking.vpc_id : var.network_id
   network_public_subnets = var.create_vpc ? module.networking.public_subnets : var.network_public_subnets
@@ -59,6 +60,14 @@ locals {
   network_elasticache_subnet_group_name = module.networking.elasticache_subnet_group_name
 }
 
+module "msk" {
+  source    = "./modules/msk"
+  namespace = var.namespace
+
+  private_subnets = local.network_private_subnets
+  vpc_id          = local.network_id
+}
+
 module "database" {
   source = "./modules/database"
 
diff --git a/modules/msk/main.tf b/modules/msk/main.tf
index 5610b4a2..a53a4848 100644
--- a/modules/msk/main.tf
+++ b/modules/msk/main.tf
@@ -1,47 +1,56 @@
-resource "aws_security_group" "msk_brokers_sg" {
-  name        = "msk-brokers-sg"
-  vpc_id      = data.aws_vpc.existing_vpc.id
-  description = "Security group for MSK brokers"
+# Security group for MSK (allows traffic within your VPC)
+resource "aws_security_group" "msk" {
+  name        = "${var.namespace}-msk-sg"
+  vpc_id      = var.vpc_id
+  description = "Allow MSK traffic within the VPC"
 
-  # Restrict inbound traffic to only necessary ports from your VPC CIDR
   ingress {
-    from_port   = 2181 # Zookeeper
-    to_port     = 2181
-    protocol    = "tcp"
-    cidr_blocks = [data.aws_vpc.existing_vpc.cidr_block] 
+    from_port = 9092
+    to_port   = 9092
+    protocol  = "tcp"
+    self      = true
   }
 
-  # Add more ingress rules as needed for monitoring, etc.
-
   egress {
     from_port   = 0
     to_port     = 0
     protocol    = "-1"
     cidr_blocks = ["0.0.0.0/0"]
   }
-
-  tags = {
-    Name = "msk-brokers-sg"
-  }
 }
 
 resource "aws_msk_cluster" "default" {
-  cluster_name                   = "${var.namespace}"
-  kafka_version          = "3.4.0" # Choose your desired Kafka version
-  number_of_broker_nodes = 3
+  cluster_name           = var.namespace
+  kafka_version          = "3.6.0"
+  number_of_broker_nodes = length(var.private_subnets)
 
   broker_node_group_info {
-    instance_type    = "kafka.m5.large"  # Adjust instance type as needed
-    client_subnets   = data.aws_subnets.private_subnets.ids
-    security_groups  = [aws_security_group.msk_brokers_sg.id]
-    # ebs_volume_size  = 50 # In GB
+    instance_type = "kafka.m5.large"
+
+    client_subnets  = var.private_subnets
+    security_groups = [aws_security_group.msk.id]
+
+    storage_info {
+      ebs_storage_info {
+        volume_size = 20
+      }
+    }
   }
 
   encryption_info {
     encryption_in_transit {
-      client_broker = "TLS" 
+      client_broker = "TLS"
     }
   }
 
-  depends_on = [aws_security_group.msk_brokers_sg] 
+  depends_on = [aws_security_group.msk]
+}
+
+output "zookeeper_connect_string" {
+  value = aws_msk_cluster.default.zookeeper_connect_string
+}
+
+output "bootstrap_brokers_tls" {
+  description = "TLS connection host:port pairs"
+  value       = aws_msk_cluster.default.bootstrap_brokers_tls
 }
\ No newline at end of file
diff --git a/modules/msk/variables.tf b/modules/msk/variables.tf
index 6857747f..a1ac111d 100644
--- a/modules/msk/variables.tf
+++ b/modules/msk/variables.tf
@@ -1,3 +1,11 @@
 variable "namespace" {
   type = string
+}
+
+variable "vpc_id" {
+  type = string
+}
+
+variable "private_subnets" {
+  type = list(string)
 }
\ No newline at end of file

From c39dbce2915cdc78f7d410fd229b46fa96f7bac0 Mon Sep 17 00:00:00 2001
From: Justin Brooks <jsbroks@gmail.com>
Date: Thu, 15 Feb 2024 11:10:59 -0500
Subject: [PATCH 8/8] add variables

---
 modules/msk/main.tf      |  6 +++---
 modules/msk/variables.tf | 15 +++++++++++++++
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/modules/msk/main.tf b/modules/msk/main.tf
index a53a4848..bff43b4c 100644
--- a/modules/msk/main.tf
+++ b/modules/msk/main.tf
@@ -21,18 +21,18 @@ resource "aws_security_group" "msk" {
 
 resource "aws_msk_cluster" "default" {
   cluster_name           = var.namespace
-  kafka_version          = "3.6.0"
+  kafka_version          = var.kafka_version
   number_of_broker_nodes = length(var.private_subnets)
 
   broker_node_group_info {
-    instance_type = "kafka.m5.large"
+    instance_type = var.instance_type
 
     client_subnets  = var.private_subnets
     security_groups = [aws_security_group.msk.id]
 
     storage_info {
       ebs_storage_info {
-        volume_size = 20
+        volume_size = var.volume_size
       }
     }
   }
diff --git a/modules/msk/variables.tf b/modules/msk/variables.tf
index a1ac111d..47d5a0cf 100644
--- a/modules/msk/variables.tf
+++ b/modules/msk/variables.tf
@@ -8,4 +8,19 @@ variable "vpc_id" {
 
 variable "private_subnets" {
   type = list(string)
+}
+
+variable "instance_type" {
+  type    = string
+  default = "kafka.m5.large"
+}
+
+variable "volume_size" {
+  type    = number
+  default = 20
+}
+
+variable "kafka_version" {
+  type    = string
+  default = "3.6.0"
 }
\ No newline at end of file