Refactor Vault resources (#38)

Co-authored-by: Filippo Morelli <[email protected]>

bootstrap/collector.py

@@ -176,8 +176,8 @@ def validate_or_prompt_email(message, value=None, default=None, required=True):
     return validate_or_prompt_email(message, None, default, required)
 
 
-def validate_or_prompt_password(message, value=None, default=None, required=True):
-    """Validate the given password or prompt until a valid value is provided."""
+def validate_or_prompt_secret(message, value=None, default=None, required=True):
+    """Validate the given secret or prompt until a valid value is provided."""
     if value is None:
         value = click.prompt(message, default=default, hide_input=True)
     try:
@@ -186,7 +186,7 @@ def validate_or_prompt_password(message, value=None, default=None, required=True
     except validators.ValidationFailure:
         pass
     click.echo(error("Please type at least 8 chars!"))
-    return validate_or_prompt_password(message, None, default, required)
+    return validate_or_prompt_secret(message, None, default, required)
 
 
 def validate_or_prompt_path(message, value=None, default=None, required=True):
@@ -298,7 +298,7 @@ def clean_terraform_backend(
         terraform_cloud_hostname = validate_or_prompt_domain(
             "Terraform host name", terraform_cloud_hostname, default="app.terraform.io"
         )
-        terraform_cloud_token = validate_or_prompt_password(
+        terraform_cloud_token = validate_or_prompt_secret(
             "Terraform Cloud User token", terraform_cloud_token
         )
         terraform_cloud_organization = terraform_cloud_organization or click.prompt(
@@ -337,16 +337,21 @@ def clean_terraform_backend(
 
 def clean_vault_data(vault_token, vault_url, quiet=False):
     """Return the Vault data, if applicable."""
-    if vault_token or (
-        vault_token is None
+    if vault_url or (
+        vault_url is None
         and click.confirm(
             "Do you want to use Vault for secrets management?",
         )
     ):
-        vault_token = validate_or_prompt_password("Vault token", vault_token)
+        vault_token = validate_or_prompt_secret(
+            "Vault token (leave blank to perform a browser-based OIDC authentication)",
+            vault_token,
+            default="",
+            required=False,
+        )
         quiet or click.confirm(
             warning(
-                "Make sure the Vault token has enough permissions to enable the "
+                "Make sure your Vault permissions allow to enable the "
                 "project secrets backends and manage the project secrets. Continue?"
             ),
             abort=True,

bootstrap/constants.py

@@ -1,32 +1,69 @@
-#!/usr/bin/env python
 """Web project initialization CLI constants."""
 
+from typing import Dict
+
 # Stacks
 
+# BEWARE: stack names must be suitable for inclusion in Vault paths
+
+DEV_STACK_NAME = "development"
+
 DEV_STACK_SLUG = "dev"
 
+STAGE_STACK_NAME = "staging"
+
 STAGE_STACK_SLUG = "stage"
 
+MAIN_STACK_NAME = "main"
+
 MAIN_STACK_SLUG = "main"
 
+STACKS_CHOICES = {
+    "1": [{"name": MAIN_STACK_NAME, "slug": MAIN_STACK_SLUG}],
+    "2": [
+        {"name": DEV_STACK_NAME, "slug": DEV_STACK_SLUG},
+        {"name": MAIN_STACK_NAME, "slug": MAIN_STACK_SLUG},
+    ],
+    "3": [
+        {"name": DEV_STACK_NAME, "slug": DEV_STACK_SLUG},
+        {"name": STAGE_STACK_NAME, "slug": STAGE_STACK_SLUG},
+        {"name": MAIN_STACK_NAME, "slug": MAIN_STACK_SLUG},
+    ],
+}
+
 # Environments
 
+# BEWARE: environment names must be suitable for inclusion in Vault paths
+
 DEV_ENV_NAME = "development"
 
 DEV_ENV_SLUG = "dev"
 
+DEV_ENV_STACK_CHOICES: Dict[str, str] = {
+    "1": MAIN_STACK_SLUG,
+}
+
 STAGE_ENV_NAME = "staging"
 
 STAGE_ENV_SLUG = "stage"
 
+STAGE_ENV_STACK_CHOICES: Dict[str, str] = {
+    "1": MAIN_STACK_SLUG,
+    "2": DEV_STACK_SLUG,
+}
+
 PROD_ENV_NAME = "production"
 
 PROD_ENV_SLUG = "prod"
 
+PROD_ENV_STACK_CHOICES: Dict[str, str] = {}
+
 # Env vars
 
 GITLAB_TOKEN_ENV_VAR = "GITLAB_PRIVATE_TOKEN"
 
+VAULT_TOKEN_ENV_VAR = "VAULT_TOKEN"
+
 # Deployment type
 
 DEPLOYMENT_TYPE_DIGITALOCEAN = "digitalocean-k8s"

bootstrap/runner.py

@@ -5,6 +5,7 @@
 import subprocess
 from dataclasses import dataclass, field
 from functools import partial
+from operator import itemgetter
 from pathlib import Path
 from time import time
 
@@ -15,12 +16,16 @@
 from bootstrap.constants import (
     DEV_ENV_NAME,
     DEV_ENV_SLUG,
+    DEV_ENV_STACK_CHOICES,
     DEV_STACK_SLUG,
     MAIN_STACK_SLUG,
     PROD_ENV_NAME,
     PROD_ENV_SLUG,
+    PROD_ENV_STACK_CHOICES,
+    STACKS_CHOICES,
     STAGE_ENV_NAME,
     STAGE_ENV_SLUG,
+    STAGE_ENV_STACK_CHOICES,
     STAGE_STACK_SLUG,
     TERRAFORM_BACKEND_TFC,
 )
@@ -74,8 +79,8 @@ class Runner:
     terraform_dir: Path | None = None
     logs_dir: Path | None = None
     run_id: str = field(init=False)
-    stacks_environments: dict = field(init=False, default_factory=dict)
-    environments_stacks: dict = field(init=False, default_factory=dict)
+    stacks: list = field(init=False, default_factory=list)
+    envs: list = field(init=False, default_factory=list)
     gitlab_variables: dict = field(init=False, default_factory=dict)
     tfvars: dict = field(init=False, default_factory=dict)
     vault_secrets: dict = field(init=False, default_factory=dict)
@@ -87,52 +92,46 @@ def __post_init__(self):
         self.run_id = f"{time():.0f}"
         self.terraform_dir = self.terraform_dir or Path(f".terraform/{self.run_id}")
         self.logs_dir = self.logs_dir or Path(f".logs/{self.run_id}")
-        self.set_stacks_environments()
-        self.set_environments_stacks()
+        self.set_stacks()
+        self.set_envs()
         self.collect_tfvars()
         self.collect_gitlab_variables()
 
-    def set_stacks_environments(self):
-        """Set the environments distribution per stack."""
-        dev_env = {
-            "name": DEV_ENV_NAME,
-            "url": self.project_url_dev,
-        }
-        stage_env = {
-            "name": STAGE_ENV_NAME,
-            "url": self.project_url_stage,
-        }
-        prod_env = {
-            "name": PROD_ENV_NAME,
-            "url": self.project_url_prod,
-        }
-        if self.environment_distribution == "1":
-            self.stacks_environments = {
-                MAIN_STACK_SLUG: {
-                    DEV_ENV_SLUG: dev_env,
-                    STAGE_ENV_SLUG: stage_env,
-                    PROD_ENV_SLUG: prod_env,
-                }
-            }
-        elif self.environment_distribution == "2":
-            self.stacks_environments = {
-                DEV_STACK_SLUG: {DEV_ENV_SLUG: dev_env, STAGE_ENV_SLUG: stage_env},
-                MAIN_STACK_SLUG: {PROD_ENV_SLUG: prod_env},
-            }
-        elif self.environment_distribution == "3":
-            self.stacks_environments = {
-                DEV_STACK_SLUG: {DEV_ENV_SLUG: dev_env},
-                STAGE_STACK_SLUG: {STAGE_ENV_SLUG: stage_env},
-                MAIN_STACK_SLUG: {PROD_ENV_SLUG: prod_env},
-            }
-
-    def set_environments_stacks(self):
-        """Set a dict with environments to stacks mapping."""
-        self.environments_stacks = {
-            env_slug: stack_slug
-            for stack_slug, stack_envs in self.stacks_environments.items()
-            for env_slug in stack_envs
-        }
+    def set_stacks(self):
+        """Set the stacks."""
+        self.stacks = STACKS_CHOICES[self.environment_distribution]
+
+    def set_envs(self):
+        """Set the envs."""
+        self.envs = [
+            {
+                "basic_auth_enabled": True,
+                "name": DEV_ENV_NAME,
+                "slug": DEV_ENV_SLUG,
+                "stack_slug": DEV_ENV_STACK_CHOICES.get(
+                    self.environment_distribution, DEV_STACK_SLUG
+                ),
+                "url": self.project_url_dev,
+            },
+            {
+                "basic_auth_enabled": True,
+                "name": STAGE_ENV_NAME,
+                "slug": STAGE_ENV_SLUG,
+                "stack_slug": STAGE_ENV_STACK_CHOICES.get(
+                    self.environment_distribution, STAGE_STACK_SLUG
+                ),
+                "url": self.project_url_stage,
+            },
+            {
+                "basic_auth_enabled": False,
+                "name": PROD_ENV_NAME,
+                "slug": PROD_ENV_SLUG,
+                "stack_slug": PROD_ENV_STACK_CHOICES.get(
+                    self.environment_distribution, MAIN_STACK_SLUG
+                ),
+                "url": self.project_url_prod,
+            },
+        ]
 
     def register_gitlab_variable(
         self, level, var_name, var_value=None, masked=False, protected=True
@@ -166,7 +165,7 @@ def collect_gitlab_variables(self):
                 ("SENTRY_URL", self.sentry_url),
                 ("SENTRY_ENABLED", "true"),
             )
-        if not self.vault_token:
+        if not self.vault_url:
             self.collect_gitlab_variables_secrets()
 
     def collect_gitlab_variables_secrets(self):
@@ -207,33 +206,28 @@ def collect_tfvars(self):
             ("internal_backend_url", self.internal_backend_url),
             ("service_slug", self.service_slug),
         )
-        for stack_slug, stack_envs in self.stacks_environments.items():
-            for env_slug, env_data in stack_envs.items():
-                self.register_environment_tfvars(
-                    ("environment", env_data["name"]),
-                    ("project_url", env_data["url"]),
-                    ("stack_slug", stack_slug),
-                    env_slug=env_slug,
-                )
+        for env in self.envs:
+            self.register_environment_tfvars(
+                ("environment", env["name"]),
+                ("project_url", env["url"]),
+                ("stack_slug", env["stack_slug"]),
+                env_slug=env["slug"],
+            )
 
-    def register_vault_environment_secret(self, env_slug, name, data):
+    def register_vault_environment_secret(self, env_name, secret_name, secret_data):
         """Register a Vault environment secret locally."""
-        self.vault_secrets[f"envs/{env_slug}/{name}"] = data
+        self.vault_secrets[f"envs/{env_name}/{secret_name}"] = secret_data
 
-    def collect_vault_environment_secrets(self, env_slug):
+    def collect_vault_environment_secrets(self, env_name):
         """Collect the Vault secrets for the given environment."""
         # Sentry env vars are used by the GitLab CI/CD
         self.sentry_dsn and self.register_vault_environment_secret(
-            env_slug, f"sentry_{self.service_slug}", dict(sentry_dsn=self.sentry_dsn)
+            env_name, f"{self.service_slug}/sentry", dict(sentry_dsn=self.sentry_dsn)
         )
 
     def collect_vault_secrets(self):
         """Collect Vault secrets."""
-        [
-            self.collect_vault_environment_secrets(env_slug)
-            for stack_envs in self.stacks_environments.values()
-            for env_slug in stack_envs
-        ]
+        [self.collect_vault_environment_secrets(env["name"]) for env in self.envs]
 
     def init_service(self):
         """Initialize the service."""
@@ -242,14 +236,11 @@ def init_service(self):
             os.path.dirname(os.path.dirname(__file__)),
             extra_context={
                 "deployment_type": self.deployment_type,
-                "environments_stacks": self.environments_stacks,
                 "internal_service_port": self.internal_service_port,
                 "project_dirname": self.project_dirname,
                 "project_name": self.project_name,
                 "project_slug": self.project_slug,
-                "project_url_dev": self.project_url_dev,
-                "project_url_prod": self.project_url_prod,
-                "project_url_stage": self.project_url_stage,
+                "resources": {"envs": self.envs, "stacks": self.stacks},
                 "service_slug": self.service_slug,
                 "terraform_backend": self.terraform_backend,
                 "terraform_cloud_organization": self.terraform_cloud_organization,
@@ -276,12 +267,12 @@ def init_terraform_cloud(self):
             TF_VAR_create_organization=self.terraform_cloud_organization_create
             and "true"
             or "false",
+            TF_VAR_environments=json.dumps(list(map(itemgetter("slug"), self.envs))),
             TF_VAR_hostname=self.terraform_cloud_hostname,
             TF_VAR_organization_name=self.terraform_cloud_organization,
             TF_VAR_project_name=self.project_name,
             TF_VAR_project_slug=self.project_slug,
             TF_VAR_service_slug=self.service_slug,
-            TF_VAR_stacks="[]",
             TF_VAR_terraform_cloud_token=self.terraform_cloud_token,
         )
         self.run_terraform("terraform-cloud", env)
@@ -309,8 +300,8 @@ def init_vault(self):
         env = dict(
             TF_VAR_project_slug=self.project_slug,
             TF_VAR_secrets=json.dumps(self.vault_secrets),
-            VAULT_ADDR=self.vault_url,
-            VAULT_TOKEN=self.vault_token,
+            TF_VAR_vault_address=self.vault_url,
+            TF_VAR_vault_token=self.vault_token,
         )
         self.run_terraform("vault", env)
 
@@ -475,10 +466,10 @@ def run(self):
         click.echo(highlight(f"Initializing the {self.service_slug} service:"))
         self.init_service()
         self.create_env_file()
-        if self.gitlab_group_path:
-            self.init_gitlab()
         if self.terraform_backend == TERRAFORM_BACKEND_TFC:
             self.init_terraform_cloud()
-        if self.vault_token:
+        if self.gitlab_group_path:
+            self.init_gitlab()
+        if self.vault_url:
             self.init_vault()
         self.change_output_owner()