From 8aee09ec803ddee88edb9238693b94260483d077 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Thu, 3 Oct 2024 09:44:18 -0700 Subject: [PATCH 01/61] Add copier files --- .template-infra/{{_copier_conf.answers_file}}.jinja | 2 ++ copier.yml | 3 +++ 2 files changed, 5 insertions(+) create mode 100644 .template-infra/{{_copier_conf.answers_file}}.jinja create mode 100644 copier.yml diff --git a/.template-infra/{{_copier_conf.answers_file}}.jinja b/.template-infra/{{_copier_conf.answers_file}}.jinja new file mode 100644 index 00000000..a96840d6 --- /dev/null +++ b/.template-infra/{{_copier_conf.answers_file}}.jinja @@ -0,0 +1,2 @@ +# Changes here will be overwritten by Copier +{{ _copier_answers|to_nice_yaml -}} diff --git a/copier.yml b/copier.yml new file mode 100644 index 00000000..ef6548c8 --- /dev/null +++ b/copier.yml @@ -0,0 +1,3 @@ +app_name: + type: str + help: The name of the app From a44b00370087430ba84c01bf7167632160b0393b Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Thu, 3 Oct 2024 13:57:48 -0700 Subject: [PATCH 02/61] Templatize app_name --- .github/workflows/{cd-app.yml => cd-{{app_name}}.yml} | 0 ...nment-checks.yml => ci-{{app_name}}-pr-environment-checks.yml} | 0 ...ent-destroy.yml => ci-{{app_name}}-pr-environment-destroy.yml} | 0 ...rability-scans.yml => ci-{{app_name}}-vulnerability-scans.yml} | 0 infra/{app => {{app_name}}}/app-config/build_repository.tf | 0 infra/{app => {{app_name}}}/app-config/dev.tf | 0 infra/{app => {{app_name}}}/app-config/env-config/database.tf | 0 .../app-config/env-config/environment_variables.tf | 0 .../app-config/env-config/file_upload_jobs.tf | 0 .../app-config/env-config/identity_provider.tf | 0 infra/{app => {{app_name}}}/app-config/env-config/main.tf | 0 .../{app => {{app_name}}}/app-config/env-config/notifications.tf | 0 infra/{app => {{app_name}}}/app-config/env-config/outputs.tf | 0 .../{app => {{app_name}}}/app-config/env-config/scheduled_jobs.tf | 0 infra/{app => {{app_name}}}/app-config/env-config/variables.tf | 0 infra/{app => {{app_name}}}/app-config/feature_flags.tf | 0 infra/{app => {{app_name}}}/app-config/main.tf | 0 infra/{app => {{app_name}}}/app-config/outputs.tf | 0 infra/{app => {{app_name}}}/app-config/prod.tf | 0 infra/{app => {{app_name}}}/app-config/staging.tf | 0 infra/{app => {{app_name}}}/build-repository/main.tf | 0 infra/{app => {{app_name}}}/database/main.tf | 0 infra/{app => {{app_name}}}/database/outputs.tf | 0 infra/{app => {{app_name}}}/database/variables.tf | 0 infra/{app => {{app_name}}}/service/image_tag.tf | 0 infra/{app => {{app_name}}}/service/main.tf | 0 infra/{app => {{app_name}}}/service/outputs.tf | 0 infra/{app => {{app_name}}}/service/secrets.tf | 0 infra/{app => {{app_name}}}/service/variables.tf | 0 29 files changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{cd-app.yml => cd-{{app_name}}.yml} (100%) rename .github/workflows/{ci-app-pr-environment-checks.yml => ci-{{app_name}}-pr-environment-checks.yml} (100%) rename .github/workflows/{ci-app-pr-environment-destroy.yml => ci-{{app_name}}-pr-environment-destroy.yml} (100%) rename .github/workflows/{ci-app-vulnerability-scans.yml => ci-{{app_name}}-vulnerability-scans.yml} (100%) rename infra/{app => {{app_name}}}/app-config/build_repository.tf (100%) rename infra/{app => {{app_name}}}/app-config/dev.tf (100%) rename infra/{app => {{app_name}}}/app-config/env-config/database.tf (100%) rename infra/{app => {{app_name}}}/app-config/env-config/environment_variables.tf (100%) rename infra/{app => {{app_name}}}/app-config/env-config/file_upload_jobs.tf (100%) rename infra/{app => {{app_name}}}/app-config/env-config/identity_provider.tf (100%) rename infra/{app => {{app_name}}}/app-config/env-config/main.tf (100%) rename infra/{app => {{app_name}}}/app-config/env-config/notifications.tf (100%) rename infra/{app => {{app_name}}}/app-config/env-config/outputs.tf (100%) rename infra/{app => {{app_name}}}/app-config/env-config/scheduled_jobs.tf (100%) rename infra/{app => {{app_name}}}/app-config/env-config/variables.tf (100%) rename infra/{app => {{app_name}}}/app-config/feature_flags.tf (100%) rename infra/{app => {{app_name}}}/app-config/main.tf (100%) rename infra/{app => {{app_name}}}/app-config/outputs.tf (100%) rename infra/{app => {{app_name}}}/app-config/prod.tf (100%) rename infra/{app => {{app_name}}}/app-config/staging.tf (100%) rename infra/{app => {{app_name}}}/build-repository/main.tf (100%) rename infra/{app => {{app_name}}}/database/main.tf (100%) rename infra/{app => {{app_name}}}/database/outputs.tf (100%) rename infra/{app => {{app_name}}}/database/variables.tf (100%) rename infra/{app => {{app_name}}}/service/image_tag.tf (100%) rename infra/{app => {{app_name}}}/service/main.tf (100%) rename infra/{app => {{app_name}}}/service/outputs.tf (100%) rename infra/{app => {{app_name}}}/service/secrets.tf (100%) rename infra/{app => {{app_name}}}/service/variables.tf (100%) diff --git a/.github/workflows/cd-app.yml b/.github/workflows/cd-{{app_name}}.yml similarity index 100% rename from .github/workflows/cd-app.yml rename to .github/workflows/cd-{{app_name}}.yml diff --git a/.github/workflows/ci-app-pr-environment-checks.yml b/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml similarity index 100% rename from .github/workflows/ci-app-pr-environment-checks.yml rename to .github/workflows/ci-{{app_name}}-pr-environment-checks.yml diff --git a/.github/workflows/ci-app-pr-environment-destroy.yml b/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml similarity index 100% rename from .github/workflows/ci-app-pr-environment-destroy.yml rename to .github/workflows/ci-{{app_name}}-pr-environment-destroy.yml diff --git a/.github/workflows/ci-app-vulnerability-scans.yml b/.github/workflows/ci-{{app_name}}-vulnerability-scans.yml similarity index 100% rename from .github/workflows/ci-app-vulnerability-scans.yml rename to .github/workflows/ci-{{app_name}}-vulnerability-scans.yml diff --git a/infra/app/app-config/build_repository.tf b/infra/{{app_name}}/app-config/build_repository.tf similarity index 100% rename from infra/app/app-config/build_repository.tf rename to infra/{{app_name}}/app-config/build_repository.tf diff --git a/infra/app/app-config/dev.tf b/infra/{{app_name}}/app-config/dev.tf similarity index 100% rename from infra/app/app-config/dev.tf rename to infra/{{app_name}}/app-config/dev.tf diff --git a/infra/app/app-config/env-config/database.tf b/infra/{{app_name}}/app-config/env-config/database.tf similarity index 100% rename from infra/app/app-config/env-config/database.tf rename to infra/{{app_name}}/app-config/env-config/database.tf diff --git a/infra/app/app-config/env-config/environment_variables.tf b/infra/{{app_name}}/app-config/env-config/environment_variables.tf similarity index 100% rename from infra/app/app-config/env-config/environment_variables.tf rename to infra/{{app_name}}/app-config/env-config/environment_variables.tf diff --git a/infra/app/app-config/env-config/file_upload_jobs.tf b/infra/{{app_name}}/app-config/env-config/file_upload_jobs.tf similarity index 100% rename from infra/app/app-config/env-config/file_upload_jobs.tf rename to infra/{{app_name}}/app-config/env-config/file_upload_jobs.tf diff --git a/infra/app/app-config/env-config/identity_provider.tf b/infra/{{app_name}}/app-config/env-config/identity_provider.tf similarity index 100% rename from infra/app/app-config/env-config/identity_provider.tf rename to infra/{{app_name}}/app-config/env-config/identity_provider.tf diff --git a/infra/app/app-config/env-config/main.tf b/infra/{{app_name}}/app-config/env-config/main.tf similarity index 100% rename from infra/app/app-config/env-config/main.tf rename to infra/{{app_name}}/app-config/env-config/main.tf diff --git a/infra/app/app-config/env-config/notifications.tf b/infra/{{app_name}}/app-config/env-config/notifications.tf similarity index 100% rename from infra/app/app-config/env-config/notifications.tf rename to infra/{{app_name}}/app-config/env-config/notifications.tf diff --git a/infra/app/app-config/env-config/outputs.tf b/infra/{{app_name}}/app-config/env-config/outputs.tf similarity index 100% rename from infra/app/app-config/env-config/outputs.tf rename to infra/{{app_name}}/app-config/env-config/outputs.tf diff --git a/infra/app/app-config/env-config/scheduled_jobs.tf b/infra/{{app_name}}/app-config/env-config/scheduled_jobs.tf similarity index 100% rename from infra/app/app-config/env-config/scheduled_jobs.tf rename to infra/{{app_name}}/app-config/env-config/scheduled_jobs.tf diff --git a/infra/app/app-config/env-config/variables.tf b/infra/{{app_name}}/app-config/env-config/variables.tf similarity index 100% rename from infra/app/app-config/env-config/variables.tf rename to infra/{{app_name}}/app-config/env-config/variables.tf diff --git a/infra/app/app-config/feature_flags.tf b/infra/{{app_name}}/app-config/feature_flags.tf similarity index 100% rename from infra/app/app-config/feature_flags.tf rename to infra/{{app_name}}/app-config/feature_flags.tf diff --git a/infra/app/app-config/main.tf b/infra/{{app_name}}/app-config/main.tf similarity index 100% rename from infra/app/app-config/main.tf rename to infra/{{app_name}}/app-config/main.tf diff --git a/infra/app/app-config/outputs.tf b/infra/{{app_name}}/app-config/outputs.tf similarity index 100% rename from infra/app/app-config/outputs.tf rename to infra/{{app_name}}/app-config/outputs.tf diff --git a/infra/app/app-config/prod.tf b/infra/{{app_name}}/app-config/prod.tf similarity index 100% rename from infra/app/app-config/prod.tf rename to infra/{{app_name}}/app-config/prod.tf diff --git a/infra/app/app-config/staging.tf b/infra/{{app_name}}/app-config/staging.tf similarity index 100% rename from infra/app/app-config/staging.tf rename to infra/{{app_name}}/app-config/staging.tf diff --git a/infra/app/build-repository/main.tf b/infra/{{app_name}}/build-repository/main.tf similarity index 100% rename from infra/app/build-repository/main.tf rename to infra/{{app_name}}/build-repository/main.tf diff --git a/infra/app/database/main.tf b/infra/{{app_name}}/database/main.tf similarity index 100% rename from infra/app/database/main.tf rename to infra/{{app_name}}/database/main.tf diff --git a/infra/app/database/outputs.tf b/infra/{{app_name}}/database/outputs.tf similarity index 100% rename from infra/app/database/outputs.tf rename to infra/{{app_name}}/database/outputs.tf diff --git a/infra/app/database/variables.tf b/infra/{{app_name}}/database/variables.tf similarity index 100% rename from infra/app/database/variables.tf rename to infra/{{app_name}}/database/variables.tf diff --git a/infra/app/service/image_tag.tf b/infra/{{app_name}}/service/image_tag.tf similarity index 100% rename from infra/app/service/image_tag.tf rename to infra/{{app_name}}/service/image_tag.tf diff --git a/infra/app/service/main.tf b/infra/{{app_name}}/service/main.tf similarity index 100% rename from infra/app/service/main.tf rename to infra/{{app_name}}/service/main.tf diff --git a/infra/app/service/outputs.tf b/infra/{{app_name}}/service/outputs.tf similarity index 100% rename from infra/app/service/outputs.tf rename to infra/{{app_name}}/service/outputs.tf diff --git a/infra/app/service/secrets.tf b/infra/{{app_name}}/service/secrets.tf similarity index 100% rename from infra/app/service/secrets.tf rename to infra/{{app_name}}/service/secrets.tf diff --git a/infra/app/service/variables.tf b/infra/{{app_name}}/service/variables.tf similarity index 100% rename from infra/app/service/variables.tf rename to infra/{{app_name}}/service/variables.tf From 2811ba09fedc503963ec1aeacb162ac0c8548d68 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Fri, 18 Oct 2024 13:10:16 -0700 Subject: [PATCH 03/61] Rename e2e/app to e2e/{{app_name}} --- e2e/{app => {{app_name}}}/playwright.config.js | 0 e2e/{app => {{app_name}}}/tests/index.spec.js | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename e2e/{app => {{app_name}}}/playwright.config.js (100%) rename e2e/{app => {{app_name}}}/tests/index.spec.js (100%) diff --git a/e2e/app/playwright.config.js b/e2e/{{app_name}}/playwright.config.js similarity index 100% rename from e2e/app/playwright.config.js rename to e2e/{{app_name}}/playwright.config.js diff --git a/e2e/app/tests/index.spec.js b/e2e/{{app_name}}/tests/index.spec.js similarity index 100% rename from e2e/app/tests/index.spec.js rename to e2e/{{app_name}}/tests/index.spec.js From 10eea5b951a546870f519480b9813ce114e2cc96 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Mon, 28 Oct 2024 15:41:43 -0400 Subject: [PATCH 04/61] Update cd-{{app_name}}.yml to parameterize job name --- ...cd-{{app_name}}.yml => cd-{{app_name}}.yml.jinja} | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) rename .github/workflows/{cd-{{app_name}}.yml => cd-{{app_name}}.yml.jinja} (69%) diff --git a/.github/workflows/cd-{{app_name}}.yml b/.github/workflows/cd-{{app_name}}.yml.jinja similarity index 69% rename from .github/workflows/cd-{{app_name}}.yml rename to .github/workflows/cd-{{app_name}}.yml.jinja index 313d6e1a..e37eecdc 100644 --- a/.github/workflows/cd-{{app_name}}.yml +++ b/.github/workflows/cd-{{app_name}}.yml.jinja @@ -1,7 +1,7 @@ -name: Deploy App +name: Deploy {{ app_name }} # Need to set a default value for when the workflow is triggered from a git push # which bypasses the default configuration for inputs -run-name: Deploy ${{ inputs.version || 'main' }} to App ${{ inputs.environment || 'dev' }} +run-name: Deploy ${{'{{'}}inputs.version || 'main' {{'}}'}} to {{ app_name }} ${{'{{'}} inputs.environment || 'dev' {{'}}'}} on: # !! Uncomment the following lines once you've set up the dev environment and ready to turn on continuous deployment @@ -9,7 +9,7 @@ on: # branches: # - "main" # paths: - # - "app/**" + # - "{{ app_name }}/**" # - "bin/**" # - "infra/**" workflow_dispatch: @@ -33,6 +33,6 @@ jobs: name: Deploy uses: ./.github/workflows/deploy.yml with: - app_name: "app" - environment: ${{ inputs.environment || 'dev' }} - version: ${{ inputs.version || 'main' }} + app_name: "{{ app_name }}" + environment: ${{'{{'}} inputs.environment || 'dev' {{'}}'}} + version: ${{'{{'}} inputs.version || 'main' {{'}}'}} From 37cc0d1a8617a792aabe9a8e4cd171e0b66ffb18 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Mon, 28 Oct 2024 17:48:44 -0400 Subject: [PATCH 05/61] Parameterize workflows/ci-{{app_name}}* content for app_name --- ... ci-{{app_name}}-pr-environment-checks.yml.jinja} | 8 ++++---- ...ci-{{app_name}}-pr-environment-destroy.yml.jinja} | 6 +++--- ...=> ci-{{app_name}}-vulnerability-scans.yml.jinja} | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) rename .github/workflows/{ci-{{app_name}}-pr-environment-checks.yml => ci-{{app_name}}-pr-environment-checks.yml.jinja} (70%) rename .github/workflows/{ci-{{app_name}}-pr-environment-destroy.yml => ci-{{app_name}}-pr-environment-destroy.yml.jinja} (75%) rename .github/workflows/{ci-{{app_name}}-vulnerability-scans.yml => ci-{{app_name}}-vulnerability-scans.yml.jinja} (55%) diff --git a/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml b/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja similarity index 70% rename from .github/workflows/ci-{{app_name}}-pr-environment-checks.yml rename to .github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja index 0a250d35..10375131 100644 --- a/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml +++ b/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja @@ -1,4 +1,4 @@ -name: CI App PR Environment Checks +name: CI {{ app_name }} PR Environment Checks on: workflow_dispatch: inputs: @@ -16,7 +16,7 @@ jobs: uses: ./.github/workflows/pr-environment-checks.yml if: github.event_name == 'workflow_dispatch' || github.event.pull_request.state == 'open' with: - app_name: "app" + app_name: "{{ app_name }}" environment: "dev" - pr_number: ${{ inputs.pr_number || github.event.number }} - commit_hash: ${{ inputs.commit_hash || github.event.pull_request.head.sha }} + pr_number: ${{'{{'}} inputs.pr_number || github.event.number {{'}}'}} + commit_hash: ${{'{{'}} inputs.commit_hash || github.event.pull_request.head.sha {{'}}'}} diff --git a/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml b/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja similarity index 75% rename from .github/workflows/ci-{{app_name}}-pr-environment-destroy.yml rename to .github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja index ad4cbb01..ec210acc 100644 --- a/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml +++ b/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja @@ -1,4 +1,4 @@ -name: CI App PR Environment Destroy +name: CI {{ app_name }} PR Environment Destroy on: workflow_dispatch: inputs: @@ -13,6 +13,6 @@ jobs: name: " " # GitHub UI is noisy when calling reusable workflows, so use whitespace for name to reduce noise uses: ./.github/workflows/pr-environment-destroy.yml with: - app_name: "app" + app_name: "{{ app_name }}" environment: "dev" - pr_number: ${{ inputs.pr_number || github.event.number }} + pr_number: ${{'{{'}} inputs.pr_number || github.event.number {{'}}'}} diff --git a/.github/workflows/ci-{{app_name}}-vulnerability-scans.yml b/.github/workflows/ci-{{app_name}}-vulnerability-scans.yml.jinja similarity index 55% rename from .github/workflows/ci-{{app_name}}-vulnerability-scans.yml rename to .github/workflows/ci-{{app_name}}-vulnerability-scans.yml.jinja index e20236f4..c5e29824 100644 --- a/.github/workflows/ci-{{app_name}}-vulnerability-scans.yml +++ b/.github/workflows/ci-{{app_name}}-vulnerability-scans.yml.jinja @@ -1,26 +1,26 @@ -name: CI Vulnerability Scans +name: CI Vulnerability Scans - {{ app_name }} on: push: branches: - main paths: - - app/** + - {{ app_name }}/** - .grype.yml - .hadolint.yaml - .trivyignore - - .github/workflows/ci-app-vulnerability-scans.yml + - .github/workflows/ci-{{ app_name}}-vulnerability-scans.yml pull_request: paths: - - app/** + - {{ app_name }}/** - .grype.yml - .hadolint.yaml - .trivyignore - - .github/workflows/ci-app-vulnerability-scans.yml + - .github/workflows/ci-{{ app_name}}-vulnerability-scans.yml jobs: vulnerability-scans: name: Vulnerability Scans uses: ./.github/workflows/vulnerability-scans.yml with: - app_name: "app" + app_name: "{{ app_name }}" From 11076e55aafbf422514926c950e194045af8f72c Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Fri, 1 Nov 2024 10:07:57 -0400 Subject: [PATCH 06/61] WIP try local port setting --- copier.yml | 5 +++++ .../{playwright.config.js => playwright.config.js.jinja} | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) rename e2e/{{app_name}}/{playwright.config.js => playwright.config.js.jinja} (75%) diff --git a/copier.yml b/copier.yml index ef6548c8..419c5807 100644 --- a/copier.yml +++ b/copier.yml @@ -1,3 +1,8 @@ app_name: type: str help: The name of the app + +local_port: + type: int + help: The port to be used in local development + default: 3000 diff --git a/e2e/{{app_name}}/playwright.config.js b/e2e/{{app_name}}/playwright.config.js.jinja similarity index 75% rename from e2e/{{app_name}}/playwright.config.js rename to e2e/{{app_name}}/playwright.config.js.jinja index 77a72fca..b00fde61 100644 --- a/e2e/{{app_name}}/playwright.config.js +++ b/e2e/{{app_name}}/playwright.config.js.jinja @@ -6,7 +6,7 @@ export default defineConfig(deepMerge( baseConfig, { use: { - baseURL: baseConfig.use.baseURL || "localhost:3000" + baseURL: baseConfig.use.baseURL || "localhost:{{ local_port }}" }, } )); From 11205583517d7cb41ca1316a6b68ac1e3b40d7c8 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Fri, 1 Nov 2024 10:24:03 -0400 Subject: [PATCH 07/61] Have app vuln scans run on any changes to underlying action (as was original intention I believe) --- .github/workflows/ci-{{app_name}}-vulnerability-scans.yml.jinja | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-{{app_name}}-vulnerability-scans.yml.jinja b/.github/workflows/ci-{{app_name}}-vulnerability-scans.yml.jinja index c5e29824..e1ecefa1 100644 --- a/.github/workflows/ci-{{app_name}}-vulnerability-scans.yml.jinja +++ b/.github/workflows/ci-{{app_name}}-vulnerability-scans.yml.jinja @@ -9,6 +9,7 @@ on: - .grype.yml - .hadolint.yaml - .trivyignore + - .github/workflows/vulnerability-scans.yml - .github/workflows/ci-{{ app_name}}-vulnerability-scans.yml pull_request: paths: @@ -16,6 +17,7 @@ on: - .grype.yml - .hadolint.yaml - .trivyignore + - .github/workflows/vulnerability-scans.yml - .github/workflows/ci-{{ app_name}}-vulnerability-scans.yml jobs: From 92522203b4adb05c4829c75ff7718e79550f96b5 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Fri, 1 Nov 2024 18:19:52 -0400 Subject: [PATCH 08/61] Support conditional blocks based on if dev env is available --- .github/workflows/cd-{{app_name}}.yml.jinja | 15 ++++++++++++- ...service.yml => ci-infra-service.yml.jinja} | 22 ++++++++++++++++++- ...app_name}}-pr-environment-checks.yml.jinja | 10 ++++++++- ...pp_name}}-pr-environment-destroy.yml.jinja | 11 +++++++++- copier.yml | 9 ++++++++ template-only-docs/set-up-cd.md | 9 ++++++-- template-only-docs/set-up-ci.md | 7 +++++- 7 files changed, 76 insertions(+), 7 deletions(-) rename .github/workflows/{ci-infra-service.yml => ci-infra-service.yml.jinja} (64%) diff --git a/.github/workflows/cd-{{app_name}}.yml.jinja b/.github/workflows/cd-{{app_name}}.yml.jinja index e37eecdc..601accbe 100644 --- a/.github/workflows/cd-{{app_name}}.yml.jinja +++ b/.github/workflows/cd-{{app_name}}.yml.jinja @@ -4,7 +4,19 @@ name: Deploy {{ app_name }} run-name: Deploy ${{'{{'}}inputs.version || 'main' {{'}}'}} to {{ app_name }} ${{'{{'}} inputs.environment || 'dev' {{'}}'}} on: - # !! Uncomment the following lines once you've set up the dev environment and ready to turn on continuous deployment + {% if is_dev_env_setup %} + push: + branches: + - "main" + paths: + - "{{ app_name }}/**" + - "bin/**" + - "infra/**" + {% else %} + # !! Once you've set up the dev environment and are ready to enable continuous + # deployment, run `nava-platform update --data is_dev_env_setup=true` to + # enable these lines. They are here as comments for context. + # # push: # branches: # - "main" @@ -12,6 +24,7 @@ on: # - "{{ app_name }}/**" # - "bin/**" # - "infra/**" + {% endif %} workflow_dispatch: inputs: environment: diff --git a/.github/workflows/ci-infra-service.yml b/.github/workflows/ci-infra-service.yml.jinja similarity index 64% rename from .github/workflows/ci-infra-service.yml rename to .github/workflows/ci-infra-service.yml.jinja index 7c486c99..a3f362a1 100644 --- a/.github/workflows/ci-infra-service.yml +++ b/.github/workflows/ci-infra-service.yml.jinja @@ -1,7 +1,26 @@ name: CI Infra Service Checks on: - # !! Uncomment to trigger automated infra tests once dev environment is set up + {% if is_dev_env_setup %} + push: + branches: + - main + paths: + - infra/*/service/** + - infra/modules/** + - infra/test/** + - .github/workflows/ci-infra-service.yml + pull_request: + paths: + - infra/*/service/** + - infra/modules/** + - infra/test/** + - .github/workflows/ci-infra-service.yml + {% else %} + # !! Once you've set up the dev environment and are ready to enable automated + # infra tests, run `nava-platform update --data is_dev_env_setup=true` to + # enable these lines. They are here as comments for context. + # # push: # branches: # - main @@ -16,6 +35,7 @@ on: # - infra/modules/** # - infra/test/** # - .github/workflows/ci-infra-service.yml + {% endif %} workflow_dispatch: jobs: diff --git a/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja b/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja index 10375131..6a47d6a5 100644 --- a/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja +++ b/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja @@ -8,8 +8,16 @@ on: commit_hash: required: true type: string - # !! Uncomment the following lines once you've set up the dev environment and are ready to enable PR environments + {% if is_dev_env_setup %} + pull_request: + {% else %} + # !! Once you've set up the dev environment and are ready to enable PR + # environments, run `nava-platform update --data is_dev_env_setup=true` to + # enable these lines. They are here as comments for context. + # # pull_request: + {% endif %} + jobs: update: name: " " # GitHub UI is noisy when calling reusable workflows, so use whitespace for name to reduce noise diff --git a/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja b/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja index ec210acc..04e9e044 100644 --- a/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja +++ b/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja @@ -5,9 +5,18 @@ on: pr_number: required: true type: string - # !! Uncomment the following lines once you've set up the dev environment and are ready to enable PR environments + {% if is_dev_env_setup %} + pull_request: + types: [closed] + {% else %} + # !! Once you've set up the dev environment and are ready to enable PR + # environments, run `nava-platform update --data is_dev_env_setup=true` to + # enable these lines. They are here as comments for context. + # # pull_request: # types: [closed] + {% endif %} + jobs: destroy: name: " " # GitHub UI is noisy when calling reusable workflows, so use whitespace for name to reduce noise diff --git a/copier.yml b/copier.yml index 419c5807..d76502d8 100644 --- a/copier.yml +++ b/copier.yml @@ -6,3 +6,12 @@ local_port: type: int help: The port to be used in local development default: 3000 + +is_dev_env_setup: + type: bool + help: If there's a dev environment deployed, to enable various CI/CD. + default: false + +_envops: + trim_blocks: true + lstrip_blocks: true diff --git a/template-only-docs/set-up-cd.md b/template-only-docs/set-up-cd.md index d9443771..e716febe 100644 --- a/template-only-docs/set-up-cd.md +++ b/template-only-docs/set-up-cd.md @@ -1,6 +1,11 @@ # Set up CD -Once you have set up your application environments, you can enable continuous deployment in [cd-app.yml](../.github/workflows/cd-app.yml) by searching for `!!` and following the instructions to: +Once you have set up your application environments (at least `dev`), you can +enable continuous deployment by running: +```sh +nava-platform update --data is_dev_env_setup=true +``` + +TODO still needed? 1. Update the `role-to-assume` with the GitHub actions ARN. -2. Uncomment the `on: push: ["main]` workflow trigger. This will trigger the deployment workflow on every merge to `main`. diff --git a/template-only-docs/set-up-ci.md b/template-only-docs/set-up-ci.md index 7264adfa..14bb439c 100644 --- a/template-only-docs/set-up-ci.md +++ b/template-only-docs/set-up-ci.md @@ -8,8 +8,13 @@ Some checks are disabled until you've completed certain setup steps: ### After setting up the application environment -After [setting up the app environment](/docs/infra/set-up-app-env.md): +After [setting up the app environment](/docs/infra/set-up-app-env.md), run: +```sh +nava-platform update --data is_dev_env_setup=true +``` + +TODO still needed? - Uncomment the infra end-to-end tests by searching for `!!` in [ci-infra-service.yml](/.github/workflows/ci-infra-service.yml). You can verify that CI is running and passing by clicking into the Actions tab in GitHub. Note that this repo only contains CI for infra. Application CI (`ci-app.yml`) is included as part of the application templates. - Uncomment the push trigger in [cd-app.yml](/.github/workflows/cd-app.yml) - If you setup your AWS account in a different region than `us-east-1`, update the `aws-region` workflow settings to match your region. From 0097c5bc969511eca8a95ba323b994cc2a54546c Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Mon, 4 Nov 2024 17:03:49 -0500 Subject: [PATCH 09/61] Make infra service checks per-app? --- ...=> ci-{{app_name}}-infra-service.yml.jinja} | 18 +++++++++--------- Makefile | 3 ++- infra/test/infra_test.go | 12 +++++++----- 3 files changed, 18 insertions(+), 15 deletions(-) rename .github/workflows/{ci-infra-service.yml.jinja => ci-{{app_name}}-infra-service.yml.jinja} (75%) diff --git a/.github/workflows/ci-infra-service.yml.jinja b/.github/workflows/ci-{{app_name}}-infra-service.yml.jinja similarity index 75% rename from .github/workflows/ci-infra-service.yml.jinja rename to .github/workflows/ci-{{app_name}}-infra-service.yml.jinja index a3f362a1..faa81b36 100644 --- a/.github/workflows/ci-infra-service.yml.jinja +++ b/.github/workflows/ci-{{app_name}}-infra-service.yml.jinja @@ -1,4 +1,4 @@ -name: CI Infra Service Checks +name: CI Infra Service Checks - {{ app_name }} on: {% if is_dev_env_setup %} @@ -6,16 +6,16 @@ on: branches: - main paths: - - infra/*/service/** + - infra/{{ app_name }}/service/** - infra/modules/** - infra/test/** - - .github/workflows/ci-infra-service.yml + - .github/workflows/ci-{{ app_name }}-infra-service.yml pull_request: paths: - - infra/*/service/** + - infra/{{ app_name }}/service/** - infra/modules/** - infra/test/** - - .github/workflows/ci-infra-service.yml + - .github/workflows/ci-{{ app_name }}-infra-service.yml {% else %} # !! Once you've set up the dev environment and are ready to enable automated # infra tests, run `nava-platform update --data is_dev_env_setup=true` to @@ -25,13 +25,13 @@ on: # branches: # - main # paths: - # - infra/*/service/** + # - infra/{{ app_name }}/service/** # - infra/modules/** # - infra/test/** # - .github/workflows/ci-infra-service.yml # pull_request: # paths: - # - infra/*/service/** + # - infra/{{ app_name }}/service/** # - infra/modules/** # - infra/test/** # - .github/workflows/ci-infra-service.yml @@ -62,9 +62,9 @@ jobs: - name: Configure AWS credentials uses: ./.github/actions/configure-aws-credentials with: - app_name: app + app_name: {{ app_name }} # Run infra CI on dev environment environment: dev - name: Run Terratest - run: make infra-test-service + run: make infra-test-service APP_NAME={{ app_name }} diff --git a/Makefile b/Makefile index 1e1b6a73..36b70b51 100644 --- a/Makefile +++ b/Makefile @@ -220,7 +220,8 @@ infra-format: ## Format infra code terraform fmt -recursive infra infra-test-service: ## Run service layer infra test suite - cd infra/test && go test -run TestService -v -timeout 30m + @:$(call check_defined, APP_NAME, "the name of subdirectory of /infra that holds the application's infrastructure code") + cd infra/test && APP_NAME=$(APP_NAME) go test -run TestService -v -timeout 30m ############# ## Linting ## diff --git a/infra/test/infra_test.go b/infra/test/infra_test.go index a264fbec..64bf8cbe 100644 --- a/infra/test/infra_test.go +++ b/infra/test/infra_test.go @@ -2,6 +2,7 @@ package test import ( "fmt" + "os" "strings" "testing" "time" @@ -14,6 +15,7 @@ import ( var uniqueId = strings.ToLower(random.UniqueId()) var workspaceName = fmt.Sprintf("t-%s", uniqueId) +var testAppName = os.Getenv("APP_NAME") func TestService(t *testing.T) { BuildAndPublish(t) @@ -25,7 +27,7 @@ func TestService(t *testing.T) { }) terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{ Reconfigure: true, - TerraformDir: "../app/service/", + TerraformDir: fmt.Sprintf("../%s/service/", testAppName), Vars: map[string]interface{}{ "environment_name": "dev", "image_tag": imageTag, @@ -59,14 +61,14 @@ func BuildAndPublish(t *testing.T) { // after which we add BackendConfig: []string{"dev.s3.tfbackend": terraform.KeyOnly} to terraformOptions // and replace the call to terraform.RunTerraformCommand with terraform.Init TerraformInit(t, &terraform.Options{ - TerraformDir: "../app/build-repository/", + TerraformDir: fmt.Sprintf("../%s/build-repository/", testAppName), }, "shared.s3.tfbackend") fmt.Println("::endgroup::") fmt.Println("::group::Build release") shell.RunCommand(t, shell.Command{ Command: "make", - Args: []string{"release-build", "APP_NAME=app"}, + Args: []string{"release-build", fmt.Sprintf("APP_NAME=%s", testAppName)}, WorkingDir: "../../", }) fmt.Println("::endgroup::") @@ -74,7 +76,7 @@ func BuildAndPublish(t *testing.T) { fmt.Println("::group::Publish release") shell.RunCommand(t, shell.Command{ Command: "make", - Args: []string{"release-publish", "APP_NAME=app"}, + Args: []string{"release-publish", fmt.Sprintf("APP_NAME=%s", testAppName)}, WorkingDir: "../../", }) fmt.Println("::endgroup::") @@ -82,7 +84,7 @@ func BuildAndPublish(t *testing.T) { func WaitForServiceToBeStable(t *testing.T, workspaceName string) { fmt.Println("::group::Wait for service to be stable") - appName := "app" + appName := testAppName environmentName := "dev" serviceName := fmt.Sprintf("%s-%s-%s", workspaceName, appName, environmentName) shell.RunCommand(t, shell.Command{ From fcff0d15efbd081feefbd39061e63e0423921012 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Mon, 4 Nov 2024 17:24:21 -0500 Subject: [PATCH 10/61] Dummy change for updating --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 36b70b51..2e483215 100644 --- a/Makefile +++ b/Makefile @@ -223,6 +223,7 @@ infra-test-service: ## Run service layer infra test suite @:$(call check_defined, APP_NAME, "the name of subdirectory of /infra that holds the application's infrastructure code") cd infra/test && APP_NAME=$(APP_NAME) go test -run TestService -v -timeout 30m + ############# ## Linting ## ############# From 0733d9659ea4062559e81ba849d7119a93f8a200 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Tue, 5 Nov 2024 12:28:34 -0500 Subject: [PATCH 11/61] Use `/health` endpoint for check in infra_test.go --- Makefile | 1 - infra/test/infra_test.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 2e483215..36b70b51 100644 --- a/Makefile +++ b/Makefile @@ -223,7 +223,6 @@ infra-test-service: ## Run service layer infra test suite @:$(call check_defined, APP_NAME, "the name of subdirectory of /infra that holds the application's infrastructure code") cd infra/test && APP_NAME=$(APP_NAME) go test -run TestService -v -timeout 30m - ############# ## Linting ## ############# diff --git a/infra/test/infra_test.go b/infra/test/infra_test.go index 64bf8cbe..76b5971d 100644 --- a/infra/test/infra_test.go +++ b/infra/test/infra_test.go @@ -98,7 +98,7 @@ func WaitForServiceToBeStable(t *testing.T, workspaceName string) { func RunEndToEndTests(t *testing.T, terraformOptions *terraform.Options) { fmt.Println("::group::Check service for healthy status 200") serviceEndpoint := terraform.Output(t, terraformOptions, "service_endpoint") - http_helper.HttpGetWithRetryWithCustomValidation(t, serviceEndpoint, nil, 5, 1*time.Second, func(responseStatus int, responseBody string) bool { + http_helper.HttpGetWithRetryWithCustomValidation(t, serviceEndpoint+"/health", nil, 5, 1*time.Second, func(responseStatus int, responseBody string) bool { return responseStatus == 200 }) fmt.Println("::endgroup::") From f2d5e344183873793926a9a95c7b0d69ca4c3702 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Tue, 5 Nov 2024 15:32:23 -0500 Subject: [PATCH 12/61] Disable TLS checks in infra_test.go --- infra/test/infra_test.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/infra/test/infra_test.go b/infra/test/infra_test.go index 76b5971d..e3fb30ba 100644 --- a/infra/test/infra_test.go +++ b/infra/test/infra_test.go @@ -1,6 +1,7 @@ package test import ( + "crypto/tls" "fmt" "os" "strings" @@ -98,7 +99,16 @@ func WaitForServiceToBeStable(t *testing.T, workspaceName string) { func RunEndToEndTests(t *testing.T, terraformOptions *terraform.Options) { fmt.Println("::group::Check service for healthy status 200") serviceEndpoint := terraform.Output(t, terraformOptions, "service_endpoint") - http_helper.HttpGetWithRetryWithCustomValidation(t, serviceEndpoint+"/health", nil, 5, 1*time.Second, func(responseStatus int, responseBody string) bool { + + // if the service is using the `enable_https` option, there will be issues + // with certs using `service_endpoint`, like: + // + // tls: failed to verify certificate: x509: certificate is valid for .-.navateam.com, not ---..elb.amazonaws.com. Sleeping for 1s and will try again. + // + // so have the test skip over invalid certs + tlsConfig := tls.Config{InsecureSkipVerify: true} + + http_helper.HttpGetWithRetryWithCustomValidation(t, serviceEndpoint+"/health", &tlsConfig, 5, 1*time.Second, func(responseStatus int, responseBody string) bool { return responseStatus == 200 }) fmt.Println("::endgroup::") From dd8a1c73fdaf9cb5f76fb6c1be1be2614e7c746f Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Tue, 5 Nov 2024 13:30:48 -0500 Subject: [PATCH 13/61] Update docs for broken links and consistently use --- .github/workflows/README.md | 12 ++++++------ bin/account-ids-by-name | 2 +- docs/e2e/e2e-checks.md | 6 +++--- docs/feature-flags.md | 2 +- docs/infra/environment-variables-and-secrets.md | 6 +++--- docs/infra/pull-request-environments.md | 4 ++-- docs/infra/set-up-app-build-repository.md | 4 ++-- docs/infra/set-up-app-env.md | 8 ++++---- docs/infra/set-up-network.md | 2 +- docs/infra/set-up-public-internet-access.md | 2 +- infra/README.md | 4 ++-- template-only-docs/set-up-cd.md | 3 +-- template-only-docs/set-up-ci.md | 5 +---- template-only-docs/set-up-pr-environments.md | 4 ++-- 14 files changed, 30 insertions(+), 34 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index db3692f6..1ef4a949 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -8,12 +8,12 @@ The CI/CD for this project uses [reusable Github Actions workflows](https://docs Each app should have: -- `ci-[app_name]`: must be created; should run linting and testing -- `ci-[app_name]-vulnerability-scans`: calls `vulnerability-scans` +- `ci-`: must be created; should run linting and testing +- `ci--vulnerability-scans`: calls `vulnerability-scans` - Based on [ci-app-vulnerability-scans](https://github.com/navapbc/template-infra/blob/main/.github/workflows/ci-app-vulnerability-scans.yml) -- `ci-[app_name]-pr-environment-checks.yml`: calls `pr-environment-checks.yml` to create or update a pull request environment (see [pull request environments](/docs/infra/pull-request-environments.md)) +- `ci--pr-environment-checks.yml`: calls `pr-environment-checks.yml` to create or update a pull request environment (see [pull request environments](/docs/infra/pull-request-environments.md)) - Based on [ci-app-pr-environment-checks.yml](/.github/workflows/ci-app-pr-environment-checks.yml) -- `ci-[app_name]-pr-environment-destroy.yml`: calls `pr-environment-destroy.yml` to destroy the pull request environment (see [pull request environments](/docs/infra/pull-request-environments.md)) +- `ci--pr-environment-destroy.yml`: calls `pr-environment-destroy.yml` to destroy the pull request environment (see [pull request environments](/docs/infra/pull-request-environments.md)) - Based on [ci-app-pr-environment-destroy.yml](https://github.com/navapbc/template-infra/blob/main/.github/workflows/ci-app-pr-environment-destroy.yml) ### App-agnostic workflows @@ -26,7 +26,7 @@ Each app should have: Each app should have: -- `cd-[app_name]`: deploys an application +- `cd-`: deploys an application - Based on [`cd-app`](https://github.com/navapbc/template-infra/blob/main/.github/workflows/cd-app.yml) The CD workflow uses these reusable workflows: @@ -47,4 +47,4 @@ graph TD ## ⛑️ Helper workflows -- [`check-ci-cd-auth`](./check-ci-cd-auth.yml): verifes that the project's Github repo is able to connect to AWS +- [`check-ci-cd-auth`](./check-ci-cd-auth.yml): verifies that the project's Github repo is able to connect to AWS diff --git a/bin/account-ids-by-name b/bin/account-ids-by-name index 0169934e..5eb6d076 100755 --- a/bin/account-ids-by-name +++ b/bin/account-ids-by-name @@ -5,7 +5,7 @@ set -euo pipefail # We use script dir to make this script agnostic to where it's called from. -# This is needed since this script its called from infra//build-repository +# This is needed since this script its called from infra//build-repository # in an external data source script_dir=$(dirname "$0") diff --git a/docs/e2e/e2e-checks.md b/docs/e2e/e2e-checks.md index 02c71d92..9cd25f93 100644 --- a/docs/e2e/e2e-checks.md +++ b/docs/e2e/e2e-checks.md @@ -88,9 +88,9 @@ The [E2E Tests Workflow](../../.github/workflows/e2e-tests.yml) takes a `service ## Configuration The E2E tests are configured using the following files: -- [Base Configuration](../../e2e/playwright.config.js) -- [App-specific Configuration](../../e2e/app/playwright.config.js) +- Base Configuration in `e2e/playwright.config.js` +- App-specific Configuration in `e2e//playwright.config.js` The app-specific configuration files extend the common base configuration. -By default when running `make e2e-test APP_NAME=app BASE_URL=http://localhost:3000 ` - you don't necessarily need to pass an `BASE_URL` since the default is defined in the app-specific playwright config (`./e2e/app/playwright.config.js`). +By default when running `make e2e-test APP_NAME=app BASE_URL=http://localhost:3000 ` - you don't necessarily need to pass an `BASE_URL` since the default is defined in the app-specific playwright config (`./e2e//playwright.config.js`). diff --git a/docs/feature-flags.md b/docs/feature-flags.md index 4c053b74..aef66a76 100644 --- a/docs/feature-flags.md +++ b/docs/feature-flags.md @@ -8,7 +8,7 @@ This project leverages [Amazon CloudWatch Evidently](https://docs.aws.amazon.com ## Creating feature flags -The list of feature flags for an application is defined in the `feature_flags` property in its app-config module (in `/infra/[app_name]/app-config/feature-flags.tf`). To create a new feature flag, add a new string to that list. To remove a feature flag, remove the feature flag from the list. The set of feature flags will be updated on the next `terraform apply` of the service layer, or during the next deploy of the application. +The list of feature flags for an application is defined in the `feature_flags` property in its app-config module (in `/infra//app-config/feature-flags.tf`). To create a new feature flag, add a new string to that list. To remove a feature flag, remove the feature flag from the list. The set of feature flags will be updated on the next `terraform apply` of the service layer, or during the next deploy of the application. ## Querying feature flags in the application diff --git a/docs/infra/environment-variables-and-secrets.md b/docs/infra/environment-variables-and-secrets.md index f5dd71c6..16385eb9 100644 --- a/docs/infra/environment-variables-and-secrets.md +++ b/docs/infra/environment-variables-and-secrets.md @@ -8,7 +8,7 @@ Applications may need application specific configuration as environment variable > ⚠️ Note: Do not put sensitive information such as credentials as regular environment variables. The method described in this section will embed the environment variables and their values in the ECS task definition's container definitions, so anyone with access to view the task definition will be able to see the values of the environment variables. For configuring secrets, see the section below on [Secrets](#secrets) -Environment variables are defined in the `app-config` module in the [environment_variables.tf file](/infra/app/app-config/env-config/environment_variables.tf). Modify the `default_extra_environment_variables` map to define extra environment variables specific to the application. Map keys define the environment variable name, and values define the default value for the variable across application environments. For example: +Environment variables are defined in `infra//app-config/env-config/environment_variables.tf`. Modify the `default_extra_environment_variables` map to define extra environment variables specific to the application. Map keys define the environment variable name, and values define the default value for the variable across application environments. For example: ```terraform # environment_variables.tf @@ -21,7 +21,7 @@ locals { } ``` -To override the default values for a particular environment, modify the `app-config/[environment].tf file` for the environment, and pass overrides to the `env-config` module using the `service_override_extra_environment_variables` variable. For example: +To override the default values for a particular environment, modify the `app-config/.tf` file for the environment, and pass overrides to the `env-config` module using the `service_override_extra_environment_variables` variable. For example: ```terraform # dev.tf @@ -40,7 +40,7 @@ module "dev_config" { Secrets are a specific category of environment variables that need to be handled sensitively. Examples of secrets are authentication credentials such as API keys for external services. Secrets first need to be stored in AWS SSM Parameter Store as a `SecureString`. This section then describes how to make those secrets accessible to the ECS task as environment variables through the `secrets` configuration in the container definition (see AWS documentation on [retrieving Secrets Manager secrets through environment variables](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/secrets-envvar-secrets-manager.html)). -Secrets are defined in the same file that non-sensitive environment variables are defined, in the `app-config` module in the [environment_variables.tf file](/infra/app/app-config/env-config/environment_variables.tf). Modify the `secrets` map to define the secrets that the application will have access to. For each secret, the map key defines the environment variable name. The `manage_method` property, which can be set to `"generated"` or `"manual"`, defines whether or not to generate a random secret or to reference an existing secret that was manually created and stored into AWS SSM. The `secret_store_name` property defines the SSM parameter name that stores the secret value. If `manage_method = "generated"`, then `secret_store_name` is where terraform will store the secret. If `manage_method = "manual"`, then `secret_store_name` is where terraform will look for the existing secret. For example: +Secrets are defined in the same file that non-sensitive environment variables are defined, in `infra//app-config/env-config/environment_variables.tf`. Modify the `secrets` map to define the secrets that the application will have access to. For each secret, the map key defines the environment variable name. The `manage_method` property, which can be set to `"generated"` or `"manual"`, defines whether or not to generate a random secret or to reference an existing secret that was manually created and stored into AWS SSM. The `secret_store_name` property defines the SSM parameter name that stores the secret value. If `manage_method = "generated"`, then `secret_store_name` is where terraform will store the secret. If `manage_method = "manual"`, then `secret_store_name` is where terraform will look for the existing secret. For example: ```terraform # environment_variables.tf diff --git a/docs/infra/pull-request-environments.md b/docs/infra/pull-request-environments.md index 4953140e..4051d048 100644 --- a/docs/infra/pull-request-environments.md +++ b/docs/infra/pull-request-environments.md @@ -28,7 +28,7 @@ Pull request environments are created by GitHub Actions workflows. There are two Using these reusable workflows, configure PR environments for each application with application-specific workflows: -- `ci-[app_name]-pr-environment-checks.yml` +- `ci--pr-environment-checks.yml` - Based on [ci-app-pr-environment-checks.yml](https://github.com/navapbc/template-infra/blob/main/.github/workflows/ci-app-pr-environment-checks.yml) -- `ci-[app_name]-pr-environment-destroy.yml` +- `ci--pr-environment-destroy.yml` - Based on [ci-app-pr-environment-destroy.yml](https://github.com/navapbc/template-infra/blob/main/.github/workflows/ci-app-pr-environment-destroy.yml) diff --git a/docs/infra/set-up-app-build-repository.md b/docs/infra/set-up-app-build-repository.md index eb30bb49..d9305bc6 100644 --- a/docs/infra/set-up-app-build-repository.md +++ b/docs/infra/set-up-app-build-repository.md @@ -7,7 +7,7 @@ The application build repository setup process will create infrastructure resour Before setting up the application's build repository you'll need to have: 1. [Set up the AWS account](./set-up-aws-account.md) -2. [Configure the app](/infra/app/app-config/main.tf) +2. Configure the app in `infra//app-config/main.tf`. ## 1. Configure backend @@ -17,7 +17,7 @@ To create the `tfbackend` file for the build repository using the backend config make infra-configure-app-build-repository APP_NAME=app ``` -Pass in the name of the app folder within `infra`. By default this is `app`. +Pass in the name of the app folder within `infra`. ## 2. Create build repository resources diff --git a/docs/infra/set-up-app-env.md b/docs/infra/set-up-app-env.md index 702b9223..f43bda68 100644 --- a/docs/infra/set-up-app-env.md +++ b/docs/infra/set-up-app-env.md @@ -10,7 +10,7 @@ Before setting up the application's environments you'll need to have: 1. [A compatible application in the app folder](https://github.com/navapbc/template-infra/blob/main/template-only-docs/application-requirements.md) 2. [Set up the AWS account that this environment is going to use](/docs/infra/set-up-aws-account.md). -3. [Configure the app](/infra/app/app-config/main.tf). +3. Configure the app in `infra//app-config/main.tf`. 1. Make sure you update `has_database` to `true` or `false` (defaults to `true`) depending on whether or not your application has a database to integrate with. 2. Make sure you update `has_external_non_aws_service` to `true` or `false` depending on whether your application utilizes any non-AWS services. Other applications within the same git repo count as external services, so if your application makes API calls to another application service in the same git repo, set `has_external_non_aws_service` to `true`. 3. If you're configuring your production environment, make sure to update the `service_cpu`, `service_memory`, and `service_desired_instance_count` settings based on the project's needs. If your application is sensitive to performance, consider doing a load test. @@ -29,10 +29,10 @@ To create the `tfbackend` and `tfvars` files for the new application environment make infra-configure-app-service APP_NAME=app ENVIRONMENT= ``` -`APP_NAME` needs to be the name of the application folder within the `infra` folder. It defaults to `app`. -`ENVIRONMENT` needs to be the name of the environment you are creating. This will create a file called `.s3.tfbackend` in the `infra/app/service` module directory. +`APP_NAME` needs to be the name of the application folder within the `infra` folder. +`ENVIRONMENT` needs to be the name of the environment you are creating. This will create a file called `.s3.tfbackend` in the `infra//service` module directory. -Depending on the value of `has_database` in the [app-config module](/infra/app/app-config/main.tf), the application will be configured with or without database access. +Depending on the value of `has_database` in the app-config module (`infra//app-config/main.tf`), the application will be configured with or without database access. ## 2. Build and publish the application to the application build repository diff --git a/docs/infra/set-up-network.md b/docs/infra/set-up-network.md index f5b7e3c5..cba96afd 100644 --- a/docs/infra/set-up-network.md +++ b/docs/infra/set-up-network.md @@ -14,7 +14,7 @@ Before setting up the network you'll need to have: 2. Optionally adjust the configuration for the networks you want to have on your project in the [project-config module](/infra/project-config/networks.tf). By default, there are three networks defined, one for each application environment. If you have multiple apps and want your applications in separate networks, you may want to give the networks differentiating names (e.g. "foo-dev", "foo-prod", "bar-dev", "bar-prod", instead of just "dev", "prod"). 1. Optionally, [configure custom domains](/docs/infra/set-up-custom-domains.md). You can also come back to setting up custom domains at a later time. 2. Optionally, [configure HTTPS support](/docs/infra/https-support.md). You can also come back to setting up HTTPS support at a later time. -3. [Configure the app](/infra/app/app-config/main.tf). +3. Configure the app in `infra//app-config/main.tf`. 1. Update `has_database` to `true` or `false` depending on whether or not your application has a database to integrate with. This setting determines whether or not to create VPC endpoints needed by the database layer. 2. Update `has_external_non_aws_service` to `true` or `false` depending on whether or not your application makes calls over the public internet. Set this to `true` (a) if your application makes calls to a SaaS service, or (b) if your application needs to call services from another application in the same git repo. This setting determines whether or not to create NAT gateways, which allows the service in the private subnet to make requests to the internet. For more information, see [set up network access to the public internet](./set-up-public-internet-access.md) 3. If you made changes to the configuration of the networks in the optional step 2 above and or to the default application environments: Update `network_name` for your application environments. This mapping ensures that each network is configured appropriately based on the application(s) in that network (see `local.apps_in_network` in [/infra/networks/main.tf](/infra/networks/main.tf)) Failure to set the network name properly means that the network layer may not receive the correct application configurations for `has_database` and `has_external_non_aws_service`. diff --git a/docs/infra/set-up-public-internet-access.md b/docs/infra/set-up-public-internet-access.md index 58a9f574..7f6e4387 100644 --- a/docs/infra/set-up-public-internet-access.md +++ b/docs/infra/set-up-public-internet-access.md @@ -13,7 +13,7 @@ Note: To access services that are provided directly by AWS, you can access them ## 1. Configure `has_external_non_aws_service` property in app-config module -In the `infra//app-config` module, set `has_external_non_aws_service` to `true`. +In the `infra//app-config` module, set `has_external_non_aws_service` to `true`. ## 2. Create or update the network diff --git a/infra/README.md b/infra/README.md index 0d104cac..942a1a68 100644 --- a/infra/README.md +++ b/infra/README.md @@ -10,7 +10,7 @@ The structure for the infrastructure code looks like this: infra/ Infrastructure code project-config/ Project-level configuration for account-level resources and resource tags accounts/ [Root module] IaC and IAM resources - [app_name]/ Application directory: infrastructure for the main application + / Application directory: infrastructure for the main application modules/ Reusable child modules networks/ [Root module] Account level network config (shared across all apps, environments, and terraform workspaces) ``` @@ -55,7 +55,7 @@ This project has the following AWS environments: - `staging` - `prod` -The environments share the same root modules but will have different configurations. Backend configuration is saved as [`.tfbackend`](https://developer.hashicorp.com/terraform/language/backend#file) files. Most `.tfbackend` files are named after the environment. For example, the `[app_name]/service` infrastructure resources for the `dev` environment are configured via `dev.s3.tfbackend`. Resources for a module that are shared across environments, such as the build-repository, use `shared.s3.tfbackend`. Resources that are shared across the entire account (e.g. /infra/accounts) use `..s3.tfbackend`. +The environments share the same root modules but will have different configurations. Backend configuration is saved as [`.tfbackend`](https://developer.hashicorp.com/terraform/language/backend#file) files. Most `.tfbackend` files are named after the environment. For example, the `/service` infrastructure resources for the `dev` environment are configured via `dev.s3.tfbackend`. Resources for a module that are shared across environments, such as the build-repository, use `shared.s3.tfbackend`. Resources that are shared across the entire account (e.g. /infra/accounts) use `..s3.tfbackend`. ### 🔀 Project workflow diff --git a/template-only-docs/set-up-cd.md b/template-only-docs/set-up-cd.md index e716febe..ae7b173e 100644 --- a/template-only-docs/set-up-cd.md +++ b/template-only-docs/set-up-cd.md @@ -7,5 +7,4 @@ enable continuous deployment by running: nava-platform update --data is_dev_env_setup=true ``` -TODO still needed? -1. Update the `role-to-assume` with the GitHub actions ARN. +And update the `role-to-assume` with the GitHub actions ARN. diff --git a/template-only-docs/set-up-ci.md b/template-only-docs/set-up-ci.md index 14bb439c..21be3426 100644 --- a/template-only-docs/set-up-ci.md +++ b/template-only-docs/set-up-ci.md @@ -14,7 +14,4 @@ After [setting up the app environment](/docs/infra/set-up-app-env.md), run: nava-platform update --data is_dev_env_setup=true ``` -TODO still needed? -- Uncomment the infra end-to-end tests by searching for `!!` in [ci-infra-service.yml](/.github/workflows/ci-infra-service.yml). You can verify that CI is running and passing by clicking into the Actions tab in GitHub. Note that this repo only contains CI for infra. Application CI (`ci-app.yml`) is included as part of the application templates. -- Uncomment the push trigger in [cd-app.yml](/.github/workflows/cd-app.yml) -- If you setup your AWS account in a different region than `us-east-1`, update the `aws-region` workflow settings to match your region. +If you setup your AWS account in a different region than `us-east-1`, update the `aws-region` workflow settings to match your region. diff --git a/template-only-docs/set-up-pr-environments.md b/template-only-docs/set-up-pr-environments.md index 09e3d5d1..9aae59a9 100644 --- a/template-only-docs/set-up-pr-environments.md +++ b/template-only-docs/set-up-pr-environments.md @@ -2,7 +2,7 @@ [Pull request environments](/docs/infra/pull-request-environments.md) are temporary environments that exist during a pull request. Enable them after [setting up the app environment](/docs/infra/set-up-app-env.md): -- In [ci-app-pr-environment-checks.yml](/.github/workflows/ci-app-pr-environment-checks.yml) and [ci-app-pr-environment-destroy.yml](/.github/workflows/ci-app-pr-environment-destroy.yml), search for `!!`. -- Uncomment the PR environment triggers. +- In [ci-{{app_name}}-pr-environment-checks.yml](/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml) and [ci-{{app_name}}-pr-environment-destroy.yml](/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml), search for `!!`. +- Uncomment the PR environment triggers for spot testing. Run `nava-platform update --data is_dev_env_setup=true` to consistently enable things. You can verify that PR environments are working by opening a new PR and waiting for the "PR Environment Update" job to finish. From a023ba342ee06bc239239e1486b0b57451f7d943 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Tue, 5 Nov 2024 15:47:24 -0500 Subject: [PATCH 14/61] Update decision docs for template usage --- ...e-infra-config-from-tfvars-files-into-config-module.md | 8 ++++---- docs/decisions/infra/0010-feature-flags-system-design.md | 6 +++--- docs/decisions/infra/0011-network-layer-design.md | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/decisions/infra/0008-consolidate-infra-config-from-tfvars-files-into-config-module.md b/docs/decisions/infra/0008-consolidate-infra-config-from-tfvars-files-into-config-module.md index 3174021d..6fcbe441 100644 --- a/docs/decisions/infra/0008-consolidate-infra-config-from-tfvars-files-into-config-module.md +++ b/docs/decisions/infra/0008-consolidate-infra-config-from-tfvars-files-into-config-module.md @@ -8,20 +8,20 @@ Technical Story: [Replace configure scripts with project/app config variables #3 ## Context -Currently, application infrastructure configuration is split across config modules (see [app-config](/infra/app/app-config/)) as well as .tfvars files in each of the application's infra layers - infra/app/build-repository, infra/app/database, and infra/app/service. As @kyeah pointed out, it’s easy to make mistakes when configuration is spread across multiple files, and expressed a desire to manage tfvars across environments all in a single file the way that some applications do for application configuration. Also, as @acouch [pointed out](https://github.com/navapbc/template-infra/pull/282#discussion_r1219930653), there is a lot of duplicate code with the configure scripts (setup-current-account.sh, configure-app-build-repository.sh, configure-app-database.sh, configure-app-service.sh) that configure the backend config and variable files for each infrastructure layer, which increases the burden of maintaining the configuration scripts. +Currently, application infrastructure configuration is split across config modules (see `infra//app-config/`) as well as .tfvars files in each of the application's infra layers - infra//build-repository, infra//database, and infra//service. As @kyeah pointed out, it’s easy to make mistakes when configuration is spread across multiple files, and expressed a desire to manage tfvars across environments all in a single file the way that some applications do for application configuration. Also, as @acouch [pointed out](https://github.com/navapbc/template-infra/pull/282#discussion_r1219930653), there is a lot of duplicate code with the configure scripts (setup-current-account.sh, configure-app-build-repository.sh, configure-app-database.sh, configure-app-service.sh) that configure the backend config and variable files for each infrastructure layer, which increases the burden of maintaining the configuration scripts. ## Overview This ADR proposes the following: -* Move all environment configuration into [app-config](/infra/app/app-config/) modules +* Move all environment configuration into `infra//app-config/` modules * Remove the need for .tfvars files * Remove the configuration scripts that are currently used for configuring each infrastructure layer Benefits: -* All configuration can now be managed in the [app-config](/infra/app/app-config/) module. -* All dependencies between root modules can be managed explicitly via the [app-config](/infra/app/app-config/) module. +* All configuration can now be managed in the `infra//app-config/` module. +* All dependencies between root modules can be managed explicitly via the `infra//app-config/` module. * Custom configuration scripts no longer need to be maintained * Eliminates the need to specify -var-file option when running terraform apply, which reduces the need for terraform wrapper scripts diff --git a/docs/decisions/infra/0010-feature-flags-system-design.md b/docs/decisions/infra/0010-feature-flags-system-design.md index 092cac70..07a5b074 100644 --- a/docs/decisions/infra/0010-feature-flags-system-design.md +++ b/docs/decisions/infra/0010-feature-flags-system-design.md @@ -23,7 +23,7 @@ One key design question is how features should be managed once defined. How shou ### Option 1. Manage features using app-config module as part of service layer -Define features in [app-config](/infra/app/app-config/), and use that configuration in the [service layer](/infra/app/service/) to create and configure the features in AWS Evidently. +Define features in app-config (`infra//app-config/`), and use that configuration in the service layer (`infra//service/`) to create and configure the features in AWS Evidently. * Everything is defined in code and in one place. * Feature and feature configurations are updated automatically as part of service deploys or can be done manually via a `terraform apply`. @@ -46,13 +46,13 @@ features = { ### Option 2. Manage features using app-config as part of a separate infrastructure layer -Define features in [app-config](/infra/app/app-config/main.tf). Create the features in the [service layer](/infra/app/service/) but set things like throttle percentages (for gradual rollouts) in a separate infrastructure layer. +Define features in app-config (`infra//app-config/main.tf`). Create the features in the service layer (`infra//service/`) but set things like throttle percentages (for gradual rollouts) in a separate infrastructure layer. * Allows for separation of permissions. For example, individuals can have permission to update feature launch throttle percentages without having permission to create, edit, or delete the features themselves. ### Option 3. Manage features in AWS Console outside of Terraform -Define features in [app-config](/infra/app/app-config/main.tf) and create them in the [service layer](/infra/app/service), but set things like throttle percentages (for gradual rollouts) outside of Terraform (e.g. via AWS Console). Use `lifecycle { ignore_changes = [entity_overrides] }` in the terraform configuration for the `aws_evidently_feature` resources to ignore settings that are managed via the AWS Console. +Define features in app-config (`infra//app-config/main.tf`) and create them in the service layer (`/infra//service/`), but set things like throttle percentages (for gradual rollouts) outside of Terraform (e.g. via AWS Console). Use `lifecycle { ignore_changes = [entity_overrides] }` in the terraform configuration for the `aws_evidently_feature` resources to ignore settings that are managed via the AWS Console. * Empowers non-technical roles like business owners and product managers to enable and disable feature flags and adjust feature launch throttle percentages without needing to depend on the development team. * A no-code approach using the AWS Console GUI means that it's possible to leverage the full set of functionality offered by AWS CloudWatch Evidently, including things like scheduled launches, with minimal training and without needing to learn how to do it in code. diff --git a/docs/decisions/infra/0011-network-layer-design.md b/docs/decisions/infra/0011-network-layer-design.md index 863e5b64..f3252498 100644 --- a/docs/decisions/infra/0011-network-layer-design.md +++ b/docs/decisions/infra/0011-network-layer-design.md @@ -19,7 +19,7 @@ We aim to achieve these requirements without adding complexity to the other laye ### Network configuration -Define the configuration for networks in a new property `network_configs` in the [project-config module](/infra/project-config/main.tf). `network_configs` is a map from the network name to the network configuration. The network name is a name the project team chooses to serve as a human-readable identifier to reference the network. To keep network configuration DRY and reuse common configurations between networks, create a sub-module `network-config` under the project-config module, analogous to the [env-config module](/infra/app/app-config/env-config/) under the [app-config module](/infra/app/app-config/). The `project-config` module might look something like this: +Define the configuration for networks in a new property `network_configs` in the project-config module (`infra/project-config/main.tf`). `network_configs` is a map from the network name to the network configuration. The network name is a name the project team chooses to serve as a human-readable identifier to reference the network. To keep network configuration DRY and reuse common configurations between networks, create a sub-module `network-config` under the project-config module, analogous to the env-config module (`infra//app-config/env-config/`) under the app-config module (`/infra//app-config/`). The `project-config` module might look something like this: ```terraform # project-config/main.tf @@ -59,7 +59,7 @@ Add a "network_name" name tag to the VPC with the name of the network. The VPC t ## Application-specific network configuration -In order to determine which VPC to use for each application environment, add a `network_name` property to the [environment config](/infra/app/app-config/env-config/). The network name will be used in the database and service layers by the `aws_vpc` data source: +In order to determine which VPC to use for each application environment, add a `network_name` property to the environment config (`infra//app-config/env-config/`). The network name will be used in the database and service layers by the `aws_vpc` data source: ```terraform data "aws_vpc" "network" { From 81f514d8eba281e7d2b31d0c5bb5a6f047761b3d Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Tue, 5 Nov 2024 16:15:45 -0500 Subject: [PATCH 15/61] More broken doc link updates --- .github/workflows/README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 1ef4a949..02f56914 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -2,6 +2,8 @@ The CI/CD for this project uses [reusable Github Actions workflows](https://docs.github.com/en/actions/using-workflows/reusing-workflows). +TODO: update the workflow links to point to blob/main before merge + ## 🧪 CI ### Per app workflows @@ -10,11 +12,11 @@ Each app should have: - `ci-`: must be created; should run linting and testing - `ci--vulnerability-scans`: calls `vulnerability-scans` - - Based on [ci-app-vulnerability-scans](https://github.com/navapbc/template-infra/blob/main/.github/workflows/ci-app-vulnerability-scans.yml) + - Based on [ci-{{app_name}}-vulnerability-scans](https://github.com/navapbc/template-infra/blob/lorenyu%2Fplatform-cli/.github/workflows/ci-{{app_name}}-vulnerability-scans.yml.jinja) - `ci--pr-environment-checks.yml`: calls `pr-environment-checks.yml` to create or update a pull request environment (see [pull request environments](/docs/infra/pull-request-environments.md)) - - Based on [ci-app-pr-environment-checks.yml](/.github/workflows/ci-app-pr-environment-checks.yml) + - Based on [ci-{{app_name}}-pr-environment-checks.yml](https://github.com/navapbc/template-infra/blob/lorenyu%2Fplatform-cli/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja) - `ci--pr-environment-destroy.yml`: calls `pr-environment-destroy.yml` to destroy the pull request environment (see [pull request environments](/docs/infra/pull-request-environments.md)) - - Based on [ci-app-pr-environment-destroy.yml](https://github.com/navapbc/template-infra/blob/main/.github/workflows/ci-app-pr-environment-destroy.yml) + - Based on [ci-{{app_name}}-pr-environment-destroy.yml](https://github.com/navapbc/template-infra/blob/lorenyu%2Fplatform-cli/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja) ### App-agnostic workflows @@ -27,7 +29,7 @@ Each app should have: Each app should have: - `cd-`: deploys an application - - Based on [`cd-app`](https://github.com/navapbc/template-infra/blob/main/.github/workflows/cd-app.yml) + - Based on [`cd-{{app_name}}`](https://github.com/navapbc/template-infra/blob/lorenyu%2Fplatform-cli/.github/workflows/cd-{{app_name}}.yml.jinja) The CD workflow uses these reusable workflows: From 37a2962f9c12fbc86afa4125ad068b3ab4d88e85 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Tue, 5 Nov 2024 16:18:17 -0500 Subject: [PATCH 16/61] More broken doc link updates --- template-only-docs/set-up-pr-environments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template-only-docs/set-up-pr-environments.md b/template-only-docs/set-up-pr-environments.md index 9aae59a9..bcfe92bc 100644 --- a/template-only-docs/set-up-pr-environments.md +++ b/template-only-docs/set-up-pr-environments.md @@ -2,7 +2,7 @@ [Pull request environments](/docs/infra/pull-request-environments.md) are temporary environments that exist during a pull request. Enable them after [setting up the app environment](/docs/infra/set-up-app-env.md): -- In [ci-{{app_name}}-pr-environment-checks.yml](/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml) and [ci-{{app_name}}-pr-environment-destroy.yml](/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml), search for `!!`. +- In [ci-{{app_name}}-pr-environment-checks.yml](/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja) and [ci-{{app_name}}-pr-environment-destroy.yml](/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja), search for `!!`. - Uncomment the PR environment triggers for spot testing. Run `nava-platform update --data is_dev_env_setup=true` to consistently enable things. You can verify that PR environments are working by opening a new PR and waiting for the "PR Environment Update" job to finish. From b6f626e1fe31145f6df42c683b65e9c789333f88 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Tue, 5 Nov 2024 17:49:47 -0500 Subject: [PATCH 17/61] Update template-infra's infra check to install example project to run against --- .github/workflows/template-only-ci-infra.yml | 32 +++++++++++++++++++- template-only-test/template_infra_test.go | 18 ++--------- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/.github/workflows/template-only-ci-infra.yml b/.github/workflows/template-only-ci-infra.yml index 48603244..99d1e2da 100644 --- a/.github/workflows/template-only-ci-infra.yml +++ b/.github/workflows/template-only-ci-infra.yml @@ -32,14 +32,42 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - name: Checkout template-infra repo + uses: actions/checkout@v4 + with: + path: template-infra + # TODO: Revert to checking out main once we've merged the platform-cli branch into the main branch + ref: lorenyu/platform-cli + - uses: hashicorp/setup-terraform@v2 with: terraform_version: 1.8.3 terraform_wrapper: false + - uses: actions/setup-go@v3 with: go-version: ">=1.19.0" + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install nava-platform CLI + run: pipx install --python python3.13 git+https://github.com/navapbc/platform-cli + + - name: Install infra template + run: | + nava-platform infra install --template-uri ./template-infra \ + --data app_name=app \ + --data local_port=3000 \ + --data is_dev_env_setup=true \ + project-dir + + - name: Copy template-only things to project copy of template + run: | + cp -vr ./template-infra/template-only* ./project-dir + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v3 with: @@ -56,7 +84,9 @@ jobs: aws-secret-access-key: ${{ secrets.TESTER_AWS_SECRET_ACCESS_KEY }} - name: Run Terratest + working-directory: project-dir run: make -f template-only.mak test env: # GitHub token needed for GitHub CLI which is used during tests to check GitHub Actions auth GH_TOKEN: ${{ secrets.PLATFORM_TESTER_GITHUB_TOKEN }} + IMAGE_TAG: ${{ github.sha }} diff --git a/template-only-test/template_infra_test.go b/template-only-test/template_infra_test.go index 33049c11..f0bc5fdd 100644 --- a/template-only-test/template_infra_test.go +++ b/template-only-test/template_infra_test.go @@ -2,6 +2,7 @@ package test import ( "fmt" + "os" "strings" "testing" "time" @@ -18,6 +19,8 @@ import ( var uniqueId = strings.ToLower(random.UniqueId()) var projectName = fmt.Sprintf("plt-tst-act-%s", uniqueId) +var imageTag = os.Getenv("IMAGE_TAG") + func TestEndToEnd(t *testing.T) { defer TeardownAccount(t) SetUpProject(t, projectName) @@ -113,13 +116,6 @@ func SetUpDevEnvironment(t *testing.T) { WorkingDir: "../", }) - // Get current commit hash, which should be the one that was deployed as part of validating the build-repository - imageTag := shell.RunCommandAndGetOutput(t, shell.Command{ - Command: "git", - Args: []string{"rev-parse", "HEAD"}, - WorkingDir: "./", - }) - shell.RunCommand(t, shell.Command{ Command: "make", Args: []string{"infra-update-app-service", "APP_NAME=app", "ENVIRONMENT=dev"}, @@ -241,11 +237,3 @@ func TeardownDevEnvironment(t *testing.T) { }) fmt.Println("::endgroup::") } - -func GetCurrentCommitHash(t *testing.T) string { - return shell.RunCommandAndGetOutput(t, shell.Command{ - Command: "git", - Args: []string{"rev-parse", "HEAD"}, - WorkingDir: "./", - }) -} From 25904e1071516e7921deda28773de86bd58cd264 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Wed, 6 Nov 2024 17:51:01 -0500 Subject: [PATCH 18/61] Pull infra/project-config/main.tf into template --- .github/workflows/template-only-ci-infra.yml | 13 ++++- copier.yml | 56 +++++++++++++++++++ .../project-config/{main.tf => main.tf.jinja} | 8 +-- template-only-bin/set-up-project | 39 ------------- template-only-test/template_infra_test.go | 9 +-- 5 files changed, 74 insertions(+), 51 deletions(-) rename infra/project-config/{main.tf => main.tf.jinja} (78%) diff --git a/.github/workflows/template-only-ci-infra.yml b/.github/workflows/template-only-ci-infra.yml index d47e2dea..15a34829 100644 --- a/.github/workflows/template-only-ci-infra.yml +++ b/.github/workflows/template-only-ci-infra.yml @@ -58,7 +58,17 @@ jobs: - name: Install infra template run: | + # Note: projectName can't be too long since S3 bucket names have a 63 + # character max length, $RANDOM has a max value of 32767 + unique_id=$(printf '%.5d' $((RANDOM))) + project_name=plt-tst-act-${unique_id} + echo "PROJECT_NAME=${project_name}" >> "$GITHUB_ENV" + nava-platform infra install --template-uri ./template-infra \ + --data project_name="${project_name}" \ + --data owner=navapbc \ + --data code_repository_url=${{ github.repositoryUrl }} \ + --data default_region=us-east-1 \ --data app_name=app \ --data local_port=3000 \ --data is_dev_env_setup=true \ @@ -75,7 +85,7 @@ jobs: # Use access key credentials for the template infra test workflow # instead of using GitHub OIDC because only one GitHub OIDC provider - # can be added to the Platform accoutn, and we want to create a + # can be added to the Platform account, and we want to create a # GitHub OIDC provider as part of the template test. # (see https://docs.aws.amazon.com/cli/latest/reference/iam/create-open-id-connect-provider.html) # @@ -90,3 +100,4 @@ jobs: # GitHub token needed for GitHub CLI which is used during tests to check GitHub Actions auth GH_TOKEN: ${{ secrets.PLATFORM_TESTER_GITHUB_TOKEN }} IMAGE_TAG: ${{ github.sha }} + PROJECT_NAME: ${{ env.PROJECT_NAME }} diff --git a/copier.yml b/copier.yml index d76502d8..54bf857b 100644 --- a/copier.yml +++ b/copier.yml @@ -1,3 +1,59 @@ +# +# Base vars +# +project_name: + type: str + help: The slugified name of the project (lower case, dashes, and underscores) + +owner: + type: str + help: Project owner slug (used for tagging infra resources) + +code_repository_url: + type: str + help: URL of project source code repository + +default_region: + type: str + help: Default AWS region for project + default: us-east-2 + choices: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-south-2 + - ap-southeast-3 + - ap-southeast-5 + - ap-southeast-4 + - ap-south-1 + - ap-northeast-3 + - ap-northeast-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-northeast-1 + - ca-central-1 + - ca-west-1 + - cn-north-1 + - cn-northwest-1 + - eu-central-1 + - eu-west-1 + - eu-west-2 + - eu-south-1 + - eu-west-3 + - eu-south-2 + - eu-north-1 + - eu-central-2 + - il-central-1 + - me-south-1 + - me-central-1 + - sa-east-1 + +# +# App vars +# app_name: type: str help: The name of the app diff --git a/infra/project-config/main.tf b/infra/project-config/main.tf.jinja similarity index 78% rename from infra/project-config/main.tf rename to infra/project-config/main.tf.jinja index e3ecf2df..419c43ec 100644 --- a/infra/project-config/main.tf +++ b/infra/project-config/main.tf.jinja @@ -1,18 +1,18 @@ locals { # Machine readable project name (lower case letters, dashes, and underscores) # This will be used in names of AWS resources - project_name = "" + project_name = "{{ project_name }}" # Project owner (e.g. navapbc). Used for tagging infra resources. - owner = "" + owner = "{{ owner }}" # URL of project source code repository - code_repository_url = "" + code_repository_url = "{{ code_repository_url }}" # Default AWS region for project (e.g. us-east-1, us-east-2, us-west-1). # This is dependent on where your project is located (if regional) # otherwise us-east-1 is a good default - default_region = "" + default_region = "{{ default_region }}" github_actions_role_name = "${local.project_name}-github-actions" diff --git a/template-only-bin/set-up-project b/template-only-bin/set-up-project index 2432dd9a..74b2ba5e 100755 --- a/template-only-bin/set-up-project +++ b/template-only-bin/set-up-project @@ -1,45 +1,6 @@ #!/bin/bash set -euo pipefail -project_name="$1" -owner="$2" -default_region="$3" -repo_url=$(git remote get-url origin) - -echo "Account configuration" -echo "=====================" -echo "PROJECT_NAME=${project_name}" -echo "REPO_URL=${repo_url}" -echo - -cd infra/project-config - -echo "-------------------------------------" -echo "Replace placeholder values in main.tf" -echo "-------------------------------------" - -# First replace the placeholder value for in main.tf -# The project name is used to define unique names for the infrastructure -# resources that are created in subsequent infra setup steps. -sed -i.bak "s//${project_name}/" main.tf - -# Then replace the placeholder value for in main.tf -# The repository is needed to set up the GitHub OpenID Connect provider -# in AWS which allows GitHub Actions to authenticate with our AWS account -# when called from our repository only. -# Use '|' as the regex delimiter for sed instead of '/' since -# REPO_URL will have a '/' in it -sed -i.bak "s||${repo_url}|" main.tf - -# Replace remaining placeholder values -sed -i.bak "s//${owner}/" main.tf -sed -i.bak "s//${default_region}/" main.tf - -# Remove the backup file created by sed -rm main.tf.bak - -cd - - # Set has_database to false for template only CI since database setup takes too long sed -i.bak "s/has_database = true/has_database = false/" infra/app/app-config/main.tf rm infra/app/app-config/main.tf.bak diff --git a/template-only-test/template_infra_test.go b/template-only-test/template_infra_test.go index f0bc5fdd..b1722241 100644 --- a/template-only-test/template_infra_test.go +++ b/template-only-test/template_infra_test.go @@ -3,22 +3,17 @@ package test import ( "fmt" "os" - "strings" "testing" "time" "github.com/gruntwork-io/terratest/modules/aws" http_helper "github.com/gruntwork-io/terratest/modules/http-helper" - "github.com/gruntwork-io/terratest/modules/random" "github.com/gruntwork-io/terratest/modules/shell" "github.com/gruntwork-io/terratest/modules/terraform" "github.com/stretchr/testify/assert" ) -// Note: projectName can't be too long since S3 bucket names have a 63 character max length -var uniqueId = strings.ToLower(random.UniqueId()) -var projectName = fmt.Sprintf("plt-tst-act-%s", uniqueId) - +var projectName = os.Getenv("PROJECT_NAME") var imageTag = os.Getenv("IMAGE_TAG") func TestEndToEnd(t *testing.T) { @@ -60,7 +55,7 @@ func SetUpProject(t *testing.T, projectName string) { fmt.Println("::group::Configuring project") shell.RunCommand(t, shell.Command{ Command: "make", - Args: []string{"-f", "template-only.mak", "set-up-project", fmt.Sprintf("PROJECT_NAME=%s", projectName)}, + Args: []string{"-f", "template-only.mak", "set-up-project"}, WorkingDir: "../", }) fmt.Println("::endgroup::") From 4ad924aaa69f6dcfbd3e1aae9b600af91d43a303 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Thu, 7 Nov 2024 11:24:08 -0500 Subject: [PATCH 19/61] Fix doc link lint due to project-config templating --- .github/workflows/template-only-ci-infra.yml | 4 ++-- docs/infra/set-up-aws-account.md | 1 + infra/README.md | 10 +++++----- template-only.mak | 4 +--- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/template-only-ci-infra.yml b/.github/workflows/template-only-ci-infra.yml index 15a34829..45d827c2 100644 --- a/.github/workflows/template-only-ci-infra.yml +++ b/.github/workflows/template-only-ci-infra.yml @@ -58,7 +58,7 @@ jobs: - name: Install infra template run: | - # Note: projectName can't be too long since S3 bucket names have a 63 + # Note: project_name can't be too long since S3 bucket names have a 63 # character max length, $RANDOM has a max value of 32767 unique_id=$(printf '%.5d' $((RANDOM))) project_name=plt-tst-act-${unique_id} @@ -66,7 +66,7 @@ jobs: nava-platform infra install --template-uri ./template-infra \ --data project_name="${project_name}" \ - --data owner=navapbc \ + --data owner=platform-admins \ --data code_repository_url=${{ github.repositoryUrl }} \ --data default_region=us-east-1 \ --data app_name=app \ diff --git a/docs/infra/set-up-aws-account.md b/docs/infra/set-up-aws-account.md index 827246d7..18bcb158 100644 --- a/docs/infra/set-up-aws-account.md +++ b/docs/infra/set-up-aws-account.md @@ -9,6 +9,7 @@ The AWS account setup process will: ## Prerequisites * You'll need to have [set up infrastructure tools](./set-up-infrastructure-tools.md), like Terraform, AWS CLI, and AWS authentication. + * You'll also need to make sure the [project is configured](/infra/project-config/main.tf). ## Instructions diff --git a/infra/README.md b/infra/README.md index 942a1a68..20bdcf41 100644 --- a/infra/README.md +++ b/infra/README.md @@ -70,11 +70,11 @@ Generally, you should use the Make targets or the underlying bin scripts, but yo To set up this project for the first time (i.e., it has never been deployed to the target AWS account): 1. [Install this template](/README.md#installation) into an application that meets the [Application Requirements](/README.md#application-requirements) -2. [Configure the project](/infra/project-config/main.tf) (These values will be used in subsequent infra setup steps to namespace resources and add infrastructure tags.) -3. [Set up infrastructure developer tools](/docs/infra/set-up-infrastructure-tools.md) -4. [Set up AWS account](/docs/infra/set-up-aws-account.md) -5. [Set up the virtual network (VPC)](/docs/infra/set-up-network.md) -6. For each application: + 1. You may need to tweak the generated [project configuration](/infra/project-config/main.tf) depending on your needs. +2. [Set up infrastructure developer tools](/docs/infra/set-up-infrastructure-tools.md) +3. [Set up AWS account](/docs/infra/set-up-aws-account.md) +4. [Set up the virtual network (VPC)](/docs/infra/set-up-network.md) +5. For each application: 1. [Set up application build repository](/docs/infra/set-up-app-build-repository.md) 2. [Set up application database](/docs/infra/set-up-database.md) 3. [Set up application environment](/docs/infra/set-up-app-env.md) diff --git a/template-only.mak b/template-only.mak index 10eb3ee6..2ca51928 100644 --- a/template-only.mak +++ b/template-only.mak @@ -2,8 +2,6 @@ # and is not intended to be used by projects that are using the template PROJECT_NAME ?= platform-template-infra -PROJECT_OWNER ?= platform-admins -PROJECT_REGION ?= us-east-1 .PHONY : \ clean \ @@ -18,7 +16,7 @@ test: cd template-only-test && PROJECT_NAME=$(PROJECT_NAME) go test -v -timeout 30m set-up-project: - ./template-only-bin/set-up-project $(PROJECT_NAME) $(PROJECT_OWNER) $(PROJECT_REGION) + ./template-only-bin/set-up-project $(PROJECT_NAME) clean: rm -fr infra/accounts/account/.terraform infra/app/envs/dev/.terraform infra/app/envs/staging/.terraform infra/app/envs/prod/.terraform infra/app/build-repository/.terraform From f968f029578836baf5b0d54fe1b9f0cd2973bfd5 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Thu, 7 Nov 2024 15:24:17 -0500 Subject: [PATCH 20/61] Fix IMAGE_TAG when not in a git repo Need to define DATE before using it. --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 36b70b51..8a01179e 100644 --- a/Makefile +++ b/Makefile @@ -238,6 +238,10 @@ lint-markdown: ## Lint Markdown docs for broken links # does not conflict with other images during local development IMAGE_NAME := $(PROJECT_ROOT)-$(APP_NAME) +# Generate an informational tag so we can see where every image comes from. +DATE := $(shell date -u '+%Y%m%d.%H%M%S') +INFO_TAG := $(DATE).$(USER) + GIT_REPO_AVAILABLE := $(shell git rev-parse --is-inside-work-tree 2>/dev/null) # Generate a unique tag based solely on the git hash. @@ -248,10 +252,6 @@ else IMAGE_TAG := "unknown-dev.$(DATE)" endif -# Generate an informational tag so we can see where every image comes from. -DATE := $(shell date -u '+%Y%m%d.%H%M%S') -INFO_TAG := $(DATE).$(USER) - release-build: ## Build release for $APP_NAME and tag it with current git hash @:$(call check_defined, APP_NAME, the name of subdirectory of /infra that holds the application's infrastructure code) cd $(APP_NAME) && $(MAKE) release-build \ From c44ae5a460399248c75ddda8ec4fe5f00f9bef7e Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Thu, 7 Nov 2024 15:30:23 -0500 Subject: [PATCH 21/61] template_infra_test.go: Build the IMAGE_TAG specified --- template-only-test/template_infra_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/template-only-test/template_infra_test.go b/template-only-test/template_infra_test.go index b1722241..8675a6c8 100644 --- a/template-only-test/template_infra_test.go +++ b/template-only-test/template_infra_test.go @@ -147,14 +147,14 @@ func ValidateBuildRepository(t *testing.T) { err := shell.RunCommandE(t, shell.Command{ Command: "make", - Args: []string{"release-build", "APP_NAME=app"}, + Args: []string{"release-build", "APP_NAME=app", fmt.Sprintf("IMAGE_TAG=%s", imageTag)}, WorkingDir: "../", }) assert.NoError(t, err, "Could not build release") err = shell.RunCommandE(t, shell.Command{ Command: "make", - Args: []string{"release-publish", "APP_NAME=app"}, + Args: []string{"release-publish", "APP_NAME=app", fmt.Sprintf("IMAGE_TAG=%s", imageTag)}, WorkingDir: "../", }) assert.NoError(t, err, "Could not publish release") From 536a572c63dac50093277647e07c0bd26b7d1042 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Thu, 7 Nov 2024 15:58:50 -0500 Subject: [PATCH 22/61] bin/check-github-actions-auth: be explicit about the repo to check --- bin/check-github-actions-auth | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bin/check-github-actions-auth b/bin/check-github-actions-auth index 202d5fbd..52f300bf 100755 --- a/bin/check-github-actions-auth +++ b/bin/check-github-actions-auth @@ -10,11 +10,11 @@ set -euo pipefail account_name="$1" +code_repository=$(terraform -chdir="infra/project-config" output --raw code_repository) + # This is used later to determine the run id of the workflow run # See comment below about "Getting workflow run id" -prev_run_create_time=$(gh run list --workflow check-ci-cd-auth.yml --limit 1 --json createdAt --jq ".[].createdAt") - -code_repository=$(terraform -chdir="infra/project-config" output --raw code_repository) +prev_run_create_time=$(gh run list --repo "${code_repository}" --workflow check-ci-cd-auth.yml --limit 1 --json createdAt --jq ".[].createdAt") echo "=========================" echo "Check GitHub Actions Auth" @@ -50,7 +50,7 @@ echo "::endgroup::" ## Run workflow ## ################## -gh workflow run check-ci-cd-auth.yml --field "aws_region=${aws_region}" --field "role_to_assume=${aws_role_to_assume}" +gh workflow run check-ci-cd-auth.yml --repo "${code_repository}" --field "aws_region=${aws_region}" --field "role_to_assume=${aws_role_to_assume}" ######################### ## Get workflow run id ## @@ -76,13 +76,13 @@ echo "Previous workflow run created at ${prev_run_create_time}" echo "Check workflow run create time until we find a newer workflow run" while : ; do echo -n "." - run_create_time=$(gh run list --workflow check-ci-cd-auth.yml --limit 1 --json createdAt --jq ".[].createdAt") + run_create_time=$(gh run list --repo "${code_repository}" --workflow check-ci-cd-auth.yml --limit 1 --json createdAt --jq ".[].createdAt") [[ "${run_create_time}" > "${prev_run_create_time}" ]] && break done echo "Found newer workflow run created at ${run_create_time}" echo "Get id of workflow run" -workflow_run_id=$(gh run list --workflow check-ci-cd-auth.yml --limit 1 --json databaseId --jq ".[].databaseId") +workflow_run_id=$(gh run list --repo "${code_repository}" --workflow check-ci-cd-auth.yml --limit 1 --json databaseId --jq ".[].databaseId") echo "Workflow run id: ${workflow_run_id}" workflow_run_url="https://github.com/${code_repository}/actions/runs/${workflow_run_id}" @@ -91,4 +91,4 @@ echo " ${workflow_run_url}" echo "Watch workflow run until it exits" # --exit-status causes command to exit with non-zero status if run fails -gh run watch "${workflow_run_id}" --exit-status +gh run watch "${workflow_run_id}" --repo "${code_repository}" --exit-status From 2e3954590056617c61b3d427467a9360486660ce Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Thu, 7 Nov 2024 16:13:50 -0500 Subject: [PATCH 23/61] template_infra_test.go: comment on not using /health --- template-only-test/template_infra_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/template-only-test/template_infra_test.go b/template-only-test/template_infra_test.go index 8675a6c8..5329b258 100644 --- a/template-only-test/template_infra_test.go +++ b/template-only-test/template_infra_test.go @@ -180,6 +180,8 @@ func ValidateDevEnvironment(t *testing.T) { TerraformDir: "../infra/app/service/", }) serviceEndpoint := terraform.Output(t, terraformOptions, "service_endpoint") + // Not checking the /health endpoint as we don't deploy the database for + // this testing, so that endpoint will fail as currently coded http_helper.HttpGetWithRetryWithCustomValidation(t, serviceEndpoint, nil, 10, 3*time.Second, func(responseStatus int, responseBody string) bool { return responseStatus == 200 }) From c53219c911a7e9a408f836aa80438cd6741dd346 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Thu, 7 Nov 2024 16:28:59 -0500 Subject: [PATCH 24/61] Prefix all copier variables with template part name (base or app) --- .github/workflows/cd-{{app_name}}.yml.jinja | 4 ++-- .../ci-{{app_name}}-infra-service.yml.jinja | 4 ++-- ...ci-{{app_name}}-pr-environment-checks.yml.jinja | 4 ++-- ...i-{{app_name}}-pr-environment-destroy.yml.jinja | 4 ++-- .github/workflows/template-only-ci-infra.yml | 14 ++++++++------ copier.yml | 12 ++++++------ e2e/{{app_name}}/playwright.config.js.jinja | 2 +- infra/project-config/main.tf.jinja | 8 ++++---- template-only-docs/set-up-cd.md | 2 +- template-only-docs/set-up-ci.md | 2 +- template-only-docs/set-up-pr-environments.md | 2 +- 11 files changed, 30 insertions(+), 28 deletions(-) diff --git a/.github/workflows/cd-{{app_name}}.yml.jinja b/.github/workflows/cd-{{app_name}}.yml.jinja index 601accbe..a7930bbd 100644 --- a/.github/workflows/cd-{{app_name}}.yml.jinja +++ b/.github/workflows/cd-{{app_name}}.yml.jinja @@ -4,7 +4,7 @@ name: Deploy {{ app_name }} run-name: Deploy ${{'{{'}}inputs.version || 'main' {{'}}'}} to {{ app_name }} ${{'{{'}} inputs.environment || 'dev' {{'}}'}} on: - {% if is_dev_env_setup %} + {% if app_is_dev_env_setup %} push: branches: - "main" @@ -14,7 +14,7 @@ on: - "infra/**" {% else %} # !! Once you've set up the dev environment and are ready to enable continuous - # deployment, run `nava-platform update --data is_dev_env_setup=true` to + # deployment, run `nava-platform update --data app_is_dev_env_setup=true` to # enable these lines. They are here as comments for context. # # push: diff --git a/.github/workflows/ci-{{app_name}}-infra-service.yml.jinja b/.github/workflows/ci-{{app_name}}-infra-service.yml.jinja index ec743807..a4ddd7e5 100644 --- a/.github/workflows/ci-{{app_name}}-infra-service.yml.jinja +++ b/.github/workflows/ci-{{app_name}}-infra-service.yml.jinja @@ -1,7 +1,7 @@ name: CI Infra Service Checks - {{ app_name }} on: - {% if is_dev_env_setup %} + {% if app_is_dev_env_setup %} push: branches: - main @@ -18,7 +18,7 @@ on: - .github/workflows/ci-{{ app_name }}-infra-service.yml {% else %} # !! Once you've set up the dev environment and are ready to enable automated - # infra tests, run `nava-platform update --data is_dev_env_setup=true` to + # infra tests, run `nava-platform update --data app_is_dev_env_setup=true` to # enable these lines. They are here as comments for context. # # push: diff --git a/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja b/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja index 6a47d6a5..c0d191cd 100644 --- a/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja +++ b/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja @@ -8,11 +8,11 @@ on: commit_hash: required: true type: string - {% if is_dev_env_setup %} + {% if app_is_dev_env_setup %} pull_request: {% else %} # !! Once you've set up the dev environment and are ready to enable PR - # environments, run `nava-platform update --data is_dev_env_setup=true` to + # environments, run `nava-platform update --data app_is_dev_env_setup=true` to # enable these lines. They are here as comments for context. # # pull_request: diff --git a/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja b/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja index 04e9e044..4b9cdb8f 100644 --- a/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja +++ b/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja @@ -5,12 +5,12 @@ on: pr_number: required: true type: string - {% if is_dev_env_setup %} + {% if app_is_dev_env_setup %} pull_request: types: [closed] {% else %} # !! Once you've set up the dev environment and are ready to enable PR - # environments, run `nava-platform update --data is_dev_env_setup=true` to + # environments, run `nava-platform update --data app_is_dev_env_setup=true` to # enable these lines. They are here as comments for context. # # pull_request: diff --git a/.github/workflows/template-only-ci-infra.yml b/.github/workflows/template-only-ci-infra.yml index 45d827c2..c3220f42 100644 --- a/.github/workflows/template-only-ci-infra.yml +++ b/.github/workflows/template-only-ci-infra.yml @@ -64,14 +64,16 @@ jobs: project_name=plt-tst-act-${unique_id} echo "PROJECT_NAME=${project_name}" >> "$GITHUB_ENV" + # TODO: add --data-file support to CLI and have as many of these + # settings in more maintainable separate file nava-platform infra install --template-uri ./template-infra \ - --data project_name="${project_name}" \ - --data owner=platform-admins \ - --data code_repository_url=${{ github.repositoryUrl }} \ - --data default_region=us-east-1 \ + --data base_project_name="${project_name}" \ + --data base_owner=platform-admins \ + --data base_code_repository_url=${{ github.repositoryUrl }} \ + --data base_default_region=us-east-1 \ --data app_name=app \ - --data local_port=3000 \ - --data is_dev_env_setup=true \ + --data app_local_port=3000 \ + --data app_is_dev_env_setup=true \ project-dir - name: Copy template-only things to project copy of template diff --git a/copier.yml b/copier.yml index 54bf857b..4f3b347c 100644 --- a/copier.yml +++ b/copier.yml @@ -1,19 +1,19 @@ # # Base vars # -project_name: +base_project_name: type: str help: The slugified name of the project (lower case, dashes, and underscores) -owner: +base_owner: type: str help: Project owner slug (used for tagging infra resources) -code_repository_url: +base_code_repository_url: type: str help: URL of project source code repository -default_region: +base_default_region: type: str help: Default AWS region for project default: us-east-2 @@ -58,12 +58,12 @@ app_name: type: str help: The name of the app -local_port: +app_local_port: type: int help: The port to be used in local development default: 3000 -is_dev_env_setup: +app_is_dev_env_setup: type: bool help: If there's a dev environment deployed, to enable various CI/CD. default: false diff --git a/e2e/{{app_name}}/playwright.config.js.jinja b/e2e/{{app_name}}/playwright.config.js.jinja index b00fde61..19dea871 100644 --- a/e2e/{{app_name}}/playwright.config.js.jinja +++ b/e2e/{{app_name}}/playwright.config.js.jinja @@ -6,7 +6,7 @@ export default defineConfig(deepMerge( baseConfig, { use: { - baseURL: baseConfig.use.baseURL || "localhost:{{ local_port }}" + baseURL: baseConfig.use.baseURL || "localhost:{{ app_local_port }}" }, } )); diff --git a/infra/project-config/main.tf.jinja b/infra/project-config/main.tf.jinja index 419c43ec..084d1563 100644 --- a/infra/project-config/main.tf.jinja +++ b/infra/project-config/main.tf.jinja @@ -1,18 +1,18 @@ locals { # Machine readable project name (lower case letters, dashes, and underscores) # This will be used in names of AWS resources - project_name = "{{ project_name }}" + project_name = "{{ base_project_name }}" # Project owner (e.g. navapbc). Used for tagging infra resources. - owner = "{{ owner }}" + owner = "{{ base_owner }}" # URL of project source code repository - code_repository_url = "{{ code_repository_url }}" + code_repository_url = "{{ base_code_repository_url }}" # Default AWS region for project (e.g. us-east-1, us-east-2, us-west-1). # This is dependent on where your project is located (if regional) # otherwise us-east-1 is a good default - default_region = "{{ default_region }}" + default_region = "{{ base_default_region }}" github_actions_role_name = "${local.project_name}-github-actions" diff --git a/template-only-docs/set-up-cd.md b/template-only-docs/set-up-cd.md index ae7b173e..cc5a54d2 100644 --- a/template-only-docs/set-up-cd.md +++ b/template-only-docs/set-up-cd.md @@ -4,7 +4,7 @@ Once you have set up your application environments (at least `dev`), you can enable continuous deployment by running: ```sh -nava-platform update --data is_dev_env_setup=true +nava-platform update --data app_is_dev_env_setup=true ``` And update the `role-to-assume` with the GitHub actions ARN. diff --git a/template-only-docs/set-up-ci.md b/template-only-docs/set-up-ci.md index 21be3426..d9e81095 100644 --- a/template-only-docs/set-up-ci.md +++ b/template-only-docs/set-up-ci.md @@ -11,7 +11,7 @@ Some checks are disabled until you've completed certain setup steps: After [setting up the app environment](/docs/infra/set-up-app-env.md), run: ```sh -nava-platform update --data is_dev_env_setup=true +nava-platform update --data app_is_dev_env_setup=true ``` If you setup your AWS account in a different region than `us-east-1`, update the `aws-region` workflow settings to match your region. diff --git a/template-only-docs/set-up-pr-environments.md b/template-only-docs/set-up-pr-environments.md index bcfe92bc..f48e502b 100644 --- a/template-only-docs/set-up-pr-environments.md +++ b/template-only-docs/set-up-pr-environments.md @@ -3,6 +3,6 @@ [Pull request environments](/docs/infra/pull-request-environments.md) are temporary environments that exist during a pull request. Enable them after [setting up the app environment](/docs/infra/set-up-app-env.md): - In [ci-{{app_name}}-pr-environment-checks.yml](/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja) and [ci-{{app_name}}-pr-environment-destroy.yml](/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja), search for `!!`. -- Uncomment the PR environment triggers for spot testing. Run `nava-platform update --data is_dev_env_setup=true` to consistently enable things. +- Uncomment the PR environment triggers for spot testing. Run `nava-platform update --data app_is_dev_env_setup=true` to consistently enable things. You can verify that PR environments are working by opening a new PR and waiting for the "PR Environment Update" job to finish. From 5ab08b33464d753cbb8a5d7d02e0bf432da7ef7d Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Thu, 7 Nov 2024 17:03:29 -0500 Subject: [PATCH 25/61] Only ask relevant questions for different template The CLI tool will inject the `template` answer, so generally the end users will never see/answer it directly. --- copier.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/copier.yml b/copier.yml index 4f3b347c..4ea87b44 100644 --- a/copier.yml +++ b/copier.yml @@ -1,21 +1,32 @@ +template: + type: str + choices: + - base + - app + # # Base vars # base_project_name: type: str help: The slugified name of the project (lower case, dashes, and underscores) + when: &base + "{{ template == 'base' }}" base_owner: type: str help: Project owner slug (used for tagging infra resources) + when: *base base_code_repository_url: type: str help: URL of project source code repository + when: *base base_default_region: type: str help: Default AWS region for project + when: *base default: us-east-2 choices: - us-east-1 @@ -57,16 +68,20 @@ base_default_region: app_name: type: str help: The name of the app + when: &app + "{{ template == 'app' }}" app_local_port: type: int help: The port to be used in local development default: 3000 + when: *app app_is_dev_env_setup: type: bool help: If there's a dev environment deployed, to enable various CI/CD. default: false + when: *app _envops: trim_blocks: true From 4c61efd8930ea077ade74759fc39a2d4abb1cefa Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Fri, 8 Nov 2024 16:34:41 -0500 Subject: [PATCH 26/61] misc. doc updates --- docs/releases.md | 9 +++++---- template-only-docs/application-requirements.md | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/releases.md b/docs/releases.md index 68a05e62..ef550805 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -5,12 +5,13 @@ To build a release, run ```bash -make release-build +make release-build APP_NAME= ``` -This builds the release from [app/Dockerfile](../app/Dockerfile). The Dockerfile -needs to have a build stage called `release` to act as the build target. -(See [Name your build stages](https://docs.docker.com/build/building/multi-stage/#name-your-build-stages)) +This calls the `release-build` target in `/Makefile` with some +parameters to build an image. The `/Dockerfile` should have a build +stage called `release` to act as the build target. (See [Name your build +stages](https://docs.docker.com/build/building/multi-stage/#name-your-build-stages)) ## Publishing a release diff --git a/template-only-docs/application-requirements.md b/template-only-docs/application-requirements.md index d3caace1..72edb3bb 100644 --- a/template-only-docs/application-requirements.md +++ b/template-only-docs/application-requirements.md @@ -6,7 +6,7 @@ In order to use the template infrastructure, you need an application that meets * The application folder needs to have a `Makefile` that has a build target `release-build` that takes in a Makefile variable `OPTS`, and passes those `OPTS` as options to the `docker build` command. The top level [Makefile](/Makefile) in this repo will call the application's `release-build` make target passing in release tags to tag the docker image with. * The web application needs to listen on the port defined by the environment variable `PORT`, rather than hardcode the `PORT`. This allows the infrastructure to configure the application to listen on a container port specified by the infrastructure. See [The Twelve-Factor App](https://12factor.net/) to learn more about designing applications to be portable to different infrastructure environments using environment variables. * The web application needs to have a health check endpoint at `/health` that returns an HTTP 200 OK response when the application is healthy and ready to accept requests. -* The Docker image needs to have `wget` installed. This is used in the container task defintion's healthcheck configuration in order to ping the application's `/health` endpoint. If you want to use a different healthcheck command (e.g. `curl`) then you'll need to modify the `healthCheck` configuration in the `aws_ecs_task_definition` resource in [modules/service/main.tf](/infra/modules/service/main.tf). +* The Docker image needs to have `wget` installed. This is used in the container task definition's healthcheck configuration in order to ping the application's `/health` endpoint. If you want to use a different healthcheck command (e.g. `curl`) then you'll need to modify the `healthCheck` configuration in the `aws_ecs_task_definition` resource in [modules/service/main.tf](/infra/modules/service/main.tf). ## Database Requirements From aba8076d6ded06f638ce68331ff8ce396353e006 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Fri, 8 Nov 2024 11:47:17 -0500 Subject: [PATCH 27/61] move app/ to template-only-app/ --- .github/workflows/template-only-ci-infra.yml | 4 ++++ copier.yml | 4 ++++ {app => template-only-app}/Dockerfile | 0 {app => template-only-app}/Makefile | 0 {app => template-only-app}/app.py | 0 {app => template-only-app}/db-migrate | 0 {app => template-only-app}/db.py | 0 {app => template-only-app}/feature_flags.py | 0 {app => template-only-app}/migrations.sql | 0 {app => template-only-app}/requirements.txt | 0 {app => template-only-app}/storage.py | 0 {app => template-only-app}/templates/index.html | 0 template-only-docs/application-requirements.md | 2 +- {{app_name}}/Makefile | 9 +++++++++ 14 files changed, 18 insertions(+), 1 deletion(-) rename {app => template-only-app}/Dockerfile (100%) rename {app => template-only-app}/Makefile (100%) rename {app => template-only-app}/app.py (100%) rename {app => template-only-app}/db-migrate (100%) rename {app => template-only-app}/db.py (100%) rename {app => template-only-app}/feature_flags.py (100%) rename {app => template-only-app}/migrations.sql (100%) rename {app => template-only-app}/requirements.txt (100%) rename {app => template-only-app}/storage.py (100%) rename {app => template-only-app}/templates/index.html (100%) create mode 100644 {{app_name}}/Makefile diff --git a/.github/workflows/template-only-ci-infra.yml b/.github/workflows/template-only-ci-infra.yml index c3220f42..be0f029c 100644 --- a/.github/workflows/template-only-ci-infra.yml +++ b/.github/workflows/template-only-ci-infra.yml @@ -80,6 +80,10 @@ jobs: run: | cp -vr ./template-infra/template-only* ./project-dir + - name: Install example app for infra tests + run: | + mv -vf ./project-dir/template-only-app ./project-dir/app + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: diff --git a/copier.yml b/copier.yml index 4ea87b44..7e9f9658 100644 --- a/copier.yml +++ b/copier.yml @@ -86,3 +86,7 @@ app_is_dev_env_setup: _envops: trim_blocks: true lstrip_blocks: true + +_skip_if_exists: + - "/{{ app_name }}/" + - "/{{ app_name }}/Makefile" diff --git a/app/Dockerfile b/template-only-app/Dockerfile similarity index 100% rename from app/Dockerfile rename to template-only-app/Dockerfile diff --git a/app/Makefile b/template-only-app/Makefile similarity index 100% rename from app/Makefile rename to template-only-app/Makefile diff --git a/app/app.py b/template-only-app/app.py similarity index 100% rename from app/app.py rename to template-only-app/app.py diff --git a/app/db-migrate b/template-only-app/db-migrate similarity index 100% rename from app/db-migrate rename to template-only-app/db-migrate diff --git a/app/db.py b/template-only-app/db.py similarity index 100% rename from app/db.py rename to template-only-app/db.py diff --git a/app/feature_flags.py b/template-only-app/feature_flags.py similarity index 100% rename from app/feature_flags.py rename to template-only-app/feature_flags.py diff --git a/app/migrations.sql b/template-only-app/migrations.sql similarity index 100% rename from app/migrations.sql rename to template-only-app/migrations.sql diff --git a/app/requirements.txt b/template-only-app/requirements.txt similarity index 100% rename from app/requirements.txt rename to template-only-app/requirements.txt diff --git a/app/storage.py b/template-only-app/storage.py similarity index 100% rename from app/storage.py rename to template-only-app/storage.py diff --git a/app/templates/index.html b/template-only-app/templates/index.html similarity index 100% rename from app/templates/index.html rename to template-only-app/templates/index.html diff --git a/template-only-docs/application-requirements.md b/template-only-docs/application-requirements.md index 72edb3bb..effaf9bf 100644 --- a/template-only-docs/application-requirements.md +++ b/template-only-docs/application-requirements.md @@ -22,7 +22,7 @@ If your application needs a database, it must also: ## Example Application -The infra template includes an example "hello, world" application that works with the template. The source code for this test application is at [app](/app). +The infra template includes an example "hello, world" application that works with the template. The source code for this test application is at [template-only-app](/template-only-app). A live demo of the test application is fully deployed by the repo, which is used for testing the infra template. Please check [that repo's README](https://github.com/navapbc/platform-test?tab=readme-ov-file#environment-urls) to locate a URL for seeing the live demo. diff --git a/{{app_name}}/Makefile b/{{app_name}}/Makefile new file mode 100644 index 00000000..4794397a --- /dev/null +++ b/{{app_name}}/Makefile @@ -0,0 +1,9 @@ +.PHONY: \ + release-build + +release-build: + docker buildx build \ + --target release \ + --platform=linux/amd64 \ + $(OPTS) \ + . From 46b57773ee99618a2acaa9e24e3242d1a1ac820d Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Tue, 12 Nov 2024 11:24:06 -0500 Subject: [PATCH 28/61] Fix documented update commands --- .github/workflows/cd-{{app_name}}.yml.jinja | 7 +++++-- .github/workflows/ci-{{app_name}}-infra-service.yml.jinja | 7 +++++-- .../ci-{{app_name}}-pr-environment-checks.yml.jinja | 7 +++++-- .../ci-{{app_name}}-pr-environment-destroy.yml.jinja | 7 +++++-- template-only-docs/set-up-cd.md | 2 +- template-only-docs/set-up-ci.md | 2 +- template-only-docs/set-up-pr-environments.md | 2 +- 7 files changed, 23 insertions(+), 11 deletions(-) diff --git a/.github/workflows/cd-{{app_name}}.yml.jinja b/.github/workflows/cd-{{app_name}}.yml.jinja index a7930bbd..94bbd232 100644 --- a/.github/workflows/cd-{{app_name}}.yml.jinja +++ b/.github/workflows/cd-{{app_name}}.yml.jinja @@ -14,8 +14,11 @@ on: - "infra/**" {% else %} # !! Once you've set up the dev environment and are ready to enable continuous - # deployment, run `nava-platform update --data app_is_dev_env_setup=true` to - # enable these lines. They are here as comments for context. + # deployment, run: + # + # nava-platform infra update --data app_is_dev_env_setup=true . + # + # to enable these lines. They are here as comments for context. # # push: # branches: diff --git a/.github/workflows/ci-{{app_name}}-infra-service.yml.jinja b/.github/workflows/ci-{{app_name}}-infra-service.yml.jinja index a4ddd7e5..3857ad31 100644 --- a/.github/workflows/ci-{{app_name}}-infra-service.yml.jinja +++ b/.github/workflows/ci-{{app_name}}-infra-service.yml.jinja @@ -18,8 +18,11 @@ on: - .github/workflows/ci-{{ app_name }}-infra-service.yml {% else %} # !! Once you've set up the dev environment and are ready to enable automated - # infra tests, run `nava-platform update --data app_is_dev_env_setup=true` to - # enable these lines. They are here as comments for context. + # infra tests, run: + # + # nava-platform infra update --data app_is_dev_env_setup=true . + # + # to enable these lines. They are here as comments for context. # # push: # branches: diff --git a/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja b/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja index c0d191cd..13ed1a3e 100644 --- a/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja +++ b/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja @@ -12,8 +12,11 @@ on: pull_request: {% else %} # !! Once you've set up the dev environment and are ready to enable PR - # environments, run `nava-platform update --data app_is_dev_env_setup=true` to - # enable these lines. They are here as comments for context. + # environments, run: + # + # nava-platform infra update --data app_is_dev_env_setup=true . + # + # to enable these lines. They are here as comments for context. # # pull_request: {% endif %} diff --git a/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja b/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja index 4b9cdb8f..3f86174e 100644 --- a/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja +++ b/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja @@ -10,8 +10,11 @@ on: types: [closed] {% else %} # !! Once you've set up the dev environment and are ready to enable PR - # environments, run `nava-platform update --data app_is_dev_env_setup=true` to - # enable these lines. They are here as comments for context. + # environments, run: + # + # nava-platform infra update --data app_is_dev_env_setup=true . + # + # to enable these lines. They are here as comments for context. # # pull_request: # types: [closed] diff --git a/template-only-docs/set-up-cd.md b/template-only-docs/set-up-cd.md index cc5a54d2..1aef1471 100644 --- a/template-only-docs/set-up-cd.md +++ b/template-only-docs/set-up-cd.md @@ -4,7 +4,7 @@ Once you have set up your application environments (at least `dev`), you can enable continuous deployment by running: ```sh -nava-platform update --data app_is_dev_env_setup=true +nava-platform infra update --data app_is_dev_env_setup=true . ``` And update the `role-to-assume` with the GitHub actions ARN. diff --git a/template-only-docs/set-up-ci.md b/template-only-docs/set-up-ci.md index d9e81095..34a96049 100644 --- a/template-only-docs/set-up-ci.md +++ b/template-only-docs/set-up-ci.md @@ -11,7 +11,7 @@ Some checks are disabled until you've completed certain setup steps: After [setting up the app environment](/docs/infra/set-up-app-env.md), run: ```sh -nava-platform update --data app_is_dev_env_setup=true +nava-platform infra update --data app_is_dev_env_setup=true . ``` If you setup your AWS account in a different region than `us-east-1`, update the `aws-region` workflow settings to match your region. diff --git a/template-only-docs/set-up-pr-environments.md b/template-only-docs/set-up-pr-environments.md index f48e502b..18e09e3b 100644 --- a/template-only-docs/set-up-pr-environments.md +++ b/template-only-docs/set-up-pr-environments.md @@ -3,6 +3,6 @@ [Pull request environments](/docs/infra/pull-request-environments.md) are temporary environments that exist during a pull request. Enable them after [setting up the app environment](/docs/infra/set-up-app-env.md): - In [ci-{{app_name}}-pr-environment-checks.yml](/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja) and [ci-{{app_name}}-pr-environment-destroy.yml](/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja), search for `!!`. -- Uncomment the PR environment triggers for spot testing. Run `nava-platform update --data app_is_dev_env_setup=true` to consistently enable things. +- Uncomment the PR environment triggers for spot testing. Run `nava-platform infra update --data app_is_dev_env_setup=true .` to consistently enable things. You can verify that PR environments are working by opening a new PR and waiting for the "PR Environment Update" job to finish. From a602c819229c2a85aa35ea8ecfd965834d89164d Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Tue, 12 Nov 2024 12:09:02 -0500 Subject: [PATCH 29/61] app_is_dev_env_setup -> app_has_dev_env_setup Until we think of something better. --- .github/workflows/cd-{{app_name}}.yml.jinja | 4 ++-- .github/workflows/ci-{{app_name}}-infra-service.yml.jinja | 4 ++-- .../workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja | 4 ++-- .../ci-{{app_name}}-pr-environment-destroy.yml.jinja | 4 ++-- .github/workflows/template-only-ci-infra.yml | 2 +- copier.yml | 2 +- template-only-docs/set-up-cd.md | 2 +- template-only-docs/set-up-ci.md | 2 +- template-only-docs/set-up-pr-environments.md | 2 +- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/cd-{{app_name}}.yml.jinja b/.github/workflows/cd-{{app_name}}.yml.jinja index 94bbd232..0442a04b 100644 --- a/.github/workflows/cd-{{app_name}}.yml.jinja +++ b/.github/workflows/cd-{{app_name}}.yml.jinja @@ -4,7 +4,7 @@ name: Deploy {{ app_name }} run-name: Deploy ${{'{{'}}inputs.version || 'main' {{'}}'}} to {{ app_name }} ${{'{{'}} inputs.environment || 'dev' {{'}}'}} on: - {% if app_is_dev_env_setup %} + {% if app_has_dev_env_setup %} push: branches: - "main" @@ -16,7 +16,7 @@ on: # !! Once you've set up the dev environment and are ready to enable continuous # deployment, run: # - # nava-platform infra update --data app_is_dev_env_setup=true . + # nava-platform infra update --data app_has_dev_env_setup=true . # # to enable these lines. They are here as comments for context. # diff --git a/.github/workflows/ci-{{app_name}}-infra-service.yml.jinja b/.github/workflows/ci-{{app_name}}-infra-service.yml.jinja index 3857ad31..406e008f 100644 --- a/.github/workflows/ci-{{app_name}}-infra-service.yml.jinja +++ b/.github/workflows/ci-{{app_name}}-infra-service.yml.jinja @@ -1,7 +1,7 @@ name: CI Infra Service Checks - {{ app_name }} on: - {% if app_is_dev_env_setup %} + {% if app_has_dev_env_setup %} push: branches: - main @@ -20,7 +20,7 @@ on: # !! Once you've set up the dev environment and are ready to enable automated # infra tests, run: # - # nava-platform infra update --data app_is_dev_env_setup=true . + # nava-platform infra update --data app_has_dev_env_setup=true . # # to enable these lines. They are here as comments for context. # diff --git a/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja b/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja index 13ed1a3e..764ea91b 100644 --- a/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja +++ b/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja @@ -8,13 +8,13 @@ on: commit_hash: required: true type: string - {% if app_is_dev_env_setup %} + {% if app_has_dev_env_setup %} pull_request: {% else %} # !! Once you've set up the dev environment and are ready to enable PR # environments, run: # - # nava-platform infra update --data app_is_dev_env_setup=true . + # nava-platform infra update --data app_has_dev_env_setup=true . # # to enable these lines. They are here as comments for context. # diff --git a/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja b/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja index 3f86174e..62bb4579 100644 --- a/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja +++ b/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja @@ -5,14 +5,14 @@ on: pr_number: required: true type: string - {% if app_is_dev_env_setup %} + {% if app_has_dev_env_setup %} pull_request: types: [closed] {% else %} # !! Once you've set up the dev environment and are ready to enable PR # environments, run: # - # nava-platform infra update --data app_is_dev_env_setup=true . + # nava-platform infra update --data app_has_dev_env_setup=true . # # to enable these lines. They are here as comments for context. # diff --git a/.github/workflows/template-only-ci-infra.yml b/.github/workflows/template-only-ci-infra.yml index be0f029c..f8cce0a8 100644 --- a/.github/workflows/template-only-ci-infra.yml +++ b/.github/workflows/template-only-ci-infra.yml @@ -73,7 +73,7 @@ jobs: --data base_default_region=us-east-1 \ --data app_name=app \ --data app_local_port=3000 \ - --data app_is_dev_env_setup=true \ + --data app_has_dev_env_setup=true \ project-dir - name: Copy template-only things to project copy of template diff --git a/copier.yml b/copier.yml index 7e9f9658..aa3b58ad 100644 --- a/copier.yml +++ b/copier.yml @@ -77,7 +77,7 @@ app_local_port: default: 3000 when: *app -app_is_dev_env_setup: +app_has_dev_env_setup: type: bool help: If there's a dev environment deployed, to enable various CI/CD. default: false diff --git a/template-only-docs/set-up-cd.md b/template-only-docs/set-up-cd.md index 1aef1471..83c9c3e4 100644 --- a/template-only-docs/set-up-cd.md +++ b/template-only-docs/set-up-cd.md @@ -4,7 +4,7 @@ Once you have set up your application environments (at least `dev`), you can enable continuous deployment by running: ```sh -nava-platform infra update --data app_is_dev_env_setup=true . +nava-platform infra update --data app_has_dev_env_setup=true . ``` And update the `role-to-assume` with the GitHub actions ARN. diff --git a/template-only-docs/set-up-ci.md b/template-only-docs/set-up-ci.md index 34a96049..dd26da74 100644 --- a/template-only-docs/set-up-ci.md +++ b/template-only-docs/set-up-ci.md @@ -11,7 +11,7 @@ Some checks are disabled until you've completed certain setup steps: After [setting up the app environment](/docs/infra/set-up-app-env.md), run: ```sh -nava-platform infra update --data app_is_dev_env_setup=true . +nava-platform infra update --data app_has_dev_env_setup=true . ``` If you setup your AWS account in a different region than `us-east-1`, update the `aws-region` workflow settings to match your region. diff --git a/template-only-docs/set-up-pr-environments.md b/template-only-docs/set-up-pr-environments.md index 18e09e3b..1df2db04 100644 --- a/template-only-docs/set-up-pr-environments.md +++ b/template-only-docs/set-up-pr-environments.md @@ -3,6 +3,6 @@ [Pull request environments](/docs/infra/pull-request-environments.md) are temporary environments that exist during a pull request. Enable them after [setting up the app environment](/docs/infra/set-up-app-env.md): - In [ci-{{app_name}}-pr-environment-checks.yml](/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja) and [ci-{{app_name}}-pr-environment-destroy.yml](/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja), search for `!!`. -- Uncomment the PR environment triggers for spot testing. Run `nava-platform infra update --data app_is_dev_env_setup=true .` to consistently enable things. +- Uncomment the PR environment triggers for spot testing. Run `nava-platform infra update --data app_has_dev_env_setup=true .` to consistently enable things. You can verify that PR environments are working by opening a new PR and waiting for the "PR Environment Update" job to finish. From 3f8617140ed7329056cb31c9abaedf2d074cff47 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Tue, 12 Nov 2024 12:27:03 -0500 Subject: [PATCH 30/61] Question tweaks --- copier.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/copier.yml b/copier.yml index aa3b58ad..41eb0012 100644 --- a/copier.yml +++ b/copier.yml @@ -10,6 +10,10 @@ template: base_project_name: type: str help: The slugified name of the project (lower case, dashes, and underscores) + validator: >- + {% if not (base_project_name | regex_search('^[a-z0-9\-_]+$')) %} + The project name can not be empty and should only contain lower case letters, digits, dashes, and underscores. + {% endif %} when: &base "{{ template == 'base' }}" @@ -21,11 +25,12 @@ base_owner: base_code_repository_url: type: str help: URL of project source code repository + default: "https://github.com/{{ base_owner }}/{{ base_project_name }}" when: *base base_default_region: type: str - help: Default AWS region for project + help: Default AWS region for the project when: *base default: us-east-2 choices: @@ -68,18 +73,22 @@ base_default_region: app_name: type: str help: The name of the app + validator: >- + {% if not (app_name | regex_search('^[a-z0-9\-_]+$')) %} + The app name can not be empty and should only contain lower case letters, digits, dashes, and underscores. + {% endif %} when: &app "{{ template == 'app' }}" app_local_port: type: int - help: The port to be used in local development + help: "The port to be used in local development of '{{ app_name }}'" default: 3000 when: *app app_has_dev_env_setup: type: bool - help: If there's a dev environment deployed, to enable various CI/CD. + help: "Does '{{ app_name }}' have a dev environment deployed? (to enable various CI/CD)" default: false when: *app From 5532b1067ef2c41694d4040696121e7c2383e2d9 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Tue, 12 Nov 2024 13:37:11 -0500 Subject: [PATCH 31/61] fixes for app/ -> template-only-app/ --- .github/workflows/template-only-ci-app.yml | 4 ++-- .github/workflows/template-only-ci-infra.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/template-only-ci-app.yml b/.github/workflows/template-only-ci-app.yml index 67f7fe70..06920f70 100644 --- a/.github/workflows/template-only-ci-app.yml +++ b/.github/workflows/template-only-ci-app.yml @@ -6,12 +6,12 @@ on: - main pull_request: paths: - - app/** + - template-only-app/** - .github/workflows/template-only-ci-app.yml defaults: run: - working-directory: ./app + working-directory: ./template-only-app jobs: # Run the build to make sure it doesn't fail diff --git a/.github/workflows/template-only-ci-infra.yml b/.github/workflows/template-only-ci-infra.yml index f8cce0a8..7b872dda 100644 --- a/.github/workflows/template-only-ci-infra.yml +++ b/.github/workflows/template-only-ci-infra.yml @@ -12,7 +12,7 @@ on: - template-only-infra/** - template-only-test/** - .github/workflows/template-only-ci-infra.yml - - app/Dockerfile + - template-only-app/Dockerfile workflow_dispatch: # For now, only allow one workflow run at a time, since this provisions an OIDC From 6ad105d177b5d8ebf85ef6817523a590c374e4a3 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Tue, 12 Nov 2024 13:40:25 -0500 Subject: [PATCH 32/61] More app -> doc updates --- bin/create-or-update-database-roles | 5 +++-- docs/infra/destroy-infrastructure.md | 2 +- docs/infra/making-infra-changes.md | 16 ++++++++-------- docs/infra/set-up-database.md | 4 ++-- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/bin/create-or-update-database-roles b/bin/create-or-update-database-roles index a0865901..29403d72 100755 --- a/bin/create-or-update-database-roles +++ b/bin/create-or-update-database-roles @@ -2,8 +2,9 @@ # ----------------------------------------------------------------------------- # Script that invokes the database role-manager AWS Lambda function to create # or update the Postgres user roles for a particular environment. -# The Lambda function is created by the infra/app/database root module and is -# defined in the infra/app/database child module. +# +# The Lambda function is created by the infra/modules/database root module and is +# defined in the infra//database child module. # # Positional parameters: # app_name (required) – the name of subdirectory of /infra that holds the diff --git a/docs/infra/destroy-infrastructure.md b/docs/infra/destroy-infrastructure.md index 0635e330..abd68b93 100644 --- a/docs/infra/destroy-infrastructure.md +++ b/docs/infra/destroy-infrastructure.md @@ -4,7 +4,7 @@ To destroy everything you'll need to undeploy all the infrastructure in reverse ## Instructions -1. First, destroy all your environments. Within `/infra/app/service` run the following, replacing `dev` with the environment you're destroying. +1. First, destroy all your environments. Within `/infra//service` run the following, replacing `dev` with the environment you're destroying. ```bash $ terraform init --backend-config=dev.s3.tfbackend diff --git a/docs/infra/making-infra-changes.md b/docs/infra/making-infra-changes.md index 3133c7b2..6c9c7235 100644 --- a/docs/infra/making-infra-changes.md +++ b/docs/infra/making-infra-changes.md @@ -45,9 +45,9 @@ TF_CLI_ARGS_apply='-var=image_tag=abcdef1' make infra-update-app-service APP_NAM An alternative to using the Makefile is to directly use the terraform wrapper scripts that the Makefile uses: ```bash -project-root$ ./bin/terraform-init app/service dev -project-root$ ./bin/terraform-apply app/service dev -project-root$ ./bin/terraform-init-and-apply app/service dev # calls init and apply in the same script +project-root$ ./bin/terraform-init /service dev +project-root$ ./bin/terraform-apply /service dev +project-root$ ./bin/terraform-init-and-apply /service dev # calls init and apply in the same script ``` Look in the script files for more details on usage. @@ -57,16 +57,16 @@ Look in the script files for more details on usage. Finally, if the wrapper scripts don't meet your needs, you can always run `terraform` directly. You may need to do this if you are running terraform commands other than `terraform plan` and `terraform apply`, such as `terraform import`, `terraform taint`, etc. To do this, you'll need to remember to run `terraform init` with the appropriate `tfbackend` file since the root modules are shared across multiple backends. For example, to make changes to the application's service resources in the dev environment: ```bash -project-root$ cd infra/app/service -infra/app-service$ terraform init -backend-config=dev.s3.tfbackend -infra/app-service$ terraform apply -var-file=dev.tfvars +project-root$ cd infra//service +infra//service$ terraform init -backend-config=dev.s3.tfbackend +infra//service$ terraform apply -var-file=dev.tfvars ``` or you can run the commands from the project root by using the `-chdir` flag. ```bash -project-root$ terraform init -chdir=infra/app/service -backend-config=dev.s3.tfbackend -project-root$ terraform apply -chdir=infra/app/service -var="environment_name=dev" +project-root$ terraform init -chdir=infra//service -backend-config=dev.s3.tfbackend +project-root$ terraform apply -chdir=infra//service -var="environment_name=dev" ``` ## See also diff --git a/docs/infra/set-up-database.md b/docs/infra/set-up-database.md index 45210cbc..d62459d2 100644 --- a/docs/infra/set-up-database.md +++ b/docs/infra/set-up-database.md @@ -28,7 +28,7 @@ make infra-configure-app-database APP_NAME= ENVIRONMENT= ``` `APP_NAME` needs to be the name of the application folder within the `infra` folder. By default, this is `app`. -`ENVIRONMENT` needs to be the name of the environment you are creating. This will create a file called `.s3.tfbackend` in the `infra/app/service` module directory. +`ENVIRONMENT` needs to be the name of the environment you are creating. This will create a file called `.s3.tfbackend` in the `infra//service` module directory. ### (Optional) Enable any database extensions that require `rds_superuser` @@ -37,7 +37,7 @@ To enable some extensions, such as [pgvector](https://github.com/pgvector/pgvect For example, to enable the pgvector extension: ```terraform -# infra/app/app-config/env-config/main.tf +# infra//app-config/env-config/main.tf database_config = { ... From de5a35815f63db60d9858dcbd4324a9e32048bb0 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Tue, 12 Nov 2024 15:07:40 -0500 Subject: [PATCH 33/61] Template-ize infra/networks/main.tf Not a normal template --- infra/networks/{main.tf => main.tf.jinja} | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) rename infra/networks/{main.tf => main.tf.jinja} (82%) diff --git a/infra/networks/main.tf b/infra/networks/main.tf.jinja similarity index 82% rename from infra/networks/main.tf rename to infra/networks/main.tf.jinja index de966c20..ce58d014 100644 --- a/infra/networks/main.tf +++ b/infra/networks/main.tf.jinja @@ -1,3 +1,12 @@ +{# + This is a special file template that is owned/maintained as a part of the + "base" template, but is re-rendered when a new app is added, as it is supposed + to import each app config module. This is a little hacky and depends on + special support in the nava-platform CLI. +#} + +{%- set app_names = app_names | default([]) -%} + locals { tags = merge(module.project_config.default_tags, { network_name = var.network_name @@ -10,7 +19,7 @@ locals { # List of configuration for all applications, even ones that are not in the current network # If project has multiple applications, add other app configs to this list - app_configs = [module.app_config] + app_configs = [{% for app_name in app_names -%} {{ "module.%s_config" | format(app_name) }}{{", " if not loop.last}} {%- endfor %}] # List of configuration for applications that are in the current network # An application is in the current network if at least one of its environments @@ -64,10 +73,13 @@ module "project_config" { source = "../project-config" } -module "app_config" { - source = "../app/app-config" +{% for app_name in app_names %} +module "{{ app_name }}_config" { + source = "../{{ app_name }}/app-config" } +{% endfor -%} + module "network" { source = "../modules/network" name = var.network_name From 306b1a6d0f4988fa6bef9c04d7b5ca122555b877 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Wed, 13 Nov 2024 10:44:29 -0500 Subject: [PATCH 34/61] Ignore broken link for networks/main.tf that will exist for installed templates --- docs/infra/set-up-network.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/infra/set-up-network.md b/docs/infra/set-up-network.md index cba96afd..10c59236 100644 --- a/docs/infra/set-up-network.md +++ b/docs/infra/set-up-network.md @@ -17,6 +17,7 @@ Before setting up the network you'll need to have: 3. Configure the app in `infra//app-config/main.tf`. 1. Update `has_database` to `true` or `false` depending on whether or not your application has a database to integrate with. This setting determines whether or not to create VPC endpoints needed by the database layer. 2. Update `has_external_non_aws_service` to `true` or `false` depending on whether or not your application makes calls over the public internet. Set this to `true` (a) if your application makes calls to a SaaS service, or (b) if your application needs to call services from another application in the same git repo. This setting determines whether or not to create NAT gateways, which allows the service in the private subnet to make requests to the internet. For more information, see [set up network access to the public internet](./set-up-public-internet-access.md) + 3. If you made changes to the configuration of the networks in the optional step 2 above and or to the default application environments: Update `network_name` for your application environments. This mapping ensures that each network is configured appropriately based on the application(s) in that network (see `local.apps_in_network` in [/infra/networks/main.tf](/infra/networks/main.tf)) Failure to set the network name properly means that the network layer may not receive the correct application configurations for `has_database` and `has_external_non_aws_service`. ## 1. Configure backend From e8436f566d5268a19cdedb277c18cb23b75a7193 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Tue, 12 Nov 2024 18:30:00 -0500 Subject: [PATCH 35/61] Update example app in platform-test in CD --- .github/workflows/template-only-cd.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/template-only-cd.yml b/.github/workflows/template-only-cd.yml index 52907212..5cca4131 100644 --- a/.github/workflows/template-only-cd.yml +++ b/.github/workflows/template-only-cd.yml @@ -75,6 +75,16 @@ jobs: working-directory: project-repo run: nava-platform infra update --template-uri ../template-infra --version lorenyu/platform-cli . + - name: Install example app + if: "${{ matrix.project_repo == 'navapbc/platform-test' }}" + run: | + mv -vf ./template-infra/template-only-app ./project-repo/app + cd project-repo + if [[ $(git status app --porcelain | wc -l) -ne 0 ]]; then + git add app + git commit --message "Update app/ from example" + fi + - name: Push changes to project repo working-directory: project-repo run: git push From 35a50049c3fee1672d93843bc69bce9586b22eb3 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Wed, 13 Nov 2024 12:44:38 -0500 Subject: [PATCH 36/61] misc. doc fix --- docs/infra/destroy-infrastructure.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/infra/destroy-infrastructure.md b/docs/infra/destroy-infrastructure.md index abd68b93..1de6539e 100644 --- a/docs/infra/destroy-infrastructure.md +++ b/docs/infra/destroy-infrastructure.md @@ -43,7 +43,7 @@ To destroy everything you'll need to undeploy all the infrastructure in reverse # Comment out or delete the backend block backend "s3" { ... - }2 + } ``` 4. Then run the following from within the `infra/accounts` directory to copy the `tfstate` back to a local `tfstate` file: From d15e09c44a4a643a34aba4570b35eec5da21a6a0 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Thu, 14 Nov 2024 10:21:55 -0500 Subject: [PATCH 37/61] doc updates --- infra/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/infra/README.md b/infra/README.md index 20bdcf41..378811a0 100644 --- a/infra/README.md +++ b/infra/README.md @@ -10,9 +10,10 @@ The structure for the infrastructure code looks like this: infra/ Infrastructure code project-config/ Project-level configuration for account-level resources and resource tags accounts/ [Root module] IaC and IAM resources - / Application directory: infrastructure for the main application + / Application directory(-ies): infrastructure for the application modules/ Reusable child modules networks/ [Root module] Account level network config (shared across all apps, environments, and terraform workspaces) + test/ Infrastructure tests ``` Each application directory contains the following: From 1cbab232c1cb4821ed20794fbdfe20cdad10d5e5 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Thu, 14 Nov 2024 11:28:06 -0500 Subject: [PATCH 38/61] Update template CI/CD stuff to not checkout lorenyu/platform-cli Once this PR is merged, things should use `main`/whatever branch it's running from. --- .github/workflows/template-only-cd.yml | 24 ++------------------ .github/workflows/template-only-ci-infra.yml | 2 -- 2 files changed, 2 insertions(+), 24 deletions(-) diff --git a/.github/workflows/template-only-cd.yml b/.github/workflows/template-only-cd.yml index 5cca4131..17293ee2 100644 --- a/.github/workflows/template-only-cd.yml +++ b/.github/workflows/template-only-cd.yml @@ -10,25 +10,6 @@ on: concurrency: platform-template-only-cd jobs: - # TODO: Get rid of this job once we've merged the platform-cli branch into the main branch - update-platform-cli-branch: - name: Update platform-cli branch - runs-on: ubuntu-latest - steps: - - name: Checkout template-infra repo - uses: actions/checkout@v4 - with: - path: template-infra - ref: lorenyu/platform-cli - # Fetch history of all branches so we can merge main into the feature branch - fetch-depth: 0 - - name: Update - working-directory: template-infra - run: | - git config user.name nava-platform-bot - git config user.email platform-admins@navapbc.com - git merge origin/main - git push deploy: name: Deploy to ${{ matrix.project_repo }} runs-on: ubuntu-latest @@ -45,10 +26,9 @@ jobs: uses: actions/checkout@v4 with: path: template-infra - # TODO: Revert to checking out main once we've merged the platform-cli branch into the main branch - ref: lorenyu/platform-cli # Fetch history because the Platform CLI needs it to do the update fetch-depth: 0 + - name: Checkout project repo uses: actions/checkout@v4 with: @@ -73,7 +53,7 @@ jobs: - name: Update infra template working-directory: project-repo - run: nava-platform infra update --template-uri ../template-infra --version lorenyu/platform-cli . + run: nava-platform infra update --template-uri ../template-infra . - name: Install example app if: "${{ matrix.project_repo == 'navapbc/platform-test' }}" diff --git a/.github/workflows/template-only-ci-infra.yml b/.github/workflows/template-only-ci-infra.yml index 7b872dda..7771e3c5 100644 --- a/.github/workflows/template-only-ci-infra.yml +++ b/.github/workflows/template-only-ci-infra.yml @@ -36,8 +36,6 @@ jobs: uses: actions/checkout@v4 with: path: template-infra - # TODO: Revert to checking out main once we've merged the platform-cli branch into the main branch - ref: lorenyu/platform-cli - uses: hashicorp/setup-terraform@v2 with: From a666a9ce290f17a094e9371b281778524b8f442e Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Thu, 14 Nov 2024 12:11:07 -0500 Subject: [PATCH 39/61] Reduce some duplication on platform-cli install --- .github/workflows/template-only-cd.yml | 2 +- .github/workflows/template-only-ci-infra.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/template-only-cd.yml b/.github/workflows/template-only-cd.yml index 17293ee2..60fb08fa 100644 --- a/.github/workflows/template-only-cd.yml +++ b/.github/workflows/template-only-cd.yml @@ -43,7 +43,7 @@ jobs: python-version: '3.13' - name: Install nava-platform CLI - run: pipx install --python python3.13 git+https://github.com/navapbc/platform-cli + run: pipx install --python "$(which python)" git+https://github.com/navapbc/platform-cli - name: Configure git working-directory: project-repo diff --git a/.github/workflows/template-only-ci-infra.yml b/.github/workflows/template-only-ci-infra.yml index 7771e3c5..d76dd031 100644 --- a/.github/workflows/template-only-ci-infra.yml +++ b/.github/workflows/template-only-ci-infra.yml @@ -52,7 +52,7 @@ jobs: python-version: '3.13' - name: Install nava-platform CLI - run: pipx install --python python3.13 git+https://github.com/navapbc/platform-cli + run: pipx install --python "$(which python)" git+https://github.com/navapbc/platform-cli - name: Install infra template run: | From a0a33b4cf9aeaa1422b3a1a2056ab3b7b97934e2 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Thu, 14 Nov 2024 12:11:55 -0500 Subject: [PATCH 40/61] Remove TODO from non-template-only docs Tracked as TODO on the PR --- .github/workflows/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 02f56914..95e2fa45 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -2,8 +2,6 @@ The CI/CD for this project uses [reusable Github Actions workflows](https://docs.github.com/en/actions/using-workflows/reusing-workflows). -TODO: update the workflow links to point to blob/main before merge - ## 🧪 CI ### Per app workflows From cddd7dbfb52fb5cb0dba2aba2bbb634aad1a36b3 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Thu, 14 Nov 2024 12:18:15 -0500 Subject: [PATCH 41/61] Fix a couple template-only CI things from recent change --- .github/workflows/template-only-cd.yml | 1 - .github/workflows/template-only-ci-infra.yml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/template-only-cd.yml b/.github/workflows/template-only-cd.yml index 60fb08fa..794507aa 100644 --- a/.github/workflows/template-only-cd.yml +++ b/.github/workflows/template-only-cd.yml @@ -13,7 +13,6 @@ jobs: deploy: name: Deploy to ${{ matrix.project_repo }} runs-on: ubuntu-latest - needs: update-platform-cli-branch strategy: fail-fast: true matrix: diff --git a/.github/workflows/template-only-ci-infra.yml b/.github/workflows/template-only-ci-infra.yml index d76dd031..f10b519c 100644 --- a/.github/workflows/template-only-ci-infra.yml +++ b/.github/workflows/template-only-ci-infra.yml @@ -36,6 +36,7 @@ jobs: uses: actions/checkout@v4 with: path: template-infra + fetch-depth: 0 - uses: hashicorp/setup-terraform@v2 with: From 66b4f27db6b2ee8312e5393650c637c8d40a2a01 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Thu, 14 Nov 2024 12:39:16 -0500 Subject: [PATCH 42/61] Specify `--version` in CI for platform-cli Since platform-cli defaults to `lorenyu/platform-cli` branch still. This should keep things working through the transition period. --- .github/workflows/template-only-cd.yml | 2 +- .github/workflows/template-only-ci-infra.yml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/template-only-cd.yml b/.github/workflows/template-only-cd.yml index 794507aa..3ceb6c3f 100644 --- a/.github/workflows/template-only-cd.yml +++ b/.github/workflows/template-only-cd.yml @@ -52,7 +52,7 @@ jobs: - name: Update infra template working-directory: project-repo - run: nava-platform infra update --template-uri ../template-infra . + run: nava-platform infra update --template-uri ../template-infra --version HEAD . - name: Install example app if: "${{ matrix.project_repo == 'navapbc/platform-test' }}" diff --git a/.github/workflows/template-only-ci-infra.yml b/.github/workflows/template-only-ci-infra.yml index f10b519c..2d529764 100644 --- a/.github/workflows/template-only-ci-infra.yml +++ b/.github/workflows/template-only-ci-infra.yml @@ -36,7 +36,6 @@ jobs: uses: actions/checkout@v4 with: path: template-infra - fetch-depth: 0 - uses: hashicorp/setup-terraform@v2 with: @@ -65,7 +64,7 @@ jobs: # TODO: add --data-file support to CLI and have as many of these # settings in more maintainable separate file - nava-platform infra install --template-uri ./template-infra \ + nava-platform infra install --template-uri ./template-infra --version HEAD \ --data base_project_name="${project_name}" \ --data base_owner=platform-admins \ --data base_code_repository_url=${{ github.repositoryUrl }} \ From 874163984e965fc83ba950a579134515431d653b Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Thu, 14 Nov 2024 16:19:36 -0500 Subject: [PATCH 43/61] More doc updates --- docs/infra/pull-request-environments.md | 4 ++-- docs/infra/set-up-database.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/infra/pull-request-environments.md b/docs/infra/pull-request-environments.md index 4051d048..e0b00cd7 100644 --- a/docs/infra/pull-request-environments.md +++ b/docs/infra/pull-request-environments.md @@ -29,6 +29,6 @@ Pull request environments are created by GitHub Actions workflows. There are two Using these reusable workflows, configure PR environments for each application with application-specific workflows: - `ci--pr-environment-checks.yml` - - Based on [ci-app-pr-environment-checks.yml](https://github.com/navapbc/template-infra/blob/main/.github/workflows/ci-app-pr-environment-checks.yml) + - Based on [ci-{{app_name}}-pr-environment-checks.yml](https://github.com/navapbc/template-infra/blob/lorenyu%2Fplatform-cli/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja) - `ci--pr-environment-destroy.yml` - - Based on [ci-app-pr-environment-destroy.yml](https://github.com/navapbc/template-infra/blob/main/.github/workflows/ci-app-pr-environment-destroy.yml) + - Based on [ci-{{app_name}}-pr-environment-destroy.yml](https://github.com/navapbc/template-infra/blob/lorenyu%2Fplatform-cli/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja) diff --git a/docs/infra/set-up-database.md b/docs/infra/set-up-database.md index d62459d2..d82f7104 100644 --- a/docs/infra/set-up-database.md +++ b/docs/infra/set-up-database.md @@ -27,7 +27,7 @@ To create the `tfbackend` file for the new application environment, run make infra-configure-app-database APP_NAME= ENVIRONMENT= ``` -`APP_NAME` needs to be the name of the application folder within the `infra` folder. By default, this is `app`. +`APP_NAME` needs to be the name of the application folder within the `infra` folder. `ENVIRONMENT` needs to be the name of the environment you are creating. This will create a file called `.s3.tfbackend` in the `infra//service` module directory. ### (Optional) Enable any database extensions that require `rds_superuser` From a56021af3d96f76adec75bfea22b46a0d76a4839 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Thu, 14 Nov 2024 17:49:49 -0500 Subject: [PATCH 44/61] bin/update-pr-environment: Support multiple apps in comments --- bin/update-pr-environment | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/update-pr-environment b/bin/update-pr-environment index 54b57965..15e13411 100755 --- a/bin/update-pr-environment +++ b/bin/update-pr-environment @@ -37,17 +37,17 @@ aws ecs wait services-stable --cluster "${cluster_name}" --services "${service_n service_endpoint="$(terraform -chdir="infra/${app_name}/service" output -raw service_endpoint)" pr_info=$(cat < -## Preview environment + +## Preview environment for ${app_name} - Service endpoint: ${service_endpoint} - Deployed commit: ${image_tag} - + EOF ) pr_body="$(gh pr view "${pr_number}" --json body | jq --raw-output .body)" -if [[ $pr_body == *""*""* ]]; then - pr_body="${pr_body//*/$pr_info}" +if [[ $pr_body == *""*""* ]]; then + pr_body="${pr_body//*/$pr_info}" else pr_body="${pr_body}"$'\n\n'"${pr_info}" fi From 4336bd03a35d2e2427ddc2ba7426c6a4311acf53 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Mon, 18 Nov 2024 10:48:07 -0500 Subject: [PATCH 45/61] Clean up older single-app section of PR Env info if present --- bin/update-pr-environment | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bin/update-pr-environment b/bin/update-pr-environment index 15e13411..5c6730ea 100755 --- a/bin/update-pr-environment +++ b/bin/update-pr-environment @@ -46,6 +46,13 @@ EOF ) pr_body="$(gh pr view "${pr_number}" --json body | jq --raw-output .body)" + +# clean up older single-app section if present +if [[ $pr_body == *""*""* ]]; then + pr_body="${pr_body//*}" +fi + +# update or add the environment info if [[ $pr_body == *""*""* ]]; then pr_body="${pr_body//*/$pr_info}" else From 490209487e3eebbd0eb20be5e7fa492ab8e36607 Mon Sep 17 00:00:00 2001 From: Kai Siren Date: Wed, 27 Nov 2024 15:18:31 -0800 Subject: [PATCH 46/61] remove feature flags due to evidently deprecation --- infra/{{app_name}}/app-config/feature_flags.tf | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 infra/{{app_name}}/app-config/feature_flags.tf diff --git a/infra/{{app_name}}/app-config/feature_flags.tf b/infra/{{app_name}}/app-config/feature_flags.tf deleted file mode 100644 index 6224b253..00000000 --- a/infra/{{app_name}}/app-config/feature_flags.tf +++ /dev/null @@ -1,3 +0,0 @@ -locals { - feature_flags = ["foo", "bar"] -} From ccc8cab26a2232121536742eb042c8b185ab348b Mon Sep 17 00:00:00 2001 From: Kai Siren Date: Wed, 27 Nov 2024 15:19:55 -0800 Subject: [PATCH 47/61] remove feature flags due to evidently deprecation (part 2) --- template-only-app/feature_flags.py | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 template-only-app/feature_flags.py diff --git a/template-only-app/feature_flags.py b/template-only-app/feature_flags.py deleted file mode 100644 index 51d1888b..00000000 --- a/template-only-app/feature_flags.py +++ /dev/null @@ -1,20 +0,0 @@ -import logging -import os - -import boto3 - -logger = logging.getLogger() - - -def is_feature_enabled(feature_name: str) -> bool: - feature_flags_project_name = os.environ.get("FEATURE_FLAGS_PROJECT") - - logger.info("Getting Evidently client") - client = boto3.client("evidently") - - response = client.evaluate_feature( - entityId="anonymous", - feature=feature_name, - project=feature_flags_project_name, - ) - return response["value"]["boolValue"] From 2693314d37ce007f72b9930470824b2db5bc29d1 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Tue, 19 Nov 2024 15:34:28 -0500 Subject: [PATCH 48/61] Build out docs/release.md a little bit --- docs/releases.md | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/releases.md b/docs/releases.md index ef550805..a708572e 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -13,10 +13,30 @@ parameters to build an image. The `/Dockerfile` should have a build stage called `release` to act as the build target. (See [Name your build stages](https://docs.docker.com/build/building/multi-stage/#name-your-build-stages)) +You may pass `IMAGE_NAME` and `IMAGE_TAG` arguments if wanting to control those +aspects of the built image, but typically leave them at the defaults, which will +be based on `APP_NAME` and latest commit hash, respectively. + ## Publishing a release -TODO +```bash +make release-publish APP_NAME= +``` ## Deploying a release -TODO +```bash +make release-deploy APP_NAME= ENVIRONMENT= +``` + +## All together + +Typically the release process is automated based on merges to release branches +or through invoked GitHub Actions, but if wanting to make a test deploy for some +local application changes, could do something like: + +```sh +make release-build release-publish release-deploy APP_NAME= ENVIRONMENT= +``` + +(you may also need `release-run-database-migrations` in there before `release-deploy`, but be careful) From 494f7c32f81aa5f96cc986fc89c49520dbe22e5a Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Mon, 2 Dec 2024 11:32:29 -0500 Subject: [PATCH 49/61] Exclude template-only files that aren't tagged as such in the name By specifying `_exclude` in `copier.yml`, we need to re-add the relevant default excludes to the list (`.git` and `copier.yml`). https://copier.readthedocs.io/en/stable/configuring/#exclude --- copier.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/copier.yml b/copier.yml index 41eb0012..fb149e5f 100644 --- a/copier.yml +++ b/copier.yml @@ -99,3 +99,11 @@ _envops: _skip_if_exists: - "/{{ app_name }}/" - "/{{ app_name }}/Makefile" + +_exclude: + - /.git + - /copier.yml + - /CODEOWNERS + - /CONTRIBUTING.md + - /LICENSE.md + - /README.md From 82ec64ad6bc6bc16f68cf6f67ba85a6432e22618 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Mon, 2 Dec 2024 14:26:23 -0500 Subject: [PATCH 50/61] Remove old install/update scripts --- README.md | 34 +++++++--------- .../download-and-install-template | 11 ------ template-only-bin/install-template | 36 ----------------- template-only-bin/update-template | 39 ------------------- 4 files changed, 15 insertions(+), 105 deletions(-) delete mode 100755 template-only-bin/download-and-install-template delete mode 100755 template-only-bin/install-template delete mode 100755 template-only-bin/update-template diff --git a/README.md b/README.md index 2c4eefd2..77e37904 100644 --- a/README.md +++ b/README.md @@ -22,10 +22,13 @@ This template assumes that you have an application to deploy. See [application r ## Installation -To get started using the infrastructure template on your project, run the following command in your project's directory to execute the [download and install script](https://github.com/navapbc/template-infra/tree/main/template-only-bin/download-and-install-template), which clones the template repository, copies the template files to your repository, and removes any files that are only relevant to the template itself: +To get started using the infrastructure template on your project, clone this +repository to a local directory next to your project, [install the nava-platform +tool](https://github.com/navapbc/platform-cli), and then run the following +command in your project's root directory: -```bash -curl https://raw.githubusercontent.com/navapbc/template-infra/main/template-only-bin/download-and-install-template | bash -s +```sh +nava-platform infra install --template-uri ../template-infra . ``` Now you're ready to set up the various pieces of your infrastructure. @@ -42,24 +45,17 @@ After downloading and installing the template into your project: ## Updates -There are multiple ways to receive template updates on your project. For most updates, you can simply run the [update-template](/template-only-bin/update-template) script +With the [nava-platform tool +installed](https://github.com/navapbc/platform-cli), pull updates to your local +`template-infra` repository, then run the following in your project's root +directory: -```bash -curl https://raw.githubusercontent.com/navapbc/template-infra/main/template-only-bin/update-template | bash -s +```sh +nava-platform infra update . ``` -If the update fails the simplest option may be to re-run the installation script above and manually review the changes. +If the update fails, the tool will provide some guidance, but effectively the +next step will be apply the updates in smaller pieces with manual merge conflict +resolution. **Remember:** Make sure to read the release notes in case there are breaking changes you need to address. - -### Renamed applications - -If you renamed your application from `infra/app` to something else like `infra/foo`, then first rename your app back to `infra/app` before applying the updates e.g. - -```bash -mv foo app -mv infra/foo infra/app -curl https://raw.githubusercontent.com/navapbc/template-infra/main/template-only-bin/update-template | bash -s -mv infra/app infra/foo -mv app foo -``` diff --git a/template-only-bin/download-and-install-template b/template-only-bin/download-and-install-template deleted file mode 100755 index 37c8be20..00000000 --- a/template-only-bin/download-and-install-template +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -set -euo pipefail - -echo "Fetch latest version of template-infra" -git clone --single-branch --branch main --depth 1 git@github.com:navapbc/template-infra.git - -echo "Install template" -./template-infra/template-only-bin/install-template - -echo "Clean up template-infra folder" -rm -fr template-infra diff --git a/template-only-bin/install-template b/template-only-bin/install-template deleted file mode 100755 index 1c457a73..00000000 --- a/template-only-bin/install-template +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# -# This script installs the template-infra to your project. Run -# this script from your project's root directory. -set -euox pipefail - -cur_dir=$(pwd) -script_dir=$(dirname "$0") -template_dir="${script_dir}/.." - -echo "Copy files from template-infra" -cd "${template_dir}" -# Note: Keep this list of paths in sync with INCLUDE_PATHS in update-template -cp -r \ - .github \ - bin \ - docs \ - infra \ - e2e \ - Makefile \ - .dockleconfig \ - .gitignore \ - .grype.yml \ - .hadolint.yaml \ - .trivyignore \ - .terraform-version \ - "${cur_dir}" - -# Store template version in a file -git rev-parse HEAD > "${cur_dir}/.template-version" - -cd - - -echo "Remove files relevant only to template development" -# Note: Keep this list of paths in sync with EXCLUDE_OPT in update-template -rm .github/workflows/template-only-* diff --git a/template-only-bin/update-template b/template-only-bin/update-template deleted file mode 100755 index d842bc21..00000000 --- a/template-only-bin/update-template +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# ----------------------------------------------------------------------------- -# This script updates template-infra in your project. Run -# this script from your project's root directory. -# -# Positional parameters: -# target_version (optional) – the version of template-infra to upgrade to. -# Defaults to main. -# ----------------------------------------------------------------------------- -set -euo pipefail - -target_version=${1:-"main"} - -current_version=$(cat .template-version) - -echo "Clone template-infra" -git clone https://github.com/navapbc/template-infra.git - -echo "Creating patch" -cd template-infra -git checkout "${target_version}" - -# Get version hash to update .template-version after patch is successful -target_version_hash=$(git rev-parse HEAD) - -# Note: Keep this list in sync with the files copied in install-template -git diff "${current_version}" "${target_version}" --binary -- .github bin docs infra e2e Makefile .dockleconfig .gitignore .grype.yml .hadolint.yaml .trivyignore .terraform-version > update.patch -cd - - -echo "Applying patch" -# Note: Keep this list in sync with the removed files in install-template -exclude_opt="--exclude=.github/workflows/template-only-*" -git apply "${exclude_opt}" --allow-empty template-infra/update.patch - -echo "Saving new template version to .template-infra" -echo "${target_version_hash}" > .template-version - -echo "Clean up template-infra folder" -rm -fr template-infra From e61b207334c87319d2fdbb75d1d20faf11dc1466 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Mon, 9 Dec 2024 18:15:57 -0500 Subject: [PATCH 51/61] Fix destroy-pr-environment for multi-app PR env setup --- bin/destroy-pr-environment | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/bin/destroy-pr-environment b/bin/destroy-pr-environment index 92c97eb1..ebd29714 100755 --- a/bin/destroy-pr-environment +++ b/bin/destroy-pr-environment @@ -34,16 +34,22 @@ echo "Delete workspace: ${workspace}" terraform -chdir="infra/${app_name}/service" workspace delete "${workspace}" pr_info=$(cat < + ## Preview environment ♻️ Environment destroyed ♻️ - + EOF ) pr_body="$(gh pr view "${pr_number}" --json body | jq --raw-output .body)" + +# clean up older single-app section if present if [[ $pr_body == *""*""* ]]; then - pr_body="${pr_body//*/$pr_info}" + pr_body="${pr_body//*}" +fi + +if [[ $pr_body == *""*""* ]]; then + pr_body="${pr_body//*/$pr_info}" else pr_body="${pr_body}"$'\n\n'"${pr_info}" fi From 88164a56262f19d096090a2a2d15ec8597ec1f7e Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Tue, 10 Dec 2024 09:41:03 -0500 Subject: [PATCH 52/61] e2e doc fixes/consistency --- docs/e2e/e2e-checks.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/e2e/e2e-checks.md b/docs/e2e/e2e-checks.md index a0a1c85c..91218e01 100644 --- a/docs/e2e/e2e-checks.md +++ b/docs/e2e/e2e-checks.md @@ -7,7 +7,7 @@ This repository uses [Playwright](https://playwright.dev/) to perform end-to-end By default in CI, tests are sharded across 3 concurrent runs to reduce total runtime. As the test suite grows, consider increasing the shard count to further optimize execution time. This is set in the [workflow file](../../.github/workflows/e2e-tests.yml#L22). ## Folder Structure -In order to support e2e for multiple apps, the folder structure will include a base playwright config (`./e2e/playwright.config.js`), and app-specific derived playwright config that override the base config. See the example folder structure below: +In order to support e2e for multiple apps, the folder structure will include a base playwright config (`/e2e/playwright.config.js`), and app-specific derived playwright config that override the base config. See the example folder structure below: ``` - e2e - playwright.config.js @@ -23,7 +23,7 @@ In order to support e2e for multiple apps, the folder structure will include a b Some highlights: >- By default, the base config is defined to run on a minimal browser-set (desktop and mobile chrome). Browsers can be added in the app-specific playwright config. ->- Snapshots will be output locally (in the `./e2e` folder or the container) - or in the artifacts of the CI job +>- Snapshots will be output locally (in the `/e2e` folder or the container) - or in the artifacts of the CI job >- HTML reports are output to the `playwright-report` folder >- Accessibility testing can be performed using the `@axe-core/playwright` package (https://playwright.dev/docs/accessibility-testing) @@ -55,7 +55,7 @@ Then, run the tests with your app name and base url: ```bash make e2e-test-native APP_NAME=app ``` ->* `BASE_URL` is optional for both `e2e-test-native` and `e2e-test-native-ui` targets. It will by default use the [app-specific](../../e2e/app/playwright.config.js) `baseURL` +>* `BASE_URL` is optional for both `e2e-test-native` and `e2e-test-native-ui` targets. It will by default use the app-specific (`/e2e//playwright.config.js`) `baseURL` ### Run tests in UI mode @@ -68,7 +68,7 @@ make e2e-test-native-ui APP_NAME=app #### Run tests in parallel -The following commands split test execution into 3 separate shards, with results consolidated into a merged report located in `./e2e/blob-report`. This setup emulates how the sharded tests run in CI. +The following commands split test execution into 3 separate shards, with results consolidated into a merged report located in `/e2e/blob-report`. This setup emulates how the sharded tests run in CI. ``` # ensure app is running on port 3000 @@ -82,7 +82,7 @@ make e2e-clean-report # clean the report folders ``` ### Viewing the report -If running in docker, the report will be copied from the container to your local `./e2e/playwright-report` folder. If running natively, the report will also appear in this same folder. +If running in docker, the report will be copied from the container to your local `/e2e/playwright-report` folder. If running natively, the report will also appear in this same folder. To quickly view the report, you can run: @@ -114,9 +114,9 @@ The [E2E Tests Workflow](../../.github/workflows/e2e-tests.yml) takes a `service ## Configuration The E2E tests are configured using the following files: -- Base Configuration in `e2e/playwright.config.js` -- App-specific Configuration in `e2e//playwright.config.js` +- Base Configuration in `/e2e/playwright.config.js` +- App-specific Configuration in `/e2e//playwright.config.js` The app-specific configuration files extend the common base configuration. -By default when running `make e2e-test APP_NAME=app BASE_URL=http://localhost:3000 ` - you don't necessarily need to pass an `BASE_URL` since the default is defined in the app-specific playwright config (`./e2e//playwright.config.js`). +By default when running `make e2e-test APP_NAME=app BASE_URL=http://localhost:3000 ` - you don't necessarily need to pass an `BASE_URL` since the default is defined in the app-specific playwright config (`/e2e//playwright.config.js`). From 7fb04958b175a3df7508623d83bd4216753204f8 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Tue, 24 Dec 2024 15:59:15 -0500 Subject: [PATCH 53/61] Fix custom setup-terraform usage in `template-only-ci-infra.yml` --- .github/actions/setup-terraform/action.yml | 12 ++++++++---- .github/workflows/template-only-ci-infra.yml | 4 +++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/actions/setup-terraform/action.yml b/.github/actions/setup-terraform/action.yml index cd09f02d..80d17aef 100644 --- a/.github/actions/setup-terraform/action.yml +++ b/.github/actions/setup-terraform/action.yml @@ -1,12 +1,16 @@ -name: 'Set up Terraform' -description: 'Set up Terraform with the version stored in the .terraform-version file' +name: "Set up Terraform" +description: "Set up Terraform with the version stored in the .terraform-version file" +inputs: + version-file: + description: "File containing the terraform version to use." + default: ".terraform-version" runs: using: "composite" steps: - - name: Get .terraform-version + - name: Get Terraform version id: get-terraform-version run: | - terraform_version="$(cat .terraform-version)" + terraform_version="$(cat ${{ inputs.version-file }})" echo "Terraform version: ${terraform_version}" echo "terraform_version=${terraform_version}" >> "$GITHUB_OUTPUT" shell: bash diff --git a/.github/workflows/template-only-ci-infra.yml b/.github/workflows/template-only-ci-infra.yml index 967e03b6..9c77738f 100644 --- a/.github/workflows/template-only-ci-infra.yml +++ b/.github/workflows/template-only-ci-infra.yml @@ -38,7 +38,9 @@ jobs: path: template-infra - name: Set up Terraform - uses: ./.github/actions/setup-terraform + uses: ./template-infra/.github/actions/setup-terraform + with: + version-file: ./template-infra/.terraform-version - uses: actions/setup-go@v5 with: From 4c38ade241586b921aea24f447e2e6319407b5c2 Mon Sep 17 00:00:00 2001 From: Kai Siren Date: Thu, 2 Jan 2025 15:49:01 -0800 Subject: [PATCH 54/61] adds: infra/{{app_name}}/service/identity_provider.tf --- .../{{app_name}}/service/identity_provider.tf | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 infra/{{app_name}}/service/identity_provider.tf diff --git a/infra/{{app_name}}/service/identity_provider.tf b/infra/{{app_name}}/service/identity_provider.tf new file mode 100644 index 00000000..e2af836b --- /dev/null +++ b/infra/{{app_name}}/service/identity_provider.tf @@ -0,0 +1,53 @@ +locals { + # If this is a temporary environment, re-use an existing Cognito user pool. Otherwise, create a new one. + identity_provider_user_pool_id = module.app_config.enable_identity_provider ? ( + local.is_temporary ? module.existing_identity_provider[0].user_pool_id : module.identity_provider[0].user_pool_id + ) : null + identity_provider_environment_variables = module.app_config.enable_identity_provider ? { + COGNITO_USER_POOL_ID = local.identity_provider_user_pool_id, + COGNITO_CLIENT_ID = module.identity_provider_client[0].client_id + } : {} +} + +# If the app has `enable_identity_provider` set to true AND this is not a temporary +# environment, then create a new identity provider. +module "identity_provider" { + count = module.app_config.enable_identity_provider && !local.is_temporary ? 1 : 0 + source = "../../modules/identity-provider/resources" + + is_temporary = local.is_temporary + + name = local.identity_provider_config.identity_provider_name + password_minimum_length = local.identity_provider_config.password_policy.password_minimum_length + temporary_password_validity_days = local.identity_provider_config.password_policy.temporary_password_validity_days + verification_email_message = local.identity_provider_config.verification_email.verification_email_message + verification_email_subject = local.identity_provider_config.verification_email.verification_email_subject + domain_name = local.network_config.domain_config.hosted_zone + domain_identity_arn = local.notifications_config == null ? null : local.domain_identity_arn + sender_email = local.notifications_config == null ? null : local.notifications_config.sender_email + sender_display_name = local.notifications_config == null ? null : local.notifications_config.sender_display_name + reply_to_email = local.notifications_config == null ? null : local.notifications_config.reply_to_email +} + +# If the app has `enable_identity_provider` set to true AND this *is* a temporary +# environment, then use an existing identity provider. +module "existing_identity_provider" { + count = module.app_config.enable_identity_provider && local.is_temporary ? 1 : 0 + source = "../../modules/identity-provider/data" + + name = local.identity_provider_config.identity_provider_name +} + +# If the app has `enable_identity_provider` set to true, create a new identity provider +# client for the service. A new client is created for all environments, including +# temporary environments. +module "identity_provider_client" { + count = module.app_config.enable_identity_provider ? 1 : 0 + source = "../../modules/identity-provider-client/resources" + + callback_urls = local.identity_provider_config.client.callback_urls + logout_urls = local.identity_provider_config.client.logout_urls + name = "${local.prefix}${local.identity_provider_config.identity_provider_name}" + + user_pool_id = local.identity_provider_user_pool_id +} From 93fc8ded012ee93ab961976e6888d625776b9eeb Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Tue, 24 Dec 2024 16:50:36 -0500 Subject: [PATCH 55/61] Print generated project name in `template-only-ci-infra.yml` --- .github/workflows/template-only-ci-infra.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/template-only-ci-infra.yml b/.github/workflows/template-only-ci-infra.yml index 9c77738f..bfc0e389 100644 --- a/.github/workflows/template-only-ci-infra.yml +++ b/.github/workflows/template-only-ci-infra.yml @@ -60,6 +60,7 @@ jobs: # character max length, $RANDOM has a max value of 32767 unique_id=$(printf '%.5d' $((RANDOM))) project_name=plt-tst-act-${unique_id} + echo "PROJECT_NAME=${project_name}" echo "PROJECT_NAME=${project_name}" >> "$GITHUB_ENV" # TODO: add --data-file support to CLI and have as many of these From e4f9cecb12558485477701a0f4fda38ff845d994 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Tue, 31 Dec 2024 13:43:13 -0500 Subject: [PATCH 56/61] Support per-app vulnerability scan configs Application templates can provide overrides in their own directories, with the scanning falling back to the top-level project configs otherwise. --- .github/actions/first-file/action.yml | 58 +++++++++++++++++++++++ .github/workflows/vulnerability-scans.yml | 43 ++++++++++++++++- 2 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 .github/actions/first-file/action.yml diff --git a/.github/actions/first-file/action.yml b/.github/actions/first-file/action.yml new file mode 100644 index 00000000..ea313f0f --- /dev/null +++ b/.github/actions/first-file/action.yml @@ -0,0 +1,58 @@ +name: Return first file that exists +description: Check given list of files in order and return first one that exists. + +inputs: + files: + required: true + type: string + description: | + The list of files to check, in the order to check for them. + + File names should be properly quoted\escaped and either space or newline + separated. + + Either: + ```yaml + files: my_file.txt some_other_file.txt + ``` + + Or: + ```yaml + files: |- + my_file.txt + some_other_file.txt + ``` + +outputs: + found_file: + description: "Path of first file found." + value: ${{ steps.find-file.outputs.found_file }} + +runs: + using: "composite" + steps: + - name: Get file list + id: file-list + shell: bash + run: | + # Get file list + # https://github.com/actions/runner/issues/1877 + files=$(printf %s "${{ inputs.files }}" | tr '\n' ' ') + echo "File list: ${files}" + echo "files=${files}" >> "$GITHUB_OUTPUT" + + - name: Check file list + id: find-file + shell: bash + run: | + # Check file list + # https://github.com/actions/runner/issues/1877 + for f in ${{ steps.file-list.outputs.files }}; do + if [[ -e "${f}" ]]; then + found_file="${f}" + break + fi + done + + echo "found_file=${found_file}" + echo "found_file=${found_file}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/vulnerability-scans.yml b/.github/workflows/vulnerability-scans.yml index 232f34b6..fb4a838b 100644 --- a/.github/workflows/vulnerability-scans.yml +++ b/.github/workflows/vulnerability-scans.yml @@ -20,6 +20,13 @@ jobs: steps: - uses: actions/checkout@v4 + - uses: ./.github/actions/first-file + id: hadolint-config + with: + files: |- + ${{ inputs.app_name }}/.hadolint.yaml + .hadolint.yaml + # Scans Dockerfile for any bad practices or issues - name: Scan Dockerfile by hadolint uses: hadolint/hadolint-action@v3.1.0 @@ -28,6 +35,7 @@ jobs: format: tty failure-threshold: warning output-file: hadolint-results.txt + config: ${{ steps.hadolint-config.outputs.found_file }} - name: Save output to workflow summary if: always() # Runs even if there is a failure @@ -39,6 +47,18 @@ jobs: steps: - uses: actions/checkout@v4 + - uses: ./.github/actions/first-file + id: trivy-ignore + with: + files: |- + ${{ inputs.app_name }}/.trivyignore + .trivyignore + + - uses: ./.github/actions/first-file + id: trivy-secret + with: + files: ${{ inputs.app_name }}/trivy-secret.yaml trivy-secret.yaml + - name: Build and tag Docker image for scanning id: build-image run: | @@ -57,6 +77,9 @@ jobs: ignore-unfixed: true vuln-type: os scanners: vuln,secret + trivyignores: ${{ steps.trivy-ignore.outputs.found_file }} + env: + TRIVY_SECRET_CONFIG: ${{ steps.trivy-secret.outputs.found_file }} - name: Save output to workflow summary if: always() # Runs even if there is a failure @@ -69,6 +92,13 @@ jobs: steps: - uses: actions/checkout@v4 + - uses: ./.github/actions/first-file + id: grype-config + with: + files: |- + ${{ inputs.app_name }}/.grype.yml + .grype.yml + - name: Build and tag Docker image for scanning id: build-image run: | @@ -82,6 +112,8 @@ jobs: with: image: ${{ steps.build-image.outputs.image }} output-format: table + env: + GRYPE_CONFIG: ${{ steps.grype-config.outputs.found_file }} - name: Save output to workflow summary if: always() # Runs even if there is a failure @@ -93,6 +125,13 @@ jobs: steps: - uses: actions/checkout@v4 + - uses: ./.github/actions/first-file + id: dockle-config + with: + files: |- + ${{ inputs.app_name }}/.dockleconfig + .dockleconfig + - name: Build and tag Docker image for scanning id: build-image run: | @@ -105,8 +144,8 @@ jobs: # variable, this will save the variable in this file to env for Dockle - name: Set any acceptable Dockle files run: | - if grep -q "^DOCKLE_ACCEPT_FILES=.*" .dockleconfig; then - grep -s '^DOCKLE_ACCEPT_FILES=' .dockleconfig >> "$GITHUB_ENV" + if grep -q "^DOCKLE_ACCEPT_FILES=.*" ${{ steps.dockle-config.outputs.found_file }}; then + grep -s '^DOCKLE_ACCEPT_FILES=' ${{ steps.dockle-config.outputs.found_file }} >> "$GITHUB_ENV" fi - name: Run Dockle container linter From 455e4b34ff3cf0848117d057d305483ae704e0ff Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Mon, 6 Jan 2025 10:53:22 -0500 Subject: [PATCH 57/61] Fix template-only-app install in CI --- .github/workflows/template-only-ci-infra.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/template-only-ci-infra.yml b/.github/workflows/template-only-ci-infra.yml index bfc0e389..0c5ed7a4 100644 --- a/.github/workflows/template-only-ci-infra.yml +++ b/.github/workflows/template-only-ci-infra.yml @@ -81,7 +81,7 @@ jobs: - name: Install example app for infra tests run: | - mv -vf ./project-dir/template-only-app ./project-dir/app + cp -vfr ./project-dir/template-only-app/* ./project-dir/app - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 From c7b79d25768a20f2fb021862ffe9b9654b6b58f0 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Mon, 13 Jan 2025 19:11:30 -0500 Subject: [PATCH 58/61] Update docs to recommend `infra update --answers-only` --- .github/workflows/cd-{{app_name}}.yml.jinja | 2 +- .github/workflows/ci-{{app_name}}-infra-service.yml.jinja | 2 +- .../workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja | 2 +- .../workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja | 2 +- template-only-docs/set-up-cd.md | 2 +- template-only-docs/set-up-ci.md | 2 +- template-only-docs/set-up-pr-environments.md | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cd-{{app_name}}.yml.jinja b/.github/workflows/cd-{{app_name}}.yml.jinja index 0442a04b..64001cbc 100644 --- a/.github/workflows/cd-{{app_name}}.yml.jinja +++ b/.github/workflows/cd-{{app_name}}.yml.jinja @@ -16,7 +16,7 @@ on: # !! Once you've set up the dev environment and are ready to enable continuous # deployment, run: # - # nava-platform infra update --data app_has_dev_env_setup=true . + # nava-platform infra update --answers-only --data app_has_dev_env_setup=true . # # to enable these lines. They are here as comments for context. # diff --git a/.github/workflows/ci-{{app_name}}-infra-service.yml.jinja b/.github/workflows/ci-{{app_name}}-infra-service.yml.jinja index 09e288a2..d06f3e6b 100644 --- a/.github/workflows/ci-{{app_name}}-infra-service.yml.jinja +++ b/.github/workflows/ci-{{app_name}}-infra-service.yml.jinja @@ -20,7 +20,7 @@ on: # !! Once you've set up the dev environment and are ready to enable automated # infra tests, run: # - # nava-platform infra update --data app_has_dev_env_setup=true . + # nava-platform infra update --answers-only --data app_has_dev_env_setup=true . # # to enable these lines. They are here as comments for context. # diff --git a/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja b/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja index 764ea91b..899ff195 100644 --- a/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja +++ b/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja @@ -14,7 +14,7 @@ on: # !! Once you've set up the dev environment and are ready to enable PR # environments, run: # - # nava-platform infra update --data app_has_dev_env_setup=true . + # nava-platform infra update --answers-only --data app_has_dev_env_setup=true . # # to enable these lines. They are here as comments for context. # diff --git a/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja b/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja index cfaee5e5..137c93e1 100644 --- a/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja +++ b/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja @@ -12,7 +12,7 @@ on: # !! Once you've set up the dev environment and are ready to enable PR # environments, run: # - # nava-platform infra update --data app_has_dev_env_setup=true . + # nava-platform infra update --answers-only --data app_has_dev_env_setup=true . # # to enable these lines. They are here as comments for context. # diff --git a/template-only-docs/set-up-cd.md b/template-only-docs/set-up-cd.md index 83c9c3e4..aa149274 100644 --- a/template-only-docs/set-up-cd.md +++ b/template-only-docs/set-up-cd.md @@ -4,7 +4,7 @@ Once you have set up your application environments (at least `dev`), you can enable continuous deployment by running: ```sh -nava-platform infra update --data app_has_dev_env_setup=true . +nava-platform infra update --answers-only --data app_has_dev_env_setup=true . ``` And update the `role-to-assume` with the GitHub actions ARN. diff --git a/template-only-docs/set-up-ci.md b/template-only-docs/set-up-ci.md index dd26da74..ba0b1be9 100644 --- a/template-only-docs/set-up-ci.md +++ b/template-only-docs/set-up-ci.md @@ -11,7 +11,7 @@ Some checks are disabled until you've completed certain setup steps: After [setting up the app environment](/docs/infra/set-up-app-env.md), run: ```sh -nava-platform infra update --data app_has_dev_env_setup=true . +nava-platform infra update --answers-only --data app_has_dev_env_setup=true . ``` If you setup your AWS account in a different region than `us-east-1`, update the `aws-region` workflow settings to match your region. diff --git a/template-only-docs/set-up-pr-environments.md b/template-only-docs/set-up-pr-environments.md index 1df2db04..f9c78a94 100644 --- a/template-only-docs/set-up-pr-environments.md +++ b/template-only-docs/set-up-pr-environments.md @@ -3,6 +3,6 @@ [Pull request environments](/docs/infra/pull-request-environments.md) are temporary environments that exist during a pull request. Enable them after [setting up the app environment](/docs/infra/set-up-app-env.md): - In [ci-{{app_name}}-pr-environment-checks.yml](/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja) and [ci-{{app_name}}-pr-environment-destroy.yml](/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja), search for `!!`. -- Uncomment the PR environment triggers for spot testing. Run `nava-platform infra update --data app_has_dev_env_setup=true .` to consistently enable things. +- Uncomment the PR environment triggers for spot testing. Run `nava-platform infra update --answers-only --data app_has_dev_env_setup=true .` to consistently enable things. You can verify that PR environments are working by opening a new PR and waiting for the "PR Environment Update" job to finish. From 4df7757126da03f8acd92a4f22f1afc7dc6edebe Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Mon, 13 Jan 2025 19:12:05 -0500 Subject: [PATCH 59/61] platform-cli supports remote infra repos, don't recommend cloning --- README.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 77e37904..51c709b2 100644 --- a/README.md +++ b/README.md @@ -22,13 +22,12 @@ This template assumes that you have an application to deploy. See [application r ## Installation -To get started using the infrastructure template on your project, clone this -repository to a local directory next to your project, [install the nava-platform +To get started, [install the nava-platform tool](https://github.com/navapbc/platform-cli), and then run the following command in your project's root directory: ```sh -nava-platform infra install --template-uri ../template-infra . +nava-platform infra install . ``` Now you're ready to set up the various pieces of your infrastructure. @@ -46,9 +45,8 @@ After downloading and installing the template into your project: ## Updates With the [nava-platform tool -installed](https://github.com/navapbc/platform-cli), pull updates to your local -`template-infra` repository, then run the following in your project's root -directory: +installed](https://github.com/navapbc/platform-cli), run the following in your +project's root directory: ```sh nava-platform infra update . From 94f22c17dc478e7da0108df1c3df131d98aac831 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Mon, 13 Jan 2025 19:12:34 -0500 Subject: [PATCH 60/61] Fix terraform CLI script example arguments Should be prefixed with `infra/` --- docs/infra/making-infra-changes.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/infra/making-infra-changes.md b/docs/infra/making-infra-changes.md index 6c9c7235..88d5976c 100644 --- a/docs/infra/making-infra-changes.md +++ b/docs/infra/making-infra-changes.md @@ -45,9 +45,9 @@ TF_CLI_ARGS_apply='-var=image_tag=abcdef1' make infra-update-app-service APP_NAM An alternative to using the Makefile is to directly use the terraform wrapper scripts that the Makefile uses: ```bash -project-root$ ./bin/terraform-init /service dev -project-root$ ./bin/terraform-apply /service dev -project-root$ ./bin/terraform-init-and-apply /service dev # calls init and apply in the same script +project-root$ ./bin/terraform-init infra//service dev +project-root$ ./bin/terraform-apply infra//service dev +project-root$ ./bin/terraform-init-and-apply infra//service dev # calls init and apply in the same script ``` Look in the script files for more details on usage. From fd380bc34eeb8363f4d05ead3ad558bd3d28f761 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Tue, 14 Jan 2025 15:13:55 -0500 Subject: [PATCH 61/61] Update doc links to main in prep for merge --- .github/workflows/README.md | 8 ++++---- docs/infra/pull-request-environments.md | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 95e2fa45..9ae197a3 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -10,11 +10,11 @@ Each app should have: - `ci-`: must be created; should run linting and testing - `ci--vulnerability-scans`: calls `vulnerability-scans` - - Based on [ci-{{app_name}}-vulnerability-scans](https://github.com/navapbc/template-infra/blob/lorenyu%2Fplatform-cli/.github/workflows/ci-{{app_name}}-vulnerability-scans.yml.jinja) + - Based on [ci-{{app_name}}-vulnerability-scans](https://github.com/navapbc/template-infra/blob/main/.github/workflows/ci-{{app_name}}-vulnerability-scans.yml.jinja) - `ci--pr-environment-checks.yml`: calls `pr-environment-checks.yml` to create or update a pull request environment (see [pull request environments](/docs/infra/pull-request-environments.md)) - - Based on [ci-{{app_name}}-pr-environment-checks.yml](https://github.com/navapbc/template-infra/blob/lorenyu%2Fplatform-cli/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja) + - Based on [ci-{{app_name}}-pr-environment-checks.yml](https://github.com/navapbc/template-infra/blob/main/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja) - `ci--pr-environment-destroy.yml`: calls `pr-environment-destroy.yml` to destroy the pull request environment (see [pull request environments](/docs/infra/pull-request-environments.md)) - - Based on [ci-{{app_name}}-pr-environment-destroy.yml](https://github.com/navapbc/template-infra/blob/lorenyu%2Fplatform-cli/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja) + - Based on [ci-{{app_name}}-pr-environment-destroy.yml](https://github.com/navapbc/template-infra/blob/main/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja) ### App-agnostic workflows @@ -27,7 +27,7 @@ Each app should have: Each app should have: - `cd-`: deploys an application - - Based on [`cd-{{app_name}}`](https://github.com/navapbc/template-infra/blob/lorenyu%2Fplatform-cli/.github/workflows/cd-{{app_name}}.yml.jinja) + - Based on [`cd-{{app_name}}`](https://github.com/navapbc/template-infra/blob/main/.github/workflows/cd-{{app_name}}.yml.jinja) The CD workflow uses these reusable workflows: diff --git a/docs/infra/pull-request-environments.md b/docs/infra/pull-request-environments.md index e0b00cd7..513e3e54 100644 --- a/docs/infra/pull-request-environments.md +++ b/docs/infra/pull-request-environments.md @@ -29,6 +29,6 @@ Pull request environments are created by GitHub Actions workflows. There are two Using these reusable workflows, configure PR environments for each application with application-specific workflows: - `ci--pr-environment-checks.yml` - - Based on [ci-{{app_name}}-pr-environment-checks.yml](https://github.com/navapbc/template-infra/blob/lorenyu%2Fplatform-cli/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja) + - Based on [ci-{{app_name}}-pr-environment-checks.yml](https://github.com/navapbc/template-infra/blob/main/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja) - `ci--pr-environment-destroy.yml` - - Based on [ci-{{app_name}}-pr-environment-destroy.yml](https://github.com/navapbc/template-infra/blob/lorenyu%2Fplatform-cli/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja) + - Based on [ci-{{app_name}}-pr-environment-destroy.yml](https://github.com/navapbc/template-infra/blob/main/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja)