From 38bc56f1c56a87b8b8dbaf19a97bdcbb27865971 Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Thu, 9 Sep 2021 16:35:24 -0500 Subject: [PATCH 01/26] RDD: Introduce import blocks to support multiple includes --- docs/_docs/02_features/auto-init.md | 2 +- docs/_docs/02_features/auto-retry.md | 2 +- docs/_docs/02_features/aws-auth.md | 2 +- .../02_features/before-and-after-hooks.md | 2 +- docs/_docs/02_features/caching.md | 2 +- docs/_docs/02_features/debugging.md | 2 +- ...rm-commands-on-multiple-modules-at-once.md | 2 +- docs/_docs/02_features/inputs.md | 2 +- .../02_features/keep-your-cli-flags-dry.md | 2 +- ...eep-your-remote-state-configuration-dry.md | 2 +- .../keep-your-terragrunt-architecture-dry.md | 347 ++++++++++++++++++ docs/_docs/02_features/locals.md | 2 +- docs/_docs/02_features/lock-file-handling.md | 2 +- .../work-with-multiple-aws-accounts.md | 2 +- docs/_docs/04_reference/built-in-functions.md | 37 ++ .../config-blocks-and-attributes.md | 55 ++- 16 files changed, 444 insertions(+), 21 deletions(-) create mode 100644 docs/_docs/02_features/keep-your-terragrunt-architecture-dry.md diff --git a/docs/_docs/02_features/auto-init.md b/docs/_docs/02_features/auto-init.md index b698d7c04c..49ac9b029c 100644 --- a/docs/_docs/02_features/auto-init.md +++ b/docs/_docs/02_features/auto-init.md @@ -5,7 +5,7 @@ category: features categories_url: features excerpt: Auto-Init is a feature of Terragrunt that makes it so that terragrunt init does not need to be called explicitly before other terragrunt commands. tags: ["CLI"] -order: 209 +order: 245 nav_title: Documentation nav_title_link: /docs/ --- diff --git a/docs/_docs/02_features/auto-retry.md b/docs/_docs/02_features/auto-retry.md index cbd205cba0..c4282581b1 100644 --- a/docs/_docs/02_features/auto-retry.md +++ b/docs/_docs/02_features/auto-retry.md @@ -5,7 +5,7 @@ category: features categories_url: features excerpt: Auto-Retry is a feature of terragrunt that will automatically address situations where a terraform command needs to be re-run. tags: ["CLI"] -order: 210 +order: 250 nav_title: Documentation nav_title_link: /docs/ --- diff --git a/docs/_docs/02_features/aws-auth.md b/docs/_docs/02_features/aws-auth.md index 882878db0e..41da15b3ba 100644 --- a/docs/_docs/02_features/aws-auth.md +++ b/docs/_docs/02_features/aws-auth.md @@ -5,7 +5,7 @@ category: features categories_url: features excerpt: Learn how the Terragrunt works with AWS Credentials and AWS IAM policies. tags: ["AWS"] -order: 213 +order: 260 nav_title: Documentation nav_title_link: /docs/ --- diff --git a/docs/_docs/02_features/before-and-after-hooks.md b/docs/_docs/02_features/before-and-after-hooks.md index b6d452d6c6..4a2dc0a082 100644 --- a/docs/_docs/02_features/before-and-after-hooks.md +++ b/docs/_docs/02_features/before-and-after-hooks.md @@ -5,7 +5,7 @@ category: features categories_url: features excerpt: Learn how to execute custom code before or after running Terraform. tags: ["hooks"] -order: 208 +order: 240 nav_title: Documentation nav_title_link: /docs/ --- diff --git a/docs/_docs/02_features/caching.md b/docs/_docs/02_features/caching.md index a888466fb5..671e4cc7f5 100644 --- a/docs/_docs/02_features/caching.md +++ b/docs/_docs/02_features/caching.md @@ -5,7 +5,7 @@ category: features categories_url: features excerpt: Learn more about caching in Terragrunt. tags: ["caching"] -order: 211 +order: 255 nav_title: Documentation nav_title_link: /docs/ --- diff --git a/docs/_docs/02_features/debugging.md b/docs/_docs/02_features/debugging.md index 8461b297ca..bb1af029be 100644 --- a/docs/_docs/02_features/debugging.md +++ b/docs/_docs/02_features/debugging.md @@ -5,7 +5,7 @@ category: features categories_url: features excerpt: Learn how to debug issues with terragrunt and terraform. tags: ["DRY", "Use cases", "CLI"] -order: 220 +order: 265 nav_title: Documentation nav_title_link: /docs/ --- diff --git a/docs/_docs/02_features/execute-terraform-commands-on-multiple-modules-at-once.md b/docs/_docs/02_features/execute-terraform-commands-on-multiple-modules-at-once.md index 2aade2378a..dca408f126 100644 --- a/docs/_docs/02_features/execute-terraform-commands-on-multiple-modules-at-once.md +++ b/docs/_docs/02_features/execute-terraform-commands-on-multiple-modules-at-once.md @@ -5,7 +5,7 @@ category: features categories_url: features excerpt: Learn how to avoid tedious tasks of running commands on each module separately. tags: ["DRY", "Modules", "Use cases", "CLI"] -order: 203 +order: 220 nav_title: Documentation nav_title_link: /docs/ --- diff --git a/docs/_docs/02_features/inputs.md b/docs/_docs/02_features/inputs.md index 3b1bbf4a24..db2ed188c6 100644 --- a/docs/_docs/02_features/inputs.md +++ b/docs/_docs/02_features/inputs.md @@ -5,7 +5,7 @@ category: features categories_url: features excerpt: Learn how to use inputs. tags: ["inputs"] -order: 205 +order: 230 nav_title: Documentation nav_title_link: /docs/ --- diff --git a/docs/_docs/02_features/keep-your-cli-flags-dry.md b/docs/_docs/02_features/keep-your-cli-flags-dry.md index 34a47d72c3..315a79a0e2 100644 --- a/docs/_docs/02_features/keep-your-cli-flags-dry.md +++ b/docs/_docs/02_features/keep-your-cli-flags-dry.md @@ -5,7 +5,7 @@ category: features categories_url: features excerpt: Learn how to keep CLI flags DRY with "extra_arguments" block in your "terragrunt.hcl". tags: ["DRY", "Use cases", "CLI"] -order: 202 +order: 215 nav_title: Documentation nav_title_link: /docs/ --- diff --git a/docs/_docs/02_features/keep-your-remote-state-configuration-dry.md b/docs/_docs/02_features/keep-your-remote-state-configuration-dry.md index 73e5aacdb4..daa3eea0f4 100644 --- a/docs/_docs/02_features/keep-your-remote-state-configuration-dry.md +++ b/docs/_docs/02_features/keep-your-remote-state-configuration-dry.md @@ -5,7 +5,7 @@ category: features categories_url: features excerpt: Learn how to create and manage remote state configuration. tags: ["DRY", "remote", "Use cases", "CLI"] -order: 201 +order: 205 nav_title: Documentation nav_title_link: /docs/ --- diff --git a/docs/_docs/02_features/keep-your-terragrunt-architecture-dry.md b/docs/_docs/02_features/keep-your-terragrunt-architecture-dry.md new file mode 100644 index 0000000000..6c36178012 --- /dev/null +++ b/docs/_docs/02_features/keep-your-terragrunt-architecture-dry.md @@ -0,0 +1,347 @@ +--- +layout: collection-browser-doc +title: Keep your Terragrunt Architecture DRY +category: features +categories_url: features +excerpt: Learn how to use multiple terragrunt configurations to DRY up your architecture. +tags: ["DRY", "Use cases", "backend"] +order: 210 +nav_title: Documentation +nav_title_link: /docs/ +--- + +## Keep your Terragrunt Architecture DRY + + - [Motivation](#motivation) + + - [Using import to DRY common Terragrunt config](#using-import-to-dry-common-terragrunt-config) + + - [Using read\_terragrunt\_config to DRY parent configurations](#using-read_terragrunt_config-to-dry-parent-configurations) + + +### Motivation + +As covered in [Keep your Terraform code DRY]({{site.baseurl}}/docs/features/keep-your-terraform-code-dry) and [Keep your +remote state configuration DRY]({{site.baseurl}}/docs/features/keep-your-remote-state-configuration-dry), it becomes +important to define base Terragrunt configuration files that are included in the child config. For example, you might +have a **root** Terragrunt configuration that defines the remote state and provider configurations: + +```hcl +remote_state { + backend = "s3" + config = { + bucket = "my-terraform-state" + key = "${path_relative_to_include()}/terraform.tfstate" + region = "us-east-1" + encrypt = true + dynamodb_table = "my-lock-table" + } +} + +generate "provider" { + path = "provider.tf" + if_exists = "overwrite_terragrunt" + contents = < stage -> prod. + module_version = { + qa = "v0.2.0" + stage = "v0.1.0" + prod = "v0.1.0" + } +} + +terraform { + source = "github.com//modules.git//app?ref=${local.module_version[local.env_name]}" +} + +dependency "vpc" { + config_path = "../vpc" +} + +dependency "mysql" { + config_path = "../mysql" +} + +inputs = { + env = local.env_name + basename = "example-app" + vpc_id = dependency.vpc.outputs.vpc_id + subnet_ids = dependency.vpc.outputs.subnet_ids + mysql_endpoint = dependency.mysql.outputs.endpoint +} +``` + +With this configuration, `env_vars` is loaded based on which folder is being invoked. For example, when Terragrunt is +invoked in the `prod/app/terragrunt.hcl` folder, `prod/env.hcl` is loaded, while `qa/env.hcl` is loaded when +Terragrunt is invoked in the `qa/app/terragrunt.hcl` folder. + +Now we can keep the same child config even if we have different versions to deploy per environment. As a bonus, we can +further reduce our child config to eliminate the `env` input variable since that is loaded in the `env.hcl` context: + +```hcl +import "root" { + path = find_in_parent_folders() +} + +import "env" { + path = "${get_terragrunt_dir()}/../../_env/app.hcl" +} +``` diff --git a/docs/_docs/02_features/locals.md b/docs/_docs/02_features/locals.md index 5b18c29ddf..abebbae6fe 100644 --- a/docs/_docs/02_features/locals.md +++ b/docs/_docs/02_features/locals.md @@ -5,7 +5,7 @@ category: features categories_url: features excerpt: Learn how to use locals. tags: ["locals"] -order: 206 +order: 235 nav_title: Documentation nav_title_link: /docs/ --- diff --git a/docs/_docs/02_features/lock-file-handling.md b/docs/_docs/02_features/lock-file-handling.md index 75018bebe2..c2e8a81f54 100644 --- a/docs/_docs/02_features/lock-file-handling.md +++ b/docs/_docs/02_features/lock-file-handling.md @@ -5,7 +5,7 @@ category: features categories_url: features excerpt: Learn how to Terragrunt handles the Terraform lock file tags: ["CLI", "DRY"] -order: 230 +order: 270 nav_title: Documentation nav_title_link: /docs/ --- diff --git a/docs/_docs/02_features/work-with-multiple-aws-accounts.md b/docs/_docs/02_features/work-with-multiple-aws-accounts.md index fbeb523de7..7a95cb9eed 100644 --- a/docs/_docs/02_features/work-with-multiple-aws-accounts.md +++ b/docs/_docs/02_features/work-with-multiple-aws-accounts.md @@ -5,7 +5,7 @@ category: features categories_url: features excerpt: Learn how the Terragrunt may help you to work with mulitple AWS accounts. tags: ["AWS", "Use cases", "CLI", "AWS IAM"] -order: 204 +order: 225 nav_title: Documentation nav_title_link: /docs/ --- diff --git a/docs/_docs/04_reference/built-in-functions.md b/docs/_docs/04_reference/built-in-functions.md index 4bc0590ea7..dddd678604 100644 --- a/docs/_docs/04_reference/built-in-functions.md +++ b/docs/_docs/04_reference/built-in-functions.md @@ -172,6 +172,24 @@ remote_state { The resulting `key` will be `prod/mysql/terraform.tfstate` for the prod `mysql` module and `stage/mysql/terraform.tfstate` for the stage `mysql` module. +If you have `import` blocks, this function requires a `name` parameter when used in the child config to specify which +`import` block to base the relative path on. + +Example: + +```hcl +import "root" { + path = find_in_parent_folders() +} +import "region" { + path = find_in_parent_folders("region.hcl") +} + +terraform { + source = "../modules/${path_relative_to_include("root")}" +} +``` + ## path\_relative\_from\_include `path_relative_from_include()` returns the relative path between the `path` specified in its `include` block and the current `terragrunt.hcl` file (it is the counterpart of `path_relative_to_include()`). For example, consider the following folder structure: @@ -231,6 +249,25 @@ Another use case would be to add extra argument to include the `common.tfvars` f This allows proper retrieval of the `common.tfvars` from whatever the level of subdirectories we have. +If you have `import` blocks, this function requires a `name` parameter when used in the child config to specify which +`import` block to base the relative path on. + +Example: + +```hcl +import "root" { + path = find_in_parent_folders() +} +import "region" { + path = find_in_parent_folders("region.hcl") +} + +terraform { + source = "../modules/${path_relative_from_include("root")}" +} +``` + + ## get\_env `get_env(NAME)` return the value of variable named `NAME` or throws exceptions if that variable is not set. Example: diff --git a/docs/_docs/04_reference/config-blocks-and-attributes.md b/docs/_docs/04_reference/config-blocks-and-attributes.md index a6bf5c351a..f0b19f55a8 100644 --- a/docs/_docs/04_reference/config-blocks-and-attributes.md +++ b/docs/_docs/04_reference/config-blocks-and-attributes.md @@ -21,7 +21,7 @@ The following is a reference of all the supported blocks and attributes in the c - [terraform](#terraform) - [remote_state](#remote_state) -- [include](#include) +- [import/include](#import-include) - [locals](#locals) - [dependency](#dependency) - [dependencies](#dependencies) @@ -369,27 +369,38 @@ remote_state { -### include +### import/include -The `include` block is used to specify inheritance of Terragrunt configuration files. The included config (also called +The `import` and `include` blocks are used to specify inheritance of Terragrunt configuration files. The included config (also called the `parent`) will be merged with the current configuration (also called the `child`) before processing. You can learn more about the inheritance properties of Terragrunt in the [Filling in remote state settings with Terragrunt section](/docs/features/keep-your-remote-state-configuration-dry/#filling-in-remote-state-settings-with-terragrunt) of the "Keep your remote state configuration DRY" use case overview. -The `include` block supports the following arguments: +`include` blocks are a special case of the `import` block, where there is no label associated with it (equivalent to +`import "" {}`). This is a special shorthand you can use if you only have a single import. While you can have multiple +`import` blocks in a single config, you can only have one `include` block per config. +Both `import` and `include` blocks supports the following arguments: + +- `name` (label; only for `import`): You can define multiple `import` blocks in a single terragrunt config. Each import + must be labeled with a unique name to differentiate it from the other imports. E.g., if you had a block `import + "remote" {}`, you can reference the relevant exposed data with the expression `include.remote`. Defaults to `""` for + `include` blocks. - `path` (attribute): Specifies the path to a Terragrunt configuration file (the `parent` config) that should be merged with this configuration (the `child` config). -- `expose` (attribute, optional): Specifies whether or not the included config should be parsed and exposed as a - variable. When `true`, you can reference the data of the included config under the variable `include`. Defaults to - `false`. +- `expose` (attribute, optional): Specifies whether or not the imported config should be parsed and exposed as a + variable. When `true`, you can reference the data of the imported config under the variable `include`. Defaults to + `false`. Note that the `include` variable is a map of `import` labels to the parsed configuration value when there are + multiple `import` blocks, while it will be the actual parsed configuration value when there is only one + `import`/`include` block. - `merge_strategy` (attribute, optional): Specifies how the included config should be merged. Valid values are: `no_merge` (do not merge the included config), `shallow` (do a shallow merge - default), `deep` (do a deep merge of the included config). -Example: +Examples: +_Single import_ ```hcl # If you have the following folder structure, and the following contents for ./child/terragrunt.hcl, this will include # and merge the items in the terragrunt.hcl file at the root. @@ -408,6 +419,34 @@ inputs = { } ``` +_Multiple imports_ +```hcl +# If you have the following folder structure, and the following contents for ./child/terragrunt.hcl, this will import +# and merge the items in the terragrunt.hcl file at the root, while only loading the data in the region.hcl +# configuration. +# +# . +# ├── terragrunt.hcl +# ├── region.hcl +# └── child +# └── terragrunt.hcl +import "remote_state" { + path = find_in_parent_folders() + expose = true +} + +import "region" { + path = find_in_parent_folders("region.hcl") + expose = true + merge_strategy = "no_merge" +} + +inputs = { + remote_state_config = include.remote_state.remote_state + region = include.region.region +} +``` + **What is deep merge?** When the `merge_strategy` for the `include` block is set to `deep`, Terragrunt will perform a deep merge of the included From 0bb98255dc21fd75900e3e44cda247c7a3d22595 Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Fri, 10 Sep 2021 11:44:47 -0500 Subject: [PATCH 02/26] Go back to include instead of import --- .../keep-your-terragrunt-architecture-dry.md | 30 +++++++------- docs/_docs/04_reference/built-in-functions.md | 16 ++++---- .../config-blocks-and-attributes.md | 40 +++++++++---------- 3 files changed, 41 insertions(+), 45 deletions(-) diff --git a/docs/_docs/02_features/keep-your-terragrunt-architecture-dry.md b/docs/_docs/02_features/keep-your-terragrunt-architecture-dry.md index 6c36178012..c9179fef79 100644 --- a/docs/_docs/02_features/keep-your-terragrunt-architecture-dry.md +++ b/docs/_docs/02_features/keep-your-terragrunt-architecture-dry.md @@ -14,7 +14,7 @@ nav_title_link: /docs/ - [Motivation](#motivation) - - [Using import to DRY common Terragrunt config](#using-import-to-dry-common-terragrunt-config) + - [Using include to DRY common Terragrunt config](#using-include-to-dry-common-terragrunt-config) - [Using read\_terragrunt\_config to DRY parent configurations](#using-read_terragrunt_config-to-dry-parent-configurations) @@ -60,7 +60,7 @@ include { } ``` -This pattern is useful for global configuration blocks that need to be imported in all of your modules, but what if you +This pattern is useful for global configuration blocks that need to be included in all of your modules, but what if you have Terragrunt configurations that are only relevant to subsets of your module? For example, consider the following terragrunt file structure, which defines three environments (`prod`, `qa`, and `stage`) with the same infrastructure in each one (an app, a MySQL database, and a VPC): @@ -95,11 +95,9 @@ adjustment to the `instance_type` parameter for each environment. These identica `terragrunt.hcl` configuration because they are only relevant to the `app` configurations, and not `mysql` or `vpc`. However, it is cumbersome to copy paste these settings across all three environments. -To solve this, you can use [import instead of -include]({{site.baseurl}}/docs/reference/config-blocks-and-attributes#import-include), which supports merging multiple -included configurations unlike `include` blocks. +To solve this, you can use [multiple include blocks]({{site.baseurl}}/docs/reference/config-blocks-and-attributes#include). -### Using import to DRY common Terragrunt config +### Using include to DRY common Terragrunt config Suppose your `qa/app/terragrunt.hcl` configuration looks like the following: @@ -186,16 +184,16 @@ inputs = { } ``` -Note that everything is defined except for the `env` input variable. We now modify `qa/app/terragrunt.hcl` to import -this alongside the root configuration by using `import` blocks instead of `include`, significantly reducing our per +Note that everything is defined except for the `env` input variable. We now modify `qa/app/terragrunt.hcl` to include +this alongside the root configuration by using multiple `include` blocks, significantly reducing our per environment configuration: ```hcl -import "root" { +include "root" { path = find_in_parent_folders() } -import "env" { +include "env" { path = "${get_terragrunt_dir()}/../../_env/app.hcl" } @@ -206,8 +204,8 @@ inputs = { ### Using read\_terragrunt\_config to DRY parent configurations -In the previous section, we covered using `import` to DRY common component configurations. While powerful, `import` has -a limitation where the imported configuration is statically merged into the child configuration. +In the previous section, we covered using `include` to DRY common component configurations. While powerful, `include` has +a limitation where the included configuration is statically merged into the child configuration. In our example, note that the `_env/app.hcl` file hardcodes the `app `module version to `v0.1.0` (relevant section pasted below for convenience): @@ -225,11 +223,11 @@ What if we want to deploy a different version for each environment? One way you the following: ```hcl -import "root" { +include "root" { path = find_in_parent_folders() } -import "env" { +include "env" { path = "${get_terragrunt_dir()}/../../_env/app.hcl" } @@ -337,11 +335,11 @@ Now we can keep the same child config even if we have different versions to depl further reduce our child config to eliminate the `env` input variable since that is loaded in the `env.hcl` context: ```hcl -import "root" { +include "root" { path = find_in_parent_folders() } -import "env" { +include "env" { path = "${get_terragrunt_dir()}/../../_env/app.hcl" } ``` diff --git a/docs/_docs/04_reference/built-in-functions.md b/docs/_docs/04_reference/built-in-functions.md index dddd678604..70b0b2bb8f 100644 --- a/docs/_docs/04_reference/built-in-functions.md +++ b/docs/_docs/04_reference/built-in-functions.md @@ -172,16 +172,16 @@ remote_state { The resulting `key` will be `prod/mysql/terraform.tfstate` for the prod `mysql` module and `stage/mysql/terraform.tfstate` for the stage `mysql` module. -If you have `import` blocks, this function requires a `name` parameter when used in the child config to specify which -`import` block to base the relative path on. +If you have `include` blocks, this function requires a `name` parameter when used in the child config to specify which +`include` block to base the relative path on. Example: ```hcl -import "root" { +include "root" { path = find_in_parent_folders() } -import "region" { +include "region" { path = find_in_parent_folders("region.hcl") } @@ -249,16 +249,16 @@ Another use case would be to add extra argument to include the `common.tfvars` f This allows proper retrieval of the `common.tfvars` from whatever the level of subdirectories we have. -If you have `import` blocks, this function requires a `name` parameter when used in the child config to specify which -`import` block to base the relative path on. +If you have `include` blocks, this function requires a `name` parameter when used in the child config to specify which +`include` block to base the relative path on. Example: ```hcl -import "root" { +include "root" { path = find_in_parent_folders() } -import "region" { +include "region" { path = find_in_parent_folders("region.hcl") } diff --git a/docs/_docs/04_reference/config-blocks-and-attributes.md b/docs/_docs/04_reference/config-blocks-and-attributes.md index f0b19f55a8..4116000754 100644 --- a/docs/_docs/04_reference/config-blocks-and-attributes.md +++ b/docs/_docs/04_reference/config-blocks-and-attributes.md @@ -21,7 +21,7 @@ The following is a reference of all the supported blocks and attributes in the c - [terraform](#terraform) - [remote_state](#remote_state) -- [import/include](#import-include) +- [include](#include) - [locals](#locals) - [dependency](#dependency) - [dependencies](#dependencies) @@ -369,38 +369,36 @@ remote_state { -### import/include +### include -The `import` and `include` blocks are used to specify inheritance of Terragrunt configuration files. The included config (also called +The `include` block is used to specify inheritance of Terragrunt configuration files. The included config (also called the `parent`) will be merged with the current configuration (also called the `child`) before processing. You can learn more about the inheritance properties of Terragrunt in the [Filling in remote state settings with Terragrunt section](/docs/features/keep-your-remote-state-configuration-dry/#filling-in-remote-state-settings-with-terragrunt) of the "Keep your remote state configuration DRY" use case overview. -`include` blocks are a special case of the `import` block, where there is no label associated with it (equivalent to -`import "" {}`). This is a special shorthand you can use if you only have a single import. While you can have multiple -`import` blocks in a single config, you can only have one `include` block per config. +You can have more than one `include` block, but each one must have a unique label. Note that a bare `include` block with +no label (`include {}`) is a short hand for an `include` block that uses the label `""` (equivalent to `include "" {}`). -Both `import` and `include` blocks supports the following arguments: +`include` blocks support the following arguments: -- `name` (label; only for `import`): You can define multiple `import` blocks in a single terragrunt config. Each import - must be labeled with a unique name to differentiate it from the other imports. E.g., if you had a block `import - "remote" {}`, you can reference the relevant exposed data with the expression `include.remote`. Defaults to `""` for - `include` blocks. +- `name` (label): You can define multiple `include` blocks in a single terragrunt config. Each include block + must be labeled with a unique name to differentiate it from the other includes. E.g., if you had a block `include + "remote" {}`, you can reference the relevant exposed data with the expression `include.remote`. - `path` (attribute): Specifies the path to a Terragrunt configuration file (the `parent` config) that should be merged with this configuration (the `child` config). -- `expose` (attribute, optional): Specifies whether or not the imported config should be parsed and exposed as a - variable. When `true`, you can reference the data of the imported config under the variable `include`. Defaults to - `false`. Note that the `include` variable is a map of `import` labels to the parsed configuration value when there are - multiple `import` blocks, while it will be the actual parsed configuration value when there is only one - `import`/`include` block. +- `expose` (attribute, optional): Specifies whether or not the included config should be parsed and exposed as a + variable. When `true`, you can reference the data of the included config under the variable `include`. Defaults to + `false`. Note that the `include` variable is a map of `include` labels to the parsed configuration value when there are + multiple `include` blocks, while it will be the actual parsed configuration value when there is only one + `include` block. - `merge_strategy` (attribute, optional): Specifies how the included config should be merged. Valid values are: `no_merge` (do not merge the included config), `shallow` (do a shallow merge - default), `deep` (do a deep merge of the included config). Examples: -_Single import_ +_Single include_ ```hcl # If you have the following folder structure, and the following contents for ./child/terragrunt.hcl, this will include # and merge the items in the terragrunt.hcl file at the root. @@ -419,9 +417,9 @@ inputs = { } ``` -_Multiple imports_ +_Multiple includes_ ```hcl -# If you have the following folder structure, and the following contents for ./child/terragrunt.hcl, this will import +# If you have the following folder structure, and the following contents for ./child/terragrunt.hcl, this will include # and merge the items in the terragrunt.hcl file at the root, while only loading the data in the region.hcl # configuration. # @@ -430,12 +428,12 @@ _Multiple imports_ # ├── region.hcl # └── child # └── terragrunt.hcl -import "remote_state" { +include "remote_state" { path = find_in_parent_folders() expose = true } -import "region" { +include "region" { path = find_in_parent_folders("region.hcl") expose = true merge_strategy = "no_merge" From 2e6099f838d89f395f516f03ebf012c2e5283cd4 Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Thu, 9 Sep 2021 12:04:36 -0500 Subject: [PATCH 03/26] WIP: initial attempt to support multiple include blocks --- config/config.go | 12 +-- config/config_helpers.go | 168 ++++++++++++++++++++++++++----------- config/config_partial.go | 88 ++++++++++++-------- config/cty_helpers.go | 24 ++++-- config/dependency.go | 6 +- config/include.go | 173 ++++++++++++++++++++++----------------- config/locals.go | 4 +- 7 files changed, 303 insertions(+), 172 deletions(-) diff --git a/config/config.go b/config/config.go index f7193bb7c4..572ba01a25 100644 --- a/config/config.go +++ b/config/config.go @@ -189,6 +189,7 @@ type terragruntGenerateBlock struct { // IncludeConfig represents the configuration settings for a parent Terragrunt configuration file that you can // "include" in a child Terragrunt configuration file type IncludeConfig struct { + Name string `hcl:",label"` Path string `hcl:"path,attr"` Expose *bool `hcl:"expose,attr"` MergeStrategy *string `hcl:"merge_strategy,attr"` @@ -587,7 +588,7 @@ func ParseConfigString( } // Decode just the Base blocks. See the function docs for DecodeBaseBlocks for more info on what base blocks are. - localsAsCty, terragruntInclude, trackInclude, err := DecodeBaseBlocks(terragruntOptions, parser, file, filename, includeFromChild) + localsAsCty, trackInclude, err := DecodeBaseBlocks(terragruntOptions, parser, file, filename, includeFromChild) if err != nil { return nil, err } @@ -602,7 +603,7 @@ func ParseConfigString( if dependencyOutputs == nil { // Decode just the `dependency` blocks, retrieving the outputs from the target terragrunt config in the // process. - retrievedOutputs, err := decodeAndRetrieveOutputs(file, filename, terragruntOptions, terragruntInclude.Include, contextExtensions) + retrievedOutputs, err := decodeAndRetrieveOutputs(file, filename, terragruntOptions, trackInclude, contextExtensions) if err != nil { return nil, err } @@ -625,11 +626,10 @@ func ParseConfigString( } // If this file includes another, parse and merge it. Otherwise just return this config. - if terragruntInclude.Include != nil { - return handleInclude(config, terragruntInclude.Include, terragruntOptions, contextExtensions.DecodedDependencies) - } else { - return config, nil + if trackInclude != nil { + return handleInclude(config, trackInclude, terragruntOptions, contextExtensions.DecodedDependencies) } + return config, nil } func decodeAsTerragruntConfigFile( diff --git a/config/config_helpers.go b/config/config_helpers.go index 817d32b95a..00eea6bff2 100644 --- a/config/config_helpers.go +++ b/config/config_helpers.go @@ -70,9 +70,13 @@ type EnvVar struct { // TrackInclude is used to differentiate between an included config in the current parsing context, and an included // config that was passed through from a previous parsing context. type TrackInclude struct { - // Current is used to specify another config that should be imported and merged before the final - // TerragruntConfig is returned. - Current *IncludeConfig + // CurrentList is used to track the list of configs that should be imported and merged before the final + // TerragruntConfig is returned. This preserves the order of the blocks as they appear in the config, so that we can + // merge the included config in the right order. + CurrentList []IncludeConfig + + // CurrentMap is the map version of CurrentList that maps the block labels to the included config. + CurrentMap map[string]IncludeConfig // Original is used to track the original included config, and is used for resolving the include related // functions. @@ -82,7 +86,7 @@ type TrackInclude struct { // EvalContextExtensions provides various extensions to the evaluation context to enhance the parsing capabilities. type EvalContextExtensions struct { // TrackInclude represents contexts of included configurations. - TrackInclude TrackInclude + TrackInclude *TrackInclude // Locals are preevaluated variable bindings that can be used by reference in the code. Locals *cty.Value @@ -106,27 +110,27 @@ func CreateTerragruntEvalContext( } terragruntFunctions := map[string]function.Function{ - "find_in_parent_folders": wrapStringSliceToStringAsFuncImpl(findInParentFolders, extensions.TrackInclude.Original, terragruntOptions), - "path_relative_to_include": wrapVoidToStringAsFuncImpl(pathRelativeToInclude, extensions.TrackInclude.Original, terragruntOptions), - "path_relative_from_include": wrapVoidToStringAsFuncImpl(pathRelativeFromInclude, extensions.TrackInclude.Original, terragruntOptions), - "get_env": wrapStringSliceToStringAsFuncImpl(getEnvironmentVariable, extensions.TrackInclude.Original, terragruntOptions), - "run_cmd": wrapStringSliceToStringAsFuncImpl(runCommand, extensions.TrackInclude.Original, terragruntOptions), + "find_in_parent_folders": wrapStringSliceToStringAsFuncImpl(findInParentFolders, extensions.TrackInclude, terragruntOptions), + "path_relative_to_include": wrapVoidToStringAsFuncImpl(pathRelativeToInclude, extensions.TrackInclude, terragruntOptions), + "path_relative_from_include": wrapStringSliceToStringAsFuncImpl(pathRelativeFromInclude, extensions.TrackInclude, terragruntOptions), + "get_env": wrapStringSliceToStringAsFuncImpl(getEnvironmentVariable, extensions.TrackInclude, terragruntOptions), + "run_cmd": wrapStringSliceToStringAsFuncImpl(runCommand, extensions.TrackInclude, terragruntOptions), "read_terragrunt_config": readTerragruntConfigAsFuncImpl(terragruntOptions), - "get_platform": wrapVoidToStringAsFuncImpl(getPlatform, extensions.TrackInclude.Original, terragruntOptions), - "get_terragrunt_dir": wrapVoidToStringAsFuncImpl(getTerragruntDir, extensions.TrackInclude.Original, terragruntOptions), - "get_original_terragrunt_dir": wrapVoidToStringAsFuncImpl(getOriginalTerragruntDir, extensions.TrackInclude.Original, terragruntOptions), - "get_terraform_command": wrapVoidToStringAsFuncImpl(getTerraformCommand, extensions.TrackInclude.Original, terragruntOptions), - "get_terraform_cli_args": wrapVoidToStringSliceAsFuncImpl(getTerraformCliArgs, extensions.TrackInclude.Original, terragruntOptions), - "get_parent_terragrunt_dir": wrapVoidToStringAsFuncImpl(getParentTerragruntDir, extensions.TrackInclude.Original, terragruntOptions), - "get_aws_account_id": wrapVoidToStringAsFuncImpl(getAWSAccountID, extensions.TrackInclude.Original, terragruntOptions), - "get_aws_caller_identity_arn": wrapVoidToStringAsFuncImpl(getAWSCallerIdentityARN, extensions.TrackInclude.Original, terragruntOptions), - "get_aws_caller_identity_user_id": wrapVoidToStringAsFuncImpl(getAWSCallerIdentityUserID, extensions.TrackInclude.Original, terragruntOptions), + "get_platform": wrapVoidToStringAsFuncImpl(getPlatform, extensions.TrackInclude, terragruntOptions), + "get_terragrunt_dir": wrapVoidToStringAsFuncImpl(getTerragruntDir, extensions.TrackInclude, terragruntOptions), + "get_original_terragrunt_dir": wrapVoidToStringAsFuncImpl(getOriginalTerragruntDir, extensions.TrackInclude, terragruntOptions), + "get_terraform_command": wrapVoidToStringAsFuncImpl(getTerraformCommand, extensions.TrackInclude, terragruntOptions), + "get_terraform_cli_args": wrapVoidToStringSliceAsFuncImpl(getTerraformCliArgs, extensions.TrackInclude, terragruntOptions), + "get_parent_terragrunt_dir": wrapStringSliceToStringAsFuncImpl(getParentTerragruntDir, extensions.TrackInclude, terragruntOptions), + "get_aws_account_id": wrapVoidToStringAsFuncImpl(getAWSAccountID, extensions.TrackInclude, terragruntOptions), + "get_aws_caller_identity_arn": wrapVoidToStringAsFuncImpl(getAWSCallerIdentityARN, extensions.TrackInclude, terragruntOptions), + "get_aws_caller_identity_user_id": wrapVoidToStringAsFuncImpl(getAWSCallerIdentityUserID, extensions.TrackInclude, terragruntOptions), "get_terraform_commands_that_need_vars": wrapStaticValueToStringSliceAsFuncImpl(TERRAFORM_COMMANDS_NEED_VARS), "get_terraform_commands_that_need_locking": wrapStaticValueToStringSliceAsFuncImpl(TERRAFORM_COMMANDS_NEED_LOCKING), "get_terraform_commands_that_need_input": wrapStaticValueToStringSliceAsFuncImpl(TERRAFORM_COMMANDS_NEED_INPUT), "get_terraform_commands_that_need_parallelism": wrapStaticValueToStringSliceAsFuncImpl(TERRAFORM_COMMANDS_NEED_PARALLELISM), - "sops_decrypt_file": wrapStringSliceToStringAsFuncImpl(sopsDecryptFile, extensions.TrackInclude.Original, terragruntOptions), - "get_terragrunt_source_cli_flag": wrapVoidToStringAsFuncImpl(getTerragruntSourceCliFlag, extensions.TrackInclude.Original, terragruntOptions), + "sops_decrypt_file": wrapStringSliceToStringAsFuncImpl(sopsDecryptFile, extensions.TrackInclude, terragruntOptions), + "get_terragrunt_source_cli_flag": wrapVoidToStringAsFuncImpl(getTerragruntSourceCliFlag, extensions.TrackInclude, terragruntOptions), } functions := map[string]function.Function{} @@ -147,29 +151,47 @@ func CreateTerragruntEvalContext( if extensions.DecodedDependencies != nil { ctx.Variables["dependency"] = *extensions.DecodedDependencies } - if extensions.TrackInclude.Current != nil && extensions.TrackInclude.Current.GetExpose() { - includedConfig, err := parseIncludedConfig(extensions.TrackInclude.Current, terragruntOptions, extensions.DecodedDependencies) - if err != nil { - return ctx, err + if len(extensions.TrackInclude.CurrentMap) > 0 { + // For each include block, check if we want to expose the included config, and if so, add under the include + // variable. + exposedIncludeMap := map[string]cty.Value{} + for key, included := range extensions.TrackInclude.CurrentMap { + if included.GetExpose() { + parsedIncluded, err := parseIncludedConfig(&included, terragruntOptions, extensions.DecodedDependencies) + if err != nil { + return ctx, err + } + parsedIncludedCty, err := terragruntConfigAsCty(parsedIncluded) + if err != nil { + return ctx, err + } + exposedIncludeMap[key] = parsedIncludedCty + } } - // MAINTAINER'S NOTE: For now, we don't support multiple include blocks in a terragrunt config. In the future, - // when we support multiple include blocks, expose each included config indexed by the label. - ctx.Variables["include"], err = terragruntConfigAsCty(includedConfig) - if err != nil { - return ctx, err + if len(exposedIncludeMap) == 1 { + // If we have only one exposed include map, then flatten the map as a shorthand + for _, val := range exposedIncludeMap { + ctx.Variables["include"] = val + } + } else { + var err error + ctx.Variables["include"], err = convertValuesMapToCtyVal(exposedIncludeMap) + if err != nil { + return ctx, err + } } } return ctx, nil } // Return the OS platform -func getPlatform(include *IncludeConfig, terragruntOptions *options.TerragruntOptions) (string, error) { +func getPlatform(trackInclude *TrackInclude, terragruntOptions *options.TerragruntOptions) (string, error) { return runtime.GOOS, nil } // Return the directory where the Terragrunt configuration file lives -func getTerragruntDir(include *IncludeConfig, terragruntOptions *options.TerragruntOptions) (string, error) { +func getTerragruntDir(trackInclude *TrackInclude, terragruntOptions *options.TerragruntOptions) (string, error) { terragruntConfigFileAbsPath, err := filepath.Abs(terragruntOptions.TerragruntConfigPath) if err != nil { return "", errors.WithStackTrace(err) @@ -182,7 +204,7 @@ func getTerragruntDir(include *IncludeConfig, terragruntOptions *options.Terragr // Terragrunt config is being read from another e.g., if /terraform-code/terragrunt.hcl // calls read_terragrunt_config("/foo/bar.hcl"), and within bar.hcl, you call get_original_terragrunt_dir(), you'll // get back /terraform-code. -func getOriginalTerragruntDir(include *IncludeConfig, terragruntOptions *options.TerragruntOptions) (string, error) { +func getOriginalTerragruntDir(trackIncude *TrackInclude, terragruntOptions *options.TerragruntOptions) (string, error) { terragruntConfigFileAbsPath, err := filepath.Abs(terragruntOptions.OriginalTerragruntConfigPath) if err != nil { return "", errors.WithStackTrace(err) @@ -192,8 +214,8 @@ func getOriginalTerragruntDir(include *IncludeConfig, terragruntOptions *options } // Return the parent directory where the Terragrunt configuration file lives -func getParentTerragruntDir(include *IncludeConfig, terragruntOptions *options.TerragruntOptions) (string, error) { - parentPath, err := pathRelativeFromInclude(include, terragruntOptions) +func getParentTerragruntDir(params []string, trackInclude *TrackInclude, terragruntOptions *options.TerragruntOptions) (string, error) { + parentPath, err := pathRelativeFromInclude(params, trackInclude, terragruntOptions) if err != nil { return "", errors.WithStackTrace(err) } @@ -234,7 +256,7 @@ var runCommandCache = NewStringCache() // runCommand is a helper function that runs a command and returns the stdout as the interporation // for each `run_cmd` in locals section, function is called twice // result -func runCommand(args []string, include *IncludeConfig, terragruntOptions *options.TerragruntOptions) (string, error) { +func runCommand(args []string, trackInclude *TrackInclude, terragruntOptions *options.TerragruntOptions) (string, error) { if len(args) == 0 { return "", errors.WithStackTrace(EmptyStringNotAllowed("parameter to the run_cmd function")) } @@ -279,7 +301,7 @@ func runCommand(args []string, include *IncludeConfig, terragruntOptions *option return value, nil } -func getEnvironmentVariable(parameters []string, include *IncludeConfig, terragruntOptions *options.TerragruntOptions) (string, error) { +func getEnvironmentVariable(parameters []string, trackInclude *TrackInclude, terragruntOptions *options.TerragruntOptions) (string, error) { parameterMap, err := parseGetEnvParameters(parameters) if err != nil { @@ -299,7 +321,11 @@ func getEnvironmentVariable(parameters []string, include *IncludeConfig, terragr // Find a parent Terragrunt configuration file in the parent folders above the current Terragrunt configuration file // and return its path -func findInParentFolders(params []string, include *IncludeConfig, terragruntOptions *options.TerragruntOptions) (string, error) { +func findInParentFolders( + params []string, + trackInclude *TrackInclude, + terragruntOptions *options.TerragruntOptions, +) (string, error) { numParams := len(params) var fileToFindParam string @@ -355,12 +381,12 @@ func findInParentFolders(params []string, include *IncludeConfig, terragruntOpti // Return the relative path between the included Terragrunt configuration file and the current Terragrunt configuration // file -func pathRelativeToInclude(include *IncludeConfig, terragruntOptions *options.TerragruntOptions) (string, error) { - if include == nil { +func pathRelativeToInclude(trackInclude *TrackInclude, terragruntOptions *options.TerragruntOptions) (string, error) { + if trackInclude == nil || trackInclude.Original == nil { return ".", nil } - includePath := filepath.Dir(include.Path) + includePath := filepath.Dir(trackInclude.Original.Path) currentPath := filepath.Dir(terragruntOptions.TerragruntConfigPath) if !filepath.IsAbs(includePath) { @@ -371,12 +397,19 @@ func pathRelativeToInclude(include *IncludeConfig, terragruntOptions *options.Te } // Return the relative path from the current Terragrunt configuration to the included Terragrunt configuration file -func pathRelativeFromInclude(include *IncludeConfig, terragruntOptions *options.TerragruntOptions) (string, error) { - if include == nil { +func pathRelativeFromInclude(params []string, trackInclude *TrackInclude, terragruntOptions *options.TerragruntOptions) (string, error) { + if trackInclude == nil { return ".", nil } - includePath := filepath.Dir(include.Path) + included, err := getSelectedIncludeBlock(trackInclude.CurrentMap, params) + if err != nil { + return "", err + } else if included == nil { + return ".", nil + } + + includePath := filepath.Dir(included.Path) currentPath := filepath.Dir(terragruntOptions.TerragruntConfigPath) if !filepath.IsAbs(includePath) { @@ -387,17 +420,17 @@ func pathRelativeFromInclude(include *IncludeConfig, terragruntOptions *options. } // getTerraformCommand returns the current terraform command in execution -func getTerraformCommand(include *IncludeConfig, terragruntOptions *options.TerragruntOptions) (string, error) { +func getTerraformCommand(trackInclude *TrackInclude, terragruntOptions *options.TerragruntOptions) (string, error) { return terragruntOptions.TerraformCommand, nil } // getTerraformCliArgs returns cli args for terraform -func getTerraformCliArgs(include *IncludeConfig, terragruntOptions *options.TerragruntOptions) ([]string, error) { +func getTerraformCliArgs(trackInclude *TrackInclude, terragruntOptions *options.TerragruntOptions) ([]string, error) { return terragruntOptions.TerraformCliArgs, nil } // Return the AWS account id associated to the current set of credentials -func getAWSAccountID(include *IncludeConfig, terragruntOptions *options.TerragruntOptions) (string, error) { +func getAWSAccountID(trackInclude *TrackInclude, terragruntOptions *options.TerragruntOptions) (string, error) { accountID, err := aws_helper.GetAWSAccountID(nil, terragruntOptions) if err == nil { return accountID, nil @@ -406,7 +439,7 @@ func getAWSAccountID(include *IncludeConfig, terragruntOptions *options.Terragru } // Return the ARN of the AWS identity associated with the current set of credentials -func getAWSCallerIdentityARN(include *IncludeConfig, terragruntOptions *options.TerragruntOptions) (string, error) { +func getAWSCallerIdentityARN(trackInclude *TrackInclude, terragruntOptions *options.TerragruntOptions) (string, error) { identityARN, err := aws_helper.GetAWSIdentityArn(nil, terragruntOptions) if err == nil { return identityARN, nil @@ -415,7 +448,7 @@ func getAWSCallerIdentityARN(include *IncludeConfig, terragruntOptions *options. } // Return the UserID of the AWS identity associated with the current set of credentials -func getAWSCallerIdentityUserID(include *IncludeConfig, terragruntOptions *options.TerragruntOptions) (string, error) { +func getAWSCallerIdentityUserID(trackInclude *TrackInclude, terragruntOptions *options.TerragruntOptions) (string, error) { userID, err := aws_helper.GetAWSUserID(nil, terragruntOptions) if err == nil { return userID, nil @@ -573,7 +606,7 @@ func getModulePathFromSourceUrl(sourceUrl string) (string, error) { var sopsCache = make(map[string]string) // decrypts and returns sops encrypted utf-8 yaml or json data as a string -func sopsDecryptFile(params []string, include *IncludeConfig, terragruntOptions *options.TerragruntOptions) (string, error) { +func sopsDecryptFile(params []string, trackInclude *TrackInclude, terragruntOptions *options.TerragruntOptions) (string, error) { numParams := len(params) var sourceFile string @@ -619,10 +652,37 @@ func sopsDecryptFile(params []string, include *IncludeConfig, terragruntOptions } // Return the location of the Terraform files provided via --terragrunt-source -func getTerragruntSourceCliFlag(include *IncludeConfig, terragruntOptions *options.TerragruntOptions) (string, error) { +func getTerragruntSourceCliFlag(trackInclude *TrackInclude, terragruntOptions *options.TerragruntOptions) (string, error) { return terragruntOptions.Source, nil } +// Return the selected include block based on a label passed in as a function param. Note that the assumption is that: +// - If there are no include blocks, no param is required and nil is returned. +// - If there is only one include block, no param is required and that is automatically returned. +// - If there is more than one include block, 1 param is required to use as the label name to lookup the include block +// to use. +func getSelectedIncludeBlock(includeMap map[string]IncludeConfig, params []string) (*IncludeConfig, error) { + if len(includeMap) == 0 { + return nil, nil + } else if len(includeMap) == 1 { + for _, val := range includeMap { + return &val, nil + } + } + + numParams := len(params) + if numParams != 1 { + return nil, errors.WithStackTrace(WrongNumberOfParams{Func: "path_relative_from_include", Expected: "1", Actual: numParams}) + } + + includeName := params[0] + included, hasKey := includeMap[includeName] + if !hasKey { + return nil, errors.WithStackTrace(InvalidIncludeKey{name: includeName}) + } + return &included, nil +} + // Custom error types type WrongNumberOfParams struct { Func string @@ -726,3 +786,11 @@ type InvalidSopsFormat struct { func (err InvalidSopsFormat) Error() string { return fmt.Sprintf("File %s is not a valid format or encoding. Terragrunt will only decrypt yaml or json files in UTF-8 encoding.", err.SourceFilePath) } + +type InvalidIncludeKey struct { + name string +} + +func (err InvalidIncludeKey) Error() string { + return fmt.Sprintf("There is no include block in the current config with the label '%s'", err.name) +} diff --git a/config/config_partial.go b/config/config_partial.go index c269991060..226efab7bb 100644 --- a/config/config_partial.go +++ b/config/config_partial.go @@ -3,6 +3,7 @@ package config import ( "fmt" "path/filepath" + "strings" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclparse" @@ -29,8 +30,8 @@ const ( // terragruntInclude is a struct that can be used to only decode the include block. type terragruntInclude struct { - Include *IncludeConfig `hcl:"include,block"` - Remain hcl.Body `hcl:",remain"` + Include []IncludeConfig `hcl:"include,block"` + Remain hcl.Body `hcl:",remain"` } // terragruntDependencies is a struct that can be used to only decode the dependencies block. @@ -98,21 +99,21 @@ func DecodeBaseBlocks( hclFile *hcl.File, filename string, includeFromChild *IncludeConfig, -) (*cty.Value, *terragruntInclude, TrackInclude, error) { - // Decode just the `include` block, and verify that it's allowed here - terragruntInclude, err := decodeAsTerragruntInclude( +) (*cty.Value, *TrackInclude, error) { + // Decode just the `include` blocks, and verify that it's allowed here + terragruntIncludeList, err := decodeAsTerragruntInclude( hclFile, filename, terragruntOptions, EvalContextExtensions{}, ) if err != nil { - return nil, nil, TrackInclude{}, err + return nil, nil, err } - trackInclude, err := getTrackInclude(terragruntInclude, includeFromChild, terragruntOptions) + trackInclude, err := getTrackInclude(terragruntIncludeList, includeFromChild, terragruntOptions) if err != nil { - return nil, nil, TrackInclude{}, err + return nil, nil, err } // Evaluate all the expressions in the locals block separately and generate the variables list to use in the @@ -125,34 +126,57 @@ func DecodeBaseBlocks( trackInclude, ) if err != nil { - return nil, nil, trackInclude, err + return nil, trackInclude, err } localsAsCty, err := convertValuesMapToCtyVal(locals) if err != nil { - return nil, nil, trackInclude, err + return nil, trackInclude, err } - return &localsAsCty, terragruntInclude, trackInclude, nil + return &localsAsCty, trackInclude, nil } -func getTrackInclude(terragruntInclude *terragruntInclude, includeFromChild *IncludeConfig, terragruntOptions *options.TerragruntOptions) (TrackInclude, error) { - if terragruntInclude.Include != nil && includeFromChild != nil { - return TrackInclude{}, errors.WithStackTrace(TooManyLevelsOfInheritance{ +// getTrackInclude converts the terragrunt include blocks into TrackInclude structs that differentiate between an +// included config in the current parsing context, and an included config that was passed through from a previous +// parsing context. +func getTrackInclude( + terragruntIncludeList []IncludeConfig, + includeFromChild *IncludeConfig, + terragruntOptions *options.TerragruntOptions, +) (*TrackInclude, error) { + includedPaths := []string{} + terragruntIncludeMap := make(map[string]IncludeConfig, len(terragruntIncludeList)) + for _, tgInc := range terragruntIncludeList { + includedPaths = append(includedPaths, tgInc.Path) + terragruntIncludeMap[tgInc.Name] = tgInc + } + + hasInclude := len(terragruntIncludeList) > 0 + if hasInclude && includeFromChild != nil { + // tgInc appears in a parent that is already included, which means a nested include block. This is not + // something we currently support. + err := errors.WithStackTrace(TooManyLevelsOfInheritance{ ConfigPath: terragruntOptions.TerragruntConfigPath, FirstLevelIncludePath: includeFromChild.Path, - SecondLevelIncludePath: terragruntInclude.Include.Path, + SecondLevelIncludePath: strings.Join(includedPaths, ","), }) - } else if terragruntInclude.Include != nil && includeFromChild == nil { - return TrackInclude{ - Current: terragruntInclude.Include, - Original: terragruntInclude.Include, - }, nil - + return nil, err + } else if hasInclude && includeFromChild == nil { + // Current parsing context where there is no included config already loaded. + trackInc := TrackInclude{ + CurrentList: terragruntIncludeList, + CurrentMap: terragruntIncludeMap, + Original: nil, + } + return &trackInc, nil } else { - return TrackInclude{ - Current: terragruntInclude.Include, - Original: includeFromChild, - }, nil + // Parsing context where there is an included config already loaded. + trackInc := TrackInclude{ + CurrentList: terragruntIncludeList, + CurrentMap: terragruntIncludeMap, + Original: includeFromChild, + } + return &trackInc, nil } } @@ -205,7 +229,7 @@ func PartialParseConfigString( } // Decode just the Base blocks. See the function docs for DecodeBaseBlocks for more info on what base blocks are. - localsAsCty, terragruntInclude, trackInclude, err := DecodeBaseBlocks(terragruntOptions, parser, file, filename, includeFromChild) + localsAsCty, trackInclude, err := DecodeBaseBlocks(terragruntOptions, parser, file, filename, includeFromChild) if err != nil { return nil, err } @@ -323,8 +347,8 @@ func PartialParseConfigString( } // If this file includes another, parse and merge the partial blocks. Otherwise just return this config. - if terragruntInclude.Include != nil { - return handleIncludePartial(&output, terragruntInclude.Include, terragruntOptions, decodeList) + if len(trackInclude.CurrentList) > 0 { + return handleIncludePartial(&output, trackInclude, terragruntOptions, decodeList) } return &output, nil } @@ -358,13 +382,13 @@ func decodeAsTerragruntInclude( filename string, terragruntOptions *options.TerragruntOptions, extensions EvalContextExtensions, -) (*terragruntInclude, error) { - terragruntInclude := terragruntInclude{} - err := decodeHcl(file, filename, &terragruntInclude, terragruntOptions, extensions) +) ([]IncludeConfig, error) { + tgInc := terragruntInclude{} + err := decodeHcl(file, filename, &tgInc, terragruntOptions, extensions) if err != nil { return nil, err } - return &terragruntInclude, nil + return tgInc.Include, nil } // Custom error types diff --git a/config/cty_helpers.go b/config/cty_helpers.go index d497fec789..75856c7622 100644 --- a/config/cty_helpers.go +++ b/config/cty_helpers.go @@ -16,7 +16,11 @@ import ( // Create a cty Function that takes as input parameters a slice of strings (var args, so this slice could be of any // length) and returns as output a string. The implementation of the function calls the given toWrap function, passing // it the input parameters string slice as well as the given include and terragruntOptions. -func wrapStringSliceToStringAsFuncImpl(toWrap func(params []string, include *IncludeConfig, terragruntOptions *options.TerragruntOptions) (string, error), include *IncludeConfig, terragruntOptions *options.TerragruntOptions) function.Function { +func wrapStringSliceToStringAsFuncImpl( + toWrap func(params []string, trackInclude *TrackInclude, terragruntOptions *options.TerragruntOptions) (string, error), + trackInclude *TrackInclude, + terragruntOptions *options.TerragruntOptions, +) function.Function { return function.New(&function.Spec{ VarParam: &function.Parameter{Type: cty.String}, Type: function.StaticReturnType(cty.String), @@ -25,7 +29,7 @@ func wrapStringSliceToStringAsFuncImpl(toWrap func(params []string, include *Inc if err != nil { return cty.StringVal(""), err } - out, err := toWrap(params, include, terragruntOptions) + out, err := toWrap(params, trackInclude, terragruntOptions) if err != nil { return cty.StringVal(""), err } @@ -36,11 +40,15 @@ func wrapStringSliceToStringAsFuncImpl(toWrap func(params []string, include *Inc // Create a cty Function that takes no input parameters and returns as output a string. The implementation of the // function calls the given toWrap function, passing it the given include and terragruntOptions. -func wrapVoidToStringAsFuncImpl(toWrap func(include *IncludeConfig, terragruntOptions *options.TerragruntOptions) (string, error), include *IncludeConfig, terragruntOptions *options.TerragruntOptions) function.Function { +func wrapVoidToStringAsFuncImpl( + toWrap func(trackInclude *TrackInclude, terragruntOptions *options.TerragruntOptions) (string, error), + trackInclude *TrackInclude, + terragruntOptions *options.TerragruntOptions, +) function.Function { return function.New(&function.Spec{ Type: function.StaticReturnType(cty.String), Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { - out, err := toWrap(include, terragruntOptions) + out, err := toWrap(trackInclude, terragruntOptions) if err != nil { return cty.StringVal(""), err } @@ -51,11 +59,15 @@ func wrapVoidToStringAsFuncImpl(toWrap func(include *IncludeConfig, terragruntOp // Create a cty Function that takes no input parameters and returns as output a string slice. The implementation of the // function calls the given toWrap function, passing it the given include and terragruntOptions. -func wrapVoidToStringSliceAsFuncImpl(toWrap func(include *IncludeConfig, terragruntOptions *options.TerragruntOptions) ([]string, error), include *IncludeConfig, terragruntOptions *options.TerragruntOptions) function.Function { +func wrapVoidToStringSliceAsFuncImpl( + toWrap func(trackInclude *TrackInclude, terragruntOptions *options.TerragruntOptions) ([]string, error), + trackInclude *TrackInclude, + terragruntOptions *options.TerragruntOptions, +) function.Function { return function.New(&function.Spec{ Type: function.StaticReturnType(cty.List(cty.String)), Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { - outVals, err := toWrap(include, terragruntOptions) + outVals, err := toWrap(trackInclude, terragruntOptions) if err != nil || len(outVals) == 0 { return cty.ListValEmpty(cty.String), err } diff --git a/config/dependency.go b/config/dependency.go index 6fdde5a67f..2d621c8467 100644 --- a/config/dependency.go +++ b/config/dependency.go @@ -119,7 +119,7 @@ func decodeAndRetrieveOutputs( file *hcl.File, filename string, terragruntOptions *options.TerragruntOptions, - includeConfig *IncludeConfig, + trackInclude *TrackInclude, extensions EvalContextExtensions, ) (*cty.Value, error) { decodedDependency := terragruntDependency{} @@ -128,8 +128,8 @@ func decodeAndRetrieveOutputs( } // Merge in included dependencies - if includeConfig != nil { - mergedDecodedDependency, err := handleIncludeForDependency(decodedDependency, *includeConfig, terragruntOptions) + if trackInclude != nil { + mergedDecodedDependency, err := handleIncludeForDependency(decodedDependency, trackInclude, terragruntOptions) if err != nil { return nil, err } diff --git a/config/include.go b/config/include.go index 64f0a472d5..41432c39cc 100644 --- a/config/include.go +++ b/config/include.go @@ -31,82 +31,96 @@ func parseIncludedConfig(includedConfig *IncludeConfig, terragruntOptions *optio // user. func handleInclude( config *TerragruntConfig, - includeConfig *IncludeConfig, + trackInclude *TrackInclude, terragruntOptions *options.TerragruntOptions, dependencyOutputs *cty.Value, ) (*TerragruntConfig, error) { - if includeConfig == nil { + if trackInclude == nil { return nil, fmt.Errorf("You reached an impossible condition. This is most likely a bug in terragrunt. Please open an issue at github.com/gruntwork-io/terragrunt with this error message. Code: HANDLE_INCLUDE_NIL_INCLUDE_CONFIG") } - mergeStrategy, err := includeConfig.GetMergeStrategy() - if err != nil { - return config, err - } - - includedConfig, err := parseIncludedConfig(includeConfig, terragruntOptions, dependencyOutputs) - if err != nil { - return nil, err - } + // We merge in the include blocks in reverse order here. The expectation is that the bottom most elements override + // those in earlier includes, so we need to merge bottom up instead of top down to ensure this. + includeList := trackInclude.CurrentList + baseConfig := config + for i := len(includeList) - 1; i >= 0; i-- { + includeConfig := includeList[i] + mergeStrategy, err := includeConfig.GetMergeStrategy() + if err != nil { + return config, err + } - switch mergeStrategy { - case NoMerge: - terragruntOptions.Logger.Debugf("Included config %s has strategy no merge: not merging config in.", includeConfig.Path) - return config, nil - case ShallowMerge: - terragruntOptions.Logger.Debugf("Included config %s has strategy shallow merge: merging config in (shallow).", includeConfig.Path) - includedConfig.Merge(config, terragruntOptions) - return includedConfig, nil - case DeepMerge: - terragruntOptions.Logger.Debugf("Included config %s has strategy deep merge: merging config in (deep).", includeConfig.Path) - if err := includedConfig.DeepMerge(config, terragruntOptions); err != nil { + parsedIncludeConfig, err := parseIncludedConfig(&includeConfig, terragruntOptions, dependencyOutputs) + if err != nil { return nil, err } - return includedConfig, nil - } - return nil, fmt.Errorf("You reached an impossible condition. This is most likely a bug in terragrunt. Please open an issue at github.com/gruntwork-io/terragrunt with this error message. Code: UNKNOWN_MERGE_STRATEGY_%s", mergeStrategy) + switch mergeStrategy { + case NoMerge: + terragruntOptions.Logger.Debugf("Included config %s has strategy no merge: not merging config in.", includeConfig.Path) + case ShallowMerge: + terragruntOptions.Logger.Debugf("Included config %s has strategy shallow merge: merging config in (shallow).", includeConfig.Path) + parsedIncludeConfig.Merge(baseConfig, terragruntOptions) + baseConfig = parsedIncludeConfig + case DeepMerge: + terragruntOptions.Logger.Debugf("Included config %s has strategy deep merge: merging config in (deep).", includeConfig.Path) + if err := parsedIncludeConfig.DeepMerge(baseConfig, terragruntOptions); err != nil { + return nil, err + } + baseConfig = parsedIncludeConfig + default: + return nil, fmt.Errorf("You reached an impossible condition. This is most likely a bug in terragrunt. Please open an issue at github.com/gruntwork-io/terragrunt with this error message. Code: UNKNOWN_MERGE_STRATEGY_%s", mergeStrategy) + } + } + return baseConfig, nil } // handleIncludePartial merges the a partially parsed include config into the child config according to the strategy // specified by the user. func handleIncludePartial( config *TerragruntConfig, - includeConfig *IncludeConfig, + trackInclude *TrackInclude, terragruntOptions *options.TerragruntOptions, decodeList []PartialDecodeSectionType, ) (*TerragruntConfig, error) { - if includeConfig == nil { + if trackInclude == nil { return nil, fmt.Errorf("You reached an impossible condition. This is most likely a bug in terragrunt. Please open an issue at github.com/gruntwork-io/terragrunt with this error message. Code: HANDLE_INCLUDE_PARTIAL_NIL_INCLUDE_CONFIG") } - mergeStrategy, err := includeConfig.GetMergeStrategy() - if err != nil { - return config, err - } - - includedConfig, err := partialParseIncludedConfig(includeConfig, terragruntOptions, decodeList) - if err != nil { - return nil, err - } + // We merge in the include blocks in reverse order here. The expectation is that the bottom most elements override + // those in earlier includes, so we need to merge bottom up instead of top down to ensure this. + includeList := trackInclude.CurrentList + baseConfig := config + for i := len(includeList) - 1; i >= 0; i-- { + includeConfig := includeList[i] + mergeStrategy, err := includeConfig.GetMergeStrategy() + if err != nil { + return nil, err + } - switch mergeStrategy { - case NoMerge: - terragruntOptions.Logger.Debugf("[Partial] Included config %s has strategy no merge: not merging config in.", includeConfig.Path) - return config, nil - case ShallowMerge: - terragruntOptions.Logger.Debugf("[Partial] Included config %s has strategy shallow merge: merging config in (shallow).", includeConfig.Path) - includedConfig.Merge(config, terragruntOptions) - return includedConfig, nil - case DeepMerge: - terragruntOptions.Logger.Debugf("[Partial] Included config %s has strategy deep merge: merging config in (deep).", includeConfig.Path) - if err := includedConfig.DeepMerge(config, terragruntOptions); err != nil { + parsedIncludeConfig, err := partialParseIncludedConfig(&includeConfig, terragruntOptions, decodeList) + if err != nil { return nil, err } - return includedConfig, nil - } - return nil, fmt.Errorf("You reached an impossible condition. This is most likely a bug in terragrunt. Please open an issue at github.com/gruntwork-io/terragrunt with this error message. Code: UNKNOWN_MERGE_STRATEGY_%s_PARTIAL", mergeStrategy) + switch mergeStrategy { + case NoMerge: + terragruntOptions.Logger.Debugf("[Partial] Included config %s has strategy no merge: not merging config in.", includeConfig.Path) + case ShallowMerge: + terragruntOptions.Logger.Debugf("[Partial] Included config %s has strategy shallow merge: merging config in (shallow).", includeConfig.Path) + parsedIncludeConfig.Merge(baseConfig, terragruntOptions) + baseConfig = parsedIncludeConfig + case DeepMerge: + terragruntOptions.Logger.Debugf("[Partial] Included config %s has strategy deep merge: merging config in (deep).", includeConfig.Path) + if err := parsedIncludeConfig.DeepMerge(baseConfig, terragruntOptions); err != nil { + return nil, err + } + baseConfig = parsedIncludeConfig + default: + return nil, fmt.Errorf("You reached an impossible condition. This is most likely a bug in terragrunt. Please open an issue at github.com/gruntwork-io/terragrunt with this error message. Code: UNKNOWN_MERGE_STRATEGY_%s_PARTIAL", mergeStrategy) + } + } + return baseConfig, nil } // handleIncludeForDependency is a partial merge of the included config to handle dependencies. This only merges the @@ -115,34 +129,47 @@ func handleIncludePartial( // child. func handleIncludeForDependency( childDecodedDependency terragruntDependency, - includeConfig IncludeConfig, + trackInclude *TrackInclude, terragruntOptions *options.TerragruntOptions, ) (*terragruntDependency, error) { - mergeStrategy, err := includeConfig.GetMergeStrategy() - if err != nil { - return nil, err - } + if trackInclude == nil { + return nil, fmt.Errorf("You reached an impossible condition. This is most likely a bug in terragrunt. Please open an issue at github.com/gruntwork-io/terragrunt with this error message. Code: HANDLE_INCLUDE_DEPENDENCY_NIL_INCLUDE_CONFIG") + } + // We merge in the include blocks in reverse order here. The expectation is that the bottom most elements override + // those in earlier includes, so we need to merge bottom up instead of top down to ensure this. + includeList := trackInclude.CurrentList + baseDependencyBlock := childDecodedDependency.Dependencies + for i := len(includeList) - 1; i >= 0; i-- { + includeConfig := includeList[i] + mergeStrategy, err := includeConfig.GetMergeStrategy() + if err != nil { + return nil, err + } - includedPartialParse, err := partialParseIncludedConfig(&includeConfig, terragruntOptions, []PartialDecodeSectionType{DependencyBlock}) - if err != nil { - return nil, err - } + includedPartialParse, err := partialParseIncludedConfig(&includeConfig, terragruntOptions, []PartialDecodeSectionType{DependencyBlock}) + if err != nil { + return nil, err + } - switch mergeStrategy { - case NoMerge: - terragruntOptions.Logger.Debugf("Included config %s has strategy no merge: not merging config in for dependency.", includeConfig.Path) - return &childDecodedDependency, nil - case ShallowMerge: - terragruntOptions.Logger.Debugf("Included config %s has strategy shallow merge: merging config in (shallow) for dependency.", includeConfig.Path) - mergedDependencyBlock := mergeDependencyBlocks(includedPartialParse.TerragruntDependencies, childDecodedDependency.Dependencies) - return &terragruntDependency{Dependencies: mergedDependencyBlock}, nil - case DeepMerge: - terragruntOptions.Logger.Debugf("Included config %s has strategy deep merge: merging config in (deep) for dependency.", includeConfig.Path) - mergedDependencyBlock, err := deepMergeDependencyBlocks(includedPartialParse.TerragruntDependencies, childDecodedDependency.Dependencies) - return &terragruntDependency{Dependencies: mergedDependencyBlock}, err + switch mergeStrategy { + case NoMerge: + terragruntOptions.Logger.Debugf("Included config %s has strategy no merge: not merging config in for dependency.", includeConfig.Path) + case ShallowMerge: + terragruntOptions.Logger.Debugf("Included config %s has strategy shallow merge: merging config in (shallow) for dependency.", includeConfig.Path) + mergedDependencyBlock := mergeDependencyBlocks(includedPartialParse.TerragruntDependencies, baseDependencyBlock) + baseDependencyBlock = mergedDependencyBlock + case DeepMerge: + terragruntOptions.Logger.Debugf("Included config %s has strategy deep merge: merging config in (deep) for dependency.", includeConfig.Path) + mergedDependencyBlock, err := deepMergeDependencyBlocks(includedPartialParse.TerragruntDependencies, baseDependencyBlock) + if err != nil { + return nil, err + } + baseDependencyBlock = mergedDependencyBlock + default: + return nil, fmt.Errorf("You reached an impossible condition. This is most likely a bug in terragrunt. Please open an issue at github.com/gruntwork-io/terragrunt with this error message. Code: UNKNOWN_MERGE_STRATEGY_%s_DEPENDENCY", mergeStrategy) + } } - - return nil, fmt.Errorf("You reached an impossible condition. This is most likely a bug in terragrunt. Please open an issue at github.com/gruntwork-io/terragrunt with this error message. Code: UNKNOWN_MERGE_STRATEGY_%s_DEPENDENCY", mergeStrategy) + return &terragruntDependency{Dependencies: baseDependencyBlock}, nil } // Merge performs a shallow merge of the given sourceConfig into the targetConfig. sourceConfig will override common diff --git a/config/locals.go b/config/locals.go index 9eb11e0541..a26d7050cd 100644 --- a/config/locals.go +++ b/config/locals.go @@ -45,7 +45,7 @@ func evaluateLocalsBlock( parser *hclparse.Parser, hclFile *hcl.File, filename string, - trackInclude TrackInclude, + trackInclude *TrackInclude, ) (map[string]cty.Value, error) { diagsWriter := util.GetDiagnosticsWriter(parser) @@ -117,7 +117,7 @@ func attemptEvaluateLocals( filename string, locals []*Local, evaluatedLocals map[string]cty.Value, - trackInclude TrackInclude, + trackInclude *TrackInclude, diagsWriter hcl.DiagnosticWriter, ) (unevaluatedLocals []*Local, newEvaluatedLocals map[string]cty.Value, evaluated bool, err error) { // The HCL2 parser and especially cty conversions will panic in many types of errors, so we have to recover from From 9317b5dffaedb2a7c57aea4df43277e0c07cc74b Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Thu, 9 Sep 2021 13:50:23 -0500 Subject: [PATCH 04/26] Fix panic error --- config/config_helpers.go | 2 +- config/hcl_parser.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config_helpers.go b/config/config_helpers.go index 00eea6bff2..14e571598e 100644 --- a/config/config_helpers.go +++ b/config/config_helpers.go @@ -151,7 +151,7 @@ func CreateTerragruntEvalContext( if extensions.DecodedDependencies != nil { ctx.Variables["dependency"] = *extensions.DecodedDependencies } - if len(extensions.TrackInclude.CurrentMap) > 0 { + if extensions.TrackInclude != nil && len(extensions.TrackInclude.CurrentList) > 0 { // For each include block, check if we want to expose the included config, and if so, add under the include // variable. exposedIncludeMap := map[string]cty.Value{} diff --git a/config/hcl_parser.go b/config/hcl_parser.go index 471a068b66..7b98196609 100644 --- a/config/hcl_parser.go +++ b/config/hcl_parser.go @@ -75,7 +75,7 @@ func ParseAndDecodeVarFile(hclContents string, filename string, out interface{}) // those panics here and convert them to normal errors defer func() { if recovered := recover(); recovered != nil { - err = PanicWhileParsingConfig{RecoveredValue: recovered, ConfigFile: filename} + err = errors.WithStackTrace(PanicWhileParsingConfig{RecoveredValue: recovered, ConfigFile: filename}) } }() From 5ba9a67ca23eb4437cb7a925c6e9c9b8ee47ba97 Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Thu, 9 Sep 2021 14:42:50 -0500 Subject: [PATCH 05/26] Introduce new import block, instead of include for backward compatibility --- config/config.go | 26 +++++++--- config/config_helpers.go | 8 ++-- config/config_helpers_test.go | 90 ++++++++++++++++++++++++----------- config/config_partial.go | 85 ++++++++++----------------------- config/include.go | 67 +++++++++++++++++++++----- config/locals_test.go | 8 ++-- 6 files changed, 170 insertions(+), 114 deletions(-) diff --git a/config/config.go b/config/config.go index 572ba01a25..5879e54df6 100644 --- a/config/config.go +++ b/config/config.go @@ -65,6 +65,7 @@ type terragruntConfigFile struct { TerragruntVersionConstraint *string `hcl:"terragrunt_version_constraint,attr"` Inputs *cty.Value `hcl:"inputs,attr"` Include *IncludeConfig `hcl:"include,block"` + Import *ImportConfig `hcl:"import,block"` // We allow users to configure remote state (backend) via blocks: // @@ -187,26 +188,37 @@ type terragruntGenerateBlock struct { } // IncludeConfig represents the configuration settings for a parent Terragrunt configuration file that you can -// "include" in a child Terragrunt configuration file +// "include" in a child Terragrunt configuration file. +// This exists for backward compatibility reasons, and is equivalent to an import block with no label ("" for the +// label). You can only have one include block per terragrunt config. type IncludeConfig struct { + Path string `hcl:"path,attr"` + Expose *bool `hcl:"expose,attr"` + MergeStrategy *string `hcl:"merge_strategy,attr"` +} + +// ImportConfig represents the configuration settings for a parent Terragrunt configuration file that you can +// import into a child Terragrunt configuration file. You can have more than one import config. Note that this replaces +// "include". +type ImportConfig struct { Name string `hcl:",label"` Path string `hcl:"path,attr"` Expose *bool `hcl:"expose,attr"` MergeStrategy *string `hcl:"merge_strategy,attr"` } -func (cfg *IncludeConfig) String() string { - return fmt.Sprintf("IncludeConfig{Path = %s, Expose = %v, MergeStrategy = %v}", cfg.Path, cfg.Expose, cfg.MergeStrategy) +func (cfg *ImportConfig) String() string { + return fmt.Sprintf("ImportConfig{Path = %s, Expose = %v, MergeStrategy = %v}", cfg.Path, cfg.Expose, cfg.MergeStrategy) } -func (cfg *IncludeConfig) GetExpose() bool { +func (cfg *ImportConfig) GetExpose() bool { if cfg == nil || cfg.Expose == nil { return false } return *cfg.Expose } -func (cfg *IncludeConfig) GetMergeStrategy() (MergeStrategyType, error) { +func (cfg *ImportConfig) GetMergeStrategy() (MergeStrategyType, error) { if cfg.MergeStrategy == nil { return ShallowMerge, nil } @@ -533,7 +545,7 @@ func ReadTerragruntConfig(terragruntOptions *options.TerragruntOptions) (*Terrag // Parse the Terragrunt config file at the given path. If the include parameter is not nil, then treat this as a config // included in some other config file when resolving relative paths. -func ParseConfigFile(filename string, terragruntOptions *options.TerragruntOptions, include *IncludeConfig, dependencyOutputs *cty.Value) (*TerragruntConfig, error) { +func ParseConfigFile(filename string, terragruntOptions *options.TerragruntOptions, include *ImportConfig, dependencyOutputs *cty.Value) (*TerragruntConfig, error) { configString, err := util.ReadFileAsString(filename) if err != nil { return nil, err @@ -576,7 +588,7 @@ func ParseConfigFile(filename string, terragruntOptions *options.TerragruntOptio func ParseConfigString( configString string, terragruntOptions *options.TerragruntOptions, - includeFromChild *IncludeConfig, + includeFromChild *ImportConfig, filename string, dependencyOutputs *cty.Value, ) (*TerragruntConfig, error) { diff --git a/config/config_helpers.go b/config/config_helpers.go index 14e571598e..dac8423ab2 100644 --- a/config/config_helpers.go +++ b/config/config_helpers.go @@ -73,14 +73,14 @@ type TrackInclude struct { // CurrentList is used to track the list of configs that should be imported and merged before the final // TerragruntConfig is returned. This preserves the order of the blocks as they appear in the config, so that we can // merge the included config in the right order. - CurrentList []IncludeConfig + CurrentList []ImportConfig // CurrentMap is the map version of CurrentList that maps the block labels to the included config. - CurrentMap map[string]IncludeConfig + CurrentMap map[string]ImportConfig // Original is used to track the original included config, and is used for resolving the include related // functions. - Original *IncludeConfig + Original *ImportConfig } // EvalContextExtensions provides various extensions to the evaluation context to enhance the parsing capabilities. @@ -661,7 +661,7 @@ func getTerragruntSourceCliFlag(trackInclude *TrackInclude, terragruntOptions *o // - If there is only one include block, no param is required and that is automatically returned. // - If there is more than one include block, 1 param is required to use as the label name to lookup the include block // to use. -func getSelectedIncludeBlock(includeMap map[string]IncludeConfig, params []string) (*IncludeConfig, error) { +func getSelectedIncludeBlock(includeMap map[string]ImportConfig, params []string) (*ImportConfig, error) { if len(includeMap) == 0 { return nil, nil } else if len(includeMap) == 1 { diff --git a/config/config_helpers_test.go b/config/config_helpers_test.go index 9df37680a0..c48e5dc27e 100644 --- a/config/config_helpers_test.go +++ b/config/config_helpers_test.go @@ -18,7 +18,7 @@ func TestPathRelativeToInclude(t *testing.T) { t.Parallel() testCases := []struct { - include *IncludeConfig + include *ImportConfig terragruntOptions *options.TerragruntOptions expectedPath string }{ @@ -28,39 +28,49 @@ func TestPathRelativeToInclude(t *testing.T) { ".", }, { - &IncludeConfig{Path: "../" + DefaultTerragruntConfigPath}, + &ImportConfig{Path: "../" + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, helpers.RootFolder+"child/"+DefaultTerragruntConfigPath), "child", }, { - &IncludeConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}, + &ImportConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, helpers.RootFolder+"child/"+DefaultTerragruntConfigPath), "child", }, { - &IncludeConfig{Path: "../../../" + DefaultTerragruntConfigPath}, + &ImportConfig{Path: "../../../" + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/sub-sub-child/"+DefaultTerragruntConfigPath), "child/sub-child/sub-sub-child", }, { - &IncludeConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}, + &ImportConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/sub-sub-child/"+DefaultTerragruntConfigPath), "child/sub-child/sub-sub-child", }, { - &IncludeConfig{Path: "../../other-child/" + DefaultTerragruntConfigPath}, + &ImportConfig{Path: "../../other-child/" + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/"+DefaultTerragruntConfigPath), "../child/sub-child", }, { - &IncludeConfig{Path: "../../" + DefaultTerragruntConfigPath}, + &ImportConfig{Path: "../../" + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, "../child/sub-child/"+DefaultTerragruntConfigPath), "child/sub-child", }, } for _, testCase := range testCases { - actualPath, actualErr := pathRelativeToInclude(testCase.include, testCase.terragruntOptions) + var trackInclude *TrackInclude = nil + if testCase.include != nil { + trackInclude = &TrackInclude{ + CurrentList: []ImportConfig{*testCase.include}, + CurrentMap: map[string]ImportConfig{ + "": *testCase.include, + }, + Original: testCase.include, + } + } + actualPath, actualErr := pathRelativeToInclude(trackInclude, testCase.terragruntOptions) assert.Nil(t, actualErr, "For include %v and options %v, unexpected error: %v", testCase.include, testCase.terragruntOptions, actualErr) assert.Equal(t, testCase.expectedPath, actualPath, "For include %v and options %v", testCase.include, testCase.terragruntOptions) } @@ -70,7 +80,7 @@ func TestPathRelativeFromInclude(t *testing.T) { t.Parallel() testCases := []struct { - include *IncludeConfig + include *ImportConfig terragruntOptions *options.TerragruntOptions expectedPath string }{ @@ -80,39 +90,50 @@ func TestPathRelativeFromInclude(t *testing.T) { ".", }, { - &IncludeConfig{Path: "../" + DefaultTerragruntConfigPath}, + &ImportConfig{Path: "../" + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, helpers.RootFolder+"child/"+DefaultTerragruntConfigPath), "..", }, { - &IncludeConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}, + &ImportConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, helpers.RootFolder+"child/"+DefaultTerragruntConfigPath), "..", }, { - &IncludeConfig{Path: "../../../" + DefaultTerragruntConfigPath}, + &ImportConfig{Path: "../../../" + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/sub-sub-child/"+DefaultTerragruntConfigPath), "../../..", }, { - &IncludeConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}, + &ImportConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/sub-sub-child/"+DefaultTerragruntConfigPath), "../../..", }, { - &IncludeConfig{Path: "../../other-child/" + DefaultTerragruntConfigPath}, + &ImportConfig{Path: "../../other-child/" + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/"+DefaultTerragruntConfigPath), "../../other-child", }, { - &IncludeConfig{Path: "../../" + DefaultTerragruntConfigPath}, + &ImportConfig{Path: "../../" + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, "../child/sub-child/"+DefaultTerragruntConfigPath), "../..", }, } for _, testCase := range testCases { - actualPath, actualErr := pathRelativeFromInclude(testCase.include, testCase.terragruntOptions) + var trackInclude *TrackInclude = nil + if testCase.include != nil { + trackInclude = &TrackInclude{ + CurrentList: []ImportConfig{*testCase.include}, + CurrentMap: map[string]ImportConfig{ + "": *testCase.include, + }, + Original: testCase.include, + } + } + // TODO: update to test for multiple include blocks + actualPath, actualErr := pathRelativeFromInclude([]string{}, trackInclude, testCase.terragruntOptions) assert.Nil(t, actualErr, "For include %v and options %v, unexpected error: %v", testCase.include, testCase.terragruntOptions, actualErr) assert.Equal(t, testCase.expectedPath, actualPath, "For include %v and options %v", testCase.include, testCase.terragruntOptions) } @@ -271,7 +292,7 @@ func TestResolveTerragruntInterpolation(t *testing.T) { testCases := []struct { str string - include *IncludeConfig + include *ImportConfig terragruntOptions *options.TerragruntOptions expectedOut string expectedErr string @@ -285,7 +306,7 @@ func TestResolveTerragruntInterpolation(t *testing.T) { }, { "terraform { source = path_relative_to_include() }", - &IncludeConfig{Path: "../" + DefaultTerragruntConfigPath}, + &ImportConfig{Path: "../" + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, "/root/child/"+DefaultTerragruntConfigPath), "child", "", @@ -338,7 +359,7 @@ func TestResolveEnvInterpolationConfigString(t *testing.T) { testCases := []struct { str string - include *IncludeConfig + include *ImportConfig terragruntOptions *options.TerragruntOptions expectedOut string expectedErr string @@ -430,7 +451,7 @@ func TestResolveCommandsInterpolationConfigString(t *testing.T) { testCases := []struct { str string - include *IncludeConfig + include *ImportConfig terragruntOptions *options.TerragruntOptions expectedFooInput []string }{ @@ -492,7 +513,7 @@ func TestResolveCliArgsInterpolationConfigString(t *testing.T) { } testCase := struct { str string - include *IncludeConfig + include *ImportConfig terragruntOptions *options.TerragruntOptions expectedFooInput []string }{ @@ -588,7 +609,7 @@ func TestGetParentTerragruntDir(t *testing.T) { parentDir := filepath.ToSlash(filepath.Dir(currentDir)) testCases := []struct { - include *IncludeConfig + include *ImportConfig terragruntOptions *options.TerragruntOptions expectedPath string }{ @@ -598,39 +619,50 @@ func TestGetParentTerragruntDir(t *testing.T) { helpers.RootFolder + "child", }, { - &IncludeConfig{Path: "../" + DefaultTerragruntConfigPath}, + &ImportConfig{Path: "../" + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, helpers.RootFolder+"child/"+DefaultTerragruntConfigPath), helpers.RootFolder, }, { - &IncludeConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}, + &ImportConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, helpers.RootFolder+"child/"+DefaultTerragruntConfigPath), helpers.RootFolder, }, { - &IncludeConfig{Path: "../../../" + DefaultTerragruntConfigPath}, + &ImportConfig{Path: "../../../" + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/sub-sub-child/"+DefaultTerragruntConfigPath), helpers.RootFolder, }, { - &IncludeConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}, + &ImportConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/sub-sub-child/"+DefaultTerragruntConfigPath), helpers.RootFolder, }, { - &IncludeConfig{Path: "../../other-child/" + DefaultTerragruntConfigPath}, + &ImportConfig{Path: "../../other-child/" + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/"+DefaultTerragruntConfigPath), fmt.Sprintf("%s/other-child", filepath.VolumeName(parentDir)), }, { - &IncludeConfig{Path: "../../" + DefaultTerragruntConfigPath}, + &ImportConfig{Path: "../../" + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, "../child/sub-child/"+DefaultTerragruntConfigPath), parentDir, }, } for _, testCase := range testCases { - actualPath, actualErr := getParentTerragruntDir(testCase.include, testCase.terragruntOptions) + var trackInclude *TrackInclude = nil + if testCase.include != nil { + trackInclude = &TrackInclude{ + CurrentList: []ImportConfig{*testCase.include}, + CurrentMap: map[string]ImportConfig{ + "": *testCase.include, + }, + Original: testCase.include, + } + } + // TODO: update to test for multiple include blocks + actualPath, actualErr := getParentTerragruntDir([]string{}, trackInclude, testCase.terragruntOptions) assert.Nil(t, actualErr, "For include %v and options %v, unexpected error: %v", testCase.include, testCase.terragruntOptions, actualErr) assert.Equal(t, testCase.expectedPath, actualPath, "For include %v and options %v", testCase.include, testCase.terragruntOptions) } diff --git a/config/config_partial.go b/config/config_partial.go index 226efab7bb..29b1b8766b 100644 --- a/config/config_partial.go +++ b/config/config_partial.go @@ -3,7 +3,6 @@ package config import ( "fmt" "path/filepath" - "strings" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclparse" @@ -30,8 +29,9 @@ const ( // terragruntInclude is a struct that can be used to only decode the include block. type terragruntInclude struct { - Include []IncludeConfig `hcl:"include,block"` - Remain hcl.Body `hcl:",remain"` + Include *IncludeConfig `hcl:"include,block"` + Import []ImportConfig `hcl:"import,block"` + Remain hcl.Body `hcl:",remain"` } // terragruntDependencies is a struct that can be used to only decode the dependencies block. @@ -98,9 +98,9 @@ func DecodeBaseBlocks( parser *hclparse.Parser, hclFile *hcl.File, filename string, - includeFromChild *IncludeConfig, + includeFromChild *ImportConfig, ) (*cty.Value, *TrackInclude, error) { - // Decode just the `include` blocks, and verify that it's allowed here + // Decode just the `include` and `import` blocks, and verify that it's allowed here terragruntIncludeList, err := decodeAsTerragruntInclude( hclFile, filename, @@ -136,54 +136,10 @@ func DecodeBaseBlocks( return &localsAsCty, trackInclude, nil } -// getTrackInclude converts the terragrunt include blocks into TrackInclude structs that differentiate between an -// included config in the current parsing context, and an included config that was passed through from a previous -// parsing context. -func getTrackInclude( - terragruntIncludeList []IncludeConfig, - includeFromChild *IncludeConfig, - terragruntOptions *options.TerragruntOptions, -) (*TrackInclude, error) { - includedPaths := []string{} - terragruntIncludeMap := make(map[string]IncludeConfig, len(terragruntIncludeList)) - for _, tgInc := range terragruntIncludeList { - includedPaths = append(includedPaths, tgInc.Path) - terragruntIncludeMap[tgInc.Name] = tgInc - } - - hasInclude := len(terragruntIncludeList) > 0 - if hasInclude && includeFromChild != nil { - // tgInc appears in a parent that is already included, which means a nested include block. This is not - // something we currently support. - err := errors.WithStackTrace(TooManyLevelsOfInheritance{ - ConfigPath: terragruntOptions.TerragruntConfigPath, - FirstLevelIncludePath: includeFromChild.Path, - SecondLevelIncludePath: strings.Join(includedPaths, ","), - }) - return nil, err - } else if hasInclude && includeFromChild == nil { - // Current parsing context where there is no included config already loaded. - trackInc := TrackInclude{ - CurrentList: terragruntIncludeList, - CurrentMap: terragruntIncludeMap, - Original: nil, - } - return &trackInc, nil - } else { - // Parsing context where there is an included config already loaded. - trackInc := TrackInclude{ - CurrentList: terragruntIncludeList, - CurrentMap: terragruntIncludeMap, - Original: includeFromChild, - } - return &trackInc, nil - } -} - func PartialParseConfigFile( filename string, terragruntOptions *options.TerragruntOptions, - include *IncludeConfig, + include *ImportConfig, decodeList []PartialDecodeSectionType, ) (*TerragruntConfig, error) { configString, err := util.ReadFileAsString(filename) @@ -217,7 +173,7 @@ func PartialParseConfigFile( func PartialParseConfigString( configString string, terragruntOptions *options.TerragruntOptions, - includeFromChild *IncludeConfig, + includeFromChild *ImportConfig, filename string, decodeList []PartialDecodeSectionType, ) (*TerragruntConfig, error) { @@ -353,7 +309,7 @@ func PartialParseConfigString( return &output, nil } -func partialParseIncludedConfig(includedConfig *IncludeConfig, terragruntOptions *options.TerragruntOptions, decodeList []PartialDecodeSectionType) (*TerragruntConfig, error) { +func partialParseIncludedConfig(includedConfig *ImportConfig, terragruntOptions *options.TerragruntOptions, decodeList []PartialDecodeSectionType) (*TerragruntConfig, error) { if includedConfig.Path == "" { return nil, errors.WithStackTrace(IncludedConfigMissingPath(terragruntOptions.TerragruntConfigPath)) } @@ -372,23 +328,34 @@ func partialParseIncludedConfig(includedConfig *IncludeConfig, terragruntOptions ) } -// This decodes only the `include` block of a terragrunt config, so its value can be used while decoding the rest of the -// config. -// For consistency, `include` in the call to `decodeHcl` is always assumed to be nil. -// Either it really is nil (parsing the child config), or it shouldn't be used anyway (the parent config shouldn't have -// an include block) +// This decodes only the `include` and `import` blocks of a terragrunt config, so its value can be used while decoding +// the rest of the config. +// For consistency, `include` in the call to `decodeHcl` is always assumed to be nil. Either it really is nil (parsing +// the child config), or it shouldn't be used anyway (the parent config shouldn't have an include block). func decodeAsTerragruntInclude( file *hcl.File, filename string, terragruntOptions *options.TerragruntOptions, extensions EvalContextExtensions, -) ([]IncludeConfig, error) { +) ([]ImportConfig, error) { tgInc := terragruntInclude{} err := decodeHcl(file, filename, &tgInc, terragruntOptions, extensions) if err != nil { return nil, err } - return tgInc.Include, nil + + // Convert the legacy include block into an ImportConfig with no label ("" for label). include blocks are always the + // top most import block. + if tgInc.Include != nil { + includeAsImport := ImportConfig{ + Name: "", + Path: tgInc.Include.Path, + Expose: tgInc.Include.Expose, + MergeStrategy: tgInc.Include.MergeStrategy, + } + tgInc.Import = append([]ImportConfig{includeAsImport}, tgInc.Import...) + } + return tgInc.Import, nil } // Custom error types diff --git a/config/include.go b/config/include.go index 41432c39cc..204f908f77 100644 --- a/config/include.go +++ b/config/include.go @@ -3,6 +3,7 @@ package config import ( "fmt" "path/filepath" + "strings" "github.com/imdario/mergo" "github.com/zclconf/go-cty/cty" @@ -13,7 +14,7 @@ import ( ) // Parse the config of the given include, if one is specified -func parseIncludedConfig(includedConfig *IncludeConfig, terragruntOptions *options.TerragruntOptions, dependencyOutputs *cty.Value) (*TerragruntConfig, error) { +func parseIncludedConfig(includedConfig *ImportConfig, terragruntOptions *options.TerragruntOptions, dependencyOutputs *cty.Value) (*TerragruntConfig, error) { if includedConfig.Path == "" { return nil, errors.WithStackTrace(IncludedConfigMissingPath(terragruntOptions.TerragruntConfigPath)) } @@ -50,7 +51,7 @@ func handleInclude( return config, err } - parsedIncludeConfig, err := parseIncludedConfig(&includeConfig, terragruntOptions, dependencyOutputs) + parsedImportConfig, err := parseIncludedConfig(&includeConfig, terragruntOptions, dependencyOutputs) if err != nil { return nil, err } @@ -60,14 +61,14 @@ func handleInclude( terragruntOptions.Logger.Debugf("Included config %s has strategy no merge: not merging config in.", includeConfig.Path) case ShallowMerge: terragruntOptions.Logger.Debugf("Included config %s has strategy shallow merge: merging config in (shallow).", includeConfig.Path) - parsedIncludeConfig.Merge(baseConfig, terragruntOptions) - baseConfig = parsedIncludeConfig + parsedImportConfig.Merge(baseConfig, terragruntOptions) + baseConfig = parsedImportConfig case DeepMerge: terragruntOptions.Logger.Debugf("Included config %s has strategy deep merge: merging config in (deep).", includeConfig.Path) - if err := parsedIncludeConfig.DeepMerge(baseConfig, terragruntOptions); err != nil { + if err := parsedImportConfig.DeepMerge(baseConfig, terragruntOptions); err != nil { return nil, err } - baseConfig = parsedIncludeConfig + baseConfig = parsedImportConfig default: return nil, fmt.Errorf("You reached an impossible condition. This is most likely a bug in terragrunt. Please open an issue at github.com/gruntwork-io/terragrunt with this error message. Code: UNKNOWN_MERGE_STRATEGY_%s", mergeStrategy) } @@ -98,7 +99,7 @@ func handleIncludePartial( return nil, err } - parsedIncludeConfig, err := partialParseIncludedConfig(&includeConfig, terragruntOptions, decodeList) + parsedImportConfig, err := partialParseIncludedConfig(&includeConfig, terragruntOptions, decodeList) if err != nil { return nil, err } @@ -108,14 +109,14 @@ func handleIncludePartial( terragruntOptions.Logger.Debugf("[Partial] Included config %s has strategy no merge: not merging config in.", includeConfig.Path) case ShallowMerge: terragruntOptions.Logger.Debugf("[Partial] Included config %s has strategy shallow merge: merging config in (shallow).", includeConfig.Path) - parsedIncludeConfig.Merge(baseConfig, terragruntOptions) - baseConfig = parsedIncludeConfig + parsedImportConfig.Merge(baseConfig, terragruntOptions) + baseConfig = parsedImportConfig case DeepMerge: terragruntOptions.Logger.Debugf("[Partial] Included config %s has strategy deep merge: merging config in (deep).", includeConfig.Path) - if err := parsedIncludeConfig.DeepMerge(baseConfig, terragruntOptions); err != nil { + if err := parsedImportConfig.DeepMerge(baseConfig, terragruntOptions); err != nil { return nil, err } - baseConfig = parsedIncludeConfig + baseConfig = parsedImportConfig default: return nil, fmt.Errorf("You reached an impossible condition. This is most likely a bug in terragrunt. Please open an issue at github.com/gruntwork-io/terragrunt with this error message. Code: UNKNOWN_MERGE_STRATEGY_%s_PARTIAL", mergeStrategy) } @@ -497,3 +498,47 @@ func mergeHooks(terragruntOptions *options.TerragruntOptions, childHooks []Hook, } *parentHooks = result } + +// getTrackInclude converts the terragrunt include blocks into TrackInclude structs that differentiate between an +// included config in the current parsing context, and an included config that was passed through from a previous +// parsing context. +func getTrackInclude( + terragruntIncludeList []ImportConfig, + includeFromChild *ImportConfig, + terragruntOptions *options.TerragruntOptions, +) (*TrackInclude, error) { + includedPaths := []string{} + terragruntIncludeMap := make(map[string]ImportConfig, len(terragruntIncludeList)) + for _, tgInc := range terragruntIncludeList { + includedPaths = append(includedPaths, tgInc.Path) + terragruntIncludeMap[tgInc.Name] = tgInc + } + + hasInclude := len(terragruntIncludeList) > 0 + if hasInclude && includeFromChild != nil { + // tgInc appears in a parent that is already included, which means a nested include block. This is not + // something we currently support. + err := errors.WithStackTrace(TooManyLevelsOfInheritance{ + ConfigPath: terragruntOptions.TerragruntConfigPath, + FirstLevelIncludePath: includeFromChild.Path, + SecondLevelIncludePath: strings.Join(includedPaths, ","), + }) + return nil, err + } else if hasInclude && includeFromChild == nil { + // Current parsing context where there is no included config already loaded. + trackInc := TrackInclude{ + CurrentList: terragruntIncludeList, + CurrentMap: terragruntIncludeMap, + Original: nil, + } + return &trackInc, nil + } else { + // Parsing context where there is an included config already loaded. + trackInc := TrackInclude{ + CurrentList: terragruntIncludeList, + CurrentMap: terragruntIncludeMap, + Original: includeFromChild, + } + return &trackInc, nil + } +} diff --git a/config/locals_test.go b/config/locals_test.go index 7db762ad97..9d83ae5f97 100644 --- a/config/locals_test.go +++ b/config/locals_test.go @@ -22,7 +22,7 @@ func TestEvaluateLocalsBlock(t *testing.T) { file, err := parseHcl(parser, LocalsTestConfig, mockFilename) require.NoError(t, err) - evaluatedLocals, err := evaluateLocalsBlock(terragruntOptions, parser, file, mockFilename, TrackInclude{}) + evaluatedLocals, err := evaluateLocalsBlock(terragruntOptions, parser, file, mockFilename, nil) require.NoError(t, err) var actualRegion string @@ -67,7 +67,7 @@ func TestEvaluateLocalsBlockMultiDeepReference(t *testing.T) { file, err := parseHcl(parser, LocalsTestMultiDeepReferenceConfig, mockFilename) require.NoError(t, err) - evaluatedLocals, err := evaluateLocalsBlock(terragruntOptions, parser, file, mockFilename, TrackInclude{}) + evaluatedLocals, err := evaluateLocalsBlock(terragruntOptions, parser, file, mockFilename, nil) require.NoError(t, err) expected := "a" @@ -106,7 +106,7 @@ func TestEvaluateLocalsBlockImpossibleWillFail(t *testing.T) { file, err := parseHcl(parser, LocalsTestImpossibleConfig, mockFilename) require.NoError(t, err) - _, err = evaluateLocalsBlock(terragruntOptions, parser, file, mockFilename, TrackInclude{}) + _, err = evaluateLocalsBlock(terragruntOptions, parser, file, mockFilename, nil) require.Error(t, err) switch errors.Unwrap(err).(type) { @@ -126,7 +126,7 @@ func TestEvaluateLocalsBlockMultipleLocalsBlocksWillFail(t *testing.T) { file, err := parseHcl(parser, MultipleLocalsBlockConfig, mockFilename) require.NoError(t, err) - _, err = evaluateLocalsBlock(terragruntOptions, parser, file, mockFilename, TrackInclude{}) + _, err = evaluateLocalsBlock(terragruntOptions, parser, file, mockFilename, nil) require.Error(t, err) } From a6ff274772817b3de470be051ee7d8a8582ec9c0 Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Thu, 9 Sep 2021 14:46:11 -0500 Subject: [PATCH 06/26] gofmt --- config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.go b/config/config.go index 5879e54df6..037b24aee2 100644 --- a/config/config.go +++ b/config/config.go @@ -65,7 +65,7 @@ type terragruntConfigFile struct { TerragruntVersionConstraint *string `hcl:"terragrunt_version_constraint,attr"` Inputs *cty.Value `hcl:"inputs,attr"` Include *IncludeConfig `hcl:"include,block"` - Import *ImportConfig `hcl:"import,block"` + Import *ImportConfig `hcl:"import,block"` // We allow users to configure remote state (backend) via blocks: // From b482a7433b0bc75e0d855ec601ddfe84f8e5f1a6 Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Thu, 9 Sep 2021 15:03:46 -0500 Subject: [PATCH 07/26] Fix path_relative_to_include in child config --- config/config_helpers.go | 47 +++++++++++++++++++++++------------ config/config_helpers_test.go | 3 ++- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/config/config_helpers.go b/config/config_helpers.go index dac8423ab2..8f5b03cebe 100644 --- a/config/config_helpers.go +++ b/config/config_helpers.go @@ -111,7 +111,7 @@ func CreateTerragruntEvalContext( terragruntFunctions := map[string]function.Function{ "find_in_parent_folders": wrapStringSliceToStringAsFuncImpl(findInParentFolders, extensions.TrackInclude, terragruntOptions), - "path_relative_to_include": wrapVoidToStringAsFuncImpl(pathRelativeToInclude, extensions.TrackInclude, terragruntOptions), + "path_relative_to_include": wrapStringSliceToStringAsFuncImpl(pathRelativeToInclude, extensions.TrackInclude, terragruntOptions), "path_relative_from_include": wrapStringSliceToStringAsFuncImpl(pathRelativeFromInclude, extensions.TrackInclude, terragruntOptions), "get_env": wrapStringSliceToStringAsFuncImpl(getEnvironmentVariable, extensions.TrackInclude, terragruntOptions), "run_cmd": wrapStringSliceToStringAsFuncImpl(runCommand, extensions.TrackInclude, terragruntOptions), @@ -342,12 +342,12 @@ func findInParentFolders( } previousDir, err := filepath.Abs(filepath.Dir(terragruntOptions.TerragruntConfigPath)) - previousDir = filepath.ToSlash(previousDir) - if err != nil { return "", errors.WithStackTrace(err) } + previousDir = filepath.ToSlash(previousDir) + fileToFindStr := DefaultTerragruntConfigPath if fileToFindParam != "" { fileToFindStr = fileToFindParam @@ -380,13 +380,28 @@ func findInParentFolders( } // Return the relative path between the included Terragrunt configuration file and the current Terragrunt configuration -// file -func pathRelativeToInclude(trackInclude *TrackInclude, terragruntOptions *options.TerragruntOptions) (string, error) { - if trackInclude == nil || trackInclude.Original == nil { +// file. Name param is required and used to lookup the relevant import block when called in a child config with multiple +// import blocks. +func pathRelativeToInclude(params []string, trackInclude *TrackInclude, terragruntOptions *options.TerragruntOptions) (string, error) { + if trackInclude == nil { + return ".", nil + } + + var included ImportConfig + if trackInclude.Original != nil { + included = *trackInclude.Original + } else if len(trackInclude.CurrentList) > 0 { + // Called in child context, so we need to select the right include file. + selected, err := getSelectedImportBlock(trackInclude.CurrentMap, params) + if err != nil { + return "", err + } + included = *selected + } else { return ".", nil } - includePath := filepath.Dir(trackInclude.Original.Path) + includePath := filepath.Dir(included.Path) currentPath := filepath.Dir(terragruntOptions.TerragruntConfigPath) if !filepath.IsAbs(includePath) { @@ -402,7 +417,7 @@ func pathRelativeFromInclude(params []string, trackInclude *TrackInclude, terrag return ".", nil } - included, err := getSelectedIncludeBlock(trackInclude.CurrentMap, params) + included, err := getSelectedImportBlock(trackInclude.CurrentMap, params) if err != nil { return "", err } else if included == nil { @@ -661,11 +676,11 @@ func getTerragruntSourceCliFlag(trackInclude *TrackInclude, terragruntOptions *o // - If there is only one include block, no param is required and that is automatically returned. // - If there is more than one include block, 1 param is required to use as the label name to lookup the include block // to use. -func getSelectedIncludeBlock(includeMap map[string]ImportConfig, params []string) (*ImportConfig, error) { - if len(includeMap) == 0 { +func getSelectedImportBlock(importMap map[string]ImportConfig, params []string) (*ImportConfig, error) { + if len(importMap) == 0 { return nil, nil - } else if len(includeMap) == 1 { - for _, val := range includeMap { + } else if len(importMap) == 1 { + for _, val := range importMap { return &val, nil } } @@ -675,12 +690,12 @@ func getSelectedIncludeBlock(includeMap map[string]ImportConfig, params []string return nil, errors.WithStackTrace(WrongNumberOfParams{Func: "path_relative_from_include", Expected: "1", Actual: numParams}) } - includeName := params[0] - included, hasKey := includeMap[includeName] + importName := params[0] + imported, hasKey := importMap[importName] if !hasKey { - return nil, errors.WithStackTrace(InvalidIncludeKey{name: includeName}) + return nil, errors.WithStackTrace(InvalidIncludeKey{name: importName}) } - return &included, nil + return &imported, nil } // Custom error types diff --git a/config/config_helpers_test.go b/config/config_helpers_test.go index c48e5dc27e..1ccb4941b8 100644 --- a/config/config_helpers_test.go +++ b/config/config_helpers_test.go @@ -70,7 +70,8 @@ func TestPathRelativeToInclude(t *testing.T) { Original: testCase.include, } } - actualPath, actualErr := pathRelativeToInclude(trackInclude, testCase.terragruntOptions) + // TODO: update to test for multiple include blocks + actualPath, actualErr := pathRelativeToInclude([]string{}, trackInclude, testCase.terragruntOptions) assert.Nil(t, actualErr, "For include %v and options %v, unexpected error: %v", testCase.include, testCase.terragruntOptions, actualErr) assert.Equal(t, testCase.expectedPath, actualPath, "For include %v and options %v", testCase.include, testCase.terragruntOptions) } From ac27dc4ff5346037ed52de87a0d8a7ed7283ef6d Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Thu, 9 Sep 2021 15:14:39 -0500 Subject: [PATCH 08/26] Fix bug where path_relative_from_include did not consider the parent context --- config/config_helpers.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/config/config_helpers.go b/config/config_helpers.go index 8f5b03cebe..8080ccd521 100644 --- a/config/config_helpers.go +++ b/config/config_helpers.go @@ -392,7 +392,7 @@ func pathRelativeToInclude(params []string, trackInclude *TrackInclude, terragru included = *trackInclude.Original } else if len(trackInclude.CurrentList) > 0 { // Called in child context, so we need to select the right include file. - selected, err := getSelectedImportBlock(trackInclude.CurrentMap, params) + selected, err := getSelectedImportBlock(*trackInclude, params) if err != nil { return "", err } @@ -417,7 +417,7 @@ func pathRelativeFromInclude(params []string, trackInclude *TrackInclude, terrag return ".", nil } - included, err := getSelectedImportBlock(trackInclude.CurrentMap, params) + included, err := getSelectedImportBlock(*trackInclude, params) if err != nil { return "", err } else if included == nil { @@ -672,14 +672,23 @@ func getTerragruntSourceCliFlag(trackInclude *TrackInclude, terragruntOptions *o } // Return the selected include block based on a label passed in as a function param. Note that the assumption is that: +// - If the Original attribute is set, we are in the parent context so return that. // - If there are no include blocks, no param is required and nil is returned. // - If there is only one include block, no param is required and that is automatically returned. // - If there is more than one include block, 1 param is required to use as the label name to lookup the include block // to use. -func getSelectedImportBlock(importMap map[string]ImportConfig, params []string) (*ImportConfig, error) { +func getSelectedImportBlock(trackInclude TrackInclude, params []string) (*ImportConfig, error) { + importMap := trackInclude.CurrentMap + + if trackInclude.Original != nil { + return trackInclude.Original, nil + } + if len(importMap) == 0 { return nil, nil - } else if len(importMap) == 1 { + } + + if len(importMap) == 1 { for _, val := range importMap { return &val, nil } From d3ef5b368a8fbc42f4227ecc3471103359f29e10 Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Fri, 10 Sep 2021 10:47:51 -0500 Subject: [PATCH 09/26] Implement multiple imports test for path_relative_from_include unit test --- config/config_helpers_test.go | 50 +++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/config/config_helpers_test.go b/config/config_helpers_test.go index 1ccb4941b8..cd0758f49a 100644 --- a/config/config_helpers_test.go +++ b/config/config_helpers_test.go @@ -81,60 +81,82 @@ func TestPathRelativeFromInclude(t *testing.T) { t.Parallel() testCases := []struct { - include *ImportConfig + include map[string]ImportConfig + params []string terragruntOptions *options.TerragruntOptions expectedPath string }{ { + nil, nil, terragruntOptionsForTest(t, helpers.RootFolder+"child/"+DefaultTerragruntConfigPath), ".", }, { - &ImportConfig{Path: "../" + DefaultTerragruntConfigPath}, + map[string]ImportConfig{"": ImportConfig{Path: "../" + DefaultTerragruntConfigPath}}, + nil, terragruntOptionsForTest(t, helpers.RootFolder+"child/"+DefaultTerragruntConfigPath), "..", }, { - &ImportConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}, + map[string]ImportConfig{"": ImportConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}}, + nil, terragruntOptionsForTest(t, helpers.RootFolder+"child/"+DefaultTerragruntConfigPath), "..", }, { - &ImportConfig{Path: "../../../" + DefaultTerragruntConfigPath}, + map[string]ImportConfig{"": ImportConfig{Path: "../../../" + DefaultTerragruntConfigPath}}, + nil, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/sub-sub-child/"+DefaultTerragruntConfigPath), "../../..", }, { - &ImportConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}, + map[string]ImportConfig{"": ImportConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}}, + nil, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/sub-sub-child/"+DefaultTerragruntConfigPath), "../../..", }, { - &ImportConfig{Path: "../../other-child/" + DefaultTerragruntConfigPath}, + map[string]ImportConfig{"": ImportConfig{Path: "../../other-child/" + DefaultTerragruntConfigPath}}, + nil, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/"+DefaultTerragruntConfigPath), "../../other-child", }, { - &ImportConfig{Path: "../../" + DefaultTerragruntConfigPath}, + map[string]ImportConfig{"": ImportConfig{Path: "../../" + DefaultTerragruntConfigPath}}, + nil, terragruntOptionsForTest(t, "../child/sub-child/"+DefaultTerragruntConfigPath), "../..", }, + { + map[string]ImportConfig{ + "root": ImportConfig{Path: "../../" + DefaultTerragruntConfigPath}, + "child": ImportConfig{Path: "../../other-child/" + DefaultTerragruntConfigPath}, + }, + []string{"child"}, + terragruntOptionsForTest(t, "../child/sub-child/"+DefaultTerragruntConfigPath), + "../../other-child", + }, } for _, testCase := range testCases { var trackInclude *TrackInclude = nil if testCase.include != nil { + currentList := make([]ImportConfig, len(testCase.include)) + i := 0 + for _, val := range testCase.include { + currentList[i] = val + i++ + } trackInclude = &TrackInclude{ - CurrentList: []ImportConfig{*testCase.include}, - CurrentMap: map[string]ImportConfig{ - "": *testCase.include, - }, - Original: testCase.include, + CurrentList: currentList, + CurrentMap: testCase.include, + } + if len(testCase.params) == 0 { + trackInclude.Original = ¤tList[0] } } - // TODO: update to test for multiple include blocks - actualPath, actualErr := pathRelativeFromInclude([]string{}, trackInclude, testCase.terragruntOptions) + actualPath, actualErr := pathRelativeFromInclude(testCase.params, trackInclude, testCase.terragruntOptions) assert.Nil(t, actualErr, "For include %v and options %v, unexpected error: %v", testCase.include, testCase.terragruntOptions, actualErr) assert.Equal(t, testCase.expectedPath, actualPath, "For include %v and options %v", testCase.include, testCase.terragruntOptions) } From 5e177dba709497b1e439cebb01c914997bb0ff33 Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Fri, 10 Sep 2021 11:39:11 -0500 Subject: [PATCH 10/26] Use a parse and update approach for dealing with bare include blocks --- config/config.go | 45 ++++++++----------- config/config_helpers.go | 14 +++--- config/config_helpers_test.go | 68 ++++++++++++++-------------- config/config_partial.go | 85 +++++++++++++++++++++++++---------- config/include.go | 28 ++++++------ 5 files changed, 134 insertions(+), 106 deletions(-) diff --git a/config/config.go b/config/config.go index 037b24aee2..ca5e9c6f60 100644 --- a/config/config.go +++ b/config/config.go @@ -64,8 +64,6 @@ type terragruntConfigFile struct { TerraformVersionConstraint *string `hcl:"terraform_version_constraint,attr"` TerragruntVersionConstraint *string `hcl:"terragrunt_version_constraint,attr"` Inputs *cty.Value `hcl:"inputs,attr"` - Include *IncludeConfig `hcl:"include,block"` - Import *ImportConfig `hcl:"import,block"` // We allow users to configure remote state (backend) via blocks: // @@ -113,17 +111,21 @@ type terragruntConfigFile struct { RetryMaxAttempts *int `hcl:"retry_max_attempts,optional"` RetrySleepIntervalSec *int `hcl:"retry_sleep_interval_sec,optional"` - // This struct is used for validating and parsing the entire terragrunt config. Since locals are evaluated in a - // completely separate cycle, it should not be evaluated here. Otherwise, we can't support self referencing other - // elements in the same block. - Locals *terragruntLocal `hcl:"locals,block"` + // This struct is used for validating and parsing the entire terragrunt config. Since locals and include are + // evaluated in a completely separate cycle, it should not be evaluated here. Otherwise, we can't support self + // referencing other elements in the same block. + Locals *terragruntLocal `hcl:"locals,block"` + Include *terragruntIncludeIgnore `hcl:"include,block"` } -// We use a struct designed to not parse the block, as locals are parsed and decoded using a special routine that allows -// references to the other locals in the same block. +// We use a struct designed to not parse the block, as locals and includes are parsed and decoded using a special +// routine that allows references to the other locals in the same block. type terragruntLocal struct { Remain hcl.Body `hcl:",remain"` } +type terragruntIncludeIgnore struct { + Remain hcl.Body `hcl:",remain"` +} // Configuration for Terraform remote state as parsed from a terragrunt.hcl config file type remoteStateConfigFile struct { @@ -188,37 +190,26 @@ type terragruntGenerateBlock struct { } // IncludeConfig represents the configuration settings for a parent Terragrunt configuration file that you can -// "include" in a child Terragrunt configuration file. -// This exists for backward compatibility reasons, and is equivalent to an import block with no label ("" for the -// label). You can only have one include block per terragrunt config. +// include into a child Terragrunt configuration file. You can have more than one include config. type IncludeConfig struct { + Name string `hcl:"name,label"` Path string `hcl:"path,attr"` Expose *bool `hcl:"expose,attr"` MergeStrategy *string `hcl:"merge_strategy,attr"` } -// ImportConfig represents the configuration settings for a parent Terragrunt configuration file that you can -// import into a child Terragrunt configuration file. You can have more than one import config. Note that this replaces -// "include". -type ImportConfig struct { - Name string `hcl:",label"` - Path string `hcl:"path,attr"` - Expose *bool `hcl:"expose,attr"` - MergeStrategy *string `hcl:"merge_strategy,attr"` -} - -func (cfg *ImportConfig) String() string { - return fmt.Sprintf("ImportConfig{Path = %s, Expose = %v, MergeStrategy = %v}", cfg.Path, cfg.Expose, cfg.MergeStrategy) +func (cfg *IncludeConfig) String() string { + return fmt.Sprintf("IncludeConfig{Path = %s, Expose = %v, MergeStrategy = %v}", cfg.Path, cfg.Expose, cfg.MergeStrategy) } -func (cfg *ImportConfig) GetExpose() bool { +func (cfg *IncludeConfig) GetExpose() bool { if cfg == nil || cfg.Expose == nil { return false } return *cfg.Expose } -func (cfg *ImportConfig) GetMergeStrategy() (MergeStrategyType, error) { +func (cfg *IncludeConfig) GetMergeStrategy() (MergeStrategyType, error) { if cfg.MergeStrategy == nil { return ShallowMerge, nil } @@ -545,7 +536,7 @@ func ReadTerragruntConfig(terragruntOptions *options.TerragruntOptions) (*Terrag // Parse the Terragrunt config file at the given path. If the include parameter is not nil, then treat this as a config // included in some other config file when resolving relative paths. -func ParseConfigFile(filename string, terragruntOptions *options.TerragruntOptions, include *ImportConfig, dependencyOutputs *cty.Value) (*TerragruntConfig, error) { +func ParseConfigFile(filename string, terragruntOptions *options.TerragruntOptions, include *IncludeConfig, dependencyOutputs *cty.Value) (*TerragruntConfig, error) { configString, err := util.ReadFileAsString(filename) if err != nil { return nil, err @@ -588,7 +579,7 @@ func ParseConfigFile(filename string, terragruntOptions *options.TerragruntOptio func ParseConfigString( configString string, terragruntOptions *options.TerragruntOptions, - includeFromChild *ImportConfig, + includeFromChild *IncludeConfig, filename string, dependencyOutputs *cty.Value, ) (*TerragruntConfig, error) { diff --git a/config/config_helpers.go b/config/config_helpers.go index 8080ccd521..eaaa7a7637 100644 --- a/config/config_helpers.go +++ b/config/config_helpers.go @@ -73,14 +73,14 @@ type TrackInclude struct { // CurrentList is used to track the list of configs that should be imported and merged before the final // TerragruntConfig is returned. This preserves the order of the blocks as they appear in the config, so that we can // merge the included config in the right order. - CurrentList []ImportConfig + CurrentList []IncludeConfig // CurrentMap is the map version of CurrentList that maps the block labels to the included config. - CurrentMap map[string]ImportConfig + CurrentMap map[string]IncludeConfig // Original is used to track the original included config, and is used for resolving the include related // functions. - Original *ImportConfig + Original *IncludeConfig } // EvalContextExtensions provides various extensions to the evaluation context to enhance the parsing capabilities. @@ -387,12 +387,12 @@ func pathRelativeToInclude(params []string, trackInclude *TrackInclude, terragru return ".", nil } - var included ImportConfig + var included IncludeConfig if trackInclude.Original != nil { included = *trackInclude.Original } else if len(trackInclude.CurrentList) > 0 { // Called in child context, so we need to select the right include file. - selected, err := getSelectedImportBlock(*trackInclude, params) + selected, err := getSelectedIncludeBlock(*trackInclude, params) if err != nil { return "", err } @@ -417,7 +417,7 @@ func pathRelativeFromInclude(params []string, trackInclude *TrackInclude, terrag return ".", nil } - included, err := getSelectedImportBlock(*trackInclude, params) + included, err := getSelectedIncludeBlock(*trackInclude, params) if err != nil { return "", err } else if included == nil { @@ -677,7 +677,7 @@ func getTerragruntSourceCliFlag(trackInclude *TrackInclude, terragruntOptions *o // - If there is only one include block, no param is required and that is automatically returned. // - If there is more than one include block, 1 param is required to use as the label name to lookup the include block // to use. -func getSelectedImportBlock(trackInclude TrackInclude, params []string) (*ImportConfig, error) { +func getSelectedIncludeBlock(trackInclude TrackInclude, params []string) (*IncludeConfig, error) { importMap := trackInclude.CurrentMap if trackInclude.Original != nil { diff --git a/config/config_helpers_test.go b/config/config_helpers_test.go index cd0758f49a..ece7ddaed7 100644 --- a/config/config_helpers_test.go +++ b/config/config_helpers_test.go @@ -18,7 +18,7 @@ func TestPathRelativeToInclude(t *testing.T) { t.Parallel() testCases := []struct { - include *ImportConfig + include *IncludeConfig terragruntOptions *options.TerragruntOptions expectedPath string }{ @@ -28,32 +28,32 @@ func TestPathRelativeToInclude(t *testing.T) { ".", }, { - &ImportConfig{Path: "../" + DefaultTerragruntConfigPath}, + &IncludeConfig{Path: "../" + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, helpers.RootFolder+"child/"+DefaultTerragruntConfigPath), "child", }, { - &ImportConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}, + &IncludeConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, helpers.RootFolder+"child/"+DefaultTerragruntConfigPath), "child", }, { - &ImportConfig{Path: "../../../" + DefaultTerragruntConfigPath}, + &IncludeConfig{Path: "../../../" + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/sub-sub-child/"+DefaultTerragruntConfigPath), "child/sub-child/sub-sub-child", }, { - &ImportConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}, + &IncludeConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/sub-sub-child/"+DefaultTerragruntConfigPath), "child/sub-child/sub-sub-child", }, { - &ImportConfig{Path: "../../other-child/" + DefaultTerragruntConfigPath}, + &IncludeConfig{Path: "../../other-child/" + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/"+DefaultTerragruntConfigPath), "../child/sub-child", }, { - &ImportConfig{Path: "../../" + DefaultTerragruntConfigPath}, + &IncludeConfig{Path: "../../" + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, "../child/sub-child/"+DefaultTerragruntConfigPath), "child/sub-child", }, @@ -63,8 +63,8 @@ func TestPathRelativeToInclude(t *testing.T) { var trackInclude *TrackInclude = nil if testCase.include != nil { trackInclude = &TrackInclude{ - CurrentList: []ImportConfig{*testCase.include}, - CurrentMap: map[string]ImportConfig{ + CurrentList: []IncludeConfig{*testCase.include}, + CurrentMap: map[string]IncludeConfig{ "": *testCase.include, }, Original: testCase.include, @@ -81,7 +81,7 @@ func TestPathRelativeFromInclude(t *testing.T) { t.Parallel() testCases := []struct { - include map[string]ImportConfig + include map[string]IncludeConfig params []string terragruntOptions *options.TerragruntOptions expectedPath string @@ -93,45 +93,45 @@ func TestPathRelativeFromInclude(t *testing.T) { ".", }, { - map[string]ImportConfig{"": ImportConfig{Path: "../" + DefaultTerragruntConfigPath}}, + map[string]IncludeConfig{"": IncludeConfig{Path: "../" + DefaultTerragruntConfigPath}}, nil, terragruntOptionsForTest(t, helpers.RootFolder+"child/"+DefaultTerragruntConfigPath), "..", }, { - map[string]ImportConfig{"": ImportConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}}, + map[string]IncludeConfig{"": IncludeConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}}, nil, terragruntOptionsForTest(t, helpers.RootFolder+"child/"+DefaultTerragruntConfigPath), "..", }, { - map[string]ImportConfig{"": ImportConfig{Path: "../../../" + DefaultTerragruntConfigPath}}, + map[string]IncludeConfig{"": IncludeConfig{Path: "../../../" + DefaultTerragruntConfigPath}}, nil, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/sub-sub-child/"+DefaultTerragruntConfigPath), "../../..", }, { - map[string]ImportConfig{"": ImportConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}}, + map[string]IncludeConfig{"": IncludeConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}}, nil, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/sub-sub-child/"+DefaultTerragruntConfigPath), "../../..", }, { - map[string]ImportConfig{"": ImportConfig{Path: "../../other-child/" + DefaultTerragruntConfigPath}}, + map[string]IncludeConfig{"": IncludeConfig{Path: "../../other-child/" + DefaultTerragruntConfigPath}}, nil, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/"+DefaultTerragruntConfigPath), "../../other-child", }, { - map[string]ImportConfig{"": ImportConfig{Path: "../../" + DefaultTerragruntConfigPath}}, + map[string]IncludeConfig{"": IncludeConfig{Path: "../../" + DefaultTerragruntConfigPath}}, nil, terragruntOptionsForTest(t, "../child/sub-child/"+DefaultTerragruntConfigPath), "../..", }, { - map[string]ImportConfig{ - "root": ImportConfig{Path: "../../" + DefaultTerragruntConfigPath}, - "child": ImportConfig{Path: "../../other-child/" + DefaultTerragruntConfigPath}, + map[string]IncludeConfig{ + "root": IncludeConfig{Path: "../../" + DefaultTerragruntConfigPath}, + "child": IncludeConfig{Path: "../../other-child/" + DefaultTerragruntConfigPath}, }, []string{"child"}, terragruntOptionsForTest(t, "../child/sub-child/"+DefaultTerragruntConfigPath), @@ -142,7 +142,7 @@ func TestPathRelativeFromInclude(t *testing.T) { for _, testCase := range testCases { var trackInclude *TrackInclude = nil if testCase.include != nil { - currentList := make([]ImportConfig, len(testCase.include)) + currentList := make([]IncludeConfig, len(testCase.include)) i := 0 for _, val := range testCase.include { currentList[i] = val @@ -315,7 +315,7 @@ func TestResolveTerragruntInterpolation(t *testing.T) { testCases := []struct { str string - include *ImportConfig + include *IncludeConfig terragruntOptions *options.TerragruntOptions expectedOut string expectedErr string @@ -329,7 +329,7 @@ func TestResolveTerragruntInterpolation(t *testing.T) { }, { "terraform { source = path_relative_to_include() }", - &ImportConfig{Path: "../" + DefaultTerragruntConfigPath}, + &IncludeConfig{Path: "../" + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, "/root/child/"+DefaultTerragruntConfigPath), "child", "", @@ -382,7 +382,7 @@ func TestResolveEnvInterpolationConfigString(t *testing.T) { testCases := []struct { str string - include *ImportConfig + include *IncludeConfig terragruntOptions *options.TerragruntOptions expectedOut string expectedErr string @@ -474,7 +474,7 @@ func TestResolveCommandsInterpolationConfigString(t *testing.T) { testCases := []struct { str string - include *ImportConfig + include *IncludeConfig terragruntOptions *options.TerragruntOptions expectedFooInput []string }{ @@ -536,7 +536,7 @@ func TestResolveCliArgsInterpolationConfigString(t *testing.T) { } testCase := struct { str string - include *ImportConfig + include *IncludeConfig terragruntOptions *options.TerragruntOptions expectedFooInput []string }{ @@ -632,7 +632,7 @@ func TestGetParentTerragruntDir(t *testing.T) { parentDir := filepath.ToSlash(filepath.Dir(currentDir)) testCases := []struct { - include *ImportConfig + include *IncludeConfig terragruntOptions *options.TerragruntOptions expectedPath string }{ @@ -642,32 +642,32 @@ func TestGetParentTerragruntDir(t *testing.T) { helpers.RootFolder + "child", }, { - &ImportConfig{Path: "../" + DefaultTerragruntConfigPath}, + &IncludeConfig{Path: "../" + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, helpers.RootFolder+"child/"+DefaultTerragruntConfigPath), helpers.RootFolder, }, { - &ImportConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}, + &IncludeConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, helpers.RootFolder+"child/"+DefaultTerragruntConfigPath), helpers.RootFolder, }, { - &ImportConfig{Path: "../../../" + DefaultTerragruntConfigPath}, + &IncludeConfig{Path: "../../../" + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/sub-sub-child/"+DefaultTerragruntConfigPath), helpers.RootFolder, }, { - &ImportConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}, + &IncludeConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/sub-sub-child/"+DefaultTerragruntConfigPath), helpers.RootFolder, }, { - &ImportConfig{Path: "../../other-child/" + DefaultTerragruntConfigPath}, + &IncludeConfig{Path: "../../other-child/" + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/"+DefaultTerragruntConfigPath), fmt.Sprintf("%s/other-child", filepath.VolumeName(parentDir)), }, { - &ImportConfig{Path: "../../" + DefaultTerragruntConfigPath}, + &IncludeConfig{Path: "../../" + DefaultTerragruntConfigPath}, terragruntOptionsForTest(t, "../child/sub-child/"+DefaultTerragruntConfigPath), parentDir, }, @@ -677,8 +677,8 @@ func TestGetParentTerragruntDir(t *testing.T) { var trackInclude *TrackInclude = nil if testCase.include != nil { trackInclude = &TrackInclude{ - CurrentList: []ImportConfig{*testCase.include}, - CurrentMap: map[string]ImportConfig{ + CurrentList: []IncludeConfig{*testCase.include}, + CurrentMap: map[string]IncludeConfig{ "": *testCase.include, }, Original: testCase.include, diff --git a/config/config_partial.go b/config/config_partial.go index 29b1b8766b..1ad2d9cbd3 100644 --- a/config/config_partial.go +++ b/config/config_partial.go @@ -6,6 +6,7 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclparse" + "github.com/hashicorp/hcl/v2/hclwrite" "github.com/zclconf/go-cty/cty" "github.com/gruntwork-io/terragrunt/errors" @@ -27,11 +28,10 @@ const ( RemoteStateBlock ) -// terragruntInclude is a struct that can be used to only decode the include block. -type terragruntInclude struct { - Include *IncludeConfig `hcl:"include,block"` - Import []ImportConfig `hcl:"import,block"` - Remain hcl.Body `hcl:",remain"` +// terragruntIncludeMultiple is a struct that can be used to only decode the include block with labels. +type terragruntIncludeMultiple struct { + Include []IncludeConfig `hcl:"include,block"` + Remain hcl.Body `hcl:",remain"` } // terragruntDependencies is a struct that can be used to only decode the dependencies block. @@ -98,7 +98,7 @@ func DecodeBaseBlocks( parser *hclparse.Parser, hclFile *hcl.File, filename string, - includeFromChild *ImportConfig, + includeFromChild *IncludeConfig, ) (*cty.Value, *TrackInclude, error) { // Decode just the `include` and `import` blocks, and verify that it's allowed here terragruntIncludeList, err := decodeAsTerragruntInclude( @@ -139,7 +139,7 @@ func DecodeBaseBlocks( func PartialParseConfigFile( filename string, terragruntOptions *options.TerragruntOptions, - include *ImportConfig, + include *IncludeConfig, decodeList []PartialDecodeSectionType, ) (*TerragruntConfig, error) { configString, err := util.ReadFileAsString(filename) @@ -173,7 +173,7 @@ func PartialParseConfigFile( func PartialParseConfigString( configString string, terragruntOptions *options.TerragruntOptions, - includeFromChild *ImportConfig, + includeFromChild *IncludeConfig, filename string, decodeList []PartialDecodeSectionType, ) (*TerragruntConfig, error) { @@ -309,7 +309,7 @@ func PartialParseConfigString( return &output, nil } -func partialParseIncludedConfig(includedConfig *ImportConfig, terragruntOptions *options.TerragruntOptions, decodeList []PartialDecodeSectionType) (*TerragruntConfig, error) { +func partialParseIncludedConfig(includedConfig *IncludeConfig, terragruntOptions *options.TerragruntOptions, decodeList []PartialDecodeSectionType) (*TerragruntConfig, error) { if includedConfig.Path == "" { return nil, errors.WithStackTrace(IncludedConfigMissingPath(terragruntOptions.TerragruntConfigPath)) } @@ -328,34 +328,65 @@ func partialParseIncludedConfig(includedConfig *ImportConfig, terragruntOptions ) } -// This decodes only the `include` and `import` blocks of a terragrunt config, so its value can be used while decoding -// the rest of the config. +// This decodes only the `include` blocks of a terragrunt config, so its value can be used while decoding the rest of +// the config. // For consistency, `include` in the call to `decodeHcl` is always assumed to be nil. Either it really is nil (parsing // the child config), or it shouldn't be used anyway (the parent config shouldn't have an include block). +// +// We take a two pass approach to parsing include blocks to support include blocks without a label. Ideally we can parse +// include blocks with and without labels in a single pass, but the HCL parser is fairly restrictive when it comes to +// parsing blocks with labels, requiring the exact number of expected labels in the parsing step. +// To handle this restriction, we first see if there are any include blocks without any labels, and if there is, we +// modify it in the file object to inject the label as "". func decodeAsTerragruntInclude( file *hcl.File, filename string, terragruntOptions *options.TerragruntOptions, extensions EvalContextExtensions, -) ([]ImportConfig, error) { - tgInc := terragruntInclude{} - err := decodeHcl(file, filename, &tgInc, terragruntOptions, extensions) +) ([]IncludeConfig, error) { + updatedBytes, isUpdated, err := updateBareIncludeBlock(file, filename) if err != nil { return nil, err } + if isUpdated { + // Code was updated, so we need to reparse the new updated contents. This is necessarily because the blocks + // returned by hclparse does not support editing, and so we have to go through hclwrite, which leads to a + // different AST representation. + file, err = parseHcl(hclparse.NewParser(), string(updatedBytes), filename) + if err != nil { + return nil, err + } + } - // Convert the legacy include block into an ImportConfig with no label ("" for label). include blocks are always the - // top most import block. - if tgInc.Include != nil { - includeAsImport := ImportConfig{ - Name: "", - Path: tgInc.Include.Path, - Expose: tgInc.Include.Expose, - MergeStrategy: tgInc.Include.MergeStrategy, + tgInc := terragruntIncludeMultiple{} + if err := decodeHcl(file, filename, &tgInc, terragruntOptions, extensions); err != nil { + return nil, err + } + return tgInc.Include, nil +} + +// updateBareIncludeBlock searches the parsed terragrunt contents for a bare include block (include without a label), +// and convert it to one with empty string as the label. This is necessary because the hcl parser is strictly enforces +// label counts when parsing out labels with a go struct. +// +// Returns the updated contents, a boolean indicated whether anything changed, and an error (if any). +func updateBareIncludeBlock(file *hcl.File, filename string) ([]byte, bool, error) { + hclFile, err := hclwrite.ParseConfig(file.Bytes, filename, hcl.InitialPos) + if err != nil { + return nil, false, errors.WithStackTrace(err) + } + + codeWasUpdated := false + for _, block := range hclFile.Body().Blocks() { + if block.Type() == "include" && len(block.Labels()) == 0 { + if codeWasUpdated { + return nil, false, errors.WithStackTrace(MultipleBareIncludeBlocksErr{}) + } + block.SetLabels([]string{""}) + codeWasUpdated = true } - tgInc.Import = append([]ImportConfig{includeAsImport}, tgInc.Import...) } - return tgInc.Import, nil + return hclFile.Bytes(), codeWasUpdated, nil } // Custom error types @@ -367,3 +398,9 @@ type InvalidPartialBlockName struct { func (err InvalidPartialBlockName) Error() string { return fmt.Sprintf("Unrecognized partial block code %d. This is most likely an error in terragrunt. Please file a bug report on the project repository.", err.sectionCode) } + +type MultipleBareIncludeBlocksErr struct{} + +func (err MultipleBareIncludeBlocksErr) Error() string { + return "Multiple bare include blocks (include blocks without label) is not supported." +} diff --git a/config/include.go b/config/include.go index 204f908f77..7193251360 100644 --- a/config/include.go +++ b/config/include.go @@ -14,7 +14,7 @@ import ( ) // Parse the config of the given include, if one is specified -func parseIncludedConfig(includedConfig *ImportConfig, terragruntOptions *options.TerragruntOptions, dependencyOutputs *cty.Value) (*TerragruntConfig, error) { +func parseIncludedConfig(includedConfig *IncludeConfig, terragruntOptions *options.TerragruntOptions, dependencyOutputs *cty.Value) (*TerragruntConfig, error) { if includedConfig.Path == "" { return nil, errors.WithStackTrace(IncludedConfigMissingPath(terragruntOptions.TerragruntConfigPath)) } @@ -51,7 +51,7 @@ func handleInclude( return config, err } - parsedImportConfig, err := parseIncludedConfig(&includeConfig, terragruntOptions, dependencyOutputs) + parsedIncludeConfig, err := parseIncludedConfig(&includeConfig, terragruntOptions, dependencyOutputs) if err != nil { return nil, err } @@ -61,14 +61,14 @@ func handleInclude( terragruntOptions.Logger.Debugf("Included config %s has strategy no merge: not merging config in.", includeConfig.Path) case ShallowMerge: terragruntOptions.Logger.Debugf("Included config %s has strategy shallow merge: merging config in (shallow).", includeConfig.Path) - parsedImportConfig.Merge(baseConfig, terragruntOptions) - baseConfig = parsedImportConfig + parsedIncludeConfig.Merge(baseConfig, terragruntOptions) + baseConfig = parsedIncludeConfig case DeepMerge: terragruntOptions.Logger.Debugf("Included config %s has strategy deep merge: merging config in (deep).", includeConfig.Path) - if err := parsedImportConfig.DeepMerge(baseConfig, terragruntOptions); err != nil { + if err := parsedIncludeConfig.DeepMerge(baseConfig, terragruntOptions); err != nil { return nil, err } - baseConfig = parsedImportConfig + baseConfig = parsedIncludeConfig default: return nil, fmt.Errorf("You reached an impossible condition. This is most likely a bug in terragrunt. Please open an issue at github.com/gruntwork-io/terragrunt with this error message. Code: UNKNOWN_MERGE_STRATEGY_%s", mergeStrategy) } @@ -99,7 +99,7 @@ func handleIncludePartial( return nil, err } - parsedImportConfig, err := partialParseIncludedConfig(&includeConfig, terragruntOptions, decodeList) + parsedIncludeConfig, err := partialParseIncludedConfig(&includeConfig, terragruntOptions, decodeList) if err != nil { return nil, err } @@ -109,14 +109,14 @@ func handleIncludePartial( terragruntOptions.Logger.Debugf("[Partial] Included config %s has strategy no merge: not merging config in.", includeConfig.Path) case ShallowMerge: terragruntOptions.Logger.Debugf("[Partial] Included config %s has strategy shallow merge: merging config in (shallow).", includeConfig.Path) - parsedImportConfig.Merge(baseConfig, terragruntOptions) - baseConfig = parsedImportConfig + parsedIncludeConfig.Merge(baseConfig, terragruntOptions) + baseConfig = parsedIncludeConfig case DeepMerge: terragruntOptions.Logger.Debugf("[Partial] Included config %s has strategy deep merge: merging config in (deep).", includeConfig.Path) - if err := parsedImportConfig.DeepMerge(baseConfig, terragruntOptions); err != nil { + if err := parsedIncludeConfig.DeepMerge(baseConfig, terragruntOptions); err != nil { return nil, err } - baseConfig = parsedImportConfig + baseConfig = parsedIncludeConfig default: return nil, fmt.Errorf("You reached an impossible condition. This is most likely a bug in terragrunt. Please open an issue at github.com/gruntwork-io/terragrunt with this error message. Code: UNKNOWN_MERGE_STRATEGY_%s_PARTIAL", mergeStrategy) } @@ -503,12 +503,12 @@ func mergeHooks(terragruntOptions *options.TerragruntOptions, childHooks []Hook, // included config in the current parsing context, and an included config that was passed through from a previous // parsing context. func getTrackInclude( - terragruntIncludeList []ImportConfig, - includeFromChild *ImportConfig, + terragruntIncludeList []IncludeConfig, + includeFromChild *IncludeConfig, terragruntOptions *options.TerragruntOptions, ) (*TrackInclude, error) { includedPaths := []string{} - terragruntIncludeMap := make(map[string]ImportConfig, len(terragruntIncludeList)) + terragruntIncludeMap := make(map[string]IncludeConfig, len(terragruntIncludeList)) for _, tgInc := range terragruntIncludeList { includedPaths = append(includedPaths, tgInc.Path) terragruntIncludeMap[tgInc.Name] = tgInc From 7158540c2713db9646b5c58a0d957b0a3fa70cc2 Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Fri, 10 Sep 2021 12:31:40 -0500 Subject: [PATCH 11/26] Implement multiple imports test for path_relative_to_include unit test --- config/config_helpers_test.go | 82 ++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 35 deletions(-) diff --git a/config/config_helpers_test.go b/config/config_helpers_test.go index ece7ddaed7..e4db785009 100644 --- a/config/config_helpers_test.go +++ b/config/config_helpers_test.go @@ -18,60 +18,67 @@ func TestPathRelativeToInclude(t *testing.T) { t.Parallel() testCases := []struct { - include *IncludeConfig + include map[string]IncludeConfig + params []string terragruntOptions *options.TerragruntOptions expectedPath string }{ { + nil, nil, terragruntOptionsForTest(t, helpers.RootFolder+"child/"+DefaultTerragruntConfigPath), ".", }, { - &IncludeConfig{Path: "../" + DefaultTerragruntConfigPath}, + map[string]IncludeConfig{"": IncludeConfig{Path: "../" + DefaultTerragruntConfigPath}}, + nil, terragruntOptionsForTest(t, helpers.RootFolder+"child/"+DefaultTerragruntConfigPath), "child", }, { - &IncludeConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}, + map[string]IncludeConfig{"": IncludeConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}}, + nil, terragruntOptionsForTest(t, helpers.RootFolder+"child/"+DefaultTerragruntConfigPath), "child", }, { - &IncludeConfig{Path: "../../../" + DefaultTerragruntConfigPath}, + map[string]IncludeConfig{"": IncludeConfig{Path: "../../../" + DefaultTerragruntConfigPath}}, + nil, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/sub-sub-child/"+DefaultTerragruntConfigPath), "child/sub-child/sub-sub-child", }, { - &IncludeConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}, + map[string]IncludeConfig{"": IncludeConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}}, + nil, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/sub-sub-child/"+DefaultTerragruntConfigPath), "child/sub-child/sub-sub-child", }, { - &IncludeConfig{Path: "../../other-child/" + DefaultTerragruntConfigPath}, + map[string]IncludeConfig{"": IncludeConfig{Path: "../../other-child/" + DefaultTerragruntConfigPath}}, + nil, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/"+DefaultTerragruntConfigPath), "../child/sub-child", }, { - &IncludeConfig{Path: "../../" + DefaultTerragruntConfigPath}, + map[string]IncludeConfig{"": IncludeConfig{Path: "../../" + DefaultTerragruntConfigPath}}, + nil, terragruntOptionsForTest(t, "../child/sub-child/"+DefaultTerragruntConfigPath), "child/sub-child", }, + { + map[string]IncludeConfig{ + "root": IncludeConfig{Path: "../../" + DefaultTerragruntConfigPath}, + "child": IncludeConfig{Path: "../../other-child/" + DefaultTerragruntConfigPath}, + }, + []string{"child"}, + terragruntOptionsForTest(t, "../child/sub-child/"+DefaultTerragruntConfigPath), + "../child/sub-child", + }, } for _, testCase := range testCases { - var trackInclude *TrackInclude = nil - if testCase.include != nil { - trackInclude = &TrackInclude{ - CurrentList: []IncludeConfig{*testCase.include}, - CurrentMap: map[string]IncludeConfig{ - "": *testCase.include, - }, - Original: testCase.include, - } - } - // TODO: update to test for multiple include blocks - actualPath, actualErr := pathRelativeToInclude([]string{}, trackInclude, testCase.terragruntOptions) + trackInclude := getTrackIncludeFromTestData(testCase.include, testCase.params) + actualPath, actualErr := pathRelativeToInclude(testCase.params, trackInclude, testCase.terragruntOptions) assert.Nil(t, actualErr, "For include %v and options %v, unexpected error: %v", testCase.include, testCase.terragruntOptions, actualErr) assert.Equal(t, testCase.expectedPath, actualPath, "For include %v and options %v", testCase.include, testCase.terragruntOptions) } @@ -140,22 +147,7 @@ func TestPathRelativeFromInclude(t *testing.T) { } for _, testCase := range testCases { - var trackInclude *TrackInclude = nil - if testCase.include != nil { - currentList := make([]IncludeConfig, len(testCase.include)) - i := 0 - for _, val := range testCase.include { - currentList[i] = val - i++ - } - trackInclude = &TrackInclude{ - CurrentList: currentList, - CurrentMap: testCase.include, - } - if len(testCase.params) == 0 { - trackInclude.Original = ¤tList[0] - } - } + trackInclude := getTrackIncludeFromTestData(testCase.include, testCase.params) actualPath, actualErr := pathRelativeFromInclude(testCase.params, trackInclude, testCase.terragruntOptions) assert.Nil(t, actualErr, "For include %v and options %v, unexpected error: %v", testCase.include, testCase.terragruntOptions, actualErr) assert.Equal(t, testCase.expectedPath, actualPath, "For include %v and options %v", testCase.include, testCase.terragruntOptions) @@ -978,3 +970,23 @@ func getKeys(valueMap map[string]cty.Value) map[string]bool { } return keys } + +func getTrackIncludeFromTestData(includeMap map[string]IncludeConfig, params []string) *TrackInclude { + if len(includeMap) == 0 { + return nil + } + currentList := make([]IncludeConfig, len(includeMap)) + i := 0 + for _, val := range includeMap { + currentList[i] = val + i++ + } + trackInclude := &TrackInclude{ + CurrentList: currentList, + CurrentMap: includeMap, + } + if len(params) == 0 { + trackInclude.Original = ¤tList[0] + } + return trackInclude +} From c7d29b1ada6f494ef4796acbc4feee9250b80efd Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Fri, 10 Sep 2021 12:31:58 -0500 Subject: [PATCH 12/26] Support include reencoding for json config files --- config/config_partial.go | 125 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/config/config_partial.go b/config/config_partial.go index 1ad2d9cbd3..ae6cea8a4b 100644 --- a/config/config_partial.go +++ b/config/config_partial.go @@ -1,6 +1,7 @@ package config import ( + "encoding/json" "fmt" "path/filepath" @@ -371,6 +372,10 @@ func decodeAsTerragruntInclude( // // Returns the updated contents, a boolean indicated whether anything changed, and an error (if any). func updateBareIncludeBlock(file *hcl.File, filename string) ([]byte, bool, error) { + if filepath.Ext(filename) == ".json" { + return updateBareIncludeBlockJSON(file.Bytes) + } + hclFile, err := hclwrite.ParseConfig(file.Bytes, filename, hcl.InitialPos) if err != nil { return nil, false, errors.WithStackTrace(err) @@ -389,6 +394,118 @@ func updateBareIncludeBlock(file *hcl.File, filename string) ([]byte, bool, erro return hclFile.Bytes(), codeWasUpdated, nil } +// updateBareIncludeBlockJSON implements the logic for updateBareIncludeBlock when the terragrunt.hcl configuration is +// encoded in json. The json version of this function is fairly complex due to the flexibility in how the blocks are +// encoded. That is, all of the following are valid encodings of a terragrunt.hcl.json file that has a bare include +// block: +// +// Case 1: a single include block as top level: +// { +// "include": { +// "path": "foo" +// } +// } +// +// Case 2: a single include block in list: +// { +// "include": [ +// {"path": "foo"} +// ] +// } +// +// Case 3: mixed bare and labeled include block as list: +// { +// "include": [ +// {"path": "foo"}, +// { +// "labeled": {"path": "bar"} +// } +// ] +// } +// +// For simplicity of implementation, we focus on handling Case 1 and 2, and ignore Case 3. If we see Case 3, we will +// error out. Instead, the user should handle this case explicitly using the object encoding instead of list encoding: +// { +// "include": { +// "": {"path": "foo"}, +// "labeled": {"path": "bar"} +// } +// } +// If the multiple include blocks are encoded in this way in the json configuration, nothing needs to be done by this +// function. +func updateBareIncludeBlockJSON(fileBytes []byte) ([]byte, bool, error) { + var parsed map[string]interface{} + if err := json.Unmarshal(fileBytes, &parsed); err != nil { + return nil, false, errors.WithStackTrace(err) + } + includeBlock, hasKey := parsed["include"] + if !hasKey { + // No include block, so don't do anything + return fileBytes, false, nil + } + switch typed := includeBlock.(type) { + case []interface{}: + if len(typed) == 0 { + // No include block, so don't do anything + return nil, false, nil + } else if len(typed) > 1 { + // Could be multiple bare includes, or Case 3. We simplify the handling of this case by erroring out, + // ignoring the possibility of Case 3, which, while valid HCL encoding, is too complex to detect and handle + // here. Instead we will recommend users use the object encoding. + return nil, false, errors.WithStackTrace(MultipleBareIncludeBlocksErr{}) + } + + // Make sure this is Case 2, and not Case 3 with a single labeled block. If Case 2, update to inject the labeled + // version. Otherwise, return without modifying. + singleBlock := typed[0] + if jsonIsIncludeBlock(singleBlock) { + return updateSingleBareIncludeInParsedJSON(parsed, singleBlock) + } + return nil, false, nil + case map[string]interface{}: + if len(typed) == 0 { + // No include block, so don't do anything + return nil, false, nil + } + + // We will only update the include block if we detect the object to represent an include block. Otherwise, the + // blocks are labeled so we can pass forward to the tg parser step. + if jsonIsIncludeBlock(typed) { + return updateSingleBareIncludeInParsedJSON(parsed, typed) + } + return nil, false, nil + } + + return nil, false, errors.WithStackTrace(IncludeIsNotABlockErr{parsed: includeBlock}) +} + +// updateBareIncludeInParsedJSON replaces the include attribute into a block with the label "" in the json. Note that we +// can directly assign to the map with the single "" key without worrying about the possibility of other include blocks +// since we will only call this function if there is only one include block, and that is a bare block with no labels. +func updateSingleBareIncludeInParsedJSON(parsed map[string]interface{}, newVal interface{}) ([]byte, bool, error) { + parsed["include"] = map[string]interface{}{"": newVal} + updatedBytes, err := json.Marshal(parsed) + return updatedBytes, true, errors.WithStackTrace(err) +} + +// jsonIsIncludeBlock checks if the arbitrary json data is the include block. The data is determined to be an include +// block if: +// - It is an object +// - Has the 'path' attribute +// - The 'path' attribute is a string +func jsonIsIncludeBlock(jsonData interface{}) bool { + typed, isMap := jsonData.(map[string]interface{}) + if isMap { + pathAttr, hasPath := typed["path"] + if hasPath { + _, pathIsString := pathAttr.(string) + return pathIsString + } + return false + } + return false +} + // Custom error types type InvalidPartialBlockName struct { @@ -404,3 +521,11 @@ type MultipleBareIncludeBlocksErr struct{} func (err MultipleBareIncludeBlocksErr) Error() string { return "Multiple bare include blocks (include blocks without label) is not supported." } + +type IncludeIsNotABlockErr struct { + parsed interface{} +} + +func (err IncludeIsNotABlockErr) Error() string { + return fmt.Sprintf("Parsed include is not a block: %v", err.parsed) +} From 9678826834631543e2a7241555f0cdc5ac87665e Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Fri, 10 Sep 2021 13:46:11 -0500 Subject: [PATCH 13/26] Implement multiple imports test for get_parent_terragrunt_dir unit test --- config/config_helpers_test.go | 45 ++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/config/config_helpers_test.go b/config/config_helpers_test.go index e4db785009..3ee112f310 100644 --- a/config/config_helpers_test.go +++ b/config/config_helpers_test.go @@ -624,60 +624,67 @@ func TestGetParentTerragruntDir(t *testing.T) { parentDir := filepath.ToSlash(filepath.Dir(currentDir)) testCases := []struct { - include *IncludeConfig + include map[string]IncludeConfig + params []string terragruntOptions *options.TerragruntOptions expectedPath string }{ { + nil, nil, terragruntOptionsForTest(t, helpers.RootFolder+"child/"+DefaultTerragruntConfigPath), helpers.RootFolder + "child", }, { - &IncludeConfig{Path: "../" + DefaultTerragruntConfigPath}, + map[string]IncludeConfig{"": IncludeConfig{Path: "../" + DefaultTerragruntConfigPath}}, + nil, terragruntOptionsForTest(t, helpers.RootFolder+"child/"+DefaultTerragruntConfigPath), helpers.RootFolder, }, { - &IncludeConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}, + map[string]IncludeConfig{"": IncludeConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}}, + nil, terragruntOptionsForTest(t, helpers.RootFolder+"child/"+DefaultTerragruntConfigPath), helpers.RootFolder, }, { - &IncludeConfig{Path: "../../../" + DefaultTerragruntConfigPath}, + map[string]IncludeConfig{"": IncludeConfig{Path: "../../../" + DefaultTerragruntConfigPath}}, + nil, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/sub-sub-child/"+DefaultTerragruntConfigPath), helpers.RootFolder, }, { - &IncludeConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}, + map[string]IncludeConfig{"": IncludeConfig{Path: helpers.RootFolder + DefaultTerragruntConfigPath}}, + nil, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/sub-sub-child/"+DefaultTerragruntConfigPath), helpers.RootFolder, }, { - &IncludeConfig{Path: "../../other-child/" + DefaultTerragruntConfigPath}, + map[string]IncludeConfig{"": IncludeConfig{Path: "../../other-child/" + DefaultTerragruntConfigPath}}, + nil, terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/"+DefaultTerragruntConfigPath), fmt.Sprintf("%s/other-child", filepath.VolumeName(parentDir)), }, { - &IncludeConfig{Path: "../../" + DefaultTerragruntConfigPath}, + map[string]IncludeConfig{"": IncludeConfig{Path: "../../" + DefaultTerragruntConfigPath}}, + nil, terragruntOptionsForTest(t, "../child/sub-child/"+DefaultTerragruntConfigPath), parentDir, }, + { + map[string]IncludeConfig{ + "root": IncludeConfig{Path: "../../" + DefaultTerragruntConfigPath}, + "child": IncludeConfig{Path: "../../other-child/" + DefaultTerragruntConfigPath}, + }, + []string{"child"}, + terragruntOptionsForTest(t, helpers.RootFolder+"child/sub-child/"+DefaultTerragruntConfigPath), + fmt.Sprintf("%s/other-child", filepath.VolumeName(parentDir)), + }, } for _, testCase := range testCases { - var trackInclude *TrackInclude = nil - if testCase.include != nil { - trackInclude = &TrackInclude{ - CurrentList: []IncludeConfig{*testCase.include}, - CurrentMap: map[string]IncludeConfig{ - "": *testCase.include, - }, - Original: testCase.include, - } - } - // TODO: update to test for multiple include blocks - actualPath, actualErr := getParentTerragruntDir([]string{}, trackInclude, testCase.terragruntOptions) + trackInclude := getTrackIncludeFromTestData(testCase.include, testCase.params) + actualPath, actualErr := getParentTerragruntDir(testCase.params, trackInclude, testCase.terragruntOptions) assert.Nil(t, actualErr, "For include %v and options %v, unexpected error: %v", testCase.include, testCase.terragruntOptions, actualErr) assert.Equal(t, testCase.expectedPath, actualPath, "For include %v and options %v", testCase.include, testCase.terragruntOptions) } From f2bfd418b8b824b6a461dab30507936e0b411230 Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Fri, 10 Sep 2021 14:39:02 -0500 Subject: [PATCH 14/26] WIP: initial integration test - multiple include with deep merge --- config/config.go | 7 +- config/config_partial.go | 176 ------------------ config/hcl_parser.go | 21 +++ config/include.go | 159 ++++++++++++++++ .../child/terragrunt.hcl | 36 ++++ .../vpc/terragrunt.hcl | 3 + .../child/terragrunt.hcl | 38 ++++ .../deep-merge-overlapping/vpc/terragrunt.hcl | 3 + .../modules/empty/main.tf | 1 + .../modules/reflect/main.tf | 47 +++++ .../terragrunt_inputs.hcl | 9 + .../terragrunt_inputs_override.hcl | 8 + .../terragrunt_vpc_dep.hcl | 14 ++ .../terragrunt_vpc_dep_override.hcl | 11 ++ test/integration_include_test.go | 62 ++++++ 15 files changed, 417 insertions(+), 178 deletions(-) create mode 100644 test/fixture-include-multiple/deep-merge-nonoverlapping/child/terragrunt.hcl create mode 100644 test/fixture-include-multiple/deep-merge-nonoverlapping/vpc/terragrunt.hcl create mode 100644 test/fixture-include-multiple/deep-merge-overlapping/child/terragrunt.hcl create mode 100644 test/fixture-include-multiple/deep-merge-overlapping/vpc/terragrunt.hcl create mode 100644 test/fixture-include-multiple/modules/empty/main.tf create mode 100644 test/fixture-include-multiple/modules/reflect/main.tf create mode 100644 test/fixture-include-multiple/terragrunt_inputs.hcl create mode 100644 test/fixture-include-multiple/terragrunt_inputs_override.hcl create mode 100644 test/fixture-include-multiple/terragrunt_vpc_dep.hcl create mode 100644 test/fixture-include-multiple/terragrunt_vpc_dep_override.hcl diff --git a/config/config.go b/config/config.go index ca5e9c6f60..02ffeafb3b 100644 --- a/config/config.go +++ b/config/config.go @@ -114,8 +114,10 @@ type terragruntConfigFile struct { // This struct is used for validating and parsing the entire terragrunt config. Since locals and include are // evaluated in a completely separate cycle, it should not be evaluated here. Otherwise, we can't support self // referencing other elements in the same block. - Locals *terragruntLocal `hcl:"locals,block"` - Include *terragruntIncludeIgnore `hcl:"include,block"` + // We don't want to use the special Remain keyword here, as that would cause the checker to support parsing config + // that have extraneous, unsupported blocks and attributes. + Locals *terragruntLocal `hcl:"locals,block"` + Include []terragruntIncludeIgnore `hcl:"include,block"` } // We use a struct designed to not parse the block, as locals and includes are parsed and decoded using a special @@ -124,6 +126,7 @@ type terragruntLocal struct { Remain hcl.Body `hcl:",remain"` } type terragruntIncludeIgnore struct { + Name string `hcl:"name,label"` Remain hcl.Body `hcl:",remain"` } diff --git a/config/config_partial.go b/config/config_partial.go index ae6cea8a4b..8f2921c74d 100644 --- a/config/config_partial.go +++ b/config/config_partial.go @@ -1,13 +1,11 @@ package config import ( - "encoding/json" "fmt" "path/filepath" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclparse" - "github.com/hashicorp/hcl/v2/hclwrite" "github.com/zclconf/go-cty/cty" "github.com/gruntwork-io/terragrunt/errors" @@ -333,32 +331,12 @@ func partialParseIncludedConfig(includedConfig *IncludeConfig, terragruntOptions // the config. // For consistency, `include` in the call to `decodeHcl` is always assumed to be nil. Either it really is nil (parsing // the child config), or it shouldn't be used anyway (the parent config shouldn't have an include block). -// -// We take a two pass approach to parsing include blocks to support include blocks without a label. Ideally we can parse -// include blocks with and without labels in a single pass, but the HCL parser is fairly restrictive when it comes to -// parsing blocks with labels, requiring the exact number of expected labels in the parsing step. -// To handle this restriction, we first see if there are any include blocks without any labels, and if there is, we -// modify it in the file object to inject the label as "". func decodeAsTerragruntInclude( file *hcl.File, filename string, terragruntOptions *options.TerragruntOptions, extensions EvalContextExtensions, ) ([]IncludeConfig, error) { - updatedBytes, isUpdated, err := updateBareIncludeBlock(file, filename) - if err != nil { - return nil, err - } - if isUpdated { - // Code was updated, so we need to reparse the new updated contents. This is necessarily because the blocks - // returned by hclparse does not support editing, and so we have to go through hclwrite, which leads to a - // different AST representation. - file, err = parseHcl(hclparse.NewParser(), string(updatedBytes), filename) - if err != nil { - return nil, err - } - } - tgInc := terragruntIncludeMultiple{} if err := decodeHcl(file, filename, &tgInc, terragruntOptions, extensions); err != nil { return nil, err @@ -366,146 +344,6 @@ func decodeAsTerragruntInclude( return tgInc.Include, nil } -// updateBareIncludeBlock searches the parsed terragrunt contents for a bare include block (include without a label), -// and convert it to one with empty string as the label. This is necessary because the hcl parser is strictly enforces -// label counts when parsing out labels with a go struct. -// -// Returns the updated contents, a boolean indicated whether anything changed, and an error (if any). -func updateBareIncludeBlock(file *hcl.File, filename string) ([]byte, bool, error) { - if filepath.Ext(filename) == ".json" { - return updateBareIncludeBlockJSON(file.Bytes) - } - - hclFile, err := hclwrite.ParseConfig(file.Bytes, filename, hcl.InitialPos) - if err != nil { - return nil, false, errors.WithStackTrace(err) - } - - codeWasUpdated := false - for _, block := range hclFile.Body().Blocks() { - if block.Type() == "include" && len(block.Labels()) == 0 { - if codeWasUpdated { - return nil, false, errors.WithStackTrace(MultipleBareIncludeBlocksErr{}) - } - block.SetLabels([]string{""}) - codeWasUpdated = true - } - } - return hclFile.Bytes(), codeWasUpdated, nil -} - -// updateBareIncludeBlockJSON implements the logic for updateBareIncludeBlock when the terragrunt.hcl configuration is -// encoded in json. The json version of this function is fairly complex due to the flexibility in how the blocks are -// encoded. That is, all of the following are valid encodings of a terragrunt.hcl.json file that has a bare include -// block: -// -// Case 1: a single include block as top level: -// { -// "include": { -// "path": "foo" -// } -// } -// -// Case 2: a single include block in list: -// { -// "include": [ -// {"path": "foo"} -// ] -// } -// -// Case 3: mixed bare and labeled include block as list: -// { -// "include": [ -// {"path": "foo"}, -// { -// "labeled": {"path": "bar"} -// } -// ] -// } -// -// For simplicity of implementation, we focus on handling Case 1 and 2, and ignore Case 3. If we see Case 3, we will -// error out. Instead, the user should handle this case explicitly using the object encoding instead of list encoding: -// { -// "include": { -// "": {"path": "foo"}, -// "labeled": {"path": "bar"} -// } -// } -// If the multiple include blocks are encoded in this way in the json configuration, nothing needs to be done by this -// function. -func updateBareIncludeBlockJSON(fileBytes []byte) ([]byte, bool, error) { - var parsed map[string]interface{} - if err := json.Unmarshal(fileBytes, &parsed); err != nil { - return nil, false, errors.WithStackTrace(err) - } - includeBlock, hasKey := parsed["include"] - if !hasKey { - // No include block, so don't do anything - return fileBytes, false, nil - } - switch typed := includeBlock.(type) { - case []interface{}: - if len(typed) == 0 { - // No include block, so don't do anything - return nil, false, nil - } else if len(typed) > 1 { - // Could be multiple bare includes, or Case 3. We simplify the handling of this case by erroring out, - // ignoring the possibility of Case 3, which, while valid HCL encoding, is too complex to detect and handle - // here. Instead we will recommend users use the object encoding. - return nil, false, errors.WithStackTrace(MultipleBareIncludeBlocksErr{}) - } - - // Make sure this is Case 2, and not Case 3 with a single labeled block. If Case 2, update to inject the labeled - // version. Otherwise, return without modifying. - singleBlock := typed[0] - if jsonIsIncludeBlock(singleBlock) { - return updateSingleBareIncludeInParsedJSON(parsed, singleBlock) - } - return nil, false, nil - case map[string]interface{}: - if len(typed) == 0 { - // No include block, so don't do anything - return nil, false, nil - } - - // We will only update the include block if we detect the object to represent an include block. Otherwise, the - // blocks are labeled so we can pass forward to the tg parser step. - if jsonIsIncludeBlock(typed) { - return updateSingleBareIncludeInParsedJSON(parsed, typed) - } - return nil, false, nil - } - - return nil, false, errors.WithStackTrace(IncludeIsNotABlockErr{parsed: includeBlock}) -} - -// updateBareIncludeInParsedJSON replaces the include attribute into a block with the label "" in the json. Note that we -// can directly assign to the map with the single "" key without worrying about the possibility of other include blocks -// since we will only call this function if there is only one include block, and that is a bare block with no labels. -func updateSingleBareIncludeInParsedJSON(parsed map[string]interface{}, newVal interface{}) ([]byte, bool, error) { - parsed["include"] = map[string]interface{}{"": newVal} - updatedBytes, err := json.Marshal(parsed) - return updatedBytes, true, errors.WithStackTrace(err) -} - -// jsonIsIncludeBlock checks if the arbitrary json data is the include block. The data is determined to be an include -// block if: -// - It is an object -// - Has the 'path' attribute -// - The 'path' attribute is a string -func jsonIsIncludeBlock(jsonData interface{}) bool { - typed, isMap := jsonData.(map[string]interface{}) - if isMap { - pathAttr, hasPath := typed["path"] - if hasPath { - _, pathIsString := pathAttr.(string) - return pathIsString - } - return false - } - return false -} - // Custom error types type InvalidPartialBlockName struct { @@ -515,17 +353,3 @@ type InvalidPartialBlockName struct { func (err InvalidPartialBlockName) Error() string { return fmt.Sprintf("Unrecognized partial block code %d. This is most likely an error in terragrunt. Please file a bug report on the project repository.", err.sectionCode) } - -type MultipleBareIncludeBlocksErr struct{} - -func (err MultipleBareIncludeBlocksErr) Error() string { - return "Multiple bare include blocks (include blocks without label) is not supported." -} - -type IncludeIsNotABlockErr struct { - parsed interface{} -} - -func (err IncludeIsNotABlockErr) Error() string { - return fmt.Sprintf("Parsed include is not a block: %v", err.parsed) -} diff --git a/config/hcl_parser.go b/config/hcl_parser.go index 7b98196609..dd89f02f37 100644 --- a/config/hcl_parser.go +++ b/config/hcl_parser.go @@ -40,6 +40,12 @@ func parseHcl(parser *hclparse.Parser, hcl string, filename string) (file *hcl.F } // decodeHcl uses the HCL2 parser to decode the parsed HCL into the struct specified by out. +// +// Note that we take a two pass approach to support parsing include blocks without a label. Ideally we can parse include +// blocks with and without labels in a single pass, but the HCL parser is fairly restrictive when it comes to parsing +// blocks with labels, requiring the exact number of expected labels in the parsing step. To handle this restriction, +// we first see if there are any include blocks without any labels, and if there is, we modify it in the file object to +// inject the label as "". func decodeHcl( file *hcl.File, filename string, @@ -55,6 +61,21 @@ func decodeHcl( } }() + // Check if we need to update the file to label any bare include blocks. + updatedBytes, isUpdated, err := updateBareIncludeBlock(file, filename) + if err != nil { + return err + } + if isUpdated { + // Code was updated, so we need to reparse the new updated contents. This is necessarily because the blocks + // returned by hclparse does not support editing, and so we have to go through hclwrite, which leads to a + // different AST representation. + file, err = parseHcl(hclparse.NewParser(), string(updatedBytes), filename) + if err != nil { + return err + } + } + evalContext, err := CreateTerragruntEvalContext(filename, terragruntOptions, extensions) if err != nil { return err diff --git a/config/include.go b/config/include.go index 7193251360..8121176f19 100644 --- a/config/include.go +++ b/config/include.go @@ -1,10 +1,13 @@ package config import ( + "encoding/json" "fmt" "path/filepath" "strings" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclwrite" "github.com/imdario/mergo" "github.com/zclconf/go-cty/cty" @@ -542,3 +545,159 @@ func getTrackInclude( return &trackInc, nil } } + +// updateBareIncludeBlock searches the parsed terragrunt contents for a bare include block (include without a label), +// and convert it to one with empty string as the label. This is necessary because the hcl parser is strictly enforces +// label counts when parsing out labels with a go struct. +// +// Returns the updated contents, a boolean indicated whether anything changed, and an error (if any). +func updateBareIncludeBlock(file *hcl.File, filename string) ([]byte, bool, error) { + if filepath.Ext(filename) == ".json" { + return updateBareIncludeBlockJSON(file.Bytes) + } + + hclFile, err := hclwrite.ParseConfig(file.Bytes, filename, hcl.InitialPos) + if err != nil { + return nil, false, errors.WithStackTrace(err) + } + + codeWasUpdated := false + for _, block := range hclFile.Body().Blocks() { + if block.Type() == "include" && len(block.Labels()) == 0 { + if codeWasUpdated { + return nil, false, errors.WithStackTrace(MultipleBareIncludeBlocksErr{}) + } + block.SetLabels([]string{""}) + codeWasUpdated = true + } + } + return hclFile.Bytes(), codeWasUpdated, nil +} + +// updateBareIncludeBlockJSON implements the logic for updateBareIncludeBlock when the terragrunt.hcl configuration is +// encoded in json. The json version of this function is fairly complex due to the flexibility in how the blocks are +// encoded. That is, all of the following are valid encodings of a terragrunt.hcl.json file that has a bare include +// block: +// +// Case 1: a single include block as top level: +// { +// "include": { +// "path": "foo" +// } +// } +// +// Case 2: a single include block in list: +// { +// "include": [ +// {"path": "foo"} +// ] +// } +// +// Case 3: mixed bare and labeled include block as list: +// { +// "include": [ +// {"path": "foo"}, +// { +// "labeled": {"path": "bar"} +// } +// ] +// } +// +// For simplicity of implementation, we focus on handling Case 1 and 2, and ignore Case 3. If we see Case 3, we will +// error out. Instead, the user should handle this case explicitly using the object encoding instead of list encoding: +// { +// "include": { +// "": {"path": "foo"}, +// "labeled": {"path": "bar"} +// } +// } +// If the multiple include blocks are encoded in this way in the json configuration, nothing needs to be done by this +// function. +func updateBareIncludeBlockJSON(fileBytes []byte) ([]byte, bool, error) { + var parsed map[string]interface{} + if err := json.Unmarshal(fileBytes, &parsed); err != nil { + return nil, false, errors.WithStackTrace(err) + } + includeBlock, hasKey := parsed["include"] + if !hasKey { + // No include block, so don't do anything + return fileBytes, false, nil + } + switch typed := includeBlock.(type) { + case []interface{}: + if len(typed) == 0 { + // No include block, so don't do anything + return nil, false, nil + } else if len(typed) > 1 { + // Could be multiple bare includes, or Case 3. We simplify the handling of this case by erroring out, + // ignoring the possibility of Case 3, which, while valid HCL encoding, is too complex to detect and handle + // here. Instead we will recommend users use the object encoding. + return nil, false, errors.WithStackTrace(MultipleBareIncludeBlocksErr{}) + } + + // Make sure this is Case 2, and not Case 3 with a single labeled block. If Case 2, update to inject the labeled + // version. Otherwise, return without modifying. + singleBlock := typed[0] + if jsonIsIncludeBlock(singleBlock) { + return updateSingleBareIncludeInParsedJSON(parsed, singleBlock) + } + return nil, false, nil + case map[string]interface{}: + if len(typed) == 0 { + // No include block, so don't do anything + return nil, false, nil + } + + // We will only update the include block if we detect the object to represent an include block. Otherwise, the + // blocks are labeled so we can pass forward to the tg parser step. + if jsonIsIncludeBlock(typed) { + return updateSingleBareIncludeInParsedJSON(parsed, typed) + } + return nil, false, nil + } + + return nil, false, errors.WithStackTrace(IncludeIsNotABlockErr{parsed: includeBlock}) +} + +// updateBareIncludeInParsedJSON replaces the include attribute into a block with the label "" in the json. Note that we +// can directly assign to the map with the single "" key without worrying about the possibility of other include blocks +// since we will only call this function if there is only one include block, and that is a bare block with no labels. +func updateSingleBareIncludeInParsedJSON(parsed map[string]interface{}, newVal interface{}) ([]byte, bool, error) { + parsed["include"] = map[string]interface{}{"": newVal} + updatedBytes, err := json.Marshal(parsed) + return updatedBytes, true, errors.WithStackTrace(err) +} + +// jsonIsIncludeBlock checks if the arbitrary json data is the include block. The data is determined to be an include +// block if: +// - It is an object +// - Has the 'path' attribute +// - The 'path' attribute is a string +func jsonIsIncludeBlock(jsonData interface{}) bool { + typed, isMap := jsonData.(map[string]interface{}) + if isMap { + pathAttr, hasPath := typed["path"] + if hasPath { + _, pathIsString := pathAttr.(string) + return pathIsString + } + return false + } + return false +} + +// Custom error types + +type MultipleBareIncludeBlocksErr struct{} + +func (err MultipleBareIncludeBlocksErr) Error() string { + return "Multiple bare include blocks (include blocks without label) is not supported." +} + +type IncludeIsNotABlockErr struct { + parsed interface{} +} + +func (err IncludeIsNotABlockErr) Error() string { + return fmt.Sprintf("Parsed include is not a block: %v", err.parsed) +} diff --git a/test/fixture-include-multiple/deep-merge-nonoverlapping/child/terragrunt.hcl b/test/fixture-include-multiple/deep-merge-nonoverlapping/child/terragrunt.hcl new file mode 100644 index 0000000000..347f188a91 --- /dev/null +++ b/test/fixture-include-multiple/deep-merge-nonoverlapping/child/terragrunt.hcl @@ -0,0 +1,36 @@ +terraform { + source = "${get_terragrunt_dir()}/../../modules/reflect" +} + +include "inputs" { + path = find_in_parent_folders("terragrunt_inputs.hcl") + merge_strategy = "deep" +} + +include "vpc_dep" { + path = find_in_parent_folders("terragrunt_vpc_dep.hcl") + merge_strategy = "deep" +} + +dependency "vpc" { + config_path = "../vpc" + mock_outputs = { + attribute = "mock" + new_attribute = "new val" + list_attr = ["mock", "foo"] + map_attr = { + bar = "baz" + } + } +} + +inputs = { + attribute = "mock" + new_attribute = "new val" + list_attr = ["mock", "foo"] + map_attr = { + bar = "baz" + } + + dep_out = dependency.vpc.outputs +} diff --git a/test/fixture-include-multiple/deep-merge-nonoverlapping/vpc/terragrunt.hcl b/test/fixture-include-multiple/deep-merge-nonoverlapping/vpc/terragrunt.hcl new file mode 100644 index 0000000000..ce9847b76e --- /dev/null +++ b/test/fixture-include-multiple/deep-merge-nonoverlapping/vpc/terragrunt.hcl @@ -0,0 +1,3 @@ +terraform { + source = "${get_terragrunt_dir()}/../../modules/empty" +} diff --git a/test/fixture-include-multiple/deep-merge-overlapping/child/terragrunt.hcl b/test/fixture-include-multiple/deep-merge-overlapping/child/terragrunt.hcl new file mode 100644 index 0000000000..aab08eadde --- /dev/null +++ b/test/fixture-include-multiple/deep-merge-overlapping/child/terragrunt.hcl @@ -0,0 +1,38 @@ +terraform { + source = "${get_terragrunt_dir()}/../../modules/reflect" +} + +include "inputs" { + path = find_in_parent_folders("terragrunt_inputs.hcl") + merge_strategy = "deep" +} + +include "vpc_dep" { + path = find_in_parent_folders("terragrunt_vpc_dep.hcl") + merge_strategy = "deep" +} + +include "inputs_override" { + path = find_in_parent_folders("terragrunt_inputs_override.hcl") + merge_strategy = "deep" +} + +include "vpc_dep_override" { + path = find_in_parent_folders("terragrunt_vpc_dep_override.hcl") + merge_strategy = "deep" +} + +dependency "vpc" { + config_path = "../vpc" + mock_outputs = { + attribute = "mock" + list_attr = ["foo"] + } +} + +inputs = { + attribute = "mock" + list_attr = ["foo"] + + dep_out = dependency.vpc.outputs +} diff --git a/test/fixture-include-multiple/deep-merge-overlapping/vpc/terragrunt.hcl b/test/fixture-include-multiple/deep-merge-overlapping/vpc/terragrunt.hcl new file mode 100644 index 0000000000..ce9847b76e --- /dev/null +++ b/test/fixture-include-multiple/deep-merge-overlapping/vpc/terragrunt.hcl @@ -0,0 +1,3 @@ +terraform { + source = "${get_terragrunt_dir()}/../../modules/empty" +} diff --git a/test/fixture-include-multiple/modules/empty/main.tf b/test/fixture-include-multiple/modules/empty/main.tf new file mode 100644 index 0000000000..bb7b160deb --- /dev/null +++ b/test/fixture-include-multiple/modules/empty/main.tf @@ -0,0 +1 @@ +# Intentionally empty diff --git a/test/fixture-include-multiple/modules/reflect/main.tf b/test/fixture-include-multiple/modules/reflect/main.tf new file mode 100644 index 0000000000..772fb1da2c --- /dev/null +++ b/test/fixture-include-multiple/modules/reflect/main.tf @@ -0,0 +1,47 @@ +variable "attribute" { + type = string +} + +variable "new_attribute" { + type = string +} + +variable "old_attribute" { + type = string +} + +variable "list_attr" { + type = list(string) +} + +variable "map_attr" { + type = map(string) +} + +variable "dep_out" { + type = any +} + +output "attribute" { + value = var.attribute +} + +output "new_attribute" { + value = var.new_attribute +} + +output "old_attribute" { + value = var.old_attribute +} + +output "list_attr" { + value = var.list_attr +} + +output "map_attr" { + value = var.map_attr +} + +output "dep_out" { + value = var.dep_out +} diff --git a/test/fixture-include-multiple/terragrunt_inputs.hcl b/test/fixture-include-multiple/terragrunt_inputs.hcl new file mode 100644 index 0000000000..1d7c64b4f6 --- /dev/null +++ b/test/fixture-include-multiple/terragrunt_inputs.hcl @@ -0,0 +1,9 @@ +inputs = { + attribute = "hello" + old_attribute = "old val" + list_attr = ["hello"] + map_attr = { + foo = "bar" + test = dependency.vpc.outputs.new_attribute + } +} diff --git a/test/fixture-include-multiple/terragrunt_inputs_override.hcl b/test/fixture-include-multiple/terragrunt_inputs_override.hcl new file mode 100644 index 0000000000..e96483d918 --- /dev/null +++ b/test/fixture-include-multiple/terragrunt_inputs_override.hcl @@ -0,0 +1,8 @@ +inputs = { + attribute = "will be replaced" + new_attribute = "new val" + list_attr = ["mock"] + map_attr = { + bar = "baz" + } +} diff --git a/test/fixture-include-multiple/terragrunt_vpc_dep.hcl b/test/fixture-include-multiple/terragrunt_vpc_dep.hcl new file mode 100644 index 0000000000..3275c95150 --- /dev/null +++ b/test/fixture-include-multiple/terragrunt_vpc_dep.hcl @@ -0,0 +1,14 @@ +dependency "vpc" { + # This will get overridden by child terragrunt.hcl configs + config_path = "" + + mock_outputs = { + attribute = "hello" + old_attribute = "old val" + list_attr = ["hello"] + map_attr = { + foo = "bar" + } + } + mock_outputs_allowed_terraform_commands = ["apply", "plan", "destroy", "output"] +} diff --git a/test/fixture-include-multiple/terragrunt_vpc_dep_override.hcl b/test/fixture-include-multiple/terragrunt_vpc_dep_override.hcl new file mode 100644 index 0000000000..e8a5b90d9f --- /dev/null +++ b/test/fixture-include-multiple/terragrunt_vpc_dep_override.hcl @@ -0,0 +1,11 @@ +dependency "vpc" { + config_path = "" + mock_outputs = { + attribute = "will be replaced" + new_attribute = "new val" + list_attr = ["mock"] + map_attr = { + bar = "baz" + } + } +} diff --git a/test/integration_include_test.go b/test/integration_include_test.go index 0128bd2ac4..be47c7d114 100644 --- a/test/integration_include_test.go +++ b/test/integration_include_test.go @@ -4,6 +4,8 @@ import ( "bytes" "encoding/json" "fmt" + "io/ioutil" + "path/filepath" "strings" "testing" @@ -21,6 +23,7 @@ const ( includeNoMergeFixturePath = "qa/my-app" includeExposeFixturePath = "fixture-include-expose/" includeChildFixturePath = "child" + includeMultipleFixturePath = "fixture-include-multiple/" ) func TestTerragruntWorksWithIncludeLocals(t *testing.T) { @@ -108,6 +111,65 @@ func TestTerragruntWorksWithIncludeDeepMerge(t *testing.T) { ) } +func TestTerragruntWorksWithMultipleInclude(t *testing.T) { + t.Parallel() + + files, err := ioutil.ReadDir(includeMultipleFixturePath) + require.NoError(t, err) + + testCases := []string{} + for _, finfo := range files { + if finfo.IsDir() && filepath.Base(finfo.Name()) != "modules" { + testCases = append(testCases, finfo.Name()) + } + } + + for _, testCase := range testCases { + // Capture range variable to avoid it changing across parallel test runs + testCase := testCase + + t.Run(filepath.Base(testCase), func(t *testing.T) { + t.Parallel() + + childPath := filepath.Join(includeMultipleFixturePath, testCase, includeDeepFixtureChildPath) + cleanupTerraformFolder(t, childPath) + runTerragrunt(t, fmt.Sprintf("terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-log-level debug --terragrunt-working-dir %s", childPath)) + + stdout := bytes.Buffer{} + stderr := bytes.Buffer{} + err := runTerragruntCommand(t, fmt.Sprintf("terragrunt output -no-color -json --terragrunt-non-interactive --terragrunt-log-level debug --terragrunt-working-dir %s", childPath), &stdout, &stderr) + require.NoError(t, err) + + outputs := map[string]TerraformOutput{} + require.NoError(t, json.Unmarshal([]byte(stdout.String()), &outputs)) + validateMultipleIncludeTestOutput(t, outputs) + }) + } +} + +func validateMultipleIncludeTestOutput(t *testing.T, outputs map[string]TerraformOutput) { + assert.Equal(t, "mock", outputs["attribute"].Value.(string)) + assert.Equal(t, "new val", outputs["new_attribute"].Value.(string)) + assert.Equal(t, "old val", outputs["old_attribute"].Value.(string)) + assert.Equal(t, []interface{}{"hello", "mock", "foo"}, outputs["list_attr"].Value.([]interface{})) + assert.Equal(t, map[string]interface{}{"foo": "bar", "bar": "baz", "test": "new val"}, outputs["map_attr"].Value.(map[string]interface{})) + + assert.Equal( + t, + map[string]interface{}{ + "attribute": "mock", + "new_attribute": "new val", + "old_attribute": "old val", + "list_attr": []interface{}{"hello", "mock", "foo"}, + "map_attr": map[string]interface{}{ + "foo": "bar", + "bar": "baz", + }, + }, + outputs["dep_out"].Value.(map[string]interface{}), + ) +} + func validateIncludeRemoteStateReflection(t *testing.T, s3BucketName string, keyPath string, configPath string, workingDir string) { stdout := bytes.Buffer{} stderr := bytes.Buffer{} From ee500e9634c93377efab19964a4df18f56a82499 Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Fri, 10 Sep 2021 15:21:50 -0500 Subject: [PATCH 15/26] Add comprehensive testing of multiple includes --- .../expose/child/terragrunt.hcl | 47 +++++++++++++++++++ .../expose/vpc/terragrunt.hcl | 3 ++ .../has-bare-include/child/terragrunt.hcl | 36 ++++++++++++++ .../has-bare-include/vpc/terragrunt.hcl | 3 ++ .../child/terragrunt.hcl | 43 +++++++++++++++++ .../vpc/terragrunt.hcl | 3 ++ .../shallow-merge/child/terragrunt.hcl | 33 +++++++++++++ .../shallow-merge/vpc/terragrunt.hcl | 3 ++ .../terragrunt_inputs.hcl | 2 +- .../terragrunt_inputs_final.hcl | 10 ++++ .../terragrunt_vpc_dep_for_expose.hcl | 8 ++++ 11 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 test/fixture-include-multiple/expose/child/terragrunt.hcl create mode 100644 test/fixture-include-multiple/expose/vpc/terragrunt.hcl create mode 100644 test/fixture-include-multiple/has-bare-include/child/terragrunt.hcl create mode 100644 test/fixture-include-multiple/has-bare-include/vpc/terragrunt.hcl create mode 100644 test/fixture-include-multiple/shallow-deep-merge-overlapping/child/terragrunt.hcl create mode 100644 test/fixture-include-multiple/shallow-deep-merge-overlapping/vpc/terragrunt.hcl create mode 100644 test/fixture-include-multiple/shallow-merge/child/terragrunt.hcl create mode 100644 test/fixture-include-multiple/shallow-merge/vpc/terragrunt.hcl create mode 100644 test/fixture-include-multiple/terragrunt_inputs_final.hcl create mode 100644 test/fixture-include-multiple/terragrunt_vpc_dep_for_expose.hcl diff --git a/test/fixture-include-multiple/expose/child/terragrunt.hcl b/test/fixture-include-multiple/expose/child/terragrunt.hcl new file mode 100644 index 0000000000..6e85a75b32 --- /dev/null +++ b/test/fixture-include-multiple/expose/child/terragrunt.hcl @@ -0,0 +1,47 @@ +terraform { + source = "${get_terragrunt_dir()}/../../modules/reflect" +} + +include "inputs_override" { + path = find_in_parent_folders("terragrunt_inputs_override.hcl") + expose = true + merge_strategy = "no_merge" +} + +include "vpc_dep" { + path = find_in_parent_folders("terragrunt_vpc_dep_for_expose.hcl") + expose = true + merge_strategy = "no_merge" +} + +dependency "vpc" { + config_path = include.vpc_dep.dependency.vpc.config_path + mock_outputs = merge( + include.vpc_dep.dependency.vpc.mock_outputs, + { + attribute = "mock" + new_attribute = "new val" + list_attr = ["hello", "mock", "foo"] + map_attr = { + foo = "bar" + bar = "baz" + } + }, + ) + mock_outputs_allowed_terraform_commands = include.vpc_dep.dependency.vpc.mock_outputs_allowed_terraform_commands +} + +inputs = merge( + include.inputs_override.inputs, + { + attribute = "mock" + old_attribute = "old val" + list_attr = ["hello", "mock", "foo"] + map_attr = { + bar = "baz" + foo = "bar" + test = dependency.vpc.outputs.new_attribute + } + dep_out = dependency.vpc.outputs + }, +) diff --git a/test/fixture-include-multiple/expose/vpc/terragrunt.hcl b/test/fixture-include-multiple/expose/vpc/terragrunt.hcl new file mode 100644 index 0000000000..ce9847b76e --- /dev/null +++ b/test/fixture-include-multiple/expose/vpc/terragrunt.hcl @@ -0,0 +1,3 @@ +terraform { + source = "${get_terragrunt_dir()}/../../modules/empty" +} diff --git a/test/fixture-include-multiple/has-bare-include/child/terragrunt.hcl b/test/fixture-include-multiple/has-bare-include/child/terragrunt.hcl new file mode 100644 index 0000000000..d90d5a58e3 --- /dev/null +++ b/test/fixture-include-multiple/has-bare-include/child/terragrunt.hcl @@ -0,0 +1,36 @@ +terraform { + source = "${get_terragrunt_dir()}/../../modules/reflect" +} + +include { + path = find_in_parent_folders("terragrunt_inputs.hcl") + merge_strategy = "deep" +} + +include "vpc_dep" { + path = find_in_parent_folders("terragrunt_vpc_dep.hcl") + merge_strategy = "deep" +} + +dependency "vpc" { + config_path = "../vpc" + mock_outputs = { + attribute = "mock" + new_attribute = "new val" + list_attr = ["mock", "foo"] + map_attr = { + bar = "baz" + } + } +} + +inputs = { + attribute = "mock" + new_attribute = "new val" + list_attr = ["mock", "foo"] + map_attr = { + bar = "baz" + } + + dep_out = dependency.vpc.outputs +} diff --git a/test/fixture-include-multiple/has-bare-include/vpc/terragrunt.hcl b/test/fixture-include-multiple/has-bare-include/vpc/terragrunt.hcl new file mode 100644 index 0000000000..ce9847b76e --- /dev/null +++ b/test/fixture-include-multiple/has-bare-include/vpc/terragrunt.hcl @@ -0,0 +1,3 @@ +terraform { + source = "${get_terragrunt_dir()}/../../modules/empty" +} diff --git a/test/fixture-include-multiple/shallow-deep-merge-overlapping/child/terragrunt.hcl b/test/fixture-include-multiple/shallow-deep-merge-overlapping/child/terragrunt.hcl new file mode 100644 index 0000000000..1d5be3a224 --- /dev/null +++ b/test/fixture-include-multiple/shallow-deep-merge-overlapping/child/terragrunt.hcl @@ -0,0 +1,43 @@ +terraform { + source = "${get_terragrunt_dir()}/../../modules/reflect" +} + +include "inputs" { + path = find_in_parent_folders("terragrunt_inputs.hcl") + merge_strategy = "deep" +} + +include "inputs_override" { + path = find_in_parent_folders("terragrunt_inputs_override.hcl") +} + +# NOTE: This shallow merge is expected to be a noop, as the deep merge between vpc_dep and the child config completes +# the expected dependency.vpc block. +include "vpc_dep_override" { + path = find_in_parent_folders("terragrunt_vpc_dep_override.hcl") +} + +include "vpc_dep" { + path = find_in_parent_folders("terragrunt_vpc_dep.hcl") + merge_strategy = "deep" +} + + +dependency "vpc" { + config_path = "../vpc" + mock_outputs = { + attribute = "mock" + new_attribute = "new val" + list_attr = ["mock", "foo"] + map_attr = { + bar = "baz" + } + } +} + +inputs = { + attribute = "mock" + list_attr = ["mock", "foo"] + + dep_out = dependency.vpc.outputs +} diff --git a/test/fixture-include-multiple/shallow-deep-merge-overlapping/vpc/terragrunt.hcl b/test/fixture-include-multiple/shallow-deep-merge-overlapping/vpc/terragrunt.hcl new file mode 100644 index 0000000000..ce9847b76e --- /dev/null +++ b/test/fixture-include-multiple/shallow-deep-merge-overlapping/vpc/terragrunt.hcl @@ -0,0 +1,3 @@ +terraform { + source = "${get_terragrunt_dir()}/../../modules/empty" +} diff --git a/test/fixture-include-multiple/shallow-merge/child/terragrunt.hcl b/test/fixture-include-multiple/shallow-merge/child/terragrunt.hcl new file mode 100644 index 0000000000..554e5793be --- /dev/null +++ b/test/fixture-include-multiple/shallow-merge/child/terragrunt.hcl @@ -0,0 +1,33 @@ +terraform { + source = "${get_terragrunt_dir()}/../../modules/reflect" +} + +include "inputs" { + path = find_in_parent_folders("terragrunt_inputs.hcl") +} + +include "inputs_final" { + path = find_in_parent_folders("terragrunt_inputs_final.hcl") +} + +include "vpc_dep" { + path = find_in_parent_folders("terragrunt_vpc_dep.hcl") +} + +dependency "vpc" { + config_path = "../vpc" + mock_outputs = { + attribute = "mock" + old_attribute = "old val" + new_attribute = "new val" + list_attr = ["hello", "mock", "foo"] + map_attr = { + foo = "bar" + bar = "baz" + } + } +} + +inputs = { + dep_out = dependency.vpc.outputs +} diff --git a/test/fixture-include-multiple/shallow-merge/vpc/terragrunt.hcl b/test/fixture-include-multiple/shallow-merge/vpc/terragrunt.hcl new file mode 100644 index 0000000000..ce9847b76e --- /dev/null +++ b/test/fixture-include-multiple/shallow-merge/vpc/terragrunt.hcl @@ -0,0 +1,3 @@ +terraform { + source = "${get_terragrunt_dir()}/../../modules/empty" +} diff --git a/test/fixture-include-multiple/terragrunt_inputs.hcl b/test/fixture-include-multiple/terragrunt_inputs.hcl index 1d7c64b4f6..9d0bcf84d6 100644 --- a/test/fixture-include-multiple/terragrunt_inputs.hcl +++ b/test/fixture-include-multiple/terragrunt_inputs.hcl @@ -3,7 +3,7 @@ inputs = { old_attribute = "old val" list_attr = ["hello"] map_attr = { - foo = "bar" + foo = "bar" test = dependency.vpc.outputs.new_attribute } } diff --git a/test/fixture-include-multiple/terragrunt_inputs_final.hcl b/test/fixture-include-multiple/terragrunt_inputs_final.hcl new file mode 100644 index 0000000000..03e875e9f7 --- /dev/null +++ b/test/fixture-include-multiple/terragrunt_inputs_final.hcl @@ -0,0 +1,10 @@ +inputs = { + attribute = "mock" + new_attribute = "new val" + list_attr = ["hello", "mock", "foo"] + map_attr = { + bar = "baz" + foo = "bar" + test = dependency.vpc.outputs.new_attribute + } +} diff --git a/test/fixture-include-multiple/terragrunt_vpc_dep_for_expose.hcl b/test/fixture-include-multiple/terragrunt_vpc_dep_for_expose.hcl new file mode 100644 index 0000000000..8affdb8230 --- /dev/null +++ b/test/fixture-include-multiple/terragrunt_vpc_dep_for_expose.hcl @@ -0,0 +1,8 @@ +dependency "vpc" { + config_path = "${get_terragrunt_dir()}/../vpc" + mock_outputs = { + attribute = "hello" + old_attribute = "old val" + } + mock_outputs_allowed_terraform_commands = ["apply", "plan", "destroy", "output"] +} From 2489e2e1c55cec5dbb76ce5f96e435b6d1348c0c Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Fri, 10 Sep 2021 16:27:14 -0500 Subject: [PATCH 16/26] Add json test for multiple includes --- .../json/child/terragrunt.hcl.json | 37 +++++++++++++++++++ .../json/vpc/terragrunt.hcl | 3 ++ 2 files changed, 40 insertions(+) create mode 100644 test/fixture-include-multiple/json/child/terragrunt.hcl.json create mode 100644 test/fixture-include-multiple/json/vpc/terragrunt.hcl diff --git a/test/fixture-include-multiple/json/child/terragrunt.hcl.json b/test/fixture-include-multiple/json/child/terragrunt.hcl.json new file mode 100644 index 0000000000..289eb6535d --- /dev/null +++ b/test/fixture-include-multiple/json/child/terragrunt.hcl.json @@ -0,0 +1,37 @@ +{ + "terraform": { + "source": "${get_terragrunt_dir()}/../../modules/reflect" + }, + "include": { + "": { + "path": "${find_in_parent_folders(\"terragrunt_inputs.hcl\")}", + "merge_strategy": "deep" + }, + "vpc_dep": { + "path": "${find_in_parent_folders(\"terragrunt_vpc_dep.hcl\")}", + "merge_strategy": "deep" + } + }, + "dependency": { + "vpc": { + "config_path": "../vpc", + "mock_outputs": { + "attribute": "mock", + "new_attribute": "new val", + "list_attr": ["mock", "foo"], + "map_attr": { + "bar": "baz" + } + } + } + }, + "inputs": { + "attribute": "mock", + "new_attribute": "new val", + "list_attr": ["mock", "foo"], + "map_attr": { + "bar": "baz" + }, + "dep_out": "${dependency.vpc.outputs}" + } +} diff --git a/test/fixture-include-multiple/json/vpc/terragrunt.hcl b/test/fixture-include-multiple/json/vpc/terragrunt.hcl new file mode 100644 index 0000000000..ce9847b76e --- /dev/null +++ b/test/fixture-include-multiple/json/vpc/terragrunt.hcl @@ -0,0 +1,3 @@ +terraform { + source = "${get_terragrunt_dir()}/../../modules/empty" +} From d90638555cc2ddf38d3803fee6db531ea82b1b4b Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Fri, 10 Sep 2021 16:42:52 -0500 Subject: [PATCH 17/26] Require label when referencing exposed include --- config/config_helpers.go | 30 +++---------------- config/cty_helpers.go | 19 ++++++++++++ .../config-blocks-and-attributes.md | 6 ++-- .../child/terragrunt.hcl | 2 +- test/fixture-include/qa/my-app/terragrunt.hcl | 2 +- .../stage/my-app/terragrunt.hcl | 2 +- 6 files changed, 28 insertions(+), 33 deletions(-) diff --git a/config/config_helpers.go b/config/config_helpers.go index eaaa7a7637..7916ff49b7 100644 --- a/config/config_helpers.go +++ b/config/config_helpers.go @@ -154,33 +154,11 @@ func CreateTerragruntEvalContext( if extensions.TrackInclude != nil && len(extensions.TrackInclude.CurrentList) > 0 { // For each include block, check if we want to expose the included config, and if so, add under the include // variable. - exposedIncludeMap := map[string]cty.Value{} - for key, included := range extensions.TrackInclude.CurrentMap { - if included.GetExpose() { - parsedIncluded, err := parseIncludedConfig(&included, terragruntOptions, extensions.DecodedDependencies) - if err != nil { - return ctx, err - } - parsedIncludedCty, err := terragruntConfigAsCty(parsedIncluded) - if err != nil { - return ctx, err - } - exposedIncludeMap[key] = parsedIncludedCty - } - } - - if len(exposedIncludeMap) == 1 { - // If we have only one exposed include map, then flatten the map as a shorthand - for _, val := range exposedIncludeMap { - ctx.Variables["include"] = val - } - } else { - var err error - ctx.Variables["include"], err = convertValuesMapToCtyVal(exposedIncludeMap) - if err != nil { - return ctx, err - } + exposedInclude, err := includeMapAsCtyVal(extensions.TrackInclude.CurrentMap, terragruntOptions, extensions.DecodedDependencies) + if err != nil { + return ctx, err } + ctx.Variables["include"] = exposedInclude } return ctx, nil } diff --git a/config/cty_helpers.go b/config/cty_helpers.go index 75856c7622..a33254efc6 100644 --- a/config/cty_helpers.go +++ b/config/cty_helpers.go @@ -189,3 +189,22 @@ func generateTypeFromValuesMap(valMap map[string]cty.Value) cty.Type { } return cty.Object(outType) } + +// includeMapAsCtyVal converts the include map into a cty.Value struct that can be exposed to the child config. +func includeMapAsCtyVal(includeMap map[string]IncludeConfig, terragruntOptions *options.TerragruntOptions, decodedDependencies *cty.Value) (cty.Value, error) { + exposedIncludeMap := map[string]cty.Value{} + for key, included := range includeMap { + if included.GetExpose() { + parsedIncluded, err := parseIncludedConfig(&included, terragruntOptions, decodedDependencies) + if err != nil { + return cty.NilVal, err + } + parsedIncludedCty, err := terragruntConfigAsCty(parsedIncluded) + if err != nil { + return cty.NilVal, err + } + exposedIncludeMap[key] = parsedIncludedCty + } + } + return convertValuesMapToCtyVal(exposedIncludeMap) +} diff --git a/docs/_docs/04_reference/config-blocks-and-attributes.md b/docs/_docs/04_reference/config-blocks-and-attributes.md index 4116000754..324cc8da5a 100644 --- a/docs/_docs/04_reference/config-blocks-and-attributes.md +++ b/docs/_docs/04_reference/config-blocks-and-attributes.md @@ -389,9 +389,7 @@ no label (`include {}`) is a short hand for an `include` block that uses the lab with this configuration (the `child` config). - `expose` (attribute, optional): Specifies whether or not the included config should be parsed and exposed as a variable. When `true`, you can reference the data of the included config under the variable `include`. Defaults to - `false`. Note that the `include` variable is a map of `include` labels to the parsed configuration value when there are - multiple `include` blocks, while it will be the actual parsed configuration value when there is only one - `include` block. + `false`. Note that the `include` variable is a map of `include` labels to the parsed configuration value. - `merge_strategy` (attribute, optional): Specifies how the included config should be merged. Valid values are: `no_merge` (do not merge the included config), `shallow` (do a shallow merge - default), `deep` (do a deep merge of the included config). @@ -413,7 +411,7 @@ include { } inputs = { - remote_state_config = include.remote_state + remote_state_config = include[""].remote_state } ``` diff --git a/test/fixture-include-expose/child/terragrunt.hcl b/test/fixture-include-expose/child/terragrunt.hcl index 3be01a4fec..7e989135f4 100644 --- a/test/fixture-include-expose/child/terragrunt.hcl +++ b/test/fixture-include-expose/child/terragrunt.hcl @@ -5,7 +5,7 @@ include { locals { environment = "test" - parent_region = "${include.locals.region}-${local.environment}" + parent_region = "${include[""].locals.region}-${local.environment}" } inputs = { diff --git a/test/fixture-include/qa/my-app/terragrunt.hcl b/test/fixture-include/qa/my-app/terragrunt.hcl index 4b7a2ec110..6617d250f5 100644 --- a/test/fixture-include/qa/my-app/terragrunt.hcl +++ b/test/fixture-include/qa/my-app/terragrunt.hcl @@ -7,5 +7,5 @@ include { } inputs = { - reflect = include.remote_state + reflect = include[""].remote_state } diff --git a/test/fixture-include/stage/my-app/terragrunt.hcl b/test/fixture-include/stage/my-app/terragrunt.hcl index ff6cfe065e..bca887680a 100644 --- a/test/fixture-include/stage/my-app/terragrunt.hcl +++ b/test/fixture-include/stage/my-app/terragrunt.hcl @@ -4,5 +4,5 @@ include { } inputs = { - reflect = include.remote_state + reflect = include[""].remote_state } From 355c354dc8c2417968d39bb10f4c25abc883d8a8 Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Mon, 13 Sep 2021 09:03:50 -0500 Subject: [PATCH 18/26] Be clear that nested includes are not supported yet --- docs/_docs/04_reference/config-blocks-and-attributes.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/_docs/04_reference/config-blocks-and-attributes.md b/docs/_docs/04_reference/config-blocks-and-attributes.md index 324cc8da5a..4296d2683f 100644 --- a/docs/_docs/04_reference/config-blocks-and-attributes.md +++ b/docs/_docs/04_reference/config-blocks-and-attributes.md @@ -394,6 +394,11 @@ no label (`include {}`) is a short hand for an `include` block that uses the lab `no_merge` (do not merge the included config), `shallow` (do a shallow merge - default), `deep` (do a deep merge of the included config). +**NOTE**: At this time, Terragrunt only supports a single level of `include` blocks. That is, Terragrunt will error out +if an included config also has an `include` block defined. If you are interested in this feature, please follow +https://github.com/gruntwork-io/terragrunt/issues/1566 to be notified when nested `include` blocks are supported. + + Examples: _Single include_ From afb9d2f199dcf56d405f475fe96d355155a39c49 Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Mon, 13 Sep 2021 09:15:46 -0500 Subject: [PATCH 19/26] Support backward compatible interface where single bare include exposes at top level --- config/cty_helpers.go | 42 ++++++++++++++----- config/include.go | 6 ++- .../config-blocks-and-attributes.md | 4 +- .../{ => mixed-with-bare}/child/main.tf | 0 .../mixed-with-bare/child/terragrunt.hcl | 17 ++++++++ .../multiple/child/main.tf | 5 +++ .../multiple/child/terragrunt.hcl | 17 ++++++++ .../single-bare/child/main.tf | 5 +++ .../single-bare/child/terragrunt.hcl | 13 ++++++ .../single/child/main.tf | 5 +++ .../{ => single}/child/terragrunt.hcl | 4 +- .../fixture-include-expose/terragrunt_env.hcl | 3 ++ test/fixture-include/qa/my-app/terragrunt.hcl | 4 +- .../stage/my-app/terragrunt.hcl | 4 +- test/integration_include_test.go | 37 +++++++++++----- 15 files changed, 136 insertions(+), 30 deletions(-) rename test/fixture-include-expose/{ => mixed-with-bare}/child/main.tf (100%) create mode 100644 test/fixture-include-expose/mixed-with-bare/child/terragrunt.hcl create mode 100644 test/fixture-include-expose/multiple/child/main.tf create mode 100644 test/fixture-include-expose/multiple/child/terragrunt.hcl create mode 100644 test/fixture-include-expose/single-bare/child/main.tf create mode 100644 test/fixture-include-expose/single-bare/child/terragrunt.hcl create mode 100644 test/fixture-include-expose/single/child/main.tf rename test/fixture-include-expose/{ => single}/child/terragrunt.hcl (60%) create mode 100644 test/fixture-include-expose/terragrunt_env.hcl diff --git a/config/cty_helpers.go b/config/cty_helpers.go index a33254efc6..3aa103d7fa 100644 --- a/config/cty_helpers.go +++ b/config/cty_helpers.go @@ -190,21 +190,43 @@ func generateTypeFromValuesMap(valMap map[string]cty.Value) cty.Type { return cty.Object(outType) } -// includeMapAsCtyVal converts the include map into a cty.Value struct that can be exposed to the child config. +// includeMapAsCtyVal converts the include map into a cty.Value struct that can be exposed to the child config. For +// backward compatibility, this function will return the included config object if the config only defines a single bare +// include block that is exposed. func includeMapAsCtyVal(includeMap map[string]IncludeConfig, terragruntOptions *options.TerragruntOptions, decodedDependencies *cty.Value) (cty.Value, error) { + bareInclude, hasBareInclude := includeMap[bareIncludeKey] + if len(includeMap) == 1 && hasBareInclude { + terragruntOptions.Logger.Debug("Detected single bare include block - exposing as top level") + return includeConfigAsCtyVal(bareInclude, terragruntOptions, decodedDependencies) + } + exposedIncludeMap := map[string]cty.Value{} for key, included := range includeMap { - if included.GetExpose() { - parsedIncluded, err := parseIncludedConfig(&included, terragruntOptions, decodedDependencies) - if err != nil { - return cty.NilVal, err - } - parsedIncludedCty, err := terragruntConfigAsCty(parsedIncluded) - if err != nil { - return cty.NilVal, err - } + parsedIncludedCty, err := includeConfigAsCtyVal(included, terragruntOptions, decodedDependencies) + if err != nil { + return cty.NilVal, err + } + if parsedIncludedCty != cty.NilVal { + terragruntOptions.Logger.Debugf("Exposing include block '%s'", key) exposedIncludeMap[key] = parsedIncludedCty } } return convertValuesMapToCtyVal(exposedIncludeMap) } + +// includeConfigAsCtyVal returns the parsed include block as a cty.Value object if expose is true. Otherwise, return +// the nil representation of cty.Value. +func includeConfigAsCtyVal(includeConfig IncludeConfig, terragruntOptions *options.TerragruntOptions, decodedDependencies *cty.Value) (cty.Value, error) { + if includeConfig.GetExpose() { + parsedIncluded, err := parseIncludedConfig(&includeConfig, terragruntOptions, decodedDependencies) + if err != nil { + return cty.NilVal, err + } + parsedIncludedCty, err := terragruntConfigAsCty(parsedIncluded) + if err != nil { + return cty.NilVal, err + } + return parsedIncludedCty, nil + } + return cty.NilVal, nil +} diff --git a/config/include.go b/config/include.go index 8121176f19..7a3a4f01e7 100644 --- a/config/include.go +++ b/config/include.go @@ -16,6 +16,8 @@ import ( "github.com/gruntwork-io/terragrunt/util" ) +const bareIncludeKey = "" + // Parse the config of the given include, if one is specified func parseIncludedConfig(includedConfig *IncludeConfig, terragruntOptions *options.TerragruntOptions, dependencyOutputs *cty.Value) (*TerragruntConfig, error) { if includedConfig.Path == "" { @@ -567,7 +569,7 @@ func updateBareIncludeBlock(file *hcl.File, filename string) ([]byte, bool, erro if codeWasUpdated { return nil, false, errors.WithStackTrace(MultipleBareIncludeBlocksErr{}) } - block.SetLabels([]string{""}) + block.SetLabels([]string{bareIncludeKey}) codeWasUpdated = true } } @@ -663,7 +665,7 @@ func updateBareIncludeBlockJSON(fileBytes []byte) ([]byte, bool, error) { // can directly assign to the map with the single "" key without worrying about the possibility of other include blocks // since we will only call this function if there is only one include block, and that is a bare block with no labels. func updateSingleBareIncludeInParsedJSON(parsed map[string]interface{}, newVal interface{}) ([]byte, bool, error) { - parsed["include"] = map[string]interface{}{"": newVal} + parsed["include"] = map[string]interface{}{bareIncludeKey: newVal} updatedBytes, err := json.Marshal(parsed) return updatedBytes, true, errors.WithStackTrace(err) } diff --git a/docs/_docs/04_reference/config-blocks-and-attributes.md b/docs/_docs/04_reference/config-blocks-and-attributes.md index 4296d2683f..7e5f2673d0 100644 --- a/docs/_docs/04_reference/config-blocks-and-attributes.md +++ b/docs/_docs/04_reference/config-blocks-and-attributes.md @@ -410,13 +410,13 @@ _Single include_ # ├── terragrunt.hcl # └── child # └── terragrunt.hcl -include { +include "root" { path = find_in_parent_folders() expose = true } inputs = { - remote_state_config = include[""].remote_state + remote_state_config = include.root.remote_state } ``` diff --git a/test/fixture-include-expose/child/main.tf b/test/fixture-include-expose/mixed-with-bare/child/main.tf similarity index 100% rename from test/fixture-include-expose/child/main.tf rename to test/fixture-include-expose/mixed-with-bare/child/main.tf diff --git a/test/fixture-include-expose/mixed-with-bare/child/terragrunt.hcl b/test/fixture-include-expose/mixed-with-bare/child/terragrunt.hcl new file mode 100644 index 0000000000..3f6f07fec1 --- /dev/null +++ b/test/fixture-include-expose/mixed-with-bare/child/terragrunt.hcl @@ -0,0 +1,17 @@ +include { + path = find_in_parent_folders() + expose = true +} + +include "env" { + path = find_in_parent_folders("terragrunt_env.hcl") + expose = true +} + +locals { + parent_region = "${include[""].locals.region}-${include.env.locals.environment}" +} + +inputs = { + region = local.parent_region +} diff --git a/test/fixture-include-expose/multiple/child/main.tf b/test/fixture-include-expose/multiple/child/main.tf new file mode 100644 index 0000000000..5e6547a1c8 --- /dev/null +++ b/test/fixture-include-expose/multiple/child/main.tf @@ -0,0 +1,5 @@ +variable "region" {} + +output "region" { + value = var.region +} diff --git a/test/fixture-include-expose/multiple/child/terragrunt.hcl b/test/fixture-include-expose/multiple/child/terragrunt.hcl new file mode 100644 index 0000000000..c95358a51b --- /dev/null +++ b/test/fixture-include-expose/multiple/child/terragrunt.hcl @@ -0,0 +1,17 @@ +include "root" { + path = find_in_parent_folders() + expose = true +} + +include "env" { + path = find_in_parent_folders("terragrunt_env.hcl") + expose = true +} + +locals { + parent_region = "${include.root.locals.region}-${include.env.locals.environment}" +} + +inputs = { + region = local.parent_region +} diff --git a/test/fixture-include-expose/single-bare/child/main.tf b/test/fixture-include-expose/single-bare/child/main.tf new file mode 100644 index 0000000000..5e6547a1c8 --- /dev/null +++ b/test/fixture-include-expose/single-bare/child/main.tf @@ -0,0 +1,5 @@ +variable "region" {} + +output "region" { + value = var.region +} diff --git a/test/fixture-include-expose/single-bare/child/terragrunt.hcl b/test/fixture-include-expose/single-bare/child/terragrunt.hcl new file mode 100644 index 0000000000..0308f33700 --- /dev/null +++ b/test/fixture-include-expose/single-bare/child/terragrunt.hcl @@ -0,0 +1,13 @@ +include { + path = find_in_parent_folders() + expose = true +} + +locals { + environment = "test" + parent_region = "${include.locals.region}-${local.environment}" +} + +inputs = { + region = local.parent_region +} diff --git a/test/fixture-include-expose/single/child/main.tf b/test/fixture-include-expose/single/child/main.tf new file mode 100644 index 0000000000..5e6547a1c8 --- /dev/null +++ b/test/fixture-include-expose/single/child/main.tf @@ -0,0 +1,5 @@ +variable "region" {} + +output "region" { + value = var.region +} diff --git a/test/fixture-include-expose/child/terragrunt.hcl b/test/fixture-include-expose/single/child/terragrunt.hcl similarity index 60% rename from test/fixture-include-expose/child/terragrunt.hcl rename to test/fixture-include-expose/single/child/terragrunt.hcl index 7e989135f4..06cecccf25 100644 --- a/test/fixture-include-expose/child/terragrunt.hcl +++ b/test/fixture-include-expose/single/child/terragrunt.hcl @@ -1,11 +1,11 @@ -include { +include "root" { path = find_in_parent_folders() expose = true } locals { environment = "test" - parent_region = "${include[""].locals.region}-${local.environment}" + parent_region = "${include.root.locals.region}-${local.environment}" } inputs = { diff --git a/test/fixture-include-expose/terragrunt_env.hcl b/test/fixture-include-expose/terragrunt_env.hcl new file mode 100644 index 0000000000..a5612e5fa6 --- /dev/null +++ b/test/fixture-include-expose/terragrunt_env.hcl @@ -0,0 +1,3 @@ +locals { + environment = "test" +} diff --git a/test/fixture-include/qa/my-app/terragrunt.hcl b/test/fixture-include/qa/my-app/terragrunt.hcl index 6617d250f5..ee617f2b63 100644 --- a/test/fixture-include/qa/my-app/terragrunt.hcl +++ b/test/fixture-include/qa/my-app/terragrunt.hcl @@ -1,4 +1,4 @@ -include { +include "root" { path = "${find_in_parent_folders()}" expose = true @@ -7,5 +7,5 @@ include { } inputs = { - reflect = include[""].remote_state + reflect = include.root.remote_state } diff --git a/test/fixture-include/stage/my-app/terragrunt.hcl b/test/fixture-include/stage/my-app/terragrunt.hcl index bca887680a..8707d7aefd 100644 --- a/test/fixture-include/stage/my-app/terragrunt.hcl +++ b/test/fixture-include/stage/my-app/terragrunt.hcl @@ -1,8 +1,8 @@ -include { +include "root" { path = "${find_in_parent_folders()}" expose = true } inputs = { - reflect = include[""].remote_state + reflect = include.root.remote_state } diff --git a/test/integration_include_test.go b/test/integration_include_test.go index be47c7d114..1685041847 100644 --- a/test/integration_include_test.go +++ b/test/integration_include_test.go @@ -29,18 +29,35 @@ const ( func TestTerragruntWorksWithIncludeLocals(t *testing.T) { t.Parallel() - childPath := util.JoinPath(includeExposeFixturePath, includeChildFixturePath) - cleanupTerraformFolder(t, childPath) - runTerragrunt(t, fmt.Sprintf("terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-log-level debug --terragrunt-working-dir %s", childPath)) - - stdout := bytes.Buffer{} - stderr := bytes.Buffer{} - err := runTerragruntCommand(t, fmt.Sprintf("terragrunt output -no-color -json --terragrunt-non-interactive --terragrunt-log-level debug --terragrunt-working-dir %s", childPath), &stdout, &stderr) + files, err := ioutil.ReadDir(includeExposeFixturePath) require.NoError(t, err) - outputs := map[string]TerraformOutput{} - require.NoError(t, json.Unmarshal([]byte(stdout.String()), &outputs)) - assert.Equal(t, "us-west-1-test", outputs["region"].Value.(string)) + testCases := []string{} + for _, finfo := range files { + if finfo.IsDir() { + testCases = append(testCases, finfo.Name()) + } + } + + for _, testCase := range testCases { + // Capture range variable to avoid it changing across parallel test runs + testCase := testCase + + t.Run(filepath.Base(testCase), func(t *testing.T) { + childPath := filepath.Join(includeExposeFixturePath, testCase, includeChildFixturePath) + cleanupTerraformFolder(t, childPath) + runTerragrunt(t, fmt.Sprintf("terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-log-level debug --terragrunt-working-dir %s", childPath)) + + stdout := bytes.Buffer{} + stderr := bytes.Buffer{} + err := runTerragruntCommand(t, fmt.Sprintf("terragrunt output -no-color -json --terragrunt-non-interactive --terragrunt-log-level debug --terragrunt-working-dir %s", childPath), &stdout, &stderr) + require.NoError(t, err) + + outputs := map[string]TerraformOutput{} + require.NoError(t, json.Unmarshal([]byte(stdout.String()), &outputs)) + assert.Equal(t, "us-west-1-test", outputs["region"].Value.(string)) + }) + } } func TestTerragruntWorksWithIncludeShallowMerge(t *testing.T) { From ad8311bddae75eaee74ba5f92ac66403306cb4da Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Mon, 13 Sep 2021 09:31:28 -0500 Subject: [PATCH 20/26] Remove unnecessary return --- config/include.go | 1 - 1 file changed, 1 deletion(-) diff --git a/config/include.go b/config/include.go index 7a3a4f01e7..2fc4fce6e9 100644 --- a/config/include.go +++ b/config/include.go @@ -683,7 +683,6 @@ func jsonIsIncludeBlock(jsonData interface{}) bool { _, pathIsString := pathAttr.(string) return pathIsString } - return false } return false } From ca1a45d29b4a468cd07990b88d1911f33cd79c00 Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Mon, 13 Sep 2021 09:55:48 -0500 Subject: [PATCH 21/26] Mention expose in include docs --- .../keep-your-terragrunt-architecture-dry.md | 110 ++++++++++++++---- 1 file changed, 88 insertions(+), 22 deletions(-) diff --git a/docs/_docs/02_features/keep-your-terragrunt-architecture-dry.md b/docs/_docs/02_features/keep-your-terragrunt-architecture-dry.md index c9179fef79..7a0b2315d1 100644 --- a/docs/_docs/02_features/keep-your-terragrunt-architecture-dry.md +++ b/docs/_docs/02_features/keep-your-terragrunt-architecture-dry.md @@ -16,6 +16,8 @@ nav_title_link: /docs/ - [Using include to DRY common Terragrunt config](#using-include-to-dry-common-terragrunt-config) + - [Using exposed includes to override common configurations](#using-exposed-includes-to-override-common-configurations) + - [Using read\_terragrunt\_config to DRY parent configurations](#using-read_terragrunt_config-to-dry-parent-configurations) @@ -202,12 +204,12 @@ inputs = { } ``` -### Using read\_terragrunt\_config to DRY parent configurations +### Using exposed includes to override common configurations In the previous section, we covered using `include` to DRY common component configurations. While powerful, `include` has a limitation where the included configuration is statically merged into the child configuration. -In our example, note that the `_env/app.hcl` file hardcodes the `app `module version to `v0.1.0` (relevant section +In our example, note that the `_env/app.hcl` file hardcodes the `app` module version to `v0.1.0` (relevant section pasted below for convenience): ```hcl @@ -241,9 +243,79 @@ inputs = { } ``` -While this works, we now have duplicated the source URL. To avoid repeating the source URL, we can use -`read_terragrunt_config` to load additional context into the the parent configuration by taking advantage of the folder -structure. +While this works, we now have duplicated the source URL. To avoid repeating the source URL, we can use exposed includes +to reference data defined in the parent configurations. To do this, we will refactor our parent configuration to expose +the source URL as a local variable instead of defining it into the `terraform` block: + +```hcl +locals { + source_base_url = "github.com//modules.git//app" +} + +# ... other blocks and attributes omitted for brevity ... +``` + +We then set the `expose` attribute to `true` on the `include` block in the child configuration so that we can reference +the defined data in the parent configuration. Using that, we can construct the terraform source URL without having to +repeat the module source: + +```hcl +include "root" { + path = find_in_parent_folders() +} + +include "env" { + path = "${get_terragrunt_dir()}/../../_env/app.hcl" + expose = true +} + +# Construct the terraform.source attribute using the source_base_url and custom version v0.2.0 +terraform { + source = "${include.env.locals.source_base_url}?ref=v0.2.0" +} + +inputs = { + env = "qa" +} +``` + + +### Using read\_terragrunt\_config to DRY parent configurations + +In the previous two sections, we covered using `include` to DRY common component configurations through static merges +with the child configuration. What if you want to dynamically update the parent configuration without having to define +the override blocks in the child config? + +In our example, the child configuration defines the `env` input in its configuration (pasted below for convenience): + +```hcl +# ... other blocks omitted for brevity ... + +inputs = { + env = "qa" +} +``` + +What if some inputs depend on this `env` input? For example, what if we want to append the `env` to the `name` input +prior to passing to terraform? One way is to define the override parameters in the child config instead of the parent: + +```hcl +# ... other blocks omitted for brevity ... + +include "env" { + path = "${get_terragrunt_dir()}/../../_env/app.hcl" + expose = true +} + +inputs = { + env = "qa" + basename = "${include.env.locals.basename}-qa" +} +``` + +While this works, you could lose all the DRY advantages of the include block if you have many configurations that depend +on the `env` input. Instead, you can use `read_terragrunt_config` to load additional context into the the parent +configuration by taking advantage of the folder structure, and define the env based logic in the parent configuration. To do this, we will introduce a new `env.hcl` configuration in each environment: @@ -287,8 +359,7 @@ locals { } ``` -We can then load the `env.hcl` file in the `_env/app.hcl` file to change the version based on which environment is -loaded: +We can then load the `env.hcl` file in the `_env/app.hcl` file to load the `env` string: ```hcl locals { @@ -297,17 +368,7 @@ locals { env_vars = read_terragrunt_config(find_in_parent_folders("env.hcl")) env_name = local.env_vars.locals.env - # Centrally manage what version of the app module is used in each environment. This makes it easier to promote - # a version from dev -> stage -> prod. - module_version = { - qa = "v0.2.0" - stage = "v0.1.0" - prod = "v0.1.0" - } -} - -terraform { - source = "github.com//modules.git//app?ref=${local.module_version[local.env_name]}" + source_base_url = "github.com//modules.git//app" } dependency "vpc" { @@ -320,7 +381,7 @@ dependency "mysql" { inputs = { env = local.env_name - basename = "example-app" + basename = "example-app-${local.env_name}" vpc_id = dependency.vpc.outputs.vpc_id subnet_ids = dependency.vpc.outputs.subnet_ids mysql_endpoint = dependency.mysql.outputs.endpoint @@ -331,8 +392,7 @@ With this configuration, `env_vars` is loaded based on which folder is being inv invoked in the `prod/app/terragrunt.hcl` folder, `prod/env.hcl` is loaded, while `qa/env.hcl` is loaded when Terragrunt is invoked in the `qa/app/terragrunt.hcl` folder. -Now we can keep the same child config even if we have different versions to deploy per environment. As a bonus, we can -further reduce our child config to eliminate the `env` input variable since that is loaded in the `env.hcl` context: +Now we can clean up the child config to eliminate the `env` input variable since that is loaded in the `env.hcl` context: ```hcl include "root" { @@ -340,6 +400,12 @@ include "root" { } include "env" { - path = "${get_terragrunt_dir()}/../../_env/app.hcl" + path = "${get_terragrunt_dir()}/../../_env/app.hcl" + expose = true +} + +# Construct the terraform.source attribute using the source_base_url and custom version v0.2.0 +terraform { + source = "${include.env.locals.source_base_url}?ref=v0.2.0" } ``` From 30c9947009306f9060fea38aee03d13fb41fe311 Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Mon, 13 Sep 2021 10:07:25 -0500 Subject: [PATCH 22/26] Update docs to remove references of bare include --- docs/_docs/01_getting-started/configuration.md | 2 +- docs/_docs/01_getting-started/quick-start.md | 4 ++-- .../keep-your-remote-state-configuration-dry.md | 2 +- docs/_docs/02_features/keep-your-terraform-code-dry.md | 2 +- .../keep-your-terragrunt-architecture-dry.md | 4 ++-- docs/_docs/04_reference/built-in-functions.md | 10 +++++----- .../_docs/04_reference/config-blocks-and-attributes.md | 9 +++++---- 7 files changed, 17 insertions(+), 16 deletions(-) diff --git a/docs/_docs/01_getting-started/configuration.md b/docs/_docs/01_getting-started/configuration.md index e6ef866111..28021043fa 100644 --- a/docs/_docs/01_getting-started/configuration.md +++ b/docs/_docs/01_getting-started/configuration.md @@ -17,7 +17,7 @@ Terragrunt configuration is defined in a `terragrunt.hcl` file. This uses the sa Here’s an example: ``` hcl -include { +include "root" { path = find_in_parent_folders() } diff --git a/docs/_docs/01_getting-started/quick-start.md b/docs/_docs/01_getting-started/quick-start.md index b9f7654910..0b5d08bf2d 100644 --- a/docs/_docs/01_getting-started/quick-start.md +++ b/docs/_docs/01_getting-started/quick-start.md @@ -201,7 +201,7 @@ The final step is to update each of the child `terragrunt.hcl` files to tell the ``` hcl # stage/mysql/terragrunt.hcl -include { +include "root" { path = find_in_parent_folders() } ``` @@ -217,7 +217,7 @@ $ terragrunt apply Terragrunt will automatically find the `mysql` module’s `terragrunt.hcl` file, configure the `backend` using the settings from the root `terragrunt.hcl` file, and, thanks to the `path_relative_to_include()` function, will set the `key` to `stage/mysql/terraform.tfstate`. If you run `terragrunt apply` in `stage/frontend-app`, it’ll do the same, except it will set the `key` to `stage/frontend-app/terraform.tfstate`. -You can now add as many child modules as you want, each with a `terragrunt.hcl` with the `include { …​ }` block, and each of those modules will automatically inherit the proper `backend` configuration\! +You can now add as many child modules as you want, each with a `terragrunt.hcl` with the `include "root" { …​ }` block, and each of those modules will automatically inherit the proper `backend` configuration\! ### Keep your provider configuration DRY diff --git a/docs/_docs/02_features/keep-your-remote-state-configuration-dry.md b/docs/_docs/02_features/keep-your-remote-state-configuration-dry.md index daa3eea0f4..137ae7a596 100644 --- a/docs/_docs/02_features/keep-your-remote-state-configuration-dry.md +++ b/docs/_docs/02_features/keep-your-remote-state-configuration-dry.md @@ -94,7 +94,7 @@ remote_state { In each of the **child** `terragrunt.hcl` files, such as `mysql/terragrunt.hcl`, you can tell Terragrunt to automatically include all the settings from the root `terragrunt.hcl` file as follows: ``` hcl -include { +include "root" { path = find_in_parent_folders() } ``` diff --git a/docs/_docs/02_features/keep-your-terraform-code-dry.md b/docs/_docs/02_features/keep-your-terraform-code-dry.md index 38194fdb1f..0abb32c710 100644 --- a/docs/_docs/02_features/keep-your-terraform-code-dry.md +++ b/docs/_docs/02_features/keep-your-terraform-code-dry.md @@ -309,7 +309,7 @@ To include this in the child configurations (e.g `mysql/terragrunt.hcl`), you wo include this configuration using the `include` block: ```hcl -include { +include "root" { path = find_in_parent_folders() } ``` diff --git a/docs/_docs/02_features/keep-your-terragrunt-architecture-dry.md b/docs/_docs/02_features/keep-your-terragrunt-architecture-dry.md index 7a0b2315d1..c9a8964169 100644 --- a/docs/_docs/02_features/keep-your-terragrunt-architecture-dry.md +++ b/docs/_docs/02_features/keep-your-terragrunt-architecture-dry.md @@ -57,7 +57,7 @@ You can then include this in each of your **child** `terragrunt.hcl` files using infrastructure module you need to deploy: ```hcl -include { +include "root" { path = find_in_parent_folders() } ``` @@ -104,7 +104,7 @@ To solve this, you can use [multiple include blocks]({{site.baseurl}}/docs/refer Suppose your `qa/app/terragrunt.hcl` configuration looks like the following: ```hcl -include { +include "root" { path = find_in_parent_folders() } diff --git a/docs/_docs/04_reference/built-in-functions.md b/docs/_docs/04_reference/built-in-functions.md index 70b0b2bb8f..3817941956 100644 --- a/docs/_docs/04_reference/built-in-functions.md +++ b/docs/_docs/04_reference/built-in-functions.md @@ -97,7 +97,7 @@ file("assets/mysql/assets.txt") `find_in_parent_folders()` searches up the directory tree from the current `terragrunt.hcl` file and returns the absolute path to the first `terragrunt.hcl` in a parent folder or exit with an error if no such file is found. This is primarily useful in an `include` block to automatically find the path to a parent `terragrunt.hcl` file: ``` hcl -include { +include "root" { path = find_in_parent_folders() } ``` @@ -105,7 +105,7 @@ include { The function takes an optional `name` parameter that allows you to specify a different filename to search for: ``` hcl -include { +include "root" { path = find_in_parent_folders("some-other-file-name.hcl") } ``` @@ -113,7 +113,7 @@ include { You can also pass an optional second `fallback` parameter which causes the function to return the fallback value (instead of exiting with an error) if the file in the `name` parameter cannot be found: ``` hcl -include { +include "root" { path = find_in_parent_folders("some-other-file-name.hcl", "fallback.hcl") } ``` @@ -152,7 +152,7 @@ finding the `env.hcl` file in the `prod` directory. Imagine `prod/mysql/terragrunt.hcl` and `stage/mysql/terragrunt.hcl` include all settings from the root `terragrunt.hcl` file: ``` hcl -include { +include "root" { path = find_in_parent_folders() } ``` @@ -212,7 +212,7 @@ terraform { Imagine `terragrunt/mysql/terragrunt.hcl` and `terragrunt/secrets/mysql/terragrunt.hcl` include all settings from the root `terragrunt.hcl` file: ``` hcl -include { +include "root" { path = find_in_parent_folders() } ``` diff --git a/docs/_docs/04_reference/config-blocks-and-attributes.md b/docs/_docs/04_reference/config-blocks-and-attributes.md index 7e5f2673d0..0eda7d4a47 100644 --- a/docs/_docs/04_reference/config-blocks-and-attributes.md +++ b/docs/_docs/04_reference/config-blocks-and-attributes.md @@ -377,8 +377,9 @@ more about the inheritance properties of Terragrunt in the [Filling in remote st section](/docs/features/keep-your-remote-state-configuration-dry/#filling-in-remote-state-settings-with-terragrunt) of the "Keep your remote state configuration DRY" use case overview. -You can have more than one `include` block, but each one must have a unique label. Note that a bare `include` block with -no label (`include {}`) is a short hand for an `include` block that uses the label `""` (equivalent to `include "" {}`). +You can have more than one `include` block, but each one must have a unique label. It is recommended to always label +your `include` blocks. Bare includes (`include` block with no label - e.g., `include {}`) are currently supported for +backward compatibility, but is deprecated usage and support may be removed in the future. `include` blocks support the following arguments: @@ -487,7 +488,7 @@ inputs = { _child config_ ```hcl -include { +include "root" { path = find_in_parent_folders() merge_strategy = "deep" } @@ -547,7 +548,7 @@ inputs = { _child terragrunt.hcl_ ```hcl -include { +include "root" { path = find_in_parent_folders() merge_strategy = "deep" } From 70a65722a3f561564782da99a0ce685548c78e48 Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Mon, 13 Sep 2021 14:46:38 -0500 Subject: [PATCH 23/26] Regression test for include expose with dependencies --- .../fixture-include-expose/with-dependency/child/main.tf | 5 +++++ .../with-dependency/child/terragrunt.hcl | 9 +++++++++ test/fixture-include-expose/with-dependency/dep/main.tf | 3 +++ .../with-dependency/dep/terragrunt.hcl | 1 + .../with-dependency/terragrunt.hcl | 7 +++++++ test/integration_include_test.go | 2 +- 6 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 test/fixture-include-expose/with-dependency/child/main.tf create mode 100644 test/fixture-include-expose/with-dependency/child/terragrunt.hcl create mode 100644 test/fixture-include-expose/with-dependency/dep/main.tf create mode 100644 test/fixture-include-expose/with-dependency/dep/terragrunt.hcl create mode 100644 test/fixture-include-expose/with-dependency/terragrunt.hcl diff --git a/test/fixture-include-expose/with-dependency/child/main.tf b/test/fixture-include-expose/with-dependency/child/main.tf new file mode 100644 index 0000000000..5e6547a1c8 --- /dev/null +++ b/test/fixture-include-expose/with-dependency/child/main.tf @@ -0,0 +1,5 @@ +variable "region" {} + +output "region" { + value = var.region +} diff --git a/test/fixture-include-expose/with-dependency/child/terragrunt.hcl b/test/fixture-include-expose/with-dependency/child/terragrunt.hcl new file mode 100644 index 0000000000..27bb535030 --- /dev/null +++ b/test/fixture-include-expose/with-dependency/child/terragrunt.hcl @@ -0,0 +1,9 @@ +include "root" { + path = find_in_parent_folders() + expose = true + merge_strategy = "deep" +} + +inputs = { + region = "${include.root.locals.region}-${dependency.dep.outputs.env}" +} diff --git a/test/fixture-include-expose/with-dependency/dep/main.tf b/test/fixture-include-expose/with-dependency/dep/main.tf new file mode 100644 index 0000000000..8a7d11aff2 --- /dev/null +++ b/test/fixture-include-expose/with-dependency/dep/main.tf @@ -0,0 +1,3 @@ +output "env" { + value = "test" +} diff --git a/test/fixture-include-expose/with-dependency/dep/terragrunt.hcl b/test/fixture-include-expose/with-dependency/dep/terragrunt.hcl new file mode 100644 index 0000000000..bb7b160deb --- /dev/null +++ b/test/fixture-include-expose/with-dependency/dep/terragrunt.hcl @@ -0,0 +1 @@ +# Intentionally empty diff --git a/test/fixture-include-expose/with-dependency/terragrunt.hcl b/test/fixture-include-expose/with-dependency/terragrunt.hcl new file mode 100644 index 0000000000..a456d201bb --- /dev/null +++ b/test/fixture-include-expose/with-dependency/terragrunt.hcl @@ -0,0 +1,7 @@ +locals { + region = "us-west-1" +} + +dependency "dep" { + config_path = "${get_terragrunt_dir()}/../dep" +} diff --git a/test/integration_include_test.go b/test/integration_include_test.go index 1685041847..e2bb750164 100644 --- a/test/integration_include_test.go +++ b/test/integration_include_test.go @@ -46,7 +46,7 @@ func TestTerragruntWorksWithIncludeLocals(t *testing.T) { t.Run(filepath.Base(testCase), func(t *testing.T) { childPath := filepath.Join(includeExposeFixturePath, testCase, includeChildFixturePath) cleanupTerraformFolder(t, childPath) - runTerragrunt(t, fmt.Sprintf("terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-log-level debug --terragrunt-working-dir %s", childPath)) + runTerragrunt(t, fmt.Sprintf("terragrunt run-all apply -auto-approve --terragrunt-include-external-dependencies --terragrunt-non-interactive --terragrunt-log-level debug --terragrunt-working-dir %s", childPath)) stdout := bytes.Buffer{} stderr := bytes.Buffer{} From fb5d6041d28e250d2e5444fa53fceb4365da2c02 Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Mon, 13 Sep 2021 15:01:15 -0500 Subject: [PATCH 24/26] Support for dependencies in include when expose is true --- config/config.go | 2 +- config/config_helpers.go | 7 ++++++- config/config_partial.go | 20 ++++++++++++++++---- config/cty_helpers.go | 23 ++++++++++++++++++----- config/include.go | 12 ++++++++++-- config/locals.go | 8 ++++++-- 6 files changed, 57 insertions(+), 15 deletions(-) diff --git a/config/config.go b/config/config.go index 02ffeafb3b..f2ac036653 100644 --- a/config/config.go +++ b/config/config.go @@ -594,7 +594,7 @@ func ParseConfigString( } // Decode just the Base blocks. See the function docs for DecodeBaseBlocks for more info on what base blocks are. - localsAsCty, trackInclude, err := DecodeBaseBlocks(terragruntOptions, parser, file, filename, includeFromChild) + localsAsCty, trackInclude, err := DecodeBaseBlocks(terragruntOptions, parser, file, filename, includeFromChild, nil) if err != nil { return nil, err } diff --git a/config/config_helpers.go b/config/config_helpers.go index 7916ff49b7..e8571b6506 100644 --- a/config/config_helpers.go +++ b/config/config_helpers.go @@ -96,6 +96,11 @@ type EvalContextExtensions struct { // - outputs: The map of outputs from the terraform state obtained by running `terragrunt output` on that target // config. DecodedDependencies *cty.Value + + // PartialParseDecodeList is the list of sections that are being decoded in the current config. This can be used to + // indicate/detect that the current parsing context is partial, meaning that not all configuration values are + // expected to be available. + PartialParseDecodeList []PartialDecodeSectionType } // Create an EvalContext for the HCL2 parser. We can define functions and variables in this context that the HCL2 parser @@ -154,7 +159,7 @@ func CreateTerragruntEvalContext( if extensions.TrackInclude != nil && len(extensions.TrackInclude.CurrentList) > 0 { // For each include block, check if we want to expose the included config, and if so, add under the include // variable. - exposedInclude, err := includeMapAsCtyVal(extensions.TrackInclude.CurrentMap, terragruntOptions, extensions.DecodedDependencies) + exposedInclude, err := includeMapAsCtyVal(extensions.TrackInclude.CurrentMap, terragruntOptions, extensions.DecodedDependencies, extensions.PartialParseDecodeList) if err != nil { return ctx, err } diff --git a/config/config_partial.go b/config/config_partial.go index 8f2921c74d..c16bc54a1d 100644 --- a/config/config_partial.go +++ b/config/config_partial.go @@ -98,13 +98,14 @@ func DecodeBaseBlocks( hclFile *hcl.File, filename string, includeFromChild *IncludeConfig, + decodeList []PartialDecodeSectionType, ) (*cty.Value, *TrackInclude, error) { // Decode just the `include` and `import` blocks, and verify that it's allowed here terragruntIncludeList, err := decodeAsTerragruntInclude( hclFile, filename, terragruntOptions, - EvalContextExtensions{}, + EvalContextExtensions{PartialParseDecodeList: decodeList}, ) if err != nil { return nil, nil, err @@ -123,6 +124,7 @@ func DecodeBaseBlocks( hclFile, filename, trackInclude, + decodeList, ) if err != nil { return nil, trackInclude, err @@ -184,19 +186,29 @@ func PartialParseConfigString( } // Decode just the Base blocks. See the function docs for DecodeBaseBlocks for more info on what base blocks are. - localsAsCty, trackInclude, err := DecodeBaseBlocks(terragruntOptions, parser, file, filename, includeFromChild) + localsAsCty, trackInclude, err := DecodeBaseBlocks(terragruntOptions, parser, file, filename, includeFromChild, decodeList) if err != nil { return nil, err } // Initialize evaluation context extensions from base blocks. contextExtensions := EvalContextExtensions{ - Locals: localsAsCty, - TrackInclude: trackInclude, + Locals: localsAsCty, + TrackInclude: trackInclude, + PartialParseDecodeList: decodeList, } output := TerragruntConfig{IsPartial: true} + // Set parsed Locals on the parsed config + if contextExtensions.Locals != nil && *contextExtensions.Locals != cty.NilVal { + localsParsed, err := parseCtyValueToMap(*contextExtensions.Locals) + if err != nil { + return nil, err + } + output.Locals = localsParsed + } + // Now loop through each requested block / component to decode from the terragrunt config, decode them, and merge // them into the output TerragruntConfig struct. for _, decode := range decodeList { diff --git a/config/cty_helpers.go b/config/cty_helpers.go index 3aa103d7fa..c2fb30ad83 100644 --- a/config/cty_helpers.go +++ b/config/cty_helpers.go @@ -193,16 +193,24 @@ func generateTypeFromValuesMap(valMap map[string]cty.Value) cty.Type { // includeMapAsCtyVal converts the include map into a cty.Value struct that can be exposed to the child config. For // backward compatibility, this function will return the included config object if the config only defines a single bare // include block that is exposed. -func includeMapAsCtyVal(includeMap map[string]IncludeConfig, terragruntOptions *options.TerragruntOptions, decodedDependencies *cty.Value) (cty.Value, error) { +// NOTE: When evaluated in a partial parse context, only the partially parsed context is available in the expose. This +// ensures that we can parse the child config without having access to dependencies when constructing the dependency +// graph. +func includeMapAsCtyVal( + includeMap map[string]IncludeConfig, + terragruntOptions *options.TerragruntOptions, + decodedDependencies *cty.Value, + decodeList []PartialDecodeSectionType, +) (cty.Value, error) { bareInclude, hasBareInclude := includeMap[bareIncludeKey] if len(includeMap) == 1 && hasBareInclude { terragruntOptions.Logger.Debug("Detected single bare include block - exposing as top level") - return includeConfigAsCtyVal(bareInclude, terragruntOptions, decodedDependencies) + return includeConfigAsCtyVal(bareInclude, terragruntOptions, decodedDependencies, decodeList) } exposedIncludeMap := map[string]cty.Value{} for key, included := range includeMap { - parsedIncludedCty, err := includeConfigAsCtyVal(included, terragruntOptions, decodedDependencies) + parsedIncludedCty, err := includeConfigAsCtyVal(included, terragruntOptions, decodedDependencies, decodeList) if err != nil { return cty.NilVal, err } @@ -216,9 +224,14 @@ func includeMapAsCtyVal(includeMap map[string]IncludeConfig, terragruntOptions * // includeConfigAsCtyVal returns the parsed include block as a cty.Value object if expose is true. Otherwise, return // the nil representation of cty.Value. -func includeConfigAsCtyVal(includeConfig IncludeConfig, terragruntOptions *options.TerragruntOptions, decodedDependencies *cty.Value) (cty.Value, error) { +func includeConfigAsCtyVal( + includeConfig IncludeConfig, + terragruntOptions *options.TerragruntOptions, + decodedDependencies *cty.Value, + decodeList []PartialDecodeSectionType, +) (cty.Value, error) { if includeConfig.GetExpose() { - parsedIncluded, err := parseIncludedConfig(&includeConfig, terragruntOptions, decodedDependencies) + parsedIncluded, err := parseIncludedConfig(&includeConfig, terragruntOptions, decodedDependencies, decodeList) if err != nil { return cty.NilVal, err } diff --git a/config/include.go b/config/include.go index 2fc4fce6e9..c41a0e0ec0 100644 --- a/config/include.go +++ b/config/include.go @@ -19,7 +19,12 @@ import ( const bareIncludeKey = "" // Parse the config of the given include, if one is specified -func parseIncludedConfig(includedConfig *IncludeConfig, terragruntOptions *options.TerragruntOptions, dependencyOutputs *cty.Value) (*TerragruntConfig, error) { +func parseIncludedConfig( + includedConfig *IncludeConfig, + terragruntOptions *options.TerragruntOptions, + dependencyOutputs *cty.Value, + decodeList []PartialDecodeSectionType, +) (*TerragruntConfig, error) { if includedConfig.Path == "" { return nil, errors.WithStackTrace(IncludedConfigMissingPath(terragruntOptions.TerragruntConfigPath)) } @@ -30,6 +35,9 @@ func parseIncludedConfig(includedConfig *IncludeConfig, terragruntOptions *optio includePath = util.JoinPath(filepath.Dir(terragruntOptions.TerragruntConfigPath), includePath) } + if len(decodeList) > 0 { + return PartialParseConfigFile(includePath, terragruntOptions, includedConfig, decodeList) + } return ParseConfigFile(includePath, terragruntOptions, includedConfig, dependencyOutputs) } @@ -56,7 +64,7 @@ func handleInclude( return config, err } - parsedIncludeConfig, err := parseIncludedConfig(&includeConfig, terragruntOptions, dependencyOutputs) + parsedIncludeConfig, err := parseIncludedConfig(&includeConfig, terragruntOptions, dependencyOutputs, nil) if err != nil { return nil, err } diff --git a/config/locals.go b/config/locals.go index a26d7050cd..393fa2a532 100644 --- a/config/locals.go +++ b/config/locals.go @@ -46,6 +46,7 @@ func evaluateLocalsBlock( hclFile *hcl.File, filename string, trackInclude *TrackInclude, + decodeList []PartialDecodeSectionType, ) (map[string]cty.Value, error) { diagsWriter := util.GetDiagnosticsWriter(parser) @@ -87,6 +88,7 @@ func evaluateLocalsBlock( locals, evaluatedLocals, trackInclude, + decodeList, diagsWriter, ) if err != nil { @@ -118,6 +120,7 @@ func attemptEvaluateLocals( locals []*Local, evaluatedLocals map[string]cty.Value, trackInclude *TrackInclude, + decodeList []PartialDecodeSectionType, diagsWriter hcl.DiagnosticWriter, ) (unevaluatedLocals []*Local, newEvaluatedLocals map[string]cty.Value, evaluated bool, err error) { // The HCL2 parser and especially cty conversions will panic in many types of errors, so we have to recover from @@ -142,8 +145,9 @@ func attemptEvaluateLocals( filename, terragruntOptions, EvalContextExtensions{ - TrackInclude: trackInclude, - Locals: &evaluatedLocalsAsCty, + TrackInclude: trackInclude, + Locals: &evaluatedLocalsAsCty, + PartialParseDecodeList: decodeList, }, ) if err != nil { From 85c595c516c6fb484b562db00d7a9e8c4b8d566b Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Mon, 13 Sep 2021 15:07:53 -0500 Subject: [PATCH 25/26] Fix build for locals testing --- config/locals_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/config/locals_test.go b/config/locals_test.go index 9d83ae5f97..ef9313ab0e 100644 --- a/config/locals_test.go +++ b/config/locals_test.go @@ -22,7 +22,7 @@ func TestEvaluateLocalsBlock(t *testing.T) { file, err := parseHcl(parser, LocalsTestConfig, mockFilename) require.NoError(t, err) - evaluatedLocals, err := evaluateLocalsBlock(terragruntOptions, parser, file, mockFilename, nil) + evaluatedLocals, err := evaluateLocalsBlock(terragruntOptions, parser, file, mockFilename, nil, nil) require.NoError(t, err) var actualRegion string @@ -67,7 +67,7 @@ func TestEvaluateLocalsBlockMultiDeepReference(t *testing.T) { file, err := parseHcl(parser, LocalsTestMultiDeepReferenceConfig, mockFilename) require.NoError(t, err) - evaluatedLocals, err := evaluateLocalsBlock(terragruntOptions, parser, file, mockFilename, nil) + evaluatedLocals, err := evaluateLocalsBlock(terragruntOptions, parser, file, mockFilename, nil, nil) require.NoError(t, err) expected := "a" @@ -106,7 +106,7 @@ func TestEvaluateLocalsBlockImpossibleWillFail(t *testing.T) { file, err := parseHcl(parser, LocalsTestImpossibleConfig, mockFilename) require.NoError(t, err) - _, err = evaluateLocalsBlock(terragruntOptions, parser, file, mockFilename, nil) + _, err = evaluateLocalsBlock(terragruntOptions, parser, file, mockFilename, nil, nil) require.Error(t, err) switch errors.Unwrap(err).(type) { @@ -126,7 +126,7 @@ func TestEvaluateLocalsBlockMultipleLocalsBlocksWillFail(t *testing.T) { file, err := parseHcl(parser, MultipleLocalsBlockConfig, mockFilename) require.NoError(t, err) - _, err = evaluateLocalsBlock(terragruntOptions, parser, file, mockFilename, nil) + _, err = evaluateLocalsBlock(terragruntOptions, parser, file, mockFilename, nil, nil) require.Error(t, err) } From a8bf61d289f41171a6acdd8d5bb142b3006908de Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Mon, 13 Sep 2021 18:07:26 -0500 Subject: [PATCH 26/26] Partial parse includes locals --- config/config_partial_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config_partial_test.go b/config/config_partial_test.go index 2fa15eebe2..2bfea91a12 100644 --- a/config/config_partial_test.go +++ b/config/config_partial_test.go @@ -27,13 +27,13 @@ dependencies { require.NotNil(t, terragruntConfig.Dependencies) assert.Equal(t, len(terragruntConfig.Dependencies.Paths), 1) assert.Equal(t, terragruntConfig.Dependencies.Paths[0], "../app1") + assert.Equal(t, map[string]interface{}{"app1": "../app1"}, terragruntConfig.Locals) assert.False(t, terragruntConfig.Skip) assert.Nil(t, terragruntConfig.PreventDestroy) assert.Nil(t, terragruntConfig.Terraform) assert.Nil(t, terragruntConfig.RemoteState) assert.Nil(t, terragruntConfig.Inputs) - assert.Nil(t, terragruntConfig.Locals) } func TestPartialParseDoesNotResolveIgnoredBlock(t *testing.T) {