Implement --terragrunt-source-map (#1674)

* [skip ci]

* Implement --terragrunt-source-map

* Update docs to clarify dependency block and literalness of matches

cli/args.go

@@ -95,6 +95,11 @@ func parseTerragruntOptionsFromArgs(terragruntVersion string, args []string, wri
 		return nil, err
 	}
 
+	terraformSourceMap, err := parseMutliStringKeyValueArg(args, OPT_TERRAGRUNT_SOURCE_MAP, nil)
+	if err != nil {
+		return nil, err
+	}
+
 	sourceUpdate := parseBooleanArg(args, OPT_TERRAGRUNT_SOURCE_UPDATE, os.Getenv("TERRAGRUNT_SOURCE_UPDATE") == "true" || os.Getenv("TERRAGRUNT_SOURCE_UPDATE") == "1")
 
 	ignoreDependencyErrors := parseBooleanArg(args, OPT_TERRAGRUNT_IGNORE_DEPENDENCY_ERRORS, false)
@@ -166,6 +171,7 @@ func parseTerragruntOptionsFromArgs(terragruntVersion string, args []string, wri
 	opts.Logger.Logger.SetOutput(errWriter)
 	opts.RunTerragrunt = RunTerragrunt
 	opts.Source = terraformSource
+	opts.SourceMap = terraformSourceMap
 	opts.SourceUpdate = sourceUpdate
 	opts.TerragruntVersion, err = version.NewVersion(terragruntVersion)
 	if err != nil {

cli/cli_app.go

@@ -32,6 +32,7 @@ const OPT_NON_INTERACTIVE = "terragrunt-non-interactive"
 const OPT_WORKING_DIR = "terragrunt-working-dir"
 const OPT_DOWNLOAD_DIR = "terragrunt-download-dir"
 const OPT_TERRAGRUNT_SOURCE = "terragrunt-source"
+const OPT_TERRAGRUNT_SOURCE_MAP = "terragrunt-source-map"
 const OPT_TERRAGRUNT_SOURCE_UPDATE = "terragrunt-source-update"
 const OPT_TERRAGRUNT_IAM_ROLE = "terragrunt-iam-role"
 const OPT_TERRAGRUNT_IGNORE_DEPENDENCY_ERRORS = "terragrunt-ignore-dependency-errors"
@@ -67,6 +68,7 @@ var ALL_TERRAGRUNT_STRING_OPTS = []string{
 	OPT_WORKING_DIR,
 	OPT_DOWNLOAD_DIR,
 	OPT_TERRAGRUNT_SOURCE,
+	OPT_TERRAGRUNT_SOURCE_MAP,
 	OPT_TERRAGRUNT_IAM_ROLE,
 	OPT_TERRAGRUNT_EXCLUDE_DIR,
 	OPT_TERRAGRUNT_INCLUDE_DIR,
@@ -406,7 +408,11 @@ func RunTerragrunt(terragruntOptions *options.TerragruntOptions) error {
 	}
 
 	updatedTerragruntOptions := terragruntOptions
-	if sourceUrl := config.GetTerraformSourceUrl(terragruntOptions, terragruntConfig); sourceUrl != "" {
+	sourceUrl, err := config.GetTerraformSourceUrl(terragruntOptions, terragruntConfig)
+	if err != nil {
+		return err
+	}
+	if sourceUrl != "" {
 		updatedTerragruntOptions, err = downloadTerraformSource(sourceUrl, terragruntOptions, terragruntConfig)
 		if err != nil {
 			return err

config/config.go

@@ -2,14 +2,16 @@ package config
 
 import (
 	"fmt"
-	"github.com/mitchellh/mapstructure"
+	"net/url"
 	"os"
 	"path/filepath"
 	"reflect"
 	"strings"
 
+	"github.com/hashicorp/go-getter"
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/hclparse"
+	"github.com/mitchellh/mapstructure"
 	"github.com/sirupsen/logrus"
 	"github.com/zclconf/go-cty/cty"
 
@@ -317,14 +319,79 @@ func (conf *TerraformExtraArguments) GetVarFiles(logger *logrus.Entry) []string
 // There are two ways a user can tell Terragrunt that it needs to download Terraform configurations from a specific
 // URL: via a command-line option or via an entry in the Terragrunt configuration. If the user used one of these, this
 // method returns the source URL or an empty string if there is no source url
-func GetTerraformSourceUrl(terragruntOptions *options.TerragruntOptions, terragruntConfig *TerragruntConfig) string {
+func GetTerraformSourceUrl(terragruntOptions *options.TerragruntOptions, terragruntConfig *TerragruntConfig) (string, error) {
 	if terragruntOptions.Source != "" {
-		return terragruntOptions.Source
+		return terragruntOptions.Source, nil
 	} else if terragruntConfig.Terraform != nil && terragruntConfig.Terraform.Source != nil {
-		return *terragruntConfig.Terraform.Source
+		return adjustSourceWithMap(terragruntOptions.SourceMap, *terragruntConfig.Terraform.Source, terragruntOptions.OriginalTerragruntConfigPath)
 	} else {
-		return ""
+		return "", nil
+	}
+}
+
+// adjustSourceWithMap implements the --terragrunt-source-map feature. This function will check if the URL portion of a
+// terraform source matches any entry in the provided source map and if it does, replace it with the configured source
+// in the map. Note that this only performs literal matches with the URL portion.
+//
+// Example:
+// Suppose terragrunt is called with:
+//
+//   --terragrunt-source-map git::ssh://[email protected]/gruntwork-io/i-dont-exist.git=/path/to/local-modules
+//
+// and the terraform source is:
+//
+//   git::ssh://[email protected]/gruntwork-io/i-dont-exist.git//fixture-source-map/modules/app?ref=master
+//
+// This function will take that source and transform it to:
+//
+//   /path/to/local-modules/fixture-source-map/modules/app
+//
+func adjustSourceWithMap(sourceMap map[string]string, source string, modulePath string) (string, error) {
+	// Skip logic if source map is not configured
+	if len(sourceMap) == 0 {
+		return source, nil
 	}
+
+	// use go-getter to split the module source string into a valid URL and subdirectory (if // is present)
+	moduleUrl, moduleSubdir := getter.SourceDirSubdir(source)
+
+	// if both URL and subdir are missing, something went terribly wrong
+	if moduleUrl == "" && moduleSubdir == "" {
+		return "", errors.WithStackTrace(InvalidSourceUrlWithMap{ModulePath: modulePath, ModuleSourceUrl: source})
+	}
+
+	// If module URL is missing, return the source as is as it will not match anything in the map.
+	if moduleUrl == "" {
+		return source, nil
+	}
+
+	// Before looking up in sourceMap, make sure to drop any query parameters.
+	moduleUrlParsed, err := url.Parse(moduleUrl)
+	if err != nil {
+		return source, err
+	}
+	moduleUrlParsed.RawQuery = ""
+	moduleUrlQuery := moduleUrlParsed.String()
+
+	// Check if there is an entry to replace the URL portion in the map. Return the source as is if there is no entry in
+	// the map.
+	sourcePath, hasKey := sourceMap[moduleUrlQuery]
+	if hasKey == false {
+		return source, nil
+	}
+
+	// Since there is a source mapping, replace the module URL portion with the entry in the map, and join with the
+	// subdir.
+	// If subdir is missing, check if we can obtain a valid module name from the URL portion.
+	if moduleSubdir == "" {
+		moduleSubdirFromUrl, err := getModulePathFromSourceUrl(moduleUrl)
+		if err != nil {
+			return moduleSubdirFromUrl, err
+		}
+		moduleSubdir = moduleSubdirFromUrl
+	}
+	return util.JoinTerraformModulePath(sourcePath, moduleSubdir), nil
+
 }
 
 // Return the default hcl path to use for the Terragrunt configuration file in the given directory

config/config_helpers.go

@@ -657,6 +657,15 @@ func (err InvalidSourceUrl) Error() string {
 	return fmt.Sprintf("The --terragrunt-source parameter is set to '%s', but the source URL in the module at '%s' is invalid: '%s'. Note that the module URL must have a double-slash to separate the repo URL from the path within the repo!", err.TerragruntSource, err.ModulePath, err.ModuleSourceUrl)
 }
 
+type InvalidSourceUrlWithMap struct {
+	ModulePath      string
+	ModuleSourceUrl string
+}
+
+func (err InvalidSourceUrlWithMap) Error() string {
+	return fmt.Sprintf("The --terragrunt-source-map parameter was passed in, but the source URL in the module at '%s' is invalid: '%s'. Note that the module URL must have a double-slash to separate the repo URL from the path within the repo!", err.ModulePath, err.ModuleSourceUrl)
+}
+
 type ErrorParsingModulePath struct {
 	ModuleSourceUrl string
 }

config/dependency.go

@@ -452,7 +452,10 @@ func terragruntAlreadyInit(terragruntOptions *options.TerragruntOptions, configP
 		return false, "", err
 	}
 	var workingDir string
-	sourceUrl := GetTerraformSourceUrl(terragruntOptions, terraformBlockTGConfig)
+	sourceUrl, err := GetTerraformSourceUrl(terragruntOptions, terraformBlockTGConfig)
+	if err != nil {
+		return false, "", err
+	}
 	if sourceUrl == "" || sourceUrl == "." {
 		// When there is no source URL, there is no download process and the working dir is the same as the directory
 		// where the config is.

docs/_docs/04_reference/cli-options.md

@@ -389,6 +389,7 @@ prefix `--terragrunt-` (e.g., `--terragrunt-config`). The currently available op
 - [terragrunt-working-dir](#terragrunt-working-dir)
 - [terragrunt-download-dir](#terragrunt-download-dir)
 - [terragrunt-source](#terragrunt-source)
+- [terragrunt-source-map](#terragrunt-source-map)
 - [terragrunt-source-update](#terragrunt-source-update)
 - [terragrunt-ignore-dependency-errors](#terragrunt-ignore-dependency-errors)
 - [terragrunt-iam-role](#terragrunt-iam-role)
@@ -501,6 +502,35 @@ for all of your Terraform modules, and for each module processed by the `xxx-all
 append the path of `source` parameter in each module to the `--terragrunt-source` parameter you passed in.
 
 
+### terragrunt-source-map
+
+**CLI Arg**: `--terragrunt-source-map`<br/>
+**Requires an argument**: `--terragrunt-source-map git::ssh://github.com=/path/to/local-terraform-code`
+
+Can be supplied multiple times: `--terragrunt-source-map source1=dest1 --terragrunt-source-map source2=dest2`
+
+The `--terragrunt-source-map source=dest` param replaces any `source` URL (including the source URL of a config pulled
+in with `dependency` blocks) that has root `source` with `dest`.
+
+For example:
+
+```
+terragrunt apply --terragrunt-source-map github.com/org/modules.git:/local/path/to/modules
+```
+
+The above would replace `terraform { source = "github.com/org/modules.git//xxx" }` with `terraform { source = /local/path/to/modules//xxx }` regardless of
+whether you were running `apply`, or `run-all`, or using a `dependency`.
+
+**NOTE**: This setting is ignored if you pass in `--terragrunt-source`.
+
+Note that this only performs literal matches on the URL portion. For example, a map key of
+`ssh://[email protected]/gruntwork-io/terragrunt.git` will only match terragrunt configurations with source `source =
+"ssh://[email protected]/gruntwork-io/terragrunt.git//xxx"` and not sources of the form `source =
+"git::ssh://[email protected]/gruntwork-io/terragrunt.git//xxx"`. The latter requires a map key of
+`git::ssh://[email protected]/gruntwork-io/terragrunt.git`.
+
+
+
 ### terragrunt-source-update
 
 **CLI Arg**: `--terragrunt-source-update`<br/>

options/options.go

@@ -91,6 +91,10 @@ type TerragruntOptions struct {
 	// Terraform in that temporary folder
 	Source string
 
+	// Map to replace terraform source locations. This will replace occurences of the given source with the target
+	// value.
+	SourceMap map[string]string
+
 	// If set to true, delete the contents of the temporary folder before downloading Terraform source code into it
 	SourceUpdate bool
 
@@ -190,6 +194,7 @@ func NewTerragruntOptions(terragruntConfigPath string) (*TerragruntOptions, erro
 		LogLevel:                    DEFAULT_LOG_LEVEL,
 		Env:                         map[string]string{},
 		Source:                      "",
+		SourceMap:                   map[string]string{},
 		SourceUpdate:                false,
 		DownloadDir:                 downloadDir,
 		IgnoreDependencyErrors:      false,
@@ -265,6 +270,7 @@ func (terragruntOptions *TerragruntOptions) Clone(terragruntConfigPath string) *
 		LogLevel:                     terragruntOptions.LogLevel,
 		Env:                          util.CloneStringMap(terragruntOptions.Env),
 		Source:                       terragruntOptions.Source,
+		SourceMap:                    terragruntOptions.SourceMap,
 		SourceUpdate:                 terragruntOptions.SourceUpdate,
 		DownloadDir:                  terragruntOptions.DownloadDir,
 		Debug:                        terragruntOptions.Debug,

test/fixture-source-map/modules/app/main.tf

@@ -0,0 +1,7 @@
+variable "name" {}
+
+variable "vpc_id" {}
+
+output "app_url" {
+  value = "https://${var.name}.${var.vpc_id}.foo.io"
+}

test/fixture-source-map/modules/vpc/main.tf

@@ -0,0 +1,5 @@
+variable "name" {}
+
+output "vpc_id" {
+  value = "vpc-${var.name}-asdf1234"
+}

test/fixture-source-map/multiple-match/terragrunt-vpc/terragrunt.hcl

@@ -0,0 +1,7 @@
+terraform {
+  source = "git::ssh://[email protected]/gruntwork-io/another-dont-exist.git//fixture-source-map/modules/vpc?ref=master"
+}
+
+inputs = {
+  name = "terragrunt"
+}

test/fixture-source-map/multiple-match/terratest-vpc/terragrunt.hcl

@@ -0,0 +1,7 @@
+terraform {
+  source = "git::ssh://[email protected]/gruntwork-io/i-dont-exist.git//fixture-source-map/modules/vpc?ref=master"
+}
+
+inputs = {
+  name = "terratest"
+}

test/fixture-source-map/multiple-only-one-match/terragrunt-vpc/terragrunt.hcl

@@ -0,0 +1,7 @@
+terraform {
+  source = "../../modules/vpc"
+}
+
+inputs = {
+  name = "terragrunt"
+}

test/fixture-source-map/multiple-only-one-match/terratest-vpc/terragrunt.hcl

@@ -0,0 +1,7 @@
+terraform {
+  source = "git::ssh://[email protected]/gruntwork-io/i-dont-exist.git//fixture-source-map/modules/vpc?ref=master"
+}
+
+inputs = {
+  name = "terratest"
+}

test/fixture-source-map/multiple-with-dependency-same-url/app/terragrunt.hcl

@@ -0,0 +1,12 @@
+terraform {
+  source = "git::ssh://[email protected]/gruntwork-io/i-dont-exist.git//fixture-source-map/modules/app?ref=master"
+}
+
+dependency "vpc" {
+  config_path = "../vpc"
+}
+
+inputs = {
+  name   = "terragrunt"
+  vpc_id = dependency.vpc.outputs.vpc_id
+}

test/fixture-source-map/multiple-with-dependency-same-url/vpc/terragrunt.hcl

@@ -0,0 +1,7 @@
+terraform {
+  source = "git::ssh://[email protected]/gruntwork-io/i-dont-exist.git//fixture-source-map/modules/vpc?ref=master"
+}
+
+inputs = {
+  name = "terragrunt"
+}

test/fixture-source-map/multiple-with-dependency/app/terragrunt.hcl

@@ -0,0 +1,12 @@
+terraform {
+  source = "git::ssh://[email protected]/gruntwork-io/another-dont-exist.git//fixture-source-map/modules/app?ref=master"
+}
+
+dependency "vpc" {
+  config_path = "../vpc"
+}
+
+inputs = {
+  name   = "terragrunt"
+  vpc_id = dependency.vpc.outputs.vpc_id
+}

test/fixture-source-map/multiple-with-dependency/vpc/terragrunt.hcl

@@ -0,0 +1,7 @@
+terraform {
+  source = "git::ssh://[email protected]/gruntwork-io/i-dont-exist.git//fixture-source-map/modules/vpc?ref=master"
+}
+
+inputs = {
+  name = "terragrunt"
+}

test/fixture-source-map/single/terragrunt.hcl

@@ -0,0 +1,7 @@
+terraform {
+  source = "git::ssh://[email protected]/gruntwork-io/i-dont-exist.git//fixture-source-map/modules/vpc?ref=master"
+}
+
+inputs = {
+  name = "terragrunt"
+}