Engine verification (#3300)

* Add engine checks

* Add engine checking

* Add test to check engine checksum

* Checksum read block

* spaces cleanup

* Tests update

* engine source

* Engine update

* engine source

* Tofu version check update

* Cleanup

* Engine update

* Added test for signature checking

* Extracted public key file

* Test cleanup

* File download update

* Tests update

.circleci/config.yml

@@ -8,7 +8,7 @@ env: &env
   environment:
     GRUNTWORK_INSTALLER_VERSION: v0.0.39
     MODULE_CI_VERSION: v0.57.0
-    ENGINE_VERSION: "v0.0.2"
+    TOFU_ENGINE_VERSION: "v0.0.4"
 
 defaults: &defaults
   docker:
@@ -61,8 +61,8 @@ install_tofu_engine: &install_tofu_engine
     # Download the OpenTofu Engine binary
     set -x
     export REPO="gruntwork-io/terragrunt-engine-opentofu"
-    export ASSET_NAME="terragrunt-iac-engine-opentofu_rpc_${ENGINE_VERSION}_linux_amd64.zip"
-    wget -O "engine.zip" "https://github.com/${REPO}/releases/download/${ENGINE_VERSION}/${ASSET_NAME}"
+    export ASSET_NAME="terragrunt-iac-engine-opentofu_rpc_${TOFU_ENGINE_VERSION}_linux_amd64.zip"
+    wget -O "engine.zip" "https://github.com/${REPO}/releases/download/${TOFU_ENGINE_VERSION}/${ASSET_NAME}"
     unzip -o "engine.zip"
 
 setup_test_environment: &setup_test_environment

docs/_docs/02_features/engine.md

@@ -80,6 +80,14 @@ The cached engines are stored in the following directory:
 
 If you need to use a different path, set the environment variable `TG_ENGINE_CACHE_PATH` accordingly.
 
+Downloaded engines are checked for integrity using the SHA256 checksum GPG key.
+If the checksum does not match, the engine is not executed.
+To disable this feature, set the environment variable:
+
+```sh
+export TG_ENGINE_SKIP_CHECK=0 
+```
+
 Due to the fact that this functionality is still experimental, and not recommended for general production usage, set the following environment variable to opt-in to this functionality:
 
 ```sh

engine/engine.go

@@ -10,6 +10,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"sync"
 
@@ -40,7 +41,9 @@ const (
 	EngineCacheDir                                   = "terragrunt/plugins/iac-engine"
 	PrefixTrim                                       = "terragrunt-"
 	FileNameFormat                                   = "terragrunt-iac-%s_%s_%s_%s_%s"
+	ChecksumFileNameFormat                           = "terragrunt-iac-%s_%s_%s_SHA256SUMS"
 	EngineCachePathEnv                               = "TG_ENGINE_CACHE_PATH"
+	EngineSkipCheckEnv                               = "TG_ENGINE_SKIP_CHECK"
 	TerraformCommandContextKey      engineClientsKey = iota
 	LocksContextKey                 engineLocksKey   = iota
 )
@@ -152,28 +155,48 @@ func DownloadEngine(ctx context.Context, opts *options.TerragruntOptions) error
 		return nil
 	}
 	downloadFile := filepath.Join(path, enginePackageName(e))
-	var downloadURL string
-	if strings.HasPrefix(e.Source, "http") {
+
+	downloads := make(map[string]string)
+	checksumFile := ""
+	checksumSigFile := ""
+	if strings.Contains(e.Source, "://") {
 		// if source starts with absolute path, download as is
-		downloadURL = e.Source
+		downloads[e.Source] = downloadFile
 	} else {
-		// Archive support documented in https://github.com/hashicorp/go-getter?tab=readme-ov-file#unarchiving
-		downloadURL = fmt.Sprintf("https://%s/releases/download/%s/%s",
-			e.Source, e.Version, enginePackageName(e))
-	}
+		baseURL := fmt.Sprintf("https://%s/releases/download/%s", e.Source, e.Version)
+
+		// URLs and their corresponding local paths
+		checksumFile = filepath.Join(path, engineChecksumName(e))
+		checksumSigFile = filepath.Join(path, engineChecksumSigName(e))
+		downloads[fmt.Sprintf("%s/%s", baseURL, enginePackageName(e))] = downloadFile
+		downloads[fmt.Sprintf("%s/%s", baseURL, engineChecksumName(e))] = checksumFile
+		downloads[fmt.Sprintf("%s/%s.sig", baseURL, engineChecksumName(e))] = checksumSigFile
+	}
+
+	for url, path := range downloads {
+		opts.Logger.Infof("Downloading %s to %s", url, path)
+		client := &getter.Client{
+			Ctx:           ctx,
+			Src:           url,
+			Dst:           path,
+			Mode:          getter.ClientModeFile,
+			Decompressors: map[string]getter.Decompressor{},
+		}
 
-	client := &getter.Client{
-		Ctx:           ctx,
-		Src:           downloadURL,
-		Dst:           downloadFile,
-		Mode:          getter.ClientModeFile,
-		Decompressors: map[string]getter.Decompressor{},
+		if err := client.Get(); err != nil {
+			return errors.WithStackTrace(err)
+		}
 	}
 
-	if err := client.Get(); err != nil {
-		return errors.WithStackTrace(err)
+	if !skipEngineCheck() && checksumFile != "" && checksumSigFile != "" {
+		opts.Logger.Infof("Verifying checksum for %s", downloadFile)
+		if err := verifyFile(downloadFile, checksumFile, checksumSigFile); err != nil {
+			return errors.WithStackTrace(err)
+		}
+	} else {
+		opts.Logger.Warnf("Skipping verification for %s", downloadFile)
 	}
-	opts.Logger.Infof("Engine downloaded to %s", downloadFile)
+
 	if err := extractArchive(opts, downloadFile, localEngineFile); err != nil {
 		return errors.WithStackTrace(err)
 	}
@@ -262,6 +285,19 @@ func engineFileName(e *options.EngineOptions) string {
 	return fmt.Sprintf(FileNameFormat, engineName, e.Type, e.Version, platform, arch)
 }
 
+// engineChecksumName returns the file name of engine checksum file
+func engineChecksumName(e *options.EngineOptions) string {
+	engineName := filepath.Base(e.Source)
+
+	engineName = strings.TrimPrefix(engineName, PrefixTrim)
+	return fmt.Sprintf(ChecksumFileNameFormat, engineName, e.Type, e.Version)
+}
+
+// engineChecksumSigName returns the file name of engine checksum file signature
+func engineChecksumSigName(e *options.EngineOptions) string {
+	return fmt.Sprintf("%s.sig", engineChecksumName(e))
+}
+
 // enginePackageName returns the package name for the engine.
 func enginePackageName(e *options.EngineOptions) string {
 	return fmt.Sprintf("%s.zip", engineFileName(e))
@@ -307,11 +343,8 @@ func downloadLocksFromContext(ctx context.Context) (*util.KeyLocks, error) {
 
 // IsEngineEnabled returns true if the experimental engine is enabled.
 func IsEngineEnabled() bool {
-	switch strings.ToLower(os.Getenv(EnableExperimentalEngineEnvName)) {
-	case "1", "yes", "true", "on":
-		return true
-	}
-	return false
+	ok, _ := strconv.ParseBool(os.Getenv(EnableExperimentalEngineEnvName)) //nolint:errcheck
+	return ok
 }
 
 // Shutdown shuts down the experimental engine.
@@ -345,6 +378,16 @@ func createEngine(terragruntOptions *options.TerragruntOptions) (*proto.EngineCl
 		return nil, nil, errors.WithStackTrace(err)
 	}
 	localEnginePath := filepath.Join(path, engineFileName(terragruntOptions.Engine))
+	localChecksumFile := filepath.Join(path, engineChecksumName(terragruntOptions.Engine))
+	localChecksumSigFile := filepath.Join(path, engineChecksumSigName(terragruntOptions.Engine))
+	// validate engine before loading if verification is not disabled
+	if !skipEngineCheck() && util.FileExists(localEnginePath) && util.FileExists(localChecksumFile) && util.FileExists(localChecksumSigFile) {
+		if err := verifyFile(localEnginePath, localChecksumFile, localChecksumSigFile); err != nil {
+			return nil, nil, errors.WithStackTrace(err)
+		}
+	} else {
+		terragruntOptions.Logger.Warnf("Skipping verification for %s", localEnginePath)
+	}
 	terragruntOptions.Logger.Debugf("Creating engine %s", localEnginePath)
 
 	logger := hclog.NewInterceptLogger(&hclog.LoggerOptions{
@@ -578,3 +621,9 @@ func convertMetaToProtobuf(meta map[string]interface{}) (map[string]*anypb.Any,
 	}
 	return protoMeta, nil
 }
+
+// skipChecksumCheck returns true if the engine checksum check is skipped.
+func skipEngineCheck() bool {
+	ok, _ := strconv.ParseBool(os.Getenv(EngineSkipCheckEnv)) //nolint:errcheck
+	return ok
+}

engine/public_keys.go

@@ -0,0 +1,43 @@
+package engine
+
+const PublicKey = `-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQGNBGaBbooBDADTCKKFW1uV5krG++w0u4QA7r2H6t39NfEKb8bbssM2oFIiTsEY
+6WQbddDAbzA9KFyIA47yga1nB3tOgih+4QwZF/Wctw63sfeKQ/kdT/p3lSwI1Rbq
+BuWJ0pSrZCsS8ldxNuel2Imnr3rZtB+jAWrfJio10T3paCy8HGE470ehXYpqlcUJ
+rOUxR4PTcLnWY0PrNfMgljXyFMLvqe1sG0LuPIH3ZbGZOzmdVyo/ngeJ9fluP8DC
+XZKEXqzGe58m6iJmDBUuRRV+LPVo8NrrVfF7waQrlGjaE4GZvvsmApxXv/iM9DIg
+NpZWE3vTH/pBSsc0HapVWD/DzYQpXhwcKWdF2wtpRrYOLFiXmdTHBga3xPDpXrID
+vPghYlW19j60A9o2MRzbjnPHvNwHKv5XPBIhLcoyWnNCp6WASTbiBRwDx3miM1ZT
+euQPagG68aGabkWdEH1Pa33ZEF5oDH7j9C9ALJlUhrk5zgFSRN5GKcm00K209g2M
+dlnvgWBjoUwYU0MAEQEAAbQdR3J1bnR3b3JrIDxpbmZvQGdydW50d29yay5pbz6J
+Ac4EEwEKADgWIQQbc6gAIzjCuyjbMPSvWWjac5v8XAUCZoFuigIbAwULCQgHAgYV
+CgkICwIEFgIDAQIeAQIXgAAKCRCvWWjac5v8XAwWC/9IptEC3WhW7j8BdBjDVy5W
+jaGb75PlL8pkQFBrfNPxiLGxuLi6xuON6zSIGtKZe4XTjwnVniyYyiyfSojrKCRT
+YCctVVvgoBaylybk8ppCysyID9xs0YqrhdCZvJyH+yLAXTdmkddzj906hRkW+xmq
+7XLA2emNxv6P4mHJr9pd4aa+aloZceRZ0OgUju+8E/ZTvW6A5YYExSFoNPBlG9nn
+WrT/D6aO9gqyMzN6w888p+jo+6s3JDQ6WEnf5s2Ha8g1k/Fg0Tk6YrbhcaYVQHEW
+WrSj9wVrXWa7RjRrZTREOMe9zLI3YHIsmBM0KNgHzvmyyhPsw1hR7MBJJfHKfpJ9
+SitdkyCFWlI/UZITEAcADZkRpvaixUvzIXAsk30aWGonCaXsqrdJwmpdLRJkC8xd
+W6D6rxDhqdVyxDRi7jas6mtOk7Ao7wFMDuedX1TB8yqkhvx96FaoG22Qoy4cma7C
+Zz2jO2+ix/xztd/wq56jl0DjgKqpk06lECy/9+niyim5AY0EZoFuigEMAL7fKX6T
+e1K3K1e/WcaqGNFUGYWxlZZoGhihUAotWYeseleQB5RUmj9lwazI9zH3pteke+lV
+VwPqRD9djsQOv/B28Q6YpOd7sDbqxM6GSXED61sBAsJyDvmm0p5X7bbKJeRxhrhV
+FJjFf9F3t5gZb5Kff0vNYzCPmemT7UFaNUwDbE9wjRl5oKZfyDeUBBXB9H8aFE0J
+wLyFTnPKSpedJx7IlTbnCCzhTn0H7TKAVNYwRpSYN2GOChMiowkJrqD22G9HVZth
+g/sBJlmAFvLy8Ed8ktbZ426Xm44WRS6MFglZJJKZSEXOdSla8F4GT0Zxd6hsc6A8
+bLHQw34mFVmGZ6Q81+z2L3MV+zA3Dug3kEgRpH6g++KCX3+gpgsEugxli176rO1M
+CtZMnyR1fWBI9W8CuNm4MysImBHdOO73IUIsT2wiv4RTTGaLhU0YIfIEyojAFgEm
+S9BKCgF4BTP9FTOxxMZmINFTzDqi/b51qPBxBs6DXa/E7muOePzclQIBowARAQAB
+iQG2BBgBCgAgFiEEG3OoACM4wrso2zD0r1lo2nOb/FwFAmaBbooCGwwACgkQr1lo
+2nOb/FzonAwAw1jzHGUMAIPuLZAQNhrhj05ZbuC2A7TvWiQba9W1HPHFUZJgrxKW
+KNPaWb8oCQR8JDJlWqiZG6hWTAJ66suPrLF0KNnbiZ5Us4+o7Nv5q1i4lxpJRgoY
+FuCDZbQHXPn3jzSEDQPSA62+ZyRGxXfpqgVPpT8IPzAdCRAhMuUZb62h+WX2ey91
+rnRFIXOlPTbrOMPLaMnGBjDnsWuCQmxBCXeevBh3u8q4Wa1xCZiqN8T7PSUusalu
+xita/w5ZA+Tzwxe8VqrgutCJdj5m5OxXX4v10xbbyPfhhnaahMduGL60MV/noxy6
+9TFlXIhgDj8dxV8wt8Tv/GZSSUALaBuPs9U7Q+fGiPFpC48Q75y0uS0QWXm0taxs
+Rm6AMowi8dJEq1BIKvdEO2lDJ1uSw5Xcamj6Nu0JrM8tc1uCFNYOAEHw2bDHIcHK
++uFqASiTa9QRj1SpSRkbPJB3yuPgUr1DgEoolSlMOnUiI46K3I9APD54nuShVbVq
+iE6bHk4c9kBU
+=TmYc
+-----END PGP PUBLIC KEY BLOCK-----`

engine/verification.go

@@ -0,0 +1,54 @@
+package engine
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/ProtonMail/go-crypto/openpgp"
+	"github.com/gruntwork-io/go-commons/errors"
+	"github.com/gruntwork-io/terragrunt/util"
+)
+
+// verifyFile verifies the checksums file and the signature file of the passed file
+func verifyFile(checkedFile, checksumsFile, signatureFile string) error {
+	checksums, err := os.ReadFile(checksumsFile)
+	if err != nil {
+		return errors.WithStackTrace(err)
+	}
+	checksumsSignature, err := os.ReadFile(signatureFile)
+	if err != nil {
+		return errors.WithStackTrace(err)
+	}
+	// validate first checksum file signature
+	keyring, err := openpgp.ReadArmoredKeyRing(strings.NewReader(PublicKey))
+	if err != nil {
+		return errors.WithStackTrace(err)
+	}
+	_, err = openpgp.CheckDetachedSignature(keyring, bytes.NewReader(checksums), bytes.NewReader(checksumsSignature), nil)
+	if err != nil {
+		return errors.WithStackTrace(err)
+	}
+	// verify checksums
+	// calculate checksum of package file
+	packageChecksum, err := util.FileSHA256(checkedFile)
+	if err != nil {
+		return errors.WithStackTrace(err)
+	}
+	// match expected checksum
+	expectedChecksum := util.MatchSha256Checksum(checksums, []byte(filepath.Base(checkedFile)))
+	if expectedChecksum == nil {
+		return errors.Errorf("checksum list has no entry for %s", checkedFile)
+	}
+	var expectedSHA256Sum [sha256.Size]byte
+	if _, err := hex.Decode(expectedSHA256Sum[:], expectedChecksum); err != nil {
+		return errors.WithStackTrace(err)
+	}
+	if !bytes.Equal(expectedSHA256Sum[:], packageChecksum) {
+		return errors.Errorf("checksum list has unexpected SHA-256 hash %x (expected %x)", packageChecksum, expectedSHA256Sum)
+	}
+	return nil
+}

terraform/getproviders/package_authentication.go

@@ -8,6 +8,8 @@ import (
 	"os"
 	"strings"
 
+	"github.com/gruntwork-io/terragrunt/util"
+
 	"github.com/gruntwork-io/go-commons/errors"
 	"github.com/gruntwork-io/terragrunt/pkg/log"
 
@@ -159,14 +161,7 @@ func NewMatchingChecksumAuthentication(document []byte, filename string, wantSHA
 func (auth matchingChecksumAuthentication) Authenticate(location string) (*PackageAuthenticationResult, error) {
 	// Find the checksum in the list with matching filename. The document is in the form "0123456789abcdef filename.zip".
 	filename := []byte(auth.Filename)
-	var checksum []byte
-	for _, line := range bytes.Split(auth.Document, []byte("\n")) {
-		parts := bytes.Fields(line)
-		if len(parts) > 1 && bytes.Equal(parts[1], filename) {
-			checksum = parts[0]
-			break
-		}
-	}
+	checksum := util.MatchSha256Checksum(auth.Document, filename)
 	if checksum == nil {
 		return nil, errors.Errorf("checksum list has no SHA-256 hash for %q", auth.Filename)
 	}

test/fixture-engine/opentofu-engine/terragrunt.hcl

@@ -1,6 +1,6 @@
 engine {
   source  = "github.com/gruntwork-io/terragrunt-engine-opentofu"
-  version = "v0.0.2"
+  version = "v0.0.4"
   type    = "rpc"
 }
 

test/fixture-engine/opentofu-run-all/terragrunt.hcl

@@ -1,5 +1,5 @@
 engine {
   source  = "github.com/gruntwork-io/terragrunt-engine-opentofu"
-  version = "v0.0.2"
+  version = "v0.0.4"
   type    = "rpc"
 }

test/fixture-engine/remote-engine/terragrunt.hcl

@@ -1,6 +1,6 @@
 engine {
   source  = "__hardcoded_url__"
-  version = "v0.0.2"
+  version = "v0.0.4"
 }
 
 inputs = {