feat: auto-plan projects when modules change

cmd/server.go

@@ -23,12 +23,13 @@ import (
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/moby/moby/pkg/fileutils"
 	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
 	"github.com/runatlantis/atlantis/server"
 	"github.com/runatlantis/atlantis/server/core/config/valid"
 	"github.com/runatlantis/atlantis/server/events/vcs/bitbucketcloud"
 	"github.com/runatlantis/atlantis/server/logging"
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
 )
 
 // To add a new flag you must:
@@ -46,6 +47,8 @@ const (
 	AllowRepoConfigFlag              = "allow-repo-config"
 	AtlantisURLFlag                  = "atlantis-url"
 	AutomergeFlag                    = "automerge"
+	AutoplanModules                  = "autoplan-modules"
+	AutoplanModulesFromProjects      = "autoplan-modules-from-projects"
 	AutoplanFileListFlag             = "autoplan-file-list"
 	BitbucketBaseURLFlag             = "bitbucket-base-url"
 	BitbucketTokenFlag               = "bitbucket-token"
@@ -177,6 +180,13 @@ var stringFlags = map[string]stringFlag{
 	AtlantisURLFlag: {
 		description: "URL that Atlantis can be reached at. Defaults to http://$(hostname):$port where $port is from --" + PortFlag + ". Supports a base path ex. https://example.com/basepath.",
 	},
+	AutoplanModulesFromProjects: {
+		description: "Comma separated list of file patterns to select projects Atlantis will index for module dependencies." +
+			" Indexed projects will automatically be planned if a module they depend on is modified." +
+			" Patterns use the dockerignore (https://docs.docker.com/engine/reference/builder/#dockerignore-file) syntax." +
+			" A custom Workflow that uses autoplan 'when_modified' will ignore this value.",
+		defaultValue: "",
+	},
 	AutoplanFileListFlag: {
 		description: "Comma separated list of file patterns that Atlantis will use to check if a directory contains modified files that should trigger project planning." +
 			" Patterns use the dockerignore (https://docs.docker.com/engine/reference/builder/#dockerignore-file) syntax." +
@@ -374,6 +384,10 @@ var boolFlags = map[string]boolFlag{
 		defaultValue: false,
 		hidden:       true,
 	},
+	AutoplanModules: {
+		description:  "Automatically plan projects that have a changed module from the local repository.",
+		defaultValue: false,
+	},
 	AutomergeFlag: {
 		description:  "Automatically merge pull requests when all plans are successfully applied.",
 		defaultValue: false,

cmd/server_test.go

@@ -22,13 +22,14 @@ import (
 	"testing"
 
 	homedir "github.com/mitchellh/go-homedir"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"gopkg.in/yaml.v2"
+
 	"github.com/runatlantis/atlantis/server"
 	"github.com/runatlantis/atlantis/server/events/vcs/fixtures"
 	"github.com/runatlantis/atlantis/server/logging"
 	. "github.com/runatlantis/atlantis/testing"
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-	"gopkg.in/yaml.v2"
 )
 
 // passedConfig is set to whatever config ended up being passed to NewServer.
@@ -764,6 +765,23 @@ func TestExecute_RepoWhitelistDeprecation(t *testing.T) {
 	Equals(t, "*", passedConfig.RepoAllowlist)
 }
 
+func TestExecute_AutoDetectModulesFromProjects_Env(t *testing.T) {
+	t.Setenv("ATLANTIS_AUTOPLAN_MODULES_FROM_PROJECTS", "**/init.tf")
+	c := setupWithDefaults(map[string]interface{}{}, t)
+	err := c.Execute()
+	Ok(t, err)
+	Equals(t, "**/init.tf", passedConfig.AutoplanModulesFromProjects)
+}
+
+func TestExecute_AutoDetectModulesFromProjects(t *testing.T) {
+	c := setupWithDefaults(map[string]interface{}{
+		AutoplanModulesFromProjects: "**/*.tf",
+	}, t)
+	err := c.Execute()
+	Ok(t, err)
+	Equals(t, "**/*.tf", passedConfig.AutoplanModulesFromProjects)
+}
+
 func TestExecute_AutoplanFileList(t *testing.T) {
 	cases := []struct {
 		description string

runatlantis.io/docs/autoplanning.md

@@ -8,7 +8,7 @@ The algorithm it uses is as follows:
 1. Get the directories that those files are in
 1. If the directory path doesn't contain `modules/` then try to run `plan` in that directory
 1. If it does contain `modules/` look at the directory one level above `modules/`. If it
-contains a `main.tf` run plan in that directory, otherwise ignore the change.
+contains a `main.tf` run plan in that directory, otherwise ignore the change (see below for exceptions).
 
 ## Example
 Given the directory structure:
@@ -27,6 +27,7 @@ Given the directory structure:
 * If `project1/main.tf` were modified, we would run `plan` in `project1`
 * If `modules/module1/main.tf` were modified, we would not automatically run `plan` because we couldn't determine the location of the terraform project
     * You could use an [atlantis.yaml](repo-level-atlantis-yaml.html#configuring-planning) file to specify which projects to plan when this module changed
+    * You could enable [module autoplanning](server-configuration.html#autoplan-modules) which indexes projects to their local module dependencies.
     * Or you could manually plan with `atlantis plan -d <dir>`
 * If `project1/modules/module1/main.tf` were modified, we would look one level above `project1/modules`
 into `project1/`, see that there was a `main.tf` file and so run plan in `project1/`

runatlantis.io/docs/server-configuration.md

@@ -134,6 +134,53 @@ Values are chosen in this order:
   * Autoplan when any `*.tf` files or `.yml` files in subfolder of `project1` is modified.
     * `--autoplan-file-list='**/*.tf,project2/**/*.yml'`
 
+
+::: warning NOTE
+By default, changes to modules will not trigger autoplanning. See the flags below. 
+:::
+
+### `--autoplan-modules`
+
+```bash
+atlantis server --autoplan-modules
+```
+
+Defaults to `false`. When set to `true`, Atlantis will trace the local modules of included projects.
+Included project are projects with files included by `--autoplan-file-list`. 
+After tracing, Atlantis will plan any project that includes a changed module. This is equivalent to setting
+`--autoplan-modules-from-projects` to the value of `--autoplan-file-list`. See below.
+
+### `--autoplan-modules-from-projects`
+
+```bash
+atlantis server --autoplan-modules-from-projects='**/init.tf'
+```
+
+Enables auto-planing of projects when a module dependency in the same repository has changed. 
+This is a list of file patterns like `autoplan-file-list`. 
+
+These patterns select **projects** to index based on the files matched. The index maps modules to the projects that depends on them,
+including projects that include the module via other modules. When a module file matching `autoplan-file-list` changes,
+all indexed projects will be planned.
+
+Current default is "" (disabled).
+
+Examples:
+
+ * `**/*.tf` - will index all projects that have a `.tf` file in their directory, and plan them whenever an in-repo module dependency has changed.
+ * `**/*.tf,!foo,!bar` - will index all projects containing `.tf` except `foo` and `bar` and plan them whenever an in-repo module dependency has changed.
+   This allows projects to opt-out of auto-planning when a module dependency changes.
+
+::: warning NOTE
+Modules that are not selected by autoplan-file-list will not be indexed and dependant projects will not be planned. This
+flag allows the *projects* to index to be selected, but the trigger for a plan must be a file in `autoplan-file-list`.
+:::
+
+::: warning NOTE
+This flag overrides `--autoplan-modules`. If you wish to disable auto-planning of modules, set this flag to an empty string,
+and set `--autoplan-modules` to `false`.
+:::
+
 ### `--azuredevops-hostname`
   ```bash
   atlantis server --azuredevops-hostname="dev.azure.com"

server/controllers/events/events_controller_e2e_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/hashicorp/go-getter"
 	"github.com/hashicorp/go-version"
 	. "github.com/petergtz/pegomock"
+
 	"github.com/runatlantis/atlantis/server"
 	events_controllers "github.com/runatlantis/atlantis/server/controllers/events"
 	"github.com/runatlantis/atlantis/server/core/config"
@@ -972,6 +973,7 @@ func setupE2E(t *testing.T, repoDir string) (events_controllers.VCSEventsControl
 		commentParser,
 		false,
 		false,
+		"",
 		"**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl",
 		statsScope,
 		logger,

server/controllers/events/events_controller_test.go

@@ -17,6 +17,15 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+
 	. "github.com/petergtz/pegomock"
 	events_controllers "github.com/runatlantis/atlantis/server/controllers/events"
 	"github.com/runatlantis/atlantis/server/controllers/events/mocks"
@@ -29,14 +38,6 @@ import (
 	"github.com/runatlantis/atlantis/server/metrics"
 	. "github.com/runatlantis/atlantis/testing"
 	gitlab "github.com/xanzy/go-gitlab"
-	"io"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"path/filepath"
-	"reflect"
-	"strings"
-	"testing"
 )
 
 const githubHeader = "X-Github-Event"

server/events/modules.go

@@ -0,0 +1,161 @@
+package events
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/hashicorp/terraform-config-inspect/tfconfig"
+	"github.com/moby/moby/pkg/fileutils"
+)
+
+type module struct {
+	// path to the module
+	path string
+	// dependencies of this module
+	dependencies map[string]bool
+	// projects that depend on this module
+	projects map[string]bool
+}
+
+func (m *module) String() string {
+	if m == nil {
+		return "nil"
+	}
+	return fmt.Sprintf("%+v", *m)
+}
+
+type ModuleProjects interface {
+	// DependentProjects returns all projects that depend on the module at moduleDir
+	DependentProjects(moduleDir string) []string
+}
+
+type moduleInfo map[string]*module
+
+var _ ModuleProjects = moduleInfo{}
+
+func (m moduleInfo) String() string {
+	return fmt.Sprintf("%+v", map[string]*module(m))
+}
+
+func (m moduleInfo) DependentProjects(moduleDir string) (projectPaths []string) {
+	if m == nil || m[moduleDir] == nil {
+		return nil
+	}
+	for project := range m[moduleDir].projects {
+		projectPaths = append(projectPaths, project)
+	}
+	return projectPaths
+}
+
+type tfFs struct {
+	fs.FS
+}
+
+func (t tfFs) Open(name string) (tfconfig.File, error) {
+	return t.FS.Open(name)
+}
+
+func (t tfFs) ReadFile(name string) ([]byte, error) {
+	return fs.ReadFile(t.FS, name)
+}
+
+func (t tfFs) ReadDir(dirname string) ([]os.FileInfo, error) {
+	ls, err := fs.ReadDir(t.FS, dirname)
+	if err != nil {
+		return nil, err
+	}
+	var infos []os.FileInfo
+	for _, l := range ls {
+		info, err := l.Info()
+		if err != nil {
+			return nil, fmt.Errorf("failed to get info for %s: %w", l.Name(), err)
+		}
+		infos = append(infos, info)
+	}
+	return infos, err
+}
+
+var _ tfconfig.FS = tfFs{}
+
+func (m moduleInfo) load(files fs.FS, dir string, projects ...string) (_ *module, diags tfconfig.Diagnostics) {
+	if _, set := m[dir]; !set {
+		tfFiles := tfFs{files}
+		var mod *tfconfig.Module
+		mod, diags = tfconfig.LoadModuleFromFilesystem(tfFiles, dir)
+
+		deps := make(map[string]bool)
+		if mod != nil {
+			for _, c := range mod.ModuleCalls {
+				mPath := path.Join(dir, c.Source)
+				if !tfconfig.IsModuleDirOnFilesystem(tfFiles, mPath) {
+					continue
+				}
+				deps[mPath] = true
+			}
+		}
+
+		m[dir] = &module{
+			path:         dir,
+			dependencies: deps,
+			projects:     make(map[string]bool),
+		}
+	}
+	// set projects on my dependencies
+	for dep := range m[dir].dependencies {
+		_, err := m.load(files, dep, projects...)
+		if err != nil {
+			diags = append(diags, err...)
+		}
+	}
+	// add projects to the list of dependant projects
+	for _, p := range projects {
+		m[dir].projects[p] = true
+	}
+	return m[dir], diags
+}
+
+// FindModuleProjects returns a mapping of modules to projects that depend on them.
+func FindModuleProjects(absRepoDir string, autoplanModuleDependants string) (ModuleProjects, error) {
+	return findModuleDependants(os.DirFS(absRepoDir), autoplanModuleDependants)
+}
+
+func findModuleDependants(files fs.FS, autoplanModuleDependants string) (ModuleProjects, error) {
+	if autoplanModuleDependants == "" {
+		return moduleInfo{}, nil
+	}
+	// find all the projects matching autoplanModuleDependants
+	filter, _ := fileutils.NewPatternMatcher(strings.Split(autoplanModuleDependants, ","))
+	var projects []string
+	err := fs.WalkDir(files, ".", func(rel string, info fs.DirEntry, err error) error {
+		if match, _ := filter.Matches(rel); match {
+			if projectDir := getProjectDirFromFs(files, rel); projectDir != "" {
+				projects = append(projects, projectDir)
+			}
+		}
+		return err
+	})
+	if err != nil {
+		return nil, fmt.Errorf("find projects for module dependants: %w", err)
+	}
+
+	result := make(moduleInfo)
+	var diags tfconfig.Diagnostics
+	// for each project, find the modules it depends on, their deps, etc.
+	for _, projectDir := range projects {
+		if _, err := result.load(files, projectDir, projectDir); err != nil {
+			diags = append(diags, err...)
+		}
+	}
+	// if there are any errors, prefer one with a source location
+	if diags.HasErrors() {
+		for _, d := range diags {
+			if d.Pos != nil {
+				return nil, fmt.Errorf("%s:%d - %s: %s", d.Pos.Filename, d.Pos.Line, d.Summary, d.Detail)
+			}
+		}
+	}
+	return result, diags.Err()
+}