diff --git a/cmd/server.go b/cmd/server.go index be5dd3b8f6..98d7a0257b 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -23,12 +23,13 @@ import ( homedir "github.com/mitchellh/go-homedir" "github.com/moby/moby/pkg/fileutils" "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "github.com/runatlantis/atlantis/server" "github.com/runatlantis/atlantis/server/core/config/valid" "github.com/runatlantis/atlantis/server/events/vcs/bitbucketcloud" "github.com/runatlantis/atlantis/server/logging" - "github.com/spf13/cobra" - "github.com/spf13/viper" ) // To add a new flag you must: @@ -46,6 +47,8 @@ const ( AllowRepoConfigFlag = "allow-repo-config" AtlantisURLFlag = "atlantis-url" AutomergeFlag = "automerge" + AutoplanModules = "autoplan-modules" + AutoplanModulesFromProjects = "autoplan-modules-from-projects" AutoplanFileListFlag = "autoplan-file-list" BitbucketBaseURLFlag = "bitbucket-base-url" BitbucketTokenFlag = "bitbucket-token" @@ -177,6 +180,13 @@ var stringFlags = map[string]stringFlag{ AtlantisURLFlag: { description: "URL that Atlantis can be reached at. Defaults to http://$(hostname):$port where $port is from --" + PortFlag + ". Supports a base path ex. https://example.com/basepath.", }, + AutoplanModulesFromProjects: { + description: "Comma separated list of file patterns to select projects Atlantis will index for module dependencies." + + " Indexed projects will automatically be planned if a module they depend on is modified." + + " Patterns use the dockerignore (https://docs.docker.com/engine/reference/builder/#dockerignore-file) syntax." + + " A custom Workflow that uses autoplan 'when_modified' will ignore this value.", + defaultValue: "", + }, AutoplanFileListFlag: { description: "Comma separated list of file patterns that Atlantis will use to check if a directory contains modified files that should trigger project planning." + " Patterns use the dockerignore (https://docs.docker.com/engine/reference/builder/#dockerignore-file) syntax." + @@ -374,6 +384,10 @@ var boolFlags = map[string]boolFlag{ defaultValue: false, hidden: true, }, + AutoplanModules: { + description: "Automatically plan projects that have a changed module from the local repository.", + defaultValue: false, + }, AutomergeFlag: { description: "Automatically merge pull requests when all plans are successfully applied.", defaultValue: false, diff --git a/cmd/server_test.go b/cmd/server_test.go index 09941e4aba..e2741fc50a 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -22,13 +22,14 @@ import ( "testing" homedir "github.com/mitchellh/go-homedir" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "gopkg.in/yaml.v2" + "github.com/runatlantis/atlantis/server" "github.com/runatlantis/atlantis/server/events/vcs/fixtures" "github.com/runatlantis/atlantis/server/logging" . "github.com/runatlantis/atlantis/testing" - "github.com/spf13/cobra" - "github.com/spf13/viper" - "gopkg.in/yaml.v2" ) // passedConfig is set to whatever config ended up being passed to NewServer. @@ -764,6 +765,23 @@ func TestExecute_RepoWhitelistDeprecation(t *testing.T) { Equals(t, "*", passedConfig.RepoAllowlist) } +func TestExecute_AutoDetectModulesFromProjects_Env(t *testing.T) { + t.Setenv("ATLANTIS_AUTOPLAN_MODULES_FROM_PROJECTS", "**/init.tf") + c := setupWithDefaults(map[string]interface{}{}, t) + err := c.Execute() + Ok(t, err) + Equals(t, "**/init.tf", passedConfig.AutoplanModulesFromProjects) +} + +func TestExecute_AutoDetectModulesFromProjects(t *testing.T) { + c := setupWithDefaults(map[string]interface{}{ + AutoplanModulesFromProjects: "**/*.tf", + }, t) + err := c.Execute() + Ok(t, err) + Equals(t, "**/*.tf", passedConfig.AutoplanModulesFromProjects) +} + func TestExecute_AutoplanFileList(t *testing.T) { cases := []struct { description string diff --git a/runatlantis.io/docs/autoplanning.md b/runatlantis.io/docs/autoplanning.md index 9ababb10fd..3461013980 100644 --- a/runatlantis.io/docs/autoplanning.md +++ b/runatlantis.io/docs/autoplanning.md @@ -8,7 +8,7 @@ The algorithm it uses is as follows: 1. Get the directories that those files are in 1. If the directory path doesn't contain `modules/` then try to run `plan` in that directory 1. If it does contain `modules/` look at the directory one level above `modules/`. If it -contains a `main.tf` run plan in that directory, otherwise ignore the change. +contains a `main.tf` run plan in that directory, otherwise ignore the change (see below for exceptions). ## Example Given the directory structure: @@ -27,6 +27,7 @@ Given the directory structure: * If `project1/main.tf` were modified, we would run `plan` in `project1` * If `modules/module1/main.tf` were modified, we would not automatically run `plan` because we couldn't determine the location of the terraform project * You could use an [atlantis.yaml](repo-level-atlantis-yaml.html#configuring-planning) file to specify which projects to plan when this module changed + * You could enable [module autoplanning](server-configuration.html#autoplan-modules) which indexes projects to their local module dependencies. * Or you could manually plan with `atlantis plan -d ` * If `project1/modules/module1/main.tf` were modified, we would look one level above `project1/modules` into `project1/`, see that there was a `main.tf` file and so run plan in `project1/` diff --git a/runatlantis.io/docs/server-configuration.md b/runatlantis.io/docs/server-configuration.md index 9081f74719..2986ca5d51 100644 --- a/runatlantis.io/docs/server-configuration.md +++ b/runatlantis.io/docs/server-configuration.md @@ -144,6 +144,53 @@ Values are chosen in this order: * Autoplan when any `*.tf` files or `.yml` files in subfolder of `project1` is modified. * `--autoplan-file-list='**/*.tf,project2/**/*.yml'` + +::: warning NOTE +By default, changes to modules will not trigger autoplanning. See the flags below. +::: + +### `--autoplan-modules` + +```bash +atlantis server --autoplan-modules +``` + +Defaults to `false`. When set to `true`, Atlantis will trace the local modules of included projects. +Included project are projects with files included by `--autoplan-file-list`. +After tracing, Atlantis will plan any project that includes a changed module. This is equivalent to setting +`--autoplan-modules-from-projects` to the value of `--autoplan-file-list`. See below. + +### `--autoplan-modules-from-projects` + +```bash +atlantis server --autoplan-modules-from-projects='**/init.tf' +``` + +Enables auto-planing of projects when a module dependency in the same repository has changed. +This is a list of file patterns like `autoplan-file-list`. + +These patterns select **projects** to index based on the files matched. The index maps modules to the projects that depends on them, +including projects that include the module via other modules. When a module file matching `autoplan-file-list` changes, +all indexed projects will be planned. + +Current default is "" (disabled). + +Examples: + + * `**/*.tf` - will index all projects that have a `.tf` file in their directory, and plan them whenever an in-repo module dependency has changed. + * `**/*.tf,!foo,!bar` - will index all projects containing `.tf` except `foo` and `bar` and plan them whenever an in-repo module dependency has changed. + This allows projects to opt-out of auto-planning when a module dependency changes. + +::: warning NOTE +Modules that are not selected by autoplan-file-list will not be indexed and dependant projects will not be planned. This +flag allows the *projects* to index to be selected, but the trigger for a plan must be a file in `autoplan-file-list`. +::: + +::: warning NOTE +This flag overrides `--autoplan-modules`. If you wish to disable auto-planning of modules, set this flag to an empty string, +and set `--autoplan-modules` to `false`. +::: + ### `--azuredevops-hostname` ```bash atlantis server --azuredevops-hostname="dev.azure.com" diff --git a/server/controllers/events/events_controller_e2e_test.go b/server/controllers/events/events_controller_e2e_test.go index 1f101fe2d2..b9a2ebf867 100644 --- a/server/controllers/events/events_controller_e2e_test.go +++ b/server/controllers/events/events_controller_e2e_test.go @@ -16,6 +16,7 @@ import ( "github.com/hashicorp/go-getter" "github.com/hashicorp/go-version" . "github.com/petergtz/pegomock" + "github.com/runatlantis/atlantis/server" events_controllers "github.com/runatlantis/atlantis/server/controllers/events" "github.com/runatlantis/atlantis/server/core/config" @@ -972,6 +973,7 @@ func setupE2E(t *testing.T, repoDir string) (events_controllers.VCSEventsControl commentParser, false, false, + "", "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", statsScope, logger, diff --git a/server/controllers/events/events_controller_test.go b/server/controllers/events/events_controller_test.go index 84c5a32875..c974da309c 100644 --- a/server/controllers/events/events_controller_test.go +++ b/server/controllers/events/events_controller_test.go @@ -17,6 +17,15 @@ import ( "bytes" "errors" "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + . "github.com/petergtz/pegomock" events_controllers "github.com/runatlantis/atlantis/server/controllers/events" "github.com/runatlantis/atlantis/server/controllers/events/mocks" @@ -29,14 +38,6 @@ import ( "github.com/runatlantis/atlantis/server/metrics" . "github.com/runatlantis/atlantis/testing" gitlab "github.com/xanzy/go-gitlab" - "io" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "reflect" - "strings" - "testing" ) const githubHeader = "X-Github-Event" diff --git a/server/events/modules.go b/server/events/modules.go new file mode 100644 index 0000000000..b7144bbac8 --- /dev/null +++ b/server/events/modules.go @@ -0,0 +1,161 @@ +package events + +import ( + "fmt" + "io/fs" + "os" + "path" + "strings" + + "github.com/hashicorp/terraform-config-inspect/tfconfig" + "github.com/moby/moby/pkg/fileutils" +) + +type module struct { + // path to the module + path string + // dependencies of this module + dependencies map[string]bool + // projects that depend on this module + projects map[string]bool +} + +func (m *module) String() string { + if m == nil { + return "nil" + } + return fmt.Sprintf("%+v", *m) +} + +type ModuleProjects interface { + // DependentProjects returns all projects that depend on the module at moduleDir + DependentProjects(moduleDir string) []string +} + +type moduleInfo map[string]*module + +var _ ModuleProjects = moduleInfo{} + +func (m moduleInfo) String() string { + return fmt.Sprintf("%+v", map[string]*module(m)) +} + +func (m moduleInfo) DependentProjects(moduleDir string) (projectPaths []string) { + if m == nil || m[moduleDir] == nil { + return nil + } + for project := range m[moduleDir].projects { + projectPaths = append(projectPaths, project) + } + return projectPaths +} + +type tfFs struct { + fs.FS +} + +func (t tfFs) Open(name string) (tfconfig.File, error) { + return t.FS.Open(name) +} + +func (t tfFs) ReadFile(name string) ([]byte, error) { + return fs.ReadFile(t.FS, name) +} + +func (t tfFs) ReadDir(dirname string) ([]os.FileInfo, error) { + ls, err := fs.ReadDir(t.FS, dirname) + if err != nil { + return nil, err + } + var infos []os.FileInfo + for _, l := range ls { + info, err := l.Info() + if err != nil { + return nil, fmt.Errorf("failed to get info for %s: %w", l.Name(), err) + } + infos = append(infos, info) + } + return infos, err +} + +var _ tfconfig.FS = tfFs{} + +func (m moduleInfo) load(files fs.FS, dir string, projects ...string) (_ *module, diags tfconfig.Diagnostics) { + if _, set := m[dir]; !set { + tfFiles := tfFs{files} + var mod *tfconfig.Module + mod, diags = tfconfig.LoadModuleFromFilesystem(tfFiles, dir) + + deps := make(map[string]bool) + if mod != nil { + for _, c := range mod.ModuleCalls { + mPath := path.Join(dir, c.Source) + if !tfconfig.IsModuleDirOnFilesystem(tfFiles, mPath) { + continue + } + deps[mPath] = true + } + } + + m[dir] = &module{ + path: dir, + dependencies: deps, + projects: make(map[string]bool), + } + } + // set projects on my dependencies + for dep := range m[dir].dependencies { + _, err := m.load(files, dep, projects...) + if err != nil { + diags = append(diags, err...) + } + } + // add projects to the list of dependant projects + for _, p := range projects { + m[dir].projects[p] = true + } + return m[dir], diags +} + +// FindModuleProjects returns a mapping of modules to projects that depend on them. +func FindModuleProjects(absRepoDir string, autoplanModuleDependants string) (ModuleProjects, error) { + return findModuleDependants(os.DirFS(absRepoDir), autoplanModuleDependants) +} + +func findModuleDependants(files fs.FS, autoplanModuleDependants string) (ModuleProjects, error) { + if autoplanModuleDependants == "" { + return moduleInfo{}, nil + } + // find all the projects matching autoplanModuleDependants + filter, _ := fileutils.NewPatternMatcher(strings.Split(autoplanModuleDependants, ",")) + var projects []string + err := fs.WalkDir(files, ".", func(rel string, info fs.DirEntry, err error) error { + if match, _ := filter.Matches(rel); match { + if projectDir := getProjectDirFromFs(files, rel); projectDir != "" { + projects = append(projects, projectDir) + } + } + return err + }) + if err != nil { + return nil, fmt.Errorf("find projects for module dependants: %w", err) + } + + result := make(moduleInfo) + var diags tfconfig.Diagnostics + // for each project, find the modules it depends on, their deps, etc. + for _, projectDir := range projects { + if _, err := result.load(files, projectDir, projectDir); err != nil { + diags = append(diags, err...) + } + } + // if there are any errors, prefer one with a source location + if diags.HasErrors() { + for _, d := range diags { + if d.Pos != nil { + return nil, fmt.Errorf("%s:%d - %s: %s", d.Pos.Filename, d.Pos.Line, d.Summary, d.Detail) + } + } + } + return result, diags.Err() +} diff --git a/server/events/modules_test.go b/server/events/modules_test.go new file mode 100644 index 0000000000..3f7770c033 --- /dev/null +++ b/server/events/modules_test.go @@ -0,0 +1,71 @@ +package events + +import ( + "embed" + "fmt" + "io/fs" + "sort" + "testing" + + "github.com/stretchr/testify/assert" +) + +//go:embed testdata/fs +var repos embed.FS + +func Test_findModuleDependants(t *testing.T) { + + type args struct { + files fs.FS + autoplanModuleDependants string + } + a, err := fs.Sub(repos, "testdata/fs/repoA") + assert.NoError(t, err) + b, err := fs.Sub(repos, "testdata/fs/repoB") + assert.NoError(t, err) + + tests := []struct { + name string + args args + want map[string][]string + wantErr assert.ErrorAssertionFunc + }{ + { + name: "repoA", + args: args{ + files: a, + autoplanModuleDependants: "**/init.tf", + }, + want: map[string][]string{ + "modules/bar": {"baz", "qux/quxx"}, + "modules/foo": {"qux/quxx"}, + }, + wantErr: assert.NoError, + }, + { + name: "repoB", + args: args{ + files: b, + autoplanModuleDependants: "**/init.tf", + }, + want: map[string][]string{ + "modules/bar": {"dev/quxx", "prod/quxx"}, + "modules/foo": {"dev/quxx", "prod/quxx"}, + }, + wantErr: assert.NoError, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := findModuleDependants(tt.args.files, tt.args.autoplanModuleDependants) + if !tt.wantErr(t, err, fmt.Sprintf("findModuleDependants(%v, %v)", tt.args.files, tt.args.autoplanModuleDependants)) { + return + } + for k, v := range tt.want { + projects := got.DependentProjects(k) + sort.Strings(projects) + assert.Equalf(t, v, projects, "%v.DownstreamProjects(%v)", got, k) + } + }) + } +} diff --git a/server/events/project_command_builder.go b/server/events/project_command_builder.go index a8af5669e7..f2362fc170 100644 --- a/server/events/project_command_builder.go +++ b/server/events/project_command_builder.go @@ -5,11 +5,13 @@ import ( "os" "sort" + "github.com/uber-go/tally" + "github.com/runatlantis/atlantis/server/core/config/valid" "github.com/runatlantis/atlantis/server/logging" - "github.com/uber-go/tally" "github.com/pkg/errors" + "github.com/runatlantis/atlantis/server/core/config" "github.com/runatlantis/atlantis/server/events/command" "github.com/runatlantis/atlantis/server/events/vcs" @@ -44,6 +46,7 @@ func NewInstrumentedProjectCommandBuilder( commentBuilder CommentBuilder, skipCloneNoChanges bool, EnableRegExpCmd bool, + AutoDetectModuleFiles string, AutoplanFileList string, scope tally.Scope, logger logging.SimpleLogging, @@ -61,6 +64,7 @@ func NewInstrumentedProjectCommandBuilder( commentBuilder, skipCloneNoChanges, EnableRegExpCmd, + AutoDetectModuleFiles, AutoplanFileList, scope, logger, @@ -81,21 +85,23 @@ func NewProjectCommandBuilder( commentBuilder CommentBuilder, skipCloneNoChanges bool, EnableRegExpCmd bool, + AutoDetectModuleFiles string, AutoplanFileList string, scope tally.Scope, logger logging.SimpleLogging, ) *DefaultProjectCommandBuilder { return &DefaultProjectCommandBuilder{ - ParserValidator: parserValidator, - ProjectFinder: projectFinder, - VCSClient: vcsClient, - WorkingDir: workingDir, - WorkingDirLocker: workingDirLocker, - GlobalCfg: globalCfg, - PendingPlanFinder: pendingPlanFinder, - SkipCloneNoChanges: skipCloneNoChanges, - EnableRegExpCmd: EnableRegExpCmd, - AutoplanFileList: AutoplanFileList, + ParserValidator: parserValidator, + ProjectFinder: projectFinder, + VCSClient: vcsClient, + WorkingDir: workingDir, + WorkingDirLocker: workingDirLocker, + GlobalCfg: globalCfg, + PendingPlanFinder: pendingPlanFinder, + SkipCloneNoChanges: skipCloneNoChanges, + EnableRegExpCmd: EnableRegExpCmd, + AutoDetectModuleFiles: AutoDetectModuleFiles, + AutoplanFileList: AutoplanFileList, ProjectCommandContextBuilder: NewProjectCommandContextBuilder( policyChecksSupported, commentBuilder, @@ -157,6 +163,7 @@ type DefaultProjectCommandBuilder struct { ProjectCommandContextBuilder ProjectCommandContextBuilder SkipCloneNoChanges bool EnableRegExpCmd bool + AutoDetectModuleFiles string AutoplanFileList string EnableDiffMarkdownFormat bool } @@ -305,7 +312,13 @@ func (p *DefaultProjectCommandBuilder) buildPlanAllCommands(ctx *command.Context // If there is no config file, then we'll plan each project that // our algorithm determines was modified. ctx.Log.Info("found no %s file", config.AtlantisYAMLFilename) - modifiedProjects := p.ProjectFinder.DetermineProjects(ctx.Log, modifiedFiles, ctx.Pull.BaseRepo.FullName, repoDir, p.AutoplanFileList) + // build a module index for projects that are explicitly included + moduleInfo, err := FindModuleProjects(repoDir, p.AutoDetectModuleFiles) + if err != nil { + ctx.Log.Warn("error(s) loading project module dependencies: %s", err) + } + ctx.Log.Debug("moduleInfo for %s (matching %q) = %v", repoDir, p.AutoDetectModuleFiles, moduleInfo) + modifiedProjects := p.ProjectFinder.DetermineProjects(ctx.Log, modifiedFiles, ctx.Pull.BaseRepo.FullName, repoDir, p.AutoplanFileList, moduleInfo) ctx.Log.Info("automatically determined that there were %d projects modified in this pull request: %s", len(modifiedProjects), modifiedProjects) for _, mp := range modifiedProjects { ctx.Log.Debug("determining config for project at dir: %q", mp.Path) diff --git a/server/events/project_command_builder_internal_test.go b/server/events/project_command_builder_internal_test.go index cd83042216..e8b3b29d79 100644 --- a/server/events/project_command_builder_internal_test.go +++ b/server/events/project_command_builder_internal_test.go @@ -7,6 +7,7 @@ import ( version "github.com/hashicorp/go-version" . "github.com/petergtz/pegomock" + "github.com/runatlantis/atlantis/server/core/config" "github.com/runatlantis/atlantis/server/core/config/valid" "github.com/runatlantis/atlantis/server/events/command" @@ -624,6 +625,7 @@ projects: &CommentParser{}, false, false, + "", "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", statsScope, logger, @@ -825,6 +827,7 @@ projects: &CommentParser{}, false, true, + "", "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", statsScope, logger, @@ -1054,6 +1057,7 @@ workflows: &CommentParser{}, false, false, + "", "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", statsScope, logger, diff --git a/server/events/project_command_builder_test.go b/server/events/project_command_builder_test.go index 12bb03aad5..85ff284bd4 100644 --- a/server/events/project_command_builder_test.go +++ b/server/events/project_command_builder_test.go @@ -8,6 +8,7 @@ import ( "testing" . "github.com/petergtz/pegomock" + "github.com/runatlantis/atlantis/server/core/config" "github.com/runatlantis/atlantis/server/core/config/valid" "github.com/runatlantis/atlantis/server/events" @@ -157,6 +158,7 @@ projects: &events.CommentParser{}, false, false, + "", "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", scope, logger, @@ -424,6 +426,7 @@ projects: &events.CommentParser{}, false, true, + "", "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", scope, logger, @@ -578,6 +581,7 @@ projects: &events.CommentParser{}, false, false, + "", "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", scope, logger, @@ -668,6 +672,7 @@ func TestDefaultProjectCommandBuilder_BuildMultiApply(t *testing.T) { &events.CommentParser{}, false, false, + "", "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", scope, logger, @@ -752,6 +757,7 @@ projects: &events.CommentParser{}, false, false, + "", "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", scope, logger, @@ -830,6 +836,7 @@ func TestDefaultProjectCommandBuilder_EscapeArgs(t *testing.T) { &events.CommentParser{}, false, false, + "", "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", scope, logger, @@ -1033,6 +1040,7 @@ projects: &events.CommentParser{}, false, false, + "", "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", scope, logger, @@ -1100,6 +1108,7 @@ projects: &events.CommentParser{}, true, false, + "", "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", scope, logger, @@ -1157,6 +1166,7 @@ func TestDefaultProjectCommandBuilder_WithPolicyCheckEnabled_BuildAutoplanComman &events.CommentParser{}, false, false, + "", "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", scope, logger, @@ -1238,6 +1248,7 @@ func TestDefaultProjectCommandBuilder_BuildVersionCommand(t *testing.T) { &events.CommentParser{}, false, false, + "", "**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl", scope, logger, diff --git a/server/events/project_finder.go b/server/events/project_finder.go index fed2d7eba0..fffb83fba3 100644 --- a/server/events/project_finder.go +++ b/server/events/project_finder.go @@ -15,6 +15,7 @@ package events import ( "fmt" + "io/fs" "os" "path" "path/filepath" @@ -24,6 +25,7 @@ import ( "github.com/moby/moby/pkg/fileutils" "github.com/pkg/errors" + "github.com/runatlantis/atlantis/server/events/models" "github.com/runatlantis/atlantis/server/logging" @@ -38,7 +40,7 @@ type ProjectFinder interface { // DetermineProjects returns the list of projects that were modified based on // the modifiedFiles. The list will be de-duplicated. // absRepoDir is the path to the cloned repo on disk. - DetermineProjects(log logging.SimpleLogging, modifiedFiles []string, repoFullName string, absRepoDir string, autoplanFileList string) []models.Project + DetermineProjects(log logging.SimpleLogging, modifiedFiles []string, repoFullName string, absRepoDir string, autoplanFileList string, moduleInfo ModuleProjects) []models.Project // DetermineProjectsViaConfig returns the list of projects that were modified // based on modifiedFiles and the repo's config. // absRepoDir is the path to the cloned repo on disk. @@ -134,7 +136,7 @@ var ignoredFilenameFragments = []string{"terraform.tfstate", "terraform.tfstate. type DefaultProjectFinder struct{} // See ProjectFinder.DetermineProjects. -func (p *DefaultProjectFinder) DetermineProjects(log logging.SimpleLogging, modifiedFiles []string, repoFullName string, absRepoDir string, autoplanFileList string) []models.Project { +func (p *DefaultProjectFinder) DetermineProjects(log logging.SimpleLogging, modifiedFiles []string, repoFullName string, absRepoDir string, autoplanFileList string, moduleInfo ModuleProjects) []models.Project { var projects []models.Project modifiedTerraformFiles := p.filterToFileList(log, modifiedFiles, autoplanFileList) @@ -146,9 +148,13 @@ func (p *DefaultProjectFinder) DetermineProjects(log logging.SimpleLogging, modi var dirs []string for _, modifiedFile := range modifiedTerraformFiles { - projectDir := p.getProjectDir(modifiedFile, absRepoDir) + projectDir := getProjectDir(modifiedFile, absRepoDir) if projectDir != "" { dirs = append(dirs, projectDir) + } else if moduleInfo != nil { + downstreamProjects := moduleInfo.DependentProjects(path.Dir(modifiedFile)) + log.Debug("found downstream projects for %q: %v", modifiedFile, downstreamProjects) + dirs = append(dirs, downstreamProjects...) } } uniqueDirs := p.unique(dirs) @@ -270,7 +276,11 @@ func (p *DefaultProjectFinder) shouldIgnore(fileName string) bool { // if the root is valid by looking for a main.tf file. It returns a relative // path to the repo. If the project is at the root returns ".". If modified file // doesn't lead to a valid project path, returns an empty string. -func (p *DefaultProjectFinder) getProjectDir(modifiedFilePath string, repoDir string) string { +func getProjectDir(modifiedFilePath string, repoDir string) string { + return getProjectDirFromFs(os.DirFS(repoDir), modifiedFilePath) +} + +func getProjectDirFromFs(files fs.FS, modifiedFilePath string) string { dir := path.Dir(modifiedFilePath) if path.Base(dir) == "env" { // If the modified file was inside an env/ directory, we treat this @@ -286,7 +296,7 @@ func (p *DefaultProjectFinder) getProjectDir(modifiedFilePath string, repoDir st // Surrounding dir with /'s so we can match on /modules/ even if dir is // "modules" or "project1/modules" - if strings.Contains("/"+dir+"/", "/modules/") { + if isModule(dir) { // We treat changes inside modules/ folders specially. There are two cases: // 1. modules folder inside project: // root/ @@ -318,7 +328,7 @@ func (p *DefaultProjectFinder) getProjectDir(modifiedFilePath string, repoDir st modulesParent := modulesSplit[0] // Now we check whether there is a main.tf in the parent. - if _, err := os.Stat(filepath.Join(repoDir, modulesParent, "main.tf")); os.IsNotExist(err) { + if _, err := fs.Stat(files, filepath.Join(modulesParent, "main.tf")); errors.Is(err, fs.ErrNotExist) { return "" } return path.Clean(modulesParent) @@ -329,6 +339,10 @@ func (p *DefaultProjectFinder) getProjectDir(modifiedFilePath string, repoDir st return dir } +func isModule(dir string) bool { + return strings.Contains("/"+dir+"/", "/modules/") +} + // unique de-duplicates strs. func (p *DefaultProjectFinder) unique(strs []string) []string { hash := make(map[string]bool) diff --git a/server/events/project_finder_test.go b/server/events/project_finder_test.go index f26e12451a..c88b0f123a 100644 --- a/server/events/project_finder_test.go +++ b/server/events/project_finder_test.go @@ -292,7 +292,7 @@ func TestDetermineProjects(t *testing.T) { } for _, c := range cases { t.Run(c.description, func(t *testing.T) { - projects := m.DetermineProjects(noopLogger, c.files, modifiedRepo, c.repoDir, c.autoplanFileList) + projects := m.DetermineProjects(noopLogger, c.files, modifiedRepo, c.repoDir, c.autoplanFileList, nil) // Extract the paths from the projects. We use a slice here instead of a // map so we can test whether there are duplicates returned. diff --git a/server/events/testdata/fs/repoA/baz/init.tf b/server/events/testdata/fs/repoA/baz/init.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/server/events/testdata/fs/repoA/baz/mods.tf b/server/events/testdata/fs/repoA/baz/mods.tf new file mode 100644 index 0000000000..88871cc19f --- /dev/null +++ b/server/events/testdata/fs/repoA/baz/mods.tf @@ -0,0 +1,3 @@ +module "bar" { + source = "../modules/bar" +} diff --git a/server/events/testdata/fs/repoA/modules/bar/bar.tf b/server/events/testdata/fs/repoA/modules/bar/bar.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/server/events/testdata/fs/repoA/modules/foo/foo.tf b/server/events/testdata/fs/repoA/modules/foo/foo.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/server/events/testdata/fs/repoA/modules/foo/mods.tf b/server/events/testdata/fs/repoA/modules/foo/mods.tf new file mode 100644 index 0000000000..fc5f7d817f --- /dev/null +++ b/server/events/testdata/fs/repoA/modules/foo/mods.tf @@ -0,0 +1,3 @@ +module "bar" { + source = "../bar" +} diff --git a/server/events/testdata/fs/repoA/qux/quxx/init.tf b/server/events/testdata/fs/repoA/qux/quxx/init.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/server/events/testdata/fs/repoA/qux/quxx/mods.tf b/server/events/testdata/fs/repoA/qux/quxx/mods.tf new file mode 100644 index 0000000000..7627c2d9d1 --- /dev/null +++ b/server/events/testdata/fs/repoA/qux/quxx/mods.tf @@ -0,0 +1,3 @@ +module "foo" { + source = "../../modules/foo" +} \ No newline at end of file diff --git a/server/events/testdata/fs/repoB/dev/quxx/init.tf b/server/events/testdata/fs/repoB/dev/quxx/init.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/server/events/testdata/fs/repoB/dev/quxx/mods.tf b/server/events/testdata/fs/repoB/dev/quxx/mods.tf new file mode 100644 index 0000000000..0e1c73f19b --- /dev/null +++ b/server/events/testdata/fs/repoB/dev/quxx/mods.tf @@ -0,0 +1,3 @@ +module "foo" { + source = "../../modules/foo" +} diff --git a/server/events/testdata/fs/repoB/modules/bar/bar.tf b/server/events/testdata/fs/repoB/modules/bar/bar.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/server/events/testdata/fs/repoB/modules/foo/foo.tf b/server/events/testdata/fs/repoB/modules/foo/foo.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/server/events/testdata/fs/repoB/modules/foo/mods.tf b/server/events/testdata/fs/repoB/modules/foo/mods.tf new file mode 100644 index 0000000000..fc5f7d817f --- /dev/null +++ b/server/events/testdata/fs/repoB/modules/foo/mods.tf @@ -0,0 +1,3 @@ +module "bar" { + source = "../bar" +} diff --git a/server/events/testdata/fs/repoB/prod/quxx/init.tf b/server/events/testdata/fs/repoB/prod/quxx/init.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/server/events/testdata/fs/repoB/prod/quxx/mods.tf b/server/events/testdata/fs/repoB/prod/quxx/mods.tf new file mode 100644 index 0000000000..7627c2d9d1 --- /dev/null +++ b/server/events/testdata/fs/repoB/prod/quxx/mods.tf @@ -0,0 +1,3 @@ +module "foo" { + source = "../../modules/foo" +} \ No newline at end of file diff --git a/server/server.go b/server/server.go index 96c8469ba6..fa2eff1638 100644 --- a/server/server.go +++ b/server/server.go @@ -33,6 +33,9 @@ import ( "time" "github.com/mitchellh/go-homedir" + "github.com/uber-go/tally" + "github.com/uber-go/tally/prometheus" + cfg "github.com/runatlantis/atlantis/server/core/config" "github.com/runatlantis/atlantis/server/core/config/valid" "github.com/runatlantis/atlantis/server/core/db" @@ -40,12 +43,13 @@ import ( "github.com/runatlantis/atlantis/server/jobs" "github.com/runatlantis/atlantis/server/metrics" "github.com/runatlantis/atlantis/server/scheduled" - "github.com/uber-go/tally" - "github.com/uber-go/tally/prometheus" assetfs "github.com/elazarl/go-bindata-assetfs" "github.com/gorilla/mux" "github.com/pkg/errors" + "github.com/urfave/cli" + "github.com/urfave/negroni" + "github.com/runatlantis/atlantis/server/controllers" events_controllers "github.com/runatlantis/atlantis/server/controllers/events" "github.com/runatlantis/atlantis/server/controllers/templates" @@ -63,8 +67,6 @@ import ( "github.com/runatlantis/atlantis/server/events/webhooks" "github.com/runatlantis/atlantis/server/logging" "github.com/runatlantis/atlantis/server/static" - "github.com/urfave/cli" - "github.com/urfave/negroni" ) const ( @@ -318,6 +320,12 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { } } + // default the project files used to generate the module index to the autoplan-file-list if autoplan-modules is true + // but no files are specified + if userConfig.AutoplanModules && userConfig.AutoplanModulesFromProjects == "" { + userConfig.AutoplanModulesFromProjects = userConfig.AutoplanFileList + } + var webhooksConfig []webhooks.Config for _, c := range userConfig.Webhooks { config := webhooks.Config{ @@ -533,6 +541,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { commentParser, userConfig.SkipCloneNoChanges, userConfig.EnableRegExpCmd, + userConfig.AutoplanModulesFromProjects, userConfig.AutoplanFileList, statsScope, logger, diff --git a/server/user_config.go b/server/user_config.go index 3abd34d86c..a635bd0593 100644 --- a/server/user_config.go +++ b/server/user_config.go @@ -13,6 +13,8 @@ type UserConfig struct { AtlantisURL string `mapstructure:"atlantis-url"` Automerge bool `mapstructure:"automerge"` AutoplanFileList string `mapstructure:"autoplan-file-list"` + AutoplanModules bool `mapstructure:"autoplan-modules"` + AutoplanModulesFromProjects string `mapstructure:"autoplan-modules-from-projects"` AzureDevopsToken string `mapstructure:"azuredevops-token"` AzureDevopsUser string `mapstructure:"azuredevops-user"` AzureDevopsWebhookPassword string `mapstructure:"azuredevops-webhook-password"`