Skip to content

Commit

Permalink
feat: auto-plan projects when modules change (runatlantis#2507)
Browse files Browse the repository at this point in the history
* feat: auto-plan projects when modules change

Fixes runatlantis#920

* rename Downstream->Dependent; docs cleanup

* chore: gofmt -s
  • Loading branch information
iamnoah authored and krrrr38 committed Dec 16, 2022
1 parent f35a0cf commit fe000fa
Show file tree
Hide file tree
Showing 29 changed files with 423 additions and 37 deletions.
18 changes: 16 additions & 2 deletions cmd/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,13 @@ import (
homedir "github.com/mitchellh/go-homedir"
"github.com/moby/moby/pkg/fileutils"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/viper"

"github.com/runatlantis/atlantis/server"
"github.com/runatlantis/atlantis/server/core/config/valid"
"github.com/runatlantis/atlantis/server/events/vcs/bitbucketcloud"
"github.com/runatlantis/atlantis/server/logging"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)

// To add a new flag you must:
Expand All @@ -46,6 +47,8 @@ const (
AllowRepoConfigFlag = "allow-repo-config"
AtlantisURLFlag = "atlantis-url"
AutomergeFlag = "automerge"
AutoplanModules = "autoplan-modules"
AutoplanModulesFromProjects = "autoplan-modules-from-projects"
AutoplanFileListFlag = "autoplan-file-list"
BitbucketBaseURLFlag = "bitbucket-base-url"
BitbucketTokenFlag = "bitbucket-token"
Expand Down Expand Up @@ -177,6 +180,13 @@ var stringFlags = map[string]stringFlag{
AtlantisURLFlag: {
description: "URL that Atlantis can be reached at. Defaults to http://$(hostname):$port where $port is from --" + PortFlag + ". Supports a base path ex. https://example.com/basepath.",
},
AutoplanModulesFromProjects: {
description: "Comma separated list of file patterns to select projects Atlantis will index for module dependencies." +
" Indexed projects will automatically be planned if a module they depend on is modified." +
" Patterns use the dockerignore (https://docs.docker.com/engine/reference/builder/#dockerignore-file) syntax." +
" A custom Workflow that uses autoplan 'when_modified' will ignore this value.",
defaultValue: "",
},
AutoplanFileListFlag: {
description: "Comma separated list of file patterns that Atlantis will use to check if a directory contains modified files that should trigger project planning." +
" Patterns use the dockerignore (https://docs.docker.com/engine/reference/builder/#dockerignore-file) syntax." +
Expand Down Expand Up @@ -374,6 +384,10 @@ var boolFlags = map[string]boolFlag{
defaultValue: false,
hidden: true,
},
AutoplanModules: {
description: "Automatically plan projects that have a changed module from the local repository.",
defaultValue: false,
},
AutomergeFlag: {
description: "Automatically merge pull requests when all plans are successfully applied.",
defaultValue: false,
Expand Down
24 changes: 21 additions & 3 deletions cmd/server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,14 @@ import (
"testing"

homedir "github.com/mitchellh/go-homedir"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"gopkg.in/yaml.v2"

"github.com/runatlantis/atlantis/server"
"github.com/runatlantis/atlantis/server/events/vcs/fixtures"
"github.com/runatlantis/atlantis/server/logging"
. "github.com/runatlantis/atlantis/testing"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"gopkg.in/yaml.v2"
)

// passedConfig is set to whatever config ended up being passed to NewServer.
Expand Down Expand Up @@ -764,6 +765,23 @@ func TestExecute_RepoWhitelistDeprecation(t *testing.T) {
Equals(t, "*", passedConfig.RepoAllowlist)
}

func TestExecute_AutoDetectModulesFromProjects_Env(t *testing.T) {
t.Setenv("ATLANTIS_AUTOPLAN_MODULES_FROM_PROJECTS", "**/init.tf")
c := setupWithDefaults(map[string]interface{}{}, t)
err := c.Execute()
Ok(t, err)
Equals(t, "**/init.tf", passedConfig.AutoplanModulesFromProjects)
}

func TestExecute_AutoDetectModulesFromProjects(t *testing.T) {
c := setupWithDefaults(map[string]interface{}{
AutoplanModulesFromProjects: "**/*.tf",
}, t)
err := c.Execute()
Ok(t, err)
Equals(t, "**/*.tf", passedConfig.AutoplanModulesFromProjects)
}

func TestExecute_AutoplanFileList(t *testing.T) {
cases := []struct {
description string
Expand Down
3 changes: 2 additions & 1 deletion runatlantis.io/docs/autoplanning.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ The algorithm it uses is as follows:
1. Get the directories that those files are in
1. If the directory path doesn't contain `modules/` then try to run `plan` in that directory
1. If it does contain `modules/` look at the directory one level above `modules/`. If it
contains a `main.tf` run plan in that directory, otherwise ignore the change.
contains a `main.tf` run plan in that directory, otherwise ignore the change (see below for exceptions).

## Example
Given the directory structure:
Expand All @@ -27,6 +27,7 @@ Given the directory structure:
* If `project1/main.tf` were modified, we would run `plan` in `project1`
* If `modules/module1/main.tf` were modified, we would not automatically run `plan` because we couldn't determine the location of the terraform project
* You could use an [atlantis.yaml](repo-level-atlantis-yaml.html#configuring-planning) file to specify which projects to plan when this module changed
* You could enable [module autoplanning](server-configuration.html#autoplan-modules) which indexes projects to their local module dependencies.
* Or you could manually plan with `atlantis plan -d <dir>`
* If `project1/modules/module1/main.tf` were modified, we would look one level above `project1/modules`
into `project1/`, see that there was a `main.tf` file and so run plan in `project1/`
Expand Down
47 changes: 47 additions & 0 deletions runatlantis.io/docs/server-configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,53 @@ Values are chosen in this order:
* Autoplan when any `*.tf` files or `.yml` files in subfolder of `project1` is modified.
* `--autoplan-file-list='**/*.tf,project2/**/*.yml'`


::: warning NOTE
By default, changes to modules will not trigger autoplanning. See the flags below.
:::

### `--autoplan-modules`

```bash
atlantis server --autoplan-modules
```

Defaults to `false`. When set to `true`, Atlantis will trace the local modules of included projects.
Included project are projects with files included by `--autoplan-file-list`.
After tracing, Atlantis will plan any project that includes a changed module. This is equivalent to setting
`--autoplan-modules-from-projects` to the value of `--autoplan-file-list`. See below.

### `--autoplan-modules-from-projects`

```bash
atlantis server --autoplan-modules-from-projects='**/init.tf'
```

Enables auto-planing of projects when a module dependency in the same repository has changed.
This is a list of file patterns like `autoplan-file-list`.

These patterns select **projects** to index based on the files matched. The index maps modules to the projects that depends on them,
including projects that include the module via other modules. When a module file matching `autoplan-file-list` changes,
all indexed projects will be planned.

Current default is "" (disabled).

Examples:

* `**/*.tf` - will index all projects that have a `.tf` file in their directory, and plan them whenever an in-repo module dependency has changed.
* `**/*.tf,!foo,!bar` - will index all projects containing `.tf` except `foo` and `bar` and plan them whenever an in-repo module dependency has changed.
This allows projects to opt-out of auto-planning when a module dependency changes.

::: warning NOTE
Modules that are not selected by autoplan-file-list will not be indexed and dependant projects will not be planned. This
flag allows the *projects* to index to be selected, but the trigger for a plan must be a file in `autoplan-file-list`.
:::

::: warning NOTE
This flag overrides `--autoplan-modules`. If you wish to disable auto-planning of modules, set this flag to an empty string,
and set `--autoplan-modules` to `false`.
:::

### `--azuredevops-hostname`
```bash
atlantis server --azuredevops-hostname="dev.azure.com"
Expand Down
2 changes: 2 additions & 0 deletions server/controllers/events/events_controller_e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/hashicorp/go-getter"
"github.com/hashicorp/go-version"
. "github.com/petergtz/pegomock"

"github.com/runatlantis/atlantis/server"
events_controllers "github.com/runatlantis/atlantis/server/controllers/events"
"github.com/runatlantis/atlantis/server/core/config"
Expand Down Expand Up @@ -972,6 +973,7 @@ func setupE2E(t *testing.T, repoDir string) (events_controllers.VCSEventsControl
commentParser,
false,
false,
"",
"**/*.tf,**/*.tfvars,**/*.tfvars.json,**/terragrunt.hcl,**/.terraform.lock.hcl",
statsScope,
logger,
Expand Down
17 changes: 9 additions & 8 deletions server/controllers/events/events_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,15 @@ import (
"bytes"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"reflect"
"strings"
"testing"

. "github.com/petergtz/pegomock"
events_controllers "github.com/runatlantis/atlantis/server/controllers/events"
"github.com/runatlantis/atlantis/server/controllers/events/mocks"
Expand All @@ -29,14 +38,6 @@ import (
"github.com/runatlantis/atlantis/server/metrics"
. "github.com/runatlantis/atlantis/testing"
gitlab "github.com/xanzy/go-gitlab"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
)

const githubHeader = "X-Github-Event"
Expand Down
161 changes: 161 additions & 0 deletions server/events/modules.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
package events

import (
"fmt"
"io/fs"
"os"
"path"
"strings"

"github.com/hashicorp/terraform-config-inspect/tfconfig"
"github.com/moby/moby/pkg/fileutils"
)

type module struct {
// path to the module
path string
// dependencies of this module
dependencies map[string]bool
// projects that depend on this module
projects map[string]bool
}

func (m *module) String() string {
if m == nil {
return "nil"
}
return fmt.Sprintf("%+v", *m)
}

type ModuleProjects interface {
// DependentProjects returns all projects that depend on the module at moduleDir
DependentProjects(moduleDir string) []string
}

type moduleInfo map[string]*module

var _ ModuleProjects = moduleInfo{}

func (m moduleInfo) String() string {
return fmt.Sprintf("%+v", map[string]*module(m))
}

func (m moduleInfo) DependentProjects(moduleDir string) (projectPaths []string) {
if m == nil || m[moduleDir] == nil {
return nil
}
for project := range m[moduleDir].projects {
projectPaths = append(projectPaths, project)
}
return projectPaths
}

type tfFs struct {
fs.FS
}

func (t tfFs) Open(name string) (tfconfig.File, error) {
return t.FS.Open(name)
}

func (t tfFs) ReadFile(name string) ([]byte, error) {
return fs.ReadFile(t.FS, name)
}

func (t tfFs) ReadDir(dirname string) ([]os.FileInfo, error) {
ls, err := fs.ReadDir(t.FS, dirname)
if err != nil {
return nil, err
}
var infos []os.FileInfo
for _, l := range ls {
info, err := l.Info()
if err != nil {
return nil, fmt.Errorf("failed to get info for %s: %w", l.Name(), err)
}
infos = append(infos, info)
}
return infos, err
}

var _ tfconfig.FS = tfFs{}

func (m moduleInfo) load(files fs.FS, dir string, projects ...string) (_ *module, diags tfconfig.Diagnostics) {
if _, set := m[dir]; !set {
tfFiles := tfFs{files}
var mod *tfconfig.Module
mod, diags = tfconfig.LoadModuleFromFilesystem(tfFiles, dir)

deps := make(map[string]bool)
if mod != nil {
for _, c := range mod.ModuleCalls {
mPath := path.Join(dir, c.Source)
if !tfconfig.IsModuleDirOnFilesystem(tfFiles, mPath) {
continue
}
deps[mPath] = true
}
}

m[dir] = &module{
path: dir,
dependencies: deps,
projects: make(map[string]bool),
}
}
// set projects on my dependencies
for dep := range m[dir].dependencies {
_, err := m.load(files, dep, projects...)
if err != nil {
diags = append(diags, err...)
}
}
// add projects to the list of dependant projects
for _, p := range projects {
m[dir].projects[p] = true
}
return m[dir], diags
}

// FindModuleProjects returns a mapping of modules to projects that depend on them.
func FindModuleProjects(absRepoDir string, autoplanModuleDependants string) (ModuleProjects, error) {
return findModuleDependants(os.DirFS(absRepoDir), autoplanModuleDependants)
}

func findModuleDependants(files fs.FS, autoplanModuleDependants string) (ModuleProjects, error) {
if autoplanModuleDependants == "" {
return moduleInfo{}, nil
}
// find all the projects matching autoplanModuleDependants
filter, _ := fileutils.NewPatternMatcher(strings.Split(autoplanModuleDependants, ","))
var projects []string
err := fs.WalkDir(files, ".", func(rel string, info fs.DirEntry, err error) error {
if match, _ := filter.Matches(rel); match {
if projectDir := getProjectDirFromFs(files, rel); projectDir != "" {
projects = append(projects, projectDir)
}
}
return err
})
if err != nil {
return nil, fmt.Errorf("find projects for module dependants: %w", err)
}

result := make(moduleInfo)
var diags tfconfig.Diagnostics
// for each project, find the modules it depends on, their deps, etc.
for _, projectDir := range projects {
if _, err := result.load(files, projectDir, projectDir); err != nil {
diags = append(diags, err...)
}
}
// if there are any errors, prefer one with a source location
if diags.HasErrors() {
for _, d := range diags {
if d.Pos != nil {
return nil, fmt.Errorf("%s:%d - %s: %s", d.Pos.Filename, d.Pos.Line, d.Summary, d.Detail)
}
}
}
return result, diags.Err()
}
Loading

0 comments on commit fe000fa

Please sign in to comment.