Improve logging for the template function atmos.Component. Generate backend config and provider override files in atmos.Component function

examples/quick-start-advanced/Dockerfile

@@ -6,7 +6,7 @@ ARG GEODESIC_OS=debian
 # https://atmos.tools/
 # https://github.com/cloudposse/atmos
 # https://github.com/cloudposse/atmos/releases
-ARG ATMOS_VERSION=1.86.1
+ARG ATMOS_VERSION=1.86.2
 
 # Terraform: https://github.com/hashicorp/terraform/releases
 ARG TF_VERSION=1.9.4

go.mod

@@ -41,7 +41,7 @@ require (
 	github.com/stretchr/testify v1.9.0
 	github.com/zclconf/go-cty v1.15.0
 	gopkg.in/yaml.v2 v2.4.0
-	mvdan.cc/sh/v3 v3.8.0
+	mvdan.cc/sh/v3 v3.9.0
 )
 
 require (
@@ -237,7 +237,7 @@ require (
 	golang.org/x/oauth2 v0.20.0 // indirect
 	golang.org/x/sync v0.8.0 // indirect
 	golang.org/x/sys v0.24.0 // indirect
-	golang.org/x/term v0.22.0 // indirect
+	golang.org/x/term v0.23.0 // indirect
 	golang.org/x/text v0.16.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect

go.sum

@@ -552,6 +552,8 @@ github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTM
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
 github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
+github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI=
+github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
@@ -1521,8 +1523,8 @@ golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9sn
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
-golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk=
-golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
+golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU=
+golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1906,8 +1908,8 @@ k8s.io/client-go v0.26.2 h1:s1WkVujHX3kTp4Zn4yGNFK+dlDXy1bAAkIl+cFAiuYI=
 k8s.io/client-go v0.26.2/go.mod h1:u5EjOuSyBa09yqqyY7m3abZeovO/7D/WehVVlZ2qcqU=
 k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 h1:kmDqav+P+/5e1i9tFfHq1qcF3sOrDp+YEkVDAHu7Jwk=
 k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
-mvdan.cc/sh/v3 v3.8.0 h1:ZxuJipLZwr/HLbASonmXtcvvC9HXY9d2lXZHnKGjFc8=
-mvdan.cc/sh/v3 v3.8.0/go.mod h1:w04623xkgBVo7/IUK89E0g8hBykgEpN0vgOj3RJr6MY=
+mvdan.cc/sh/v3 v3.9.0 h1:it14fyjCdQUk4jf/aYxLO3FG8jFarR9GzMCtnlvvD7c=
+mvdan.cc/sh/v3 v3.9.0/go.mod h1:cdBk8bgoiBI7lSZqK5JhUuq7OB64VQ7fgm85xelw3Nk=
 nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
 oras.land/oras-go/v2 v2.3.1 h1:lUC6q8RkeRReANEERLfH86iwGn55lbSWP20egdFHVec=
 oras.land/oras-go/v2 v2.3.1/go.mod h1:5AQXVEu1X/FKp1F9DMOb5ZItZBOa0y5dha0yCm4NR9c=

internal/exec/template_funcs_component.go

@@ -3,6 +3,7 @@ package exec
 import (
 	"context"
 	"fmt"
+	"path"
 	"sync"
 
 	"github.com/hashicorp/terraform-exec/tfexec"
@@ -24,7 +25,6 @@ func componentFunc(cliConfig schema.CliConfiguration, component string, stack st
 	// If the result for the component in the stack already exists in the cache, return it
 	existingSections, found := componentFuncSyncMap.Load(stackSlug)
 	if found && existingSections != nil {
-
 		if cliConfig.Logs.Level == u.LogLevelTrace {
 			u.LogTrace(cliConfig, fmt.Sprintf("Found the result of the template function 'atmos.Component(%s, %s)' in the cache", component, stack))
 
@@ -72,37 +72,100 @@ func componentFunc(cliConfig schema.CliConfiguration, component string, stack st
 		return nil, fmt.Errorf("the component '%s' in the stack '%s' has an invalid 'component_info.component_path' section", component, stack)
 	}
 
+	// Auto-generate backend file
+	if cliConfig.Components.Terraform.AutoGenerateBackendFile {
+		backendFileName := path.Join(componentPath, "backend.tf.json")
+
+		u.LogTrace(cliConfig, "\nWriting the backend config to file:")
+		u.LogTrace(cliConfig, backendFileName)
+
+		backendTypeSection, ok := sections["backend_type"].(string)
+		if !ok {
+			return nil, fmt.Errorf("the component '%s' in the stack '%s' has an invalid 'backend_type' section", component, stack)
+		}
+
+		backendSection, ok := sections["backend"].(map[any]any)
+		if !ok {
+			return nil, fmt.Errorf("the component '%s' in the stack '%s' has an invalid 'backend' section", component, stack)
+		}
+
+		componentBackendConfig, err := generateComponentBackendConfig(backendTypeSection, backendSection, terraformWorkspace)
+		if err != nil {
+			return nil, err
+		}
+
+		err = u.WriteToFileAsJSON(backendFileName, componentBackendConfig, 0644)
+		if err != nil {
+			return nil, err
+		}
+
+		u.LogTrace(cliConfig, "\nWrote the backend config to file:")
+		u.LogTrace(cliConfig, backendFileName)
+	}
+
+	// Generate `providers_override.tf.json` file if the `providers` section is configured
+	providersSection, ok := sections["providers"].(map[any]any)
+
+	if ok && len(providersSection) > 0 {
+		providerOverrideFileName := path.Join(componentPath, "providers_override.tf.json")
+
+		u.LogTrace(cliConfig, "\nWriting the provider overrides to file:")
+		u.LogTrace(cliConfig, providerOverrideFileName)
+
+		var providerOverrides = generateComponentProviderOverrides(providersSection)
+		err = u.WriteToFileAsJSON(providerOverrideFileName, providerOverrides, 0644)
+		if err != nil {
+			return nil, err
+		}
+
+		u.LogTrace(cliConfig, "\nWrote the provider overrides to file:")
+		u.LogTrace(cliConfig, providerOverrideFileName)
+	}
+
+	// Initialize Terraform/OpenTofu
 	tf, err := tfexec.NewTerraform(componentPath, executable)
 	if err != nil {
 		return nil, err
 	}
 
 	ctx := context.Background()
 
+	// 'terraform init'
+	u.LogTrace(cliConfig, fmt.Sprintf("\nExecuting 'terraform init %s -s %s'", component, stack))
 	err = tf.Init(ctx, tfexec.Upgrade(false))
 	if err != nil {
 		return nil, err
 	}
+	u.LogTrace(cliConfig, fmt.Sprintf("\nExecuted 'terraform init %s -s %s'", component, stack))
 
+	// Terraform workspace
+	u.LogTrace(cliConfig, fmt.Sprintf("\nExecuting 'terraform workspace new %s' for component '%s' in stack '%s'", terraformWorkspace, component, stack))
 	err = tf.WorkspaceNew(ctx, terraformWorkspace)
 	if err != nil {
+		u.LogTrace(cliConfig, fmt.Sprintf("\nWorkspace exists. Executing 'terraform workspace select %s' for component '%s' in stack '%s'", terraformWorkspace, component, stack))
 		err = tf.WorkspaceSelect(ctx, terraformWorkspace)
 		if err != nil {
 			return nil, err
 		}
+		u.LogTrace(cliConfig, fmt.Sprintf("\nExecuted 'terraform workspace select %s' for component '%s' in stack '%s'", terraformWorkspace, component, stack))
+	} else {
+		u.LogTrace(cliConfig, fmt.Sprintf("\nExecuted 'terraform workspace new %s' for component '%s' in stack '%s'", terraformWorkspace, component, stack))
 	}
 
+	// Terraform output
+	u.LogTrace(cliConfig, fmt.Sprintf("\nExecuting 'terraform output %s -s %s'", component, stack))
 	outputMeta, err := tf.Output(ctx)
 	if err != nil {
 		return nil, err
 	}
+	u.LogTrace(cliConfig, fmt.Sprintf("\nExecuted 'terraform output %s -s %s'", component, stack))
 
 	if cliConfig.Logs.Level == u.LogLevelTrace {
 		y, err2 := u.ConvertToYAML(outputMeta)
 		if err2 != nil {
 			u.LogError(err2)
 		} else {
-			u.LogTrace(cliConfig, fmt.Sprintf("\nResult of 'atmos terraform output %s -s %s' before processing it:\n%s\n", component, stack, y))
+			u.LogTrace(cliConfig, fmt.Sprintf("\nResult of 'terraform output %s -s %s' before processing it:\n%s\n", component, stack, y))
 		}
 	}
 

website/docs/integrations/atlantis.mdx

@@ -673,7 +673,7 @@ on:
     branches: [ main ]
 
 env:
-  ATMOS_VERSION: 1.86.1
+  ATMOS_VERSION: 1.86.2
   ATMOS_CLI_CONFIG_PATH: ./
 
 jobs:

website/docs/integrations/github-actions/setup-atmos.mdx

@@ -33,6 +33,6 @@ jobs:
         uses: cloudposse/github-action-setup-atmos
         with:
           # Make sure to pin to the latest version of atmos
-          atmos_version: 1.86.1
+          atmos_version: 1.86.2
 ```
 </File>

website/docs/quick-start/advanced/provision.md

@@ -5,7 +5,7 @@ sidebar_label: Provision
 ---
 
 Having configured the Terraform components, the Atmos components catalog, all the mixins and defaults, and the Atmos top-level stacks, we can now
-provision the components into the stacks.
+provision the components in the stacks.
 
 The `vpc` Atmos components use the remote state from the `vpc-flow-logs-bucket` components, therefore the `vpc-flow-logs-bucket` components must
 be provisioned first.
@@ -69,7 +69,7 @@ Let's consider what Atmos does when executing the command `atmos terraform apply
   context variables specified in the command (`-s` flag) with the context variables defined in the stack configurations, finally finding the matching
   stack
 
-- Atmos finds the component `vpc` in the stack, processing all the inline configs and all the imports
+- Atmos finds the component `vpc` in the stack, processing all the inline configs and all the configs from the imports
 
 - Atmos deep-merges all the catalog imports for the `vpc` component and then deep-merges all the variables for the component defined in all
   sections (global `vars`, terraform `vars`, base components `vars`, component `vars`), producing the final variables for the `vpc` component in