Introduce Atmos YAML functions

atmos.yaml

@@ -15,7 +15,7 @@
 # are independent settings (supporting both absolute and relative paths).
 # If 'base_path' is provided, 'components.terraform.base_path', 'components.helmfile.base_path', 'stacks.base_path' and 'workflows.base_path'
 # are considered paths relative to 'base_path'.
-base_path: "./"
+base_path: "."
 
 vendor:
   # Path to vendor configuration file or directory containing vendor files

examples/quick-start-advanced/Dockerfile

@@ -6,10 +6,10 @@ ARG GEODESIC_OS=debian
 # https://atmos.tools/
 # https://github.com/cloudposse/atmos
 # https://github.com/cloudposse/atmos/releases
-ARG ATMOS_VERSION=1.109.0
+ARG ATMOS_VERSION=1.111.0
 
 # Terraform: https://github.com/hashicorp/terraform/releases
-ARG TF_VERSION=1.9.8
+ARG TF_VERSION=1.5.7
 
 FROM cloudposse/geodesic:${GEODESIC_VERSION}-${GEODESIC_OS}
 

examples/tests/stacks/catalog/terraform/template-functions-test/defaults.yaml

@@ -7,6 +7,9 @@ components:
       metadata:
         # Point to the Terraform component
         component: "test/template-functions-test"
+      settings:
+        spacelift:
+          workspace_enabled: false
       vars:
         enabled: true
         name: "test"

examples/tests/stacks/catalog/terraform/template-functions-test2/defaults.yaml

@@ -6,9 +6,27 @@ components:
       metadata:
         # Point to the Terraform component
         component: "test/template-functions-test2"
+      settings:
+        spacelift:
+          workspace_enabled: false
       vars:
         enabled: true
         name: "test2"
         # Examples of using Atmos template functions to get the outputs of another Atmos component
-        test_label_id: '{{ (atmos.Component "template-functions-test" .stack).outputs.test_label_id }}'
-        test_label_id_2: '{{ (atmos.Component "template-functions-test" .stack).outputs.test_label_id }}'
+        test_1: '{{ (atmos.Component "template-functions-test" .stack).outputs.test_label_id }}'
+        test_2: '{{ (atmos.Component "template-functions-test" .stack).outputs.test_label_id }}'
+        test_3: '{{ toJson (atmos.Component "template-functions-test" .stack).outputs.test_list }}'
+        test_4: '{{ toJson (atmos.Component "template-functions-test" .stack).outputs.test_map }}'
+        test_5: '{{ toJson (atmos.Component "template-functions-test" .stack).outputs.test_list }}'
+        test_10: !terraform.output template-functions-test {{ .stack }} test_label_id
+        test_11: !terraform.output template-functions-test {{ .stack }} test_label_id
+        test_12: !terraform.output template-functions-test {{ .stack }} test_list
+        test_13: !terraform.output template-functions-test {{ .stack }} test_map
+        test_17: !template '{{ .stack }}'
+        test_18: !template '{{ toJson (atmos.Component "template-functions-test" .stack).outputs.test_list }}'
+        test_19: !template '{{ toJson (atmos.Component "template-functions-test" .stack).outputs.test_map }}'
+        test_20: !exec echo 42
+        test_21: !exec echo 43
+        test_30: !exec atmos terraform output template-functions-test -s {{ .stack }} --skip-init -- -json test_label_id
+        test_31: !exec atmos terraform output template-functions-test -s {{ .stack }} --skip-init -- -json test_map
+        test_32: !exec atmos terraform output template-functions-test -s {{ .stack }} --skip-init -- -json test_list

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/arsham/figurine v1.3.0
 	github.com/bmatcuk/doublestar/v4 v4.7.1
 	github.com/charmbracelet/bubbles v0.20.0
-	github.com/charmbracelet/bubbletea v1.2.3
+	github.com/charmbracelet/bubbletea v1.2.4
 	github.com/charmbracelet/glamour v0.8.0
 	github.com/charmbracelet/lipgloss v1.0.0
 	github.com/elewis787/boa v0.1.2
@@ -22,7 +22,7 @@ require (
 	github.com/hashicorp/go-getter v1.7.6
 	github.com/hashicorp/hcl v1.0.0
 	github.com/hashicorp/hcl/v2 v2.23.0
-	github.com/hashicorp/terraform-config-inspect v0.0.0-20241107133921-3adb156ecfe2
+	github.com/hashicorp/terraform-config-inspect v0.0.0-20241129133400-c404f8227ea6
 	github.com/hashicorp/terraform-exec v0.21.0
 	github.com/ivanpirog/coloredcobra v1.0.1
 	github.com/json-iterator/go v1.1.12
@@ -40,8 +40,8 @@ require (
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.19.0
-	github.com/stretchr/testify v1.9.0
-	github.com/zclconf/go-cty v1.15.0
+	github.com/stretchr/testify v1.10.0
+	github.com/zclconf/go-cty v1.15.1
 	golang.org/x/term v0.26.0
 	gopkg.in/yaml.v3 v3.0.1
 	mvdan.cc/sh/v3 v3.10.0

go.sum

@@ -395,8 +395,8 @@ github.com/chainguard-dev/git-urls v1.0.2 h1:pSpT7ifrpc5X55n4aTTm7FFUE+ZQHKiqpiw
 github.com/chainguard-dev/git-urls v1.0.2/go.mod h1:rbGgj10OS7UgZlbzdUQIQpT0k/D4+An04HJY7Ol+Y/o=
 github.com/charmbracelet/bubbles v0.20.0 h1:jSZu6qD8cRQ6k9OMfR1WlM+ruM8fkPWkHvQWD9LIutE=
 github.com/charmbracelet/bubbles v0.20.0/go.mod h1:39slydyswPy+uVOHZ5x/GjwVAFkCsV8IIVy+4MhzwwU=
-github.com/charmbracelet/bubbletea v1.2.3 h1:d9MdMsANIYZB5pE1KkRqaUV6GfsiWm+/9z4fTuGVm9I=
-github.com/charmbracelet/bubbletea v1.2.3/go.mod h1:Qr6fVQw+wX7JkWWkVyXYk/ZUQ92a6XNekLXa3rR18MM=
+github.com/charmbracelet/bubbletea v1.2.4 h1:KN8aCViA0eps9SCOThb2/XPIlea3ANJLUkv3KnQRNCE=
+github.com/charmbracelet/bubbletea v1.2.4/go.mod h1:Qr6fVQw+wX7JkWWkVyXYk/ZUQ92a6XNekLXa3rR18MM=
 github.com/charmbracelet/glamour v0.8.0 h1:tPrjL3aRcQbn++7t18wOpgLyl8wrOHUEDS7IZ68QtZs=
 github.com/charmbracelet/glamour v0.8.0/go.mod h1:ViRgmKkf3u5S7uakt2czJ272WSg2ZenlYEZXT2x7Bjw=
 github.com/charmbracelet/lipgloss v1.0.0 h1:O7VkGDvqEdGi93X+DeqsQ7PKHDgtQfF8j8/O2qFMQNg=
@@ -813,8 +813,8 @@ github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR
 github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0=
 github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY=
 github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=
-github.com/hashicorp/terraform-config-inspect v0.0.0-20241107133921-3adb156ecfe2 h1:h+wcDC5wTiM+5YKjuLtUSWJtdtMVJNCRCPHoWieLpM4=
-github.com/hashicorp/terraform-config-inspect v0.0.0-20241107133921-3adb156ecfe2/go.mod h1:Gz/z9Hbn+4KSp8A2FBtNszfLSdT2Tn/uAKGuVqqWmDI=
+github.com/hashicorp/terraform-config-inspect v0.0.0-20241129133400-c404f8227ea6 h1:146llE+6P/9YO8RcHRehzGNiS9+OoirKW9/aML6/JIA=
+github.com/hashicorp/terraform-config-inspect v0.0.0-20241129133400-c404f8227ea6/go.mod h1:Gz/z9Hbn+4KSp8A2FBtNszfLSdT2Tn/uAKGuVqqWmDI=
 github.com/hashicorp/terraform-exec v0.21.0 h1:uNkLAe95ey5Uux6KJdua6+cv8asgILFVWkd/RG0D2XQ=
 github.com/hashicorp/terraform-exec v0.21.0/go.mod h1:1PPeMYou+KDUSSeRE9szMZ/oHf4fYUmB923Wzbq1ICg=
 github.com/hashicorp/terraform-json v0.22.1 h1:xft84GZR0QzjPVWs4lRUwvTcPnegqlyS7orfb5Ltvec=
@@ -1168,8 +1168,9 @@ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1F
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
 github.com/tchap/go-patricia/v2 v2.3.1 h1:6rQp39lgIYZ+MHmdEq4xzuk1t7OdC35z/xm0BGhTkes=
@@ -1205,8 +1206,8 @@ github.com/yuin/goldmark v1.7.4 h1:BDXOHExt+A7gwPCJgPIIq7ENvceR7we7rOS9TNoLZeg=
 github.com/yuin/goldmark v1.7.4/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
 github.com/yuin/goldmark-emoji v1.0.3 h1:aLRkLHOuBR2czCY4R8olwMjID+tENfhyFDMCRhbIQY4=
 github.com/yuin/goldmark-emoji v1.0.3/go.mod h1:tTkZEbwu5wkPmgTcitqddVxY9osFZiavD+r4AzQrh1U=
-github.com/zclconf/go-cty v1.15.0 h1:tTCRWxsexYUmtt/wVxgDClUe+uQusuI443uL6e+5sXQ=
-github.com/zclconf/go-cty v1.15.0/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE=
+github.com/zclconf/go-cty v1.15.1 h1:RgQYm4j2EvoBRXOPxhUvxPzRrGDo1eCOhHXuGfrj5S0=
+github.com/zclconf/go-cty v1.15.1/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE=
 github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo=
 github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM=
 github.com/zealic/xignore v0.3.3 h1:EpLXUgZY/JEzFkTc+Y/VYypzXtNz+MSOMVCGW5Q4CKQ=

internal/exec/describe_stacks.go

@@ -362,7 +362,12 @@ func ExecuteDescribeStacks(
 									u.LogErrorAndExit(cliConfig, err)
 								}
 
-								componentSection = componentSectionConverted
+								componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted)
+								if err != nil {
+									return nil, err
+								}
+
+								componentSection = componentSectionFinal
 							}
 
 							// Add sections
@@ -542,7 +547,12 @@ func ExecuteDescribeStacks(
 									u.LogErrorAndExit(cliConfig, err)
 								}
 
-								componentSection = componentSectionConverted
+								componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted)
+								if err != nil {
+									return nil, err
+								}
+
+								componentSection = componentSectionFinal
 							}
 
 							// Add sections

internal/exec/template_funcs_component.go

@@ -1,15 +1,11 @@
 package exec
 
 import (
-	"context"
 	"fmt"
-	"path"
 	"sync"
 
-	"github.com/hashicorp/terraform-exec/tfexec"
 	"github.com/samber/lo"
 
-	cfg "github.com/cloudposse/atmos/pkg/config"
 	"github.com/cloudposse/atmos/pkg/schema"
 	u "github.com/cloudposse/atmos/pkg/utils"
 )
@@ -48,167 +44,9 @@ func componentFunc(cliConfig schema.CliConfiguration, component string, stack st
 		return nil, err
 	}
 
-	outputProcessed := map[string]any{}
-	componentAbstract := false
-	componentEnabled := true
-
-	metadataSection, ok := sections[cfg.MetadataSectionName]
-	if ok {
-		metadata, ok2 := metadataSection.(map[string]any)
-		if ok2 {
-			componentAbstract = IsComponentAbstract(metadata)
-		}
-	}
-
-	varsSection, ok := sections[cfg.VarsSectionName]
-	if ok {
-		vars, ok2 := varsSection.(map[string]any)
-		if ok2 {
-			componentEnabled = IsComponentEnabled(vars)
-		}
-	}
-
-	// Don't process Terraform output for disabled and abstract components
-	if componentEnabled && !componentAbstract {
-		executable, ok := sections[cfg.CommandSectionName].(string)
-		if !ok {
-			return nil, fmt.Errorf("the component '%s' in the stack '%s' does not have 'command' (executable) defined", component, stack)
-		}
-
-		terraformWorkspace, ok := sections[cfg.WorkspaceSectionName].(string)
-		if !ok {
-			return nil, fmt.Errorf("the component '%s' in the stack '%s' does not have Terraform/OpenTofu workspace defined", component, stack)
-		}
-
-		componentInfo, ok := sections["component_info"]
-		if !ok {
-			return nil, fmt.Errorf("the component '%s' in the stack '%s' does not have 'component_info' defined", component, stack)
-		}
-
-		componentInfoMap, ok := componentInfo.(map[string]any)
-		if !ok {
-			return nil, fmt.Errorf("the component '%s' in the stack '%s' has an invalid 'component_info' section", component, stack)
-		}
-
-		componentPath, ok := componentInfoMap["component_path"].(string)
-		if !ok {
-			return nil, fmt.Errorf("the component '%s' in the stack '%s' has an invalid 'component_info.component_path' section", component, stack)
-		}
-
-		// Auto-generate backend file
-		if cliConfig.Components.Terraform.AutoGenerateBackendFile {
-			backendFileName := path.Join(componentPath, "backend.tf.json")
-
-			u.LogTrace(cliConfig, "\nWriting the backend config to file:")
-			u.LogTrace(cliConfig, backendFileName)
-
-			backendTypeSection, ok := sections["backend_type"].(string)
-			if !ok {
-				return nil, fmt.Errorf("the component '%s' in the stack '%s' has an invalid 'backend_type' section", component, stack)
-			}
-
-			backendSection, ok := sections["backend"].(map[string]any)
-			if !ok {
-				return nil, fmt.Errorf("the component '%s' in the stack '%s' has an invalid 'backend' section", component, stack)
-			}
-
-			componentBackendConfig, err := generateComponentBackendConfig(backendTypeSection, backendSection, terraformWorkspace)
-			if err != nil {
-				return nil, err
-			}
-
-			err = u.WriteToFileAsJSON(backendFileName, componentBackendConfig, 0644)
-			if err != nil {
-				return nil, err
-			}
-
-			u.LogTrace(cliConfig, "\nWrote the backend config to file:")
-			u.LogTrace(cliConfig, backendFileName)
-		}
-
-		// Generate `providers_override.tf.json` file if the `providers` section is configured
-		providersSection, ok := sections["providers"].(map[string]any)
-
-		if ok && len(providersSection) > 0 {
-			providerOverrideFileName := path.Join(componentPath, "providers_override.tf.json")
-
-			u.LogTrace(cliConfig, "\nWriting the provider overrides to file:")
-			u.LogTrace(cliConfig, providerOverrideFileName)
-
-			var providerOverrides = generateComponentProviderOverrides(providersSection)
-			err = u.WriteToFileAsJSON(providerOverrideFileName, providerOverrides, 0644)
-			if err != nil {
-				return nil, err
-			}
-
-			u.LogTrace(cliConfig, "\nWrote the provider overrides to file:")
-			u.LogTrace(cliConfig, providerOverrideFileName)
-		}
-
-		// Initialize Terraform/OpenTofu
-		tf, err := tfexec.NewTerraform(componentPath, executable)
-		if err != nil {
-			return nil, err
-		}
-
-		ctx := context.Background()
-
-		// 'terraform init'
-		// Before executing `terraform init`, delete the `.terraform/environment` file from the component directory
-		cleanTerraformWorkspace(cliConfig, componentPath)
-
-		u.LogTrace(cliConfig, fmt.Sprintf("\nExecuting 'terraform init %s -s %s'", component, stack))
-		err = tf.Init(ctx, tfexec.Upgrade(false))
-		if err != nil {
-			return nil, err
-		}
-		u.LogTrace(cliConfig, fmt.Sprintf("\nExecuted 'terraform init %s -s %s'", component, stack))
-
-		// Terraform workspace
-		u.LogTrace(cliConfig, fmt.Sprintf("\nExecuting 'terraform workspace new %s' for component '%s' in stack '%s'", terraformWorkspace, component, stack))
-		err = tf.WorkspaceNew(ctx, terraformWorkspace)
-		if err != nil {
-			u.LogTrace(cliConfig, fmt.Sprintf("\nWorkspace exists. Executing 'terraform workspace select %s' for component '%s' in stack '%s'", terraformWorkspace, component, stack))
-			err = tf.WorkspaceSelect(ctx, terraformWorkspace)
-			if err != nil {
-				return nil, err
-			}
-			u.LogTrace(cliConfig, fmt.Sprintf("\nExecuted 'terraform workspace select %s' for component '%s' in stack '%s'", terraformWorkspace, component, stack))
-		} else {
-			u.LogTrace(cliConfig, fmt.Sprintf("\nExecuted 'terraform workspace new %s' for component '%s' in stack '%s'", terraformWorkspace, component, stack))
-		}
-
-		// Terraform output
-		u.LogTrace(cliConfig, fmt.Sprintf("\nExecuting 'terraform output %s -s %s'", component, stack))
-		outputMeta, err := tf.Output(ctx)
-		if err != nil {
-			return nil, err
-		}
-		u.LogTrace(cliConfig, fmt.Sprintf("\nExecuted 'terraform output %s -s %s'", component, stack))
-
-		if cliConfig.Logs.Level == u.LogLevelTrace {
-			y, err2 := u.ConvertToYAML(outputMeta)
-			if err2 != nil {
-				u.LogError(cliConfig, err2)
-			} else {
-				u.LogTrace(cliConfig, fmt.Sprintf("\nResult of 'terraform output %s -s %s' before processing it:\n%s\n", component, stack, y))
-			}
-		}
-
-		outputProcessed = lo.MapEntries(outputMeta, func(k string, v tfexec.OutputMeta) (string, any) {
-			s := string(v.Value)
-			u.LogTrace(cliConfig, fmt.Sprintf("Converting the variable '%s' with the value\n%s\nfrom JSON to 'Go' data type\n", k, s))
-
-			d, err2 := u.ConvertFromJSON(s)
-
-			if err2 != nil {
-				u.LogError(cliConfig, err2)
-			} else {
-				u.LogTrace(cliConfig, fmt.Sprintf("Converted the variable '%s' with the value\n%s\nfrom JSON to 'Go' data type\nResult: %v\n", k, s, d))
-			}
-
-			return k, d
-		})
+	outputProcessed, err := execTerraformOutput(cliConfig, component, stack, sections)
+	if err != nil {
+		return nil, err
 	}
 
 	outputs := map[string]any{

internal/exec/terraform_generate_backends.go

@@ -254,7 +254,12 @@ func ExecuteTerraformGenerateBackends(
 					u.LogErrorAndExit(cliConfig, err)
 				}
 
-				componentSection = componentSectionConverted
+				componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted)
+				if err != nil {
+					return err
+				}
+
+				componentSection = componentSectionFinal
 
 				if i, ok := componentSection[cfg.BackendSectionName].(map[string]any); ok {
 					backendSection = i

internal/exec/terraform_generate_varfiles.go

@@ -270,7 +270,12 @@ func ExecuteTerraformGenerateVarfiles(
 					u.LogErrorAndExit(cliConfig, err)
 				}
 
-				componentSection = componentSectionConverted
+				componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted)
+				if err != nil {
+					return err
+				}
+
+				componentSection = componentSectionFinal
 
 				if i, ok := componentSection[cfg.VarsSectionName].(map[string]any); ok {
 					varsSection = i