From 15bad9a7f6409cf5abf3762643ae2cd65c026007 Mon Sep 17 00:00:00 2001 From: Andriy Knysh Date: Mon, 20 Jan 2025 12:52:30 -0500 Subject: [PATCH] Introduce Atmos YAML functions `!include` and `!env` (#943) * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates * updates --- examples/quick-start-advanced/Dockerfile | 2 +- go.mod | 14 +- go.sum | 28 +- internal/exec/describe_affected.go | 2 +- internal/exec/describe_component.go | 2 +- internal/exec/describe_config.go | 2 +- internal/exec/describe_dependents.go | 2 +- internal/exec/describe_stacks.go | 2 +- internal/exec/describe_workflows.go | 2 +- internal/exec/file_utils.go | 34 +++ internal/exec/go_getter_utils.go | 246 ++++++++++++++++++ internal/exec/stack_processor_utils.go | 37 +-- internal/exec/terraform_outputs.go | 4 +- internal/exec/validate_stacks.go | 19 +- internal/exec/vendor_component_utils.go | 16 +- internal/exec/vendor_model.go | 25 +- internal/exec/vendor_model_component.go | 53 ++-- internal/exec/vendor_utils.go | 165 +----------- internal/exec/yaml_func_env.go | 58 +++++ internal/exec/yaml_func_include.go | 66 +++++ internal/exec/yaml_func_store.go | 1 + internal/exec/yaml_func_utils.go | 8 +- pkg/config/const.go | 3 +- pkg/utils/file_utils.go | 71 +++++ pkg/utils/hcl_utils.go | 11 + pkg/utils/json_utils.go | 10 + pkg/utils/string_utils.go | 18 ++ pkg/utils/version_utils.go | 1 + pkg/utils/yaml_utils.go | 117 ++++++++- pkg/utils/yq_test.go | 2 +- pkg/utils/yq_utils.go | 2 +- .../template-functions-test2/defaults.yaml | 14 + .../stacks/yaml-functions/env.mdx | 79 ++++++ .../stacks/yaml-functions/exec.mdx | 1 - .../stacks/yaml-functions/include.mdx | 172 ++++++++++++ .../stacks/yaml-functions/yaml-functions.mdx | 19 ++ website/docs/integrations/atlantis.mdx | 2 +- 37 files changed, 1001 insertions(+), 309 deletions(-) create mode 100644 internal/exec/go_getter_utils.go create mode 100644 internal/exec/yaml_func_env.go create mode 100644 internal/exec/yaml_func_include.go create mode 100644 website/docs/core-concepts/stacks/yaml-functions/env.mdx create mode 100644 website/docs/core-concepts/stacks/yaml-functions/include.mdx diff --git a/examples/quick-start-advanced/Dockerfile b/examples/quick-start-advanced/Dockerfile index 0ede88f8c..9da10491f 100644 --- a/examples/quick-start-advanced/Dockerfile +++ b/examples/quick-start-advanced/Dockerfile @@ -6,7 +6,7 @@ ARG GEODESIC_OS=debian # https://atmos.tools/ # https://github.com/cloudposse/atmos # https://github.com/cloudposse/atmos/releases -ARG ATMOS_VERSION=1.149.0 +ARG ATMOS_VERSION=1.153.0 # Terraform: https://github.com/hashicorp/terraform/releases ARG TF_VERSION=1.5.7 diff --git a/go.mod b/go.mod index c4fe25d35..a1527e94e 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,8 @@ require ( github.com/alecthomas/chroma v0.10.0 github.com/arsham/figurine v1.3.0 github.com/aws/aws-sdk-go-v2 v1.33.0 - github.com/aws/aws-sdk-go-v2/config v1.29.0 - github.com/aws/aws-sdk-go-v2/service/ssm v1.56.6 + github.com/aws/aws-sdk-go-v2/config v1.29.1 + github.com/aws/aws-sdk-go-v2/service/ssm v1.56.7 github.com/bmatcuk/doublestar/v4 v4.8.0 github.com/charmbracelet/bubbles v0.20.0 github.com/charmbracelet/bubbletea v1.2.4 @@ -17,7 +17,7 @@ require ( github.com/charmbracelet/huh v0.6.0 github.com/charmbracelet/lipgloss v1.0.0 github.com/charmbracelet/log v0.4.0 - github.com/creack/pty v1.1.23 + github.com/creack/pty v1.1.24 github.com/editorconfig-checker/editorconfig-checker/v3 v3.1.2 github.com/elewis787/boa v0.1.2 github.com/fatih/color v1.18.0 @@ -89,7 +89,7 @@ require ( github.com/atotto/clipboard v0.1.4 // indirect github.com/aws/aws-sdk-go v1.44.206 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.1 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.17.53 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.54 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24 // indirect github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.14 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28 // indirect @@ -101,9 +101,9 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.5 // indirect github.com/aws/aws-sdk-go-v2/service/s3 v1.26.10 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.24.10 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.9 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.8 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.11 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.9 // indirect github.com/aws/smithy-go v1.22.1 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect diff --git a/go.sum b/go.sum index c695c1df1..02b3dcf37 100644 --- a/go.sum +++ b/go.sum @@ -749,12 +749,12 @@ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.1 h1:SdK4Ppk5IzLs64ZM github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.1/go.mod h1:n8Bs1ElDD2wJ9kCRTczA83gYbBmjSwZp3umc6zF4EeM= github.com/aws/aws-sdk-go-v2/config v1.15.3/go.mod h1:9YL3v07Xc/ohTsxFXzan9ZpFpdTOFl4X65BAKYaz8jg= github.com/aws/aws-sdk-go-v2/config v1.15.9/go.mod h1:rv/l/TbZo67kp99v/3Kb0qV6Fm1KEtKyruEV2GvVfgs= -github.com/aws/aws-sdk-go-v2/config v1.29.0 h1:Vk/u4jof33or1qAQLdofpjKV7mQQT7DcUpnYx8kdmxY= -github.com/aws/aws-sdk-go-v2/config v1.29.0/go.mod h1:iXAZK3Gxvpq3tA+B9WaDYpZis7M8KFgdrDPMmHrgbJM= +github.com/aws/aws-sdk-go-v2/config v1.29.1 h1:JZhGawAyZ/EuJeBtbQYnaoftczcb2drR2Iq36Wgz4sQ= +github.com/aws/aws-sdk-go-v2/config v1.29.1/go.mod h1:7bR2YD5euaxBhzt2y/oDkt3uNRb6tjFp98GlTFueRwk= github.com/aws/aws-sdk-go-v2/credentials v1.11.2/go.mod h1:j8YsY9TXTm31k4eFhspiQicfXPLZ0gYXA50i4gxPE8g= github.com/aws/aws-sdk-go-v2/credentials v1.12.4/go.mod h1:7g+GGSp7xtR823o1jedxKmqRZGqLdoHQfI4eFasKKxs= -github.com/aws/aws-sdk-go-v2/credentials v1.17.53 h1:lwrVhiEDW5yXsuVKlFVUnR2R50zt2DklhOyeLETqDuE= -github.com/aws/aws-sdk-go-v2/credentials v1.17.53/go.mod h1:CkqM1bIw/xjEpBMhBnvqUXYZbpCFuj6dnCAyDk2AtAY= +github.com/aws/aws-sdk-go-v2/credentials v1.17.54 h1:4UmqeOqJPvdvASZWrKlhzpRahAulBfyTJQUaYy4+hEI= +github.com/aws/aws-sdk-go-v2/credentials v1.17.54/go.mod h1:RTdfo0P0hbbTxIhmQrOsC/PquBZGabEPnCaxxKRPSnI= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.3/go.mod h1:uk1vhHHERfSVCUnqSqz8O48LBYDSC+k6brng09jcMOk= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.5/go.mod h1:WAPnuhG5IQ/i6DETFl5NmX3kKqCzw7aau9NHAGcm4QE= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24 h1:5grmdTdMsovn9kPZPI23Hhvp0ZyNm5cRO+IZFIYiAfw= @@ -797,18 +797,18 @@ github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.4/go.mod h1:PJc8s+lxyU github.com/aws/aws-sdk-go-v2/service/sns v1.17.4/go.mod h1:kElt+uCcXxcqFyc+bQqZPFD9DME/eC6oHBXvFzQ9Bcw= github.com/aws/aws-sdk-go-v2/service/sqs v1.18.3/go.mod h1:skmQo0UPvsjsuYYSYMVmrPc1HWCbHUJyrCEp+ZaLzqM= github.com/aws/aws-sdk-go-v2/service/ssm v1.24.1/go.mod h1:NR/xoKjdbRJ+qx0pMR4mI+N/H1I1ynHwXnO6FowXJc0= -github.com/aws/aws-sdk-go-v2/service/ssm v1.56.6 h1:MVtHLOXm24FJxqyXg4Jq9Ca/tBIK/pHuCkpGHvhOyVA= -github.com/aws/aws-sdk-go-v2/service/ssm v1.56.6/go.mod h1:8HjMkoX1B6HEsxGMPLu6hnx3135hwxpi6eI9aErNTAg= +github.com/aws/aws-sdk-go-v2/service/ssm v1.56.7 h1:vv7lah/6QrqHry4gcYPCcy7ByAmBAtGNjPfTf4HTH/s= +github.com/aws/aws-sdk-go-v2/service/ssm v1.56.7/go.mod h1:8HjMkoX1B6HEsxGMPLu6hnx3135hwxpi6eI9aErNTAg= github.com/aws/aws-sdk-go-v2/service/sso v1.11.3/go.mod h1:7UQ/e69kU7LDPtY40OyoHYgRmgfGM4mgsLYtcObdveU= github.com/aws/aws-sdk-go-v2/service/sso v1.11.7/go.mod h1:TFVe6Rr2joVLsYQ1ABACXgOC6lXip/qpX2x5jWg/A9w= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.10 h1:DyZUj3xSw3FR3TXSwDhPhuZkkT14QHBiacdbUVcD0Dg= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.10/go.mod h1:Ro744S4fKiCCuZECXgOi760TiYylUM8ZBf6OGiZzJtY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.9 h1:I1TsPEs34vbpOnR81GIcAq4/3Ud+jRHVGwx6qLQUHLs= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.9/go.mod h1:Fzsj6lZEb8AkTE5S68OhcbBqeWPsR8RnGuKPr8Todl8= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.11 h1:kuIyu4fTT38Kj7YCC7ouNbVZSSpqkZ+LzIfhCr6Dg+I= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.11/go.mod h1:Ro744S4fKiCCuZECXgOi760TiYylUM8ZBf6OGiZzJtY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10 h1:l+dgv/64iVlQ3WsBbnn+JSbkj01jIi+SM0wYsj3y/hY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10/go.mod h1:Fzsj6lZEb8AkTE5S68OhcbBqeWPsR8RnGuKPr8Todl8= github.com/aws/aws-sdk-go-v2/service/sts v1.16.3/go.mod h1:bfBj0iVmsUyUg4weDB4NxktD9rDGeKSVWnjTnwbx9b8= github.com/aws/aws-sdk-go-v2/service/sts v1.16.6/go.mod h1:rP1rEOKAGZoXp4iGDxSXFvODAtXpm34Egf0lL0eshaQ= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.8 h1:pqEJQtlKWvnv3B6VRt60ZmsHy3SotlEBvfUBPB1KVcM= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.8/go.mod h1:f6vjfZER1M17Fokn0IzssOTMT2N8ZSq+7jnNF0tArvw= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.9 h1:BRVDbewN6VZcwr+FBOszDKvYeXY1kJ+GGMCcpghlw0U= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.9/go.mod h1:f6vjfZER1M17Fokn0IzssOTMT2N8ZSq+7jnNF0tArvw= github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= @@ -921,8 +921,8 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.23 h1:4M6+isWdcStXEf15G/RbrMPOQj1dZ7HPZCGwE4kOeP0= -github.com/creack/pty v1.1.23/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= +github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/cyphar/filepath-securejoin v0.3.6 h1:4d9N5ykBnSp5Xn2JkhocYDkOpURL/18CYMpo6xB9uWM= github.com/cyphar/filepath-securejoin v0.3.6/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/exec/describe_affected.go b/internal/exec/describe_affected.go index e492ff11b..89c58a282 100644 --- a/internal/exec/describe_affected.go +++ b/internal/exec/describe_affected.go @@ -279,7 +279,7 @@ func ExecuteDescribeAffectedCmd(cmd *cobra.Command, args []string) error { return err } - res, err := u.EvaluateYqExpression(atmosConfig, affected, a.Query) + res, err := u.EvaluateYqExpression(&atmosConfig, affected, a.Query) if err != nil { return err } diff --git a/internal/exec/describe_component.go b/internal/exec/describe_component.go index a52796fec..406809dc1 100644 --- a/internal/exec/describe_component.go +++ b/internal/exec/describe_component.go @@ -63,7 +63,7 @@ func ExecuteDescribeComponentCmd(cmd *cobra.Command, args []string) error { return err } - res, err = u.EvaluateYqExpression(atmosConfig, componentSection, query) + res, err = u.EvaluateYqExpression(&atmosConfig, componentSection, query) if err != nil { return err } diff --git a/internal/exec/describe_config.go b/internal/exec/describe_config.go index e71a9cd12..19a8caaa0 100644 --- a/internal/exec/describe_config.go +++ b/internal/exec/describe_config.go @@ -34,7 +34,7 @@ func ExecuteDescribeConfigCmd(cmd *cobra.Command, args []string) error { var res any if query != "" { - res, err = u.EvaluateYqExpression(atmosConfig, atmosConfig, query) + res, err = u.EvaluateYqExpression(&atmosConfig, atmosConfig, query) if err != nil { return err } diff --git a/internal/exec/describe_dependents.go b/internal/exec/describe_dependents.go index d0d62e554..f4907316d 100644 --- a/internal/exec/describe_dependents.go +++ b/internal/exec/describe_dependents.go @@ -67,7 +67,7 @@ func ExecuteDescribeDependentsCmd(cmd *cobra.Command, args []string) error { var res any if query != "" { - res, err = u.EvaluateYqExpression(atmosConfig, dependents, query) + res, err = u.EvaluateYqExpression(&atmosConfig, dependents, query) if err != nil { return err } diff --git a/internal/exec/describe_stacks.go b/internal/exec/describe_stacks.go index ba2090f9d..b83baed17 100644 --- a/internal/exec/describe_stacks.go +++ b/internal/exec/describe_stacks.go @@ -116,7 +116,7 @@ func ExecuteDescribeStacksCmd(cmd *cobra.Command, args []string) error { var res any if query != "" { - res, err = u.EvaluateYqExpression(atmosConfig, finalStacksMap, query) + res, err = u.EvaluateYqExpression(&atmosConfig, finalStacksMap, query) if err != nil { return err } diff --git a/internal/exec/describe_workflows.go b/internal/exec/describe_workflows.go index eb0306c93..794c09336 100644 --- a/internal/exec/describe_workflows.go +++ b/internal/exec/describe_workflows.go @@ -69,7 +69,7 @@ func ExecuteDescribeWorkflowsCmd(cmd *cobra.Command, args []string) error { } if query != "" { - res, err = u.EvaluateYqExpression(atmosConfig, res, query) + res, err = u.EvaluateYqExpression(&atmosConfig, res, query) if err != nil { return err } diff --git a/internal/exec/file_utils.go b/internal/exec/file_utils.go index 4d65f8610..005b1d658 100644 --- a/internal/exec/file_utils.go +++ b/internal/exec/file_utils.go @@ -3,7 +3,11 @@ package exec import ( "fmt" "io" + "net/url" "os" + "path/filepath" + "runtime" + "strings" "github.com/cloudposse/atmos/pkg/schema" u "github.com/cloudposse/atmos/pkg/utils" @@ -63,3 +67,33 @@ func printOrWriteToFile( return nil } + +// SanitizeFileName replaces invalid characters and query strings with underscores for Windows. +func SanitizeFileName(uri string) string { + // Parse the URI to handle paths and query strings properly + parsed, err := url.Parse(uri) + if err != nil { + // Fallback to basic filepath.Base if URI parsing fails + return filepath.Base(uri) + } + + // Extract the path component of the URI + base := filepath.Base(parsed.Path) + + // This logic applies only to Windows + if runtime.GOOS != "windows" { + return base + } + + // Replace invalid characters for Windows + base = strings.Map(func(r rune) rune { + switch r { + case '\\', '/', ':', '*', '?', '"', '<', '>', '|': + return '_' + default: + return r + } + }, base) + + return base +} diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go new file mode 100644 index 000000000..badfb3a67 --- /dev/null +++ b/internal/exec/go_getter_utils.go @@ -0,0 +1,246 @@ +// https://github.com/hashicorp/go-getter + +package exec + +import ( + "context" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + "github.com/google/uuid" + "github.com/hashicorp/go-getter" + + "github.com/cloudposse/atmos/pkg/schema" + u "github.com/cloudposse/atmos/pkg/utils" +) + +// ValidateURI validates URIs +func ValidateURI(uri string) error { + if uri == "" { + return fmt.Errorf("URI cannot be empty") + } + // Maximum length check + if len(uri) > 2048 { + return fmt.Errorf("URI exceeds maximum length of 2048 characters") + } + // Add more validation as needed + // Validate URI format + if strings.Contains(uri, "..") { + return fmt.Errorf("URI cannot contain path traversal sequences") + } + if strings.Contains(uri, " ") { + return fmt.Errorf("URI cannot contain spaces") + } + // Validate characters + if strings.ContainsAny(uri, "<>|&;$") { + return fmt.Errorf("URI contains invalid characters") + } + // Validate scheme-specific format + if strings.HasPrefix(uri, "oci://") { + if !strings.Contains(uri[6:], "/") { + return fmt.Errorf("invalid OCI URI format") + } + } else if strings.Contains(uri, "://") { + scheme := strings.Split(uri, "://")[0] + if !IsValidScheme(scheme) { + return fmt.Errorf("unsupported URI scheme: %s", scheme) + } + } + return nil +} + +// IsValidScheme checks if the URL scheme is valid +func IsValidScheme(scheme string) bool { + validSchemes := map[string]bool{ + "http": true, + "https": true, + "git": true, + "ssh": true, + "git::https": true, + } + return validSchemes[scheme] +} + +// CustomGitHubDetector intercepts GitHub URLs and transforms them +// into something like git::https://@github.com/... so we can +// do a git-based clone with a token. +type CustomGitHubDetector struct { + AtmosConfig schema.AtmosConfiguration +} + +// Detect implements the getter.Detector interface for go-getter v1. +func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { + if len(src) == 0 { + return "", false, nil + } + + if !strings.Contains(src, "://") { + src = "https://" + src + } + + parsedURL, err := url.Parse(src) + if err != nil { + u.LogDebug(d.AtmosConfig, fmt.Sprintf("Failed to parse URL %q: %v\n", src, err)) + return "", false, fmt.Errorf("failed to parse URL %q: %w", src, err) + } + + if strings.ToLower(parsedURL.Host) != "github.com" { + u.LogDebug(d.AtmosConfig, fmt.Sprintf("Host is %q, not 'github.com', skipping token injection\n", parsedURL.Host)) + return "", false, nil + } + + parts := strings.SplitN(parsedURL.Path, "/", 4) + if len(parts) < 3 { + u.LogDebug(d.AtmosConfig, fmt.Sprintf("URL path %q doesn't look like /owner/repo\n", parsedURL.Path)) + return "", false, fmt.Errorf("invalid GitHub URL %q", parsedURL.Path) + } + + atmosGitHubToken := os.Getenv("ATMOS_GITHUB_TOKEN") + gitHubToken := os.Getenv("GITHUB_TOKEN") + + var usedToken string + var tokenSource string + + // 1. If ATMOS_GITHUB_TOKEN is set, always use that + if atmosGitHubToken != "" { + usedToken = atmosGitHubToken + tokenSource = "ATMOS_GITHUB_TOKEN" + u.LogDebug(d.AtmosConfig, "ATMOS_GITHUB_TOKEN is set\n") + } else { + // 2. Otherwise, only inject GITHUB_TOKEN if cfg.Settings.InjectGithubToken == true + if d.AtmosConfig.Settings.InjectGithubToken && gitHubToken != "" { + usedToken = gitHubToken + tokenSource = "GITHUB_TOKEN" + u.LogTrace(d.AtmosConfig, "InjectGithubToken=true and GITHUB_TOKEN is set, using it\n") + } else { + u.LogTrace(d.AtmosConfig, "No ATMOS_GITHUB_TOKEN or GITHUB_TOKEN found\n") + } + } + + if usedToken != "" { + user := parsedURL.User.Username() + pass, _ := parsedURL.User.Password() + if user == "" && pass == "" { + u.LogDebug(d.AtmosConfig, fmt.Sprintf("Injecting token from %s for %s\n", tokenSource, src)) + parsedURL.User = url.UserPassword("x-access-token", usedToken) + } else { + u.LogDebug(d.AtmosConfig, "Credentials found, skipping token injection\n") + } + } + + finalURL := "git::" + parsedURL.String() + + return finalURL, true, nil +} + +// RegisterCustomDetectors prepends the custom detector so it runs before +// the built-in ones. Any code that calls go-getter should invoke this. +func RegisterCustomDetectors(atmosConfig schema.AtmosConfiguration) { + getter.Detectors = append( + []getter.Detector{ + &CustomGitHubDetector{AtmosConfig: atmosConfig}, + }, + getter.Detectors..., + ) +} + +// GoGetterGet downloads packages (files and folders) from different sources using `go-getter` and saves them into the destination +func GoGetterGet( + atmosConfig schema.AtmosConfiguration, + src string, + dest string, + clientMode getter.ClientMode, + timeout time.Duration, +) error { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + // Register custom detectors + RegisterCustomDetectors(atmosConfig) + + client := &getter.Client{ + Ctx: ctx, + Src: src, + // Destination where the files will be stored. This will create the directory if it doesn't exist + Dst: dest, + Mode: clientMode, + } + + if err := client.Get(); err != nil { + return err + } + + return nil +} + +// DownloadDetectFormatAndParseFile downloads a remote file, detects the format of the file (JSON, YAML, HCL) and parses the file into a Go type +func DownloadDetectFormatAndParseFile(atmosConfig schema.AtmosConfiguration, file string) (any, error) { + tempDir := os.TempDir() + f := filepath.Join(tempDir, uuid.New().String()) + + if err := GoGetterGet(atmosConfig, file, f, getter.ClientModeFile, time.Second*30); err != nil { + return nil, fmt.Errorf("failed to download the file '%s': %w", file, err) + } + + res, err := u.DetectFormatAndParseFile(f) + if err != nil { + return nil, fmt.Errorf("failed to parse file '%s': %w", file, err) + } + + return res, nil +} + +/* +Supported schemes: + +file, dir, tar, zip +http, https +git, hg +s3, gcs +oci +scp, sftp +Shortcuts like github.com, bitbucket.org + +- File-related Schemes: +file - Local filesystem paths +dir - Local directories +tar - Tar files, potentially compressed (tar.gz, tar.bz2, etc.) +zip - Zip files + +- HTTP/HTTPS: +http - HTTP URLs +https - HTTPS URLs + +- Git: +git - Git repositories, which can be accessed via HTTPS or SSH + +- Mercurial: +hg - Mercurial repositories, accessed via HTTP/S or SSH + +- Amazon S3: +s3 - Amazon S3 bucket URLs + +- Google Cloud Storage: +gcs - Google Cloud Storage URLs + +- OCI: +oci - Open Container Initiative (OCI) images + +- Other Protocols: +scp - Secure Copy Protocol for SSH-based transfers +sftp - SSH File Transfer Protocol + +- GitHub/Bitbucket/Other Shortcuts: +github.com - Direct GitHub repository shortcuts +bitbucket.org - Direct Bitbucket repository shortcuts + +- Composite Schemes: +go-getter allows for composite schemes, where multiple operations can be combined. For example: +git::https://github.com/user/repo - Forces the use of git over an HTTPS URL. +tar::http://example.com/archive.tar.gz - Treats the HTTP resource as a tarball. + +*/ diff --git a/internal/exec/stack_processor_utils.go b/internal/exec/stack_processor_utils.go index 9ab228cb3..c1dc6a18e 100644 --- a/internal/exec/stack_processor_utils.go +++ b/internal/exec/stack_processor_utils.go @@ -218,7 +218,7 @@ func ProcessYAMLConfigFile( } } - stackConfigMap, err := u.UnmarshalYAML[schema.AtmosSectionMapType](stackManifestTemplatesProcessed) + stackConfigMap, err := u.UnmarshalYAMLFromFile[schema.AtmosSectionMapType](&atmosConfig, stackManifestTemplatesProcessed, filePath) if err != nil { if atmosConfig.Logs.Level == u.LogLevelTrace || atmosConfig.Logs.Level == u.LogLevelDebug { stackManifestTemplatesErrorMessage = fmt.Sprintf("\n\n%s", stackYamlConfig) @@ -1778,37 +1778,6 @@ func FindComponentDependenciesLegacy( return unique, nil } -// resolveRelativePath checks if a path is relative to the current directory and if so, -// resolves it relative to the current file's directory. It ensures the resolved path -// exists within the base path. -func resolveRelativePath(path string, currentFilePath string) string { - if path == "" { - return path - } - - // Convert all paths to use forward slashes for consistency in processing - normalizedPath := filepath.ToSlash(path) - normalizedCurrentFilePath := filepath.ToSlash(currentFilePath) - - // Atmos import paths are generally relative paths, however, there are two types of relative paths: - // 1. Paths relative to the base path (most common) - e.g. "mixins/region/us-east-2" - // 2. Paths relative to the current file's directory (less common) - e.g. "./_defaults" imports will be relative to `./` - // - // Here we check if the path starts with "." or ".." to identify if it's relative to the current file. - // If it is, we'll convert it to be relative to the file doing the import, rather than the `base_path`. - parts := strings.Split(normalizedPath, "/") - firstElement := filepath.Clean(parts[0]) - if firstElement == "." || firstElement == ".." { - // Join the current local path with the current stack file path - baseDir := filepath.Dir(normalizedCurrentFilePath) - relativePath := filepath.Join(baseDir, normalizedPath) - // Return in original format, OS-specific - return filepath.FromSlash(relativePath) - } - // For non-relative paths, return as-is in original format - return path -} - // ProcessImportSection processes the `import` section in stack manifests // The `import` section can contain: // 1. Project-relative paths (e.g. "mixins/region/us-east-2") @@ -1839,7 +1808,7 @@ func ProcessImportSection(stackMap map[string]any, filePath string) ([]schema.St var importObj schema.StackImport err := mapstructure.Decode(imp, &importObj) if err == nil { - importObj.Path = resolveRelativePath(importObj.Path, filePath) + importObj.Path = u.ResolveRelativePath(importObj.Path, filePath) result = append(result, importObj) continue } @@ -1853,7 +1822,7 @@ func ProcessImportSection(stackMap map[string]any, filePath string) ([]schema.St return nil, fmt.Errorf("invalid empty import in the file '%s'", filePath) } - s = resolveRelativePath(s, filePath) + s = u.ResolveRelativePath(s, filePath) result = append(result, schema.StackImport{Path: s}) } diff --git a/internal/exec/terraform_outputs.go b/internal/exec/terraform_outputs.go index 09e575be3..4a0ac6bfc 100644 --- a/internal/exec/terraform_outputs.go +++ b/internal/exec/terraform_outputs.go @@ -267,7 +267,7 @@ func getTerraformOutputVariable( val = "." + val } - res, err := u.EvaluateYqExpression(*atmosConfig, outputs, val) + res, err := u.EvaluateYqExpression(atmosConfig, outputs, val) if err != nil { u.LogErrorAndExit(*atmosConfig, fmt.Errorf("error evaluating terrform output '%s' for the component '%s' in the stack '%s':\n%v", @@ -293,7 +293,7 @@ func getStaticRemoteStateOutput( val = "." + val } - res, err := u.EvaluateYqExpression(*atmosConfig, remoteStateSection, val) + res, err := u.EvaluateYqExpression(atmosConfig, remoteStateSection, val) if err != nil { u.LogErrorAndExit(*atmosConfig, fmt.Errorf("error evaluating the 'static' remote state backend output '%s' for the component '%s' in the stack '%s':\n%v", diff --git a/internal/exec/validate_stacks.go b/internal/exec/validate_stacks.go index b440b1749..328abcd80 100644 --- a/internal/exec/validate_stacks.go +++ b/internal/exec/validate_stacks.go @@ -1,7 +1,6 @@ package exec import ( - "context" "embed" "fmt" "io/fs" @@ -373,37 +372,29 @@ func checkComponentStackMap(componentStackMap map[string]map[string][]string) ([ // downloadSchemaFromURL downloads the Atmos JSON Schema file from the provided URL func downloadSchemaFromURL(atmosConfig schema.AtmosConfiguration) (string, error) { - manifestURL := atmosConfig.Schemas.Atmos.Manifest parsedURL, err := url.Parse(manifestURL) if err != nil { return "", fmt.Errorf("invalid URL '%s': %w", manifestURL, err) } + if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" { return "", fmt.Errorf("unsupported URL scheme '%s' for schema manifest", parsedURL.Scheme) } + tempDir := os.TempDir() fileName, err := u.GetFileNameFromURL(manifestURL) if err != nil || fileName == "" { return "", fmt.Errorf("failed to get the file name from the URL '%s': %w", manifestURL, err) } - atmosManifestJsonSchemaFilePath := filepath.Join(tempDir, fileName) - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - // Register custom detectors - RegisterCustomDetectors(atmosConfig) + atmosManifestJsonSchemaFilePath := filepath.Join(tempDir, fileName) - client := &getter.Client{ - Ctx: ctx, - Dst: atmosManifestJsonSchemaFilePath, - Src: manifestURL, - Mode: getter.ClientModeFile, - } - if err = client.Get(); err != nil { + if err = GoGetterGet(atmosConfig, manifestURL, atmosManifestJsonSchemaFilePath, getter.ClientModeFile, time.Second*30); err != nil { return "", fmt.Errorf("failed to download the Atmos JSON Schema file '%s' from the URL '%s': %w", fileName, manifestURL, err) } + return atmosManifestJsonSchemaFilePath, nil } diff --git a/internal/exec/vendor_component_utils.go b/internal/exec/vendor_component_utils.go index 761c52a84..5ca0f4c38 100644 --- a/internal/exec/vendor_component_utils.go +++ b/internal/exec/vendor_component_utils.go @@ -12,12 +12,13 @@ import ( "github.com/Masterminds/sprig/v3" tea "github.com/charmbracelet/bubbletea" - cfg "github.com/cloudposse/atmos/pkg/config" - "github.com/cloudposse/atmos/pkg/schema" - u "github.com/cloudposse/atmos/pkg/utils" "github.com/hairyhenderson/gomplate/v3" cp "github.com/otiai10/copy" "golang.org/x/term" + + cfg "github.com/cloudposse/atmos/pkg/config" + "github.com/cloudposse/atmos/pkg/schema" + u "github.com/cloudposse/atmos/pkg/utils" ) // findComponentConfigFile identifies the component vendoring config file (`component.yaml` or `component.yml`) @@ -102,6 +103,7 @@ func ExecuteStackVendorInternal( ) error { return fmt.Errorf("command 'atmos vendor pull --stack ' is not supported yet") } + func copyComponentToDestination(atmosConfig schema.AtmosConfiguration, tempDir, componentPath string, vendorComponentSpec schema.VendorComponentSpec, sourceIsLocalFile bool, uri string) error { // Copy from the temp folder to the destination folder and skip the excluded files copyOptions := cp.Options{ @@ -181,7 +183,7 @@ func copyComponentToDestination(atmosConfig schema.AtmosConfiguration, tempDir, componentPath2 := componentPath if sourceIsLocalFile { if filepath.Ext(componentPath) == "" { - componentPath2 = filepath.Join(componentPath, sanitizeFileName(uri)) + componentPath2 = filepath.Join(componentPath, SanitizeFileName(uri)) } } @@ -190,6 +192,7 @@ func copyComponentToDestination(atmosConfig schema.AtmosConfiguration, tempDir, } return nil } + func ExecuteComponentVendorInternal( atmosConfig schema.AtmosConfiguration, vendorComponentSpec schema.VendorComponentSpec, @@ -222,6 +225,7 @@ func ExecuteComponentVendorInternal( } else { uri = vendorComponentSpec.Source.Uri } + useOciScheme := false useLocalFileSystem := false sourceIsLocalFile := false @@ -245,6 +249,7 @@ func ExecuteComponentVendorInternal( } } } + var pType pkgType if useOciScheme { pType = pkgTypeOci @@ -264,8 +269,10 @@ func ExecuteComponentVendorInternal( vendorComponentSpec: vendorComponentSpec, IsComponent: true, } + var packages []pkgComponentVendor packages = append(packages, componentPkg) + // Process mixins if len(vendorComponentSpec.Mixins) > 0 { for _, mixin := range vendorComponentSpec.Mixins { @@ -338,6 +345,7 @@ func ExecuteComponentVendorInternal( packages = append(packages, pkg) } } + // Run TUI to process packages if len(packages) > 0 { model, err := newModelComponentVendorInternal(packages, dryRun, atmosConfig) diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index 9f91fcc7d..20e873f24 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -1,7 +1,6 @@ package exec import ( - "context" "errors" "fmt" "os" @@ -13,11 +12,12 @@ import ( "github.com/charmbracelet/bubbles/spinner" tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" + "github.com/hashicorp/go-getter" + cp "github.com/otiai10/copy" + "github.com/cloudposse/atmos/pkg/schema" "github.com/cloudposse/atmos/pkg/ui/theme" u "github.com/cloudposse/atmos/pkg/utils" - "github.com/hashicorp/go-getter" - cp "github.com/otiai10/copy" ) type pkgType int @@ -113,6 +113,7 @@ func (m *modelVendor) Init() tea.Cmd { } return tea.Batch(ExecuteInstall(m.packages[0], m.dryRun, m.atmosConfig), m.spinner.Tick) } + func (m *modelVendor) Update(msg tea.Msg) (tea.Model, tea.Cmd) { switch msg := msg.(type) { case tea.WindowSizeMsg: @@ -120,6 +121,7 @@ func (m *modelVendor) Update(msg tea.Msg) (tea.Model, tea.Cmd) { if m.width > 120 { m.width = 120 } + case tea.KeyMsg: switch msg.String() { case "ctrl+c", "esc", "q": @@ -194,7 +196,6 @@ func (m *modelVendor) Update(msg tea.Msg) (tea.Model, tea.Cmd) { } func (m modelVendor) View() string { - n := len(m.packages) w := lipgloss.Width(fmt.Sprintf("%d", n)) if m.done { @@ -235,6 +236,7 @@ func max(a, b int) int { } return b } + func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.AtmosConfiguration) tea.Cmd { return func() tea.Msg { if dryRun { @@ -262,22 +264,11 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos } defer removeTempDir(atmosConfig, tempDir) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer cancel() switch p.pkgType { case pkgTypeRemote: // Use go-getter to download remote packages - // Register custom detectors - RegisterCustomDetectors(atmosConfig) - - client := &getter.Client{ - Ctx: ctx, - Dst: tempDir, - Src: p.uri, - Mode: getter.ClientModeAny, - } - if err := client.Get(); err != nil { + if err := GoGetterGet(atmosConfig, p.uri, tempDir, getter.ClientModeAny, 10*time.Minute); err != nil { return installedPkgMsg{ err: fmt.Errorf("failed to download package: %w", err), name: p.name, @@ -301,7 +292,7 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos OnSymlink: func(src string) cp.SymlinkAction { return cp.Deep }, } if p.sourceIsLocalFile { - tempDir = filepath.Join(tempDir, sanitizeFileName(p.uri)) + tempDir = filepath.Join(tempDir, SanitizeFileName(p.uri)) } if err := cp.Copy(p.uri, tempDir, copyOptions); err != nil { return installedPkgMsg{ diff --git a/internal/exec/vendor_model_component.go b/internal/exec/vendor_model_component.go index 9d2f02c92..108fe56c8 100644 --- a/internal/exec/vendor_model_component.go +++ b/internal/exec/vendor_model_component.go @@ -1,7 +1,6 @@ package exec import ( - "context" "fmt" "os" "path/filepath" @@ -11,10 +10,11 @@ import ( "github.com/charmbracelet/bubbles/progress" "github.com/charmbracelet/bubbles/spinner" tea "github.com/charmbracelet/bubbletea" - "github.com/cloudposse/atmos/pkg/schema" - "github.com/cloudposse/atmos/pkg/ui/theme" "github.com/hashicorp/go-getter" cp "github.com/otiai10/copy" + + "github.com/cloudposse/atmos/pkg/schema" + "github.com/cloudposse/atmos/pkg/ui/theme" ) type pkgComponentVendor struct { @@ -106,6 +106,7 @@ func downloadComponentAndInstall(p *pkgComponentVendor, dryRun bool, atmosConfig } } } + func installComponent(p *pkgComponentVendor, atmosConfig schema.AtmosConfiguration) error { // Create temp folder @@ -116,30 +117,20 @@ func installComponent(p *pkgComponentVendor, atmosConfig schema.AtmosConfigurati if err != nil { return fmt.Errorf("Failed to create temp directory %s", err) } + // Ensure directory permissions are restricted if err := os.Chmod(tempDir, 0700); err != nil { return fmt.Errorf("failed to set temp directory permissions: %w", err) } + defer removeTempDir(atmosConfig, tempDir) switch p.pkgType { case pkgTypeRemote: - tempDir = filepath.Join(tempDir, sanitizeFileName(p.uri)) - - // Register custom detectors - RegisterCustomDetectors(atmosConfig) - - client := &getter.Client{ - Ctx: context.Background(), - // Define the destination where the files will be stored. This will create the directory if it doesn't exist - Dst: tempDir, - // Source - Src: p.uri, - Mode: getter.ClientModeAny, - } + tempDir = filepath.Join(tempDir, SanitizeFileName(p.uri)) - if err = client.Get(); err != nil { - return fmt.Errorf("Failed to download package %s error %s", p.name, err) + if err = GoGetterGet(atmosConfig, p.uri, tempDir, getter.ClientModeAny, 10*time.Minute); err != nil { + return fmt.Errorf("failed to download package %s error %s", p.name, err) } case pkgTypeOci: @@ -162,7 +153,7 @@ func installComponent(p *pkgComponentVendor, atmosConfig schema.AtmosConfigurati tempDir2 := tempDir if p.sourceIsLocalFile { - tempDir2 = filepath.Join(tempDir, sanitizeFileName(p.uri)) + tempDir2 = filepath.Join(tempDir, SanitizeFileName(p.uri)) } if err = cp.Copy(p.uri, tempDir2, copyOptions); err != nil { @@ -177,8 +168,8 @@ func installComponent(p *pkgComponentVendor, atmosConfig schema.AtmosConfigurati } return nil - } + func installMixin(p *pkgComponentVendor, atmosConfig schema.AtmosConfiguration) error { tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) if err != nil { @@ -186,30 +177,20 @@ func installMixin(p *pkgComponentVendor, atmosConfig schema.AtmosConfiguration) } defer removeTempDir(atmosConfig, tempDir) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer cancel() + switch p.pkgType { case pkgTypeRemote: - - // Register custom detectors - RegisterCustomDetectors(atmosConfig) - - client := &getter.Client{ - Ctx: ctx, - Dst: filepath.Join(tempDir, p.mixinFilename), - Src: p.uri, - Mode: getter.ClientModeFile, + if err = GoGetterGet(atmosConfig, p.uri, filepath.Join(tempDir, p.mixinFilename), getter.ClientModeFile, 10*time.Minute); err != nil { + return fmt.Errorf("failed to download package %s error %s", p.name, err) } - if err = client.Get(); err != nil { - return fmt.Errorf("Failed to download package %s error %s", p.name, err) - } case pkgTypeOci: // Download the Image from the OCI-compatible registry, extract the layers from the tarball, and write to the destination directory err = processOciImage(atmosConfig, p.uri, tempDir) if err != nil { - return fmt.Errorf("Failed to process OCI image %s error %s", p.name, err) + return fmt.Errorf("failed to process OCI image %s error %s", p.name, err) } + case pkgTypeLocal: if p.uri == "" { return fmt.Errorf("local mixin URI cannot be empty") @@ -219,8 +200,8 @@ func installMixin(p *pkgComponentVendor, atmosConfig schema.AtmosConfiguration) default: return fmt.Errorf("unknown package type %s package %s", p.pkgType.String(), p.name) - } + // Copy from the temp folder to the destination folder copyOptions := cp.Options{ // Preserve the atime and the mtime of the entries diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index a9e0fd3aa..ee6c6520b 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -2,16 +2,13 @@ package exec import ( "fmt" - "net/url" "os" "path/filepath" - "runtime" "sort" "strings" "github.com/bmatcuk/doublestar/v4" tea "github.com/charmbracelet/bubbletea" - "github.com/hashicorp/go-getter" cp "github.com/otiai10/copy" "github.com/samber/lo" "github.com/spf13/cobra" @@ -364,7 +361,7 @@ func ExecuteAtmosVendorInternal( if err != nil { return err } - err = validateURI(uri) + err = ValidateURI(uri) if err != nil { return err } @@ -526,37 +523,6 @@ func determineSourceType(uri *string, vendorConfigFilePath string) (bool, bool, return useOciScheme, useLocalFileSystem, sourceIsLocalFile } -// sanitizeFileName replaces invalid characters and query strings with underscores for Windows. -func sanitizeFileName(uri string) string { - - // Parse the URI to handle paths and query strings properly - parsed, err := url.Parse(uri) - if err != nil { - // Fallback to basic filepath.Base if URI parsing fails - return filepath.Base(uri) - } - - // Extract the path component of the URI - base := filepath.Base(parsed.Path) - - // This logic applies only to Windows - if runtime.GOOS != "windows" { - return base - } - - // Replace invalid characters for Windows - base = strings.Map(func(r rune) rune { - switch r { - case '\\', '/', ':', '*', '?', '"', '<', '>', '|': - return '_' - default: - return r - } - }, base) - - return base -} - func copyToTarget(atmosConfig schema.AtmosConfiguration, tempDir, targetPath string, s *schema.AtmosVendorSource, sourceIsLocalFile bool, uri string) error { copyOptions := cp.Options{ Skip: generateSkipFunction(atmosConfig, tempDir, s), @@ -568,7 +534,7 @@ func copyToTarget(atmosConfig schema.AtmosConfiguration, tempDir, targetPath str // Adjust the target path if it's a local file with no extension if sourceIsLocalFile && filepath.Ext(targetPath) == "" { // Sanitize the URI for safe filenames, especially on Windows - sanitizedBase := sanitizeFileName(uri) + sanitizedBase := SanitizeFileName(uri) targetPath = filepath.Join(targetPath, sanitizedBase) } @@ -642,130 +608,3 @@ func generateSkipFunction(atmosConfig schema.AtmosConfiguration, tempDir string, return false, nil } } - -func validateURI(uri string) error { - if uri == "" { - return fmt.Errorf("URI cannot be empty") - } - // Maximum length check - if len(uri) > 2048 { - return fmt.Errorf("URI exceeds maximum length of 2048 characters") - } - // Add more validation as needed - // Validate URI format - if strings.Contains(uri, "..") { - return fmt.Errorf("URI cannot contain path traversal sequences") - } - if strings.Contains(uri, " ") { - return fmt.Errorf("URI cannot contain spaces") - } - // Validate characters - if strings.ContainsAny(uri, "<>|&;$") { - return fmt.Errorf("URI contains invalid characters") - } - // Validate scheme-specific format - if strings.HasPrefix(uri, "oci://") { - if !strings.Contains(uri[6:], "/") { - return fmt.Errorf("invalid OCI URI format") - } - } else if strings.Contains(uri, "://") { - scheme := strings.Split(uri, "://")[0] - if !isValidScheme(scheme) { - return fmt.Errorf("unsupported URI scheme: %s", scheme) - } - } - return nil -} -func isValidScheme(scheme string) bool { - validSchemes := map[string]bool{ - "http": true, - "https": true, - "git": true, - "ssh": true, - "git::https": true, - } - return validSchemes[scheme] -} - -// CustomGitHubDetector intercepts GitHub URLs and transforms them -// into something like git::https://@github.com/... so we can -// do a git-based clone with a token. -type CustomGitHubDetector struct { - AtmosConfig schema.AtmosConfiguration -} - -// Detect implements the getter.Detector interface for go-getter v1. -func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { - if len(src) == 0 { - return "", false, nil - } - - if !strings.Contains(src, "://") { - src = "https://" + src - } - - parsedURL, err := url.Parse(src) - if err != nil { - u.LogDebug(d.AtmosConfig, fmt.Sprintf("Failed to parse URL %q: %v\n", src, err)) - return "", false, fmt.Errorf("failed to parse URL %q: %w", src, err) - } - - if strings.ToLower(parsedURL.Host) != "github.com" { - u.LogDebug(d.AtmosConfig, fmt.Sprintf("Host is %q, not 'github.com', skipping token injection\n", parsedURL.Host)) - return "", false, nil - } - - parts := strings.SplitN(parsedURL.Path, "/", 4) - if len(parts) < 3 { - u.LogDebug(d.AtmosConfig, fmt.Sprintf("URL path %q doesn't look like /owner/repo\n", parsedURL.Path)) - return "", false, fmt.Errorf("invalid GitHub URL %q", parsedURL.Path) - } - - atmosGitHubToken := os.Getenv("ATMOS_GITHUB_TOKEN") - gitHubToken := os.Getenv("GITHUB_TOKEN") - - var usedToken string - var tokenSource string - - // 1. If ATMOS_GITHUB_TOKEN is set, always use that - if atmosGitHubToken != "" { - usedToken = atmosGitHubToken - tokenSource = "ATMOS_GITHUB_TOKEN" - u.LogDebug(d.AtmosConfig, "ATMOS_GITHUB_TOKEN is set\n") - } else { - // 2. Otherwise, only inject GITHUB_TOKEN if cfg.Settings.InjectGithubToken == true - if d.AtmosConfig.Settings.InjectGithubToken && gitHubToken != "" { - usedToken = gitHubToken - tokenSource = "GITHUB_TOKEN" - u.LogTrace(d.AtmosConfig, "InjectGithubToken=true and GITHUB_TOKEN is set, using it\n") - } else { - u.LogTrace(d.AtmosConfig, "No ATMOS_GITHUB_TOKEN or GITHUB_TOKEN found\n") - } - } - - if usedToken != "" { - user := parsedURL.User.Username() - pass, _ := parsedURL.User.Password() - if user == "" && pass == "" { - u.LogDebug(d.AtmosConfig, fmt.Sprintf("Injecting token from %s for %s\n", tokenSource, src)) - parsedURL.User = url.UserPassword("x-access-token", usedToken) - } else { - u.LogDebug(d.AtmosConfig, "Credentials found, skipping token injection\n") - } - } - - finalURL := "git::" + parsedURL.String() - - return finalURL, true, nil -} - -// RegisterCustomDetectors prepends the custom detector so it runs before -// the built-in ones. Any code that calls go-getter should invoke this. -func RegisterCustomDetectors(atmosConfig schema.AtmosConfiguration) { - getter.Detectors = append( - []getter.Detector{ - &CustomGitHubDetector{AtmosConfig: atmosConfig}, - }, - getter.Detectors..., - ) -} diff --git a/internal/exec/yaml_func_env.go b/internal/exec/yaml_func_env.go new file mode 100644 index 000000000..97b33a616 --- /dev/null +++ b/internal/exec/yaml_func_env.go @@ -0,0 +1,58 @@ +package exec + +import ( + "fmt" + "os" + "strings" + + "github.com/cloudposse/atmos/pkg/config" + "github.com/cloudposse/atmos/pkg/schema" + u "github.com/cloudposse/atmos/pkg/utils" +) + +func processTagEnv( + atmosConfig schema.AtmosConfiguration, + input string, + currentStack string, +) any { + u.LogTrace(atmosConfig, fmt.Sprintf("Executing Atmos YAML function: %s", input)) + + str, err := getStringAfterTag(input, config.AtmosYamlFuncEnv) + if err != nil { + u.LogErrorAndExit(atmosConfig, err) + } + + var envVarName string + envVarDefault := "" + var envVarExists bool + + parts, err := u.SplitStringByDelimiter(str, ' ') + if err != nil { + e := fmt.Errorf("error executing the YAML function: %s\n%v", input, err) + u.LogErrorAndExit(atmosConfig, e) + } + + partsLen := len(parts) + + if partsLen == 2 { + envVarName = strings.TrimSpace(parts[0]) + envVarDefault = strings.TrimSpace(parts[1]) + } else if partsLen == 1 { + envVarName = strings.TrimSpace(parts[0]) + } else { + err = fmt.Errorf("invalid number of arguments in the Atmos YAML function: %s. The function accepts 1 or 2 arguments", input) + u.LogErrorAndExit(atmosConfig, err) + } + + res, envVarExists := os.LookupEnv(envVarName) + + if envVarExists { + return res + } + + if envVarDefault != "" { + return envVarDefault + } + + return nil +} diff --git a/internal/exec/yaml_func_include.go b/internal/exec/yaml_func_include.go new file mode 100644 index 000000000..9dc6db8ae --- /dev/null +++ b/internal/exec/yaml_func_include.go @@ -0,0 +1,66 @@ +package exec + +import ( + "fmt" + "strings" + + "github.com/cloudposse/atmos/pkg/schema" + u "github.com/cloudposse/atmos/pkg/utils" +) + +func processTagInclude( + atmosConfig schema.AtmosConfiguration, + input string, + fileType string, + currentStack string, +) any { + str, err := getStringAfterTag(input, fileType) + if err != nil { + u.LogErrorAndExit(atmosConfig, err) + } + + u.LogTrace(atmosConfig, fmt.Sprintf("Executing Atmos YAML function: !include %s", str)) + + var f string + q := "" + + parts, err := u.SplitStringByDelimiter(str, ' ') + if err != nil { + e := fmt.Errorf("error executing the YAML function: !include %s\n%v", str, err) + u.LogErrorAndExit(atmosConfig, e) + } + + partsLen := len(parts) + + if partsLen == 2 { + f = strings.TrimSpace(parts[0]) + q = strings.TrimSpace(parts[1]) + } else if partsLen == 1 { + f = strings.TrimSpace(parts[0]) + } else { + err = fmt.Errorf("invalid number of arguments in the Atmos YAML function: !include %s. The function accepts 1 or 2 arguments", str) + u.LogErrorAndExit(atmosConfig, err) + } + + var res any + + if fileType == u.AtmosYamlFuncIncludeLocalFile { + res, err = u.DetectFormatAndParseFile(f) + } else if fileType == u.AtmosYamlFuncIncludeGoGetter { + res, err = DownloadDetectFormatAndParseFile(atmosConfig, f) + } + + if err != nil { + e := fmt.Errorf("error evaluating the YAML function: !include %s\n%v", str, err) + u.LogErrorAndExit(atmosConfig, e) + } + + if q != "" { + res, err = u.EvaluateYqExpression(&atmosConfig, res, q) + if err != nil { + u.LogErrorAndExit(atmosConfig, err) + } + } + + return res +} diff --git a/internal/exec/yaml_func_store.go b/internal/exec/yaml_func_store.go index 3b765a1aa..583d3cf4a 100644 --- a/internal/exec/yaml_func_store.go +++ b/internal/exec/yaml_func_store.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/charmbracelet/log" + "github.com/cloudposse/atmos/pkg/schema" u "github.com/cloudposse/atmos/pkg/utils" ) diff --git a/internal/exec/yaml_func_utils.go b/internal/exec/yaml_func_utils.go index 014821cc1..6ac6928fa 100644 --- a/internal/exec/yaml_func_utils.go +++ b/internal/exec/yaml_func_utils.go @@ -70,8 +70,14 @@ func processCustomTags( return processTagStore(atmosConfig, input, currentStack) case strings.HasPrefix(input, u.AtmosYamlFuncTerraformOutput): return processTagTerraformOutput(atmosConfig, input, currentStack) + case strings.HasPrefix(input, u.AtmosYamlFuncEnv): + return processTagEnv(atmosConfig, input, currentStack) + case strings.HasPrefix(input, u.AtmosYamlFuncIncludeGoGetter): + return processTagInclude(atmosConfig, input, u.AtmosYamlFuncIncludeGoGetter, currentStack) + case strings.HasPrefix(input, u.AtmosYamlFuncIncludeLocalFile): + return processTagInclude(atmosConfig, input, u.AtmosYamlFuncIncludeLocalFile, currentStack) default: - // If any other YAML explicit type (not currently supported by Atmos) is used, return it w/o processing + // If any other YAML explicit tag (not currently supported by Atmos) is used, return it w/o processing return input } } diff --git a/pkg/config/const.go b/pkg/config/const.go index 6174c73fb..54b24e983 100644 --- a/pkg/config/const.go +++ b/pkg/config/const.go @@ -78,7 +78,8 @@ const ( AtmosProDefaultEndpoint = "api" // Atmos YAML functions - AtmosYamlFuncTemplate = "!template" AtmosYamlFuncExec = "!exec" + AtmosYamlFuncTemplate = "!template" AtmosYamlFuncTerraformOutput = "!terraform.output" + AtmosYamlFuncEnv = "!env" ) diff --git a/pkg/utils/file_utils.go b/pkg/utils/file_utils.go index d55b56976..d556446b0 100644 --- a/pkg/utils/file_utils.go +++ b/pkg/utils/file_utils.go @@ -1,12 +1,16 @@ package utils import ( + "encoding/json" "fmt" "io/fs" "net/url" "os" "path/filepath" "strings" + + "github.com/hashicorp/hcl" + "gopkg.in/yaml.v3" ) // IsDirectory checks if the path is a directory @@ -243,3 +247,70 @@ func GetFileNameFromURL(rawURL string) (string, error) { } return fileName, nil } + +// ResolveRelativePath checks if a path is relative to the current directory and if so, +// resolves it relative to the current file's directory. It ensures the resolved path +// exists within the base path. +func ResolveRelativePath(path string, basePath string) string { + if path == "" { + return path + } + + // Convert all paths to use forward slashes for consistency in processing + normalizedPath := filepath.ToSlash(path) + normalizedBasePath := filepath.ToSlash(basePath) + + // Atmos import paths are generally relative paths, however, there are two types of relative paths: + // 1. Paths relative to the base path (most common) - e.g. "mixins/region/us-east-2" + // 2. Paths relative to the current file's directory (less common) - e.g. "./_defaults" imports will be relative to `./` + // + // Here we check if the path starts with "." or ".." to identify if it's relative to the current file. + // If it is, we'll convert it to be relative to the file doing the import, rather than the `base_path`. + parts := strings.Split(normalizedPath, "/") + firstElement := filepath.Clean(parts[0]) + if firstElement == "." || firstElement == ".." { + // Join the current local path with the current stack file path + baseDir := filepath.Dir(normalizedBasePath) + relativePath := filepath.Join(baseDir, normalizedPath) + // Return in original format, OS-specific + return filepath.FromSlash(relativePath) + } + + // For non-relative paths, return as-is in original format + return path +} + +// DetectFormatAndParseFile detects the format of the file (JSON, YAML, HCL) and parses the file into a Go type +// For all other formats, it just reads the file and returns the content as a string +func DetectFormatAndParseFile(filename string) (any, error) { + var v any + var err error + + d, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + + data := string(d) + + if IsHCL(data) { + err = hcl.Unmarshal(d, &v) + if err != nil { + return nil, err + } + } else if IsJSON(data) { + err = json.Unmarshal(d, &v) + if err != nil { + return nil, err + } + } else if IsYAML(data) { + err = yaml.Unmarshal(d, &v) + if err != nil { + return nil, err + } + } else { + v = data + } + + return v, nil +} diff --git a/pkg/utils/hcl_utils.go b/pkg/utils/hcl_utils.go index 601346fed..bd3a6f499 100644 --- a/pkg/utils/hcl_utils.go +++ b/pkg/utils/hcl_utils.go @@ -4,6 +4,7 @@ import ( "os" "strings" + "github.com/hashicorp/hcl" "github.com/hashicorp/hcl/hcl/ast" "github.com/hashicorp/hcl/hcl/printer" jsonParser "github.com/hashicorp/hcl/json/parser" @@ -140,3 +141,13 @@ func WriteTerraformBackendConfigToFileAsHcl( return nil } + +// IsHCL checks if data is in HCL format +func IsHCL(data string) bool { + if strings.TrimSpace(data) == "" { + return false + } + + var hclData any + return hcl.Unmarshal([]byte(data), &hclData) == nil +} diff --git a/pkg/utils/json_utils.go b/pkg/utils/json_utils.go index 86ae7cba5..4fe709b1a 100644 --- a/pkg/utils/json_utils.go +++ b/pkg/utils/json_utils.go @@ -133,3 +133,13 @@ func JSONToMapOfInterfaces(input string) (schema.AtmosSectionMapType, error) { } return data, nil } + +// IsJSON checks if data is in JSON format +func IsJSON(data string) bool { + if strings.TrimSpace(data) == "" { + return false + } + + var js json.RawMessage + return json.Unmarshal([]byte(data), &js) == nil +} diff --git a/pkg/utils/string_utils.go b/pkg/utils/string_utils.go index 8bc20dd04..26d7a6c1e 100644 --- a/pkg/utils/string_utils.go +++ b/pkg/utils/string_utils.go @@ -1,5 +1,10 @@ package utils +import ( + "encoding/csv" + "strings" +) + // UniqueStrings returns a unique subset of the string slice provided func UniqueStrings(input []string) []string { u := make([]string, 0, len(input)) @@ -14,3 +19,16 @@ func UniqueStrings(input []string) []string { return u } + +// SplitStringByDelimiter splits a string by the delimiter, not splitting inside quotes +func SplitStringByDelimiter(str string, delimiter rune) ([]string, error) { + r := csv.NewReader(strings.NewReader(str)) + r.Comma = delimiter + + parts, err := r.Read() + if err != nil { + return nil, err + } + + return parts, nil +} diff --git a/pkg/utils/version_utils.go b/pkg/utils/version_utils.go index a4712ef63..d21a65fbe 100644 --- a/pkg/utils/version_utils.go +++ b/pkg/utils/version_utils.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/charmbracelet/lipgloss" + "github.com/cloudposse/atmos/pkg/ui/theme" "github.com/cloudposse/atmos/pkg/version" ) diff --git a/pkg/utils/yaml_utils.go b/pkg/utils/yaml_utils.go index a7af8a2c9..4fc4c4c87 100644 --- a/pkg/utils/yaml_utils.go +++ b/pkg/utils/yaml_utils.go @@ -1,7 +1,10 @@ package utils import ( + "fmt" "os" + "path/filepath" + "strings" "gopkg.in/yaml.v3" @@ -14,6 +17,12 @@ const ( AtmosYamlFuncStore = "!store" AtmosYamlFuncTemplate = "!template" AtmosYamlFuncTerraformOutput = "!terraform.output" + AtmosYamlFuncEnv = "!env" + AtmosYamlFuncInclude = "!include" + + // For internal use by Atmos when processing the `!include` function + AtmosYamlFuncIncludeLocalFile = "!include-local-file" + AtmosYamlFuncIncludeGoGetter = "!include-go-getter" ) var ( @@ -22,6 +31,8 @@ var ( AtmosYamlFuncStore, AtmosYamlFuncTemplate, AtmosYamlFuncTerraformOutput, + AtmosYamlFuncEnv, + AtmosYamlFuncInclude, } ) @@ -75,26 +86,103 @@ func ConvertToYAML(data any) (string, error) { return string(y), nil } -func processCustomTags(node *yaml.Node) error { +func processCustomTags(atmosConfig *schema.AtmosConfiguration, node *yaml.Node, file string) error { if node.Kind == yaml.DocumentNode && len(node.Content) > 0 { - return processCustomTags(node.Content[0]) + return processCustomTags(atmosConfig, node.Content[0], file) } for i := 0; i < len(node.Content); i++ { n := node.Content[i] if SliceContainsString(AtmosYamlTags, n.Tag) { - n.Value = n.Tag + " " + n.Value + val, err := getValueWithTag(atmosConfig, n, file) + if err != nil { + return err + } + n.Value = val } - if err := processCustomTags(n); err != nil { + if err := processCustomTags(atmosConfig, n, file); err != nil { return err } } return nil } +func getNodeValue(tag string, f string, q string) string { + t := tag + "-local-file \"" + f + "\"" + if q == "" { + return strings.TrimSpace(t) + } + return strings.TrimSpace(t + " \"" + q + "\"") +} + +func getValueWithTag(atmosConfig *schema.AtmosConfiguration, n *yaml.Node, file string) (string, error) { + tag := strings.TrimSpace(n.Tag) + val := strings.TrimSpace(n.Value) + + if tag == AtmosYamlFuncInclude { + var f string + q := "" + + parts, err := SplitStringByDelimiter(val, ' ') + if err != nil { + return "", err + } + + partsLen := len(parts) + + if partsLen == 2 { + f = strings.TrimSpace(parts[0]) + q = strings.TrimSpace(parts[1]) + } else if partsLen == 1 { + f = strings.TrimSpace(parts[0]) + } else { + err := fmt.Errorf("invalid number of arguments in the Atmos YAML function: %s %s. The function accepts 1 or 2 arguments", tag, val) + return "", err + } + + // If absolute path is provided, check if the file exists + if filepath.IsAbs(f) { + if !FileExists(f) { + return "", fmt.Errorf("the function '%s %s' points to a file that does not exist", tag, val) + } + return getNodeValue(tag, f, q), nil + } + + // Detect relative paths (relative to the manifest file) and convert to absolute paths + resolved := ResolveRelativePath(f, file) + if FileExists(resolved) { + resolvedAbsolutePath, err := filepath.Abs(resolved) + if err != nil { + return "", fmt.Errorf("error converting the file path to an ansolute path in the function '%s %s': %v", tag, val, err) + } + return getNodeValue(tag, resolvedAbsolutePath, q), nil + } + + // Check if the `!include` function points to an Atmos stack manifest relative to the `base_path` defined in `atmos.yaml` + atmosManifestPath := filepath.Join(atmosConfig.BasePath, f) + if FileExists(atmosManifestPath) { + atmosManifestAbsolutePath, err := filepath.Abs(atmosManifestPath) + if err != nil { + return "", fmt.Errorf("error converting the file path to an ansolute path in the function '%s %s': %v", tag, val, err) + } + return getNodeValue(tag, atmosManifestAbsolutePath, q), nil + } + + return strings.TrimSpace(tag + "-go-getter " + f + " " + q), nil + } + + return strings.TrimSpace(tag + " " + val), nil +} + +// UnmarshalYAML unmarshals YAML into a Go type func UnmarshalYAML[T any](input string) (T, error) { + return UnmarshalYAMLFromFile[T](&schema.AtmosConfiguration{}, input, "") +} + +// UnmarshalYAMLFromFile unmarshals YAML downloaded from a file into a Go type +func UnmarshalYAMLFromFile[T any](atmosConfig *schema.AtmosConfiguration, input string, file string) (T, error) { var zeroValue T var node yaml.Node b := []byte(input) @@ -104,7 +192,7 @@ func UnmarshalYAML[T any](input string) (T, error) { return zeroValue, err } - if err := processCustomTags(&node); err != nil { + if err := processCustomTags(atmosConfig, &node, file); err != nil { return zeroValue, err } @@ -116,3 +204,22 @@ func UnmarshalYAML[T any](input string) (T, error) { return data, nil } + +// IsYAML checks if data is in YAML format +func IsYAML(data string) bool { + if strings.TrimSpace(data) == "" { + return false + } + + var yml any + err := yaml.Unmarshal([]byte(data), &yml) + if err != nil { + return false + } + + // Ensure that the parsed result is not nil and has some meaningful content + _, isMap := yml.(map[string]any) + _, isSlice := yml.([]any) + + return isMap || isSlice +} diff --git a/pkg/utils/yq_test.go b/pkg/utils/yq_test.go index c8937aa47..932c4f27f 100644 --- a/pkg/utils/yq_test.go +++ b/pkg/utils/yq_test.go @@ -43,7 +43,7 @@ vars: vpc_flow_logs_traffic_type: ALL ` - atmosConfig := schema.AtmosConfiguration{ + atmosConfig := &schema.AtmosConfiguration{ Logs: schema.Logs{ Level: "Trace", }, diff --git a/pkg/utils/yq_utils.go b/pkg/utils/yq_utils.go index 4e8169f73..00ffeae07 100644 --- a/pkg/utils/yq_utils.go +++ b/pkg/utils/yq_utils.go @@ -31,7 +31,7 @@ func (n logBackend) IsEnabledFor(level logging.Level, s string) bool { return false } -func EvaluateYqExpression(atmosConfig schema.AtmosConfiguration, data any, yq string) (any, error) { +func EvaluateYqExpression(atmosConfig *schema.AtmosConfiguration, data any, yq string) (any, error) { // Use the `yqlib` default (chatty) logger only when Atmos Logs Level is set to `Trace` // Otherwise, use the no-op logging backend if atmosConfig.Logs.Level != LogLevelTrace { diff --git a/tests/fixtures/scenarios/complete/stacks/catalog/terraform/template-functions-test2/defaults.yaml b/tests/fixtures/scenarios/complete/stacks/catalog/terraform/template-functions-test2/defaults.yaml index c0e8b02f5..ae2ab094a 100644 --- a/tests/fixtures/scenarios/complete/stacks/catalog/terraform/template-functions-test2/defaults.yaml +++ b/tests/fixtures/scenarios/complete/stacks/catalog/terraform/template-functions-test2/defaults.yaml @@ -51,3 +51,17 @@ components: test_58b: !terraform.output template-functions-test3 .val6.i1 # test_59: !terraform.output does_not_exist val6 # test_60: !terraform.output template-functions-test3 invalid-val + test70: !include ./dev.yaml + test71: !include ./dev.yaml .components.terraform.vpc.vars + test72: !include ./dev.yaml .components.terraform.vpc.vars.ipv4_primary_cidr_block + test73: !include stacks/catalog/vpc/ue2.yaml + test74: !include stacks/catalog/vpc/ue2.yaml .import + test75: !include stacks/catalog/vpc/ue2.yaml .components.terraform.vpc.vars.availability_zones + test76: !include 'stacks/catalog/vpc/ue2.yaml .components.terraform.vpc.vars.availability_zones[2]' + test77: !include a.txt + test78: !include '"b c.txt" ".[1]"' + test79: !include a.hcl + # test80: !env USER + test81: !include https://raw.githubusercontent.com/cloudposse/atmos/refs/heads/main/examples/quick-start-advanced/stacks/catalog/vpc-flow-logs-bucket/defaults.yaml + test82: !include https://raw.githubusercontent.com/cloudposse/atmos/refs/heads/main/examples/quick-start-advanced/stacks/mixins/region/us-east-2.yaml .vars + test83: !include https://api.github.com/meta .api diff --git a/website/docs/core-concepts/stacks/yaml-functions/env.mdx b/website/docs/core-concepts/stacks/yaml-functions/env.mdx new file mode 100644 index 000000000..c869d5282 --- /dev/null +++ b/website/docs/core-concepts/stacks/yaml-functions/env.mdx @@ -0,0 +1,79 @@ +--- +title: "!env" +sidebar_position: 5 +sidebar_label: "!env" +description: Retrieve environment variables and assign them to the sections in Atmos stack manifests +--- + +import Intro from '@site/src/components/Intro' + + + The `!env` Atmos YAML function is used to retrieve environment variables + and assign them to the sections in Atmos stack manifests. + + +## Usage + +The `!env` function can be called with either one or two parameters: + +```yaml + # Get the value of an environment variable. + # If the environment variable is not present in the environment, `null` will be assigned + !env + + # Get the value of an environment variable. + # If the environment variable is not present in the environment, the `default-value` will be assigned + !env +``` + +## Arguments + +
+
`env-var-name`
+
+ Environment variable name +
+ +
`default-value`
+
(Optional) Default value to use if the environment variable is not present in the environment
+
+ +If the function is called with one argument (the name of the environment variable), and the environment variable is +not present, `null` will be assigned to the corresponding section in the Atmos manifest. + +If the function is called with two arguments (the name of the environment variable and the default value), and the +environment variable is not present, the default value will be assigned to the corresponding section in the Atmos manifest. + +## Examples + +```yaml +vars: + # `api_key` will be set to `null` if the environment variable `API_KEY` is not present in the environment + api_key: !env API_KEY + # `app_name` will be set to the default value `my-app` if the environment variable `APP_NAME` is not present in the environment + app_name: !env APP_NAME my-app + +settings: + # `provisioned_by_user` will be set to `null` if the environment variable `ATMOS_USER` is not present in the environment + provisioned_by_user: !env ATMOS_USER +``` + +## Handling default values with spaces + +If you need to provide default values with spaces, enclose them in double quotes and use single quotes around the whole expression. + +For example: + +```yaml + # `app_name` will be set to the default value `my app` if the environment variable `APP_NAME` is not present in the environment + app_name: !env 'APP_NAME "my app"' + + # `app_description` will be set to the default value `my app description` if the environment variable `APP_DESCRIPTION` is not present in the environment + app_description: !env 'APP_DESCRIPTION "my app description"' +``` + +:::tip +You can use [Atmos Stack Manifest Templating](/core-concepts/stacks/templates) in the environment variable names and default values when calling the `!env` YAML function. +Atmos processes the templates first, and then executes the `!env` function, allowing you to provide the parameters to +the function dynamically. +::: diff --git a/website/docs/core-concepts/stacks/yaml-functions/exec.mdx b/website/docs/core-concepts/stacks/yaml-functions/exec.mdx index 0385addbf..ce5f2c6f5 100644 --- a/website/docs/core-concepts/stacks/yaml-functions/exec.mdx +++ b/website/docs/core-concepts/stacks/yaml-functions/exec.mdx @@ -5,7 +5,6 @@ sidebar_label: "!exec" description: Execute shell scripts and assign the results to the sections in Atmos stack manifests --- -import File from '@site/src/components/File' import Intro from '@site/src/components/Intro' diff --git a/website/docs/core-concepts/stacks/yaml-functions/include.mdx b/website/docs/core-concepts/stacks/yaml-functions/include.mdx new file mode 100644 index 000000000..e2f0c7bf7 --- /dev/null +++ b/website/docs/core-concepts/stacks/yaml-functions/include.mdx @@ -0,0 +1,172 @@ +--- +title: "!include" +sidebar_position: 6 +sidebar_label: "!include" +description: Download local or remote files from different sources, and assign the file contents or individual values to Atmos stack manifests +--- + +import Intro from '@site/src/components/Intro' +import File from '@site/src/components/File' + + + The `!include` Atmos YAML function allows downloading local or remote files from different sources, + and assigning the file contents or individual values to the sections in Atmos stack manifests. + + +## Usage + +The `!include` function can be called with either one or two parameters: + +```yaml + # Download the file and assign its content to the variable + !include + + # Download the file, filter the content using the YQ expression, + # and assign the result to the variable + !include +``` + +## Arguments + +
+
`file-path`
+
+ Path to a local or remote file +
+ +
`yq-expression`
+
(Optional) [YQ](https://mikefarah.gitbook.io/yq) expression to retrieve individual values from the file
+
+ +## Examples + + +```yaml +components: + terraform: + my-component: + vars: + # Include a local file with the path relative to the current Atmos manifest + values: !include ./values.yaml + # Include a local file with the path relative to the current Atmos manifest and query the `vars.ipv4_primary_cidr_block` value from the file using YQ + ipv4_primary_cidr_block: !include ./vpc_config.yaml .vars.ipv4_primary_cidr_block + # Include a local file relative to the `base_path` setting in `atmos.yaml` + vpc_defaults: !include stacks/catalog/vpc/defaults.yaml + # Include a local file in HCL format with Terraform variables + hcl_values: !include ./values.hcl + # Include a local file in HCL format with Terraform variables + tfvars_values: !include ../components/terraform/vpc/vpc.tfvars + # Include a local Markdown file + description: !include ./description.md + # Include a local text file + text: !include a.txt + # Include a local text file with spaces in the file name + text2: !include '"my config.txt"' + # Include a local text file on Windows with spaces in the file name, and get the `config.tests` value from the file + tests: !include '"~/My Documents/dev/tests.yaml" .config.tests' + # Download and include a remote YAML file using HTTPS protocol, and query the `vars` section from the file + region_values: !include https://raw.githubusercontent.com/cloudposse/atmos/refs/heads/main/examples/quick-start-advanced/stacks/mixins/region/us-east-2.yaml .vars + # Download and include a remote JSON file and query the `api` section from the file + allowed_ips: !include https://api.github.com/meta .api + settings: + config: + # Include a local JSON file and query the `user_id` variable from the file + user_id: !include ./user_config.json .user_id +``` + + +## Description + +The YAML standard provides [anchors and aliases](https://yaml.org/spec/1.2.2/#3222-anchors-and-aliases), that allow you +to reuse and reference pieces of your YAML file, making it more efficient and reducing duplication. + +Atmos supports YAML anchors and aliases, but the biggest limitation is that they are only available within the file in +which they are defined. You cannot reuse anchors across different files. + +The `!include` Atmos YAML function overcomes this limitation by allowing you to include the content or individual values +from different local and remote sources. The `!include` function also provides the following features: + +- Supports local files with absolute and relative paths. + +- Supports the remote protocols provided by the [`go-getter`](https://github.com/hashicorp/go-getter) library. + +- Allows you to use [YQ](https://mikefarah.gitbook.io/yq) expressions to query and filter the content of the files to retrieve individual values. + +- Automatically detects the format of the files regardless of the file extensions. It supports files in JSON, YAML + and [HCL](https://github.com/hashicorp/hcl) (`tfvars`) formats, and automatically converts them into correct + YAML structures (simple and complex types like maps and lists are supported). + All other files are returned unchanged, allowing you, for example, to include text and + [Markdown](https://www.markdownguide.org/) files as strings in Atmos manifests. + +## Supported File Protocols + +The `!include` function supports the following local file paths: + - absolute paths (e.g., `/Users/me/Documents/values.yaml`) + - paths relative to the current Atmos manifest where the `!include` function is executed (e.g., `./values.yaml`, `../config/values.yaml`) + - paths relative to the [`base_path`](/cli/configuration/#base-path) defined in `atmos.yaml` CLI config file (e.g., `stacks/catalog/vpc/defaults.yaml`) + +To download remote files from different sources, the `!include` function uses [`go-getter`](https://github.com/hashicorp/go-getter) +(used by [Terraform](https://www.terraform.io/) for downloading modules) and supports the following protocols: + +- `tar` - Tar files, potentially compressed (tar.gz, tar.bz2, etc.) +- `zip` - Zip files +- `http` - HTTP URLs +- `https` - HTTPS URLs +- `git` - Git repositories, which can be accessed via HTTPS or SSH +- `hg` - Mercurial repositories, accessed via HTTP/S or SSH +- `s3` - Amazon S3 bucket URLs +- `gcs` - Google Cloud Storage URLs +- `oci` - Open Container Initiative (OCI) images +- `scp` - Secure Copy Protocol for SSH-based transfers +- `sftp` - SSH File Transfer Protocol + +:::tip +You can use [Atmos Stack Manifest Templating](/core-concepts/stacks/templates) in the `!include` YAML function parameters. +Atmos processes the templates first, and then executes the `!include` function, allowing you to provide the parameters to +the function dynamically. +::: + +## Using YQ Expressions to retrieve individual values from files + +To retrieve individual values from complex types such as maps and lists, or do any kind of filtering or querying, +you can utilize [YQ](https://mikefarah.gitbook.io/yq) expressions. + +For example: + +- Retrieve the first item from a list + +```yaml +subnet_id1: !include .private_subnet_ids[0] +``` + +- Read a key from a map + +```yaml +username: !include .config_map.username +``` + +For more details, review the following docs: + +- [YQ Guide](https://mikefarah.gitbook.io/yq) +- [YQ Recipes](https://mikefarah.gitbook.io/yq/recipes) + +## Handling file paths and YQ expressions with spaces + +If you have spaces in the file names or YQ expressions, enclose the file paths and YQ expressions in double quotes +and use single quotes around the whole expression. + +For example, on Windows: + +```yaml + vars: + values: !include '"~/My Documents/dev/values.yaml"' + config: !include '"~/My Documents/dev/config.json" ""' +``` + +On macOS and Linux: + +```yaml + vars: + values: !include './values.yaml ""' + description: !include '"component description.md"' +``` diff --git a/website/docs/core-concepts/stacks/yaml-functions/yaml-functions.mdx b/website/docs/core-concepts/stacks/yaml-functions/yaml-functions.mdx index 6057be54e..7f04c15b8 100644 --- a/website/docs/core-concepts/stacks/yaml-functions/yaml-functions.mdx +++ b/website/docs/core-concepts/stacks/yaml-functions/yaml-functions.mdx @@ -43,6 +43,16 @@ YAML supports three types of data: core, defined, and user-defined. [access component outputs (remote state) directly within Atmos stack manifests](/core-concepts/share-data/remote-state). Note that this requires initializing each component (`terraform init`), which downloads all Terraform providers and may significantly impact performance. + - The [__`!store`__](/core-concepts/stacks/yaml-functions/store) YAML function allows reading the values from a + remote [store](/core-concepts/projects/configuration/stores) (e.g. SSM Parameter Store, Artifactory, etc.) + into Atmos stack manifests + + - The [__`!env`__](/core-concepts/stacks/yaml-functions/env) YAML function is used to retrieve environment variables + and assign them to the sections in Atmos stack manifests + + - The [__`!include`__](/core-concepts/stacks/yaml-functions/include) YAML function allows downloading local or remote files from different sources, + and assigning the file contents or individual values to the sections in Atmos stack manifests + :::tip You can combine [Atmos Stack Manifest Templating](/core-concepts/stacks/templates) with Atmos YAML functions within the same stack configuration. Atmos processes templates first, followed by YAML functions, enabling you to dynamically provide parameters to the YAML functions. @@ -106,6 +116,15 @@ components: # Get the `test_map` output of type map from the `component1` component in the current stack test_10: !terraform.output component1 {{ .stack }} test_map + + # Retrieve the value of an environment variable + api_key: !env API_KEY + + # Include a local file + config: !include ./dev-config.yaml + + # Download a remote file, query data from the file using a YQ expression, and assign the result to the variable + allowed_ips: !include https://api.github.com/meta .api ``` diff --git a/website/docs/integrations/atlantis.mdx b/website/docs/integrations/atlantis.mdx index 14b93a289..87df50367 100644 --- a/website/docs/integrations/atlantis.mdx +++ b/website/docs/integrations/atlantis.mdx @@ -673,7 +673,7 @@ on: branches: [ main ] env: - ATMOS_VERSION: 1.149.0 + ATMOS_VERSION: 1.153.0 ATMOS_CLI_CONFIG_PATH: ./ jobs: