From 5db60f09df28c3309601d02c06dc279e4c1c6d47 Mon Sep 17 00:00:00 2001 From: Rohit Nadig Date: Fri, 31 Jan 2025 02:38:00 +0000 Subject: [PATCH 1/3] Enhance deploy to output JSON instead of plain-english --- cmd/deploy.go | 15 ++++----- cmd/deploy_test.go | 2 +- cmd/destroy.go | 4 ++- cmd/export.go | 3 +- cmd/utils.go | 14 +++++++++ pkg/shell/terraform.go | 69 ++++++++++++++++++++++++++++++++++++------ 6 files changed, 87 insertions(+), 20 deletions(-) diff --git a/cmd/deploy.go b/cmd/deploy.go index 07e630eb5e..f3ffdc0136 100644 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -28,10 +28,11 @@ import ( ) func addDeployFlags(c *cobra.Command) *cobra.Command { - return addGroupSelectionFlags( - addAutoApproveFlag( - addArtifactsDirFlag( - addCreateFlags(c)))) + return addJsonOutputFlag( + addGroupSelectionFlags( + addAutoApproveFlag( + addArtifactsDirFlag( + addCreateFlags(c))))) } func init() { @@ -93,7 +94,7 @@ func doDeploy(deplRoot string) { moduleDir := filepath.Join(groupDir, subPath) checkErr(deployPackerGroup(moduleDir, getApplyBehavior()), ctx) case config.TerraformKind: - checkErr(deployTerraformGroup(groupDir, artDir, getApplyBehavior()), ctx) + checkErr(deployTerraformGroup(groupDir, artDir, getApplyBehavior(), getJsonOutputBehavior()), ctx) default: checkErr( config.BpError{ @@ -152,10 +153,10 @@ func deployPackerGroup(moduleDir string, applyBehavior shell.ApplyBehavior) erro return nil } -func deployTerraformGroup(groupDir string, artifactsDir string, applyBehavior shell.ApplyBehavior) error { +func deployTerraformGroup(groupDir string, artifactsDir string, applyBehavior shell.ApplyBehavior, outputFormat shell.OutputFormat) error { tf, err := shell.ConfigureTerraform(groupDir) if err != nil { return err } - return shell.ExportOutputs(tf, artifactsDir, applyBehavior) + return shell.ExportOutputs(tf, artifactsDir, applyBehavior, outputFormat) } diff --git a/cmd/deploy_test.go b/cmd/deploy_test.go index e7d9c2ac1c..a865f279d4 100644 --- a/cmd/deploy_test.go +++ b/cmd/deploy_test.go @@ -28,7 +28,7 @@ func (s *MySuite) TestDeployGroups(c *C) { pathEnv := os.Getenv("PATH") os.Setenv("PATH", "") - err = deployTerraformGroup(".", getArtifactsDir("."), shell.NeverApply) + err = deployTerraformGroup(".", getArtifactsDir("."), shell.NeverApply, shell.TextOutput) c.Check(err, NotNil) err = deployPackerGroup(".", shell.NeverApply) diff --git a/cmd/destroy.go b/cmd/destroy.go index b8794c3034..0acd62c8eb 100644 --- a/cmd/destroy.go +++ b/cmd/destroy.go @@ -106,7 +106,9 @@ func destroyTerraformGroup(groupDir string) error { return err } - return shell.Destroy(tf, getApplyBehavior()) + // Always output text when destroying the cluster + // The current implementation outputs JSON only for the "deploy" command + return shell.Destroy(tf, getApplyBehavior(), shell.TextOutput) } func destroyChoice(nextGroup config.GroupName) bool { diff --git a/cmd/export.go b/cmd/export.go index f4bb1adb4a..7d55a54c89 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -71,5 +71,6 @@ func runExportCmd(cmd *cobra.Command, args []string) { tf, err := shell.ConfigureTerraform(groupDir) checkErr(err, ctx) - checkErr(shell.ExportOutputs(tf, artifactsDir, shell.NeverApply), ctx) + // Always output text when exporting (never JSON) + checkErr(shell.ExportOutputs(tf, artifactsDir, shell.NeverApply, shell.TextOutput), ctx) } diff --git a/cmd/utils.go b/cmd/utils.go index 65bb2959e9..81e5d3aa87 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -82,6 +82,20 @@ func filterYaml(cmd *cobra.Command, args []string, toComplete string) ([]string, return []string{"yaml", "yml"}, cobra.ShellCompDirectiveFilterFileExt } +var flagJsonOutput bool + +func getJsonOutputBehavior() shell.OutputFormat { + if flagJsonOutput { + return shell.JsonOutput + } + return shell.TextOutput +} + +func addJsonOutputFlag(c *cobra.Command) *cobra.Command { + c.Flags().BoolVar(&flagJsonOutput, "json-output", false, "Output errors in JSON format") + return c +} + var flagSkipGroups []string var flagOnlyGroups []string diff --git a/pkg/shell/terraform.go b/pkg/shell/terraform.go index 289518f79c..489bf31dfc 100644 --- a/pkg/shell/terraform.go +++ b/pkg/shell/terraform.go @@ -36,6 +36,20 @@ import ( "github.com/zclconf/go-cty/cty/gocty" ) +// OutputFormat determines the format in which the errors are reported. +// Current supported format are text (default option) and JSON. Future +// format could be protobuf +type OutputFormat uint + +// 2 output formats are currently supported: +// - Text +// - JSON +// - Future option is ProtoBuf +const ( + TextOutput OutputFormat = iota + JsonOutput +) + // ApplyBehavior abstracts behaviors for making changes to cloud infrastructure // when gcluster believes that they may be necessary type ApplyBehavior uint @@ -224,6 +238,31 @@ func promptForApply(tf *tfexec.Terraform, path string, b ApplyBehavior) bool { } } +// This function applies the terraform plan, but generates outputs in JSON format +// (instead of text) +func applyPlanJsonOutput(tf *tfexec.Terraform, path string) error { + planFileOpt := tfexec.DirOrPlan(path) + logging.Info("Running terraform apply on deployment group %s", tf.WorkingDir()) + // To do: Make file name as a user input + // Make the JSON file name unique by having the Terraform group as a substring + jsonFilename := "cluster_toolkit_output-" + strings.ReplaceAll(tf.WorkingDir(), "/", ".") + ".json" + logging.Info("Writing to JSON file %s", jsonFilename) + jsonFile, err := os.Create(jsonFilename) + defer jsonFile.Close() + if err != nil { + logging.Info("Cannot create JSON output file %s", jsonFilename) + return err + } + tf.SetStdout(os.Stdout) + tf.SetStderr(os.Stderr) + if err := tf.ApplyJSON(context.Background(), jsonFile, planFileOpt); err != nil { + return err + } + tf.SetStdout(nil) + tf.SetStderr(nil) + return nil +} + func applyPlanConsoleOutput(tf *tfexec.Terraform, path string) error { planFileOpt := tfexec.DirOrPlan(path) logging.Info("Running terraform apply on deployment group %s", tf.WorkingDir()) @@ -240,7 +279,7 @@ func applyPlanConsoleOutput(tf *tfexec.Terraform, path string) error { // generate a Terraform plan to apply or destroy a module // recall "destroy" is just an alias for "apply -destroy"! // apply the plan automatically or after prompting the user -func applyOrDestroy(tf *tfexec.Terraform, b ApplyBehavior, destroy bool) error { +func applyOrDestroy(tf *tfexec.Terraform, b ApplyBehavior, of OutputFormat, destroy bool) error { action := "adding or changing" pastTense := "applied" if destroy { @@ -256,6 +295,7 @@ func applyOrDestroy(tf *tfexec.Terraform, b ApplyBehavior, destroy bool) error { // capture Terraform plan in a file f, err := os.CreateTemp("", "plan-)") if err != nil { + logging.Info("deploy.go.0000: applyOrDestroy()") return err } defer os.Remove(f.Name()) @@ -263,7 +303,6 @@ func applyOrDestroy(tf *tfexec.Terraform, b ApplyBehavior, destroy bool) error { if err != nil { return err } - var apply bool if wantsChange { logging.Info("Deployment group %s requires %s cloud infrastructure", tf.WorkingDir(), action) @@ -276,15 +315,25 @@ func applyOrDestroy(tf *tfexec.Terraform, b ApplyBehavior, destroy bool) error { return nil } - if err := applyPlanConsoleOutput(tf, f.Name()); err != nil { - return err + switch of { + + case JsonOutput: + if err := applyPlanJsonOutput(tf, f.Name()); err != nil { + return err + } + case TextOutput: // Text output to the console is also the default choice + fallthrough + default: + if err := applyPlanConsoleOutput(tf, f.Name()); err != nil { + return err + } } return nil } -func getOutputs(tf *tfexec.Terraform, b ApplyBehavior) (map[string]cty.Value, error) { - err := applyOrDestroy(tf, b, false) +func getOutputs(tf *tfexec.Terraform, b ApplyBehavior, o OutputFormat) (map[string]cty.Value, error) { + err := applyOrDestroy(tf, b, o, false) if err != nil { return nil, err } @@ -302,11 +351,11 @@ func outputsFile(artifactsDir string, group config.GroupName) string { // ExportOutputs will run terraform output and capture data needed for // subsequent deployment groups -func ExportOutputs(tf *tfexec.Terraform, artifactsDir string, applyBehavior ApplyBehavior) error { +func ExportOutputs(tf *tfexec.Terraform, artifactsDir string, applyBehavior ApplyBehavior, outputFormat OutputFormat) error { thisGroup := config.GroupName(filepath.Base(tf.WorkingDir())) filepath := outputsFile(artifactsDir, thisGroup) - outputValues, err := getOutputs(tf, applyBehavior) + outputValues, err := getOutputs(tf, applyBehavior, outputFormat) if err != nil { return err } @@ -428,8 +477,8 @@ func ImportInputs(groupDir string, artifactsDir string, bp config.Blueprint) err } // Destroy destroys all infrastructure in the module working directory -func Destroy(tf *tfexec.Terraform, b ApplyBehavior) error { - return applyOrDestroy(tf, b, true) +func Destroy(tf *tfexec.Terraform, b ApplyBehavior, o OutputFormat) error { + return applyOrDestroy(tf, b, o, true) } func TfVersion() (string, error) { From 9eb4d73f322b5849515fc3529e11dd196da48696 Mon Sep 17 00:00:00 2001 From: Rohit Nadig Date: Fri, 31 Jan 2025 02:38:00 +0000 Subject: [PATCH 2/3] Enhance deploy to output JSON instead of plain-english --- cmd/deploy.go | 2 +- cmd/utils.go | 4 ++-- pkg/shell/terraform.go | 29 ++++++++++++++++++++--------- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/cmd/deploy.go b/cmd/deploy.go index f3ffdc0136..c9ca595fa6 100644 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -94,7 +94,7 @@ func doDeploy(deplRoot string) { moduleDir := filepath.Join(groupDir, subPath) checkErr(deployPackerGroup(moduleDir, getApplyBehavior()), ctx) case config.TerraformKind: - checkErr(deployTerraformGroup(groupDir, artDir, getApplyBehavior(), getJsonOutputBehavior()), ctx) + checkErr(deployTerraformGroup(groupDir, artDir, getApplyBehavior(), getOutputFormat()), ctx) default: checkErr( config.BpError{ diff --git a/cmd/utils.go b/cmd/utils.go index 81e5d3aa87..17a49c229f 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -84,7 +84,7 @@ func filterYaml(cmd *cobra.Command, args []string, toComplete string) ([]string, var flagJsonOutput bool -func getJsonOutputBehavior() shell.OutputFormat { +func getOutputFormat() shell.OutputFormat { if flagJsonOutput { return shell.JsonOutput } @@ -92,7 +92,7 @@ func getJsonOutputBehavior() shell.OutputFormat { } func addJsonOutputFlag(c *cobra.Command) *cobra.Command { - c.Flags().BoolVar(&flagJsonOutput, "json-output", false, "Output errors in JSON format") + c.Flags().BoolVar(&flagJsonOutput, "json-output", false, "Write program output in JSON format (instead of default: english text)") return c } diff --git a/pkg/shell/terraform.go b/pkg/shell/terraform.go index 489bf31dfc..a5187d738c 100644 --- a/pkg/shell/terraform.go +++ b/pkg/shell/terraform.go @@ -29,6 +29,7 @@ import ( "os/exec" "path/filepath" "regexp" + "runtime" "strings" "github.com/hashicorp/terraform-exec/tfexec" @@ -41,15 +42,14 @@ import ( // format could be protobuf type OutputFormat uint -// 2 output formats are currently supported: -// - Text -// - JSON -// - Future option is ProtoBuf +// Future option could be ProtoBuf const ( TextOutput OutputFormat = iota JsonOutput ) +const supportedOutputFormats = "Text , JSON " + // ApplyBehavior abstracts behaviors for making changes to cloud infrastructure // when gcluster believes that they may be necessary type ApplyBehavior uint @@ -243,10 +243,22 @@ func promptForApply(tf *tfexec.Terraform, path string, b ApplyBehavior) bool { func applyPlanJsonOutput(tf *tfexec.Terraform, path string) error { planFileOpt := tfexec.DirOrPlan(path) logging.Info("Running terraform apply on deployment group %s", tf.WorkingDir()) + + replaceChar := "" + switch runtime.GOOS { + case "darwin": + fallthrough + case "linux": + replaceChar = "/" + case "windows": + replaceChar = "\\" + } + // To do: Make file name as a user input // Make the JSON file name unique by having the Terraform group as a substring - jsonFilename := "cluster_toolkit_output-" + strings.ReplaceAll(tf.WorkingDir(), "/", ".") + ".json" - logging.Info("Writing to JSON file %s", jsonFilename) + jsonFilename := "cluster_toolkit_output-" + strings.ReplaceAll(tf.WorkingDir(), replaceChar, ".") + ".json" + + logging.Info("Writing output to JSON file %s", jsonFilename) jsonFile, err := os.Create(jsonFilename) defer jsonFile.Close() if err != nil { @@ -316,17 +328,16 @@ func applyOrDestroy(tf *tfexec.Terraform, b ApplyBehavior, of OutputFormat, dest } switch of { - case JsonOutput: if err := applyPlanJsonOutput(tf, f.Name()); err != nil { return err } case TextOutput: // Text output to the console is also the default choice - fallthrough - default: if err := applyPlanConsoleOutput(tf, f.Name()); err != nil { return err } + default: + panic(fmt.Sprintf("Unsupported output format requested. Supported output formats: %s", supportedOutputFormats)) } return nil From b2e6b390c0c876cc7840c79367c42b875a987c1d Mon Sep 17 00:00:00 2001 From: Rohit Nadig Date: Fri, 31 Jan 2025 02:38:00 +0000 Subject: [PATCH 3/3] Enhance deploy to output JSON instead of plain-english --- pkg/shell/terraform.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/shell/terraform.go b/pkg/shell/terraform.go index a5187d738c..85f8869156 100644 --- a/pkg/shell/terraform.go +++ b/pkg/shell/terraform.go @@ -307,7 +307,6 @@ func applyOrDestroy(tf *tfexec.Terraform, b ApplyBehavior, of OutputFormat, dest // capture Terraform plan in a file f, err := os.CreateTemp("", "plan-)") if err != nil { - logging.Info("deploy.go.0000: applyOrDestroy()") return err } defer os.Remove(f.Name())