[PLAT-15437][PLAT-15445][YBA CLI] Changes to backup restore set of co…

…mmands

Summary:
- Add KMS config UUID as an option for create restore
- Add `parallelism` and `enableVerboseLogs`, `category` (to select between `yb_backup` and `ybc`) from `yb_backup.py` fields to create backup
- Add `force` flag to list backup and restore to help in automation

```
./yba backup create
Create an universe backup in YugabyteDB Anywhere

Usage:
  yba backup create [flags]

Flags:
      --universe-name string           [Required] Universe name. Name of the universe to be backed up
      --storage-config-name string     [Required] Storage config to be used for taking the backup
      --table-type string              [Required] Table type. Allowed values: ysql, ycql, yedis
      --time-before-delete-in-ms int   [Optional] Retention time of the backup in milliseconds
      --keyspace-info stringArray      [Optional] Keyspace info to perform backup operation.If no keyspace info is provided, then all the keyspaces of the table type specified are backed up. If the user wants to take backup of a subset of keyspaces, then the user has to specify the keyspace info. Provide the following semicolon separated fields as key value pairs, enclose the string with quotes: ""keyspace-name=<keyspace-name>;table-names=<table-name1>,<table-name2>,<table-name3>;table-ids=<table-id1>,<table-id2>,<table-id3>"". The table-names and table-ids attributes have to be specified as comma separated values. Keyspace name is required value. Table names and Table ids are optional values and are needed only for YCQL.Example: --keyspace-info "keyspace-name=cassandra;table-names=table1,table2;table-ids=1e683b86-7858-44d1-a1f6-406f50a4e56e,19a34a5e-3a19-4070-9d79-805ed713ce7d" --keyspace-info "keyspace-name=cassandra2;table-names=table3,table4;table-ids=e5b83a7c-130c-40c0-95ff-ec1d9ecff616,bc92d473-2e10-4f76-8bd1-9ca9741890fd"
      --use-tablespaces                [Optional] Backup tablespaces information as part of the backup
      --sse                            [Optional] Enable sse while persisting the data in AWS S3 (default true)
      --category string                [Optional] Category of the backup. Allowed values: YB_BACKUP_SCRIPT, YB_CONTROLLER
      --enable-verbose-logs            [Optional] Enable verbose logging while taking backup via "yb_backup" script. (default false)
      --parallelism int                [Optional] Number of concurrent commands to run on nodes over SSH via "yb_backup" script. (default 8)
      --base-backup-uuid string        [Optional] Base Backup UUID for taking incremental backups
  -h, --help                           help for create

Global Flags:
  -a, --apiToken string    YugabyteDB Anywhere api token.
      --config string      Config file, defaults to $HOME/.yba-cli.yaml
      --debug              Use debug mode, same as --logLevel debug.
      --disable-color      Disable colors in output. (default false)
  -H, --host string        YugabyteDB Anywhere Host (default "http://localhost:9000")
  -l, --logLevel string    Select the desired log level format. Allowed values: debug, info, warn, error, fatal. (default "info")
  -o, --output string      Select the desired output format. Allowed values: table, json, pretty. (default "table")
      --timeout duration   Wait command timeout, example: 5m, 1h. (default 168h0m0s)
      --wait               Wait until the task is completed, otherwise it will exit immediately. (default true)
```

```
./yba backup restore create
Restore an universe backup in YugabyteDB Anywhere

Usage:
  yba backup restore create [flags]

Flags:
      --universe-name string         [Required] Target universe name to perform restore operation.
      --storage-config-name string   [Required] Storage config to be used for taking the backup
      --kms-config string            [Optional] Key management service config name. For a successful restore, the KMS configuration used for restore should be the same KMS configuration used during backup creation.
      --keyspace-info stringArray    [Required] Keyspace info to perform restore operation. Provide the following semicolon separated fields as key value pairs, and enclose the string with quotes: "'keyspace-name=<keyspace-name>;storage-location=<storage_location>;backup-type=<ycql/ysql>;use-tablespaces=<use-tablespaces>;selective-restore=<selective-restore>;table-name-list=<table-name1>,<table-name2>'". The table-name-list attribute has to be specified as comma separated values. Keyspace name, storage-location and backup-type are required values. The attribute use-tablespaces, selective-restore and table-name-list are optional values. Attributes selective-restore and table-name-list and are needed only for YCQL. The attribute use-tablespaces is to be used if needed only in the case of YSQL. Example: --keyspace-info 'keyspace-name=cassandra1;storage-location=s3://bucket/location1;backup-type=ycql;selective-restore=true;table-name-list=table1,table2' --keyspace-info 'keyspace-name=postgres;storage-location=s3://bucket/location2backup-type=ysql;use-tablespaces=true'
  -h, --help                         help for create

Global Flags:
  -a, --apiToken string    YugabyteDB Anywhere api token.
      --config string      Config file, defaults to $HOME/.yba-cli.yaml
      --debug              Use debug mode, same as --logLevel debug.
      --disable-color      Disable colors in output. (default false)
  -H, --host string        YugabyteDB Anywhere Host (default "http://localhost:9000")
  -l, --logLevel string    Select the desired log level format. Allowed values: debug, info, warn, error, fatal. (default "info")
  -o, --output string      Select the desired output format. Allowed values: table, json, pretty. (default "table")
      --timeout duration   Wait command timeout, example: 5m, 1h. (default 168h0m0s)
      --wait               Wait until the task is completed, otherwise it will exit immediately. (default true)
```

Test Plan:
Manually test commands
```
./yba backup create --universe-name dkumar-cli --storage-config-name dkumar-test-s3 --table-type ysql --time-before-delete-in-ms 3600000 --keyspace-info "keyspace-name=test1" --keyspace-info "keyspace-name=test2" --category YB_CONTROLLER
```

```
./yba backup restore create --universe-name dkumar-cli --storage-config-name dkumar-test-s3 --keyspace-info "keyspace-name=test11;storage-location=<>;backup-type=ysql" --keyspace-info "keyspace-name=test21;storage-location=<>;backup-type=ysql"
```

Reviewers: vpatibandla, sneelakantan

Reviewed By: vpatibandla

Subscribers: yugaware

Differential Revision: https://phorge.dev.yugabyte.com/D38362

managed/yba-cli/cmd/backup/create_backup.go

@@ -139,6 +139,9 @@ var createBackupCmd = &cobra.Command{
 		}
 
 		tableTypeFlag, err := cmd.Flags().GetString("table-type")
+		if err != nil {
+			logrus.Fatalf(formatter.Colorize(err.Error()+"\n", formatter.RedColor))
+		}
 		if !strings.EqualFold(tableTypeFlag, "ysql") &&
 			!strings.EqualFold(tableTypeFlag, "ycql") &&
 			!strings.EqualFold(tableTypeFlag, "yedis") {
@@ -164,6 +167,16 @@ var createBackupCmd = &cobra.Command{
 			logrus.Fatalf(formatter.Colorize(err.Error()+"\n", formatter.RedColor))
 		}
 
+		enableVerboseLogs, err := cmd.Flags().GetBool("enable-verbose-logs")
+		if err != nil {
+			logrus.Fatalf(formatter.Colorize(err.Error()+"\n", formatter.RedColor))
+		}
+
+		parallelism, err := cmd.Flags().GetInt("parallelism")
+		if err != nil {
+			logrus.Fatalf(formatter.Colorize(err.Error()+"\n", formatter.RedColor))
+		}
+
 		var keyspaceTableList []ybaclient.KeyspaceTable
 		keyspaces, err := cmd.Flags().GetStringArray("keyspace-info")
 		if err != nil {
@@ -175,6 +188,11 @@ var createBackupCmd = &cobra.Command{
 			logrus.Fatalf(formatter.Colorize(err.Error()+"\n", formatter.RedColor))
 		}
 
+		category, err := cmd.Flags().GetString("category")
+		if err != nil {
+			logrus.Fatalf(formatter.Colorize(err.Error()+"\n", formatter.RedColor))
+		}
+
 		baseBackupUUID, err := cmd.Flags().GetString("base-backup-uuid")
 		if err != nil {
 			logrus.Fatalf(formatter.Colorize(err.Error()+"\n", formatter.RedColor))
@@ -185,12 +203,15 @@ var createBackupCmd = &cobra.Command{
 			UniverseUUID:      universeUUID,
 			CustomerUUID:      util.GetStringPointer(authAPI.CustomerUUID),
 			StorageConfigUUID: storageUUID,
+			BackupCategory:    util.GetStringPointer(strings.ToUpper(category)),
 			BackupType:        util.GetStringPointer(backupType),
 			KeyspaceTableList: &keyspaceTableList,
 			UseTablespaces:    util.GetBoolPointer(useTablespaces),
 			Sse:               util.GetBoolPointer(sse),
 			TimeBeforeDelete:  util.GetInt64Pointer(timeBeforeDeleteInMs),
 			ExpiryTimeUnit:    util.GetStringPointer("MILLISECONDS"),
+			EnableVerboseLogs: util.GetBoolPointer(enableVerboseLogs),
+			Parallelism:       util.GetInt32Pointer(int32(parallelism)),
 		}
 
 		if (len(strings.TrimSpace(baseBackupUUID))) > 0 {
@@ -245,7 +266,7 @@ var createBackupCmd = &cobra.Command{
 				backupListRequest := authAPI.ListBackups().PageBackupsRequest(backupAPIQuery)
 				r, response, err := backupListRequest.Execute()
 				if err != nil {
-					errMessage := util.ErrorFromHTTPResponse(response, err, "Backup", "Describe Backup")
+					errMessage := util.ErrorFromHTTPResponse(response, err, "Backup", "Create - Describe Backup")
 					logrus.Fatalf(formatter.Colorize(errMessage.Error()+"\n", formatter.RedColor))
 				}
 
@@ -362,6 +383,12 @@ func init() {
 		"[Optional] Backup tablespaces information as part of the backup")
 	createBackupCmd.Flags().Bool("sse", true,
 		"[Optional] Enable sse while persisting the data in AWS S3")
+	createBackupCmd.Flags().String("category", "",
+		"[Optional] Category of the backup. Allowed values: YB_BACKUP_SCRIPT, YB_CONTROLLER")
+	createBackupCmd.Flags().Bool("enable-verbose-logs", false,
+		"[Optional] Enable verbose logging while taking backup via \"yb_backup\" script. (default false)")
+	createBackupCmd.Flags().Int("parallelism", 8,
+		"[Optional] Number of concurrent commands to run on nodes over SSH via \"yb_backup\" script.")
 	createBackupCmd.Flags().String("base-backup-uuid", "",
 		"[Optional] Base Backup UUID for taking incremental backups")
 }

managed/yba-cli/cmd/backup/describe_backup.go

@@ -85,6 +85,22 @@ var describeBackupCmd = &cobra.Command{
 			}
 		}
 
+		backup.KMSConfigs = make([]util.KMSConfig, 0)
+		kmsConfigs, response, err := authAPI.ListKMSConfigs().Execute()
+		if err != nil {
+			errMessage := util.ErrorFromHTTPResponse(response, err,
+				"Backup", "Describe - Get KMS Configurations")
+			logrus.Fatalf(formatter.Colorize(errMessage.Error()+"\n", formatter.RedColor))
+		}
+
+		for _, k := range kmsConfigs {
+			kmsConfig, err := util.ConvertToKMSConfig(k)
+			if err != nil {
+				logrus.Fatalf(formatter.Colorize(err.Error()+"\n", formatter.RedColor))
+			}
+			backup.KMSConfigs = append(backup.KMSConfigs, kmsConfig)
+		}
+
 		if len(r.GetEntities()) > 0 && util.IsOutputType(formatter.TableFormatKey) {
 			fullBackupContext := *backup.NewFullBackupContext()
 			fullBackupContext.Output = os.Stdout

managed/yba-cli/cmd/backup/edit_backup.go

@@ -142,7 +142,6 @@ var editBackupCmd = &cobra.Command{
 		fullBackupContext.Format = backup.NewBackupFormat(viper.GetString("output"))
 		fullBackupContext.SetFullBackup(r.GetEntities()[0])
 		fullBackupContext.Write()
-		return
 	},
 }
 

managed/yba-cli/cmd/backup/list_backup.go

@@ -5,7 +5,6 @@
 package backup
 
 import (
-	"fmt"
 	"os"
 	"strings"
 
@@ -25,6 +24,7 @@ var listBackupCmd = &cobra.Command{
 	Short: "List YugabyteDB Anywhere backups",
 	Long:  "List backups in YugabyteDB Anywhere",
 	Run: func(cmd *cobra.Command, args []string) {
+		viper.BindPFlag("force", cmd.Flags().Lookup("force"))
 		authAPI := ybaAuthClient.NewAuthAPIClientAndCustomer()
 
 		universeUUIDs, err := cmd.Flags().GetString("universe-uuids")
@@ -82,7 +82,8 @@ var listBackupCmd = &cobra.Command{
 		}
 
 		backupListRequest := authAPI.ListBackups().PageBackupsRequest(backupAPIQuery)
-
+		backups := make([]ybaclient.BackupResp, 0)
+		force := viper.GetBool("force")
 		for {
 			// Execute backup list request
 			r, response, err := backupListRequest.Execute()
@@ -94,28 +95,34 @@ var listBackupCmd = &cobra.Command{
 			// Check if backups found
 			if len(r.GetEntities()) < 1 {
 				if util.IsOutputType(formatter.TableFormatKey) {
-					logrus.Infoln("No backups found\n")
+					logrus.Info("No backups found\n")
 				} else {
-					logrus.Infoln("[]\n")
+					logrus.Info("[]\n")
 				}
 				return
 			}
 
 			// Write backup entities
-			backup.Write(backupCtx, r.GetEntities())
+			if force {
+				backups = append(backups, r.GetEntities()...)
+			} else {
+				backup.Write(backupCtx, r.GetEntities())
+			}
 
 			// Check if there are more pages
 			hasNext := r.GetHasNext()
 			if !hasNext {
-				if util.IsOutputType(formatter.TableFormatKey) {
-					logrus.Infoln("No more backups present\n")
+				if util.IsOutputType(formatter.TableFormatKey) && !force {
+					logrus.Info("No more backups present\n")
 				}
 				break
 			}
 
-			// Prompt user for more entries
-			if !promptForMoreEntries() {
-				break
+			err = util.ConfirmCommand(
+				"List more entries",
+				viper.GetBool("force"))
+			if err != nil {
+				logrus.Fatal(formatter.Colorize(err.Error()+"\n", formatter.RedColor))
 			}
 
 			offset += int32(len(r.GetEntities()))
@@ -124,21 +131,18 @@ var listBackupCmd = &cobra.Command{
 			backupAPIQuery.Offset = offset
 			backupListRequest = authAPI.ListBackups().PageBackupsRequest(backupAPIQuery)
 		}
+		if force {
+			backup.Write(backupCtx, backups)
+		}
 	},
 }
 
-// Function to prompt user for more entries
-func promptForMoreEntries() bool {
-	var input string
-	fmt.Print("More entries? (yes/no): ")
-	fmt.Scanln(&input)
-	return strings.ToLower(input) == "yes"
-}
-
 func init() {
 	listBackupCmd.Flags().SortFlags = false
 	listBackupCmd.Flags().String("universe-uuids", "",
 		"[Optional] Comma separated list of universe uuids")
 	listBackupCmd.Flags().String("universe-names", "",
 		"[Optional] Comma separated list of universe names")
+	listBackupCmd.Flags().BoolP("force", "f", false,
+		"[Optional] Bypass the prompt for non-interactive usage.")
 }

managed/yba-cli/cmd/backup/restore/create_restore.go

@@ -22,8 +22,8 @@ import (
 	"github.com/yugabyte/yugabyte-db/managed/yba-cli/internal/formatter/backup/restore"
 )
 
-// createBackupRestoreCmd represents the universe backup command
-var createBackupRestoreCmd = &cobra.Command{
+// createRestoreCmd represents the universe backup command
+var createRestoreCmd = &cobra.Command{
 	Use:   "create",
 	Short: "Restore a YugabyteDB Anywhere universe backup",
 	Long:  "Restore an universe backup in YugabyteDB Anywhere",
@@ -50,13 +50,12 @@ var createBackupRestoreCmd = &cobra.Command{
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		var response *http.Response
-		authAPI, err := ybaAuthClient.NewAuthAPIClient()
+		authAPI := ybaAuthClient.NewAuthAPIClientAndCustomer()
+
+		universeNameFlag, err := cmd.Flags().GetString("universe-name")
 		if err != nil {
 			logrus.Fatalf(formatter.Colorize(err.Error()+"\n", formatter.RedColor))
 		}
-		authAPI.GetCustomerUUID()
-
-		universeNameFlag, err := cmd.Flags().GetString("universe-name")
 
 		universeListRequest := authAPI.ListUniverses()
 		universeListRequest = universeListRequest.Name(universeNameFlag)
@@ -107,7 +106,7 @@ var createBackupRestoreCmd = &cobra.Command{
 		rList = storageConfigsName
 
 		if len(rList) < 1 {
-			fmt.Println("No storage configurations found")
+			fmt.Println("No storage configurations with name " + storageName + " found")
 			return
 		}
 
@@ -116,6 +115,41 @@ var createBackupRestoreCmd = &cobra.Command{
 			storageUUID = rList[0].GetConfigUUID()
 		}
 
+		kmsConfigUUID := ""
+		kmsConfigName, err := cmd.Flags().GetString("kms-config")
+		if err != nil {
+			logrus.Fatalf(formatter.Colorize(err.Error()+"\n", formatter.RedColor))
+		}
+		// find kmsConfigUUID from the name
+		kmsConfigs, response, err := authAPI.ListKMSConfigs().Execute()
+		if err != nil {
+			errMessage := util.ErrorFromHTTPResponse(response, err,
+				"Restore", "Create - Fetch KMS Configs")
+			logrus.Fatalf(formatter.Colorize(errMessage.Error()+"\n", formatter.RedColor))
+		}
+		for _, k := range kmsConfigs {
+			metadataInterface := k["metadata"]
+			if metadataInterface != nil {
+				metadata := metadataInterface.(map[string]interface{})
+				kmsName := metadata["name"]
+				if kmsName != nil && strings.Compare(kmsName.(string), kmsConfigName) == 0 {
+					configUUID := metadata["configUUID"]
+					if configUUID != nil {
+						kmsConfigUUID = configUUID.(string)
+					}
+				}
+			}
+		}
+		if len(strings.TrimSpace(kmsConfigName)) != 0 &&
+			len(strings.TrimSpace(kmsConfigUUID)) == 0 {
+			logrus.Fatalf(
+				formatter.Colorize(
+					fmt.Sprintf("No KMS configuration with name: %s found\n",
+						kmsConfigName),
+					formatter.RedColor,
+				))
+		}
+
 		backupInfos, err := cmd.Flags().GetStringArray("keyspace-info")
 		if err != nil {
 			logrus.Fatalf(formatter.Colorize(err.Error()+"\n", formatter.RedColor))
@@ -133,13 +167,8 @@ var createBackupRestoreCmd = &cobra.Command{
 			CustomerUUID:          util.GetStringPointer(authAPI.CustomerUUID),
 			StorageConfigUUID:     util.GetStringPointer(storageUUID),
 			BackupStorageInfoList: &result,
+			KmsConfigUUID:         util.GetStringPointer(kmsConfigUUID),
 		}
-		data, err = json.Marshal(requestBody)
-		if err != nil {
-			fmt.Println("Error marshalling:", err)
-			return
-		}
-		logrus.Debug(string(data))
 
 		rCreate, response, err := authAPI.RestoreBackup().Backup(requestBody).Execute()
 		if err != nil {
@@ -286,14 +315,18 @@ func buildBackupInfoList(backupInfos []string) (res []ybaclient.BackupStorageInf
 }
 
 func init() {
-	createBackupRestoreCmd.Flags().SortFlags = false
-	createBackupRestoreCmd.Flags().String("universe-name", "",
-		"[Required] Universe name. Name of the universe to be backed up")
-	createBackupRestoreCmd.MarkFlagRequired("universe-name")
-	createBackupRestoreCmd.Flags().String("storage-config-name", "",
+	createRestoreCmd.Flags().SortFlags = false
+	createRestoreCmd.Flags().String("universe-name", "",
+		"[Required] Target universe name to perform restore operation.")
+	createRestoreCmd.MarkFlagRequired("universe-name")
+	createRestoreCmd.Flags().String("storage-config-name", "",
 		"[Required] Storage config to be used for taking the backup")
-	createBackupRestoreCmd.MarkFlagRequired("storage-config-name")
-	createBackupRestoreCmd.Flags().StringArray("keyspace-info", []string{},
+	createRestoreCmd.MarkFlagRequired("storage-config-name")
+	createRestoreCmd.Flags().String("kms-config", "",
+		"[Optional] Key management service config name. "+
+			"For a successful restore, the KMS configuration used for restore should be the same "+
+			"KMS configuration used during backup creation.")
+	createRestoreCmd.Flags().StringArray("keyspace-info", []string{},
 		"[Required] Keyspace info to perform restore operation."+
 			" Provide the following semicolon separated fields as key value pairs, and"+
 			" enclose the string with quotes: "+
@@ -305,11 +338,11 @@ func init() {
 				"values. ", formatter.GreenColor)+
 			"The attribute use-tablespaces, selective-restore and table-name-list are optional values. "+
 			"Attributes selective-restore and table-name-list and are needed only for YCQL. "+
-			"The attribute use-tablespaces is to be used if needed only in the case of YSQL"+
+			"The attribute use-tablespaces is to be used if needed only in the case of YSQL. "+
 			"Example: --keyspace-info 'keyspace-name=cassandra1;storage-location=s3://bucket/location1;"+
 			"backup-type=ycql;selective-restore=true;table-name-list=table1,table2' "+
 			"--keyspace-info 'keyspace-name=postgres;storage-location=s3://bucket/location2"+
 			"backup-type=ysql;use-tablespaces=true'")
-	createBackupRestoreCmd.MarkFlagRequired("keyspace-info")
+	createRestoreCmd.MarkFlagRequired("keyspace-info")
 
 }

managed/yba-cli/cmd/backup/restore/describe_restore.go

@@ -22,11 +22,7 @@ var describeRestoreCmd = &cobra.Command{
 	Short:   "List YugabyteDB Anywhere restores",
 	Long:    "List restores in YugabyteDB Anywhere",
 	Run: func(cmd *cobra.Command, args []string) {
-		authAPI, err := ybaAuthClient.NewAuthAPIClient()
-		if err != nil {
-			logrus.Fatalf(formatter.Colorize(err.Error()+"\n", formatter.RedColor))
-		}
-		authAPI.GetCustomerUUID()
+		authAPI := ybaAuthClient.NewAuthAPIClientAndCustomer()
 
 		restoreUUID, err := cmd.Flags().GetString("uuid")
 		if err != nil {