Make CSI snapshot creation timeout configurable for backup and schedule.

Signed-off-by: Xun Jiang <[email protected]>

changelogs/unreleased/5102-jxun

@@ -0,0 +1 @@
+Make CSI snapshot creation timeout configurable.

config/crd/v1/bases/velero.io_backups.yaml

@@ -37,6 +37,12 @@ spec:
           spec:
             description: BackupSpec defines the specification for a Velero backup.
             properties:
+              csiSnapshotCreationTimeout:
+                description: CSISnapshotCreationTimeout specifies the time used to
+                  wait for CSI VolumeSnapshot status turns to ReadyToUse during creation,
+                  before returning error as timeout.
+                nullable: true
+                type: string
               defaultVolumesToRestic:
                 description: DefaultVolumesToRestic specifies whether restic should
                   be used to take a backup of all pod volumes by default.

config/crd/v1/bases/velero.io_schedules.yaml

@@ -61,6 +61,12 @@ spec:
                 description: Template is the definition of the Backup to be run on
                   the provided schedule
                 properties:
+                  csiSnapshotCreationTimeout:
+                    description: CSISnapshotCreationTimeout specifies the time used
+                      to wait for CSI VolumeSnapshot status turns to ReadyToUse during
+                      creation, before returning error as timeout.
+                    nullable: true
+                    type: string
                   defaultVolumesToRestic:
                     description: DefaultVolumesToRestic specifies whether restic should
                       be used to take a backup of all pod volumes by default.

pkg/apis/velero/v1/backup.go

@@ -110,6 +110,12 @@ type BackupSpec struct {
 	// +optional
 	// +nullable
 	OrderedResources map[string]string `json:"orderedResources,omitempty"`
+
+	// CSISnapshotCreationTimeout specifies the time used to wait for CSI VolumeSnapshot status turns to
+	// ReadyToUse during creation, before returning error as timeout.
+	// +optional
+	// +nullable
+	CSISnapshotCreationTimeout metav1.Duration `json:"csiSnapshotCreationTimeout,omitempty"`
 }
 
 // BackupHooks contains custom behaviors that should be executed at different phases of the backup.

pkg/builder/backup_builder.go

@@ -233,3 +233,9 @@ func (b *BackupBuilder) OrderedResources(orders map[string]string) *BackupBuilde
 	b.object.Spec.OrderedResources = orders
 	return b
 }
+
+// CSISnapshotCreationTimeout sets the Backup's CSISnapshotCreationTimeout
+func (b *BackupBuilder) CSISnapshotCreationTimeout(timeout time.Duration) *BackupBuilder {
+	b.object.Spec.CSISnapshotCreationTimeout.Duration = timeout
+	return b
+}

pkg/cmd/cli/backup/create.go

@@ -82,22 +82,23 @@ func NewCreateCommand(f client.Factory, use string) *cobra.Command {
 }
 
 type CreateOptions struct {
-	Name                    string
-	TTL                     time.Duration
-	SnapshotVolumes         flag.OptionalBool
-	DefaultVolumesToRestic  flag.OptionalBool
-	IncludeNamespaces       flag.StringArray
-	ExcludeNamespaces       flag.StringArray
-	IncludeResources        flag.StringArray
-	ExcludeResources        flag.StringArray
-	Labels                  flag.Map
-	Selector                flag.LabelSelector
-	IncludeClusterResources flag.OptionalBool
-	Wait                    bool
-	StorageLocation         string
-	SnapshotLocations       []string
-	FromSchedule            string
-	OrderedResources        string
+	Name                       string
+	TTL                        time.Duration
+	SnapshotVolumes            flag.OptionalBool
+	DefaultVolumesToRestic     flag.OptionalBool
+	IncludeNamespaces          flag.StringArray
+	ExcludeNamespaces          flag.StringArray
+	IncludeResources           flag.StringArray
+	ExcludeResources           flag.StringArray
+	Labels                     flag.Map
+	Selector                   flag.LabelSelector
+	IncludeClusterResources    flag.OptionalBool
+	Wait                       bool
+	StorageLocation            string
+	SnapshotLocations          []string
+	FromSchedule               string
+	OrderedResources           string
+	CSISnapshotCreationTimeout time.Duration
 
 	client veleroclient.Interface
 }
@@ -122,6 +123,7 @@ func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) {
 	flags.StringSliceVar(&o.SnapshotLocations, "volume-snapshot-locations", o.SnapshotLocations, "List of locations (at most one per provider) where volume snapshots should be stored.")
 	flags.VarP(&o.Selector, "selector", "l", "Only back up resources matching this label selector.")
 	flags.StringVar(&o.OrderedResources, "ordered-resources", "", "Mapping Kinds to an ordered list of specific resources of that Kind.  Resource names are separated by commas and their names are in format 'namespace/resourcename'. For cluster scope resource, simply use resource name. Key-value pairs in the mapping are separated by semi-colon.  Example: 'pods=ns1/pod1,ns1/pod2;persistentvolumeclaims=ns1/pvc4,ns1/pvc8'.  Optional.")
+	flags.DurationVar(&o.CSISnapshotCreationTimeout, "csi-snapshot-creation-timeout", o.CSISnapshotCreationTimeout, "How long to wait for CSI snapshot creation before timeout.")
 	f := flags.VarPF(&o.SnapshotVolumes, "snapshot-volumes", "", "Take snapshots of PersistentVolumes as part of the backup.")
 	// this allows the user to just specify "--snapshot-volumes" as shorthand for "--snapshot-volumes=true"
 	// like a normal bool flag
@@ -332,7 +334,8 @@ func (o *CreateOptions) BuildBackup(namespace string) (*velerov1api.Backup, erro
 			LabelSelector(o.Selector.LabelSelector).
 			TTL(o.TTL).
 			StorageLocation(o.StorageLocation).
-			VolumeSnapshotLocations(o.SnapshotLocations...)
+			VolumeSnapshotLocations(o.SnapshotLocations...).
+			CSISnapshotCreationTimeout(o.CSISnapshotCreationTimeout)
 		if len(o.OrderedResources) > 0 {
 			orders, err := ParseOrderedResources(o.OrderedResources)
 			if err != nil {

pkg/cmd/cli/backup/create_test.go

@@ -19,6 +19,7 @@ package backup
 import (
 	"context"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -35,17 +36,19 @@ func TestCreateOptions_BuildBackup(t *testing.T) {
 	o.Labels.Set("velero.io/test=true")
 	o.OrderedResources = "pods=p1,p2;persistentvolumeclaims=pvc1,pvc2"
 	orders, err := ParseOrderedResources(o.OrderedResources)
+	o.CSISnapshotCreationTimeout = 20 * time.Minute
 	assert.NoError(t, err)
 
 	backup, err := o.BuildBackup(testNamespace)
 	assert.NoError(t, err)
 
 	assert.Equal(t, velerov1api.BackupSpec{
-		TTL:                     metav1.Duration{Duration: o.TTL},
-		IncludedNamespaces:      []string(o.IncludeNamespaces),
-		SnapshotVolumes:         o.SnapshotVolumes.Value,
-		IncludeClusterResources: o.IncludeClusterResources.Value,
-		OrderedResources:        orders,
+		TTL:                        metav1.Duration{Duration: o.TTL},
+		IncludedNamespaces:         []string(o.IncludeNamespaces),
+		SnapshotVolumes:            o.SnapshotVolumes.Value,
+		IncludeClusterResources:    o.IncludeClusterResources.Value,
+		OrderedResources:           orders,
+		CSISnapshotCreationTimeout: metav1.Duration{Duration: o.CSISnapshotCreationTimeout},
 	}, backup.Spec)
 
 	assert.Equal(t, map[string]string{

pkg/cmd/cli/schedule/create.go

@@ -133,18 +133,19 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error {
 		},
 		Spec: api.ScheduleSpec{
 			Template: api.BackupSpec{
-				IncludedNamespaces:      o.BackupOptions.IncludeNamespaces,
-				ExcludedNamespaces:      o.BackupOptions.ExcludeNamespaces,
-				IncludedResources:       o.BackupOptions.IncludeResources,
-				ExcludedResources:       o.BackupOptions.ExcludeResources,
-				IncludeClusterResources: o.BackupOptions.IncludeClusterResources.Value,
-				LabelSelector:           o.BackupOptions.Selector.LabelSelector,
-				SnapshotVolumes:         o.BackupOptions.SnapshotVolumes.Value,
-				TTL:                     metav1.Duration{Duration: o.BackupOptions.TTL},
-				StorageLocation:         o.BackupOptions.StorageLocation,
-				VolumeSnapshotLocations: o.BackupOptions.SnapshotLocations,
-				DefaultVolumesToRestic:  o.BackupOptions.DefaultVolumesToRestic.Value,
-				OrderedResources:        orders,
+				IncludedNamespaces:         o.BackupOptions.IncludeNamespaces,
+				ExcludedNamespaces:         o.BackupOptions.ExcludeNamespaces,
+				IncludedResources:          o.BackupOptions.IncludeResources,
+				ExcludedResources:          o.BackupOptions.ExcludeResources,
+				IncludeClusterResources:    o.BackupOptions.IncludeClusterResources.Value,
+				LabelSelector:              o.BackupOptions.Selector.LabelSelector,
+				SnapshotVolumes:            o.BackupOptions.SnapshotVolumes.Value,
+				TTL:                        metav1.Duration{Duration: o.BackupOptions.TTL},
+				StorageLocation:            o.BackupOptions.StorageLocation,
+				VolumeSnapshotLocations:    o.BackupOptions.SnapshotLocations,
+				DefaultVolumesToRestic:     o.BackupOptions.DefaultVolumesToRestic.Value,
+				OrderedResources:           orders,
+				CSISnapshotCreationTimeout: metav1.Duration{Duration: o.BackupOptions.CSISnapshotCreationTimeout},
 			},
 			Schedule:                   o.Schedule,
 			UseOwnerReferencesInBackup: &o.UseOwnerReferencesInBackup,

pkg/cmd/server/server.go

@@ -102,28 +102,30 @@ const (
 	// the default TTL for a backup
 	defaultBackupTTL = 30 * 24 * time.Hour
 
+	defaultCSISnapshotCreationTimeout = 10 * time.Minute
+
 	// defaultCredentialsDirectory is the path on disk where credential
 	// files will be written to
 	defaultCredentialsDirectory = "/tmp/credentials"
 )
 
 type serverConfig struct {
 	// TODO(2.0) Deprecate defaultBackupLocation
-	pluginDir, metricsAddress, defaultBackupLocation                        string
-	backupSyncPeriod, podVolumeOperationTimeout, resourceTerminatingTimeout time.Duration
-	defaultBackupTTL, storeValidationFrequency                              time.Duration
-	restoreResourcePriorities                                               []string
-	defaultVolumeSnapshotLocations                                          map[string]string
-	restoreOnly                                                             bool
-	disabledControllers                                                     []string
-	clientQPS                                                               float32
-	clientBurst                                                             int
-	clientPageSize                                                          int
-	profilerAddress                                                         string
-	formatFlag                                                              *logging.FormatFlag
-	defaultResticMaintenanceFrequency                                       time.Duration
-	garbageCollectionFrequency                                              time.Duration
-	defaultVolumesToRestic                                                  bool
+	pluginDir, metricsAddress, defaultBackupLocation                              string
+	backupSyncPeriod, podVolumeOperationTimeout, resourceTerminatingTimeout       time.Duration
+	defaultBackupTTL, storeValidationFrequency, defaultCSISnapshotCreationTimeout time.Duration
+	restoreResourcePriorities                                                     []string
+	defaultVolumeSnapshotLocations                                                map[string]string
+	restoreOnly                                                                   bool
+	disabledControllers                                                           []string
+	clientQPS                                                                     float32
+	clientBurst                                                                   int
+	clientPageSize                                                                int
+	profilerAddress                                                               string
+	formatFlag                                                                    *logging.FormatFlag
+	defaultResticMaintenanceFrequency                                             time.Duration
+	garbageCollectionFrequency                                                    time.Duration
+	defaultVolumesToRestic                                                        bool
 }
 
 type controllerRunInfo struct {
@@ -142,6 +144,7 @@ func NewCommand(f client.Factory) *cobra.Command {
 			defaultVolumeSnapshotLocations:    make(map[string]string),
 			backupSyncPeriod:                  defaultBackupSyncPeriod,
 			defaultBackupTTL:                  defaultBackupTTL,
+			defaultCSISnapshotCreationTimeout: defaultCSISnapshotCreationTimeout,
 			storeValidationFrequency:          defaultStoreValidationFrequency,
 			podVolumeOperationTimeout:         defaultPodVolumeOperationTimeout,
 			restoreResourcePriorities:         defaultRestorePriorities,
@@ -650,6 +653,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string
 			s.config.defaultBackupLocation,
 			s.config.defaultVolumesToRestic,
 			s.config.defaultBackupTTL,
+			s.config.defaultCSISnapshotCreationTimeout,
 			s.sharedInformerFactory.Velero().V1().VolumeSnapshotLocations().Lister(),
 			defaultVolumeSnapshotLocations,
 			s.metrics,