Merge branch 'vmware-tanzu:main' into main

.gitignore

@@ -46,7 +46,7 @@ _tiltbuild
 tilt-resources/tilt-settings.json
 tilt-resources/velero_v1_backupstoragelocation.yaml
 tilt-resources/deployment.yaml
-tilt-resources/restic.yaml
+tilt-resources/node-agent.yaml
 tilt-resources/cloud
 
 test/e2e/report.xml

MAINTAINERS.md

@@ -30,9 +30,10 @@
 
 | Feature Area | Lead |
 | ----------------------------- | :---------------------: |
-| Architect | Dave Smith-Uchida (dsu-igeek) |
-| Technical Lead | Daniel Jiang (reasonerjt) |
+| Architect | Dave Smith-Uchida [dsu-igeek](https://github.com/dsu-igeek) |
+| Technical Lead | Daniel Jiang [reasonerjt](https://github.com/reasonerjt) |
 | Kubernetes CSI Liaison |  |
 | Deployment |  |
-| Community Management | Orlin Vasilev (OrlinVasilev) |
-| Product Management | Eleanor Millman (eleanor-millman) |
+| Community Management | Orlin Vasilev [OrlinVasilev](https://github.com/OrlinVasilev) |
+| Product Management | Pradeep Kumar Chaturvedi [pradeepkchaturvedi](https://github.com/pradeepkchaturvedi) |
+

Tiltfile

@@ -17,7 +17,7 @@ k8s_yaml([
 # default values
 settings = {
     "default_registry": "docker.io/velero",
-    "enable_restic": False,
+    "use_node_agent": False,
     "enable_debug": False,
     "debug_continue_on_start": True,  # Continue the velero process by default when in debug mode
     "create_backup_locations": False,
@@ -34,9 +34,9 @@ k8s_yaml(kustomize('tilt-resources'))
 k8s_yaml('tilt-resources/deployment.yaml')
 if settings.get("enable_debug"):
     k8s_resource('velero', port_forwards = '2345')
-    # TODO: Need to figure out how to apply port forwards for all restic pods
-if settings.get("enable_restic"):
-    k8s_yaml('tilt-resources/restic.yaml')
+    # TODO: Need to figure out how to apply port forwards for all node-agent pods
+if settings.get("use_node_agent"):
+    k8s_yaml('tilt-resources/node-agent.yaml')
 if settings.get("create_backup_locations"):
     k8s_yaml('tilt-resources/velero_v1_backupstoragelocation.yaml')
 if settings.get("setup-minio"):

changelogs/unreleased/5390-lyndon

@@ -0,0 +1 @@
+Rename Velero daemonset from "restic" to "node-agent"

changelogs/unreleased/5429-reasonerjt

@@ -0,0 +1 @@
+Skip the exclusion check for additional resources returned by BIA

changelogs/unreleased/5441-sseago

@@ -0,0 +1 @@
+moved RIA execute input/output structs back to velero package

design/CLI/PoC/overlays/plugins/restic.yaml → design/CLI/PoC/overlays/plugins/node-agent.yaml

@@ -5,22 +5,22 @@ metadata:
   creationTimestamp: null
   labels:
     component: velero
-  name: restic
+  name: node-agent
   namespace: velero
 spec:
   selector:
     matchLabels:
-      name: restic
+      name: node-agent
   template:
     metadata:
       creationTimestamp: null
       labels:
         component: velero
-        name: restic
+        name: node-agent
     spec:
       containers:
         - args:
-            - restic
+            - node-agent
             - server
           command:
             - /velero
@@ -43,7 +43,7 @@ spec:
               value: /credentials/cloud
           image: velero/velero:latest
           imagePullPolicy: Always
-          name: restic
+          name: node-agent
           resources: {}
           volumeMounts:
             - mountPath: /host_pods

pkg/backup/backup.go

@@ -416,7 +416,7 @@ func (kb *kubernetesBackupper) BackupWithResolvers(log logrus.FieldLogger,
 }
 
 func (kb *kubernetesBackupper) backupItem(log logrus.FieldLogger, gr schema.GroupResource, itemBackupper *itemBackupper, unstructured *unstructured.Unstructured, preferredGVR schema.GroupVersionResource) bool {
-	backedUpItem, err := itemBackupper.backupItem(log, unstructured, gr, preferredGVR)
+	backedUpItem, err := itemBackupper.backupItem(log, unstructured, gr, preferredGVR, false)
 	if aggregate, ok := err.(kubeerrs.Aggregate); ok {
 		log.WithField("name", unstructured.GetName()).Infof("%d errors encountered backup up item", len(aggregate.Errors()))
 		// log each error separately so we get error location info in the log, and an

pkg/backup/item_backupper.go

@@ -24,6 +24,8 @@ import (
 	"strings"
 	"time"
 
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	corev1api "k8s.io/api/core/v1"
@@ -47,6 +49,11 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/volume"
 )
 
+const (
+	mustIncludeAdditionalItemAnnotation = "backup.velero.io/must-include-additional-items"
+	excludeFromBackupLabel              = "velero.io/exclude-from-backup"
+)
+
 // itemBackupper can back up individual items to a tar writer.
 type itemBackupper struct {
 	backupRequest           *Request
@@ -61,16 +68,11 @@ type itemBackupper struct {
 	snapshotLocationVolumeSnapshotters map[string]vsv1.VolumeSnapshotter
 }
 
-const (
-	// veleroExcludeFromBackupLabel labeled item should be exclude by velero in backup job.
-	veleroExcludeFromBackupLabel = "velero.io/exclude-from-backup"
-)
-
 // backupItem backs up an individual item to tarWriter. The item may be excluded based on the
 // namespaces IncludesExcludes list.
 // In addition to the error return, backupItem also returns a bool indicating whether the item
 // was actually backed up.
-func (ib *itemBackupper) backupItem(logger logrus.FieldLogger, obj runtime.Unstructured, groupResource schema.GroupResource, preferredGVR schema.GroupVersionResource) (bool, error) {
+func (ib *itemBackupper) backupItem(logger logrus.FieldLogger, obj runtime.Unstructured, groupResource schema.GroupResource, preferredGVR schema.GroupVersionResource, mustInclude bool) (bool, error) {
 	metadata, err := meta.Accessor(obj)
 	if err != nil {
 		return false, err
@@ -83,28 +85,30 @@ func (ib *itemBackupper) backupItem(logger logrus.FieldLogger, obj runtime.Unstr
 	log = log.WithField("resource", groupResource.String())
 	log = log.WithField("namespace", namespace)
 
-	if metadata.GetLabels()[veleroExcludeFromBackupLabel] == "true" {
-		log.Infof("Excluding item because it has label %s=true", veleroExcludeFromBackupLabel)
-		return false, nil
-	}
-
-	// NOTE: we have to re-check namespace & resource includes/excludes because it's possible that
-	// backupItem can be invoked by a custom action.
-	if namespace != "" && !ib.backupRequest.NamespaceIncludesExcludes.ShouldInclude(namespace) {
-		log.Info("Excluding item because namespace is excluded")
-		return false, nil
-	}
-
-	// NOTE: we specifically allow namespaces to be backed up even if IncludeClusterResources is
-	// false.
-	if namespace == "" && groupResource != kuberesource.Namespaces && ib.backupRequest.Spec.IncludeClusterResources != nil && !*ib.backupRequest.Spec.IncludeClusterResources {
-		log.Info("Excluding item because resource is cluster-scoped and backup.spec.includeClusterResources is false")
-		return false, nil
-	}
+	if mustInclude {
+		log.Infof("Skipping the exclusion checks for this resource")
+	} else {
+		if metadata.GetLabels()[excludeFromBackupLabel] == "true" {
+			log.Infof("Excluding item because it has label %s=true", excludeFromBackupLabel)
+			return false, nil
+		}
+		// NOTE: we have to re-check namespace & resource includes/excludes because it's possible that
+		// backupItem can be invoked by a custom action.
+		if namespace != "" && !ib.backupRequest.NamespaceIncludesExcludes.ShouldInclude(namespace) {
+			log.Info("Excluding item because namespace is excluded")
+			return false, nil
+		}
+		// NOTE: we specifically allow namespaces to be backed up even if IncludeClusterResources is
+		// false.
+		if namespace == "" && groupResource != kuberesource.Namespaces && ib.backupRequest.Spec.IncludeClusterResources != nil && !*ib.backupRequest.Spec.IncludeClusterResources {
+			log.Info("Excluding item because resource is cluster-scoped and backup.spec.includeClusterResources is false")
+			return false, nil
+		}
 
-	if !ib.backupRequest.ResourceIncludesExcludes.ShouldInclude(groupResource.String()) {
-		log.Info("Excluding item because resource is excluded")
-		return false, nil
+		if !ib.backupRequest.ResourceIncludesExcludes.ShouldInclude(groupResource.String()) {
+			log.Info("Excluding item because resource is excluded")
+			return false, nil
+		}
 	}
 
 	if metadata.GetDeletionTimestamp() != nil {
@@ -320,7 +324,8 @@ func (ib *itemBackupper) executeActions(
 		if err != nil {
 			return nil, errors.Wrapf(err, "error executing custom action (groupResource=%s, namespace=%s, name=%s)", groupResource.String(), namespace, name)
 		}
-		obj = updatedItem
+		u := &unstructured.Unstructured{Object: updatedItem.UnstructuredContent()}
+		mustInclude := u.GetAnnotations()[mustIncludeAdditionalItemAnnotation] == "true"
 
 		for _, additionalItem := range additionalItemIdentifiers {
 			gvr, resource, err := ib.discoveryHelper.ResourceFor(additionalItem.GroupResource.WithVersion(""))
@@ -334,6 +339,7 @@ func (ib *itemBackupper) executeActions(
 			}
 
 			item, err := client.Get(additionalItem.Name, metav1.GetOptions{})
+
 			if apierrors.IsNotFound(err) {
 				log.WithFields(logrus.Fields{
 					"groupResource": additionalItem.GroupResource,
@@ -346,12 +352,17 @@ func (ib *itemBackupper) executeActions(
 				return nil, errors.WithStack(err)
 			}
 
-			if _, err = ib.backupItem(log, item, gvr.GroupResource(), gvr); err != nil {
+			if _, err = ib.backupItem(log, item, gvr.GroupResource(), gvr, mustInclude); err != nil {
 				return nil, err
 			}
 		}
+		// remove the annotation as it's for communication between BIA and velero server,
+		// we don't want the resource be restored with this annotation.
+		if _, ok := u.GetAnnotations()[mustIncludeAdditionalItemAnnotation]; ok {
+			delete(u.GetAnnotations(), mustIncludeAdditionalItemAnnotation)
+		}
+		obj = u
 	}
-
 	return obj, nil
 }
 

pkg/cmd/cli/debug/debug.go

@@ -152,7 +152,7 @@ func NewCommand(f client.Factory) *cobra.Command {
 	c := &cobra.Command{
 		Use:   "debug",
 		Short: "Generate debug bundle",
-		Long: `Generate a tarball containing the logs of velero deployment, plugin logs, restic DaemonSet, 
+		Long: `Generate a tarball containing the logs of velero deployment, plugin logs, node-agent DaemonSet, 
 specs of resources created by velero server, and optionally the logs of backup and restore.`,
 		Run: func(c *cobra.Command, args []string) {
 			flags := c.Flags()

pkg/cmd/cli/install/install.go

@@ -55,17 +55,17 @@ type InstallOptions struct {
 	VeleroPodMemRequest             string
 	VeleroPodCPULimit               string
 	VeleroPodMemLimit               string
-	ResticPodCPURequest             string
-	ResticPodMemRequest             string
-	ResticPodCPULimit               string
-	ResticPodMemLimit               string
+	NodeAgentPodCPURequest          string
+	NodeAgentPodMemRequest          string
+	NodeAgentPodCPULimit            string
+	NodeAgentPodMemLimit            string
 	RestoreOnly                     bool
 	SecretFile                      string
 	NoSecret                        bool
 	DryRun                          bool
 	BackupStorageConfig             flag.Map
 	VolumeSnapshotConfig            flag.Map
-	UseRestic                       bool
+	UseNodeAgent                    bool
 	Wait                            bool
 	UseVolumeSnapshots              bool
 	DefaultRepoMaintenanceFrequency time.Duration
@@ -95,23 +95,23 @@ func (o *InstallOptions) BindFlags(flags *pflag.FlagSet) {
 	flags.StringVar(&o.VeleroPodMemRequest, "velero-pod-mem-request", o.VeleroPodMemRequest, `Memory request for Velero pod. A value of "0" is treated as unbounded. Optional.`)
 	flags.StringVar(&o.VeleroPodCPULimit, "velero-pod-cpu-limit", o.VeleroPodCPULimit, `CPU limit for Velero pod. A value of "0" is treated as unbounded. Optional.`)
 	flags.StringVar(&o.VeleroPodMemLimit, "velero-pod-mem-limit", o.VeleroPodMemLimit, `Memory limit for Velero pod. A value of "0" is treated as unbounded. Optional.`)
-	flags.StringVar(&o.ResticPodCPURequest, "restic-pod-cpu-request", o.ResticPodCPURequest, `CPU request for restic pod. A value of "0" is treated as unbounded. Optional.`)
-	flags.StringVar(&o.ResticPodMemRequest, "restic-pod-mem-request", o.ResticPodMemRequest, `Memory request for restic pod. A value of "0" is treated as unbounded. Optional.`)
-	flags.StringVar(&o.ResticPodCPULimit, "restic-pod-cpu-limit", o.ResticPodCPULimit, `CPU limit for restic pod. A value of "0" is treated as unbounded. Optional.`)
-	flags.StringVar(&o.ResticPodMemLimit, "restic-pod-mem-limit", o.ResticPodMemLimit, `Memory limit for restic pod. A value of "0" is treated as unbounded. Optional.`)
+	flags.StringVar(&o.NodeAgentPodCPURequest, "node-agent-pod-cpu-request", o.NodeAgentPodCPURequest, `CPU request for node-agent pod. A value of "0" is treated as unbounded. Optional.`)
+	flags.StringVar(&o.NodeAgentPodMemRequest, "node-agent-pod-mem-request", o.NodeAgentPodMemRequest, `Memory request for node-agent pod. A value of "0" is treated as unbounded. Optional.`)
+	flags.StringVar(&o.NodeAgentPodCPULimit, "node-agent-pod-cpu-limit", o.NodeAgentPodCPULimit, `CPU limit for node-agent pod. A value of "0" is treated as unbounded. Optional.`)
+	flags.StringVar(&o.NodeAgentPodMemLimit, "node-agent-pod-mem-limit", o.NodeAgentPodMemLimit, `Memory limit for node-agent pod. A value of "0" is treated as unbounded. Optional.`)
 	flags.Var(&o.BackupStorageConfig, "backup-location-config", "Configuration to use for the backup storage location. Format is key1=value1,key2=value2")
 	flags.Var(&o.VolumeSnapshotConfig, "snapshot-location-config", "Configuration to use for the volume snapshot location. Format is key1=value1,key2=value2")
 	flags.BoolVar(&o.UseVolumeSnapshots, "use-volume-snapshots", o.UseVolumeSnapshots, "Whether or not to create snapshot location automatically. Set to false if you do not plan to create volume snapshots via a storage provider.")
 	flags.BoolVar(&o.RestoreOnly, "restore-only", o.RestoreOnly, "Run the server in restore-only mode. Optional.")
 	flags.BoolVar(&o.DryRun, "dry-run", o.DryRun, "Generate resources, but don't send them to the cluster. Use with -o. Optional.")
-	flags.BoolVar(&o.UseRestic, "use-restic", o.UseRestic, "Create restic daemonset. Optional.")
+	flags.BoolVar(&o.UseNodeAgent, "use-node-agent", o.UseNodeAgent, "Create Velero node-agent daemonset. Optional. Velero node-agent hosts Velero modules that need to run in one or more nodes(i.e. Restic, Kopia).")
 	flags.BoolVar(&o.Wait, "wait", o.Wait, "Wait for Velero deployment to be ready. Optional.")
 	flags.DurationVar(&o.DefaultRepoMaintenanceFrequency, "default-repo-maintain-frequency", o.DefaultRepoMaintenanceFrequency, "How often 'maintain' is run for backup repositories by default. Optional.")
 	flags.DurationVar(&o.GarbageCollectionFrequency, "garbage-collection-frequency", o.GarbageCollectionFrequency, "How often the garbage collection runs for expired backups.(default 1h)")
 	flags.Var(&o.Plugins, "plugins", "Plugin container images to install into the Velero Deployment")
 	flags.BoolVar(&o.CRDsOnly, "crds-only", o.CRDsOnly, "Only generate CustomResourceDefinition resources. Useful for updating CRDs for an existing Velero install.")
 	flags.StringVar(&o.CACertFile, "cacert", o.CACertFile, "File containing a certificate bundle to use when verifying TLS connections to the object store. Optional.")
-	flags.StringVar(&o.Features, "features", o.Features, "Comma separated list of Velero feature flags to be set on the Velero deployment and the restic daemonset, if restic is enabled")
+	flags.StringVar(&o.Features, "features", o.Features, "Comma separated list of Velero feature flags to be set on the Velero deployment and the node-agent daemonset, if node-agent is enabled")
 	flags.BoolVar(&o.DefaultVolumesToFsBackup, "default-volumes-to-fs-backup", o.DefaultVolumesToFsBackup, "Bool flag to configure Velero server to use pod volume file system backup by default for all volumes on all backups. Optional.")
 	flags.StringVar(&o.UploaderType, "uploader-type", o.UploaderType, fmt.Sprintf("The type of uploader to transfer the data of pod volumes, the supported values are '%s', '%s'", uploader.ResticType, uploader.KopiaType))
 }
@@ -130,10 +130,10 @@ func NewInstallOptions() *InstallOptions {
 		VeleroPodMemRequest:       install.DefaultVeleroPodMemRequest,
 		VeleroPodCPULimit:         install.DefaultVeleroPodCPULimit,
 		VeleroPodMemLimit:         install.DefaultVeleroPodMemLimit,
-		ResticPodCPURequest:       install.DefaultResticPodCPURequest,
-		ResticPodMemRequest:       install.DefaultResticPodMemRequest,
-		ResticPodCPULimit:         install.DefaultResticPodCPULimit,
-		ResticPodMemLimit:         install.DefaultResticPodMemLimit,
+		NodeAgentPodCPURequest:    install.DefaultNodeAgentPodCPURequest,
+		NodeAgentPodMemRequest:    install.DefaultNodeAgentPodMemRequest,
+		NodeAgentPodCPULimit:      install.DefaultNodeAgentPodCPULimit,
+		NodeAgentPodMemLimit:      install.DefaultNodeAgentPodMemLimit,
 		// Default to creating a VSL unless we're told otherwise
 		UseVolumeSnapshots:       true,
 		NoDefaultBackupLocation:  false,
@@ -171,7 +171,7 @@ func (o *InstallOptions) AsVeleroOptions() (*install.VeleroOptions, error) {
 	if err != nil {
 		return nil, err
 	}
-	resticPodResources, err := kubeutil.ParseResourceRequirements(o.ResticPodCPURequest, o.ResticPodMemRequest, o.ResticPodCPULimit, o.ResticPodMemLimit)
+	nodeAgentPodResources, err := kubeutil.ParseResourceRequirements(o.NodeAgentPodCPURequest, o.NodeAgentPodMemRequest, o.NodeAgentPodCPULimit, o.NodeAgentPodMemLimit)
 	if err != nil {
 		return nil, err
 	}
@@ -186,10 +186,10 @@ func (o *InstallOptions) AsVeleroOptions() (*install.VeleroOptions, error) {
 		PodLabels:                       o.PodLabels.Data(),
 		ServiceAccountAnnotations:       o.ServiceAccountAnnotations.Data(),
 		VeleroPodResources:              veleroPodResources,
-		ResticPodResources:              resticPodResources,
+		NodeAgentPodResources:           nodeAgentPodResources,
 		SecretData:                      secretData,
 		RestoreOnly:                     o.RestoreOnly,
-		UseRestic:                       o.UseRestic,
+		UseNodeAgent:                    o.UseNodeAgent,
 		UseVolumeSnapshots:              o.UseVolumeSnapshots,
 		BSLConfig:                       o.BackupStorageConfig.Data(),
 		VSLConfig:                       o.VolumeSnapshotConfig.Data(),
@@ -216,7 +216,7 @@ A prefix within the bucket and configuration for the backup store location may a
 Additionally, volume snapshot information for the same provider may be supplied.
 
 All required CustomResourceDefinitions will be installed to the server, as well as the
-Velero Deployment and associated Restic DaemonSet.
+Velero Deployment and associated node-agent DaemonSet.
 
 The provided secret data will be created in a Secret named 'cloud-credentials'.
 
@@ -302,8 +302,8 @@ func (o *InstallOptions) Run(c *cobra.Command, f client.Factory) error {
 			return errors.Wrap(err, errorMsg)
 		}
 
-		if o.UseRestic {
-			fmt.Println("Waiting for Velero restic daemonset to be ready.")
+		if o.UseNodeAgent {
+			fmt.Println("Waiting for node-agent daemonset to be ready.")
 			if _, err = install.DaemonSetIsReady(dynamicFactory, o.Namespace); err != nil {
 				return errors.Wrap(err, errorMsg)
 			}
@@ -393,8 +393,8 @@ func (o *InstallOptions) Validate(c *cobra.Command, args []string, f client.Fact
 		}
 	}
 
-	if o.DefaultVolumesToFsBackup && !o.UseRestic {
-		return errors.New("--use-restic is required when using --default-volumes-to-fs-backup")
+	if o.DefaultVolumesToFsBackup && !o.UseNodeAgent {
+		return errors.New("--use-node-agent is required when using --default-volumes-to-fs-backup")
 	}
 
 	switch {

pkg/cmd/cli/nodeagent/node_agent.go

@@ -0,0 +1,37 @@
+/*
+Copyright The Velero Contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nodeagent
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/vmware-tanzu/velero/pkg/client"
+)
+
+func NewCommand(f client.Factory) *cobra.Command {
+	c := &cobra.Command{
+		Use:   "node-agent",
+		Short: "Work with node-agent",
+		Long:  "Work with node-agent",
+	}
+
+	c.AddCommand(
+		NewServerCommand(f),
+	)
+
+	return c
+}