From bc6414672e7e39ba699f10fd18cf48fd81404bed Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Thu, 28 Nov 2024 16:02:03 +0800 Subject: [PATCH 1/3] disable block volume data mover on windows Signed-off-by: Lyndon-Li --- pkg/controller/data_upload_controller.go | 12 ++-- pkg/util/kube/node.go | 18 ++++++ pkg/util/kube/node_test.go | 78 ++++++++++++++++++++++++ pkg/util/kube/pvc_pv.go | 7 ++- pkg/util/kube/pvc_pv_test.go | 9 ++- 5 files changed, 118 insertions(+), 6 deletions(-) diff --git a/pkg/controller/data_upload_controller.go b/pkg/controller/data_upload_controller.go index 6b3464983a..5d1073b2f7 100644 --- a/pkg/controller/data_upload_controller.go +++ b/pkg/controller/data_upload_controller.go @@ -803,14 +803,18 @@ func (r *DataUploadReconciler) setupExposeParam(du *velerov2alpha1api.DataUpload return nil, errors.Wrapf(err, "failed to get PVC %s/%s", du.Spec.SourceNamespace, du.Spec.SourcePVC) } - nodeOS, err := kube.GetPVCAttachingNodeOS(pvc, r.kubeClient.CoreV1(), r.kubeClient.StorageV1(), r.logger) + accessMode := exposer.AccessModeFileSystem + if pvc.Spec.VolumeMode != nil && *pvc.Spec.VolumeMode == corev1.PersistentVolumeBlock { + accessMode = exposer.AccessModeBlock + } + + nodeOS, err := kube.GetPVCAttachingNodeOS(pvc, (accessMode == exposer.AccessModeBlock), r.kubeClient.CoreV1(), r.kubeClient.StorageV1(), r.logger) if err != nil { return nil, errors.Wrapf(err, "failed to get attaching node OS for PVC %s/%s", du.Spec.SourceNamespace, du.Spec.SourcePVC) } - accessMode := exposer.AccessModeFileSystem - if pvc.Spec.VolumeMode != nil && *pvc.Spec.VolumeMode == corev1.PersistentVolumeBlock { - accessMode = exposer.AccessModeBlock + if err := kube.HasNodeWithOS(context.Background(), nodeOS, r.kubeClient.CoreV1()); err != nil { + return nil, errors.Wrapf(err, "no appropriate node to run data upload for PVC %s/%s", du.Spec.SourceNamespace, du.Spec.SourcePVC) } hostingPodLabels := map[string]string{velerov1api.DataUploadLabel: du.Name} diff --git a/pkg/util/kube/node.go b/pkg/util/kube/node.go index 62ecd4a2fe..da68183a55 100644 --- a/pkg/util/kube/node.go +++ b/pkg/util/kube/node.go @@ -17,6 +17,7 @@ package kube import ( "context" + "fmt" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -99,3 +100,20 @@ func GetNodeOS(ctx context.Context, nodeName string, nodeClient corev1client.Cor return node.Labels[NodeOSLabel], nil } + +func HasNodeWithOS(ctx context.Context, os string, nodeClient corev1client.CoreV1Interface) error { + if os == "" { + return errors.New("invalid node OS") + } + + nodes, err := nodeClient.Nodes().List(ctx, metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", NodeOSLabel, os)}) + if err != nil { + return errors.Wrapf(err, "error listing nodes with OS %s", os) + } + + if len(nodes.Items) == 0 { + return errors.Errorf("node with OS %s doesn't exist", os) + } + + return nil +} diff --git a/pkg/util/kube/node_test.go b/pkg/util/kube/node_test.go index e24aa6284f..a26285f5ff 100644 --- a/pkg/util/kube/node_test.go +++ b/pkg/util/kube/node_test.go @@ -20,6 +20,7 @@ import ( "context" "testing" + "github.com/pkg/errors" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -27,6 +28,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/builder" kubeClientFake "k8s.io/client-go/kubernetes/fake" + clientTesting "k8s.io/client-go/testing" clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake" velerotest "github.com/vmware-tanzu/velero/pkg/test" @@ -181,3 +183,79 @@ func TestGetNodeOSType(t *testing.T) { }) } } + +func TestHasNodeWithOS(t *testing.T) { + nodeNoOSLabel := builder.ForNode("fake-node-1").Result() + nodeWindows := builder.ForNode("fake-node-2").Labels(map[string]string{"kubernetes.io/os": "windows"}).Result() + nodeLinux := builder.ForNode("fake-node-3").Labels(map[string]string{"kubernetes.io/os": "linux"}).Result() + + scheme := runtime.NewScheme() + corev1.AddToScheme(scheme) + + tests := []struct { + name string + kubeClientObj []runtime.Object + kubeReactors []reactor + os string + err string + }{ + { + name: "os is empty", + err: "invalid node OS", + }, + { + name: "error to list node", + kubeReactors: []reactor{ + { + verb: "list", + resource: "nodes", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, errors.New("fake-list-error") + }, + }, + }, + os: "linux", + err: "error listing nodes with OS linux: fake-list-error", + }, + { + name: "no expected node - no node", + os: "linux", + err: "node with OS linux doesn't exist", + }, + { + name: "no expected node - no node with label", + kubeClientObj: []runtime.Object{ + nodeNoOSLabel, + nodeWindows, + }, + os: "linux", + err: "node with OS linux doesn't exist", + }, + { + name: "succeed", + kubeClientObj: []runtime.Object{ + nodeNoOSLabel, + nodeWindows, + nodeLinux, + }, + os: "windows", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fakeKubeClient := kubeClientFake.NewSimpleClientset(test.kubeClientObj...) + + for _, reactor := range test.kubeReactors { + fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) + } + + err := HasNodeWithOS(context.TODO(), test.os, fakeKubeClient.CoreV1()) + if test.err != "" { + assert.EqualError(t, err, test.err) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/pkg/util/kube/pvc_pv.go b/pkg/util/kube/pvc_pv.go index bfa7cb32a0..eab07f25b5 100644 --- a/pkg/util/kube/pvc_pv.go +++ b/pkg/util/kube/pvc_pv.go @@ -429,11 +429,16 @@ func DiagnosePV(pv *corev1api.PersistentVolume) string { return diag } -func GetPVCAttachingNodeOS(pvc *corev1api.PersistentVolumeClaim, nodeClient corev1client.CoreV1Interface, +func GetPVCAttachingNodeOS(pvc *corev1api.PersistentVolumeClaim, blockAccess bool, nodeClient corev1client.CoreV1Interface, storageClient storagev1.StorageV1Interface, log logrus.FieldLogger) (string, error) { var nodeOS string var scFsType string + if blockAccess { + log.Infof("Use linux node for block access for PVC %s/%s", pvc.Namespace, pvc.Name) + return NodeOSLinux, nil + } + if value := pvc.Annotations[KubeAnnSelectedNode]; value != "" { os, err := GetNodeOS(context.Background(), value, nodeClient) if err != nil { diff --git a/pkg/util/kube/pvc_pv_test.go b/pkg/util/kube/pvc_pv_test.go index 5d00917897..ea3195966b 100644 --- a/pkg/util/kube/pvc_pv_test.go +++ b/pkg/util/kube/pvc_pv_test.go @@ -1610,6 +1610,7 @@ func TestGetPVCAttachingNodeOS(t *testing.T) { name string pvc *corev1api.PersistentVolumeClaim kubeClientObj []runtime.Object + blockAccess bool expectedNodeOS string err string }{ @@ -1662,6 +1663,12 @@ func TestGetPVCAttachingNodeOS(t *testing.T) { }, expectedNodeOS: NodeOSWindows, }, + { + name: "block access", + pvc: pvcObjWithBoth, + blockAccess: true, + expectedNodeOS: NodeOSLinux, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -1669,7 +1676,7 @@ func TestGetPVCAttachingNodeOS(t *testing.T) { var kubeClient kubernetes.Interface = fakeKubeClient - nodeOS, err := GetPVCAttachingNodeOS(test.pvc, kubeClient.CoreV1(), kubeClient.StorageV1(), velerotest.NewLogger()) + nodeOS, err := GetPVCAttachingNodeOS(test.pvc, test.blockAccess, kubeClient.CoreV1(), kubeClient.StorageV1(), velerotest.NewLogger()) if err != nil { assert.EqualError(t, err, test.err) From d2a25cd446c2aa2efd2023eb8da6859b07d03ca1 Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Thu, 28 Nov 2024 17:10:58 +0800 Subject: [PATCH 2/3] fs uploader skip system folders on windows Signed-off-by: Lyndon-Li --- pkg/uploader/kopia/progress.go | 4 +++- pkg/uploader/kopia/snapshot.go | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/uploader/kopia/progress.go b/pkg/uploader/kopia/progress.go index 7f0619f574..9f2498379c 100644 --- a/pkg/uploader/kopia/progress.go +++ b/pkg/uploader/kopia/progress.go @@ -138,7 +138,9 @@ func (p *Progress) HashingFile(fname string) {} func (p *Progress) ExcludedFile(fname string, numBytes int64) {} // ExcludedDir statistic the dir been excluded currently -func (p *Progress) ExcludedDir(dirname string) {} +func (p *Progress) ExcludedDir(dirname string) { + p.log.Infof("Excluded dir %s", dirname) +} // FinishedHashingFile which will called when specific file finished hash func (p *Progress) FinishedHashingFile(fname string, numBytes int64) { diff --git a/pkg/uploader/kopia/snapshot.go b/pkg/uploader/kopia/snapshot.go index c80ab155f8..fce620eb73 100644 --- a/pkg/uploader/kopia/snapshot.go +++ b/pkg/uploader/kopia/snapshot.go @@ -127,6 +127,10 @@ func setupPolicy(ctx context.Context, rep repo.RepositoryWriter, sourceInfo snap curPolicy.UploadPolicy.ParallelUploadAboveSize = newOptionalInt64(2 << 30) } + if runtime.GOOS == "windows" { + curPolicy.FilesPolicy.IgnoreRules = []string{"/System Volume Information/", "/$Recycle.Bin/"} + } + err := setPolicyFunc(ctx, rep, sourceInfo, curPolicy) if err != nil { return nil, errors.Wrap(err, "error to set policy") From cb22dfc4826146a30b5cceb526b463c65b700302 Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Thu, 2 Jan 2025 11:35:32 +0800 Subject: [PATCH 3/3] fs uploader and block uploader support Windows nodes Signed-off-by: Lyndon-Li --- changelogs/unreleased/8569-Lyndon-Li | 1 + pkg/controller/data_upload_controller.go | 12 ++++++------ pkg/controller/data_upload_controller_test.go | 5 ++++- pkg/util/kube/pvc_pv.go | 6 +++--- pkg/util/kube/pvc_pv_test.go | 17 +++++++++++++---- 5 files changed, 27 insertions(+), 14 deletions(-) create mode 100644 changelogs/unreleased/8569-Lyndon-Li diff --git a/changelogs/unreleased/8569-Lyndon-Li b/changelogs/unreleased/8569-Lyndon-Li new file mode 100644 index 0000000000..336a9856c9 --- /dev/null +++ b/changelogs/unreleased/8569-Lyndon-Li @@ -0,0 +1 @@ +fs uploader and block uploader support Windows nodes \ No newline at end of file diff --git a/pkg/controller/data_upload_controller.go b/pkg/controller/data_upload_controller.go index 5d1073b2f7..66f5b67f76 100644 --- a/pkg/controller/data_upload_controller.go +++ b/pkg/controller/data_upload_controller.go @@ -803,12 +803,7 @@ func (r *DataUploadReconciler) setupExposeParam(du *velerov2alpha1api.DataUpload return nil, errors.Wrapf(err, "failed to get PVC %s/%s", du.Spec.SourceNamespace, du.Spec.SourcePVC) } - accessMode := exposer.AccessModeFileSystem - if pvc.Spec.VolumeMode != nil && *pvc.Spec.VolumeMode == corev1.PersistentVolumeBlock { - accessMode = exposer.AccessModeBlock - } - - nodeOS, err := kube.GetPVCAttachingNodeOS(pvc, (accessMode == exposer.AccessModeBlock), r.kubeClient.CoreV1(), r.kubeClient.StorageV1(), r.logger) + nodeOS, err := kube.GetPVCAttachingNodeOS(pvc, r.kubeClient.CoreV1(), r.kubeClient.StorageV1(), r.logger) if err != nil { return nil, errors.Wrapf(err, "failed to get attaching node OS for PVC %s/%s", du.Spec.SourceNamespace, du.Spec.SourcePVC) } @@ -817,6 +812,11 @@ func (r *DataUploadReconciler) setupExposeParam(du *velerov2alpha1api.DataUpload return nil, errors.Wrapf(err, "no appropriate node to run data upload for PVC %s/%s", du.Spec.SourceNamespace, du.Spec.SourcePVC) } + accessMode := exposer.AccessModeFileSystem + if pvc.Spec.VolumeMode != nil && *pvc.Spec.VolumeMode == corev1.PersistentVolumeBlock { + accessMode = exposer.AccessModeBlock + } + hostingPodLabels := map[string]string{velerov1api.DataUploadLabel: du.Name} for _, k := range util.ThirdPartyLabels { if v, err := nodeagent.GetLabelValue(context.Background(), r.kubeClient, du.Namespace, k, nodeOS); err != nil { diff --git a/pkg/controller/data_upload_controller_test.go b/pkg/controller/data_upload_controller_test.go index 8e3b1688b6..f480a692c5 100644 --- a/pkg/controller/data_upload_controller_test.go +++ b/pkg/controller/data_upload_controller_test.go @@ -59,6 +59,7 @@ import ( velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/boolptr" + "github.com/vmware-tanzu/velero/pkg/util/kube" ) const dataUploadName = "dataupload-1" @@ -187,6 +188,8 @@ func initDataUploaderReconcilerWithError(needError ...error) (*DataUploadReconci }, } + node := builder.ForNode("fake-node").Labels(map[string]string{kube.NodeOSLabel: kube.NodeOSLinux}).Result() + dataPathMgr := datapath.NewManager(1) now, err := time.Parse(time.RFC1123, time.RFC1123) @@ -229,7 +232,7 @@ func initDataUploaderReconcilerWithError(needError ...error) (*DataUploadReconci } fakeSnapshotClient := snapshotFake.NewSimpleClientset(vsObject, vscObj) - fakeKubeClient := clientgofake.NewSimpleClientset(daemonSet) + fakeKubeClient := clientgofake.NewSimpleClientset(daemonSet, node) return NewDataUploadReconciler( fakeClient, diff --git a/pkg/util/kube/pvc_pv.go b/pkg/util/kube/pvc_pv.go index eab07f25b5..e91e5dab3b 100644 --- a/pkg/util/kube/pvc_pv.go +++ b/pkg/util/kube/pvc_pv.go @@ -429,13 +429,13 @@ func DiagnosePV(pv *corev1api.PersistentVolume) string { return diag } -func GetPVCAttachingNodeOS(pvc *corev1api.PersistentVolumeClaim, blockAccess bool, nodeClient corev1client.CoreV1Interface, +func GetPVCAttachingNodeOS(pvc *corev1api.PersistentVolumeClaim, nodeClient corev1client.CoreV1Interface, storageClient storagev1.StorageV1Interface, log logrus.FieldLogger) (string, error) { var nodeOS string var scFsType string - if blockAccess { - log.Infof("Use linux node for block access for PVC %s/%s", pvc.Namespace, pvc.Name) + if pvc.Spec.VolumeMode != nil && *pvc.Spec.VolumeMode == corev1api.PersistentVolumeBlock { + log.Infof("Use linux node for block mode PVC %s/%s", pvc.Namespace, pvc.Name) return NodeOSLinux, nil } diff --git a/pkg/util/kube/pvc_pv_test.go b/pkg/util/kube/pvc_pv_test.go index ea3195966b..2a5c2d8266 100644 --- a/pkg/util/kube/pvc_pv_test.go +++ b/pkg/util/kube/pvc_pv_test.go @@ -1564,6 +1564,17 @@ func TestGetPVCAttachingNodeOS(t *testing.T) { }, } + blockMode := corev1api.PersistentVolumeBlock + pvcObjBlockMode := &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "fake-namespace", + Name: "fake-pvc", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + VolumeMode: &blockMode, + }, + } + pvcObjWithNode := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-namespace", @@ -1610,7 +1621,6 @@ func TestGetPVCAttachingNodeOS(t *testing.T) { name string pvc *corev1api.PersistentVolumeClaim kubeClientObj []runtime.Object - blockAccess bool expectedNodeOS string err string }{ @@ -1665,8 +1675,7 @@ func TestGetPVCAttachingNodeOS(t *testing.T) { }, { name: "block access", - pvc: pvcObjWithBoth, - blockAccess: true, + pvc: pvcObjBlockMode, expectedNodeOS: NodeOSLinux, }, } @@ -1676,7 +1685,7 @@ func TestGetPVCAttachingNodeOS(t *testing.T) { var kubeClient kubernetes.Interface = fakeKubeClient - nodeOS, err := GetPVCAttachingNodeOS(test.pvc, test.blockAccess, kubeClient.CoreV1(), kubeClient.StorageV1(), velerotest.NewLogger()) + nodeOS, err := GetPVCAttachingNodeOS(test.pvc, kubeClient.CoreV1(), kubeClient.StorageV1(), velerotest.NewLogger()) if err != nil { assert.EqualError(t, err, test.err)