feat: support aznfs mount helper

Makefile

@@ -111,7 +111,7 @@ blob: blobfuse-proxy
 blob-windows:
 	CGO_ENABLED=0 GOOS=windows go build -a -ldflags ${LDFLAGS} -mod vendor -o _output/blobplugin.exe ./pkg/blobplugin
 
-.PHONT: blob-darwin
+.PHONY: blob-darwin
 blob-darwin:
 	CGO_ENABLED=0 GOOS=darwin go build -a -ldflags ${LDFLAGS} -mod vendor -o _output/blobplugin ./pkg/blobplugin
 
@@ -182,4 +182,4 @@ delete-metrics-svc:
 
 .PHONY: blobfuse-proxy
 blobfuse-proxy:
-	CGO_ENABLED=0 GOOS=linux go build -mod vendor -ldflags="-s -w" -o _output/${ARCH}/blobfuse-proxy ./pkg/blobfuse-proxy
+	CGO_ENABLED=0 GOOS=linux GOARCH=$(ARCH) go build -mod vendor -ldflags="-s -w" -o _output/${ARCH}/blobfuse-proxy ./pkg/blobfuse-proxy

charts/README.md

@@ -148,6 +148,7 @@ The following table lists the configurable parameters of the latest Azure Blob S
 | `linux.distro`                                        | configure ssl certificates for different Linux distribution(available values: `debian`, `fedora`)             | `debian`
 | `workloadIdentity.clientID` | client ID of workload identity | ''
 | `workloadIdentity.tenantID` | [optional] If the AAD application or user-assigned managed identity is not in the same tenant as the cluster then set tenantID with the AAD application or user-assigned managed identity tenant ID | ''
+| `node.enableAZNFS` | enable [AZNFS mount helper](https://github.com/Azure/AZNFS-mount/) for NFS protocol | true
 
 ## troubleshooting
  - Add `--wait -v=5 --debug` in `helm install` command to get detailed error

charts/latest/blob-csi-driver/templates/csi-blob-node.yaml

@@ -35,9 +35,9 @@ spec:
       imagePullSecrets:
 {{ toYaml .Values.imagePullSecrets | indent 8 }}
       {{- end }}
-{{- if .Values.node.enableBlobfuseProxy }}
+      {{- if or .Values.node.enableBlobfuseProxy .Values.node.enableAZNFS }}
       hostPID: true
-{{- end }}
+      {{- end }}
       hostNetwork: true
       dnsPolicy: Default
       serviceAccountName: {{ .Values.serviceAccount.node }}
@@ -170,6 +170,7 @@ spec:
             - "--append-timestamp-cache-dir={{ .Values.node.appendTimeStampInCacheDir }}"
             - "--mount-permissions={{ .Values.node.mountPermissions }}"
             - "--allow-inline-volume-key-access-with-idenitity={{ .Values.node.allowInlineVolumeKeyAccessWithIdentity }}"
+            - "--enable-aznfs={{ .Values.node.enableAZNFS }}"
           ports:
             - containerPort: {{ .Values.node.livenessProbe.healthPort }}
               name: healthz
@@ -238,7 +239,31 @@ spec:
               mountPath: /etc/pki/ca-trust/extracted
               readOnly: true
             {{- end }}
+            {{- if .Values.node.enableAZNFS }}
+            - mountPath: /opt/microsoft/aznfs/data
+              name: aznfs-data
+            {{- end }}
           resources: {{- toYaml .Values.node.resources.blob | nindent 12 }}
+{{- if .Values.node.enableAZNFS }}
+        - name: aznfswatchdog
+{{- if hasPrefix "/" .Values.image.blob.repository }}
+          image: "{{ .Values.image.baseRepo }}{{ .Values.image.blob.repository }}:{{ .Values.image.blob.tag }}"
+{{- else }}
+          image: "{{ .Values.image.blob.repository }}:{{ .Values.image.blob.tag }}"
+{{- end }}
+          command:
+            - "aznfswatchdog"
+          imagePullPolicy: {{ .Values.image.blob.pullPolicy }}
+          securityContext:
+            privileged: true
+          resources: {{- toYaml .Values.node.resources.aznfswatchdog | nindent 12 }}
+          volumeMounts:
+            - mountPath: /opt/microsoft/aznfs/data
+              name: aznfs-data
+            - mountPath: {{ .Values.linux.kubelet }}/
+              mountPropagation: Bidirectional
+              name: mountpoint-dir
+{{- end }}
       volumes:
 {{- if .Values.node.enableBlobfuseProxy }}
         - name: host-usr
@@ -280,6 +305,12 @@ spec:
           hostPath:
             path: /etc/pki/ca-trust/extracted
         {{- end }}
+        {{- if .Values.node.enableAZNFS }}
+        - hostPath:
+            path: /opt/microsoft/aznfs/data
+            type: DirectoryOrCreate
+          name: aznfs-data
+        {{- end }}
       {{- if .Values.securityContext }}
       securityContext: {{- toYaml .Values.securityContext | nindent 8 }}
       {{- end }}

charts/latest/blob-csi-driver/values.yaml

@@ -145,10 +145,17 @@ node:
       requests:
         cpu: 10m
         memory: 20Mi
+    aznfswatchdog:
+      limits:
+        memory: 100Mi
+      requests:
+        cpu: 10m
+        memory: 20Mi
   affinity: {}
   nodeSelector: {}
   tolerations:
     - operator: "Exists"
+  enableAZNFS: true
 
 feature:
   fsGroupPolicy: ReadWriteOnceWithFSType

pkg/blob/blob.go

@@ -104,6 +104,7 @@ const (
 	Fuse                           = "fuse"
 	Fuse2                          = "fuse2"
 	NFS                            = "nfs"
+	AZNFS                          = "aznfs"
 	vnetResourceGroupField         = "vnetresourcegroup"
 	vnetNameField                  = "vnetname"
 	subnetNameField                = "subnetname"
@@ -168,6 +169,7 @@ type DriverOptions struct {
 	MountPermissions                       uint64
 	KubeAPIQPS                             float64
 	KubeAPIBurst                           int
+	EnableAZNFS                            bool
 }
 
 // Driver implements all interfaces of CSI drivers
@@ -192,6 +194,7 @@ type Driver struct {
 	mountPermissions                       uint64
 	kubeAPIQPS                             float64
 	kubeAPIBurst                           int
+	enableAZNFS                            bool
 	mounter                                *mount.SafeFormatAndMount
 	volLockMap                             *util.LockMap
 	// A map storing all volumes with ongoing operations so that additional operations
@@ -229,6 +232,7 @@ func NewDriver(options *DriverOptions) *Driver {
 		mountPermissions:                       options.MountPermissions,
 		kubeAPIQPS:                             options.KubeAPIQPS,
 		kubeAPIBurst:                           options.KubeAPIBurst,
+		enableAZNFS:                            options.EnableAZNFS,
 	}
 	d.Name = options.DriverName
 	d.Version = driverVersion

pkg/blob/nodeserver.go

@@ -314,10 +314,15 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe
 		klog.V(2).Infof("target %v\nprotocol %v\n\nvolumeId %v\ncontext %v\nmountflags %v\nserverAddress %v",
 			targetPath, protocol, volumeID, attrib, mountFlags, serverAddress)
 
+		mountType := AZNFS
+		if !d.enableAZNFS {
+			mountType = NFS
+		}
+
 		source := fmt.Sprintf("%s:/%s/%s", serverAddress, accountName, containerName)
 		mountOptions := util.JoinMountOptions(mountFlags, []string{"sec=sys,vers=3,nolock"})
 		if err := wait.PollImmediate(1*time.Second, 2*time.Minute, func() (bool, error) {
-			return true, d.mounter.MountSensitive(source, targetPath, NFS, mountOptions, []string{})
+			return true, d.mounter.MountSensitive(source, targetPath, mountType, mountOptions, []string{})
 		}); err != nil {
 			var helpLinkMsg string
 			if d.appendMountErrorHelpLink {

pkg/blobfuse-proxy/init.sh

@@ -26,99 +26,104 @@ HOST_CMD="nsenter --mount=/proc/1/ns/mnt"
 
 DISTRIBUTION=$($HOST_CMD cat /etc/os-release | grep ^ID= | cut -d'=' -f2 | tr -d '"')
 echo "Linux distribution: $DISTRIBUTION"
+ARCH=$($HOST_CMD uname -m)
+echo "Linux Arch is $(uname -m)"
 
-if [ "${DISTRIBUTION}" = "ubuntu" ] && { [ "${INSTALL_BLOBFUSE}" = "true" ] || [ "${INSTALL_BLOBFUSE2}" = "true" ]; }
+if [ "${ARCH}" != "aarch64" ]
 then
-  release=$($HOST_CMD lsb_release -rs)
-  echo "Ubuntu release: $release"
-
-  if [ "$(expr "$release" \< "22.04")" -eq 1 ]
+  if [ "${DISTRIBUTION}" = "ubuntu" ] && { [ "${INSTALL_BLOBFUSE}" = "true" ] || [ "${INSTALL_BLOBFUSE2}" = "true" ]; }
   then
-    cp /blobfuse-proxy/packages-microsoft-prod-18.04.deb /host/etc/packages-microsoft-prod.deb
-  else
-    cp /blobfuse-proxy/packages-microsoft-prod-22.04.deb /host/etc/packages-microsoft-prod.deb
-  fi
-
-  # when running dpkg -i /etc/packages-microsoft-prod.deb, need to enter y to continue. 
-  # refer to https://stackoverflow.com/questions/45349571/how-to-install-deb-with-dpkg-non-interactively
-  yes | $HOST_CMD dpkg -i /etc/packages-microsoft-prod.deb && $HOST_CMD apt update
-
-  pkg_list=""
-  if [ "${INSTALL_BLOBFUSE}" = "true" ] && [ "$(expr "$release" \< "22.04")" -eq 1 ]
-  then
-    pkg_list="${pkg_list} fuse"
-    # install blobfuse with latest version or specific version
-    if [ -z "${BLOBFUSE_VERSION}" ]; then
-      echo "install blobfuse with latest version"
-      pkg_list="${pkg_list} blobfuse"
+    release=$($HOST_CMD lsb_release -rs)
+    echo "Ubuntu release: $release"
+
+    if [ "$(expr "$release" \< "22.04")" -eq 1 ]
+    then
+      cp /blobfuse-proxy/packages-microsoft-prod-18.04.deb /host/etc/packages-microsoft-prod.deb
     else
-      pkg_list="${pkg_list} blobfuse=${BLOBFUSE_VERSION}"
+      cp /blobfuse-proxy/packages-microsoft-prod-22.04.deb /host/etc/packages-microsoft-prod.deb
     fi
-  fi
-
-  if [ "${INSTALL_BLOBFUSE2}" = "true" ]
-  then
-    if [ "$(expr "$release" \< "22.04")" -eq 1 ]; then
-      echo "install fuse for blobfuse2"
+
+    # when running dpkg -i /etc/packages-microsoft-prod.deb, need to enter y to continue. 
+    # refer to https://stackoverflow.com/questions/45349571/how-to-install-deb-with-dpkg-non-interactively
+    yes | $HOST_CMD dpkg -i /etc/packages-microsoft-prod.deb && $HOST_CMD apt update
+
+    pkg_list=""
+    if [ "${INSTALL_BLOBFUSE}" = "true" ] && [ "$(expr "$release" \< "22.04")" -eq 1 ]
+    then
       pkg_list="${pkg_list} fuse"
-    else
-      echo "install fuse3 for blobfuse2, current release is $release"
-      pkg_list="${pkg_list} fuse3"
+      # install blobfuse with latest version or specific version
+      if [ -z "${BLOBFUSE_VERSION}" ]; then
+        echo "install blobfuse with latest version"
+        pkg_list="${pkg_list} blobfuse"
+      else
+        pkg_list="${pkg_list} blobfuse=${BLOBFUSE_VERSION}"
+      fi
     fi
 
-    # install blobfuse2 with latest version or specific version
-    if [ -z "${BLOBFUSE2_VERSION}" ]; then
-      echo "install blobfuse2 with latest version"
-      pkg_list="${pkg_list} blobfuse2"
-    else
-      pkg_list="${pkg_list} blobfuse2=${BLOBFUSE2_VERSION}"
+    if [ "${INSTALL_BLOBFUSE2}" = "true" ]
+    then
+      if [ "$(expr "$release" \< "22.04")" -eq 1 ]; then
+        echo "install fuse for blobfuse2"
+        pkg_list="${pkg_list} fuse"
+      else
+        echo "install fuse3 for blobfuse2, current release is $release"
+        pkg_list="${pkg_list} fuse3"
+      fi
+
+      # install blobfuse2 with latest version or specific version
+      if [ -z "${BLOBFUSE2_VERSION}" ]; then
+        echo "install blobfuse2 with latest version"
+        pkg_list="${pkg_list} blobfuse2"
+      else
+        pkg_list="${pkg_list} blobfuse2=${BLOBFUSE2_VERSION}"
+      fi
     fi
+    echo "begin to install ${pkg_list}"
+    $HOST_CMD apt-get install -y $pkg_list
+    $HOST_CMD rm -f /etc/packages-microsoft-prod.deb
   fi
-  echo "begin to install ${pkg_list}"
-  $HOST_CMD apt-get install -y $pkg_list
-  $HOST_CMD rm -f /etc/packages-microsoft-prod.deb
-fi
 
-updateBlobfuseProxy="true"
-if [ -f "/host/usr/bin/blobfuse-proxy" ];then
-  old=$(sha256sum /host/usr/bin/blobfuse-proxy | awk '{print $1}')
-  new=$(sha256sum /blobfuse-proxy/blobfuse-proxy | awk '{print $1}')
-  if [ "$old" = "$new" ];then
-    updateBlobfuseProxy="false"
-    echo "no need to update blobfuse-proxy"
+  updateBlobfuseProxy="true"
+  if [ -f "/host/usr/bin/blobfuse-proxy" ];then
+    old=$(sha256sum /host/usr/bin/blobfuse-proxy | awk '{print $1}')
+    new=$(sha256sum /blobfuse-proxy/blobfuse-proxy | awk '{print $1}')
+    if [ "$old" = "$new" ];then
+      updateBlobfuseProxy="false"
+      echo "no need to update blobfuse-proxy"
+    fi
   fi
-fi
 
-if [ "$updateBlobfuseProxy" = "true" ];then
-  echo "copy blobfuse-proxy...."
-  rm -rf /host/var/lib/kubelet/plugins/blob.csi.azure.com/blobfuse-proxy.sock
-  rm -rf /host/usr/bin/blobfuse-proxy
-  cp /blobfuse-proxy/blobfuse-proxy /host/usr/bin/blobfuse-proxy
-  chmod 755 /host/usr/bin/blobfuse-proxy
-fi
+  if [ "$updateBlobfuseProxy" = "true" ];then
+    echo "copy blobfuse-proxy...."
+    rm -rf /host/var/lib/kubelet/plugins/blob.csi.azure.com/blobfuse-proxy.sock
+    rm -rf /host/usr/bin/blobfuse-proxy
+    cp /blobfuse-proxy/blobfuse-proxy /host/usr/bin/blobfuse-proxy
+    chmod 755 /host/usr/bin/blobfuse-proxy
+  fi
 
-updateService="true"
-if [ -f "/host/usr/lib/systemd/system/blobfuse-proxy.service" ];then
-  old=$(sha256sum /host/usr/lib/systemd/system/blobfuse-proxy.service | awk '{print $1}')
-  new=$(sha256sum /blobfuse-proxy/blobfuse-proxy.service | awk '{print $1}')
-  if [ "$old" = "$new" ];then
-    updateService="false"
-    echo "no need to update blobfuse-proxy.service"
+  updateService="true"
+  if [ -f "/host/usr/lib/systemd/system/blobfuse-proxy.service" ];then
+    old=$(sha256sum /host/usr/lib/systemd/system/blobfuse-proxy.service | awk '{print $1}')
+    new=$(sha256sum /blobfuse-proxy/blobfuse-proxy.service | awk '{print $1}')
+    if [ "$old" = "$new" ];then
+      updateService="false"
+      echo "no need to update blobfuse-proxy.service"
+    fi
   fi
-fi
 
-if [ "$updateService" = "true" ];then
-  echo "copy blobfuse-proxy.service...."
-  mkdir -p /host/usr/lib/systemd/system
-  cp /blobfuse-proxy/blobfuse-proxy.service /host/usr/lib/systemd/system/blobfuse-proxy.service
-fi
+  if [ "$updateService" = "true" ];then
+    echo "copy blobfuse-proxy.service...."
+    mkdir -p /host/usr/lib/systemd/system
+    cp /blobfuse-proxy/blobfuse-proxy.service /host/usr/lib/systemd/system/blobfuse-proxy.service
+  fi
 
-if [ "${INSTALL_BLOBFUSE_PROXY}" = "true" ];then
-  if [ "$updateBlobfuseProxy" = "true" ] || [ "$updateService" = "true" ];then
-    echo "start blobfuse-proxy...."
-    $HOST_CMD systemctl daemon-reload
-    $HOST_CMD systemctl enable blobfuse-proxy.service
-    $HOST_CMD systemctl restart blobfuse-proxy.service
+  if [ "${INSTALL_BLOBFUSE_PROXY}" = "true" ];then
+    if [ "$updateBlobfuseProxy" = "true" ] || [ "$updateService" = "true" ];then
+      echo "start blobfuse-proxy...."
+      $HOST_CMD systemctl daemon-reload
+      $HOST_CMD systemctl enable blobfuse-proxy.service
+      $HOST_CMD systemctl restart blobfuse-proxy.service
+    fi
   fi
 fi
 

pkg/blobplugin/Dockerfile

@@ -28,9 +28,16 @@ RUN chmod +x /blobfuse-proxy/init.sh && \
  chmod +x /blobfuse-proxy/blobfuse-proxy.service && \
  chmod +x /blobfuse-proxy/blobfuse-proxy
 
-RUN apt update && apt upgrade -y && apt-mark unhold libcap2 && clean-install ca-certificates uuid-dev util-linux mount udev wget e2fsprogs nfs-common netbase
+# packages that are only needed by aznfs: procps conntrack iptables bind9-host iproute2 bash netcat sysvinit-utils.
+RUN apt update && apt upgrade -y && apt-mark unhold libcap2 && clean-install ca-certificates uuid-dev util-linux mount udev wget e2fsprogs nfs-common netbase procps conntrack iptables bind9-host iproute2 bash netcat sysvinit-utils
+
+# install aznfs
+RUN if [ "$ARCH" = "amd64" ] ; then \
+  wget -O aznfs.tar.gz https://github.com/Azure/AZNFS-mount/releases/download/1.0.8/aznfs-1.0.8-1.x86_64.tar.gz; \
+else \
+  wget -O aznfs.tar.gz https://github.com/Azure/AZNFS-mount/releases/download/1.0.8/aznfs-1.0.8-1.arm64.tar.gz;fi
+RUN tar xvzf aznfs.tar.gz -C / && rm aznfs.tar.gz
 
-ARG ARCH=amd64
 RUN if [ "$ARCH" = "amd64" ] ; then \
   clean-install libcurl4-gnutls-dev && \
   wget -O /blobfuse-proxy/packages-microsoft-prod-22.04.deb https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb && \

pkg/blobplugin/main.go

@@ -58,6 +58,7 @@ var (
 	kubeAPIQPS                             = flag.Float64("kube-api-qps", 25.0, "QPS to use while communicating with the kubernetes apiserver.")
 	kubeAPIBurst                           = flag.Int("kube-api-burst", 50, "Burst to use while communicating with the kubernetes apiserver.")
 	appendMountErrorHelpLink               = flag.Bool("append-mount-error-help-link", true, "Whether to include a link for help with mount errors when a mount error occurs.")
+	enableAZNFS                            = flag.Bool("enable-aznfs", true, "replace nfs mount with aznfs mount")
 )
 
 func main() {
@@ -97,6 +98,7 @@ func handle() {
 		AppendMountErrorHelpLink:               *appendMountErrorHelpLink,
 		KubeAPIQPS:                             *kubeAPIQPS,
 		KubeAPIBurst:                           *kubeAPIBurst,
+		EnableAZNFS:                            *enableAZNFS,
 	}
 	driver := blob.NewDriver(&driverOptions)
 	if driver == nil {

test/external-e2e/run.sh

@@ -40,7 +40,6 @@ setup_e2e_binaries() {
 }
 
 print_logs() {
-    bash ./hack/verify-examples.sh
     echo "print out driver logs ..."
     bash ./test/utils/blob_log.sh $DRIVER
 }
@@ -80,3 +79,6 @@ if [ ! -z ${EXTERNAL_E2E_TEST_NFS} ]; then
         -storage.testdriver=$PROJECT_ROOT/test/external-e2e/testdriver-nfs.yaml \
         --kubeconfig=$KUBECONFIG
 fi
+
+echo "begin to run verify-examples.sh ...."
+bash ./hack/verify-examples.sh