feat: Add SAS support, NodePublishVolume added, using csi-lib-sas

Signed-off-by: Joe Skazinski <[email protected]>

go.mod

@@ -4,6 +4,7 @@ go 1.16
 
 require (
 	github.com/Seagate/csi-lib-iscsi v1.0.3
+	github.com/Seagate/csi-lib-sas v0.0.0-00010101000000-000000000000
 	github.com/Seagate/seagate-exos-x-api-go v1.0.8-0.20220531203625-3d1a38b18ac6
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/container-storage-interface/spec v1.4.0
@@ -25,3 +26,4 @@ require (
 
 // replace github.com/Seagate/seagate-exos-x-api-go => ../seagate-exos-x-api-go
 // replace github.com/Seagate/csi-lib-iscsi => ../csi-lib-iscsi
+replace github.com/Seagate/csi-lib-sas => ../csi-lib-sas

go.sum

@@ -127,13 +127,16 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
+github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE=
+github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -752,6 +755,8 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
 k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
+k8s.io/klog/v2 v2.70.0 h1:GMmmjoFOrNepPN0ZeGCzvD2Gh5IKRwdFx8W5PBxVTQU=
+k8s.io/klog/v2 v2.70.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

pkg/node/node.go

@@ -140,10 +140,8 @@ func (node *Node) NodePublishVolume(ctx context.Context, req *csi.NodePublishVol
 	klog.Infof("NodePublishVolume called with volume name %s", volumeName)
 
 	config := make(map[string]string)
-	if storageProtocol == common.StorageProtocolISCSI {
-		config["iscsiInfoPath"] = node.getIscsiInfoPath(volumeName)
-		klog.V(2).Infof("NodePublishVolume iscsiInfoPath (%v)", config["iscsiInfoPath"])
-	}
+	config["connectorInfoPath"] = node.getConnectorInfoPath(storageProtocol, volumeName)
+	klog.V(2).Infof("NodePublishVolume connectorInfoPath (%v)", config["connectorInfoPath"])
 
 	// Get storage handler
 	storageNode, err := storage.NewStorageNode(storageProtocol, config)
@@ -165,8 +163,8 @@ func (node *Node) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublis
 	klog.Infof("NodeUnpublishVolume volume %s at target path %s", volumeName, req.GetTargetPath())
 
 	config := make(map[string]string)
-	config["iscsiInfoPath"] = node.getIscsiInfoPath(volumeName)
-	klog.V(2).Infof("NodeUnpublishVolume iscsiInfoPath (%v)", config["iscsiInfoPath"])
+	config["connectorInfoPath"] = node.getConnectorInfoPath(storageProtocol, volumeName)
+	klog.V(2).Infof("NodeUnpublishVolume connectorInfoPath (%v)", config["connectorInfoPath"])
 
 	// Get storage handler
 	storageNode, err := storage.NewStorageNode(storageProtocol, config)
@@ -188,8 +186,8 @@ func (node *Node) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolum
 	klog.Infof("NodeExpandVolume volume %s at volume path %s", volumeName, req.GetVolumePath())
 
 	config := make(map[string]string)
-	config["iscsiInfoPath"] = node.getIscsiInfoPath(volumeName)
-	klog.V(2).Infof("NodeExpandVolume iscsiInfoPath (%v)", config["iscsiInfoPath"])
+	config["connectorInfoPath"] = node.getConnectorInfoPath(storageProtocol, volumeName)
+	klog.V(2).Infof("NodeExpandVolume connectorInfoPath (%v)", config["connectorInfoPath"])
 
 	// Get storage handler
 	storageNode, err := storage.NewStorageNode(storageProtocol, config)
@@ -228,9 +226,9 @@ func (node *Node) Probe(ctx context.Context, req *csi.ProbeRequest) (*csi.ProbeR
 	return &csi.ProbeResponse{Ready: &wrappers.BoolValue{Value: true}}, nil
 }
 
-// getIscsiInfoPath
-func (node *Node) getIscsiInfoPath(volumeID string) string {
-	return fmt.Sprintf("%s/iscsi-%s.json", node.runPath, volumeID)
+// getConnectorInfoPath
+func (node *Node) getConnectorInfoPath(storageProtocol, volumeID string) string {
+	return fmt.Sprintf("%s/%s-%s.json", node.runPath, storageProtocol, volumeID)
 }
 
 // checkHostBinary: Determine if a binary image is installed or not
@@ -244,7 +242,7 @@ func checkHostBinary(name string) error {
 	return nil
 }
 
-// readInitiatorName: Extract the initiaotr name from /etc/iscsi file
+// readInitiatorName: Extract the initiator name from /etc/iscsi file
 func readInitiatorName() (string, error) {
 	initiatorNameFilePath := "/etc/iscsi/initiatorname.iscsi"
 	file, err := os.Open(initiatorNameFilePath)

pkg/storage/iscsiNode.go

@@ -206,11 +206,11 @@ func (iscsi *iscsiStorage) NodePublishVolume(ctx context.Context, req *csi.NodeP
 		return nil, errors.New("device has already been mounted in several locations, please unmount first")
 	}
 
-	klog.Infof("saving ISCSI connection info in %s", iscsi.iscsiInfoPath)
-	if _, err := os.Stat(iscsi.iscsiInfoPath); err == nil {
-		klog.Warningf("iscsi connection file already exists: %s", iscsi.iscsiInfoPath)
+	klog.Infof("saving ISCSI connection info in %s", iscsi.connectorInfoPath)
+	if _, err := os.Stat(iscsi.connectorInfoPath); err == nil {
+		klog.Warningf("iscsi connection file already exists: %s", iscsi.connectorInfoPath)
 	}
-	err = iscsilib.PersistConnector(&connector, iscsi.iscsiInfoPath)
+	err = iscsilib.PersistConnector(&connector, iscsi.connectorInfoPath)
 	if err != nil {
 		return nil, status.Error(codes.Internal, err.Error())
 	}
@@ -259,8 +259,8 @@ func (iscsi *iscsiStorage) NodeUnpublishVolume(ctx context.Context, req *csi.Nod
 		klog.Warningf("assuming that volume is already unmounted: %v", err)
 	}
 
-	klog.Infof("loading ISCSI connection info from %s", iscsi.iscsiInfoPath)
-	connector, err := iscsilib.GetConnectorFromFile(iscsi.iscsiInfoPath)
+	klog.Infof("loading ISCSI connection info from %s", iscsi.connectorInfoPath)
+	connector, err := iscsilib.GetConnectorFromFile(iscsi.connectorInfoPath)
 	if err != nil {
 		if os.IsNotExist(err) {
 			klog.Warning(errors.Wrap(err, "assuming that ISCSI connection is already closed"))
@@ -301,8 +301,8 @@ func (iscsi *iscsiStorage) NodeUnpublishVolume(ctx context.Context, req *csi.Nod
 		return nil, err
 	}
 
-	klog.Infof("deleting ISCSI connection info file %s", iscsi.iscsiInfoPath)
-	os.Remove(iscsi.iscsiInfoPath)
+	klog.Infof("deleting ISCSI connection info file %s", iscsi.connectorInfoPath)
+	os.Remove(iscsi.connectorInfoPath)
 
 	klog.Info("successfully detached ISCSI device")
 	return &csi.NodeUnpublishVolumeResponse{}, nil
@@ -329,7 +329,7 @@ func (iscsi *iscsiStorage) NodeExpandVolume(ctx context.Context, req *csi.NodeEx
 		return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("node expand volume requires volume path"))
 	}
 
-	connector, err := iscsilib.GetConnectorFromFile(iscsi.iscsiInfoPath)
+	connector, err := iscsilib.GetConnectorFromFile(iscsi.connectorInfoPath)
 	klog.V(3).Infof("GetConnectorFromFile(%s) connector: %v, err: %v", volumeName, connector, err)
 
 	if err != nil {

pkg/storage/sasNode.go

@@ -20,10 +20,18 @@ package storage
 
 import (
 	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
 
+	saslib "github.com/Seagate/csi-lib-sas/sas"
+	"github.com/Seagate/seagate-exos-x-csi/pkg/common"
 	"github.com/container-storage-interface/spec/lib/go/csi"
+	"github.com/pkg/errors"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+	"k8s.io/klog"
 )
 
 // NodeStageVolume mounts the volume to a staging path on the node. This is
@@ -43,7 +51,92 @@ func (sas *sasStorage) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnsta
 
 // NodePublishVolume mounts the volume mounted to the staging path to the target path
 func (sas *sasStorage) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
-	return nil, status.Error(codes.Unimplemented, "NodePublishVolume is not implemented")
+	if len(req.GetVolumeId()) == 0 {
+		return nil, status.Error(codes.InvalidArgument, "cannot publish volume with empty id")
+	}
+	if len(req.GetTargetPath()) == 0 {
+		return nil, status.Error(codes.InvalidArgument, "cannot publish volume at an empty path")
+	}
+	if req.GetVolumeCapability() == nil {
+		return nil, status.Error(codes.InvalidArgument, "cannot publish volume without capabilities")
+	}
+
+	volumeName, _ := common.VolumeIdGetName(req.GetVolumeId())
+	wwn, _ := common.VolumeIdGetWwn(req.GetVolumeId())
+	lun, _ := req.GetPublishContext()["lun"]
+
+	// Ensure that NodePublishVolume is only called once per volume
+	addGatekeeper(volumeName)
+	defer removeGatekeeper(volumeName)
+
+	klog.V(1).Infof("[START] publish volume (%s) wwn (%s) target (%s) lun (%s)", volumeName, wwn, req.GetTargetPath(), lun)
+
+	// Initiate SAS attachment
+	klog.Info("initiating SAS connection...")
+	connector := saslib.Connector{Lun: lun, TargetWWNs: []string{wwn}}
+	path, err := saslib.Attach(ctx, connector, &saslib.OSioHandler{})
+	if err != nil {
+		return nil, status.Error(codes.Unavailable, err.Error())
+	}
+	klog.Infof("attached device at %s", path)
+
+	fsType := req.GetVolumeContext()[common.FsTypeConfigKey]
+	err = ensureFsType(fsType, path)
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	corrupted := false
+	if err = checkFs(path, "Publish"); err != nil {
+		corrupted = true
+	}
+
+	if connector.Multipath {
+		klog.Infof("device is using multipath, device=%v, wwn=%v, corrupted=%v", path, wwn, corrupted)
+	} else {
+		klog.Infof("device is NOT using multipath, device=%v, wwn=%v, corrupted=%v", path, wwn, corrupted)
+	}
+
+	if corrupted {
+		klog.Infof("device corruption (publish), device=%v, volume=%s, multipath=%v, wwn=%v, corrupted=%v", connector.DevicePath, volumeName, connector.Multipath, wwn, corrupted)
+		debugCorruption("$$", path)
+		return nil, status.Errorf(codes.DataLoss, "(publish) filesystem (%v) seems to be corrupted: %v", path, err)
+	}
+
+	out, err := exec.Command("findmnt", "--output", "TARGET", "--noheadings", path).Output()
+	mountpoints := strings.Split(strings.Trim(string(out), "\n"), "\n")
+	if err != nil || len(mountpoints) == 0 {
+		klog.V(1).Infof("mount -t %s %s %s", fsType, path, req.GetTargetPath())
+		os.Mkdir(req.GetTargetPath(), 00755)
+		if _, err = os.Stat(path); errors.Is(err, os.ErrNotExist) {
+			klog.Infof("targetpath does not exist:%s", req.GetTargetPath())
+		}
+		out, err = exec.Command("mount", "-t", fsType, path, req.GetTargetPath()).CombinedOutput()
+		if err != nil {
+			return nil, status.Error(codes.Internal, string(out))
+		}
+	} else if len(mountpoints) == 1 {
+		if mountpoints[0] == req.GetTargetPath() {
+			klog.Infof("volume %s already mounted", req.GetTargetPath())
+		} else {
+			errStr := fmt.Sprintf("device has already been mounted somewhere else (%s instead of %s), please unmount first", mountpoints[0], req.GetTargetPath())
+			return nil, status.Error(codes.Internal, errStr)
+		}
+	} else if len(mountpoints) > 1 {
+		return nil, errors.New("device has already been mounted in several locations, please unmount first")
+	}
+
+	klog.Infof("saving SAS connection info in %s", sas.connectorInfoPath)
+	if _, err := os.Stat(sas.connectorInfoPath); err == nil {
+		klog.Warningf("sas connection file already exists: %s", sas.connectorInfoPath)
+	}
+	err = saslib.PersistConnector(ctx, &connector, sas.connectorInfoPath)
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	klog.Infof("successfully mounted volume at %s", req.GetTargetPath())
+	return &csi.NodePublishVolumeResponse{}, nil
 }
 
 // NodeUnpublishVolume unmounts the volume from the target path

pkg/storage/storageService.go

@@ -36,16 +36,18 @@ type commonService struct {
 }
 
 type fcStorage struct {
-	cs commonService
+	cs                commonService
+	connectorInfoPath string
 }
 
 type iscsiStorage struct {
-	cs            commonService
-	iscsiInfoPath string
+	cs                commonService
+	connectorInfoPath string
 }
 
 type sasStorage struct {
-	cs commonService
+	cs                commonService
+	connectorInfoPath string
 }
 
 // buildCommonService:
@@ -63,15 +65,15 @@ func NewStorageNode(storageProtocol string, config map[string]string) (StorageOp
 		storageProtocol = strings.TrimSpace(storageProtocol)
 		klog.V(2).Infof("NewStorageNode for (%s)", storageProtocol)
 		if storageProtocol == common.StorageProtocolFC {
-			return &fcStorage{cs: comnserv}, nil
+			return &fcStorage{cs: comnserv, connectorInfoPath: config["connectorInfoPath"]}, nil
 		} else if storageProtocol == common.StorageProtocolSAS {
-			return &sasStorage{cs: comnserv}, nil
+			return &sasStorage{cs: comnserv, connectorInfoPath: config["connectorInfoPath"]}, nil
 		} else if storageProtocol == common.StorageProtocolISCSI {
-			return &iscsiStorage{cs: comnserv, iscsiInfoPath: config["iscsiInfoPath"]}, nil
+			return &iscsiStorage{cs: comnserv, connectorInfoPath: config["connectorInfoPath"]}, nil
 		} else {
 			klog.Warningf("Invalid or no storage protocol specified (%s)", storageProtocol)
 			klog.Warningf("Expecting storageProtocol (iscsi, fc, sas, etc) in StorageClass YAML. Default of (%s) used.", common.StorageProtocolISCSI)
-			return &iscsiStorage{cs: comnserv, iscsiInfoPath: config["iscsiInfoPath"]}, nil
+			return &iscsiStorage{cs: comnserv, connectorInfoPath: config["connectorInfoPath"]}, nil
 		}
 	}
 	return nil, err