Hs 312/sas device not found

Makefile

@@ -9,7 +9,7 @@ endif
 ifdef VERSION
 VERSION := $(VERSION)
 else
-VERSION := v1.5.1
+VERSION := v1.5.3
 endif
 
 VERSION_FLAG = -X github.com/Seagate/seagate-exos-x-csi/pkg/common.Version=$(VERSION)

cmd/controller/controller.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/Seagate/seagate-exos-x-csi/pkg/common"
 	"github.com/Seagate/seagate-exos-x-csi/pkg/controller"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 var bind = flag.String("bind", fmt.Sprintf("unix:///var/run/%s/csi-controller.sock", common.PluginName), "RPC bind URI (can be a UNIX socket path or any URI)")

cmd/node/node.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/Seagate/seagate-exos-x-csi/pkg/common"
 	"github.com/Seagate/seagate-exos-x-csi/pkg/node"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 var bind = flag.String("bind", fmt.Sprintf("unix:///var/run/%s/csi-node.sock", common.PluginName), "RPC bind URI (can be a UNIX socket path or any URI)")
@@ -24,7 +24,6 @@ func main() {
 		}
 	}
 
-
 	klog.Infof("starting storage node plugin (%s)", common.Version)
 	node.New().Start(*bind)
 }

go.mod

@@ -4,8 +4,8 @@ go 1.19
 
 require (
 	github.com/Seagate/csi-lib-iscsi v1.0.3
-	github.com/Seagate/csi-lib-sas v1.0.1
-	github.com/Seagate/seagate-exos-x-api-go v1.0.9
+	github.com/Seagate/csi-lib-sas v1.0.2
+	github.com/Seagate/seagate-exos-x-api-go v1.0.11
 	github.com/container-storage-interface/spec v1.4.0
 	github.com/golang/protobuf v1.5.2
 	github.com/google/uuid v1.3.0
@@ -16,7 +16,7 @@ require (
 	github.com/prometheus/client_golang v1.13.0
 	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
 	google.golang.org/grpc v1.50.0
-	k8s.io/klog v1.0.0
+	k8s.io/klog/v2 v2.70.0
 )
 
 require (
@@ -37,7 +37,6 @@ require (
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	k8s.io/klog/v2 v2.70.0 // indirect
 )
 
 // replace github.com/Seagate/seagate-exos-x-api-go => ../seagate-exos-x-api-go

go.sum

@@ -36,10 +36,10 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
 github.com/Seagate/csi-lib-iscsi v1.0.3 h1:PEPCySqKmdRwgyM4vb/cugFVyNbo5JL+f/J7W4Q/WVw=
 github.com/Seagate/csi-lib-iscsi v1.0.3/go.mod h1:sp7ftl8BMVgMNybv3sw8V20MJsX0bl2w5BoSFlBRPkY=
-github.com/Seagate/csi-lib-sas v1.0.1 h1:7qIFiQ7YORsvKlGxCL4XguadMBmIBeis45BoB29PMg0=
-github.com/Seagate/csi-lib-sas v1.0.1/go.mod h1:lX/OnO0sLm4vXCFwsPADyzZxSFkmXv5t+WleiYNkN8U=
-github.com/Seagate/seagate-exos-x-api-go v1.0.9 h1:ZnNaU3GQ7/vo8u8qPdi+nTHdxRguBfaE+pRWgvIGphE=
-github.com/Seagate/seagate-exos-x-api-go v1.0.9/go.mod h1:BAbcvtjsD9uKUdJbypktpa/gQ75TH3wWqC8L4Exd3ac=
+github.com/Seagate/csi-lib-sas v1.0.2 h1:rR/tPmQMYt7nwor5YC1LInxrYvueLjCxFKHxkh0qL5A=
+github.com/Seagate/csi-lib-sas v1.0.2/go.mod h1:lX/OnO0sLm4vXCFwsPADyzZxSFkmXv5t+WleiYNkN8U=
+github.com/Seagate/seagate-exos-x-api-go v1.0.11 h1:TpOnQAdcd269J/6X2z0IXMAEN6DgZotOEiDhK0rRWPY=
+github.com/Seagate/seagate-exos-x-api-go v1.0.11/go.mod h1:BAbcvtjsD9uKUdJbypktpa/gQ75TH3wWqC8L4Exd3ac=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
 github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
@@ -117,7 +117,6 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
-github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE=
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
@@ -742,8 +741,6 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
-k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
-k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/klog/v2 v2.70.0 h1:GMmmjoFOrNepPN0ZeGCzvD2Gh5IKRwdFx8W5PBxVTQU=
 k8s.io/klog/v2 v2.70.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

helm/csi-charts/values.yaml

@@ -12,7 +12,7 @@ image:
   repository: ghcr.io/seagate/seagate-exos-x-csi
   # -- Tag to use for nodes and controller
   # @default -- Uses Chart.appVersion value by default if tag does not specify a new version.
-  tag: "v1.5.1"
+  tag: "v1.5.3"
   # -- Default is set to IfNotPresent, to override use Always here to always pull the specified version
   pullPolicy: Always
 

pkg/common/driver.go

@@ -17,7 +17,7 @@ import (
 	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	"github.com/prometheus/client_golang/prometheus"
 	"google.golang.org/grpc"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 // PluginName is the public name to be used in storage class etc.
@@ -141,7 +141,7 @@ func (driver *Driver) Start(bind string) {
 
 	var ll klog.Level = 0
 	for i := 0; i < 10; i++ {
-		if klog.V(klog.Level(i)) {
+		if klog.V(klog.Level(i)).Enabled() {
 			ll = klog.Level(i)
 		} else {
 			break

pkg/common/stringlock.go

@@ -21,7 +21,7 @@ package common
 import (
 	"sync"
 
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 type MyMutex struct {

pkg/common/system.go

@@ -23,7 +23,7 @@ import (
 	"strings"
 	"unicode"
 
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 // ValidateName verifies that the string only includes spaces and printable UTF-8 characters except: " , < \

pkg/controller/controller.go

@@ -13,7 +13,7 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 const (

pkg/controller/expander.go

@@ -9,7 +9,7 @@ import (
 	"github.com/container-storage-interface/spec/lib/go/csi"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 // ControllerExpandVolume expands a volume to the given new size

pkg/controller/provisioner.go

@@ -10,7 +10,7 @@ import (
 	"github.com/container-storage-interface/spec/lib/go/csi"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 var (

pkg/controller/publisher.go

@@ -12,7 +12,7 @@ import (
 	"github.com/container-storage-interface/spec/lib/go/csi"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 // getConnectorInfoPath

pkg/controller/snapshotter.go

@@ -15,7 +15,7 @@ import (
 	"github.com/golang/protobuf/ptypes/timestamp"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 // CreateSnapshot creates a snapshot of the given volume

pkg/exporter/exporter.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 // Exporter : Configuration (from command-line)

pkg/node/node.go

@@ -20,7 +20,7 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 // Node is the implementation of csi.NodeServer
@@ -33,7 +33,7 @@ type Node struct {
 
 // New is a convenience function for creating a node driver
 func New() *Node {
-	if klog.V(8) {
+	if klog.V(8).Enabled() {
 		iscsi.EnableDebugLogging(os.Stderr)
 	}
 

pkg/storage/fcNode.go

@@ -31,7 +31,7 @@ import (
 	"github.com/pkg/errors"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 // NodeStageVolume mounts the volume to a staging path on the node. This is

pkg/storage/iscsiNode.go

@@ -33,7 +33,7 @@ import (
 	"github.com/pkg/errors"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 // Configuration constants

pkg/storage/sasNode.go

@@ -24,16 +24,20 @@ import (
 	"os"
 	"os/exec"
 	"strings"
+	"time"
 
 	saslib "github.com/Seagate/csi-lib-sas/sas"
 	"github.com/Seagate/seagate-exos-x-csi/pkg/common"
 	"github.com/container-storage-interface/spec/lib/go/csi"
 	"github.com/pkg/errors"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
+// Map of device WWNs to timestamp of when they were unpublished from the node
+var globalRemovedDevicesMap = map[string]time.Time{}
+
 // NodeStageVolume mounts the volume to a staging path on the node. This is
 // called by the CO before NodePublishVolume and is used to temporary mount the
 // volume to a staging path. Once mounted, NodePublishVolume will make sure to
@@ -63,7 +67,7 @@ func (sas *sasStorage) NodePublishVolume(ctx context.Context, req *csi.NodePubli
 
 	volumeName, _ := common.VolumeIdGetName(req.GetVolumeId())
 	wwn, _ := common.VolumeIdGetWwn(req.GetVolumeId())
-	lun, _ := req.GetPublishContext()["lun"]
+	lun := req.GetPublishContext()["lun"]
 
 	// Ensure that NodePublishVolume is only called once per volume
 	AddGatekeeper(volumeName)
@@ -80,6 +84,11 @@ func (sas *sasStorage) NodePublishVolume(ctx context.Context, req *csi.NodePubli
 	}
 	klog.Infof("attached device at %s", path)
 
+	// if current wwn has been published before, remove it from our list of previously unpublished wwns
+	delete(globalRemovedDevicesMap, wwn)
+	// check if previously unpublished devices were rediscovered by the scsi subsystem during Attach
+	checkPreviouslyRemovedDevices(ctx)
+
 	fsType := req.GetVolumeContext()[common.FsTypeConfigKey]
 	err = EnsureFsType(fsType, path)
 	if err != nil {
@@ -215,6 +224,23 @@ func (sas *sasStorage) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnp
 		return nil, status.Errorf(codes.DataLoss, "(unpublish) filesystem seems to be corrupted: %v", err)
 	}
 
+	if !connector.Multipath {
+		// If we didn't discover the multipath device initially, double check that we didn't just miss it
+		// Detach the discovered devices if they are found
+		klog.V(3).Info("Device saved as non-multipath. Searching for additional devices before Detach")
+		if connector.IoHandler == nil {
+			connector.IoHandler = &saslib.OSioHandler{}
+		}
+		discoveredMpathName, devices := saslib.FindDiskById(klog.FromContext(ctx), wwn, connector.IoHandler)
+		if (discoveredMpathName != connector.OSPathName) && (len(devices) > 0) {
+			klog.V(0).Infof("Found additional linked devices: %s, %v", discoveredMpathName, devices)
+			klog.V(0).Infof("Replacing original connector info prior to Detach, device: %s=>%s, linked device paths: %v=>%v", connector.OSPathName, discoveredMpathName, connector.OSDevicePaths, devices)
+			connector.OSPathName = discoveredMpathName
+			connector.OSDevicePaths = devices
+			connector.Multipath = true
+		}
+	}
+
 	klog.Info("DisconnectVolume, detaching SAS device")
 	err = saslib.Detach(ctx, connector.OSPathName, connector.IoHandler)
 
@@ -225,10 +251,26 @@ func (sas *sasStorage) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnp
 	klog.Infof("deleting SAS connection info file %s", sas.connectorInfoPath)
 	os.Remove(sas.connectorInfoPath)
 
+	globalRemovedDevicesMap[connector.VolumeWWN] = time.Now()
+
 	klog.Info("successfully detached SAS device")
 	return &csi.NodeUnpublishVolumeResponse{}, nil
 }
 
+func checkPreviouslyRemovedDevices(ctx context.Context) error {
+	klog.Info("Checking previously removed devices")
+	for wwn, _ := range globalRemovedDevicesMap {
+		klog.Infof("Checking for rediscovery of wwn:%s", wwn)
+
+		dm, devices := saslib.FindDiskById(klog.FromContext(ctx), wwn, &saslib.OSioHandler{})
+		if dm != "" {
+			klog.Infof("Rediscovery found for wwn:%s -- mpath device: %s, devices: %v", wwn, dm, devices)
+			saslib.Detach(ctx, dm, &saslib.OSioHandler{})
+		}
+	}
+	return nil
+}
+
 // NodeGetVolumeStats return info about a given volume
 // Will not be called as the plugin does not have the GET_VOLUME_STATS capability
 func (sas *sasStorage) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) {

pkg/storage/storageService.go

@@ -29,7 +29,7 @@ import (
 	"github.com/Seagate/seagate-exos-x-csi/pkg/common"
 	"github.com/container-storage-interface/spec/lib/go/csi"
 	"github.com/pkg/errors"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 type StorageOperations interface {
@@ -64,7 +64,7 @@ func buildCommonService(config map[string]string) (commonService, error) {
 	return commonserv, nil
 }
 
-//NewStorageNode : To return specific implementation of storage
+// NewStorageNode : To return specific implementation of storage
 func NewStorageNode(storageProtocol string, config map[string]string) (StorageOperations, error) {
 	comnserv, err := buildCommonService(config)
 	if err == nil {