diff --git a/README.md b/README.md index 2bf19e08..7d29e936 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ The Seagate Exos X CSI Driver supports the following storage arrays - Seagate Exos X and AssuredSAN (4006/5005/4005/3005) - Dell PowerVault ME4 and ME5 Series -iSCSI, SAS, and FC host interfaces are supported. +iSCSI, SAS, and FC host interfaces are supported for both block and filesystem mount types [![Go Report Card](https://goreportcard.com/badge/github.com/Seagate/seagate-exos-x-csi)](https://goreportcard.com/report/github.com/Seagate/seagate-exos-x-csi) @@ -31,7 +31,7 @@ This project implements the **Container Storage Interface** in order to facilita This CSI driver is an open-source project under the Apache 2.0 [license](./LICENSE). ## Key Features -- Manage persistent volumes backed by iSCSI protocols on Exos X enclosures +- Manage persistent volumes on Exos X enclosures - Control multiple Exos X systems within a single Kubernetes cluster - Manage Exos X snapshots and clones, including restoring from snapshots - Clone, extend and manage persistent volumes created outside of the Exos CSI Driver diff --git a/example/block-volume-pod.yaml b/example/block-volume-pod.yaml new file mode 100644 index 00000000..d86026ef --- /dev/null +++ b/example/block-volume-pod.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: systems-pvc +spec: + accessModes: + - ReadWriteOnce + volumeMode: Block + storageClassName: block-vol-storageclass + resources: + requests: + storage: 5Gi +--- +apiVersion: v1 +kind: Pod +metadata: + name: test-pod +spec: + containers: + - image: ghcr.io/seagate/seagate-exos-x-testapp + command: ["/bin/sh", "-c", "while sleep 60; do echo hello > /vol/test && ls -l /vol && cat /vol/test && rm /vol/test; done"] + name: test-pod-container + volumeDevices: + - devicePath: /block-vol + name: volume + ports: + - containerPort: 8080 + volumes: + - name: volume + persistentVolumeClaim: + claimName: systems-pvc \ No newline at end of file diff --git a/example/block-volume-storageclass.yaml b/example/block-volume-storageclass.yaml new file mode 100644 index 00000000..7e91dee6 --- /dev/null +++ b/example/block-volume-storageclass.yaml @@ -0,0 +1,21 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +provisioner: csi-exos-x.seagate.com # Check pkg/driver.go, Required for the plugin to recognize this storage class as handled by itself. +volumeBindingMode: WaitForFirstConsumer # Prefer this value to avoid unschedulable pods (https://kubernetes.io/docs/concepts/storage/storage-classes/#volume-binding-mode) +allowVolumeExpansion: true +metadata: + name: block-vol-storageclass +parameters: + # Secrets name and namespace, they can be the same for provisioner, controller-publish and controller-expand sections. + csi.storage.k8s.io/provisioner-secret-name: seagate-exos-x-csi-secrets + csi.storage.k8s.io/provisioner-secret-namespace: default + csi.storage.k8s.io/controller-publish-secret-name: seagate-exos-x-csi-secrets + csi.storage.k8s.io/controller-publish-secret-namespace: default + csi.storage.k8s.io/controller-expand-secret-name: seagate-exos-x-csi-secrets + csi.storage.k8s.io/controller-expand-secret-namespace: default + csi.storage.k8s.io/node-publish-secret-name: seagate-exos-x-csi-secrets + csi.storage.k8s.io/node-publish-secret-namespace: default + pool: A # Pool to use on the IQN to provision volumes + volPrefix: stx # Desired prefix for volume naming, an underscore is appended + storageProtocol: iscsi # iscsi, fc or sas + AccessType: block \ No newline at end of file diff --git a/pkg/common/driver.go b/pkg/common/driver.go index 97c125d4..da32df42 100644 --- a/pkg/common/driver.go +++ b/pkg/common/driver.go @@ -56,6 +56,11 @@ const ( NodeServicePortEnvVar = "CSI_NODE_SERVICE_PORT" ) +var SupportedAccessModes = [2]csi.VolumeCapability_AccessMode_Mode{ + csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY, +} + // Driver contains main resources needed by the driver and references the underlying specific driver type Driver struct { Server *grpc.Server diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 31bfee2d..df7fba3b 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -31,17 +31,6 @@ const ( invalidArgumentErrorCode = -10058 ) -var volumeCapabilities = []*csi.VolumeCapability{ - { - AccessType: &csi.VolumeCapability_Mount{ - Mount: &csi.VolumeCapability_MountVolume{}, - }, - AccessMode: &csi.VolumeCapability_AccessMode{ - Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, - }, - }, -} - var csiMutexes = map[string]*sync.Mutex{ "/csi.v1.Controller/CreateVolume": {}, "/csi.v1.Controller/ControllerPublishVolume": {}, @@ -170,8 +159,7 @@ func (controller *Controller) ControllerGetCapabilities(ctx context.Context, req return &csi.ControllerGetCapabilitiesResponse{Capabilities: csc}, nil } -// ValidateVolumeCapabilities checks whether the volume capabilities requested -// are supported. +// ValidateVolumeCapabilities checks whether a provisioned volume supports the capabilities requested func (controller *Controller) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { volumeName, _ := common.VolumeIdGetName(req.GetVolumeId()) @@ -188,7 +176,7 @@ func (controller *Controller) ValidateVolumeCapabilities(ctx context.Context, re return &csi.ValidateVolumeCapabilitiesResponse{ Confirmed: &csi.ValidateVolumeCapabilitiesResponse_Confirmed{ - VolumeCapabilities: volumeCapabilities, + VolumeCapabilities: req.GetVolumeCapabilities(), }, }, nil } @@ -301,19 +289,27 @@ func runPreflightChecks(parameters map[string]string, capabilities *[]*csi.Volum return status.Error(codes.InvalidArgument, "missing volume capabilities") } for _, capability := range *capabilities { - if capability.GetAccessMode().GetMode() != csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER { - return status.Error(codes.FailedPrecondition, "storage only supports ReadWriteOnce access mode") + accessMode := capability.GetAccessMode().GetMode() + accessModeSupported := false + for _, mode := range common.SupportedAccessModes { + if accessMode == mode { + accessModeSupported = true + } + } + if !accessModeSupported { + return status.Errorf(codes.FailedPrecondition, "driver does not support access mode %v", accessMode) } - if capability.GetMount().GetFsType() == "" { - if err := checkIfKeyExistsInConfig(common.FsTypeConfigKey); err != nil { - return status.Error(codes.FailedPrecondition, "no fstype specified in storage class") - } else { - klog.InfoS("storage class parameter "+common.FsTypeConfigKey+" is deprecated. Please migrate to 'csi.storage.k8s.io/fstype'", "parameter", common.FsTypeConfigKey) + if mount := capability.GetMount(); mount != nil { + if mount.GetFsType() == "" { + if err := checkIfKeyExistsInConfig(common.FsTypeConfigKey); err != nil { + return status.Error(codes.FailedPrecondition, "no fstype specified in storage class") + } else { + klog.InfoS("storage class parameter "+common.FsTypeConfigKey+" is deprecated. Please migrate to 'csi.storage.k8s.io/fstype'", "parameter", common.FsTypeConfigKey) + } } } } } - return nil } diff --git a/pkg/controller/provisioner.go b/pkg/controller/provisioner.go index 81b01a86..4f618c46 100644 --- a/pkg/controller/provisioner.go +++ b/pkg/controller/provisioner.go @@ -13,17 +13,6 @@ import ( "k8s.io/klog/v2" ) -var ( - volumeCaps = []csi.VolumeCapability_AccessMode{ - { - Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, - }, - { - Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY, - }, - } -) - // Extract available SAS addresses for Nodes from topology segments // This will contain all SAS initiators for all nodes unless the storage class // has specified allowed or preferred topologies @@ -203,23 +192,22 @@ func getSizeStr(size int64) string { // isValidVolumeCapabilities validates the given VolumeCapability array is valid func isValidVolumeCapabilities(volCaps []*csi.VolumeCapability) error { if len(volCaps) == 0 { - return fmt.Errorf("CreateVolume Volume capabilities must be provided") + return fmt.Errorf("volume capabilities to validate not provided") } - hasSupport := func(cap *csi.VolumeCapability) error { - if blk := cap.GetBlock(); blk != nil { - return fmt.Errorf("driver only supports mount access type volume capability") - } - for _, c := range volumeCaps { - if c.GetMode() == cap.AccessMode.GetMode() { - return nil + + hasSupport := func(cap *csi.VolumeCapability) bool { + for _, supportedMode := range common.SupportedAccessModes { + // we currently support block and mount volumes with both supported access modes, so don't check mount types + if cap.GetAccessMode().Mode == supportedMode { + return true } } - return fmt.Errorf("driver does not support access mode %v", cap.AccessMode.GetMode()) + return false } for _, c := range volCaps { - if err := hasSupport(c); err != nil { - return err + if !hasSupport(c) { + return fmt.Errorf("driver does not support access mode %v", c.GetAccessMode()) } } return nil diff --git a/pkg/node/node.go b/pkg/node/node.go index 657b1e5a..3da394d0 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -162,14 +162,34 @@ func (node *Node) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapab return &csi.NodeGetCapabilitiesResponse{Capabilities: csc}, nil } -// NodePublishVolume mounts the volume mounted to the staging path to the target path +// NodePublishVolume mounts the device to the target path func (node *Node) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { - + if len(req.GetVolumeId()) == 0 { + return nil, status.Error(codes.InvalidArgument, "cannot publish volume with empty id") + } + if len(req.GetTargetPath()) == 0 { + return nil, status.Error(codes.InvalidArgument, "cannot publish volume at an empty path") + } + if req.GetVolumeCapability() == nil { + return nil, status.Error(codes.InvalidArgument, "cannot publish volume without capabilities") + } + if req.GetVolumeCapability().GetBlock() != nil && + req.GetVolumeCapability().GetMount() != nil { + return nil, status.Error(codes.InvalidArgument, "cannot have both block and mount access type") + } + if req.GetVolumeCapability().GetBlock() == nil && + req.GetVolumeCapability().GetMount() == nil { + return nil, status.Error(codes.InvalidArgument, "volume access type not specified, must be either block or mount") + } // Extract the volume name and the storage protocol from the augmented volume id volumeName, _ := common.VolumeIdGetName(req.GetVolumeId()) storageProtocol, _ := common.VolumeIdGetStorageProtocol(req.GetVolumeId()) - klog.Infof("NodePublishVolume called with volume name %s", volumeName) + // Ensure that NodePublishVolume is only called once per volume + storage.AddGatekeeper(volumeName) + defer storage.RemoveGatekeeper(volumeName) + + klog.InfoS("NodePublishVolume call", "volumeName", volumeName) config := make(map[string]string) config["connectorInfoPath"] = node.getConnectorInfoPath(storageProtocol, volumeName) @@ -177,35 +197,63 @@ func (node *Node) NodePublishVolume(ctx context.Context, req *csi.NodePublishVol // Get storage handler storageNode, err := storage.NewStorageNode(storageProtocol, config) - if storageNode != nil { - return storageNode.NodePublishVolume(ctx, req) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + // Do any required device discovery and return path of the device on the node fs + path, err := storageNode.AttachStorage(ctx, req) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) } - klog.Errorf("NodePublishVolume error for storage protocol (%v): %v", storageProtocol, err) - return nil, status.Errorf(codes.Internal, "Unable to process for storage protocol (%v)", storageProtocol) + if req.GetVolumeCapability().GetMount() != nil { + err = storage.MountFilesystem(req, path) + } else if req.GetVolumeCapability().GetBlock() != nil { + err = storage.MountDevice(req, path) + } + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + return &csi.NodePublishVolumeResponse{}, nil } -// NodeUnpublishVolume unmounts the volume from the target path +// NodeUnpublishVolume unmounts the volume from the target path and removes devices func (node *Node) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { + if len(req.GetVolumeId()) == 0 { + return nil, status.Error(codes.InvalidArgument, "cannot unpublish volume with an empty volume id") + } + if len(req.GetTargetPath()) == 0 { + return nil, status.Error(codes.InvalidArgument, "cannot unpublish volume with an empty target path") + } // Extract the volume name and the storage protocol from the augmented volume id volumeName, _ := common.VolumeIdGetName(req.GetVolumeId()) storageProtocol, _ := common.VolumeIdGetStorageProtocol(req.GetVolumeId()) - klog.Infof("NodeUnpublishVolume volume %s at target path %s", volumeName, req.GetTargetPath()) + // Ensure that NodeUnpublishVolume is only called once per volume + storage.AddGatekeeper(volumeName) + defer storage.RemoveGatekeeper(volumeName) + + klog.InfoS("NodeUnpublishVolume volume", "volumeName", volumeName, "targetPath", req.GetTargetPath()) config := make(map[string]string) config["connectorInfoPath"] = node.getConnectorInfoPath(storageProtocol, volumeName) - klog.V(2).Infof("NodeUnpublishVolume connectorInfoPath (%v)", config["connectorInfoPath"]) + klog.V(2).InfoS("NodeUnpublishVolume", "connectorInfoPath", config["connectorInfoPath"]) // Get storage handler storageNode, err := storage.NewStorageNode(storageProtocol, config) - if storageNode != nil { - return storageNode.NodeUnpublishVolume(ctx, req) + if storageNode == nil { + klog.ErrorS(err, "Error creating storage node") + return nil, status.Errorf(codes.Internal, "unable to create storage node") + } + storage.Unmount(req.GetTargetPath()) + err = storageNode.DetachStorage(ctx, req) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) } - klog.Errorf("NodeUnpublishVolume error for storage protocol (%v): %v", storageProtocol, err) - return nil, status.Errorf(codes.Internal, "Unable to process for storage protocol (%v)", storageProtocol) + return &csi.NodeUnpublishVolumeResponse{}, nil } // NodeExpandVolume finalizes volume expansion on the node diff --git a/pkg/storage/fcNode.go b/pkg/storage/fcNode.go index 634cbba5..f35f8e4e 100644 --- a/pkg/storage/fcNode.go +++ b/pkg/storage/fcNode.go @@ -22,11 +22,11 @@ import ( "bufio" "context" "fmt" + "io/fs" "os" "os/exec" "path/filepath" "strings" - "time" fclib "github.com/Seagate/csi-lib-sas/sas" "github.com/Seagate/seagate-exos-x-csi/pkg/common" @@ -52,167 +52,47 @@ func (fc *fcStorage) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstage return nil, status.Error(codes.Unimplemented, "NodeUnstageVolume is not implemented") } -// NodePublishVolume mounts the volume mounted to the staging path to the target path -func (fc *fcStorage) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { - if len(req.GetVolumeId()) == 0 { - return nil, status.Error(codes.InvalidArgument, "cannot publish volume with empty id") - } - if len(req.GetTargetPath()) == 0 { - return nil, status.Error(codes.InvalidArgument, "cannot publish volume at an empty path") - } - if req.GetVolumeCapability() == nil { - return nil, status.Error(codes.InvalidArgument, "cannot publish volume without capabilities") - } - - volumeName, _ := common.VolumeIdGetName(req.GetVolumeId()) +func (fc *fcStorage) AttachStorage(ctx context.Context, req *csi.NodePublishVolumeRequest) (string, error) { + klog.InfoS("initiating FC connection...") wwn, _ := common.VolumeIdGetWwn(req.GetVolumeId()) - lun, _ := req.GetPublishContext()["lun"] - - // Ensure that NodePublishVolume is only called once per volume - AddGatekeeper(volumeName) - defer RemoveGatekeeper(volumeName) - - klog.V(1).Infof("[START] publish volume (%s) wwn (%s) target (%s) lun (%s)", volumeName, wwn, req.GetTargetPath(), lun) - - // Initiate FC attachment - klog.Info("initiating FC connection...") - connector := fclib.Connector{VolumeWWN: wwn} - path, err := fclib.Attach(ctx, &connector, &fclib.OSioHandler{}) - if err != nil { - return nil, status.Error(codes.Unavailable, err.Error()) - } - klog.Infof("attached device at %s", path) - - // if current wwn has been published before, remove it from our list of previously unpublished wwns - delete(GlobalRemovedDevicesMap, wwn) - // check if previously unpublished devices were rediscovered by the scsi subsystem during Attach - checkPreviouslyRemovedDevices(ctx) - - fsType := GetFsType(req) - err = EnsureFsType(fsType, path) + connector := &fclib.Connector{VolumeWWN: wwn} + path, err := fclib.Attach(ctx, connector, &fclib.OSioHandler{}) if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - - corrupted := false - if err = CheckFs(path, fsType, "Publish"); err != nil { - corrupted = true - } - - if connector.Multipath { - klog.Infof("device is using multipath, device=%v, wwn=%v, corrupted=%v", path, wwn, corrupted) - } else { - klog.Infof("device is NOT using multipath, device=%v, wwn=%v, corrupted=%v", path, wwn, corrupted) - } - - if corrupted { - klog.Infof("device corruption (publish), device=%v, volume=%s, multipath=%v, wwn=%v, corrupted=%v", connector.OSPathName, volumeName, connector.Multipath, wwn, corrupted) - DebugCorruption("$$", path) - return nil, status.Errorf(codes.DataLoss, "(publish) filesystem (%v) seems to be corrupted: %v", path, err) - } - - out, err := exec.Command("findmnt", "--output", "TARGET", "--noheadings", path).Output() - mountpoints := strings.Split(strings.Trim(string(out), "\n"), "\n") - if err != nil || len(mountpoints) == 0 { - klog.V(1).Infof("mount -t %s %s %s", fsType, path, req.GetTargetPath()) - os.Mkdir(req.GetTargetPath(), 00755) - if _, err = os.Stat(path); errors.Is(err, os.ErrNotExist) { - klog.Infof("targetpath does not exist:%s", req.GetTargetPath()) - } - out, err = exec.Command("mount", "-t", fsType, path, req.GetTargetPath()).CombinedOutput() - if err != nil { - return nil, status.Error(codes.Internal, string(out)) - } - } else if len(mountpoints) == 1 { - if mountpoints[0] == req.GetTargetPath() { - klog.Infof("volume %s already mounted", req.GetTargetPath()) - } else { - errStr := fmt.Sprintf("device has already been mounted somewhere else (%s instead of %s), please unmount first", mountpoints[0], req.GetTargetPath()) - return nil, status.Error(codes.Internal, errStr) - } - } else if len(mountpoints) > 1 { - return nil, errors.New("device has already been mounted in several locations, please unmount first") - } - - klog.Infof("saving FC connection info in %s", fc.connectorInfoPath) - if _, err := os.Stat(fc.connectorInfoPath); err == nil { - klog.Warningf("fc connection file already exists: %s", fc.connectorInfoPath) + return path, err } + klog.InfoS("attached device", "path", path) err = connector.Persist(ctx, fc.connectorInfoPath) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - - klog.Infof("successfully mounted volume at %s", req.GetTargetPath()) - return &csi.NodePublishVolumeResponse{}, nil - + return path, err } -// NodeUnpublishVolume unmounts the volume from the target path -func (fc *fcStorage) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { - if len(req.GetVolumeId()) == 0 { - return nil, status.Error(codes.InvalidArgument, "cannot unpublish volume with an empty volume id") - } - if len(req.GetTargetPath()) == 0 { - return nil, status.Error(codes.InvalidArgument, "cannot unpublish volume with an empty target path") - } - - volumeName, _ := common.VolumeIdGetName(req.GetVolumeId()) - - // Ensure that NodeUnpublishVolume is only called once per volume - AddGatekeeper(volumeName) - defer RemoveGatekeeper(volumeName) - - klog.Infof("[START] unpublishing volume (%s) at target path %s", volumeName, req.GetTargetPath()) - - _, err := os.Stat(req.GetTargetPath()) - if err == nil { - klog.Infof("unmounting volume at %s", req.GetTargetPath()) - klog.V(4).Infof("command: %s %s", "mountpoint", req.GetTargetPath()) - out, err := exec.Command("mountpoint", req.GetTargetPath()).CombinedOutput() - if err == nil { - klog.V(4).Infof("command: %s %s", "umount -l", req.GetTargetPath()) - out, err := exec.Command("umount", "-l", req.GetTargetPath()).CombinedOutput() - if err != nil { - return nil, status.Error(codes.Internal, string(out)) - } - } else { - klog.Warningf("assuming that volume is already unmounted: %s", out) - } - - err = os.Remove(req.GetTargetPath()) - if err != nil && !os.IsNotExist(err) { - return nil, status.Error(codes.Internal, err.Error()) - } - } else { - klog.Warningf("assuming that volume is already unmounted: %v", err) - } - - klog.Infof("loading FC connection info from %s", fc.connectorInfoPath) +func (fc *fcStorage) DetachStorage(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) error { + klog.InfoS("loading FC connection info from file", "connectorInfoPath", fc.connectorInfoPath) connector, err := fclib.GetConnectorFromFile(fc.connectorInfoPath) if err != nil { - if os.IsNotExist(err) { - klog.Warning(errors.Wrap(err, "assuming that FC connection was already closed")) - return &csi.NodeUnpublishVolumeResponse{}, nil + if errors.Is(err, fs.ErrNotExist) { + klog.ErrorS(err, "assuming that FC connection was already closed") + return nil + } else { + return err } - return nil, status.Error(codes.Internal, err.Error()) } - klog.Infof("connector.OSPathName (%s)", connector.OSPathName) + klog.InfoS("connector.OSPathName", "connector.OSPathName", connector.OSPathName) if IsVolumeInUse(connector.OSPathName) { - klog.Info("volume is still in use on the node, thus it will not be detached") - return &csi.NodeUnpublishVolumeResponse{}, nil + klog.InfoS("volume is still in use on the node, thus it will not be detached") + return nil } _, err = os.Stat(connector.OSPathName) - if err != nil && os.IsNotExist(err) { - klog.Warningf("assuming that volume is already disconnected: %s", err) - return &csi.NodeUnpublishVolumeResponse{}, nil + if err != nil && errors.Is(err, fs.ErrNotExist) { + klog.ErrorS(err, "assuming that volume is already disconnected") + return nil } wwn, _ := common.VolumeIdGetWwn(req.GetVolumeId()) - out, err := exec.Command("ls", "-l", fmt.Sprintf("/dev/disk/by-id/dm-name-3%s", wwn)).CombinedOutput() - klog.Infof("check for dm-name: ls -l %s, err = %v, out = \n%s", fmt.Sprintf("/dev/disk/by-id/dm-name-3%s", wwn), err, string(out)) + diskByIdPath := fmt.Sprintf("/dev/disk/by-id/dm-name-3%s", wwn) + out, err := exec.Command("ls", "-l", diskByIdPath).CombinedOutput() + klog.InfoS("check for dm-name", "command", fmt.Sprintf("ls -l %s, err = %v, out = \n%s", diskByIdPath, err, string(out))) if !connector.Multipath { // If we didn't discover the multipath device initially, double check that we didn't just miss it @@ -223,29 +103,36 @@ func (fc *fcStorage) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpub } discoveredMpathName, devices := fclib.FindDiskById(klog.FromContext(ctx), wwn, connector.IoHandler) if (discoveredMpathName != connector.OSPathName) && (len(devices) > 0) { - klog.V(0).InfoS("Found additional linked devices", "path", discoveredMpathName, "devices", devices) - klog.V(0).InfoS("Replacing original connector info prior to Detach", "originalMultipathDevice", connector.OSPathName, - "discoveredMultipathDevice", discoveredMpathName, "originalLinkedDevices", connector.OSDevicePaths, "discoveredLinkedDevices", devices) + klog.V(0).InfoS("Found additional linked devices", "discoveredMpathName", discoveredMpathName, "devices", devices) + klog.V(0).InfoS("Replacing original connector info prior to Detach", + "originalDevice", connector.OSPathName, "newDevice", discoveredMpathName, + "originalDevicePaths", connector.OSDevicePaths, "newDevicePaths", devices) connector.OSPathName = discoveredMpathName connector.OSDevicePaths = devices connector.Multipath = true } } - klog.Info("DisconnectVolume, detaching device") + klog.InfoS("DisconnectVolume, detaching device") err = fclib.Detach(ctx, connector.OSPathName, connector.IoHandler) - if err != nil { - return nil, err + klog.ErrorS(err, "error detaching FC connection") + return err } - klog.Infof("deleting FC connection info file %s", fc.connectorInfoPath) + klog.InfoS("deleting FC connection info file", "fc.connectorInfoPath", fc.connectorInfoPath) os.Remove(fc.connectorInfoPath) + return nil +} - GlobalRemovedDevicesMap[connector.VolumeWWN] = time.Now() +// NodePublishVolume mounts the volume mounted to the staging path to the target path +func (fc *fcStorage) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "FC specific NodePublishVolume not implemented") +} - klog.Info("successfully detached FC device") - return &csi.NodeUnpublishVolumeResponse{}, nil +// NodeUnpublishVolume unmounts the volume from the target path +func (fc *fcStorage) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "FC specific NodeUnpublishVolume not implemented") } // NodeGetVolumeStats return info about a given volume @@ -285,13 +172,14 @@ func (fc *fcStorage) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVo klog.V(2).Info("device is NOT using multipath") } - klog.Infof("expanding filesystem using resize2fs on device %s", connector.OSPathName) - output, err := exec.Command("resize2fs", connector.OSPathName).CombinedOutput() - if err != nil { - klog.V(2).Info("could not resize filesystem: %v", output) - return nil, fmt.Errorf("could not resize filesystem: %v", output) + if req.GetVolumeCapability().GetMount() != nil { + klog.Infof("expanding filesystem using resize2fs on device %s", connector.OSPathName) + output, err := exec.Command("resize2fs", connector.OSPathName).CombinedOutput() + if err != nil { + klog.V(2).InfoS("could not resize filesystem", "resize2fs output", output) + return nil, fmt.Errorf("could not resize filesystem: %v", output) + } } - return &csi.NodeExpandVolumeResponse{}, nil } diff --git a/pkg/storage/iscsiNode.go b/pkg/storage/iscsiNode.go index 3faf386b..a6a6222e 100644 --- a/pkg/storage/iscsiNode.go +++ b/pkg/storage/iscsiNode.go @@ -31,7 +31,6 @@ import ( iscsilib "github.com/Seagate/csi-lib-iscsi/iscsi" "github.com/Seagate/seagate-exos-x-csi/pkg/common" "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/pkg/errors" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "k8s.io/klog/v2" @@ -59,39 +58,20 @@ func (iscsi *iscsiStorage) NodeUnstageVolume(ctx context.Context, req *csi.NodeU return nil, status.Error(codes.Unimplemented, "NodeUnstageVolume is not implemented") } -// NodePublishVolume mounts the volume mounted to the staging path to the target path -func (iscsi *iscsiStorage) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { - if len(req.GetVolumeId()) == 0 { - return nil, status.Error(codes.InvalidArgument, "cannot publish volume with empty id") - } - if len(req.GetTargetPath()) == 0 { - return nil, status.Error(codes.InvalidArgument, "cannot publish volume at an empty path") - } - if req.GetVolumeCapability() == nil { - return nil, status.Error(codes.InvalidArgument, "cannot publish volume without capabilities") - } - - volumeName, _ := common.VolumeIdGetName(req.GetVolumeId()) +func (iscsi *iscsiStorage) AttachStorage(ctx context.Context, req *csi.NodePublishVolumeRequest) (string, error) { wwn, _ := common.VolumeIdGetWwn(req.GetVolumeId()) - - // Ensure that NodePublishVolume is only called once per volume - AddGatekeeper(volumeName) - defer RemoveGatekeeper(volumeName) - - klog.V(1).Infof("[START] publishing volume (%s) wwn (%s) target (%s)", volumeName, wwn, req.GetTargetPath()) - iqn := req.GetVolumeContext()["iqn"] portals := strings.Split(req.GetVolumeContext()["portals"], ",") - klog.Infof("iSCSI iqn: %s, portals: %v", iqn, portals) + klog.InfoS("iSCSI connection info:", "iqn", iqn, "portals", portals) lun, _ := strconv.ParseInt(req.GetPublishContext()["lun"], 10, 32) - klog.Infof("lun-%d, LUN: %d", lun, lun) + klog.InfoS("LUN:", "lun", lun) - klog.Info("initiating ISCSI connection...") + klog.InfoS("initiating ISCSI connection...") targets := make([]iscsilib.TargetInfo, 0) for _, portal := range portals { if portal != "" { - klog.V(1).Infof("-- add iqn (%v) portal (%v)", iqn, portal) + klog.V(1).InfoS("-- add iqn and portal targets", "iqn", iqn, "portal", portal) targets = append(targets, iscsilib.TargetInfo{ Iqn: iqn, Portal: portal, @@ -99,10 +79,10 @@ func (iscsi *iscsiStorage) NodePublishVolume(ctx context.Context, req *csi.NodeP // test and produce a warning if path already exists before iscsi login devicePath := fmt.Sprintf("/dev/disk/by-path/ip-%s:3260-iscsi-%s-lun-%d", portal, iqn, lun) _, err := os.Stat(devicePath) - klog.V(4).Infof("[TEST] os stat device: exist %v device %v", !os.IsNotExist(err), devicePath) + klog.V(4).InfoS("[TEST] os stat device:", "exist", !os.IsNotExist(err), "device", devicePath) if !os.IsNotExist(err) { _, err := os.Stat(devicePath) - klog.V(4).Infof("WARNING: device exists (%v) before iscsi login, os.Stat err=%v", devicePath, err) + klog.V(4).InfoS("WARNING: device exists before iscsi login:", "devicePath", devicePath, "os.Stat error", err) } } } @@ -130,7 +110,7 @@ func (iscsi *iscsiStorage) NodePublishVolume(ctx context.Context, req *csi.NodeP } klog.V(4).InfoS("iscsi connector setup", "AuthType", authType, "Targets", targets, "Lun", lun) - connector := iscsilib.Connector{ + connector := &iscsilib.Connector{ AuthType: authType, Targets: targets, Lun: int32(lun), @@ -141,29 +121,29 @@ func (iscsi *iscsiStorage) NodePublishVolume(ctx context.Context, req *csi.NodeP RetryCount: 20, } - path, err := iscsilib.Connect(&connector) + path, err := iscsilib.Connect(connector) if err != nil { - return nil, status.Error(codes.Unavailable, err.Error()) + return "", err } - klog.Infof("attached device at %s", path) + klog.InfoS("attached device:", "path", path) exists := true out, err := exec.Command("ls", "-l", fmt.Sprintf("/dev/disk/by-id/dm-name-3%s", wwn)).CombinedOutput() - klog.V(1).Infof("ls -l %s, err = %v, out = \n%s", fmt.Sprintf("/dev/disk/by-id/dm-name-3%s", wwn), err, string(out)) + klog.V(1).InfoS("ls command output", "command", fmt.Sprintf("ls -l /dev/disk/by-id/dm-name-3%s", wwn), "err", err, "out", out) if err != nil { exists = false } // wait here until the dm-name exists, for debugging - if exists == false { + if !exists { attempts := 1 for attempts < (maxDmnameAttempts + 1) { // Force a reload of all existing multipath maps output, err := exec.Command("multipath", "-r").CombinedOutput() - klog.V(4).Infof("## (publish) multipath -r: err=%v, output=\n%v", output, err) + klog.V(4).InfoS("## (publish) multipath -r output", "err", err, "output", output) out, err := exec.Command("ls", "-l", fmt.Sprintf("/dev/disk/by-id/dm-name-3%s", wwn)).CombinedOutput() - klog.V(1).Infof("[%d] check for dm-name exists: ls -l %s, err = %v, out = \n%s", attempts, fmt.Sprintf("/dev/disk/by-id/dm-name-3%s", wwn), err, string(out)) + klog.V(1).InfoS("check for dm-name exists", "attempt", attempts, "command", fmt.Sprintf("ls -l /dev/disk/by-id/dm-name-3%s", wwn), "err", err, "out", out) if err == nil { exists = true break @@ -172,126 +152,43 @@ func (iscsi *iscsiStorage) NodePublishVolume(ctx context.Context, req *csi.NodeP attempts++ } } - - fsType := GetFsType(req) - err = EnsureFsType(fsType, path) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - - corrupted := false - if err = CheckFs(path, fsType, "Publish"); err != nil { - corrupted = true - } - - if connector.Multipath { - klog.Infof("device is using multipath, device=%v, wwn=%v, exists=%v, corrupted=%v", path, wwn, exists, corrupted) - } else { - klog.Infof("device is NOT using multipath, device=%v, wwn=%v, exists=%v, corrupted=%v", path, wwn, exists, corrupted) - } - - if corrupted { - klog.Infof("device corruption (publish), device=%v, volume=%s, multipath=%v, wwn=%v, exists=%v, corrupted=%v", connector.DevicePath, volumeName, connector.Multipath, wwn, exists, corrupted) - DebugCorruption("$$", path) - return nil, status.Errorf(codes.DataLoss, "(publish) filesystem (%v) seems to be corrupted: %v", path, err) - } - - out, err = exec.Command("findmnt", "--output", "TARGET", "--noheadings", path).Output() - mountpoints := strings.Split(strings.Trim(string(out), "\n"), "\n") - if err != nil || len(mountpoints) == 0 { - klog.V(1).Infof("mount -t %s %s %s", fsType, path, req.GetTargetPath()) - os.Mkdir(req.GetTargetPath(), 00755) - if _, err = os.Stat(path); errors.Is(err, os.ErrNotExist) { - klog.Infof("targetpath does not exist:%s", req.GetTargetPath()) - } - out, err = exec.Command("mount", "-t", fsType, path, req.GetTargetPath()).CombinedOutput() - if err != nil { - return nil, status.Error(codes.Internal, string(out)) - } - } else if len(mountpoints) == 1 { - if mountpoints[0] == req.GetTargetPath() { - klog.Infof("volume %s already mounted", req.GetTargetPath()) - } else { - errStr := fmt.Sprintf("device has already been mounted somewhere else (%s instead of %s), please unmount first", mountpoints[0], req.GetTargetPath()) - return nil, status.Error(codes.Internal, errStr) - } - } else if len(mountpoints) > 1 { - return nil, errors.New("device has already been mounted in several locations, please unmount first") + if _, err := os.Stat(iscsi.connectorInfoPath); err == nil { + klog.InfoS("iscsi connection file already exists", "connectorInfoPath", iscsi.connectorInfoPath) } - klog.Infof("saving ISCSI connection info in %s", iscsi.connectorInfoPath) + klog.InfoS("saving ISCSI connection info", "connectorInfoPath", iscsi.connectorInfoPath) if _, err := os.Stat(iscsi.connectorInfoPath); err == nil { - klog.Warningf("iscsi connection file already exists: %s", iscsi.connectorInfoPath) + klog.InfoS("iscsi connection file already exists", "connectorInfoPath", iscsi.connectorInfoPath) } - err = iscsilib.PersistConnector(&connector, iscsi.connectorInfoPath) + err = iscsilib.PersistConnector(connector, iscsi.connectorInfoPath) if err != nil { - return nil, status.Error(codes.Internal, err.Error()) + return "", err } - klog.Infof("successfully mounted volume at %s", req.GetTargetPath()) - return &csi.NodePublishVolumeResponse{}, nil + return path, nil } -// NodeUnpublishVolume unmounts the volume from the target path -func (iscsi *iscsiStorage) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { - if len(req.GetVolumeId()) == 0 { - return nil, status.Error(codes.InvalidArgument, "cannot unpublish volume with an empty volume id") - } - if len(req.GetTargetPath()) == 0 { - return nil, status.Error(codes.InvalidArgument, "cannot unpublish volume with an empty target path") - } - - volumeName, _ := common.VolumeIdGetName(req.GetVolumeId()) - - // Ensure that NodeUnpublishVolume is only called once per volume - AddGatekeeper(volumeName) - defer RemoveGatekeeper(volumeName) - - klog.Infof("[START] unpublishing volume (%s) at target path %s", volumeName, req.GetTargetPath()) - - _, err := os.Stat(req.GetTargetPath()) - if err == nil { - klog.Infof("unmounting volume at %s", req.GetTargetPath()) - klog.V(4).Infof("command: %s %s", "mountpoint", req.GetTargetPath()) - out, err := exec.Command("mountpoint", req.GetTargetPath()).CombinedOutput() - if err == nil { - klog.V(4).Infof("command: %s %s", "umount -l", req.GetTargetPath()) - out, err := exec.Command("umount", "-l", req.GetTargetPath()).CombinedOutput() - if err != nil { - return nil, status.Error(codes.Internal, string(out)) - } - } else { - klog.Warningf("assuming that volume is already unmounted: %s", out) - } - - err = os.Remove(req.GetTargetPath()) - if err != nil && !os.IsNotExist(err) { - return nil, status.Error(codes.Internal, err.Error()) - } - } else { - klog.Warningf("assuming that volume is already unmounted: %v", err) - } - +func (iscsi *iscsiStorage) DetachStorage(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) error { klog.Infof("loading ISCSI connection info from %s", iscsi.connectorInfoPath) connector, err := iscsilib.GetConnectorFromFile(iscsi.connectorInfoPath) if err != nil { if os.IsNotExist(err) { - klog.Warning(errors.Wrap(err, "assuming that ISCSI connection is already closed")) - return &csi.NodeUnpublishVolumeResponse{}, nil + klog.InfoS("assuming that ISCSI connection is already closed") + return nil } - return nil, status.Error(codes.Internal, err.Error()) + return status.Error(codes.Internal, err.Error()) } - klog.Infof("connector.DevicePath (%s)", connector.DevicePath) + klog.InfoS("connector.DevicePath", "connector.DevicePath", connector.DevicePath) if IsVolumeInUse(connector.DevicePath) { klog.Info("volume is still in use on the node, thus it will not be detached") - return &csi.NodeUnpublishVolumeResponse{}, nil + return nil } _, err = os.Stat(connector.DevicePath) if err != nil && os.IsNotExist(err) { - klog.Warningf("assuming that volume is already disconnected: %s", err) - return &csi.NodeUnpublishVolumeResponse{}, nil + klog.InfoS("connector.devicePath does not exist, assuming that volume is already disconnected") + return nil } wwn, _ := common.VolumeIdGetWwn(req.GetVolumeId()) @@ -301,14 +198,21 @@ func (iscsi *iscsiStorage) NodeUnpublishVolume(ctx context.Context, req *csi.Nod klog.Info("DisconnectVolume, detaching ISCSI device") err = iscsilib.DisconnectVolume(*connector) if err != nil { - return nil, err + return err } klog.Infof("deleting ISCSI connection info file %s", iscsi.connectorInfoPath) os.Remove(iscsi.connectorInfoPath) + return nil +} + +func (iscsi *iscsiStorage) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "iSCSI specific NodePublishVolume not implemented") +} - klog.Info("successfully detached ISCSI device") - return &csi.NodeUnpublishVolumeResponse{}, nil +// NodeUnpublishVolume unmounts the volume from the target path +func (iscsi *iscsiStorage) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "iSCSI specific NodeUnpublishVolume not implemented") } // NodeGetVolumeStats return info about a given volume @@ -348,11 +252,13 @@ func (iscsi *iscsiStorage) NodeExpandVolume(ctx context.Context, req *csi.NodeEx klog.V(2).Info("device is NOT using multipath") } - klog.Infof("expanding filesystem using resize2fs on device %s", connector.DevicePath) - output, err := exec.Command("resize2fs", connector.DevicePath).CombinedOutput() - if err != nil { - klog.V(2).Info("could not resize filesystem: %v", output) - return nil, fmt.Errorf("could not resize filesystem: %v", output) + if req.GetVolumeCapability().GetMount() != nil { + klog.Infof("expanding filesystem using resize2fs on device %s", connector.DevicePath) + output, err := exec.Command("resize2fs", connector.DevicePath).CombinedOutput() + if err != nil { + klog.V(2).InfoS("could not resize filesystem", "resize2fs output", output) + return nil, fmt.Errorf("could not resize filesystem: %v", output) + } } return &csi.NodeExpandVolumeResponse{}, nil diff --git a/pkg/storage/sasNode.go b/pkg/storage/sasNode.go index 86388781..3a53c855 100644 --- a/pkg/storage/sasNode.go +++ b/pkg/storage/sasNode.go @@ -22,12 +22,12 @@ import ( "bufio" "context" "fmt" + "io/fs" "os" "os/exec" "path/filepath" "strconv" "strings" - "time" saslib "github.com/Seagate/csi-lib-sas/sas" "github.com/Seagate/seagate-exos-x-csi/pkg/common" @@ -120,178 +120,60 @@ func GetSASInitiators() ([]string, error) { return specifiedSASAddrs, nil } -// NodePublishVolume mounts the volume mounted to the staging path to the target path -func (sas *sasStorage) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { - if len(req.GetVolumeId()) == 0 { - return nil, status.Error(codes.InvalidArgument, "cannot publish volume with empty id") - } - if len(req.GetTargetPath()) == 0 { - return nil, status.Error(codes.InvalidArgument, "cannot publish volume at an empty path") - } - if req.GetVolumeCapability() == nil { - return nil, status.Error(codes.InvalidArgument, "cannot publish volume without capabilities") - } - - volumeName, _ := common.VolumeIdGetName(req.GetVolumeId()) +func (sas *sasStorage) AttachStorage(ctx context.Context, req *csi.NodePublishVolumeRequest) (string, error) { + klog.InfoS("initiating SAS connection...") wwn, _ := common.VolumeIdGetWwn(req.GetVolumeId()) - lun := req.GetPublishContext()["lun"] - - // Ensure that NodePublishVolume is only called once per volume - AddGatekeeper(volumeName) - defer RemoveGatekeeper(volumeName) - - klog.V(1).Infof("[START] publish volume (%s) wwn (%s) target (%s) lun (%s)", volumeName, wwn, req.GetTargetPath(), lun) - - // Initiate SAS attachment - klog.Info("initiating SAS connection...") connector := saslib.Connector{VolumeWWN: wwn} path, err := saslib.Attach(ctx, &connector, &saslib.OSioHandler{}) if err != nil { - return nil, status.Error(codes.Unavailable, err.Error()) - } - klog.Infof("attached device at %s", path) - - // if current wwn has been published before, remove it from our list of previously unpublished wwns - delete(GlobalRemovedDevicesMap, wwn) - // check if previously unpublished devices were rediscovered by the scsi subsystem during Attach - checkPreviouslyRemovedDevices(ctx) - - fsType := GetFsType(req) - err = EnsureFsType(fsType, path) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - - corrupted := false - if err = CheckFs(path, fsType, "Publish"); err != nil { - corrupted = true - } - - if connector.Multipath { - klog.Infof("device is using multipath, device=%v, wwn=%v, corrupted=%v", path, wwn, corrupted) - } else { - klog.Infof("device is NOT using multipath, device=%v, wwn=%v, corrupted=%v", path, wwn, corrupted) - } - - if corrupted { - klog.Infof("device corruption (publish), device=%v, volume=%s, multipath=%v, wwn=%v, corrupted=%v", connector.OSPathName, volumeName, connector.Multipath, wwn, corrupted) - DebugCorruption("$$", path) - return nil, status.Errorf(codes.DataLoss, "(publish) filesystem (%v) seems to be corrupted: %v", path, err) - } - - out, err := exec.Command("findmnt", "--output", "TARGET", "--noheadings", path).Output() - mountpoints := strings.Split(strings.Trim(string(out), "\n"), "\n") - if err != nil || len(mountpoints) == 0 { - klog.V(1).Infof("mount -t %s %s %s", fsType, path, req.GetTargetPath()) - os.Mkdir(req.GetTargetPath(), 00755) - if _, err = os.Stat(path); errors.Is(err, os.ErrNotExist) { - klog.Infof("targetpath does not exist:%s", req.GetTargetPath()) - } - out, err = exec.Command("mount", "-t", fsType, path, req.GetTargetPath()).CombinedOutput() - if err != nil { - return nil, status.Error(codes.Internal, string(out)) - } - } else if len(mountpoints) == 1 { - if mountpoints[0] == req.GetTargetPath() { - klog.Infof("volume %s already mounted", req.GetTargetPath()) - } else { - errStr := fmt.Sprintf("device has already been mounted somewhere else (%s instead of %s), please unmount first", mountpoints[0], req.GetTargetPath()) - return nil, status.Error(codes.Internal, errStr) - } - } else if len(mountpoints) > 1 { - return nil, errors.New("device has already been mounted in several locations, please unmount first") - } - - klog.Infof("saving SAS connection info in %s", sas.connectorInfoPath) - if _, err := os.Stat(sas.connectorInfoPath); err == nil { - klog.Warningf("sas connection file already exists: %s", sas.connectorInfoPath) + return path, status.Error(codes.Unavailable, err.Error()) } + klog.InfoS("attached device", "path", path) err = connector.Persist(ctx, sas.connectorInfoPath) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - - klog.Infof("successfully mounted volume at %s", req.GetTargetPath()) - return &csi.NodePublishVolumeResponse{}, nil + return path, err } -// NodeUnpublishVolume unmounts the volume from the target path -func (sas *sasStorage) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { - if len(req.GetVolumeId()) == 0 { - return nil, status.Error(codes.InvalidArgument, "cannot unpublish volume with an empty volume id") - } - if len(req.GetTargetPath()) == 0 { - return nil, status.Error(codes.InvalidArgument, "cannot unpublish volume with an empty target path") - } - - volumeName, _ := common.VolumeIdGetName(req.GetVolumeId()) - - // Ensure that NodeUnpublishVolume is only called once per volume - AddGatekeeper(volumeName) - defer RemoveGatekeeper(volumeName) - - klog.Infof("[START] unpublishing volume (%s) at target path %s", volumeName, req.GetTargetPath()) - - _, err := os.Stat(req.GetTargetPath()) - if err == nil { - klog.Infof("unmounting volume at %s", req.GetTargetPath()) - klog.V(4).Infof("command: %s %s", "mountpoint", req.GetTargetPath()) - out, err := exec.Command("mountpoint", req.GetTargetPath()).CombinedOutput() - if err == nil { - klog.V(4).Infof("command: %s %s", "umount -l", req.GetTargetPath()) - out, err := exec.Command("umount", "-l", req.GetTargetPath()).CombinedOutput() - if err != nil { - return nil, status.Error(codes.Internal, string(out)) - } - } else { - klog.Warningf("assuming that volume is already unmounted: %s", out) - } - - err = os.Remove(req.GetTargetPath()) - if err != nil && !os.IsNotExist(err) { - return nil, status.Error(codes.Internal, err.Error()) - } - } else { - klog.Warningf("assuming that volume is already unmounted: %v", err) - } - - klog.Infof("loading SAS connection info from %s", sas.connectorInfoPath) +func (sas *sasStorage) DetachStorage(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) error { + klog.InfoS("loading SAS connection info from file", "connectorInfoPath", sas.connectorInfoPath) connector, err := saslib.GetConnectorFromFile(sas.connectorInfoPath) if err != nil { - if os.IsNotExist(err) { - klog.Warning(errors.Wrap(err, "assuming that SAS connection was already closed")) - return &csi.NodeUnpublishVolumeResponse{}, nil + if errors.Is(err, fs.ErrNotExist) { + klog.ErrorS(err, "assuming that SAS connection was already closed") + return nil } - return nil, status.Error(codes.Internal, err.Error()) + return status.Error(codes.Internal, err.Error()) } - klog.Infof("connector.OSPathName (%s)", connector.OSPathName) + klog.InfoS("connector.OSPathName", "connector.OSPathName", connector.OSPathName) if IsVolumeInUse(connector.OSPathName) { - klog.Info("volume is still in use on the node, thus it will not be detached") - return &csi.NodeUnpublishVolumeResponse{}, nil + klog.InfoS("volume is still in use on the node, thus it will not be detached") + return nil } _, err = os.Stat(connector.OSPathName) - if err != nil && os.IsNotExist(err) { - klog.Warningf("assuming that volume is already disconnected: %s", err) - return &csi.NodeUnpublishVolumeResponse{}, nil + if err != nil && errors.Is(err, fs.ErrNotExist) { + klog.ErrorS(err, "assuming that volume is already disconnected") + return nil } wwn, _ := common.VolumeIdGetWwn(req.GetVolumeId()) - out, err := exec.Command("ls", "-l", fmt.Sprintf("/dev/disk/by-id/dm-name-3%s", wwn)).CombinedOutput() - klog.Infof("check for dm-name: ls -l %s, err = %v, out = \n%s", fmt.Sprintf("/dev/disk/by-id/dm-name-3%s", wwn), err, string(out)) + diskByIdPath := fmt.Sprintf("/dev/disk/by-id/dm-name-3%s", wwn) + out, err := exec.Command("ls", "-l", diskByIdPath).CombinedOutput() + klog.InfoS("check for dm-name", "command", fmt.Sprintf("ls -l %s, err = %v, out = \n%s", diskByIdPath, err, string(out))) if !connector.Multipath { // If we didn't discover the multipath device initially, double check that we didn't just miss it // Detach the discovered devices if they are found - klog.V(3).Info("Device saved as non-multipath. Searching for additional devices before Detach") + klog.V(3).InfoS("Device saved as non-multipath. Searching for additional devices before Detach") if connector.IoHandler == nil { connector.IoHandler = &saslib.OSioHandler{} } discoveredMpathName, devices := saslib.FindDiskById(klog.FromContext(ctx), wwn, connector.IoHandler) if (discoveredMpathName != connector.OSPathName) && (len(devices) > 0) { - klog.V(0).Infof("Found additional linked devices: %s, %v", discoveredMpathName, devices) - klog.V(0).Infof("Replacing original connector info prior to Detach, device: %s=>%s, linked device paths: %v=>%v", connector.OSPathName, discoveredMpathName, connector.OSDevicePaths, devices) + klog.V(0).InfoS("Found additional linked devices", "discoveredMpathName", discoveredMpathName, "devices", devices) + klog.V(0).InfoS("Replacing original connector info prior to Detach", + "originalDevice", connector.OSPathName, "newDevice", discoveredMpathName, + "originalDevicePaths", connector.OSDevicePaths, "newDevicePaths", devices) connector.OSPathName = discoveredMpathName connector.OSDevicePaths = devices connector.Multipath = true @@ -300,18 +182,22 @@ func (sas *sasStorage) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnp klog.Info("DisconnectVolume, detaching SAS device") err = saslib.Detach(ctx, connector.OSPathName, connector.IoHandler) - if err != nil { - return nil, err + return err } - klog.Infof("deleting SAS connection info file %s", sas.connectorInfoPath) + klog.InfoS("deleting SAS connection info file", "sas.connectorInfoPath", sas.connectorInfoPath) os.Remove(sas.connectorInfoPath) + return nil +} - GlobalRemovedDevicesMap[connector.VolumeWWN] = time.Now() +func (sas *sasStorage) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "SAS specific NodePublishVolume not implemented") +} - klog.Info("successfully detached SAS device") - return &csi.NodeUnpublishVolumeResponse{}, nil +// NodeUnpublishVolume unmounts the volume from the target path +func (sas *sasStorage) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "SAS specific NodeUnpublishVolume not implemented") } func checkPreviouslyRemovedDevices(ctx context.Context) error { @@ -365,11 +251,13 @@ func (sas *sasStorage) NodeExpandVolume(ctx context.Context, req *csi.NodeExpand klog.V(2).Info("device is NOT using multipath") } - klog.Infof("expanding filesystem using resize2fs on device %s", connector.OSPathName) - output, err := exec.Command("resize2fs", connector.OSPathName).CombinedOutput() - if err != nil { - klog.V(2).Info("could not resize filesystem: %v", output) - return nil, fmt.Errorf("could not resize filesystem: %v", output) + if req.GetVolumeCapability().GetMount() != nil { + klog.Infof("expanding filesystem using resize2fs on device %s", connector.OSPathName) + output, err := exec.Command("resize2fs", connector.OSPathName).CombinedOutput() + if err != nil { + klog.V(2).InfoS("could not resize filesystem", "resize2fs output", output) + return nil, fmt.Errorf("could not resize filesystem: %v", output) + } } return &csi.NodeExpandVolumeResponse{}, nil diff --git a/pkg/storage/storageService.go b/pkg/storage/storageService.go index 98ffff03..4fb5c9d8 100644 --- a/pkg/storage/storageService.go +++ b/pkg/storage/storageService.go @@ -21,6 +21,7 @@ package storage import ( "context" "fmt" + "os" "os/exec" "regexp" "strings" @@ -29,6 +30,8 @@ import ( "github.com/Seagate/seagate-exos-x-csi/pkg/common" "github.com/container-storage-interface/spec/lib/go/csi" "github.com/pkg/errors" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "k8s.io/klog/v2" ) @@ -39,6 +42,8 @@ const ( type StorageOperations interface { csi.NodeServer + AttachStorage(ctx context.Context, req *csi.NodePublishVolumeRequest) (string, error) + DetachStorage(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) error } type commonService struct { @@ -224,6 +229,87 @@ func EnsureFsType(fsType string, disk string) error { return nil } +func MountFilesystem(req *csi.NodePublishVolumeRequest, path string) error { + fsType := GetFsType(req) + err := EnsureFsType(fsType, path) + if err != nil { + return status.Error(codes.Internal, err.Error()) + } + + if err = CheckFs(path, fsType, "Publish"); err != nil { + return err + } + + out, err := exec.Command("findmnt", "--output", "TARGET", "--noheadings", path).Output() + mountpoints := strings.Split(strings.Trim(string(out), "\n"), "\n") + if err != nil || len(mountpoints) == 0 { + klog.V(1).InfoS("mount", "command", fmt.Sprintf("mount -t %s %s %s", fsType, path, req.GetTargetPath())) + os.Mkdir(req.GetTargetPath(), 00755) + if _, err = os.Stat(path); errors.Is(err, os.ErrNotExist) { + klog.InfoS("targetpath does not exist", "targetPath", req.GetTargetPath()) + } + out, err = exec.Command("mount", "-t", fsType, path, req.GetTargetPath()).CombinedOutput() + if err != nil { + return status.Error(codes.Internal, string(out)) + } + } else if len(mountpoints) == 1 { + if mountpoints[0] == req.GetTargetPath() { + klog.InfoS("volume already mounted", "targetPath", req.GetTargetPath()) + } else { + errStr := fmt.Sprintf("device has already been mounted somewhere else (%s instead of %s), please unmount first", mountpoints[0], req.GetTargetPath()) + return status.Error(codes.Internal, errStr) + } + } else if len(mountpoints) > 1 { + return errors.New("device has already been mounted in several locations, please unmount first") + } + + klog.InfoS("successfully mounted volume", "targetPath", req.GetTargetPath()) + return nil +} + +func MountDevice(req *csi.NodePublishVolumeRequest, path string) error { + deviceFile, err := os.Create(req.GetTargetPath()) + if err != nil { + klog.ErrorS(err, "could not create file", "TargetPath", req.GetTargetPath()) + return err + } + deviceFile.Chmod(00755) + deviceFile.Close() + out, err := exec.Command("mount", "-o", "bind", path, req.GetTargetPath()).CombinedOutput() + if err != nil { + return status.Error(codes.Internal, string(out)) + } + return nil +} + +// Unmount a given path, usually req.GetVolumePath() from NodeUnpublishVolume +// used for both block and filesystem mount types +func Unmount(path string) error { + _, err := os.Stat(path) + if err == nil { + klog.InfoS("unmounting volume", "path", path) + klog.V(4).InfoS("mountpoint command", "command", "mountpoint "+path) + out, err := exec.Command("mountpoint", path).CombinedOutput() + if err == nil { + klog.V(4).InfoS("umount command", "command", "umount -l "+path) + out, err := exec.Command("umount", "-l", path).CombinedOutput() + if err != nil { + return status.Error(codes.Internal, string(out)) + } + } else { + klog.ErrorS(err, "assuming that volume is already unmounted", "mountpoint_output", out) + } + + err = os.Remove(path) + if err != nil && !os.IsNotExist(err) { + return status.Error(codes.Internal, err.Error()) + } + } else { + klog.ErrorS(err, "assuming that volume is already unmounted") + } + return nil +} + // IsVolumeInUse: Use findmnt to determine if the device path is mounted or not. func IsVolumeInUse(devicePath string) bool { _, err := exec.Command("findmnt", devicePath).CombinedOutput()