Skip to content

Commit

Permalink
fix(storage/backend): adjust nvme to parent member
Browse files Browse the repository at this point in the history 
Signed-off-by: Artsiom Koltun <[email protected]>
  • Loading branch information
@artek-koltun @sandersms
artek-koltun authored and sandersms committed Jan 18, 2024
1 parent 1fc498e commit b9be140
Show file tree
Hide file tree
Showing 8 changed files with 161 additions and 120 deletions.
22 changes: 11 additions & 11 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,18 +92,18 @@ docker run --network=host --rm -it namely/grpc-cli call --json_input --json_outp
docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 GetNvmeNamespace "{name : '//storage.opiproject.org/subsystems/subsystem2/namespaces/namespace1'}"
docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 StatsNvmeNamespace "{name : '//storage.opiproject.org/subsystems/subsystem2/namespaces/namespace1'}"
docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 CreateNvmeRemoteController "{nvme_remote_controller : {multipath: 'NVME_MULTIPATH_MULTIPATH', tcp: {hdgst: false, ddgst: false}}, nvme_remote_controller_id: 'nvmetcp12'}"
docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 ListNvmeRemoteControllers "{parent : 'todo'}"
docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 GetNvmeRemoteController "{name: '//storage.opiproject.org/volumes/nvmetcp12'}"
docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 CreateNvmePath "{nvme_path : {controller_name_ref: '//storage.opiproject.org/volumes/nvmetcp12', traddr:'11.11.11.2', trtype:'NVME_TRANSPORT_TCP', fabrics:{subnqn:'nqn.2016-06.com.opi.spdk.target0', trsvcid:'4444', adrfam:'NVME_ADRFAM_IPV4', hostnqn:'nqn.2014-08.org.nvmexpress:uuid:feb98abe-d51f-40c8-b348-2753f3571d3c'}}, nvme_path_id: 'nvmetcp12path0'}"
docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 ListNvmeRemoteControllers "{}"
docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 GetNvmeRemoteController "{name: '//storage.opiproject.org/nvmeRemoteControllers/nvmetcp12'}"
docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 CreateNvmePath "{parent: '//storage.opiproject.org/nvmeRemoteControllers/nvmetcp12', nvme_path : {traddr:'11.11.11.2', trtype:'NVME_TRANSPORT_TCP', fabrics:{subnqn:'nqn.2016-06.com.opi.spdk.target0', trsvcid:'4444', adrfam:'NVME_ADRFAM_IPV4', hostnqn:'nqn.2014-08.org.nvmexpress:uuid:feb98abe-d51f-40c8-b348-2753f3571d3c'}}, nvme_path_id: 'nvmetcp12path0'}"
docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 CreateNvmeRemoteController "{nvme_remote_controller : {multipath: 'NVME_MULTIPATH_DISABLE'}, nvme_remote_controller_id: 'nvmepcie13'}"
docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 CreateNvmePath "{nvme_path : {controller_name_ref: '//storage.opiproject.org/volumes/nvmepcie13', traddr:'0000:01:00.0', trtype:'NVME_TRANSPORT_PCIE'}, nvme_path_id: 'nvmepcie13path0'}"

docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 ListNvmePaths "{parent : 'todo'}"
docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 DeleteNvmePath "{name: '//storage.opiproject.org/volumes/nvmepcie13path0'}"
docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 DeleteNvmeRemoteController "{name: '//storage.opiproject.org/volumes/nvmepcie13'}"
docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 GetNvmePath "{name: '//storage.opiproject.org/volumes/nvmetcp12path0'}"
docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 DeleteNvmePath "{name: '//storage.opiproject.org/volumes/nvmetcp12path0'}"
docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 DeleteNvmeRemoteController "{name: '//storage.opiproject.org/volumes/nvmetcp12'}"
docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 CreateNvmePath "{parent: '//storage.opiproject.org/nvmeRemoteControllers/nvmepcie13', nvme_path : {traddr:'0000:01:00.0', trtype:'NVME_TRANSPORT_PCIE'}, nvme_path_id: 'nvmepcie13path0'}"

docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 ListNvmePaths "{parent : '//storage.opiproject.org/nvmeRemoteControllers/nvmepcie13'}"
docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 DeleteNvmePath "{name: '//storage.opiproject.org/nvmeRemoteControllers/nvmepcie13/nvmePaths/nvmepcie13path0'}"
docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 DeleteNvmeRemoteController "{name: '//storage.opiproject.org/nvmeRemoteControllers/nvmepcie13'}"
docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 GetNvmePath "{name: '//storage.opiproject.org/nvmeRemoteControllers/nvmetcp12/nvmePaths/nvmetcp12path0'}"
docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 DeleteNvmePath "{name: '//storage.opiproject.org/nvmeRemoteControllers/nvmetcp12/nvmePaths/nvmetcp12path0'}"
docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 DeleteNvmeRemoteController "{name: '//storage.opiproject.org/nvmeRemoteControllers/nvmetcp12'}"
docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 DeleteNvmeNamespace "{name : '//storage.opiproject.org/subsystems/subsystem2/namespaces/namespace1'}"
docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 DeleteNvmeController "{name : '//storage.opiproject.org/subsystems/subsystem2/controllers/controller1'}"
docker run --network=host --rm -it namely/grpc-cli call --json_input --json_output 10.10.10.10:50051 DeleteNvmeSubsystem "{name : '//storage.opiproject.org/subsystems/subsystem2'}"
Expand Down
6 changes: 3 additions & 3 deletions pkg/backend/nvme_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ func (s *Server) CreateNvmeRemoteController(_ context.Context, in *pb.CreateNvme
log.Printf("client provided the ID of a resource %v, ignoring the name field %v", in.NvmeRemoteControllerId, in.NvmeRemoteController.Name)
resourceID = in.NvmeRemoteControllerId
}
in.NvmeRemoteController.Name = utils.ResourceIDToVolumeName(resourceID)
in.NvmeRemoteController.Name = utils.ResourceIDToRemoteControllerName(resourceID)
// idempotent API when called with same key, should return same object
volume, ok := s.Volumes.NvmeControllers[in.NvmeRemoteController.Name]
if ok {
Expand Down Expand Up @@ -92,15 +92,15 @@ func (s *Server) UpdateNvmeRemoteController(_ context.Context, in *pb.UpdateNvme
return nil, err
}
// fetch object from the database
volume, ok := s.Volumes.NvmeControllers[in.NvmeRemoteController.Name]
controller, ok := s.Volumes.NvmeControllers[in.NvmeRemoteController.Name]
if !ok {
if in.AllowMissing {
log.Printf("TODO: in case of AllowMissing, create a new resource, don;t return error")
}
err := status.Errorf(codes.NotFound, "unable to find key %s", in.NvmeRemoteController.Name)
return nil, err
}
resourceID := path.Base(volume.Name)
resourceID := utils.GetRemoteControllerIDFromNvmeRemoteName(controller.Name)
// update_mask = 2
if err := fieldmask.Validate(in.UpdateMask, in.NvmeRemoteController); err != nil {
return nil, err
Expand Down
81 changes: 48 additions & 33 deletions pkg/backend/nvme_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import (

var (
testNvmeCtrlID = "opi-nvme8"
testNvmeCtrlName = utils.ResourceIDToVolumeName(testNvmeCtrlID)
testNvmeCtrlName = utils.ResourceIDToRemoteControllerName(testNvmeCtrlID)
testNvmeCtrl = pb.NvmeRemoteController{
Tcp: &pb.TcpController{
Hdgst: false,
Expand Down Expand Up @@ -178,38 +178,46 @@ func TestBackEnd_ListNvmeRemoteControllers(t *testing.T) {
in: testNvmeCtrlID,
out: []*pb.NvmeRemoteController{
{
Name: utils.ResourceIDToVolumeName("OpiNvme12"),
Name: utils.ResourceIDToRemoteControllerName("OpiNvme12"),
},
{
Name: utils.ResourceIDToVolumeName("OpiNvme13"),
Name: utils.ResourceIDToRemoteControllerName("OpiNvme13"),
},
},
errCode: codes.OK,
errMsg: "",
size: 0,
token: "",
existingControllers: map[string]*pb.NvmeRemoteController{
utils.ResourceIDToVolumeName("OpiNvme12"): {Name: utils.ResourceIDToVolumeName("OpiNvme12")},
utils.ResourceIDToVolumeName("OpiNvme13"): {Name: utils.ResourceIDToVolumeName("OpiNvme13")},
utils.ResourceIDToRemoteControllerName("OpiNvme12"): {
Name: utils.ResourceIDToRemoteControllerName("OpiNvme12"),
},
utils.ResourceIDToRemoteControllerName("OpiNvme13"): {
Name: utils.ResourceIDToRemoteControllerName("OpiNvme13"),
},
},
},
"pagination overflow": {
in: testNvmeCtrlID,
out: []*pb.NvmeRemoteController{
{
Name: utils.ResourceIDToVolumeName("OpiNvme12"),
Name: utils.ResourceIDToRemoteControllerName("OpiNvme12"),
},
{
Name: utils.ResourceIDToVolumeName("OpiNvme13"),
Name: utils.ResourceIDToRemoteControllerName("OpiNvme13"),
},
},
errCode: codes.OK,
errMsg: "",
size: 1000,
token: "",
existingControllers: map[string]*pb.NvmeRemoteController{
utils.ResourceIDToVolumeName("OpiNvme12"): {Name: utils.ResourceIDToVolumeName("OpiNvme12")},
utils.ResourceIDToVolumeName("OpiNvme13"): {Name: utils.ResourceIDToVolumeName("OpiNvme13")},
utils.ResourceIDToRemoteControllerName("OpiNvme12"): {
Name: utils.ResourceIDToRemoteControllerName("OpiNvme12"),
},
utils.ResourceIDToRemoteControllerName("OpiNvme13"): {
Name: utils.ResourceIDToRemoteControllerName("OpiNvme13"),
},
},
},
"pagination negative": {
Expand All @@ -220,8 +228,12 @@ func TestBackEnd_ListNvmeRemoteControllers(t *testing.T) {
size: -10,
token: "",
existingControllers: map[string]*pb.NvmeRemoteController{
utils.ResourceIDToVolumeName("OpiNvme12"): {Name: utils.ResourceIDToVolumeName("OpiNvme12")},
utils.ResourceIDToVolumeName("OpiNvme13"): {Name: utils.ResourceIDToVolumeName("OpiNvme13")},
utils.ResourceIDToRemoteControllerName("OpiNvme12"): {
Name: utils.ResourceIDToRemoteControllerName("OpiNvme12"),
},
utils.ResourceIDToRemoteControllerName("OpiNvme13"): {
Name: utils.ResourceIDToRemoteControllerName("OpiNvme13"),
},
},
},
"pagination error": {
Expand All @@ -232,51 +244,54 @@ func TestBackEnd_ListNvmeRemoteControllers(t *testing.T) {
size: 0,
token: "unknown-pagination-token",
existingControllers: map[string]*pb.NvmeRemoteController{
utils.ResourceIDToVolumeName("OpiNvme12"): {Name: utils.ResourceIDToVolumeName("OpiNvme12")},
utils.ResourceIDToVolumeName("OpiNvme13"): {Name: utils.ResourceIDToVolumeName("OpiNvme13")},
utils.ResourceIDToRemoteControllerName("OpiNvme12"): {
Name: utils.ResourceIDToRemoteControllerName("OpiNvme12"),
},
utils.ResourceIDToRemoteControllerName("OpiNvme13"): {
Name: utils.ResourceIDToRemoteControllerName("OpiNvme13"),
},
},
},
"pagination": {
in: testNvmeCtrlID,
out: []*pb.NvmeRemoteController{
{
Name: utils.ResourceIDToVolumeName("OpiNvme12"),
Name: utils.ResourceIDToRemoteControllerName("OpiNvme12"),
},
},
errCode: codes.OK,
errMsg: "",
size: 1,
token: "",
existingControllers: map[string]*pb.NvmeRemoteController{
utils.ResourceIDToVolumeName("OpiNvme12"): {Name: utils.ResourceIDToVolumeName("OpiNvme12")},
utils.ResourceIDToVolumeName("OpiNvme13"): {Name: utils.ResourceIDToVolumeName("OpiNvme13")},
utils.ResourceIDToRemoteControllerName("OpiNvme12"): {
Name: utils.ResourceIDToRemoteControllerName("OpiNvme12"),
},
utils.ResourceIDToRemoteControllerName("OpiNvme13"): {
Name: utils.ResourceIDToRemoteControllerName("OpiNvme13"),
},
},
},
"pagination offset": {
in: testNvmeCtrlID,
out: []*pb.NvmeRemoteController{
{
Name: utils.ResourceIDToVolumeName("OpiNvme13"),
Name: utils.ResourceIDToRemoteControllerName("OpiNvme13"),
},
},
errCode: codes.OK,
errMsg: "",
size: 1,
token: "existing-pagination-token",
existingControllers: map[string]*pb.NvmeRemoteController{
utils.ResourceIDToVolumeName("OpiNvme12"): {Name: utils.ResourceIDToVolumeName("OpiNvme12")},
utils.ResourceIDToVolumeName("OpiNvme13"): {Name: utils.ResourceIDToVolumeName("OpiNvme13")},
utils.ResourceIDToRemoteControllerName("OpiNvme12"): {
Name: utils.ResourceIDToRemoteControllerName("OpiNvme12"),
},
utils.ResourceIDToRemoteControllerName("OpiNvme13"): {
Name: utils.ResourceIDToRemoteControllerName("OpiNvme13"),
},
},
},
"no required field": {
in: "",
out: []*pb.NvmeRemoteController{},
errCode: codes.Unknown,
errMsg: "missing required field: parent",
size: 0,
token: "",
existingControllers: map[string]*pb.NvmeRemoteController{},
},
}

// run tests
Expand All @@ -290,7 +305,7 @@ func TestBackEnd_ListNvmeRemoteControllers(t *testing.T) {
testEnv.opiSpdkServer.Volumes.NvmeControllers[k] = utils.ProtoClone(v)
}

request := &pb.ListNvmeRemoteControllersRequest{Parent: tt.in, PageSize: tt.size, PageToken: tt.token}
request := &pb.ListNvmeRemoteControllersRequest{PageSize: tt.size, PageToken: tt.token}
response, err := testEnv.client.ListNvmeRemoteControllers(testEnv.ctx, request)

if !utils.EqualProtoSlices(response.GetNvmeRemoteControllers(), tt.out) {
Expand Down Expand Up @@ -464,21 +479,21 @@ func TestBackEnd_DeleteNvmeRemoteController(t *testing.T) {
missing: false,
},
"valid request with unknown key": {
in: utils.ResourceIDToVolumeName("unknown-id"),
in: utils.ResourceIDToRemoteControllerName("unknown-id"),
out: nil,
errCode: codes.NotFound,
errMsg: fmt.Sprintf("unable to find key %v", utils.ResourceIDToVolumeName("unknown-id")),
errMsg: fmt.Sprintf("unable to find key %v", utils.ResourceIDToRemoteControllerName("unknown-id")),
missing: false,
},
"unknown key with missing allowed": {
in: utils.ResourceIDToVolumeName("unknown-id"),
in: utils.ResourceIDToRemoteControllerName("unknown-id"),
out: &emptypb.Empty{},
errCode: codes.OK,
errMsg: "",
missing: true,
},
"malformed name": {
in: utils.ResourceIDToVolumeName("-ABC-DEF"),
in: utils.ResourceIDToRemoteControllerName("-ABC-DEF"),
out: &emptypb.Empty{},
errCode: codes.Unknown,
errMsg: fmt.Sprintf("segment '%s': not a valid DNS name", "-ABC-DEF"),
Expand Down
29 changes: 18 additions & 11 deletions pkg/backend/nvme_path.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,17 +45,20 @@ func (s *Server) CreateNvmePath(ctx context.Context, in *pb.CreateNvmePathReques
log.Printf("client provided the ID of a resource %v, ignoring the name field %v", in.NvmePathId, in.NvmePath.Name)
resourceID = in.NvmePathId
}
in.NvmePath.Name = utils.ResourceIDToVolumeName(resourceID)
in.NvmePath.Name = utils.ResourceIDToNvmePathName(
utils.GetRemoteControllerIDFromNvmeRemoteName(in.Parent),
resourceID,
)

nvmePath, ok := s.Volumes.NvmePaths[in.NvmePath.Name]
if ok {
log.Printf("Already existing NvmePath with id %v", in.NvmePath.Name)
return nvmePath, nil
}

controller, ok := s.Volumes.NvmeControllers[in.NvmePath.ControllerNameRef]
controller, ok := s.Volumes.NvmeControllers[in.Parent]
if !ok {
err := status.Errorf(codes.NotFound, "unable to find NvmeRemoteController by key %s", in.NvmePath.ControllerNameRef)
err := status.Errorf(codes.NotFound, "unable to find NvmeRemoteController by key %s", in.Parent)
return nil, err
}

Expand Down Expand Up @@ -85,7 +88,7 @@ func (s *Server) CreateNvmePath(ctx context.Context, in *pb.CreateNvmePathReques
psk = keyFile
}
params := spdk.BdevNvmeAttachControllerParams{
Name: path.Base(controller.Name),
Name: utils.GetRemoteControllerIDFromNvmeRemoteName(controller.Name),
Trtype: s.opiTransportToSpdk(in.NvmePath.GetTrtype()),
Traddr: in.NvmePath.GetTraddr(),
Adrfam: utils.OpiAdressFamilyToSpdk(in.NvmePath.GetFabrics().GetAdrfam()),
Expand Down Expand Up @@ -123,14 +126,17 @@ func (s *Server) DeleteNvmePath(ctx context.Context, in *pb.DeleteNvmePathReques
err := status.Errorf(codes.NotFound, "unable to find key %s", in.Name)
return nil, err
}
controller, ok := s.Volumes.NvmeControllers[nvmePath.ControllerNameRef]
controllerName := utils.ResourceIDToRemoteControllerName(
utils.GetRemoteControllerIDFromNvmeRemoteName(in.Name),
)
controller, ok := s.Volumes.NvmeControllers[controllerName]
if !ok {
err := status.Errorf(codes.Internal, "unable to find NvmeRemoteController by key %s", nvmePath.ControllerNameRef)
err := status.Errorf(codes.Internal, "unable to find NvmeRemoteController by key %s", controllerName)
return nil, err
}

params := spdk.BdevNvmeDetachControllerParams{
Name: path.Base(controller.Name),
Name: utils.GetRemoteControllerIDFromNvmeRemoteName(controller.Name),
Trtype: s.opiTransportToSpdk(nvmePath.GetTrtype()),
Traddr: nvmePath.GetTraddr(),
Adrfam: utils.OpiAdressFamilyToSpdk(nvmePath.GetFabrics().GetAdrfam()),
Expand All @@ -145,7 +151,7 @@ func (s *Server) DeleteNvmePath(ctx context.Context, in *pb.DeleteNvmePathReques
}
log.Printf("Received from SPDK: %v", result)
if !result {
msg := fmt.Sprintf("Could not delete Nvme Path: %s", path.Base(in.Name))
msg := fmt.Sprintf("Could not delete Nvme Path: %s", in.Name)
return nil, status.Errorf(codes.InvalidArgument, msg)
}

Expand All @@ -161,15 +167,15 @@ func (s *Server) UpdateNvmePath(_ context.Context, in *pb.UpdateNvmePathRequest)
return nil, err
}
// fetch object from the database
volume, ok := s.Volumes.NvmePaths[in.NvmePath.Name]
nvmePath, ok := s.Volumes.NvmePaths[in.NvmePath.Name]
if !ok {
if in.AllowMissing {
log.Printf("TODO: in case of AllowMissing, create a new resource, don;t return error")
}
err := status.Errorf(codes.NotFound, "unable to find key %s", in.NvmePath.Name)
return nil, err
}
resourceID := path.Base(volume.Name)
resourceID := path.Base(nvmePath.Name)
// update_mask = 2
if err := fieldmask.Validate(in.UpdateMask, in.NvmePath); err != nil {
return nil, err
Expand Down Expand Up @@ -278,8 +284,9 @@ func (s *Server) opiMultipathToSpdk(multipath pb.NvmeMultipath) string {

func (s *Server) numberOfPathsForController(controllerName string) int {
numberOfPaths := 0
prefix := controllerName + "/"
for _, path := range s.Volumes.NvmePaths {
if path.ControllerNameRef == controllerName {
if strings.HasPrefix(path.Name, prefix) {
numberOfPaths++
}
}
Expand Down
Loading

0 comments on commit b9be140

Please sign in to comment.