From de5b9a076ff114f24f87d3c3040807186a16ca60 Mon Sep 17 00:00:00 2001 From: Periyasamy Palanisamy Date: Thu, 16 Dec 2021 15:08:48 +0100 Subject: [PATCH 1/3] make skip driver check as device config parameter Signed-off-by: Periyasamy Palanisamy --- .../common/resourcepool/common.go | 21 +++++---- pkg/sriov/config/config.go | 8 ++++ pkg/sriov/pci/pool.go | 47 +++++++++---------- 3 files changed, 42 insertions(+), 34 deletions(-) diff --git a/pkg/networkservice/common/resourcepool/common.go b/pkg/networkservice/common/resourcepool/common.go index 1d4835af..a32b3680 100644 --- a/pkg/networkservice/common/resourcepool/common.go +++ b/pkg/networkservice/common/resourcepool/common.go @@ -22,6 +22,7 @@ package resourcepool import ( "context" + "strconv" "sync" "github.com/pkg/errors" @@ -57,10 +58,10 @@ type resourcePoolConfig struct { selectedVFs map[string]string } -func (s *resourcePoolConfig) selectVF(connID string, vfConfig *vfconfig.VFConfig, tokenID string) (vf sriov.PCIFunction, err error) { +func (s *resourcePoolConfig) selectVF(connID string, vfConfig *vfconfig.VFConfig, tokenID string) (vf sriov.PCIFunction, skipDriverCheck bool, err error) { vfPCIAddr, err := s.resourcePool.Select(tokenID, s.driverType) if err != nil { - return nil, errors.Wrapf(err, "failed to select VF for: %v", s.driverType) + return nil, false, errors.Wrapf(err, "failed to select VF for: %v", s.driverType) } s.selectedVFs[connID] = vfPCIAddr @@ -72,25 +73,27 @@ func (s *resourcePoolConfig) selectVF(connID string, vfConfig *vfconfig.VFConfig pf, err := s.pciPool.GetPCIFunction(pfPCIAddr) if err != nil { - return nil, errors.Wrapf(err, "failed to get PF: %v", pfPCIAddr) + return nil, true, errors.Wrapf(err, "failed to get PF: %v", pfPCIAddr) } vfConfig.PFInterfaceName, err = pf.GetNetInterfaceName() if err != nil { - return nil, errors.Errorf("failed to get PF net interface name: %v", pfPCIAddr) + return nil, true, errors.Errorf("failed to get PF net interface name: %v", pfPCIAddr) } vf, err := s.pciPool.GetPCIFunction(vfPCIAddr) if err != nil { - return nil, errors.Wrapf(err, "failed to get VF: %v", vfPCIAddr) + return nil, true, errors.Wrapf(err, "failed to get VF: %v", vfPCIAddr) } vfConfig.VFNum = i - return vf, err + skipDriverCheck, _ = strconv.ParseBool(pfCfg.SkipDriverCheck) + + return vf, skipDriverCheck, err } } - return nil, errors.Errorf("no VF with selected PCI address exists: %v", s.selectedVFs[connID]) + return nil, true, errors.Errorf("no VF with selected PCI address exists: %v", s.selectedVFs[connID]) } func (s *resourcePoolConfig) close(conn *networkservice.Connection) error { @@ -113,7 +116,7 @@ func assignVF(ctx context.Context, logger log.Logger, conn *networkservice.Conne vfConfig := &vfconfig.VFConfig{} logger.Infof("trying to select VF for %v", resourcePool.driverType) - vf, err := resourcePool.selectVF(conn.GetId(), vfConfig, tokenID) + vf, skipDriverCheck, err := resourcePool.selectVF(conn.GetId(), vfConfig, tokenID) if err != nil { return err } @@ -131,7 +134,7 @@ func assignVF(ctx context.Context, logger log.Logger, conn *networkservice.Conne switch resourcePool.driverType { case sriov.KernelDriver: vfConfig.VFInterfaceName, err = vf.GetNetInterfaceName() - if err != nil { + if err != nil && !skipDriverCheck { return errors.Wrapf(err, "failed to get VF net interface name: %v", vf.GetPCIAddress()) } case sriov.VFIOPCIDriver: diff --git a/pkg/sriov/config/config.go b/pkg/sriov/config/config.go index 722eecfd..09eacd7d 100644 --- a/pkg/sriov/config/config.go +++ b/pkg/sriov/config/config.go @@ -20,6 +20,7 @@ package config import ( "context" "fmt" + "strconv" "strings" "github.com/networkservicemesh/sdk/pkg/tools/log/logruslogger" @@ -55,6 +56,7 @@ type PhysicalFunction struct { VFKernelDriver string `yaml:"vfKernelDriver"` Capabilities []string `yaml:"capabilities"` ServiceDomains []string `yaml:"serviceDomains"` + SkipDriverCheck string `default:"false" yaml:"skipDriverCheck"` VirtualFunctions []*VirtualFunction `yaml:"virtualFunctions"` } @@ -84,6 +86,9 @@ func (pf *PhysicalFunction) String() string { _, _ = sb.WriteString(strings.Join(strs, " ")) _, _ = sb.WriteString("]") + _, _ = sb.WriteString(" SkipDriverCheck:") + _, _ = sb.WriteString(pf.SkipDriverCheck) + _, _ = sb.WriteString("}") return sb.String() } @@ -116,6 +121,9 @@ func ReadConfig(ctx context.Context, configFile string) (*Config, error) { if len(pfCfg.ServiceDomains) == 0 { return nil, errors.Errorf("%s has no ServiceDomains set", pciAddr) } + if _, err := strconv.ParseBool(pfCfg.SkipDriverCheck); err != nil { + return nil, errors.Errorf("%s has invalid SkipDriverCheck set", pciAddr) + } } logger.WithField("Config", "ReadConfig").Infof("unmarshalled Config: %+v", cfg) diff --git a/pkg/sriov/pci/pool.go b/pkg/sriov/pci/pool.go index da3c61f0..a256710c 100644 --- a/pkg/sriov/pci/pool.go +++ b/pkg/sriov/pci/pool.go @@ -52,26 +52,20 @@ type Pool struct { functions map[string]*function // pciAddr -> *function functionsByIOMMUGroup map[uint][]*function // iommuGroup -> []*function vfioDir string - skipDriverCheck bool } type function struct { - function pciFunction - kernelDriver string + function pciFunction + kernelDriver string + skipDriverCheck bool } // NewPool returns a new PCI Pool func NewPool(pciDevicesPath, pciDriversPath, vfioDir string, cfg *config.Config) (*Pool, error) { - return NewPCIPool(pciDevicesPath, pciDriversPath, vfioDir, cfg, false) -} - -// NewPCIPool returns a new PCI Pool -func NewPCIPool(pciDevicesPath, pciDriversPath, vfioDir string, cfg *config.Config, skipDriverCheck bool) (*Pool, error) { p := &Pool{ functions: map[string]*function{}, functionsByIOMMUGroup: map[uint][]*function{}, vfioDir: vfioDir, - skipDriverCheck: skipDriverCheck, } for pfPCIAddr, pfCfg := range cfg.PhysicalFunctions { @@ -80,12 +74,14 @@ func NewPCIPool(pciDevicesPath, pciDriversPath, vfioDir string, cfg *config.Conf return nil, err } - if err := p.addFunction(&pf.Function, pfCfg.PFKernelDriver); err != nil { + skipDriverCheck, _ := strconv.ParseBool(pfCfg.SkipDriverCheck) + + if err := p.addFunction(&pf.Function, pfCfg.PFKernelDriver, skipDriverCheck); err != nil { return nil, err } for _, vf := range pf.GetVirtualFunctions() { - if err := p.addFunction(vf, pfCfg.VFKernelDriver); err != nil { + if err := p.addFunction(vf, pfCfg.VFKernelDriver, skipDriverCheck); err != nil { return nil, err } } @@ -99,7 +95,6 @@ func NewTestPool(physicalFunctions map[string]*sriovtest.PCIPhysicalFunction, cf p := &Pool{ functions: map[string]*function{}, functionsByIOMMUGroup: map[uint][]*function{}, - skipDriverCheck: true, } for pfPCIAddr, pfCfg := range cfg.PhysicalFunctions { @@ -108,20 +103,21 @@ func NewTestPool(physicalFunctions map[string]*sriovtest.PCIPhysicalFunction, cf return nil, errors.Errorf("PF doesn't exist: %v", pfPCIAddr) } - _ = p.addFunction(&pf.PCIFunction, pfCfg.PFKernelDriver) + _ = p.addFunction(&pf.PCIFunction, pfCfg.PFKernelDriver, true) for _, vf := range pf.Vfs { - _ = p.addFunction(vf, pfCfg.VFKernelDriver) + _ = p.addFunction(vf, pfCfg.VFKernelDriver, true) } } return p, nil } -func (p *Pool) addFunction(pcif pciFunction, kernelDriver string) (err error) { +func (p *Pool) addFunction(pcif pciFunction, kernelDriver string, skipDriverCheck bool) (err error) { f := &function{ - function: pcif, - kernelDriver: kernelDriver, + function: pcif, + kernelDriver: kernelDriver, + skipDriverCheck: skipDriverCheck, } p.functions[pcif.GetPCIAddress()] = f @@ -162,7 +158,7 @@ func (p *Pool) BindDriver(ctx context.Context, iommuGroup uint, driverType sriov } for _, f := range p.functionsByIOMMUGroup[iommuGroup] { - if err := p.waitDriverGettingBound(ctx, f.function, driverType); err != nil { + if err := p.waitDriverGettingBound(ctx, f.function, driverType, f.skipDriverCheck); err != nil { return err } } @@ -170,10 +166,11 @@ func (p *Pool) BindDriver(ctx context.Context, iommuGroup uint, driverType sriov return nil } -func (p *Pool) waitDriverGettingBound(ctx context.Context, pcif pciFunction, driverType sriov.DriverType) error { +func (p *Pool) waitDriverGettingBound(ctx context.Context, pcif pciFunction, driverType sriov.DriverType, + skipDriverCheck bool) error { timeoutCh := time.After(driverBindTimeout) for { - var driverCheck func(pciFunction) error + var driverCheck func(pciFunction, bool) error switch driverType { case sriov.KernelDriver: driverCheck = p.kernelDriverCheck @@ -183,7 +180,7 @@ func (p *Pool) waitDriverGettingBound(ctx context.Context, pcif pciFunction, dri return errors.Errorf("driver type is not supported: %v", driverType) } - err := driverCheck(pcif) + err := driverCheck(pcif, skipDriverCheck) if err == nil { return nil } @@ -198,8 +195,8 @@ func (p *Pool) waitDriverGettingBound(ctx context.Context, pcif pciFunction, dri } } -func (p *Pool) kernelDriverCheck(pcif pciFunction) error { - if p.skipDriverCheck { +func (p *Pool) kernelDriverCheck(pcif pciFunction, skipDriverCheck bool) error { + if skipDriverCheck { return nil } @@ -207,8 +204,8 @@ func (p *Pool) kernelDriverCheck(pcif pciFunction) error { return err } -func (p *Pool) vfioDriverCheck(pcif pciFunction) error { - if p.skipDriverCheck { +func (p *Pool) vfioDriverCheck(pcif pciFunction, skipDriverCheck bool) error { + if skipDriverCheck { return nil } From 5826a9b361a72888f3e396cd2011f7fa52a587d3 Mon Sep 17 00:00:00 2001 From: Periyasamy Palanisamy Date: Thu, 16 Dec 2021 15:33:33 +0100 Subject: [PATCH 2/3] fix unit test Signed-off-by: Periyasamy Palanisamy --- pkg/networkservice/common/resourcepool/config.yml | 2 ++ pkg/sriov/config/config.go | 2 +- pkg/sriov/config/config.yml | 2 ++ pkg/sriov/config/config_test.go | 3 +++ pkg/sriov/resource/config.yml | 3 +++ pkg/sriov/token/config.yml | 2 ++ 6 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pkg/networkservice/common/resourcepool/config.yml b/pkg/networkservice/common/resourcepool/config.yml index 4da02cfb..0972a9f8 100644 --- a/pkg/networkservice/common/resourcepool/config.yml +++ b/pkg/networkservice/common/resourcepool/config.yml @@ -1,6 +1,7 @@ --- physicalFunctions: 0000:00:01.0: + skipDriverCheck: true pfKernelDriver: pf-1-driver vfKernelDriver: vf-1-driver capabilities: @@ -14,6 +15,7 @@ physicalFunctions: - address: 0000:00:01.2 iommuGroup: 1 0000:00:02.0: + skipDriverCheck: true pfKernelDriver: pf-2-driver vfKernelDriver: vf-2-driver capabilities: diff --git a/pkg/sriov/config/config.go b/pkg/sriov/config/config.go index 09eacd7d..69db4dfa 100644 --- a/pkg/sriov/config/config.go +++ b/pkg/sriov/config/config.go @@ -56,7 +56,7 @@ type PhysicalFunction struct { VFKernelDriver string `yaml:"vfKernelDriver"` Capabilities []string `yaml:"capabilities"` ServiceDomains []string `yaml:"serviceDomains"` - SkipDriverCheck string `default:"false" yaml:"skipDriverCheck"` + SkipDriverCheck string `yaml:"skipDriverCheck"` VirtualFunctions []*VirtualFunction `yaml:"virtualFunctions"` } diff --git a/pkg/sriov/config/config.yml b/pkg/sriov/config/config.yml index 31352d38..2292dd26 100644 --- a/pkg/sriov/config/config.yml +++ b/pkg/sriov/config/config.yml @@ -1,6 +1,7 @@ --- physicalFunctions: 0000:01:00.0: + skipDriverCheck: true pfKernelDriver: pf-driver vfKernelDriver: vf-driver capabilities: @@ -14,6 +15,7 @@ physicalFunctions: - address: 0000:01:00.2 iommuGroup: 2 0000:02:00.0: + skipDriverCheck: true pfKernelDriver: pf-driver vfKernelDriver: vf-driver capabilities: diff --git a/pkg/sriov/config/config_test.go b/pkg/sriov/config/config_test.go index e2344eb1..1d38177d 100644 --- a/pkg/sriov/config/config_test.go +++ b/pkg/sriov/config/config_test.go @@ -41,6 +41,7 @@ const ( vf21PciAddr = "0000:02:00.1" vf22PciAddr = "0000:02:00.2" vf23PciAddr = "0000:02:00.3" + skipDriverCheck = "true" ) func TestReadConfigFile(t *testing.T) { @@ -68,6 +69,7 @@ func TestReadConfigFile(t *testing.T) { IOMMUGroup: 2, }, }, + SkipDriverCheck: skipDriverCheck, }, pf2PciAddr: { PFKernelDriver: pfKernelDriver, @@ -94,6 +96,7 @@ func TestReadConfigFile(t *testing.T) { IOMMUGroup: 3, }, }, + SkipDriverCheck: skipDriverCheck, }, }, }, cfg) diff --git a/pkg/sriov/resource/config.yml b/pkg/sriov/resource/config.yml index ae1b9a5e..bb59d64d 100644 --- a/pkg/sriov/resource/config.yml +++ b/pkg/sriov/resource/config.yml @@ -1,6 +1,7 @@ --- physicalFunctions: 0000:01:00.0: + skipDriverCheck: true pfKernelDriver: pf-driver vfKernelDriver: vf-driver capabilities: @@ -12,6 +13,7 @@ physicalFunctions: - address: 0000:01:00.1 iommuGroup: 1 0000:02:00.0: + skipDriverCheck: true pfKernelDriver: pf-driver vfKernelDriver: vf-driver capabilities: @@ -25,6 +27,7 @@ physicalFunctions: - address: 0000:02:00.2 iommuGroup: 2 0000:03:00.0: + skipDriverCheck: true pfKernelDriver: pf-driver vfKernelDriver: vf-driver capabilities: diff --git a/pkg/sriov/token/config.yml b/pkg/sriov/token/config.yml index 12265efc..a1dbf0b5 100644 --- a/pkg/sriov/token/config.yml +++ b/pkg/sriov/token/config.yml @@ -1,6 +1,7 @@ --- physicalFunctions: 0000:01:00.0: + skipDriverCheck: true pfKernelDriver: pf-driver vfKernelDriver: vf-driver capabilities: @@ -12,6 +13,7 @@ physicalFunctions: - address: 0000:01:00.1 iommuGroup: 1 0000:02:00.0: + skipDriverCheck: true pfKernelDriver: pf-driver vfKernelDriver: vf-driver capabilities: From c2f1c74d786c7e2b579d4fe62fbfc1d376b4e99c Mon Sep 17 00:00:00 2001 From: Periyasamy Palanisamy Date: Thu, 16 Dec 2021 16:20:21 +0100 Subject: [PATCH 3/3] fix lint errors Signed-off-by: Periyasamy Palanisamy --- pkg/sriov/config/config.go | 2 ++ pkg/sriov/config/config_test.go | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/sriov/config/config.go b/pkg/sriov/config/config.go index 69db4dfa..eaf78948 100644 --- a/pkg/sriov/config/config.go +++ b/pkg/sriov/config/config.go @@ -1,5 +1,7 @@ // Copyright (c) 2020-2021 Doc.ai and/or its affiliates. // +// Copyright (c) 2021 Nordix Foundation. +// // SPDX-License-Identifier: Apache-2.0 // // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/pkg/sriov/config/config_test.go b/pkg/sriov/config/config_test.go index 1d38177d..7cae8bf1 100644 --- a/pkg/sriov/config/config_test.go +++ b/pkg/sriov/config/config_test.go @@ -1,4 +1,6 @@ -// Copyright (c) 2020 Doc.ai and/or its affiliates. +// Copyright (c) 2020-2021 Doc.ai and/or its affiliates. +// +// Copyright (c) 2021 Nordix Foundation. // // SPDX-License-Identifier: Apache-2.0 //