Skip to content

Commit

Permalink
Move IB VF configuration to the IB interface
Browse files Browse the repository at this point in the history 
switch sriov module to use the new interface

Signed-off-by: amaslennikov <[email protected]>
  • Loading branch information
@almaslennikov
almaslennikov committed Jul 23, 2024
1 parent b51ba3b commit 7116dc0
Show file tree
Hide file tree
Showing 5 changed files with 52 additions and 53 deletions.
29 changes: 15 additions & 14 deletions pkg/helper/mock/mock_helper.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

26 changes: 4 additions & 22 deletions pkg/host/internal/sriov/sriov.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ type sriov struct {
networkHelper types.NetworkInterface
udevHelper types.UdevInterface
vdpaHelper types.VdpaInterface
infinibandHelper types.InfinibandInterface
netlinkLib netlinkPkg.NetlinkLib
dputilsLib dputilsPkg.DPUtilsLib
sriovnetLib sriovnetPkg.SriovnetLib
Expand All @@ -49,6 +50,7 @@ func New(utilsHelper utils.CmdInterface,
networkHelper types.NetworkInterface,
udevHelper types.UdevInterface,
vdpaHelper types.VdpaInterface,
infinibandHelper types.InfinibandInterface,
netlinkLib netlinkPkg.NetlinkLib,
dputilsLib dputilsPkg.DPUtilsLib,
sriovnetLib sriovnetPkg.SriovnetLib,
Expand All @@ -58,6 +60,7 @@ func New(utilsHelper utils.CmdInterface,
networkHelper: networkHelper,
udevHelper: udevHelper,
vdpaHelper: vdpaHelper,
infinibandHelper: infinibandHelper,
netlinkLib: netlinkLib,
dputilsLib: dputilsLib,
sriovnetLib: sriovnetLib,
Expand Down Expand Up @@ -163,27 +166,6 @@ func (s *sriov) getVfInfo(vfAddr string, pfName string, eswitchMode string, devi
return vf
}

func (s *sriov) SetVfGUID(vfAddr string, pfLink netlink.Link) error {
log.Log.Info("SetVfGUID()", "vf", vfAddr)
vfID, err := s.dputilsLib.GetVFID(vfAddr)
if err != nil {
log.Log.Error(err, "SetVfGUID(): unable to get VF id", "address", vfAddr)
return err
}
guid := utils.GenerateRandomGUID()
if err := s.netlinkLib.LinkSetVfNodeGUID(pfLink, vfID, guid); err != nil {
return err
}
if err := s.netlinkLib.LinkSetVfPortGUID(pfLink, vfID, guid); err != nil {
return err
}
if err = s.kernelHelper.Unbind(vfAddr); err != nil {
return err
}

return nil
}

func (s *sriov) VFIsReady(pciAddr string) (netlink.Link, error) {
log.Log.Info("VFIsReady()", "device", pciAddr)
var err error
Expand Down Expand Up @@ -491,7 +473,7 @@ func (s *sriov) configSriovVFDevices(iface *sriovnetworkv1.Interface) error {
linkType = s.GetLinkType(iface.Name)
}
if strings.EqualFold(linkType, consts.LinkTypeIB) {
if err = s.SetVfGUID(addr, pfLink); err != nil {
if err := s.infinibandHelper.ConfigureVfGUID(addr, iface.PciAddress, vfID, pfLink); err != nil {
return err
}
} else {
Expand Down
12 changes: 11 additions & 1 deletion pkg/host/internal/sriov/sriov_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ var _ = Describe("SRIOV", func() {
hostMock = hostMockPkg.NewMockHostManagerInterface(testCtrl)
storeManagerMode = hostStoreMockPkg.NewMockManagerInterface(testCtrl)

s = New(nil, hostMock, hostMock, hostMock, hostMock, netlinkLibMock, dputilsLibMock, sriovnetLibMock, ghwLibMock)
s = New(nil, hostMock, hostMock, hostMock, hostMock, nil, netlinkLibMock, dputilsLibMock, sriovnetLibMock, ghwLibMock)
})

AfterEach(func() {
Expand Down Expand Up @@ -227,6 +227,7 @@ var _ = Describe("SRIOV", func() {
pfLinkMock.EXPECT().Attrs().Return(&netlink.LinkAttrs{Flags: 0, EncapType: "ether"})
netlinkLibMock.EXPECT().IsLinkAdminStateUp(pfLinkMock).Return(false)
netlinkLibMock.EXPECT().LinkSetUp(pfLinkMock).Return(nil)
netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes()

dputilsLibMock.EXPECT().GetVFID("0000:d8:00.2").Return(0, nil).Times(2)
hostMock.EXPECT().HasDriver("0000:d8:00.2").Return(false, "")
Expand Down Expand Up @@ -304,6 +305,7 @@ var _ = Describe("SRIOV", func() {
vf0LinkMock := netlinkMockPkg.NewMockLink(testCtrl)
netlinkLibMock.EXPECT().LinkSetVfNodeGUID(vf0LinkMock, 0, gomock.Any()).Return(nil)
netlinkLibMock.EXPECT().LinkSetVfPortGUID(vf0LinkMock, 0, gomock.Any()).Return(nil)
netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes()

storeManagerMode.EXPECT().SaveLastPfAppliedStatus(gomock.Any()).Return(nil)

Expand Down Expand Up @@ -370,6 +372,7 @@ var _ = Describe("SRIOV", func() {
hostMock.EXPECT().AddVfRepresentorUdevRule("0000:d8:00.0", "enp216s0f0np0", "7cfe90ff2cc0", "p0").Return(nil)
hostMock.EXPECT().CreateVDPADevice("0000:d8:00.2", "vhost_vdpa")
hostMock.EXPECT().LoadUdevRules().Return(nil)
netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes()

storeManagerMode.EXPECT().SaveLastPfAppliedStatus(gomock.Any()).Return(nil)

Expand Down Expand Up @@ -397,6 +400,8 @@ var _ = Describe("SRIOV", func() {

It("externally managed - wrong VF count", func() {
dputilsLibMock.EXPECT().GetVFconfigured("0000:d8:00.0").Return(0)
netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes()

Expect(s.ConfigSriovInterfaces(storeManagerMode,
[]sriovnetworkv1.Interface{{
Name: "enp216s0f0np0",
Expand All @@ -421,6 +426,8 @@ var _ = Describe("SRIOV", func() {
netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return(
&netlink.DevlinkDevice{Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}},
nil)
netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes()

hostMock.EXPECT().GetNetdevMTU("0000:d8:00.0")
Expect(s.ConfigSriovInterfaces(storeManagerMode,
[]sriovnetworkv1.Interface{{
Expand Down Expand Up @@ -456,6 +463,7 @@ var _ = Describe("SRIOV", func() {
netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return(
&netlink.DevlinkDevice{Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}},
nil)
netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes()
hostMock.EXPECT().RemoveDisableNMUdevRule("0000:d8:00.0").Return(nil)
hostMock.EXPECT().RemovePersistPFNameUdevRule("0000:d8:00.0").Return(nil)
hostMock.EXPECT().RemoveVfRepresentorUdevRule("0000:d8:00.0").Return(nil)
Expand All @@ -474,6 +482,7 @@ var _ = Describe("SRIOV", func() {
helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/devices/0000:d8:00.0/sriov_numvfs", "0")
})
It("reset device - skip external", func() {
netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes()
storeManagerMode.EXPECT().LoadPfsStatus("0000:d8:00.0").Return(&sriovnetworkv1.Interface{
Name: "enp216s0f0np0",
PciAddress: "0000:d8:00.0",
Expand Down Expand Up @@ -502,6 +511,7 @@ var _ = Describe("SRIOV", func() {
netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return(
&netlink.DevlinkDevice{Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}},
nil)
netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes()
hostMock.EXPECT().RemoveDisableNMUdevRule("0000:d8:00.0").Return(nil)
hostMock.EXPECT().RemovePersistPFNameUdevRule("0000:d8:00.0").Return(nil)
hostMock.EXPECT().RemoveVfRepresentorUdevRule("0000:d8:00.0").Return(nil)
Expand Down
29 changes: 15 additions & 14 deletions pkg/host/mock/mock_host.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 7 additions & 2 deletions pkg/host/types/interfaces.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,8 @@ type NetworkInterface interface {
EnableHwTcOffload(ifaceName string) error
// GetNetDevLinkAdminState returns the admin state of the interface.
GetNetDevLinkAdminState(ifaceName string) string
// GetPciAddressFromInterfaceName parses sysfs to get pci address of an interface by name
GetPciAddressFromInterfaceName(interfaceName string) (string, error)
}

type ServiceInterface interface {
Expand All @@ -135,8 +137,6 @@ type SriovInterface interface {
// SetSriovNumVfs changes the number of virtual functions allocated for a specific
// physical function base on pci address
SetSriovNumVfs(pciAddr string, numVfs int) error
// SetVfGUID sets the GUID for a virtual function
SetVfGUID(vfAddr string, pfLink netlink.Link) error
// VFIsReady returns the interface virtual function if the device is ready
VFIsReady(pciAddr string) (netlink.Link, error)
// SetVfAdminMac sets the virtual function administrative mac address via the physical function
Expand Down Expand Up @@ -204,3 +204,8 @@ type BridgeInterface interface {
// The function detach interface from managed bridges only.
DetachInterfaceFromManagedBridge(pciAddr string) error
}

type InfinibandInterface interface {
// ConfigureVfGUID configures and sets a GUID for an IB VF device
ConfigureVfGUID(vfAddr string, pfAddr string, vfID int, pfLink netlink.Link) error
}

0 comments on commit 7116dc0

Please sign in to comment.