diff --git a/pkg/consts/constants.go b/pkg/consts/constants.go index af217cb01..cbfe9ad98 100644 --- a/pkg/consts/constants.go +++ b/pkg/consts/constants.go @@ -138,6 +138,9 @@ const ( // ManageSoftwareBridgesFeatureGate: enables management of software bridges by the operator ManageSoftwareBridgesFeatureGate = "manageSoftwareBridges" + + // The path to the file on the host filesystem that contains the IB GUID distribution for IB VFs + InfinibandGUIDConfigFilePath = SriovConfBasePath + "/infiniband/guids" ) const ( diff --git a/pkg/host/internal/infiniband/guid.go b/pkg/host/internal/infiniband/guid.go new file mode 100644 index 000000000..9cb2bffb6 --- /dev/null +++ b/pkg/host/internal/infiniband/guid.go @@ -0,0 +1,62 @@ +package infiniband + +import ( + "fmt" + "math/rand" + "net" +) + +// GUID address is an uint64 encapsulation for network hardware address +type GUID uint64 + +const ( + guidLength = 8 + byteBitLen = 8 + byteMask = 0xff +) + +// ParseGUID parses string only as GUID 64 bit +func ParseGUID(s string) (GUID, error) { + ha, err := net.ParseMAC(s) + if err != nil { + return 0, err + } + if len(ha) != guidLength { + return 0, fmt.Errorf("invalid GUID address %s", s) + } + var guid uint64 + for idx, octet := range ha { + guid |= uint64(octet) << uint(byteBitLen*(guidLength-1-idx)) + } + return GUID(guid), nil +} + +// String returns the string representation of GUID +func (g GUID) String() string { + return g.HardwareAddr().String() +} + +// HardwareAddr returns GUID representation as net.HardwareAddr +func (g GUID) HardwareAddr() net.HardwareAddr { + value := uint64(g) + ha := make(net.HardwareAddr, guidLength) + for idx := guidLength - 1; idx >= 0; idx-- { + ha[idx] = byte(value & byteMask) + value >>= byteBitLen + } + + return ha +} + +func generateRandomGUID() net.HardwareAddr { + guid := make(net.HardwareAddr, 8) + + // First field is 0x01 - xfe to avoid all zero and all F invalid guids + guid[0] = byte(1 + rand.Intn(0xfe)) + + for i := 1; i < len(guid); i++ { + guid[i] = byte(rand.Intn(0x100)) + } + + return guid +} diff --git a/pkg/host/internal/infiniband/guid_test.go b/pkg/host/internal/infiniband/guid_test.go new file mode 100644 index 000000000..f31d5d3d0 --- /dev/null +++ b/pkg/host/internal/infiniband/guid_test.go @@ -0,0 +1,29 @@ +package infiniband + +import ( + "net" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("GUID", func() { + It("should parse and process GUIDs correctly", func() { + guidStr := "00:01:02:03:04:05:06:08" + nextGuidStr := "00:01:02:03:04:05:06:09" + + guid, err := ParseGUID(guidStr) + Expect(err).NotTo(HaveOccurred()) + + Expect(guid.String()).To(Equal(guidStr)) + Expect((guid + 1).String()).To(Equal(nextGuidStr)) + }) + It("should represent GUID as HW address", func() { + guidStr := "00:01:02:03:04:05:06:08" + + guid, err := ParseGUID(guidStr) + Expect(err).NotTo(HaveOccurred()) + + Expect(guid.HardwareAddr()).To(Equal(net.HardwareAddr{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08})) + }) +}) diff --git a/pkg/host/internal/infiniband/ib_guid_config.go b/pkg/host/internal/infiniband/ib_guid_config.go new file mode 100644 index 000000000..261b40b5e --- /dev/null +++ b/pkg/host/internal/infiniband/ib_guid_config.go @@ -0,0 +1,126 @@ +package infiniband + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" +) + +type ibPfGUIDJSONConfig struct { + PciAddress string `json:"pciAddress,omitempty"` + PfGUID string `json:"pfGuid,omitempty"` + GUIDs []string `json:"guids,omitempty"` + GUIDsRange *GUIDRangeJSON `json:"guidsRange,omitempty"` +} + +type GUIDRangeJSON struct { + Start string `json:"start,omitempty"` + End string `json:"end,omitempty"` +} + +type ibPfGUIDConfig struct { + GUIDs []GUID + GUIDRange *GUIDRange +} + +type GUIDRange struct { + Start GUID + End GUID +} + +func getIbGUIDConfig(configPath string, netlinkLib netlink.NetlinkLib, networkHelper types.NetworkInterface) (map[string]ibPfGUIDConfig, error) { + links, err := netlinkLib.LinkList() + if err != nil { + return nil, err + } + rawConfigs, err := readJSONConfig(configPath) + if err != nil { + return nil, err + } + resultConfigs := map[string]ibPfGUIDConfig{} + // Parse JSON config into an internal struct + for _, rawConfig := range rawConfigs { + pciAddress, err := getPfPciAddressFromRawConfig(rawConfig, links, networkHelper) + if err != nil { + return nil, fmt.Errorf("failed to extract pci address from ib guid config: %w", err) + } + if len(rawConfig.GUIDs) == 0 && (rawConfig.GUIDsRange == nil || (rawConfig.GUIDsRange.Start == "" || rawConfig.GUIDsRange.End == "")) { + return nil, fmt.Errorf("either guid list or guid range should be provided, got none") + } + if len(rawConfig.GUIDs) != 0 && rawConfig.GUIDsRange != nil { + return nil, fmt.Errorf("either guid list or guid range should be provided, got both") + } + if rawConfig.GUIDsRange != nil && ((rawConfig.GUIDsRange.Start != "" && rawConfig.GUIDsRange.End == "") || (rawConfig.GUIDsRange.Start == "" && rawConfig.GUIDsRange.End != "")) { + return nil, fmt.Errorf("both guid rangeStart and rangeEnd should be provided, got one") + } + if len(rawConfig.GUIDs) != 0 { + var guids []GUID + for _, guidStr := range rawConfig.GUIDs { + guid, err := ParseGUID(guidStr) + if err != nil { + return nil, fmt.Errorf("failed to parse ib guid %s: %w", guidStr, err) + } + guids = append(guids, guid) + } + resultConfigs[pciAddress] = ibPfGUIDConfig{ + GUIDs: guids, + } + continue + } + + rangeStart, err := ParseGUID(rawConfig.GUIDsRange.Start) + if err != nil { + return nil, fmt.Errorf("failed to parse ib guid range start: %w", err) + } + rangeEnd, err := ParseGUID(rawConfig.GUIDsRange.End) + if err != nil { + return nil, fmt.Errorf("failed to parse ib guid range end: %w", err) + } + if rangeEnd < rangeStart { + return nil, fmt.Errorf("range end cannot be less then range start") + } + resultConfigs[pciAddress] = ibPfGUIDConfig{ + GUIDRange: &GUIDRange{ + Start: rangeStart, + End: rangeEnd, + }, + } + } + return resultConfigs, nil +} + +// readJSONConfig reads the file at the given path and unmarshals the contents into an array of ibPfGUIDJSONConfig structs +func readJSONConfig(configPath string) ([]ibPfGUIDJSONConfig, error) { + data, err := os.ReadFile(utils.GetHostExtensionPath(configPath)) + if err != nil { + return nil, fmt.Errorf("failed to read ib guid config: %w", err) + } + var configs []ibPfGUIDJSONConfig + if err := json.Unmarshal(data, &configs); err != nil { + return nil, fmt.Errorf("failed to unmarshal content of ib guid config: %w", err) + } + return configs, nil +} + +func getPfPciAddressFromRawConfig(pfRawConfig ibPfGUIDJSONConfig, links []netlink.Link, networkHelper types.NetworkInterface) (string, error) { + if pfRawConfig.PciAddress != "" && pfRawConfig.PfGUID != "" { + return "", fmt.Errorf("either PCI address or PF GUID required to describe an interface, both provided") + } + if pfRawConfig.PciAddress == "" && pfRawConfig.PfGUID == "" { + return "", fmt.Errorf("either PCI address or PF GUID required to describe an interface, none provided") + } + if pfRawConfig.PciAddress != "" { + return pfRawConfig.PciAddress, nil + } + // PfGUID is provided, need to resolve the pci address + for _, link := range links { + if link.Attrs().HardwareAddr.String() == pfRawConfig.PfGUID { + return networkHelper.GetPciAddressFromInterfaceName(link.Attrs().Name) + } + } + return "", fmt.Errorf("no matching link found for pf guid: %s", pfRawConfig.PfGUID) +} diff --git a/pkg/host/internal/infiniband/ib_guid_config_test.go b/pkg/host/internal/infiniband/ib_guid_config_test.go new file mode 100644 index 000000000..63f6b1683 --- /dev/null +++ b/pkg/host/internal/infiniband/ib_guid_config_test.go @@ -0,0 +1,285 @@ +package infiniband + +import ( + "fmt" + + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/vishvananda/netlink" + + netlinkLibPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" + netlinkMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink/mock" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/network" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/fakefilesystem" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/helpers" +) + +var _ = Describe("IbGuidConfig", func() { + Describe("readJSONConfig", Ordered, func() { + var ( + createJsonConfig func(string) string + ) + + BeforeEach(func() { + createJsonConfig = func(content string) string { + configPath := "/config.json" + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/host"}, + Files: map[string][]byte{"/host" + configPath: []byte(content)}, + }) + + return configPath + } + }) + + It("should correctly decode a JSON configuration file with all fields present", func() { + mockJsonConfig := `[{"pciAddress":"0000:00:00.0","pfGuid":"00:00:00:00:00:00:00:00","guids":["00:01:02:03:04:05:06:07", "00:01:02:03:04:05:06:08"],"guidsRange":{"start":"00:01:02:03:04:05:06:08","end":"00:01:02:03:04:05:06:FF"}}]` + + configPath := createJsonConfig(mockJsonConfig) + + configs, err := readJSONConfig(configPath) + Expect(err).NotTo(HaveOccurred()) + Expect(configs).To(HaveLen(1)) + Expect(configs[0].PciAddress).To(Equal("0000:00:00.0")) + Expect(configs[0].GUIDs).To(ContainElement("00:01:02:03:04:05:06:07")) + Expect(configs[0].GUIDs).To(ContainElement("00:01:02:03:04:05:06:08")) + Expect(configs[0].GUIDsRange.Start).To(Equal("00:01:02:03:04:05:06:08")) + Expect(configs[0].GUIDsRange.End).To(Equal("00:01:02:03:04:05:06:FF")) + + }) + It("should correctly decode a JSON configuration file with one field present", func() { + mockJsonConfig := `[{"pciAddress":"0000:00:00.0"}]` + + configPath := createJsonConfig(mockJsonConfig) + + configs, err := readJSONConfig(configPath) + Expect(err).NotTo(HaveOccurred()) + Expect(configs).To(HaveLen(1)) + Expect(configs[0].PciAddress).To(Equal("0000:00:00.0")) + }) + It("should correctly decode a JSON array with several elements", func() { + mockJsonConfig := `[{"pciAddress":"0000:00:00.0","guids":["00:01:02:03:04:05:06:07"]},{"pfGuid":"00:00:00:00:00:00:00:00","guidsRange":{"start":"00:01:02:03:04:05:06:08","end":"00:01:02:03:04:05:06:FF"}}]` + + configPath := createJsonConfig(mockJsonConfig) + + configs, err := readJSONConfig(configPath) + Expect(err).NotTo(HaveOccurred()) + Expect(configs).To(HaveLen(2)) + Expect(configs[0].PciAddress).To(Equal("0000:00:00.0")) + Expect(configs[1].PfGUID).To(Equal("00:00:00:00:00:00:00:00")) + Expect(configs[1].GUIDsRange.Start).To(Equal("00:01:02:03:04:05:06:08")) + Expect(configs[1].GUIDsRange.End).To(Equal("00:01:02:03:04:05:06:FF")) + }) + It("should fail on a non-array JSON", func() { + mockJsonConfig := `{"pciAddress":"0000:00:00.0", "newField": "newValue"}` + + configPath := createJsonConfig(mockJsonConfig) + + _, err := readJSONConfig(configPath) + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("getPfPciAddressFromRawConfig", func() { + var ( + networkHelper types.NetworkInterface + ) + BeforeEach(func() { + networkHelper = network.New(nil, nil, nil, nil) + }) + It("should return same pci address when pci address is provided", func() { + pci, err := getPfPciAddressFromRawConfig(ibPfGUIDJSONConfig{PciAddress: "pciAddress"}, nil, networkHelper) + Expect(err).NotTo(HaveOccurred()) + Expect(pci).To(Equal("pciAddress")) + }) + It("should find correct pci address when pf guid is given", func() { + pfGuid := generateRandomGUID() + + testCtrl := gomock.NewController(GinkgoT()) + pfLinkMock := netlinkMockPkg.NewMockLink(testCtrl) + pfLinkMock.EXPECT().Attrs().Return(&netlink.LinkAttrs{Name: "ib216s0f0", HardwareAddr: pfGuid}).Times(2) + + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/sys/bus/pci/0000:3b:00.0", "/sys/class/net/ib216s0f0"}, + Symlinks: map[string]string{"/sys/class/net/ib216s0f0/device": "/sys/bus/pci/0000:3b:00.0"}, + }) + + pci, err := getPfPciAddressFromRawConfig(ibPfGUIDJSONConfig{PfGUID: pfGuid.String()}, []netlinkLibPkg.Link{pfLinkMock}, networkHelper) + Expect(err).NotTo(HaveOccurred()) + Expect(pci).To(Equal("0000:3b:00.0")) + + testCtrl.Finish() + }) + It("should return an error when no matching link is found", func() { + pfGuidDesired := generateRandomGUID() + pfGuidActual := generateRandomGUID() + + testCtrl := gomock.NewController(GinkgoT()) + pfLinkMock := netlinkMockPkg.NewMockLink(testCtrl) + pfLinkMock.EXPECT().Attrs().Return(&netlink.LinkAttrs{Name: "ib216s0f0", HardwareAddr: pfGuidActual}).Times(1) + + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/sys/bus/pci/0000:3b:00.0", "/sys/class/net/ib216s0f0"}, + Symlinks: map[string]string{"/sys/class/net/ib216s0f0/device": "/sys/bus/pci/0000:3b:00.0"}, + }) + + _, err := getPfPciAddressFromRawConfig(ibPfGUIDJSONConfig{PfGUID: pfGuidDesired.String()}, []netlinkLibPkg.Link{pfLinkMock}, networkHelper) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(MatchRegexp(`no matching link found for pf guid:.*`))) + + testCtrl.Finish() + }) + It("should return an error when too many parameters are provided", func() { + _, err := getPfPciAddressFromRawConfig(ibPfGUIDJSONConfig{PfGUID: "pfGuid", PciAddress: "pciAddress"}, nil, networkHelper) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError("either PCI address or PF GUID required to describe an interface, both provided")) + }) + It("should return an error when too few parameters are provided", func() { + _, err := getPfPciAddressFromRawConfig(ibPfGUIDJSONConfig{}, nil, networkHelper) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError("either PCI address or PF GUID required to describe an interface, none provided")) + }) + }) + + Describe("getIbGUIDConfig", func() { + Describe("Tests without common mocks", func() { + It("should parse correct json config and return a map", func() { + pfGuid, _ := ParseGUID(generateRandomGUID().String()) + vfGuid1, _ := ParseGUID(generateRandomGUID().String()) + vfGuid2, _ := ParseGUID(generateRandomGUID().String()) + rangeStart, err := ParseGUID("00:01:02:03:04:05:06:08") + Expect(err).NotTo(HaveOccurred()) + rangeEnd, err := ParseGUID("00:01:02:03:04:05:06:FF") + Expect(err).NotTo(HaveOccurred()) + + configPath := "/config.json" + configStr := fmt.Sprintf(`[{"pciAddress":"0000:3b:00.1","guids":["%s", "%s"]},{"pfGuid":"%s","guidsRange":{"start":"%s","end":"%s"}}]`, vfGuid1.String(), vfGuid2.String(), pfGuid.String(), rangeStart.String(), rangeEnd.String()) + + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/sys/bus/pci/0000:3b:00.0", "/sys/class/net/ib216s0f0", "/host"}, + Symlinks: map[string]string{"/sys/class/net/ib216s0f0/device": "/sys/bus/pci/0000:3b:00.0"}, + Files: map[string][]byte{"/host" + configPath: []byte(configStr)}, + }) + + testCtrl := gomock.NewController(GinkgoT()) + + pfLinkMock := netlinkMockPkg.NewMockLink(testCtrl) + pfLinkMock.EXPECT().Attrs().Return(&netlink.LinkAttrs{Name: "ib216s0f0", HardwareAddr: pfGuid.HardwareAddr()}).Times(2) + + netlinkLibMock := netlinkMockPkg.NewMockNetlinkLib(testCtrl) + netlinkLibMock.EXPECT().LinkList().Return([]netlinkLibPkg.Link{pfLinkMock}, nil).Times(1) + + networkHelper := network.New(nil, nil, nil, nil) + + config, err := getIbGUIDConfig(configPath, netlinkLibMock, networkHelper) + Expect(err).NotTo(HaveOccurred()) + Expect(config["0000:3b:00.1"].GUIDs[0]).To(Equal(vfGuid1)) + Expect(config["0000:3b:00.1"].GUIDs[1]).To(Equal(vfGuid2)) + Expect(config["0000:3b:00.0"].GUIDRange).To(Not(BeNil())) + Expect(config["0000:3b:00.0"].GUIDRange.Start).To(Equal(rangeStart)) + Expect(config["0000:3b:00.0"].GUIDRange.End).To(Equal(rangeEnd)) + + testCtrl.Finish() + }) + }) + + Describe("Tests with common mocks", func() { + var ( + netlinkLibMock *netlinkMockPkg.MockNetlinkLib + testCtrl *gomock.Controller + + createJsonConfig func(string) string + + networkHelper types.NetworkInterface + ) + + BeforeEach(func() { + createJsonConfig = func(content string) string { + configPath := "/config.json" + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/host"}, + Files: map[string][]byte{"/host" + configPath: []byte(content)}, + }) + + return configPath + } + + testCtrl = gomock.NewController(GinkgoT()) + netlinkLibMock = netlinkMockPkg.NewMockNetlinkLib(testCtrl) + netlinkLibMock.EXPECT().LinkList().Return([]netlinkLibPkg.Link{}, nil).Times(1) + + networkHelper = network.New(nil, nil, nil, nil) + }) + + AfterEach(func() { + testCtrl.Finish() + }) + + It("should return an error when invalid json config is provided", func() { + configPath := createJsonConfig(`[invalid file]`) + + _, err := getIbGUIDConfig(configPath, netlinkLibMock, networkHelper) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(MatchRegexp("failed to unmarshal content of ib guid config.*"))) + }) + It("should return an error when failed to determine pf's pci address", func() { + configPath := createJsonConfig(`[{"guids":["00:01:02:03:04:05:06:07"]}]`) + + _, err := getIbGUIDConfig(configPath, netlinkLibMock, networkHelper) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(MatchRegexp("failed to extract pci address from ib guid config.*"))) + }) + It("should return an error when both guids and rangeStart are provided", func() { + configPath := createJsonConfig(`[{"pciAddress": "someaddress", "guids":["00:01:02:03:04:05:06:07"], "guidsRange":{"start": "00:01:02:03:04:05:06:AA"}}]`) + + _, err := getIbGUIDConfig(configPath, netlinkLibMock, networkHelper) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(MatchRegexp("either guid list or guid range should be provided, got both.*"))) + }) + It("should return an error when both guids and rangeEnd are provided", func() { + configPath := createJsonConfig(`[{"pciAddress": "someaddress", "guids":["00:01:02:03:04:05:06:07"], "guidsRange":{"end": "00:01:02:03:04:05:06:AA"}}]`) + + _, err := getIbGUIDConfig(configPath, netlinkLibMock, networkHelper) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(MatchRegexp("either guid list or guid range should be provided, got both.*"))) + }) + It("should return an error when neither guids nor range are provided", func() { + configPath := createJsonConfig(`[{"pciAddress": "someaddress"}]`) + + _, err := getIbGUIDConfig(configPath, netlinkLibMock, networkHelper) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(MatchRegexp("either guid list or guid range should be provided, got none.*"))) + }) + It("should return an error when invalid guid list is provided", func() { + configPath := createJsonConfig(`[{"pciAddress": "someaddress", "guids":["invalid_guid"]}]`) + + _, err := getIbGUIDConfig(configPath, netlinkLibMock, networkHelper) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(MatchRegexp("failed to parse ib guid invalid_guid.*"))) + }) + It("should return an error when invalid guid range start is provided", func() { + configPath := createJsonConfig(`[{"pciAddress": "someaddress", "guidsRange": {"start":"invalid range start", "end":"00:01:02:03:04:05:06:FF"}}]`) + + _, err := getIbGUIDConfig(configPath, netlinkLibMock, networkHelper) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(MatchRegexp("failed to parse ib guid range start.*"))) + }) + It("should return an error when invalid guid range end is provided", func() { + configPath := createJsonConfig(`[{"pciAddress": "someaddress", "guidsRange": {"start":"00:01:02:03:04:05:06:08", "end":"invalid range end"}}]`) + + _, err := getIbGUIDConfig(configPath, netlinkLibMock, networkHelper) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(MatchRegexp("failed to parse ib guid range end.*"))) + }) + It("should return an error when guid range end is less than range start", func() { + configPath := createJsonConfig(`[{"pciAddress": "someaddress", "guidsRange": {"start":"00:01:02:03:04:05:06:FF", "end":"00:01:02:03:04:05:06:AA"}}]`) + + _, err := getIbGUIDConfig(configPath, netlinkLibMock, networkHelper) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(MatchRegexp("range end cannot be less then range start.*"))) + }) + }) + }) +}) diff --git a/pkg/host/internal/infiniband/ib_guid_pool.go b/pkg/host/internal/infiniband/ib_guid_pool.go new file mode 100644 index 000000000..fa0b52528 --- /dev/null +++ b/pkg/host/internal/infiniband/ib_guid_pool.go @@ -0,0 +1,63 @@ +package infiniband + +import ( + "fmt" + "net" + + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" +) + +// ibGUIDPool is an interface that returns the GUID, allocated for a specific VF id of the specific PF +type ibGUIDPool interface { + // GetVFGUID returns the GUID, allocated for a specific VF id of the specific PF + // If no guid pool exists for the given pfPciAddr, returns an error + // If no guids are available for the given VF id, returns an error + GetVFGUID(pfPciAddr string, vfID int) (net.HardwareAddr, error) +} + +type ibGUIDPoolImpl struct { + guidConfigs map[string]ibPfGUIDConfig +} + +// newIbGUIDPool returns an instance of ibGUIDPool +func newIbGUIDPool(configPath string, netlinkLib netlink.NetlinkLib, networkHelper types.NetworkInterface) (ibGUIDPool, error) { + // All validation for the config file is done in the getIbGUIDConfig function + configs, err := getIbGUIDConfig(configPath, netlinkLib, networkHelper) + if err != nil { + return nil, fmt.Errorf("failed to create ib guid pool: %w", err) + } + + return &ibGUIDPoolImpl{guidConfigs: configs}, nil +} + +// GetVFGUID returns the GUID, allocated for a specific VF id of the specific PF +// If no guid pool exists for the given pfPciAddr, returns an error +// If no guids are available for the given VF id, returns an error +func (p *ibGUIDPoolImpl) GetVFGUID(pfPciAddr string, vfID int) (net.HardwareAddr, error) { + config, exists := p.guidConfigs[pfPciAddr] + if !exists { + return nil, fmt.Errorf("no guid pool for pci address: %s", pfPciAddr) + } + + if len(config.GUIDs) != 0 { + if vfID >= len(config.GUIDs) { + return nil, fmt.Errorf("no guid allocation found for VF id: %d on pf %s", vfID, pfPciAddr) + } + + guid := config.GUIDs[vfID] + + return guid.HardwareAddr(), nil + } + + if config.GUIDRange != nil { + nextGUID := config.GUIDRange.Start + GUID(vfID) + if nextGUID > config.GUIDRange.End { + return nil, fmt.Errorf("no guid allocation found for VF id: %d on pf %s", vfID, pfPciAddr) + } + + return nextGUID.HardwareAddr(), nil + } + + return nil, fmt.Errorf("no guid pool for pci address: %s", pfPciAddr) +} diff --git a/pkg/host/internal/infiniband/ib_guid_pool_test.go b/pkg/host/internal/infiniband/ib_guid_pool_test.go new file mode 100644 index 000000000..86878d21a --- /dev/null +++ b/pkg/host/internal/infiniband/ib_guid_pool_test.go @@ -0,0 +1,79 @@ +package infiniband + +import ( + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + netlinkLibPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" + netlinkMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink/mock" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/network" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/fakefilesystem" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/helpers" +) + +var _ = Describe("ibGUIDPool", Ordered, func() { + var ( + netlinkLibMock *netlinkMockPkg.MockNetlinkLib + testCtrl *gomock.Controller + + createJsonConfig func(string) string + + guidPool ibGUIDPool + ) + + BeforeAll(func() { + var err error + + createJsonConfig = func(content string) string { + configPath := "/config.json" + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/host"}, + Files: map[string][]byte{"/host" + configPath: []byte(content)}, + }) + + return configPath + } + + testCtrl = gomock.NewController(GinkgoT()) + netlinkLibMock = netlinkMockPkg.NewMockNetlinkLib(testCtrl) + netlinkLibMock.EXPECT().LinkList().Return([]netlinkLibPkg.Link{}, nil).Times(1) + + configPath := createJsonConfig( + `[{"pciAddress":"0000:3b:00.0","guids":["00:00:00:00:00:00:00:00", "00:00:00:00:00:00:00:01"]}, + {"pciAddress":"0000:3b:00.1","guidsRange":{"start":"00:00:00:00:00:00:01:00","end":"00:00:00:00:00:00:01:02"}}]`) + + guidPool, err = newIbGUIDPool(configPath, netlinkLibMock, network.New(nil, nil, nil, nil)) + Expect(err).NotTo(HaveOccurred()) + }) + + DescribeTable("check GetVFGUID function", + func(pfAddr string, vfID int, expectedError string, expectedGUID string) { + guid, err := guidPool.GetVFGUID(pfAddr, vfID) + if expectedError != "" { + Expect(err).To(MatchError(expectedError)) + } else { + Expect(err).NotTo(HaveOccurred()) + } + if expectedGUID != "" { + Expect(guid.String()).To(Equal(expectedGUID)) + } + }, + + Entry("Should get the first GUID out of the array", "0000:3b:00.0", 0, "", "00:00:00:00:00:00:00:00"), + Entry("Should get the last GUID out of the array", "0000:3b:00.0", 1, "", "00:00:00:00:00:00:00:01"), + Entry("Should get the same result when called again for an array GUID", "0000:3b:00.0", 0, "", "00:00:00:00:00:00:00:00"), + Entry("Should return a pool exhausted error when given a VF ID immediately out of bounds of a GUID array", "0000:3b:00.0", 2, "no guid allocation found for VF id: 2 on pf 0000:3b:00.0", ""), + Entry("Should return a pool exhausted error when given a VF ID, grossly out of bounds of a GUID array", "0000:3b:00.0", 5, "no guid allocation found for VF id: 5 on pf 0000:3b:00.0", ""), + Entry("Should get a correct GUID from the GUID range #1", "0000:3b:00.1", 0, "", "00:00:00:00:00:00:01:00"), + Entry("Should get a correct GUID from the GUID range #2", "0000:3b:00.1", 1, "", "00:00:00:00:00:00:01:01"), + Entry("Should get a correct GUID from the GUID range #3", "0000:3b:00.1", 2, "", "00:00:00:00:00:00:01:02"), + Entry("Should return a pool exhausted error when given a VF ID immediately out of bounds of a GUID range", "0000:3b:00.1", 3, "no guid allocation found for VF id: 3 on pf 0000:3b:00.1", ""), + Entry("Should return a pool exhausted error when given a VF ID, grossly out of bounds of a GUID range", "0000:3b:00.1", 5, "no guid allocation found for VF id: 5 on pf 0000:3b:00.1", ""), + Entry("Should get the same result when called again for an array GUID", "0000:3b:00.1", 1, "", "00:00:00:00:00:00:01:01"), + ) + + AfterAll(func() { + testCtrl.Finish() + }) +}) diff --git a/pkg/host/internal/infiniband/infiniband.go b/pkg/host/internal/infiniband/infiniband.go new file mode 100644 index 000000000..f54957bc3 --- /dev/null +++ b/pkg/host/internal/infiniband/infiniband.go @@ -0,0 +1,67 @@ +package infiniband + +import ( + "errors" + "fmt" + "io/fs" + "net" + + "github.com/vishvananda/netlink" + "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + netlinkLibPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" +) + +// New creates and returns an InfinibandInterface object, that handles IB VF GUID configuration +func New(netlinkLib netlinkLibPkg.NetlinkLib, kernelHelper types.KernelInterface, networkHelper types.NetworkInterface) (types.InfinibandInterface, error) { + guidPool, err := newIbGUIDPool(consts.InfinibandGUIDConfigFilePath, netlinkLib, networkHelper) + if err != nil { + // if config file doesn't exist, fallback to the random GUID generation + if errors.Is(err, fs.ErrNotExist) { + log.Log.Info("infiniband.New(): ib guid config doesn't exist, continuing without it", "config path", consts.InfinibandGUIDConfigFilePath) + return &infiniband{guidPool: nil, netlinkLib: netlinkLib, kernelHelper: kernelHelper}, nil + } + + return nil, fmt.Errorf("failed to create the ib guid pool: %w", err) + } + + return &infiniband{guidPool: guidPool, netlinkLib: netlinkLib, kernelHelper: kernelHelper}, nil +} + +type infiniband struct { + guidPool ibGUIDPool + netlinkLib netlinkLibPkg.NetlinkLib + kernelHelper types.KernelInterface +} + +// ConfigureVfGUID configures and sets a GUID for an IB VF device +func (i *infiniband) ConfigureVfGUID(vfAddr string, pfAddr string, vfID int, pfLink netlink.Link) error { + log.Log.Info("ConfigureVfGUID(): configure vf guid", "vfAddr", vfAddr, "pfAddr", pfAddr, "vfID", vfID) + + guid := generateRandomGUID() + + if i.guidPool != nil { + guidFromPool, err := i.guidPool.GetVFGUID(pfAddr, vfID) + if err != nil { + log.Log.Info("ConfigureVfGUID(): failed to get GUID from IB GUID pool", "address", vfAddr, "error", err) + return err + } + guid = guidFromPool + } + log.Log.Info("ConfigureVfGUID(): set vf guid", "address", vfAddr, "guid", guid) + + return i.applyVfGUIDToInterface(guid, vfAddr, vfID, pfLink) +} + +func (i *infiniband) applyVfGUIDToInterface(guid net.HardwareAddr, vfAddr string, vfID int, pfLink netlink.Link) error { + if err := i.netlinkLib.LinkSetVfNodeGUID(pfLink, vfID, guid); err != nil { + return err + } + if err := i.netlinkLib.LinkSetVfPortGUID(pfLink, vfID, guid); err != nil { + return err + } + + return nil +} diff --git a/pkg/host/internal/infiniband/infiniband_test.go b/pkg/host/internal/infiniband/infiniband_test.go new file mode 100644 index 000000000..58b3a8fec --- /dev/null +++ b/pkg/host/internal/infiniband/infiniband_test.go @@ -0,0 +1,106 @@ +package infiniband + +import ( + "net" + + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + netlinkLibPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" + netlinkMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink/mock" + hostMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/mock" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/fakefilesystem" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/helpers" +) + +var _ = Describe("infiniband interface implementation", func() { + var ( + testCtrl *gomock.Controller + netlinkLibMock *netlinkMockPkg.MockNetlinkLib + hostMock *hostMockPkg.MockHostManagerInterface + ) + BeforeEach(func() { + testCtrl = gomock.NewController(GinkgoT()) + netlinkLibMock = netlinkMockPkg.NewMockNetlinkLib(testCtrl) + hostMock = hostMockPkg.NewMockHostManagerInterface(testCtrl) + }) + AfterEach(func() { + testCtrl.Finish() + }) + It("should create infiniband helper if guid config path is empty", func() { + netlinkLibMock.EXPECT().LinkList().Return([]netlinkLibPkg.Link{}, nil) + _, err := New(netlinkLibMock, hostMock, hostMock) + Expect(err).NotTo(HaveOccurred()) + }) + It("should assign guids if guid pool is nil", func() { + netlinkLibMock.EXPECT().LinkList().Return([]netlinkLibPkg.Link{}, nil) + var generatedGUID string + pfLinkMock := netlinkMockPkg.NewMockLink(testCtrl) + netlinkLibMock.EXPECT().LinkSetVfNodeGUID(pfLinkMock, 0, gomock.Any()).DoAndReturn( + func(link netlinkLibPkg.Link, vf int, nodeguid net.HardwareAddr) error { + // save generated GUID to validate that it is valid + generatedGUID = nodeguid.String() + return nil + }) + netlinkLibMock.EXPECT().LinkSetVfPortGUID(pfLinkMock, 0, gomock.Any()).Return(nil) + ib, err := New(netlinkLibMock, hostMock, hostMock) + Expect(err).NotTo(HaveOccurred()) + err = ib.ConfigureVfGUID("0000:d8:00.2", "0000:d8:00.0", 0, pfLinkMock) + Expect(err).NotTo(HaveOccurred()) + // validate that generated GUID is valid + _, err = ParseGUID(generatedGUID) + Expect(err).NotTo(HaveOccurred()) + }) + It("should assign guids if guid pool is not nil", func() { + var assignedGUID string + pfLinkMock := netlinkMockPkg.NewMockLink(testCtrl) + netlinkLibMock.EXPECT().LinkSetVfNodeGUID(pfLinkMock, 0, gomock.Any()).DoAndReturn( + func(link netlinkLibPkg.Link, vf int, nodeguid net.HardwareAddr) error { + // save generated GUID to validate that it is valid + assignedGUID = nodeguid.String() + return nil + }) + netlinkLibMock.EXPECT().LinkSetVfPortGUID(pfLinkMock, 0, gomock.Any()).Return(nil) + + guid, _ := ParseGUID("00:00:00:00:00:00:00:01") + pool := &ibGUIDPoolImpl{guidConfigs: map[string]ibPfGUIDConfig{"0000:d8:00.0": {GUIDs: []GUID{guid}}}} + + ib := &infiniband{guidPool: pool, netlinkLib: netlinkLibMock, kernelHelper: hostMock} + + err := ib.ConfigureVfGUID("0000:d8:00.2", "0000:d8:00.0", 0, pfLinkMock) + Expect(err).NotTo(HaveOccurred()) + // validate that generated GUID is valid + resultGUID, err := ParseGUID(assignedGUID) + Expect(err).NotTo(HaveOccurred()) + Expect(resultGUID).To(Equal(guid)) + }) + It("should read guids from the file", func() { + var assignedGUID string + netlinkLibMock.EXPECT().LinkList().Return([]netlinkLibPkg.Link{}, nil) + pfLinkMock := netlinkMockPkg.NewMockLink(testCtrl) + netlinkLibMock.EXPECT().LinkSetVfNodeGUID(pfLinkMock, 0, gomock.Any()).DoAndReturn( + func(link netlinkLibPkg.Link, vf int, nodeguid net.HardwareAddr) error { + // save generated GUID to validate that it is valid + assignedGUID = nodeguid.String() + return nil + }) + netlinkLibMock.EXPECT().LinkSetVfPortGUID(pfLinkMock, 0, gomock.Any()).Return(nil) + + mockJsonConfig := `[{"pciAddress":"0000:d8:00.0","guids":["00:01:02:03:04:05:06:07"]}]` + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/host" + consts.SriovConfBasePath + "/infiniband"}, + Files: map[string][]byte{"/host" + consts.InfinibandGUIDConfigFilePath: []byte(mockJsonConfig)}, + }) + + ib, err := New(netlinkLibMock, hostMock, hostMock) + Expect(err).NotTo(HaveOccurred()) + err = ib.ConfigureVfGUID("0000:d8:00.2", "0000:d8:00.0", 0, pfLinkMock) + Expect(err).NotTo(HaveOccurred()) + // validate that generated GUID is valid + resultGUID, err := ParseGUID(assignedGUID) + Expect(err).NotTo(HaveOccurred()) + Expect(resultGUID.String()).To(Equal("00:01:02:03:04:05:06:07")) + }) +}) diff --git a/pkg/host/internal/infiniband/suite_test.go b/pkg/host/internal/infiniband/suite_test.go new file mode 100644 index 000000000..fb584a6e0 --- /dev/null +++ b/pkg/host/internal/infiniband/suite_test.go @@ -0,0 +1,21 @@ +package infiniband + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "go.uber.org/zap/zapcore" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +func TestInfiniband(t *testing.T) { + log.SetLogger(zap.New( + zap.WriteTo(GinkgoWriter), + zap.Level(zapcore.Level(-2)), + zap.UseDevMode(true))) + RegisterFailHandler(Fail) + RunSpecs(t, "Package Infiniband Suite") +} diff --git a/pkg/host/internal/network/network.go b/pkg/host/internal/network/network.go index d04980eb8..b3014f9e9 100644 --- a/pkg/host/internal/network/network.go +++ b/pkg/host/internal/network/network.go @@ -404,3 +404,20 @@ func (n *network) GetNetDevLinkAdminState(ifaceName string) string { return consts.LinkAdminStateDown } + +// GetPciAddressFromInterfaceName parses sysfs to get pci address of an interface by name +func (n *network) GetPciAddressFromInterfaceName(interfaceName string) (string, error) { + log.Log.V(2).Info("GetPciAddressFromInterfaceName(): get pci address", "interface", interfaceName) + sysfsPath := filepath.Join(vars.FilesystemRoot, consts.SysClassNet, interfaceName, "device") + + pciDevDir, err := os.Readlink(sysfsPath) + + if err != nil { + log.Log.Error(err, "GetPciAddressFromInterfaceName(): failed to get pci device dir", "interface", interfaceName) + return "", err + } + + pciAddress := filepath.Base(pciDevDir) + log.Log.V(2).Info("GetPciAddressFromInterfaceName(): result", "interface", interfaceName, "pci address", pciAddress) + return pciAddress, nil +} diff --git a/pkg/host/internal/network/network_test.go b/pkg/host/internal/network/network_test.go index 350423745..19eb3f438 100644 --- a/pkg/host/internal/network/network_test.go +++ b/pkg/host/internal/network/network_test.go @@ -271,4 +271,16 @@ var _ = Describe("Network", func() { Expect(index).To(Equal(-1)) }) }) + Context("GetPciAddressFromInterfaceName", func() { + It("Should get PCI address from sys fs", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/sys/bus/pci/0000:3b:00.0", "/sys/class/net/ib216s0f0"}, + Symlinks: map[string]string{"/sys/class/net/ib216s0f0/device": "/sys/bus/pci/0000:3b:00.0"}, + }) + + pci, err := n.GetPciAddressFromInterfaceName("ib216s0f0") + Expect(err).NotTo(HaveOccurred()) + Expect(pci).To(Equal("0000:3b:00.0")) + }) + }) }) diff --git a/pkg/host/internal/sriov/sriov_test.go b/pkg/host/internal/sriov/sriov_test.go index 6b81f9957..6d2e43a71 100644 --- a/pkg/host/internal/sriov/sriov_test.go +++ b/pkg/host/internal/sriov/sriov_test.go @@ -227,7 +227,7 @@ var _ = Describe("SRIOV", func() { pfLinkMock.EXPECT().Attrs().Return(&netlink.LinkAttrs{Flags: 0, EncapType: "ether"}) netlinkLibMock.EXPECT().IsLinkAdminStateUp(pfLinkMock).Return(false) netlinkLibMock.EXPECT().LinkSetUp(pfLinkMock).Return(nil) - netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() + netlinkLibMock.EXPECT().LinkList().Return(nil, nil).Times(0) dputilsLibMock.EXPECT().GetVFID("0000:d8:00.2").Return(0, nil).Times(2) hostMock.EXPECT().HasDriver("0000:d8:00.2").Return(false, "") diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index e475bcff6..73d3ca995 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -3,8 +3,6 @@ package utils import ( "bytes" "fmt" - "math/rand" - "net" "os" "os/exec" "path/filepath" @@ -65,19 +63,6 @@ func (u *utilsHelper) RunCommand(command string, args ...string) (string, string return stdout.String(), stderr.String(), err } -func GenerateRandomGUID() net.HardwareAddr { - guid := make(net.HardwareAddr, 8) - - // First field is 0x01 - xfe to avoid all zero and all F invalid guids - guid[0] = byte(1 + rand.Intn(0xfe)) - - for i := 1; i < len(guid); i++ { - guid[i] = byte(rand.Intn(0x100)) - } - - return guid -} - func IsCommandNotFound(err error) bool { if exitErr, ok := err.(*exec.ExitError); ok { if status, ok := exitErr.Sys().(syscall.WaitStatus); ok && status.ExitStatus() == 127 {