From 62726d3005fced82db46fee4ba5a07ca4cc6787c Mon Sep 17 00:00:00 2001 From: cyclinder Date: Fri, 10 Mar 2023 18:37:24 +0800 Subject: [PATCH] fix: gw field to adding route --- Makefile | 1 + pkg/networking/networking.go | 2 +- pkg/networking/route.go | 42 ++++++++++++++++++++++++++++++------ plugins/veth/veth.go | 21 ++++++++++++------ 4 files changed, 52 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index ad8bf83..6f868a9 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ build: @mkdir -p ./.tmp/bin ; \ for plugin in `ls ./plugins/` ; do \ echo "\033[35m ==> building $${plugin} to $(ROOT_DIR)/.tmp/bin/${plugin} \033[0m" ; \ + echo "\033[35m ==> $(GO_BUILD_FLAGS) $(GO_BUILD) $(GO_BUILD_LDFLGAS) -o ./.tmp/bin/$${plugin} ./plugins/$${plugin} \033[0m"; \ $(GO_BUILD_FLAGS) $(GO_BUILD) $(GO_BUILD_LDFLGAS) -o ./.tmp/bin/$${plugin} ./plugins/$${plugin} ; \ done diff --git a/pkg/networking/networking.go b/pkg/networking/networking.go index 0b07a8d..69dabb9 100644 --- a/pkg/networking/networking.go +++ b/pkg/networking/networking.go @@ -20,7 +20,7 @@ import ( var DefaultInterfacesToExclude = []string{ "docker.*", "cbr.*", "dummy.*", "virbr.*", "lxcbr.*", "veth.*", "lo", - "cali.*", "tunl.*", "flannel.*", "kube-ipvs.*", "cni.*", + "cali.*", "tunl.*", "flannel.*", "kube-ipvs.*", "cni.*", "vx-submariner", } func GetIPFamily(prevResult cnitypes.Result) (int, error) { diff --git a/pkg/networking/route.go b/pkg/networking/route.go index 7251328..f069443 100644 --- a/pkg/networking/route.go +++ b/pkg/networking/route.go @@ -1,14 +1,15 @@ package networking import ( + "fmt" "github.com/vishvananda/netlink" "go.uber.org/zap" "net" "os" ) -func AddRouteTable(logger *zap.Logger, ruleTable int, iface string, destinations []string) error { - link, err := netlink.LinkByName(iface) +func AddRouteTable(logger *zap.Logger, ruleTable int, scope netlink.Scope, device string, destinations []string, v4Gw, v6Gw net.IP) error { + link, err := netlink.LinkByName(device) if err != nil { logger.Error(err.Error()) return err @@ -21,15 +22,44 @@ func AddRouteTable(logger *zap.Logger, ruleTable int, iface string, destinations return err } - if err = netlink.RouteAdd(&netlink.Route{ + route := &netlink.Route{ LinkIndex: link.Attrs().Index, - Scope: netlink.SCOPE_LINK, + Scope: scope, Dst: ipNet, Table: ruleTable, - }); err != nil && !os.IsExist(err) { - logger.Error("failed to add route", zap.String("interface", iface), zap.String("dst", ipNet.String()), zap.Error(err)) + } + + if ipNet.IP.To4() != nil && v4Gw != nil { + route.Gw = v4Gw + } + + if ipNet.IP.To4() == nil && v6Gw != nil { + route.Gw = v6Gw + } + + if err = netlink.RouteAdd(route); err != nil && !os.IsExist(err) { + logger.Error("failed to RouteAdd", zap.String("route", route.String()), zap.Error(err)) return err } } return nil } + +func GetGatewayIP(addrs []netlink.Addr) (v4Gw, v6Gw net.IP, err error) { + for _, addr := range addrs { + routes, err := netlink.RouteGet(addr.IP) + if err != nil { + return nil, nil, fmt.Errorf("failed to RouteGet Pod IP(%s): %v", addr.IP.String(), err) + } + + if len(routes) > 0 { + if addr.IP.To4() != nil && v4Gw == nil { + v4Gw = routes[0].Src + } + if addr.IP.To4() == nil && v6Gw == nil { + v6Gw = routes[0].Src + } + } + } + return +} diff --git a/plugins/veth/veth.go b/plugins/veth/veth.go index 626410a..1d298f6 100644 --- a/plugins/veth/veth.go +++ b/plugins/veth/veth.go @@ -111,9 +111,9 @@ func cmdAdd(args *skel.CmdArgs) error { } if !isfirstInterface { - logger.Info("Calling veth plugin not for the first time", zap.Any("config", conf)) + logger.Info("Calling veth plugin not for the first time", zap.Any("config", conf), zap.String("netns", netns.Path())) } else { - logger.Info("Calling veth plugin for first time", zap.Any("config", conf)) + logger.Info("Calling veth plugin for first time", zap.Any("config", conf), zap.String("netns", netns.Path())) } var hostVethPairName string @@ -270,11 +270,17 @@ func setupNeighborhood(logger *zap.Logger, netns ns.NetNS, hostVethPairName stri // setupRoutes setup routes for pod and host // equivalent to: `ip route add $route` func setupRoutes(logger *zap.Logger, netns ns.NetNS, ruleTable int, hostVethPairName string, ipAddressOnNode, preInterfaceIPAddress []netlink.Addr, conf *ptypes.Veth) error { - err := netns.Do(func(_ ns.NetNS) error { + v4Gw, v6Gw, err := networking.GetGatewayIP(preInterfaceIPAddress) + if err != nil { + logger.Error("failed to GetGatewayIP", zap.Error(err)) + return err + } + + err = netns.Do(func(_ ns.NetNS) error { var err error // traffic sent to the node is forwarded via veth0 // eq: "ip r add dev veth0 table " - if err = networking.AddRouteTable(logger, ruleTable, defaultConVeth, networking.AddrsToString(ipAddressOnNode)); err != nil { + if err = networking.AddRouteTable(logger, ruleTable, netlink.SCOPE_LINK, defaultConVeth, networking.AddrsToString(ipAddressOnNode), nil, nil); err != nil { logger.Error("failed to AddRouteTable for ipAddressOnNode", zap.Error(err)) return fmt.Errorf("failed to AddRouteTable for ipAddressOnNode: %v", err) } @@ -283,7 +289,7 @@ func setupRoutes(logger *zap.Logger, netns ns.NetNS, ruleTable int, hostVethPair // eq: ip route add dev veth0 localCIDRs := append(conf.ClusterCIDR, conf.ServiceCIDR...) localCIDRs = append(localCIDRs, conf.AdditionalCIDR...) - if err := networking.AddRouteTable(logger, ruleTable, defaultConVeth, localCIDRs); err != nil { + if err = networking.AddRouteTable(logger, ruleTable, netlink.SCOPE_UNIVERSE, defaultConVeth, localCIDRs, v4Gw, v6Gw); err != nil { logger.Error("failed to AddRouteTable for localCIDRs", zap.Error(err)) return fmt.Errorf("failed to AddRouteTable for localCIDRs: %v", err) } @@ -292,7 +298,7 @@ func setupRoutes(logger *zap.Logger, netns ns.NetNS, ruleTable int, hostVethPair // make sure that all traffic to second NIC to lookup table <> // eq: ip rule add to lookup table if ruleTable != unix.RT_TABLE_MAIN { - if err := networking.AddToRuleTable(preInterfaceIPAddress, ruleTable); err != nil { + if err = networking.AddToRuleTable(preInterfaceIPAddress, ruleTable); err != nil { logger.Error("failed to AddToRuleTable", zap.Error(err)) return fmt.Errorf("failed to AddToRuleTable: %v", err) } @@ -307,7 +313,8 @@ func setupRoutes(logger *zap.Logger, netns ns.NetNS, ruleTable int, hostVethPair // set routes for host // equivalent: ip add dev veth-peer on host - if err = networking.AddRouteTable(logger, unix.RT_TABLE_MAIN, hostVethPairName, networking.AddrsToString(preInterfaceIPAddress)); err != nil { + if err = networking.AddRouteTable(logger, unix.RT_TABLE_MAIN, netlink.SCOPE_UNIVERSE, hostVethPairName, networking.AddrsToString(preInterfaceIPAddress), + nil, nil); err != nil { logger.Error("failed to AddRouteTable for preInterfaceIPAddress", zap.Error(err)) return fmt.Errorf("failed to AddRouteTable for preInterfaceIPAddress: %v", err) }