diff --git a/pkg/controllers/proxy/linux_networking.go b/pkg/controllers/proxy/linux_networking.go index 503023eca4..047f7315e5 100644 --- a/pkg/controllers/proxy/linux_networking.go +++ b/pkg/controllers/proxy/linux_networking.go @@ -5,7 +5,6 @@ import ( "fmt" "net" "os" - "os/exec" "path" "strconv" "strings" @@ -18,12 +17,15 @@ import ( "github.com/moby/ipvs" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" + "golang.org/x/sys/unix" "k8s.io/klog/v2" ) const ( - ipv4NetMaskBits = 32 - ipv6NetMaskBits = 128 + ipv4NetMaskBits = 32 + ipv4DefaultRoute = "0.0.0.0/0" + ipv6NetMaskBits = 128 + ipv6DefaultRoute = "::/0" // TODO: it's bad to rely on eth0 here. While this is inside the container's namespace and is determined by the // container runtime and so far we've been able to count on this being reliably set to eth0, it is possible that @@ -65,7 +67,6 @@ type netlinkCalls interface { func (ln *linuxNetworking) ipAddrDel(iface netlink.Link, ip string, nodeIP string) error { var netMask net.IPMask - var ipRouteCmdArgs []string parsedIP := net.ParseIP(ip) parsedNodeIP := net.ParseIP(nodeIP) if parsedIP.To4() != nil { @@ -75,7 +76,6 @@ func (ln *linuxNetworking) ipAddrDel(iface netlink.Link, ip string, nodeIP strin } netMask = net.CIDRMask(ipv4NetMaskBits, ipv4NetMaskBits) - ipRouteCmdArgs = make([]string, 0) } else { // If the IP family of the NodeIP and the VIP IP don't match, we can't proceed if parsedNodeIP.To4() != nil { @@ -88,7 +88,6 @@ func (ln *linuxNetworking) ipAddrDel(iface netlink.Link, ip string, nodeIP strin } netMask = net.CIDRMask(ipv6NetMaskBits, ipv6NetMaskBits) - ipRouteCmdArgs = []string{"-6"} } naddr := &netlink.Addr{IPNet: &net.IPNet{IP: parsedIP, Mask: netMask}, Scope: syscall.RT_SCOPE_LINK} @@ -106,13 +105,20 @@ func (ln *linuxNetworking) ipAddrDel(iface netlink.Link, ip string, nodeIP strin // Delete VIP addition to "local" rt table also, fail silently if not found (DSR special case) // #nosec G204 - ipRouteCmdArgs = append(ipRouteCmdArgs, "route", "delete", "local", ip, "dev", KubeDummyIf, - "table", "local", "proto", "kernel", "scope", "host", "src", nodeIP, "table", "local") - out, err := exec.Command("ip", ipRouteCmdArgs...).CombinedOutput() + nRoute := &netlink.Route{ + Type: unix.RTN_LOCAL, + Dst: &net.IPNet{IP: parsedIP, Mask: netMask}, + LinkIndex: iface.Attrs().Index, + Table: syscall.RT_TABLE_LOCAL, + Protocol: unix.RTPROT_KERNEL, + Scope: syscall.RT_SCOPE_HOST, + Src: parsedNodeIP, + } + err = netlink.RouteDel(nRoute) if err != nil { - if !strings.Contains(string(out), "No such process") { - klog.Errorf("Failed to delete route to service VIP %s configured on %s. Error: %v, Output: %s", - ip, KubeDummyIf, err, out) + if !strings.Contains(err.Error(), "no such process") { + klog.Errorf("Failed to delete route to service VIP %s configured on %s. Error: %v", + ip, iface.Attrs().Name, err) } else { klog.Warningf("got a No such process error while trying to remove route: %v (this is not normally bad "+ "enough to stop processing)", err) @@ -128,7 +134,6 @@ func (ln *linuxNetworking) ipAddrDel(iface netlink.Link, ip string, nodeIP strin // inside the container. func (ln *linuxNetworking) ipAddrAdd(iface netlink.Link, ip string, nodeIP string, addRoute bool) error { var netMask net.IPMask - var ipRouteCmdArgs []string parsedIP := net.ParseIP(ip) parsedNodeIP := net.ParseIP(nodeIP) if parsedIP.To4() != nil { @@ -138,7 +143,6 @@ func (ln *linuxNetworking) ipAddrAdd(iface netlink.Link, ip string, nodeIP strin } netMask = net.CIDRMask(ipv4NetMaskBits, ipv4NetMaskBits) - ipRouteCmdArgs = make([]string, 0) } else { // If we're supposed to add a route and the IP family of the NodeIP and the VIP IP don't match, we can't proceed if addRoute && parsedNodeIP.To4() != nil { @@ -146,10 +150,10 @@ func (ln *linuxNetworking) ipAddrAdd(iface netlink.Link, ip string, nodeIP strin } netMask = net.CIDRMask(ipv6NetMaskBits, ipv6NetMaskBits) - ipRouteCmdArgs = []string{"-6"} } - naddr := &netlink.Addr{IPNet: &net.IPNet{IP: parsedIP, Mask: netMask}, Scope: syscall.RT_SCOPE_LINK} + ipPrefix := &net.IPNet{IP: parsedIP, Mask: netMask} + naddr := &netlink.Addr{IPNet: ipPrefix, Scope: syscall.RT_SCOPE_LINK} err := netlink.AddrAdd(iface, naddr) if err != nil && err.Error() != IfaceHasAddr { klog.Errorf("failed to assign cluster ip %s to dummy interface: %s", naddr.IPNet.IP.String(), err.Error()) @@ -164,16 +168,24 @@ func (ln *linuxNetworking) ipAddrAdd(iface netlink.Link, ip string, nodeIP strin return nil } - // TODO: netlink.RouteReplace which is replacement for below command is not working as expected. Call succeeds but - // route is not replaced. For now do it with command. - // #nosec G204 - ipRouteCmdArgs = append(ipRouteCmdArgs, "route", "replace", "local", ip, "dev", KubeDummyIf, - "table", "local", "proto", "kernel", "scope", "host", "src", nodeIP, "table", "local") - - out, err := exec.Command("ip", ipRouteCmdArgs...).CombinedOutput() + kubeDummyLink, err := netlink.LinkByName(KubeDummyIf) + if err != nil { + klog.Errorf("failed to get %s link due to %v", KubeDummyIf, err) + return err + } + nRoute := &netlink.Route{ + Type: unix.RTN_LOCAL, + Dst: ipPrefix, + LinkIndex: kubeDummyLink.Attrs().Index, + Table: syscall.RT_TABLE_LOCAL, + Protocol: unix.RTPROT_KERNEL, + Scope: syscall.RT_SCOPE_HOST, + Src: parsedNodeIP, + } + err = netlink.RouteReplace(nRoute) if err != nil { - klog.Errorf("Failed to replace route to service VIP %s configured on %s. Error: %v, Output: %s", - ip, KubeDummyIf, err, out) + klog.Errorf("Failed to replace route to service VIP %s configured on %s. Error: %v", + ip, KubeDummyIf, err) return err } @@ -431,24 +443,56 @@ func (ln *linuxNetworking) setupPolicyRoutingForDSR(setupIPv4, setupIPv6 bool) e return fmt.Errorf("failed to setup policy routing required for DSR due to %v", err) } + loNetLink, err := netlink.LinkByName("lo") + if err != nil { + return fmt.Errorf("failed to get loopback interface due to %v", err) + } + if setupIPv4 { - out, err := exec.Command("ip", "route", "list", "table", customDSRRouteTableID).Output() - if err != nil || !strings.Contains(string(out), " lo ") { - if err = exec.Command("ip", "route", "add", "local", "default", "dev", "lo", "table", - customDSRRouteTableID).Run(); err != nil { - return fmt.Errorf("failed to add route in custom route table due to: %v", err) + nFamily := netlink.FAMILY_V4 + _, defaultRouteCIDR, err := net.ParseCIDR(ipv4DefaultRoute) + if err != nil { + //nolint:goconst // This is a static value and should not be changed + return fmt.Errorf("failed to parse default (%s) route (this is statically defined, so if you see this "+ + "error please report because something has gone very wrong) due to: %v", ipv4DefaultRoute, err) + } + nRoute := &netlink.Route{ + Type: unix.RTN_LOCAL, + Dst: defaultRouteCIDR, + LinkIndex: loNetLink.Attrs().Index, + Table: customDSRRouteTableID, + } + routes, err := netlink.RouteListFiltered(nFamily, nRoute, netlink.RT_FILTER_TABLE|netlink.RT_FILTER_OIF) + if err != nil || len(routes) < 1 { + err = netlink.RouteAdd(nRoute) + if err != nil { + return fmt.Errorf("failed to add route to custom route table for DSR due to: %v", err) } } } + if setupIPv6 { - out, err := exec.Command("ip", "-6", "route", "list", "table", customDSRRouteTableID).Output() - if err != nil || !strings.Contains(string(out), " lo ") { - if err = exec.Command("ip", "-6", "route", "add", "local", "default", "dev", "lo", "table", - customDSRRouteTableID).Run(); err != nil { - return fmt.Errorf("failed to add route in custom route table due to: %v", err) + nFamily := netlink.FAMILY_V6 + _, defaultRouteCIDR, err := net.ParseCIDR(ipv6DefaultRoute) + if err != nil { + return fmt.Errorf("failed to parse default (%s) route (this is statically defined, so if you see this "+ + "error please report because something has gone very wrong) due to: %v", ipv6DefaultRoute, err) + } + nRoute := &netlink.Route{ + Type: unix.RTN_LOCAL, + Dst: defaultRouteCIDR, + LinkIndex: loNetLink.Attrs().Index, + Table: customDSRRouteTableID, + } + routes, err := netlink.RouteListFiltered(nFamily, nRoute, netlink.RT_FILTER_TABLE|netlink.RT_FILTER_OIF) + if err != nil || len(routes) < 1 { + err = netlink.RouteAdd(nRoute) + if err != nil { + return fmt.Errorf("failed to add route to custom route table for DSR due to: %v", err) } } } + return nil } @@ -456,7 +500,6 @@ func (ln *linuxNetworking) setupPolicyRoutingForDSR(setupIPv4, setupIPv6 bool) e // directly responds back with source IP as external IP kernel will treat as martian packet. // To prevent martian packets add route to external IP through the `kube-bridge` interface // setupRoutesForExternalIPForDSR: setups routing so that kernel does not think return packets as martians - func (ln *linuxNetworking) setupRoutesForExternalIPForDSR(serviceInfoMap serviceInfoMap, setupIPv4, setupIPv6 bool) error { err := utils.RouteTableAdd(externalIPRouteTableID, externalIPRouteTableName) @@ -464,27 +507,45 @@ func (ln *linuxNetworking) setupRoutesForExternalIPForDSR(serviceInfoMap service return fmt.Errorf("failed to setup policy routing required for DSR due to %v", err) } - setupIPRulesAndRoutes := func(ipArgs []string) error { - out, err := runIPCommandsWithArgs(ipArgs, "rule", "list").Output() + setupIPRulesAndRoutes := func(isIPv6 bool) error { + nFamily := netlink.FAMILY_V4 + _, defaultPrefixCIDR, err := net.ParseCIDR(ipv4DefaultRoute) + if isIPv6 { + nFamily = netlink.FAMILY_V6 + _, defaultPrefixCIDR, err = net.ParseCIDR(ipv6DefaultRoute) + } + if err != nil { + return fmt.Errorf("failed to parse default route (this is statically defined, so if you see this "+ + "error please report because something has gone very wrong) due to: %v", err) + } + + nRule := &netlink.Rule{ + Priority: defaultDSRPolicyRulePriority, + Src: defaultPrefixCIDR, + Table: externalIPRouteTableID, + } + rules, err := netlink.RuleListFiltered(nFamily, nRule, + netlink.RT_FILTER_TABLE|netlink.RT_FILTER_SRC|netlink.RT_FILTER_PRIORITY) if err != nil { - return fmt.Errorf("failed to verify if `ip rule add prio 32765 from all lookup external_ip` exists due to: %v", - err) + return fmt.Errorf("failed to list rule for external IP's and verify if `ip rule add prio 32765 from all "+ + "lookup external_ip` exists due to: %v", err) } - if !(strings.Contains(string(out), externalIPRouteTableName) || - strings.Contains(string(out), externalIPRouteTableID)) { - err = runIPCommandsWithArgs(ipArgs, "rule", "add", "prio", "32765", "from", "all", "lookup", - externalIPRouteTableID).Run() + if len(rules) < 1 { + err = netlink.RuleAdd(nRule) if err != nil { klog.Infof("Failed to add policy rule `ip rule add prio 32765 from all lookup external_ip` due to %v", - err.Error()) + err) return fmt.Errorf("failed to add policy rule `ip rule add prio 32765 from all lookup external_ip` "+ "due to %v", err) } } - out, _ = runIPCommandsWithArgs(ipArgs, "route", "list", "table", externalIPRouteTableID).Output() - outStr := string(out) + kubeBridgeLink, err := netlink.LinkByName(KubeBridgeIf) + if err != nil { + return fmt.Errorf("failed to get kube-bridge interface due to %v", err) + } + activeExternalIPs := make(map[string]bool) for _, svc := range serviceInfoMap { for _, externalIP := range svc.externalIPs { @@ -497,9 +558,21 @@ func (ln *linuxNetworking) setupRoutesForExternalIPForDSR(serviceInfoMap service activeExternalIPs[externalIP] = true - if !strings.Contains(outStr, externalIP) { - if err = runIPCommandsWithArgs(ipArgs, "route", "add", externalIP, "dev", "kube-bridge", "table", - externalIPRouteTableID).Run(); err != nil { + nSrcIP := net.ParseIP(externalIP) + nRoute := &netlink.Route{ + Src: nSrcIP, + LinkIndex: kubeBridgeLink.Attrs().Index, + Table: externalIPRouteTableID, + } + + routes, err := netlink.RouteListFiltered(nFamily, nRoute, + netlink.RT_FILTER_SRC|netlink.RT_FILTER_TABLE|netlink.RT_FILTER_OIF) + if err != nil { + return fmt.Errorf("failed to list route for external IP's due to: %s", err) + } + if len(routes) < 1 { + err = netlink.RouteAdd(nRoute) + if err != nil { klog.Errorf("Failed to add route for %s in custom route table for external IP's due to: %v", externalIP, err) continue @@ -509,19 +582,18 @@ func (ln *linuxNetworking) setupRoutesForExternalIPForDSR(serviceInfoMap service } // check if there are any pbr in externalIPRouteTableID for external IP's - if len(outStr) > 0 { - // clean up stale external IPs - for _, line := range strings.Split(strings.Trim(outStr, "\n"), "\n") { - route := strings.Split(strings.Trim(line, " "), " ") - ip := route[0] - if !activeExternalIPs[ip] { - args := []string{"route", "del", "table", externalIPRouteTableID} - args = append(args, route...) - if err = runIPCommandsWithArgs(ipArgs, args...).Run(); err != nil { - klog.Errorf("Failed to del route for %v in custom route table for external IP's due to: %s", - ip, err) - continue - } + routes, err := netlink.RouteList(nil, nFamily) + if err != nil { + return fmt.Errorf("failed to list route for external IP's due to: %s", err) + } + for idx, route := range routes { + ip := route.Src.String() + if !activeExternalIPs[ip] { + err = netlink.RouteDel(&routes[idx]) + if err != nil { + klog.Errorf("Failed to del route for %v in custom route table for external IP's due to: %s", + ip, err) + continue } } } @@ -530,13 +602,13 @@ func (ln *linuxNetworking) setupRoutesForExternalIPForDSR(serviceInfoMap service } if setupIPv4 { - err = setupIPRulesAndRoutes([]string{}) + err = setupIPRulesAndRoutes(false) if err != nil { return err } } if setupIPv6 { - err = setupIPRulesAndRoutes([]string{"-6"}) + err = setupIPRulesAndRoutes(true) if err != nil { return err } diff --git a/pkg/controllers/proxy/network_services_controller.go b/pkg/controllers/proxy/network_services_controller.go index cbe57ffcd0..67009ab59f 100644 --- a/pkg/controllers/proxy/network_services_controller.go +++ b/pkg/controllers/proxy/network_services_controller.go @@ -32,6 +32,7 @@ const ( KubeDummyIf = "kube-dummy-if" KubeTunnelIfv4 = "kube-tunnel-if" KubeTunnelIfv6 = "kube-tunnel-v6" + KubeBridgeIf = "kube-bridge" IfaceNotFound = "Link not found" IfaceHasAddr = "file exists" IfaceHasNoAddr = "cannot assign requested address" @@ -41,11 +42,13 @@ const ( IpvsSvcFSched2 = "flag-2" IpvsSvcFSched3 = "flag-3" - customDSRRouteTableID = "78" - customDSRRouteTableName = "kube-router-dsr" - externalIPRouteTableID = "79" - externalIPRouteTableName = "external_ip" - kubeRouterProxyName = "kube-router" + customDSRRouteTableID = 78 + customDSRRouteTableName = "kube-router-dsr" + externalIPRouteTableID = 79 + externalIPRouteTableName = "external_ip" + kubeRouterProxyName = "kube-router" + defaultTrafficDirectorRulePriority = 32764 + defaultDSRPolicyRulePriority = 32765 // Taken from https://github.com/torvalds/linux/blob/master/include/uapi/linux/ip_vs.h#L21 ipvsPersistentFlagHex = 0x0001 @@ -1721,23 +1724,35 @@ func (nsc *NetworkServicesController) cleanupMangleTableRule(ip string, protocol // http://www.austintek.com/LVS/LVS-HOWTO/HOWTO/LVS-HOWTO.routing_to_VIP-less_director.html // routeVIPTrafficToDirector: setups policy routing so that FWMARKed packets are delivered locally func routeVIPTrafficToDirector(fwmark string, family v1.IPFamily) error { - ipArgs := make([]string, 0) + nFamily := netlink.FAMILY_V4 if family == v1.IPv6Protocol { - ipArgs = append(ipArgs, "-6") + nFamily = netlink.FAMILY_V6 } - out, err := runIPCommandsWithArgs(ipArgs, "rule", "list").Output() + iFWMark, err := strconv.Atoi(fwmark) if err != nil { - return errors.New("Failed to verify if `ip rule` exists due to: " + err.Error()) + return fmt.Errorf("failed to convert fwmark to integer due to: %v", err) } - if !strings.Contains(string(out), fwmark+" ") { - err = runIPCommandsWithArgs(ipArgs, "rule", "add", "prio", "32764", "fwmark", fwmark, "table", - customDSRRouteTableID).Run() + + nRule := &netlink.Rule{ + Mark: iFWMark, + Table: customDSRRouteTableID, + Priority: defaultTrafficDirectorRulePriority, + } + + routes, err := netlink.RuleListFiltered(nFamily, nRule, netlink.RT_FILTER_MARK|netlink.RT_FILTER_TABLE) + if err != nil { + return fmt.Errorf("failed to verify if `ip rule` exists due to: %v", err) + } + + if len(routes) < 1 { + err = netlink.RuleAdd(nRule) if err != nil { - return errors.New("Failed to add policy rule to lookup traffic to VIP through the custom " + - " routing table due to " + err.Error()) + return fmt.Errorf("failed to add policy rule to lookup traffic to VIP through the custom "+ + "routing table due to %v", err) } } + return nil } diff --git a/pkg/controllers/proxy/utils.go b/pkg/controllers/proxy/utils.go index 1e37294a06..2a5ed82a03 100644 --- a/pkg/controllers/proxy/utils.go +++ b/pkg/controllers/proxy/utils.go @@ -6,7 +6,6 @@ import ( "fmt" "hash/fnv" "net" - "os/exec" "runtime" "strconv" "strings" @@ -559,14 +558,6 @@ func getIPVSFirewallInputChainRule(family v1.IPFamily) []string { "-j", ipvsFirewallChainName} } -// runIPCommandsWithArgs extend the exec.Command interface to allow passing an additional array of arguments to ip -func runIPCommandsWithArgs(ipArgs []string, additionalArgs ...string) *exec.Cmd { - var allArgs []string - allArgs = append(allArgs, ipArgs...) - allArgs = append(allArgs, additionalArgs...) - return exec.Command("ip", allArgs...) -} - // getLabelFromMap checks the list of passed labels for the service.kubernetes.io/service-proxy-name // label and if it exists, returns it otherwise returns an error func getLabelFromMap(label string, labels map[string]string) (string, error) { diff --git a/pkg/controllers/routing/network_routes_controller.go b/pkg/controllers/routing/network_routes_controller.go index d6450d994e..688ef1e12c 100644 --- a/pkg/controllers/routing/network_routes_controller.go +++ b/pkg/controllers/routing/network_routes_controller.go @@ -33,7 +33,7 @@ import ( const ( IfaceNotFound = "Link not found" - customRouteTableID = "77" + customRouteTableID = 77 customRouteTableName = "kube-router" podSubnetsIPSetName = "kube-router-pod-subnets" nodeAddrsIPSetName = "kube-router-node-ips" @@ -753,11 +753,11 @@ func (nrc *NetworkRoutingController) cleanupTunnel(destinationSubnet *net.IPNet, // setupOverlayTunnel attempts to create a tunnel link and corresponding routes for IPIP based overlay networks func (nrc *NetworkRoutingController) setupOverlayTunnel(tunnelName string, nextHop net.IP, nextHopSubnet *net.IPNet) (netlink.Link, error) { - var out []byte link, err := netlink.LinkByName(tunnelName) var bestIPForFamily net.IP var ipipMode, fouLinkType string + var nFamily int isIPv6 := false ipBase := make([]string, 0) strFormattedEncapPort := strconv.FormatInt(int64(nrc.overlayEncapPort), 10) @@ -766,12 +766,14 @@ func (nrc *NetworkRoutingController) setupOverlayTunnel(tunnelName string, nextH bestIPForFamily = utils.FindBestIPv4NodeAddress(nrc.primaryIP, nrc.nodeIPv4Addrs) ipipMode = encapTypeIPIP fouLinkType = ipipModev4 + nFamily = netlink.FAMILY_V4 } else { // Need to activate the ip command in IPv6 mode ipBase = append(ipBase, "-6") bestIPForFamily = utils.FindBestIPv6NodeAddress(nrc.primaryIP, nrc.nodeIPv6Addrs) ipipMode = ipipModev6 fouLinkType = "ip6tnl" + nFamily = netlink.FAMILY_V6 isIPv6 = true } if nil == bestIPForFamily { @@ -799,13 +801,17 @@ func (nrc *NetworkRoutingController) setupOverlayTunnel(tunnelName string, nextH // If we are transitioning from FoU to IPIP we also need to clean up the old FoU port if it exists if fouPortAndProtoExist(nrc.overlayEncapPort, isIPv6) { - fouArgs := ipBase - fouArgs = append(fouArgs, "fou", "del", "port", strFormattedEncapPort) - out, err := exec.Command("ip", fouArgs...).CombinedOutput() + nFOU := &netlink.Fou{ + Family: nFamily, + Port: int(nrc.overlayEncapPort), + EncapType: netlink.FOU_ENCAP_GUE, + } + + err = netlink.FouDel(*nFOU) if err != nil { - klog.Warningf("failed to clean up previous FoU tunnel port (this is only a warning because it "+ - "won't stop kube-router from working for now, but still shouldn't have happened) - error: "+ - "%v, output %s", err, out) + klog.Errorf("failed to clean up previous FoU tunnel (%s) port (this is only a warning because "+ + "it won't stop kube-router from working for now, but still shouldn't have happened) - "+ + "error: %v", tunnelName, err) } } } @@ -835,13 +841,16 @@ func (nrc *NetworkRoutingController) setupOverlayTunnel(tunnelName string, nextH case encapTypeFOU: // Ensure that the FOU tunnel port is set correctly if !fouPortAndProtoExist(nrc.overlayEncapPort, isIPv6) { - fouArgs := ipBase - fouArgs = append(fouArgs, "fou", "add", "port", strFormattedEncapPort, "gue") - out, err := exec.Command("ip", fouArgs...).CombinedOutput() + nFOU := netlink.Fou{ + Family: nFamily, + Port: int(nrc.overlayEncapPort), + EncapType: netlink.FOU_ENCAP_GUE, + } + err = netlink.FouAdd(nFOU) if err != nil { - //nolint:goconst // don't need to make error messages a constant + //nolint:goconst // This does not need to be abstracted return nil, fmt.Errorf("route not injected for the route advertised by the node %s "+ - "Failed to set FoU tunnel port - error: %s, output: %s", tunnelName, err, string(out)) + "Failed to set FoU tunnel port - error: %s", tunnelName, err) } } @@ -856,6 +865,10 @@ func (nrc *NetworkRoutingController) setupOverlayTunnel(tunnelName string, nextH } klog.V(2).Infof("Executing the following command to create tunnel: ip %s", cmdArgs) + // TODO: we should make this call via the netlink library whenever they support ip6ip6 tunnels. Right now, it + // doesn't appear like the project supports them. They support something else called an ip6tun which seems + // similar, but we would probably need to figure out what combination of primitives we need to serialize in + // order to get the same effect we get below. out, err := exec.Command("ip", cmdArgs...).CombinedOutput() if err != nil { return nil, fmt.Errorf("route not injected for the route advertised by the node %s "+ @@ -875,17 +888,16 @@ func (nrc *NetworkRoutingController) setupOverlayTunnel(tunnelName string, nextH // Now that the tunnel link exists, we need to add a route to it, so the node knows where to send traffic bound for // this interface - //nolint:gocritic // we understand that we are appending to a new slice - cmdArgs := append(ipBase, "route", "list", "table", customRouteTableID) - out, err = exec.Command("ip", cmdArgs...).CombinedOutput() - // This used to be "dev "+tunnelName+" scope" but this isn't consistent with IPv6's output, so we changed it to just - // "dev "+tunnelName, but at this point I'm unsure if there was a good reason for adding scope on before, so that's - // why this comment is here. - if err != nil || !strings.Contains(string(out), "dev "+tunnelName) { - //nolint:gocritic // we understand that we are appending to a new slice - cmdArgs = append(ipBase, "route", "add", nextHop.String(), "dev", tunnelName, "table", customRouteTableID) - if out, err = exec.Command("ip", cmdArgs...).CombinedOutput(); err != nil { - return nil, fmt.Errorf("failed to add route in custom route table, err: %s, output: %s", err, string(out)) + nRoute := &netlink.Route{ + Gw: nextHop, + LinkIndex: link.Attrs().Index, + Table: customRouteTableID, + } + foundRoutes, err := netlink.RouteListFiltered(netlink.FAMILY_ALL, nRoute, + netlink.RT_FILTER_TABLE|netlink.RT_FILTER_OIF) + if err != nil || len(foundRoutes) < 1 { + if err = netlink.RouteAdd(nRoute); err != nil { + return nil, fmt.Errorf("failed to list routes in custom route table %d, error: %v", customRouteTableID, err) } } diff --git a/pkg/controllers/routing/pbr.go b/pkg/controllers/routing/pbr.go index cd93a27edb..a9c34995f7 100644 --- a/pkg/controllers/routing/pbr.go +++ b/pkg/controllers/routing/pbr.go @@ -2,31 +2,44 @@ package routing import ( "fmt" - "os/exec" - "strings" + "net" "github.com/cloudnativelabs/kube-router/v2/pkg/utils" + "github.com/vishvananda/netlink" +) + +const ( + PBRRuleAdd = iota + PBRRuleDel ) // ipRuleAbstraction used for abstracting iproute2 rule additions between IPv4 and IPv6 for both add and del operations. // ipProtocol is the iproute2 protocol specified as a string ("-4" or "-6"). ipOp is the rule operation specified as a // string ("add" or "del). The cidr is the IPv4 / IPv6 source CIDR string that when received will be used to lookup // routes in a custom table. -func ipRuleAbstraction(ipProtocol, ipOp, cidr string) error { - out, err := exec.Command("ip", ipProtocol, "rule", "list").Output() +func ipRuleAbstraction(ipFamily int, ipOp int, cidr string) error { + _, nSrc, err := net.ParseCIDR(cidr) + if err != nil { + return fmt.Errorf("failed to parse CIDR: %s", err.Error()) + } + + nRule := &netlink.Rule{ + Family: ipFamily, + Src: nSrc, + Table: customRouteTableID, + } + rules, err := netlink.RuleListFiltered(ipFamily, nRule, netlink.RT_FILTER_SRC) if err != nil { - return fmt.Errorf("failed to verify if `ip rule` exists: %s", err.Error()) + return fmt.Errorf("failed to list rules: %s", err.Error()) } - if strings.Contains(string(out), cidr) && ipOp == "del" { - err = exec.Command("ip", ipProtocol, "rule", ipOp, "from", cidr, "lookup", customRouteTableID).Run() - if err != nil { - return fmt.Errorf("failed to add ip rule due to: %s", err.Error()) + if ipOp == PBRRuleDel && len(rules) > 0 { + if err := netlink.RuleDel(nRule); err != nil { + return fmt.Errorf("failed to delete rule: %s", err.Error()) } - } else if !strings.Contains(string(out), cidr) && ipOp == "add" { - err = exec.Command("ip", ipProtocol, "rule", ipOp, "from", cidr, "lookup", customRouteTableID).Run() - if err != nil { - return fmt.Errorf("failed to add ip rule due to: %s", err.Error()) + } else if ipOp == PBRRuleAdd && len(rules) < 1 { + if err := netlink.RuleAdd(nRule); err != nil { + return fmt.Errorf("failed to add rule: %s", err.Error()) } } @@ -43,14 +56,14 @@ func (nrc *NetworkRoutingController) enablePolicyBasedRouting() error { if nrc.isIPv4Capable { for _, ipv4CIDR := range nrc.podIPv4CIDRs { - if err := ipRuleAbstraction("-4", "add", ipv4CIDR); err != nil { + if err := ipRuleAbstraction(netlink.FAMILY_V4, PBRRuleAdd, ipv4CIDR); err != nil { return err } } } if nrc.isIPv6Capable { for _, ipv6CIDR := range nrc.podIPv6CIDRs { - if err := ipRuleAbstraction("-6", "add", ipv6CIDR); err != nil { + if err := ipRuleAbstraction(netlink.FAMILY_V6, PBRRuleAdd, ipv6CIDR); err != nil { return err } } @@ -67,14 +80,14 @@ func (nrc *NetworkRoutingController) disablePolicyBasedRouting() error { if nrc.isIPv4Capable { for _, ipv4CIDR := range nrc.podIPv4CIDRs { - if err := ipRuleAbstraction("-4", "del", ipv4CIDR); err != nil { + if err := ipRuleAbstraction(netlink.FAMILY_V4, PBRRuleDel, ipv4CIDR); err != nil { return err } } } if nrc.isIPv6Capable { for _, ipv6CIDR := range nrc.podIPv6CIDRs { - if err := ipRuleAbstraction("-6", "del", ipv6CIDR); err != nil { + if err := ipRuleAbstraction(netlink.FAMILY_V6, PBRRuleDel, ipv6CIDR); err != nil { return err } } diff --git a/pkg/controllers/routing/utils.go b/pkg/controllers/routing/utils.go index 455240e9fa..d9a330dc84 100644 --- a/pkg/controllers/routing/utils.go +++ b/pkg/controllers/routing/utils.go @@ -1,13 +1,11 @@ package routing import ( - "bufio" "crypto/sha256" "encoding/base64" "errors" "fmt" "net" - "os/exec" "regexp" "strconv" "strings" @@ -336,41 +334,20 @@ func (nrc *NetworkRoutingController) getBGPRouteInfoForVIP(vip string) (subnet u // where the only thing that distinguishes them is the -6 or not on the end // WARNING we're parsing a CLI tool here not an API, this may break at some point in the future func fouPortAndProtoExist(port uint16, isIPv6 bool) bool { - const ipRoute2IPv6Prefix = "-6" - strPort := strconv.FormatInt(int64(port), 10) - fouArgs := make([]string, 0) - klog.V(2).Infof("Checking FOU Port and Proto... %s - %t", strPort, isIPv6) + klog.V(2).Infof("Checking FOU Port and Proto... %d - %t", port, isIPv6) + fouFamily := netlink.FAMILY_V4 if isIPv6 { - fouArgs = append(fouArgs, ipRoute2IPv6Prefix) + fouFamily = netlink.FAMILY_V6 } - fouArgs = append(fouArgs, "fou", "show") - - out, err := exec.Command("ip", fouArgs...).CombinedOutput() - // iproute2 returns an error if no fou configuration exists + fous, err := netlink.FouList(fouFamily) if err != nil { + klog.Errorf("failed to list fou ports: %v", err) return false } - strOut := string(out) - klog.V(2).Infof("Combined output of ip fou show: %s", strOut) - scanner := bufio.NewScanner(strings.NewReader(strOut)) - - // loop over all lines of output - for scanner.Scan() { - scannedLine := scanner.Text() - // if the output doesn't contain our port at all, then continue - if !strings.Contains(scannedLine, strPort) { - continue - } - - // if this is IPv6 port and it has the correct IPv6 suffix (see example above) then return true - if isIPv6 && strings.HasSuffix(scannedLine, ipRoute2IPv6Prefix) { - return true - } - - // if this is not IPv6 and it does not have an IPv6 suffix (see example above) then return true - if !isIPv6 && !strings.HasSuffix(scannedLine, ipRoute2IPv6Prefix) { + for _, fou := range fous { + if fou.Port == int(port) && fou.EncapType == netlink.FOU_ENCAP_GUE { return true } } @@ -387,18 +364,17 @@ func fouPortAndProtoExist(port uint16, isIPv6 bool) bool { // Output for a normal IPIP tunnel looks like: // ipip ipip remote local dev ttl inherit ... func linkFOUEnabled(linkName string) bool { - const fouEncapEnabled = "encap gue" - cmdArgs := []string{"-details", "link", "show", linkName} - - out, err := exec.Command("ip", cmdArgs...).CombinedOutput() + const fouEncapType = "gue" + nLink, err := netlink.LinkByName(linkName) if err != nil { - klog.Warningf("recevied an error while trying to look at the link details of %s, this shouldn't have happened", - linkName) + klog.Errorf("recevied an error while trying to look at the link details of %s, this shouldn't have happened: "+ + "%v", linkName, err) return false + } - if strings.Contains(string(out), fouEncapEnabled) { + if nLink.Attrs().EncapType == fouEncapType { return true } diff --git a/pkg/utils/linux_routing.go b/pkg/utils/linux_routing.go index 78f2f3b527..0256a7880b 100644 --- a/pkg/utils/linux_routing.go +++ b/pkg/utils/linux_routing.go @@ -22,7 +22,7 @@ var ( ) // RouteTableAdd adds a new named table to iproute's rt_tables configuration file -func RouteTableAdd(tableNumber, tableName string) error { +func RouteTableAdd(tableNumber int, tableName string) error { var rtTablesLoc string for _, possibleLoc := range rtTablesPosLoc { _, err := os.Stat(possibleLoc) @@ -47,7 +47,7 @@ func RouteTableAdd(tableNumber, tableName string) error { return fmt.Errorf("failed to open: %s", err.Error()) } defer CloseCloserDisregardError(f) - if _, err = f.WriteString(tableNumber + " " + tableName + "\n"); err != nil { + if _, err = f.WriteString(fmt.Sprint(tableNumber) + " " + tableName + "\n"); err != nil { return fmt.Errorf("failed to write: %s", err.Error()) } }