diff --git a/v2/config/pod/coil-egress-controller.yaml b/v2/config/pod/coil-egress-controller.yaml index 7e8c1765..ae6b7dbd 100644 --- a/v2/config/pod/coil-egress-controller.yaml +++ b/v2/config/pod/coil-egress-controller.yaml @@ -15,6 +15,7 @@ spec: labels: app.kubernetes.io/component: coil-egress-controller spec: + hostNetwork: true priorityClassName: system-cluster-critical tolerations: - key: node-role.kubernetes.io/master diff --git a/v2/e2e/coil_test.go b/v2/e2e/coil_test.go index c8039250..1835f4cb 100644 --- a/v2/e2e/coil_test.go +++ b/v2/e2e/coil_test.go @@ -19,14 +19,16 @@ import ( ) var ( - testIPv6 = os.Getenv(testIPv6Key) == "true" + enableIPv6Tests = os.Getenv(testIPv6Key) == "true" + enableIPAMTests = os.Getenv(testIPAMKey) == "true" + enableEgressTests = os.Getenv(testEgressKey) == "true" ) var _ = Describe("coil", func() { - if os.Getenv(testIPAMKey) == "true" { + if enableIPAMTests { Context("when the IPAM features are enabled", testIPAM) } - if os.Getenv(testEgressKey) == "true" { + if enableEgressTests { Context("when egress feature is enabled", testEgress) } Context("when coild is deployed", testCoild) @@ -71,7 +73,7 @@ func testIPAM() { It("should allow pods on different nodes to communicate", func() { By("creating the default pool") manifest := "manifests/default_pool.yaml" - if testIPv6 { + if enableIPv6Tests { manifest = "manifests/default_pool_v6.yaml" } kubectlSafe(nil, "apply", "-f", manifest) @@ -116,7 +118,7 @@ func testIPAM() { By("checking communication between pods on different nodes") var testURL string - if testIPv6 { + if enableIPv6Tests { testURL = fmt.Sprintf("http://[%s]:8000", httpdIP) } else { testURL = fmt.Sprintf("http://%s:8000", httpdIP) @@ -198,7 +200,7 @@ func testIPAM() { It("should export routes to routing table 119", func() { var ipOpt string - if testIPv6 { + if enableIPv6Tests { ipOpt = "-6" } else { ipOpt = "-4" @@ -389,7 +391,7 @@ func testEgress() { It("should allow NAT traffic over foo-over-udp tunnel", func() { var fakeIP, fakeURL, ipOpt string - if testIPv6 { + if enableIPv6Tests { fakeIP = "2606:4700:4700::9999" fakeURL = fmt.Sprintf("http://[%s]", fakeIP) ipOpt = "-6" @@ -404,44 +406,50 @@ func testEgress() { Expect(err).NotTo(HaveOccurred()) _, err = runOnNode("coil-control-plane", "ip", "link", "set", "dummy-fake", "up") Expect(err).NotTo(HaveOccurred()) - if testIPv6 { + if enableIPv6Tests { _, err = runOnNode("coil-control-plane", "ip", "address", "add", fakeIP+"/128", "dev", "dummy-fake", "nodad") } else { _, err = runOnNode("coil-control-plane", "ip", "address", "add", fakeIP+"/32", "dev", "dummy-fake") } Expect(err).NotTo(HaveOccurred()) + natAddresses := []string{} + if !enableIPAMTests { + natAddresses = getNATAddresses("egress") + } + By("running HTTP server on coil-control-plane") - if os.Getenv(testIPAMKey) == "true" { + if enableIPAMTests { go runOnNode("coil-control-plane", "/usr/local/bin/echotest") } else { - go runOnNode("coil-control-plane", "/usr/local/bin/echotest", "reply") + go runOnNode("coil-control-plane", "/usr/local/bin/echotest", "--reply-remote") } time.Sleep(100 * time.Millisecond) By("sending and receiving HTTP request from nat-client") data := make([]byte, 1<<20) // 1 MiB - resp := kubectlSafe(data, "exec", "-i", "nat-client", "--", "curl", "-sf", "-T", "-", fakeURL) - Expect(resp).To(HaveLen(1 << 20)) + testNAT(data, "nat-client", fakeURL, natAddresses, enableIPAMTests) By("running the same test 100 times") for i := 0; i < 100; i++ { time.Sleep(1 * time.Millisecond) - resp := kubectlSafe(data, "exec", "-i", "nat-client", "--", "curl", "-sf", "-T", "-", fakeURL) - Expect(resp).To(HaveLen(1 << 20)) + testNAT(data, "nat-client", fakeURL, natAddresses, enableIPAMTests) + } + + natAddresses = []string{} + if !enableIPAMTests { + natAddresses = getNATAddresses("egress-sport-auto") } By("sending and receiving HTTP request from nat-client-sport-auto") data = make([]byte, 1<<20) // 1 MiB - resp = kubectlSafe(data, "exec", "-i", "nat-client-sport-auto", "--", "curl", "-sf", "-T", "-", fakeURL) - Expect(resp).To(HaveLen(1 << 20)) + testNAT(data, "nat-client-sport-auto", fakeURL, natAddresses, enableIPAMTests) By("running the same test 100 times with nat-client-sport-auto") for i := 0; i < 100; i++ { time.Sleep(1 * time.Millisecond) - resp := kubectlSafe(data, "exec", "-i", "nat-client-sport-auto", "--", "curl", "-sf", "-T", "-", fakeURL) - Expect(resp).To(HaveLen(1 << 20)) + testNAT(data, "nat-client-sport-auto", fakeURL, natAddresses, enableIPAMTests) } By("creating a dummy pod don't use egress") @@ -541,15 +549,19 @@ func testEgress() { Expect(fouCount).To(Equal(1)) By("sending and receiving HTTP request from nat-client") + + natAddresses = []string{} + if !enableIPAMTests { + natAddresses = getNATAddresses("egress") + } + data = make([]byte, 1<<20) // 1 MiB - resp = kubectlSafe(data, "exec", "-i", "nat-client", "--", "curl", "-sf", "-T", "-", fakeURL) - Expect(resp).To(HaveLen(1 << 20)) + testNAT(data, "nat-client", fakeURL, natAddresses, enableIPAMTests) By("running the same test 100 times") for i := 0; i < 100; i++ { time.Sleep(1 * time.Millisecond) - resp := kubectlSafe(data, "exec", "-i", "nat-client", "--", "curl", "-sf", "-T", "-", fakeURL) - Expect(resp).To(HaveLen(1 << 20)) + testNAT(data, "nat-client", fakeURL, natAddresses, enableIPAMTests) } }) } @@ -565,3 +577,30 @@ func testCoild() { Expect(mfs).NotTo(BeEmpty()) }) } + +func testNAT(data []byte, clientPod, fakeURL string, natAddresses []string, ipamEnabled bool) { + resp := kubectlSafe(data, "exec", "-i", clientPod, "--", "curl", "-sf", "-T", "-", fakeURL) + + if !enableIPAMTests { + respStr := string(resp) + idx := strings.Index(respStr, "|") + ipAddr := respStr[:idx] + resp = []byte(respStr[idx+1:]) + Expect(natAddresses).To(ContainElement(ipAddr)) + } + Expect(resp).To(HaveLen(1 << 20)) +} + +func getNATAddresses(name string) []string { + eps := &corev1.Endpoints{} + err := getResource("internet", "endpoints", name, "", eps) + Expect(err).ToNot(HaveOccurred()) + + natAddresses := []string{} + for _, s := range eps.Subsets { + for _, a := range s.Addresses { + natAddresses = append(natAddresses, a.IP) + } + } + return natAddresses +} diff --git a/v2/e2e/controller_test.go b/v2/e2e/controller_test.go index 34fefa77..3e480940 100644 --- a/v2/e2e/controller_test.go +++ b/v2/e2e/controller_test.go @@ -82,7 +82,7 @@ func testCoilEgressController() { node := pods.Items[0].Spec.NodeName address := fmt.Sprintf("http://%s:9396/metrics", pods.Items[0].Status.PodIP) - if testIPv6 { + if enableIPv6Tests { address = fmt.Sprintf("http://[%s]:9396/metrics", pods.Items[0].Status.PodIP) } diff --git a/v2/e2e/echo-server/main.go b/v2/e2e/echo-server/main.go index b01e264d..6131aae7 100644 --- a/v2/e2e/echo-server/main.go +++ b/v2/e2e/echo-server/main.go @@ -1,13 +1,14 @@ package main import ( + "flag" "io" + "net" "net/http" - "os" ) type echoHandler struct { - withReply bool + withRemoteAddrReply bool } func (h echoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { @@ -18,23 +19,27 @@ func (h echoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { } w.Header().Set("content-type", "application/octet-stream") - w.Write(body) - if h.withReply { - w.Write([]byte(req.Host)) - } + if h.withRemoteAddrReply { + remote, _, err := net.SplitHostPort(req.RemoteAddr) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + remote += "|" + w.Write([]byte(remote)) + } + w.Write(body) } func main() { - withReply := false - if len(os.Args) > 1 { - if os.Args[1] == "reply" { - withReply = true - } - } + var withRemoteAddress bool + flag.BoolVar(&withRemoteAddress, "reply-remote", false, "if set, echo-server will reply with remote host address (default: false)") + flag.Parse() + s := &http.Server{ Handler: echoHandler{ - withReply: withReply, + withRemoteAddrReply: withRemoteAddress, }, } s.ListenAndServe()