From 925b87929840d71aacb814116c968c2833546bae Mon Sep 17 00:00:00 2001 From: Patryk Strusiewicz-Surmacki Date: Thu, 17 Oct 2024 11:30:31 +0200 Subject: [PATCH 1/2] More changes Signed-off-by: Patryk Strusiewicz-Surmacki --- v2/e2e/coil_test.go | 21 +++++++++++++-------- v2/e2e/echo-server/main.go | 21 ++++++++++++++++++--- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/v2/e2e/coil_test.go b/v2/e2e/coil_test.go index 2e589c60..c8039250 100644 --- a/v2/e2e/coil_test.go +++ b/v2/e2e/coil_test.go @@ -400,19 +400,24 @@ func testEgress() { } By("setting a fake global address to coil-control-plane") - runOnNode("coil-control-plane", "ip", "link", "add", "dummy-fake", "type", "dummy") - // Expect(err).NotTo(HaveOccurred()) - runOnNode("coil-control-plane", "ip", "link", "set", "dummy-fake", "up") - // Expect(err).NotTo(HaveOccurred()) + _, err := runOnNode("coil-control-plane", "ip", "link", "add", "dummy-fake", "type", "dummy") + Expect(err).NotTo(HaveOccurred()) + _, err = runOnNode("coil-control-plane", "ip", "link", "set", "dummy-fake", "up") + Expect(err).NotTo(HaveOccurred()) if testIPv6 { - runOnNode("coil-control-plane", "ip", "address", "add", fakeIP+"/128", "dev", "dummy-fake", "nodad") + _, err = runOnNode("coil-control-plane", "ip", "address", "add", fakeIP+"/128", "dev", "dummy-fake", "nodad") } else { - runOnNode("coil-control-plane", "ip", "address", "add", fakeIP+"/32", "dev", "dummy-fake") + _, err = runOnNode("coil-control-plane", "ip", "address", "add", fakeIP+"/32", "dev", "dummy-fake") } - // Expect(err).NotTo(HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) By("running HTTP server on coil-control-plane") - go runOnNode("coil-control-plane", "/usr/local/bin/echotest") + if os.Getenv(testIPAMKey) == "true" { + go runOnNode("coil-control-plane", "/usr/local/bin/echotest") + } else { + go runOnNode("coil-control-plane", "/usr/local/bin/echotest", "reply") + } + time.Sleep(100 * time.Millisecond) By("sending and receiving HTTP request from nat-client") diff --git a/v2/e2e/echo-server/main.go b/v2/e2e/echo-server/main.go index 09c20ac8..b01e264d 100644 --- a/v2/e2e/echo-server/main.go +++ b/v2/e2e/echo-server/main.go @@ -3,11 +3,14 @@ package main import ( "io" "net/http" + "os" ) -type echoHandler struct{} +type echoHandler struct { + withReply bool +} -func (echoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { +func (h echoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { body, err := io.ReadAll(req.Body) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) @@ -16,11 +19,23 @@ func (echoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { w.Header().Set("content-type", "application/octet-stream") w.Write(body) + if h.withReply { + w.Write([]byte(req.Host)) + } + } func main() { + withReply := false + if len(os.Args) > 1 { + if os.Args[1] == "reply" { + withReply = true + } + } s := &http.Server{ - Handler: echoHandler{}, + Handler: echoHandler{ + withReply: withReply, + }, } s.ListenAndServe() } From d4532a10571029760281cbb41347a1afa8496f01 Mon Sep 17 00:00:00 2001 From: Patryk Strusiewicz-Surmacki Date: Thu, 17 Oct 2024 16:52:05 +0200 Subject: [PATCH 2/2] Added remote replay tests Signed-off-by: Patryk Strusiewicz-Surmacki --- v2/config/pod/coil-egress-controller.yaml | 1 + v2/e2e/coil_test.go | 83 +++++++++++++++++------ v2/e2e/controller_test.go | 2 +- v2/e2e/echo-server/main.go | 31 +++++---- 4 files changed, 81 insertions(+), 36 deletions(-) diff --git a/v2/config/pod/coil-egress-controller.yaml b/v2/config/pod/coil-egress-controller.yaml index 7e8c1765..ae6b7dbd 100644 --- a/v2/config/pod/coil-egress-controller.yaml +++ b/v2/config/pod/coil-egress-controller.yaml @@ -15,6 +15,7 @@ spec: labels: app.kubernetes.io/component: coil-egress-controller spec: + hostNetwork: true priorityClassName: system-cluster-critical tolerations: - key: node-role.kubernetes.io/master diff --git a/v2/e2e/coil_test.go b/v2/e2e/coil_test.go index c8039250..1835f4cb 100644 --- a/v2/e2e/coil_test.go +++ b/v2/e2e/coil_test.go @@ -19,14 +19,16 @@ import ( ) var ( - testIPv6 = os.Getenv(testIPv6Key) == "true" + enableIPv6Tests = os.Getenv(testIPv6Key) == "true" + enableIPAMTests = os.Getenv(testIPAMKey) == "true" + enableEgressTests = os.Getenv(testEgressKey) == "true" ) var _ = Describe("coil", func() { - if os.Getenv(testIPAMKey) == "true" { + if enableIPAMTests { Context("when the IPAM features are enabled", testIPAM) } - if os.Getenv(testEgressKey) == "true" { + if enableEgressTests { Context("when egress feature is enabled", testEgress) } Context("when coild is deployed", testCoild) @@ -71,7 +73,7 @@ func testIPAM() { It("should allow pods on different nodes to communicate", func() { By("creating the default pool") manifest := "manifests/default_pool.yaml" - if testIPv6 { + if enableIPv6Tests { manifest = "manifests/default_pool_v6.yaml" } kubectlSafe(nil, "apply", "-f", manifest) @@ -116,7 +118,7 @@ func testIPAM() { By("checking communication between pods on different nodes") var testURL string - if testIPv6 { + if enableIPv6Tests { testURL = fmt.Sprintf("http://[%s]:8000", httpdIP) } else { testURL = fmt.Sprintf("http://%s:8000", httpdIP) @@ -198,7 +200,7 @@ func testIPAM() { It("should export routes to routing table 119", func() { var ipOpt string - if testIPv6 { + if enableIPv6Tests { ipOpt = "-6" } else { ipOpt = "-4" @@ -389,7 +391,7 @@ func testEgress() { It("should allow NAT traffic over foo-over-udp tunnel", func() { var fakeIP, fakeURL, ipOpt string - if testIPv6 { + if enableIPv6Tests { fakeIP = "2606:4700:4700::9999" fakeURL = fmt.Sprintf("http://[%s]", fakeIP) ipOpt = "-6" @@ -404,44 +406,50 @@ func testEgress() { Expect(err).NotTo(HaveOccurred()) _, err = runOnNode("coil-control-plane", "ip", "link", "set", "dummy-fake", "up") Expect(err).NotTo(HaveOccurred()) - if testIPv6 { + if enableIPv6Tests { _, err = runOnNode("coil-control-plane", "ip", "address", "add", fakeIP+"/128", "dev", "dummy-fake", "nodad") } else { _, err = runOnNode("coil-control-plane", "ip", "address", "add", fakeIP+"/32", "dev", "dummy-fake") } Expect(err).NotTo(HaveOccurred()) + natAddresses := []string{} + if !enableIPAMTests { + natAddresses = getNATAddresses("egress") + } + By("running HTTP server on coil-control-plane") - if os.Getenv(testIPAMKey) == "true" { + if enableIPAMTests { go runOnNode("coil-control-plane", "/usr/local/bin/echotest") } else { - go runOnNode("coil-control-plane", "/usr/local/bin/echotest", "reply") + go runOnNode("coil-control-plane", "/usr/local/bin/echotest", "--reply-remote") } time.Sleep(100 * time.Millisecond) By("sending and receiving HTTP request from nat-client") data := make([]byte, 1<<20) // 1 MiB - resp := kubectlSafe(data, "exec", "-i", "nat-client", "--", "curl", "-sf", "-T", "-", fakeURL) - Expect(resp).To(HaveLen(1 << 20)) + testNAT(data, "nat-client", fakeURL, natAddresses, enableIPAMTests) By("running the same test 100 times") for i := 0; i < 100; i++ { time.Sleep(1 * time.Millisecond) - resp := kubectlSafe(data, "exec", "-i", "nat-client", "--", "curl", "-sf", "-T", "-", fakeURL) - Expect(resp).To(HaveLen(1 << 20)) + testNAT(data, "nat-client", fakeURL, natAddresses, enableIPAMTests) + } + + natAddresses = []string{} + if !enableIPAMTests { + natAddresses = getNATAddresses("egress-sport-auto") } By("sending and receiving HTTP request from nat-client-sport-auto") data = make([]byte, 1<<20) // 1 MiB - resp = kubectlSafe(data, "exec", "-i", "nat-client-sport-auto", "--", "curl", "-sf", "-T", "-", fakeURL) - Expect(resp).To(HaveLen(1 << 20)) + testNAT(data, "nat-client-sport-auto", fakeURL, natAddresses, enableIPAMTests) By("running the same test 100 times with nat-client-sport-auto") for i := 0; i < 100; i++ { time.Sleep(1 * time.Millisecond) - resp := kubectlSafe(data, "exec", "-i", "nat-client-sport-auto", "--", "curl", "-sf", "-T", "-", fakeURL) - Expect(resp).To(HaveLen(1 << 20)) + testNAT(data, "nat-client-sport-auto", fakeURL, natAddresses, enableIPAMTests) } By("creating a dummy pod don't use egress") @@ -541,15 +549,19 @@ func testEgress() { Expect(fouCount).To(Equal(1)) By("sending and receiving HTTP request from nat-client") + + natAddresses = []string{} + if !enableIPAMTests { + natAddresses = getNATAddresses("egress") + } + data = make([]byte, 1<<20) // 1 MiB - resp = kubectlSafe(data, "exec", "-i", "nat-client", "--", "curl", "-sf", "-T", "-", fakeURL) - Expect(resp).To(HaveLen(1 << 20)) + testNAT(data, "nat-client", fakeURL, natAddresses, enableIPAMTests) By("running the same test 100 times") for i := 0; i < 100; i++ { time.Sleep(1 * time.Millisecond) - resp := kubectlSafe(data, "exec", "-i", "nat-client", "--", "curl", "-sf", "-T", "-", fakeURL) - Expect(resp).To(HaveLen(1 << 20)) + testNAT(data, "nat-client", fakeURL, natAddresses, enableIPAMTests) } }) } @@ -565,3 +577,30 @@ func testCoild() { Expect(mfs).NotTo(BeEmpty()) }) } + +func testNAT(data []byte, clientPod, fakeURL string, natAddresses []string, ipamEnabled bool) { + resp := kubectlSafe(data, "exec", "-i", clientPod, "--", "curl", "-sf", "-T", "-", fakeURL) + + if !enableIPAMTests { + respStr := string(resp) + idx := strings.Index(respStr, "|") + ipAddr := respStr[:idx] + resp = []byte(respStr[idx+1:]) + Expect(natAddresses).To(ContainElement(ipAddr)) + } + Expect(resp).To(HaveLen(1 << 20)) +} + +func getNATAddresses(name string) []string { + eps := &corev1.Endpoints{} + err := getResource("internet", "endpoints", name, "", eps) + Expect(err).ToNot(HaveOccurred()) + + natAddresses := []string{} + for _, s := range eps.Subsets { + for _, a := range s.Addresses { + natAddresses = append(natAddresses, a.IP) + } + } + return natAddresses +} diff --git a/v2/e2e/controller_test.go b/v2/e2e/controller_test.go index 34fefa77..3e480940 100644 --- a/v2/e2e/controller_test.go +++ b/v2/e2e/controller_test.go @@ -82,7 +82,7 @@ func testCoilEgressController() { node := pods.Items[0].Spec.NodeName address := fmt.Sprintf("http://%s:9396/metrics", pods.Items[0].Status.PodIP) - if testIPv6 { + if enableIPv6Tests { address = fmt.Sprintf("http://[%s]:9396/metrics", pods.Items[0].Status.PodIP) } diff --git a/v2/e2e/echo-server/main.go b/v2/e2e/echo-server/main.go index b01e264d..6131aae7 100644 --- a/v2/e2e/echo-server/main.go +++ b/v2/e2e/echo-server/main.go @@ -1,13 +1,14 @@ package main import ( + "flag" "io" + "net" "net/http" - "os" ) type echoHandler struct { - withReply bool + withRemoteAddrReply bool } func (h echoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { @@ -18,23 +19,27 @@ func (h echoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { } w.Header().Set("content-type", "application/octet-stream") - w.Write(body) - if h.withReply { - w.Write([]byte(req.Host)) - } + if h.withRemoteAddrReply { + remote, _, err := net.SplitHostPort(req.RemoteAddr) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + remote += "|" + w.Write([]byte(remote)) + } + w.Write(body) } func main() { - withReply := false - if len(os.Args) > 1 { - if os.Args[1] == "reply" { - withReply = true - } - } + var withRemoteAddress bool + flag.BoolVar(&withRemoteAddress, "reply-remote", false, "if set, echo-server will reply with remote host address (default: false)") + flag.Parse() + s := &http.Server{ Handler: echoHandler{ - withReply: withReply, + withRemoteAddrReply: withRemoteAddress, }, } s.ListenAndServe()