From da5c0c9eb9cff9e8baebabb0d9f16acd371c38db Mon Sep 17 00:00:00 2001 From: Leonard Cohnen Date: Wed, 26 Jul 2023 19:20:13 +0200 Subject: [PATCH 1/4] connectivity: fix tunnel feature defaults The default value is not necessarily reflected in the ConfigMap. Therefore, we need to set the right values when the keys are not found. Accoding to the documented defaults in Cilium's deployment, tunneling with vxlan is enabled if not specified otherwise. Signed-off-by: Leonard Cohnen --- connectivity/check/features.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/connectivity/check/features.go b/connectivity/check/features.go index f1fd4abcb2..645b158a0b 100644 --- a/connectivity/check/features.go +++ b/connectivity/check/features.go @@ -212,7 +212,7 @@ func (ct *ConnectivityTest) extractFeaturesFromConfigMap(ctx context.Context, cl } if versioncheck.MustCompile("<1.14.0")(ct.CiliumVersion) { - mode = "disabled" + mode = "vxlan" if v, ok := cm.Data["tunnel"]; ok { mode = v } @@ -221,13 +221,16 @@ func (ct *ConnectivityTest) extractFeaturesFromConfigMap(ctx context.Context, cl Mode: mode, } } else { - mode = "native" + mode = "tunnel" if v, ok := cm.Data["routing-mode"]; ok { mode = v } - tunnelProto := "" + + tunnelProto := "vxlan" if mode != "native" { - tunnelProto = cm.Data["tunnel-protocol"] + if v, ok := cm.Data["tunnel-protocol"]; ok { + tunnelProto = v + } } result[FeatureTunnel] = FeatureStatus{ From 7c54c5fc7b20b26e4bc434a7b58635161d929b30 Mon Sep 17 00:00:00 2001 From: Leonard Cohnen Date: Wed, 26 Jul 2023 19:23:16 +0200 Subject: [PATCH 2/4] connectivity: document feature detection order Signed-off-by: Leonard Cohnen --- connectivity/check/features.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/connectivity/check/features.go b/connectivity/check/features.go index 645b158a0b..dbb2590d9b 100644 --- a/connectivity/check/features.go +++ b/connectivity/check/features.go @@ -190,6 +190,9 @@ func parseBoolStatus(s string) bool { return false } +// extractFeaturesFromConfigMap extracts features from the Cilium ConfigMap. +// Note that there is no rule regarding if the default value is reflected +// in the ConfigMap or not. func (ct *ConnectivityTest) extractFeaturesFromConfigMap(ctx context.Context, client *k8s.Client, result FeatureSet) error { cm, err := client.GetConfigMap(ctx, ct.params.CiliumNamespace, defaults.ConfigMapName, metav1.GetOptions{}) if err != nil { @@ -265,6 +268,10 @@ func (ct *ConnectivityTest) extractFeaturesFromConfigMap(ctx context.Context, cl return nil } +// extractFeaturesFromRuntimeConfig extracts features from the Cilium runtime config. +// The downside of this approach is that the `DaemonConfig` struct is not stable. +// If there are changes to it in the future, we will likely have to maintain +// version-specific copies of the struct in the Cilium-CLI. func (ct *ConnectivityTest) extractFeaturesFromRuntimeConfig(ctx context.Context, ciliumPod Pod, result FeatureSet) error { namespace := ciliumPod.Pod.Namespace @@ -521,6 +528,9 @@ func (ct *ConnectivityTest) detectFeatures(ctx context.Context) error { for _, ciliumPod := range ct.ciliumPods { features := FeatureSet{} + // If unsure from which source to retrieve the information from, + // prefer "CiliumStatus" over "ConfigMap" over "RuntimeConfig". + // See the corresponding functions for more information. err := ct.extractFeaturesFromConfigMap(ctx, ciliumPod.K8sClient, features) if err != nil { return err From f13fdc2f82a1c43632ee9252df7bee5dab919212 Mon Sep 17 00:00:00 2001 From: Leonard Cohnen Date: Wed, 26 Jul 2023 19:32:07 +0200 Subject: [PATCH 3/4] connectivity: fix node-to-node encryption tests When tunneling is enabled, and the feature is correctly picked up the the cilium-cli, the encryption test always listens on the tunneling interface for leaked packets. This is wrong as e.g. node-to-node communication is not encapsulated and therefore does not pass the cilium_vxlan interface. Signed-off-by: Leonard Cohnen --- connectivity/tests/encryption.go | 68 ++++++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 16 deletions(-) diff --git a/connectivity/tests/encryption.go b/connectivity/tests/encryption.go index daee87bb2a..346369bc26 100644 --- a/connectivity/tests/encryption.go +++ b/connectivity/tests/encryption.go @@ -13,6 +13,8 @@ import ( "sync" "time" + "github.com/cilium/cilium/pkg/defaults" + "github.com/cilium/cilium-cli/connectivity/check" ) @@ -23,26 +25,59 @@ const ( requestICMPEcho ) -// getInterNodeIface determines on which netdev iface to capture pkts. In the -// case of tunneling, we don't expect to see unencrypted pkts on a corresponding -// tunneling iface, so the choice is obvious. In the native routing mode, we run -// "ip route get $DST_IP" from the client pod's node. +// getInterNodeIface determines on which netdev iface to capture pkts. +// We run "ip route get $DST_IP" from the client pod's node to see to +// which interface the traffic is routed to. Additionally, we translate +// the interface name to the tunneling interface name, if the route goes +// through "cilium_host" and tunneling is enabled. func getInterNodeIface(ctx context.Context, t *check.Test, clientHost *check.Pod, dstIP string) string { - tunnelFeat, ok := t.Context().Feature(check.FeatureTunnel) - if ok && tunnelFeat.Enabled { + cmd := []string{ + "/bin/sh", "-c", + fmt.Sprintf("ip -o route get %s | grep -oE 'dev [^ ]*' | cut -d' ' -f2", + dstIP), + } + t.Debugf("Running %s", strings.Join(cmd, " ")) + dev, err := clientHost.K8sClient.ExecInPod(ctx, clientHost.Pod.Namespace, + clientHost.Pod.Name, "", cmd) + if err != nil { + t.Fatalf("Failed to get IP route: %s", err) + } + + device := strings.TrimRight(dev.String(), "\n\r") + + // When tunneling is enabled, and the traffic is routed to the cilium IP space + // we want to capture on the tunnel interface. + if tunnelFeat, ok := t.Context().Feature(check.FeatureTunnel); ok && + tunnelFeat.Enabled && device == defaults.HostDevice { return "cilium_" + tunnelFeat.Mode // E.g. cilium_vxlan } - cmd := []string{"/bin/sh", "-c", - fmt.Sprintf("ip -o r g %s | grep -oE 'dev [^ ]*' | cut -d' ' -f2", - dstIP)} + return device +} + +// getSourceAddress determines the source IP address we want to use for +// capturing packet. If direct routing is used, the source IP is the client IP. +func getSourceAddress(ctx context.Context, t *check.Test, client, + clientHost *check.Pod, ipFam check.IPFamily, dstIP string, +) string { + if tunnelStatus, ok := t.Context().Feature(check.FeatureTunnel); ok && + !tunnelStatus.Enabled { + return client.Address(ipFam) + } + + cmd := []string{ + "/bin/sh", "-c", + fmt.Sprintf("ip -o route get %s | grep -oE 'src [^ ]*' | cut -d' ' -f2", + dstIP), + } t.Debugf("Running %s", strings.Join(cmd, " ")) - dev, err := clientHost.K8sClient.ExecInPod(ctx, clientHost.Pod.Namespace, + srcIP, err := clientHost.K8sClient.ExecInPod(ctx, clientHost.Pod.Namespace, clientHost.Pod.Name, "", cmd) if err != nil { t.Fatalf("Failed to get IP route: %s", err) } - return strings.TrimRight(dev.String(), "\n\r") + + return strings.TrimRight(srcIP.String(), "\n\r") } // PodToPodEncryption is a test case which checks the following: @@ -85,11 +120,11 @@ func (s *podToPodEncryption) Run(ctx context.Context, t *check.Test) { } func testNoTrafficLeak(ctx context.Context, t *check.Test, s check.Scenario, - client, server, clientHost *check.Pod, reqType requestType, ipFam check.IPFamily) { - + client, server, clientHost *check.Pod, reqType requestType, ipFam check.IPFamily, +) { dstAddr := server.Address(ipFam) iface := getInterNodeIface(ctx, t, clientHost, dstAddr) - t.Debugf("Detected %s iface for communication among client and server nodes", iface) + srcAddr := getSourceAddress(ctx, t, client, clientHost, ipFam, dstAddr) bgStdout := &safeBuffer{} bgStderr := &safeBuffer{} @@ -117,10 +152,11 @@ func testNoTrafficLeak(ctx context.Context, t *check.Test, s check.Scenario, // Unfortunately, we cannot use "host %s and host %s" filter here, // as IPsec recirculates replies to the iface netdev, which would // make tcpdump to capture the pkts (false positive). - fmt.Sprintf("src host %s and dst host %s and %s", client.Address(ipFam), dstAddr, protoFilter), + fmt.Sprintf("src host %s and dst host %s and %s", srcAddr, dstAddr, protoFilter), // Only one pkt is enough, as we don't expect any unencrypted pkt // to be captured - "-c", "1"} + "-c", "1", + } t.Debugf("Running in bg: %s", strings.Join(cmd, " ")) err := clientHost.K8sClient.ExecInPodWithWriters(ctx, killCmdCtx, clientHost.Pod.Namespace, clientHost.Pod.Name, "", cmd, bgStdout, bgStderr) From 9e4012997db44aad914bd8fd7d6c5483319361a4 Mon Sep 17 00:00:00 2001 From: Leonard Cohnen Date: Thu, 27 Jul 2023 10:42:50 +0200 Subject: [PATCH 4/4] connectivity: print all captured packets Signed-off-by: Leonard Cohnen --- connectivity/tests/encryption.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/connectivity/tests/encryption.go b/connectivity/tests/encryption.go index 346369bc26..ed7ea41aac 100644 --- a/connectivity/tests/encryption.go +++ b/connectivity/tests/encryption.go @@ -148,14 +148,11 @@ func testNoTrafficLeak(ctx context.Context, t *check.Test, s check.Scenario, // its captures. cmd := []string{ "tcpdump", "-i", iface, "--immediate-mode", "-w", fmt.Sprintf("/tmp/%s.pcap", t.Name()), - // Capture pod egress traffic. + // Capture egress traffic. // Unfortunately, we cannot use "host %s and host %s" filter here, // as IPsec recirculates replies to the iface netdev, which would // make tcpdump to capture the pkts (false positive). fmt.Sprintf("src host %s and dst host %s and %s", srcAddr, dstAddr, protoFilter), - // Only one pkt is enough, as we don't expect any unencrypted pkt - // to be captured - "-c", "1", } t.Debugf("Running in bg: %s", strings.Join(cmd, " ")) err := clientHost.K8sClient.ExecInPodWithWriters(ctx, killCmdCtx, @@ -213,7 +210,17 @@ func testNoTrafficLeak(ctx context.Context, t *check.Test, s check.Scenario, t.Fatalf("Failed to retrieve tcpdump pkt count: %s", err) } if !strings.HasPrefix(count.String(), "0 packets") { - t.Fatalf("Captured unencrypted pkt (count=%s)", count.String()) + t.Failf("Captured unencrypted pkt (count=%s)", strings.TrimRight(count.String(), "\n\r")) + + // If debug mode is enabled, dump the captured pkts + if t.Context().Params().Debug { + cmd := []string{"/bin/sh", "-c", fmt.Sprintf("tcpdump -r /tmp/%s.pcap 2>/dev/null", t.Name())} + out, err := clientHost.K8sClient.ExecInPod(ctx, clientHost.Pod.Namespace, clientHost.Pod.Name, "", cmd) + if err != nil { + t.Fatalf("Failed to retrieve tcpdump output: %s", err) + } + t.Debugf("Captured pkts:\n%s", out.String()) + } } }