Skip to content

Commit

Permalink
connectivity: Check for unexpected packet drops
Browse files Browse the repository at this point in the history
Convert the check for missed tail call packet drops into a more general
check for any unexpected packets drops. Current expected packet drop
reasons are Policy denied and Policy denied by denylist only. Others
will be added in subsequent commits.

Contrary to the Missed tail call check who only ran in the context of
the conn-disrupt test, this new check is always run by default.

Signed-off-by: Paul Chaignon <[email protected]>
  • Loading branch information
pchaigno committed Dec 6, 2023
1 parent 21bcf7c commit e7c8ab4
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 24 deletions.
4 changes: 2 additions & 2 deletions connectivity/suite.go
Original file line number Diff line number Diff line change
Expand Up @@ -259,10 +259,10 @@ func Run(ctx context.Context, ct *check.ConnectivityTest, addExtraTests func(*ch
// include --include-conn-disrupt-test"
return ct.Run(ctx)
}

ct.NewTest("no-missed-tail-calls").WithScenarios(tests.NoMissedTailCalls())
}

ct.NewTest("no-unexpected-packet-drops").WithScenarios(tests.NoUnexpectedPacketDrops())

// Run all tests without any policies in place.
noPoliciesScenarios := []check.Scenario{
tests.PodToPod(),
Expand Down
34 changes: 12 additions & 22 deletions connectivity/tests/errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ package tests

import (
"context"
"strconv"
"strings"
"time"

Expand Down Expand Up @@ -75,43 +74,34 @@ func (n *noErrorsInLogs) Run(ctx context.Context, t *check.Test) {

}

// NoMissedTailCalls checks whether there were no drops due to missed (BPF)
// tail calls.
func NoMissedTailCalls() check.Scenario {
return &noMissedTailCalls{}
// NoUnexpectedPacketDrops checks whether there were no drops due to expected
// packet drops.
func NoUnexpectedPacketDrops() check.Scenario {
return &noUnexpectedPacketDrops{}
}

type noMissedTailCalls struct{}
type noUnexpectedPacketDrops struct{}

func (n *noMissedTailCalls) Name() string {
return "no-missed-tail-calls"
func (n *noUnexpectedPacketDrops) Name() string {
return "no-unexpected-packet-drops"
}

func (n *noMissedTailCalls) Run(ctx context.Context, t *check.Test) {
func (n *noUnexpectedPacketDrops) Run(ctx context.Context, t *check.Test) {
ct := t.Context()
cmd := []string{
"/bin/sh", "-c",
"cilium metrics list -o json | jq '.[] | select( .name == \"cilium_drop_count_total\" and .labels.reason == \"Missed tail call\" ).value'",
"cilium metrics list -o json | jq '.[] | select((.name == \"cilium_drop_count_total\") and (.labels.reason | IN(\"Policy denied\", \"Policy denied by denylist\") | not))'",
}

for _, pod := range ct.CiliumPods() {
pod := pod
stdout, err := pod.K8sClient.ExecInPod(ctx, pod.Pod.Namespace, pod.Pod.Name, defaults.AgentContainerName, cmd)
if err != nil {
t.Fatalf("Error fetching missed tail call drop counts: %s", err)
t.Fatalf("Error fetching packet drop counts: %s", err)
}
countStr := strings.TrimSpace(stdout.String())
if countStr == "" {
return
}

count, err := strconv.Atoi(countStr)
if err != nil {
t.Fatalf("Failed to convert missed tail call drops %q to int: %s", countStr, err)
}

if count != 0 {
t.Fatalf("Detected drops due to missed tail calls: %d", count)
if countStr != "" {
t.Fatalf("Found unexpected packet drops:\n%s", countStr)
}
}

Expand Down

0 comments on commit e7c8ab4

Please sign in to comment.