add /test-by-label to run all jobs matching a label selector

pkg/pjutil/filter.go

@@ -23,12 +23,15 @@ import (
 
 	"github.com/sirupsen/logrus"
 
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"sigs.k8s.io/prow/pkg/config"
 )
 
 var TestAllRe = regexp.MustCompile(`(?m)^/test all,?($|\s.*)`)
 
+var TestByLabelsRe = regexp.MustCompile(`(?m)^/test-by-label (.+)$`)
+
 // RetestRe provides the regex for `/retest`
 var RetestRe = regexp.MustCompile(`(?m)^/retest\s*$`)
 
@@ -149,6 +152,51 @@ func (tf *TestAllFilter) Name() string {
 	return "test-all-filter"
 }
 
+// TestByLabelFilter builds a filter for the automatic behavior of
+// `/testlabel selector`. This filter works the same way as TestAllFilter, but
+// additionally filters jobs down to match the given label selector.
+type TestByLabelFilter struct {
+	base      *TestAllFilter
+	selectors []labels.Selector
+}
+
+func NewTestByLabelFilter(command string) (*TestByLabelFilter, error) {
+	matches := TestByLabelsRe.FindAllStringSubmatch(command, -1)
+
+	var selectors []labels.Selector
+	for _, match := range matches {
+		selector, err := labels.Parse(match[1])
+		if err != nil {
+			return nil, fmt.Errorf("invalid selector %q: %w", match[1], err)
+		}
+		selectors = append(selectors, selector)
+	}
+
+	return &TestByLabelFilter{
+		base:      NewTestAllFilter(),
+		selectors: selectors,
+	}, nil
+}
+
+func (tf *TestByLabelFilter) ShouldRun(p config.Presubmit) (bool, bool, bool) {
+	shouldRun, forced, def := tf.base.ShouldRun(p)
+	if !shouldRun {
+		return shouldRun, forced, def
+	}
+
+	for _, selector := range tf.selectors {
+		if selector.Matches(labels.Set(p.Labels)) {
+			return true, false, false
+		}
+	}
+
+	return false, false, false
+}
+
+func (tf *TestByLabelFilter) Name() string {
+	return "test-by-label-filter"
+}
+
 // AggregateFilter builds a filter that evaluates the child filters in order
 // and returns the first match
 type AggregateFilter struct {
@@ -292,5 +340,13 @@ func PresubmitFilter(honorOkToTest bool, contextGetter contextGetter, body strin
 		logger.Debug("Using test-all filter.")
 		filters = append(filters, NewTestAllFilter())
 	}
+	if TestByLabelsRe.MatchString(body) {
+		logger.Debug("Using test-by-label filter.")
+		filter, err := NewTestByLabelFilter(body)
+		if err != nil {
+			return nil, fmt.Errorf("cannot build test-label filter: %w", err)
+		}
+		filters = append(filters, filter)
+	}
 	return NewAggregateFilter(filters), nil
 }

pkg/plugins/trigger/generic-comment.go

@@ -61,6 +61,7 @@ func handleGenericComment(c Client, trigger plugins.Trigger, gc github.GenericCo
 		!pjutil.RetestRequiredRe.MatchString(gc.Body) &&
 		!pjutil.OkToTestRe.MatchString(gc.Body) &&
 		!pjutil.TestAllRe.MatchString(gc.Body) &&
+		!pjutil.TestByLabelsRe.MatchString(gc.Body) &&
 		!pjutil.MayNeedHelpComment(gc.Body) {
 		matched := false
 		for _, presubmit := range presubmits {
@@ -191,6 +192,8 @@ type GitHubClient interface {
 //   - if we got a /test all or an /ok-to-test, we want to consider any job
 //     that doesn't explicitly require a human trigger comment; jobs will
 //     default to not run unless we can determine that they should
+//   - if we got a /test-by-label selector, it work the same as "/test all", but
+//     restricted to those jobs matching the given label selector
 //
 // If a comment that we get matches more than one of the above patterns, we
 // consider the set of matching presubmits the union of the results from the

pkg/plugins/trigger/generic-comment_test.go

@@ -18,7 +18,6 @@ package trigger
 
 import (
 	"fmt"
-	"log"
 	"reflect"
 	"strings"
 	"testing"
@@ -1323,7 +1322,7 @@ func TestHandleGenericComment(t *testing.T) {
 				}
 			}
 			if err := c.Config.SetPresubmits(presubmits); err != nil {
-				t.Fatalf("%s: failed to set presubmits: %v", tc.name, err)
+				t.Fatalf("failed to set presubmits: %v", err)
 			}
 
 			event := github.GenericCommentEvent{
@@ -1345,23 +1344,24 @@ func TestHandleGenericComment(t *testing.T) {
 			}
 			trigger.SetDefaults()
 
-			log.Printf("running case %s", tc.name)
 			// In some cases handleGenericComment can be called twice for the same event.
 			// For instance on Issue/PR creation and modification.
 			// Let's call it twice to ensure idempotency.
 			if err := handleGenericComment(c, trigger, event); err != nil {
-				t.Fatalf("%s: didn't expect error: %s", tc.name, err)
+				t.Fatalf("didn't expect error: %v", err)
 			}
 			validate(t, fakeProwJobClient.Fake.Actions(), g, tc)
 			if err := handleGenericComment(c, trigger, event); err != nil {
-				t.Fatalf("%s: didn't expect error: %s", tc.name, err)
+				t.Fatalf("didn't expect error: %v", err)
 			}
 			validate(t, fakeProwJobClient.Fake.Actions(), g, tc)
 		})
 	}
 }
 
 func validate(t *testing.T, actions []clienttesting.Action, g *fakegithub.FakeClient, tc testcase) {
+	t.Helper()
+
 	startedContexts := sets.New[string]()
 	for _, action := range actions {
 		switch action := action.(type) {
@@ -1459,23 +1459,23 @@ func TestRetestFilter(t *testing.T) {
 	for _, testCase := range testCases {
 		t.Run(testCase.name, func(t *testing.T) {
 			if len(testCase.presubmits) != len(testCase.expected) {
-				t.Fatalf("%s: have %d presubmits but only %d expected filter outputs", testCase.name, len(testCase.presubmits), len(testCase.expected))
+				t.Fatalf("have %d presubmits but only %d expected filter outputs", len(testCase.presubmits), len(testCase.expected))
 			}
 			if err := config.SetPresubmitRegexes(testCase.presubmits); err != nil {
-				t.Fatalf("%s: could not set presubmit regexes: %v", testCase.name, err)
+				t.Fatalf("could not set presubmit regexes: %v", err)
 			}
 			filter := pjutil.NewRetestFilter(testCase.failedContexts, testCase.allContexts)
 			for i, presubmit := range testCase.presubmits {
 				actualFiltered, actualForced, actualDefault := filter.ShouldRun(presubmit)
 				expectedFiltered, expectedForced, expectedDefault := testCase.expected[i][0], testCase.expected[i][1], testCase.expected[i][2]
 				if actualFiltered != expectedFiltered {
-					t.Errorf("%s: filter did not evaluate correctly, expected %v but got %v for %v", testCase.name, expectedFiltered, actualFiltered, presubmit.Name)
+					t.Errorf("filter did not evaluate correctly, expected %v but got %v for %v", expectedFiltered, actualFiltered, presubmit.Name)
 				}
 				if actualForced != expectedForced {
-					t.Errorf("%s: filter did not determine forced correctly, expected %v but got %v for %v", testCase.name, expectedForced, actualForced, presubmit.Name)
+					t.Errorf("filter did not determine forced correctly, expected %v but got %v for %v", expectedForced, actualForced, presubmit.Name)
 				}
 				if actualDefault != expectedDefault {
-					t.Errorf("%s: filter did not determine default correctly, expected %v but got %v for %v", testCase.name, expectedDefault, actualDefault, presubmit.Name)
+					t.Errorf("filter did not determine default correctly, expected %v but got %v for %v", expectedDefault, actualDefault, presubmit.Name)
 				}
 			}
 		})

pkg/plugins/trigger/trigger.go

@@ -128,6 +128,13 @@ func helpProvider(config *plugins.Configuration, enabledRepos []config.OrgRepo)
 		WhoCanUse:   "Anyone can trigger this command on a trusted PR.",
 		Examples:    []string{"/test all", "/test pull-bazel-test"},
 	})
+	pluginHelp.AddCommand(pluginhelp.Command{
+		Usage:       "/test-by-label label=selector",
+		Description: "Manually starts all jobs matching the given label selector.",
+		Featured:    true,
+		WhoCanUse:   "Anyone can trigger this command on a trusted PR.",
+		Examples:    []string{"/test-by-label provider=kubevirt", "/test-by-label release=31"},
+	})
 	pluginHelp.AddCommand(pluginhelp.Command{
 		Usage:       "/retest",
 		Description: "Rerun test jobs that have failed.",