Mutation tests only

.github/mut_blacklist

@@ -0,0 +1,6 @@
+54f1ea08e7395cf6768c102d5677d764
+bc6bebd9df1b01a984c0ffab3c50b8de
+d7ce4ca9bca24c65de535f680084fdeb
+00a067ff2145e46f285ddf7f0f842788
+cebc341f1ea73b8dc8907be3bb97a1a0
+d18e7ef6485ad32dd1b4935777f433e0

.github/mut_config.yml

@@ -0,0 +1,7 @@
+skip_without_test: false
+skip_with_build_tags: false
+json_output: true
+silent_mode: false
+min_msi: 0.90 # should be >0.95
+exclude_dirs:
+  - messages/proto

.github/workflows/main.yml

@@ -47,6 +47,27 @@ jobs:
       - name: Run Go Test with race
         run: go test -race -shuffle=on -timeout 2m ./...
 
+  mutating:
+    name: Mutation tests
+    runs-on: ubuntu-latest
+    if: false # skipped, should be restored and fixed
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19.x
+
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Install dependencies
+        run: make install-deps
+
+      - name: Mutating testing
+        run: make mut
+
   reproducible-builds:
     runs-on: ubuntu-latest
     steps:

Makefile

@@ -1,4 +1,4 @@
-.PHONY: lint lint-all build-dummy install-deps
+.PHONY: lint lint-all build-dummy install-deps mut
 
 FIRST_COMMIT ?= $(shell git rev-list --max-parents=0 HEAD)
 
@@ -13,4 +13,10 @@ lint-all:
 	./build/bin/golangci-lint run --config ./.golangci.yml --new-from-rev=$(FIRST_COMMIT)
 
 install-deps:
+	go get github.com/JekaMas/go-mutesting/cmd/[email protected]
+	go install github.com/JekaMas/go-mutesting/...
 	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./build/bin v1.50.1
+
+mut:
+	MUTATION_TEST=on go-mutesting --blacklist=".github/mut_blacklist" --config=".github/mut_config.yml" ./...
+	@echo MSI: `jq '.stats.msi' report.json`

core/ibft_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"os"
 	"sync"
 	"testing"
 	"time"
@@ -2292,3 +2293,30 @@ func TestIBFT_ExtendRoundTimer(t *testing.T) {
 	// Make sure the round timeout was extended
 	assert.Equal(t, additionalTimeout, i.additionalTimeout)
 }
+
+// A dummy test that is needed only to run all tests with mutations
+func TestDummyForMutations(t *testing.T) {
+	t.Parallel()
+
+	if len(os.Getenv("MUTATION_TEST")) == 0 {
+		t.SkipNow()
+	}
+
+	tests := map[string]func(t *testing.T){
+		"TestConsensus_ValidFlow":    TestConsensus_ValidFlow,
+		"TestConsensus_InvalidBlock": TestConsensus_InvalidBlock,
+		"TestDropMaxFaultyPlusOne":   TestDropMaxFaultyPlusOne,
+		"TestDropMaxFaulty":          TestDropMaxFaulty,
+
+		// property bases tests
+		//"TestProperty_MajorityHonestNodes": TestProperty_MajorityHonestNodes,
+	}
+
+	for name, test := range tests {
+		t.Parallel()
+
+		test := test
+
+		t.Run(name, test)
+	}
+}

deps/dummy.go

@@ -0,0 +1,5 @@
+package deps
+
+import (
+	_ "github.com/JekaMas/go-mutesting"
+)

go.mod

@@ -3,6 +3,7 @@ module github.com/0xPolygon/go-ibft
 go 1.19
 
 require (
+	github.com/JekaMas/go-mutesting v1.1.1
 	github.com/google/uuid v1.3.0
 	github.com/stretchr/testify v1.8.1
 	go.uber.org/goleak v1.2.0
@@ -14,5 +15,8 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
+	golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
+	golang.org/x/tools v0.1.12 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,3 +1,5 @@
+github.com/JekaMas/go-mutesting v1.1.1 h1:jdgkCgMRxFXvluWGTrkSKjdBPeEczkYQoUM5P4MfaOw=
+github.com/JekaMas/go-mutesting v1.1.1/go.mod h1:4MvW+K744lDEkfvAP0hazhPthWMcnsQur7QhLebcb7A=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -22,7 +24,12 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o
 go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
 go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
-golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=