Flaky test detector

.github/workflows/flaky.yml

@@ -0,0 +1,62 @@
+# This GitHub Actions workflow finds new tests in pull requests
+# and runs them repeatedly to check for flakiness.
+name: 'Flaky Test Detector'
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  stress-new-test-additions:
+    name: Stress new tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: "go.mod"
+
+      - name: Find New Tests
+        id: find_new_tests
+        run: go run scripts/gitTestDetector.go
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+
+      - name: Run New Tests
+        if: steps.find_new_tests.outputs.new_tests != ''
+        env:
+          RUN_COUNT: 100
+          TOTAL_TIMEOUT: 100s
+        run: |
+          echo "Found new tests to run:"
+          echo "${{ steps.find_new_tests.outputs.new_tests }}" | tr ' ' '\n'
+          fail=0
+          for test_info in ${{ steps.find_new_tests.outputs.new_tests }}; do
+            test_package=$(echo $test_info | cut -d':' -f1)
+            test_name=$(echo $test_info | cut -d':' -f2)
+
+            echo "---"
+            echo "Running test '$test_package/$test_name' for up to ${TOTAL_TIMEOUT}"
+
+            exit_code=0
+            timeout ${TOTAL_TIMEOUT} go test -v -failfast -count=${RUN_COUNT} ./$test_package -run "^${test_name}$" || exit_code=$?
+
+            if [ $exit_code -ne 0 ]; then
+              if [ $exit_code -eq 124 ]; then
+                echo "::warning title=Test Timeout::Test '$test_name' did not complete ${RUN_COUNT} runs within the time limit."
+              else
+                echo "::error title=Flaky test detected:: Test '$test_name' failed with exit code $exit_code during one of its ${RUN_COUNT} runs."
+                fail=1
+              fi
+            else
+              echo "Test '$test_name' passed all ${RUN_COUNT} runs within the time limit."
+            fi
+          done
+          exit $fail

scripts/gitTestDetector.go

@@ -0,0 +1,120 @@
+package main
+
+import (
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+func main() {
+	baseCommit := os.Getenv("BASE_SHA")
+	headCommit := os.Getenv("HEAD_SHA")
+
+	if baseCommit == "" || headCommit == "" {
+		fmt.Println("Warning: BASE_SHA or HEAD_SHA not set. Falling back to 'HEAD~1...HEAD'.")
+		baseCommit = "HEAD~1"
+		headCommit = "HEAD"
+	}
+
+	fmt.Printf("Checking for new test cases between %s and %s\n", baseCommit, headCommit)
+
+	diffCmd := exec.Command("git", "diff", "--name-only",
+		baseCommit, headCommit, "--", "*_test.go")
+	diffOutput, err := diffCmd.Output()
+	if err != nil {
+		log.Fatalf("Failed to run git diff: %v", err)
+	}
+
+	changedTestFiles := strings.Split(strings.TrimSpace(string(diffOutput)), "\n")
+	if len(changedTestFiles) == 0 || changedTestFiles[0] == "" {
+		fmt.Println("No `_test.go` files were changed in this pull request.")
+		setOutput("new_tests", "")
+		os.Exit(0)
+	}
+	fmt.Printf("Found changed test files: %v\n", changedTestFiles)
+
+	var newTests []string
+
+	for _, file := range changedTestFiles {
+		oldContent, _ := getFileContentAtCommit(baseCommit, file)
+		newContent, err := getFileContentAtCommit(headCommit, file)
+		if err != nil {
+			fmt.Printf("Could not get new content for %s: %v\n", file, err)
+			continue
+		}
+
+		oldTests, _ := getTestFunctions(oldContent)
+		newTestsMap, err := getTestFunctions(newContent)
+		if err != nil {
+			fmt.Printf("Could not parse new content for %s: %v\n", file, err)
+			continue
+		}
+
+		testPackage := filepath.Dir(file)
+
+		for testName := range newTestsMap {
+			if !oldTests[testName] {
+				fmt.Printf("Found new test: '%s' in file '%s'\n", testName, file)
+				newTests = append(newTests, fmt.Sprintf("%s:%s", testPackage, testName))
+			}
+		}
+	}
+
+	if len(newTests) > 0 {
+		setOutput("new_tests", strings.Join(newTests, " "))
+	} else {
+		fmt.Println("No new test functions were found in the changed files.")
+		setOutput("new_tests", "")
+	}
+}
+
+// setOutput appends a key-value pair to the GITHUB_OUTPUT file.
+func setOutput(name, value string) {
+	outputFile := os.Getenv("GITHUB_OUTPUT")
+	if outputFile == "" {
+		fmt.Println("GITHUB_OUTPUT not set. Skipping setting output.")
+		return
+	}
+	f, err := os.OpenFile(outputFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		log.Fatalf("Failed to open GITHUB_OUTPUT file: %v", err)
+	}
+	defer f.Close()
+	if _, err := f.WriteString(fmt.Sprintf("%s=%s\n", name, value)); err != nil {
+		log.Fatalf("Failed to write to GITHUB_OUTPUT file: %v", err)
+	}
+}
+
+// getFileContentAtCommit retrieves the content of a file at a specific git commit.
+func getFileContentAtCommit(commitHash, filePath string) (string, error) {
+	cmd := exec.Command("git", "show", fmt.Sprintf("%s:%s", commitHash, filePath))
+	output, err := cmd.Output()
+	if err != nil {
+		return "", nil
+	}
+	return string(output), nil
+}
+
+// getTestFunctions parses Go source code and returns a map of test function names.
+func getTestFunctions(source string) (map[string]bool, error) {
+	fset := token.NewFileSet()
+	node, err := parser.ParseFile(fset, "src.go", source, 0)
+	if err != nil {
+		return nil, err
+	}
+	tests := make(map[string]bool)
+	ast.Inspect(node, func(n ast.Node) bool {
+		fn, ok := n.(*ast.FuncDecl)
+		if ok && strings.HasPrefix(fn.Name.Name, "Test") {
+			tests[fn.Name.Name] = true
+		}
+		return true
+	})
+	return tests, nil
+}

server/pse/pse_test.go

@@ -22,6 +22,10 @@ import (
 	"testing"
 )
 
+func TestInServerPSE(t *testing.T) {
+
+}
+
 func TestPSEmulationCPU(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		t.Skipf("Skipping this test on Windows")

server/raft_test.go

@@ -3781,3 +3781,18 @@ func TestNRGChainOfBlocksStopAndCatchUp(t *testing.T) {
 		}
 	}
 }
+
+// func TestFlakyDetectorFail(t *testing.T) {
+// 	if rand.Intn(10) == 0 {
+// 		t.FailNow()
+// 	}
+// }
+
+func TestFlakyDetectorCI(t *testing.T) {
+	// TEST ONLY
+}
+
+func TestFlakyDetectorCILong(t *testing.T) {
+	// TEST ONLY
+	time.Sleep(10 * time.Second)
+}