tests: Clean up LifecycleFuzzTest

Makes the test less indirect and also adds stronger assertions on the
precise order in which lifecycle methods are called.

Author: fmeum

examples/junit/src/test/java/com/example/BUILD.bazel

@@ -1,5 +1,11 @@
 load("//bazel:fuzz_target.bzl", "java_fuzz_target_test")
 
+java_library(
+    name = "test_successful_exception",
+    srcs = ["TestSuccessfulException.java"],
+    visibility = ["//src/test/java/com/code_intelligence/jazzer/junit:__subpackages__"],
+)
+
 java_binary(
     name = "ExampleFuzzTests",
     testonly = True,
@@ -9,11 +15,13 @@ java_binary(
         "//src/test/java/com/code_intelligence/jazzer/junit:__pkg__",
     ],
     deps = [
+        ":test_successful_exception",
         "//deploy:jazzer",
         "//deploy:jazzer-api",
         "//deploy:jazzer-junit",
         "//examples/junit/src/main/java/com/example:parser",
         "//examples/junit/src/test/resources:example_seed_corpora",
+        "@maven//:com_google_truth_truth",
         "@maven//:org_junit_jupiter_junit_jupiter_api",
         "@maven//:org_junit_jupiter_junit_jupiter_params",
         "@maven//:org_mockito_mockito_core",
@@ -63,18 +71,20 @@ java_fuzz_target_test(
 java_fuzz_target_test(
     name = "LifecycleFuzzTest",
     srcs = ["LifecycleFuzzTest.java"],
-    allowed_findings = ["java.io.IOException"],
+    allowed_findings = ["com.example.TestSuccessfulException"],
     fuzzer_args = [
-        "-runs=0",
+        "-runs=3",
     ],
     target_class = "com.example.LifecycleFuzzTest",
     verify_crash_reproducer = False,
     runtime_deps = [
         ":junit_runtime",
     ],
     deps = [
+        ":test_successful_exception",
         "//examples/junit/src/main/java/com/example:parser",
         "//src/main/java/com/code_intelligence/jazzer/junit:fuzz_test",
+        "@maven//:com_google_truth_truth",
         "@maven//:org_junit_jupiter_junit_jupiter_api",
     ],
 )

examples/junit/src/test/java/com/example/LifecycleFuzzTest.java

@@ -16,73 +16,127 @@
 
 package com.example;
 
+import static com.google.common.truth.Truth.assertThat;
+import static java.util.Arrays.asList;
+import static java.util.Collections.unmodifiableList;
+
+import com.code_intelligence.jazzer.api.FuzzerSecurityIssueLow;
 import com.code_intelligence.jazzer.junit.FuzzTest;
+import com.example.LifecycleFuzzTest.LifecycleCallbacks1;
+import com.example.LifecycleFuzzTest.LifecycleCallbacks2;
+import com.example.LifecycleFuzzTest.LifecycleCallbacks3;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.MethodOrderer;
 import org.junit.jupiter.api.TestMethodOrder;
+import org.junit.jupiter.api.extension.AfterEachCallback;
+import org.junit.jupiter.api.extension.BeforeEachCallback;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.api.extension.ExtensionContext;
 import org.junit.jupiter.api.extension.TestInstancePostProcessor;
 
 @TestMethodOrder(MethodOrderer.MethodName.class)
 @ExtendWith(LifecycleFuzzTest.LifecycleInstancePostProcessor.class)
+@ExtendWith(LifecycleCallbacks1.class)
+@ExtendWith(LifecycleCallbacks2.class)
+@ExtendWith(LifecycleCallbacks3.class)
 class LifecycleFuzzTest {
-  // In fuzzing mode, the test is invoked once on the empty input and once with Jazzer.
-  private static final int EXPECTED_EACH_COUNT =
-      System.getenv().getOrDefault("JAZZER_FUZZ", "").isEmpty() ? 1 : 2;
-
-  private static int beforeAllCount = 0;
-  private static int beforeEachGlobalCount = 0;
-  private static int afterEachGlobalCount = 0;
-  private static int afterAllCount = 0;
+  private static final ArrayList<String> events = new ArrayList<>();
 
   private boolean beforeEachCalledOnInstance = false;
   private boolean testInstancePostProcessorCalledOnInstance = false;
 
   @BeforeAll
   static void beforeAll() {
-    beforeAllCount++;
+    events.add("beforeAll");
   }
 
   @BeforeEach
-  void beforeEach() {
-    beforeEachGlobalCount++;
+  void beforeEach1() {
+    events.add("beforeEach1");
     beforeEachCalledOnInstance = true;
   }
 
+  @BeforeEach
+  void beforeEach2() {
+    events.add("beforeEach2");
+  }
+
+  @BeforeEach
+  void beforeEach3() {
+    events.add("beforeEach3");
+  }
+
   @Disabled
   @FuzzTest
   void disabledFuzz(byte[] data) {
+    events.add("disabledFuzz");
     throw new AssertionError("This test should not be executed");
   }
 
-  @FuzzTest(maxDuration = "1s")
+  @FuzzTest(maxExecutions = 3)
   void lifecycleFuzz(byte[] data) {
-    Assertions.assertEquals(1, beforeAllCount);
-    Assertions.assertEquals(beforeEachGlobalCount, afterEachGlobalCount + 1);
-    Assertions.assertTrue(beforeEachCalledOnInstance);
-    Assertions.assertTrue(testInstancePostProcessorCalledOnInstance);
+    events.add("lifecycleFuzz");
+    assertThat(beforeEachCalledOnInstance).isTrue();
+    assertThat(testInstancePostProcessorCalledOnInstance).isTrue();
+  }
+
+  @AfterEach
+  void afterEach1() {
+    events.add("afterEach1");
   }
 
   @AfterEach
-  void afterEach() {
-    afterEachGlobalCount++;
+  void afterEach2() {
+    events.add("afterEach2");
+  }
+
+  @AfterEach
+  void afterEach3() {
+    events.add("afterEach3");
   }
 
   @AfterAll
-  static void afterAll() throws IOException {
-    afterAllCount++;
-    Assertions.assertEquals(1, beforeAllCount);
-    Assertions.assertEquals(EXPECTED_EACH_COUNT, beforeEachGlobalCount);
-    Assertions.assertEquals(EXPECTED_EACH_COUNT, afterEachGlobalCount);
-    Assertions.assertEquals(1, afterAllCount);
-    throw new IOException();
+  static void afterAll() throws TestSuccessfulException {
+    events.add("afterAll");
+
+    boolean isRegressionTest = "".equals(System.getenv("JAZZER_FUZZ"));
+    boolean isFuzzingFromCommandLine = System.getenv("JAZZER_FUZZ") == null;
+    boolean isFuzzingFromJUnit = !isFuzzingFromCommandLine && !isRegressionTest;
+
+    final List<String> expectedBeforeEachEvents = unmodifiableList(asList("beforeEachCallback1",
+        "beforeEachCallback2", "beforeEachCallback3", "beforeEach1", "beforeEach2", "beforeEach3"));
+    final List<String> expectedAfterEachEvents = unmodifiableList(asList("afterEach1", "afterEach2",
+        "afterEach3", "afterEachCallback3", "afterEachCallback2", "afterEachCallback1"));
+
+    ArrayList<String> expectedEvents = new ArrayList<>();
+    expectedEvents.add("beforeAll");
+
+    // When run from the command-line, the fuzz test is not separately executed on the empty seed.
+    if (isRegressionTest || isFuzzingFromJUnit) {
+      expectedEvents.addAll(expectedBeforeEachEvents);
+      expectedEvents.add("lifecycleFuzz");
+      expectedEvents.addAll(expectedAfterEachEvents);
+    }
+    if (isFuzzingFromJUnit || isFuzzingFromCommandLine) {
+      expectedEvents.addAll(expectedBeforeEachEvents);
+      // TODO: Fuzz tests currently don't run before each and after each methods between fuzz test
+      // invocations.
+      expectedEvents.addAll(Collections.nCopies(3, "lifecycleFuzz"));
+      expectedEvents.addAll(expectedAfterEachEvents);
+    }
+
+    expectedEvents.add("afterAll");
+
+    assertThat(events).containsExactlyElementsIn(expectedEvents).inOrder();
+    throw new TestSuccessfulException("Lifecycle methods invoked as expected");
   }
 
   static class LifecycleInstancePostProcessor implements TestInstancePostProcessor {
@@ -91,4 +145,40 @@ public void postProcessTestInstance(Object o, ExtensionContext extensionContext)
       ((LifecycleFuzzTest) o).testInstancePostProcessorCalledOnInstance = true;
     }
   }
+
+  static class LifecycleCallbacks1 implements BeforeEachCallback, AfterEachCallback {
+    @Override
+    public void beforeEach(ExtensionContext extensionContext) {
+      events.add("beforeEachCallback1");
+    }
+
+    @Override
+    public void afterEach(ExtensionContext extensionContext) {
+      events.add("afterEachCallback1");
+    }
+  }
+
+  static class LifecycleCallbacks2 implements BeforeEachCallback, AfterEachCallback {
+    @Override
+    public void beforeEach(ExtensionContext extensionContext) {
+      events.add("beforeEachCallback2");
+    }
+
+    @Override
+    public void afterEach(ExtensionContext extensionContext) {
+      events.add("afterEachCallback2");
+    }
+  }
+
+  static class LifecycleCallbacks3 implements BeforeEachCallback, AfterEachCallback {
+    @Override
+    public void beforeEach(ExtensionContext extensionContext) {
+      events.add("beforeEachCallback3");
+    }
+
+    @Override
+    public void afterEach(ExtensionContext extensionContext) {
+      events.add("afterEachCallback3");
+    }
+  }
 }

examples/junit/src/test/java/com/example/TestSuccessfulException.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2023 Code Intelligence GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.example;
+
+/**
+ * An exception thrown by a java_fuzz_target_test if the test run is considered successful.
+ *
+ * <p>Use this instead of a generic exception to ensure that tests do not pass if such a generic
+ * exception is thrown unexpectedly.
+ * <p>Use this instead of {@link com.code_intelligence.jazzer.api.FuzzerSecurityIssueLow} and other
+ * Jazzer-specific exceptions as using them in tests leads to classloader issues: The exception
+ * classes may be loaded both in the bootstrap and the system classloader depending on when exactly
+ * the agent (and with it the bootstrap jar) is installed, which can cause in `instanceof` checks
+ * failing unexpectedly.
+ */
+public class TestSuccessfulException extends Exception {
+  public TestSuccessfulException(String message) {
+    super(message);
+  }
+}

src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel

@@ -245,6 +245,7 @@ java_test(
             "@maven//:org_junit_jupiter_junit_jupiter_engine",
         ],
         deps = [
+            "//examples/junit/src/test/java/com/example:test_successful_exception",
             "//src/main/java/com/code_intelligence/jazzer/api:hooks",
             "@maven//:junit_junit",
             "@maven//:org_junit_platform_junit_platform_engine",

src/test/java/com/code_intelligence/jazzer/junit/LifecycleTest.java

@@ -32,6 +32,8 @@
 import static org.junit.platform.testkit.engine.EventType.STARTED;
 import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf;
 
+import com.code_intelligence.jazzer.api.FuzzerSecurityIssueLow;
+import com.example.TestSuccessfulException;
 import java.io.IOException;
 import java.nio.file.Path;
 import org.junit.Before;
@@ -80,7 +82,7 @@ public void fuzzingEnabled() {
         event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ)),
             finishedSuccessfully()),
         event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)),
-            finishedWithFailure(instanceOf(IOException.class))),
+            finishedWithFailure(instanceOf(TestSuccessfulException.class))),
         event(type(FINISHED), container(ENGINE), finishedSuccessfully()));
 
     results.testEvents().assertEventsMatchExactly(
@@ -116,7 +118,7 @@ public void fuzzingDisabled() {
             container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))),
         event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))),
         event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)),
-            finishedWithFailure(instanceOf(IOException.class))),
+            finishedWithFailure(instanceOf(TestSuccessfulException.class))),
         event(type(FINISHED), container(ENGINE), finishedSuccessfully()));
 
     results.testEvents().assertEventsMatchExactly(