improve conformance comparison report for new tests

.github/workflows/conformance-report.yml

@@ -115,10 +115,10 @@ jobs:
           O="$COMPARISON_REPORT_NAME"
           ARGS="-t $T -d $D -bf $BF -bt $BT -tf $TF -tt $TT -o $O"
           gradle :test:partiql-tests-runner:run --args="$ARGS"
-      - name: "Artifact CROSS-COMMIT-REPORT-PARTIQLCORE-EXTENDED Report (appends to comparison_report.md)"
+      - name: "Artifact CROSS-COMMIT-REPORT-PARTIQLEXTENDED Report (appends to comparison_report.md)"
         continue-on-error: true
         run: |
-          T="CROSS-COMMIT-REPORT-PARTIQLCORE-EXTENDED"
+          T="CROSS-COMMIT-REPORT-PARTIQLEXTENDED"
           D="PartiQLExtended"
           BF="$GITHUB_WORKSPACE/artifact/partiqlextended/$CONFORMANCE_REPORT_NAME"
           BT="${{ github.event.pull_request.base.sha }}"
@@ -155,10 +155,10 @@ jobs:
           L="$COMMENT_SIZE_LIMIT"
           ARGS="-t $T -d $D -bf $BF -bt $BT -tf $TF -tt $TT -o $O -l $L"
           gradle :test:partiql-tests-runner:run --args="$ARGS"
-      - name: "Comment CROSS-COMMIT-REPORT-PARTIQLCORE-EXTENDED (appends to comparison_report_limited.md)"
+      - name: "Comment CROSS-COMMIT-REPORT-PARTIQLEXTENDED (appends to comparison_report_limited.md)"
         continue-on-error: true
         run: |
-          T="CROSS-COMMIT-REPORT-PARTIQLCORE-EXTENDED"
+          T="CROSS-COMMIT-REPORT-PARTIQLEXTENDED"
           D="PartiQLExtended"
           BF="$GITHUB_WORKSPACE/artifact/partiqlextended/$CONFORMANCE_REPORT_NAME"          
           BT="${{ github.event.pull_request.base.sha }}"

test/partiql-tests-runner/src/main/kotlin/org/partiql/runner/ReportAnalyzer.kt

@@ -18,13 +18,17 @@ class ReportAnalyzer(
         const val TARGET = "TARGET"
     }
 
+    private val firstTotal = first.passingSet + first.failingSet + first.ignoredSet
     private val passingInBoth = first.passingSet.intersect(second.passingSet)
     private val failingInBoth = first.failingSet.intersect(second.failingSet)
     private val ignoredInBoth = first.ignoredSet.intersect(second.ignoredSet)
     private val passingFirstFailingSecond = first.passingSet.intersect(second.failingSet)
     private val passingFirstIgnoredSecond = first.passingSet.intersect(second.ignoredSet)
     private val failureFirstPassingSecond = first.failingSet.intersect(second.passingSet)
     private val ignoredFirstPassingSecond = first.ignoredSet.intersect(second.passingSet)
+    private val newPassing = second.passingSet - firstTotal
+    private val newFailing = second.failingSet - firstTotal
+    private val newIgnored = second.ignoredSet - firstTotal
     private val firstPassingSize = first.passingSet.size
     private val firstFailingSize = first.failingSet.size
     private val firstIgnoreSize = first.ignoredSet.size
@@ -46,6 +50,7 @@ class ReportAnalyzer(
             appendTitle(this)
             appendTable(this)
             appendSummary(this)
+            appendOptionalNewTests(this, limit)
             appendOptionalNowFailingTests(this, limit, passingFirstFailingSecond, "FAILING")
             appendOptionalNowFailingTests(this, limit, passingFirstIgnoredSecond, "IGNORED")
             appendOptionalNowPassingTests(this, limit, failureFirstPassingSecond, "FAILING")
@@ -104,11 +109,14 @@ class ReportAnalyzer(
         out.appendMarkdown("## Testing Details")
         out.appendLine("- **Base Commit**: ${first.commitId}")
         out.appendLine("- **Target Commit**: ${second.commitId}")
-
+        out.appendLine("- **Java Version**: ${System.getProperty("java.version")}")
         out.appendMarkdown("## Result Details")
         if (passingFirstFailingSecond.isNotEmpty() || passingFirstIgnoredSecond.isNotEmpty()) {
             out.appendLine("- **$ICON_X REGRESSION DETECTED. See *Now Failing/Ignored Tests*. $ICON_X**")
         }
+        out.appendLine("- **New passing tests**: ${newPassing.count()}")
+        out.appendLine("- **New failing tests**: ${newFailing.count()}")
+        out.appendLine("- **New ignored tests**: ${newIgnored.count()}")
         out.appendLine("- **Passing in both**: ${passingInBoth.count()}")
         out.appendLine("- **Failing in both**: ${failingInBoth.count()}")
         out.appendLine("- **Ignored in both**: ${ignoredInBoth.count()}")
@@ -118,6 +126,34 @@ class ReportAnalyzer(
         out.appendLine("- **IGNORED in $BASE but now PASSING in $TARGET**: ${ignoredFirstPassingSecond.count()}")
     }
 
+    private fun appendOptionalNewTests(out: Appendable, limit: Int) {
+        if (this.newPassing.isNotEmpty() || this.newFailing.isNotEmpty() || this.newIgnored.isNotEmpty()) {
+            out.appendMarkdown("## New Tests Added")
+            // character count limitation with comments in GitHub
+            // also, not ideal to list out hundreds of test names
+            if (newPassing.size + newFailing.size + newIgnored.size < limit) {
+                out.appendMarkdown("<details><summary>Click here to see</summary>")
+                var i: Int = 0
+                out.appendLine("### Passing $ICON_CHECK")
+                newPassing.forEach { testName ->
+                    out.appendLine("${i++}. $testName")
+                }
+                out.appendLine("### Failing $ICON_X")
+                newFailing.forEach { testName ->
+                    out.appendLine("${i++}. $testName")
+                }
+                out.appendLine("### Ignored $ICON_DIAMOND_ORANGE")
+                newIgnored.forEach { testName ->
+                    out.appendLine("${i++}. $testName")
+                }
+
+                out.appendMarkdown("</details>")
+            } else {
+                out.appendMarkdown("The complete list can be found in GitHub CI summary, either from Step Summary or in the Artifact.")
+            }
+        }
+    }
+
     private fun appendOptionalNowFailingTests(out: Appendable, limit: Int, set: Set<String>, description: String) {
         if (set.isNotEmpty()) {
             out.appendMarkdown("## Now $description Tests $ICON_X")

test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ConformanceTestBase.kt

@@ -1,7 +1,9 @@
 package org.partiql.runner
 
+import org.junit.jupiter.api.TestInstance
 import org.partiql.runner.test.TestRunner
 
+@TestInstance(TestInstance.Lifecycle.PER_CLASS) // Annotation is need to avoid skipList being recreated for each test.
 abstract class ConformanceTestBase<T, V> {
     abstract val runner: TestRunner<T, V>