diff --git a/.github/workflows/conformance-report.yml b/.github/workflows/conformance-report.yml
index a02f0a825..945c8a480 100644
--- a/.github/workflows/conformance-report.yml
+++ b/.github/workflows/conformance-report.yml
@@ -115,10 +115,10 @@ jobs:
O="$COMPARISON_REPORT_NAME"
ARGS="-t $T -d $D -bf $BF -bt $BT -tf $TF -tt $TT -o $O"
gradle :test:partiql-tests-runner:run --args="$ARGS"
- - name: "Artifact CROSS-COMMIT-REPORT-PARTIQLCORE-EXTENDED Report (appends to comparison_report.md)"
+ - name: "Artifact CROSS-COMMIT-REPORT-PARTIQLEXTENDED Report (appends to comparison_report.md)"
continue-on-error: true
run: |
- T="CROSS-COMMIT-REPORT-PARTIQLCORE-EXTENDED"
+ T="CROSS-COMMIT-REPORT-PARTIQLEXTENDED"
D="PartiQLExtended"
BF="$GITHUB_WORKSPACE/artifact/partiqlextended/$CONFORMANCE_REPORT_NAME"
BT="${{ github.event.pull_request.base.sha }}"
@@ -155,10 +155,10 @@ jobs:
L="$COMMENT_SIZE_LIMIT"
ARGS="-t $T -d $D -bf $BF -bt $BT -tf $TF -tt $TT -o $O -l $L"
gradle :test:partiql-tests-runner:run --args="$ARGS"
- - name: "Comment CROSS-COMMIT-REPORT-PARTIQLCORE-EXTENDED (appends to comparison_report_limited.md)"
+ - name: "Comment CROSS-COMMIT-REPORT-PARTIQLEXTENDED (appends to comparison_report_limited.md)"
continue-on-error: true
run: |
- T="CROSS-COMMIT-REPORT-PARTIQLCORE-EXTENDED"
+ T="CROSS-COMMIT-REPORT-PARTIQLEXTENDED"
D="PartiQLExtended"
BF="$GITHUB_WORKSPACE/artifact/partiqlextended/$CONFORMANCE_REPORT_NAME"
BT="${{ github.event.pull_request.base.sha }}"
diff --git a/test/partiql-tests b/test/partiql-tests
index c40ef9738..67d22dcd5 160000
--- a/test/partiql-tests
+++ b/test/partiql-tests
@@ -1 +1 @@
-Subproject commit c40ef9738ed812ac8322c8c63aa3cbe3e2fee18f
+Subproject commit 67d22dcd5a7eeaaa2624be9d1bb3da6a8b9faf0a
diff --git a/test/partiql-tests-runner/src/main/kotlin/org/partiql/runner/ReportAnalyzer.kt b/test/partiql-tests-runner/src/main/kotlin/org/partiql/runner/ReportAnalyzer.kt
index 1b71f9197..d9355a9cd 100644
--- a/test/partiql-tests-runner/src/main/kotlin/org/partiql/runner/ReportAnalyzer.kt
+++ b/test/partiql-tests-runner/src/main/kotlin/org/partiql/runner/ReportAnalyzer.kt
@@ -18,6 +18,7 @@ class ReportAnalyzer(
const val TARGET = "TARGET"
}
+ private val firstTotal = first.passingSet + first.failingSet + first.ignoredSet
private val passingInBoth = first.passingSet.intersect(second.passingSet)
private val failingInBoth = first.failingSet.intersect(second.failingSet)
private val ignoredInBoth = first.ignoredSet.intersect(second.ignoredSet)
@@ -25,6 +26,9 @@ class ReportAnalyzer(
private val passingFirstIgnoredSecond = first.passingSet.intersect(second.ignoredSet)
private val failureFirstPassingSecond = first.failingSet.intersect(second.passingSet)
private val ignoredFirstPassingSecond = first.ignoredSet.intersect(second.passingSet)
+ private val newPassing = second.passingSet - firstTotal
+ private val newFailing = second.failingSet - firstTotal
+ private val newIgnored = second.ignoredSet - firstTotal
private val firstPassingSize = first.passingSet.size
private val firstFailingSize = first.failingSet.size
private val firstIgnoreSize = first.ignoredSet.size
@@ -46,6 +50,7 @@ class ReportAnalyzer(
appendTitle(this)
appendTable(this)
appendSummary(this)
+ appendOptionalNewTests(this, limit)
appendOptionalNowFailingTests(this, limit, passingFirstFailingSecond, "FAILING")
appendOptionalNowFailingTests(this, limit, passingFirstIgnoredSecond, "IGNORED")
appendOptionalNowPassingTests(this, limit, failureFirstPassingSecond, "FAILING")
@@ -104,11 +109,14 @@ class ReportAnalyzer(
out.appendMarkdown("## Testing Details")
out.appendLine("- **Base Commit**: ${first.commitId}")
out.appendLine("- **Target Commit**: ${second.commitId}")
-
+ out.appendLine("- **Java Version**: ${System.getProperty("java.version")}")
out.appendMarkdown("## Result Details")
if (passingFirstFailingSecond.isNotEmpty() || passingFirstIgnoredSecond.isNotEmpty()) {
out.appendLine("- **$ICON_X REGRESSION DETECTED. See *Now Failing/Ignored Tests*. $ICON_X**")
}
+ out.appendLine("- **New passing tests**: ${newPassing.count()}")
+ out.appendLine("- **New failing tests**: ${newFailing.count()}")
+ out.appendLine("- **New ignored tests**: ${newIgnored.count()}")
out.appendLine("- **Passing in both**: ${passingInBoth.count()}")
out.appendLine("- **Failing in both**: ${failingInBoth.count()}")
out.appendLine("- **Ignored in both**: ${ignoredInBoth.count()}")
@@ -118,6 +126,34 @@ class ReportAnalyzer(
out.appendLine("- **IGNORED in $BASE but now PASSING in $TARGET**: ${ignoredFirstPassingSecond.count()}")
}
+ private fun appendOptionalNewTests(out: Appendable, limit: Int) {
+ if (this.newPassing.isNotEmpty() || this.newFailing.isNotEmpty() || this.newIgnored.isNotEmpty()) {
+ out.appendMarkdown("## New Tests Added")
+ // character count limitation with comments in GitHub
+ // also, not ideal to list out hundreds of test names
+ if (newPassing.size + newFailing.size + newIgnored.size < limit) {
+ out.appendMarkdown("Click here to see
")
+ var i: Int = 0
+ out.appendLine("### Passing $ICON_CHECK")
+ newPassing.forEach { testName ->
+ out.appendLine("${i++}. $testName")
+ }
+ out.appendLine("### Failing $ICON_X")
+ newFailing.forEach { testName ->
+ out.appendLine("${i++}. $testName")
+ }
+ out.appendLine("### Ignored $ICON_DIAMOND_ORANGE")
+ newIgnored.forEach { testName ->
+ out.appendLine("${i++}. $testName")
+ }
+
+ out.appendMarkdown(" ")
+ } else {
+ out.appendMarkdown("The complete list can be found in GitHub CI summary, either from Step Summary or in the Artifact.")
+ }
+ }
+ }
+
private fun appendOptionalNowFailingTests(out: Appendable, limit: Int, set: Set, description: String) {
if (set.isNotEmpty()) {
out.appendMarkdown("## Now $description Tests $ICON_X")
diff --git a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ConformanceTestBase.kt b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ConformanceTestBase.kt
index 38c545bd9..22a10010f 100644
--- a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ConformanceTestBase.kt
+++ b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ConformanceTestBase.kt
@@ -1,7 +1,9 @@
package org.partiql.runner
+import org.junit.jupiter.api.TestInstance
import org.partiql.runner.test.TestRunner
+@TestInstance(TestInstance.Lifecycle.PER_CLASS) // Annotation is need to avoid skipList being recreated for each test.
abstract class ConformanceTestBase {
abstract val runner: TestRunner