diff --git a/.github/workflows/conformance-report.yml b/.github/workflows/conformance-report.yml index a02f0a825..945c8a480 100644 --- a/.github/workflows/conformance-report.yml +++ b/.github/workflows/conformance-report.yml @@ -115,10 +115,10 @@ jobs: O="$COMPARISON_REPORT_NAME" ARGS="-t $T -d $D -bf $BF -bt $BT -tf $TF -tt $TT -o $O" gradle :test:partiql-tests-runner:run --args="$ARGS" - - name: "Artifact CROSS-COMMIT-REPORT-PARTIQLCORE-EXTENDED Report (appends to comparison_report.md)" + - name: "Artifact CROSS-COMMIT-REPORT-PARTIQLEXTENDED Report (appends to comparison_report.md)" continue-on-error: true run: | - T="CROSS-COMMIT-REPORT-PARTIQLCORE-EXTENDED" + T="CROSS-COMMIT-REPORT-PARTIQLEXTENDED" D="PartiQLExtended" BF="$GITHUB_WORKSPACE/artifact/partiqlextended/$CONFORMANCE_REPORT_NAME" BT="${{ github.event.pull_request.base.sha }}" @@ -155,10 +155,10 @@ jobs: L="$COMMENT_SIZE_LIMIT" ARGS="-t $T -d $D -bf $BF -bt $BT -tf $TF -tt $TT -o $O -l $L" gradle :test:partiql-tests-runner:run --args="$ARGS" - - name: "Comment CROSS-COMMIT-REPORT-PARTIQLCORE-EXTENDED (appends to comparison_report_limited.md)" + - name: "Comment CROSS-COMMIT-REPORT-PARTIQLEXTENDED (appends to comparison_report_limited.md)" continue-on-error: true run: | - T="CROSS-COMMIT-REPORT-PARTIQLCORE-EXTENDED" + T="CROSS-COMMIT-REPORT-PARTIQLEXTENDED" D="PartiQLExtended" BF="$GITHUB_WORKSPACE/artifact/partiqlextended/$CONFORMANCE_REPORT_NAME" BT="${{ github.event.pull_request.base.sha }}" diff --git a/test/partiql-tests b/test/partiql-tests index c40ef9738..67d22dcd5 160000 --- a/test/partiql-tests +++ b/test/partiql-tests @@ -1 +1 @@ -Subproject commit c40ef9738ed812ac8322c8c63aa3cbe3e2fee18f +Subproject commit 67d22dcd5a7eeaaa2624be9d1bb3da6a8b9faf0a diff --git a/test/partiql-tests-runner/src/main/kotlin/org/partiql/runner/ReportAnalyzer.kt b/test/partiql-tests-runner/src/main/kotlin/org/partiql/runner/ReportAnalyzer.kt index 1b71f9197..d9355a9cd 100644 --- a/test/partiql-tests-runner/src/main/kotlin/org/partiql/runner/ReportAnalyzer.kt +++ b/test/partiql-tests-runner/src/main/kotlin/org/partiql/runner/ReportAnalyzer.kt @@ -18,6 +18,7 @@ class ReportAnalyzer( const val TARGET = "TARGET" } + private val firstTotal = first.passingSet + first.failingSet + first.ignoredSet private val passingInBoth = first.passingSet.intersect(second.passingSet) private val failingInBoth = first.failingSet.intersect(second.failingSet) private val ignoredInBoth = first.ignoredSet.intersect(second.ignoredSet) @@ -25,6 +26,9 @@ class ReportAnalyzer( private val passingFirstIgnoredSecond = first.passingSet.intersect(second.ignoredSet) private val failureFirstPassingSecond = first.failingSet.intersect(second.passingSet) private val ignoredFirstPassingSecond = first.ignoredSet.intersect(second.passingSet) + private val newPassing = second.passingSet - firstTotal + private val newFailing = second.failingSet - firstTotal + private val newIgnored = second.ignoredSet - firstTotal private val firstPassingSize = first.passingSet.size private val firstFailingSize = first.failingSet.size private val firstIgnoreSize = first.ignoredSet.size @@ -46,6 +50,7 @@ class ReportAnalyzer( appendTitle(this) appendTable(this) appendSummary(this) + appendOptionalNewTests(this, limit) appendOptionalNowFailingTests(this, limit, passingFirstFailingSecond, "FAILING") appendOptionalNowFailingTests(this, limit, passingFirstIgnoredSecond, "IGNORED") appendOptionalNowPassingTests(this, limit, failureFirstPassingSecond, "FAILING") @@ -104,11 +109,14 @@ class ReportAnalyzer( out.appendMarkdown("## Testing Details") out.appendLine("- **Base Commit**: ${first.commitId}") out.appendLine("- **Target Commit**: ${second.commitId}") - + out.appendLine("- **Java Version**: ${System.getProperty("java.version")}") out.appendMarkdown("## Result Details") if (passingFirstFailingSecond.isNotEmpty() || passingFirstIgnoredSecond.isNotEmpty()) { out.appendLine("- **$ICON_X REGRESSION DETECTED. See *Now Failing/Ignored Tests*. $ICON_X**") } + out.appendLine("- **New passing tests**: ${newPassing.count()}") + out.appendLine("- **New failing tests**: ${newFailing.count()}") + out.appendLine("- **New ignored tests**: ${newIgnored.count()}") out.appendLine("- **Passing in both**: ${passingInBoth.count()}") out.appendLine("- **Failing in both**: ${failingInBoth.count()}") out.appendLine("- **Ignored in both**: ${ignoredInBoth.count()}") @@ -118,6 +126,34 @@ class ReportAnalyzer( out.appendLine("- **IGNORED in $BASE but now PASSING in $TARGET**: ${ignoredFirstPassingSecond.count()}") } + private fun appendOptionalNewTests(out: Appendable, limit: Int) { + if (this.newPassing.isNotEmpty() || this.newFailing.isNotEmpty() || this.newIgnored.isNotEmpty()) { + out.appendMarkdown("## New Tests Added") + // character count limitation with comments in GitHub + // also, not ideal to list out hundreds of test names + if (newPassing.size + newFailing.size + newIgnored.size < limit) { + out.appendMarkdown("
Click here to see") + var i: Int = 0 + out.appendLine("### Passing $ICON_CHECK") + newPassing.forEach { testName -> + out.appendLine("${i++}. $testName") + } + out.appendLine("### Failing $ICON_X") + newFailing.forEach { testName -> + out.appendLine("${i++}. $testName") + } + out.appendLine("### Ignored $ICON_DIAMOND_ORANGE") + newIgnored.forEach { testName -> + out.appendLine("${i++}. $testName") + } + + out.appendMarkdown("
") + } else { + out.appendMarkdown("The complete list can be found in GitHub CI summary, either from Step Summary or in the Artifact.") + } + } + } + private fun appendOptionalNowFailingTests(out: Appendable, limit: Int, set: Set, description: String) { if (set.isNotEmpty()) { out.appendMarkdown("## Now $description Tests $ICON_X") diff --git a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ConformanceTestBase.kt b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ConformanceTestBase.kt index 38c545bd9..22a10010f 100644 --- a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ConformanceTestBase.kt +++ b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ConformanceTestBase.kt @@ -1,7 +1,9 @@ package org.partiql.runner +import org.junit.jupiter.api.TestInstance import org.partiql.runner.test.TestRunner +@TestInstance(TestInstance.Lifecycle.PER_CLASS) // Annotation is need to avoid skipList being recreated for each test. abstract class ConformanceTestBase { abstract val runner: TestRunner