TEST: WIP: Automated summary PR commenting

.github/kokoro/run.sh

@@ -78,6 +78,25 @@ echo "----------------------------------------"
 )
 echo "----------------------------------------"
 
+echo
+echo "========================================"
+echo "Running Analysis"
+echo "----------------------------------------"
+{
+	ANALYZER=$GIT_CHECKOUT"/tools/report_analyzer.py"
+	OUT_DIR=$GIT_CHECKOUT"/out/report/"
+	COMPARE_REPORT=$OUT_DIR"/report.csv"
+	BASE_REPORT=$OUT_DIR"/base_report.csv"
+	CHANGES_SUMMARY_JSON=$OUT_DIR"/tests_summary.json"
+	CHANGES_SUMMARY_MD=$OUT_DIR"/tests_summary.md"
+
+	# Get base report from sv-tests master run
+	wget https://symbiflow.github.io/sv-tests-results/report.csv -O $BASE_REPORT
+
+	python $ANALYZER $COMPARE_REPORT $BASE_REPORT -o $CHANGES_SUMMARY_JSON -t $CHANGES_SUMMARY_MD
+}
+echo "----------------------------------------"
+
 if [[ $KOKORO_TYPE = continuous ]]; then
 	#   - "make report USE_ALL_RUNNERS=1"
 	#   - "touch out/report/.nojekyll"

.github/workflows/summary.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+ANALYZER=$PWD"/tools/report_analyzer.py"
+OUT_DIR=$PWD"/out/report/"
+COMPARE_REPORT=$OUT_DIR"/tests_report.csv"
+BASE_REPORT=$OUT_DIR"/base_report.csv"
+CHANGES_SUMMARY_JSON=$OUT_DIR"/tests_summary.json"
+CHANGES_SUMMARY_MD=$OUT_DIR"/tests_summary.md"
+
+set -x
+set -e
+
+# Get a conda environment
+wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
+bash miniconda.sh -b -p $HOME/miniconda
+source "$HOME/miniconda/etc/profile.d/conda.sh"
+hash -r
+conda config --set always_yes yes --set changeps1 no
+
+conda env create --file conf/environment.yml
+conda activate sv-test-env
+hash -r
+conda info -a
+
+# Get base report from sv-tests master run
+wget https://symbiflow.github.io/sv-tests-results/report.csv -O $BASE_REPORT
+
+# Delete headers from all report.csv
+for file in $(find ./out/report_* -name "*.csv" -print); do
+	sed -i.backup 1,1d $file
+done
+
+# concatenate test reports
+cat $(find ./out/report_* -name "*.csv" -print) >> $COMPARE_REPORT
+
+# Insert header at the first line of concatenated report
+sed -i 1i\ $(cat $(find ./out/report_* -name "*.csv.backup" -print | head -1) | head -1) $COMPARE_REPORT
+
+python $ANALYZER $COMPARE_REPORT $BASE_REPORT -o $CHANGES_SUMMARY_JSON -t $CHANGES_SUMMARY_MD
+
+set +e
+set +x
+

.github/workflows/sv-tests-ci.yml

@@ -64,3 +64,78 @@ jobs:
       - name: Run
         run:
           ./.github/workflows/run.sh
+      - name: Prepare Report
+        run:
+          ./.github/workflows/tool_report_prepare.sh
+      - uses: actions/upload-artifact@v2
+        with:
+          name: report_${{ matrix.env.JOB_NAME }}
+          path: |
+            ./out/report/${{ matrix.env.JOB_NAME }}_report.csv
+
+  Summary:
+    name: Summary
+    runs-on: ubuntu-18.04
+    needs: Run
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+      - name: Prepare output directories
+        run: mkdir -p out/report
+      - uses: actions/download-artifact@v2
+        with:
+          path: ./out/
+      - name: Summary
+        run:
+          ./.github/workflows/summary.sh
+      - uses: actions/upload-artifact@v2
+        with:
+          name: tests_summary
+          path: |
+            ./out/report/tests_summary.json
+            ./out/report/tests_summary.md
+            ./out/report/tests_report.csv
+            ./out/report/base_report.csv
+      - id: get-artifacts-to-delete
+        run: |
+          artifacts=$(find ./out -type d -name 'report_*' -exec basename {} \;)
+          echo $artifacts
+          artifacts="${artifacts//'%'/'%25'}"
+          artifacts="${artifacts//$'\n'/'%0A'}"
+          artifacts="${artifacts//$'\r'/'%0D'}"
+          echo ::set-output name=artifacts::$artifacts
+          echo $artifacts
+      - name: Delete Old Artifacts
+        uses: geekyeggo/delete-artifact@v1
+        with:
+          name: ${{ steps.get-artifacts-to-delete.outputs.artifacts }}
+
+  Comment:
+    if: github.event_name == 'pull_request'
+    name: Comment
+    runs-on: ubuntu-18.04
+    needs: Summary
+    steps:
+      - name: Prepare output directories
+        run: mkdir -p out/report
+      - uses: actions/download-artifact@v2
+        with:
+          name: tests_summary
+          path: ./out/report
+      - id: get-comment-body
+        run: |
+          body=$(cat ./out/report/tests_summary.md)
+          body="${body//'%'/'%25'}"
+          body="${body//$'\n'/'%0A'}"
+          body="${body//$'\r'/'%0D'}"
+          echo ::set-output name=body::$body
+      - name: Post comment
+        uses: KeisukeYamashita/create-comment@v1
+        with:
+          check-only-first-line: "true"
+          unique: "true"
+          token: ${{ secrets.GITHUB_TOKEN }}
+          comment: ${{ steps.get-comment-body.outputs.body }}

.github/workflows/tool_report_prepare.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+OUT_DIR=$PWD"/out/report/"
+COMPARE_REPORT=$OUT_DIR"/report.csv"
+
+set -x
+set -e
+
+source "$HOME/miniconda/etc/profile.d/conda.sh"
+hash -r
+conda activate sv-test-env
+hash -r
+
+mv $COMPARE_REPORT $OUT_DIR"/"$JOB_NAME"_report.csv"
+
+set +e
+set +x

conf/requirements.txt

@@ -6,4 +6,5 @@ yapf==0.30.0
 psutil
 mako
 make_var
+pytablewriter
 git+https://github.com/lowRISC/fusesoc.git@ot

tools/report_analyzer.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021 The SymbiFlow Authors.
+#
+# Use of this source code is governed by a ISC-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/ISC
+#
+# SPDX-License-Identifier: ISC
+
+import argparse
+import csv
+import json
+import sys
+import pytablewriter
+
+relevant_headers = ["Tool", "TestName", "Pass"]
+
+
+def get_data(csv_path):
+    with open(csv_path, newline="") as csv_file:
+        report = list(csv.DictReader(csv_file))
+        header = report[0].keys()
+
+        assert set(relevant_headers).issubset(
+            header), "Lack of crucial headers in CSV report " + csv_path
+
+        tools = set(row["Tool"] for row in report)
+
+        sorted_report = {}
+        for tool in tools:
+            sorted_report[tool] = {}
+
+        for row in report:
+            sorted_report[row["Tool"]][row["TestName"]] = row["Pass"]
+
+    return sorted_report
+
+
+def check_tool(tool_reportA, tool_reportB, tool_name):
+    results = {
+        "new_passes": [],
+        "new_failures": [],
+        "added": [],
+        "removed": [],
+        "summary": {},
+    }
+
+    testsA = set(tool_reportA.keys())
+    testsB = set(tool_reportB.keys())
+
+    tests_added = testsA.difference(testsB)
+    tests_removed = testsB.difference(testsA)
+
+    tests_comparable = testsA.intersection(testsB)
+
+    added_cnt = len(tests_added)
+    removed_cnt = len(tests_removed)
+    no_change_cnt = 0
+
+    for test in tests_comparable:
+        res = check_test(tool_reportA[test], tool_reportB[test])
+        if (res == -1):
+            results["new_failures"].append(test)
+        elif (res == 1):
+            results["new_passes"].append(test)
+        else:
+            no_change_cnt += 1
+
+    fail_cnt = len(results["new_failures"])
+    pass_cnt = len(results["new_passes"])
+
+    for added_test in tests_added:
+        results["added"].append(added_test)
+
+    for removed_test in tests_removed:
+        results["removed"].append(removed_test)
+
+    results["summary"]["new_failures"] = fail_cnt
+    results["summary"]["new_passes"] = pass_cnt
+    results["summary"]["added"] = added_cnt
+    results["summary"]["removed"] = removed_cnt
+    results["summary"]["not_affected"] = no_change_cnt
+
+    return results
+
+
+def check_test(test_reportA, test_reportB):
+    if (test_reportA == test_reportB):
+        return 0
+    elif (test_reportA == "True" and test_reportB == "False"):
+        return 1
+    elif (test_reportA == "False" and test_reportB == "True"):
+        return -1
+    else:
+        raise ValueError(
+            "unknown test result occured: A -> " + test_reportA + " B -> " +
+            test_reportB)
+
+
+def check_reports(reportA, reportB):
+    summary = {
+        "comparable_tools": {},
+        "added_tools": [],
+        "removed_tools": [],
+    }
+
+    toolsA = set(reportA.keys())
+    toolsB = set(reportB.keys())
+    tools = toolsA.intersection(toolsB)
+
+    if (toolsA != toolsB):
+        tools_added = toolsA.difference(toolsB)
+        tools_removed = toolsB.difference(toolsA)
+        summary["added_tools"] = list(tools_added)
+        summary["removed_tools"] = list(tools_removed)
+
+    for tool in tools:
+        tool_results = check_tool(reportA[tool], reportB[tool], tool)
+        summary["comparable_tools"][tool] = tool_results
+
+    return summary
+
+
+def prepare_comment(summary, table_path):
+    tools = list(summary["comparable_tools"].keys())
+    cols = list(summary["comparable_tools"][tools[0]]["summary"].keys())
+    cols.insert(0, "tool")
+
+    matrix = []
+    for tool in tools:
+        vals = list(summary["comparable_tools"][tool]["summary"].values())
+        vals.insert(0, tool)
+        matrix.append(vals)
+
+    writer = pytablewriter.MarkdownTableWriter()
+    writer.table_name = "Compared test results"
+    writer.header_list = cols
+    writer.value_matrix = matrix
+    writer.write_table()
+    with open(table_path, "w") as f:
+        writer.stream = f
+        writer.write_table()
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "report_compare",
+        help="csv report that will be compared with base report")
+    parser.add_argument(
+        "report_base", help="csv report that will be a base of comparison")
+    parser.add_argument(
+        "-o",
+        "--output",
+        dest="output_path",
+        default="report_summary.json",
+        help="path to output json file, defaults to \"report_summary.json\"")
+    parser.add_argument(
+        "-t",
+        "--table",
+        dest="table_path",
+        default="report_summary.md",
+        help=
+        "path to output md file with summary, defaults to \"report_summary.md\""
+    )
+    args = parser.parse_args()
+
+    reportA = get_data(args.report_compare)
+    reportB = get_data(args.report_base)
+
+    summary = check_reports(reportA, reportB)
+
+    prepare_comment(summary, args.table_path)
+
+    with open(args.output_path, "w") as json_file:
+        json.dump(summary, json_file, indent=4)
+
+
+if __name__ == "__main__":
+    main()