diff --git a/.github/kokoro/run.sh b/.github/kokoro/run.sh index 13d7039276767..ab47097de2f18 100755 --- a/.github/kokoro/run.sh +++ b/.github/kokoro/run.sh @@ -78,6 +78,25 @@ echo "----------------------------------------" ) echo "----------------------------------------" +echo +echo "========================================" +echo "Running Analysis" +echo "----------------------------------------" +{ + ANALYZER=$GIT_CHECKOUT"/tools/report_analyzer.py" + OUT_DIR=$GIT_CHECKOUT"/out/report/" + COMPARE_REPORT=$OUT_DIR"/report.csv" + BASE_REPORT=$OUT_DIR"/base_report.csv" + CHANGES_SUMMARY_JSON=$OUT_DIR"/tests_summary.json" + CHANGES_SUMMARY_MD=$OUT_DIR"/tests_summary.md" + + # Get base report from sv-tests master run + wget https://symbiflow.github.io/sv-tests-results/report.csv -O $BASE_REPORT + + python $ANALYZER $COMPARE_REPORT $BASE_REPORT -o $CHANGES_SUMMARY_JSON -t $CHANGES_SUMMARY_MD +} +echo "----------------------------------------" + if [[ $KOKORO_TYPE = continuous ]]; then # - "make report USE_ALL_RUNNERS=1" # - "touch out/report/.nojekyll" diff --git a/.github/workflows/summary.sh b/.github/workflows/summary.sh new file mode 100755 index 0000000000000..478d793223470 --- /dev/null +++ b/.github/workflows/summary.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +ANALYZER=$PWD"/tools/report_analyzer.py" +OUT_DIR=$PWD"/out/report/" +COMPARE_REPORT=$OUT_DIR"/tests_report.csv" +BASE_REPORT=$OUT_DIR"/base_report.csv" +CHANGES_SUMMARY_JSON=$OUT_DIR"/tests_summary.json" +CHANGES_SUMMARY_MD=$OUT_DIR"/tests_summary.md" + +set -x +set -e + +# Get a conda environment +wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh +bash miniconda.sh -b -p $HOME/miniconda +source "$HOME/miniconda/etc/profile.d/conda.sh" +hash -r +conda config --set always_yes yes --set changeps1 no + +conda env create --file conf/environment.yml +conda activate sv-test-env +hash -r +conda info -a + +# Get base report from sv-tests master run +wget https://symbiflow.github.io/sv-tests-results/report.csv -O $BASE_REPORT + +# Delete headers from all report.csv +for file in $(find ./out/report_* -name "*.csv" -print); do + sed -i.backup 1,1d $file +done + +# concatenate test reports +cat $(find ./out/report_* -name "*.csv" -print) >> $COMPARE_REPORT + +# Insert header at the first line of concatenated report +sed -i 1i\ $(cat $(find ./out/report_* -name "*.csv.backup" -print | head -1) | head -1) $COMPARE_REPORT + +python $ANALYZER $COMPARE_REPORT $BASE_REPORT -o $CHANGES_SUMMARY_JSON -t $CHANGES_SUMMARY_MD + +set +e +set +x + diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml index a46b6f592ccbb..a0b62306464b6 100644 --- a/.github/workflows/sv-tests-ci.yml +++ b/.github/workflows/sv-tests-ci.yml @@ -64,3 +64,78 @@ jobs: - name: Run run: ./.github/workflows/run.sh + - name: Prepare Report + run: + ./.github/workflows/tool_report_prepare.sh + - uses: actions/upload-artifact@v2 + with: + name: report_${{ matrix.env.JOB_NAME }} + path: | + ./out/report/${{ matrix.env.JOB_NAME }}_report.csv + + Summary: + name: Summary + runs-on: ubuntu-18.04 + needs: Run + steps: + - name: Checkout code + uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Prepare output directories + run: mkdir -p out/report + - uses: actions/download-artifact@v2 + with: + path: ./out/ + - name: Summary + run: + ./.github/workflows/summary.sh + - uses: actions/upload-artifact@v2 + with: + name: tests_summary + path: | + ./out/report/tests_summary.json + ./out/report/tests_summary.md + ./out/report/tests_report.csv + ./out/report/base_report.csv + - id: get-artifacts-to-delete + run: | + artifacts=$(find ./out -type d -name 'report_*' -exec basename {} \;) + echo $artifacts + artifacts="${artifacts//'%'/'%25'}" + artifacts="${artifacts//$'\n'/'%0A'}" + artifacts="${artifacts//$'\r'/'%0D'}" + echo ::set-output name=artifacts::$artifacts + echo $artifacts + - name: Delete Old Artifacts + uses: geekyeggo/delete-artifact@v1 + with: + name: ${{ steps.get-artifacts-to-delete.outputs.artifacts }} + + Comment: + if: github.event_name == 'pull_request' + name: Comment + runs-on: ubuntu-18.04 + needs: Summary + steps: + - name: Prepare output directories + run: mkdir -p out/report + - uses: actions/download-artifact@v2 + with: + name: tests_summary + path: ./out/report + - id: get-comment-body + run: | + body=$(cat ./out/report/tests_summary.md) + body="${body//'%'/'%25'}" + body="${body//$'\n'/'%0A'}" + body="${body//$'\r'/'%0D'}" + echo ::set-output name=body::$body + - name: Post comment + uses: KeisukeYamashita/create-comment@v1 + with: + check-only-first-line: "true" + unique: "true" + token: ${{ secrets.GITHUB_TOKEN }} + comment: ${{ steps.get-comment-body.outputs.body }} diff --git a/.github/workflows/tool_report_prepare.sh b/.github/workflows/tool_report_prepare.sh new file mode 100755 index 0000000000000..d799dd76eaba8 --- /dev/null +++ b/.github/workflows/tool_report_prepare.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +OUT_DIR=$PWD"/out/report/" +COMPARE_REPORT=$OUT_DIR"/report.csv" + +set -x +set -e + +source "$HOME/miniconda/etc/profile.d/conda.sh" +hash -r +conda activate sv-test-env +hash -r + +mv $COMPARE_REPORT $OUT_DIR"/"$JOB_NAME"_report.csv" + +set +e +set +x diff --git a/conf/requirements.txt b/conf/requirements.txt index 31e0ca3c2fdca..3e758b6c5b5e7 100644 --- a/conf/requirements.txt +++ b/conf/requirements.txt @@ -6,4 +6,5 @@ yapf==0.30.0 psutil mako make_var +pytablewriter git+https://github.com/lowRISC/fusesoc.git@ot diff --git a/tools/report_analyzer.py b/tools/report_analyzer.py new file mode 100755 index 0000000000000..df2bba7ab2bba --- /dev/null +++ b/tools/report_analyzer.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (C) 2021 The SymbiFlow Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + +import argparse +import csv +import json +import sys +import pytablewriter + +relevant_headers = ["Tool", "TestName", "Pass"] + + +def get_data(csv_path): + with open(csv_path, newline="") as csv_file: + report = list(csv.DictReader(csv_file)) + header = report[0].keys() + + assert set(relevant_headers).issubset( + header), "Lack of crucial headers in CSV report " + csv_path + + tools = set(row["Tool"] for row in report) + + sorted_report = {} + for tool in tools: + sorted_report[tool] = {} + + for row in report: + sorted_report[row["Tool"]][row["TestName"]] = row["Pass"] + + return sorted_report + + +def check_tool(tool_reportA, tool_reportB, tool_name): + results = { + "new_passes": [], + "new_failures": [], + "added": [], + "removed": [], + "summary": {}, + } + + testsA = set(tool_reportA.keys()) + testsB = set(tool_reportB.keys()) + + tests_added = testsA.difference(testsB) + tests_removed = testsB.difference(testsA) + + tests_comparable = testsA.intersection(testsB) + + added_cnt = len(tests_added) + removed_cnt = len(tests_removed) + no_change_cnt = 0 + + for test in tests_comparable: + res = check_test(tool_reportA[test], tool_reportB[test]) + if (res == -1): + results["new_failures"].append(test) + elif (res == 1): + results["new_passes"].append(test) + else: + no_change_cnt += 1 + + fail_cnt = len(results["new_failures"]) + pass_cnt = len(results["new_passes"]) + + for added_test in tests_added: + results["added"].append(added_test) + + for removed_test in tests_removed: + results["removed"].append(removed_test) + + results["summary"]["new_failures"] = fail_cnt + results["summary"]["new_passes"] = pass_cnt + results["summary"]["added"] = added_cnt + results["summary"]["removed"] = removed_cnt + results["summary"]["not_affected"] = no_change_cnt + + return results + + +def check_test(test_reportA, test_reportB): + if (test_reportA == test_reportB): + return 0 + elif (test_reportA == "True" and test_reportB == "False"): + return 1 + elif (test_reportA == "False" and test_reportB == "True"): + return -1 + else: + raise ValueError( + "unknown test result occured: A -> " + test_reportA + " B -> " + + test_reportB) + + +def check_reports(reportA, reportB): + summary = { + "comparable_tools": {}, + "added_tools": [], + "removed_tools": [], + } + + toolsA = set(reportA.keys()) + toolsB = set(reportB.keys()) + tools = toolsA.intersection(toolsB) + + if (toolsA != toolsB): + tools_added = toolsA.difference(toolsB) + tools_removed = toolsB.difference(toolsA) + summary["added_tools"] = list(tools_added) + summary["removed_tools"] = list(tools_removed) + + for tool in tools: + tool_results = check_tool(reportA[tool], reportB[tool], tool) + summary["comparable_tools"][tool] = tool_results + + return summary + + +def prepare_comment(summary, table_path): + tools = list(summary["comparable_tools"].keys()) + cols = list(summary["comparable_tools"][tools[0]]["summary"].keys()) + cols.insert(0, "tool") + + matrix = [] + for tool in tools: + vals = list(summary["comparable_tools"][tool]["summary"].values()) + vals.insert(0, tool) + matrix.append(vals) + + writer = pytablewriter.MarkdownTableWriter() + writer.table_name = "Compared test results" + writer.header_list = cols + writer.value_matrix = matrix + writer.write_table() + with open(table_path, "w") as f: + writer.stream = f + writer.write_table() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "report_compare", + help="csv report that will be compared with base report") + parser.add_argument( + "report_base", help="csv report that will be a base of comparison") + parser.add_argument( + "-o", + "--output", + dest="output_path", + default="report_summary.json", + help="path to output json file, defaults to \"report_summary.json\"") + parser.add_argument( + "-t", + "--table", + dest="table_path", + default="report_summary.md", + help= + "path to output md file with summary, defaults to \"report_summary.md\"" + ) + args = parser.parse_args() + + reportA = get_data(args.report_compare) + reportB = get_data(args.report_base) + + summary = check_reports(reportA, reportB) + + prepare_comment(summary, args.table_path) + + with open(args.output_path, "w") as json_file: + json.dump(summary, json_file, indent=4) + + +if __name__ == "__main__": + main()