Try to produce SARIF from linux clang

This is just horrible.

Author: LebedevRI

.ci/clang-sarif-merge.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+
+import argparse
+import functools
+import glob
+import json
+import os
+
+
+def create_parser():
+    parser = argparse.ArgumentParser(description="merge many clang sarif's")
+
+    parser.add_argument(
+        "-o",
+        "--output",
+        dest="output",
+        required=True,
+        help="the merged SARIF file",
+    )
+
+    parser.add_argument(
+        "dir",
+        help="Directory containing individual sarif's",
+    )
+
+    return parser
+
+
+def merge_two_sarifs(x, y):
+    assert x.keys() == y.keys()
+    res = dict()
+    for key in x.keys():
+        a = x[key]
+        b = y[key]
+        assert type(a) == type(b)
+        if isinstance(a, str):
+            assert a == b
+            res[key] = a
+        elif isinstance(a, list):
+            res[key] = a + b
+        elif isinstance(a, dict):
+            res[key] = merge_two_sarifs(a, b)
+        else:
+            assert False
+    return res
+
+
+def main():
+    # Parse the command line flags
+    parser = create_parser()
+    args, unknown_args = parser.parse_known_args()
+    assert not unknown_args
+    files = glob.glob(os.path.join(args.dir, "*.json"), recursive=True)
+    datas = []
+    for f in files:
+        with open(f, "r") as f:
+            datas.append(json.load(f))
+    result = dict()
+    if len(datas) > 0:
+        result = functools.reduce(merge_two_sarifs, datas)
+    if "runs" in result:
+        result["runs"] = [functools.reduce(merge_two_sarifs, result["runs"])]
+    with open(args.output, "w") as f:
+        json.dump(result, f)
+
+
+if __name__ == "__main__":
+    main()

.ci/clang-sarif-wrapper.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+# set -x
+set +e
+
+SARIFDIR="$1"
+mkdir -p "$SARIFDIR"
+
+shift
+
+INVOCATIONHASH=$(echo "$*" | sha1sum --text - | cut -d' ' -f 1)
+
+LOGNAME="$SARIFDIR/$INVOCATIONHASH.json"
+
+$@ -fdiagnostics-format=sarif -Wno-sarif-format-unstable 2>"$LOGNAME"
+RES=$?
+
+LC=$(wc -l "$LOGNAME" | cut -d' ' -f 1)
+if [ $LC -eq 3 ]; then
+   # Got no warnings.
+   exit $RES
+elif [ $LC -eq 4 ]; then
+   touch "$SARIFDIR/.warnings"
+   OUT="$(cat "$LOGNAME")"
+   # Drop last line, which is: "$N (warning|error)[s] generated."
+   echo "$OUT" | head -n 2 | tail -n 1 > "$LOGNAME"
+   exit $RES
+else
+   /bin/false # ???
+fi

.github/workflows/CI-linux.yml

@@ -46,13 +46,15 @@ jobs:
   linux:
     runs-on: ubuntu-latest
     env:
+      JOB_NAME: ${{ runner.os }}.${{ inputs.distro }}.${{ inputs.compiler-family }}${{ inputs.compiler-version }}.${{ inputs.flavor }}
       SONAR_SCANNER_VERSION: 5.0.1.3006
-    name: ${{ inputs.distro }}.${{ inputs.compiler-family }}${{ inputs.compiler-version }}.${{ inputs.flavor }}
+    name: ${{ env.JOB_NAME }}
     container:
       image: ${{ inputs.distro }}
       options: --tmpfs /tmp:exec --tmpfs /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}:exec
     steps:
       - name: Configure APT
+        env:
         timeout-minutes: 1
         run: |
           set -xe
@@ -137,7 +139,8 @@ jobs:
           fi
           if [ "$COMPILER_FAMILY" = "LLVM" ]; then
             eatmydata apt install clang-${{ inputs.compiler-version }} \
-                                      libomp-${{ inputs.compiler-version }}-dev
+                                  libomp-${{ inputs.compiler-version }}-dev \
+                                  python3
           fi
           if [ "$ENABLE_SAMPLE_BASED_TESTING" = "true" ]; then
             eatmydata apt install zstd
@@ -228,6 +231,7 @@ jobs:
         uses: github/codeql-action/init@v3
         with:
           languages: cpp
+          source-root: ${{ github.workspace }}/rawspeed
       - name: Set up JDK 11 (for SonarCloud static analysis)
         timeout-minutes: 1
         if: inputs.flavor == 'SonarCloudStaticAnalysis' && github.repository == 'darktable-org/rawspeed' && github.event_name != 'pull_request' && github.ref_type == 'branch' && (github.ref_name == 'develop' || github.ref_name == 'stable')
@@ -264,6 +268,7 @@ jobs:
         env:
           CC: ${{ inputs.compiler-CC }}
           CXX: ${{ inputs.compiler-CXX }}
+          FAMILY: ${{ inputs.compiler-family }}
           CLANG_TIDY: ${{ inputs.compiler-CLANG_TIDY }}
           GCOV: ${{ inputs.compiler-GCOV }}
           SRC_DIR: ${{ github.workspace }}/rawspeed
@@ -279,14 +284,16 @@ jobs:
           cmake -E make_directory "${INSTALL_PREFIX}"
           export ECO="${ECO} -DRAWSPEED_REFERENCE_SAMPLE_ARCHIVE=${RPUU_DST}"
           export ECO="${ECO} -DCMAKE_CXX_CLANG_TIDY_EXPORT_FIXES_DIR=${BUILD_DIR}/clang-tidy/"
-          if [ "$FLAVOR" = "ClangTidy" ] || [ "$FLAVOR" = "ClangStaticAnalysis" ] || [ "$FLAVOR" = "ClangCTUStaticAnalysis" ] || [ "$FLAVOR" = "CodeQLAnalysis" ]; then
+          if [ "$FAMILY" = "LLVM" ]; then
             export ECO="${ECO} -DRAWSPEED_ENABLE_WERROR=OFF"
+            export ECO="${ECO} -DCMAKE_CXX_COMPILER_LAUNCHER='${SRC_DIR}/.ci/clang-sarif-wrapper.sh;${GITHUB_WORKSPACE}/clang_report/'"
           fi
           "${SRC_DIR}/.ci/ci-script.sh"
       - name: Build
         id: build
         timeout-minutes: ${{ inputs.flavor != 'ClangTidy' && (inputs.flavor != 'CodeQLAnalysis' && 7 || 12) || 25 }}
         env:
+          FAMILY: ${{ inputs.compiler-family }}
           SRC_DIR: ${{ github.workspace }}/rawspeed
           BUILD_DIR: ${{ github.workspace }}/rawspeed-build
           INSTALL_PREFIX: ${{ github.workspace }}/rawspeed-install
@@ -295,6 +302,29 @@ jobs:
         run: |
           set -xe
           "${SRC_DIR}/.ci/ci-script.sh"
+          if [ "$FLAVOR" = "ClangTidy" ] || [ "$FLAVOR" = "ClangStaticAnalysis" ] || [ "$FLAVOR" = "ClangCTUStaticAnalysis" ] || [ "$FLAVOR" = "CodeQLAnalysis" ]; then
+            exit 0
+          else
+            MARKER="${GITHUB_WORKSPACE}/clang_report/.warnings"
+            [ -f "$MARKER" ] && exit 1 || exit 0
+          fi
+      - name: Merge clang SARIF reports
+        if: inputs.compiler-family == 'LLVM' && !cancelled() && steps.build.conclusion != 'skipped'
+        timeout-minutes: 1
+        env:
+          SRC_DIR: ${{ github.workspace }}/rawspeed
+          JOB_NAME: ${{ env.JOB_NAME }}
+        run: |
+          set -xe
+          "${SRC_DIR}/.ci/clang-sarif-merge.py" --output="${GITHUB_WORKSPACE}/clang_report.json" "${GITHUB_WORKSPACE}/clang_report/"
+      - name: Upload results of clang compile [SARIF]
+        timeout-minutes: 1
+        if: inputs.compiler-family == 'LLVM' && !cancelled() && steps.build.conclusion != 'skipped'
+        uses: github/codeql-action/upload-sarif@v3
+        with:
+          sarif_file: "${{ github.workspace }}/clang_report.json"
+          checkout_path: "${{ github.workspace }}/rawspeed"
+          category: ${{ job.name }}
       - name: Test (unit tests)
         timeout-minutes: 1
         env: