Make vulnerabilities scanner less strict

Author: dnestoro

.github/workflows/new-library-versions-dispatcher.yml

@@ -0,0 +1,66 @@
+name: Orchestrate Batches
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  actions: write
+
+jobs:
+  calculate-matrix:
+    name: "📋 Get list of all supported libraries with newer versions"
+    permissions: write-all
+    runs-on: "ubuntu-22.04"
+    timeout-minutes: 5
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - name: "☁️ Checkout repository"
+        uses: actions/checkout@v4
+      - name: "🔧 Prepare environment"
+        uses: graalvm/setup-graalvm@v1
+        with:
+          java-version: '21'
+          distribution: 'graalvm'
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: "🕸️ Populate matrix"
+        id: set-matrix
+        run: |
+          ./gradlew fetchExistingLibrariesWithNewerVersions --matrixLimit=200
+
+  orchestrate:
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    needs: get-all-libraries
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.calculate-matrix.outputs.matrix) }}
+    steps:
+      - name: Calculate and Dispatch Batches
+        run: |
+          
+          TOTAL=2000
+          BATCH_SIZE=250
+          for START in $(seq 0 $BATCH_SIZE $((TOTAL - 1))); do
+            END=$((START + BATCH_SIZE - 1))
+            if (( END >= TOTAL )); then END=$((TOTAL - 1)); fi
+
+            echo "Dispatching batch: $START to $END"
+
+            gh workflow run run-batch.yml \
+              --ref main \
+              -f batch_start=$START \
+              -f batch_end=$END
+
+            # Optional: sleep if you want to throttle how quickly batches start
+            # sleep 10
+          done
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        # Install the GitHub CLI (gh)
+      - name: Install GitHub CLI
+        run: |
+          sudo apt-get update
+          sudo apt-get install gh

tests/tck-build-logic/src/main/java/org/graalvm/internal/tck/DockerUtils.java

@@ -1,7 +1,6 @@
 package org.graalvm.internal.tck;
 
 import java.io.BufferedReader;
-import java.io.File;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.net.MalformedURLException;
@@ -10,10 +9,9 @@
 import java.net.URL;
 import java.nio.file.*;
 import java.util.*;
-import java.util.stream.Collectors;
 
 public class DockerUtils {
-    private static final String ALLOWED_DOCKER_IMAGES = "/allowed-docker-images";
+    public static final String ALLOWED_DOCKER_IMAGES = "/allowed-docker-images";
 
     private static URL getDockerfileDirectory() {
         URL url = DockerUtils.class.getResource(ALLOWED_DOCKER_IMAGES);
@@ -47,7 +45,7 @@ private static String imageNameFromFile(URL dockerFile) throws IOException, URIS
         return images.get(0);
     }
 
-    private static String fileNameFromJar(URL jarFile) {
+    public static String fileNameFromJar(URL jarFile) {
         return jarFile.toString().split("!")[1];
     }
 
@@ -78,4 +76,8 @@ public static Set<String> getAllAllowedImages() {
         }
     }
 
+    public static String getImageName(String imageWithVersion) {
+        return imageWithVersion.split(":")[0];
+    }
+
 }

tests/tck-build-logic/src/main/java/org/graalvm/internal/tck/GrypeTask.java

@@ -10,10 +10,13 @@
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.*;
+import java.util.stream.Collectors;
 
-import static org.graalvm.internal.tck.DockerUtils.extractImagesNames;
-import static org.graalvm.internal.tck.DockerUtils.getAllAllowedImages;
 
 public abstract class GrypeTask extends DefaultTask {
 
@@ -33,88 +36,190 @@ void setNewCommit(String newCommit) {
     private String newCommit;
     private String baseCommit;
 
-    private final String jqMatcher = " | jq -c '.matches | .[] | .vulnerability | select(.severity | (contains(\"High\") or contains(\"Critical\")))'";
+    private static final String JQ_MATCHER = " | jq -c '.matches | .[] | .vulnerability | select(.severity | (contains(\"High\") or contains(\"Critical\")))'";
+    private static final String DOCKERFILE_DIRECTORY = "allowed-docker-images";
 
-    private List<URL> getChangedImages(String base, String head){
-        ByteArrayOutputStream baos = new ByteArrayOutputStream();
-        getExecOperations().exec(spec -> {
-            spec.setStandardOutput(baos);
-            spec.commandLine("git", "diff", "--name-only", "--diff-filter=ACMRT", base, head);
-        });
+    private record Vulnerabilities(int critical, int high){}
 
-        String output = baos.toString(StandardCharsets.UTF_8);
-        String dockerfileDirectory = "allowed-docker-images";
-        List<URL> diffFiles = Arrays.stream(output.split("\\r?\\n"))
-                .filter(path -> path.contains(dockerfileDirectory))
-                .map(path -> path.substring(path.lastIndexOf("/") + 1))
-                .map(DockerUtils::getDockerFile)
-                .toList();
+    private record DockerImage(String image, Vulnerabilities vulnerabilities) {
+        public String getImageName() {
+            return DockerUtils.getImageName(image);
+        }
 
-        if (diffFiles.isEmpty()) {
-            throw new RuntimeException("There are no changed or new docker image founded. " +
-                    "This task should be executed only if there are changes in allowed-docker-images directory.");
+        public boolean isVulnerableImage() {
+            return vulnerabilities.critical() > 0 || vulnerabilities.high() > 0;
+        }
+
+        public boolean isLessVulnerable(DockerImage other) {
+            // first check number of critical vulnerabilities
+            if (this.vulnerabilities.critical() < other.vulnerabilities().critical()) {
+                return true;
+            }
+
+            // if number of critical vulnerabilities is the same => check number of high vulnerabilities
+            return this.vulnerabilities.critical() == other.vulnerabilities().critical() && this.vulnerabilities.high() < other.vulnerabilities().high();
         }
 
-        return diffFiles;
+        public void printVulnerabilityStatus() {
+            System.out.println("Image: " + image + " contains " + vulnerabilities.critical() + " critical and " + vulnerabilities.high() + " high vulnerabilities");
+        }
     }
 
     @TaskAction
     void run() throws IllegalStateException, IOException, URISyntaxException {
-        List<String> vulnerableImages = new ArrayList<>();
-        Set<String> allowedImages;
-        if (baseCommit == null && newCommit == null) {
-            allowedImages = getAllAllowedImages();
+        boolean scanAllAllowedImages = baseCommit == null && newCommit == null;
+        if (scanAllAllowedImages) {
+            scanAllImages();
         } else {
-            allowedImages = extractImagesNames(getChangedImages(baseCommit, newCommit));
+            scanChangedImages();
+        }
+    }
+
+    /**
+     * Re-scans all images from allowed images list
+     */
+    private void scanAllImages() {
+        Set<DockerImage> imagesToCheck = DockerUtils.getAllAllowedImages().stream().map(this::makeDockerImage).collect(Collectors.toSet());
+        List<DockerImage> vulnerableImages = imagesToCheck.stream().filter(DockerImage::isVulnerableImage).toList();
+
+        if (!vulnerableImages.isEmpty()) {
+            vulnerableImages.forEach(DockerImage::printVulnerabilityStatus);
+            throw new IllegalStateException("Highly vulnerable images found. Please check the list of vulnerable images provided above.");
         }
+    }
 
-        boolean shouldFail = false;
-        for (String image : allowedImages) {
-            System.out.println("Checking image: " + image);
-            String[] command = { "-c",  "grype -o json " + image + jqMatcher };
-
-            ByteArrayOutputStream execOutput = new ByteArrayOutputStream();
-            getExecOperations().exec(execSpec -> {
-                execSpec.setExecutable("/bin/sh");
-                execSpec.setArgs(List.of(command));
-                execSpec.setStandardOutput(execOutput);
-            });
-
-            ByteArrayInputStream inputStream = new ByteArrayInputStream(execOutput.toByteArray());
-            try (BufferedReader stdOut = new BufferedReader(new InputStreamReader(inputStream))) {
-                int numberOfHigh = 0;
-                int numberOfCritical = 0;
-                String line;
-                while ((line = stdOut.readLine()) != null) {
-                    if (line.contains("\"severity\":\"High\"")) {
-                        numberOfHigh++;
-                    }else if (line.contains("\"severity\":\"Critical\"")) {
-                        numberOfCritical++;
+    /**
+     * Scans images that have been changed between org.graalvm.internal.tck.GrypeTask#baseCommit and org.graalvm.internal.tck.GrypeTask#newCommit.
+     * If changed images are less vulnerable than previously allowed images, they won't be reported as vulnerable
+     */
+    private void scanChangedImages() throws IOException, URISyntaxException {
+        Set<DockerImage> imagesToCheck = getChangedImages().stream().map(this::makeDockerImage).collect(Collectors.toSet());
+        List<DockerImage> vulnerableImages = imagesToCheck.stream().filter(DockerImage::isVulnerableImage).toList();
+
+        if (!vulnerableImages.isEmpty()) {
+            int acceptedImages = 0;
+            Set<String> currentlyAllowedImages = getAllowedImagesFromMaster();
+
+            for (DockerImage image : vulnerableImages) {
+                image.printVulnerabilityStatus();
+
+                // get allowed image with the same name, if it exists
+                Optional<String> existingAllowedImage = currentlyAllowedImages.stream()
+                        .filter(allowedImage -> DockerUtils.getImageName(allowedImage).equalsIgnoreCase(image.getImageName()))
+                        .findFirst();
+
+                // check if a new image is less vulnerable than the existing one
+                if (existingAllowedImage.isPresent()) {
+                    DockerImage imageToCompare = makeDockerImage(existingAllowedImage.get());
+                    imageToCompare.printVulnerabilityStatus();
+
+                    if (image.isLessVulnerable(imageToCompare)) {
+                        System.out.println("Accepting: " + image.image() + " because it has less vulnerabilities than existing: " + imageToCompare.image());
+                        acceptedImages++;
                     }
                 }
+            }
 
-                if (numberOfHigh > 0 || numberOfCritical > 0) {
-                    vulnerableImages.add("Image: " + image + " contains " + numberOfCritical + " critical, and " + numberOfHigh + " high vulnerabilities");
-                }
+            if (acceptedImages < vulnerableImages.size()) {
+                throw new IllegalStateException("Highly vulnerable images found. Please check the list of vulnerable images provided above.");
+            }
+        }
+    }
+
+    private DockerImage makeDockerImage(String image) {
+        System.out.println("Generating info for docker image: " + image);
+        return new DockerImage(image, getVulnerabilities(image));
+    }
+
+    private Vulnerabilities getVulnerabilities(String image) {
+        int numberOfHigh = 0;
+        int numberOfCritical = 0;
+        String[] command = {"-c", "grype -o json " + image + JQ_MATCHER};
+
+        // call Grype to get vulnerabilities
+        ByteArrayOutputStream execOutput = new ByteArrayOutputStream();
+        getExecOperations().exec(execSpec -> {
+            execSpec.setExecutable("/bin/sh");
+            execSpec.setArgs(List.of(command));
+            execSpec.setStandardOutput(execOutput);
+        });
 
-                if (numberOfHigh > 4 || numberOfCritical > 0) {
-                    shouldFail = true;
+        // parse Grype output
+        ByteArrayInputStream inputStream = new ByteArrayInputStream(execOutput.toByteArray());
+        try (BufferedReader stdOut = new BufferedReader(new InputStreamReader(inputStream))) {
+            String line;
+            while ((line = stdOut.readLine()) != null) {
+                if (line.contains("\"severity\":\"High\"")) {
+                    numberOfHigh++;
+                } else if (line.contains("\"severity\":\"Critical\"")) {
+                    numberOfCritical++;
                 }
             }
 
             inputStream.close();
             execOutput.close();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
         }
 
-        if (!vulnerableImages.isEmpty()) {
-            System.err.println("Vulnerable images found:");
-            System.err.println("===========================================================");
-            vulnerableImages.forEach(System.err::println);
-        }
+        return new Vulnerabilities(numberOfCritical, numberOfHigh);
+    }
 
-        if (shouldFail) {
-            throw new IllegalStateException("Highly vulnerable images found. Please check the list of vulnerable images provided above.");
+    /**
+     * Get all docker images introduced between two commits
+     */
+    private Set<String> getChangedImages() throws IOException, URISyntaxException {
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        getExecOperations().exec(spec -> {
+            spec.setStandardOutput(baos);
+            spec.commandLine("git", "diff", "--name-only", "--diff-filter=ACMRT", baseCommit, newCommit);
+        });
+
+        String output = baos.toString(StandardCharsets.UTF_8);
+        List<URL> diffFiles = Arrays.stream(output.split("\\r?\\n"))
+                .filter(path -> path.contains(DOCKERFILE_DIRECTORY))
+                .map(path -> path.substring(path.lastIndexOf("/") + 1))
+                .map(DockerUtils::getDockerFile)
+                .toList();
+
+        if (diffFiles.isEmpty()) {
+            throw new RuntimeException("There are no changed or new docker image founded. " +
+                    "This task should be executed only if there are changes in allowed-docker-images directory.");
         }
+
+        return DockerUtils.extractImagesNames(diffFiles);
     }
 
+    /**
+     * Return all allowed docker images from master branch
+     */
+    private Set<String> getAllowedImagesFromMaster() throws URISyntaxException, IOException {
+        URL url = GrypeTask.class.getResource(DockerUtils.ALLOWED_DOCKER_IMAGES);
+        if (url == null) {
+            throw new RuntimeException("Cannot find allowed-docker-images directory");
+        }
+
+        Set<String> allowedImages = new HashSet<>();
+        try (FileSystem fs = FileSystems.newFileSystem(url.toURI(), Collections.emptyMap())) {
+            List<String> files = Files.walk(fs.getPath(DockerUtils.ALLOWED_DOCKER_IMAGES))
+                    .filter(Files::isRegularFile)
+                    .map(Path::toString)
+                    .map(path -> path.substring(path.lastIndexOf("/") + 1))
+                    .map(DockerUtils::getDockerFile)
+                    .map(DockerUtils::fileNameFromJar)
+                    .toList();
+
+            for (String file : files) {
+                ByteArrayOutputStream baos = new ByteArrayOutputStream();
+                getExecOperations().exec(spec -> {
+                    spec.setStandardOutput(baos);
+                    spec.commandLine("git", "show", "master:tests/tck-build-logic/src/main/resources" + file);
+                });
+
+                allowedImages.add(baos.toString());
+            }
+        }
+
+        return allowedImages.stream().map(line -> line.substring("FROM".length()).trim()).collect(Collectors.toSet());
+    }
 }