diff --git a/.github/workflows/benches.yml b/.github/workflows/benches.yml new file mode 100644 index 000000000..dbbe9957a --- /dev/null +++ b/.github/workflows/benches.yml @@ -0,0 +1,284 @@ +name: Benchmark ML-KEM and ML-DSA + +on: + merge_group: + pull_request_target: + workflow_dispatch: + push: + branches: ["main"] + +jobs: + bench-mlkem: + strategy: + fail-fast: true + matrix: + bits: [32, 64] + os: + - macos-latest + - ubuntu-latest + - windows-latest + exclude: + # There's no such thing as 32-bit macOS + - bits: 32 + os: "macos-latest" + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash + working-directory: libcrux-ml-kem + + steps: + - uses: actions/checkout@v4 + with: + repository: 'cryspen/libcrux' + + # set up rust env + + - name: Update dependencies + run: cargo update + + - run: echo "RUST_TARGET_FLAG=" > $GITHUB_ENV + if: ${{ matrix.bits == 64 }} + + - name: 🛠️ Setup Ubuntu x86 + if: ${{ matrix.bits == 32 && matrix.os == 'ubuntu-latest' }} + run: | + rustup target add i686-unknown-linux-gnu + sudo apt-get update + sudo apt-get install -y gcc-multilib g++-multilib + + - name: 🛠️ Setup macOS + if: ${{ matrix.os == 'macos-latest' }} + run: | + rustup target add aarch64-apple-darwin + + # Set up 32 bit systems + + - name: 🛠️ Config Windows x86 + run: echo "RUST_TARGET_FLAG=--target=i686-pc-windows-msvc" > $GITHUB_ENV + if: ${{ matrix.bits == 32 && matrix.os == 'windows-latest' }} + + - name: 🛠️ Config Linux x86 + run: | + echo "RUST_TARGET_FLAG=--target=i686-unknown-linux-gnu" > $GITHUB_ENV + if: ${{ matrix.bits == 32 && matrix.os == 'ubuntu-latest' }} + + # run benchmarks + + - name: 🏃🏻‍♀️ Benchmarks + run: cargo bench --verbose $RUST_TARGET_FLAG -- --output-format bencher | tee bench.txt + + # extract benchmark json + + - name: Extract benchmarks + uses: cryspen/benchmark-data-extract-transform@v2 + with: + name: ML-KEM Benchmark + tool: 'cargo' + os: ${{ matrix.os }}_${{ matrix.bits }} + output-file-path: libcrux-ml-kem/bench.txt + data-out-path: benchdata-mlkem-${{matrix.os}}-${{matrix.bits}}.json + + - uses: actions/upload-artifact@v4 + with: + name: benchdata-mlkem-${{matrix.os}}-${{matrix.bits}}.json + path: benchdata-mlkem-${{matrix.os}}-${{matrix.bits}}.json + + bench-mldsa: + strategy: + fail-fast: true + matrix: + bits: [32, 64] + os: + - macos-latest + - ubuntu-latest + - windows-latest + exclude: + # There's no such thing as 32-bit macOS + - bits: 32 + os: "macos-latest" + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash + working-directory: libcrux-ml-dsa + + steps: + - uses: actions/checkout@v4 + with: + repository: 'cryspen/libcrux' + + # set up rust env + + - name: Update dependencies + run: cargo update + + - run: echo "RUST_TARGET_FLAG=" > $GITHUB_ENV + if: ${{ matrix.bits == 64 }} + + - name: 🛠️ Setup Ubuntu x86 + if: ${{ matrix.bits == 32 && matrix.os == 'ubuntu-latest' }} + run: | + rustup target add i686-unknown-linux-gnu + sudo apt-get update + sudo apt-get install -y gcc-multilib g++-multilib + + - name: 🛠️ Setup macOS + if: ${{ matrix.os == 'macos-latest' }} + run: | + rustup target add aarch64-apple-darwin + + # Set up 32 bit systems + + - name: 🛠️ Config Windows x86 + run: echo "RUST_TARGET_FLAG=--target=i686-pc-windows-msvc" > $GITHUB_ENV + if: ${{ matrix.bits == 32 && matrix.os == 'windows-latest' }} + + - name: 🛠️ Config Linux x86 + run: | + echo "RUST_TARGET_FLAG=--target=i686-unknown-linux-gnu" > $GITHUB_ENV + if: ${{ matrix.bits == 32 && matrix.os == 'ubuntu-latest' }} + + # run benchmarks + + - name: 🏃🏻‍♀️ Benchmarks + run: cargo bench --bench manual44 --bench manual65 --bench manual87 --verbose $RUST_TARGET_FLAG -- --output-format bencher | tee bench.txt + + # extract benchmark json + + - name: Extract benchmarks + uses: cryspen/benchmark-data-extract-transform@v2 + with: + name: ML-DSA Benchmark + tool: 'cargo' + os: ${{ matrix.os }}_${{ matrix.bits }} + output-file-path: libcrux-ml-dsa/bench.txt + data-out-path: benchdata-mldsa-${{matrix.os}}-${{matrix.bits}}.json + + - uses: actions/upload-artifact@v4 + with: + name: benchdata-mldsa-${{matrix.os}}-${{matrix.bits}}.json + path: benchdata-mldsa-${{matrix.os}}-${{matrix.bits}}.json + + merge: + permissions: + pull-requests: write + runs-on: ubuntu-latest + needs: [bench-mlkem, bench-mldsa] + steps: + - uses: actions/checkout@v4 + with: + repository: 'cryspen/libcrux' + + - run: mkdir benchdata + - uses: actions/download-artifact@v5 + with: + pattern: benchdata-*.json + path: benchdata/ + merge-multiple: true + + - name: Join per-run data + run: | + jq -s add benchdata/benchdata-mlkem-* > benchdata-mlkem.json + jq -s add benchdata/benchdata-mldsa-* > benchdata-mldsa.json + + # check if the PR branch is on a fork. if it is, post comments. + # otherwise, generate page and merge it. + - name: Set IS_FORK environment variable + run: echo "IS_FORK=${{ github.event.pull_request.head.repo.fork }}" >> $GITHUB_ENV + + # if not a fork, use page generation action + - name: Upload ML-DSA benchmarks + if: env.IS_FORK == 'false' + uses: cryspen/benchmark-upload-and-plot-action@v3 + with: + name: ML-DSA Benchmark + input-data-path: benchdata-mldsa.json + github-token: ${{ secrets.GITHUB_TOKEN }} + gh-repository: github.com/${{ github.repository }} + group-by: label,keySize,os + schema: implementation,keySize,label,hardware,os + auto-push: false + fail-on-alert: true + alert-threshold: 200% + + - name: TODO remove previous checkout + run: rm -rf benchmark-data-repository + + - name: Upload ML-KEM benchmarks + if: env.IS_FORK == 'false' + uses: cryspen/benchmark-upload-and-plot-action@v3 + with: + name: ML-KEM Benchmark + input-data-path: benchdata-mlkem.json + github-token: ${{ secrets.GITHUB_TOKEN }} + gh-repository: github.com/${{ github.repository }} + group-by: os,keySize + schema: category,keySize,name,platform,api,os + auto-push: false + fail-on-alert: true + alert-threshold: 200% + + # if a fork, use comment-based action + - uses: cryspen/benchmark-comment-and-plot-action@main + if: env.IS_FORK == 'true' + with: + id: mldsa + gh-repository: cryspen/libcrux + group-by: label,keySize,os + schema: implementation,keySize,label,hardware,os + input-data-path: benchdata-mldsa.json + name: ML-DSA Benchmark + output-md: report-mldsa.md + + - uses: cryspen/benchmark-comment-and-plot-action@main + if: env.IS_FORK == 'true' + with: + id: mlkem + gh-repository: cryspen/libcrux + group-by: os,keySize + schema: category,keySize,name,platform,api,os + input-data-path: benchdata-mlkem.json + name: ML-KEM Benchmark + output-md: report-mlkem.md + + - name: Build comment + if: env.IS_FORK == 'true' + id: build_comment + run: | + BASE_HASH="${{ github.event.pull_request.base.sha }}" + BASE_HASH_SHORT=$(echo "${{ github.event.pull_request.base.sha }}" | cut -c1-7) + + PR_HEAD_HASH="${{ github.event.pull_request.head.sha }}" + PR_HEAD_HASH_SHORT=$(echo "${{ github.event.pull_request.head.sha }}" | cut -c1-7) + + PR_COMMIT_URL="${{ github.server_url }}/${{ github.repository }}/commit/${PR_HEAD_HASH}" + BASE_COMMIT_URL="${{ github.server_url }}/${{ github.repository }}/commit/${BASE_HASH}" + + cat >"$GITHUB_OUTPUT" < $GITHUB_ENV - if: ${{ matrix.bits == 32 && matrix.os == 'windows-latest' }} - - - name: 🛠️ Config Linux x86 - run: | - echo "RUST_TARGET_FLAG=--target=i686-unknown-linux-gnu" > $GITHUB_ENV - if: ${{ matrix.bits == 32 && matrix.os == 'ubuntu-latest' }} - - # Benchmarks ... - - - name: 🏃🏻‍♀️ Benchmarks - run: cargo bench --bench manual44 --bench manual65 --bench manual87 --verbose $RUST_TARGET_FLAG -- --output-format bencher | tee bench.txt - - - name: Extract benchmarks - uses: cryspen/benchmark-data-extract-transform@v2 - with: - name: ML-DSA Benchmark - tool: 'cargo' - os: ${{ matrix.os }}_${{ matrix.bits }} - output-file-path: libcrux-ml-dsa/bench.txt - data-out-path: libcrux-ml-dsa/bench-processed.json - - - name: Upload benchmarks - uses: cryspen/benchmark-upload-and-plot-action@v3 - with: - name: ML-DSA Benchmark - input-data-path: libcrux-ml-dsa/bench-processed.json - github-token: ${{ secrets.GITHUB_TOKEN }} - gh-repository: github.com/${{ github.repository }} - group-by: label,keySize,os - schema: implementation,keySize,label,hardware,os - auto-push: true - fail-on-alert: true - alert-threshold: 200% - - mldsa-bench-status: - if: ${{ always() }} - needs: [benchmark] - runs-on: ubuntu-latest - steps: - - name: Successful - if: ${{ !(contains(needs.*.result, 'failure')) }} - run: exit 0 - - name: Failing - if: ${{ (contains(needs.*.result, 'failure')) }} - run: exit 1 diff --git a/.github/workflows/mlkem-bench.yml b/.github/workflows/mlkem-bench.yml deleted file mode 100644 index 1f3c038fc..000000000 --- a/.github/workflows/mlkem-bench.yml +++ /dev/null @@ -1,114 +0,0 @@ -name: ML-KEM - Benchmark - -on: - # run benchmarks and fail on regression - merge_group: - # run on PR but skip - pull_request: - branches: ["main", "dev"] - workflow_dispatch: - # run benchmarks once merged and push data - push: - branches: ["main"] -env: - CARGO_TERM_COLOR: always - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - benchmark: - if: ${{ github.event_name != 'pull_request' }} - strategy: - fail-fast: true - matrix: - bits: [32, 64] - os: - - macos-13 - - macos-latest - - ubuntu-latest - - windows-latest - exclude: - # There's no such thing as 32-bit macOS - - bits: 32 - os: "macos-latest" - - bits: 32 - os: "macos-13" - - runs-on: ${{ matrix.os }} - defaults: - run: - shell: bash - working-directory: libcrux-ml-kem - - steps: - - uses: actions/checkout@v5 - - - name: Update dependencies - run: cargo update - - - run: echo "RUST_TARGET_FLAG=" > $GITHUB_ENV - if: ${{ matrix.bits == 64 }} - - - name: 🛠️ Setup Ubuntu x86 - if: ${{ matrix.bits == 32 && matrix.os == 'ubuntu-latest' }} - run: | - rustup target add i686-unknown-linux-gnu - sudo apt-get update - sudo apt-get install -y gcc-multilib g++-multilib - - - name: 🛠️ Setup macOS - if: ${{ matrix.os == 'macos-latest' }} - run: | - rustup target add aarch64-apple-darwin - - # Set up 32 bit systems - - - name: 🛠️ Config Windows x86 - run: echo "RUST_TARGET_FLAG=--target=i686-pc-windows-msvc" > $GITHUB_ENV - if: ${{ matrix.bits == 32 && matrix.os == 'windows-latest' }} - - - name: 🛠️ Config Linux x86 - run: | - echo "RUST_TARGET_FLAG=--target=i686-unknown-linux-gnu" > $GITHUB_ENV - if: ${{ matrix.bits == 32 && matrix.os == 'ubuntu-latest' }} - - # Benchmarks ... - - - name: 🏃🏻‍♀️ Benchmarks - run: cargo bench --verbose $RUST_TARGET_FLAG -- --output-format bencher | tee bench.txt - - - name: Extract benchmarks - uses: cryspen/benchmark-data-extract-transform@v2 - with: - name: ML-KEM Benchmark - tool: 'cargo' - os: ${{ matrix.os }}_${{ matrix.bits }} - output-file-path: libcrux-ml-kem/bench.txt - data-out-path: libcrux-ml-kem/bench-processed.json - - - name: Upload benchmarks - uses: cryspen/benchmark-upload-and-plot-action@v3 - with: - name: ML-KEM Benchmark - input-data-path: libcrux-ml-kem/bench-processed.json - github-token: ${{ secrets.GITHUB_TOKEN }} - gh-repository: github.com/${{ github.repository }} - group-by: os,keySize - schema: category,keySize,name,platform,api,os - auto-push: true - fail-on-alert: true - alert-threshold: 200% - - mlkem-bench-status: - if: ${{ always() }} - needs: [benchmark] - runs-on: ubuntu-latest - steps: - - name: Successful - if: ${{ !(contains(needs.*.result, 'failure')) }} - run: exit 0 - - name: Failing - if: ${{ (contains(needs.*.result, 'failure')) }} - run: exit 1