From 9d8ee74558550c65e1a80802329cb1f09218454f Mon Sep 17 00:00:00 2001 From: Connor Tsui Date: Wed, 18 Dec 2024 09:56:02 -0500 Subject: [PATCH 1/6] add tpch benchmark workflow --- .github/workflows/tpch.yaml | 49 +++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/tpch.yaml diff --git a/.github/workflows/tpch.yaml b/.github/workflows/tpch.yaml new file mode 100644 index 00000000..a5771f70 --- /dev/null +++ b/.github/workflows/tpch.yaml @@ -0,0 +1,49 @@ +name: Rust TPC-H Benchmarks + +# Do not run this workflow on pull request since this workflow has permission to modify contents. +on: + push: + branches: + - main + # Hack for testing (REMOVE) + - continuous-bench-attempt + +permissions: + # deployments permission to deploy GitHub pages website + deployments: write + # contents permission to update benchmark contents in gh-pages branch + contents: write + +jobs: + benchmark: + name: Run Rust benchmark example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + # Setup Rust. + - run: rustup toolchain install stable --profile minimal --no-self-update + + # Make sure to use cached compilation objects if they exist. + - uses: Swatinem/rust-cache@v2 + + # Run the benchmark and store the result in `output.txt`. + - name: Run benchmark + run: cargo bench --bench planner_bench tpch/ -- --output-format bencher | tee output.txt + + # Run the `github-action-benchmark` action. + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Rust Benchmark + tool: "cargo" + output-file-path: output.txt + gh-pages-branch: gh-pages + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: "125%" + summary-always: true + comment-on-alert: true + fail-on-alert: true + alert-comment-cc-users: "@connortsui20" From a4af10fdcd6ed07f2663fa285fcb94acd5aa4936 Mon Sep 17 00:00:00 2001 From: Connor Tsui Date: Wed, 18 Dec 2024 12:00:53 -0500 Subject: [PATCH 2/6] remove debug println --- optd-core/src/cascades/optimizer.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/optd-core/src/cascades/optimizer.rs b/optd-core/src/cascades/optimizer.rs index d1b66607..e971343b 100644 --- a/optd-core/src/cascades/optimizer.rs +++ b/optd-core/src/cascades/optimizer.rs @@ -261,10 +261,10 @@ impl> CascadesOptimizer { let plan_space = self.memo.estimated_plan_space(); if let Some(partial_explore_space) = self.prop.partial_explore_space { if plan_space - plan_space_begin > partial_explore_space { - println!( - "plan space size budget used, not applying logical rules any more. current plan space: {}", - plan_space - ); + // println!( + // "plan space size budget used, not applying logical rules any more. current plan space: {}", + // plan_space + // ); self.ctx.budget_used = true; if self.prop.panic_on_budget { panic!("plan space size budget used"); From 2b2963dff8f3125ca78bebac5c404f9d6f3e4406 Mon Sep 17 00:00:00 2001 From: Connor Tsui Date: Wed, 18 Dec 2024 13:05:48 -0500 Subject: [PATCH 3/6] cleanup --- .github/workflows/tpch.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tpch.yaml b/.github/workflows/tpch.yaml index a5771f70..c967ee7b 100644 --- a/.github/workflows/tpch.yaml +++ b/.github/workflows/tpch.yaml @@ -5,8 +5,6 @@ on: push: branches: - main - # Hack for testing (REMOVE) - - continuous-bench-attempt permissions: # deployments permission to deploy GitHub pages website @@ -40,8 +38,9 @@ jobs: output-file-path: output.txt gh-pages-branch: gh-pages github-token: ${{ secrets.GITHUB_TOKEN }} + # Automatically push the results to the GitHub pages branch. auto-push: true - # Show alert with commit comment on detecting possible performance regression + # Show alert with commit comment on detecting possible performance regression. alert-threshold: "125%" summary-always: true comment-on-alert: true From 3d87f81fa86f683a3dbcd42661fbb3198695f5a3 Mon Sep 17 00:00:00 2001 From: Connor Tsui Date: Wed, 18 Dec 2024 18:01:58 -0500 Subject: [PATCH 4/6] push to the dedicated benchmark results repository --- .github/workflows/tpch.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tpch.yaml b/.github/workflows/tpch.yaml index c967ee7b..77ded84f 100644 --- a/.github/workflows/tpch.yaml +++ b/.github/workflows/tpch.yaml @@ -5,6 +5,7 @@ on: push: branches: - main + - continuous-bench-attempt permissions: # deployments permission to deploy GitHub pages website @@ -37,7 +38,7 @@ jobs: tool: "cargo" output-file-path: output.txt gh-pages-branch: gh-pages - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }} # Automatically push the results to the GitHub pages branch. auto-push: true # Show alert with commit comment on detecting possible performance regression. @@ -46,3 +47,5 @@ jobs: comment-on-alert: true fail-on-alert: true alert-comment-cc-users: "@connortsui20" + # Push to dedicated benchmark result repository. + gh-repository: "https://github.com/cmu-db/optd-benchmark" From f3ff5bc58aaa23b4441d833dac3866f3d5f82990 Mon Sep 17 00:00:00 2001 From: Connor Tsui Date: Wed, 18 Dec 2024 18:15:21 -0500 Subject: [PATCH 5/6] add caching for CI workflow as well --- .github/workflows/CI.yaml | 2 ++ .github/workflows/tpch.yaml | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index a49b90d1..aa4eda25 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -36,6 +36,8 @@ jobs: with: profile: minimal components: rustfmt, clippy + # Make sure to use cached compilation objects if they exist. + - uses: Swatinem/rust-cache@v2 - name: Check code format uses: actions-rs/cargo@v1 with: diff --git a/.github/workflows/tpch.yaml b/.github/workflows/tpch.yaml index 77ded84f..4ab450fa 100644 --- a/.github/workflows/tpch.yaml +++ b/.github/workflows/tpch.yaml @@ -38,7 +38,7 @@ jobs: tool: "cargo" output-file-path: output.txt gh-pages-branch: gh-pages - github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }} + github-token: ${{ secrets.GITHUB_TOKEN }} # Automatically push the results to the GitHub pages branch. auto-push: true # Show alert with commit comment on detecting possible performance regression. @@ -48,4 +48,4 @@ jobs: fail-on-alert: true alert-comment-cc-users: "@connortsui20" # Push to dedicated benchmark result repository. - gh-repository: "https://github.com/cmu-db/optd-benchmark" + gh-repository: "github.com/cmu-db/optd-benchmark" From 1a5261f4aa230a49e0077aa966a942700cb00571 Mon Sep 17 00:00:00 2001 From: Connor Tsui Date: Fri, 20 Dec 2024 14:10:16 -0500 Subject: [PATCH 6/6] rename benchmark --- .github/workflows/tpch.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tpch.yaml b/.github/workflows/tpch.yaml index 4ab450fa..f921cc6a 100644 --- a/.github/workflows/tpch.yaml +++ b/.github/workflows/tpch.yaml @@ -1,6 +1,6 @@ name: Rust TPC-H Benchmarks -# Do not run this workflow on pull request since this workflow has permission to modify contents. +# Do NOT run this workflow on pull request since this workflow has permission to modify contents. on: push: branches: @@ -34,7 +34,7 @@ jobs: - name: Store benchmark result uses: benchmark-action/github-action-benchmark@v1 with: - name: Rust Benchmark + name: TPC-H Planning and Execution Benchmark tool: "cargo" output-file-path: output.txt gh-pages-branch: gh-pages @@ -48,4 +48,4 @@ jobs: fail-on-alert: true alert-comment-cc-users: "@connortsui20" # Push to dedicated benchmark result repository. - gh-repository: "github.com/cmu-db/optd-benchmark" + # gh-repository: "github.com/cmu-db/optd-benchmark" # TODO NOT WORKING