Skip to content
This repository was archived by the owner on Mar 17, 2025. It is now read-only.

Commit 95b2c65

Browse files
beelchesterbeelchester
committed
Merge remote-tracking branch 'upstream/main' into k6_migration
2 parents 7a44cfe + da28f61 commit 95b2c65

File tree

6 files changed

+151
-70
lines changed

6 files changed

+151
-70
lines changed

‎.github/workflows/bench.yml‎

Lines changed: 44 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,14 @@
11
name: "Run benchmark"
2-
32
on:
43
pull_request_target:
54
types: [assigned, opened, synchronize, reopened, edited]
65
push:
76
branches:
87
- main
9-
108
permissions:
119
contents: write
1210
pull-requests: write
1311
issues: write
14-
1512
jobs:
1613
build:
1714
runs-on: benchmarking-runner
@@ -22,6 +19,19 @@ jobs:
2219
INFLUXDB_TOKEN: ${{ secrets.INFLUXDB_TOKEN }}
2320
INFLUXDB_ORG: ${{ secrets.INFLUXDB_ORG }}
2421
INFLUXDB_URL: ${{ secrets.INFLUXDB_URL }}
22+
strategy:
23+
matrix:
24+
service:
25+
[
26+
apollo_server,
27+
caliban,
28+
netflix_dgs,
29+
gqlgen,
30+
tailcall,
31+
async_graphql,
32+
hasura,
33+
graphql_jit,
34+
]
2535
steps:
2636
- name: Checkout (GitHub)
2737
uses: actions/checkout@v4
@@ -53,9 +63,37 @@ jobs:
5363
5464
- name: Run benchmarks
5565
run: |
56-
bash ./setup.sh
57-
bash ./run_benchmarks.sh
66+
bash ./graphql/${{ matrix.service }}/setup.sh
67+
bash run_benchmarks.sh ${{ matrix.service }}
68+
- name: List benchmark files
69+
run: |
70+
ls -la bench*.txt || echo "No matching files found"
5871
72+
- name: Upload benchmark results
73+
uses: actions/upload-artifact@v3
74+
with:
75+
name: benchmark-results
76+
path: bench*.txt
77+
78+
analyze:
79+
needs: build
80+
runs-on: benchmarking-runner
81+
steps:
82+
- name: Checkout (GitHub)
83+
uses: actions/checkout@v4
84+
85+
- name: Download all benchmark results
86+
uses: actions/download-artifact@v3
87+
with:
88+
name: benchmark-results
89+
path: .
90+
91+
- name: List downloaded artifacts
92+
run: ls -la bench*.txt || echo "No matching files found"
93+
94+
- name: Analyze results
95+
run: |
96+
bash run_analyze_script.sh
5997
- name: Print benchmark results
6098
run: cat ./results.md
6199

@@ -73,4 +111,4 @@ jobs:
73111
with:
74112
branch: main
75113
commit_author: Author <[email protected]>
76-
commit_message: "Update performance results in README.md"
114+
commit_message: "[ci skip] update performance results in README.md"

‎README.md‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -58,32 +58,32 @@ Get started with the benchmarks:
5858
| Query | Server | Requests/sec | Latency (ms) | Relative |
5959
|-------:|--------:|--------------:|--------------:|---------:|
6060
| 1 | `{ posts { id userId title user { id name email }}}` |
61-
|| [Tailcall] | `29,496.60` | `3.38` | `196.74x` |
62-
|| [async-graphql] | `1,805.45` | `55.70` | `12.04x` |
63-
|| [Caliban] | `1,547.91` | `64.33` | `10.32x` |
64-
|| [GraphQL JIT] | `1,335.26` | `74.60` | `8.91x` |
65-
|| [Gqlgen] | `772.80` | `128.41` | `5.15x` |
66-
|| [Netflix DGS] | `358.06` | `177.42` | `2.39x` |
67-
|| [Apollo GraphQL] | `268.25` | `366.16` | `1.79x` |
68-
|| [Hasura] | `149.93` | `549.07` | `1.00x` |
61+
|| [Tailcall] | `29,173.60` | `3.41` | `211.81x` |
62+
|| [async-graphql] | `2,019.59` | `49.49` | `14.66x` |
63+
|| [Caliban] | `1,744.80` | `57.28` | `12.67x` |
64+
|| [GraphQL JIT] | `1,342.93` | `74.16` | `9.75x` |
65+
|| [Gqlgen] | `797.87` | `124.41` | `5.79x` |
66+
|| [Netflix DGS] | `366.18` | `181.49` | `2.66x` |
67+
|| [Apollo GraphQL] | `263.39` | `372.49` | `1.91x` |
68+
|| [Hasura] | `137.74` | `587.80` | `1.00x` |
6969
| 2 | `{ posts { title }}` |
70-
|| [Tailcall] | `58,939.60` | `1.69` | `67.77x` |
71-
|| [Caliban] | `9,238.76` | `11.16` | `10.62x` |
72-
|| [async-graphql] | `9,138.76` | `11.34` | `10.51x` |
73-
|| [Gqlgen] | `2,136.19` | `48.48` | `2.46x` |
74-
|| [Apollo GraphQL] | `1,775.87` | `56.26` | `2.04x` |
75-
|| [Netflix DGS] | `1,593.80` | `69.82` | `1.83x` |
76-
|| [GraphQL JIT] | `1,399.68` | `71.35` | `1.61x` |
77-
|| [Hasura] | `869.69` | `114.81` | `1.00x` |
70+
|| [Tailcall] | `58,485.30` | `1.70` | `67.37x` |
71+
|| [Caliban] | `9,751.71` | `10.62` | `11.23x` |
72+
|| [async-graphql] | `9,444.13` | `10.73` | `10.88x` |
73+
|| [Gqlgen] | `2,181.26` | `47.48` | `2.51x` |
74+
|| [Apollo GraphQL] | `1,760.69` | `56.72` | `2.03x` |
75+
|| [Netflix DGS] | `1,598.09` | `70.06` | `1.84x` |
76+
|| [GraphQL JIT] | `1,392.45` | `71.71` | `1.60x` |
77+
|| [Hasura] | `868.17` | `114.99` | `1.00x` |
7878
| 3 | `{ greet }` |
79-
|| [Caliban] | `67,637.00` | `1.07` | `25.83x` |
80-
|| [Tailcall] | `59,557.80` | `1.69` | `22.75x` |
81-
|| [Gqlgen] | `47,525.20` | `5.21` | `18.15x` |
82-
|| [async-graphql] | `47,150.60` | `2.21` | `18.01x` |
83-
|| [Netflix DGS] | `8,261.44` | `15.02` | `3.16x` |
84-
|| [Apollo GraphQL] | `8,186.12` | `12.54` | `3.13x` |
85-
|| [GraphQL JIT] | `5,273.45` | `18.93` | `2.01x` |
86-
|| [Hasura] | `2,618.18` | `38.13` | `1.00x` |
79+
|| [Caliban] | `67,512.70` | `1.07` | `26.49x` |
80+
|| [Tailcall] | `59,015.70` | `1.71` | `23.16x` |
81+
|| [Gqlgen] | `47,378.10` | `5.25` | `18.59x` |
82+
|| [async-graphql] | `47,361.00` | `2.25` | `18.58x` |
83+
|| [Netflix DGS] | `8,297.54` | `14.39` | `3.26x` |
84+
|| [Apollo GraphQL] | `8,128.40` | `12.43` | `3.19x` |
85+
|| [GraphQL JIT] | `5,241.04` | `19.05` | `2.06x` |
86+
|| [Hasura] | `2,548.54` | `39.16` | `1.00x` |
8787

8888
<!-- PERFORMANCE_RESULTS_END -->
8989

‎analyze.sh‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
#!/bin/bash
22

3+
# Install gnuplot
4+
sudo apt-get update && sudo apt-get install -y gnuplot
5+
36
function extractMetric() {
47
local file="$1"
58
local metric="$2"
@@ -128,4 +131,4 @@ fi
128131
# Delete the result TXT files
129132
for file in "${resultFiles[@]}"; do
130133
rm "$file"
131-
done
134+
done

‎results.md‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -3,31 +3,31 @@
33
| Query | Server | Requests/sec | Latency (ms) | Relative |
44
|-------:|--------:|--------------:|--------------:|---------:|
55
| 1 | `{ posts { id userId title user { id name email }}}` |
6-
|| [Tailcall] | `29,496.60` | `3.38` | `196.74x` |
7-
|| [async-graphql] | `1,805.45` | `55.70` | `12.04x` |
8-
|| [Caliban] | `1,547.91` | `64.33` | `10.32x` |
9-
|| [GraphQL JIT] | `1,335.26` | `74.60` | `8.91x` |
10-
|| [Gqlgen] | `772.80` | `128.41` | `5.15x` |
11-
|| [Netflix DGS] | `358.06` | `177.42` | `2.39x` |
12-
|| [Apollo GraphQL] | `268.25` | `366.16` | `1.79x` |
13-
|| [Hasura] | `149.93` | `549.07` | `1.00x` |
6+
|| [Tailcall] | `29,173.60` | `3.41` | `211.81x` |
7+
|| [async-graphql] | `2,019.59` | `49.49` | `14.66x` |
8+
|| [Caliban] | `1,744.80` | `57.28` | `12.67x` |
9+
|| [GraphQL JIT] | `1,342.93` | `74.16` | `9.75x` |
10+
|| [Gqlgen] | `797.87` | `124.41` | `5.79x` |
11+
|| [Netflix DGS] | `366.18` | `181.49` | `2.66x` |
12+
|| [Apollo GraphQL] | `263.39` | `372.49` | `1.91x` |
13+
|| [Hasura] | `137.74` | `587.80` | `1.00x` |
1414
| 2 | `{ posts { title }}` |
15-
|| [Tailcall] | `58,939.60` | `1.69` | `67.77x` |
16-
|| [Caliban] | `9,238.76` | `11.16` | `10.62x` |
17-
|| [async-graphql] | `9,138.76` | `11.34` | `10.51x` |
18-
|| [Gqlgen] | `2,136.19` | `48.48` | `2.46x` |
19-
|| [Apollo GraphQL] | `1,775.87` | `56.26` | `2.04x` |
20-
|| [Netflix DGS] | `1,593.80` | `69.82` | `1.83x` |
21-
|| [GraphQL JIT] | `1,399.68` | `71.35` | `1.61x` |
22-
|| [Hasura] | `869.69` | `114.81` | `1.00x` |
15+
|| [Tailcall] | `58,485.30` | `1.70` | `67.37x` |
16+
|| [Caliban] | `9,751.71` | `10.62` | `11.23x` |
17+
|| [async-graphql] | `9,444.13` | `10.73` | `10.88x` |
18+
|| [Gqlgen] | `2,181.26` | `47.48` | `2.51x` |
19+
|| [Apollo GraphQL] | `1,760.69` | `56.72` | `2.03x` |
20+
|| [Netflix DGS] | `1,598.09` | `70.06` | `1.84x` |
21+
|| [GraphQL JIT] | `1,392.45` | `71.71` | `1.60x` |
22+
|| [Hasura] | `868.17` | `114.99` | `1.00x` |
2323
| 3 | `{ greet }` |
24-
|| [Caliban] | `67,637.00` | `1.07` | `25.83x` |
25-
|| [Tailcall] | `59,557.80` | `1.69` | `22.75x` |
26-
|| [Gqlgen] | `47,525.20` | `5.21` | `18.15x` |
27-
|| [async-graphql] | `47,150.60` | `2.21` | `18.01x` |
28-
|| [Netflix DGS] | `8,261.44` | `15.02` | `3.16x` |
29-
|| [Apollo GraphQL] | `8,186.12` | `12.54` | `3.13x` |
30-
|| [GraphQL JIT] | `5,273.45` | `18.93` | `2.01x` |
31-
|| [Hasura] | `2,618.18` | `38.13` | `1.00x` |
24+
|| [Caliban] | `67,512.70` | `1.07` | `26.49x` |
25+
|| [Tailcall] | `59,015.70` | `1.71` | `23.16x` |
26+
|| [Gqlgen] | `47,378.10` | `5.25` | `18.59x` |
27+
|| [async-graphql] | `47,361.00` | `2.25` | `18.58x` |
28+
|| [Netflix DGS] | `8,297.54` | `14.39` | `3.26x` |
29+
|| [Apollo GraphQL] | `8,128.40` | `12.43` | `3.19x` |
30+
|| [GraphQL JIT] | `5,241.04` | `19.05` | `2.06x` |
31+
|| [Hasura] | `2,548.54` | `39.16` | `1.00x` |
3232

3333
<!-- PERFORMANCE_RESULTS_END -->

‎run_analyze_script.sh‎

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,36 @@
11
#!/bin/bash
22

3-
rm results.md
3+
# Update and install gnuplot
4+
sudo apt-get update && sudo apt-get install -y gnuplot
45

6+
# Remove existing results file
7+
rm -f results.md
8+
9+
services=("apollo" "caliban" "netflixdgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit")
10+
11+
# Loop through each benchmark (1, 2, 3)
512
for bench in 1 2 3; do
6-
if ls bench${bench}*.txt &> /dev/null; then
7-
echo "Processing files for bench${bench}:"
8-
bash analyze.sh bench${bench}*.txt
9-
echo "Files processed: $(ls bench${bench}*.txt)"
10-
else
11-
echo "No matching files found for bench${bench}*.txt"
12-
fi
13+
echo "Processing files for bench${bench}:"
14+
15+
# Construct the command for each benchmark
16+
cmd="bash analyze.sh"
17+
18+
# Loop through each service
19+
for service in "${services[@]}"; do
20+
# Convert service name to match file naming convention
21+
case $service in
22+
"apollo") file_service="apollo_server" ;;
23+
"netflixdgs") file_service="netflix_dgs" ;;
24+
*) file_service=$service ;;
25+
esac
26+
27+
# Add files for current service to the command
28+
for run in 1 2 3; do
29+
cmd+=" bench${bench}_result${run}_graphql_${file_service}_run.sh.txt"
30+
done
31+
done
32+
33+
# Execute the command
34+
echo "Executing: $cmd"
35+
eval $cmd
1336
done

‎run_benchmarks.sh‎

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ function killServerOnPort() {
1212
echo "No process found running on port $port"
1313
fi
1414
}
15+
1516
bench1Results=()
1617
bench2Results=()
1718
bench3Results=()
@@ -73,16 +74,32 @@ function runBenchmark() {
7374

7475
rm "results.md"
7576

76-
for service in "apollo_server" "caliban" "netflix_dgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit"; do
77-
runBenchmark "$service"
78-
if [ "$service" == "apollo_server" ]; then
77+
# Main script
78+
if [ $# -eq 0 ]; then
79+
echo "Usage: $0 <service_name>"
80+
echo "Available services: apollo_server, caliban, netflix_dgs, gqlgen, tailcall, async_graphql, hasura, graphql_jit"
81+
exit 1
82+
fi
83+
84+
service="$1"
85+
valid_services=("apollo_server" "caliban" "netflix_dgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit")
86+
87+
if [[ ! " ${valid_services[@]} " =~ " ${service} " ]]; then
88+
echo "Invalid service name. Available services: ${valid_services[*]}"
89+
exit 1
90+
fi
91+
92+
93+
94+
runBenchmark "graphql/${service}/run.sh"
95+
96+
if [ "$service" == "apollo_server" ]; then
7997
cd graphql/apollo_server/
8098
npm stop
8199
cd ../../
82-
elif [ "$service" == "hasura" ]; then
100+
elif [ "$service" == "hasura" ]; then
83101
bash "graphql/hasura/kill.sh"
84-
fi
85-
done
102+
fi
86103

87104
bash analyze.sh "${bench1Results[@]}"
88105
bash analyze.sh "${bench2Results[@]}"
@@ -101,4 +118,4 @@ if [[ "$UPLOAD_TO_CLOUD" == "true" ]]; then
101118
curl -o assets/posts_latency.png -H "Authorization: Bearer $GRAFANA_API_KEY" "https://tailcall.grafana.net/render/d-solo/cdqucydulbfggb?tab=queries&from=$from&to=$now&panelId=panel-6&__feature.dashboardSceneSolo&width=1000&height=500&tz=Asia%2FCalcutta" --connect-timeout 120
102119
curl -o assets/greet_req.png -H "Authorization: Bearer $GRAFANA_API_KEY" "https://tailcall.grafana.net/render/d-solo/cdqucydulbfggb?tab=queries&from=$from&to=$now&panelId=panel-8&__feature.dashboardSceneSolo&width=1000&height=500&tz=Asia%2FCalcutta" --connect-timeout 120
103120
curl -o assets/greet_latency.png -H "Authorization: Bearer $GRAFANA_API_KEY" "https://tailcall.grafana.net/render/d-solo/cdqucydulbfggb?tab=queries&from=$from&to=$now&panelId=panel-9&__feature.dashboardSceneSolo&width=1000&height=500&tz=Asia%2FCalcutta" --connect-timeout 120
104-
fi
121+
fi

0 commit comments

Comments
 (0)