Skip to content

Commit 45768a9

Browse files
feisuzhufeisuzhu
committed
[ci] Revive /benchmark
1 parent 59780c1 commit 45768a9

File tree

4 files changed

+183
-10
lines changed

4 files changed

+183
-10
lines changed

‎.github/workflows/perf.yml‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,10 @@ name: Performance Monitoring
22
on:
33
push:
44
branches:
5-
- master
6-
workflow_dispatch:
5+
- master
6+
workflow_dispatch: {}
7+
repository_dispatch:
8+
types: [benchmark-command]
79

810
env:
911
CI_IMAGE_VERSION: '202303241150'
@@ -69,10 +71,15 @@ jobs:
6971
sleep 0.5
7072
7173
ci-docker-run-gpu --name taichi-benchmark-run \
74+
-e GITHUB_TOKEN \
75+
-e GITHUB_EVENT_ACTION \
7276
-e BENCHMARK_UPLOAD_TOKEN \
77+
-v $GITHUB_EVENT_PATH:/github-event.json \
7378
registry.botmaster.tgr/taichi-test-android:${{ env.CI_IMAGE_VERSION }} \
7479
/home/dev/taichi/.github/workflows/scripts/unix-perf-mon.sh
7580
7681
env:
7782
PY: '3.8'
7883
BENCHMARK_UPLOAD_TOKEN: ${{ secrets.BENCHMARK_UPLOAD_TOKEN }}
84+
GITHUB_EVENT_ACTION: ${{ github.event.action }}
85+
GITHUB_TOKEN: ${{ secrets.GARDENER_PAT }}

‎.github/workflows/scripts/perfmon.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/usr/bin/env python3 -u
2+
3+
import os
4+
from pathlib import Path
5+
6+
if os.environ.get('IN_DOCKER'):
7+
os.chdir(Path('taichi').resolve())
8+
9+
#. $(dirname $0)/common-utils.sh
10+
11+
#[[ "$IN_DOCKER" == "true" ]] && cd tiiaichi
12+
13+
#python3 .github/workflows/scripts/build.py --write-env=/tmp/ti-env.sh
14+
#. /tmp/ti-env.sh
15+
16+
## TODO: hard code Android NDK path in Docker image, should be handled by build.py
17+
#export ANDROID_NDK_ROOT=/android-sdk/ndk-bundle
18+
19+
#python -m pip install dist/*.whl
20+
21+
#TAG=$(git describe --exact-match --tags 2>/dev/null || true)
22+
#if [ ! -z "$TAG" ]; then
23+
# MORE_TAGS="--tags type=release,release=$TAG"
24+
#else
25+
# MORE_TAGS=""
26+
#fi
27+
28+
#git clone https://github.com/taichi-dev/taichi_benchmark
29+
#cd taichi_benchmark
30+
#pip install -r requirements.txt
31+
#python run.py --upload-auth $BENCHMARK_UPLOAD_TOKEN $MORE_TAGS

‎.github/workflows/scripts/post-benchmark-to-github-pr.py‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
#!/usr/bin/env python
2+
# -*- coding: utf-8 -*-
3+
4+
import argparse
5+
import json
6+
import os
7+
import sqlite3
8+
# -- stdlib --
9+
from pathlib import Path
10+
11+
# -- third party --
12+
import requests
13+
14+
# -- own --
15+
16+
# -- code --
17+
gh = requests.Session()
18+
gh.headers.update({
19+
'Authorization': f'Bearer {os.environ["GITHUB_TOKEN"]}',
20+
'Accept': 'application/vnd.github+json',
21+
'X-GitHub-Api-Version': '2022-11-28',
22+
})
23+
24+
25+
def post_comment(repo, number, msg):
26+
gh.post(
27+
f'https://api.github.com/repos/{repo}/issues/{number}/comments',
28+
json={'body': msg},
29+
)
30+
31+
32+
def get_db():
33+
db = sqlite3.connect(':memory:')
34+
db.execute('CREATE TABLE release (name TEXT, value REAL)')
35+
db.execute('CREATE UNIQUE INDEX release_name ON release (name)')
36+
db.execute('CREATE TABLE current (name TEXT, value REAL)')
37+
db.execute('CREATE UNIQUE INDEX current_name ON current (name)')
38+
return db
39+
40+
41+
IGNORE_TAGS = {'type', 'release', 'impl'}
42+
43+
44+
def flatten_metric(m):
45+
tags = [f'{k}={v}' for k, v in m['tags'].items() if k not in IGNORE_TAGS]
46+
tags = ','.join(sorted(tags))
47+
return (f'{m["name"]}@{tags}', m['value'])
48+
49+
50+
def render_result(baseline, sha, rs):
51+
texts = []
52+
_ = texts.append
53+
_(f'## Benchmark Report')
54+
_(f'Baseline: `{baseline}`')
55+
_(f'Current: `{sha}`')
56+
_('')
57+
_('| Item | Baseline | Current | Change |')
58+
_('| --- | --- | --- | --- |')
59+
for name, cv, rv, rate in rs:
60+
if rate > 5.0:
61+
color = 'green'
62+
elif rate < -5.0:
63+
color = 'red'
64+
else:
65+
color = 'gray'
66+
67+
rate = f'{" +"[rate > 0]}{rate:.2f}'
68+
69+
_(fr'| {name} | {rv:.2f} | {cv:.2f} | $\textcolor{{{color}}}{{\textsf{{{rate}\\%}}}}$ |'
70+
)
71+
72+
return '\n'.join(texts)
73+
74+
75+
def main():
76+
parser = argparse.ArgumentParser()
77+
parser.add_argument('event')
78+
parser.add_argument('result')
79+
options = parser.parse_args()
80+
81+
db = get_db()
82+
83+
current = json.loads(Path(options.result).read_text())
84+
for item in current:
85+
db.execute('INSERT INTO current VALUES (?, ?)', flatten_metric(item))
86+
87+
ver = requests.get(
88+
'https://benchmark.taichi-lang.cn/releases?order=vnum.desc&limit=1'
89+
).json()[0]['version']
90+
release = requests.get(
91+
f'https://benchmark.taichi-lang.cn/taichi_benchmark?tags->>type=eq.release&tags->>release=eq.{ver}'
92+
).json()
93+
94+
for item in release:
95+
db.execute('INSERT INTO release VALUES (?, ?)', flatten_metric(item))
96+
97+
db.execute('INSERT INTO current VALUES (?, ?)', ['meh', 1.2])
98+
99+
rs = db.execute('''
100+
SELECT
101+
c.name AS name,
102+
c.value AS cv,
103+
COALESCE(r.value, 0.0) AS rv,
104+
COALESCE((c.value - r.value) / r.value * 100, 0.0) AS rate
105+
FROM
106+
current c
107+
LEFT JOIN release r ON (r.name = c.name)
108+
ORDER BY name
109+
''')
110+
111+
event = json.loads(Path(options.event).read_text())
112+
sha = event["client_payload"]["pull_request"]["head"]["sha"]
113+
text = render_result(ver, sha, rs)
114+
115+
post_comment(
116+
event["repository"]["full_name"],
117+
event["client_payload"]["pull_request"]["number"],
118+
text,
119+
)
120+
121+
122+
if __name__ == '__main__':
123+
main()

‎.github/workflows/scripts/unix-perf-mon.sh‎

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ export PYTHONUNBUFFERED=1
77

88
[[ "$IN_DOCKER" == "true" ]] && cd taichi
99

10+
1011
python3 .github/workflows/scripts/build.py --write-env=/tmp/ti-env.sh
1112
. /tmp/ti-env.sh
1213

@@ -15,14 +16,25 @@ export ANDROID_NDK_ROOT=/android-sdk/ndk-bundle
1516

1617
python -m pip install dist/*.whl
1718

18-
TAG=$(git describe --exact-match --tags 2>/dev/null || true)
19-
if [ ! -z "$TAG" ]; then
20-
MORE_TAGS="--tags type=release,release=$TAG"
21-
else
22-
MORE_TAGS=""
23-
fi
19+
#################################
20+
git clone --depth=1 --branch=minimal-tests https://github.com/taichi-dev/taichi_benchmark
21+
#################################
22+
# git clone https://github.com/taichi-dev/taichi_benchmark
2423

25-
git clone https://github.com/taichi-dev/taichi_benchmark
2624
cd taichi_benchmark
2725
pip install -r requirements.txt
28-
python run.py --upload-auth $BENCHMARK_UPLOAD_TOKEN $MORE_TAGS
26+
27+
28+
if [ "$GITHUB_EVENT_ACTION" == "benchmark-command" ]; then
29+
python run.py --save ../result.json
30+
cd ..
31+
python .github/workflows/scripts/post-benchmark-to-github-pr.py /github-event.json result.json
32+
else
33+
TAG=$(git describe --exact-match --tags 2>/dev/null || true)
34+
if [ ! -z "$TAG" ]; then
35+
MORE_TAGS="--tags type=release,release=$TAG"
36+
else
37+
MORE_TAGS=""
38+
fi
39+
python run.py --upload-auth $BENCHMARK_UPLOAD_TOKEN $MORE_TAGS
40+
fi

0 commit comments

Comments
 (0)