Skip to content

Commit 580d6df

Browse files
kevin314kevin314
authored
[CI] Add tests to Modal (#562)
1 parent 344e430 commit 580d6df

File tree

12 files changed

+241
-83
lines changed

12 files changed

+241
-83
lines changed

‎.buildkite/pipeline.yml‎

Lines changed: 95 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,25 +2,26 @@ env:
22
IMAGE_VERSION: "py3.12-latest"
33

44
steps:
5-
- block: "Start Build"
6-
blocked_state: "running"
7-
prompt: "Approve build?"
5+
- label: "pre-commit"
6+
command: ".buildkite/scripts/pre_commit.sh"
7+
agents:
8+
queue: "default"
9+
env:
10+
- BUILDKITE_CLEAN_CHECKOUT=true
11+
12+
- wait
813

914
- label: "Trigger Tests"
10-
command: |
11-
echo "Current working directory: $(pwd)"
12-
echo "Current branch:"
13-
git branch --show-current
14-
echo "Full diff:"
15-
git diff --name-only $BUILDKITE_PULL_REQUEST_BASE_BRANCH...HEAD
1615
plugins:
1716
- monorepo-diff#v1.4.0:
1817
diff: "git diff --name-only $BUILDKITE_PULL_REQUEST_BASE_BRANCH...HEAD"
1918
watch:
2019
- path:
2120
- "fastvideo/v1/models/encoders/**"
22-
- "fastvideo/v1/models/loaders/**"
21+
- "fastvideo/v1/models/loader/**"
2322
- "fastvideo/v1/tests/encoders/**"
23+
- "pyproject.toml"
24+
- "docker/Dockerfile.python3.12"
2425
config:
2526
command: "timeout 30m .buildkite/scripts/pr_test.sh"
2627
label: "Encoder Tests"
@@ -31,8 +32,10 @@ steps:
3132
queue: "default"
3233
- path:
3334
- "fastvideo/v1/models/vaes/**"
34-
- "fastvideo/v1/models/loaders/**"
35+
- "fastvideo/v1/models/loader/**"
3536
- "fastvideo/v1/tests/vaes/**"
37+
- "pyproject.toml"
38+
- "docker/Dockerfile.python3.12"
3639
config:
3740
command: "timeout 30m .buildkite/scripts/pr_test.sh"
3841
label: "VAE Tests"
@@ -43,10 +46,12 @@ steps:
4346
queue: "default"
4447
- path:
4548
- "fastvideo/v1/models/dits/**"
46-
- "fastvideo/v1/models/loaders/**"
49+
- "fastvideo/v1/models/loader/**"
4750
- "fastvideo/v1/tests/transformers/**"
4851
- "fastvideo/v1/layers/**"
4952
- "fastvideo/v1/attention/**"
53+
- "pyproject.toml"
54+
- "docker/Dockerfile.python3.12"
5055
config:
5156
command: "timeout 30m .buildkite/scripts/pr_test.sh"
5257
label: "Transformer Tests"
@@ -55,7 +60,8 @@ steps:
5560
- TEST_TYPE=transformer
5661
agents:
5762
queue: "default"
58-
- path: "fastvideo/v1/**/*.py"
63+
- path:
64+
- "fastvideo/v1/**/*.py"
5965
config:
6066
command: "timeout 60m .buildkite/scripts/pr_test.sh"
6167
label: "SSIM Tests"
@@ -64,3 +70,79 @@ steps:
6470
- TEST_TYPE=ssim
6571
agents:
6672
queue: "default"
73+
- path:
74+
- "fastvideo/v1/**"
75+
- "pyproject.toml"
76+
- "docker/Dockerfile.python3.12"
77+
config:
78+
command: "timeout 30m .buildkite/scripts/pr_test.sh"
79+
label: "Training Tests"
80+
env:
81+
- BUILDKITE_CLEAN_CHECKOUT=true
82+
- TEST_TYPE=training
83+
agents:
84+
queue: "default"
85+
- path:
86+
- "fastvideo/v1/**"
87+
- "csrc/attn/vsa/**"
88+
- "csrc/attn/tk/**"
89+
- "csrc/attn/setup_vsa.py"
90+
- "csrc/attn/config_vsa.py"
91+
- "csrc/attn/vsa.cpp"
92+
- "pyproject.toml"
93+
- "docker/Dockerfile.python3.12"
94+
config:
95+
command: "timeout 30m .buildkite/scripts/pr_test.sh"
96+
label: "Training Tests VSA"
97+
env:
98+
- BUILDKITE_CLEAN_CHECKOUT=true
99+
- TEST_TYPE=training_vsa
100+
agents:
101+
queue: "default"
102+
- path:
103+
- "fastvideo/v1/**"
104+
- "csrc/attn/st_attn/**"
105+
- "csrc/attn/setup_sta.py"
106+
- "csrc/attn/config_sta.py"
107+
- "csrc/attn/st_attn.cpp"
108+
- "pyproject.toml"
109+
- "docker/Dockerfile.python3.12"
110+
config:
111+
command: "timeout 30m .buildkite/scripts/pr_test.sh"
112+
label: "Inference Tests STA"
113+
env:
114+
- BUILDKITE_CLEAN_CHECKOUT=true
115+
- TEST_TYPE=inference_sta
116+
agents:
117+
queue: "default"
118+
- path:
119+
- "csrc/attn/st_attn/**"
120+
- "csrc/attn/setup_sta.py"
121+
- "csrc/attn/config_sta.py"
122+
- "csrc/attn/st_attn.cpp"
123+
- "pyproject.toml"
124+
- "docker/Dockerfile.python3.12"
125+
config:
126+
command: "timeout 30m .buildkite/scripts/pr_test.sh"
127+
label: "Precision Tests STA"
128+
env:
129+
- BUILDKITE_CLEAN_CHECKOUT=true
130+
- TEST_TYPE=precision_sta
131+
agents:
132+
queue: "default"
133+
- path:
134+
- "csrc/attn/vsa/**"
135+
- "csrc/attn/tk/**"
136+
- "csrc/attn/setup_vsa.py"
137+
- "csrc/attn/config_vsa.py"
138+
- "csrc/attn/vsa.cpp"
139+
- "pyproject.toml"
140+
- "docker/Dockerfile.python3.12"
141+
config:
142+
command: "timeout 30m .buildkite/scripts/pr_test.sh"
143+
label: "Precision Tests VSA"
144+
env:
145+
- BUILDKITE_CLEAN_CHECKOUT=true
146+
- TEST_TYPE=precision_vsa
147+
agents:
148+
queue: "default"

‎.buildkite/scripts/pr_test.sh‎

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,10 @@ log "Setting up Modal authentication from Buildkite secrets..."
3131
MODAL_TOKEN_ID=$(buildkite-agent secret get modal_token_id)
3232
MODAL_TOKEN_SECRET=$(buildkite-agent secret get modal_token_secret)
3333

34+
WANDB_API_KEY=$(buildkite-agent secret get wandb_api_key)
35+
36+
WANDB_API_KEY=$(buildkite-agent secret get wandb_api_key)
37+
3438
if [ -n "$MODAL_TOKEN_ID" ] && [ -n "$MODAL_TOKEN_SECRET" ]; then
3539
log "Retrieved Modal credentials from Buildkite secrets"
3640
python3 -m modal token set --token-id "$MODAL_TOKEN_ID" --token-secret "$MODAL_TOKEN_SECRET" --profile buildkite-ci --activate --verify
@@ -54,22 +58,44 @@ if [ -z "${TEST_TYPE:-}" ]; then
5458
fi
5559
log "Test type: $TEST_TYPE"
5660

61+
MODAL_ENV="BUILDKITE_REPO=$BUILDKITE_REPO BUILDKITE_COMMIT=$BUILDKITE_COMMIT IMAGE_VERSION=$IMAGE_VERSION"
62+
5763
case "$TEST_TYPE" in
5864
"encoder")
5965
log "Running encoder tests..."
60-
MODAL_COMMAND="python3 -m modal run $MODAL_TEST_FILE::run_encoder_tests"
66+
MODAL_COMMAND="$MODAL_ENV python3 -m modal run $MODAL_TEST_FILE::run_encoder_tests"
6167
;;
6268
"vae")
6369
log "Running VAE tests..."
64-
MODAL_COMMAND="python3 -m modal run $MODAL_TEST_FILE::run_vae_tests"
70+
MODAL_COMMAND="$MODAL_ENV python3 -m modal run $MODAL_TEST_FILE::run_vae_tests"
6571
;;
6672
"transformer")
6773
log "Running transformer tests..."
68-
MODAL_COMMAND="python3 -m modal run $MODAL_TEST_FILE::run_transformer_tests"
74+
MODAL_COMMAND="$MODAL_ENV python3 -m modal run $MODAL_TEST_FILE::run_transformer_tests"
6975
;;
7076
"ssim")
7177
log "Running SSIM tests..."
72-
MODAL_COMMAND="python3 -m modal run $MODAL_TEST_FILE::run_ssim_tests"
78+
MODAL_COMMAND="$MODAL_ENV python3 -m modal run $MODAL_TEST_FILE::run_ssim_tests"
79+
;;
80+
"training")
81+
log "Running training tests..."
82+
MODAL_COMMAND="$MODAL_ENV WANDB_API_KEY=$WANDB_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_training_tests"
83+
;;
84+
"training_vsa")
85+
log "Running training VSA tests..."
86+
MODAL_COMMAND="$MODAL_ENV WANDB_API_KEY=$WANDB_API_KEY python3 -m modal run $MODAL_TEST_FILE::run_training_tests_VSA"
87+
;;
88+
"inference_sta")
89+
log "Running inference STA tests..."
90+
MODAL_COMMAND="$MODAL_ENV python3 -m modal run $MODAL_TEST_FILE::run_inference_tests_STA"
91+
;;
92+
"precision_sta")
93+
log "Running precision STA tests..."
94+
MODAL_COMMAND="$MODAL_ENV python3 -m modal run $MODAL_TEST_FILE::run_precision_tests_STA"
95+
;;
96+
"precision_vsa")
97+
log "Running precision VSA tests..."
98+
MODAL_COMMAND="$MODAL_ENV python3 -m modal run $MODAL_TEST_FILE::run_precision_tests_VSA"
7399
;;
74100
*)
75101
log "Error: Unknown test type: $TEST_TYPE"

‎.buildkite/scripts/pre_commit.sh‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/bin/bash
2+
set -uo pipefail
3+
4+
log() {
5+
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1"
6+
}
7+
8+
log "=== Starting pre-commit checks ==="
9+
10+
cd "$(dirname "$0")/../.."
11+
PROJECT_ROOT=$(pwd)
12+
log "Project root: $PROJECT_ROOT"
13+
14+
if ! python3 -m pre_commit --version &> /dev/null; then
15+
log "pre-commit not found, installing..."
16+
python3 -m pip install --user pre-commit==4.0.1
17+
18+
if ! python3 -m pre_commit --version &> /dev/null; then
19+
log "Error: Failed to install pre-commit."
20+
exit 1
21+
fi
22+
fi
23+
24+
log "Pre-commit version: $(python3 -m pre_commit --version)"
25+
26+
log "Installing/updating pre-commit hooks..."
27+
python3 -m pre_commit install --install-hooks
28+
29+
log "Running pre-commit checks on all files..."
30+
python3 -m pre_commit run --all-files
31+
PRE_COMMIT_EXIT_CODE=$?
32+
33+
if [ $PRE_COMMIT_EXIT_CODE -eq 0 ]; then
34+
log "Pre-commit checks completed successfully"
35+
else
36+
log "Error: Pre-commit checks failed with exit code: $PRE_COMMIT_EXIT_CODE"
37+
fi
38+
39+
log "=== Pre-commit checks completed with exit code: $PRE_COMMIT_EXIT_CODE ==="
40+
exit $PRE_COMMIT_EXIT_CODE

‎fastvideo/v1/tests/encoders/test_clip_encoder.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,5 +170,5 @@ def test_clip_encoder():
170170
f"Pooler outputs differ significantly: mean diff = {mean_diff_pooler.item()}"
171171
assert max_diff_hidden < 1e-1, \
172172
f"Hidden states differ significantly: max diff = {max_diff_hidden.item()}"
173-
assert max_diff_pooler < 1e-2, \
173+
assert max_diff_pooler < 2e-2, \
174174
f"Pooler outputs differ significantly: max diff = {max_diff_pooler.item()}"

0 commit comments

Comments
 (0)