Enale buck testing in ET for llm runner #474
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: pull | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }} | |
| cancel-in-progress: true | |
| jobs: | |
| unittest-linux: | |
| name: unittest-linux | |
| uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main | |
| strategy: | |
| fail-fast: false | |
| with: | |
| runner: linux.2xlarge | |
| submodules: 'true' | |
| ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} | |
| timeout: 90 | |
| script: | | |
| set -ex | |
| cmake -DCMAKE_BUILD_TYPE=Debug test -Bbuild/test | |
| cmake --build build/test -j9 --config Debug | |
| pushd build/test && ctest && popd | |
| # Install tokenizers | |
| pip install . -v | |
| pip install pytest blobfile transformers>=4.53.1 | |
| # Run tests | |
| pytest | |
| unittest-windows: | |
| uses: pytorch/test-infra/.github/workflows/windows_job.yml@main | |
| with: | |
| runner: windows.4xlarge | |
| submodules: 'recursive' | |
| ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} | |
| script: | | |
| conda init powershell | |
| powershell -Command "& { | |
| Set-PSDebug -Trace 1 | |
| \$ErrorActionPreference = 'Stop' | |
| \$PSNativeCommandUseErrorActionPreference = \$true | |
| # Create a symbolic link to work around path length limitations. This gives a much shorter | |
| # path, as the default checkout directory is deeply nested. | |
| \$workingDir = \$PWD.Path | |
| cd \$Env:GITHUB_WORKSPACE | |
| New-Item -ItemType SymbolicLink -Path tk -Value \$workingDir | |
| cd tk | |
| # Run C++ unit tests | |
| cmake -DCMAKE_BUILD_TYPE=Debug test -Bbuild/test -T ClangCL | |
| cmake --build build/test -j9 --config Debug | |
| if (\$LASTEXITCODE -ne 0) { | |
| Write-Host "Build was not successful. Exit code: \$LASTEXITCODE." | |
| exit \$LASTEXITCODE | |
| } | |
| Push-Location build/test | |
| ctest | |
| if (\$LASTEXITCODE -ne 0) { | |
| Write-Host "Unit tests were not successful. Exit code: \$LASTEXITCODE." | |
| exit \$LASTEXITCODE | |
| } | |
| Pop-Location | |
| conda create --yes --quiet -n tokenizers python=3.12 | |
| conda activate tokenizers | |
| # Install tokenizers | |
| pip install . -v | |
| if (\$LASTEXITCODE -ne 0) { | |
| Write-Host "Python installation was unsuccessful. Exit code: \$LASTEXITCODE." | |
| exit \$LASTEXITCODE | |
| } | |
| pip install pytest blobfile transformers>=4.53.1 | |
| # Run python tests | |
| pytest | |
| if (\$LASTEXITCODE -ne 0) { | |
| Write-Host "Python tests were not successful. Exit code: \$LASTEXITCODE." | |
| Start-Sleep -Seconds 600 # Debug - keep alive to give time to SSH | |
| exit \$LASTEXITCODE | |
| } | |
| }" |