diff --git a/.github/actions/linux-uttest/action.yml b/.github/actions/linux-uttest/action.yml index 6e2cf2cee..8d7a7611e 100644 --- a/.github/actions/linux-uttest/action.yml +++ b/.github/actions/linux-uttest/action.yml @@ -102,6 +102,55 @@ runs: cp *.xml ${{ github.workspace }}/ut_log echo -e "File Path: cd pytorch/third_party/torch-xpu-ops/test/xpu" | tee -a ${{ github.workspace }}/ut_log/reproduce_op_ut.log echo -e "Reproduce Command: pytest -sv failed_case" | tee -a ${{ github.workspace }}/ut_log/reproduce_op_ut.log + - name: xpu_inductor + shell: timeout 18000 bash -xe {0} + if: ${{ inputs.ut_name == 'xpu_inductor' }} + run: | + export PYTORCH_TEST_WITH_SLOW=1 + export PYTORCH_TESTING_DEVICE_ONLY_FOR="xpu" + mkdir -p ut_log/xpu_inductor + cd pytorch + files=( + "test_codecache.py" + "test_kernel_benchmark.py" + "test_max_autotune.py" + "test_mkldnn_pattern_matcher.py" + "test_triton_kernels.py" + "test_torchinductor.py" + "test_torchinductor_opinfo.py" + "test_compile_subprocess.py" + "test_compiled_optimizers.py" + "test_compiled_autograd.py" + "test_aot_inductor.py" + ) + printf "%s\n" "${files[@]}" > test_files.txt + cat test_files.txt + while IFS= read -r line; do + ( + echo "=== Starting test: ${line} ===" + start=$(date +%s) + pytest -sv test/inductor/${line} --junit-xml=${{ github.workspace }}/ut_log/inductor_${line}.xml 2>${{ github.workspace }}/ut_log/xpu_inductor/xpu_inductor_${line}_test_error.log | \ + tee ${{ github.workspace }}/ut_log/xpu_inductor/xpu_inductor_${line}_test.log + end=$(date +%s) + echo -e "${line} duration: $((end - start))s" + echo "=== Finished test: ${line} ===" + ) & + done < test_files.txt + + wait + - name: test_xpu + shell: timeout 3600 bash -xe {0} + if: ${{ inputs.ut_name == 'test_xpu' }} + run: | + export PYTORCH_TEST_WITH_SLOW=1 + export PYTORCH_TESTING_DEVICE_ONLY_FOR="xpu" + mkdir -p ut_log/test_xpu + cd pytorch + if [ -f "test/test_xpu.py" ]; then + pytest -sv test/test_xpu.py --junit-xml=${{ github.workspace }}/ut_log/test_xpu.xml \ + 2> ${{ github.workspace }}/ut_log/test_xpu/test_xpu_error.log | \ + tee ${{ github.workspace }}/ut_log/test_xpu/test_xpu.log + fi - name: torch_xpu shell: timeout 3600 bash -xe {0} if: ${{ inputs.ut_name == 'torch_xpu' }} @@ -109,13 +158,10 @@ runs: export PYTORCH_TEST_WITH_SLOW=1 export PYTORCH_TESTING_DEVICE_ONLY_FOR="xpu" mkdir -p ut_log/torch_xpu - cd pytorch - test_cmd="python test/run_test.py --include " - for test in $(ls test/inductor | grep test); do test_cmd="${test_cmd} inductor/$test"; done - for test in $(ls test/xpu | grep test); do test_cmd="${test_cmd} xpu/$test"; done - if [ -f "test/test_xpu.py" ]; then test_cmd="${test_cmd} test_xpu.py"; fi - eval $test_cmd 2> ${{ github.workspace }}/ut_log/torch_xpu/torch_xpu_test_error.log | \ - tee ${{ github.workspace }}/ut_log/torch_xpu/torch_xpu_test.log + cd pytorch/test/xpu + pytest -sv test*.py --junit-xml=${{ github.workspace }}/ut_log/torch_xpu.xml \ + 2> ${{ github.workspace }}/ut_log/torch_xpu/torch_xpu_error.log | \ + tee ${{ github.workspace }}/ut_log/torch_xpu/torch_xpu.log - name: xpu_profiling shell: timeout 3600 bash -xe {0} if: ${{ inputs.ut_name == 'xpu_profiling' }} diff --git a/.github/scripts/check-ut.py b/.github/scripts/check-ut.py index c21f9ceda..fa6e1c179 100644 --- a/.github/scripts/check-ut.py +++ b/.github/scripts/check-ut.py @@ -106,6 +106,13 @@ def get_message(case): return " ; ".join(error_messages) if error_messages else f"{case.result[0].message.splitlines()[0]}" +def get_case_identifier(case): + """Generate a unique identifier for a test case to detect duplicates""" + category = get_category_from_case(case) + classname = get_classname(case) + name = get_name(case) + return f"{category}:{classname}:{name}" + def print_md_row(row, print_header=False, failure_list=None): if print_header: header = " | ".join([f"{key}" for key in row.keys()]) @@ -125,7 +132,16 @@ def print_failures(failure_list=None): print("### Test Failures") print_header = True + + seen_cases = set() + unique_failures = [] for case in failures: + case_id = get_case_identifier(case) + if case_id not in seen_cases: + seen_cases.add(case_id) + unique_failures.append(case) + + for case in unique_failures: print_md_row({ 'Category': get_category_from_case(case), 'Class name': get_classname(case), @@ -140,9 +156,15 @@ def generate_failures_log(): if not failures: return + seen_cases = set() + failures_by_category.clear() + for case in failures: - category = get_category_from_case(case) - failures_by_category[category].append(case) + case_id = get_case_identifier(case) + if case_id not in seen_cases: + seen_cases.add(case_id) + category = get_category_from_case(case) + failures_by_category[category].append(case) for category, category_failures in failures_by_category.items(): if not category_failures: @@ -246,8 +268,12 @@ def determine_category(ut): return 'op_transformers' elif ut == 'test_xpu': return 'test_xpu' + elif ut == 'torch_xpu': + return 'torch_xpu' elif 'op_ut' in ut: return 'op_ut' + elif 'inductor_' in ut: + return 'xpu_inductor' else: return 'unknown' @@ -342,6 +368,8 @@ def print_summary(): } for summary in summaries: + if summary['Test cases'] == 0: + continue print_md_row({ 'Category': summary['Category'], 'UT': summary['UT'], diff --git a/.github/scripts/ut_result_check.sh b/.github/scripts/ut_result_check.sh index 1aac53249..d5d8c8a4d 100644 --- a/.github/scripts/ut_result_check.sh +++ b/.github/scripts/ut_result_check.sh @@ -82,7 +82,9 @@ check_test_cases() { ["op_regression_dev1"]=1 ["op_transformers"]=237 ["op_ut"]=120408 + ["xpu_inductor"]=8330 ["test_xpu"]=69 + ["torch_xpu"]=359 ) if [[ ! -f "$log_file" ]]; then @@ -124,7 +126,7 @@ check_test_cases() { } -if [[ "${ut_suite}" == 'op_regression' || "${ut_suite}" == 'op_regression_dev1' || "${ut_suite}" == 'op_extended' || "${ut_suite}" == 'op_transformers' || "${ut_suite}" == 'op_ut' || "${ut_suite}" == 'test_xpu' ]]; then +if [[ "${ut_suite}" == 'op_regression' || "${ut_suite}" == 'op_regression_dev1' || "${ut_suite}" == 'op_extended' || "${ut_suite}" == 'op_transformers' || "${ut_suite}" == 'op_ut' || "${ut_suite}" == 'test_xpu' || "${ut_suite}" == 'xpu_inductor' || "${ut_suite}" == 'torch_xpu' ]]; then echo -e "=========================================================================" echo -e "Show Failed cases in ${ut_suite}" echo -e "=========================================================================" diff --git a/.github/workflows/nightly_ondemand.yml b/.github/workflows/nightly_ondemand.yml index d408622ef..7e097d659 100644 --- a/.github/workflows/nightly_ondemand.yml +++ b/.github/workflows/nightly_ondemand.yml @@ -95,7 +95,7 @@ jobs: echo "No such scheduler: ${{ github.event.schedule }}" exit 1 fi - ut='["basic","op_ut","xpu_profiling","xpu_distributed"]' + ut='["basic","op_ut","xpu_profiling","xpu_inductor","torch_xpu","xpu_distributed"]' suite='["huggingface","timm_models","torchbench","pt2e"]' triton='' python='3.10'