Reduce CI time (#1948)

Not much time (~20s) is saved but we should delete checkpoint folder
anyway as there is no use to upload the checkpoint folder.

Author: fegin

.github/workflows/integration_test_8gpu_features.yaml

@@ -75,3 +75,6 @@ jobs:
 
         export TEST_WITH_ROCM=$([[ "${{ matrix.gpu-arch-type }}" == "rocm" ]] && echo 1 || echo 0)
         python -m tests.integration_tests.run_tests --test_suite features $RUNNER_TEMP/artifacts-to-be-uploaded --ngpu 8
+
+        rm -rf $RUNNER_TEMP/artifacts-to-be-uploaded/*/checkpoint
+        rm -rf artifacts-to-be-uploaded/*/checkpoint

.github/workflows/integration_test_8gpu_h100.yaml

@@ -54,3 +54,4 @@ jobs:
 
         # Enable CPP stacktraces for debugging symmetric memory initialization errors.
         TORCH_SHOW_CPP_STACKTRACES=1 python -m tests.integration_tests.run_tests --test_suite h100 artifacts-to-be-uploaded --ngpu 8
+        rm -rf artifacts-to-be-uploaded/*/checkpoint

.github/workflows/integration_test_8gpu_models.yaml

@@ -53,3 +53,4 @@ jobs:
         mkdir artifacts-to-be-uploaded
         python -m tests.integration_tests.run_tests --test_suite models artifacts-to-be-uploaded --ngpu 8
         python -m tests.integration_tests.flux artifacts-to-be-uploaded/flux --ngpu 8
+        rm -rf artifacts-to-be-uploaded/*/checkpoint

.github/workflows/integration_test_8gpu_simple_fsdp.yaml

@@ -54,3 +54,4 @@ jobs:
 
         # Run the numerics unit tests of SimpleFSDP
         torchrun --nproc-per-node=8 -m pytest torchtitan/experiments/simple_fsdp/tests/test_numerics.py -v
+        rm -rf artifacts-to-be-uploaded/*/checkpoint

.github/workflows/integration_test_8gpu_torchcomms.yaml

@@ -51,3 +51,4 @@ jobs:
 
         mkdir artifacts-to-be-uploaded
         TEST_BACKEND=ncclx TRAIN_FILE=torchtitan.experiments.torchcomms.train python -m torchtitan.experiments.torchcomms.integration_tests artifacts-to-be-uploaded --ngpu 8
+        rm -rf artifacts-to-be-uploaded/*/checkpoint

.github/workflows/integration_test_8gpu_torchft.yaml

@@ -57,3 +57,4 @@ jobs:
         # Getting error - Cuda failure 217 'peer access is not supported between these two devices'
         python -m tests.integration_tests.ft artifacts-to-be-uploaded --ngpu 8
         # pkill -9 torchft_lighthouse
+        rm -rf artifacts-to-be-uploaded/*/checkpoint

.github/workflows/integration_test_8gpu_vlm.yaml

@@ -53,3 +53,4 @@ jobs:
 
         mkdir artifacts-to-be-uploaded
         python -m torchtitan.experiments.vlm.tests.integration_tests artifacts-to-be-uploaded --ngpu 4
+        rm -rf artifacts-to-be-uploaded/*/checkpoint

tests/integration_tests/run_tests.py

@@ -7,6 +7,7 @@
 import argparse
 import os
 import subprocess
+import time
 
 from torchtitan.tools.logging import logger
 
@@ -53,7 +54,7 @@ def run_single_test(test_flavor: OverrideDefinitions, full_path: str, output_dir
         if override_arg:
             cmd += " " + " ".join(override_arg)
         logger.info(
-            f"=====Integration test, flavor : {test_flavor.test_descr}, command : {cmd}====="
+            f"===== {time.strftime('%Y-%m-%d %H:%M:%S')} Integration test, flavor : {test_flavor.test_descr}, command : {cmd}====="
         )
 
         # save checkpoint (idx == 0) and load it for generation (idx == 1)