[amd][ci] Fix permission issues in the CI (#601)

Author: xuzhao9

.github/workflows/_linux-benchmark-mi350.yml

@@ -118,7 +118,7 @@ jobs:
           "
           mkdir -p benchmark-output
           cp -r ".benchmarks/${{ inputs.benchmark_name }}" benchmark-output/${TRITONBENCH_SIDE_A_ENV}
-          rm -rf .benchmarks || true
+          sudo rm -rf .benchmarks || true
           # post-process result.json
           latest_result_json=$(find ./benchmark-output/${TRITONBENCH_SIDE_A_ENV} -name "result.json"  | sort -r | head -n 1)
           python3 ./.ci/test_infra/oss_ci_benchmark_v3.py --json ${latest_result_json} \
@@ -141,7 +141,7 @@ jobs:
           "
           mkdir -p benchmark-output
           cp -r ".benchmarks/${{ inputs.benchmark_name }}" benchmark-output/${TRITONBENCH_SIDE_B_ENV}
-          rm -rf .benchmarks || true
+          sudo rm -rf .benchmarks || true
           # post-process result.json
           latest_result_json=$(find ./benchmark-output/${TRITONBENCH_SIDE_B_ENV} -name "result.json"  | sort -r | head -n 1)
           python3 ./.ci/test_infra/oss_ci_benchmark_v3.py --json ${latest_result_json} \

benchmarks/compile_time/run.py

@@ -119,7 +119,7 @@ def main() -> None:
     from tritonbench.utils.run_utils import run_in_task, setup_output_dir
 
     output_files = []
-    run_timestamp, output_dir = setup_output_dir("compile_time")
+    run_timestamp, output_dir = setup_output_dir("compile_time", ci=args.ci)
     op_args_list = {}
     if args.op:
         op_list = [args.op]

benchmarks/nightly/run.py

@@ -73,7 +73,7 @@ def run():
     from tritonbench.utils.run_utils import run_in_task, setup_output_dir
     from tritonbench.utils.scuba_utils import decorate_benchmark_data, log_benchmark
 
-    run_timestamp, output_dir = setup_output_dir("nightly")
+    run_timestamp, output_dir = setup_output_dir("nightly", ci=args.ci)
     # Run each operator
     output_files = []
     operator_benchmarks = get_operator_benchmarks()

tritonbench/utils/gpu_utils.py

@@ -4,7 +4,11 @@
 from contextlib import contextmanager
 from typing import Dict, List, Optional
 
-from tritonbench.utils.env_utils import is_mtia
+try:
+    from tritonbench.utils.env_utils import is_mtia
+except ModuleNotFoundError:
+    # In CI environment, we don't have torch installed at this point
+    is_mtia = lambda: False
 
 if is_mtia():
     from tritonbench.utils.fb.mtia_utils import MTIA_COMPUTE_SPECS, MTIA_MEMORY_SPECS

tritonbench/utils/run_utils.py

@@ -363,8 +363,11 @@ def run_in_task(
         sys.exit(1)
 
 
-def setup_output_dir(bm_name: str):
+def setup_output_dir(bm_name: str, ci: bool = False):
     current_timestamp = datetime.fromtimestamp(time.time()).strftime("%Y%m%d%H%M%S")
     output_dir = BENCHMARKS_OUTPUT_DIR.joinpath(bm_name, f"run-{current_timestamp}")
     Path.mkdir(output_dir, parents=True, exist_ok=True)
+    # set writable permission for all users (used by the ci env)
+    if ci:
+        output_dir.chmod(0o777)
     return current_timestamp, output_dir.absolute()