Results from R50 GH action on macos-latest

Author: mlcommons-bot

open/MLCommons/measurements/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/README.md

@@ -1,3 +1,3 @@
 | Model    | Scenario   |   Accuracy |   Throughput | Latency (in ms)   |
 |----------|------------|------------|--------------|-------------------|
-| resnet50 | offline    |         76 |         7.87 | -                 |
+| resnet50 | offline    |         76 |        9.061 | -                 |

open/MLCommons/measurements/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline/README.md

@@ -16,16 +16,16 @@ pip install -U mlcflow
 
 mlc rm cache -f
 
-mlc pull repo GATEOverflow@mlperf-automations --checkout=525d36468c61affa1c49bc324cdd741890d5fbf7
+mlc pull repo mlcommons@mlperf-automations --checkout=d2ebe85e5aa9c9b1b1cb9e7dd042dcf395abcd74
 
 
 ```
 *Note that if you want to use the [latest automation recipes](https://docs.mlcommons.org/inference) for MLPerf,
- you should simply reload GATEOverflow@mlperf-automations without checkout and clean MLC cache as follows:*
+ you should simply reload mlcommons@mlperf-automations without checkout and clean MLC cache as follows:*
 
 ```bash
-mlc rm repo GATEOverflow@mlperf-automations
-mlc pull repo GATEOverflow@mlperf-automations
+mlc rm repo mlcommons@mlperf-automations
+mlc pull repo mlcommons@mlperf-automations
 mlc rm cache -f
 
 ```
@@ -40,4 +40,4 @@ Model Precision: fp32
 `acc`: `76.0`, Required accuracy for closed division `>= 75.6954`
 
 ### Performance Results 
-`Samples per second`: `7.86972`
+`Samples per second`: `9.06095`

open/MLCommons/measurements/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline/accuracy_console.out

@@ -1,5 +1,5 @@
-User Conf path: /Users/runner/MLC/repos/GATEOverflow@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/8e34b4933a4c4e77ad3e1373c7d22ea9.conf
-Dataset Preprocessed path: /Users/runner/MLC/repos/local/cache/get-preprocessed-dataset-imagenet_8d80f184
+User Conf path: /Users/runner/MLC/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/013441842ddd4005924747f1c2aedc40.conf
+Dataset Preprocessed path: /Users/runner/MLC/repos/local/cache/get-preprocessed-dataset-imagenet_879c48b8
 Dataset List filepath: 
 Scenario: Offline
 Mode: AccuracyOnly

open/MLCommons/measurements/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline/mlc-deps.mmd

@@ -15,64 +15,72 @@ graph TD
     get-generic-sys-util,bb0393afa8404a11_(_md5sha1sum_) --> get,generic-sys-util,_install,_md5sha1sum
     download-file,9cdc8dc41aae437e_(_wget,_url.https://armi.in/files/caffe_ilsvrc12.tar.gz_) --> get,generic-sys-util,_md5sha1sum
     download-and-extract,c67e81a4ce2649f5_(_extract,_wget,_url.https://armi.in/files/caffe_ilsvrc12.tar.gz_) --> download,file,_wget,_url.https://armi.in/files/caffe_ilsvrc12.tar.gz
-    extract-file,3f0b76219d004817_(_keep,_path./Users/runner/MLC/repos/local/cache/download-file_imagenet-aux-da_b575c35c/caffe_ilsvrc12.tar.gz_) --> detect,os
-    extract-file,3f0b76219d004817_(_keep,_path./Users/runner/MLC/repos/local/cache/download-file_imagenet-aux-da_b575c35c/caffe_ilsvrc12.tar.gz_) --> get,generic-sys-util,_md5sha1sum
-    download-and-extract,c67e81a4ce2649f5_(_extract,_wget,_url.https://armi.in/files/caffe_ilsvrc12.tar.gz_) --> extract,file,_keep,_path./Users/runner/MLC/repos/local/cache/download-file_imagenet-aux-da_b575c35c/caffe_ilsvrc12.tar.gz
+    extract-file,3f0b76219d004817_(_keep,_path./Users/runner/MLC/repos/local/cache/download-file_imagenet-aux-da_aaa6ad52/caffe_ilsvrc12.tar.gz_) --> detect,os
+    extract-file,3f0b76219d004817_(_keep,_path./Users/runner/MLC/repos/local/cache/download-file_imagenet-aux-da_aaa6ad52/caffe_ilsvrc12.tar.gz_) --> get,generic-sys-util,_md5sha1sum
+    download-and-extract,c67e81a4ce2649f5_(_extract,_wget,_url.https://armi.in/files/caffe_ilsvrc12.tar.gz_) --> extract,file,_keep,_path./Users/runner/MLC/repos/local/cache/download-file_imagenet-aux-da_aaa6ad52/caffe_ilsvrc12.tar.gz
     get-dataset-imagenet-aux,bb2c6dd8c8c64217 --> download-and-extract,_extract,_wget,_url.https://armi.in/files/caffe_ilsvrc12.tar.gz
     app-mlperf-inference,d775cac873ee4231_(_cpp,_resnet50,_onnxruntime,_cpu,_test,_r5.1-dev_default,_offline_) --> get,dataset-aux,imagenet-aux
-    app-mlperf-inference-mlcommons-cpp,bf62405e6c7a44bf_(_resnet50,_cpu,_onnxruntime,_offline_) --> detect,os
-    app-mlperf-inference-mlcommons-cpp,bf62405e6c7a44bf_(_resnet50,_cpu,_onnxruntime,_offline_) --> detect,cpu
+    app-mlperf-inference-mlcommons-cpp,bf62405e6c7a44bf_(_resnet50,_onnxruntime,_offline,_cpu_) --> detect,os
+    detect-cpu,586c8a43320142f7 --> detect,os
+    app-mlperf-inference-mlcommons-cpp,bf62405e6c7a44bf_(_resnet50,_onnxruntime,_offline,_cpu_) --> detect,cpu
     get-sys-utils-mlc,bc90993277e84b8e --> detect,os
     get-mlperf-inference-loadgen,64c3d98d0ba04950_(_wg-inference_) --> detect,os
     get-mlperf-inference-loadgen,64c3d98d0ba04950_(_wg-inference_) --> get,python3
     get-mlperf-inference-loadgen,64c3d98d0ba04950_(_wg-inference_) --> get,mlcommons,inference,src
     get-mlperf-inference-loadgen,64c3d98d0ba04950_(_wg-inference_) --> get,compiler
+    detect-cpu,586c8a43320142f7 --> detect,os
     get-cmake,52bf974d791b4fc8 --> detect,cpu
     get-mlperf-inference-loadgen,64c3d98d0ba04950_(_wg-inference_) --> get,cmake
     get-generic-python-lib,94b62a682bc44791_(_package.wheel_) --> detect,os
+    detect-cpu,586c8a43320142f7 --> detect,os
     get-generic-python-lib,94b62a682bc44791_(_package.wheel_) --> detect,cpu
     get-generic-python-lib,94b62a682bc44791_(_package.wheel_) --> get,python3
     get-generic-python-lib,94b62a682bc44791_(_pip_) --> detect,os
+    detect-cpu,586c8a43320142f7 --> detect,os
     get-generic-python-lib,94b62a682bc44791_(_pip_) --> detect,cpu
     get-generic-python-lib,94b62a682bc44791_(_pip_) --> get,python3
     get-generic-python-lib,94b62a682bc44791_(_package.wheel_) --> get,generic-python-lib,_pip
     get-mlperf-inference-loadgen,64c3d98d0ba04950_(_wg-inference_) --> get,generic-python-lib,_package.wheel
     get-generic-python-lib,94b62a682bc44791_(_pip_) --> get,python3
     get-mlperf-inference-loadgen,64c3d98d0ba04950_(_wg-inference_) --> get,generic-python-lib,_pip
     get-generic-python-lib,94b62a682bc44791_(_package.pybind11_) --> detect,os
+    detect-cpu,586c8a43320142f7 --> detect,os
     get-generic-python-lib,94b62a682bc44791_(_package.pybind11_) --> detect,cpu
     get-generic-python-lib,94b62a682bc44791_(_package.pybind11_) --> get,python3
     get-generic-python-lib,94b62a682bc44791_(_pip_) --> get,python3
     get-generic-python-lib,94b62a682bc44791_(_package.pybind11_) --> get,generic-python-lib,_pip
     get-mlperf-inference-loadgen,64c3d98d0ba04950_(_wg-inference_) --> get,generic-python-lib,_package.pybind11
     get-generic-python-lib,94b62a682bc44791_(_package.setuptools_) --> detect,os
+    detect-cpu,586c8a43320142f7 --> detect,os
     get-generic-python-lib,94b62a682bc44791_(_package.setuptools_) --> detect,cpu
     get-generic-python-lib,94b62a682bc44791_(_package.setuptools_) --> get,python3
     get-generic-python-lib,94b62a682bc44791_(_pip_) --> get,python3
     get-generic-python-lib,94b62a682bc44791_(_package.setuptools_) --> get,generic-python-lib,_pip
     get-mlperf-inference-loadgen,64c3d98d0ba04950_(_wg-inference_) --> get,generic-python-lib,_package.setuptools
-    app-mlperf-inference-mlcommons-cpp,bf62405e6c7a44bf_(_resnet50,_cpu,_onnxruntime,_offline_) --> get,loadgen,_wg-inference
-    app-mlperf-inference-mlcommons-cpp,bf62405e6c7a44bf_(_resnet50,_cpu,_onnxruntime,_offline_) --> get,mlcommons,inference,src
+    app-mlperf-inference-mlcommons-cpp,bf62405e6c7a44bf_(_resnet50,_onnxruntime,_offline,_cpu_) --> get,loadgen,_wg-inference
+    app-mlperf-inference-mlcommons-cpp,bf62405e6c7a44bf_(_resnet50,_onnxruntime,_offline,_cpu_) --> get,mlcommons,inference,src
     get-onnxruntime-prebuilt,be02c84ff57c4244_(_cpu_) --> detect,os
-    app-mlperf-inference-mlcommons-cpp,bf62405e6c7a44bf_(_resnet50,_cpu,_onnxruntime,_offline_) --> get,lib,onnxruntime,lang-cpp,_cpu
+    app-mlperf-inference-mlcommons-cpp,bf62405e6c7a44bf_(_resnet50,_onnxruntime,_offline,_cpu_) --> get,lib,onnxruntime,lang-cpp,_cpu
     get-preprocessed-dataset-imagenet,f259d490bbaf45f5_(_NCHW_) --> get,python3
     get-dataset-imagenet-val,7afd58d287fe4f11_(_size.500,_2012_) --> detect,os
-    download-file,9cdc8dc41aae437e_(_mlcutil,_url.http://cKnowledge.org/ai/data/ILSVRC2012_img_val_500.tar_) --> detect,os
-    download-file,9cdc8dc41aae437e_(_mlcutil,_url.http://cKnowledge.org/ai/data/ILSVRC2012_img_val_500.tar_) --> get,generic-sys-util,_md5sha1sum
-    download-and-extract,c67e81a4ce2649f5_(_extract,_url.http://cKnowledge.org/ai/data/ILSVRC2012_img_val_500.tar_) --> download,file,_mlcutil,_url.http://cKnowledge.org/ai/data/ILSVRC2012_img_val_500.tar
-    extract-file,3f0b76219d004817_(_keep,_path./Users/runner/MLC/repos/local/cache/get-dataset-imagenet-val_06d478e1/ILSVRC2012_img_val_500.tar_) --> detect,os
-    extract-file,3f0b76219d004817_(_keep,_path./Users/runner/MLC/repos/local/cache/get-dataset-imagenet-val_06d478e1/ILSVRC2012_img_val_500.tar_) --> get,generic-sys-util,_md5sha1sum
-    download-and-extract,c67e81a4ce2649f5_(_extract,_url.http://cKnowledge.org/ai/data/ILSVRC2012_img_val_500.tar_) --> extract,file,_keep,_path./Users/runner/MLC/repos/local/cache/get-dataset-imagenet-val_06d478e1/ILSVRC2012_img_val_500.tar
+    download-file,9cdc8dc41aae437e_(_cmutil,_url.http://cKnowledge.org/ai/data/ILSVRC2012_img_val_500.tar_) --> detect,os
+    download-file,9cdc8dc41aae437e_(_cmutil,_url.http://cKnowledge.org/ai/data/ILSVRC2012_img_val_500.tar_) --> get,generic-sys-util,_md5sha1sum
+    download-and-extract,c67e81a4ce2649f5_(_extract,_url.http://cKnowledge.org/ai/data/ILSVRC2012_img_val_500.tar_) --> download,file,_cmutil,_url.http://cKnowledge.org/ai/data/ILSVRC2012_img_val_500.tar
+    extract-file,3f0b76219d004817_(_keep,_path./Users/runner/MLC/repos/local/cache/get-dataset-imagenet-val_f319208f/ILSVRC2012_img_val_500.tar_) --> detect,os
+    extract-file,3f0b76219d004817_(_keep,_path./Users/runner/MLC/repos/local/cache/get-dataset-imagenet-val_f319208f/ILSVRC2012_img_val_500.tar_) --> get,generic-sys-util,_md5sha1sum
+    download-and-extract,c67e81a4ce2649f5_(_extract,_url.http://cKnowledge.org/ai/data/ILSVRC2012_img_val_500.tar_) --> extract,file,_keep,_path./Users/runner/MLC/repos/local/cache/get-dataset-imagenet-val_f319208f/ILSVRC2012_img_val_500.tar
     get-dataset-imagenet-val,7afd58d287fe4f11_(_size.500,_2012_) --> download-and-extract,file,_extract,_url.http://cKnowledge.org/ai/data/ILSVRC2012_img_val_500.tar
     get-preprocessed-dataset-imagenet,f259d490bbaf45f5_(_NCHW_) --> get,dataset,image-classification,original,_size.500,_2012
     get-preprocessed-dataset-imagenet,f259d490bbaf45f5_(_NCHW_) --> get,dataset-aux,image-classification,imagenet-aux
     get-generic-python-lib,94b62a682bc44791_(_package.opencv-python-headless_) --> detect,os
+    detect-cpu,586c8a43320142f7 --> detect,os
     get-generic-python-lib,94b62a682bc44791_(_package.opencv-python-headless_) --> detect,cpu
     get-generic-python-lib,94b62a682bc44791_(_package.opencv-python-headless_) --> get,python3
     get-generic-python-lib,94b62a682bc44791_(_pip_) --> get,python3
     get-generic-python-lib,94b62a682bc44791_(_package.opencv-python-headless_) --> get,generic-python-lib,_pip
     get-preprocessed-dataset-imagenet,f259d490bbaf45f5_(_NCHW_) --> get,generic-python-lib,_package.opencv-python-headless
     get-generic-python-lib,94b62a682bc44791_(_pillow_) --> detect,os
+    detect-cpu,586c8a43320142f7 --> detect,os
     get-generic-python-lib,94b62a682bc44791_(_pillow_) --> detect,cpu
     get-generic-python-lib,94b62a682bc44791_(_pillow_) --> get,python3
     get-generic-python-lib,94b62a682bc44791_(_pip_) --> get,python3
@@ -85,24 +93,28 @@ graph TD
     get-generic-sys-util,bb0393afa8404a11_(_install,_libgl_) --> detect,sudo
     get-generic-sys-util,bb0393afa8404a11_(_libgl_) --> get,generic-sys-util,_install,_libgl
     get-preprocessed-dataset-imagenet,f259d490bbaf45f5_(_NCHW_) --> get,generic-sys-util,_libgl
-    app-mlperf-inference-mlcommons-cpp,bf62405e6c7a44bf_(_resnet50,_cpu,_onnxruntime,_offline_) --> get,dataset,preprocessed,imagenet,_NCHW
-    download-file,9cdc8dc41aae437e_(_mlcutil,_url.https://armi.in/files/resnet50_v1_op11/resnet50_v1.onnx_) --> detect,os
-    download-file,9cdc8dc41aae437e_(_mlcutil,_url.https://armi.in/files/resnet50_v1_op11/resnet50_v1.onnx_) --> get,generic-sys-util,_md5sha1sum
-    download-and-extract,c67e81a4ce2649f5_(_url.https://armi.in/files/resnet50_v1_op11/resnet50_v1.onnx_) --> download,file,_mlcutil,_url.https://armi.in/files/resnet50_v1_op11/resnet50_v1.onnx
+    app-mlperf-inference-mlcommons-cpp,bf62405e6c7a44bf_(_resnet50,_onnxruntime,_offline,_cpu_) --> get,dataset,preprocessed,imagenet,_NCHW
+    download-file,9cdc8dc41aae437e_(_cmutil,_url.https://armi.in/files/resnet50_v1_op11/resnet50_v1.onnx_) --> detect,os
+    download-file,9cdc8dc41aae437e_(_cmutil,_url.https://armi.in/files/resnet50_v1_op11/resnet50_v1.onnx_) --> get,generic-sys-util,_md5sha1sum
+    download-and-extract,c67e81a4ce2649f5_(_url.https://armi.in/files/resnet50_v1_op11/resnet50_v1.onnx_) --> download,file,_cmutil,_url.https://armi.in/files/resnet50_v1_op11/resnet50_v1.onnx
     get-ml-model-resnet50,56203e4e998b4bc0_(_onnx_) --> download-and-extract,_url.https://armi.in/files/resnet50_v1_op11/resnet50_v1.onnx
-    app-mlperf-inference-mlcommons-cpp,bf62405e6c7a44bf_(_resnet50,_cpu,_onnxruntime,_offline_) --> get,ml-model,raw,resnet50,_onnx
+    app-mlperf-inference-mlcommons-cpp,bf62405e6c7a44bf_(_resnet50,_onnxruntime,_offline,_cpu_) --> get,ml-model,raw,resnet50,_onnx
     generate-mlperf-inference-user-conf,3af4475745964b93_(_wg-inference_) --> detect,os
+    detect-cpu,586c8a43320142f7 --> detect,os
     generate-mlperf-inference-user-conf,3af4475745964b93_(_wg-inference_) --> detect,cpu
     generate-mlperf-inference-user-conf,3af4475745964b93_(_wg-inference_) --> get,python
     get-mlperf-inference-sut-configs,c2fbf72009e2445b --> get,cache,dir,_name.mlperf-inference-sut-configs
     generate-mlperf-inference-user-conf,3af4475745964b93_(_wg-inference_) --> get,sut,configs
     generate-mlperf-inference-user-conf,3af4475745964b93_(_wg-inference_) --> get,mlcommons,inference,src
-    app-mlperf-inference-mlcommons-cpp,bf62405e6c7a44bf_(_resnet50,_cpu,_onnxruntime,_offline_) --> generate,user-conf,mlperf,inference,_wg-inference
+    app-mlperf-inference-mlcommons-cpp,bf62405e6c7a44bf_(_resnet50,_onnxruntime,_offline,_cpu_) --> generate,user-conf,mlperf,inference,_wg-inference
+    detect-cpu,586c8a43320142f7 --> detect,os
     compile-program,c05042ba005a4bfa --> detect,cpu
     compile-program,c05042ba005a4bfa --> get,compiler
+    detect-cpu,586c8a43320142f7 --> detect,os
     get-compiler-flags,31be8b74a69742f8 --> detect,cpu
     compile-program,c05042ba005a4bfa --> get,compiler-flags
-    app-mlperf-inference-mlcommons-cpp,bf62405e6c7a44bf_(_resnet50,_cpu,_onnxruntime,_offline_) --> compile,cpp-program
+    app-mlperf-inference-mlcommons-cpp,bf62405e6c7a44bf_(_resnet50,_onnxruntime,_offline,_cpu_) --> compile,cpp-program
+    detect-cpu,586c8a43320142f7 --> detect,os
     benchmark-program,19f369ef47084895 --> detect,cpu
     benchmark-program-mlperf,cfff0132a8aa4018 --> benchmark-program,program
-    app-mlperf-inference-mlcommons-cpp,bf62405e6c7a44bf_(_resnet50,_cpu,_onnxruntime,_offline_) --> benchmark-mlperf
+    app-mlperf-inference-mlcommons-cpp,bf62405e6c7a44bf_(_resnet50,_onnxruntime,_offline,_cpu_) --> benchmark-mlperf