diff --git a/open/scc123-Shanxi_University/code/bert-99/README.md b/open/scc123-Shanxi_University/code/bert-99/README.md new file mode 100644 index 0000000..2fd9f95 --- /dev/null +++ b/open/scc123-Shanxi_University/code/bert-99/README.md @@ -0,0 +1 @@ +TBD \ No newline at end of file diff --git a/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/README.md b/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/README.md new file mode 100644 index 0000000..9ab3328 --- /dev/null +++ b/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/README.md @@ -0,0 +1,3 @@ +| Model | Scenario | Accuracy | Throughput | Latency (in ms) | +|---------|------------|------------|--------------|-------------------| +| bert-99 | offline | 90.876 | 83.471 | - | \ No newline at end of file diff --git a/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/README.md b/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/README.md new file mode 100644 index 0000000..73286ed --- /dev/null +++ b/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/README.md @@ -0,0 +1,54 @@ +This experiment is generated using the [MLCommons Collective Mind automation framework (CM)](https://github.com/mlcommons/cm4mlops). + +*Check [CM MLPerf docs](https://docs.mlcommons.org/inference) for more details.* + +## Host platform + +* OS version: Linux-6.1.110-1.el9.elrepo.x86_64-x86_64-with-glibc2.35 +* CPU version: x86_64 +* Python version: 3.10.12 (main, Sep 11 2024, 15:47:36) [GCC 11.4.0] +* MLCommons CM version: 3.2.6 + +## CM Run Command + +See [CM installation guide](https://docs.mlcommons.org/inference/install/). + +```bash +pip install -U cmind + +cm rm cache -f + +cm pull repo luoyueyuguang@cm4mlops --checkout=6dbb26a3da6b8ebdbc96be3be3a0e9817d3b6d26 + +cm run script \ + --tags=run-mlperf,inference,_r4.1-dev,_short \ + --model=bert-99 \ + --implementation=reference \ + --framework=pytorch \ + --category=edge \ + --scenario=Offline \ + --execution_mode=valid \ + --device=cuda \ + --quiet +``` +*Note that if you want to use the [latest automation recipes](https://docs.mlcommons.org/inference) for MLPerf (CM scripts), + you should simply reload luoyueyuguang@cm4mlops without checkout and clean CM cache as follows:* + +```bash +cm rm repo luoyueyuguang@cm4mlops +cm pull repo luoyueyuguang@cm4mlops +cm rm cache -f + +``` + +## Results + +Platform: b7828419c93b-reference-gpu-pytorch-cu124 + +Model Precision: fp32 + +### Accuracy Results +`F1`: `90.87602`, Required accuracy for closed division `>= 89.96526` + +### Performance Results +`Samples per second`: `83.4708` diff --git a/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/accuracy_console.out b/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/accuracy_console.out new file mode 100644 index 0000000..e69de29 diff --git a/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/b7828419c93b-reference-gpu-pytorch-cu124.json b/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/b7828419c93b-reference-gpu-pytorch-cu124.json new file mode 100644 index 0000000..46db96a --- /dev/null +++ b/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/b7828419c93b-reference-gpu-pytorch-cu124.json @@ -0,0 +1,7 @@ +{ + "starting_weights_filename": "https://armi.in/files/fp32/model.pytorch", + "retraining": "no", + "input_data_types": "fp32", + "weight_data_types": "fp32", + "weight_transformations": "none" +} \ No newline at end of file diff --git a/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/cm-version-info.json b/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/cm-version-info.json new file mode 100644 index 0000000..b09da21 --- /dev/null +++ b/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/cm-version-info.json @@ -0,0 +1,454 @@ +{ + "app-mlperf-inference,d775cac873ee4231:reference,bert-99,pytorch,cuda,valid,r4.1-dev_default,offline": [ + { + "detect,os": { + "script_uid": "863735b7db8c44fc", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_variations": "", + "version": "", + "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_bert-99,_pytorch,_cuda,_valid,_r4.1-dev_default,_offline )" + } + }, + { + "get,sys-utils-cm": { + "script_uid": "bc90993277e84b8e", + "script_alias": "get-sys-utils-cm", + "script_tags": "get,sys-utils-cm", + "script_variations": "", + "version": "", + "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_bert-99,_pytorch,_cuda,_valid,_r4.1-dev_default,_offline )" + } + }, + { + "get,python": { + "script_uid": "d0b5dd74373f4a62", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_variations": "", + "version": "3.10.12", + "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_bert-99,_pytorch,_cuda,_valid,_r4.1-dev_default,_offline )" + } + }, + { + "get,mlcommons,inference,src,_deeplearningexamples": { + "script_uid": "4b57186581024797", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_variations": "deeplearningexamples", + "version": "master-git-f74d16f54131d9080b9e45f234cc23e0ebaaf20c", + "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_bert-99,_pytorch,_cuda,_valid,_r4.1-dev_default,_offline )" + } + }, + { + "get,mlperf,inference,src,_deeplearningexamples": { + "script_uid": "4b57186581024797", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_variations": "deeplearningexamples", + "version": "master-git-f74d16f54131d9080b9e45f234cc23e0ebaaf20c", + "parent": "get-mlperf-inference-utils,e341e5f86d8342e5" + } + }, + { + "get,mlperf,inference,utils": { + "script_uid": "e341e5f86d8342e5", + "script_alias": "get-mlperf-inference-utils", + "script_tags": "get,mlperf,inference,util,utils,functions", + "script_variations": "", + "version": "", + "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_bert-99,_pytorch,_cuda,_valid,_r4.1-dev_default,_offline )" + } + }, + { + "get,cuda,_toolkit": { + "script_uid": "46d133d9ef92422d", + "script_alias": "get-cuda", + "script_tags": "get,cuda,cuda-compiler,cuda-lib,toolkit,lib,nvcc,get-nvcc,get-cuda,46d133d9ef92422d", + "script_variations": "toolkit", + "version": "12.4", + "parent": "get-cuda-devices,7a3ede4d3558427a ( with-pycuda )" + } + }, + { + "get,python3": { + "script_uid": "d0b5dd74373f4a62", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_variations": "", + "version": "3.10.12", + "parent": "get-cuda-devices,7a3ede4d3558427a ( with-pycuda )" + } + }, + { + "get,generic-python-lib,_package.pycuda": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "package.pycuda", + "version": "2024.1.2", + "parent": "get-cuda-devices,7a3ede4d3558427a ( with-pycuda )" + } + }, + { + "get,generic-python-lib,_package.numpy": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "package.numpy", + "version": "1.26.4", + "parent": "get-cuda-devices,7a3ede4d3558427a ( with-pycuda )" + } + }, + { + "get,cuda-devices,_with-pycuda": { + "script_uid": "7a3ede4d3558427a", + "script_alias": "get-cuda-devices", + "script_tags": "get,cuda-devices", + "script_variations": "with-pycuda", + "version": "", + "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_bert-99,_pytorch,_cuda,_valid,_r4.1-dev_default,_offline )" + } + }, + { + "get,dataset,squad,language-processing": { + "script_uid": "6651c119c3ae49b3", + "script_alias": "get-dataset-squad", + "script_tags": "get,dataset,squad,language-processing,validation,original", + "script_variations": "", + "version": "1.1", + "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_bert-99,_pytorch,_cuda,_valid,_r4.1-dev_default,_offline )" + } + }, + { + "get,dataset-aux,squad-vocab": { + "script_uid": "e38874fff5094577", + "script_alias": "get-dataset-squad-vocab", + "script_tags": "get,aux,squad,dataset-aux,language-processing,squad-aux,vocab,squad-vocab", + "script_variations": "", + "version": "", + "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_bert-99,_pytorch,_cuda,_valid,_r4.1-dev_default,_offline )" + } + }, + { + "detect,os": { + "script_uid": "863735b7db8c44fc", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_variations": "", + "version": "", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_bert-99,_pytorch,_cuda,_fp32 )" + } + }, + { + "detect,os": { + "script_uid": "863735b7db8c44fc", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_variations": "", + "version": "", + "parent": "detect-cpu,586c8a43320142f7" + } + }, + { + "detect,cpu": { + "script_uid": "586c8a43320142f7", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_variations": "", + "version": "", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_bert-99,_pytorch,_cuda,_fp32 )" + } + }, + { + "get,sys-utils-cm": { + "script_uid": "bc90993277e84b8e", + "script_alias": "get-sys-utils-cm", + "script_tags": "get,sys-utils-cm", + "script_variations": "", + "version": "", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_bert-99,_pytorch,_cuda,_fp32 )" + } + }, + { + "get,python": { + "script_uid": "d0b5dd74373f4a62", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_variations": "", + "version": "3.10.12", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_bert-99,_pytorch,_cuda,_fp32 )" + } + }, + { + "get,cuda,_cudnn": { + "script_uid": "46d133d9ef92422d", + "script_alias": "get-cuda", + "script_tags": "get,cuda,cuda-compiler,cuda-lib,toolkit,lib,nvcc,get-nvcc,get-cuda,46d133d9ef92422d", + "script_variations": "cudnn", + "version": "12.4", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_bert-99,_pytorch,_cuda,_fp32 )" + } + }, + { + "get,generic-python-lib,_torchvision_cuda": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "torchvision_cuda", + "version": "0.20.0", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_bert-99,_pytorch,_cuda,_fp32 )" + } + }, + { + "get,generic-python-lib,_transformers": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "transformers", + "version": "4.45.2", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_bert-99,_pytorch,_cuda,_fp32 )" + } + }, + { + "get,ml-model,language-processing,bert-large,raw,_fp32,_pytorch": { + "script_uid": "5e865dbdc65949d2", + "script_alias": "get-ml-model-bert-large-squad", + "script_tags": "get,ml-model,raw,bert,bert-large,bert-squad,language,language-processing", + "script_variations": "fp32,pytorch", + "version": "", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_bert-99,_pytorch,_cuda,_fp32 )" + } + }, + { + "get,dataset,squad,original": { + "script_uid": "6651c119c3ae49b3", + "script_alias": "get-dataset-squad", + "script_tags": "get,dataset,squad,language-processing,validation,original", + "script_variations": "", + "version": "1.1", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_bert-99,_pytorch,_cuda,_fp32 )" + } + }, + { + "get,dataset-aux,squad-vocab": { + "script_uid": "e38874fff5094577", + "script_alias": "get-dataset-squad-vocab", + "script_tags": "get,aux,squad,dataset-aux,language-processing,squad-aux,vocab,squad-vocab", + "script_variations": "", + "version": "", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_bert-99,_pytorch,_cuda,_fp32 )" + } + }, + { + "detect,os": { + "script_uid": "863735b7db8c44fc", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_variations": "", + "version": "", + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93" + } + }, + { + "detect,os": { + "script_uid": "863735b7db8c44fc", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_variations": "", + "version": "", + "parent": "detect-cpu,586c8a43320142f7" + } + }, + { + "detect,cpu": { + "script_uid": "586c8a43320142f7", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_variations": "", + "version": "", + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93" + } + }, + { + "get,python": { + "script_uid": "d0b5dd74373f4a62", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_variations": "", + "version": "3.10.12", + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93" + } + }, + { + "get,mlcommons,inference,src,_deeplearningexamples": { + "script_uid": "4b57186581024797", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_variations": "deeplearningexamples", + "version": "master-git-f74d16f54131d9080b9e45f234cc23e0ebaaf20c", + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93" + } + }, + { + "get,cache,dir,_name.mlperf-inference-sut-configs": { + "script_uid": "48f4622e059b45ce", + "script_alias": "get-cache-dir", + "script_tags": "get,cache,dir,directory", + "script_variations": "name.mlperf-inference-sut-configs", + "version": "", + "parent": "get-mlperf-inference-sut-configs,c2fbf72009e2445b" + } + }, + { + "get,sut,configs": { + "script_uid": "c2fbf72009e2445b", + "script_alias": "get-mlperf-inference-sut-configs", + "script_tags": "get,mlperf,inference,sut,configs,sut-configs", + "script_variations": "", + "version": "", + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93" + } + }, + { + "generate,user-conf,mlperf,inference": { + "script_uid": "3af4475745964b93", + "script_alias": "generate-mlperf-inference-user-conf", + "script_tags": "generate,mlperf,inference,user-conf,inference-user-conf", + "script_variations": "", + "version": "", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_bert-99,_pytorch,_cuda,_fp32 )" + } + }, + { + "get,loadgen": { + "script_uid": "64c3d98d0ba04950", + "script_alias": "get-mlperf-inference-loadgen", + "script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons", + "script_variations": "", + "version": "master", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_bert-99,_pytorch,_cuda,_fp32 )" + } + }, + { + "get,mlcommons,inference,src,_deeplearningexamples": { + "script_uid": "4b57186581024797", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_variations": "deeplearningexamples", + "version": "master-git-f74d16f54131d9080b9e45f234cc23e0ebaaf20c", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_bert-99,_pytorch,_cuda,_fp32 )" + } + }, + { + "get,mlcommons,inference,src": { + "script_uid": "4b57186581024797", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_variations": "", + "version": "master-git-f74d16f54131d9080b9e45f234cc23e0ebaaf20c", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_bert-99,_pytorch,_cuda,_fp32 )" + } + }, + { + "get,generic-python-lib,_package.psutil": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "package.psutil", + "version": "6.1.0", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_bert-99,_pytorch,_cuda,_fp32 )" + } + }, + { + "get,generic-python-lib,_package.pydantic": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "package.pydantic", + "version": "2.9.2", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_bert-99,_pytorch,_cuda,_fp32 )" + } + }, + { + "get,generic-python-lib,_tokenization": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "tokenization", + "version": "1.0.7", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_bert-99,_pytorch,_cuda,_fp32 )" + } + }, + { + "get,generic-python-lib,_six": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "six", + "version": "1.16.0", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_bert-99,_pytorch,_cuda,_fp32 )" + } + }, + { + "get,generic-python-lib,_package.absl-py": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "package.absl-py", + "version": "2.1.0", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_bert-99,_pytorch,_cuda,_fp32 )" + } + }, + { + "get,generic-python-lib,_boto3": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "boto3", + "version": "1.35.46", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_bert-99,_pytorch,_cuda,_fp32 )" + } + }, + { + "detect,os": { + "script_uid": "863735b7db8c44fc", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_variations": "", + "version": "", + "parent": "detect-cpu,586c8a43320142f7" + } + }, + { + "detect,cpu": { + "script_uid": "586c8a43320142f7", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_variations": "", + "version": "", + "parent": "benchmark-program,19f369ef47084895" + } + }, + { + "benchmark-program,program": { + "script_uid": "19f369ef47084895", + "script_alias": "benchmark-program", + "script_tags": "program,benchmark,benchmark-program", + "script_variations": "", + "version": "", + "parent": "benchmark-program-mlperf,cfff0132a8aa4018" + } + }, + { + "benchmark-mlperf": { + "script_uid": "cfff0132a8aa4018", + "script_alias": "benchmark-program-mlperf", + "script_tags": "mlperf,benchmark-mlperf", + "script_variations": "", + "version": "", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_bert-99,_pytorch,_cuda,_fp32 )" + } + } + ] +} \ No newline at end of file diff --git a/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/cpu_info.json b/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/cpu_info.json new file mode 100644 index 0000000..96f120e --- /dev/null +++ b/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/cpu_info.json @@ -0,0 +1,26 @@ +{ + "CM_HOST_CPU_WRITE_PROTECT_SUPPORT": "yes", + "CM_HOST_CPU_MICROCODE": "0x1000065", + "CM_HOST_CPU_FPU_SUPPORT": "yes", + "CM_HOST_CPU_FPU_EXCEPTION_SUPPORT": "yes", + "CM_HOST_CPU_BUGS": "sysret_ss_attrs null_seg spectre_v1 spectre_v2 spec_store_bypass srso", + "CM_HOST_CPU_TLB_SIZE": "1024 4K pages", + "CM_HOST_CPU_CFLUSH_SIZE": "64", + "CM_HOST_CPU_ARCHITECTURE": "x86_64", + "CM_HOST_CPU_TOTAL_CORES": "16", + "CM_HOST_CPU_ON_LINE_CPUS_LIST": "0-15", + "CM_HOST_CPU_VENDOR_ID": "AuthenticAMD", + "CM_HOST_CPU_MODEL_NAME": "AMD EPYC-Milan Processor", + "CM_HOST_CPU_FAMILY": "25", + "CM_HOST_CPU_THREADS_PER_CORE": "1", + "CM_HOST_CPU_PHYSICAL_CORES_PER_SOCKET": "1", + "CM_HOST_CPU_SOCKETS": "16", + "CM_HOST_CPU_L1D_CACHE_SIZE": "512 KiB (16 instances)", + "CM_HOST_CPU_L1I_CACHE_SIZE": "512 KiB (16 instances)", + "CM_HOST_CPU_L2_CACHE_SIZE": "8 MiB (16 instances)", + "CM_HOST_CPU_L3_CACHE_SIZE": "512 MiB (16 instances)", + "CM_HOST_CPU_NUMA_NODES": "1", + "CM_HOST_CPU_TOTAL_LOGICAL_CORES": "16", + "CM_HOST_MEMORY_CAPACITY": "60G", + "CM_HOST_DISK_CAPACITY": "234G" +} \ No newline at end of file diff --git a/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/mlperf.conf b/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/mlperf.conf new file mode 100644 index 0000000..10f7ae7 --- /dev/null +++ b/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/mlperf.conf @@ -0,0 +1,98 @@ +# The format of this config file is 'key = value'. +# The key has the format 'model.scenario.key'. Value is mostly int64_t. +# Model maybe '*' as wildcard. In that case the value applies to all models. +# All times are in milli seconds + +# Set performance_sample_count for each model. +# User can optionally set this to higher values in user.conf. +resnet50.*.performance_sample_count_override = 1024 +ssd-mobilenet.*.performance_sample_count_override = 256 +retinanet.*.performance_sample_count_override = 64 +bert.*.performance_sample_count_override = 10833 +dlrm.*.performance_sample_count_override = 204800 +dlrm-v2.*.performance_sample_count_override = 204800 +rnnt.*.performance_sample_count_override = 2513 +gptj.*.performance_sample_count_override = 13368 +llama2-70b.*.performance_sample_count_override = 24576 +stable-diffusion-xl.*.performance_sample_count_override = 5000 +# set to 0 to let entire sample set to be performance sample +3d-unet.*.performance_sample_count_override = 0 + +# Set seeds. The seeds will be distributed two weeks before the submission. +*.*.qsl_rng_seed = 3066443479025735752 +*.*.sample_index_rng_seed = 10688027786191513374 +*.*.schedule_rng_seed = 14962580496156340209 +# Set seeds for TEST_05. The seeds will be distributed two weeks before the submission. +*.*.test05_qsl_rng_seed = 16799458546791641818 +*.*.test05_sample_index_rng_seed = 5453809927556429288 +*.*.test05_schedule_rng_seed = 5435552105434836064 + + +*.SingleStream.target_latency_percentile = 90 +*.SingleStream.min_duration = 600000 + +*.MultiStream.target_latency_percentile = 99 +*.MultiStream.samples_per_query = 8 +*.MultiStream.min_duration = 600000 +*.MultiStream.min_query_count = 662 +retinanet.MultiStream.target_latency = 528 + +# 3D-UNet uses equal issue mode because it has non-uniform inputs +3d-unet.*.sample_concatenate_permutation = 1 + +# LLM benchmarks have non-uniform inputs and outputs, and use equal issue mode for all latency scenario +gptj.*.sample_concatenate_permutation = 1 +llama2-70b.*.sample_concatenate_permutation = 1 +mixtral-8x7b.*.sample_concatenate_permutation = 1 + +*.Server.target_latency = 10 +*.Server.target_latency_percentile = 99 +*.Server.target_duration = 0 +*.Server.min_duration = 600000 +resnet50.Server.target_latency = 15 +retinanet.Server.target_latency = 100 +bert.Server.target_latency = 130 +dlrm.Server.target_latency = 60 +dlrm-v2.Server.target_latency = 60 +rnnt.Server.target_latency = 1000 +gptj.Server.target_latency = 20000 +stable-diffusion-xl.Server.target_latency = 20000 +# Llama2-70b benchmarks measures token latencies +llama2-70b.*.use_token_latencies = 1 +mixtral-8x7b.*.use_token_latencies = 1 +# gptj benchmark infers token latencies +gptj.*.infer_token_latencies = 1 +gptj.*.token_latency_scaling_factor = 69 +# Only ttft and tpot are tracked for the llama2-70b & mixtral-8x7B benchmark therefore target_latency = 0 +llama2-70b.Server.target_latency = 0 +llama2-70b.Server.ttft_latency = 2000 +llama2-70b.Server.tpot_latency = 200 + +mixtral-8x7b.Server.target_latency = 0 +mixtral-8x7b.Server.ttft_latency = 2000 +mixtral-8x7b.Server.tpot_latency = 200 + +*.Offline.target_latency_percentile = 90 +*.Offline.min_duration = 600000 + +# In Offline scenario, we always have one query. But LoadGen maps this to +# min_sample_count internally in Offline scenario. If the dataset size is larger +# than 24576 we limit the min_query_count to 24576 and otherwise we use +# the dataset size as the limit + +resnet50.Offline.min_query_count = 24576 +retinanet.Offline.min_query_count = 24576 +dlrm-v2.Offline.min_query_count = 24576 +bert.Offline.min_query_count = 10833 +gptj.Offline.min_query_count = 13368 +rnnt.Offline.min_query_count = 2513 +3d-unet.Offline.min_query_count = 43 +stable-diffusion-xl.Offline.min_query_count = 5000 +llama2-70b.Offline.min_query_count = 24576 +mixtral-8x7b.Offline.min_query_count = 15000 + +# These fields should be defined and overridden by user.conf. +*.SingleStream.target_latency = 10 +*.MultiStream.target_latency = 80 +*.Server.target_qps = 1.0 +*.Offline.target_qps = 1.0 diff --git a/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/os_info.json b/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/os_info.json new file mode 100644 index 0000000..7f461b2 --- /dev/null +++ b/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/os_info.json @@ -0,0 +1,30 @@ +{ + "CM_HOST_OS_TYPE": "linux", + "CM_HOST_OS_BITS": "64", + "CM_HOST_OS_FLAVOR": "ubuntu", + "CM_HOST_OS_FLAVOR_LIKE": "debian", + "CM_HOST_OS_VERSION": "22.04", + "CM_HOST_OS_KERNEL_VERSION": "6.1.110-1.el9.elrepo.x86_64", + "CM_HOST_OS_GLIBC_VERSION": "2.35", + "CM_HOST_OS_MACHINE": "x86_64", + "CM_HOST_OS_PACKAGE_MANAGER": "apt", + "CM_HOST_OS_PACKAGE_MANAGER_INSTALL_CMD": "DEBIAN_FRONTEND=noninteractive apt-get install -y", + "CM_HOST_OS_PACKAGE_MANAGER_UPDATE_CMD": "apt-get update -y", + "+CM_HOST_OS_DEFAULT_LIBRARY_PATH": [ + "/usr/local/lib/x86_64-linux-gnu", + "/lib/x86_64-linux-gnu", + "/usr/lib/x86_64-linux-gnu", + "/usr/lib/x86_64-linux-gnu64", + "/usr/local/lib64", + "/lib64", + "/usr/lib64", + "/usr/local/lib", + "/lib", + "/usr/lib", + "/usr/x86_64-linux-gnu/lib64", + "/usr/x86_64-linux-gnu/lib" + ], + "CM_HOST_PLATFORM_FLAVOR": "x86_64", + "CM_HOST_PYTHON_BITS": "64", + "CM_HOST_SYSTEM_NAME": "b7828419c93b" +} \ No newline at end of file diff --git a/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/performance_console.out b/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/performance_console.out new file mode 100644 index 0000000..e69de29 diff --git a/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/pip_freeze.json b/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/pip_freeze.json new file mode 100644 index 0000000..2639c27 --- /dev/null +++ b/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/pip_freeze.json @@ -0,0 +1,69 @@ +{ + "pip_freeze": { + "absl-py": "2.1.0", + "annotated-types": "0.7.0", + "boto3": "1.35.46", + "botocore": "1.35.46", + "certifi": "2024.8.30", + "charset-normalizer": "3.4.0", + "cmind": "3.2.6", + "dmiparser": "5.1", + "filelock": "3.16.1", + "fsspec": "2024.10.0", + "giturlparse": "0.12.0", + "huggingface-hub": "0.26.1", + "idna": "3.10", + "Jinja2": "3.1.4", + "jmespath": "1.0.1", + "Mako": "1.3.6", + "MarkupSafe": "3.0.2", + "mpmath": "1.3.0", + "networkx": "3.4.2", + "numpy": "1.26.4", + "nvidia-cublas-cu12": "12.4.5.8", + "nvidia-cuda-cupti-cu12": "12.4.127", + "nvidia-cuda-nvrtc-cu12": "12.4.127", + "nvidia-cuda-runtime-cu12": "12.4.127", + "nvidia-cudnn-cu12": "9.1.0.70", + "nvidia-cufft-cu12": "11.2.1.3", + "nvidia-curand-cu12": "10.3.5.147", + "nvidia-cusolver-cu12": "11.6.1.9", + "nvidia-cusparse-cu12": "12.3.1.170", + "nvidia-nccl-cu12": "2.21.5", + "nvidia-nvjitlink-cu12": "12.4.127", + "nvidia-nvtx-cu12": "12.4.127", + "packaging": "24.1", + "pandas": "2.2.3", + "pillow": "11.0.0", + "pip": "22.0.2", + "platformdirs": "4.3.6", + "psutil": "6.1.0", + "pybind11": "2.13.6", + "pycuda": "2024.1.2", + "pydantic": "2.9.2", + "pydantic_core": "2.23.4", + "python-dateutil": "2.9.0.post0", + "pytools": "2024.1.14", + "pytz": "2024.2", + "PyYAML": "6.0.2", + "regex": "2024.9.11", + "requests": "2.32.3", + "s3transfer": "0.10.3", + "safetensors": "0.4.5", + "setuptools": "59.6.0", + "six": "1.16.0", + "sympy": "1.13.1", + "tabulate": "0.9.0", + "tokenization": "1.0.7", + "tokenizers": "0.20.1", + "torch": "2.5.0", + "torchvision": "0.20.0", + "tqdm": "4.66.5", + "transformers": "4.45.2", + "triton": "3.1.0", + "typing_extensions": "4.12.2", + "tzdata": "2024.2", + "urllib3": "2.2.3", + "wheel": "0.44.0" + } +} \ No newline at end of file diff --git a/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/user.conf b/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/user.conf new file mode 100644 index 0000000..6cc24f8 --- /dev/null +++ b/open/scc123-Shanxi_University/measurements/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/user.conf @@ -0,0 +1 @@ +bert.Offline.target_qps = 84.305508 diff --git a/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/README.md b/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/README.md new file mode 100644 index 0000000..429e13e --- /dev/null +++ b/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/README.md @@ -0,0 +1,3 @@ +| Model | Scenario | Accuracy | Throughput | Latency (in ms) | +|---------|------------|------------|--------------|-------------------| +| bert-99 | offline | 90.8749 | 3.269 | - | \ No newline at end of file diff --git a/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/README.md b/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/README.md new file mode 100644 index 0000000..a1ac7e9 --- /dev/null +++ b/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/README.md @@ -0,0 +1,55 @@ +This experiment is generated using the [MLCommons Collective Mind automation framework (CM)](https://github.com/mlcommons/cm4mlops). + +*Check [CM MLPerf docs](https://docs.mlcommons.org/inference) for more details.* + +## Host platform + +* OS version: Linux-6.1.110-1.el9.elrepo.x86_64-x86_64-with-glibc2.34 +* CPU version: x86_64 +* Python version: 3.9.18 (main, Aug 23 2024, 00:00:00) +[GCC 11.4.1 20231218 (Red Hat 11.4.1-3)] +* MLCommons CM version: 3.2.6 + +## CM Run Command + +See [CM installation guide](https://docs.mlcommons.org/inference/install/). + +```bash +pip install -U cmind + +cm rm cache -f + +cm pull repo mlcommons@cm4mlops --checkout=6dbb26a3da6b8ebdbc96be3be3a0e9817d3b6d26 + +cm run script \ + --tags=run-mlperf,inference,_r4.1-dev \ + --model=bert-99 \ + --implementation=reference \ + --framework=pytorch \ + --category=edge \ + --scenario=Offline \ + --execution_mode=valid \ + --device=cpu \ + --quiet +``` +*Note that if you want to use the [latest automation recipes](https://docs.mlcommons.org/inference) for MLPerf (CM scripts), + you should simply reload mlcommons@cm4mlops without checkout and clean CM cache as follows:* + +```bash +cm rm repo mlcommons@cm4mlops +cm pull repo mlcommons@cm4mlops +cm rm cache -f + +``` + +## Results + +Platform: scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config + +Model Precision: fp32 + +### Accuracy Results +`F1`: `90.87487`, Required accuracy for closed division `>= 89.96526` + +### Performance Results +`Samples per second`: `3.26888` diff --git a/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy_console.out b/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy_console.out new file mode 100644 index 0000000..e69de29 diff --git a/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/cm-version-info.json b/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/cm-version-info.json new file mode 100644 index 0000000..4b4e177 --- /dev/null +++ b/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/cm-version-info.json @@ -0,0 +1,414 @@ +{ + "app-mlperf-inference,d775cac873ee4231:reference,bert-99,pytorch,cpu,valid,r4.1-dev_default,offline": [ + { + "detect,os": { + "script_uid": "863735b7db8c44fc", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_variations": "", + "version": "", + "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_bert-99,_pytorch,_cpu,_valid,_r4.1-dev_default,_offline )" + } + }, + { + "detect,os": { + "script_uid": "863735b7db8c44fc", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_variations": "", + "version": "", + "parent": "get-sys-utils-cm,bc90993277e84b8e" + } + }, + { + "get,python": { + "script_uid": "d0b5dd74373f4a62", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_variations": "", + "version": "3.9.18", + "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_bert-99,_pytorch,_cpu,_valid,_r4.1-dev_default,_offline )" + } + }, + { + "get,mlcommons,inference,src,_deeplearningexamples": { + "script_uid": "4b57186581024797", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_variations": "deeplearningexamples", + "version": "master-git-f74d16f54131d9080b9e45f234cc23e0ebaaf20c", + "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_bert-99,_pytorch,_cpu,_valid,_r4.1-dev_default,_offline )" + } + }, + { + "get,mlperf,inference,src,_deeplearningexamples": { + "script_uid": "4b57186581024797", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_variations": "deeplearningexamples", + "version": "master-git-f74d16f54131d9080b9e45f234cc23e0ebaaf20c", + "parent": "get-mlperf-inference-utils,e341e5f86d8342e5" + } + }, + { + "get,mlperf,inference,utils": { + "script_uid": "e341e5f86d8342e5", + "script_alias": "get-mlperf-inference-utils", + "script_tags": "get,mlperf,inference,util,utils,functions", + "script_variations": "", + "version": "", + "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_bert-99,_pytorch,_cpu,_valid,_r4.1-dev_default,_offline )" + } + }, + { + "get,dataset,squad,language-processing": { + "script_uid": "6651c119c3ae49b3", + "script_alias": "get-dataset-squad", + "script_tags": "get,dataset,squad,language-processing,validation,original", + "script_variations": "", + "version": "1.1", + "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_bert-99,_pytorch,_cpu,_valid,_r4.1-dev_default,_offline )" + } + }, + { + "get,dataset-aux,squad-vocab": { + "script_uid": "e38874fff5094577", + "script_alias": "get-dataset-squad-vocab", + "script_tags": "get,aux,squad,dataset-aux,language-processing,squad-aux,vocab,squad-vocab", + "script_variations": "", + "version": "", + "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_bert-99,_pytorch,_cpu,_valid,_r4.1-dev_default,_offline )" + } + }, + { + "detect,os": { + "script_uid": "863735b7db8c44fc", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_variations": "", + "version": "", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_cpu,_bert-99,_offline,_fp32 )" + } + }, + { + "detect,os": { + "script_uid": "863735b7db8c44fc", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_variations": "", + "version": "", + "parent": "detect-cpu,586c8a43320142f7" + } + }, + { + "detect,cpu": { + "script_uid": "586c8a43320142f7", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_variations": "", + "version": "", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_cpu,_bert-99,_offline,_fp32 )" + } + }, + { + "detect,os": { + "script_uid": "863735b7db8c44fc", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_variations": "", + "version": "", + "parent": "get-sys-utils-cm,bc90993277e84b8e" + } + }, + { + "get,python": { + "script_uid": "d0b5dd74373f4a62", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_variations": "", + "version": "3.9.18", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_cpu,_bert-99,_offline,_fp32 )" + } + }, + { + "get,generic-python-lib,_torch": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "torch", + "version": "2.5.0", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_cpu,_bert-99,_offline,_fp32 )" + } + }, + { + "get,generic-python-lib,_torch": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "torch", + "version": "2.5.0", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_cpu,_bert-99,_offline,_fp32 )" + } + }, + { + "get,generic-python-lib,_torchvision": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "torchvision", + "version": "0.20.0", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_cpu,_bert-99,_offline,_fp32 )" + } + }, + { + "get,generic-python-lib,_transformers": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "transformers", + "version": "4.45.2", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_cpu,_bert-99,_offline,_fp32 )" + } + }, + { + "get,ml-model,language-processing,bert-large,raw,_pytorch,_fp32": { + "script_uid": "5e865dbdc65949d2", + "script_alias": "get-ml-model-bert-large-squad", + "script_tags": "get,ml-model,raw,bert,bert-large,bert-squad,language,language-processing", + "script_variations": "pytorch,fp32", + "version": "", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_cpu,_bert-99,_offline,_fp32 )" + } + }, + { + "get,dataset,squad,original": { + "script_uid": "6651c119c3ae49b3", + "script_alias": "get-dataset-squad", + "script_tags": "get,dataset,squad,language-processing,validation,original", + "script_variations": "", + "version": "1.1", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_cpu,_bert-99,_offline,_fp32 )" + } + }, + { + "get,dataset-aux,squad-vocab": { + "script_uid": "e38874fff5094577", + "script_alias": "get-dataset-squad-vocab", + "script_tags": "get,aux,squad,dataset-aux,language-processing,squad-aux,vocab,squad-vocab", + "script_variations": "", + "version": "", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_cpu,_bert-99,_offline,_fp32 )" + } + }, + { + "detect,os": { + "script_uid": "863735b7db8c44fc", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_variations": "", + "version": "", + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93" + } + }, + { + "detect,os": { + "script_uid": "863735b7db8c44fc", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_variations": "", + "version": "", + "parent": "detect-cpu,586c8a43320142f7" + } + }, + { + "detect,cpu": { + "script_uid": "586c8a43320142f7", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_variations": "", + "version": "", + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93" + } + }, + { + "get,python": { + "script_uid": "d0b5dd74373f4a62", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_variations": "", + "version": "3.9.18", + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93" + } + }, + { + "get,mlcommons,inference,src,_deeplearningexamples": { + "script_uid": "4b57186581024797", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_variations": "deeplearningexamples", + "version": "master-git-f74d16f54131d9080b9e45f234cc23e0ebaaf20c", + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93" + } + }, + { + "get,cache,dir,_name.mlperf-inference-sut-configs": { + "script_uid": "48f4622e059b45ce", + "script_alias": "get-cache-dir", + "script_tags": "get,cache,dir,directory", + "script_variations": "name.mlperf-inference-sut-configs", + "version": "", + "parent": "get-mlperf-inference-sut-configs,c2fbf72009e2445b" + } + }, + { + "get,sut,configs": { + "script_uid": "c2fbf72009e2445b", + "script_alias": "get-mlperf-inference-sut-configs", + "script_tags": "get,mlperf,inference,sut,configs,sut-configs", + "script_variations": "", + "version": "", + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93" + } + }, + { + "generate,user-conf,mlperf,inference": { + "script_uid": "3af4475745964b93", + "script_alias": "generate-mlperf-inference-user-conf", + "script_tags": "generate,mlperf,inference,user-conf,inference-user-conf", + "script_variations": "", + "version": "", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_cpu,_bert-99,_offline,_fp32 )" + } + }, + { + "get,loadgen": { + "script_uid": "64c3d98d0ba04950", + "script_alias": "get-mlperf-inference-loadgen", + "script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons", + "script_variations": "", + "version": "master", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_cpu,_bert-99,_offline,_fp32 )" + } + }, + { + "get,mlcommons,inference,src,_deeplearningexamples": { + "script_uid": "4b57186581024797", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_variations": "deeplearningexamples", + "version": "master-git-f74d16f54131d9080b9e45f234cc23e0ebaaf20c", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_cpu,_bert-99,_offline,_fp32 )" + } + }, + { + "get,mlcommons,inference,src": { + "script_uid": "4b57186581024797", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_variations": "", + "version": "master-git-f74d16f54131d9080b9e45f234cc23e0ebaaf20c", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_cpu,_bert-99,_offline,_fp32 )" + } + }, + { + "get,generic-python-lib,_package.psutil": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "package.psutil", + "version": "6.1.0", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_cpu,_bert-99,_offline,_fp32 )" + } + }, + { + "get,generic-python-lib,_package.pydantic": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "package.pydantic", + "version": "2.9.2", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_cpu,_bert-99,_offline,_fp32 )" + } + }, + { + "get,generic-python-lib,_tokenization": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "tokenization", + "version": "1.0.7", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_cpu,_bert-99,_offline,_fp32 )" + } + }, + { + "get,generic-python-lib,_six": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "six", + "version": "1.16.0", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_cpu,_bert-99,_offline,_fp32 )" + } + }, + { + "get,generic-python-lib,_package.absl-py": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "package.absl-py", + "version": "2.1.0", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_cpu,_bert-99,_offline,_fp32 )" + } + }, + { + "get,generic-python-lib,_boto3": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "boto3", + "version": "1.35.46", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_cpu,_bert-99,_offline,_fp32 )" + } + }, + { + "detect,os": { + "script_uid": "863735b7db8c44fc", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_variations": "", + "version": "", + "parent": "detect-cpu,586c8a43320142f7" + } + }, + { + "detect,cpu": { + "script_uid": "586c8a43320142f7", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_variations": "", + "version": "", + "parent": "benchmark-program,19f369ef47084895" + } + }, + { + "benchmark-program,program": { + "script_uid": "19f369ef47084895", + "script_alias": "benchmark-program", + "script_tags": "program,benchmark,benchmark-program", + "script_variations": "", + "version": "", + "parent": "benchmark-program-mlperf,cfff0132a8aa4018" + } + }, + { + "benchmark-mlperf": { + "script_uid": "cfff0132a8aa4018", + "script_alias": "benchmark-program-mlperf", + "script_tags": "mlperf,benchmark-mlperf", + "script_variations": "", + "version": "", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_cpu,_bert-99,_offline,_fp32 )" + } + } + ] +} \ No newline at end of file diff --git a/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/cpu_info.json b/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/cpu_info.json new file mode 100644 index 0000000..62eef30 --- /dev/null +++ b/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/cpu_info.json @@ -0,0 +1,26 @@ +{ + "CM_HOST_CPU_WRITE_PROTECT_SUPPORT": "yes", + "CM_HOST_CPU_MICROCODE": "0x1000065", + "CM_HOST_CPU_FPU_SUPPORT": "yes", + "CM_HOST_CPU_FPU_EXCEPTION_SUPPORT": "yes", + "CM_HOST_CPU_BUGS": "sysret_ss_attrs null_seg spectre_v1 spectre_v2 spec_store_bypass srso", + "CM_HOST_CPU_TLB_SIZE": "1024 4K pages", + "CM_HOST_CPU_CFLUSH_SIZE": "64", + "CM_HOST_CPU_ARCHITECTURE": "x86_64", + "CM_HOST_CPU_TOTAL_CORES": "32", + "CM_HOST_CPU_ON_LINE_CPUS_LIST": "0-31", + "CM_HOST_CPU_VENDOR_ID": "AuthenticAMD", + "CM_HOST_CPU_MODEL_NAME": "AMD EPYC-Milan Processor", + "CM_HOST_CPU_FAMILY": "25", + "CM_HOST_CPU_THREADS_PER_CORE": "1", + "CM_HOST_CPU_PHYSICAL_CORES_PER_SOCKET": "1", + "CM_HOST_CPU_SOCKETS": "32", + "CM_HOST_CPU_L1D_CACHE_SIZE": "1 MiB (32 instances)", + "CM_HOST_CPU_L1I_CACHE_SIZE": "1 MiB (32 instances)", + "CM_HOST_CPU_L2_CACHE_SIZE": "16 MiB (32 instances)", + "CM_HOST_CPU_L3_CACHE_SIZE": "1 GiB (32 instances)", + "CM_HOST_CPU_NUMA_NODES": "1", + "CM_HOST_CPU_TOTAL_LOGICAL_CORES": "32", + "CM_HOST_MEMORY_CAPACITY": "125G", + "CM_HOST_DISK_CAPACITY": "158G" +} \ No newline at end of file diff --git a/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/mlperf.conf b/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/mlperf.conf new file mode 100644 index 0000000..10f7ae7 --- /dev/null +++ b/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/mlperf.conf @@ -0,0 +1,98 @@ +# The format of this config file is 'key = value'. +# The key has the format 'model.scenario.key'. Value is mostly int64_t. +# Model maybe '*' as wildcard. In that case the value applies to all models. +# All times are in milli seconds + +# Set performance_sample_count for each model. +# User can optionally set this to higher values in user.conf. +resnet50.*.performance_sample_count_override = 1024 +ssd-mobilenet.*.performance_sample_count_override = 256 +retinanet.*.performance_sample_count_override = 64 +bert.*.performance_sample_count_override = 10833 +dlrm.*.performance_sample_count_override = 204800 +dlrm-v2.*.performance_sample_count_override = 204800 +rnnt.*.performance_sample_count_override = 2513 +gptj.*.performance_sample_count_override = 13368 +llama2-70b.*.performance_sample_count_override = 24576 +stable-diffusion-xl.*.performance_sample_count_override = 5000 +# set to 0 to let entire sample set to be performance sample +3d-unet.*.performance_sample_count_override = 0 + +# Set seeds. The seeds will be distributed two weeks before the submission. +*.*.qsl_rng_seed = 3066443479025735752 +*.*.sample_index_rng_seed = 10688027786191513374 +*.*.schedule_rng_seed = 14962580496156340209 +# Set seeds for TEST_05. The seeds will be distributed two weeks before the submission. +*.*.test05_qsl_rng_seed = 16799458546791641818 +*.*.test05_sample_index_rng_seed = 5453809927556429288 +*.*.test05_schedule_rng_seed = 5435552105434836064 + + +*.SingleStream.target_latency_percentile = 90 +*.SingleStream.min_duration = 600000 + +*.MultiStream.target_latency_percentile = 99 +*.MultiStream.samples_per_query = 8 +*.MultiStream.min_duration = 600000 +*.MultiStream.min_query_count = 662 +retinanet.MultiStream.target_latency = 528 + +# 3D-UNet uses equal issue mode because it has non-uniform inputs +3d-unet.*.sample_concatenate_permutation = 1 + +# LLM benchmarks have non-uniform inputs and outputs, and use equal issue mode for all latency scenario +gptj.*.sample_concatenate_permutation = 1 +llama2-70b.*.sample_concatenate_permutation = 1 +mixtral-8x7b.*.sample_concatenate_permutation = 1 + +*.Server.target_latency = 10 +*.Server.target_latency_percentile = 99 +*.Server.target_duration = 0 +*.Server.min_duration = 600000 +resnet50.Server.target_latency = 15 +retinanet.Server.target_latency = 100 +bert.Server.target_latency = 130 +dlrm.Server.target_latency = 60 +dlrm-v2.Server.target_latency = 60 +rnnt.Server.target_latency = 1000 +gptj.Server.target_latency = 20000 +stable-diffusion-xl.Server.target_latency = 20000 +# Llama2-70b benchmarks measures token latencies +llama2-70b.*.use_token_latencies = 1 +mixtral-8x7b.*.use_token_latencies = 1 +# gptj benchmark infers token latencies +gptj.*.infer_token_latencies = 1 +gptj.*.token_latency_scaling_factor = 69 +# Only ttft and tpot are tracked for the llama2-70b & mixtral-8x7B benchmark therefore target_latency = 0 +llama2-70b.Server.target_latency = 0 +llama2-70b.Server.ttft_latency = 2000 +llama2-70b.Server.tpot_latency = 200 + +mixtral-8x7b.Server.target_latency = 0 +mixtral-8x7b.Server.ttft_latency = 2000 +mixtral-8x7b.Server.tpot_latency = 200 + +*.Offline.target_latency_percentile = 90 +*.Offline.min_duration = 600000 + +# In Offline scenario, we always have one query. But LoadGen maps this to +# min_sample_count internally in Offline scenario. If the dataset size is larger +# than 24576 we limit the min_query_count to 24576 and otherwise we use +# the dataset size as the limit + +resnet50.Offline.min_query_count = 24576 +retinanet.Offline.min_query_count = 24576 +dlrm-v2.Offline.min_query_count = 24576 +bert.Offline.min_query_count = 10833 +gptj.Offline.min_query_count = 13368 +rnnt.Offline.min_query_count = 2513 +3d-unet.Offline.min_query_count = 43 +stable-diffusion-xl.Offline.min_query_count = 5000 +llama2-70b.Offline.min_query_count = 24576 +mixtral-8x7b.Offline.min_query_count = 15000 + +# These fields should be defined and overridden by user.conf. +*.SingleStream.target_latency = 10 +*.MultiStream.target_latency = 80 +*.Server.target_qps = 1.0 +*.Offline.target_qps = 1.0 diff --git a/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/os_info.json b/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/os_info.json new file mode 100644 index 0000000..260a848 --- /dev/null +++ b/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/os_info.json @@ -0,0 +1,26 @@ +{ + "CM_HOST_OS_TYPE": "linux", + "CM_HOST_OS_BITS": "64", + "CM_HOST_OS_FLAVOR": "rocky", + "CM_HOST_OS_FLAVOR_LIKE": "rhel centos fedora", + "CM_HOST_OS_VERSION": "9.4", + "CM_HOST_OS_KERNEL_VERSION": "6.1.110-1.el9.elrepo.x86_64", + "CM_HOST_OS_GLIBC_VERSION": "2.34", + "CM_HOST_OS_MACHINE": "x86_64", + "CM_HOST_OS_PACKAGE_MANAGER": "dnf", + "CM_HOST_OS_PACKAGE_MANAGER_INSTALL_CMD": "dnf install -y", + "CM_HOST_OS_PACKAGE_MANAGER_UPDATE_CMD": "dnf update -y", + "+CM_HOST_OS_DEFAULT_LIBRARY_PATH": [ + "/usr/x86_64-redhat-linux/lib64", + "/usr/lib64", + "/usr/local/lib64", + "/lib64", + "/usr/x86_64-redhat-linux/lib", + "/usr/local/lib", + "/lib", + "/usr/lib" + ], + "CM_HOST_PLATFORM_FLAVOR": "x86_64", + "CM_HOST_PYTHON_BITS": "64", + "CM_HOST_SYSTEM_NAME": "scc123-cpu0.novalocal" +} \ No newline at end of file diff --git a/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/performance_console.out b/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/performance_console.out new file mode 100644 index 0000000..e69de29 diff --git a/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/pip_freeze.json b/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/pip_freeze.json new file mode 100644 index 0000000..283947f --- /dev/null +++ b/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/pip_freeze.json @@ -0,0 +1,46 @@ +{ + "pip_freeze": { + "absl-py": "2.1.0", + "annotated-types": "0.7.0", + "boto3": "1.35.46", + "botocore": "1.35.46", + "certifi": "2024.8.30", + "charset-normalizer": "3.4.0", + "dmiparser": "5.1", + "filelock": "3.16.1", + "fsspec": "2024.10.0", + "huggingface-hub": "0.26.1", + "idna": "3.10", + "Jinja2": "3.1.4", + "jmespath": "1.0.1", + "MarkupSafe": "3.0.2", + "mpmath": "1.3.0", + "networkx": "3.2.1", + "numpy": "2.0.2", + "packaging": "24.1", + "pillow": "11.0.0", + "pip": "21.2.3", + "psutil": "6.1.0", + "pybind11": "2.13.6", + "pydantic": "2.9.2", + "pydantic_core": "2.23.4", + "python-dateutil": "2.9.0.post0", + "PyYAML": "6.0.2", + "regex": "2024.9.11", + "requests": "2.32.3", + "s3transfer": "0.10.3", + "safetensors": "0.4.5", + "setuptools": "53.0.0", + "six": "1.16.0", + "sympy": "1.13.1", + "tokenization": "1.0.7", + "tokenizers": "0.20.1", + "torch": "2.5.0+cpu", + "torchvision": "0.20.0+cpu", + "tqdm": "4.66.5", + "transformers": "4.45.2", + "typing_extensions": "4.12.2", + "urllib3": "1.26.20", + "wheel": "0.44.0" + } +} \ No newline at end of file diff --git a/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config.json b/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config.json new file mode 100644 index 0000000..46db96a --- /dev/null +++ b/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config.json @@ -0,0 +1,7 @@ +{ + "starting_weights_filename": "https://armi.in/files/fp32/model.pytorch", + "retraining": "no", + "input_data_types": "fp32", + "weight_data_types": "fp32", + "weight_transformations": "none" +} \ No newline at end of file diff --git a/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/user.conf b/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/user.conf new file mode 100644 index 0000000..3febce9 --- /dev/null +++ b/open/scc123-Shanxi_University/measurements/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/user.conf @@ -0,0 +1 @@ +bert.Offline.target_qps = 3.3015687999999996 diff --git a/open/scc123-Shanxi_University/model_mapping.json b/open/scc123-Shanxi_University/model_mapping.json new file mode 100644 index 0000000..0f13f35 --- /dev/null +++ b/open/scc123-Shanxi_University/model_mapping.json @@ -0,0 +1,3 @@ +{ + "bert-99": "bert-99" +} \ No newline at end of file diff --git a/open/scc123-Shanxi_University/results/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/accuracy/accuracy.txt b/open/scc123-Shanxi_University/results/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/accuracy/accuracy.txt new file mode 100644 index 0000000..1e470f7 --- /dev/null +++ b/open/scc123-Shanxi_University/results/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/accuracy/accuracy.txt @@ -0,0 +1,8 @@ +{"exact_match": 83.68968779564806, "f1": 90.87601905331469} +Reading examples... +Loading cached features from '/home/cmuser/CM/repos/local/cache/3171bd6f0a5b4bed/inference/language/bert/eval_features.pickle'... +Loading LoadGen logs... +Post-processing predictions... +Writing predictions to: /home/cmuser/CM/repos/local/cache/c4569818cd2c4937/valid_results/b7828419c93b-reference-gpu-pytorch-vdefault-cu124/bert-99/offline/accuracy/predictions.json +Evaluating predictions... +hash=0cd6156dd0b12e65913f6d05744f6a8fdf90d5989dd59054d9daf7ccd76dac64 diff --git a/open/scc123-Shanxi_University/results/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/accuracy/mlperf_log_accuracy.json b/open/scc123-Shanxi_University/results/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/accuracy/mlperf_log_accuracy.json new file mode 100644 index 0000000..08cd474 --- /dev/null +++ b/open/scc123-Shanxi_University/results/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/accuracy/mlperf_log_accuracy.json @@ -0,0 +1,7 @@ +[ +{ "seq_id" : 0, "qsl_idx" : 4119, "data" : "ACC1EEC09D68FAC0C414FDC0DE2025C1D1A5D0C07A3E24C1642502C16FA016C1695713C1760E18C10D300FC19C5520C193720CC171831EC191BB16C1ED5714C1EA7916C168BF13C1479C13C1C02912C1AA2014C1C61218C1C5AF10C101311FC1B8F319C1BB1910C1985F1DC19B511DC1B0DA14C154AA18C1FE050DC07D3CA7C0DEF7ACC00366A2C01594B0C0F48B9EC08B65FEC0F4A1F3C0FC84F3C0AD1FE4C0558703C1625C03C14DE3E7C0731E07C16C69EAC06473EEC029260CC1C16010C1DC72C4C0DEF5F1C0933EDFC06B2BC0C03EB915C1F01A0FC1B4AAD1C017840AC1FDF2C8C0F72507C1435710C126ED1BC19AD3E3C0B49CCCC034D0FFC069AA0CC1869DD0C0F6EF05C113F495C0A4AED5C0EB73CFC07A48ABC0720BC9C063CBC4C00B9BBCC0D81E01C14C6CBDC05DD308C1C360BBC063B5F0C030A309C1055613C1B520A4C0FD0604C10345DFC092D2C1C0EE28DDC02B3E13C1571D02C1FC4B12C1FE7FCBC0E05503C1F21AE8C0211DC3C0041516C083C2C3C00F4ABBC05171AAC08D64B1C032068EC0C5E5FDC0A82208C1B1FF0BC16B4304C1E0D301C1141AE8C0F56304C1139D09C1B2CCE7C0ABFE09C1FF7EEAC0B058ECC04BC20EC1F97312C19CE1D7C061D305C1FE93E8C08521A4C02247CAC04E7CEAC062917BC023A6EBC09BA2A9C01E4DE0C0B0CDFBC0CF841CC1BF2034C04176AAC086DFD0C0864EB1C0BD36BBC0054370C007CD03C10FDFF1C07E1AE9C0340BEEC0AFB906C1D63400C118D10AC11310FCC04E3EDAC074FF10C1BB3DFCC0C852CBC0E684FEC0DED507C15894E8C0D80E0DC1AEC0E7C0A40CE1C0735010C1DC4F10C1A27FB8C07938F2C0F932CDC0714C6CC0FBF2EDC0B54497C0D538BFC07864FAC035F1E6C0762C01C1EBE58CC0DDF5C7C0D07EB7C0CC5579C0C901B9C03E36BDC040FC6AC06EA1E2C0741E29C03FE2D2C0ABF6A2C031CF9CC0736BAAC0E128C9C01E15C2C06EE5E5C0602AA6C0AFD6B5C01433CDC0ADE1DAC0264BBBC0A261E3C0063F9CC0AA13E2C0974FCDC029359BC06D19C3BE209EC4C0C5CEAA40F1E687C0C17FB44009AC6FC0D759573FA732A4C097A4B83E547AB0C0955B3DBF542F61C04C3F8FC037BDD3C0758FC8BFBBE4ABC0F86B2AC03F3D3EC000722FC077DBC740172E60C083F24E40471DACC02E8300C1D4B296C0E0E4EBC04E61F0C03494F6C0BFCE12C1CB8E15C1C090F5C08259B6C0EEFB09C1C86107C1892D00C11E1C0FC136FAFBC0C66908C18ABAF8C0D082F8C0D84404C1CCC30AC12CDB0EC18BF113C13B63FBC010120FC113205AC013D5F2C079A3E5C02193EFC0AB66F9C0A0F6E0C01E23F0C045FEDEC0B254E1C0CB330EC1099C08C15CAC0CC1310214C1632E1AC1C9E0A1C0A51EF7C0ECD9F3C01B11EBC0FE9AFAC0D521E3C0C2D50CC19DF216C1A1E7EDC07F6614C1090709C1632202C1287610C1684017C15C8201C1F14E15C18491FAC00D0E13C14ADC12C1BE051BC1902605C14936E3C056630FC1A7D512C19D2CE2C0DDAF0FC1F6A106C10D49F5C0C65B0FC1815819C16598EEC04FB213C1CC5A00C16CF50AC1E94C0BC1AF911FC12BB382C0AD00DEC0904AE9C01CB3EEC05573FFC01962F1C0932192C053F8EC3E7CCB02C104AB0DC188661DC16AA921C10A731DC1E79E21C1F6861DC19B8E21C1E3821DC12C8321C109671DC128A021C1542A1DC129E121C1493A1DC11FD621C1D32E1DC18DDC21C1ED4E1DC108C121C179971DC12A6D21C115851DC1AF8121C10C681DC115B021C1055F1DC13DB621C125401DC1A6D021C12E8C1DC10F7B21C140931DC1DD6F21C1B4901DC1B36A21C1C2C91DC1AF2F21C1DDC71DC1103F21C1859D1DC1076F21C1619D1DC1376F21C158931DC1587421C1AC8D1DC18D7A21C1033B1DC16DD521C11F1F1DC177F221C169421DC18DC921C1A9331DC1D2D621C1CA751DC1729E21C1EB6D1DC1C59D21C199ED1DC1746920C1C67B1DC15A7D21C1EB681DC147A621C13CC11DC151FB20C18D6C1DC1469A21C123691DC1C1A421C121741DC155A421C1EB631DC1E3B121C156741DC1439421C15E621DC181B221C13F601DC1CDAB21C1C39D1DC11E6A21C1A98E1DC19B8521C11FB41DC17D5121C143A71DC1105D21C1A0951DC1ED7221C1FF4E1DC17BC621C17E541DC1CBB821C16E471DC1C8C721C1A72D1DC11BE121C1E04F1DC138C121C177771DC1939421C135681DC13AAE21C1A56D1DC10B9F21C1CB521DC141BE21C1615B1DC149AF21C103591DC1DCB921C1107B1DC1C28421C150601DC1B6A621C1E5791DC10E9021C15F731DC1859921C16A761DC1DA8E21C174861DC17B7B21C1F83C1DC167CD21C13B2D1DC1F5E121C124461DC1D7C721C115771DC1B99621C113D61DC1AAC920C108871DC1AF7821C12B661DC1A7AF21C113FA1DC17E4120C1F46C1DC1F09C21C1ED5E1DC167AE21C116781DC1D59621C100681DC190AA21C1C4571DC133BC21C181611DC16BB321C19C611DC1F3B621C1CE831DC17B9021C12A631DC16BBA21C1FF661DC199A921C1DA4D1DC107B721C1EE6D1DC1419B21C17DA11DC1065A21C1BF791DC1509121C153B01DC16E4F21C100A71DC1C96321C117A01DC1336921C180721DC1C89B21C13D5C1DC15DB321C1B7771DC10C8F21C17A451DC101CC21C1A2431DC152CA21C19D5C1DC15FAF21C183491DC1BBC721C11C811DC1478721C17E901DC1CF7521C194C91DC112E320C1B6681DC1BAA921C1B6861DC1C77D21C1B14D1DC150C721C1AA421DC178D721C123551DC100B821C155671DC1B0B121C1FE681DC145B421C117881DC1448021C1F4701DC14FA521C18AA71DC16E3321C1B0 + +... + +C1113A1FC194A51EC1BF4F1FC186861EC1E8511FC199D61EC1A0001FC153E81EC18BED1EC116FD1EC1E3E01EC1E8BD1EC11C1E1FC1DFD71EC1E00C1FC1ECD41EC1E00B1FC163D11EC19E151FC1C5D21EC139041FC19FE41EC104FB1EC16BE11EC1D5021FC10AF81EC1A1E41EC13AE51EC12D011FC165F51EC118EA1EC17C0C1FC173D41EC184F21EC1DEF01EC14CFB1EC1E9F51EC1084D1FC12E921EC15E141FC1B0D61EC1431D1FC14EB61EC1FBF51EC1B9DF1EC1EDE41EC178F21EC12EDC1EC113061FC123EC1EC1C2DE1EC121201FC142B71EC1CC451FC17C821EC10AE81EC14FF31EC127CF1EC1381F1FC125AF1EC1CF381FC1DBD11EC1BF0E1FC119C91EC1FF161FC153D91EC10B011FC105E21EC185F31EC174001FC17AC61EC19FFD1EC16ADC1EC145F01EC1E4EF1EC1A2101FC158C11EC1C1E41EC11B021FC15AE71EC129051FC125EE1EC191F81EC168D71EC1AE0C1FC1BDCE1EC1A9181FC135E81EC1B7EF1EC1BCF61EC146E31EC16AED1EC101F21EC119021FC105D61EC1B8241FC175AD1EC1230D1FC107CE1EC1BF0F1FC140CB1EC129181FC105C01EC193111FC13CE01EC14AFF1EC1F5F11EC12FF61EC194F41EC1430B1FC1CFDF1EC1E4E71EC198F01EC1362D1FC146A91EC1B62C1FC190B71EC1A0EB1EC1CDFC1EC1F0F81EC156E81EC19ED51EC182121FC171DB1EC1F7051FC12DE11EC15BFE1EC11BF01EC1DEEB1EC154011FC135DA1EC189F51EC1AAE61EC1E5161FC139C31EC1BFEF1EC1E0F71EC17ADD1EC18B081FC17FF11EC177F41EC154F11EC11EF21EC166EC1EC189FB1EC1BA031FC184DB1EC1A94B1FC11E8D1EC16BE61EC18F181FC160DB1EC1561F1FC1DB001FC1CFE01EC103CB1EC17F2F1FC1CEE51EC150071FC12AE51EC1ABF51EC15DE01EC1E5FF1EC12FC41EC15B351FC178C11EC1A5411FC1F61B1FC1DFB31EC1530B1FC163CC1EC1DEE51EC1D9051FC18A121FC179BD1EC1CB091FC1FBCB1EC19BE71EC10F031FC152041FC181EC1EC167721FC1976F1EC19FF91EC1ACFD1EC118111FC1B0DC1EC1BD011FC1E7F81EC1CC4B1FC1C28B1EC1AAC91EC1D4261FC117C01EC1D72A1FC1F9DA1EC1350B1FC10EED1EC1F4F01EC1ECF71EC1EDE51EC1D4011FC11CDB1EC14A0E1FC191CE1EC1C7EF1EC102FD1EC10AE31EC1160C1FC1C7E91EC1A9021FC1B8EB1EC1E5001FC168CD1EC177291FC13AF11EC13AF51EC1DFEC1EC153FD1EC118DE1EC11A191FC108FA1EC117F11EC143C91EC1BB311FC175C11EC1CE391FC1B90E1FC1D2C41EC104031FC19ED61EC1D1F01EC103EF1EC1D3151FC18CB71EC17A081FC142CD1EC108F21EC17F021FC1CAEF1EC123041FC1A4151FC120C71EC17D231FC1DBC61EC14ADA1EC1FE1B1FC19C0E1FC1A3C11EC124061FC1CBCD1EC1BEF51EC167021FC1ACFE1EC1AEE51EC1CC641FC12D731EC1603B1FC1E1A41EC147FC1EC15DEF1EC1EA211FC1E5BA1EC135091FC118DB1EC19BEB1EC1D8FF1EC13B041FC107DD1EC14CF91EC1A3EC1EC144ED1EC187001FC1F3F41EC193F21EC112051FC142E31EC17AD31EC1591E1FC1C5F41EC18EF21EC13FED1EC1F4F41EC159CC1EC197281FC114071FC151D81EC1DD161FC124D01EC1C6F01EC1C1051FC107EB1EC1AA081FC1C0F91EC1D8EB1EC18EDA1EC19F1A1FC177FA1EC117F11EC18BFC1EC16DE61EC100C31EC133321FC15CF61EC1CFF01EC133091FC119CC1EC141F61EC10DDF1EC1170B1FC14DD71EC16A591FC1E08A1EC110511FC1AE951EC11A831FC1333D1EC12EF11EC196FE1EC1D9EB1EC1AC0C1FC176EF1EC1EAFC1EC150F01EC180F01EC11ED21EC1D91A1FC1B3DF1EC1DF0E1FC10B001FC192E01EC148E31EC1CE031FC173CA1EC19F1C1FC148EE1EC168FA1EC1F9101FC1DED11EC1B8DD1EC156161FC1A9D91EC1EE1F1FC172EF1EC10EFA1EC167EA1EC120091FC170DC1EC12E211FC15BED1EC1E0021FC198F31EC107F61EC102D71EC1731C1FC129EC1EC160FB1EC1BAF81EC1DAE51EC121E11EC138051FC166E11EC13B1F1FC1EB241FC1D7B61EC126C81EC160411FC19FE61EC1A70B1FC1F1F81EC137E31EC181DC1EC1D7061FC10FDC1EC1F3141FC1A7F31EC157F31EC196E71EC1B9101FC129031FC132F91EC144F01EC15FF21EC175FC1EC1F1DF1EC108EB1EC1B7F71EC151D31EC14A201FC124FE1EC12AE31EC1020F1FC108D11EC1D8EB1EC1CD181FC172001FC1C3F21EC167451FC18BA31EC1F0F91EC107FA1EC1E6E11EC1B1171FC105CA1EC1C51D1FC159E01EC1BE151FC1EBD41EC1CC271FC1F9E11EC1F11D1FC173BE1EC182441FC1F3FF1EC133E01EC148011FC1DCDA1EC147DA1EC1870E1FC1D9E71EC1F0061FC1F2051FC1EEDC1EC1F5D61EC18E291FC11FD51EC1F1281FC120E31EC153061FC102CB1EC1EA1D1FC15CC11EC1432C1FC194C21EC1C42A1FC18DED1EC194FC1EC175F61EC14DF91EC1BAFA1EC109F51EC15E191FC115D41EC1DEC81EC155421FC1A0EB1EC1C30D1FC1F8EE1EC1FC071FC11CCF1EC16C251FC12AF51EC1F8F41EC126EB1EC1D7F91EC166DD1EC168151FC194CC1EC1B2341FC18F061FC1E5CF1EC14FFE1EC11DD41EC1D3E21EC1D6061FC125101FC1E5EE1EC1246A1FC157781EC1116E1FC12B691EC1E5DE1EC1F5161FC1F9FC1EC1C8F81EC1F9441FC146A41EC1E1D11EC1BC2D1FC135E91EC12C091FC15A041FC115E11EC175EB1EC1B90B1FC12CE01EC1DB1A1FC1B6151FC163DD1EC12F0E1FC156E61EC170C81EC1C7381FC13CE61EC14A0B1FC140EE1EC1D5F71EC10CD61EC1D5201FC1DCE31EC183261FC1F63E1FC1B9A21EC179011FC1C9011FC108291FC1D2C61EC13E031FC186E41EC1C5F01EC14FF21EC1" } +] diff --git a/open/scc123-Shanxi_University/results/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/accuracy/mlperf_log_detail.txt b/open/scc123-Shanxi_University/results/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/accuracy/mlperf_log_detail.txt new file mode 100644 index 0000000..74cb95d --- /dev/null +++ b/open/scc123-Shanxi_University/results/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/accuracy/mlperf_log_detail.txt @@ -0,0 +1,70 @@ +:::MLLOG {"key": "loadgen_version", "value": "4.1 @ f74d16f541", "time_ms": 0.005240, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 53, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "loadgen_build_date_local", "value": "2024-10-23T06:21:10.796371", "time_ms": 0.005240, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 55, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "loadgen_build_date_utc", "value": "2024-10-23T06:21:10.796378", "time_ms": 0.005240, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 56, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "loadgen_git_commit_date", "value": "2024-10-22T15:38:02-05:00", "time_ms": 0.005240, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 57, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "loadgen_git_log_message", "value": "f74d16f54131d9080b9e45f234cc23e0ebaaf20c Apply min_new_tokens=2 to mixtral-8x7b, address #1777 (#1884)\necb880167756cb4b36ad70766b8d3254bfb06d26 [Postmortem 4.1] Make mlperf.conf static in loadgen, enable automatic Pypi release (#1882)\nf5c8f1758374aeaba26b2e84d31690111cfdf054 Fix bug: Loadgen ignoring token latency targets in user conf (#1874)\n976bb1ad9c7946be79507f3ff67955c27426af52 Set correct remote repo (#1871)\n41fa8aadd1ba0ecc97f6a519d8b42b04278e5f24 Add format files github action (#1682)", "time_ms": 0.005240, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 58, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "loadgen_git_status_message", "value": "", "time_ms": 0.005240, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 60, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "loadgen_file_sha1", "value": {"/.clang-format":"012aad77e5206c89d50718c46c119d1f3cb056b2","/CMakeLists.txt":"b73434348f7860471606aaa395b570e81113cb6d","/MANIFEST.in":"8d3c4ac6c325b7b9a0fd4cf4a4108cbeff8d5025","/README.md":"20a55bb946c2c0bbb564ced2af1e48efd096b3a8","/README_BUILD.md":"5f6c6a784e9cd6995db47f9b9f70b1769909c9d8","/README_FAQ.md":"01f9ae9887f50bc030dc6107e740f40c43ca388f","/VERSION.txt":"cb67dcc41adcbb7849a0a808a501ee9ccd951d92","/__init__.py":"da39a3ee5e6b4b0d3255bfef95601890afd80709","/bindings/c_api.cc":"32181da9e161c285f8fe46ddaa49e6cba2f9f918","/bindings/c_api.h":"91f58bd79b83b278f3240174a9af747fc38aff74","/bindings/python_api.cc":"9f538d2a5390c77ae0bc3f8a351bcdb2587bc66c","/diagram_network_submission.png":"53dba8ad4272190ceb6335c12fd25e53dc02a8cb","/diagram_submission.png":"84c2f79309b237cef652aef6a187ba8e875a3952","/early_stopping.cc":"0cd7b546a389deac73f7955cd39255ed76557d62","/early_stopping.h":"158fcae6a5f47e82150d6416fa1f7bcef37e77fe","/issue_query_controller.cc":"126e952d00f4ea9efd12405fb209aa3ed585e4b2","/issue_query_controller.h":"923d9d5cdf598e3ec33d7a1110a31f7e11527ec7","/loadgen.cc":"6650091ba7a918f343b06eb7a5aa540eae87275f","/loadgen.h":"e00fdc6dbc85a8c9a8485dbcbfe2944f81251c4e","/loadgen_integration_diagram.svg":"47f748307536f80cfc606947b440dd732afc2637","/logging.cc":"197efc96d178e5d33a750d07fa7b2966417506ea","/logging.h":"ddb961df7bcc145bcd7cce8c21f7cf075350dcbe","/mlperf.conf":"0a4daef277bb3151139980e484dd5e644bf36e18","/pyproject.toml":"712fab87b72ba67ef2a068d0f9f47da65130342f","/query_dispatch_library.h":"13ad6d842200cb161d6927eb74a3fafd79c46c75","/query_sample.h":"e9187c8612bbdc972305b789feb6e15c26e96cfe","/query_sample_library.h":"8323a2225be1dff31f08ecc86b76eb3de06568bc","/requirements.txt":"a5ff7e77caa6e9e22ada90f0de0c865c987bf167","/results.cc":"34e2d2a44324cb07c884f92146ecbb8ef9d704e2","/results.h":"fce22d5a588d91fd968a6b25c27896dba87bc276","/setup.py":"a722046e05858c6d9f38f0e2b3fe425334beef28","/system_under_test.h":"18d4809589dae33317d88d9beeb5491a6e1ccdec","/test_settings.h":"476ecd4032f3bafe6f201df25d68aca4e177f659","/test_settings_internal.cc":"ce4322c849d24ffafc28a37b5e528a4cb4df227d","/test_settings_internal.h":"f1d5335b53ca610c30e0edc5d07999a27b5b4b9a","/utils.cc":"3df8fdabf6eaea4697cf25d1dcb89cae88e36efd","/utils.h":"40775e32d619ea6356826ae5ea4174c7911f6894","/version.cc":"cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0","/version.h":"9d574baa64424e9c708fcfedd3dbb0b518a65fcc","/version_generator.py":"eea9b9cb1a06cd1abe1bbdaee82f9af31527fedb"}, "time_ms": 0.005240, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 67, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "test_datetime", "value": "2024-10-23T08:04:43Z", "time_ms": 0.012122, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1198, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "sut_name", "value": "PySUT", "time_ms": 0.012122, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1199, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "get_sut_name_duration_ns", "value": 300, "time_ms": 0.012122, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1200, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "qsl_name", "value": "PyQSL", "time_ms": 0.012122, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1201, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "qsl_reported_total_count", "value": 10833, "time_ms": 0.012122, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1202, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "qsl_reported_performance_count", "value": 10833, "time_ms": 0.012122, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1203, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "requested_scenario", "value": "Offline", "time_ms": 0.019576, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 271, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "requested_test_mode", "value": "AccuracyOnly", "time_ms": 0.019576, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 272, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "requested_offline_expected_qps", "value": 84.3055, "time_ms": 0.019576, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 309, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "requested_min_duration_ms", "value": 600000, "time_ms": 0.019576, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 315, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "requested_max_duration_ms", "value": 0, "time_ms": 0.019576, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 316, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "requested_min_query_count", "value": 10833, "time_ms": 0.019576, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 317, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "requested_max_query_count", "value": 0, "time_ms": 0.019576, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 318, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "requested_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.019576, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 319, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "requested_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.019576, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 320, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "requested_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.019576, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 322, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "requested_accuracy_log_rng_seed", "value": 0, "time_ms": 0.019576, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 323, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "requested_accuracy_log_probability", "value": 0, "time_ms": 0.019576, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 325, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "requested_accuracy_log_sampling_target", "value": 0, "time_ms": 0.019576, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 327, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "requested_print_timestamps", "value": false, "time_ms": 0.019576, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 329, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "requested_performance_issue_unique", "value": false, "time_ms": 0.019576, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 330, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "requested_performance_issue_same", "value": false, "time_ms": 0.019576, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 332, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "requested_performance_issue_same_index", "value": 0, "time_ms": 0.019576, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 334, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "requested_performance_sample_count_override", "value": 10833, "time_ms": 0.019576, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 336, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "requested_sample_concatenate_permutation", "value": false, "time_ms": 0.019576, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 338, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "effective_scenario", "value": "Offline", "time_ms": 0.019697, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 417, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "effective_test_mode", "value": "AccuracyOnly", "time_ms": 0.019697, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 418, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "effective_samples_per_query", "value": 55641, "time_ms": 0.019697, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 420, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "effective_target_qps", "value": 84.3055, "time_ms": 0.019697, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 421, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "effective_target_latency_ns", "value": 0, "time_ms": 0.019697, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 422, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "effective_target_latency_percentile", "value": 0.99, "time_ms": 0.019697, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 423, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "effective_max_async_queries", "value": 1, "time_ms": 0.019697, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 425, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "effective_target_duration_ms", "value": 0, "time_ms": 0.019697, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 426, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "effective_min_duration_ms", "value": 600000, "time_ms": 0.019697, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 428, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "effective_max_duration_ms", "value": 0, "time_ms": 0.019697, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 429, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "effective_min_query_count", "value": 1, "time_ms": 0.019697, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 430, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "effective_max_query_count", "value": 0, "time_ms": 0.019697, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 431, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "effective_min_sample_count", "value": 55641, "time_ms": 0.019697, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 432, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "effective_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.019697, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 433, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "effective_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.019697, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 434, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "effective_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.019697, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 436, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "effective_accuracy_log_rng_seed", "value": 0, "time_ms": 0.019697, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 437, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "effective_accuracy_log_probability", "value": 0, "time_ms": 0.019697, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 439, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "effective_accuracy_log_sampling_target", "value": 0, "time_ms": 0.019697, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 441, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "effective_print_timestamps", "value": false, "time_ms": 0.019697, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 443, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "effective_performance_issue_unique", "value": false, "time_ms": 0.019697, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 444, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "effective_performance_issue_same", "value": false, "time_ms": 0.019697, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 446, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "effective_performance_issue_same_index", "value": 0, "time_ms": 0.019697, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 448, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "effective_performance_sample_count", "value": 10833, "time_ms": 0.019697, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 450, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "effective_sample_concatenate_permutation", "value": false, "time_ms": 0.019697, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 452, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "generic_message", "value": "Starting accuracy mode", "time_ms": 0.021540, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1090, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "loaded_qsl_set", "value": [4119,3771,2947,5174,8221,10562,9791,4633,8569,8837,7091,5810,6371,10778,540,1732,2832,4085,676,6115,887,9339,10819,344,9312,1951,5739,9692,9525,4397,638,4375,10601,8825,6906,3471,9163,7272,9940,1207,7738,1663,9660,5625,2468,396,7153,8514,1418,8510,4187,1103,9970,10200,2141,6364,9748,8412,2719,5615,2838,7835,752,6099,10486,8982,3673,7292,10573,3118,9998,6825,5831,4451,9259,4859,1199,7842,6587,920,6687,3869,1033,6262,1223,30,5009,10063,2956,1007,2185,64,77,3470,6239,6679,1470,62,3074,1622,8153,202,9235,1614,9392,4912,4968,1648,122,3772,902,3526,1772,858,265,5129,6799,6255,523,10796,8819,1230,9400,3732,1321,7882,7821,9768,6056,4282,4617,6107,10130,7756,3036,10451,2201,3976,1086,3668,5394,3177,5128,8384,7234,4039,8503,3640,10043,5766,6810,1011,7547,1718,2401,3909,8343,5923,10069,9886,532,3270,7147,10062,9680,5432,9974,7736,3655,811,1105,1820,7465,2267,9641,4046,3956,5247,5181,7625,4073,9663,288,426,6762,1617,7914,4014,2904,9151,4181,6042,8967,6981,5547,10457,9322,500,938,131,8457,1171,3814,10549,8093,10675,2159,5807,7567,8366,6518,943,6819,5889,5108,7755,5382,8456,9756,10247,9351,3150,8971,1231,2990,5380,510,4324,3049,1165,6116,8684,6992,1829,139,1021,1602,5805,4226,5966,1040,3274,4269,10376,5316,8756,9587,8231,2809,2270,7887,7390,485,8028,7880,328,6236,514,4644,6061,1522,10537,9446,8787,8577,6774,2023,7474,4904,5064,6183,5196,3361,8268,7606,5968,6182,5019,8265,4621,1525,8181,163,1352,5785,8632,1074,2392,8398,6561,125,7071,5577,995,3783,3849,9753,8651,1051,1159,3971,8903,8413,162,5854,2133,1416,9098,1759,633,5800,7671,4731,10623,5586,10578,8929,5894,6833,2909,5063,3163,5002,6278,4148,7477,8408,10104,8805,1504,8105,5746,3572,8095,312,7436,6165,2786,6286,2079,4807,10523,6117,3840,4169,8750,3016,10238,1137,10037,9427,8449,4908,3020,7183,1557,10500,2982,8020,599,1519,6873,10276,7350,8224,719,8474,9981,9198,2802,5756,1897,10801,7682,91,4160,6697,2989,10113,5765,982,48,6573,4412,5148,7681,2514,6348,620,6702,250,10219,7813,4512,3988,7538,3272,4433,4597,7580,8229,603,9284,9959,4683,711,7542,5450,1175,9859,10685,5041,9760,765,6303,3395,2908,7823,7516,237,105,8034,1757,10401,9935,5256,7800,2642,276,8636,5297,10766,5954,5725,8144,6660,3901,8078,4158,9455,7297,5959,8088,8583,4733,10345,5232,9216,8902,4667,1449,801,9231,1792,1211,6013,8566,4988,2938,4730,404,8699,2346,3059,672,3701,7231,10169,10431,5806,1373,4193,10665,10030,4335,7950,10150,842,5753,6435,5096,5922,5497,3193,9367,2158,3816,5356,65,258,4675,1277,2041,6032,10088,7814,8726,2036,7885,9303,7440,8023,1151,5749,9573,7776,572,1240,6297,6439,1343,8605,3386,3251,4631,8140,3760,2068,4957,2623,4961,10281,1467,9037,10546,4415,215,2194,3273,387,3327,2225,10739,300,4467,10823,10668,710,4228,4517,2069,8709,2409,2573,6144,9357,4844,3985,2864,2814,2615,8952,5085,5090,1937,1854,10689,5061,10374,1580,5223,4851,7888,869,3023,10178,1615,7566,7219,5276,6685,8839,10534,10532,5826,1251,9124,8308,8702,1805,3148,3529,9942,8171,2450,6782,7679,2503,9989,2748,3961,3660,9466,6832,1635,1831,9298,6457,3165,6633,6513,10354,7865,1888,1182,10643,5477,3120,7676,6178,2833,9468,4910,2657,9723,492,3347,6481,2822,4426,1018,486,826,3455,1535,1539,5782,8267,10102,2714,5573,5482,2442,537,6934,9420,1925,10526,8158,583,5921,5634,10715,2063,2865,4722,6711,3033,5830,6761,10221,3292,5093,3487,9844,5192,3127,3166,5517,8842,5986,6515,8806,4368,3880,682,1506,8542,6600,5522,10789,2775,6979,7375,9732,9488,7435,6127,2058,8763,5776,2901,551,2871,7369,6929,4164,307,7840,861,2670,8027,9410,2386,2513,1063,5122,8218,4457,1100,2437,462,5774,9629,10033,5794,4106,3616,7179,5414,4598,9925,9289,69,9948,9945,4585,6214,1953,6363,7744,6030,3012,37,4855,10794,2109,5500,1652,9019,7047,3763,10114,2735,4137,7192,6450,7896,6319,2559,9057,4150,1626,1399,3805,10481,7054,3160,3802,2753,643,4212,10435,3360,157,8424,6667,1686,8112,1983,6648,9535,2567,6149,8099,319,10795,2980,8559,3326,2116,2360,7544,5778,2589,3310,3751,3352,4366,9705,368,6088,8217,2399,6537,10120,6669,3061,1457,5252,7052,10159,6880,5593,8852,3506,5847,4877,4944,8609,5364,1816,5290,634,1661,1221,6935,702,4824,1453,8062,7092,345,7030,9428,204,3121,2188,6429,7446,10631,2354,2569,1219,5475,5727,6274,1556,8968,7730,10342,7917,1985,6856,5692,7132,1146,10468,6861,1550,2928,6585,5681,5352,5348,10265,10307,322,9139,8357,9486,7441,4023,10141,10170,1441,165,9603,5998,10586,9022,5633,1234,6283,2131,7996,5042,6246,4201,3064,3071,5674,6769,9698,386,9319,10686,4,9197,3510,428,382,6614,7688,6154,10078,2876,8185,5240,1006,8696,4353,5760,7961,8376,6405,2107,5804,9860,3812,1679,2756,9652,7964,750,1997,4091,1899,1308,4035,6656,8123,10248,1510,2846,5638,477,4076,10730,10589,8266,3218,4977,2536,5334,10054,8762,8598,3929,3811,9581,7752,552,9441,5711,6642,1129,284,976,10747,933,7929,7607,9805,5102,9979,2939,6910,2647,4835,7528,4876,6734,2706,4394,4926,8341,8734,1135,3753,9333,1291,6011,3867,5563,3343,7612,1027,5165,9700,6340,6104,3099,9537,3428,1612,1606,10634,10021,2761,1042,9310,3601,6806,4003,8336,1320,8874,6545,3842,3715,4916,177,6141,922,10477,10369,3372,8706,7261,9000,2944,1770,4537,8671,8299,4515,6709,7300,5017,1499,3864,4145,8725,2669,6292,6724,7960,6080,4257,743,1363,10444,4518,9545,10388,2296,3870,3737,984,4590,5924,149,1966,10084,6018,1340,8785,1678,4211,8663,6542,1475,6302,7202,1850,9261,10681,2339,8461,2664,7011,3441,9549,1482,9314,6828,1290,9203,7486,209,5215,2113,714,6092,3889,825,7966,5609,3787,6223,207,6965,2644,4612,9140,10280,472,5726,5208,8772,2044,2931,1775,4134,10539,7508,172,8681,8240,10189,4870,8908,7684,9379,3266,3376,8405,10311,453,8213,2034,2190,2294,8580,4421,4725,3952,2918,8999,8545,7043,542,10301,9146,9032,3597,4112,2062,3195,2850,6169,10805,4900,8382,2059,5703,10074,7541,10662,1760,2114,10749,5993,6295,5211,7792,4639,8333,3833,1847,83,10770,8312,8831,9004,371,4923,7699,3307,7834,5879,7458,6735,9799,10741,1304,1576,8657,3329,8860,6081,1744,7308,10564,7102,10611,7126,10413,6653,6846,1371,6864,2078,8362,3021,3569,10777,1939,2861,8085,4899,3288,7360,7690,1489,5526,116,7151,5145,6673,5523,9390,8191,1004,10528,8958,9118,7393,4684,7266,6966,5995,8508,3778,313,2324,3282,10165,3447,6294,6865,9833,2202,6316,365,3052,7057,10821,1194,7974,1410,1782,9986,1123,5591,7286,7552,10721,1088,1793,6524,117,1763,8854,2199,3366,6230,6135,4113,4154,2326,7696,10438,4546,6268,4991,8670,9515,9500,6272,5253,2634,10092,965,6654,316,7333,10361,7385,2359,2521,10505,6418,3731,6452,4151,2002,4407,9193,5715,5311,9681,1461,6752,10196,9063,7064,7760,10356,9534,6401,9570,6213,7243,1969,5069,4463,8379,2228,10617,10563,4001,7533,4118,8634,303,4758,6480,6863,4079,3551,2387,959,10384,8204,3739,4388,9571,5235,890,3448,3380,4022,3407,3303,8225,1710,3134,8655,8273,1442,9126,9090,767,2352,5036,6959,1116,7507,8320,206,5100,10580,4657,10013,591,5970,3983,10637,3489,6177,4329,9309,598,3799,7447,4206,7727,6138,480,3483,3979,1719,5575,5814,6507,10156,4769,9437,3749,949,10166,9129,1387,7559,21,3606,2329,10379,71,6909,4268,3231,7838,4403,4880,2289,4825,7313,3855,9482,5669,5755,1472,8584,145,7282,4105,5546,9644,1811,10620,8543,8840,6823,3084,9331,7771,5570,2129,3333,9811,1567,9316,6617,7070,2971,5115,9806,8920,6889,6597,9550,7991,10076,1289,8872,1156,963,6369,353,4520,4406,4510,327,2672,10123,881,4197,10616,7244,6791,4055,9060,7901,4398,10019,5895,6816,724,6400,3142,2649,727,7658,4411,4131,4893,3203,6901,5131,5905,7199,3408,4562,101,9214,54,2388,5032,6645,6310,973,7184,5595,4177,3578,10735,6783,1468,7622,854,1425,1272,8441,10581,2009,3999,4258,1929,7531,8205,610,8733,2698,10366,9817,7227,10638,1446,4587,10052,9114,3354,10140,677,5614,10068,3206,10826,5460,3063,1093,5083,10551,502,9391,7170,6267,10812,1555,9356,8910,4623,3970,10429,4096,9172,522,7591,8773,2791,5612,3922,6555,8079,6802,5491,4135,6640,7203,373,9221,2355,7947,5119,5940,5363,10443,646,8044,3473,2214,5335,5803,10138,5180,5525,927,5272,2372,2951,526,7209,5972,1753,6877,1496,185,843,1124,8942,5229,1918,10371,1282,6424,10228,3253,354,7137,9669,5618,2061,632,986,1991,9433,5261,5130,2246,6315,3835,5515,2976,187,7121,2230,7683,930,10571,2538,3106,1630,8453,5267,6534,6349,8372,6532,7725,8522,8596,4984,6027,1483,2281,10284,4300,7155,8557,1842,6925,251,4156,2883,10632,4008,8994,3844,3112,5932,6837,4987,4751,7133,9724,4981,6781,5405,8770,1012,9963,9551,9840,5471,3562,454,4696,198,1902,1516,8943,4082,7253,497,281,8464,3677,8635,10172,7900,5238,2747,4914,2487,9376,2618,6526,9801,463,10039,1226,7307,4673,7525,8661,1010,6727,9665,10411,1307,4267,6543,9564,4793,9552,6627,3638,3785,2658,3384,3158,9720,9818,10790,6483,9067,669,3930,7843,5427,7587,7536,7029,3125,3531,6385,1565,8749,1736,3512,5719,4868,888,8255,8535,6194,10174,8304,5939,5070,6040,588,8437,10759,8538,8206,1458,5159,3997,1259,4858,7176,9035,10403,2868,1122,5888,9842,4622,4261,10673,9324,8487,600,9479,6220,3419,195,2155,399,287,9830,9883,9328,2055,7836,10511,1451,6850,5911,3337,9778,10660,4739,5981,8347,3588,9865,4238,8957,5074,1691,6813,3860,3883,8878,7674,8731,7336,9136,7178,4565,7820,10508,2051,8164,823,7624,2648,3128,7150,7687,4830,10350,8119,416,735,4601,5587,120,2691,3920,4192,7125,10053,25,808,9563,4299,10112,5112,3643,5446,7403,8391,7114,1542,2628,8911,9279,2886,3221,10056,1912,3291,5819,3830,8896,10788,4000,5815,1183,355,9074,8086,5140,10657,2345,2550,8728,8468,697,10557,10322,8004,1473,1698,4574,4635,1921,261,5606,3409,9595,8966,5447,4599,6622,7714,5381,2778,9396,7374,10654,1346,5113,1536,6553,1306,1932,3216,656,10615,6053,96,4447,8775,8624,8685,2235,7956,4390,1420,220,6145,2255,2478,2730,7084,1155,9091,5757,8924,2955,7628,2093,1334,1068,8013,1927,7841,9731,10436,809,6519,5217,174,2169,7952,10341,1776,10629,3169,8108,3635,5671,8667,1653,10761,1309,3659,7039,8056,2827,42,524,10646,921,2283,449,9983,2561,4542,4260,1725,9512,4540,1378,7252,2161,6347,937,8455,7862,5008,10109,1246,7697,6176,2816,1824,1746,734,7320,3564,3892,9662,2526,1553,8188,7826,3837,429,6706,2945,9511,3466,9636,6688,2453,8254,8438,8321,2699,5091,3521,3229,9219,4792,1859,9449,5190,8178,3073,2781,5817,8737,1855,2849,3602,79,6723,10289,8429,3626,7006,5884,3852,2315,5409,8693,1762,4978,8377,9892,3182,1361,2626,2581,10619,115,7025,9240,2236,2317,3641,1835,7551,4979,7791,6767,8032,7159,3000,1145,2175,549,3987,1748,8906,7038,7915,9995,8563,5741,9152,5391,7122,7933,5747,1511,8215,948,15,1029,6760,6874,6170,1801,9164,9377,8369,3748,10277,7497,7145,4724,17,2694,1041,1682,3788,8184,1015,8296,2167,4999,4809,10396,9853,4018,2086,10584,10089,3002,5841,5318,689,7581,7680,3427,950,10466,2087,9890,7666,4523,8530,1538,10784,247,6301,7460,3972,684,7646,8883,8024,1238,6393,9716,618,5104,6728,6168,7797,10273,7164,670,10252,85,6602,1693,2527,4274,3552,2284,4290,6156,8784,274,1620,9382,5000,110,10290,1326,4785,8644,9321,8743,9,1878,9711,9425,8764,6691,4962,9498,10727,2959,5262,10066,372,1336,4845,7753,5342,2047,4024,2708,7710,8639,7472,8668,337,3094,4172,9903,7644,10831,10082,4625,3043,4471,9277,1133,9577,1523,8489,1841,7878,2932,9616,7479,2448,8165,6033,4630,1946,5486,9327,9442,6824,2104,6855,8319,8378,6456,6623,3915,988,2662,7009,1331,4094,2812,796,9735,2396,3797,9481,8897,3743,8803,1808,8879,6637,9233,3703,10278,3136,10094,81,2825,23,5436,5321,3824,4891,10231,1138,6722,5433,10553,2430,2984,1167,7110,9750,10009,8406,4659,8768,5678,8435,6988,1727,9161,9051,8582,4297,1815,5707,4350,243,1708,567,4490,9397,4652,10440,667,2578,9330,7469,8600,7356,9971,8486,1243,9547,6352,8556,19,9780,4502,2964,10460,2851,7626,2804,3746,58,2929,2472,4481,10015,4514,5012,6768,8136,7804,8161,5094,1206,7412,87,1022,3011,1545,6527,4404,1587,219,6490,3575,2132,2170,8614,4632,1562,7478,1779,2568,2949,10606,5660,1261,8182,8310,9657,2565,10346,822,2574,9905,4513,942,7786,1994,3533,6809,6583,3425,6015,7013,6397,4259,6497,9677,2497,3363,1940,4265,6946,3088,1401,10722,10732,5925,2685,6250,3914,4958,1876,10051,7221,26,84,2499,10705,4400,3723,5910,10250,5623,6564,4077,6725,6325,9633,9992,437,9142,605,3683,8832,8260,9272,4071,5646,3230,3596,6495,5696,4247,10351,5313,5373,4495,9682,9607,6146,358,4643,8625,3709,5207,5780,2967,5613,9068,3608,10710,6708,1898,5621,3131,4750,10125,3496,103,5264,6034,10680,9190,1349,3828,5604,9517,9102,8439,9832,8173,6119,3318,5786,3492,401,9618,6737,9246,10146,3680,7759,8436,1478,3378,3003,1660,925,9796,5173,1044,2008,3475,865,6017,718,9793,1592,9430,6229,529,7481,95,7128,9088,8572,4171,2859,8960,8950,3438,7131,5,8015,10358,67,8337,807,6106,1295,6866,9463,2720,3919,5147,1892,9815,10195,9138,266,3848,3780,1271,9627,5332,1521,10639,3556,9904,7271,3891,1213,8989,4234,563,4263,3480,10700,5545,7916,9350,9786,6511,4021,6922,1984,4254,755,6461,3650,5958,5689,3435,7761,3648,8174,8881,2716,6508,6386,5843,3153,6167,5457,8103,7101,5161,7225,3116,6335,6232,3624,6746,7037,10058,489,7847,812,6876,2727,7260,10814,4133,9238,10395,8947,4124,4669,875,4427,2431,4726,3733,3001,3539,768,8472,4334,6094,7578,6868,7276,9502,6084,7087,402,9412,9610,10335,6365,2290,9256,6903,7270,1972,9424,10448,3560,6410,8419,4920,5914,5584,9059,9529,562,4294,5088,6380,5179,5616,9946,6684,4795,4616,9561,7288,1987,7946,7108,8662,5201,7766,6575,1062,3497,9869,9122,961,8317,10115,4600,1059,8,6879,3847,7515,9775,3184,3741,3809,341,9223,6047,8395,2605,4141,2368,1109,181,2919,2252,321,8946,2397,1328,2189,2233,8146,4925,3129,5624,8067,8220,2629,8591,6744,8973,3534,1933,10737,10754,1081,4428,7665,2095,5946,8168,4176,8325,5243,10191,5745,627,2313,5089,9157,10175,1803,8250,4839,4344,1397,2785,8976,4182,7627,2439,8767,992,3594,818,2102,2101,7732,2136,10045,6248,1701,7480,10187,9937,3843,4786,8259,7733,7721,5053,6924,7265,10529,10608,8077,7425,10765,813,4107,5464,8698,8647,3665,581,980,8450,10412,9757,6155,1706,1941,8281,6977,5637,7609,8901,10709,5763,7148,7382,9695,2239,2800,9120,5495,7097,3339,9044,827,50,1384,5652,4019,9507,9295,2200,7869,9741,548,5114,8388,2264,1407,4688,395,5627,7012,2351,9849,3137,6676,1406,8049,10738,7938,6844,8757,1906,1720,824,152,1742,32,8444,3287,6875,521,4929,860,1248,7532,9617,94,6945,8107,8890,2118,7189,6022,154,1980,3854,3140,4681,4570,1392,10217,4753,9345,8277,6969,10410,6610,9759,6796,742,9170,7334,7780,717,5580,5154,4276,4306,8589,6427,10628,915,3066,1846,10121,10220,2508,4302,1543,4611,10453,2889,7579,5234,4531,3198,10398,10020,6225,5270,10618,1303,10518,8779,10461,891,4256,10506,5200,4219,9264,8721,3488,2880,7936,6805,2890,8052,6522,6509,2171,2995,1433,1839,10227,3882,6721,3807,2408,2490,1655,10605,7259,1887,1633,5018,5907,10151,3302,7634,3450,7198,10367,5511,8730,747,9227,1485,916,2122,3422,1130,5716,2056,9262,5693,6798,6626,8951,2751,4995,8045,3931,10260,3342,862,1645,3349,8241,7103,5082,6862,6982,3208,7729,4230,8507,2794,6525,5350,6103,3755,6140,7475,3405,4271,9560,8504,10318,4354,4804,4034,7982,4743,10081,5034,5204,4456,2219,5283,9225,10570,2549,7837,8810,7726,8525,4911,3469,1584,1560,2451,5714,10472,1089,7401,4989,4385,3767,8397,5864,4789,3259,4890,5461,5320,1786,1140,3776,3696,9036,5564,2049,6468,9156,10126,1935,234,9858,9891,8536,1209,4892,8258,8291,6581,6296,10502,3412,10768,1198,188,4816,2370,6120,8353,3903,2653,4473,9807,3336,10450,336,9875,10364,5056,853,9406,2684,2793,40,6549,4146,5640,8555,5304,7418,471,8630,5934,2035,8983,3445,290,3371,4401,8316,10649,5231,164,9503,7449,5802,2757,8777,816,3911,2,6857,9514,960,1621,5752,1908,466,709,3191,9230,6834,6411,6592,7152,4715,8364,918,7236,1398,262,6157,6360,2921,7615,5288,8106,8849,3554,10572,8991,2539,5869,9213,1216,10205,561,8327,7083,6887,3060,4662,10044,4191,6757,6073,7207,560,4918,5459,8732,10017,7135,7652,5535,7610,495,273,5312,3205,6574,2417,584,5836,10755,5353,759,3625,8620,420,7973,2307,66,5398,2085,10086,723,7014,519,5236,5632,8553,6916,6998,8152,1275,3831,10010,10463,838,2524,4967,9471,5851,6093,4443,4646,8309,7431,2676,10055,6830,4125,3918,5589,3010,6421,2099,6933,6436,72,8166,8163,10515,4243,5788,450,10803,9002,6970,1769,2469,3200,8104,9015,1690,6937,855,6928,9291,4941,839,9930,7156,7706,4081,3157,8792,8833,10389,3563,10,4377,3576,4697,7293,1518,6860,10329,9176,9462,8131,9614,5770,7315,192,8974,6949,9467,9960,8516,6686,10083,9837,4736,5790,5619,3704,5021,9476,2830,7667,7796,4310,5560,10482,6927,1338,9168,6215,199,6636,3334,736,2197,4126,5777,4790,8780,8041,4551,9267,2103,7162,828,8017,7190,9703,2127,2927,147,7206,2050,9966,3873,8688,7673,5456,135,431,673,7177,6377,5127,4934,6254,5626,5899,4771,473,7437,1169,6379,6983,2858,196,1437,7010,9086,3399,10046,6596,8526,9415,9897,111,5043,4298,8918,6621,935,6991,3676,9578,6304,4155,9749,4602,7003,2347,2356,3803,4396,3893,2328,8008,9080,3092,6,3868,5686,6187,8753,4117,3827,7839,7802,886,7335,6504,5407,8562,9025,8690,2280,7174,6455,4088,4782,10711,4028,9609,503,9260,9167,1101,6139,6477,4806,9054,8853,6694,2675,6907,5095,3365,10135,2635,2179,6345,3916,10650,5588,3946,7935,9270,8573,1954,2668,7883,5073,9113,7274,8506,5789,1170,3817,2025,2558,2930,6677,1237,10535,3048,5337,5718,4716,3595,5829,4484,1296,1869,1917,8270,1571,2788,5991,1215,1784,4026,10806,3362,10767,8060,7707,4475,9900,10469,10163,7968,647,5425,10308,6002,6493,1364,5762,6698,2545,3856,3924,8431,6247,4392,6777,9694,9804,2686,6563,4577,5772,3144,6822,6884,2836,8850,8094,2821,1948,9586,4533,6110,3712,4199,9103,129,10164,1348,4064,9565,6125,4138,6980,847,5975,169,10600,4323,10072,5913,6375,8035,8155,1563,8328,7201,9934,2410,9856,4423,4637,2398,6290,9363,10699,1833,4767,6129,7556,1208,3982,6244,191,7024,3183,1735,2193,10317,10626,505,445,10257,2710,10330,8720,2534,1699,7061,275,3942,2477,4419,10426,1517,2622,9286,9030,7554,179,7188,5937,9497,1586,9211,5323,6521,9632,999,405,8373,9292,8812,4986,9445,153,3298,2973,8030,9150,7407,2234,863,1548,5328,4231,1669,8363,1276,9774,6514,5054,384,802,2896,940,8293,5167,4544,10091,8984,4821,7220,7959,9005,9249,4416,4721,5592,6076,4389,45,6716,3519,10820,6719,1781,3161,5442,1313,7141,9096,10636,9843,4279,7790,10409,6433,3351,1938,374,5402,9271,7075,2805,5431,7325,6565,2773,7340,2485,8318,8193,461,5345,8187,412,7342,2476,1702,4996,8452,5548,7214,2260,5900,10525,8305,1342,7913,253,2981,9746,2958,1421,2627,1341,9765,1549,929,1840,1178,2466,9353,3252,9403,6487,4854,597,6938,6488,286,6792,762,5919,2092,2884,1724,475,6827,6211,1333,9454,6986,13,2168,1391,2067,609,5116,7378,4960,9290,1657,9620,5336,8686,3243,5885,7257,6961,201,9658,8788,3097,8502,4805,10487,911,7829,5956,5366,8415,5651,1578,6314,4237,2562,5501,2366,5504,9881,6217,4973,3028,5307,9605,1460,5583,9812,9281,8741,5133,5521,1791,4066,2285,9188,9954,7788,9717,5022,5218,4311,790,9089,2367,7997,10489,2272,5410,99,10061,8346,4930,6835,6048,7535,3091,9401,2593,2937,4829,10714,1996,10064,3424,9182,941,9928,1082,10347,3145,979,9133,3454,1711,946,7088,7078,5079,3933,4365,5938,3905,2650,1863,10688,6729,7242,6905,3301,6078,7467,4208,4240,102,2898,7866,3908,0,2320,4564,9645,3130,5168,730,9889,1750,5874,8513,2268,5242,493,8981,3411,8873,9200,6005,9460,5665,5708,9797,6328,1036,6745,4405,8295,1758,6789,468,10742,10566,6780,8567,4284,7106,3888,6747,10134,6950,10079,4363,7898,5374,1257,7745,2380,3605,895,2687,3690,6462,2529,8692,9247,7395,3283,8669,9707,9205,7983,6253,3503,7919,1053,704,2244,6779,3846,6993,3850,4924,1971,10387,240,7572,8863,7864,663,4581,1409,2378,9100,10684,1465,8200,5679,8430,5866,10602,4143,4065,7157,8933,7763,756,8091,2488,3115,8248,4356,4813,2288,9687,1393,5999,1844,7255,5349,10830,4116,1112,8544,3895,6576,8194,8834,10391,7969,10713,270,8909,4111,137,9459,7322,10103,8581,3155,7142,7191,5400,8913,8209,2912,3434,9773,879,4719,2807,9704,53,6242,6241,10666,1322,7298,10008,6086,680,8047,8898,5265,4202,5467,2493,7408,4157,10255,7722,4414,2878,8074,7421,9232,7063,8111,1670,10266,8674,7217,10279,2979,5574,6269,5317,10415,5184,2429,6674,419,1028,602,3727,7429,8638,6839,4843,9767,9621,5347,8133,7694,5617,1227,9555,3029,7363,6500,1191,3770,569,9048,5524,4940,10368,3132,7358,6003,9402,5732,2310,9557,4538,4345,8550,2484,7195,2504,7417,9600,6031,5846,6812,2555,4214,6655,8537,10751,6918,10757,8349,3244,4011,2276,1882,4060,3935,5393,7871,8386,7808,6472,3279,9192,7420,5157,8294,9566,3586,6818,3461,2903,10582,4755,932,6547,7731,7116,6503,4569,2923,910,3633,10723,9597,7027,4013,318,4554,10480,4935,6482,9895,3968,9779,6338,5187,3697,4582,1668,5137,8926,2100,8939,421,7079,8746,9413,4702,4901,3314,4101,5505,4007,1945,2963,5969,4783,1783,9823,256,4620,9153,9013,4927,9344,381,3998,9990,9381,1974,343,1916,7817,8445,664,7711,1362,956,5597,4417,9924,10746,5499,8223,10152,10186,7344,10783,919,5740,2709,10804,3018,9651,1095,6801,2717,695,1039,4704,1968,7498,5877,6665,6062,6342,1369,7364,3518,1077,3477,167,6184,5735,10006,7140,142,10370,7238,8799,8729,8082,9052,7067,8401,10349,9366,2312,1977,687,2772,2054,1413,612,3524,9634,9184,487,2026,2950,10136,2795,9524,4779,2019,5694,9852,557,4399,3006,9302,448,9064,5979,614,8551,10117,5001,8813,2045,2004,1154,2660,2894,10769,10256,218,4627,5322,6193,4038,1047,10321,10185,8329,7893,3951,2841,5700,10545,1978,8269,7370,7944,1003,7894,5343,6838,3515,2592,5720,8210,4921,1411,8822,5033,5904,726,5850,10080,7768,8330,1761,10253,1999,2046,912,7247,388,909,7637,2885,6014,556,9409,8055,3013,2750,356,2013,7530,10229,7411,7058,3058,5964,661,5758,5075,9836,6536,7720,10652,7524,4127,5014,4102,7705,3151,5532,9841,10340,7154,3284,7345,3375,6469,5518,4207,9845,400,1689,9398,10817,8610,6210,1751,10381,8534,6921,7999,6198,579,5124,5576,5607,5123,2421,10295,936,9870,7568,7811,7124,7907,411,3853,9183,106,6917,264,10050,10827,7105,8068,10781,1848,8039,6663,2847,9257,216,1113,4974,3196,611,304,2473,2790,10718,8641,10239,6376,1673,6374,9374,10406,2266,1651,9275,1450,5199,3493,729,8786,3965,2164,10365,10704,10208,1957,9965,9217,10139,733,5282,8080,3543,5275,590,1160,3542,9855,2518,2115,4956,10293,406,4864,5139,90,2580,370,9999,4425,7981,6406,4233,1031,2241,3237,1368,2321,6915,6584,6024,9548,2178,6387,649,754,8490,3414,7263,9642,2974,1402,9436,1217,3607,1,10445,2177,5224,10785,6035,7483,4860,9638,2206,1664,3234,2369,9648,2988,4099,9207,8862,1325,3217,9838,24,8606,6913,613,4303,5931,2463,2389,4701,3517,3541,10703,76,3694,9364,1845,8650,8520,7539,1716,7934,7906,1623,8466,2147,8175,10031,6726,329,10339,4814,7099,3759,10691,2464,6133,7713,3511,8759,8141,8368,10168,10610,9278,2522,10242,8130,224,2991,7810,2665,1142,6650,7115,5876,7737,5600,1412,8703,2590,9123,9408,9072,7819,10100,8038,5736,4266,2379,6486,7044,1502,484,4061,7076,4856,830,9697,361,8006,4689,9128,6434,5473,4976,7287,6089,9453,8501,2261,694,1665,5015,9854,9116,6001,238,5344,7822,10075,520,10527,944,5415,5578,3430,8048,4248,2837,5481,4618,1894,6009,6394,6147,8147,4351,4142,5620,3634,5536,8907,8980,3468,5449,7897,2920,5155,5395,3019,5222,2840,6755,7988,2654,4044,6114,3682,666,1302,737,3689,9752,1569,1785,6357,1114,675,108,10494,3388,513,2607,7824,7604,8675,6630,8212,5132,4706,9440,8955,2474,997,3170,7381,5811,5423,9952,4352,536,6807,6608,4381,3613,6090,2043,9993,8022,6463,5233,5538,3567,8628,10107,4752,2176,2510,9131,608,1934,10326,6049,7473,10002,9980,9325,8922,173,4485,2029,6396,8838,3465,5068,8116,1752,3938,2702,3111,4095,2364,5833,3051,1076,9518,9346,9961,4445,10800,1787,9308,436,9226,2645,1335,6681,10203,2632,109,189,6794,6162,3330,4732,10731,2570,2874,7353,9202,3047,1624,1049,10760,4875,1474,3649,9608,8257,5856,2064,8798,9299,2151,8005,4950,8407,2553,10736,5585,3257,2732,3721,1488,9661,4482,8935,10132,7630,5372,893,7775,10774,5823,6158,5209,3451,6891,3300,8847,8532,6843,8934,6898,6790,2606,8179,4049,8802,2680,1512,906,1111,4647,3762,6238,8988,6754,5832,6252,5058,3387,5371,4464,3207,3791,8987,1915,9554,504,6612,9795,7017,2817,6043,3181,9908,7232,8996,4938,2603,10153,2996,8961,8102,8494,8546,7196,9426,10531,5274,5378,9572,9076,7582,7739,4030,9819,2759,4619,5997,2557,8196,5858,1822,8552,5375,792,9222,3432,6101,8917,6499,2065,8660,7774,9932,1634,2053,460,2302,6753,9132,3549,8109,4132,2226,35,856,7415,1789,10690,1117,7113,10701,4075,10160,5071,6859,6895,10327,3925,1886,4449,7509,3389,1094,3621,3138,8654,4811,8794,3825,8282,3255,10427,1318,7657,3724,4749,3832,7691,3700,654,10775,6337,9337,5942,2010,9884,1312,4170,6284,4315,1647,8322,413,5086,7301,10671,10286,57,7291,8493,8827,5516,5906,5944,3083,5859,7521,8744,9754,7249,4505,4325,3481,2395,7226,7462,4472,5996,7193,8297,6354,5730,3544,3775,9949,6291,3664,2797,2215,2382,576,8029,527,964,6438,1643,3289,9708,1424,6054,10708,7651,772,10561,1065,884,8975,4503,1814,4889,6641,4252,7430,8817,140,4691,4309,9589,9736,8607,10522,6358,5483,5424,10622,8007,6096,3368,2576,6381,872,952,8533,4866,8454,1462,525,8302,8570,1305,8578,10192,3851,8998,2077,4583,7603,1981,1950,6336,9929,5882,1017,6446,10118,1091,10455,3910,1910,3168,5820,7254,9582,2248,10158,4501,3219,7586,3332,2495,10149,1400,9536,8198,1825,8617,7892,6695,1674,564,1837,8354,7450,3757,3350,6431,8710,2218,883,7237,6185,9864,4220,4166,8345,3899,1141,6516,9239,2828,8645,2299,280,10297,4149,6999,3323,6334,6890,5010,533,7576,8887,3871,10267,151,10683,8121,241,1263,4798,8306,7032,7161,10209,8138,8648,3886,2731,3246,6008,6293,8100,3735,535,10541,2575,194,398,9077,4511,2897,2801,1107,9448,4698,9977,56,5691,2799,6299,1877,2546,249,6840,369,8154,4543,1030,7511,6672,2479,3693,4817,7215,1959,9969,8892,8588,8949,2597,9591,5225,5839,8656,9544,10360,1379,770,4033,9667,1889,10119,9584,8283,1173,1268,3054,7641,1976,1180,186,1242,8339,10012,10655,10510,3278,686,4677,8761,9171,546,6318,1990,10316,7623,119,297,10223,3981,3652,70,2530,367,2505,5067,3426,5107,9593,5494,4975,6102,10542,9640,6174,2993,993,10810,7304,6270,3623,6960,2789,10659,9465,7208,232,1797,9084,4634,10213,2892,1531,10538,6240,4048,434,1696,2096,2154,5955,3906,1023,764,7620,4200,1480,5444,8855,2875,4680,2373,2509,415,4847,244,3536,7396,7854,6402,6764,5717,2612,7518,1388,2494,7990,7695,4699,725,10423,6758,1529,10825,10028,4651,9574,3622,7717,6038,34,782,3761,1360,1252,469,1332,8916,7484,6163,1355,6793,2856,5510,3793,2097,3537,9431,7678,9169,4833,6330,5980,3209,7638,4552,4100,4097,6541,5193,8253,8484,4713,1619,8236,10598,7169,5420,7904,7461,2877,6963,4387,5027,7978,7922,5764,868,9365,5698,10613,1471,7376,2475,7724,5441,7876,7772,4810,5974,6180,2341,7399,4869,8272,2551,9540,3663,4661,7387,9215,678,559,8585,9556,1172,8642,10262,1629,758,5978,5650,1058,4005,8479,3393,2314,3109,688,4087,1385,3656,5084,3632,6601,2240,773,3705,6186,6718,10490,6206,4953,5333,3990,5296,7962,4700,3875,3379,4566,6968,10025,2445,3345,3024,6765,7875,4867,7351,8380,4679,3322,10035,9288,6748,1947,7616,7548,4123,8016,7489,3823,4872,1408,5880,9180,10609,8791,9910,2762,501,2636,7971,7303,3710,8084,1871,1192,7080,1610,8560,6046,3286,7608,8403,1405,1456,171,9912,5947,6118,3610,2547,1024,9199,538,75,2767,7471,9726,3927,2537,10667,4043,6391,3214,721,6063,655,8208,2262,8748,8869,8531,2027,3236,3406,442,1572,745,2271,8858,8243,9847,5287,8127,6546,9913,7275,644,7994,1440,10414,3423,7031,1962,4244,6559,5916,5549,1383,1046,7830,8063,2015,7986,3304,8612,6704,6361,7166,3462,4209,10171,1685,7002,6730,308,9110,577,10568,7015,7048,6200,7212,7204,3804,3527,4437,4759,4010,4020,10664,6710,10161,3553,10282,9968,3260,3077,6533,7424,2986,2071,441,5647,6703,2446,4080,8054,10182,5896,7928,3584,362,8050,2586,8843,10707,5492,2815,10658,7635,9592,8608,6908,7434,7085,4774,1327,835,4693,3188,6956,8886,2162,9389,2679,7123,4884,7419,6613,2599,2770,2512,9265,9323,6675,315,4508,994,1605,2256,6321,7675,1002,1016,5303,7584,939,9626,476,10211,7573,1394,2212,6189,5687,5309,2519,2186,9559,1936,10268,6558,1365,5791,778,8678,5601,2292,5025,1235,1149,7433,4062,2881,5871,4555,39,78,9166,3311,8126,6888,10408,2625,3960,4656,4561,6849,2338,4591,5531,9619,6036,10464,4378,5775,5278,8385,5514,814,6775,3691,1153,5631,1738,10799,4966,8611,566,9820,5957,100,5963,7789,2157,945,2454,9093,7849,4225,6444,1374,10587,975,7098,8058,9947,6619,2301,5426,7998,5117,10155,5206,4371,4802,3685,9491,592,4838,3861,7949,8643,10780,5685,1218,6105,8629,550,4773,3887,1085,9941,9690,6897,4089,7284,2403,3185,7042,3189,4886,8571,10372,4446,10204,6191,10190,5220,892,681,1956,4588,7534,16,9701,3355,7389,5463,8145,8026,10246,6004,8139,8300,9676,6087,10325,7770,1641,5543,8985,2344,3897,3463,4337,5949,121,9911,5948,5163,380,4338,5743,2588,7318,3309,1269,7850,10085,9008,7879,3726,8774,4465,1582,1084,7094,2400,1591,3299,3681,3079,4841,8797,4760,1087,9671,4295,9505,193,6289,8515,3661,4952,3713,1229,2917,5257,1283,10199,392,9175,5151,5186,5259,2742,5558,7953,4610,7127,6288,7319,4110,4571,6219,2481,6926,9553,6990,3912,5875,7281,10595,9108,1438,1721,2385,1677,1680,1319,9504,615,1423,2962,9699,3025,1598,4435,383,1376,1500,10577,3504,3614,2769,6227,9003,3580,9666,7605,565,10263,2243,4723,6161,3943,2985,5072,10303,214,662,9251,7059,1594,10716,8301,8575,4253,851,9329,221,4931,4304,5659,9531,4453,5603,5149,9034,8677,3769,1067,2273,8997,10147,9404,2376,2891,7600,3841,8925,665,2306,9250,5455,9480,867,6341,9121,3774,1526,6615,9599,8963,5503,7574,6773,6067,4849,7428,10274,7285,5767,2348,9835,10216,6878,5135,1952,4374,8735,5508,7856,9349,2843,1163,8433,10027,8160,1666,10473,2184,5213,3742,3312,4222,8097,7886,4280,9485,9868,1202,10177,10625,6589,6484,9399,7670,928,1799,4665,2656,6195,2070,3590,4204,7167,6582,9483,2334,708,4965,10394,5443,1132,6578,8197,5453,6137,2391,2305,4122,10702,10596,8626,9975,846,2613,7107,5840,6501,10762,8233,9635,5216,4642,981,1273,5860,7778,8031,3295,3978,3377,1692,9623,2022,5006,2933,2362,6505,7846,8760,4139,3926,8228,4993,955,7036,5748,5838,4895,2873,4320,6831,2316,10111,4289,821,6447,7323,1992,8371,4121,1486,2222,2729,9106,1726,9943,9438,6083,7096,9919,7295,7958,8467,8711,4069,2961,1279,3075,9506,8713,4595,5744,4442,1632,2425,9991,4850,2145,4173,7451,2060,6197,2208,5824,2153,8172,10574,2182,1108,7870,6607,2471,7827,375,350,1422,10352,3592,8700,4654,5258,1317,741,6720,6234,4016,8176,3364,6644,1640,6309,4506,1505,8432,5960,8540,8037,5684,3482,9664,6398,2776,6996,4042,29,4796,9606,4589,7588,5677,9996,223,10604,9268,9332,1000,5030,8857,4104,2253,6858,8937,803,7668,2554,3619,3806,9541,9951,4563,3124,8399,7965,357,518,8483,6068,7283,6756,4235,4742,4717,8592,2142,9685,8548,7402,1547,9149,5784,3989,3932,4074,2048,5281,9318,9283,926,7932,8745,970,7942,9395,2038,9659,4474,2826,1014,1256,7223,6836,5031,10434,6919,3313,4292,1196,3720,9317,7640,9160,4194,1494,4862,6707,6512,9825,4448,10377,6057,4671,8463,2263,5369,4103,3525,1964,4949,3722,628,2595,9388,2771,8001,6221,8959,794,8167,5818,6785,10198,2327,6074,7405,104,2796,8427,5164,3194,4163,2428,4530,9301,1853,3810,3369,3566,8871,3593,5162,10696,3950,9776,776,8593,793,4384,6886,8597,10697,6506,7867,7662,6932,3070,5849,1299,9313,7089,1188,7527,8274,7617,9763,1965,1382,1278,7165,4754,132,3934,5773,3618,1861,4879,4232,1608,7457,9306,8350,6811,331,5542,8990,10048,2922,2424,9307,7175,9873,2420,1541,4434,9879,5961,379,9443,5898,8122,5731,5295,10776,4342,5821,6351,9794,9510,3798,3885,4822,5020,8183,6732,8244,786,2465,6664,7517,580,1414,5712,9073,4036,3581,5520,1967,5666,7007,805,7398,1375,2287,3014,1514,7082,2948,4840,5182,7685,9902,3858,2726,5489,3321,1069,2160,7404,7495,774,3015,1566,2165,10288,7464,4720,4575,1314,4215,9784,6609,3315,9477,8473,4763,5845,4286,10726,1603,2744,5759,7388,7496,443,5962,2544,4507,3383,10748,977,3133,5066,7945,7020,7372,7908,9721,3396,3044,4705,5051,2867,4909,3792,10614,8740,9867,6736,5040,4029,1286,6666,8875,7317,3955,5706,5928,2204,2970,9492,1771,1577,4196,1796,6150,6332,8066,10210,10514,6531,1597,3550,1998,10585,10338,3467,3513,7648,7130,478,3766,1589,8418,7858,3941,252,1390,5599,5989,10670,4660,2121,8351,10824,1083,5326,3784,2480,4330,6742,1055,6714,9734,7618,6628,6739,3459,7735,7233,896,9831,1034,6971,9972,10607,6012,1828,1179,7485,8423,2000,10488,10393,8148,306,6307,5723,3736,8021,545,1800,8561,4709,4341,5076,6557,49,5452,3152,9361,7324,5291,10230,1166,5673,10554,9590,2402,6817,6282,4560,6787,2619,3082,7172,378,9055,3528,4948,2721,3878,6323,465,1459,4690,5881,8599,6943,3078,4573,10188,989,6954,7512,2440,10679,1955,5738,4278,6300,4090,418,2325,797,7902,5562,7500,5812,6426,9675,10422,5734,5466,9828,4321,1741,10793,8344,5110,9826,5798,3173,8495,934,7692,9878,6594,10145,10478,359,1168,2900,7019,5060,10029,9714,10294,2461,6362,5496,4818,5045,8945,1395,467,5935,7392,8811,4144,138,3996,9673,22,6571,9148,3490,5469,8279,4128,10552,5965,3222,8387,4735,5675,587,4483,732,9006,8313,9916,10503,8672,8904,8717,3879,804,6395,6853,8793,5645,9263,4326,7807,703,9920,9921,621,1152,2080,9094,1654,9085,5434,228,6097,9710,6478,3745,292,1884,6743,7614,227,283,342,8367,5329,335,9729,8649,3535,7069,9212,8129,631,8568,3410,7051,3356,7090,9978,3476,6530,5512,558,1920,5688,874,10397,9335,3501,6682,2456,7749,2711,4236,3796,1144,2238,6882,9740,1559,9354,2183,6050,578,7734,7779,908,10302,3754,8541,2030,2217,2677,7045,8230,4045,7459,2594,245,1546,7210,2489,1050,10245,2211,6196,7005,3105,2394,8442,2643,1681,683,10344,3821,1581,1329,3027,10653,4438,2855,5299,9489,4828,6646,7762,4189,8143,2232,3949,1858,6854,4653,543,7173,7312,2808,8149,9147,5657,10143,10744,9355,5572,5005,6795,2371,9706,5397,114,5550,5722,4002,2902,9611,4624,1249,5994,3057,7931,8587,128,3617,3834,3936,2579,3505,5039,2734,1604,10004,9028,6222,134,2754,9643,6359,5667,5834,7338,9715,6603,8549,7546,8064,7406,9539,9079,3267,8523,5474,8278,749,10173,3890,8826,1232,2152,3609,6052,4636,547,6111,4470,7391,9771,3495,10437,4357,5642,7073,3199,5636,7976,3039,7306,8263,2432,6931,6939,1810,6231,6064,2910,5622,210,3667,7592,4179,6544,8919,553,6261,6344,7743,5103,8769,6550,10433,3442,6715,5078,2994,156,10005,8083,459,10499,7719,6126,4902,882,346,496,9696,7967,9186,6019,8594,6353,5210,1989,4180,3440,9062,9026,9861,1765,2083,7647,4668,5988,10385,236,8142,9513,3647,9762,6634,707,4780,4349,9788,6037,1973,7316,3644,7269,1293,4331,5920,2824,784,8808,7636,10501,4346,9604,3583,89,815,5118,8724,8820,622,3558,5367,3268,10496,10251,10612,9358,4492,3439,2528,4578,4998,9901,4834,6670,2381,452,728,2414,2144,205,9871,10693,2531,2693,10512,4273,9691,6586,8465,9769,9712,8864,2535,2511,4098,7560,5136,2297,8993,1174,127,8876,2274,6451,2258,7504,8488,10197,2124,3458,7321,530,5044,10362,5873,6408,9987,1128,3718,2187,8072,2935,1162,8969,4865,10233,5870,9814,2766,11,3017,9046,269,10719,2081,4459,55,10097,1148,7631,699,1631,9964,233,5950,10034,630,4183,507,146,6343,636,5796,2173,8298,6538,4678,2936,1875,5183,9375,5050,9418,4161,10593,1823,4761,2322,4641,6692,1037,10807,2458,2715,5655,594,8227,6911,6800,92,2723,6060,9962,8447,10758,364,5943,4409,1330,9276,2608,3262,5754,771,10332,2577,6814,12,1452,2768,6277,8409,4685,701,5324,5690,5340,4536,8970,1380,5185,3324,3789,8640,2084,3096,3642,6870,9730,3877,3730,2074,9798,1919,3884,6570,2954,10828,4606,6346,456,1222,9125,6957,9985,8392,6713,8226,8090,2426,9478,2977,3055,7816,2895,1435,5144,9285,7337,7948,4494,2337,2021,3233,6847,7767,5141,4072,4942,4519,8219,3957,7035,5737,10181,1857,573,6453,8070,1487,8245,3401,74,2412,10024,1767,2340,3582,6259,4382,4391,9071,8524,3277,7663,4461,6690,8738,4640,1057,2411,1766,3958,10475,2331,6209,5120,4241,3620,7851,8747,8014,4615,5327,6661,5724,7213,4174,3042,3781,8256,9050,6121,1687,8723,8928,8216,4012,1609,1025,10815,1627,2712,3460,1819,528,8275,3103,6324,6258,1714,3494,339,2746,4748,1558,5927,4130,255,8202,5952,4136,3744,8776,1493,10404,4959,4666,10569,4604,1258,7505,9352,9742,2210,9713,5191,1157,3921,4608,3631,3994,6122,9702,783,1638,7712,4431,9499,10236,7925,3175,3839,7765,3228,967,10465,8682,9189,5901,6216,8835,6577,10105,6803,4545,10101,5035,8664,3247,7289,6285,9579,2335,4441,5540,5314,10215,7815,1561,1515,8885,5594,6631,9877,6205,1817,4745,7235,8356,1570,5325,2137,7589,4823,5413,870,9038,8595,6638,1396,8242,6569,9562,5783,6639,5695,7180,1728,9872,10476,7661,1224,6893,1241,9378,3256,4203,9933,183,1530,7277,6528,5438,7181,1982,6972,2295,1503,334,4532,2293,6171,4842,4728,8492,124,7444,3752,5825,31,8000,2163,10530,1366,2548,8081,10771,63,5827,9191,10218,3258,1045,1656,3215,3630,2150,3276,4800,9414,5226,2673,7677,4255,5266,2072,6953,5004,6562,5365,1818,3162,6766,4452,10524,6985,3637,9850,10000,9672,3876,9647,1575,3699,1905,7629,8972,9744,2924,9452,10234,4737,9010,4288,1843,897,2435,7348,9305,978,2138,10591,3862,10540,4897,8402,8932,2443,2279,438,5038,3574,6535,7273,541,9598,8619,10459,3306,9043,5605,595,1270,309,9280,3080,8411,7825,6006,7801,7664,7599,9458,2506,7764,798,1520,1616,6378,3245,7877,2953,8480,819,5608,9181,3729,61,1072,9009,7482,6788,7703,136,7700,254,9615,2540,1717,3947,7794,6947,836,5519,2444,511,3548,1804,7296,5388,8948,3707,1683,4664,4175,6204,10261,7357,2251,7104,3675,9253,6265,9613,8071,880,8365,1019,10270,4469,2357,9646,6892,3507,9792,7985,8527,8701,5552,8180,7799,685,5787,10343,3628,2637,4607,6202,7831,1747,8342,8891,8110,4050,366,5387,6974,3666,6611,761,2852,1372,2423,2697,4355,4272,3122,1161,8618,3711,5037,2076,6160,5662,4277,2651,5470,7848,3954,10235,8938,8288,9439,7828,1147,9241,931,2983,2602,3639,10306,4791,7723,217,1464,4162,5951,9082,8177,1064,391,4460,4955,2377,8880,6173,10304,5903,4972,9018,8977,3457,6329,9922,10740,3241,3917,1061,1885,8708,1200,422,10127,10300,4057,9790,616,10594,876,9033,8848,5862,1729,3265,10167,2659,9386,160,7519,2905,7659,2866,7245,5302,10353,8856,7921,10474,4053,5354,2011,9422,4729,5751,3319,7514,9944,9950,4440,2560,330,6772,9725,8422,4070,8694,6370,82,1377,983,5023,4812,923,2223,9533,2839,6942,1434,8796,6440,7757,4383,2743,10695,3570,3836,2997,1901,10313,8818,282,7540,7951,640,4658,9066,6712,3190,6649,18,7251,8497,10497,3530,8859,7445,293,3135,3285,5228,7818,9159,1642,212,5077,10059,10224,9474,38,3687,8851,5630,4078,5205,2674,5273,9822,9101,5376,8046,10206,9099,10624,8498,3325,9800,2610,10036,5867,958,107,8096,3239,2779,7466,7553,257,5710,4982,7049,3335,5857,5359,427,5097,1659,9772,1903,7040,7989,3671,3800,4015,2571,9341,184,9530,6705,7872,713,10630,2143,5392,4797,9516,7023,8169,1432,998,10446,2128,1143,2641,5346,6883,2829,9336,531,2566,2192,7022,7377,3100,4287,2722,348,6510,2462,8539,4168,8882,8284,4946,8475,325,7060,5026,8396,242,6696,9738,8290,3456,974,3611,3034,3400,9070,3913,9058,5565,9612,491,9628,7903,8809,5909,4190,133,2459,10390,9461,6598,3296,1484,6651,1189,1136,10269,8716,2604,9105,2737,9229,9906,4458,4593,10509,2300,10419,2978,3444,5399,7138,4718,8884,878,9755,8727,6201,913,5246,8460,3046,10378,5541,1509,8375,10416,2966,5271,5412,6459,7443,423,4115,8895,5842,8995,1870,10678,7194,4184,5171,5134,1386,10357,10447,1588,8201,1476,3085,9936,4819,6112,6423,4114,5793,8659,10375,10392,10226,6228,10520,9994,6659,4213,8586,10157,5178,968,6417,7026,1891,5152,9371,7758,9926,1713,3758,1986,7000,10425,5742,5202,6059,301,8124,10285,9862,5195,6313,1125,9145,8861,1904,3437,966,6740,6131,4547,748,6188,1778,3108,2707,4740,1281,5530,2220,3117,267,2298,9813,4067,410,5319,6671,1650,2031,8162,4250,3449,2343,4319,1120,4328,8823,1497,8676,4534,4493,7543,1637,10786,6778,9204,1922,9654,7216,10663,9984,9722,10259,2842,5479,903,8705,10644,340,7186,10011,8075,10402,3227,3486,8059,1508,1928,2913,5628,3948,9127,834,2148,6044,10047,4301,5883,8459,10071,1777,347,9119,10550,7146,8893,6026,6382,987,1958,9674,1439,4336,617,10110,8751,3865,1739,9637,7028,6264,544,9315,7881,1646,9678,3604,3429,5902,1131,3146,425,7593,4580,3394,9917,10669,51,4054,5421,2972,4784,4216,10237,9244,9117,263,2703,4153,829,2671,9144,1300,3538,2713,969,2940,5837,123,4246,2242,3397,3559,9273,2862,8899,10122,4756,10441,1345,2552,8012,2803,9783,2374,113,10808,7117,2887,5370,9423,7118,10405,2042,5214,7520,6305,5230,5702,5893,5160,8239,8443,1963,3056,8615,1988,5048,8374,10467,3263,1351,8824,4051,3390,9007,7844,1417,3179,4370,7476,5408,9069,2486,3857,9829,3149,5458,6826,3702,7502,1931,7305,2646,3186,10418,4308,9668,4614,5683,2725,5028,2075,10096,5579,7891,1740,3443,4395,5792,1806,2934,1358,446,8199,2600,7575,4686,8603,10645,2365,6077,27,8416,8389,3532,10452,9112,6458,779,3706,5602,3491,1214,9766,10558,7160,985,9311,4747,7365,9393,8192,10131,352,3555,8718,2318,6298,9115,6414,5059,4762,1043,10292,3719,6000,9293,7993,5809,9174,6588,7874,1285,7895,4497,490,3603,4468,10023,4990,8098,8953,7955,7585,9538,10717,4198,7379,7347,6404,2452,10791,9451,9434,1900,6207,8195,444,4707,7438,3945,7309,9508,3636,9931,751,1513,7330,4526,7302,6181,4836,2969,8683,2434,10420,1255,7561,5451,1354,3156,8512,6041,7410,2695,1052,5733,326,8002,2911,6192,1186,4794,3102,2780,10124,6902,7751,1860,7119,9475,1080,2501,6738,657,3040,1404,6243,4638,7596,8286,10001,1995,4885,5016,700,1096,9373,10380,6442,8010,8622,5590,5555,4420,2265,10517,360,4766,3698,6413,5355,2098,2682,6007,7884,7455,6407,5992,9866,2449,9770,3026,8469,6652,3474,2728,6460,8092,4744,6701,483,4307,6416,1197,6172,9248,3679,9624,3141,10291,4778,7741,6399,10137,5490,1537,2502,4058,5728,6731,7569,9370,4584,9650,8771,9296,4853,6281,3213,3991,10214,4402,3308,5861,8801,2541,2806,8246,7693,7163,4479,10183,4499,2870,2818,433,5280,7423,3523,277,9234,3359,6266,2652,8400,3509,6474,6899,3627,4408,4857,2763,2692,951,5878,93,7602,3866,6123,9827,4827,4296,6166,9155,1573,1187,7248,5668,5248,10495,6987,1832,7795,3031,6226,10462,5930,4682,2556,7669,10829,3232,4195,1466,658,2736,1005,7371,6604,5653,9709,6620,203,9923,9596,9134,10184,1097,2006,841,5306,6568,1774,953,5445,6271,10536,6632,5237,3317,7016,1838,1599,9473,7491,9956,6082,6567,10498,6069,1316,9300,7654,1245,8348,200,3338,10575,8521,8420,4422,6023,4084,2433,629,8627,2765,3348,8868,9228,6100,8931,6058,7149,7228,7187,8836,8870,9519,1583,6952,2907,1879,2174,3280,5649,2941,9958,5417,2835,1228,10093,8120,6580,1267,10753,8830,889,1533,8040,451,9047,7555,6662,3248,2254,1102,5454,2792,760,555,5644,5852,7394,7299,7034,3995,10424,8434,6449,3385,9049,176,8115,7940,2120,7349,5390,1849,9385,1809,7557,4293,5338,2195,9782,1060,3418,2741,8426,6276,10811,6551,9347,9967,417,2752,653,5143,10764,10087,9520,7863,7890,1212,363,8665,8428,10298,4025,9887,6616,9693,8458,7386,6976,8752,3838,10243,2149,7229,8451,208,8888,6658,6151,6997,9185,4898,1254,3235,7523,9065,1205,3187,3872,3316,3579,4480,6989,9622,3600,7742,904,3750,7977,894,1671,753,4343,8574,2696,575,9888,1479,9031,2140,3571,3119,738,1618,36,4165,8790,7056,775,9417,5933,5664,2091,7923,6491,289,389,4738,3275,9130,6657,7650,9976,9493,7264,4439,6025,5092,6529,10275,5286,5439,6153,8944,4549,2311,4848,175,8689,7442,1737,10334,9384,8394,10599,7656,2139,1056,9173,3210,7453,8334,4218,7781,2663,458,4826,1253,1287,9135,9542,2213,5029,5983,8326,9733,8956,10621,4450,2018,10264,4032,6941,4832,9224,3782,7416,9039,5279,9177,8940,3107,8604,8065,5872,6263,4936,2231,5699,1658,9528,4516,6143,5057,2024,1723,2787,8421,6871,9097,6717,10283,8554,311,6930,8742,3598,1695,8025,5125,3072,2863,10772,4239,3795,6784,4808,1507,6842,8992,2384,8867,8410,10565,3264,5172,1066,6964,5639,1071,7503,3037,7185,3993,791,10129,9421,4291,6224,2040,1121,7182,2117,660,7773,2992,9297,5098,5635,260,41,2308,4477,6829,10003,2353,8547,10633,3779,8471,2221,7454,3331,3404,2418,6306,3098,9490,2906,10202,8331,8448,2688,1419,7833,2491,2134,10430,5537,10249,3708,2915,8687,4746,1181,1098,2275,5656,5176,1454,4557,9649,5855,3662,9686,226,8923,5848,2245,691,506,7852,625,3859,3822,10065,4907,4896,7095,2309,10323,10315,4815,8238,2330,6372,8754,9914,5816,7672,10333,9743,1008,720,298,9165,5430,10336,645,7004,10040,2404,6075,1185,2172,1079,1150,9024,1856,7211,9810,7230,231,2135,2166,3577,5488,7158,3953,4500,2250,3297,479,2482,1001,7728,9751,9484,1712,4787,2415,4339,9988,10373,7494,1443,1359,7359,3734,1350,6388,7452,9953,8101,10682,6113,1070,1429,4283,2304,2180,9416,5990,9728,7168,7120,6689,4915,7910,3290,4224,9470,9684,8758,601,6944,8564,7963,3963,8114,5865,8087,1284,8807,10305,6479,5221,8359,8011,4943,6494,2542,1715,5559,9575,4093,2621,8073,6280,5529,3101,408,7564,10648,3269,4476,795,3975,6028,6275,88,7927,5484,3340,7979,10590,2005,1104,7643,1463,3479,6973,6502,5701,1236,2585,1389,9527,2631,3065,10144,9764,7611,3293,4186,6643,8222,8338,1867,539,10676,2819,8370,1075,3358,698,5361,6443,4799,4649,9747,4281,6751,3089,10324,7046,9915,5245,3147,1310,4831,6492,8695,2823,9761,10095,3874,3686,9107,10116,7139,2090,2705,8511,9137,10225,2224,4509,9583,230,10470,3415,10743,10752,799,1106,112,3067,7889,8280,5158,8646,4846,1544,3211,5416,6948,924,8912,4525,10559,3143,10232,5254,8476,9368,409,2319,1203,2237,397,947,810,2946,10809,10399,1600,1239,295,3373,586,7134,9411,693,7262,2639,1534,9472,320,2942,155,4548,7427,1353,9585,3022,4550,2943,7570,5654,3478,279,2869,2014,619,10180,9839,5377,235,6848,2667,6079,9567,5768,170,5853,1812,845,3904,3674,8936,4947,10142,2975,4027,5709,7448,8156,6759,1118,5055,4954,2413,7053,5971,8135,8477,9982,2033,8789,7256,2624,9432,5761,3522,2564,817,3249,1649,1551,4937,2916,877,1370,9027,4063,8234,5177,8841,9543,8118,8089,8894,1970,499,8781,2611,4210,3940,5797,10007,864,4852,9326,9209,9688,9083,9522,7704,7686,9014,3845,5697,8323,4951,5676,6333,2661,10194,2683,3353,712,5437,6233,5241,4152,7501,3164,8707,9042,8264,3126,2783,6470,4362,2926,5973,4521,455,10725,8186,1942,9237,9369,6683,5672,637,7362,3391,9898,7409,6317,6437,2810,2007,8889,8125,6599,4596,7632,9040,5918,6556,8783,182,8844,1636,2587,7562,7702,624,10471,7899,740,8235,2375,9444,2701,2278,9340,8712,6951,1426,626,1667,4734,9495,8631,246,4980,4264,1481,5485,4380,1260,9918,8509,5929,4041,7750,8262,1579,3557,10822,763,3765,2525,5476,2517,6016,3974,10428,3254,1009,2700,9348,3271,2999,47,2760,9208,7086,8261,2106,8251,4710,1852,7708,7487,1501,5013,7346,6355,8829,5065,6593,9258,3413,8043,5188,9078,3068,4711,148,5953,6678,1851,8491,2704,3587,6776,3801,7613,3863,7343,5926,8930,3114,4985,3589,5890,6770,2349,314,10328,4992,5506,4873,7072,2745,439,403,9419,5493,3242,2001,1339,457,1204,9521,5330,9287,6465,5472,4223,5301,4504,2507,447,4803,6647,5513,1294,2782,1492,6441,8954,7222,8061,8637,1709,10640,2457,4006,5977,5146,1444,4270,10383,9372,2094,10519,8033,6356,7329,957,582,8390,7246,6466,2957,5138,2854,4971,6312,596,7995,2777,291,6471,7414,4004,4322,1961,474,1381,6422,4466,2407,3261,4361,2405,9245,1427,4567,1639,1590,3004,6540,5263,3695,4340,3794,5528,10038,6552,10458,6485,7909,3202,1733,6605,512,9092,130,2543,668,787,7368,7987,1265,769,5011,4613,7619,8157,2774,9304,9405,6373,3967,3433,7008,7093,5844,5799,6476,2358,5331,873,4801,8355,7250,1448,46,440,9957,1158,2089,259,7583,1491,2112,197,3226,3568,7279,7144,1292,10479,7857,1979,7980,7050,4418,674,3547,3416,1415,6415,3514,1704,2156,10016,8979,9081,10555,1827,905,8440,4251,7355,3728,917,10022,299,20,3069,1532,4592,9362,8528,6496,4983,5003,3937,5771,2755,1880,10133,5465,9158,9689,10543,10712,5985,651,5315,5249,482,3176,98,9834,3367,2196,3907,3093,8381,8722,1611,9894,8679,1428,6287,4496,5534,5987,4863,4672,10516,5721,1564,9938,1247,6179,5868,6432,5298,7111,5781,6881,1347,7754,6820,781,5533,5284,10067,2111,3446,3502,8866,5915,4359,996,4770,7785,1595,3035,3969,1834,3894,10060,8237,1794,285,2198,3898,2191,585,7384,3768,5892,7861,3172,9201,3973,8360,9342,6750,5498,9320,3139,9195,271,8978,2498,3090,9029,8057,5487,7859,5886,2422,6572,4524,3341,10417,3420,2125,9679,8478,10296,4837,2146,10201,4572,7845,432,4056,6428,6786,5383,8941,5658,3110,3684,6070,7510,8914,10042,859,4068,2249,4663,8335,3453,2952,7639,7327,5126,3180,1297,6085,6914,7645,3692,5197,4462,3670,8821,2088,8579,7490,7747,1909,6872,2681,8113,10745,6445,7205,6128,1745,10449,623,1826,2520,2860,4903,8795,2126,7311,43,10400,832,4454,9274,6159,6680,9631,6808,5611,1527,9011,5539,3818,5795,5294,3417,1220,5251,10641,705,731,4052,659,6164,3599,8211,7918,414,5863,6203,4906,3646,3087,2784,9252,5435,9360,10077,3962,10597,568,7493,2844,1498,6635,2020,9896,2965,3007,9816,5198,8816,2614,10724,7426,5268,8446,5403,7001,1874,9220,8332,1949,10750,554,3431,850,8986,8616,5396,10179,1773,4905,6851,4712,4871,8715,8602,6473,33,7331,1430,10299,2572,5750,5351,746,10635,849,4964,7698,1628,481,9939,7701,9494,2303,1528,642,2016,8927,6467,8499,10319,6885,8311,3714,954,8404,6367,9487,962,5111,4317,5150,4777,7109,4603,2853,3520,7240,7400,1722,757,8128,9243,5801,8462,1048,9218,3498,4386,3612,5024,6841,10271,7171,844,6606,8417,9909,3819,3645,2342,2282,6771,1176,2203,4765,1195,10359,7129,6350,9997,5643,4312,2277,3964,3346,5897,7970,7911,8613,4692,5080,3725,9380,8921,4376,10773,8271,8482,5509,2205,7855,8340,604,8190,9469,8159,3095,1868,10544,126,4528,8170,5189,7468,4360,1730,2406,8633,7463,5887,6108,716,7373,6962,9580,4262,4650,5357,1177,2039,2110,5109,3081,9569,4488,10687,6520,6045,639,2447,5169,840,10661,4655,2914,4314,2617,10818,7954,1139,10816,4159,4963,8800,2888,213,6936,4444,1895,4037,272,7413,4249,7339,7366,3808,7197,1703,278,1274,3500,6039,6237,3786,8778,6912,2460,7565,6517,6920,5419,9429,9625,2105,7278,44,5835,1731,2470,3716,5105,3086,10588,4129,144,2492,4881,1035,10813,6693,6804,2893,3197,9021,5596,3452,470,3320,1210,10363,5976,14,10026,1127,7112,10310,5566,5663,4313,6308,570,8905,9464,696,9907,9143,9739,534,5567,9737,8697,8009,6595,7748,6175,2811,1795,323,5705,9266,8804,3777,2857,4429,848,10309,4432,6124,9294,9758,4888,5462,10734,60,6955,7033,10162,10782,5704,10677,6489,239,1688,2845,7642,5557,8276,10148,4919,715,1613,7832,7782,4369,4221,10098,7332,2563,1821,7380,9670,7590,1672,8134,4047,8414,1447,4883,5300,10207,333,7805,5046,10355,10106,10787,8425,118,2057,5194,1013,785,3756,8500,1705,7621,1644,7926,5106,10337,5047,6368,8076,10556,4887,3167,1092,2286,6560,9254,10521,6051,6958,6256,9338,4372,2598,1596,10583,2798,1250,1090,3250,1193,4922,7367,6904,1032,2633,3984,3678,10579,9630,7571,4424,5891,9045,10533,3944,8151,6326,349,6091,10032,7920,5682,1625,6700,5362,10627,390,2960,10442,9501,2740,332,6412,5368,7522,7456,4776,5568,10176,10603,2834,9456,494,1184,7598,10802,6199,6212,4285,8470,5081,4933,8900,1926,6071,4553,3615,4333,10240,8739,3508,2738,6320,606,4676,6448,7062,3923,9653,9846,6235,4436,7653,6591,7352,5142,2582,141,7984,7937,1700,5360,9973,8303,2216,3392,5680,4674,898,7812,5440,10154,5908,6940,1495,6403,1960,143,10733,1115,385,2003,3658,6218,9387,3104,3224,9821,6590,6995,1119,7558,1749,6419,1201,6475,7268,2028,4645,159,4539,650,8755,914,5101,6625,10041,4939,5386,4379,10272,10567,6251,4413,1836,8361,2724,10090,9095,5561,9683,2123,5828,9016,7290,3041,6152,2523,6430,6136,10244,8247,1743,2496,1756,7074,9655,2609,168,2012,1883,7018,4120,8666,4874,6425,9334,1431,2247,6331,2119,10692,4109,5227,7218,7957,9785,789,7793,9255,3565,6148,4764,1357,3900,10483,5553,310,6566,9803,5087,1038,9601,5212,7798,97,5341,8719,2052,2879,9955,6539,7136,6190,8865,10729,3813,4393,4913,971,3881,296,8207,1344,4147,5982,8505,3540,692,571,3436,10049,4969,7746,2500,7224,8877,607,2764,9718,9457,9447,7873,7941,4775,4586,10698,3113,1684,2739,7397,8815,7939,80,1455,68,1790,324,1324,166,7055,508,4188,10439,10456,2733,3154,7294,8691,8828,5121,4478,8150,10672,4373,10763,6245,10348,9857,509,3045,1893,9523,393,6821,2257,10491,4882,1798,6020,3826,6109,3516,4486,1802,7354,7041,8285,9526,1190,2813,2082,1020,5289,10287,5967,1788,2181,7526,1913,6629,10108,161,5813,3201,10756,777,2616,9787,3171,9236,5769,1054,9576,10386,3381,7594,4648,7314,9435,1755,8289,7709,73,10073,5670,706,1298,6797,6390,7549,9558,4178,7597,10720,10254,4629,10212,6749,4997,3038,5219,9808,5984,8232,9546,9111,2455,6021,5912,2427,1924,2227,9383,7068,8845,5239,6273,1134,4576,593,9745,2640,3398,2363,10832,10258,6618,9041,10560,7100,4894,10312,3030,10057,1445,8018,1356,4009,424,7326,1568,9899,52,7740,6498,4092,2987,9882,8623,1830,4741,7803,6132,4605,10728,766,8252,6098,9594,2831,1862,5556,3050,9343,6894,2383,8358,2515,9023,8652,5581,229,2207,4205,2332,7383,1585,338,1907,3370,6464,1315,2666,3790,5170,9206,2333,1944,1768,2438,4695,6741,4242,2467,1574,6815,5502,2073,3076,9242,9824,6994,225,9602,10432,4714,9719,1233,1490,7529,2209,8915,7280,8518,9056,6763,7601,871,9777,5468,7267,6130,5156,4708,2017,3178,4626,4727,2899,2259,4527,268,1126,2690,8846,4768,990,7470,464,8962,9588,5052,498,5422,3374,222,8315,9194,5648,3740,5358,9450,5404,800,7633,7513,1911,10798,4108,6668,9880,6896,10382,972,2638,5292,2848,8203,1662,10651,7066,9075,1244,7537,5305,8307,2416,8736,6072,5507,2269,10314,8042,5527,4970,9893,4059,6257,3546,2718,1288,8132,10694,1469,8314,7328,7065,7972,10070,2689,9001,5203,7595,4017,7716,5277,4579,8558,3992,1707,5641,1676,4430,1754,739,4781,8393,10484,7806,1993,3657,1865,10642,8287,3204,8680,1675,6066,1601,1734,5936,9282,5478,7930,4559,991,10018,690,9154,5255,2655,7,190,7563,3717,7868,4358,7924,1301,4491,1780,1110,4275,2925,10548,3212,5661,1607,648,3977,377,5569,8069,6249,635,7992,9496,1881,831,9851,6923,2872,2229,5544,4541,8383,5285,3344,4031,8003,8292,3980,2591,6389,7649,4318,4167,4367,5554,9509,10507,7912,3421,7077,158,10779,1764,1975,3545,2350,2601,9141,3829,488,2532,7550,1593,3403,515,1078,1026,8352,28,5244,1436,3651,10513,7655,4609,7718,722,2066,8653,5428,2630,10421,4348,3225,2483,2390,407,4347,6548,3357,9017,3959,1311,4687,1864,3815,7860,5808,833,7853,5293,2032,4185,8576,8529,7310,5411,5448,9196,6311,1403,5260,3192,3485,8481,7769,5729,8249,9809,10407,5779,9162,4861,3747,5007,8565,150,1930,59,9012,6420,8601,1323,3382,10547,9104,2516,1866,7239,5308,6733,6967,4529,305,1914,5713,10320,4772,4140,2749,3591,5250,899,7081,9020,6409,8965,10656,9269,1896,3,3240,885,837,5062,3654,8658,7200,9568,4820,4917,3032,10576,3305,7492,394,5822,1262,4455,1266,4332,10128,8704,1943,901,4994,2291,852,5385,4703,7488,5551,6142,8051,2323,3484,900,7787,6383,5099,7499,4489,3561,3005,3672,9053,2678,2620,5480,6523,3986,8496,8673,8036,10492,3123,4932,6279,9394,2583,8766,3820,8782,3223,3939,6554,2596,6975,6208,8519,5429,5598,4556,3773,9885,5418,5269,6845,5945,6978,671,7361,4878,857,6065,3669,5153,6339,9781,7506,5582,10674,7783,1164,6454,6869,744,5571,1337,178,3159,4040,4487,10592,3464,4364,9407,3896,2419,351,5379,8019,10792,4217,1280,7422,10222,3499,8117,7432,435,5339,7143,9876,6029,6260,3573,7943,2998,4757,9359,86,4594,6384,5384,2037,806,1813,10493,10485,9639,9927,788,4945,6055,3688,8189,5401,8485,3053,6984,652,9087,1697,4410,7577,10647,7975,1225,10797,4670,4305,1554,9109,3402,9179,1367,1807,5406,7905,9802,9532,5049,1890,3281,8324,820,4227,10504,8964,6579,3009,7784,6134,516,6010,2584,574,3764,6900,6852,2336,6366,2108,10331,4628,1923,2441,5175,5310,2361,1099,7341,1540,7715,4245,430,9210,6867,7021,866,5941,3220,2820,10706,211,6392,2533,589,5389,9848,7241,1073,6699,4694,2436,9187,3328,3174,10193,1477,8517,2758,641,6327,1264,9178,3902,376,10241,517,3294,9789,3966,4568,6624,1873,7689,10014,8621,9727,679,1872,4229,1524,10454,7777,5629,2130,1552,907,1694,4086,9061,6322,9874,294,3629,3738,780,7258,3008,180,4316,2968,7545,5166,7660,7809,9863,4788,248,2393,3585,3472,6095,4522,3928,3238,4498,7439,4083,10099,8137,8053,4558,4928,9656,3062,302,4535,8590,8214,8765,5610,317,5917,4327,8714,8814,3653,2882], "time_ms": 0.104736, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 613, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "generated_query_count", "value": 1, "time_ms": 1.034163, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 428, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "generated_samples_per_query", "value": 55641, "time_ms": 1.034163, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 429, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "generated_query_duration", "value": 11861621, "time_ms": 1.034163, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 430, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "logger_swap_request_slots_retry_count", "value": 0, "time_ms": 132672.951580, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 898, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "logger_swap_request_slots_retry_retry_count", "value": 0, "time_ms": 132672.951580, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 900, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "logger_swap_request_slots_retry_reencounter_count", "value": 0, "time_ms": 132672.951580, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 902, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "logger_start_reading_entries_retry_count", "value": 0, "time_ms": 132672.951580, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 904, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "logger_tls_total_log_cas_fail_count", "value": 0, "time_ms": 132672.951580, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 906, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "logger_tls_total_swap_buffers_slot_retry_count", "value": 0, "time_ms": 132672.951580, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 908, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "power_begin", "value": "10-23-2024 08:04:43.202", "time_ms": 132672.951740, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 564, "pid": 7676, "tid": 7676}} +:::MLLOG {"key": "power_end", "value": "10-23-2024 08:06:55.870", "time_ms": 132672.951740, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 566, "pid": 7676, "tid": 7676}} diff --git a/open/scc123-Shanxi_University/results/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/accuracy/mlperf_log_summary.txt b/open/scc123-Shanxi_University/results/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/accuracy/mlperf_log_summary.txt new file mode 100644 index 0000000..a05d865 --- /dev/null +++ b/open/scc123-Shanxi_University/results/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/accuracy/mlperf_log_summary.txt @@ -0,0 +1,4 @@ + +No warnings encountered during test. + +No errors encountered during test. diff --git a/open/scc123-Shanxi_University/results/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/performance/run_1/mlperf_log_accuracy.json b/open/scc123-Shanxi_University/results/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/performance/run_1/mlperf_log_accuracy.json new file mode 100644 index 0000000..0d4f101 --- /dev/null +++ b/open/scc123-Shanxi_University/results/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/performance/run_1/mlperf_log_accuracy.json @@ -0,0 +1,2 @@ +[ +] diff --git a/open/scc123-Shanxi_University/results/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/performance/run_1/mlperf_log_detail.txt b/open/scc123-Shanxi_University/results/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/performance/run_1/mlperf_log_detail.txt new file mode 100644 index 0000000..0834545 --- /dev/null +++ b/open/scc123-Shanxi_University/results/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/performance/run_1/mlperf_log_detail.txt @@ -0,0 +1,87 @@ +:::MLLOG {"key": "loadgen_version", "value": "4.1 @ f74d16f541", "time_ms": 0.004879, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 53, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "loadgen_build_date_local", "value": "2024-10-23T06:21:10.796371", "time_ms": 0.004879, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 55, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "loadgen_build_date_utc", "value": "2024-10-23T06:21:10.796378", "time_ms": 0.004879, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 56, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "loadgen_git_commit_date", "value": "2024-10-22T15:38:02-05:00", "time_ms": 0.004879, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 57, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "loadgen_git_log_message", "value": "f74d16f54131d9080b9e45f234cc23e0ebaaf20c Apply min_new_tokens=2 to mixtral-8x7b, address #1777 (#1884)\necb880167756cb4b36ad70766b8d3254bfb06d26 [Postmortem 4.1] Make mlperf.conf static in loadgen, enable automatic Pypi release (#1882)\nf5c8f1758374aeaba26b2e84d31690111cfdf054 Fix bug: Loadgen ignoring token latency targets in user conf (#1874)\n976bb1ad9c7946be79507f3ff67955c27426af52 Set correct remote repo (#1871)\n41fa8aadd1ba0ecc97f6a519d8b42b04278e5f24 Add format files github action (#1682)", "time_ms": 0.004879, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 58, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "loadgen_git_status_message", "value": "", "time_ms": 0.004879, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 60, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "loadgen_file_sha1", "value": {"/.clang-format":"012aad77e5206c89d50718c46c119d1f3cb056b2","/CMakeLists.txt":"b73434348f7860471606aaa395b570e81113cb6d","/MANIFEST.in":"8d3c4ac6c325b7b9a0fd4cf4a4108cbeff8d5025","/README.md":"20a55bb946c2c0bbb564ced2af1e48efd096b3a8","/README_BUILD.md":"5f6c6a784e9cd6995db47f9b9f70b1769909c9d8","/README_FAQ.md":"01f9ae9887f50bc030dc6107e740f40c43ca388f","/VERSION.txt":"cb67dcc41adcbb7849a0a808a501ee9ccd951d92","/__init__.py":"da39a3ee5e6b4b0d3255bfef95601890afd80709","/bindings/c_api.cc":"32181da9e161c285f8fe46ddaa49e6cba2f9f918","/bindings/c_api.h":"91f58bd79b83b278f3240174a9af747fc38aff74","/bindings/python_api.cc":"9f538d2a5390c77ae0bc3f8a351bcdb2587bc66c","/diagram_network_submission.png":"53dba8ad4272190ceb6335c12fd25e53dc02a8cb","/diagram_submission.png":"84c2f79309b237cef652aef6a187ba8e875a3952","/early_stopping.cc":"0cd7b546a389deac73f7955cd39255ed76557d62","/early_stopping.h":"158fcae6a5f47e82150d6416fa1f7bcef37e77fe","/issue_query_controller.cc":"126e952d00f4ea9efd12405fb209aa3ed585e4b2","/issue_query_controller.h":"923d9d5cdf598e3ec33d7a1110a31f7e11527ec7","/loadgen.cc":"6650091ba7a918f343b06eb7a5aa540eae87275f","/loadgen.h":"e00fdc6dbc85a8c9a8485dbcbfe2944f81251c4e","/loadgen_integration_diagram.svg":"47f748307536f80cfc606947b440dd732afc2637","/logging.cc":"197efc96d178e5d33a750d07fa7b2966417506ea","/logging.h":"ddb961df7bcc145bcd7cce8c21f7cf075350dcbe","/mlperf.conf":"0a4daef277bb3151139980e484dd5e644bf36e18","/pyproject.toml":"712fab87b72ba67ef2a068d0f9f47da65130342f","/query_dispatch_library.h":"13ad6d842200cb161d6927eb74a3fafd79c46c75","/query_sample.h":"e9187c8612bbdc972305b789feb6e15c26e96cfe","/query_sample_library.h":"8323a2225be1dff31f08ecc86b76eb3de06568bc","/requirements.txt":"a5ff7e77caa6e9e22ada90f0de0c865c987bf167","/results.cc":"34e2d2a44324cb07c884f92146ecbb8ef9d704e2","/results.h":"fce22d5a588d91fd968a6b25c27896dba87bc276","/setup.py":"a722046e05858c6d9f38f0e2b3fe425334beef28","/system_under_test.h":"18d4809589dae33317d88d9beeb5491a6e1ccdec","/test_settings.h":"476ecd4032f3bafe6f201df25d68aca4e177f659","/test_settings_internal.cc":"ce4322c849d24ffafc28a37b5e528a4cb4df227d","/test_settings_internal.h":"f1d5335b53ca610c30e0edc5d07999a27b5b4b9a","/utils.cc":"3df8fdabf6eaea4697cf25d1dcb89cae88e36efd","/utils.h":"40775e32d619ea6356826ae5ea4174c7911f6894","/version.cc":"cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0","/version.h":"9d574baa64424e9c708fcfedd3dbb0b518a65fcc","/version_generator.py":"eea9b9cb1a06cd1abe1bbdaee82f9af31527fedb"}, "time_ms": 0.004879, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 67, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "test_datetime", "value": "2024-10-23T07:54:21Z", "time_ms": 0.019617, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1198, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "sut_name", "value": "PySUT", "time_ms": 0.019617, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1199, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "get_sut_name_duration_ns", "value": 371, "time_ms": 0.019617, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1200, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "qsl_name", "value": "PyQSL", "time_ms": 0.019617, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1201, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "qsl_reported_total_count", "value": 10833, "time_ms": 0.019617, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1202, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "qsl_reported_performance_count", "value": 10833, "time_ms": 0.019617, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1203, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "requested_scenario", "value": "Offline", "time_ms": 0.026871, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 271, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "requested_test_mode", "value": "PerformanceOnly", "time_ms": 0.026871, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 272, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "requested_offline_expected_qps", "value": 77.0195, "time_ms": 0.026871, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 309, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "requested_min_duration_ms", "value": 600000, "time_ms": 0.026871, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 315, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "requested_max_duration_ms", "value": 0, "time_ms": 0.026871, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 316, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "requested_min_query_count", "value": 10833, "time_ms": 0.026871, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 317, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "requested_max_query_count", "value": 0, "time_ms": 0.026871, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 318, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "requested_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.026871, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 319, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "requested_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.026871, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 320, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "requested_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.026871, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 322, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "requested_accuracy_log_rng_seed", "value": 0, "time_ms": 0.026871, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 323, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "requested_accuracy_log_probability", "value": 0, "time_ms": 0.026871, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 325, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "requested_accuracy_log_sampling_target", "value": 0, "time_ms": 0.026871, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 327, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "requested_print_timestamps", "value": false, "time_ms": 0.026871, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 329, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "requested_performance_issue_unique", "value": false, "time_ms": 0.026871, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 330, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "requested_performance_issue_same", "value": false, "time_ms": 0.026871, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 332, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "requested_performance_issue_same_index", "value": 0, "time_ms": 0.026871, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 334, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "requested_performance_sample_count_override", "value": 10833, "time_ms": 0.026871, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 336, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "requested_sample_concatenate_permutation", "value": false, "time_ms": 0.026871, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 338, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "effective_scenario", "value": "Offline", "time_ms": 0.027041, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 417, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "effective_test_mode", "value": "PerformanceOnly", "time_ms": 0.027041, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 418, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "effective_samples_per_query", "value": 50832, "time_ms": 0.027041, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 420, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "effective_target_qps", "value": 77.0195, "time_ms": 0.027041, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 421, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "effective_target_latency_ns", "value": 0, "time_ms": 0.027041, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 422, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "effective_target_latency_percentile", "value": 0.99, "time_ms": 0.027041, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 423, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "effective_max_async_queries", "value": 1, "time_ms": 0.027041, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 425, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "effective_target_duration_ms", "value": 0, "time_ms": 0.027041, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 426, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "effective_min_duration_ms", "value": 600000, "time_ms": 0.027041, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 428, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "effective_max_duration_ms", "value": 0, "time_ms": 0.027041, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 429, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "effective_min_query_count", "value": 1, "time_ms": 0.027041, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 430, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "effective_max_query_count", "value": 0, "time_ms": 0.027041, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 431, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "effective_min_sample_count", "value": 50832, "time_ms": 0.027041, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 432, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "effective_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.027041, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 433, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "effective_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.027041, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 434, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "effective_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.027041, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 436, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "effective_accuracy_log_rng_seed", "value": 0, "time_ms": 0.027041, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 437, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "effective_accuracy_log_probability", "value": 0, "time_ms": 0.027041, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 439, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "effective_accuracy_log_sampling_target", "value": 0, "time_ms": 0.027041, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 441, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "effective_print_timestamps", "value": false, "time_ms": 0.027041, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 443, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "effective_performance_issue_unique", "value": false, "time_ms": 0.027041, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 444, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "effective_performance_issue_same", "value": false, "time_ms": 0.027041, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 446, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "effective_performance_issue_same_index", "value": 0, "time_ms": 0.027041, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 448, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "effective_performance_sample_count", "value": 10833, "time_ms": 0.027041, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 450, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "effective_sample_concatenate_permutation", "value": false, "time_ms": 0.027041, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 452, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "generic_message", "value": "Starting performance mode", "time_ms": 0.035286, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 841, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "loaded_qsl_set", "value": [4119,3771,2947,5174,8221,10562,9791,4633,8569,8837,7091,5810,6371,10778,540,1732,2832,4085,676,6115,887,9339,10819,344,9312,1951,5739,9692,9525,4397,638,4375,10601,8825,6906,3471,9163,7272,9940,1207,7738,1663,9660,5625,2468,396,7153,8514,1418,8510,4187,1103,9970,10200,2141,6364,9748,8412,2719,5615,2838,7835,752,6099,10486,8982,3673,7292,10573,3118,9998,6825,5831,4451,9259,4859,1199,7842,6587,920,6687,3869,1033,6262,1223,30,5009,10063,2956,1007,2185,64,77,3470,6239,6679,1470,62,3074,1622,8153,202,9235,1614,9392,4912,4968,1648,122,3772,902,3526,1772,858,265,5129,6799,6255,523,10796,8819,1230,9400,3732,1321,7882,7821,9768,6056,4282,4617,6107,10130,7756,3036,10451,2201,3976,1086,3668,5394,3177,5128,8384,7234,4039,8503,3640,10043,5766,6810,1011,7547,1718,2401,3909,8343,5923,10069,9886,532,3270,7147,10062,9680,5432,9974,7736,3655,811,1105,1820,7465,2267,9641,4046,3956,5247,5181,7625,4073,9663,288,426,6762,1617,7914,4014,2904,9151,4181,6042,8967,6981,5547,10457,9322,500,938,131,8457,1171,3814,10549,8093,10675,2159,5807,7567,8366,6518,943,6819,5889,5108,7755,5382,8456,9756,10247,9351,3150,8971,1231,2990,5380,510,4324,3049,1165,6116,8684,6992,1829,139,1021,1602,5805,4226,5966,1040,3274,4269,10376,5316,8756,9587,8231,2809,2270,7887,7390,485,8028,7880,328,6236,514,4644,6061,1522,10537,9446,8787,8577,6774,2023,7474,4904,5064,6183,5196,3361,8268,7606,5968,6182,5019,8265,4621,1525,8181,163,1352,5785,8632,1074,2392,8398,6561,125,7071,5577,995,3783,3849,9753,8651,1051,1159,3971,8903,8413,162,5854,2133,1416,9098,1759,633,5800,7671,4731,10623,5586,10578,8929,5894,6833,2909,5063,3163,5002,6278,4148,7477,8408,10104,8805,1504,8105,5746,3572,8095,312,7436,6165,2786,6286,2079,4807,10523,6117,3840,4169,8750,3016,10238,1137,10037,9427,8449,4908,3020,7183,1557,10500,2982,8020,599,1519,6873,10276,7350,8224,719,8474,9981,9198,2802,5756,1897,10801,7682,91,4160,6697,2989,10113,5765,982,48,6573,4412,5148,7681,2514,6348,620,6702,250,10219,7813,4512,3988,7538,3272,4433,4597,7580,8229,603,9284,9959,4683,711,7542,5450,1175,9859,10685,5041,9760,765,6303,3395,2908,7823,7516,237,105,8034,1757,10401,9935,5256,7800,2642,276,8636,5297,10766,5954,5725,8144,6660,3901,8078,4158,9455,7297,5959,8088,8583,4733,10345,5232,9216,8902,4667,1449,801,9231,1792,1211,6013,8566,4988,2938,4730,404,8699,2346,3059,672,3701,7231,10169,10431,5806,1373,4193,10665,10030,4335,7950,10150,842,5753,6435,5096,5922,5497,3193,9367,2158,3816,5356,65,258,4675,1277,2041,6032,10088,7814,8726,2036,7885,9303,7440,8023,1151,5749,9573,7776,572,1240,6297,6439,1343,8605,3386,3251,4631,8140,3760,2068,4957,2623,4961,10281,1467,9037,10546,4415,215,2194,3273,387,3327,2225,10739,300,4467,10823,10668,710,4228,4517,2069,8709,2409,2573,6144,9357,4844,3985,2864,2814,2615,8952,5085,5090,1937,1854,10689,5061,10374,1580,5223,4851,7888,869,3023,10178,1615,7566,7219,5276,6685,8839,10534,10532,5826,1251,9124,8308,8702,1805,3148,3529,9942,8171,2450,6782,7679,2503,9989,2748,3961,3660,9466,6832,1635,1831,9298,6457,3165,6633,6513,10354,7865,1888,1182,10643,5477,3120,7676,6178,2833,9468,4910,2657,9723,492,3347,6481,2822,4426,1018,486,826,3455,1535,1539,5782,8267,10102,2714,5573,5482,2442,537,6934,9420,1925,10526,8158,583,5921,5634,10715,2063,2865,4722,6711,3033,5830,6761,10221,3292,5093,3487,9844,5192,3127,3166,5517,8842,5986,6515,8806,4368,3880,682,1506,8542,6600,5522,10789,2775,6979,7375,9732,9488,7435,6127,2058,8763,5776,2901,551,2871,7369,6929,4164,307,7840,861,2670,8027,9410,2386,2513,1063,5122,8218,4457,1100,2437,462,5774,9629,10033,5794,4106,3616,7179,5414,4598,9925,9289,69,9948,9945,4585,6214,1953,6363,7744,6030,3012,37,4855,10794,2109,5500,1652,9019,7047,3763,10114,2735,4137,7192,6450,7896,6319,2559,9057,4150,1626,1399,3805,10481,7054,3160,3802,2753,643,4212,10435,3360,157,8424,6667,1686,8112,1983,6648,9535,2567,6149,8099,319,10795,2980,8559,3326,2116,2360,7544,5778,2589,3310,3751,3352,4366,9705,368,6088,8217,2399,6537,10120,6669,3061,1457,5252,7052,10159,6880,5593,8852,3506,5847,4877,4944,8609,5364,1816,5290,634,1661,1221,6935,702,4824,1453,8062,7092,345,7030,9428,204,3121,2188,6429,7446,10631,2354,2569,1219,5475,5727,6274,1556,8968,7730,10342,7917,1985,6856,5692,7132,1146,10468,6861,1550,2928,6585,5681,5352,5348,10265,10307,322,9139,8357,9486,7441,4023,10141,10170,1441,165,9603,5998,10586,9022,5633,1234,6283,2131,7996,5042,6246,4201,3064,3071,5674,6769,9698,386,9319,10686,4,9197,3510,428,382,6614,7688,6154,10078,2876,8185,5240,1006,8696,4353,5760,7961,8376,6405,2107,5804,9860,3812,1679,2756,9652,7964,750,1997,4091,1899,1308,4035,6656,8123,10248,1510,2846,5638,477,4076,10730,10589,8266,3218,4977,2536,5334,10054,8762,8598,3929,3811,9581,7752,552,9441,5711,6642,1129,284,976,10747,933,7929,7607,9805,5102,9979,2939,6910,2647,4835,7528,4876,6734,2706,4394,4926,8341,8734,1135,3753,9333,1291,6011,3867,5563,3343,7612,1027,5165,9700,6340,6104,3099,9537,3428,1612,1606,10634,10021,2761,1042,9310,3601,6806,4003,8336,1320,8874,6545,3842,3715,4916,177,6141,922,10477,10369,3372,8706,7261,9000,2944,1770,4537,8671,8299,4515,6709,7300,5017,1499,3864,4145,8725,2669,6292,6724,7960,6080,4257,743,1363,10444,4518,9545,10388,2296,3870,3737,984,4590,5924,149,1966,10084,6018,1340,8785,1678,4211,8663,6542,1475,6302,7202,1850,9261,10681,2339,8461,2664,7011,3441,9549,1482,9314,6828,1290,9203,7486,209,5215,2113,714,6092,3889,825,7966,5609,3787,6223,207,6965,2644,4612,9140,10280,472,5726,5208,8772,2044,2931,1775,4134,10539,7508,172,8681,8240,10189,4870,8908,7684,9379,3266,3376,8405,10311,453,8213,2034,2190,2294,8580,4421,4725,3952,2918,8999,8545,7043,542,10301,9146,9032,3597,4112,2062,3195,2850,6169,10805,4900,8382,2059,5703,10074,7541,10662,1760,2114,10749,5993,6295,5211,7792,4639,8333,3833,1847,83,10770,8312,8831,9004,371,4923,7699,3307,7834,5879,7458,6735,9799,10741,1304,1576,8657,3329,8860,6081,1744,7308,10564,7102,10611,7126,10413,6653,6846,1371,6864,2078,8362,3021,3569,10777,1939,2861,8085,4899,3288,7360,7690,1489,5526,116,7151,5145,6673,5523,9390,8191,1004,10528,8958,9118,7393,4684,7266,6966,5995,8508,3778,313,2324,3282,10165,3447,6294,6865,9833,2202,6316,365,3052,7057,10821,1194,7974,1410,1782,9986,1123,5591,7286,7552,10721,1088,1793,6524,117,1763,8854,2199,3366,6230,6135,4113,4154,2326,7696,10438,4546,6268,4991,8670,9515,9500,6272,5253,2634,10092,965,6654,316,7333,10361,7385,2359,2521,10505,6418,3731,6452,4151,2002,4407,9193,5715,5311,9681,1461,6752,10196,9063,7064,7760,10356,9534,6401,9570,6213,7243,1969,5069,4463,8379,2228,10617,10563,4001,7533,4118,8634,303,4758,6480,6863,4079,3551,2387,959,10384,8204,3739,4388,9571,5235,890,3448,3380,4022,3407,3303,8225,1710,3134,8655,8273,1442,9126,9090,767,2352,5036,6959,1116,7507,8320,206,5100,10580,4657,10013,591,5970,3983,10637,3489,6177,4329,9309,598,3799,7447,4206,7727,6138,480,3483,3979,1719,5575,5814,6507,10156,4769,9437,3749,949,10166,9129,1387,7559,21,3606,2329,10379,71,6909,4268,3231,7838,4403,4880,2289,4825,7313,3855,9482,5669,5755,1472,8584,145,7282,4105,5546,9644,1811,10620,8543,8840,6823,3084,9331,7771,5570,2129,3333,9811,1567,9316,6617,7070,2971,5115,9806,8920,6889,6597,9550,7991,10076,1289,8872,1156,963,6369,353,4520,4406,4510,327,2672,10123,881,4197,10616,7244,6791,4055,9060,7901,4398,10019,5895,6816,724,6400,3142,2649,727,7658,4411,4131,4893,3203,6901,5131,5905,7199,3408,4562,101,9214,54,2388,5032,6645,6310,973,7184,5595,4177,3578,10735,6783,1468,7622,854,1425,1272,8441,10581,2009,3999,4258,1929,7531,8205,610,8733,2698,10366,9817,7227,10638,1446,4587,10052,9114,3354,10140,677,5614,10068,3206,10826,5460,3063,1093,5083,10551,502,9391,7170,6267,10812,1555,9356,8910,4623,3970,10429,4096,9172,522,7591,8773,2791,5612,3922,6555,8079,6802,5491,4135,6640,7203,373,9221,2355,7947,5119,5940,5363,10443,646,8044,3473,2214,5335,5803,10138,5180,5525,927,5272,2372,2951,526,7209,5972,1753,6877,1496,185,843,1124,8942,5229,1918,10371,1282,6424,10228,3253,354,7137,9669,5618,2061,632,986,1991,9433,5261,5130,2246,6315,3835,5515,2976,187,7121,2230,7683,930,10571,2538,3106,1630,8453,5267,6534,6349,8372,6532,7725,8522,8596,4984,6027,1483,2281,10284,4300,7155,8557,1842,6925,251,4156,2883,10632,4008,8994,3844,3112,5932,6837,4987,4751,7133,9724,4981,6781,5405,8770,1012,9963,9551,9840,5471,3562,454,4696,198,1902,1516,8943,4082,7253,497,281,8464,3677,8635,10172,7900,5238,2747,4914,2487,9376,2618,6526,9801,463,10039,1226,7307,4673,7525,8661,1010,6727,9665,10411,1307,4267,6543,9564,4793,9552,6627,3638,3785,2658,3384,3158,9720,9818,10790,6483,9067,669,3930,7843,5427,7587,7536,7029,3125,3531,6385,1565,8749,1736,3512,5719,4868,888,8255,8535,6194,10174,8304,5939,5070,6040,588,8437,10759,8538,8206,1458,5159,3997,1259,4858,7176,9035,10403,2868,1122,5888,9842,4622,4261,10673,9324,8487,600,9479,6220,3419,195,2155,399,287,9830,9883,9328,2055,7836,10511,1451,6850,5911,3337,9778,10660,4739,5981,8347,3588,9865,4238,8957,5074,1691,6813,3860,3883,8878,7674,8731,7336,9136,7178,4565,7820,10508,2051,8164,823,7624,2648,3128,7150,7687,4830,10350,8119,416,735,4601,5587,120,2691,3920,4192,7125,10053,25,808,9563,4299,10112,5112,3643,5446,7403,8391,7114,1542,2628,8911,9279,2886,3221,10056,1912,3291,5819,3830,8896,10788,4000,5815,1183,355,9074,8086,5140,10657,2345,2550,8728,8468,697,10557,10322,8004,1473,1698,4574,4635,1921,261,5606,3409,9595,8966,5447,4599,6622,7714,5381,2778,9396,7374,10654,1346,5113,1536,6553,1306,1932,3216,656,10615,6053,96,4447,8775,8624,8685,2235,7956,4390,1420,220,6145,2255,2478,2730,7084,1155,9091,5757,8924,2955,7628,2093,1334,1068,8013,1927,7841,9731,10436,809,6519,5217,174,2169,7952,10341,1776,10629,3169,8108,3635,5671,8667,1653,10761,1309,3659,7039,8056,2827,42,524,10646,921,2283,449,9983,2561,4542,4260,1725,9512,4540,1378,7252,2161,6347,937,8455,7862,5008,10109,1246,7697,6176,2816,1824,1746,734,7320,3564,3892,9662,2526,1553,8188,7826,3837,429,6706,2945,9511,3466,9636,6688,2453,8254,8438,8321,2699,5091,3521,3229,9219,4792,1859,9449,5190,8178,3073,2781,5817,8737,1855,2849,3602,79,6723,10289,8429,3626,7006,5884,3852,2315,5409,8693,1762,4978,8377,9892,3182,1361,2626,2581,10619,115,7025,9240,2236,2317,3641,1835,7551,4979,7791,6767,8032,7159,3000,1145,2175,549,3987,1748,8906,7038,7915,9995,8563,5741,9152,5391,7122,7933,5747,1511,8215,948,15,1029,6760,6874,6170,1801,9164,9377,8369,3748,10277,7497,7145,4724,17,2694,1041,1682,3788,8184,1015,8296,2167,4999,4809,10396,9853,4018,2086,10584,10089,3002,5841,5318,689,7581,7680,3427,950,10466,2087,9890,7666,4523,8530,1538,10784,247,6301,7460,3972,684,7646,8883,8024,1238,6393,9716,618,5104,6728,6168,7797,10273,7164,670,10252,85,6602,1693,2527,4274,3552,2284,4290,6156,8784,274,1620,9382,5000,110,10290,1326,4785,8644,9321,8743,9,1878,9711,9425,8764,6691,4962,9498,10727,2959,5262,10066,372,1336,4845,7753,5342,2047,4024,2708,7710,8639,7472,8668,337,3094,4172,9903,7644,10831,10082,4625,3043,4471,9277,1133,9577,1523,8489,1841,7878,2932,9616,7479,2448,8165,6033,4630,1946,5486,9327,9442,6824,2104,6855,8319,8378,6456,6623,3915,988,2662,7009,1331,4094,2812,796,9735,2396,3797,9481,8897,3743,8803,1808,8879,6637,9233,3703,10278,3136,10094,81,2825,23,5436,5321,3824,4891,10231,1138,6722,5433,10553,2430,2984,1167,7110,9750,10009,8406,4659,8768,5678,8435,6988,1727,9161,9051,8582,4297,1815,5707,4350,243,1708,567,4490,9397,4652,10440,667,2578,9330,7469,8600,7356,9971,8486,1243,9547,6352,8556,19,9780,4502,2964,10460,2851,7626,2804,3746,58,2929,2472,4481,10015,4514,5012,6768,8136,7804,8161,5094,1206,7412,87,1022,3011,1545,6527,4404,1587,219,6490,3575,2132,2170,8614,4632,1562,7478,1779,2568,2949,10606,5660,1261,8182,8310,9657,2565,10346,822,2574,9905,4513,942,7786,1994,3533,6809,6583,3425,6015,7013,6397,4259,6497,9677,2497,3363,1940,4265,6946,3088,1401,10722,10732,5925,2685,6250,3914,4958,1876,10051,7221,26,84,2499,10705,4400,3723,5910,10250,5623,6564,4077,6725,6325,9633,9992,437,9142,605,3683,8832,8260,9272,4071,5646,3230,3596,6495,5696,4247,10351,5313,5373,4495,9682,9607,6146,358,4643,8625,3709,5207,5780,2967,5613,9068,3608,10710,6708,1898,5621,3131,4750,10125,3496,103,5264,6034,10680,9190,1349,3828,5604,9517,9102,8439,9832,8173,6119,3318,5786,3492,401,9618,6737,9246,10146,3680,7759,8436,1478,3378,3003,1660,925,9796,5173,1044,2008,3475,865,6017,718,9793,1592,9430,6229,529,7481,95,7128,9088,8572,4171,2859,8960,8950,3438,7131,5,8015,10358,67,8337,807,6106,1295,6866,9463,2720,3919,5147,1892,9815,10195,9138,266,3848,3780,1271,9627,5332,1521,10639,3556,9904,7271,3891,1213,8989,4234,563,4263,3480,10700,5545,7916,9350,9786,6511,4021,6922,1984,4254,755,6461,3650,5958,5689,3435,7761,3648,8174,8881,2716,6508,6386,5843,3153,6167,5457,8103,7101,5161,7225,3116,6335,6232,3624,6746,7037,10058,489,7847,812,6876,2727,7260,10814,4133,9238,10395,8947,4124,4669,875,4427,2431,4726,3733,3001,3539,768,8472,4334,6094,7578,6868,7276,9502,6084,7087,402,9412,9610,10335,6365,2290,9256,6903,7270,1972,9424,10448,3560,6410,8419,4920,5914,5584,9059,9529,562,4294,5088,6380,5179,5616,9946,6684,4795,4616,9561,7288,1987,7946,7108,8662,5201,7766,6575,1062,3497,9869,9122,961,8317,10115,4600,1059,8,6879,3847,7515,9775,3184,3741,3809,341,9223,6047,8395,2605,4141,2368,1109,181,2919,2252,321,8946,2397,1328,2189,2233,8146,4925,3129,5624,8067,8220,2629,8591,6744,8973,3534,1933,10737,10754,1081,4428,7665,2095,5946,8168,4176,8325,5243,10191,5745,627,2313,5089,9157,10175,1803,8250,4839,4344,1397,2785,8976,4182,7627,2439,8767,992,3594,818,2102,2101,7732,2136,10045,6248,1701,7480,10187,9937,3843,4786,8259,7733,7721,5053,6924,7265,10529,10608,8077,7425,10765,813,4107,5464,8698,8647,3665,581,980,8450,10412,9757,6155,1706,1941,8281,6977,5637,7609,8901,10709,5763,7148,7382,9695,2239,2800,9120,5495,7097,3339,9044,827,50,1384,5652,4019,9507,9295,2200,7869,9741,548,5114,8388,2264,1407,4688,395,5627,7012,2351,9849,3137,6676,1406,8049,10738,7938,6844,8757,1906,1720,824,152,1742,32,8444,3287,6875,521,4929,860,1248,7532,9617,94,6945,8107,8890,2118,7189,6022,154,1980,3854,3140,4681,4570,1392,10217,4753,9345,8277,6969,10410,6610,9759,6796,742,9170,7334,7780,717,5580,5154,4276,4306,8589,6427,10628,915,3066,1846,10121,10220,2508,4302,1543,4611,10453,2889,7579,5234,4531,3198,10398,10020,6225,5270,10618,1303,10518,8779,10461,891,4256,10506,5200,4219,9264,8721,3488,2880,7936,6805,2890,8052,6522,6509,2171,2995,1433,1839,10227,3882,6721,3807,2408,2490,1655,10605,7259,1887,1633,5018,5907,10151,3302,7634,3450,7198,10367,5511,8730,747,9227,1485,916,2122,3422,1130,5716,2056,9262,5693,6798,6626,8951,2751,4995,8045,3931,10260,3342,862,1645,3349,8241,7103,5082,6862,6982,3208,7729,4230,8507,2794,6525,5350,6103,3755,6140,7475,3405,4271,9560,8504,10318,4354,4804,4034,7982,4743,10081,5034,5204,4456,2219,5283,9225,10570,2549,7837,8810,7726,8525,4911,3469,1584,1560,2451,5714,10472,1089,7401,4989,4385,3767,8397,5864,4789,3259,4890,5461,5320,1786,1140,3776,3696,9036,5564,2049,6468,9156,10126,1935,234,9858,9891,8536,1209,4892,8258,8291,6581,6296,10502,3412,10768,1198,188,4816,2370,6120,8353,3903,2653,4473,9807,3336,10450,336,9875,10364,5056,853,9406,2684,2793,40,6549,4146,5640,8555,5304,7418,471,8630,5934,2035,8983,3445,290,3371,4401,8316,10649,5231,164,9503,7449,5802,2757,8777,816,3911,2,6857,9514,960,1621,5752,1908,466,709,3191,9230,6834,6411,6592,7152,4715,8364,918,7236,1398,262,6157,6360,2921,7615,5288,8106,8849,3554,10572,8991,2539,5869,9213,1216,10205,561,8327,7083,6887,3060,4662,10044,4191,6757,6073,7207,560,4918,5459,8732,10017,7135,7652,5535,7610,495,273,5312,3205,6574,2417,584,5836,10755,5353,759,3625,8620,420,7973,2307,66,5398,2085,10086,723,7014,519,5236,5632,8553,6916,6998,8152,1275,3831,10010,10463,838,2524,4967,9471,5851,6093,4443,4646,8309,7431,2676,10055,6830,4125,3918,5589,3010,6421,2099,6933,6436,72,8166,8163,10515,4243,5788,450,10803,9002,6970,1769,2469,3200,8104,9015,1690,6937,855,6928,9291,4941,839,9930,7156,7706,4081,3157,8792,8833,10389,3563,10,4377,3576,4697,7293,1518,6860,10329,9176,9462,8131,9614,5770,7315,192,8974,6949,9467,9960,8516,6686,10083,9837,4736,5790,5619,3704,5021,9476,2830,7667,7796,4310,5560,10482,6927,1338,9168,6215,199,6636,3334,736,2197,4126,5777,4790,8780,8041,4551,9267,2103,7162,828,8017,7190,9703,2127,2927,147,7206,2050,9966,3873,8688,7673,5456,135,431,673,7177,6377,5127,4934,6254,5626,5899,4771,473,7437,1169,6379,6983,2858,196,1437,7010,9086,3399,10046,6596,8526,9415,9897,111,5043,4298,8918,6621,935,6991,3676,9578,6304,4155,9749,4602,7003,2347,2356,3803,4396,3893,2328,8008,9080,3092,6,3868,5686,6187,8753,4117,3827,7839,7802,886,7335,6504,5407,8562,9025,8690,2280,7174,6455,4088,4782,10711,4028,9609,503,9260,9167,1101,6139,6477,4806,9054,8853,6694,2675,6907,5095,3365,10135,2635,2179,6345,3916,10650,5588,3946,7935,9270,8573,1954,2668,7883,5073,9113,7274,8506,5789,1170,3817,2025,2558,2930,6677,1237,10535,3048,5337,5718,4716,3595,5829,4484,1296,1869,1917,8270,1571,2788,5991,1215,1784,4026,10806,3362,10767,8060,7707,4475,9900,10469,10163,7968,647,5425,10308,6002,6493,1364,5762,6698,2545,3856,3924,8431,6247,4392,6777,9694,9804,2686,6563,4577,5772,3144,6822,6884,2836,8850,8094,2821,1948,9586,4533,6110,3712,4199,9103,129,10164,1348,4064,9565,6125,4138,6980,847,5975,169,10600,4323,10072,5913,6375,8035,8155,1563,8328,7201,9934,2410,9856,4423,4637,2398,6290,9363,10699,1833,4767,6129,7556,1208,3982,6244,191,7024,3183,1735,2193,10317,10626,505,445,10257,2710,10330,8720,2534,1699,7061,275,3942,2477,4419,10426,1517,2622,9286,9030,7554,179,7188,5937,9497,1586,9211,5323,6521,9632,999,405,8373,9292,8812,4986,9445,153,3298,2973,8030,9150,7407,2234,863,1548,5328,4231,1669,8363,1276,9774,6514,5054,384,802,2896,940,8293,5167,4544,10091,8984,4821,7220,7959,9005,9249,4416,4721,5592,6076,4389,45,6716,3519,10820,6719,1781,3161,5442,1313,7141,9096,10636,9843,4279,7790,10409,6433,3351,1938,374,5402,9271,7075,2805,5431,7325,6565,2773,7340,2485,8318,8193,461,5345,8187,412,7342,2476,1702,4996,8452,5548,7214,2260,5900,10525,8305,1342,7913,253,2981,9746,2958,1421,2627,1341,9765,1549,929,1840,1178,2466,9353,3252,9403,6487,4854,597,6938,6488,286,6792,762,5919,2092,2884,1724,475,6827,6211,1333,9454,6986,13,2168,1391,2067,609,5116,7378,4960,9290,1657,9620,5336,8686,3243,5885,7257,6961,201,9658,8788,3097,8502,4805,10487,911,7829,5956,5366,8415,5651,1578,6314,4237,2562,5501,2366,5504,9881,6217,4973,3028,5307,9605,1460,5583,9812,9281,8741,5133,5521,1791,4066,2285,9188,9954,7788,9717,5022,5218,4311,790,9089,2367,7997,10489,2272,5410,99,10061,8346,4930,6835,6048,7535,3091,9401,2593,2937,4829,10714,1996,10064,3424,9182,941,9928,1082,10347,3145,979,9133,3454,1711,946,7088,7078,5079,3933,4365,5938,3905,2650,1863,10688,6729,7242,6905,3301,6078,7467,4208,4240,102,2898,7866,3908,0,2320,4564,9645,3130,5168,730,9889,1750,5874,8513,2268,5242,493,8981,3411,8873,9200,6005,9460,5665,5708,9797,6328,1036,6745,4405,8295,1758,6789,468,10742,10566,6780,8567,4284,7106,3888,6747,10134,6950,10079,4363,7898,5374,1257,7745,2380,3605,895,2687,3690,6462,2529,8692,9247,7395,3283,8669,9707,9205,7983,6253,3503,7919,1053,704,2244,6779,3846,6993,3850,4924,1971,10387,240,7572,8863,7864,663,4581,1409,2378,9100,10684,1465,8200,5679,8430,5866,10602,4143,4065,7157,8933,7763,756,8091,2488,3115,8248,4356,4813,2288,9687,1393,5999,1844,7255,5349,10830,4116,1112,8544,3895,6576,8194,8834,10391,7969,10713,270,8909,4111,137,9459,7322,10103,8581,3155,7142,7191,5400,8913,8209,2912,3434,9773,879,4719,2807,9704,53,6242,6241,10666,1322,7298,10008,6086,680,8047,8898,5265,4202,5467,2493,7408,4157,10255,7722,4414,2878,8074,7421,9232,7063,8111,1670,10266,8674,7217,10279,2979,5574,6269,5317,10415,5184,2429,6674,419,1028,602,3727,7429,8638,6839,4843,9767,9621,5347,8133,7694,5617,1227,9555,3029,7363,6500,1191,3770,569,9048,5524,4940,10368,3132,7358,6003,9402,5732,2310,9557,4538,4345,8550,2484,7195,2504,7417,9600,6031,5846,6812,2555,4214,6655,8537,10751,6918,10757,8349,3244,4011,2276,1882,4060,3935,5393,7871,8386,7808,6472,3279,9192,7420,5157,8294,9566,3586,6818,3461,2903,10582,4755,932,6547,7731,7116,6503,4569,2923,910,3633,10723,9597,7027,4013,318,4554,10480,4935,6482,9895,3968,9779,6338,5187,3697,4582,1668,5137,8926,2100,8939,421,7079,8746,9413,4702,4901,3314,4101,5505,4007,1945,2963,5969,4783,1783,9823,256,4620,9153,9013,4927,9344,381,3998,9990,9381,1974,343,1916,7817,8445,664,7711,1362,956,5597,4417,9924,10746,5499,8223,10152,10186,7344,10783,919,5740,2709,10804,3018,9651,1095,6801,2717,695,1039,4704,1968,7498,5877,6665,6062,6342,1369,7364,3518,1077,3477,167,6184,5735,10006,7140,142,10370,7238,8799,8729,8082,9052,7067,8401,10349,9366,2312,1977,687,2772,2054,1413,612,3524,9634,9184,487,2026,2950,10136,2795,9524,4779,2019,5694,9852,557,4399,3006,9302,448,9064,5979,614,8551,10117,5001,8813,2045,2004,1154,2660,2894,10769,10256,218,4627,5322,6193,4038,1047,10321,10185,8329,7893,3951,2841,5700,10545,1978,8269,7370,7944,1003,7894,5343,6838,3515,2592,5720,8210,4921,1411,8822,5033,5904,726,5850,10080,7768,8330,1761,10253,1999,2046,912,7247,388,909,7637,2885,6014,556,9409,8055,3013,2750,356,2013,7530,10229,7411,7058,3058,5964,661,5758,5075,9836,6536,7720,10652,7524,4127,5014,4102,7705,3151,5532,9841,10340,7154,3284,7345,3375,6469,5518,4207,9845,400,1689,9398,10817,8610,6210,1751,10381,8534,6921,7999,6198,579,5124,5576,5607,5123,2421,10295,936,9870,7568,7811,7124,7907,411,3853,9183,106,6917,264,10050,10827,7105,8068,10781,1848,8039,6663,2847,9257,216,1113,4974,3196,611,304,2473,2790,10718,8641,10239,6376,1673,6374,9374,10406,2266,1651,9275,1450,5199,3493,729,8786,3965,2164,10365,10704,10208,1957,9965,9217,10139,733,5282,8080,3543,5275,590,1160,3542,9855,2518,2115,4956,10293,406,4864,5139,90,2580,370,9999,4425,7981,6406,4233,1031,2241,3237,1368,2321,6915,6584,6024,9548,2178,6387,649,754,8490,3414,7263,9642,2974,1402,9436,1217,3607,1,10445,2177,5224,10785,6035,7483,4860,9638,2206,1664,3234,2369,9648,2988,4099,9207,8862,1325,3217,9838,24,8606,6913,613,4303,5931,2463,2389,4701,3517,3541,10703,76,3694,9364,1845,8650,8520,7539,1716,7934,7906,1623,8466,2147,8175,10031,6726,329,10339,4814,7099,3759,10691,2464,6133,7713,3511,8759,8141,8368,10168,10610,9278,2522,10242,8130,224,2991,7810,2665,1142,6650,7115,5876,7737,5600,1412,8703,2590,9123,9408,9072,7819,10100,8038,5736,4266,2379,6486,7044,1502,484,4061,7076,4856,830,9697,361,8006,4689,9128,6434,5473,4976,7287,6089,9453,8501,2261,694,1665,5015,9854,9116,6001,238,5344,7822,10075,520,10527,944,5415,5578,3430,8048,4248,2837,5481,4618,1894,6009,6394,6147,8147,4351,4142,5620,3634,5536,8907,8980,3468,5449,7897,2920,5155,5395,3019,5222,2840,6755,7988,2654,4044,6114,3682,666,1302,737,3689,9752,1569,1785,6357,1114,675,108,10494,3388,513,2607,7824,7604,8675,6630,8212,5132,4706,9440,8955,2474,997,3170,7381,5811,5423,9952,4352,536,6807,6608,4381,3613,6090,2043,9993,8022,6463,5233,5538,3567,8628,10107,4752,2176,2510,9131,608,1934,10326,6049,7473,10002,9980,9325,8922,173,4485,2029,6396,8838,3465,5068,8116,1752,3938,2702,3111,4095,2364,5833,3051,1076,9518,9346,9961,4445,10800,1787,9308,436,9226,2645,1335,6681,10203,2632,109,189,6794,6162,3330,4732,10731,2570,2874,7353,9202,3047,1624,1049,10760,4875,1474,3649,9608,8257,5856,2064,8798,9299,2151,8005,4950,8407,2553,10736,5585,3257,2732,3721,1488,9661,4482,8935,10132,7630,5372,893,7775,10774,5823,6158,5209,3451,6891,3300,8847,8532,6843,8934,6898,6790,2606,8179,4049,8802,2680,1512,906,1111,4647,3762,6238,8988,6754,5832,6252,5058,3387,5371,4464,3207,3791,8987,1915,9554,504,6612,9795,7017,2817,6043,3181,9908,7232,8996,4938,2603,10153,2996,8961,8102,8494,8546,7196,9426,10531,5274,5378,9572,9076,7582,7739,4030,9819,2759,4619,5997,2557,8196,5858,1822,8552,5375,792,9222,3432,6101,8917,6499,2065,8660,7774,9932,1634,2053,460,2302,6753,9132,3549,8109,4132,2226,35,856,7415,1789,10690,1117,7113,10701,4075,10160,5071,6859,6895,10327,3925,1886,4449,7509,3389,1094,3621,3138,8654,4811,8794,3825,8282,3255,10427,1318,7657,3724,4749,3832,7691,3700,654,10775,6337,9337,5942,2010,9884,1312,4170,6284,4315,1647,8322,413,5086,7301,10671,10286,57,7291,8493,8827,5516,5906,5944,3083,5859,7521,8744,9754,7249,4505,4325,3481,2395,7226,7462,4472,5996,7193,8297,6354,5730,3544,3775,9949,6291,3664,2797,2215,2382,576,8029,527,964,6438,1643,3289,9708,1424,6054,10708,7651,772,10561,1065,884,8975,4503,1814,4889,6641,4252,7430,8817,140,4691,4309,9589,9736,8607,10522,6358,5483,5424,10622,8007,6096,3368,2576,6381,872,952,8533,4866,8454,1462,525,8302,8570,1305,8578,10192,3851,8998,2077,4583,7603,1981,1950,6336,9929,5882,1017,6446,10118,1091,10455,3910,1910,3168,5820,7254,9582,2248,10158,4501,3219,7586,3332,2495,10149,1400,9536,8198,1825,8617,7892,6695,1674,564,1837,8354,7450,3757,3350,6431,8710,2218,883,7237,6185,9864,4220,4166,8345,3899,1141,6516,9239,2828,8645,2299,280,10297,4149,6999,3323,6334,6890,5010,533,7576,8887,3871,10267,151,10683,8121,241,1263,4798,8306,7032,7161,10209,8138,8648,3886,2731,3246,6008,6293,8100,3735,535,10541,2575,194,398,9077,4511,2897,2801,1107,9448,4698,9977,56,5691,2799,6299,1877,2546,249,6840,369,8154,4543,1030,7511,6672,2479,3693,4817,7215,1959,9969,8892,8588,8949,2597,9591,5225,5839,8656,9544,10360,1379,770,4033,9667,1889,10119,9584,8283,1173,1268,3054,7641,1976,1180,186,1242,8339,10012,10655,10510,3278,686,4677,8761,9171,546,6318,1990,10316,7623,119,297,10223,3981,3652,70,2530,367,2505,5067,3426,5107,9593,5494,4975,6102,10542,9640,6174,2993,993,10810,7304,6270,3623,6960,2789,10659,9465,7208,232,1797,9084,4634,10213,2892,1531,10538,6240,4048,434,1696,2096,2154,5955,3906,1023,764,7620,4200,1480,5444,8855,2875,4680,2373,2509,415,4847,244,3536,7396,7854,6402,6764,5717,2612,7518,1388,2494,7990,7695,4699,725,10423,6758,1529,10825,10028,4651,9574,3622,7717,6038,34,782,3761,1360,1252,469,1332,8916,7484,6163,1355,6793,2856,5510,3793,2097,3537,9431,7678,9169,4833,6330,5980,3209,7638,4552,4100,4097,6541,5193,8253,8484,4713,1619,8236,10598,7169,5420,7904,7461,2877,6963,4387,5027,7978,7922,5764,868,9365,5698,10613,1471,7376,2475,7724,5441,7876,7772,4810,5974,6180,2341,7399,4869,8272,2551,9540,3663,4661,7387,9215,678,559,8585,9556,1172,8642,10262,1629,758,5978,5650,1058,4005,8479,3393,2314,3109,688,4087,1385,3656,5084,3632,6601,2240,773,3705,6186,6718,10490,6206,4953,5333,3990,5296,7962,4700,3875,3379,4566,6968,10025,2445,3345,3024,6765,7875,4867,7351,8380,4679,3322,10035,9288,6748,1947,7616,7548,4123,8016,7489,3823,4872,1408,5880,9180,10609,8791,9910,2762,501,2636,7971,7303,3710,8084,1871,1192,7080,1610,8560,6046,3286,7608,8403,1405,1456,171,9912,5947,6118,3610,2547,1024,9199,538,75,2767,7471,9726,3927,2537,10667,4043,6391,3214,721,6063,655,8208,2262,8748,8869,8531,2027,3236,3406,442,1572,745,2271,8858,8243,9847,5287,8127,6546,9913,7275,644,7994,1440,10414,3423,7031,1962,4244,6559,5916,5549,1383,1046,7830,8063,2015,7986,3304,8612,6704,6361,7166,3462,4209,10171,1685,7002,6730,308,9110,577,10568,7015,7048,6200,7212,7204,3804,3527,4437,4759,4010,4020,10664,6710,10161,3553,10282,9968,3260,3077,6533,7424,2986,2071,441,5647,6703,2446,4080,8054,10182,5896,7928,3584,362,8050,2586,8843,10707,5492,2815,10658,7635,9592,8608,6908,7434,7085,4774,1327,835,4693,3188,6956,8886,2162,9389,2679,7123,4884,7419,6613,2599,2770,2512,9265,9323,6675,315,4508,994,1605,2256,6321,7675,1002,1016,5303,7584,939,9626,476,10211,7573,1394,2212,6189,5687,5309,2519,2186,9559,1936,10268,6558,1365,5791,778,8678,5601,2292,5025,1235,1149,7433,4062,2881,5871,4555,39,78,9166,3311,8126,6888,10408,2625,3960,4656,4561,6849,2338,4591,5531,9619,6036,10464,4378,5775,5278,8385,5514,814,6775,3691,1153,5631,1738,10799,4966,8611,566,9820,5957,100,5963,7789,2157,945,2454,9093,7849,4225,6444,1374,10587,975,7098,8058,9947,6619,2301,5426,7998,5117,10155,5206,4371,4802,3685,9491,592,4838,3861,7949,8643,10780,5685,1218,6105,8629,550,4773,3887,1085,9941,9690,6897,4089,7284,2403,3185,7042,3189,4886,8571,10372,4446,10204,6191,10190,5220,892,681,1956,4588,7534,16,9701,3355,7389,5463,8145,8026,10246,6004,8139,8300,9676,6087,10325,7770,1641,5543,8985,2344,3897,3463,4337,5949,121,9911,5948,5163,380,4338,5743,2588,7318,3309,1269,7850,10085,9008,7879,3726,8774,4465,1582,1084,7094,2400,1591,3299,3681,3079,4841,8797,4760,1087,9671,4295,9505,193,6289,8515,3661,4952,3713,1229,2917,5257,1283,10199,392,9175,5151,5186,5259,2742,5558,7953,4610,7127,6288,7319,4110,4571,6219,2481,6926,9553,6990,3912,5875,7281,10595,9108,1438,1721,2385,1677,1680,1319,9504,615,1423,2962,9699,3025,1598,4435,383,1376,1500,10577,3504,3614,2769,6227,9003,3580,9666,7605,565,10263,2243,4723,6161,3943,2985,5072,10303,214,662,9251,7059,1594,10716,8301,8575,4253,851,9329,221,4931,4304,5659,9531,4453,5603,5149,9034,8677,3769,1067,2273,8997,10147,9404,2376,2891,7600,3841,8925,665,2306,9250,5455,9480,867,6341,9121,3774,1526,6615,9599,8963,5503,7574,6773,6067,4849,7428,10274,7285,5767,2348,9835,10216,6878,5135,1952,4374,8735,5508,7856,9349,2843,1163,8433,10027,8160,1666,10473,2184,5213,3742,3312,4222,8097,7886,4280,9485,9868,1202,10177,10625,6589,6484,9399,7670,928,1799,4665,2656,6195,2070,3590,4204,7167,6582,9483,2334,708,4965,10394,5443,1132,6578,8197,5453,6137,2391,2305,4122,10702,10596,8626,9975,846,2613,7107,5840,6501,10762,8233,9635,5216,4642,981,1273,5860,7778,8031,3295,3978,3377,1692,9623,2022,5006,2933,2362,6505,7846,8760,4139,3926,8228,4993,955,7036,5748,5838,4895,2873,4320,6831,2316,10111,4289,821,6447,7323,1992,8371,4121,1486,2222,2729,9106,1726,9943,9438,6083,7096,9919,7295,7958,8467,8711,4069,2961,1279,3075,9506,8713,4595,5744,4442,1632,2425,9991,4850,2145,4173,7451,2060,6197,2208,5824,2153,8172,10574,2182,1108,7870,6607,2471,7827,375,350,1422,10352,3592,8700,4654,5258,1317,741,6720,6234,4016,8176,3364,6644,1640,6309,4506,1505,8432,5960,8540,8037,5684,3482,9664,6398,2776,6996,4042,29,4796,9606,4589,7588,5677,9996,223,10604,9268,9332,1000,5030,8857,4104,2253,6858,8937,803,7668,2554,3619,3806,9541,9951,4563,3124,8399,7965,357,518,8483,6068,7283,6756,4235,4742,4717,8592,2142,9685,8548,7402,1547,9149,5784,3989,3932,4074,2048,5281,9318,9283,926,7932,8745,970,7942,9395,2038,9659,4474,2826,1014,1256,7223,6836,5031,10434,6919,3313,4292,1196,3720,9317,7640,9160,4194,1494,4862,6707,6512,9825,4448,10377,6057,4671,8463,2263,5369,4103,3525,1964,4949,3722,628,2595,9388,2771,8001,6221,8959,794,8167,5818,6785,10198,2327,6074,7405,104,2796,8427,5164,3194,4163,2428,4530,9301,1853,3810,3369,3566,8871,3593,5162,10696,3950,9776,776,8593,793,4384,6886,8597,10697,6506,7867,7662,6932,3070,5849,1299,9313,7089,1188,7527,8274,7617,9763,1965,1382,1278,7165,4754,132,3934,5773,3618,1861,4879,4232,1608,7457,9306,8350,6811,331,5542,8990,10048,2922,2424,9307,7175,9873,2420,1541,4434,9879,5961,379,9443,5898,8122,5731,5295,10776,4342,5821,6351,9794,9510,3798,3885,4822,5020,8183,6732,8244,786,2465,6664,7517,580,1414,5712,9073,4036,3581,5520,1967,5666,7007,805,7398,1375,2287,3014,1514,7082,2948,4840,5182,7685,9902,3858,2726,5489,3321,1069,2160,7404,7495,774,3015,1566,2165,10288,7464,4720,4575,1314,4215,9784,6609,3315,9477,8473,4763,5845,4286,10726,1603,2744,5759,7388,7496,443,5962,2544,4507,3383,10748,977,3133,5066,7945,7020,7372,7908,9721,3396,3044,4705,5051,2867,4909,3792,10614,8740,9867,6736,5040,4029,1286,6666,8875,7317,3955,5706,5928,2204,2970,9492,1771,1577,4196,1796,6150,6332,8066,10210,10514,6531,1597,3550,1998,10585,10338,3467,3513,7648,7130,478,3766,1589,8418,7858,3941,252,1390,5599,5989,10670,4660,2121,8351,10824,1083,5326,3784,2480,4330,6742,1055,6714,9734,7618,6628,6739,3459,7735,7233,896,9831,1034,6971,9972,10607,6012,1828,1179,7485,8423,2000,10488,10393,8148,306,6307,5723,3736,8021,545,1800,8561,4709,4341,5076,6557,49,5452,3152,9361,7324,5291,10230,1166,5673,10554,9590,2402,6817,6282,4560,6787,2619,3082,7172,378,9055,3528,4948,2721,3878,6323,465,1459,4690,5881,8599,6943,3078,4573,10188,989,6954,7512,2440,10679,1955,5738,4278,6300,4090,418,2325,797,7902,5562,7500,5812,6426,9675,10422,5734,5466,9828,4321,1741,10793,8344,5110,9826,5798,3173,8495,934,7692,9878,6594,10145,10478,359,1168,2900,7019,5060,10029,9714,10294,2461,6362,5496,4818,5045,8945,1395,467,5935,7392,8811,4144,138,3996,9673,22,6571,9148,3490,5469,8279,4128,10552,5965,3222,8387,4735,5675,587,4483,732,9006,8313,9916,10503,8672,8904,8717,3879,804,6395,6853,8793,5645,9263,4326,7807,703,9920,9921,621,1152,2080,9094,1654,9085,5434,228,6097,9710,6478,3745,292,1884,6743,7614,227,283,342,8367,5329,335,9729,8649,3535,7069,9212,8129,631,8568,3410,7051,3356,7090,9978,3476,6530,5512,558,1920,5688,874,10397,9335,3501,6682,2456,7749,2711,4236,3796,1144,2238,6882,9740,1559,9354,2183,6050,578,7734,7779,908,10302,3754,8541,2030,2217,2677,7045,8230,4045,7459,2594,245,1546,7210,2489,1050,10245,2211,6196,7005,3105,2394,8442,2643,1681,683,10344,3821,1581,1329,3027,10653,4438,2855,5299,9489,4828,6646,7762,4189,8143,2232,3949,1858,6854,4653,543,7173,7312,2808,8149,9147,5657,10143,10744,9355,5572,5005,6795,2371,9706,5397,114,5550,5722,4002,2902,9611,4624,1249,5994,3057,7931,8587,128,3617,3834,3936,2579,3505,5039,2734,1604,10004,9028,6222,134,2754,9643,6359,5667,5834,7338,9715,6603,8549,7546,8064,7406,9539,9079,3267,8523,5474,8278,749,10173,3890,8826,1232,2152,3609,6052,4636,547,6111,4470,7391,9771,3495,10437,4357,5642,7073,3199,5636,7976,3039,7306,8263,2432,6931,6939,1810,6231,6064,2910,5622,210,3667,7592,4179,6544,8919,553,6261,6344,7743,5103,8769,6550,10433,3442,6715,5078,2994,156,10005,8083,459,10499,7719,6126,4902,882,346,496,9696,7967,9186,6019,8594,6353,5210,1989,4180,3440,9062,9026,9861,1765,2083,7647,4668,5988,10385,236,8142,9513,3647,9762,6634,707,4780,4349,9788,6037,1973,7316,3644,7269,1293,4331,5920,2824,784,8808,7636,10501,4346,9604,3583,89,815,5118,8724,8820,622,3558,5367,3268,10496,10251,10612,9358,4492,3439,2528,4578,4998,9901,4834,6670,2381,452,728,2414,2144,205,9871,10693,2531,2693,10512,4273,9691,6586,8465,9769,9712,8864,2535,2511,4098,7560,5136,2297,8993,1174,127,8876,2274,6451,2258,7504,8488,10197,2124,3458,7321,530,5044,10362,5873,6408,9987,1128,3718,2187,8072,2935,1162,8969,4865,10233,5870,9814,2766,11,3017,9046,269,10719,2081,4459,55,10097,1148,7631,699,1631,9964,233,5950,10034,630,4183,507,146,6343,636,5796,2173,8298,6538,4678,2936,1875,5183,9375,5050,9418,4161,10593,1823,4761,2322,4641,6692,1037,10807,2458,2715,5655,594,8227,6911,6800,92,2723,6060,9962,8447,10758,364,5943,4409,1330,9276,2608,3262,5754,771,10332,2577,6814,12,1452,2768,6277,8409,4685,701,5324,5690,5340,4536,8970,1380,5185,3324,3789,8640,2084,3096,3642,6870,9730,3877,3730,2074,9798,1919,3884,6570,2954,10828,4606,6346,456,1222,9125,6957,9985,8392,6713,8226,8090,2426,9478,2977,3055,7816,2895,1435,5144,9285,7337,7948,4494,2337,2021,3233,6847,7767,5141,4072,4942,4519,8219,3957,7035,5737,10181,1857,573,6453,8070,1487,8245,3401,74,2412,10024,1767,2340,3582,6259,4382,4391,9071,8524,3277,7663,4461,6690,8738,4640,1057,2411,1766,3958,10475,2331,6209,5120,4241,3620,7851,8747,8014,4615,5327,6661,5724,7213,4174,3042,3781,8256,9050,6121,1687,8723,8928,8216,4012,1609,1025,10815,1627,2712,3460,1819,528,8275,3103,6324,6258,1714,3494,339,2746,4748,1558,5927,4130,255,8202,5952,4136,3744,8776,1493,10404,4959,4666,10569,4604,1258,7505,9352,9742,2210,9713,5191,1157,3921,4608,3631,3994,6122,9702,783,1638,7712,4431,9499,10236,7925,3175,3839,7765,3228,967,10465,8682,9189,5901,6216,8835,6577,10105,6803,4545,10101,5035,8664,3247,7289,6285,9579,2335,4441,5540,5314,10215,7815,1561,1515,8885,5594,6631,9877,6205,1817,4745,7235,8356,1570,5325,2137,7589,4823,5413,870,9038,8595,6638,1396,8242,6569,9562,5783,6639,5695,7180,1728,9872,10476,7661,1224,6893,1241,9378,3256,4203,9933,183,1530,7277,6528,5438,7181,1982,6972,2295,1503,334,4532,2293,6171,4842,4728,8492,124,7444,3752,5825,31,8000,2163,10530,1366,2548,8081,10771,63,5827,9191,10218,3258,1045,1656,3215,3630,2150,3276,4800,9414,5226,2673,7677,4255,5266,2072,6953,5004,6562,5365,1818,3162,6766,4452,10524,6985,3637,9850,10000,9672,3876,9647,1575,3699,1905,7629,8972,9744,2924,9452,10234,4737,9010,4288,1843,897,2435,7348,9305,978,2138,10591,3862,10540,4897,8402,8932,2443,2279,438,5038,3574,6535,7273,541,9598,8619,10459,3306,9043,5605,595,1270,309,9280,3080,8411,7825,6006,7801,7664,7599,9458,2506,7764,798,1520,1616,6378,3245,7877,2953,8480,819,5608,9181,3729,61,1072,9009,7482,6788,7703,136,7700,254,9615,2540,1717,3947,7794,6947,836,5519,2444,511,3548,1804,7296,5388,8948,3707,1683,4664,4175,6204,10261,7357,2251,7104,3675,9253,6265,9613,8071,880,8365,1019,10270,4469,2357,9646,6892,3507,9792,7985,8527,8701,5552,8180,7799,685,5787,10343,3628,2637,4607,6202,7831,1747,8342,8891,8110,4050,366,5387,6974,3666,6611,761,2852,1372,2423,2697,4355,4272,3122,1161,8618,3711,5037,2076,6160,5662,4277,2651,5470,7848,3954,10235,8938,8288,9439,7828,1147,9241,931,2983,2602,3639,10306,4791,7723,217,1464,4162,5951,9082,8177,1064,391,4460,4955,2377,8880,6173,10304,5903,4972,9018,8977,3457,6329,9922,10740,3241,3917,1061,1885,8708,1200,422,10127,10300,4057,9790,616,10594,876,9033,8848,5862,1729,3265,10167,2659,9386,160,7519,2905,7659,2866,7245,5302,10353,8856,7921,10474,4053,5354,2011,9422,4729,5751,3319,7514,9944,9950,4440,2560,330,6772,9725,8422,4070,8694,6370,82,1377,983,5023,4812,923,2223,9533,2839,6942,1434,8796,6440,7757,4383,2743,10695,3570,3836,2997,1901,10313,8818,282,7540,7951,640,4658,9066,6712,3190,6649,18,7251,8497,10497,3530,8859,7445,293,3135,3285,5228,7818,9159,1642,212,5077,10059,10224,9474,38,3687,8851,5630,4078,5205,2674,5273,9822,9101,5376,8046,10206,9099,10624,8498,3325,9800,2610,10036,5867,958,107,8096,3239,2779,7466,7553,257,5710,4982,7049,3335,5857,5359,427,5097,1659,9772,1903,7040,7989,3671,3800,4015,2571,9341,184,9530,6705,7872,713,10630,2143,5392,4797,9516,7023,8169,1432,998,10446,2128,1143,2641,5346,6883,2829,9336,531,2566,2192,7022,7377,3100,4287,2722,348,6510,2462,8539,4168,8882,8284,4946,8475,325,7060,5026,8396,242,6696,9738,8290,3456,974,3611,3034,3400,9070,3913,9058,5565,9612,491,9628,7903,8809,5909,4190,133,2459,10390,9461,6598,3296,1484,6651,1189,1136,10269,8716,2604,9105,2737,9229,9906,4458,4593,10509,2300,10419,2978,3444,5399,7138,4718,8884,878,9755,8727,6201,913,5246,8460,3046,10378,5541,1509,8375,10416,2966,5271,5412,6459,7443,423,4115,8895,5842,8995,1870,10678,7194,4184,5171,5134,1386,10357,10447,1588,8201,1476,3085,9936,4819,6112,6423,4114,5793,8659,10375,10392,10226,6228,10520,9994,6659,4213,8586,10157,5178,968,6417,7026,1891,5152,9371,7758,9926,1713,3758,1986,7000,10425,5742,5202,6059,301,8124,10285,9862,5195,6313,1125,9145,8861,1904,3437,966,6740,6131,4547,748,6188,1778,3108,2707,4740,1281,5530,2220,3117,267,2298,9813,4067,410,5319,6671,1650,2031,8162,4250,3449,2343,4319,1120,4328,8823,1497,8676,4534,4493,7543,1637,10786,6778,9204,1922,9654,7216,10663,9984,9722,10259,2842,5479,903,8705,10644,340,7186,10011,8075,10402,3227,3486,8059,1508,1928,2913,5628,3948,9127,834,2148,6044,10047,4301,5883,8459,10071,1777,347,9119,10550,7146,8893,6026,6382,987,1958,9674,1439,4336,617,10110,8751,3865,1739,9637,7028,6264,544,9315,7881,1646,9678,3604,3429,5902,1131,3146,425,7593,4580,3394,9917,10669,51,4054,5421,2972,4784,4216,10237,9244,9117,263,2703,4153,829,2671,9144,1300,3538,2713,969,2940,5837,123,4246,2242,3397,3559,9273,2862,8899,10122,4756,10441,1345,2552,8012,2803,9783,2374,113,10808,7117,2887,5370,9423,7118,10405,2042,5214,7520,6305,5230,5702,5893,5160,8239,8443,1963,3056,8615,1988,5048,8374,10467,3263,1351,8824,4051,3390,9007,7844,1417,3179,4370,7476,5408,9069,2486,3857,9829,3149,5458,6826,3702,7502,1931,7305,2646,3186,10418,4308,9668,4614,5683,2725,5028,2075,10096,5579,7891,1740,3443,4395,5792,1806,2934,1358,446,8199,2600,7575,4686,8603,10645,2365,6077,27,8416,8389,3532,10452,9112,6458,779,3706,5602,3491,1214,9766,10558,7160,985,9311,4747,7365,9393,8192,10131,352,3555,8718,2318,6298,9115,6414,5059,4762,1043,10292,3719,6000,9293,7993,5809,9174,6588,7874,1285,7895,4497,490,3603,4468,10023,4990,8098,8953,7955,7585,9538,10717,4198,7379,7347,6404,2452,10791,9451,9434,1900,6207,8195,444,4707,7438,3945,7309,9508,3636,9931,751,1513,7330,4526,7302,6181,4836,2969,8683,2434,10420,1255,7561,5451,1354,3156,8512,6041,7410,2695,1052,5733,326,8002,2911,6192,1186,4794,3102,2780,10124,6902,7751,1860,7119,9475,1080,2501,6738,657,3040,1404,6243,4638,7596,8286,10001,1995,4885,5016,700,1096,9373,10380,6442,8010,8622,5590,5555,4420,2265,10517,360,4766,3698,6413,5355,2098,2682,6007,7884,7455,6407,5992,9866,2449,9770,3026,8469,6652,3474,2728,6460,8092,4744,6701,483,4307,6416,1197,6172,9248,3679,9624,3141,10291,4778,7741,6399,10137,5490,1537,2502,4058,5728,6731,7569,9370,4584,9650,8771,9296,4853,6281,3213,3991,10214,4402,3308,5861,8801,2541,2806,8246,7693,7163,4479,10183,4499,2870,2818,433,5280,7423,3523,277,9234,3359,6266,2652,8400,3509,6474,6899,3627,4408,4857,2763,2692,951,5878,93,7602,3866,6123,9827,4827,4296,6166,9155,1573,1187,7248,5668,5248,10495,6987,1832,7795,3031,6226,10462,5930,4682,2556,7669,10829,3232,4195,1466,658,2736,1005,7371,6604,5653,9709,6620,203,9923,9596,9134,10184,1097,2006,841,5306,6568,1774,953,5445,6271,10536,6632,5237,3317,7016,1838,1599,9473,7491,9956,6082,6567,10498,6069,1316,9300,7654,1245,8348,200,3338,10575,8521,8420,4422,6023,4084,2433,629,8627,2765,3348,8868,9228,6100,8931,6058,7149,7228,7187,8836,8870,9519,1583,6952,2907,1879,2174,3280,5649,2941,9958,5417,2835,1228,10093,8120,6580,1267,10753,8830,889,1533,8040,451,9047,7555,6662,3248,2254,1102,5454,2792,760,555,5644,5852,7394,7299,7034,3995,10424,8434,6449,3385,9049,176,8115,7940,2120,7349,5390,1849,9385,1809,7557,4293,5338,2195,9782,1060,3418,2741,8426,6276,10811,6551,9347,9967,417,2752,653,5143,10764,10087,9520,7863,7890,1212,363,8665,8428,10298,4025,9887,6616,9693,8458,7386,6976,8752,3838,10243,2149,7229,8451,208,8888,6658,6151,6997,9185,4898,1254,3235,7523,9065,1205,3187,3872,3316,3579,4480,6989,9622,3600,7742,904,3750,7977,894,1671,753,4343,8574,2696,575,9888,1479,9031,2140,3571,3119,738,1618,36,4165,8790,7056,775,9417,5933,5664,2091,7923,6491,289,389,4738,3275,9130,6657,7650,9976,9493,7264,4439,6025,5092,6529,10275,5286,5439,6153,8944,4549,2311,4848,175,8689,7442,1737,10334,9384,8394,10599,7656,2139,1056,9173,3210,7453,8334,4218,7781,2663,458,4826,1253,1287,9135,9542,2213,5029,5983,8326,9733,8956,10621,4450,2018,10264,4032,6941,4832,9224,3782,7416,9039,5279,9177,8940,3107,8604,8065,5872,6263,4936,2231,5699,1658,9528,4516,6143,5057,2024,1723,2787,8421,6871,9097,6717,10283,8554,311,6930,8742,3598,1695,8025,5125,3072,2863,10772,4239,3795,6784,4808,1507,6842,8992,2384,8867,8410,10565,3264,5172,1066,6964,5639,1071,7503,3037,7185,3993,791,10129,9421,4291,6224,2040,1121,7182,2117,660,7773,2992,9297,5098,5635,260,41,2308,4477,6829,10003,2353,8547,10633,3779,8471,2221,7454,3331,3404,2418,6306,3098,9490,2906,10202,8331,8448,2688,1419,7833,2491,2134,10430,5537,10249,3708,2915,8687,4746,1181,1098,2275,5656,5176,1454,4557,9649,5855,3662,9686,226,8923,5848,2245,691,506,7852,625,3859,3822,10065,4907,4896,7095,2309,10323,10315,4815,8238,2330,6372,8754,9914,5816,7672,10333,9743,1008,720,298,9165,5430,10336,645,7004,10040,2404,6075,1185,2172,1079,1150,9024,1856,7211,9810,7230,231,2135,2166,3577,5488,7158,3953,4500,2250,3297,479,2482,1001,7728,9751,9484,1712,4787,2415,4339,9988,10373,7494,1443,1359,7359,3734,1350,6388,7452,9953,8101,10682,6113,1070,1429,4283,2304,2180,9416,5990,9728,7168,7120,6689,4915,7910,3290,4224,9470,9684,8758,601,6944,8564,7963,3963,8114,5865,8087,1284,8807,10305,6479,5221,8359,8011,4943,6494,2542,1715,5559,9575,4093,2621,8073,6280,5529,3101,408,7564,10648,3269,4476,795,3975,6028,6275,88,7927,5484,3340,7979,10590,2005,1104,7643,1463,3479,6973,6502,5701,1236,2585,1389,9527,2631,3065,10144,9764,7611,3293,4186,6643,8222,8338,1867,539,10676,2819,8370,1075,3358,698,5361,6443,4799,4649,9747,4281,6751,3089,10324,7046,9915,5245,3147,1310,4831,6492,8695,2823,9761,10095,3874,3686,9107,10116,7139,2090,2705,8511,9137,10225,2224,4509,9583,230,10470,3415,10743,10752,799,1106,112,3067,7889,8280,5158,8646,4846,1544,3211,5416,6948,924,8912,4525,10559,3143,10232,5254,8476,9368,409,2319,1203,2237,397,947,810,2946,10809,10399,1600,1239,295,3373,586,7134,9411,693,7262,2639,1534,9472,320,2942,155,4548,7427,1353,9585,3022,4550,2943,7570,5654,3478,279,2869,2014,619,10180,9839,5377,235,6848,2667,6079,9567,5768,170,5853,1812,845,3904,3674,8936,4947,10142,2975,4027,5709,7448,8156,6759,1118,5055,4954,2413,7053,5971,8135,8477,9982,2033,8789,7256,2624,9432,5761,3522,2564,817,3249,1649,1551,4937,2916,877,1370,9027,4063,8234,5177,8841,9543,8118,8089,8894,1970,499,8781,2611,4210,3940,5797,10007,864,4852,9326,9209,9688,9083,9522,7704,7686,9014,3845,5697,8323,4951,5676,6333,2661,10194,2683,3353,712,5437,6233,5241,4152,7501,3164,8707,9042,8264,3126,2783,6470,4362,2926,5973,4521,455,10725,8186,1942,9237,9369,6683,5672,637,7362,3391,9898,7409,6317,6437,2810,2007,8889,8125,6599,4596,7632,9040,5918,6556,8783,182,8844,1636,2587,7562,7702,624,10471,7899,740,8235,2375,9444,2701,2278,9340,8712,6951,1426,626,1667,4734,9495,8631,246,4980,4264,1481,5485,4380,1260,9918,8509,5929,4041,7750,8262,1579,3557,10822,763,3765,2525,5476,2517,6016,3974,10428,3254,1009,2700,9348,3271,2999,47,2760,9208,7086,8261,2106,8251,4710,1852,7708,7487,1501,5013,7346,6355,8829,5065,6593,9258,3413,8043,5188,9078,3068,4711,148,5953,6678,1851,8491,2704,3587,6776,3801,7613,3863,7343,5926,8930,3114,4985,3589,5890,6770,2349,314,10328,4992,5506,4873,7072,2745,439,403,9419,5493,3242,2001,1339,457,1204,9521,5330,9287,6465,5472,4223,5301,4504,2507,447,4803,6647,5513,1294,2782,1492,6441,8954,7222,8061,8637,1709,10640,2457,4006,5977,5146,1444,4270,10383,9372,2094,10519,8033,6356,7329,957,582,8390,7246,6466,2957,5138,2854,4971,6312,596,7995,2777,291,6471,7414,4004,4322,1961,474,1381,6422,4466,2407,3261,4361,2405,9245,1427,4567,1639,1590,3004,6540,5263,3695,4340,3794,5528,10038,6552,10458,6485,7909,3202,1733,6605,512,9092,130,2543,668,787,7368,7987,1265,769,5011,4613,7619,8157,2774,9304,9405,6373,3967,3433,7008,7093,5844,5799,6476,2358,5331,873,4801,8355,7250,1448,46,440,9957,1158,2089,259,7583,1491,2112,197,3226,3568,7279,7144,1292,10479,7857,1979,7980,7050,4418,674,3547,3416,1415,6415,3514,1704,2156,10016,8979,9081,10555,1827,905,8440,4251,7355,3728,917,10022,299,20,3069,1532,4592,9362,8528,6496,4983,5003,3937,5771,2755,1880,10133,5465,9158,9689,10543,10712,5985,651,5315,5249,482,3176,98,9834,3367,2196,3907,3093,8381,8722,1611,9894,8679,1428,6287,4496,5534,5987,4863,4672,10516,5721,1564,9938,1247,6179,5868,6432,5298,7111,5781,6881,1347,7754,6820,781,5533,5284,10067,2111,3446,3502,8866,5915,4359,996,4770,7785,1595,3035,3969,1834,3894,10060,8237,1794,285,2198,3898,2191,585,7384,3768,5892,7861,3172,9201,3973,8360,9342,6750,5498,9320,3139,9195,271,8978,2498,3090,9029,8057,5487,7859,5886,2422,6572,4524,3341,10417,3420,2125,9679,8478,10296,4837,2146,10201,4572,7845,432,4056,6428,6786,5383,8941,5658,3110,3684,6070,7510,8914,10042,859,4068,2249,4663,8335,3453,2952,7639,7327,5126,3180,1297,6085,6914,7645,3692,5197,4462,3670,8821,2088,8579,7490,7747,1909,6872,2681,8113,10745,6445,7205,6128,1745,10449,623,1826,2520,2860,4903,8795,2126,7311,43,10400,832,4454,9274,6159,6680,9631,6808,5611,1527,9011,5539,3818,5795,5294,3417,1220,5251,10641,705,731,4052,659,6164,3599,8211,7918,414,5863,6203,4906,3646,3087,2784,9252,5435,9360,10077,3962,10597,568,7493,2844,1498,6635,2020,9896,2965,3007,9816,5198,8816,2614,10724,7426,5268,8446,5403,7001,1874,9220,8332,1949,10750,554,3431,850,8986,8616,5396,10179,1773,4905,6851,4712,4871,8715,8602,6473,33,7331,1430,10299,2572,5750,5351,746,10635,849,4964,7698,1628,481,9939,7701,9494,2303,1528,642,2016,8927,6467,8499,10319,6885,8311,3714,954,8404,6367,9487,962,5111,4317,5150,4777,7109,4603,2853,3520,7240,7400,1722,757,8128,9243,5801,8462,1048,9218,3498,4386,3612,5024,6841,10271,7171,844,6606,8417,9909,3819,3645,2342,2282,6771,1176,2203,4765,1195,10359,7129,6350,9997,5643,4312,2277,3964,3346,5897,7970,7911,8613,4692,5080,3725,9380,8921,4376,10773,8271,8482,5509,2205,7855,8340,604,8190,9469,8159,3095,1868,10544,126,4528,8170,5189,7468,4360,1730,2406,8633,7463,5887,6108,716,7373,6962,9580,4262,4650,5357,1177,2039,2110,5109,3081,9569,4488,10687,6520,6045,639,2447,5169,840,10661,4655,2914,4314,2617,10818,7954,1139,10816,4159,4963,8800,2888,213,6936,4444,1895,4037,272,7413,4249,7339,7366,3808,7197,1703,278,1274,3500,6039,6237,3786,8778,6912,2460,7565,6517,6920,5419,9429,9625,2105,7278,44,5835,1731,2470,3716,5105,3086,10588,4129,144,2492,4881,1035,10813,6693,6804,2893,3197,9021,5596,3452,470,3320,1210,10363,5976,14,10026,1127,7112,10310,5566,5663,4313,6308,570,8905,9464,696,9907,9143,9739,534,5567,9737,8697,8009,6595,7748,6175,2811,1795,323,5705,9266,8804,3777,2857,4429,848,10309,4432,6124,9294,9758,4888,5462,10734,60,6955,7033,10162,10782,5704,10677,6489,239,1688,2845,7642,5557,8276,10148,4919,715,1613,7832,7782,4369,4221,10098,7332,2563,1821,7380,9670,7590,1672,8134,4047,8414,1447,4883,5300,10207,333,7805,5046,10355,10106,10787,8425,118,2057,5194,1013,785,3756,8500,1705,7621,1644,7926,5106,10337,5047,6368,8076,10556,4887,3167,1092,2286,6560,9254,10521,6051,6958,6256,9338,4372,2598,1596,10583,2798,1250,1090,3250,1193,4922,7367,6904,1032,2633,3984,3678,10579,9630,7571,4424,5891,9045,10533,3944,8151,6326,349,6091,10032,7920,5682,1625,6700,5362,10627,390,2960,10442,9501,2740,332,6412,5368,7522,7456,4776,5568,10176,10603,2834,9456,494,1184,7598,10802,6199,6212,4285,8470,5081,4933,8900,1926,6071,4553,3615,4333,10240,8739,3508,2738,6320,606,4676,6448,7062,3923,9653,9846,6235,4436,7653,6591,7352,5142,2582,141,7984,7937,1700,5360,9973,8303,2216,3392,5680,4674,898,7812,5440,10154,5908,6940,1495,6403,1960,143,10733,1115,385,2003,3658,6218,9387,3104,3224,9821,6590,6995,1119,7558,1749,6419,1201,6475,7268,2028,4645,159,4539,650,8755,914,5101,6625,10041,4939,5386,4379,10272,10567,6251,4413,1836,8361,2724,10090,9095,5561,9683,2123,5828,9016,7290,3041,6152,2523,6430,6136,10244,8247,1743,2496,1756,7074,9655,2609,168,2012,1883,7018,4120,8666,4874,6425,9334,1431,2247,6331,2119,10692,4109,5227,7218,7957,9785,789,7793,9255,3565,6148,4764,1357,3900,10483,5553,310,6566,9803,5087,1038,9601,5212,7798,97,5341,8719,2052,2879,9955,6539,7136,6190,8865,10729,3813,4393,4913,971,3881,296,8207,1344,4147,5982,8505,3540,692,571,3436,10049,4969,7746,2500,7224,8877,607,2764,9718,9457,9447,7873,7941,4775,4586,10698,3113,1684,2739,7397,8815,7939,80,1455,68,1790,324,1324,166,7055,508,4188,10439,10456,2733,3154,7294,8691,8828,5121,4478,8150,10672,4373,10763,6245,10348,9857,509,3045,1893,9523,393,6821,2257,10491,4882,1798,6020,3826,6109,3516,4486,1802,7354,7041,8285,9526,1190,2813,2082,1020,5289,10287,5967,1788,2181,7526,1913,6629,10108,161,5813,3201,10756,777,2616,9787,3171,9236,5769,1054,9576,10386,3381,7594,4648,7314,9435,1755,8289,7709,73,10073,5670,706,1298,6797,6390,7549,9558,4178,7597,10720,10254,4629,10212,6749,4997,3038,5219,9808,5984,8232,9546,9111,2455,6021,5912,2427,1924,2227,9383,7068,8845,5239,6273,1134,4576,593,9745,2640,3398,2363,10832,10258,6618,9041,10560,7100,4894,10312,3030,10057,1445,8018,1356,4009,424,7326,1568,9899,52,7740,6498,4092,2987,9882,8623,1830,4741,7803,6132,4605,10728,766,8252,6098,9594,2831,1862,5556,3050,9343,6894,2383,8358,2515,9023,8652,5581,229,2207,4205,2332,7383,1585,338,1907,3370,6464,1315,2666,3790,5170,9206,2333,1944,1768,2438,4695,6741,4242,2467,1574,6815,5502,2073,3076,9242,9824,6994,225,9602,10432,4714,9719,1233,1490,7529,2209,8915,7280,8518,9056,6763,7601,871,9777,5468,7267,6130,5156,4708,2017,3178,4626,4727,2899,2259,4527,268,1126,2690,8846,4768,990,7470,464,8962,9588,5052,498,5422,3374,222,8315,9194,5648,3740,5358,9450,5404,800,7633,7513,1911,10798,4108,6668,9880,6896,10382,972,2638,5292,2848,8203,1662,10651,7066,9075,1244,7537,5305,8307,2416,8736,6072,5507,2269,10314,8042,5527,4970,9893,4059,6257,3546,2718,1288,8132,10694,1469,8314,7328,7065,7972,10070,2689,9001,5203,7595,4017,7716,5277,4579,8558,3992,1707,5641,1676,4430,1754,739,4781,8393,10484,7806,1993,3657,1865,10642,8287,3204,8680,1675,6066,1601,1734,5936,9282,5478,7930,4559,991,10018,690,9154,5255,2655,7,190,7563,3717,7868,4358,7924,1301,4491,1780,1110,4275,2925,10548,3212,5661,1607,648,3977,377,5569,8069,6249,635,7992,9496,1881,831,9851,6923,2872,2229,5544,4541,8383,5285,3344,4031,8003,8292,3980,2591,6389,7649,4318,4167,4367,5554,9509,10507,7912,3421,7077,158,10779,1764,1975,3545,2350,2601,9141,3829,488,2532,7550,1593,3403,515,1078,1026,8352,28,5244,1436,3651,10513,7655,4609,7718,722,2066,8653,5428,2630,10421,4348,3225,2483,2390,407,4347,6548,3357,9017,3959,1311,4687,1864,3815,7860,5808,833,7853,5293,2032,4185,8576,8529,7310,5411,5448,9196,6311,1403,5260,3192,3485,8481,7769,5729,8249,9809,10407,5779,9162,4861,3747,5007,8565,150,1930,59,9012,6420,8601,1323,3382,10547,9104,2516,1866,7239,5308,6733,6967,4529,305,1914,5713,10320,4772,4140,2749,3591,5250,899,7081,9020,6409,8965,10656,9269,1896,3,3240,885,837,5062,3654,8658,7200,9568,4820,4917,3032,10576,3305,7492,394,5822,1262,4455,1266,4332,10128,8704,1943,901,4994,2291,852,5385,4703,7488,5551,6142,8051,2323,3484,900,7787,6383,5099,7499,4489,3561,3005,3672,9053,2678,2620,5480,6523,3986,8496,8673,8036,10492,3123,4932,6279,9394,2583,8766,3820,8782,3223,3939,6554,2596,6975,6208,8519,5429,5598,4556,3773,9885,5418,5269,6845,5945,6978,671,7361,4878,857,6065,3669,5153,6339,9781,7506,5582,10674,7783,1164,6454,6869,744,5571,1337,178,3159,4040,4487,10592,3464,4364,9407,3896,2419,351,5379,8019,10792,4217,1280,7422,10222,3499,8117,7432,435,5339,7143,9876,6029,6260,3573,7943,2998,4757,9359,86,4594,6384,5384,2037,806,1813,10493,10485,9639,9927,788,4945,6055,3688,8189,5401,8485,3053,6984,652,9087,1697,4410,7577,10647,7975,1225,10797,4670,4305,1554,9109,3402,9179,1367,1807,5406,7905,9802,9532,5049,1890,3281,8324,820,4227,10504,8964,6579,3009,7784,6134,516,6010,2584,574,3764,6900,6852,2336,6366,2108,10331,4628,1923,2441,5175,5310,2361,1099,7341,1540,7715,4245,430,9210,6867,7021,866,5941,3220,2820,10706,211,6392,2533,589,5389,9848,7241,1073,6699,4694,2436,9187,3328,3174,10193,1477,8517,2758,641,6327,1264,9178,3902,376,10241,517,3294,9789,3966,4568,6624,1873,7689,10014,8621,9727,679,1872,4229,1524,10454,7777,5629,2130,1552,907,1694,4086,9061,6322,9874,294,3629,3738,780,7258,3008,180,4316,2968,7545,5166,7660,7809,9863,4788,248,2393,3585,3472,6095,4522,3928,3238,4498,7439,4083,10099,8137,8053,4558,4928,9656,3062,302,4535,8590,8214,8765,5610,317,5917,4327,8714,8814,3653,2882], "time_ms": 0.111820, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 613, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "generated_query_count", "value": 1, "time_ms": 2.056605, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 428, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "generated_samples_per_query", "value": 50832, "time_ms": 2.056605, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 429, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "generated_query_duration", "value": 12983730, "time_ms": 2.056605, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 430, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "generic_message", "value": "Ending naturally: Minimum query count and test duration met.", "time_ms": 608992.446173, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "issue_query_controller.cc", "line_no": 482, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "logger_swap_request_slots_retry_count", "value": 0, "time_ms": 608992.507057, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 898, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "logger_swap_request_slots_retry_retry_count", "value": 0, "time_ms": 608992.507057, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 900, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "logger_swap_request_slots_retry_reencounter_count", "value": 0, "time_ms": 608992.507057, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 902, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "logger_start_reading_entries_retry_count", "value": 0, "time_ms": 608992.507057, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 904, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "logger_tls_total_log_cas_fail_count", "value": 0, "time_ms": 608992.507057, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 906, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "logger_tls_total_swap_buffers_slot_retry_count", "value": 0, "time_ms": 608992.507057, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 908, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "power_begin", "value": "10-23-2024 07:54:21.177", "time_ms": 608992.507167, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 564, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "power_end", "value": "10-23-2024 08:04:30.157", "time_ms": 608992.507167, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 566, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "result_validity", "value": "VALID", "time_ms": 608992.872975, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 655, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "result_min_duration_met", "value": true, "time_ms": 608992.872975, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 660, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "result_min_queries_met", "value": true, "time_ms": 608992.872975, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 661, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "early_stopping_met", "value": true, "time_ms": 608992.872975, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 662, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "early_stopping_result", "value": "", "time_ms": 608992.872975, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 682, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "result_query_count", "value": 1, "time_ms": 608992.872975, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 692, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "result_samples_per_second", "value": 83.4708, "time_ms": 608992.872975, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 748, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "result_min_latency_ns", "value": 387217930, "time_ms": 608992.872975, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 754, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "result_max_latency_ns", "value": 608979647369, "time_ms": 608992.872975, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 755, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "result_mean_latency_ns", "value": 306681303425, "time_ms": 608992.872975, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 756, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "result_50.00_percentile_latency_ns", "value": 308243349625, "time_ms": 608992.872975, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 758, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "result_90.00_percentile_latency_ns", "value": 548828999693, "time_ms": 608992.872975, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 758, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "result_95.00_percentile_latency_ns", "value": 578864292714, "time_ms": 608992.872975, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 758, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "result_97.00_percentile_latency_ns", "value": 590916549729, "time_ms": 608992.872975, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 758, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "result_99.00_percentile_latency_ns", "value": 602948184527, "time_ms": 608992.872975, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 758, "pid": 6444, "tid": 6444}} +:::MLLOG {"key": "result_99.90_percentile_latency_ns", "value": 608383033212, "time_ms": 608992.872975, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 758, "pid": 6444, "tid": 6444}} diff --git a/open/scc123-Shanxi_University/results/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/performance/run_1/mlperf_log_summary.txt b/open/scc123-Shanxi_University/results/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/performance/run_1/mlperf_log_summary.txt new file mode 100644 index 0000000..c81f671 --- /dev/null +++ b/open/scc123-Shanxi_University/results/b7828419c93b-reference-gpu-pytorch-cu124/bert-99/offline/performance/run_1/mlperf_log_summary.txt @@ -0,0 +1,51 @@ +================================================ +MLPerf Results Summary +================================================ +SUT name : PySUT +Scenario : Offline +Mode : PerformanceOnly +Samples per second: 83.4708 +Result is : VALID + Min duration satisfied : Yes + Min queries satisfied : Yes + Early stopping satisfied: Yes + +================================================ +Additional Stats +================================================ +Min latency (ns) : 387217930 +Max latency (ns) : 608979647369 +Mean latency (ns) : 306681303425 +50.00 percentile latency (ns) : 308243349625 +90.00 percentile latency (ns) : 548828999693 +95.00 percentile latency (ns) : 578864292714 +97.00 percentile latency (ns) : 590916549729 +99.00 percentile latency (ns) : 602948184527 +99.90 percentile latency (ns) : 608383033212 + +================================================ +Test Parameters Used +================================================ +samples_per_query : 50832 +target_qps : 77.0195 +target_latency (ns): 0 +max_async_queries : 1 +min_duration (ms): 600000 +max_duration (ms): 0 +min_query_count : 1 +max_query_count : 0 +qsl_rng_seed : 3066443479025735752 +sample_index_rng_seed : 10688027786191513374 +schedule_rng_seed : 14962580496156340209 +accuracy_log_rng_seed : 0 +accuracy_log_probability : 0 +accuracy_log_sampling_target : 0 +print_timestamps : 0 +performance_issue_unique : 0 +performance_issue_same : 0 +performance_issue_same_index : 0 +performance_sample_count : 10833 + +No warnings encountered during test. + +No errors encountered during test. diff --git a/open/scc123-Shanxi_University/results/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy/accuracy.txt b/open/scc123-Shanxi_University/results/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy/accuracy.txt new file mode 100644 index 0000000..304ebe7 --- /dev/null +++ b/open/scc123-Shanxi_University/results/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy/accuracy.txt @@ -0,0 +1,8 @@ +{"exact_match": 83.68968779564806, "f1": 90.87487229720105} +Reading examples... +Loading cached features from '/home/rocky/CM/repos/local/cache/741f8c7dee2f4365/inference/language/bert/eval_features.pickle'... +Loading LoadGen logs... +Post-processing predictions... +Writing predictions to: /home/rocky/CM/repos/local/cache/493709de70974384/valid_results/scc123_cpu0.novalocal-reference-cpu-pytorch-v2.5.0-default_config/bert-99/offline/accuracy/predictions.json +Evaluating predictions... +hash=38aefa72b8f9d9a7ab4fc7b73f84ac4f73098ba443b2224e51cdfb3ae7b8f141 diff --git a/open/scc123-Shanxi_University/results/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy/mlperf_log_accuracy.json b/open/scc123-Shanxi_University/results/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy/mlperf_log_accuracy.json new file mode 100644 index 0000000..80b1c89 --- /dev/null +++ b/open/scc123-Shanxi_University/results/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy/mlperf_log_accuracy.json @@ -0,0 +1,7 @@ +[ +{ "seq_id" : 0, "qsl_idx" : 9772, "data" : "0D27C5C026F7D7C06270D8C0224A09C1846E11C199E516C1852BFBC0EDDA12C12B8C0BC1478413C10E2F06C1DF9B13C1A667E6C0B7AD03C1768C12C1EB0A15C16B84E2C02DE006C11C43E4C0AE3B01C1BD07E6C0CF0BCCC0868BE4C087ED04C182F719C16A201EC140CD08C169D113C1CDF979BE32F8D1C0D3BAA0C02131E2C0614F79C07561EAC0AA7B33C08DE5BFC0B7D7DBC0AF7AE8C0921266C035D1D9C0F60458C06B28D5C03DBF91C08A326AC00D5762C043AE02C166869DBF5306DFC0AB25EE409FE10FC093C4A5409157E040429563C0E9FB4F40AEC29DC0E31BE9C0DB6994C0D380E3C05269EDC037CC0DC19C4BD6BF9B8DEFBF6A0906C11612D5C0D785E7C0855BCBC08714A6C0C98CE8C0E88B49C0AB79CCC094D546C0AA77B4C0E30D87C0DD8CEBBF78D5FEC0EB6CE8C0BF93CCC0BEA4F7C0DD2E8FC01508C9C0795A80C0C66BA1C0996DCCC0646189C06307C1C0F3019EC0AF61BEC0B86A00C10F4CD2C0D00906C1B137E2C0936F11C1E7D7EAC08D3708C11E9006C1FBE211C132BBCCC0214BF3C0DD5ADFC0AF530FC19B8EE8C0EFFF00C141F208C1D34516C185C67CC0C19FD5C070EAF0C039DF07C15B71E4C0A63594C0A6E503C121250AC152C8E4C037CB0CC1157AD7C06B4EE1C0529206C1AF0B10C13E09EBC0A52E08C11100E8C0A71EBEC0B72215C1236708C19E0EE8C065FA11C1D8AED0C0AD2E0EC12FBAFDC00F5406C10E610BC14F490FC1F10DFCC05EDF17C1ABB205C144A408C1B731FDC0B5F70AC10E358FC0BD2210C1792AF6C045E708C147009FC0175B04C1963371C047C4E9C0571ED0C07FE4CCC08259BEC013577BC08588E8C03B63D7C001FABAC01BA505C11EEEBFC009C70DC1C4B6D9C0AF74D8C0D88506C1EBD413C1C62002C07F8808C01234F2C07504FBC04A16ABC08BA9F2C0E8A4A9C06DDCE1C09931AFC04B374CC040320DC14DF604C1636ACCC0C9D507C11858D0C09C6F05C19059E0C09583E0C06C2207C15BF416C1E5ACEEC0BD43CDC05A6D07C1199CFEC0B38AE2C0B35916C14FAD02C1C117F6C08DE412C1C11C1AC115D3D3C0657914C169A2F9C0353F0AC19C59EFC0036212C1029F06C14DA2F2C041F9D3C0F63ABCC01BE1E7C0DB48C9C00B8D10C0FD118DC05318A1C06D1892C0CA1EE2C0703706C102C4F6C052A8CEC0714ED2C0BDE3EDC00FB3B8C0C9A3E1C0850C0BC1625008C1BDA1D2C0DD2EEEC0C82412C1F19516C1FCA2CEC06D5DD1C01101E6C0FD67C1C043DCDFC075E902C150E099C00DF5B6C015000FC1F79702C16383C7C0E77094C04983F1C0D5FABEC0BD5E86C08722C1C0EBFAFFC0D8CF13C1678A03C1A42915C139C2EBC0D5C403C16868E7C01C6205C135E9C3C0715701C1F8C5CBC0E359D2C0A3C9E5C0FC5E03C11DF3B8C0E1219EC0D384AEC099B0F2BFB31AF4C0CB3B04C1403B1DC1F30221C14A501DC1AFD820C1F4EB1CC18A1021C10CF91CC13A0C21C16E941CC1CB3E21C18E901CC1198221C1A4731CC180A521C11A891CC1B59421C1DE821CC1F1A221C15CDF1CC1115521C146DD1CC13D5821C104C31CC1136421C138E01CC13C4121C1AFB11CC1BA5621C10D721DC1B33B1FC1E7601DC169C31FC194021DC125F820C126041DC1098A20C17D081DC189E920C1B0B31CC1136421C1E2931CC17B8521C1DA1A1DC1B28C20C1B08F1CC1779321C14F691CC12EBE21C16FCA1CC18F6721C17FD31CC1744C21C126911CC1AF8F21C1D73F1DC17FB420C194E01CC1934821C1751A1DC1AF0E21C10C3D1DC15FC620C1FC101DC1E92A21C15F0C1DC1792D21C146121DC12D0B21C1C2CC1CC1412B21C140AA1CC1937721C1FCBB1CC1134F21C11FCD1CC1772421C1465A1CC185C221C186801CC105A721C127A11CC1E77C21C19F9A1CC1088621C17FAE1CC17D7821C18ACA1CC19B5421C169DC1CC1B04521C1DAB71CC12F5D21C1FA851DC1D8AC1FC133381DC1F36F20C11BC91CC15B3C21C1413A1DC1FFBB1FC16EDB1CC1872921C141B01CC1095C21C1100D1DC1DEAE20C121DF1CC14B2821C1788F1CC1C18A21C1BEF61CC1F7D120C1668A1CC1EB9521C1BAAE1CC1997F21C1F8F11CC1014121C1B6FC1CC1D13221C1A81E1DC1A60E21C1C4111DC1012221C1D7CE1CC1106121C132DF1CC1B75B21C127EF1CC1154121C1FEDD1CC1675621C16DC41CC1F56421C190AA1CC1767E21C1983E1DC1A1B220C149D81CC1E44F21C1FEB71CC1486C21C1C2C11CC1E75F21C145FE1CC1591F21C1CAE91CC1B52921C1B9A41CC1236F21C14ABA1CC14B6521C1FABE1CC1705921C110921CC16C8821C1285E1CC1F6C021C15F701CC1D9AE21C186901CC1D89721C1E0C21CC15B6521C1C2CE1CC13D5521C1C8F71CC19D1621C16EE31CC1FC1F21C1334E1DC1991620C162641DC1BB591FC1B2FB1CC1A10121C1C6D61CC18F3E21C1A0AD1CC10B6421C111BE1CC1536021C170BA1CC1C76D21C139E21CC1F54821C16EF61CC1FD2821C1F5FB1CC1BB2721C1734C1DC1F6C720C1782C1DC1AFFB20C132271DC117E420C1F90F1DC1E32121C1E4091DC1173221C1EF291DC1BDFE20C14FEA1CC16D1921C1DCB31CC1E16321C134EA1CC11D2921C1B0951CC1BB8F21C1F6B71CC1576821C147A01CC1F57D21C138E21CC16D3921C1F3CF1CC1C94A21C115EA1CC18D2E21C1E7D21CC1294421C164D51CC17F3821C140011DC112DC20C1D8521DC1211C20C146BC1CC13F4A21C14FBA1CC1035221C1D0F51CC13AB220C12EB61CC19D6121C15C821CC1C59B21C16CD31CC1F33921C1BADE1CC1113521C1B4921CC1378221C1D9791DC1A60F20C1DC341DC1519320C1EFE21CC17B4021C1CE + +... + +C149AB13C19CEB0EC1D3BE18C11BE1F8C0AD5B0FC126490CC1E31011C1630E09C16785CAC066DE09C1BD2CEEC0433C97C05B69F5C01FB5F5C0A591E3C01FF0E2C07714B0C08FFA13C120B211C18A3F0DC18512F4C05BE013C1955D16C176FA08C18FCF0AC106E013C12C2819C1061200C14DFD17C12748D2C08A6A10C13B8A10C1ED04F2C0B28B15C1130B13C1E4530CC18B3C12C1AAB012C1C9CC1AC1FBEAE5C093730CC1F8F402C1BC83CEC05C1817C1F989EFC0C4210DC1FF4117C1A17E0DC18F6B0EC10556C6C01DA6D3C0062F15C18EBB16C1AE2606C119041DC16195FBC0AF6B18C13D8D10C165DC0EC1573FF7C07607C9C0C13BE6C094AEC5C0F171F4BFF1A0D2C0FC4206C0D7BBC7C03B29B7C0D08BCCBF52F2F2C0D6F109C1FD9CD2C0E5FE12C1A5BFA1C01DF704C17D7805C151CCD9C0BA8312C11F9612C1642407C1713A12C1C1D3EEC0F7ED10C14CD403C19CC000C18FCE0CC1DF6618C111FCFDC06CBAECC0699F17C181DAE8C0297810C18B9914C1BAFE0CC17DB21DC1BE8910C119BF10C1422F13C18F5816C1A0DC0CC1B50B1FC1DD76E8C05F730BC14AAF0CC10991FBC0FC0B0CC131F213C106ABDDC020B4FBC054F013C1B12D15C1AF61D9C07CF50CC17C6E09C1DE3411C17BF2FBC01F2CFDC094A60EC1768B12C132CD01C1553614C1465D0AC1E1E619C1505E00C1EF9310C13960FAC02DB305C1EE94F0C0BB60CFC0443BD9C0DAC5C3C050C7C1BFC0A6DBC08005FDBFDCADD5C0B997BFC081721CC079AAF5C0C78C00C18F47EAC099370FC1D20914C12BA0F1C09F3CE7C08B0D17C1DA1A05C14139EFC006FA14C194E5E6C074FE09C1C9C418C1E0500BC1818E10C18F6DE3C0BE1A18C19FD1D5C01EBE19C1CC9F01C117B70DC1DAF702C1593FD5C02CC90CC15136ECC0B0F906C12B781BC1681A00C1D95D21C1AA150BC1727E0AC159C70EC17C7D1FC12D2487C0641909C1812DF1C07AA900C1ED92F0C02F21C4C0F5EFDBC04E58E1C0919899C01CA0F3C06A2FFFC0731F0CC1A9CDE5C0B9B90FC18FFBF2C09A32F5C012E60CC1AA9A11C1C3A6E2C0596519C1F727E4C051D00BC1F591FEC03F2014C1679EF5C0E133ABC04AE90BC1C07FE2C0174DE2C02D5E11C1E96301C1F3D809C14E0F06C159F0D7C043B809C1A147D9C00C2E0DC1664015C1F749DCC0E93217C1B1C3DFC0462415C17D15E6C051ED08C1FA5B00C1DFA213C17C54EFC0CBFE17C18727EFC01DF213C15467FAC00980C4C02971BDC07156A8C090CEBE3E7778BCC04558EFBE1321CEC08FD6A5C083FB0CBFECBEE1C0CD13EFC03DA5F4C0A64410C131B7CDC0442703C1D46307C1496105C1FEDF11C12771EBC0014711C1C1C810C1D307F2C027EB11C1A48DAEC033F30AC1BD9EF7C081BBC7C0947C07C15FF30EC176640EC166BFD1C08DBBCFC08D7A01C185FEA5C0367706C1EEB188C040CCCBC0588306C105C7EBC07D4BD2C0480994C046ECD7C0D54508C1B86708C1F94CF4C0C2F50AC10565BFC0DD38DFC0DD86B4C00D8DA0C04F0715C1F603F8C0EFC100C1D8A904C1DA90C6C05F83BCC0971408C1A1C7B8C02A2DF4C00B28D0C0F32209C10524FBC0434FDCC02CF002C1810907C16BC4EFC065110AC15B1309C175D50BC1DA42E4C0C17406C120160DC1B60D12C106ABC9C08D480BC1D347C5C0A11B0FC12F58E0C0330CFFC0840904C1638117C1182BB5C0413008C16372BAC00F3603C13AAF07C1F5D41EC10DE7C2C08DFEA2C077CBF9C0406AA6C08796EDC06F7F07C1FED004C130DA02C1219EA0C0677008C1066698BF2BE2BFC0EDC9A9BF3854BFC0177AAAC0CDE7ECBF0177F1C035B7AEC06A4501C19D7B03C12AE596C0D5C00FC132DC01C19753FAC01B5E09C1992310C1420704C13AD410C10123E6C0FF2C0CC172110DC16FD216C10C7E01C1EF2F13C11B9704C1D5DCF2C010EB0AC1FF4F16C19AFE05C1D59E1AC1FD79E9C0E90613C146BB02C1A97B11C1E3F207C1114B14C18A320BC1BEC102C1F49206C14DFE19C15FEBE9C0CD580FC1AAF704C174B5F8C0743904C1D4F918C19DFE8AC0FDB3F7C0C72BB2C027BF0FC17ACFE3C07557E1C0F32FF2C009FABDC01A5BDAC02F20C1C04734B6C0412109C1DD6FC8C003BF0BC106B110C1CB6E1BC16A13E0C0D34FBCC030C30EC1792F16C1E740F2C07F6F10C16935D2C069C8F5C01BC7F2C047EE9BC078CB12C1F9CF0BC1ADECF8C0E8A200C1AEA3C8C0087206C18D49C8C0CDDAB5C0E87608C157EBD6C0746006C16BF810C16BFAEEC0EBBF21C1D85B05C1739414C1B7A30DC15B950EC126210FC1C5AF1BC1F04405C1E3C21AC1D195EAC0BA65DAC01A6512C14BB5F5C058BA0EC1702919C12A650DC120B81EC1A21207C1BC031FC16AE011C1D7E21DC1A5F5E5C0071E17C1889906C14309E5C0DADE05C1831DF8C004A003C157201FC1FF1912C18B101CC1D37B10C147C61AC12CD511C1CCAB1DC1E12DFEC0736A1CC118870DC10C1009C1B4B20DC1981F1BC1EF0FF4C013B417C19BFAE9C0DD7F18C1240C13C1BEF91BC12B7E02C121ADEBC096B00FC156381AC1529B04C1C5891AC1BF32E6C0FC2D06C1B5D0F7C06A6AC5C0774802C15B4D18C123C5EEC0327B1FC1217FBAC013C2EDC08FE7FDC0ABDC04C101660FC1F1A51EC136A80DC1264E1FC1774D0DC131231DC16CD00FC1A523FBC08E611AC1CD0318C196A801C1403524C1FA1612C1A0971CC1FDDFFCC05D2E21C12D31F9C071CD0EC1C6E214C106011BC1F14FF9C0995216C1D48614C145761DC1AA6F00C17B2224C10386FEC02B031FC1B2360EC1CBA3F6C09515BBC0870B5CC0CCC602C1BB8808C19EF11BC1E31022C17AF61BC14D2122C103D21BC1C95D22C1E7D91BC1113722C1" } +] diff --git a/open/scc123-Shanxi_University/results/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy/mlperf_log_detail.txt b/open/scc123-Shanxi_University/results/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy/mlperf_log_detail.txt new file mode 100644 index 0000000..fd1f59c --- /dev/null +++ b/open/scc123-Shanxi_University/results/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy/mlperf_log_detail.txt @@ -0,0 +1,71 @@ +:::MLLOG {"key": "error_invalid_config", "value": "can't open file /tmp/pip-req-build-m_g6hv8p/mlperf.conf", "time_ms": 18446744073707.312509, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": true, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 592, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "loadgen_version", "value": "4.1 @ NA", "time_ms": 0.006142, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 53, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "loadgen_build_date_local", "value": "2024-10-23T01:02:32.884482", "time_ms": 0.006142, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 55, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "loadgen_build_date_utc", "value": "2024-10-23T01:02:32.884489", "time_ms": 0.006142, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 56, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "loadgen_git_commit_date", "value": "NA", "time_ms": 0.006142, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 57, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "loadgen_git_log_message", "value": "NA", "time_ms": 0.006142, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 58, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "loadgen_git_status_message", "value": "NA", "time_ms": 0.006142, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 60, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "loadgen_file_sha1", "value": {"/.clang-format":"012aad77e5206c89d50718c46c119d1f3cb056b2","/CMakeLists.txt":"b73434348f7860471606aaa395b570e81113cb6d","/MANIFEST.in":"8d3c4ac6c325b7b9a0fd4cf4a4108cbeff8d5025","/README.md":"20a55bb946c2c0bbb564ced2af1e48efd096b3a8","/README_BUILD.md":"5f6c6a784e9cd6995db47f9b9f70b1769909c9d8","/README_FAQ.md":"01f9ae9887f50bc030dc6107e740f40c43ca388f","/VERSION.txt":"cb67dcc41adcbb7849a0a808a501ee9ccd951d92","/__init__.py":"da39a3ee5e6b4b0d3255bfef95601890afd80709","/bindings/c_api.cc":"32181da9e161c285f8fe46ddaa49e6cba2f9f918","/bindings/c_api.h":"91f58bd79b83b278f3240174a9af747fc38aff74","/bindings/python_api.cc":"9f538d2a5390c77ae0bc3f8a351bcdb2587bc66c","/diagram_network_submission.png":"53dba8ad4272190ceb6335c12fd25e53dc02a8cb","/diagram_submission.png":"84c2f79309b237cef652aef6a187ba8e875a3952","/early_stopping.cc":"0cd7b546a389deac73f7955cd39255ed76557d62","/early_stopping.h":"158fcae6a5f47e82150d6416fa1f7bcef37e77fe","/issue_query_controller.cc":"126e952d00f4ea9efd12405fb209aa3ed585e4b2","/issue_query_controller.h":"923d9d5cdf598e3ec33d7a1110a31f7e11527ec7","/loadgen.cc":"6650091ba7a918f343b06eb7a5aa540eae87275f","/loadgen.h":"e00fdc6dbc85a8c9a8485dbcbfe2944f81251c4e","/loadgen_integration_diagram.svg":"47f748307536f80cfc606947b440dd732afc2637","/logging.cc":"197efc96d178e5d33a750d07fa7b2966417506ea","/logging.h":"ddb961df7bcc145bcd7cce8c21f7cf075350dcbe","/mlperf.conf":"0a4daef277bb3151139980e484dd5e644bf36e18","/pyproject.toml":"712fab87b72ba67ef2a068d0f9f47da65130342f","/query_dispatch_library.h":"13ad6d842200cb161d6927eb74a3fafd79c46c75","/query_sample.h":"e9187c8612bbdc972305b789feb6e15c26e96cfe","/query_sample_library.h":"8323a2225be1dff31f08ecc86b76eb3de06568bc","/requirements.txt":"a5ff7e77caa6e9e22ada90f0de0c865c987bf167","/results.cc":"34e2d2a44324cb07c884f92146ecbb8ef9d704e2","/results.h":"fce22d5a588d91fd968a6b25c27896dba87bc276","/setup.py":"a722046e05858c6d9f38f0e2b3fe425334beef28","/system_under_test.h":"18d4809589dae33317d88d9beeb5491a6e1ccdec","/test_settings.h":"476ecd4032f3bafe6f201df25d68aca4e177f659","/test_settings_internal.cc":"ce4322c849d24ffafc28a37b5e528a4cb4df227d","/test_settings_internal.h":"f1d5335b53ca610c30e0edc5d07999a27b5b4b9a","/utils.cc":"3df8fdabf6eaea4697cf25d1dcb89cae88e36efd","/utils.h":"40775e32d619ea6356826ae5ea4174c7911f6894","/version.cc":"cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0","/version.h":"9d574baa64424e9c708fcfedd3dbb0b518a65fcc","/version_generator.py":"eea9b9cb1a06cd1abe1bbdaee82f9af31527fedb"}, "time_ms": 0.006142, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 67, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "test_datetime", "value": "2024-10-23T01:07:01Z", "time_ms": 0.006412, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1198, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "sut_name", "value": "PySUT", "time_ms": 0.006412, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1199, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "get_sut_name_duration_ns", "value": 411, "time_ms": 0.006412, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1200, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "qsl_name", "value": "PyQSL", "time_ms": 0.006412, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1201, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "qsl_reported_total_count", "value": 10833, "time_ms": 0.006412, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1202, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "qsl_reported_performance_count", "value": 10833, "time_ms": 0.006412, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1203, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "requested_scenario", "value": "Offline", "time_ms": 0.016552, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 271, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "requested_test_mode", "value": "AccuracyOnly", "time_ms": 0.016552, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 272, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "requested_offline_expected_qps", "value": 3.30157, "time_ms": 0.016552, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 309, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "requested_min_duration_ms", "value": 10000, "time_ms": 0.016552, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 315, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "requested_max_duration_ms", "value": 0, "time_ms": 0.016552, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 316, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "requested_min_query_count", "value": 100, "time_ms": 0.016552, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 317, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "requested_max_query_count", "value": 0, "time_ms": 0.016552, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 318, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "requested_qsl_rng_seed", "value": 0, "time_ms": 0.016552, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 319, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "requested_sample_index_rng_seed", "value": 0, "time_ms": 0.016552, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 320, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "requested_schedule_rng_seed", "value": 0, "time_ms": 0.016552, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 322, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "requested_accuracy_log_rng_seed", "value": 0, "time_ms": 0.016552, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 323, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "requested_accuracy_log_probability", "value": 0, "time_ms": 0.016552, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 325, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "requested_accuracy_log_sampling_target", "value": 0, "time_ms": 0.016552, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 327, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "requested_print_timestamps", "value": false, "time_ms": 0.016552, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 329, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "requested_performance_issue_unique", "value": false, "time_ms": 0.016552, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 330, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "requested_performance_issue_same", "value": false, "time_ms": 0.016552, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 332, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "requested_performance_issue_same_index", "value": 0, "time_ms": 0.016552, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 334, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "requested_performance_sample_count_override", "value": 0, "time_ms": 0.016552, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 336, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "requested_sample_concatenate_permutation", "value": false, "time_ms": 0.016552, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 338, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "effective_scenario", "value": "Offline", "time_ms": 0.016683, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 417, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "effective_test_mode", "value": "AccuracyOnly", "time_ms": 0.016683, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 418, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "effective_samples_per_query", "value": 100, "time_ms": 0.016683, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 420, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "effective_target_qps", "value": 3.30157, "time_ms": 0.016683, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 421, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "effective_target_latency_ns", "value": 0, "time_ms": 0.016683, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 422, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "effective_target_latency_percentile", "value": 0.99, "time_ms": 0.016683, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 423, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "effective_max_async_queries", "value": 1, "time_ms": 0.016683, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 425, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "effective_target_duration_ms", "value": 0, "time_ms": 0.016683, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 426, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "effective_min_duration_ms", "value": 10000, "time_ms": 0.016683, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 428, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "effective_max_duration_ms", "value": 0, "time_ms": 0.016683, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 429, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "effective_min_query_count", "value": 1, "time_ms": 0.016683, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 430, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "effective_max_query_count", "value": 0, "time_ms": 0.016683, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 431, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "effective_min_sample_count", "value": 100, "time_ms": 0.016683, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 432, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "effective_qsl_rng_seed", "value": 0, "time_ms": 0.016683, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 433, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "effective_sample_index_rng_seed", "value": 0, "time_ms": 0.016683, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 434, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "effective_schedule_rng_seed", "value": 0, "time_ms": 0.016683, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 436, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "effective_accuracy_log_rng_seed", "value": 0, "time_ms": 0.016683, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 437, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "effective_accuracy_log_probability", "value": 0, "time_ms": 0.016683, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 439, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "effective_accuracy_log_sampling_target", "value": 0, "time_ms": 0.016683, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 441, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "effective_print_timestamps", "value": false, "time_ms": 0.016683, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 443, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "effective_performance_issue_unique", "value": false, "time_ms": 0.016683, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 444, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "effective_performance_issue_same", "value": false, "time_ms": 0.016683, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 446, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "effective_performance_issue_same_index", "value": 0, "time_ms": 0.016683, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 448, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "effective_performance_sample_count", "value": 10833, "time_ms": 0.016683, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 450, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "effective_sample_concatenate_permutation", "value": false, "time_ms": 0.016683, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 452, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "generic_message", "value": "Starting accuracy mode", "time_ms": 0.029246, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1090, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "loaded_qsl_set", "value": [9772,8533,6521,10490,3029,2324,6045,6496,9363,6154,1030,2969,8111,5802,10018,1133,2097,3409,4680,39,4868,7780,8289,7294,4631,10540,9874,2476,3146,7551,8190,6721,4132,122,2590,868,4256,7881,10412,4438,6772,3786,88,5404,2200,10209,10100,2370,7769,8943,1991,7158,4606,9675,9814,6951,7100,4866,4199,794,3376,1373,8595,10723,9612,1229,1047,7106,3600,6805,2823,7677,7773,2040,8798,5281,4612,6254,6768,5391,7564,2751,2285,3174,4657,8930,3339,352,10433,7908,7546,5175,7175,9475,1351,5871,7253,2279,2029,3023,5714,8064,4178,7408,6999,46,1829,2902,8456,5123,7042,1709,2620,10561,3280,8594,3419,855,8675,4283,3558,1719,1768,9569,5776,6241,9306,9490,5169,5214,4324,4051,1394,1451,2563,7082,1921,1620,2961,4590,2349,6556,3707,9133,8308,6036,10686,1014,1527,10431,517,3672,4839,6014,916,9164,9447,6346,19,3723,1609,3563,10701,10546,2432,3349,5694,9823,9875,10459,8792,5015,8832,1518,9941,9176,10092,293,8233,448,6818,5971,7574,1245,8134,6534,7670,2829,3689,4059,6250,9368,1813,1912,3852,5277,8573,8628,10425,7577,3962,5483,8039,8862,7213,7044,6050,4344,8142,3293,7035,1435,3291,8638,3611,59,5513,6339,6510,7951,6719,1834,10460,8886,6799,8275,2809,1616,6987,6549,4642,3858,1446,2743,9044,896,5305,6406,1091,1321,2813,5852,9851,2287,1843,234,10348,8771,6940,283,2798,4459,4654,5983,3924,5677,476,6291,9809,8713,1590,583,6398,8084,6239,5040,2804,9190,3347,9780,7128,6955,4871,469,6031,9853,3251,6931,9540,4635,7692,5237,2878,1907,10725,6523,35,9723,9625,9857,840,9615,9098,4298,8790,7522,6699,1516,2545,6422,5630,912,9571,2542,3577,809,9710,8257,2636,9297,6488,7470,10122,8672,6895,10630,10567,6756,3649,2762,7599,7820,3940,5696,2258,2119,6834,7705,7598,390,2667,6701,7489,869,2406,3692,4495,5472,6362,5651,9687,9550,5911,8110,3342,8258,4352,3058,4000,9221,6190,412,3651,5982,4776,1114,9010,307,10678,1497,506,3369,9360,267,4310,5008,6421,8879,1089,8419,5809,6215,4796,3173,4566,29,2080,9032,2894,7772,7530,2581,7788,5112,6602,2284,7144,2935,4235,203,7443,8547,523,6952,5896,3810,843,200,413,1227,5279,4208,2500,8185,2114,10240,8754,10718,6850,2470,4650,10794,903,5554,8,1721,3066,473,5685,7567,1661,3635,9249,5415,4949,7283,1389,9036,9623,9919,6638,3908,9866,9869,5323,4224,6866,5722,6583,4622,7870,3346,815,1180,8837,2726,1877,10422,4176,354,7665,5363,10539,8652,5661,2539,236,3356,8836,6132,8038,64,304,5414,6590,3976,6808,7741,8001,649,3979,1296,3398,1234,4280,3283,4912,1306,9121,1554,1544,4629,5088,2411,1211,10132,9027,4990,1324,2587,252,3141,3358,4557,2594,1993,6852,5380,5614,572,8091,7992,6666,4100,6924,8376,6316,7358,1454,4120,7480,731,6575,3093,873,9599,8145,8658,4227,9122,5189,7625,2478,8988,5127,5600,7097,9641,4149,5421,5586,6113,4113,4944,3411,4842,3818,8928,2910,1961,9392,1983,10204,6544,4295,6146,1856,7909,10071,6564,10760,6541,4643,6102,5976,1214,7482,1119,7973,47,10637,2930,8386,2068,9638,3747,2295,2079,6753,1869,1315,2861,5433,4552,7540,5218,9132,7376,5675,2429,4889,3322,3406,684,1051,6578,1469,585,7474,6332,6354,1228,9698,9667,4816,7869,5017,3265,8823,163,8997,8826,10693,10184,1206,22,3425,8830,10068,10473,3842,2525,7435,445,3422,6408,4214,3805,6659,1864,3380,2224,5262,2817,8872,5504,7080,7517,6367,1475,3780,6641,5678,627,10538,3482,9708,9174,10329,3111,4006,8236,6967,768,660,5133,6118,8939,10318,4273,8469,6925,2920,2275,9367,626,2106,6696,6165,10476,2502,5203,5876,509,5615,6535,935,1403,7210,5,1461,5979,2956,7172,8269,772,2822,1379,4010,4812,8177,8344,490,9108,8453,9115,10332,7948,2425,6009,7403,7914,9950,3092,5332,320,8659,4114,2862,7139,5374,8983,6450,1082,1827,3617,1866,4164,1057,1916,7689,7652,3305,4683,3499,2316,7326,8330,5139,7295,6044,1766,4865,6565,4641,6777,10557,202,6085,1757,9090,2012,9446,6500,2197,4062,10215,690,9712,7297,7166,10258,6771,2927,5796,270,5626,6969,6372,10453,6061,5977,1450,414,10523,3984,808,6905,6376,3732,5882,4579,5679,5206,2184,4308,8131,4902,9149,3564,8036,9793,8089,2815,4182,7900,3438,3147,7539,5718,3165,2327,759,2101,10556,1411,9596,9327,7768,5181,10799,6207,7206,3296,6875,4287,172,4289,2662,6841,1056,7711,1973,1920,920,3715,9506,10505,7614,5617,5665,1326,10578,1832,1644,8164,5032,2945,10743,954,1671,567,10255,531,4112,4861,4926,7385,6520,2810,9488,2126,4315,10173,4179,6347,6773,3338,9219,3448,4961,641,6657,5200,3949,2461,4219,1746,7547,908,1938,4248,7561,9420,699,3761,9668,2711,10470,2117,1603,4288,1312,5794,5319,10088,6418,8499,8394,260,7427,5581,5344,7240,8080,5290,289,2707,5245,10527,4314,4710,3500,1533,10804,6794,4968,6256,7525,6946,9759,3516,4708,656,5985,1233,8779,6008,9326,4521,7532,4035,4044,4491,4728,2393,7709,447,5893,6357,3965,6494,3989,4917,6900,9315,10219,2024,5576,1017,9966,6484,2044,5358,1423,8400,8976,4443,9272,4428,6150,9424,2094,10498,3507,4087,7763,3249,2858,9715,3741,7702,9918,8734,8217,4660,3860,7892,8986,1123,9396,1085,9449,3444,4107,9783,10369,3145,9366,9305,3922,4077,4646,1659,7098,10415,3037,3254,9613,7970,6944,3518,9982,1004,718,9863,3970,2423,9640,8843,10386,4217,5764,10017,4027,10741,2167,33,9746,2482,4726,4980,2850,8178,7661,7800,2840,3334,8775,10321,8727,5442,2618,8686,10236,1018,4434,2687,689,6591,4565,5762,3370,729,826,7626,563,8721,4688,2916,7604,7877,7750,7014,6176,2788,3053,7440,8958,2573,7141,2774,7659,89,10382,8674,2486,8128,10832,9270,1820,10499,2570,9251,5905,6680,1883,3511,9237,8766,1022,5355,347,10336,4327,6463,9629,2447,250,6843,8942,7529,4605,2917,10313,7375,9893,4364,10420,2929,2681,937,783,3840,10246,4724,4292,10270,4209,8346,4674,7001,4786,4304,3871,7148,7941,5291,7138,5628,7052,8810,7493,9467,10658,8728,1273,2140,5190,7180,5968,2925,8925,6763,8946,7273,5738,7126,8556,6663,3047,488,5992,10744,8303,4197,3766,10590,2355,4656,10801,8148,9493,9273,8230,3025,751,7439,3912,8281,2454,2081,104,10784,2194,3319,3229,2897,9694,7726,3294,3178,2642,7721,859,1975,9183,5833,1488,6893,5844,6397,113,4675,5026,6870,3675,5975,7198,5251,5568,6292,7142,9697,9688,5115,6903,5258,10147,8915,7724,4457,9369,6461,4743,3658,2107,1923,5588,120,8115,5909,771,6266,343,7559,7965,8247,5860,8301,6049,6897,4285,7980,7759,7977,10389,4897,6180,9184,3150,9025,9925,8413,779,1108,429,6865,3239,9068,9040,4616,7085,6547,3399,8065,8296,10111,189,2452,8657,9202,10126,1410,2867,1387,2265,2263,1708,10535,501,480,7466,2908,3437,3572,2272,1546,10661,9196,3067,4705,7753,6780,258,6454,3910,3078,8254,9718,6469,8566,6363,9656,5648,8521,14,5765,7298,9928,7378,2015,7081,1656,5211,9841,667,5768,7638,6606,6731,674,9989,6229,4537,6411,4899,642,8597,3961,2057,5777,2651,1663,9296,6068,6974,1467,7353,3027,10060,5668,1860,10005,2299,9568,7850,4342,3710,9026,7996,4397,8944,9597,3182,7984,5037,3212,3763,8069,3209,8221,6524,5142,3004,9792,5196,5219,5990,3817,8314,8905,5328,5476,4216,10434,336,9351,6149,5920,5999,2919,4826,9193,3125,5946,2800,10774,6456,3153,3998,1397,8498,3100,1886,10040,10761,2319,3663,8914,3952,5956,1599,1194,1276,3049,4126,876,8612,8011,7524,1919,8881,8271,8964,4486,8278,5834,10548,898,7498,1699,4785,1635,7308,7943,721,4684,8285,9997,983,7031,2970,9808,5216,6202,2158,2241,3389,5064,9525,9560,8408,5100,10795,1307,3045,6847,654,4677,2067,10486,2255,6414,586,3361,1617,7362,2414,2912,2499,8500,8318,411,5741,6592,9791,3746,6519,1308,7949,5934,8724,8828,8277,1628,2820,2204,7673,138,3391,10247,9811,8738,219,9962,5247,7765,4815,3578,537,4662,4294,1624,462,4795,6996,10568,4137,2934,10685,6209,7477,5302,2831,7739,7017,8352,7723,176,9739,3082,7794,4621,9706,6487,5647,513,10771,10334,3227,6471,9260,8382,888,1486,2682,6938,2139,3636,9284,3522,4518,9552,3787,6789,7357,3638,9317,8932,8648,3773,1605,10220,6879,10354,4529,10797,5514,9765,1358,4300,8531,3550,7840,6656,2553,5798,3022,3660,5507,10353,8617,5745,5028,6337,7267,5459,5122,10772,2940,4798,2185,3596,5980,2036,2740,5458,3574,5197,1433,2567,2616,10069,3136,9522,3363,5074,8240,9979,1193,934,9728,4357,9331,6296,2949,8431,6230,8629,993,8423,10401,1117,10405,7879,1910,9102,7770,8856,867,7066,8970,8700,7133,1453,4395,4762,6498,7377,915,3561,3126,9600,1352,5244,7219,5807,1350,2052,2792,2472,5590,10323,287,4396,767,2489,2112,6615,8644,862,7039,6816,2369,2403,9047,9061,3377,3567,6613,10341,3430,1612,6340,9996,4589,2463,482,1735,947,10737,2603,9470,7550,2931,6888,3116,6730,2728,6650,5313,10206,6295,1356,9614,3060,6162,7197,10762,10253,1430,6729,4042,4499,3951,10560,8990,2674,4046,10698,6723,9888,5767,4038,5886,457,4803,9269,4784,5153,3005,10558,2689,6953,146,6617,10624,6891,5688,978,8146,7526,1121,10497,10803,8780,7462,6089,4447,8725,7901,4599,3982,7748,8774,7422,761,1732,5294,1844,4975,10263,10445,7447,7608,5910,7501,999,1522,9598,4128,8389,1335,3056,9134,8542,582,10747,6801,7235,6218,1523,1270,152,5940,10121,9873,3628,1763,5637,4299,10652,1760,7036,1437,3947,9099,6373,1459,700,5311,3288,5915,1816,3694,10299,4444,5300,10338,6742,9576,3223,3351,10193,9608,6186,3328,8782,2450,1,4737,5702,5577,3802,5159,7216,3055,9659,6851,9330,1365,3397,5591,3190,1175,367,4450,1188,1292,8005,6860,4769,6416,1789,4951,10629,3849,10609,9299,988,5849,6012,7771,9013,7821,3735,9543,5827,213,4319,8662,4802,3579,1897,3420,6878,2087,10726,6204,2219,5138,2142,2485,2166,9169,3161,7655,9547,9986,1445,1316,10123,1665,5149,7154,7258,1274,8995,2971,2492,7059,8846,5780,2175,927,2703,7756,4681,8156,6446,3899,5929,7275,1162,9622,2162,2077,6136,7183,5797,4701,2572,4394,9628,4551,9161,3527,8155,1540,8697,8245,9821,6570,4884,1862,212,8345,7662,4049,8515,839,4474,10048,3458,1537,1263,9631,6636,5166,1808,4891,4135,8441,1320,4835,330,4147,2021,4850,5450,4526,9346,8858,7986,10146,1419,6959,6675,1853,8764,4804,2759,5510,5589,4163,9334,7194,6981,1960,10770,3847,5887,1170,758,9294,158,3166,7125,4118,1163,7350,5572,6412,9192,9947,4005,7779,2673,5286,5539,6341,6444,4793,10596,2505,10039,3442,9374,4943,7860,2982,9860,6063,9072,8567,10075,3757,2519,339,10438,6257,10358,86,7263,1618,638,6252,3937,7620,4148,5686,9544,4433,9670,2890,10536,9528,3387,8809,9654,664,6,8656,1362,2580,9642,2023,5761,3043,334,4780,10500,7643,3040,9595,646,8829,6161,8732,402,6760,6889,369,10576,785,8670,5165,1060,4645,9178,3722,584,2235,2952,2237,4945,2210,6024,2678,3379,9100,3307,9834,6279,863,7181,10057,2942,8992,2174,4729,6509,7995,7110,5318,6058,10638,7002,9320,6688,4101,5419,8361,640,2733,3915,3245,4690,1870,2436,7887,7520,6478,15,1280,6402,8422,3571,4086,565,5960,5548,364,5420,4960,1613,7940,5298,4547,610,3688,6214,3584,1468,5595,1271,6210,10690,8368,5884,10164,421,9322,2270,1201,9440,143,7657,2343,2160,6746,4678,10606,1805,834,692,6283,1925,2135,7595,8839,3252,9635,1607,3457,5241,4014,417,3613,829,10082,3081,9832,9802,68,8205,3901,2303,9766,1962,8882,4429,10322,3210,2269,5021,4937,2281,6819,7000,906,3240,9236,6106,7393,1780,7287,5340,790,8897,7248,1452,10607,6658,84,3235,3892,8968,6197,7083,2103,9371,5526,7250,3333,8412,1987,9483,1535,854,2339,8892,7454,1223,9403,2009,3439,8611,1852,2357,9430,5959,2041,9048,173,533,6507,5753,6054,3191,8874,8936,7710,8033,6923,6183,4522,9984,8367,789,10666,835,10304,10003,1541,8154,10768,5412,3098,1689,9451,2561,5873,10691,8322,2614,4536,7637,9398,6392,2018,3733,10554,8043,9181,4848,7633,7654,9050,9433,669,4374,9976,1284,2550,3170,3929,9922,3148,1594,4846,1786,1185,10575,9586,7313,8996,1675,8609,10446,4630,2913,2779,7040,8917,2967,3475,9497,1655,1496,7712,6909,661,10241,6690,9545,5621,5385,8166,773,114,57,8448,2626,10273,5670,10356,1491,7946,2932,763,9136,10276,63,3134,10689,6915,8272,2877,5295,5951,3587,5367,9243,2073,1442,7591,8553,171,141,547,10410,2715,6840,6110,9092,5730,9227,2190,5698,478,10733,8845,593,8758,7390,387,4685,9564,3528,6727,8522,6134,4942,10166,5759,1399,24,2671,6960,7305,1490,6144,4959,2156,540,3362,9052,1064,3326,5580,9634,9604,10242,6393,6262,10754,4472,9681,4679,7325,2892,5889,3831,3028,8336,149,428,1935,5045,2458,4585,3931,9112,3872,6714,8437,10053,8770,2812,7916,9589,4760,116,800,4844,7543,5719,1484,1043,7266,1125,1135,812,10233,10355,9173,8077,8169,1063,4037,2634,1550,9143,2193,2864,7911,2797,9379,2062,5640,8849,1642,3394,10034,6233,940,10696,3413,3997,10618,6359,5517,6404,7389,890,2449,8430,9045,621,5521,456,1924,4370,7264,9627,3395,2426,904,5238,2116,2195,3841,8615,5024,2503,7861,3721,7535,9034,3357,512,7200,346,8249,7645,7727,6388,229,6026,3038,882,688,8206,8100,1505,9850,4924,7134,1863,7808,5117,3539,8003,4307,9355,10156,1106,2341,7170,9920,10705,9462,7630,542,9616,7627,3524,8016,6448,2479,2522,2153,8355,1498,7242,9685,2336,2381,2755,1034,4538,9226,1948,10530,7534,452,3143,8327,1239,6836,3754,7568,893,5352,6439,10408,1370,8921,8847,8947,6109,4948,6526,9778,9521,245,3848,1068,6628,8622,9441,9960,1507,9789,1504,6447,770,264,7500,6430,3972,4920,3486,6761,9940,7202,2320,6051,1398,755,1875,7398,7335,1028,7064,875,1360,6290,8687,3601,3003,9515,2951,1814,10819,318,8800,2127,6679,10319,7758,10094,5047,10203,7256,2569,8993,2074,1470,1087,7007,7104,6030,9065,874,3436,1803,4427,9779,2212,3823,8338,6338,1039,1885,1474,3942,7288,1055,4916,1070,9283,7370,5089,4406,625,4887,6443,4442,2398,6236,9041,4936,8508,8711,623,415,1281,9388,6203,395,9415,3517,6817,9987,6264,8794,5280,6116,79,6522,9479,9970,9532,8884,7445,4108,2530,32,6660,2893,6244,5164,2847,9975,5452,877,921,9278,5894,5529,8600,5879,5778,7115,728,4535,7473,4165,8432,10074,4085,6103,600,6892,6306,750,1627,6838,3164,8949,10394,130,1049,7124,239,8279,880,4446,4050,9514,2985,7204,1833,3090,2028,8464,3748,6320,7600,9582,5501,5735,580,4821,10807,9953,10311,2619,3021,392,5292,8545,10351,10526,9380,4822,9501,6854,308,8316,10706,5343,7412,3720,458,3657,9285,9030,10227,6585,2998,9859,9643,5046,5991,5839,5952,9325,1957,8264,5464,5908,9217,10581,3703,6921,2124,8085,2772,322,3057,5351,5706,6270,1207,10213,1587,4389,9332,10429,10600,9358,2253,10194,842,8420,1785,3337,7923,2054,3729,9645,745,3450,9665,4505,8516,1749,1407,7099,2002,313,10049,6253,9992,886,8535,475,150,8071,3545,5327,10545,7094,4271,9402,3589,4572,6907,5806,9407,9535,6028,3948,1381,8736,4947,4099,3392,9024,8000,5030,7278,8455,3771,5130,9316,10756,6932,4783,6837,5426,998,8569,2468,8584,5422,2179,6647,9658,3159,7783,961,2291,7363,1118,9459,3104,4258,4913,10252,1992,8510,9696,10029,3568,3916,3503,5866,6882,5891,8778,4206,6224,4602,3064,21,5540,3552,7650,2182,960,10514,5271,2264,5339,6548,4076,8188,4158,1366,4856,4880,5751,1591,8483,9000,10033,3865,8741,6845,5111,9461,5727,1374,3737,464,10196,9546,6205,6811,4493,4620,4981,3192,7087,1698,4640,3309,7184,10027,1415,10286,5310,6639,7931,7268,7985,7827,5357,7226,4451,3198,3272,907,4400,7874,2693,8270,1129,224,5182,5937,2456,5786,5376,6858,10031,8347,6674,2909,719,7610,9707,2280,2821,4570,454,1109,6728,2641,2083,3827,6375,527,2805,7464,2109,6703,4687,3888,4533,1693,5129,1584,1666,9938,2816,8219,4807,10104,9876,8987,5708,5002,5492,4335,1981,1218,3838,9858,4262,7663,4220,4977,2987,8168,2070,4652,7176,6441,1847,9095,548,6025,7993,4774,825,5303,2234,2723,1519,7116,9505,8903,5733,2085,10714,553,9188,10337,7713,1202,9666,622,1110,6913,6098,6919,3606,2527,4988,1575,8512,2938,9824,943,5447,8390,502,6559,208,6003,3171,10380,6573,4293,4607,725,5396,4133,7487,3207,2827,9390,8397,2225,3180,9917,9154,3381,8787,5649,3900,8602,3382,550,9155,5243,10205,5544,7716,1898,3418,9973,2980,5518,5537,2688,2846,8070,7531,5541,7074,609,6245,10397,2928,8436,3791,9216,5878,10398,3843,8742,9049,1822,997,2896,5144,5627,3401,6631,10331,3026,4967,6424,6561,5984,2261,4169,5254,8682,953,2099,2718,2576,1874,3065,7816,3884,9387,5703,6979,43,8335,6081,7102,34,3036,4970,227,7938,3755,3128,1294,9349,9775,8297,8364,10298,1041,878,6151,9329,4073,2421,2747,1650,5944,6702,1066,8099,3895,10442,10675,2512,970,3557,4935,7790,1673,9849,2617,9676,8223,5555,4702,10692,3911,10284,2102,4360,8120,6237,2230,2593,179,3177,4358,3656,4554,3386,10452,6607,1289,8495,1383,10679,10582,1553,5596,2708,9983,6126,2891,6618,3551,3168,1492,5407,4569,635,6862,4831,18,8222,5819,2141,6911,2072,9295,3262,2731,588,10769,3,8465,4725,3417,4813,4686,10035,962,1200,7636,9137,8029,9165,3472,989,7333,3824,6558,2323,3921,230,2297,2007,5099,7516,1664,3724,2962,6153,746,10537,2069,957,2996,9258,6710,5471,564,5390,10157,9460,6540,3445,1300,7292,7307,6208,4093,3073,2390,4067,4455,7483,581,8384,2146,4781,3782,3597,4928,1016,5620,8492,5020,10529,2278,1195,5270,4763,4963,4202,6966,9886,6401,4978,8470,10207,4779,2696,4047,1959,9798,8581,6645,440,8137,4008,6100,7420,952,6812,4066,9977,9796,4356,9729,8577,2239,205,9482,3907,10093,9807,5273,5662,9774,5186,8951,1176,1120,3554,1911,5409,7196,5709,7700,9086,5284,5158,6318,2283,7271,9726,1561,4306,5566,597,4402,4274,2866,9639,1906,6700,4746,5136,7706,7619,4697,722,2544,7050,7902,8101,4941,7191,5408,397,8324,6689,6942,10472,5584,788,4110,8252,685,9904,7648,8339,7959,9007,1277,6982,5646,4823,10571,4695,3740,9318,6410,8726,8960,6662,7917,6380,1257,7304,6019,9022,9418,3856,7073,2474,4377,6901,8463,10297,5338,9130,7159,5868,174,2448,9592,6718,9738,9437,1077,7846,3592,1930,8149,5793,3208,10786,10274,8634,6796,7694,5592,8808,5736,8745,10580,2473,2288,6781,6612,3006,673,3719,827,5924,981,3109,691,1754,3800,7061,10595,6365,5239,2796,6958,3502,7011,8259,1580,198,4722,7255,6079,7437,8717,7449,7451,1010,2014,1147,624,1879,9914,170,8196,3196,7528,8799,1417,3625,10468,2879,1393,9651,1647,1660,2960,6099,3365,3520,3071,10327,3833,9445,9319,9254,51,1458,7646,1244,4040,8877,6929,426,4992,9565,8117,7672,8525,521,5466,10131,9566,9937,7008,7932,1702,7734,7324,437,9503,10403,6289,902,756,5575,3941,8704,8094,2039,2159,1586,2950,4610,1704,10805,2168,7701,3650,124,5299,9601,6930,4417,6480,4562,7471,1104,8409,10717,8479,4691,1793,9624,4711,706,8963,9828,4097,1142,10814,1045,9699,1971,9946,8709,704,6743,8411,5226,10260,5029,10289,6962,3745,1918,9059,9749,10,300,4480,8050,5900,10544,10522,2240,3321,338,2601,5406,9630,7578,9529,9945,1995,2645,4430,1438,3562,7843,7381,3946,9465,9909,2769,6499,2497,2679,5826,1471,500,5209,7301,7704,6553,4017,9170,1639,9210,8585,5162,3329,3614,3042,4312,5622,3408,7742,3555,2828,6027,5329,1237,297,3009,2720,1220,7023,9232,10474,5783,3714,1654,10301,3469,5996,8216,2151,5869,2244,6037,310,720,10749,2625,7433,4170,5230,3709,8383,1332,7718,4166,98,6991,491,8643,9419,10593,10155,3995,560,1065,4563,2874,6336,5227,10444,4103,9894,3708,3414,9225,5192,2700,8004,7806,4553,529,5542,5467,2342,2806,10054,2546,10112,2773,7369,1432,3114,1258,169,3938,1868,10731,10171,6778,2509,10462,1359,9507,2555,1510,243,2130,6258,6043,782,1859,275,6927,5585,2524,1578,6502,8647,2859,388,2872,45,1954,3885,6984,3306,5090,10632,10402,959,1262,2514,6790,25,6130,9166,2551,81,10790,7922,8489,4269,2597,2030,9033,6732,3896,309,546,3069,3119,9344,4192,8690,5253,3999,1880,5050,8471,2488,6390,5202,6643,6148,7563,3775,8817,10055,8012,1581,8906,9242,8562,5389,5545,1309,3801,36,7323,10467,6271,10479,9123,4946,9157,2592,7343,9825,8560,6331,8047,6358,1982,5096,1124,8841,4787,3749,3960,6627,9767,5369,1871,5828,9674,2321,4333,4556,3077,4827,1298,9342,7069,6733,3985,3127,7791,10649,499,7309,8982,9927,5228,6366,10366,8890,1684,1240,8646,1329,10267,5011,3920,9820,5530,5721,401,7906,7411,10165,5731,7346,5392,3313,5361,6774,9427,5933,6588,1338,1448,5875,10703,9880,2609,8405,1478,4818,2359,9553,3532,10597,10216,10758,7719,9908,6873,7384,6417,7276,5635,2076,929,4908,443,6445,7987,4855,619,3753,1677,4080,1226,2944,10264,2090,8176,5097,8984,1136,5564,4481,787,187,4470,7629,7419,1037,4940,8055,1736,8184,396,1167,1112,847,4203,754,6775,4339,2464,8624,3712,6697,5752,7936,8211,4524,10809,5431,3481,6975,2290,7793,5444,1219,9195,2394,2624,3366,7751,3154,6048,3068,7934,2558,5534,4852,5634,479,8374,9151,5511,373,178,3044,7461,48,10038,337,552,9912,5441,4089,6804,3813,6528,4757,743,5667,6034,6004,5207,2782,9678,2628,6042,3893,4160,4279,534,8861,7058,10822,5016,8859,3797,3454,3490,2134,4419,5174,16,1126,10708,3793,1640,7485,9619,8720,6869,2205,5955,7095,2534,1900,4454,1711,1937,148,4747,301,5025,5417,7185,9300,5468,3669,9267,463,94,5863,4997,9397,10186,1267,6221,6114,10534,3851,4134,8072,6920,3837,6086,193,1131,4841,5927,7606,6635,8689,7450,2692,6303,2173,7252,8371,7786,6369,5084,8551,10615,3504,8564,2144,4349,979,4840,9867,4439,8834,7544,1854,4778,4748,1168,8088,5102,4129,3188,8353,2947,3101,8937,6231,5520,5222,562,3142,5259,10265,2635,7057,10141,9516,2717,6108,7979,2557,4973,9846,3441,10052,8170,7368,2885,410,8458,1357,10393,985,7228,1282,4270,10796,5229,10390,6605,8360,5790,2790,2053,7336,1092,7826,8287,10780,5747,7884,7217,7968,7211,8034,5252,8941,1344,9458,613,10589,9644,9801,7208,2034,4531,1547,3623,7293,8299,4063,4476,3605,8693,5536,9077,1406,8294,2152,6698,3932,1347,9292,3832,10085,2754,6238,3396,5054,2169,3825,1850,1169,10824,5485,6481,10261,3103,2582,8763,4272,7400,8918,6833,3506,4401,7,5114,2335,7929,5482,255,8239,6935,3544,5337,10521,7168,1255,4670,4906,4627,1080,1772,6750,2289,6691,3443,1265,3471,8676,2330,8530,4265,5194,6665,4753,9073,6876,8733,764,1915,10305,7685,10110,1395,633,1216,2180,8927,6757,5418,10478,7777,1730,6842,5205,2177,10655,2266,484,3226,6185,1160,3317,9889,5922,3881,6174,10231,10424,2977,5423,7418,1657,10108,8139,8965,9551,9248,4918,1418,7807,3576,6531,5435,3548,9781,6265,2022,8838,10050,9452,5356,1701,9740,7991,573,7199,9652,3463,2965,8549,7904,9891,6672,911,4387,615,3519,4388,1020,4876,5373,2683,1053,7004,5326,4914,6278,3986,8309,9062,5903,603,290,2818,1143,6353,6415,8171,9804,1611,6193,2108,8444,1775,5236,5972,7565,539,2254,966,9399,5416,757,10096,5457,269,945,3977,10309,6824,8452,8582,4332,4385,9378,5475,693,4676,4363,8152,4368,3957,9345,10148,5921,1696,3560,7695,2181,5573,9158,10681,8480,1752,3792,1414,4408,5210,2387,2664,2632,5183,7215,8404,7823,6220,1248,9657,8248,9261,7912,1299,295,8114,2768,1796,8263,6302,8425,4423,1483,398,9381,1161,9364,3187,6235,3284,3645,489,384,9954,10620,5094,5710,8518,6985,10223,4911,6497,4863,2744,8505,2457,4255,4608,5249,6653,5034,7254,10300,2856,6155,204,4331,10061,4900,9902,9204,4525,1006,5693,494,5395,881,6593,7841,1633,438,6072,4317,4592,8702,9720,312,6069,3643,3123,1098,2042,4413,7156,2612,1774,2953,5645,6352,2250,5289,2410,1479,3341,3368,10174,10275,1137,10601,8438,7885,237,211,10572,9035,2386,10608,3435,6486,9089,8429,10676,4151,10550,8801,28,8911,4022,4379,7463,2249,665,865,9245,5784,53,3529,5865,7329,10145,1530,5481,6994,8664,7257,6539,7214,7245,3186,4756,712,4749,9489,3295,1290,4479,8496,10623,9961,5429,7669,2607,4898,8532,257,678,1416,10542,8293,645,7649,1380,6632,6716,2125,8759,2757,7342,5086,8143,2206,5168,936,577,6182,3702,2363,5757,5462,7798,444,4501,1179,10603,8046,2630,5605,8310,4366,383,9455,4405,10625,17,3758,3384,10224,180,2780,2521,2526,6614,6432,1855,3416,7491,2157,4409,7647,9671,1130,5167,986,356,3875,4143,6633,5574,6458,6947,6581,4002,4497,1217,4487,2123,652,4362,6420,7373,10520,9519,7107,8666,7933,1904,10487,5193,7229,6242,6749,10098,5557,3122,9423,6707,10492,9936,240,6158,3270,1190,845,10324,4226,8229,8439,1447,9593,6046,6830,9669,268,6135,7187,4469,2834,6275,1364,2435,860,7797,3175,2466,2051,4422,9877,7469,569,238,1210,4392,10269,7259,4860,591,6319,6886,10650,6828,594,3484,8442,2191,3691,2760,5056,8056,7413,8242,382,7542,698,2245,6626,2149,6986,9486,1600,4504,1762,9492,8478,4232,3367,7981,1963,5075,7549,5293,5787,3974,1141,551,4672,4893,2686,2799,3299,4252,1384,965,8068,5287,6178,5760,3452,5004,8710,74,1577,10413,8559,8543,10409,2302,9214,8696,6752,7795,10742,161,5506,1278,26,9831,8007,9443,3546,4244,7760,1285,10504,3433,1184,8103,3711,2465,9203,3867,6934,9308,10084,10450,10051,1003,2574,5257,3024,424,131,10333,9287,9577,5388,10707,9275,1187,9854,4465,4138,4558,2676,2214,9021,7147,10727,8373,8831,9235,2164,2787,6047,9856,9942,6568,4180,8769,10427,7818,3816,4742,8514,8803,6164,115,3275,6073,10588,5692,9935,7037,8025,10604,10481,6087,10564,951,1895,5316,6313,8663,7613,10103,2111,8723,6902,6438,8854,6644,922,5059,5453,4276,8883,6101,6767,636,3203,5740,7872,9727,10037,3269,7824,10280,9539,3743,8633,2163,4065,7761,4029,6370,7149,10362,8350,6572,6301,818,2652,4207,10102,7320,10419,7562,10411,3777,5942,9787,31,5754,6485,6160,1525,5220,2663,526,5597,5489,4399,2229,20,10463,9182,7745,977,10635,9484,6177,7399,6125,9011,6822,6371,4934,8127,8528,2675,1940,4257,8109,10006,8311,3508,607,10605,9252,1253,5954,7051,6724,9471,10387,6916,6017,4965,3987,2992,10187,5810,7731,8994,3933,3796,6015,2540,272,4441,8497,10200,9042,10150,10436,7864,2794,9636,1313,8575,2704,7966,5141,2086,7260,5519,10376,7327,2368,4895,37,5053,2803,1703,5449,4375,441,10192,805,1480,4523,8814,519,4583,8256,7999,1336,4032,837,4140,10249,2346,10221,9265,5309,7828,4489,9606,6311,3316,10668,3537,10440,4740,3903,353,2445,8365,2933,2131,474,1634,5556,8985,194,1246,7088,4482,724,1251,2775,345,8363,6364,9129,2338,4877,4172,7958,6483,2694,4983,191,7394,3030,5836,483,8851,9015,7703,3176,2600,9477,1876,5850,3850,10036,8563,5888,810,3460,5907,4738,10831,8341,4625,8627,4829,4508,8578,5820,10117,9988,8797,6993,5606,4034,1499,363,3679,3980,3902,7890,2786,1564,10664,7982,2019,10518,10169,5653,643,7223,6269,7049,7006,7862,6305,3219,262,8182,10293,1421,9881,9852,2350,9263,4048,5057,5583,3846,735,5154,8202,4421,1994,2310,8950,8833,3730,2972,453,10577,3260,3033,1189,4909,1667,61,969,778,9520,4601,687,6094,3894,6754,2348,7728,215,2639,7737,2256,1707,9302,10254,4594,6429,4577,1792,6844,6138,7075,8590,6196,4168,4954,2602,368,1939,10715,555,2649,5043,544,5938,9241,9862,2559,4881,4528,7785,3076,10023,6070,6104,9163,7541,1997,4069,5654,1733,1031,2605,1462,6040,2307,10516,8922,9559,589,4500,2706,3292,305,3259,3423,2660,9578,3752,6078,1846,9572,5135,5505,10826,8596,9883,8781,7246,3113,2467,10340,7523,4122,2207,6505,3169,4591,8534,1081,5916,7123,9494,2440,1849,524,919,427,8815,4250,9836,72,8603,5199,10392,2428,7754,1695,2227,10491,5261,7570,6329,1401,7925,10776,7225,5365,9591,5493,872,4095,8093,4068,9080,8785,6455,4800,4598,8956,1322,8714,804,6712,9224,3015,10211,7838,5770,8743,1668,2666,5611,7459,8387,6616,7687,4766,2439,9377,341,4361,8402,5049,7112,4549,8337,9998,9510,7975,8163,6532,7379,10143,77,3429,1646,242,7068,900,4998,160,9124,9066,1632,8158,6530,9972,142,4665,2943,5898,2228,2749,7432,10262,10634,6288,391,6648,3489,774,9526,3201,7602,2883,6739,5800,8821,2870,6928,6765,1596,838,4888,9053,10028,4892,3062,5609,4593,2830,1592,492,892,5067,6649,1264,2396,3898,3096,192,4070,9517,7436,8895,1685,9215,393,8900,5841,3061,10259,3462,7676,1236,4764,5737,10457,6065,0,71,6355,10827,5455,1266,8113,849,9890,1765,2758,9168,6791,4739,6598,9055,7653,3859,9150,8102,8060,8625,3091,5269,5242,6201,1555,1734,5775,10119,1839,6543,2742,9730,6571,7707,6187,8002,3473,1933,1392,3287,4758,1536,3364,4210,2586,891,8198,9847,634,2444,10673,3967,1678,5113,2697,5083,8973,10383,1341,6668,10130,3492,10374,7928,1146,995,5883,8124,10320,1783,6170,9186,2725,9702,1385,973,1831,10823,6579,4473,879,8822,3594,6957,2451,8661,3751,4230,2684,10113,4453,4989,8795,7556,1737,10021,9985,3468,9990,6748,5803,9307,6963,8434,1970,4041,10015,6956,1602,6175,2533,3140,5563,9187,9113,1531,371,5304,7945,1250,9777,7372,10587,10641,1388,1197,7621,1371,2887,6490,7698,958,1683,2886,6075,82,8678,3263,10317,7557,2300,7605,10183,2990,1779,1809,7699,6577,6140,9878,10674,9788,5424,5001,4896,4305,3102,7077,9266,5986,10767,6642,58,1204,9964,528,8418,4159,7032,4883,2050,5707,5498,8066,10722,2712,8262,9131,3407,1769,3975,4636,3677,10627,2189,167,884,10381,407,3248,6735,9513,6315,8108,1604,5725,4020,5862,1145,7764,8819,4222,9845,7583,3906,2939,5031,5106,7146,324,4733,1466,7859,6495,7328,6082,1658,2498,7569,249,2789,6677,1177,8049,7514,9575,2946,4074,1758,10817,5180,199,7581,4281,10626,10414,4966,4415,9751,10677,2752,8620,7121,7118,3277,2484,8561,7691,8433,1740,4667,8908,5071,6687,8491,3488,6863,10594,9895,596,9958,2677,1716,694,5604,7105,3496,6475,9748,677,5109,3072,5185,5656,8762,10292,1215,2841,6501,10291,8231,8267,9524,5076,7262,2948,10602,3427,4693,4979,10793,498,8586,4411,2171,9871,3595,6596,4483,7152,8719,1892,8735,7782,1186,9879,7572,2680,8857,5371,10384,3670,4870,4237,5824,9562,5608,1291,7021,9079,2729,8598,1662,579,9736,7560,3736,497,6604,5856,5155,8250,6322,4982,608,5607,5110,8840,10441,6084,3390,181,6711,277,8218,5612,7789,10086,1570,831,3565,5195,7558,9770,472,10067,6664,9844,10076,4144,8260,409,10310,3652,3835,4580,8580,6890,2785,1152,1723,1025,2192,653,1619,9006,3318,10494,7926,1441,5082,2092,121,1402,10133,9830,317,4115,7423,5618,8165,9530,5274,4477,1349,6304,7915,6965,4378,1330,5301,1951,2035,2063,5383,1473,10176,1865,5360,10646,3626,10396,4123,5599,7137,5558,2767,7386,10821,8443,6107,2648,5936,1481,9603,6825,10138,9870,2899,4003,10016,4078,4847,3157,8018,5902,5176,9827,4550,5657,5039,8541,9218,10493,185,6470,5527,1934,10753,5638,7589,7708,8180,7084,1067,6736,1230,6667,2653,2848,6937,3607,3107,3510,8807,359,5857,9537,6449,6327,5699,1420,3131,5349,291,8755,5788,9277,10350,1867,1700,10533,8450,2638,9389,9755,8234,3297,7277,9906,1099,1706,5756,7898,8979,913,2296,8399,9207,1044,2260,5931,8685,218,4371,2246,6223,10552,8933,5224,8655,7407,9617,5838,4971,10800,7822,5831,1996,10716,6076,7365,355,9487,8291,1548,3465,5156,7967,7580,6813,4263,7671,1756,3880,4502,4223,3152,3112,679,7499,1083,2115,8969,4718,5705,10475,9742,8812,9362,4581,2691,5488,1851,5320,4516,8565,6442,9805,2882,404,8144,6839,2347,7740,10510,9438,2218,1894,10237,5515,2595,3612,9882,6576,5386,6029,4707,7919,4188,8863,1811,10551,4259,1355,2941,2513,7157,1238,6600,6846,2305,10399,2091,9109,3819,4503,6566,9339,666,3016,8446,1429,5981,3374,1890,13,1158,2547,2008,4669,914,430,5632,2098,6005,9812,9579,5213,10508,9335,5362,5456,10182,1222,6383,8730,5033,3828,8051,328,4004,1116,3118,9357,10105,10120,4460,2326,6899,3218,6726,294,10087,9763,9993,946,8540,4582,3647,5967,5830,10232,1198,3521,7664,9406,6391,8331,3680,7484,9757,3834,4372,4072,2232,3981,9256,2208,8740,10787,10177,1095,8153,1156,5377,1927,9905,10364,4437,3864,4036,7455,8485,6552,8179,2895,256,10591,3926,3621,7635,6226,231,4098,3050,7871,2975,1328,3993,2746,7594,1742,1363,7272,3770,535,10750,2554,3815,6006,2750,5312,4251,887,2640,365,370,2025,858,7679,1629,2332,7809,6067,10346,7265,828,4638,10507,2865,10214,9680,8457,4337,1232,8616,9570,10312,4236,4788,9508,7863,1164,10080,9911,6832,748,5947,5055,4490,7241,7830,5081,2588,2104,7120,3918,8935,9328,6350,3200,1165,1323,1310,7632,10531,2292,5567,5248,6954,7839,9456,10290,8459,926,206,10654,8953,9898,791,8744,5923,2724,7145,2520,10238,385,7318,5018,10669,6115,1241,436,3512,7243,7151,899,2921,4355,925,696,6260,9253,6670,2807,8507,7930,1369,8898,5160,10285,3355,6282,3556,8356,2380,8825,6052,2329,5726,6403,10773,1990,3661,3221,4407,2826,3854,8501,5348,4414,8340,8268,5359,3470,1748,4559,4927,3158,6472,5009,4613,3456,3641,1029,8226,9454,1111,4628,2736,10272,4346,2313,5108,3836,8220,2183,1838,7103,8842,6474,4996,3953,2900,2741,1413,6217,7024,168,10079,3184,1069,4696,2713,732,2811,5398,3734,10695,8998,3873,2518,6440,6077,7964,2814,5125,9414,3674,9855,7888,5711,4962,6515,9472,3385,10090,1140,7833,9499,8579,1770,885,4092,420,7519,7736,9819,3020,10765,10170,377,3097,9930,3300,8057,8160,7072,9842,3573,4700,10129,400,112,2501,6384,10234,6646,7571,10256,5091,10525,3739,1339,8315,6436,9156,9478,4064,8967,10360,963,9175,1797,10711,7623,709,8482,5317,1404,2298,2238,8523,467,5436,7810,2150,2986,4719,6926,3784,5673,6884,7434,10659,9200,10461,157,4894,3323,6683,1626,6970,7192,3089,2010,284,6192,2216,7284,7009,4215,6961,4191,1305,4561,1573,7279,4930,3604,5691,8424,7538,10570,9135,333,7851,10073,4449,8403,942,7616,697,7456,6232,3927,6737,2496,406,2148,8078,9957,1477,6428,7171,9140,7347,8026,10135,9114,2643,4874,5601,5460,8474,2415,6127,10161,6609,8401,358,9093,5912,1097,1372,1687,9167,4666,7503,6968,486,4649,2994,631,7585,1318,110,5484,4181,6492,9003,4574,2392,3821,7135,40,1243,5264,8097,9474,7316,4297,5132,9716,8683,7505,8173,8623,4025,9262,9004,139,9434,1932,5354,6023,10367,9382,1346,3654,4185,303,8265,372,3214,4009,9784,8200,1738,10640,7092,4958,10656,164,1493,9741,4195,7624,9967,5063,1153,8298,4576,3360,10598,8380,6206,8119,6874,1972,10759,4806,4905,4157,4915,6343,9019,8680,2510,9372,4261,6669,6147,1830,3667,3415,8237,4655,4886,10032,1902,4456,9969,7536,4750,5825,777,2791,2247,1062,5188,793,1648,75,8820,9361,2656,7725,10783,924,7950,6871,620,8261,6285,3250,6145,824,7289,4735,2374,5399,10435,8637,8613,2571,10782,2078,9978,2999,2333,4018,8650,4039,4015,1778,4513,1054,5859,5147,2082,9264,6243,2203,2096,423,2267,3149,1386,6080,974,10811,3804,1913,3228,8454,10109,4929,10316,8212,9185,1093,3257,6092,6493,1950,1686,6018,9056,3332,4715,99,6910,507,5964,9558,9228,4517,1826,3495,3194,6097,8058,10532,4190,3084,3290,10218,2506,10447,7453,1113,6823,7587,7315,1722,8789,5370,5553,7486,4542,1731,4548,5700,2136,7799,2776,9087,3727,5744,780,1495,1956,4782,1621,6589,8159,9043,9745,1980,7983,197,8232,5746,1908,4471,7693,3534,2061,9370,1688,10779,4187,83,4794,4024,4291,7465,10665,1958,7533,4603,4752,7628,6389,6734,1691,10019,905,9213,6314,10599,1601,8130,6972,2915,9817,10081,8174,8396,530,8850,4277,3728,323,4174,2734,5118,9031,1899,3312,9208,10662,8092,7899,5874,2495,4175,5012,2223,10735,7312,2315,3298,6798,1103,2430,6261,8902,2360,3000,5331,4412,7136,6529,10357,1235,7086,1354,8490,9231,1773,7554,3343,9356,5880,9400,3139,3812,2371,7905,10004,6563,4071,6622,3286,6468,6877,7383,422,7286,2801,5366,8977,4373,9107,1500,1718,10720,5853,9951,4907,5448,4056,3255,10506,2853,2710,6681,6308,10495,10014,6382,3693,2989,930,8760,5345,493,3372,8388,109,2257,8604,3281,9058,7055,543,1078,971,4673,8123,9141,6249,10815,4925,1343,4186,5233,4330,8253,7775,4229,8286,1589,6157,7588,186,4246,6168,5092,9800,8251,1872,6856,7831,1625,5048,6374,9731,9281,8147,87,3498,9337,2964,156,9931,9152,7332,2006,6538,5306,3403,3530,4212,1967,7953,4130,5624,50,7337,7016,9282,10149,505,9075,2585,10083,9247,2334,3132,1317,3764,6095,7896,5835,2433,360,781,6222,5943,331,4475,8975,8354,8246,9999,7231,4193,8503,10226,2248,7684,1988,8824,4398,9422,6083,3853,4055,2889,5748,6848,2455,4088,3622,10636,5410,984,8974,4343,8006,4,7013,450,416,9714,6152,8777,7944,2564,5774,190,1653,8321,321,9633,6545,1676,8320,1848,8086,7319,747,7954,1759,4183,1552,9354,8009,1485,4239,10592,5602,864,6637,2838,10519,96,5750,6640,4932,5570,7617,2784,2058,9229,514,292,6324,2654,7338,4001,6225,9754,3618,5631,6465,4488,3877,6943,1252,3608,403,1817,8484,6156,4136,2385,1739,3742,5321,3375,7356,4348,3762,10471,6396,9220,4730,8295,5687,6482,7179,8529,7812,9840,8546,4023,2419,8631,10306,8118,3686,9690,2376,8010,1790,8244,4836,5854,10342,6896,9311,2552,2959,2549,7330,5716,4090,8104,9432,4286,1969,7717,1641,6629,7784,4532,4773,5201,7475,8544,4568,7805,126,8284,4316,4109,9722,9806,713,8776,1823,5215,2685,4731,5263,4632,7913,329,2337,10372,1149,2869,10829,1807,5267,727,9417,4854,1781,7182,6198,4322,8273,2737,629,6525,4571,10070,2199,103,9009,7401,3480,6379,9915,2459,340,8079,9816,8729,4991,7163,449,10585,3830,1303,5191,10583,559,6351,8228,10375,5755,861,2832,9980,9373,10622,10115,195,10058,3424,7233,503,4106,950,1812,3928,739,3303,4323,5330,9768,4381,4242,433,10485,6586,10230,9088,7743,4875,5779,7237,1295,3936,8076,1968,7334,8926,9096,4162,8912,8040,10712,2976,5479,5350,821,1178,9177,5346,9466,8955,538,3282,4964,1538,6557,10125,7504,117,8816,7113,10611,5639,2658,6323,1058,5225,2089,1457,8539,5003,1422,9310,2364,225,9939,3225,4609,9029,628,10553,8375,630,10660,1579,6188,1409,1482,991,4278,183,7494,6661,7452,5152,803,9230,5069,119,2954,9246,8853,561,9747,7119,7844,3151,3955,9268,2462,3690,2777,1061,5060,1301,7417,4142,9286,9197,5145,1943,4228,70,5872,2565,7920,4234,8509,1964,3264,1353,9314,4155,7566,3978,7020,4466,8572,6074,7285,6964,5065,6312,7521,2001,6059,4828,6978,5655,766,9584,9555,9782,1777,2147,1529,10326,1914,4458,7162,9385,8307,9094,5427,9773,9014,6722,9276,4564,6597,4817,6293,7239,5005,8835,10315,648,2842,4121,3213,5619,5499,9444,3345,4205,55,2443,3344,1260,1955,288,4198,2187,49,4201,4150,10511,2304,9994,9067,5486,2556,5116,4369,3994,10373,7651,4736,5532,5378,5525,3630,9646,1115,3542,5823,7457,3007,8642,7690,7681,3646,4698,2880,10245,220,7349,7596,2356,9590,4745,1568,4596,10195,7952,659,2722,5494,4659,4584,2409,9435,806,7939,10751,3224,9531,399,7639,644,10428,1638,5837,9705,165,6988,10020,1154,2274,3205,8369,5121,3074,8635,7796,3311,2017,1279,7802,9409,611,10613,8748,2719,7354,681,7261,3070,10667,8757,9039,10745,3525,3079,10282,10700,3289,3278,8440,6431,6914,2469,5974,3681,4514,4597,10432,1144,8957,2397,1107,2604,5101,6179,3523,8329,9023,4418,7195,8305,1949,10788,1882,9405,5973,1952,2778,1428,460,3331,5445,2221,8962,987,6451,325,7553,2037,9884,8864,1042,10308,6434,8136,2481,7814,3012,8860,10509,3874,23,4910,7189,8194,2365,6671,7280,5314,4485,4196,9020,6131,7787,939,10781,226,1059,56,1931,2427,8571,6864,8022,9753,281,4720,6020,801,4864,967,1631,3404,3935,3352,3075,8749,8639,3130,1776,1873,8059,2222,9097,6887,7043,3879,10097,248,5104,7506,5397,1074,2122,65,7109,832,1425,1800,4744,8362,9872,3115,6692,5038,10738,10359,2911,8909,575,4194,2770,6142,508,7749,10721,8806,6786,8630,5877,8660,7921,1747,9995,7065,251,8126,4420,5935,9046,9797,7781,8468,9965,5978,4260,10616,1502,408,7270,9899,3883,9743,1597,5019,10154,5107,4851,54,4145,3108,5500,107,8191,8768,5208,3886,3105,5664,4853,1005,4624,7467,6011,9481,10724,556,6678,7161,5966,5720,4509,8416,1545,2064,9663,2424,6349,6676,1513,1724,2434,7603,3487,9171,8966,741,10283,2802,5681,10775,10663,956,1337,10349,8715,4952,5163,2301,4741,8952,2178,7997,5296,5805,6251,1096,3232,8504,5671,4575,9924,2113,1139,7848,2702,762,6769,9607,2923,5561,6776,6713,3106,5440,10377,9944,1979,5535,7321,1610,8204,10466,5769,9677,2560,4030,3181,7903,7858,7282,9948,566,9611,4326,4060,2591,2647,9081,1750,285,7735,866,2914,4045,7177,7989,9795,1926,3330,9536,7732,9892,1557,4094,5812,405,5633,8536,8695,9533,2611,4492,4957,2852,5221,7234,10124,10078,5953,4845,3526,7409,6139,7656,9701,6310,11,10344,836,1464,5250,7003,8788,738,6309,4510,4484,9695,3725,3767,5676,2317,3644,5523,5818,38,3052,8554,102,3598,2020,4972,7018,3958,5546,8366,4859,1965,8673,1815,9159,6297,1027,6939,6212,7091,6184,7111,2528,6725,5461,282,7132,214,6624,1489,4623,7448,8074,7392,1623,2215,470,10335,8668,8385,3493,2378,9963,933,9974,6119,1128,1558,6766,6159,9991,1015,8866,4052,2845,2401,7697,10633,8591,570,7640,5926,7038,6762,3359,6013,6807,4334,2033,9534,576,4376,9070,6625,9341,5451,2748,3559,3806,10642,8999,6527,8377,6112,3501,599,7961,8929,2753,9588,9512,1456,10569,153,254,2399,2695,4403,5235,8772,1710,4367,9376,3855,3110,5382,7207,3639,7311,12,8852,2438,2979,3315,2781,3244,7012,2354,7642,1614,6227,955,7414,10271,2480,6219,4648,6898,6090,6551,1449,1720,8568,10228,6247,8818,7883,4544,5795,4689,9309,10465,3160,4177,5758,9008,671,3944,7988,1033,10482,5552,3930,7331,10089,1331,4124,2407,7238,6368,6992,9211,1408,4995,9761,7355,3904,7053,378,3497,549,4213,10030,8981,708,3505,7310,3698,3863,3590,8796,7165,4321,3241,6294,2763,8475,10404,6906,7174,5066,8161,10175,9609,8280,9758,1767,5095,9682,106,4819,123,7041,10099,2668,3682,3696,7160,2233,7359,637,2331,60,1615,7552,9523,9956,3642,798,4467,3388,8948,9756,266,3267,7891,1465,5478,9794,1340,7686,8045,2105,1543,3013,2584,2491,1183,7188,6345,4082,6806,8692,10766,4993,1391,3760,9084,3803,10010,145,6387,7281,7236,326,1986,5103,4249,7122,9527,5212,5994,7478,6400,3631,4809,2088,6200,10792,3705,6976,10541,10702,9662,379,4984,1942,717,2045,6793,744,6511,819,6971,2837,6630,487,10621,2471,4694,5571,6536,3353,4320,8767,4639,5151,2577,5491,9818,8493,4253,10610,2093,10653,3094,740,3934,3137,4241,2314,5368,7314,8574,1841,4211,9259,1947,9760,4507,9343,5587,6894,133,3135,602,1520,9142,5641,5463,4221,9679,795,3759,897,8511,9201,7322,9887,8208,6163,5528,8199,4857,9838,8601,2833,7472,7025,6770,6783,4431,3839,1713,5496,1562,5495,2133,10208,9,1400,4799,7339,2765,975,1259,5663,5334,451,7886,4938,4540,3001,4463,4354,1802,8681,78,4152,2955,2202,296,3538,3017,1753,1035,2251,1076,3156,5829,5682,3968,4329,1297,9502,4328,1304,8061,4189,3878,1073,10672,10728,5087,10680,8304,3204,8032,10339,554,69,2259,2884,5297,4416,3320,7178,7582,5773,9620,9340,7497,1199,2420,3629,3671,3684,9848,8548,6802,10755,7446,5480,5177,10347,10178,4791,592,9567,1127,1936,9103,2855,8215,2460,3620,9769,247,672,1283,8133,10198,3314,9632,8699,9981,4634,9661,2100,932,3585,6941,8107,8181,3514,7852,7047,10699,4139,3393,3222,7340,1521,5364,3640,134,286,9673,2143,6423,4031,3973,6782,9735,3018,6998,8907,5791,7186,4573,7955,196,3509,1974,6459,9548,2936,590,9732,4167,1571,7842,8651,2400,10365,3032,1275,7972,10789,10489,6779,6300,2871,9412,7029,598,3120,7034,3659,2318,4527,2795,3772,2808,1427,1532,7868,5724,10830,1598,9803,4153,5341,5150,10045,1086,2352,6460,6912,1725,6933,207,2860,4767,1998,10458,3274,8323,3969,135,7856,3031,4173,1426,3199,2937,6105,62,5512,1079,10719,5881,5473,7586,9932,7205,9257,6007,8870,8784,7667,5901,6741,3048,10430,6829,1225,4264,4462,7957,8195,1976,3494,8891,2535,8978,5543,2060,4238,6477,1460,2819,1744,9057,9416,2732,1922,10345,4440,2395,6299,6211,3923,6550,93,8023,10477,9138,6584,3243,6321,814,5892,127,734,10201,1891,5430,8407,9692,10065,9574,6333,8910,4661,2621,3541,6462,10820,4313,1835,6603,6849,7117,1672,1837,5997,2306,8537,233,3099,9724,4704,1946,7978,2727,658,9557,9279,2699,2543,1122,1810,5533,7998,5275,3599,2650,5384,1221,2903,8445,5342,10303,7224,714,2312,2504,4247,2579,8931,8395,9238,1209,1024,1009,4820,8684,1007,7222,909,10002,5950,4141,5538,1887,8940,5143,7317,4771,9194,7847,5804,8017,2756,7744,5701,6652,2138,8116,10361,10172,1503,10806,8288,6437,605,647,9117,8013,2517,7875,1038,6694,9383,6758,1302,5508,2389,5579,246,8608,8274,349,657,7666,6555,1390,784,1333,5023,263,2568,6476,8462,431,4727,9191,6867,2835,9082,2176,3185,3913,2413,97,10295,968,996,9709,5231,5474,1572,9790,7062,419,3242,1542,2016,8640,9172,2262,8632,5919,8653,3706,9012,7129,10395,216,10418,4245,5941,7774,6747,8913,5816,8889,362,8811,10828,5870,9410,10279,4987,7089,8203,844,8701,7849,3451,7803,4545,9125,2709,3011,1150,3144,2705,10151,6533,5549,2004,3615,9051,9290,3653,2065,2924,3054,9323,6386,5454,7344,2854,6317,5078,765,3195,3882,9660,10739,5255,7990,1314,10619,1377,459,6880,5260,432,3678,10400,2170,10188,7755,3080,5821,2644,5353,8067,6360,8739,9776,3783,9650,3549,4478,7173,5381,1636,6491,10517,532,10566,177,3695,1157,7592,1501,10631,8075,5666,3536,1361,992,9199,7212,184,2437,5945,5569,1989,3566,2966,5278,4751,8481,3648,9336,5811,5813,9085,6997,3162,9457,4732,8487,10042,5124,1903,2589,5465,8844,8827,4950,7720,1745,4007,6599,7410,5080,6800,8751,7746,5443,1191,7046,8605,9491,2493,8372,10250,595,4595,1072,8583,10107,1075,8805,3668,7063,4284,2095,6173,9064,8129,3890,10643,3861,4053,3477,5781,4338,4955,6717,10180,7033,6918,1155,2084,1791,465,8150,8887,7351,1771,6546,3956,4904,8747,1825,3129,6032,3945,4849,6936,265,10456,2361,9350,6356,7303,9955,8317,3808,9771,7406,9563,8326,3063,6169,4266,3750,8343,4325,7190,7976,10114,5689,2322,381,6554,3964,4617,2622,7405,1743,1560,3744,8096,9581,3246,66,10455,4061,846,5058,7674,280,2412,5815,2844,3335,7641,851,4615,10670,2129,6334,1884,5890,5749,10740,5000,140,4296,2657,1231,1071,10746,3285,3738,10443,6950,4309,8044,4033,3216,6809,10687,2145,6344,1205,2824,1828,4231,8752,5010,5851,5930,1002,6759,3963,3809,8410,8869,10118,703,587,2128,8671,2209,6795,3483,2898,5680,4873,3700,516,4699,9002,4200,9785,7918,8241,6143,3870,8466,4862,376,5644,4647,9717,1101,8186,9637,9464,8214,7857,3350,6706,7729,188,6619,8015,9498,7675,9153,3238,5531,948,2038,3701,7169,8945,6503,8502,4653,10210,707,80,9815,8302,8151,726,7429,5925,2049,2761,271,5948,10007,10047,5372,9810,8414,8614,3179,4404,9083,4543,9655,9549,5690,9713,5120,10144,5987,1472,1375,2988,4445,2293,9404,4619,221,4119,4054,8712,4810,8904,6764,7575,6399,8138,4391,3256,3095,9110,7438,4496,8959,2372,73,1011,1799,42,2583,8398,4555,101,1174,1376,3591,10167,10449,9333,5171,9413,9005,2236,9391,9233,796,4879,10001,802,1714,3247,5743,4530,30,786,1476,6171,1651,848,5610,1794,4761,1670,9037,6057,466,5347,7593,4637,5861,3950,2973,3765,9365,1213,8449,6512,7203,5013,10586,7767,6281,2031,3939,4057,1896,841,4351,4340,7715,2957,3889,5594,5268,1727,2716,7395,871,2048,7402,676,9921,4539,9347,9538,6033,8691,1694,4146,6587,4811,10778,1348,4586,3046,10199,1368,9721,8476,7108,5683,4243,5403,5204,6141,2839,7792,3624,4028,4461,8467,1652,10388,485,3610,10091,4102,316,1268,1585,5623,6053,5490,495,833,9127,8417,4923,3085,1929,6814,7876,5963,601,10484,7634,10407,5497,10647,9205,7290,2922,8669,5315,8053,9071,1327,10328,3410,1319,5842,3426,7026,5742,4755,10562,6060,8876,9703,8980,4604,6274,3637,760,510,8300,7882,8325,10128,9476,6120,9900,8506,9017,7974,1132,2367,2441,8193,2672,7299,6755,8519,4792,4824,2968,8183,10648,10330,9913,10802,3019,4081,2431,223,7441,298,1858,2730,4837,776,10469,5140,7854,4789,3088,6172,7597,4161,4154,5603,90,374,4435,10813,7371,3547,10748,3455,2110,4058,4021,10512,7060,9903,8893,9868,6983,8924,6684,10697,5325,7096,6381,6248,9626,6803,6686,481,2608,4713,9602,4633,9293,655,394,982,10139,4386,2981,3820,9111,6452,8756,4986,8428,1595,3378,7733,3664,10617,1242,4218,4933,4838,8971,3570,1674,541,6022,9594,6325,6715,5961,10502,8027,7391,6506,1090,6685,3121,3220,1405,9469,3795,1526,274,8426,10379,8707,7845,7865,6453,10694,7947,4611,3633,1396,1463,1272,1941,6516,3726,4658,3768,9147,8460,650,1023,5439,3794,2487,1000,319,1105,3996,3466,1637,302,10013,850,1999,10163,6785,5234,4282,2904,3336,162,7296,4867,8427,1681,5962,5285,3327,9496,557,5272,8538,10302,8192,6021,8802,5895,683,2446,5184,9104,9511,7766,3163,2056,4096,7935,8894,2188,7576,1514,3302,1881,10709,2366,3713,244,9431,6122,1539,3632,7738,1325,1342,5446,209,159,10062,2,2382,7078,5503,3133,9733,4974,2599,4682,4410,5393,10251,2578,9691,496,6277,2405,5256,10363,9146,1036,10043,1622,9799,5470,2201,515,7817,5379,1269,7227,8266,3683,3991,10682,8486,7131,9312,7424,182,5763,9324,822,7028,4117,3868,6513,7722,1151,1013,306,9752,10423,1166,4104,2714,9693,7251,1247,2627,2901,4425,1203,7910,4587,389,3553,1824,8607,1679,5198,6419,2984,2738,2137,2161,976,6228,6056,6133,7573,2575,166,10072,4976,7615,2388,9162,9518,3474,10181,10368,9250,4560,2340,3308,6601,8042,7579,6263,4311,8804,7548,6537,5172,9244,1019,5672,439,7442,7518,1945,3897,1669,7300,1928,118,972,8415,3988,5814,7937,9395,8731,105,8654,3602,136,361,901,9255,7496,468,2211,1032,4302,7584,5425,4426,4939,6980,7079,1901,4341,6361,964,7612,2416,2511,3826,10325,1751,8645,3869,2172,3779,5307,2926,8813,7969,4956,3440,10385,10437,5041,3857,6425,9683,7479,3822,6240,811,1138,7249,3891,7056,7114,9288,9583,894,6435,7757,5625,6620,9348,3215,9750,5799,9280,332,9737,6038,2907,6827,1436,10808,1487,8885,6610,5522,1094,7971,2418,853,9313,736,3371,9126,7022,7247,522,2196,7153,9618,10044,6820,1569,5822,9500,5394,7660,10503,4885,5550,8041,314,2701,6821,6740,3206,6128,6704,670,5578,8312,2384,3665,6001,2120,1444,6117,7027,7867,6995,8095,5989,4985,9647,1088,9700,7819,5845,2868,8550,9463,8187,2243,5240,9145,3155,1261,6945,1566,4664,10378,8636,7956,2783,3790,3959,10426,7067,2623,1494,6088,5027,7747,9933,2242,9929,3662,5068,5846,2046,10480,1588,3447,1021,9105,7889,7513,4969,6286,4614,616,10212,3581,994,4723,4012,8348,1534,2066,10555,8112,2026,9298,9813,8162,9561,5560,2978,3699,5335,5178,8048,799,6427,7835,6784,9896,5917,5636,723,9901,8641,6330,5965,7361,1412,6654,8688,7140,4953,52,4204,10777,1196,1705,5487,7345,7815,2000,85,2408,3197,8786,2876,2155,6348,7364,3266,7907,8021,5593,6066,9429,9120,4511,7508,6693,9291,10024,2377,6569,9426,4464,4716,6815,7444,1212,9401,2345,536,259,8706,7388,8132,342,5559,5085,2843,6853,2286,4833,7555,9375,10296,10448,1524,7367,5134,7428,1549,2836,3582,7005,7155,9556,4303,6872,1787,4709,8359,3943,9063,7380,10136,1801,5035,10059,9653,8210,3310,10142,1512,5913,4127,4858,8558,8357,2043,4267,6738,7071,2958,125,6426,2276,95,2735,9078,6595,574,8140,6787,6284,1565,8865,5044,7019,3909,6705,10612,4692,1840,1606,5939,3354,6280,10417,7415,3189,4546,8081,6276,8665,3716,3432,5771,8370,10190,3485,7306,8328,4618,4515,8679,6745,7895,7873,5098,917,3083,1001,923,5904,6395,1643,10041,7416,5988,5847,752,7269,5105,8282,9971,518,1878,3569,8157,8517,8791,5957,9394,3193,6002,9952,2383,742,9725,7218,6517,2963,3769,4026,3704,7396,8083,5324,931,1857,2027,1836,6695,2402,6720,1563,3533,10451,9180,5051,4830,5855,6091,2629,7836,4350,4359,6634,7291,10584,7631,100,6234,6111,823,9198,7611,10011,44,6608,1511,9016,4436,9303,1764,1819,2277,5885,7832,10140,8378,10288,2875,6246,3866,5308,7130,7837,6514,4921,8225,9001,3627,3515,4384,10025,6129,3237,9541,2453,8031,1528,6881,6582,1288,6922,3230,9222,8333,8037,4512,9829,7366,3845,7683,8916,7070,2344,6272,9610,3756,775,10528,10008,1455,8030,5660,10764,1026,3829,5658,8477,702,4578,2764,5006,3919,4797,1741,3231,10704,9453,10278,10628,8451,2993,311,9968,5402,3039,632,8472,2358,2003,8024,3781,6268,7714,3535,9148,8552,558,10243,9421,8557,5438,4383,10162,8708,1439,852,2532,1224,9144,2610,3446,5157,4878,8716,5918,6093,4903,578,8125,10101,3586,7492,5119,9734,10287,7609,2273,10515,8014,8290,8698,6016,651,7776,5659,1440,10185,9648,2353,2213,3258,6405,8919,5684,6199,1443,7093,5042,8599,2490,7404,5070,9160,10064,6273,5674,8783,7425,3233,8753,9672,2379,4567,3461,5282,10371,7221,10370,9384,4790,7458,7421,9321,10160,471,1861,3138,9304,2477,3373,4345,2857,10217,6064,129,8694,7682,6181,8319,10222,5131,4534,6335,8773,9686,7143,8391,8201,2362,6096,8028,10116,4382,9139,5413,4754,6857,6479,10248,4353,3217,6751,9509,8793,4448,7963,6859,425,7811,5232,9826,2220,5477,7015,8054,3236,1100,1334,5723,6904,6489,5958,10683,10421,5801,4494,6259,7167,8587,10713,2294,7076,455,504,10579,7030,3966,8008,8703,1795,1842,5524,606,5375,2851,10524,10563,5036,9436,8122,7829,7512,10314,1978,2631,6594,5772,357,9359,3718,9425,8954,2311,9240,8667,6831,8135,7164,10127,7476,7762,6908,5170,4380,3268,5401,9450,7680,3449,3983,663,348,9711,9018,5785,3634,7897,9411,9554,1593,1182,2442,232,5739,5817,1788,10046,5928,9338,4775,9223,4759,1755,1893,9091,7894,10022,7622,797,4588,10277,8555,1172,7101,4714,9209,9408,366,5562,4365,4079,9074,4869,753,6744,10732,571,9179,5899,5223,4808,9206,9028,5428,6255,3788,6508,5914,1821,5283,3271,4626,520,3324,710,4390,8934,2670,769,8868,4084,3117,1192,8313,8082,5712,1798,3540,7274,3479,4043,1761,8167,7341,8761,3261,7468,8619,7048,7244,8524,6191,4668,10818,10197,1008,1424,10657,9916,2721,1712,2669,10158,5695,7924,4506,5840,4651,5516,222,3798,9301,8243,5509,2217,3124,6394,9786,5469,813,9689,9949,10159,5864,7853,9428,3014,6267,8172,41,9580,418,10239,8332,3992,3655,6562,5993,2881,6792,9353,5187,6810,1889,9833,4184,4336,2059,4171,7880,733,3603,2995,5146,4777,9704,9504,6326,5995,5148,3348,8991,10513,2005,10391,4520,9119,6407,3814,2032,4519,8141,9101,4156,7537,10179,9843,8073,1559,10488,2598,668,8989,442,6062,7601,8020,3925,7607,3917,3325,8334,4075,2271,7668,76,5266,715,350,8349,7374,7302,235,1630,10268,2515,10729,2637,4703,8358,944,4318,6518,7962,949,2997,5072,3685,2508,3778,8621,7488,1208,5713,6611,5643,7352,10574,5073,9621,3478,3051,27,6467,2516,7678,10573,6213,1517,3434,2698,6457,4111,4125,446,5432,10645,4872,1905,10000,6580,10812,111,2404,5437,3971,7430,2132,7942,2268,895,2906,730,1690,3676,10730,4083,6990,92,10791,1148,6826,3583,6298,10257,10266,9038,1287,4019,1382,8878,4268,9485,6121,10785,2118,2325,7825,1784,5582,5717,4393,8588,380,2615,1048,10651,7426,1567,3491,9907,9897,1608,8379,10671,10549,7209,10026,1726,5061,8392,8421,8255,10464,1171,711,6855,1515,344,7644,7618,618,5014,9744,8238,1728,9605,3887,8527,8705,1845,6216,6137,1782,6885,3183,8867,5276,7510,3428,4770,9585,3513,7230,5650,5333,6194,8899,6409,3575,8276,3421,9764,2991,6189,10244,6328,4814,4541,980,2047,10009,3811,8189,6166,8447,6466,701,9274,938,3459,2873,8406,7460,4772,3776,3279,5697,7150,4644,9822,1692,10684,3609,617,2309,9885,1134,1574,3008,273,9060,8677,147,1052,5734,2308,7688,3954,10066,3059,8972,6413,5502,3453,8526,210,639,680,8019,5161,6473,7127,3666,8589,10639,4843,1084,6035,8873,10454,3253,9934,2075,5128,8938,8292,175,10614,4994,10106,792,10752,2507,4832,151,7834,4931,2537,2918,9542,6709,8576,9468,918,3010,10352,1508,2661,8494,9448,4919,10547,3476,2646,716,5732,1367,9837,6010,990,1576,10189,2536,8901,4233,1984,6124,6682,614,9835,5137,10483,3580,8351,5782,820,7220,7878,1977,5843,675,9923,7527,4734,9861,2351,5052,2013,5998,9959,10063,686,5246,3844,2154,9664,2863,1173,3276,3273,2739,5126,7010,889,461,1434,7045,8923,3616,2071,4424,7960,1311,2373,6167,8105,1806,9069,3785,1583,5808,10810,8235,3990,1431,10095,2633,6673,5729,5598,10439,10225,4768,3731,434,2391,2974,5642,525,8750,8593,6574,3172,2375,5848,4999,241,3035,9189,9393,8224,8765,1645,3862,8592,8880,9762,1556,8920,137,6788,8227,749,6542,1050,10191,9943,3340,8520,7348,3593,5897,8875,6621,10496,8718,3002,8871,7752,3914,2226,7855,7090,5400,9289,5265,10137,2825,5906,5179,2606,4275,1181,6917,9587,10688,2055,10077,1159,5007,2766,1985,4717,10294,7515,2613,7397,5715,7590,6623,2531,5434,1509,9926,941,6868,1909,1818,2541,5336,2165,4290,10202,7927,5387,2011,9212,1254,5405,7490,2422,3087,5551,8649,8175,6055,4240,108,6655,10056,6708,7507,2121,5173,2198,1046,816,7866,2659,9076,6041,8513,8283,351,9480,9118,10798,8488,6560,8209,9271,3467,7054,5789,7509,4432,8626,7893,10565,2252,10012,5652,7658,6123,6835,91,9239,928,1286,2529,1378,910,4706,4116,3774,6949,8052,7495,5077,2596,228,155,7387,7778,154,7201,7193,5704,1944,6071,1582,375,2793,2548,8197,1953,10757,4301,4013,2849,8090,7232,10763,10736,7994,6989,477,9473,10134,1717,7360,3673,5079,10543,8207,7696,3431,2745,8461,279,4834,8746,128,2538,7382,10281,2888,9439,3789,10559,9442,3234,8213,8106,253,9573,856,10644,5867,1293,1715,6861,3543,10406,10501,7813,1256,9910,511,132,2771,1040,9839,5547,8035,9386,5616,545,8342,10307,5022,8098,2905,6433,6567,1680,5832,2494,5858,2655,1249,3807,1551,2186,1804,4801,3412,5322,5565,4468,1697,8570,10343,2282,201,10734,3383,3211,8393,386,299,5613,6307,8381,10152,883,9234,737,8606,3301,327,4105,5792,1966,8722,10416,6883,5217,8896,6342,4890,4671,6195,4901,2983,6464,7481,8618,8855,682,8473,3405,604,5093,335,9864,2562,5932,1345,1102,6651,10235,695,67,9352,7545,3400,2523,144,3402,8848,1888,5411,3464,3876,8737,1012,6973,10816,8063,3687,4452,5629,4091,2690,217,2566,3697,8888,8121,276,435,1649,6797,3034,7801,3799,9865,2483,568,612,857,4498,4663,7730,7511,3619,6000,5969,261,2665,4721,5970,8610,1506,5728,4882,2328,4712,4825,1917,9495,8087,4254,5062,2417,807,870,6977,6378,3531,1682,5766,5669,3304,4016,9719,8435,3905,5949,4225,4011,7502,10710,3202,6287,8062,3167,9106,10825,4347,662,10229,9128,830,3588,4805,6377,3717,8961,6385,7431,3041,9649,2475,315,6039,10168,278,7804,10153,817,8306,4922,6948,5288,1729,3086,2231,6504,9116,4765,4600,9054,705,9684,4131], "time_ms": 0.119826, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 613, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "generated_query_count", "value": 1, "time_ms": 1.086237, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 428, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "generated_samples_per_query", "value": 100, "time_ms": 1.086237, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 429, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "generated_query_duration", "value": 302886312, "time_ms": 1.086237, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 430, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "logger_swap_request_slots_retry_count", "value": 0, "time_ms": 3208246.260795, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 898, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "logger_swap_request_slots_retry_retry_count", "value": 0, "time_ms": 3208246.260795, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 900, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "logger_swap_request_slots_retry_reencounter_count", "value": 0, "time_ms": 3208246.260795, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 902, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "logger_start_reading_entries_retry_count", "value": 0, "time_ms": 3208246.260795, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 904, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "logger_tls_total_log_cas_fail_count", "value": 0, "time_ms": 3208246.260795, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 906, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "logger_tls_total_swap_buffers_slot_retry_count", "value": 0, "time_ms": 3208246.260795, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 908, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "power_begin", "value": "10-23-2024 01:07:01.375", "time_ms": 3208246.260875, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 564, "pid": 350126, "tid": 350126}} +:::MLLOG {"key": "power_end", "value": "10-23-2024 02:00:29.615", "time_ms": 3208246.260875, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 566, "pid": 350126, "tid": 350126}} diff --git a/open/scc123-Shanxi_University/results/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy/mlperf_log_summary.txt b/open/scc123-Shanxi_University/results/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy/mlperf_log_summary.txt new file mode 100644 index 0000000..c725ef9 --- /dev/null +++ b/open/scc123-Shanxi_University/results/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy/mlperf_log_summary.txt @@ -0,0 +1,4 @@ + +No warnings encountered during test. + +1 ERROR encountered. See detailed log. diff --git a/open/scc123-Shanxi_University/results/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/performance/run_1/mlperf_log_accuracy.json b/open/scc123-Shanxi_University/results/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/performance/run_1/mlperf_log_accuracy.json new file mode 100644 index 0000000..0d4f101 --- /dev/null +++ b/open/scc123-Shanxi_University/results/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/performance/run_1/mlperf_log_accuracy.json @@ -0,0 +1,2 @@ +[ +] diff --git a/open/scc123-Shanxi_University/results/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/performance/run_1/mlperf_log_detail.txt b/open/scc123-Shanxi_University/results/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/performance/run_1/mlperf_log_detail.txt new file mode 100644 index 0000000..7724e4a --- /dev/null +++ b/open/scc123-Shanxi_University/results/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/performance/run_1/mlperf_log_detail.txt @@ -0,0 +1,88 @@ +:::MLLOG {"key": "error_invalid_config", "value": "can't open file /tmp/pip-req-build-m_g6hv8p/mlperf.conf", "time_ms": 18446744073707.086823, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": true, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 592, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "loadgen_version", "value": "4.1 @ NA", "time_ms": 0.006442, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 53, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "loadgen_build_date_local", "value": "2024-10-23T01:02:32.884482", "time_ms": 0.006442, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 55, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "loadgen_build_date_utc", "value": "2024-10-23T01:02:32.884489", "time_ms": 0.006442, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 56, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "loadgen_git_commit_date", "value": "NA", "time_ms": 0.006442, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 57, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "loadgen_git_log_message", "value": "NA", "time_ms": 0.006442, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 58, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "loadgen_git_status_message", "value": "NA", "time_ms": 0.006442, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 60, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "loadgen_file_sha1", "value": {"/.clang-format":"012aad77e5206c89d50718c46c119d1f3cb056b2","/CMakeLists.txt":"b73434348f7860471606aaa395b570e81113cb6d","/MANIFEST.in":"8d3c4ac6c325b7b9a0fd4cf4a4108cbeff8d5025","/README.md":"20a55bb946c2c0bbb564ced2af1e48efd096b3a8","/README_BUILD.md":"5f6c6a784e9cd6995db47f9b9f70b1769909c9d8","/README_FAQ.md":"01f9ae9887f50bc030dc6107e740f40c43ca388f","/VERSION.txt":"cb67dcc41adcbb7849a0a808a501ee9ccd951d92","/__init__.py":"da39a3ee5e6b4b0d3255bfef95601890afd80709","/bindings/c_api.cc":"32181da9e161c285f8fe46ddaa49e6cba2f9f918","/bindings/c_api.h":"91f58bd79b83b278f3240174a9af747fc38aff74","/bindings/python_api.cc":"9f538d2a5390c77ae0bc3f8a351bcdb2587bc66c","/diagram_network_submission.png":"53dba8ad4272190ceb6335c12fd25e53dc02a8cb","/diagram_submission.png":"84c2f79309b237cef652aef6a187ba8e875a3952","/early_stopping.cc":"0cd7b546a389deac73f7955cd39255ed76557d62","/early_stopping.h":"158fcae6a5f47e82150d6416fa1f7bcef37e77fe","/issue_query_controller.cc":"126e952d00f4ea9efd12405fb209aa3ed585e4b2","/issue_query_controller.h":"923d9d5cdf598e3ec33d7a1110a31f7e11527ec7","/loadgen.cc":"6650091ba7a918f343b06eb7a5aa540eae87275f","/loadgen.h":"e00fdc6dbc85a8c9a8485dbcbfe2944f81251c4e","/loadgen_integration_diagram.svg":"47f748307536f80cfc606947b440dd732afc2637","/logging.cc":"197efc96d178e5d33a750d07fa7b2966417506ea","/logging.h":"ddb961df7bcc145bcd7cce8c21f7cf075350dcbe","/mlperf.conf":"0a4daef277bb3151139980e484dd5e644bf36e18","/pyproject.toml":"712fab87b72ba67ef2a068d0f9f47da65130342f","/query_dispatch_library.h":"13ad6d842200cb161d6927eb74a3fafd79c46c75","/query_sample.h":"e9187c8612bbdc972305b789feb6e15c26e96cfe","/query_sample_library.h":"8323a2225be1dff31f08ecc86b76eb3de06568bc","/requirements.txt":"a5ff7e77caa6e9e22ada90f0de0c865c987bf167","/results.cc":"34e2d2a44324cb07c884f92146ecbb8ef9d704e2","/results.h":"fce22d5a588d91fd968a6b25c27896dba87bc276","/setup.py":"a722046e05858c6d9f38f0e2b3fe425334beef28","/system_under_test.h":"18d4809589dae33317d88d9beeb5491a6e1ccdec","/test_settings.h":"476ecd4032f3bafe6f201df25d68aca4e177f659","/test_settings_internal.cc":"ce4322c849d24ffafc28a37b5e528a4cb4df227d","/test_settings_internal.h":"f1d5335b53ca610c30e0edc5d07999a27b5b4b9a","/utils.cc":"3df8fdabf6eaea4697cf25d1dcb89cae88e36efd","/utils.h":"40775e32d619ea6356826ae5ea4174c7911f6894","/version.cc":"cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0","/version.h":"9d574baa64424e9c708fcfedd3dbb0b518a65fcc","/version_generator.py":"eea9b9cb1a06cd1abe1bbdaee82f9af31527fedb"}, "time_ms": 0.006442, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 67, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "test_datetime", "value": "2024-10-23T01:06:19Z", "time_ms": 0.006743, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1198, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "sut_name", "value": "PySUT", "time_ms": 0.006743, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1199, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "get_sut_name_duration_ns", "value": 841, "time_ms": 0.006743, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1200, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "qsl_name", "value": "PyQSL", "time_ms": 0.006743, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1201, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "qsl_reported_total_count", "value": 10833, "time_ms": 0.006743, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1202, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "qsl_reported_performance_count", "value": 10833, "time_ms": 0.006743, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1203, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "requested_scenario", "value": "Offline", "time_ms": 0.016090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 271, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "requested_test_mode", "value": "PerformanceOnly", "time_ms": 0.016090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 272, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "requested_offline_expected_qps", "value": 3.0325, "time_ms": 0.016090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 309, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "requested_min_duration_ms", "value": 10000, "time_ms": 0.016090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 315, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "requested_max_duration_ms", "value": 0, "time_ms": 0.016090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 316, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "requested_min_query_count", "value": 100, "time_ms": 0.016090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 317, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "requested_max_query_count", "value": 0, "time_ms": 0.016090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 318, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "requested_qsl_rng_seed", "value": 0, "time_ms": 0.016090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 319, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "requested_sample_index_rng_seed", "value": 0, "time_ms": 0.016090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 320, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "requested_schedule_rng_seed", "value": 0, "time_ms": 0.016090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 322, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "requested_accuracy_log_rng_seed", "value": 0, "time_ms": 0.016090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 323, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "requested_accuracy_log_probability", "value": 0, "time_ms": 0.016090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 325, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "requested_accuracy_log_sampling_target", "value": 0, "time_ms": 0.016090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 327, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "requested_print_timestamps", "value": false, "time_ms": 0.016090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 329, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "requested_performance_issue_unique", "value": false, "time_ms": 0.016090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 330, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "requested_performance_issue_same", "value": false, "time_ms": 0.016090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 332, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "requested_performance_issue_same_index", "value": 0, "time_ms": 0.016090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 334, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "requested_performance_sample_count_override", "value": 0, "time_ms": 0.016090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 336, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "requested_sample_concatenate_permutation", "value": false, "time_ms": 0.016090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 338, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "effective_scenario", "value": "Offline", "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 417, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "effective_test_mode", "value": "PerformanceOnly", "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 418, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "effective_samples_per_query", "value": 100, "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 420, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "effective_target_qps", "value": 3.0325, "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 421, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "effective_target_latency_ns", "value": 0, "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 422, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "effective_target_latency_percentile", "value": 0.99, "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 423, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "effective_max_async_queries", "value": 1, "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 425, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "effective_target_duration_ms", "value": 0, "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 426, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "effective_min_duration_ms", "value": 10000, "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 428, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "effective_max_duration_ms", "value": 0, "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 429, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "effective_min_query_count", "value": 1, "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 430, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "effective_max_query_count", "value": 0, "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 431, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "effective_min_sample_count", "value": 100, "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 432, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "effective_qsl_rng_seed", "value": 0, "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 433, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "effective_sample_index_rng_seed", "value": 0, "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 434, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "effective_schedule_rng_seed", "value": 0, "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 436, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "effective_accuracy_log_rng_seed", "value": 0, "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 437, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "effective_accuracy_log_probability", "value": 0, "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 439, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "effective_accuracy_log_sampling_target", "value": 0, "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 441, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "effective_print_timestamps", "value": false, "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 443, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "effective_performance_issue_unique", "value": false, "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 444, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "effective_performance_issue_same", "value": false, "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 446, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "effective_performance_issue_same_index", "value": 0, "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 448, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "effective_performance_sample_count", "value": 10833, "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 450, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "effective_sample_concatenate_permutation", "value": false, "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 452, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "generic_message", "value": "Starting performance mode", "time_ms": 0.018855, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 841, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "loaded_qsl_set", "value": [9772,8533,6521,10490,3029,2324,6045,6496,9363,6154,1030,2969,8111,5802,10018,1133,2097,3409,4680,39,4868,7780,8289,7294,4631,10540,9874,2476,3146,7551,8190,6721,4132,122,2590,868,4256,7881,10412,4438,6772,3786,88,5404,2200,10209,10100,2370,7769,8943,1991,7158,4606,9675,9814,6951,7100,4866,4199,794,3376,1373,8595,10723,9612,1229,1047,7106,3600,6805,2823,7677,7773,2040,8798,5281,4612,6254,6768,5391,7564,2751,2285,3174,4657,8930,3339,352,10433,7908,7546,5175,7175,9475,1351,5871,7253,2279,2029,3023,5714,8064,4178,7408,6999,46,1829,2902,8456,5123,7042,1709,2620,10561,3280,8594,3419,855,8675,4283,3558,1719,1768,9569,5776,6241,9306,9490,5169,5214,4324,4051,1394,1451,2563,7082,1921,1620,2961,4590,2349,6556,3707,9133,8308,6036,10686,1014,1527,10431,517,3672,4839,6014,916,9164,9447,6346,19,3723,1609,3563,10701,10546,2432,3349,5694,9823,9875,10459,8792,5015,8832,1518,9941,9176,10092,293,8233,448,6818,5971,7574,1245,8134,6534,7670,2829,3689,4059,6250,9368,1813,1912,3852,5277,8573,8628,10425,7577,3962,5483,8039,8862,7213,7044,6050,4344,8142,3293,7035,1435,3291,8638,3611,59,5513,6339,6510,7951,6719,1834,10460,8886,6799,8275,2809,1616,6987,6549,4642,3858,1446,2743,9044,896,5305,6406,1091,1321,2813,5852,9851,2287,1843,234,10348,8771,6940,283,2798,4459,4654,5983,3924,5677,476,6291,9809,8713,1590,583,6398,8084,6239,5040,2804,9190,3347,9780,7128,6955,4871,469,6031,9853,3251,6931,9540,4635,7692,5237,2878,1907,10725,6523,35,9723,9625,9857,840,9615,9098,4298,8790,7522,6699,1516,2545,6422,5630,912,9571,2542,3577,809,9710,8257,2636,9297,6488,7470,10122,8672,6895,10630,10567,6756,3649,2762,7599,7820,3940,5696,2258,2119,6834,7705,7598,390,2667,6701,7489,869,2406,3692,4495,5472,6362,5651,9687,9550,5911,8110,3342,8258,4352,3058,4000,9221,6190,412,3651,5982,4776,1114,9010,307,10678,1497,506,3369,9360,267,4310,5008,6421,8879,1089,8419,5809,6215,4796,3173,4566,29,2080,9032,2894,7772,7530,2581,7788,5112,6602,2284,7144,2935,4235,203,7443,8547,523,6952,5896,3810,843,200,413,1227,5279,4208,2500,8185,2114,10240,8754,10718,6850,2470,4650,10794,903,5554,8,1721,3066,473,5685,7567,1661,3635,9249,5415,4949,7283,1389,9036,9623,9919,6638,3908,9866,9869,5323,4224,6866,5722,6583,4622,7870,3346,815,1180,8837,2726,1877,10422,4176,354,7665,5363,10539,8652,5661,2539,236,3356,8836,6132,8038,64,304,5414,6590,3976,6808,7741,8001,649,3979,1296,3398,1234,4280,3283,4912,1306,9121,1554,1544,4629,5088,2411,1211,10132,9027,4990,1324,2587,252,3141,3358,4557,2594,1993,6852,5380,5614,572,8091,7992,6666,4100,6924,8376,6316,7358,1454,4120,7480,731,6575,3093,873,9599,8145,8658,4227,9122,5189,7625,2478,8988,5127,5600,7097,9641,4149,5421,5586,6113,4113,4944,3411,4842,3818,8928,2910,1961,9392,1983,10204,6544,4295,6146,1856,7909,10071,6564,10760,6541,4643,6102,5976,1214,7482,1119,7973,47,10637,2930,8386,2068,9638,3747,2295,2079,6753,1869,1315,2861,5433,4552,7540,5218,9132,7376,5675,2429,4889,3322,3406,684,1051,6578,1469,585,7474,6332,6354,1228,9698,9667,4816,7869,5017,3265,8823,163,8997,8826,10693,10184,1206,22,3425,8830,10068,10473,3842,2525,7435,445,3422,6408,4214,3805,6659,1864,3380,2224,5262,2817,8872,5504,7080,7517,6367,1475,3780,6641,5678,627,10538,3482,9708,9174,10329,3111,4006,8236,6967,768,660,5133,6118,8939,10318,4273,8469,6925,2920,2275,9367,626,2106,6696,6165,10476,2502,5203,5876,509,5615,6535,935,1403,7210,5,1461,5979,2956,7172,8269,772,2822,1379,4010,4812,8177,8344,490,9108,8453,9115,10332,7948,2425,6009,7403,7914,9950,3092,5332,320,8659,4114,2862,7139,5374,8983,6450,1082,1827,3617,1866,4164,1057,1916,7689,7652,3305,4683,3499,2316,7326,8330,5139,7295,6044,1766,4865,6565,4641,6777,10557,202,6085,1757,9090,2012,9446,6500,2197,4062,10215,690,9712,7297,7166,10258,6771,2927,5796,270,5626,6969,6372,10453,6061,5977,1450,414,10523,3984,808,6905,6376,3732,5882,4579,5679,5206,2184,4308,8131,4902,9149,3564,8036,9793,8089,2815,4182,7900,3438,3147,7539,5718,3165,2327,759,2101,10556,1411,9596,9327,7768,5181,10799,6207,7206,3296,6875,4287,172,4289,2662,6841,1056,7711,1973,1920,920,3715,9506,10505,7614,5617,5665,1326,10578,1832,1644,8164,5032,2945,10743,954,1671,567,10255,531,4112,4861,4926,7385,6520,2810,9488,2126,4315,10173,4179,6347,6773,3338,9219,3448,4961,641,6657,5200,3949,2461,4219,1746,7547,908,1938,4248,7561,9420,699,3761,9668,2711,10470,2117,1603,4288,1312,5794,5319,10088,6418,8499,8394,260,7427,5581,5344,7240,8080,5290,289,2707,5245,10527,4314,4710,3500,1533,10804,6794,4968,6256,7525,6946,9759,3516,4708,656,5985,1233,8779,6008,9326,4521,7532,4035,4044,4491,4728,2393,7709,447,5893,6357,3965,6494,3989,4917,6900,9315,10219,2024,5576,1017,9966,6484,2044,5358,1423,8400,8976,4443,9272,4428,6150,9424,2094,10498,3507,4087,7763,3249,2858,9715,3741,7702,9918,8734,8217,4660,3860,7892,8986,1123,9396,1085,9449,3444,4107,9783,10369,3145,9366,9305,3922,4077,4646,1659,7098,10415,3037,3254,9613,7970,6944,3518,9982,1004,718,9863,3970,2423,9640,8843,10386,4217,5764,10017,4027,10741,2167,33,9746,2482,4726,4980,2850,8178,7661,7800,2840,3334,8775,10321,8727,5442,2618,8686,10236,1018,4434,2687,689,6591,4565,5762,3370,729,826,7626,563,8721,4688,2916,7604,7877,7750,7014,6176,2788,3053,7440,8958,2573,7141,2774,7659,89,10382,8674,2486,8128,10832,9270,1820,10499,2570,9251,5905,6680,1883,3511,9237,8766,1022,5355,347,10336,4327,6463,9629,2447,250,6843,8942,7529,4605,2917,10313,7375,9893,4364,10420,2929,2681,937,783,3840,10246,4724,4292,10270,4209,8346,4674,7001,4786,4304,3871,7148,7941,5291,7138,5628,7052,8810,7493,9467,10658,8728,1273,2140,5190,7180,5968,2925,8925,6763,8946,7273,5738,7126,8556,6663,3047,488,5992,10744,8303,4197,3766,10590,2355,4656,10801,8148,9493,9273,8230,3025,751,7439,3912,8281,2454,2081,104,10784,2194,3319,3229,2897,9694,7726,3294,3178,2642,7721,859,1975,9183,5833,1488,6893,5844,6397,113,4675,5026,6870,3675,5975,7198,5251,5568,6292,7142,9697,9688,5115,6903,5258,10147,8915,7724,4457,9369,6461,4743,3658,2107,1923,5588,120,8115,5909,771,6266,343,7559,7965,8247,5860,8301,6049,6897,4285,7980,7759,7977,10389,4897,6180,9184,3150,9025,9925,8413,779,1108,429,6865,3239,9068,9040,4616,7085,6547,3399,8065,8296,10111,189,2452,8657,9202,10126,1410,2867,1387,2265,2263,1708,10535,501,480,7466,2908,3437,3572,2272,1546,10661,9196,3067,4705,7753,6780,258,6454,3910,3078,8254,9718,6469,8566,6363,9656,5648,8521,14,5765,7298,9928,7378,2015,7081,1656,5211,9841,667,5768,7638,6606,6731,674,9989,6229,4537,6411,4899,642,8597,3961,2057,5777,2651,1663,9296,6068,6974,1467,7353,3027,10060,5668,1860,10005,2299,9568,7850,4342,3710,9026,7996,4397,8944,9597,3182,7984,5037,3212,3763,8069,3209,8221,6524,5142,3004,9792,5196,5219,5990,3817,8314,8905,5328,5476,4216,10434,336,9351,6149,5920,5999,2919,4826,9193,3125,5946,2800,10774,6456,3153,3998,1397,8498,3100,1886,10040,10761,2319,3663,8914,3952,5956,1599,1194,1276,3049,4126,876,8612,8011,7524,1919,8881,8271,8964,4486,8278,5834,10548,898,7498,1699,4785,1635,7308,7943,721,4684,8285,9997,983,7031,2970,9808,5216,6202,2158,2241,3389,5064,9525,9560,8408,5100,10795,1307,3045,6847,654,4677,2067,10486,2255,6414,586,3361,1617,7362,2414,2912,2499,8500,8318,411,5741,6592,9791,3746,6519,1308,7949,5934,8724,8828,8277,1628,2820,2204,7673,138,3391,10247,9811,8738,219,9962,5247,7765,4815,3578,537,4662,4294,1624,462,4795,6996,10568,4137,2934,10685,6209,7477,5302,2831,7739,7017,8352,7723,176,9739,3082,7794,4621,9706,6487,5647,513,10771,10334,3227,6471,9260,8382,888,1486,2682,6938,2139,3636,9284,3522,4518,9552,3787,6789,7357,3638,9317,8932,8648,3773,1605,10220,6879,10354,4529,10797,5514,9765,1358,4300,8531,3550,7840,6656,2553,5798,3022,3660,5507,10353,8617,5745,5028,6337,7267,5459,5122,10772,2940,4798,2185,3596,5980,2036,2740,5458,3574,5197,1433,2567,2616,10069,3136,9522,3363,5074,8240,9979,1193,934,9728,4357,9331,6296,2949,8431,6230,8629,993,8423,10401,1117,10405,7879,1910,9102,7770,8856,867,7066,8970,8700,7133,1453,4395,4762,6498,7377,915,3561,3126,9600,1352,5244,7219,5807,1350,2052,2792,2472,5590,10323,287,4396,767,2489,2112,6615,8644,862,7039,6816,2369,2403,9047,9061,3377,3567,6613,10341,3430,1612,6340,9996,4589,2463,482,1735,947,10737,2603,9470,7550,2931,6888,3116,6730,2728,6650,5313,10206,6295,1356,9614,3060,6162,7197,10762,10253,1430,6729,4042,4499,3951,10560,8990,2674,4046,10698,6723,9888,5767,4038,5886,457,4803,9269,4784,5153,3005,10558,2689,6953,146,6617,10624,6891,5688,978,8146,7526,1121,10497,10803,8780,7462,6089,4447,8725,7901,4599,3982,7748,8774,7422,761,1732,5294,1844,4975,10263,10445,7447,7608,5910,7501,999,1522,9598,4128,8389,1335,3056,9134,8542,582,10747,6801,7235,6218,1523,1270,152,5940,10121,9873,3628,1763,5637,4299,10652,1760,7036,1437,3947,9099,6373,1459,700,5311,3288,5915,1816,3694,10299,4444,5300,10338,6742,9576,3223,3351,10193,9608,6186,3328,8782,2450,1,4737,5702,5577,3802,5159,7216,3055,9659,6851,9330,1365,3397,5591,3190,1175,367,4450,1188,1292,8005,6860,4769,6416,1789,4951,10629,3849,10609,9299,988,5849,6012,7771,9013,7821,3735,9543,5827,213,4319,8662,4802,3579,1897,3420,6878,2087,10726,6204,2219,5138,2142,2485,2166,9169,3161,7655,9547,9986,1445,1316,10123,1665,5149,7154,7258,1274,8995,2971,2492,7059,8846,5780,2175,927,2703,7756,4681,8156,6446,3899,5929,7275,1162,9622,2162,2077,6136,7183,5797,4701,2572,4394,9628,4551,9161,3527,8155,1540,8697,8245,9821,6570,4884,1862,212,8345,7662,4049,8515,839,4474,10048,3458,1537,1263,9631,6636,5166,1808,4891,4135,8441,1320,4835,330,4147,2021,4850,5450,4526,9346,8858,7986,10146,1419,6959,6675,1853,8764,4804,2759,5510,5589,4163,9334,7194,6981,1960,10770,3847,5887,1170,758,9294,158,3166,7125,4118,1163,7350,5572,6412,9192,9947,4005,7779,2673,5286,5539,6341,6444,4793,10596,2505,10039,3442,9374,4943,7860,2982,9860,6063,9072,8567,10075,3757,2519,339,10438,6257,10358,86,7263,1618,638,6252,3937,7620,4148,5686,9544,4433,9670,2890,10536,9528,3387,8809,9654,664,6,8656,1362,2580,9642,2023,5761,3043,334,4780,10500,7643,3040,9595,646,8829,6161,8732,402,6760,6889,369,10576,785,8670,5165,1060,4645,9178,3722,584,2235,2952,2237,4945,2210,6024,2678,3379,9100,3307,9834,6279,863,7181,10057,2942,8992,2174,4729,6509,7995,7110,5318,6058,10638,7002,9320,6688,4101,5419,8361,640,2733,3915,3245,4690,1870,2436,7887,7520,6478,15,1280,6402,8422,3571,4086,565,5960,5548,364,5420,4960,1613,7940,5298,4547,610,3688,6214,3584,1468,5595,1271,6210,10690,8368,5884,10164,421,9322,2270,1201,9440,143,7657,2343,2160,6746,4678,10606,1805,834,692,6283,1925,2135,7595,8839,3252,9635,1607,3457,5241,4014,417,3613,829,10082,3081,9832,9802,68,8205,3901,2303,9766,1962,8882,4429,10322,3210,2269,5021,4937,2281,6819,7000,906,3240,9236,6106,7393,1780,7287,5340,790,8897,7248,1452,10607,6658,84,3235,3892,8968,6197,7083,2103,9371,5526,7250,3333,8412,1987,9483,1535,854,2339,8892,7454,1223,9403,2009,3439,8611,1852,2357,9430,5959,2041,9048,173,533,6507,5753,6054,3191,8874,8936,7710,8033,6923,6183,4522,9984,8367,789,10666,835,10304,10003,1541,8154,10768,5412,3098,1689,9451,2561,5873,10691,8322,2614,4536,7637,9398,6392,2018,3733,10554,8043,9181,4848,7633,7654,9050,9433,669,4374,9976,1284,2550,3170,3929,9922,3148,1594,4846,1786,1185,10575,9586,7313,8996,1675,8609,10446,4630,2913,2779,7040,8917,2967,3475,9497,1655,1496,7712,6909,661,10241,6690,9545,5621,5385,8166,773,114,57,8448,2626,10273,5670,10356,1491,7946,2932,763,9136,10276,63,3134,10689,6915,8272,2877,5295,5951,3587,5367,9243,2073,1442,7591,8553,171,141,547,10410,2715,6840,6110,9092,5730,9227,2190,5698,478,10733,8845,593,8758,7390,387,4685,9564,3528,6727,8522,6134,4942,10166,5759,1399,24,2671,6960,7305,1490,6144,4959,2156,540,3362,9052,1064,3326,5580,9634,9604,10242,6393,6262,10754,4472,9681,4679,7325,2892,5889,3831,3028,8336,149,428,1935,5045,2458,4585,3931,9112,3872,6714,8437,10053,8770,2812,7916,9589,4760,116,800,4844,7543,5719,1484,1043,7266,1125,1135,812,10233,10355,9173,8077,8169,1063,4037,2634,1550,9143,2193,2864,7911,2797,9379,2062,5640,8849,1642,3394,10034,6233,940,10696,3413,3997,10618,6359,5517,6404,7389,890,2449,8430,9045,621,5521,456,1924,4370,7264,9627,3395,2426,904,5238,2116,2195,3841,8615,5024,2503,7861,3721,7535,9034,3357,512,7200,346,8249,7645,7727,6388,229,6026,3038,882,688,8206,8100,1505,9850,4924,7134,1863,7808,5117,3539,8003,4307,9355,10156,1106,2341,7170,9920,10705,9462,7630,542,9616,7627,3524,8016,6448,2479,2522,2153,8355,1498,7242,9685,2336,2381,2755,1034,4538,9226,1948,10530,7534,452,3143,8327,1239,6836,3754,7568,893,5352,6439,10408,1370,8921,8847,8947,6109,4948,6526,9778,9521,245,3848,1068,6628,8622,9441,9960,1507,9789,1504,6447,770,264,7500,6430,3972,4920,3486,6761,9940,7202,2320,6051,1398,755,1875,7398,7335,1028,7064,875,1360,6290,8687,3601,3003,9515,2951,1814,10819,318,8800,2127,6679,10319,7758,10094,5047,10203,7256,2569,8993,2074,1470,1087,7007,7104,6030,9065,874,3436,1803,4427,9779,2212,3823,8338,6338,1039,1885,1474,3942,7288,1055,4916,1070,9283,7370,5089,4406,625,4887,6443,4442,2398,6236,9041,4936,8508,8711,623,415,1281,9388,6203,395,9415,3517,6817,9987,6264,8794,5280,6116,79,6522,9479,9970,9532,8884,7445,4108,2530,32,6660,2893,6244,5164,2847,9975,5452,877,921,9278,5894,5529,8600,5879,5778,7115,728,4535,7473,4165,8432,10074,4085,6103,600,6892,6306,750,1627,6838,3164,8949,10394,130,1049,7124,239,8279,880,4446,4050,9514,2985,7204,1833,3090,2028,8464,3748,6320,7600,9582,5501,5735,580,4821,10807,9953,10311,2619,3021,392,5292,8545,10351,10526,9380,4822,9501,6854,308,8316,10706,5343,7412,3720,458,3657,9285,9030,10227,6585,2998,9859,9643,5046,5991,5839,5952,9325,1957,8264,5464,5908,9217,10581,3703,6921,2124,8085,2772,322,3057,5351,5706,6270,1207,10213,1587,4389,9332,10429,10600,9358,2253,10194,842,8420,1785,3337,7923,2054,3729,9645,745,3450,9665,4505,8516,1749,1407,7099,2002,313,10049,6253,9992,886,8535,475,150,8071,3545,5327,10545,7094,4271,9402,3589,4572,6907,5806,9407,9535,6028,3948,1381,8736,4947,4099,3392,9024,8000,5030,7278,8455,3771,5130,9316,10756,6932,4783,6837,5426,998,8569,2468,8584,5422,2179,6647,9658,3159,7783,961,2291,7363,1118,9459,3104,4258,4913,10252,1992,8510,9696,10029,3568,3916,3503,5866,6882,5891,8778,4206,6224,4602,3064,21,5540,3552,7650,2182,960,10514,5271,2264,5339,6548,4076,8188,4158,1366,4856,4880,5751,1591,8483,9000,10033,3865,8741,6845,5111,9461,5727,1374,3737,464,10196,9546,6205,6811,4493,4620,4981,3192,7087,1698,4640,3309,7184,10027,1415,10286,5310,6639,7931,7268,7985,7827,5357,7226,4451,3198,3272,907,4400,7874,2693,8270,1129,224,5182,5937,2456,5786,5376,6858,10031,8347,6674,2909,719,7610,9707,2280,2821,4570,454,1109,6728,2641,2083,3827,6375,527,2805,7464,2109,6703,4687,3888,4533,1693,5129,1584,1666,9938,2816,8219,4807,10104,9876,8987,5708,5002,5492,4335,1981,1218,3838,9858,4262,7663,4220,4977,2987,8168,2070,4652,7176,6441,1847,9095,548,6025,7993,4774,825,5303,2234,2723,1519,7116,9505,8903,5733,2085,10714,553,9188,10337,7713,1202,9666,622,1110,6913,6098,6919,3606,2527,4988,1575,8512,2938,9824,943,5447,8390,502,6559,208,6003,3171,10380,6573,4293,4607,725,5396,4133,7487,3207,2827,9390,8397,2225,3180,9917,9154,3381,8787,5649,3900,8602,3382,550,9155,5243,10205,5544,7716,1898,3418,9973,2980,5518,5537,2688,2846,8070,7531,5541,7074,609,6245,10397,2928,8436,3791,9216,5878,10398,3843,8742,9049,1822,997,2896,5144,5627,3401,6631,10331,3026,4967,6424,6561,5984,2261,4169,5254,8682,953,2099,2718,2576,1874,3065,7816,3884,9387,5703,6979,43,8335,6081,7102,34,3036,4970,227,7938,3755,3128,1294,9349,9775,8297,8364,10298,1041,878,6151,9329,4073,2421,2747,1650,5944,6702,1066,8099,3895,10442,10675,2512,970,3557,4935,7790,1673,9849,2617,9676,8223,5555,4702,10692,3911,10284,2102,4360,8120,6237,2230,2593,179,3177,4358,3656,4554,3386,10452,6607,1289,8495,1383,10679,10582,1553,5596,2708,9983,6126,2891,6618,3551,3168,1492,5407,4569,635,6862,4831,18,8222,5819,2141,6911,2072,9295,3262,2731,588,10769,3,8465,4725,3417,4813,4686,10035,962,1200,7636,9137,8029,9165,3472,989,7333,3824,6558,2323,3921,230,2297,2007,5099,7516,1664,3724,2962,6153,746,10537,2069,957,2996,9258,6710,5471,564,5390,10157,9460,6540,3445,1300,7292,7307,6208,4093,3073,2390,4067,4455,7483,581,8384,2146,4781,3782,3597,4928,1016,5620,8492,5020,10529,2278,1195,5270,4763,4963,4202,6966,9886,6401,4978,8470,10207,4779,2696,4047,1959,9798,8581,6645,440,8137,4008,6100,7420,952,6812,4066,9977,9796,4356,9729,8577,2239,205,9482,3907,10093,9807,5273,5662,9774,5186,8951,1176,1120,3554,1911,5409,7196,5709,7700,9086,5284,5158,6318,2283,7271,9726,1561,4306,5566,597,4402,4274,2866,9639,1906,6700,4746,5136,7706,7619,4697,722,2544,7050,7902,8101,4941,7191,5408,397,8324,6689,6942,10472,5584,788,4110,8252,685,9904,7648,8339,7959,9007,1277,6982,5646,4823,10571,4695,3740,9318,6410,8726,8960,6662,7917,6380,1257,7304,6019,9022,9418,3856,7073,2474,4377,6901,8463,10297,5338,9130,7159,5868,174,2448,9592,6718,9738,9437,1077,7846,3592,1930,8149,5793,3208,10786,10274,8634,6796,7694,5592,8808,5736,8745,10580,2473,2288,6781,6612,3006,673,3719,827,5924,981,3109,691,1754,3800,7061,10595,6365,5239,2796,6958,3502,7011,8259,1580,198,4722,7255,6079,7437,8717,7449,7451,1010,2014,1147,624,1879,9914,170,8196,3196,7528,8799,1417,3625,10468,2879,1393,9651,1647,1660,2960,6099,3365,3520,3071,10327,3833,9445,9319,9254,51,1458,7646,1244,4040,8877,6929,426,4992,9565,8117,7672,8525,521,5466,10131,9566,9937,7008,7932,1702,7734,7324,437,9503,10403,6289,902,756,5575,3941,8704,8094,2039,2159,1586,2950,4610,1704,10805,2168,7701,3650,124,5299,9601,6930,4417,6480,4562,7471,1104,8409,10717,8479,4691,1793,9624,4711,706,8963,9828,4097,1142,10814,1045,9699,1971,9946,8709,704,6743,8411,5226,10260,5029,10289,6962,3745,1918,9059,9749,10,300,4480,8050,5900,10544,10522,2240,3321,338,2601,5406,9630,7578,9529,9945,1995,2645,4430,1438,3562,7843,7381,3946,9465,9909,2769,6499,2497,2679,5826,1471,500,5209,7301,7704,6553,4017,9170,1639,9210,8585,5162,3329,3614,3042,4312,5622,3408,7742,3555,2828,6027,5329,1237,297,3009,2720,1220,7023,9232,10474,5783,3714,1654,10301,3469,5996,8216,2151,5869,2244,6037,310,720,10749,2625,7433,4170,5230,3709,8383,1332,7718,4166,98,6991,491,8643,9419,10593,10155,3995,560,1065,4563,2874,6336,5227,10444,4103,9894,3708,3414,9225,5192,2700,8004,7806,4553,529,5542,5467,2342,2806,10054,2546,10112,2773,7369,1432,3114,1258,169,3938,1868,10731,10171,6778,2509,10462,1359,9507,2555,1510,243,2130,6258,6043,782,1859,275,6927,5585,2524,1578,6502,8647,2859,388,2872,45,1954,3885,6984,3306,5090,10632,10402,959,1262,2514,6790,25,6130,9166,2551,81,10790,7922,8489,4269,2597,2030,9033,6732,3896,309,546,3069,3119,9344,4192,8690,5253,3999,1880,5050,8471,2488,6390,5202,6643,6148,7563,3775,8817,10055,8012,1581,8906,9242,8562,5389,5545,1309,3801,36,7323,10467,6271,10479,9123,4946,9157,2592,7343,9825,8560,6331,8047,6358,1982,5096,1124,8841,4787,3749,3960,6627,9767,5369,1871,5828,9674,2321,4333,4556,3077,4827,1298,9342,7069,6733,3985,3127,7791,10649,499,7309,8982,9927,5228,6366,10366,8890,1684,1240,8646,1329,10267,5011,3920,9820,5530,5721,401,7906,7411,10165,5731,7346,5392,3313,5361,6774,9427,5933,6588,1338,1448,5875,10703,9880,2609,8405,1478,4818,2359,9553,3532,10597,10216,10758,7719,9908,6873,7384,6417,7276,5635,2076,929,4908,443,6445,7987,4855,619,3753,1677,4080,1226,2944,10264,2090,8176,5097,8984,1136,5564,4481,787,187,4470,7629,7419,1037,4940,8055,1736,8184,396,1167,1112,847,4203,754,6775,4339,2464,8624,3712,6697,5752,7936,8211,4524,10809,5431,3481,6975,2290,7793,5444,1219,9195,2394,2624,3366,7751,3154,6048,3068,7934,2558,5534,4852,5634,479,8374,9151,5511,373,178,3044,7461,48,10038,337,552,9912,5441,4089,6804,3813,6528,4757,743,5667,6034,6004,5207,2782,9678,2628,6042,3893,4160,4279,534,8861,7058,10822,5016,8859,3797,3454,3490,2134,4419,5174,16,1126,10708,3793,1640,7485,9619,8720,6869,2205,5955,7095,2534,1900,4454,1711,1937,148,4747,301,5025,5417,7185,9300,5468,3669,9267,463,94,5863,4997,9397,10186,1267,6221,6114,10534,3851,4134,8072,6920,3837,6086,193,1131,4841,5927,7606,6635,8689,7450,2692,6303,2173,7252,8371,7786,6369,5084,8551,10615,3504,8564,2144,4349,979,4840,9867,4439,8834,7544,1854,4778,4748,1168,8088,5102,4129,3188,8353,2947,3101,8937,6231,5520,5222,562,3142,5259,10265,2635,7057,10141,9516,2717,6108,7979,2557,4973,9846,3441,10052,8170,7368,2885,410,8458,1357,10393,985,7228,1282,4270,10796,5229,10390,6605,8360,5790,2790,2053,7336,1092,7826,8287,10780,5747,7884,7217,7968,7211,8034,5252,8941,1344,9458,613,10589,9644,9801,7208,2034,4531,1547,3623,7293,8299,4063,4476,3605,8693,5536,9077,1406,8294,2152,6698,3932,1347,9292,3832,10085,2754,6238,3396,5054,2169,3825,1850,1169,10824,5485,6481,10261,3103,2582,8763,4272,7400,8918,6833,3506,4401,7,5114,2335,7929,5482,255,8239,6935,3544,5337,10521,7168,1255,4670,4906,4627,1080,1772,6750,2289,6691,3443,1265,3471,8676,2330,8530,4265,5194,6665,4753,9073,6876,8733,764,1915,10305,7685,10110,1395,633,1216,2180,8927,6757,5418,10478,7777,1730,6842,5205,2177,10655,2266,484,3226,6185,1160,3317,9889,5922,3881,6174,10231,10424,2977,5423,7418,1657,10108,8139,8965,9551,9248,4918,1418,7807,3576,6531,5435,3548,9781,6265,2022,8838,10050,9452,5356,1701,9740,7991,573,7199,9652,3463,2965,8549,7904,9891,6672,911,4387,615,3519,4388,1020,4876,5373,2683,1053,7004,5326,4914,6278,3986,8309,9062,5903,603,290,2818,1143,6353,6415,8171,9804,1611,6193,2108,8444,1775,5236,5972,7565,539,2254,966,9399,5416,757,10096,5457,269,945,3977,10309,6824,8452,8582,4332,4385,9378,5475,693,4676,4363,8152,4368,3957,9345,10148,5921,1696,3560,7695,2181,5573,9158,10681,8480,1752,3792,1414,4408,5210,2387,2664,2632,5183,7215,8404,7823,6220,1248,9657,8248,9261,7912,1299,295,8114,2768,1796,8263,6302,8425,4423,1483,398,9381,1161,9364,3187,6235,3284,3645,489,384,9954,10620,5094,5710,8518,6985,10223,4911,6497,4863,2744,8505,2457,4255,4608,5249,6653,5034,7254,10300,2856,6155,204,4331,10061,4900,9902,9204,4525,1006,5693,494,5395,881,6593,7841,1633,438,6072,4317,4592,8702,9720,312,6069,3643,3123,1098,2042,4413,7156,2612,1774,2953,5645,6352,2250,5289,2410,1479,3341,3368,10174,10275,1137,10601,8438,7885,237,211,10572,9035,2386,10608,3435,6486,9089,8429,10676,4151,10550,8801,28,8911,4022,4379,7463,2249,665,865,9245,5784,53,3529,5865,7329,10145,1530,5481,6994,8664,7257,6539,7214,7245,3186,4756,712,4749,9489,3295,1290,4479,8496,10623,9961,5429,7669,2607,4898,8532,257,678,1416,10542,8293,645,7649,1380,6632,6716,2125,8759,2757,7342,5086,8143,2206,5168,936,577,6182,3702,2363,5757,5462,7798,444,4501,1179,10603,8046,2630,5605,8310,4366,383,9455,4405,10625,17,3758,3384,10224,180,2780,2521,2526,6614,6432,1855,3416,7491,2157,4409,7647,9671,1130,5167,986,356,3875,4143,6633,5574,6458,6947,6581,4002,4497,1217,4487,2123,652,4362,6420,7373,10520,9519,7107,8666,7933,1904,10487,5193,7229,6242,6749,10098,5557,3122,9423,6707,10492,9936,240,6158,3270,1190,845,10324,4226,8229,8439,1447,9593,6046,6830,9669,268,6135,7187,4469,2834,6275,1364,2435,860,7797,3175,2466,2051,4422,9877,7469,569,238,1210,4392,10269,7259,4860,591,6319,6886,10650,6828,594,3484,8442,2191,3691,2760,5056,8056,7413,8242,382,7542,698,2245,6626,2149,6986,9486,1600,4504,1762,9492,8478,4232,3367,7981,1963,5075,7549,5293,5787,3974,1141,551,4672,4893,2686,2799,3299,4252,1384,965,8068,5287,6178,5760,3452,5004,8710,74,1577,10413,8559,8543,10409,2302,9214,8696,6752,7795,10742,161,5506,1278,26,9831,8007,9443,3546,4244,7760,1285,10504,3433,1184,8103,3711,2465,9203,3867,6934,9308,10084,10450,10051,1003,2574,5257,3024,424,131,10333,9287,9577,5388,10707,9275,1187,9854,4465,4138,4558,2676,2214,9021,7147,10727,8373,8831,9235,2164,2787,6047,9856,9942,6568,4180,8769,10427,7818,3816,4742,8514,8803,6164,115,3275,6073,10588,5692,9935,7037,8025,10604,10481,6087,10564,951,1895,5316,6313,8663,7613,10103,2111,8723,6902,6438,8854,6644,922,5059,5453,4276,8883,6101,6767,636,3203,5740,7872,9727,10037,3269,7824,10280,9539,3743,8633,2163,4065,7761,4029,6370,7149,10362,8350,6572,6301,818,2652,4207,10102,7320,10419,7562,10411,3777,5942,9787,31,5754,6485,6160,1525,5220,2663,526,5597,5489,4399,2229,20,10463,9182,7745,977,10635,9484,6177,7399,6125,9011,6822,6371,4934,8127,8528,2675,1940,4257,8109,10006,8311,3508,607,10605,9252,1253,5954,7051,6724,9471,10387,6916,6017,4965,3987,2992,10187,5810,7731,8994,3933,3796,6015,2540,272,4441,8497,10200,9042,10150,10436,7864,2794,9636,1313,8575,2704,7966,5141,2086,7260,5519,10376,7327,2368,4895,37,5053,2803,1703,5449,4375,441,10192,805,1480,4523,8814,519,4583,8256,7999,1336,4032,837,4140,10249,2346,10221,9265,5309,7828,4489,9606,6311,3316,10668,3537,10440,4740,3903,353,2445,8365,2933,2131,474,1634,5556,8985,194,1246,7088,4482,724,1251,2775,345,8363,6364,9129,2338,4877,4172,7958,6483,2694,4983,191,7394,3030,5836,483,8851,9015,7703,3176,2600,9477,1876,5850,3850,10036,8563,5888,810,3460,5907,4738,10831,8341,4625,8627,4829,4508,8578,5820,10117,9988,8797,6993,5606,4034,1499,363,3679,3980,3902,7890,2786,1564,10664,7982,2019,10518,10169,5653,643,7223,6269,7049,7006,7862,6305,3219,262,8182,10293,1421,9881,9852,2350,9263,4048,5057,5583,3846,735,5154,8202,4421,1994,2310,8950,8833,3730,2972,453,10577,3260,3033,1189,4909,1667,61,969,778,9520,4601,687,6094,3894,6754,2348,7728,215,2639,7737,2256,1707,9302,10254,4594,6429,4577,1792,6844,6138,7075,8590,6196,4168,4954,2602,368,1939,10715,555,2649,5043,544,5938,9241,9862,2559,4881,4528,7785,3076,10023,6070,6104,9163,7541,1997,4069,5654,1733,1031,2605,1462,6040,2307,10516,8922,9559,589,4500,2706,3292,305,3259,3423,2660,9578,3752,6078,1846,9572,5135,5505,10826,8596,9883,8781,7246,3113,2467,10340,7523,4122,2207,6505,3169,4591,8534,1081,5916,7123,9494,2440,1849,524,919,427,8815,4250,9836,72,8603,5199,10392,2428,7754,1695,2227,10491,5261,7570,6329,1401,7925,10776,7225,5365,9591,5493,872,4095,8093,4068,9080,8785,6455,4800,4598,8956,1322,8714,804,6712,9224,3015,10211,7838,5770,8743,1668,2666,5611,7459,8387,6616,7687,4766,2439,9377,341,4361,8402,5049,7112,4549,8337,9998,9510,7975,8163,6532,7379,10143,77,3429,1646,242,7068,900,4998,160,9124,9066,1632,8158,6530,9972,142,4665,2943,5898,2228,2749,7432,10262,10634,6288,391,6648,3489,774,9526,3201,7602,2883,6739,5800,8821,2870,6928,6765,1596,838,4888,9053,10028,4892,3062,5609,4593,2830,1592,492,892,5067,6649,1264,2396,3898,3096,192,4070,9517,7436,8895,1685,9215,393,8900,5841,3061,10259,3462,7676,1236,4764,5737,10457,6065,0,71,6355,10827,5455,1266,8113,849,9890,1765,2758,9168,6791,4739,6598,9055,7653,3859,9150,8102,8060,8625,3091,5269,5242,6201,1555,1734,5775,10119,1839,6543,2742,9730,6571,7707,6187,8002,3473,1933,1392,3287,4758,1536,3364,4210,2586,891,8198,9847,634,2444,10673,3967,1678,5113,2697,5083,8973,10383,1341,6668,10130,3492,10374,7928,1146,995,5883,8124,10320,1783,6170,9186,2725,9702,1385,973,1831,10823,6579,4473,879,8822,3594,6957,2451,8661,3751,4230,2684,10113,4453,4989,8795,7556,1737,10021,9985,3468,9990,6748,5803,9307,6963,8434,1970,4041,10015,6956,1602,6175,2533,3140,5563,9187,9113,1531,371,5304,7945,1250,9777,7372,10587,10641,1388,1197,7621,1371,2887,6490,7698,958,1683,2886,6075,82,8678,3263,10317,7557,2300,7605,10183,2990,1779,1809,7699,6577,6140,9878,10674,9788,5424,5001,4896,4305,3102,7077,9266,5986,10767,6642,58,1204,9964,528,8418,4159,7032,4883,2050,5707,5498,8066,10722,2712,8262,9131,3407,1769,3975,4636,3677,10627,2189,167,884,10381,407,3248,6735,9513,6315,8108,1604,5725,4020,5862,1145,7764,8819,4222,9845,7583,3906,2939,5031,5106,7146,324,4733,1466,7859,6495,7328,6082,1658,2498,7569,249,2789,6677,1177,8049,7514,9575,2946,4074,1758,10817,5180,199,7581,4281,10626,10414,4966,4415,9751,10677,2752,8620,7121,7118,3277,2484,8561,7691,8433,1740,4667,8908,5071,6687,8491,3488,6863,10594,9895,596,9958,2677,1716,694,5604,7105,3496,6475,9748,677,5109,3072,5185,5656,8762,10292,1215,2841,6501,10291,8231,8267,9524,5076,7262,2948,10602,3427,4693,4979,10793,498,8586,4411,2171,9871,3595,6596,4483,7152,8719,1892,8735,7782,1186,9879,7572,2680,8857,5371,10384,3670,4870,4237,5824,9562,5608,1291,7021,9079,2729,8598,1662,579,9736,7560,3736,497,6604,5856,5155,8250,6322,4982,608,5607,5110,8840,10441,6084,3390,181,6711,277,8218,5612,7789,10086,1570,831,3565,5195,7558,9770,472,10067,6664,9844,10076,4144,8260,409,10310,3652,3835,4580,8580,6890,2785,1152,1723,1025,2192,653,1619,9006,3318,10494,7926,1441,5082,2092,121,1402,10133,9830,317,4115,7423,5618,8165,9530,5274,4477,1349,6304,7915,6965,4378,1330,5301,1951,2035,2063,5383,1473,10176,1865,5360,10646,3626,10396,4123,5599,7137,5558,2767,7386,10821,8443,6107,2648,5936,1481,9603,6825,10138,9870,2899,4003,10016,4078,4847,3157,8018,5902,5176,9827,4550,5657,5039,8541,9218,10493,185,6470,5527,1934,10753,5638,7589,7708,8180,7084,1067,6736,1230,6667,2653,2848,6937,3607,3107,3510,8807,359,5857,9537,6449,6327,5699,1420,3131,5349,291,8755,5788,9277,10350,1867,1700,10533,8450,2638,9389,9755,8234,3297,7277,9906,1099,1706,5756,7898,8979,913,2296,8399,9207,1044,2260,5931,8685,218,4371,2246,6223,10552,8933,5224,8655,7407,9617,5838,4971,10800,7822,5831,1996,10716,6076,7365,355,9487,8291,1548,3465,5156,7967,7580,6813,4263,7671,1756,3880,4502,4223,3152,3112,679,7499,1083,2115,8969,4718,5705,10475,9742,8812,9362,4581,2691,5488,1851,5320,4516,8565,6442,9805,2882,404,8144,6839,2347,7740,10510,9438,2218,1894,10237,5515,2595,3612,9882,6576,5386,6029,4707,7919,4188,8863,1811,10551,4259,1355,2941,2513,7157,1238,6600,6846,2305,10399,2091,9109,3819,4503,6566,9339,666,3016,8446,1429,5981,3374,1890,13,1158,2547,2008,4669,914,430,5632,2098,6005,9812,9579,5213,10508,9335,5362,5456,10182,1222,6383,8730,5033,3828,8051,328,4004,1116,3118,9357,10105,10120,4460,2326,6899,3218,6726,294,10087,9763,9993,946,8540,4582,3647,5967,5830,10232,1198,3521,7664,9406,6391,8331,3680,7484,9757,3834,4372,4072,2232,3981,9256,2208,8740,10787,10177,1095,8153,1156,5377,1927,9905,10364,4437,3864,4036,7455,8485,6552,8179,2895,256,10591,3926,3621,7635,6226,231,4098,3050,7871,2975,1328,3993,2746,7594,1742,1363,7272,3770,535,10750,2554,3815,6006,2750,5312,4251,887,2640,365,370,2025,858,7679,1629,2332,7809,6067,10346,7265,828,4638,10507,2865,10214,9680,8457,4337,1232,8616,9570,10312,4236,4788,9508,7863,1164,10080,9911,6832,748,5947,5055,4490,7241,7830,5081,2588,2104,7120,3918,8935,9328,6350,3200,1165,1323,1310,7632,10531,2292,5567,5248,6954,7839,9456,10290,8459,926,206,10654,8953,9898,791,8744,5923,2724,7145,2520,10238,385,7318,5018,10669,6115,1241,436,3512,7243,7151,899,2921,4355,925,696,6260,9253,6670,2807,8507,7930,1369,8898,5160,10285,3355,6282,3556,8356,2380,8825,6052,2329,5726,6403,10773,1990,3661,3221,4407,2826,3854,8501,5348,4414,8340,8268,5359,3470,1748,4559,4927,3158,6472,5009,4613,3456,3641,1029,8226,9454,1111,4628,2736,10272,4346,2313,5108,3836,8220,2183,1838,7103,8842,6474,4996,3953,2900,2741,1413,6217,7024,168,10079,3184,1069,4696,2713,732,2811,5398,3734,10695,8998,3873,2518,6440,6077,7964,2814,5125,9414,3674,9855,7888,5711,4962,6515,9472,3385,10090,1140,7833,9499,8579,1770,885,4092,420,7519,7736,9819,3020,10765,10170,377,3097,9930,3300,8057,8160,7072,9842,3573,4700,10129,400,112,2501,6384,10234,6646,7571,10256,5091,10525,3739,1339,8315,6436,9156,9478,4064,8967,10360,963,9175,1797,10711,7623,709,8482,5317,1404,2298,2238,8523,467,5436,7810,2150,2986,4719,6926,3784,5673,6884,7434,10659,9200,10461,157,4894,3323,6683,1626,6970,7192,3089,2010,284,6192,2216,7284,7009,4215,6961,4191,1305,4561,1573,7279,4930,3604,5691,8424,7538,10570,9135,333,7851,10073,4449,8403,942,7616,697,7456,6232,3927,6737,2496,406,2148,8078,9957,1477,6428,7171,9140,7347,8026,10135,9114,2643,4874,5601,5460,8474,2415,6127,10161,6609,8401,358,9093,5912,1097,1372,1687,9167,4666,7503,6968,486,4649,2994,631,7585,1318,110,5484,4181,6492,9003,4574,2392,3821,7135,40,1243,5264,8097,9474,7316,4297,5132,9716,8683,7505,8173,8623,4025,9262,9004,139,9434,1932,5354,6023,10367,9382,1346,3654,4185,303,8265,372,3214,4009,9784,8200,1738,10640,7092,4958,10656,164,1493,9741,4195,7624,9967,5063,1153,8298,4576,3360,10598,8380,6206,8119,6874,1972,10759,4806,4905,4157,4915,6343,9019,8680,2510,9372,4261,6669,6147,1830,3667,3415,8237,4655,4886,10032,1902,4456,9969,7536,4750,5825,777,2791,2247,1062,5188,793,1648,75,8820,9361,2656,7725,10783,924,7950,6871,620,8261,6285,3250,6145,824,7289,4735,2374,5399,10435,8637,8613,2571,10782,2078,9978,2999,2333,4018,8650,4039,4015,1778,4513,1054,5859,5147,2082,9264,6243,2203,2096,423,2267,3149,1386,6080,974,10811,3804,1913,3228,8454,10109,4929,10316,8212,9185,1093,3257,6092,6493,1950,1686,6018,9056,3332,4715,99,6910,507,5964,9558,9228,4517,1826,3495,3194,6097,8058,10532,4190,3084,3290,10218,2506,10447,7453,1113,6823,7587,7315,1722,8789,5370,5553,7486,4542,1731,4548,5700,2136,7799,2776,9087,3727,5744,780,1495,1956,4782,1621,6589,8159,9043,9745,1980,7983,197,8232,5746,1908,4471,7693,3534,2061,9370,1688,10779,4187,83,4794,4024,4291,7465,10665,1958,7533,4603,4752,7628,6389,6734,1691,10019,905,9213,6314,10599,1601,8130,6972,2915,9817,10081,8174,8396,530,8850,4277,3728,323,4174,2734,5118,9031,1899,3312,9208,10662,8092,7899,5874,2495,4175,5012,2223,10735,7312,2315,3298,6798,1103,2430,6261,8902,2360,3000,5331,4412,7136,6529,10357,1235,7086,1354,8490,9231,1773,7554,3343,9356,5880,9400,3139,3812,2371,7905,10004,6563,4071,6622,3286,6468,6877,7383,422,7286,2801,5366,8977,4373,9107,1500,1718,10720,5853,9951,4907,5448,4056,3255,10506,2853,2710,6681,6308,10495,10014,6382,3693,2989,930,8760,5345,493,3372,8388,109,2257,8604,3281,9058,7055,543,1078,971,4673,8123,9141,6249,10815,4925,1343,4186,5233,4330,8253,7775,4229,8286,1589,6157,7588,186,4246,6168,5092,9800,8251,1872,6856,7831,1625,5048,6374,9731,9281,8147,87,3498,9337,2964,156,9931,9152,7332,2006,6538,5306,3403,3530,4212,1967,7953,4130,5624,50,7337,7016,9282,10149,505,9075,2585,10083,9247,2334,3132,1317,3764,6095,7896,5835,2433,360,781,6222,5943,331,4475,8975,8354,8246,9999,7231,4193,8503,10226,2248,7684,1988,8824,4398,9422,6083,3853,4055,2889,5748,6848,2455,4088,3622,10636,5410,984,8974,4343,8006,4,7013,450,416,9714,6152,8777,7944,2564,5774,190,1653,8321,321,9633,6545,1676,8320,1848,8086,7319,747,7954,1759,4183,1552,9354,8009,1485,4239,10592,5602,864,6637,2838,10519,96,5750,6640,4932,5570,7617,2784,2058,9229,514,292,6324,2654,7338,4001,6225,9754,3618,5631,6465,4488,3877,6943,1252,3608,403,1817,8484,6156,4136,2385,1739,3742,5321,3375,7356,4348,3762,10471,6396,9220,4730,8295,5687,6482,7179,8529,7812,9840,8546,4023,2419,8631,10306,8118,3686,9690,2376,8010,1790,8244,4836,5854,10342,6896,9311,2552,2959,2549,7330,5716,4090,8104,9432,4286,1969,7717,1641,6629,7784,4532,4773,5201,7475,8544,4568,7805,126,8284,4316,4109,9722,9806,713,8776,1823,5215,2685,4731,5263,4632,7913,329,2337,10372,1149,2869,10829,1807,5267,727,9417,4854,1781,7182,6198,4322,8273,2737,629,6525,4571,10070,2199,103,9009,7401,3480,6379,9915,2459,340,8079,9816,8729,4991,7163,449,10585,3830,1303,5191,10583,559,6351,8228,10375,5755,861,2832,9980,9373,10622,10115,195,10058,3424,7233,503,4106,950,1812,3928,739,3303,4323,5330,9768,4381,4242,433,10485,6586,10230,9088,7743,4875,5779,7237,1295,3936,8076,1968,7334,8926,9096,4162,8912,8040,10712,2976,5479,5350,821,1178,9177,5346,9466,8955,538,3282,4964,1538,6557,10125,7504,117,8816,7113,10611,5639,2658,6323,1058,5225,2089,1457,8539,5003,1422,9310,2364,225,9939,3225,4609,9029,628,10553,8375,630,10660,1579,6188,1409,1482,991,4278,183,7494,6661,7452,5152,803,9230,5069,119,2954,9246,8853,561,9747,7119,7844,3151,3955,9268,2462,3690,2777,1061,5060,1301,7417,4142,9286,9197,5145,1943,4228,70,5872,2565,7920,4234,8509,1964,3264,1353,9314,4155,7566,3978,7020,4466,8572,6074,7285,6964,5065,6312,7521,2001,6059,4828,6978,5655,766,9584,9555,9782,1777,2147,1529,10326,1914,4458,7162,9385,8307,9094,5427,9773,9014,6722,9276,4564,6597,4817,6293,7239,5005,8835,10315,648,2842,4121,3213,5619,5499,9444,3345,4205,55,2443,3344,1260,1955,288,4198,2187,49,4201,4150,10511,2304,9994,9067,5486,2556,5116,4369,3994,10373,7651,4736,5532,5378,5525,3630,9646,1115,3542,5823,7457,3007,8642,7690,7681,3646,4698,2880,10245,220,7349,7596,2356,9590,4745,1568,4596,10195,7952,659,2722,5494,4659,4584,2409,9435,806,7939,10751,3224,9531,399,7639,644,10428,1638,5837,9705,165,6988,10020,1154,2274,3205,8369,5121,3074,8635,7796,3311,2017,1279,7802,9409,611,10613,8748,2719,7354,681,7261,3070,10667,8757,9039,10745,3525,3079,10282,10700,3289,3278,8440,6431,6914,2469,5974,3681,4514,4597,10432,1144,8957,2397,1107,2604,5101,6179,3523,8329,9023,4418,7195,8305,1949,10788,1882,9405,5973,1952,2778,1428,460,3331,5445,2221,8962,987,6451,325,7553,2037,9884,8864,1042,10308,6434,8136,2481,7814,3012,8860,10509,3874,23,4910,7189,8194,2365,6671,7280,5314,4485,4196,9020,6131,7787,939,10781,226,1059,56,1931,2427,8571,6864,8022,9753,281,4720,6020,801,4864,967,1631,3404,3935,3352,3075,8749,8639,3130,1776,1873,8059,2222,9097,6887,7043,3879,10097,248,5104,7506,5397,1074,2122,65,7109,832,1425,1800,4744,8362,9872,3115,6692,5038,10738,10359,2911,8909,575,4194,2770,6142,508,7749,10721,8806,6786,8630,5877,8660,7921,1747,9995,7065,251,8126,4420,5935,9046,9797,7781,8468,9965,5978,4260,10616,1502,408,7270,9899,3883,9743,1597,5019,10154,5107,4851,54,4145,3108,5500,107,8191,8768,5208,3886,3105,5664,4853,1005,4624,7467,6011,9481,10724,556,6678,7161,5966,5720,4509,8416,1545,2064,9663,2424,6349,6676,1513,1724,2434,7603,3487,9171,8966,741,10283,2802,5681,10775,10663,956,1337,10349,8715,4952,5163,2301,4741,8952,2178,7997,5296,5805,6251,1096,3232,8504,5671,4575,9924,2113,1139,7848,2702,762,6769,9607,2923,5561,6776,6713,3106,5440,10377,9944,1979,5535,7321,1610,8204,10466,5769,9677,2560,4030,3181,7903,7858,7282,9948,566,9611,4326,4060,2591,2647,9081,1750,285,7735,866,2914,4045,7177,7989,9795,1926,3330,9536,7732,9892,1557,4094,5812,405,5633,8536,8695,9533,2611,4492,4957,2852,5221,7234,10124,10078,5953,4845,3526,7409,6139,7656,9701,6310,11,10344,836,1464,5250,7003,8788,738,6309,4510,4484,9695,3725,3767,5676,2317,3644,5523,5818,38,3052,8554,102,3598,2020,4972,7018,3958,5546,8366,4859,1965,8673,1815,9159,6297,1027,6939,6212,7091,6184,7111,2528,6725,5461,282,7132,214,6624,1489,4623,7448,8074,7392,1623,2215,470,10335,8668,8385,3493,2378,9963,933,9974,6119,1128,1558,6766,6159,9991,1015,8866,4052,2845,2401,7697,10633,8591,570,7640,5926,7038,6762,3359,6013,6807,4334,2033,9534,576,4376,9070,6625,9341,5451,2748,3559,3806,10642,8999,6527,8377,6112,3501,599,7961,8929,2753,9588,9512,1456,10569,153,254,2399,2695,4403,5235,8772,1710,4367,9376,3855,3110,5382,7207,3639,7311,12,8852,2438,2979,3315,2781,3244,7012,2354,7642,1614,6227,955,7414,10271,2480,6219,4648,6898,6090,6551,1449,1720,8568,10228,6247,8818,7883,4544,5795,4689,9309,10465,3160,4177,5758,9008,671,3944,7988,1033,10482,5552,3930,7331,10089,1331,4124,2407,7238,6368,6992,9211,1408,4995,9761,7355,3904,7053,378,3497,549,4213,10030,8981,708,3505,7310,3698,3863,3590,8796,7165,4321,3241,6294,2763,8475,10404,6906,7174,5066,8161,10175,9609,8280,9758,1767,5095,9682,106,4819,123,7041,10099,2668,3682,3696,7160,2233,7359,637,2331,60,1615,7552,9523,9956,3642,798,4467,3388,8948,9756,266,3267,7891,1465,5478,9794,1340,7686,8045,2105,1543,3013,2584,2491,1183,7188,6345,4082,6806,8692,10766,4993,1391,3760,9084,3803,10010,145,6387,7281,7236,326,1986,5103,4249,7122,9527,5212,5994,7478,6400,3631,4809,2088,6200,10792,3705,6976,10541,10702,9662,379,4984,1942,717,2045,6793,744,6511,819,6971,2837,6630,487,10621,2471,4694,5571,6536,3353,4320,8767,4639,5151,2577,5491,9818,8493,4253,10610,2093,10653,3094,740,3934,3137,4241,2314,5368,7314,8574,1841,4211,9259,1947,9760,4507,9343,5587,6894,133,3135,602,1520,9142,5641,5463,4221,9679,795,3759,897,8511,9201,7322,9887,8208,6163,5528,8199,4857,9838,8601,2833,7472,7025,6770,6783,4431,3839,1713,5496,1562,5495,2133,10208,9,1400,4799,7339,2765,975,1259,5663,5334,451,7886,4938,4540,3001,4463,4354,1802,8681,78,4152,2955,2202,296,3538,3017,1753,1035,2251,1076,3156,5829,5682,3968,4329,1297,9502,4328,1304,8061,4189,3878,1073,10672,10728,5087,10680,8304,3204,8032,10339,554,69,2259,2884,5297,4416,3320,7178,7582,5773,9620,9340,7497,1199,2420,3629,3671,3684,9848,8548,6802,10755,7446,5480,5177,10347,10178,4791,592,9567,1127,1936,9103,2855,8215,2460,3620,9769,247,672,1283,8133,10198,3314,9632,8699,9981,4634,9661,2100,932,3585,6941,8107,8181,3514,7852,7047,10699,4139,3393,3222,7340,1521,5364,3640,134,286,9673,2143,6423,4031,3973,6782,9735,3018,6998,8907,5791,7186,4573,7955,196,3509,1974,6459,9548,2936,590,9732,4167,1571,7842,8651,2400,10365,3032,1275,7972,10789,10489,6779,6300,2871,9412,7029,598,3120,7034,3659,2318,4527,2795,3772,2808,1427,1532,7868,5724,10830,1598,9803,4153,5341,5150,10045,1086,2352,6460,6912,1725,6933,207,2860,4767,1998,10458,3274,8323,3969,135,7856,3031,4173,1426,3199,2937,6105,62,5512,1079,10719,5881,5473,7586,9932,7205,9257,6007,8870,8784,7667,5901,6741,3048,10430,6829,1225,4264,4462,7957,8195,1976,3494,8891,2535,8978,5543,2060,4238,6477,1460,2819,1744,9057,9416,2732,1922,10345,4440,2395,6299,6211,3923,6550,93,8023,10477,9138,6584,3243,6321,814,5892,127,734,10201,1891,5430,8407,9692,10065,9574,6333,8910,4661,2621,3541,6462,10820,4313,1835,6603,6849,7117,1672,1837,5997,2306,8537,233,3099,9724,4704,1946,7978,2727,658,9557,9279,2699,2543,1122,1810,5533,7998,5275,3599,2650,5384,1221,2903,8445,5342,10303,7224,714,2312,2504,4247,2579,8931,8395,9238,1209,1024,1009,4820,8684,1007,7222,909,10002,5950,4141,5538,1887,8940,5143,7317,4771,9194,7847,5804,8017,2756,7744,5701,6652,2138,8116,10361,10172,1503,10806,8288,6437,605,647,9117,8013,2517,7875,1038,6694,9383,6758,1302,5508,2389,5579,246,8608,8274,349,657,7666,6555,1390,784,1333,5023,263,2568,6476,8462,431,4727,9191,6867,2835,9082,2176,3185,3913,2413,97,10295,968,996,9709,5231,5474,1572,9790,7062,419,3242,1542,2016,8640,9172,2262,8632,5919,8653,3706,9012,7129,10395,216,10418,4245,5941,7774,6747,8913,5816,8889,362,8811,10828,5870,9410,10279,4987,7089,8203,844,8701,7849,3451,7803,4545,9125,2709,3011,1150,3144,2705,10151,6533,5549,2004,3615,9051,9290,3653,2065,2924,3054,9323,6386,5454,7344,2854,6317,5078,765,3195,3882,9660,10739,5255,7990,1314,10619,1377,459,6880,5260,432,3678,10400,2170,10188,7755,3080,5821,2644,5353,8067,6360,8739,9776,3783,9650,3549,4478,7173,5381,1636,6491,10517,532,10566,177,3695,1157,7592,1501,10631,8075,5666,3536,1361,992,9199,7212,184,2437,5945,5569,1989,3566,2966,5278,4751,8481,3648,9336,5811,5813,9085,6997,3162,9457,4732,8487,10042,5124,1903,2589,5465,8844,8827,4950,7720,1745,4007,6599,7410,5080,6800,8751,7746,5443,1191,7046,8605,9491,2493,8372,10250,595,4595,1072,8583,10107,1075,8805,3668,7063,4284,2095,6173,9064,8129,3890,10643,3861,4053,3477,5781,4338,4955,6717,10180,7033,6918,1155,2084,1791,465,8150,8887,7351,1771,6546,3956,4904,8747,1825,3129,6032,3945,4849,6936,265,10456,2361,9350,6356,7303,9955,8317,3808,9771,7406,9563,8326,3063,6169,4266,3750,8343,4325,7190,7976,10114,5689,2322,381,6554,3964,4617,2622,7405,1743,1560,3744,8096,9581,3246,66,10455,4061,846,5058,7674,280,2412,5815,2844,3335,7641,851,4615,10670,2129,6334,1884,5890,5749,10740,5000,140,4296,2657,1231,1071,10746,3285,3738,10443,6950,4309,8044,4033,3216,6809,10687,2145,6344,1205,2824,1828,4231,8752,5010,5851,5930,1002,6759,3963,3809,8410,8869,10118,703,587,2128,8671,2209,6795,3483,2898,5680,4873,3700,516,4699,9002,4200,9785,7918,8241,6143,3870,8466,4862,376,5644,4647,9717,1101,8186,9637,9464,8214,7857,3350,6706,7729,188,6619,8015,9498,7675,9153,3238,5531,948,2038,3701,7169,8945,6503,8502,4653,10210,707,80,9815,8302,8151,726,7429,5925,2049,2761,271,5948,10007,10047,5372,9810,8414,8614,3179,4404,9083,4543,9655,9549,5690,9713,5120,10144,5987,1472,1375,2988,4445,2293,9404,4619,221,4119,4054,8712,4810,8904,6764,7575,6399,8138,4391,3256,3095,9110,7438,4496,8959,2372,73,1011,1799,42,2583,8398,4555,101,1174,1376,3591,10167,10449,9333,5171,9413,9005,2236,9391,9233,796,4879,10001,802,1714,3247,5743,4530,30,786,1476,6171,1651,848,5610,1794,4761,1670,9037,6057,466,5347,7593,4637,5861,3950,2973,3765,9365,1213,8449,6512,7203,5013,10586,7767,6281,2031,3939,4057,1896,841,4351,4340,7715,2957,3889,5594,5268,1727,2716,7395,871,2048,7402,676,9921,4539,9347,9538,6033,8691,1694,4146,6587,4811,10778,1348,4586,3046,10199,1368,9721,8476,7108,5683,4243,5403,5204,6141,2839,7792,3624,4028,4461,8467,1652,10388,485,3610,10091,4102,316,1268,1585,5623,6053,5490,495,833,9127,8417,4923,3085,1929,6814,7876,5963,601,10484,7634,10407,5497,10647,9205,7290,2922,8669,5315,8053,9071,1327,10328,3410,1319,5842,3426,7026,5742,4755,10562,6060,8876,9703,8980,4604,6274,3637,760,510,8300,7882,8325,10128,9476,6120,9900,8506,9017,7974,1132,2367,2441,8193,2672,7299,6755,8519,4792,4824,2968,8183,10648,10330,9913,10802,3019,4081,2431,223,7441,298,1858,2730,4837,776,10469,5140,7854,4789,3088,6172,7597,4161,4154,5603,90,374,4435,10813,7371,3547,10748,3455,2110,4058,4021,10512,7060,9903,8893,9868,6983,8924,6684,10697,5325,7096,6381,6248,9626,6803,6686,481,2608,4713,9602,4633,9293,655,394,982,10139,4386,2981,3820,9111,6452,8756,4986,8428,1595,3378,7733,3664,10617,1242,4218,4933,4838,8971,3570,1674,541,6022,9594,6325,6715,5961,10502,8027,7391,6506,1090,6685,3121,3220,1405,9469,3795,1526,274,8426,10379,8707,7845,7865,6453,10694,7947,4611,3633,1396,1463,1272,1941,6516,3726,4658,3768,9147,8460,650,1023,5439,3794,2487,1000,319,1105,3996,3466,1637,302,10013,850,1999,10163,6785,5234,4282,2904,3336,162,7296,4867,8427,1681,5962,5285,3327,9496,557,5272,8538,10302,8192,6021,8802,5895,683,2446,5184,9104,9511,7766,3163,2056,4096,7935,8894,2188,7576,1514,3302,1881,10709,2366,3713,244,9431,6122,1539,3632,7738,1325,1342,5446,209,159,10062,2,2382,7078,5503,3133,9733,4974,2599,4682,4410,5393,10251,2578,9691,496,6277,2405,5256,10363,9146,1036,10043,1622,9799,5470,2201,515,7817,5379,1269,7227,8266,3683,3991,10682,8486,7131,9312,7424,182,5763,9324,822,7028,4117,3868,6513,7722,1151,1013,306,9752,10423,1166,4104,2714,9693,7251,1247,2627,2901,4425,1203,7910,4587,389,3553,1824,8607,1679,5198,6419,2984,2738,2137,2161,976,6228,6056,6133,7573,2575,166,10072,4976,7615,2388,9162,9518,3474,10181,10368,9250,4560,2340,3308,6601,8042,7579,6263,4311,8804,7548,6537,5172,9244,1019,5672,439,7442,7518,1945,3897,1669,7300,1928,118,972,8415,3988,5814,7937,9395,8731,105,8654,3602,136,361,901,9255,7496,468,2211,1032,4302,7584,5425,4426,4939,6980,7079,1901,4341,6361,964,7612,2416,2511,3826,10325,1751,8645,3869,2172,3779,5307,2926,8813,7969,4956,3440,10385,10437,5041,3857,6425,9683,7479,3822,6240,811,1138,7249,3891,7056,7114,9288,9583,894,6435,7757,5625,6620,9348,3215,9750,5799,9280,332,9737,6038,2907,6827,1436,10808,1487,8885,6610,5522,1094,7971,2418,853,9313,736,3371,9126,7022,7247,522,2196,7153,9618,10044,6820,1569,5822,9500,5394,7660,10503,4885,5550,8041,314,2701,6821,6740,3206,6128,6704,670,5578,8312,2384,3665,6001,2120,1444,6117,7027,7867,6995,8095,5989,4985,9647,1088,9700,7819,5845,2868,8550,9463,8187,2243,5240,9145,3155,1261,6945,1566,4664,10378,8636,7956,2783,3790,3959,10426,7067,2623,1494,6088,5027,7747,9933,2242,9929,3662,5068,5846,2046,10480,1588,3447,1021,9105,7889,7513,4969,6286,4614,616,10212,3581,994,4723,4012,8348,1534,2066,10555,8112,2026,9298,9813,8162,9561,5560,2978,3699,5335,5178,8048,799,6427,7835,6784,9896,5917,5636,723,9901,8641,6330,5965,7361,1412,6654,8688,7140,4953,52,4204,10777,1196,1705,5487,7345,7815,2000,85,2408,3197,8786,2876,2155,6348,7364,3266,7907,8021,5593,6066,9429,9120,4511,7508,6693,9291,10024,2377,6569,9426,4464,4716,6815,7444,1212,9401,2345,536,259,8706,7388,8132,342,5559,5085,2843,6853,2286,4833,7555,9375,10296,10448,1524,7367,5134,7428,1549,2836,3582,7005,7155,9556,4303,6872,1787,4709,8359,3943,9063,7380,10136,1801,5035,10059,9653,8210,3310,10142,1512,5913,4127,4858,8558,8357,2043,4267,6738,7071,2958,125,6426,2276,95,2735,9078,6595,574,8140,6787,6284,1565,8865,5044,7019,3909,6705,10612,4692,1840,1606,5939,3354,6280,10417,7415,3189,4546,8081,6276,8665,3716,3432,5771,8370,10190,3485,7306,8328,4618,4515,8679,6745,7895,7873,5098,917,3083,1001,923,5904,6395,1643,10041,7416,5988,5847,752,7269,5105,8282,9971,518,1878,3569,8157,8517,8791,5957,9394,3193,6002,9952,2383,742,9725,7218,6517,2963,3769,4026,3704,7396,8083,5324,931,1857,2027,1836,6695,2402,6720,1563,3533,10451,9180,5051,4830,5855,6091,2629,7836,4350,4359,6634,7291,10584,7631,100,6234,6111,823,9198,7611,10011,44,6608,1511,9016,4436,9303,1764,1819,2277,5885,7832,10140,8378,10288,2875,6246,3866,5308,7130,7837,6514,4921,8225,9001,3627,3515,4384,10025,6129,3237,9541,2453,8031,1528,6881,6582,1288,6922,3230,9222,8333,8037,4512,9829,7366,3845,7683,8916,7070,2344,6272,9610,3756,775,10528,10008,1455,8030,5660,10764,1026,3829,5658,8477,702,4578,2764,5006,3919,4797,1741,3231,10704,9453,10278,10628,8451,2993,311,9968,5402,3039,632,8472,2358,2003,8024,3781,6268,7714,3535,9148,8552,558,10243,9421,8557,5438,4383,10162,8708,1439,852,2532,1224,9144,2610,3446,5157,4878,8716,5918,6093,4903,578,8125,10101,3586,7492,5119,9734,10287,7609,2273,10515,8014,8290,8698,6016,651,7776,5659,1440,10185,9648,2353,2213,3258,6405,8919,5684,6199,1443,7093,5042,8599,2490,7404,5070,9160,10064,6273,5674,8783,7425,3233,8753,9672,2379,4567,3461,5282,10371,7221,10370,9384,4790,7458,7421,9321,10160,471,1861,3138,9304,2477,3373,4345,2857,10217,6064,129,8694,7682,6181,8319,10222,5131,4534,6335,8773,9686,7143,8391,8201,2362,6096,8028,10116,4382,9139,5413,4754,6857,6479,10248,4353,3217,6751,9509,8793,4448,7963,6859,425,7811,5232,9826,2220,5477,7015,8054,3236,1100,1334,5723,6904,6489,5958,10683,10421,5801,4494,6259,7167,8587,10713,2294,7076,455,504,10579,7030,3966,8008,8703,1795,1842,5524,606,5375,2851,10524,10563,5036,9436,8122,7829,7512,10314,1978,2631,6594,5772,357,9359,3718,9425,8954,2311,9240,8667,6831,8135,7164,10127,7476,7762,6908,5170,4380,3268,5401,9450,7680,3449,3983,663,348,9711,9018,5785,3634,7897,9411,9554,1593,1182,2442,232,5739,5817,1788,10046,5928,9338,4775,9223,4759,1755,1893,9091,7894,10022,7622,797,4588,10277,8555,1172,7101,4714,9209,9408,366,5562,4365,4079,9074,4869,753,6744,10732,571,9179,5899,5223,4808,9206,9028,5428,6255,3788,6508,5914,1821,5283,3271,4626,520,3324,710,4390,8934,2670,769,8868,4084,3117,1192,8313,8082,5712,1798,3540,7274,3479,4043,1761,8167,7341,8761,3261,7468,8619,7048,7244,8524,6191,4668,10818,10197,1008,1424,10657,9916,2721,1712,2669,10158,5695,7924,4506,5840,4651,5516,222,3798,9301,8243,5509,2217,3124,6394,9786,5469,813,9689,9949,10159,5864,7853,9428,3014,6267,8172,41,9580,418,10239,8332,3992,3655,6562,5993,2881,6792,9353,5187,6810,1889,9833,4184,4336,2059,4171,7880,733,3603,2995,5146,4777,9704,9504,6326,5995,5148,3348,8991,10513,2005,10391,4520,9119,6407,3814,2032,4519,8141,9101,4156,7537,10179,9843,8073,1559,10488,2598,668,8989,442,6062,7601,8020,3925,7607,3917,3325,8334,4075,2271,7668,76,5266,715,350,8349,7374,7302,235,1630,10268,2515,10729,2637,4703,8358,944,4318,6518,7962,949,2997,5072,3685,2508,3778,8621,7488,1208,5713,6611,5643,7352,10574,5073,9621,3478,3051,27,6467,2516,7678,10573,6213,1517,3434,2698,6457,4111,4125,446,5432,10645,4872,1905,10000,6580,10812,111,2404,5437,3971,7430,2132,7942,2268,895,2906,730,1690,3676,10730,4083,6990,92,10791,1148,6826,3583,6298,10257,10266,9038,1287,4019,1382,8878,4268,9485,6121,10785,2118,2325,7825,1784,5582,5717,4393,8588,380,2615,1048,10651,7426,1567,3491,9907,9897,1608,8379,10671,10549,7209,10026,1726,5061,8392,8421,8255,10464,1171,711,6855,1515,344,7644,7618,618,5014,9744,8238,1728,9605,3887,8527,8705,1845,6216,6137,1782,6885,3183,8867,5276,7510,3428,4770,9585,3513,7230,5650,5333,6194,8899,6409,3575,8276,3421,9764,2991,6189,10244,6328,4814,4541,980,2047,10009,3811,8189,6166,8447,6466,701,9274,938,3459,2873,8406,7460,4772,3776,3279,5697,7150,4644,9822,1692,10684,3609,617,2309,9885,1134,1574,3008,273,9060,8677,147,1052,5734,2308,7688,3954,10066,3059,8972,6413,5502,3453,8526,210,639,680,8019,5161,6473,7127,3666,8589,10639,4843,1084,6035,8873,10454,3253,9934,2075,5128,8938,8292,175,10614,4994,10106,792,10752,2507,4832,151,7834,4931,2537,2918,9542,6709,8576,9468,918,3010,10352,1508,2661,8494,9448,4919,10547,3476,2646,716,5732,1367,9837,6010,990,1576,10189,2536,8901,4233,1984,6124,6682,614,9835,5137,10483,3580,8351,5782,820,7220,7878,1977,5843,675,9923,7527,4734,9861,2351,5052,2013,5998,9959,10063,686,5246,3844,2154,9664,2863,1173,3276,3273,2739,5126,7010,889,461,1434,7045,8923,3616,2071,4424,7960,1311,2373,6167,8105,1806,9069,3785,1583,5808,10810,8235,3990,1431,10095,2633,6673,5729,5598,10439,10225,4768,3731,434,2391,2974,5642,525,8750,8593,6574,3172,2375,5848,4999,241,3035,9189,9393,8224,8765,1645,3862,8592,8880,9762,1556,8920,137,6788,8227,749,6542,1050,10191,9943,3340,8520,7348,3593,5897,8875,6621,10496,8718,3002,8871,7752,3914,2226,7855,7090,5400,9289,5265,10137,2825,5906,5179,2606,4275,1181,6917,9587,10688,2055,10077,1159,5007,2766,1985,4717,10294,7515,2613,7397,5715,7590,6623,2531,5434,1509,9926,941,6868,1909,1818,2541,5336,2165,4290,10202,7927,5387,2011,9212,1254,5405,7490,2422,3087,5551,8649,8175,6055,4240,108,6655,10056,6708,7507,2121,5173,2198,1046,816,7866,2659,9076,6041,8513,8283,351,9480,9118,10798,8488,6560,8209,9271,3467,7054,5789,7509,4432,8626,7893,10565,2252,10012,5652,7658,6123,6835,91,9239,928,1286,2529,1378,910,4706,4116,3774,6949,8052,7495,5077,2596,228,155,7387,7778,154,7201,7193,5704,1944,6071,1582,375,2793,2548,8197,1953,10757,4301,4013,2849,8090,7232,10763,10736,7994,6989,477,9473,10134,1717,7360,3673,5079,10543,8207,7696,3431,2745,8461,279,4834,8746,128,2538,7382,10281,2888,9439,3789,10559,9442,3234,8213,8106,253,9573,856,10644,5867,1293,1715,6861,3543,10406,10501,7813,1256,9910,511,132,2771,1040,9839,5547,8035,9386,5616,545,8342,10307,5022,8098,2905,6433,6567,1680,5832,2494,5858,2655,1249,3807,1551,2186,1804,4801,3412,5322,5565,4468,1697,8570,10343,2282,201,10734,3383,3211,8393,386,299,5613,6307,8381,10152,883,9234,737,8606,3301,327,4105,5792,1966,8722,10416,6883,5217,8896,6342,4890,4671,6195,4901,2983,6464,7481,8618,8855,682,8473,3405,604,5093,335,9864,2562,5932,1345,1102,6651,10235,695,67,9352,7545,3400,2523,144,3402,8848,1888,5411,3464,3876,8737,1012,6973,10816,8063,3687,4452,5629,4091,2690,217,2566,3697,8888,8121,276,435,1649,6797,3034,7801,3799,9865,2483,568,612,857,4498,4663,7730,7511,3619,6000,5969,261,2665,4721,5970,8610,1506,5728,4882,2328,4712,4825,1917,9495,8087,4254,5062,2417,807,870,6977,6378,3531,1682,5766,5669,3304,4016,9719,8435,3905,5949,4225,4011,7502,10710,3202,6287,8062,3167,9106,10825,4347,662,10229,9128,830,3588,4805,6377,3717,8961,6385,7431,3041,9649,2475,315,6039,10168,278,7804,10153,817,8306,4922,6948,5288,1729,3086,2231,6504,9116,4765,4600,9054,705,9684,4131], "time_ms": 0.109857, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 613, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "generated_query_count", "value": 1, "time_ms": 0.722491, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 428, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "generated_samples_per_query", "value": 100, "time_ms": 0.722491, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 429, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "generated_query_duration", "value": 329760401, "time_ms": 0.722491, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 430, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "generic_message", "value": "Ending naturally: Minimum query count and test duration met.", "time_ms": 30592.400387, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "issue_query_controller.cc", "line_no": 482, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "logger_swap_request_slots_retry_count", "value": 0, "time_ms": 30599.399891, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 898, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "logger_swap_request_slots_retry_retry_count", "value": 0, "time_ms": 30599.399891, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 900, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "logger_swap_request_slots_retry_reencounter_count", "value": 0, "time_ms": 30599.399891, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 902, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "logger_start_reading_entries_retry_count", "value": 0, "time_ms": 30599.399891, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 904, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "logger_tls_total_log_cas_fail_count", "value": 0, "time_ms": 30599.399891, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 906, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "logger_tls_total_swap_buffers_slot_retry_count", "value": 0, "time_ms": 30599.399891, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 908, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "power_begin", "value": "10-23-2024 01:06:19.626", "time_ms": 30599.400392, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 564, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "power_end", "value": "10-23-2024 01:06:50.218", "time_ms": 30599.400392, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 566, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "result_validity", "value": "VALID", "time_ms": 30599.427974, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 655, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "result_min_duration_met", "value": true, "time_ms": 30599.427974, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 660, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "result_min_queries_met", "value": true, "time_ms": 30599.427974, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 661, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "early_stopping_met", "value": true, "time_ms": 30599.427974, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 662, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "early_stopping_result", "value": "", "time_ms": 30599.427974, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 682, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "result_query_count", "value": 1, "time_ms": 30599.427974, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 692, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "result_samples_per_second", "value": 3.26888, "time_ms": 30599.427974, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 748, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "result_min_latency_ns", "value": 312527550, "time_ms": 30599.427974, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 754, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "result_max_latency_ns", "value": 30591552891, "time_ms": 30599.427974, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 755, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "result_mean_latency_ns", "value": 16036598614, "time_ms": 30599.427974, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 756, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "result_50.00_percentile_latency_ns", "value": 16653170308, "time_ms": 30599.427974, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 758, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "result_90.00_percentile_latency_ns", "value": 28040840094, "time_ms": 30599.427974, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 758, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "result_95.00_percentile_latency_ns", "value": 29467542780, "time_ms": 30599.427974, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 758, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "result_97.00_percentile_latency_ns", "value": 30052531452, "time_ms": 30599.427974, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 758, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "result_99.00_percentile_latency_ns", "value": 30591552891, "time_ms": 30599.427974, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 758, "pid": 349216, "tid": 349216}} +:::MLLOG {"key": "result_99.90_percentile_latency_ns", "value": 30591552891, "time_ms": 30599.427974, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 758, "pid": 349216, "tid": 349216}} diff --git a/open/scc123-Shanxi_University/results/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/performance/run_1/mlperf_log_summary.txt b/open/scc123-Shanxi_University/results/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/performance/run_1/mlperf_log_summary.txt new file mode 100644 index 0000000..87e30cb --- /dev/null +++ b/open/scc123-Shanxi_University/results/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/performance/run_1/mlperf_log_summary.txt @@ -0,0 +1,51 @@ +================================================ +MLPerf Results Summary +================================================ +SUT name : PySUT +Scenario : Offline +Mode : PerformanceOnly +Samples per second: 3.26888 +Result is : VALID + Min duration satisfied : Yes + Min queries satisfied : Yes + Early stopping satisfied: Yes + +================================================ +Additional Stats +================================================ +Min latency (ns) : 312527550 +Max latency (ns) : 30591552891 +Mean latency (ns) : 16036598614 +50.00 percentile latency (ns) : 16653170308 +90.00 percentile latency (ns) : 28040840094 +95.00 percentile latency (ns) : 29467542780 +97.00 percentile latency (ns) : 30052531452 +99.00 percentile latency (ns) : 30591552891 +99.90 percentile latency (ns) : 30591552891 + +================================================ +Test Parameters Used +================================================ +samples_per_query : 100 +target_qps : 3.0325 +target_latency (ns): 0 +max_async_queries : 1 +min_duration (ms): 10000 +max_duration (ms): 0 +min_query_count : 1 +max_query_count : 0 +qsl_rng_seed : 0 +sample_index_rng_seed : 0 +schedule_rng_seed : 0 +accuracy_log_rng_seed : 0 +accuracy_log_probability : 0 +accuracy_log_sampling_target : 0 +print_timestamps : 0 +performance_issue_unique : 0 +performance_issue_same : 0 +performance_issue_same_index : 0 +performance_sample_count : 10833 + +No warnings encountered during test. + +1 ERROR encountered. See detailed log. diff --git a/open/scc123-Shanxi_University/systems/b7828419c93b-reference-gpu-pytorch-cu124.json b/open/scc123-Shanxi_University/systems/b7828419c93b-reference-gpu-pytorch-cu124.json new file mode 100644 index 0000000..1de01bc --- /dev/null +++ b/open/scc123-Shanxi_University/systems/b7828419c93b-reference-gpu-pytorch-cu124.json @@ -0,0 +1,37 @@ +{ + "accelerator_frequency": "1410000 MHz", + "accelerator_host_interconnect": "N/A", + "accelerator_interconnect": "N/A", + "accelerator_interconnect_topology": "", + "accelerator_memory_capacity": "19.9959716796875 GB", + "accelerator_memory_configuration": "N/A", + "accelerator_model_name": "GRID A100X-20C", + "accelerator_on-chip_memories": "", + "accelerators_per_node": 1, + "cooling": "air", + "division": "open", + "framework": "pytorch", + "host_memory_capacity": "60G", + "host_memory_configuration": "undefined", + "host_network_card_count": "1", + "host_networking": "Gig Ethernet", + "host_networking_topology": "N/A", + "host_processor_caches": "L1d cache: 512 KiB (16 instances), L1i cache: 512 KiB (16 instances), L2 cache: 8 MiB (16 instances), L3 cache: 512 MiB (16 instances)", + "host_processor_core_count": "1", + "host_processor_frequency": "undefined", + "host_processor_interconnect": "", + "host_processor_model_name": "AMD EPYC-Milan Processor", + "host_processors_per_node": "16", + "host_storage_capacity": "234G", + "host_storage_type": "SSD", + "hw_notes": "", + "number_of_nodes": "1", + "operating_system": "Ubuntu 22.04 (linux-6.1.110-1.el9.elrepo.x86_64-glibc2.35)", + "other_software_stack": "Python: 3.10.12, LLVM-15.0.6", + "status": "available", + "submitter": "scc123-Shanxi_University", + "sw_notes": "Automated by MLCommons CM v3.2.6. ", + "system_name": "b7828419c93b", + "system_type": "edge", + "system_type_detail": "edge server" +} \ No newline at end of file diff --git a/open/scc123-Shanxi_University/systems/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config.json b/open/scc123-Shanxi_University/systems/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config.json new file mode 100644 index 0000000..7bf1348 --- /dev/null +++ b/open/scc123-Shanxi_University/systems/scc123_cpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config.json @@ -0,0 +1,37 @@ +{ + "accelerator_frequency": "", + "accelerator_host_interconnect": "N/A", + "accelerator_interconnect": "N/A", + "accelerator_interconnect_topology": "", + "accelerator_memory_capacity": "N/A", + "accelerator_memory_configuration": "N/A", + "accelerator_model_name": "N/A", + "accelerator_on-chip_memories": "", + "accelerators_per_node": "0", + "cooling": "air", + "division": "open", + "framework": "pytorch v2.5.0", + "host_memory_capacity": "125G", + "host_memory_configuration": "undefined", + "host_network_card_count": "1", + "host_networking": "Gig Ethernet", + "host_networking_topology": "N/A", + "host_processor_caches": "L1d cache: 1 MiB (32 instances), L1i cache: 1 MiB (32 instances), L2 cache: 16 MiB (32 instances), L3 cache: 1 GiB (32 instances)", + "host_processor_core_count": "1", + "host_processor_frequency": "undefined", + "host_processor_interconnect": "", + "host_processor_model_name": "AMD EPYC-Milan Processor", + "host_processors_per_node": "32", + "host_storage_capacity": "158G", + "host_storage_type": "SSD", + "hw_notes": "", + "number_of_nodes": "1", + "operating_system": "Rocky 9.4 (linux-6.1.110-1.el9.elrepo.x86_64-glibc2.34)", + "other_software_stack": "Python: 3.9.18, LLVM-15.0.6", + "status": "available", + "submitter": "scc123-Shanxi_University", + "sw_notes": "Automated by MLCommons CM v3.2.6. ", + "system_name": "scc123_cpu0.novalocal", + "system_type": "edge", + "system_type_detail": "edge server" +} \ No newline at end of file