From 2502ae0a9c0dd63206960e9a44b323b322da5715 Mon Sep 17 00:00:00 2001 From: LiangGao Date: Mon, 25 Aug 2025 13:14:15 +0800 Subject: [PATCH 1/7] Change the checker to support threshold for uint16/uint4/int4 --- onnxruntime/test/providers/checkers.cc | 71 ++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 4 deletions(-) diff --git a/onnxruntime/test/providers/checkers.cc b/onnxruntime/test/providers/checkers.cc index ff5895623fc9b..68dad482dc402 100644 --- a/onnxruntime/test/providers/checkers.cc +++ b/onnxruntime/test/providers/checkers.cc @@ -183,18 +183,27 @@ template <> struct TensorCheck { void operator()(const Tensor& expected, const Tensor& actual, const ValidateOutputParams& params, const std::string& /*provider_type*/) const { - ORT_UNUSED_PARAMETER(params); + const bool has_abs_err = params.absolute_error.has_value(); Tensor expected_sorted, actual_sorted; const Int4x2* cur_expected; const Int4x2* cur_actual; const auto size = actual.Shape().Size(); cur_expected = expected.Data(); cur_actual = actual.Data(); + double threshold; + if (has_abs_err) { + threshold = *(params.absolute_error); + } for (size_t i = 0; i < static_cast(size); ++i) { size_t r = i >> 1; size_t c = i & 0x1; - EXPECT_EQ(cur_expected[r].GetElem(c), cur_actual[r].GetElem(c)) << "i:" << i; + // TODO: the relative error is not used for int4 yet. + if (has_abs_err) { + EXPECT_NEAR(cur_expected[r].GetElem(c), cur_actual[r].GetElem(c), threshold) << "i:" << i; + } else { + EXPECT_EQ(cur_expected[r].GetElem(c), cur_actual[r].GetElem(c)) << "i:" << i; + } } } }; @@ -204,6 +213,7 @@ struct TensorCheck { void operator()(const Tensor& expected, const Tensor& actual, const ValidateOutputParams& params, const std::string& /*provider_type*/) const { ORT_UNUSED_PARAMETER(params); + const bool has_abs_err = params.absolute_error.has_value(); Tensor expected_sorted, actual_sorted; const UInt4x2* cur_expected; const UInt4x2* cur_actual; @@ -211,10 +221,20 @@ struct TensorCheck { cur_expected = expected.Data(); cur_actual = actual.Data(); + double threshold; + if (has_abs_err) { + threshold = *(params.absolute_error); + } + for (size_t i = 0; i < static_cast(size); ++i) { size_t r = i >> 1; size_t c = i & 0x1; - EXPECT_EQ(cur_expected[r].GetElem(c), cur_actual[r].GetElem(c)) << "i:" << i; + // TODO: the relative error is not used for int4 yet. + if (has_abs_err) { + EXPECT_NEAR(cur_expected[r].GetElem(c), cur_actual[r].GetElem(c), threshold) << "i:" << i; + } else { + EXPECT_EQ(cur_expected[r].GetElem(c), cur_actual[r].GetElem(c)) << "i:" << i; + } } } }; @@ -252,7 +272,7 @@ struct TensorCheck { // For any other EPs, we still expect an exact match for the results // TODO: Verify if DML can possibly have a ROUNDING_MODE parameter and conform to the other EPs #41968513 if ((provider_type == kNnapiExecutionProvider || provider_type == kDmlExecutionProvider || - provider_type == kXnnpackExecutionProvider) && + provider_type == kXnnpackExecutionProvider || provider_type == kOpenVINOExecutionProvider) && (has_abs_err || has_rel_err)) { double threshold = has_abs_err ? *(params.absolute_error) : 0.0; @@ -317,6 +337,49 @@ struct TensorCheck { } }; +template <> +struct TensorCheck { + void operator()(const Tensor& expected, + const Tensor& actual, + const ValidateOutputParams& params, + const std::string& ) const { + const bool has_abs_err = params.absolute_error.has_value(); + const bool has_rel_err = params.relative_error.has_value(); + + Tensor expected_sorted, actual_sorted; + const uint16_t* cur_expected; + const uint16_t* cur_actual; + const auto size = actual.Shape().Size(); + if (params.sort_output) { + sort_expected_and_actual_buffers(expected, expected_sorted, actual, actual_sorted); + cur_expected = expected_sorted.Data(); + cur_actual = actual_sorted.Data(); + } else { + cur_expected = expected.Data(); + cur_actual = actual.Data(); + } + + if (has_abs_err || has_rel_err) { + double threshold = has_abs_err ? *(params.absolute_error) + : 0.0; + + for (int64_t i = 0; i < size; ++i) { + if (has_rel_err) { + EXPECT_NEAR(cur_expected[i], cur_actual[i], + *(params.relative_error) * cur_expected[i]) // expected[i] is unsigned, can't be negative + << "i:" << i; + } else { // has_abs_err + EXPECT_NEAR(cur_expected[i], cur_actual[i], threshold) << "i:" << i; + } + } + } else { + for (int64_t i = 0; i < size; ++i) { + EXPECT_EQ(cur_expected[i], cur_actual[i]) << "i:" << i; + } + } + } +}; + template <> struct TensorCheck { void operator()(const Tensor& expected, From 56f498d4fa8f866c2b86d5d9508fbfcecefa78fa Mon Sep 17 00:00:00 2001 From: LiangGao Date: Mon, 25 Aug 2025 13:14:57 +0800 Subject: [PATCH 2/7] Disable loop case for OV-EP due to floating nodes --- onnxruntime/test/providers/cpu/controlflow/loop_test.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/onnxruntime/test/providers/cpu/controlflow/loop_test.cc b/onnxruntime/test/providers/cpu/controlflow/loop_test.cc index a5fd37361a255..309e604ffbb63 100644 --- a/onnxruntime/test/providers/cpu/controlflow/loop_test.cc +++ b/onnxruntime/test/providers/cpu/controlflow/loop_test.cc @@ -828,7 +828,8 @@ TEST(Loop, Opset11WithNoVariadicInputsAndOutputs) { test.AddOutput("loop_scan_out", {1}, {1.0f}); // Disable TensorRT on unsupported data type BOOL - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); + // Disable OpenVino for floating nodes + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kOpenVINOExecutionProvider}); } // Test a combination of things: From 138b9524d6661c52fa56bbf6700631d476f15a2c Mon Sep 17 00:00:00 2001 From: LiangGao Date: Mon, 25 Aug 2025 13:15:53 +0800 Subject: [PATCH 3/7] Disable the case for OV-EP due to input conflict --- onnxruntime/test/providers/cpu/tensor/slice_op.test.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/onnxruntime/test/providers/cpu/tensor/slice_op.test.cc b/onnxruntime/test/providers/cpu/tensor/slice_op.test.cc index 5b2865a3feed7..f531e2a7210c5 100644 --- a/onnxruntime/test/providers/cpu/tensor/slice_op.test.cc +++ b/onnxruntime/test/providers/cpu/tensor/slice_op.test.cc @@ -540,6 +540,10 @@ TEST(SliceTest, Slice1D_ReverseAllAxes_1) { GTEST_SKIP() << "Skipping because of the following error: Expected output shape [{4}] did not match run output shape [{0}] for output"; } + if (DefaultOpenVINOExecutionProvider().get() != nullptr) { + GTEST_SKIP() << "Skipping because of the following error: The input ends did not supported int max when step is negtive."; + } + RunSliceTest({4}, {1.0f, 2.0f, 3.0f, 4.0f}, {-1}, From a9a3d6e422240b6c3b65bbef172a0616f0fede8d Mon Sep 17 00:00:00 2001 From: LiangGao Date: Mon, 25 Aug 2025 13:16:49 +0800 Subject: [PATCH 4/7] Add threshold for uint8/int8/uint16 cases --- onnxruntime/test/contrib_ops/quantize_ops_test.cc | 2 ++ .../test/providers/cpu/tensor/quantize_linear_test.cc | 7 +++++++ onnxruntime/test/providers/cpu/tensor/resize_op_test.cc | 2 ++ 3 files changed, 11 insertions(+) diff --git a/onnxruntime/test/contrib_ops/quantize_ops_test.cc b/onnxruntime/test/contrib_ops/quantize_ops_test.cc index db685967ae5ff..de10f14ef4538 100644 --- a/onnxruntime/test/contrib_ops/quantize_ops_test.cc +++ b/onnxruntime/test/contrib_ops/quantize_ops_test.cc @@ -287,6 +287,7 @@ TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_float_int8) { 127, -127, 127, -128, 127, -128}); + test.SetOutputAbsErr("y", 1.0f); // Disable Tensorrt EP due to error: node1_quantize_scale_node: out of bounds channel axis 1. Number of input dimensions is 1. test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } @@ -311,6 +312,7 @@ TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_float_uint16) { 32769, 32765, 65535, 0, 65535, 0}); + test.SetOutputAbsErr("y", 1.0f); // Disable Tensorrt EP due to error: unsupported data type test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); diff --git a/onnxruntime/test/providers/cpu/tensor/quantize_linear_test.cc b/onnxruntime/test/providers/cpu/tensor/quantize_linear_test.cc index 8fdbf0060eaa0..b1c01fc81fa94 100644 --- a/onnxruntime/test/providers/cpu/tensor/quantize_linear_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/quantize_linear_test.cc @@ -437,6 +437,7 @@ TEST(QuantizeLinearOpTest, Uint16) { 32769, 32765, 65535, 0, 65535, 0}); + test.SetOutputAbsErr("y", 1.0f); // Disable Tensorrt EP due to error: unsupported data type test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); @@ -466,6 +467,7 @@ TEST(QuantizeLinearOpTest, Int16) { 32767, -32768, 32767, -32768, 32767, -32768}); + test.SetOutputAbsErr("y", 1.0f); // Disable Tensorrt EP due to error: unsupported data type test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); @@ -490,6 +492,7 @@ TEST(QuantizeLinearOpTest, Int4) { test.AddOutput("y", dims, {Int4x2(-8, -7), Int4x2(-1, 1), Int4x2(2, 7), Int4x2(7, unused_val)}); + test.SetOutputAbsErr("y", 1.0f); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } @@ -557,6 +560,7 @@ TEST(QuantizeLinearOpTest, OddLarge_Int4) { test.AddInput("scale", {}, {scale}, true); test.AddInput("zero_point", {}, {Int4x2(zp, unused_val)}, true); test.AddOutput("y", dims, output); + test.SetOutputAbsErr("y", 1.0f); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } @@ -583,6 +587,7 @@ TEST(QuantizeLinearOpTest, OddLarge_UInt4) { test.AddInput("scale", {}, {scale}, true); test.AddInput("zero_point", {}, {UInt4x2(zp, unused_val)}, true); test.AddOutput("y", dims, output); + test.SetOutputAbsErr("y", 1.0f); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } @@ -600,6 +605,7 @@ TEST(QuantizeLinearOpTest, Int8_NegativeZeroPoint) { test.AddInput("y_scale", {}, {.039215686f}); test.AddInput("y_zero_point", {}, {-23}); test.AddOutput("y", dims, {-23, 28, 53, 104, 127, -74, -128, -128}); + test.SetOutputAbsErr("y", 1.0f); // Disable Tensorrt EP due to the error, node1_quantize_scale_node: out of bounds channel axis 1. Number of input dimensions is 1. test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } @@ -617,6 +623,7 @@ TEST(QuantizeLinearOpTest, Int8_PositiveZeroPoint) { test.AddInput("y_scale", {}, {.039215686f}); test.AddInput("y_zero_point", {}, {23}); test.AddOutput("y", dims, {23, 74, 99, 127, 127, -28, -104, -128}); + test.SetOutputAbsErr("y", 1.0f); // Disable Tensorrt EP due to error:node1_quantize_scale_node: out of bounds channel axis 1. Number of input dimensions is 1. test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } diff --git a/onnxruntime/test/providers/cpu/tensor/resize_op_test.cc b/onnxruntime/test/providers/cpu/tensor/resize_op_test.cc index bb053bc37ce30..f3b0695bdbd9c 100644 --- a/onnxruntime/test/providers/cpu/tensor/resize_op_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/resize_op_test.cc @@ -308,6 +308,7 @@ TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_4DBilinear_uint8) { std::vector Y = {2, 4}; test.AddOutput("Y", {N, static_cast(H * scales[1]), static_cast(W * scales[2]), C}, Y); + test.SetOutputAbsErr("Y", 1.0f); // CUDA: result mismatch due to not implementing NHWC support // ROCm: results mismatch test.Run(OpTester::ExpectResult::kExpectSuccess, "", @@ -647,6 +648,7 @@ TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_4DBilinear_pytorch_half_pixe std::vector Y = {1, 7, 12}; test.AddOutput("Y", {N, sizes[1], sizes[2], C}, Y); + test.SetOutputAbsErr("Y", 1.0f); // CUDA: result mismatch due to not implementing NHWC support // ROCm: results mismatch // DML: results mismatch From 2cfa45779debcb5446aa84d69e3563f0f7a8d791 Mon Sep 17 00:00:00 2001 From: LiangGao Date: Mon, 25 Aug 2025 14:33:49 +0800 Subject: [PATCH 5/7] Fix build warning --- onnxruntime/test/providers/checkers.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onnxruntime/test/providers/checkers.cc b/onnxruntime/test/providers/checkers.cc index 68dad482dc402..5b6807ee0c4af 100644 --- a/onnxruntime/test/providers/checkers.cc +++ b/onnxruntime/test/providers/checkers.cc @@ -190,7 +190,7 @@ struct TensorCheck { const auto size = actual.Shape().Size(); cur_expected = expected.Data(); cur_actual = actual.Data(); - double threshold; + double threshold = 0.0f; if (has_abs_err) { threshold = *(params.absolute_error); } @@ -221,7 +221,7 @@ struct TensorCheck { cur_expected = expected.Data(); cur_actual = actual.Data(); - double threshold; + double threshold = 0.0f; if (has_abs_err) { threshold = *(params.absolute_error); } From 2a90792bc96626513399c9573b77ba62b8194ed7 Mon Sep 17 00:00:00 2001 From: LiangGao Date: Tue, 2 Sep 2025 15:45:36 +0800 Subject: [PATCH 6/7] New changes after rebase --- onnxruntime/test/providers/cpu/tensor/cast_op_test.cc | 3 +++ onnxruntime/test/providers/cpu/tensor/concat_op_test.cc | 2 ++ 2 files changed, 5 insertions(+) diff --git a/onnxruntime/test/providers/cpu/tensor/cast_op_test.cc b/onnxruntime/test/providers/cpu/tensor/cast_op_test.cc index 68d4f3559504a..9e3ea63fef0ce 100644 --- a/onnxruntime/test/providers/cpu/tensor/cast_op_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/cast_op_test.cc @@ -841,6 +841,9 @@ TEST(CastOpTest, Int32ToInt4x2OddNumberOfElements) { } TEST(CastOpTest, Int32ToInt4x2EmptyTensor) { + if (DefaultOpenVINOExecutionProvider().get() != nullptr) { + GTEST_SKIP() << "The OpenVINO not support 0 size input"; + } // GIVEN const std::vector empty_shape{0}; const std::vector empty_input = {}; diff --git a/onnxruntime/test/providers/cpu/tensor/concat_op_test.cc b/onnxruntime/test/providers/cpu/tensor/concat_op_test.cc index b5e13c6377ccb..5f08b6df6785d 100644 --- a/onnxruntime/test/providers/cpu/tensor/concat_op_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/concat_op_test.cc @@ -73,6 +73,7 @@ TEST(ConcatOpTest, Concat1D_2) { test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, // TensorRT: no support for dynamic shape tensor kNnapiExecutionProvider, // NNAPI: concat does not support 0 size input + kOpenVINOExecutionProvider, // OpenVINO: does not support 0 size input kQnnExecutionProvider}); // QNN: not support dynamic shape tensor } @@ -118,6 +119,7 @@ TEST(ConcatOpTest, Concat2D_3) { test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, // TensorRT: no support for dynamic shape tensor kNnapiExecutionProvider, // NNAPI: concat does not support 0 size input + kOpenVINOExecutionProvider, // OpenVINO: does not support 0 size input kQnnExecutionProvider}); // QNN: not support dynamic shape tensor } From 105bb524473e6b4fbe6e1f26022dfb8b6faadfc8 Mon Sep 17 00:00:00 2001 From: LiangGao Date: Tue, 2 Sep 2025 15:47:29 +0800 Subject: [PATCH 7/7] Change code based on review --- onnxruntime/test/providers/checkers.cc | 1 - onnxruntime/test/providers/cpu/tensor/slice_op.test.cc | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/onnxruntime/test/providers/checkers.cc b/onnxruntime/test/providers/checkers.cc index 5b6807ee0c4af..c974ca6af8a5f 100644 --- a/onnxruntime/test/providers/checkers.cc +++ b/onnxruntime/test/providers/checkers.cc @@ -212,7 +212,6 @@ template <> struct TensorCheck { void operator()(const Tensor& expected, const Tensor& actual, const ValidateOutputParams& params, const std::string& /*provider_type*/) const { - ORT_UNUSED_PARAMETER(params); const bool has_abs_err = params.absolute_error.has_value(); Tensor expected_sorted, actual_sorted; const UInt4x2* cur_expected; diff --git a/onnxruntime/test/providers/cpu/tensor/slice_op.test.cc b/onnxruntime/test/providers/cpu/tensor/slice_op.test.cc index f531e2a7210c5..657f3fe9c127a 100644 --- a/onnxruntime/test/providers/cpu/tensor/slice_op.test.cc +++ b/onnxruntime/test/providers/cpu/tensor/slice_op.test.cc @@ -541,7 +541,7 @@ TEST(SliceTest, Slice1D_ReverseAllAxes_1) { } if (DefaultOpenVINOExecutionProvider().get() != nullptr) { - GTEST_SKIP() << "Skipping because of the following error: The input ends did not supported int max when step is negtive."; + GTEST_SKIP() << "Skipping because of the following error: The input ends do not support int max when step is negative."; } RunSliceTest({4},