[SPARK-30515][SQL] Refactor SimplifyBinaryComparison to reduce the time complexity

### What changes were proposed in this pull request?

The changes in the rule `SimplifyBinaryComparison` from #27008 could bring performance regression in the optimizer when there are a large set of filter conditions.

We need to improve the implementation and reduce the time complexity.

### Why are the changes needed?

Need to fix the potential performance regression in the optimizer.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

Existing unit tests.
Also run a micor benchmark in `BinaryComparisonSimplificationSuite`
```
object Optimize extends RuleExecutor[LogicalPlan] {
    val batches =
      Batch("Constant Folding", FixedPoint(50),
        SimplifyBinaryComparison) :: Nil
  }

test("benchmark") {
  val a = Symbol("a")
  val condition = (1 to 500).map(i => EqualTo(a, a)).reduceLeft(And)
  val finalCondition = And(condition, IsNotNull(a))
  val plan = nullableRelation.where(finalCondition).analyze
  val start = System.nanoTime()
  Optimize.execute(plan)
  println((System.nanoTime() - start) /1000000)
}
```

Before the changes: 2507ms
After the changes: 3ms

Closes #27212 from gengliangwang/SimplifyBinaryComparison.

Authored-by: Gengliang Wang <[email protected]>
Signed-off-by: Takeshi Yamamuro <[email protected]>

Author: gengliangwang

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala

@@ -410,37 +410,38 @@ object SimplifyBinaryComparison
   extends Rule[LogicalPlan] with PredicateHelper with ConstraintHelper {
 
   private def canSimplifyComparison(
-      plan: LogicalPlan, left: Expression, right: Expression): Boolean = {
+      left: Expression,
+      right: Expression,
+      notNullExpressions: => ExpressionSet): Boolean = {
     if (left.semanticEquals(right)) {
-      if (!left.nullable && !right.nullable) {
-        true
-      } else {
-        // We do more checks for non-nullable cases
-        plan match {
-          case Filter(fc, _) =>
-            splitConjunctivePredicates(fc).exists { condition =>
-              condition.semanticEquals(IsNotNull(left))
-            }
-          case _ => false
-        }
-      }
+      (!left.nullable && !right.nullable) || notNullExpressions.contains(left)
     } else {
       false
     }
   }
 
   def apply(plan: LogicalPlan): LogicalPlan = plan transform {
-    case l: LogicalPlan => l transformExpressionsUp {
-      // True with equality
-      case a EqualNullSafe b if a.semanticEquals(b) => TrueLiteral
-      case a EqualTo b if canSimplifyComparison(l, a, b) => TrueLiteral
-      case a GreaterThanOrEqual b if canSimplifyComparison(l, a, b) => TrueLiteral
-      case a LessThanOrEqual b if canSimplifyComparison(l, a, b) => TrueLiteral
-
-      // False with inequality
-      case a GreaterThan b if canSimplifyComparison(l, a, b) => FalseLiteral
-      case a LessThan b if canSimplifyComparison(l, a, b) => FalseLiteral
-    }
+    case l: LogicalPlan =>
+      lazy val notNullExpressions = ExpressionSet(l match {
+        case Filter(fc, _) =>
+          splitConjunctivePredicates(fc).collect {
+            case i: IsNotNull => i.child
+          }
+        case _ => Seq.empty
+      })
+
+      l transformExpressionsUp {
+        // True with equality
+        case a EqualNullSafe b if a.semanticEquals(b) => TrueLiteral
+        case a EqualTo b if canSimplifyComparison(a, b, notNullExpressions) => TrueLiteral
+        case a GreaterThanOrEqual b if canSimplifyComparison(a, b, notNullExpressions) =>
+          TrueLiteral
+        case a LessThanOrEqual b if canSimplifyComparison(a, b, notNullExpressions) => TrueLiteral
+
+        // False with inequality
+        case a GreaterThan b if canSimplifyComparison(a, b, notNullExpressions) => FalseLiteral
+        case a LessThan b if canSimplifyComparison(a, b, notNullExpressions) => FalseLiteral
+      }
   }
 }