[EarlyCSE] Do not CSE convergent calls with memory effects

D149348 did this for readnone calls, which are handled by SimpleValue.
This patch does the same for all other CSEable calls, which are handled
by CallValue.

Differential Revision: https://reviews.llvm.org/D153151

Change-Id: Ied78587d48f12d8735f789a73f75c1c1c010618d

Author: jayfoad

llvm/lib/Transforms/Scalar/EarlyCSE.cpp

@@ -218,6 +218,19 @@ static bool matchSelectWithOptionalNotCond(Value *V, Value *&Cond, Value *&A,
   return true;
 }
 
+static unsigned hashCallInst(CallInst *CI) {
+  // Don't CSE convergent calls in different basic blocks, because they
+  // implicitly depend on the set of threads that is currently executing.
+  if (CI->isConvergent()) {
+    return hash_combine(
+        CI->getOpcode(), CI->getParent(),
+        hash_combine_range(CI->value_op_begin(), CI->value_op_end()));
+  }
+  return hash_combine(
+      CI->getOpcode(),
+      hash_combine_range(CI->value_op_begin(), CI->value_op_end()));
+}
+
 static unsigned getHashValueImpl(SimpleValue Val) {
   Instruction *Inst = Val.Inst;
   // Hash in all of the operands as pointers.
@@ -320,11 +333,8 @@ static unsigned getHashValueImpl(SimpleValue Val) {
 
   // Don't CSE convergent calls in different basic blocks, because they
   // implicitly depend on the set of threads that is currently executing.
-  if (CallInst *CI = dyn_cast<CallInst>(Inst); CI && CI->isConvergent()) {
-    return hash_combine(
-        Inst->getOpcode(), Inst->getParent(),
-        hash_combine_range(Inst->value_op_begin(), Inst->value_op_end()));
-  }
+  if (CallInst *CI = dyn_cast<CallInst>(Inst))
+    return hashCallInst(CI);
 
   // Mix in the opcode.
   return hash_combine(
@@ -524,15 +534,21 @@ unsigned DenseMapInfo<CallValue>::getHashValue(CallValue Val) {
   Instruction *Inst = Val.Inst;
 
   // Hash all of the operands as pointers and mix in the opcode.
-  return hash_combine(
-      Inst->getOpcode(),
-      hash_combine_range(Inst->value_op_begin(), Inst->value_op_end()));
+  return hashCallInst(cast<CallInst>(Inst));
 }
 
 bool DenseMapInfo<CallValue>::isEqual(CallValue LHS, CallValue RHS) {
-  Instruction *LHSI = LHS.Inst, *RHSI = RHS.Inst;
   if (LHS.isSentinel() || RHS.isSentinel())
-    return LHSI == RHSI;
+    return LHS.Inst == RHS.Inst;
+
+  CallInst *LHSI = cast<CallInst>(LHS.Inst);
+  CallInst *RHSI = cast<CallInst>(RHS.Inst);
+
+  // Convergent calls implicitly depend on the set of threads that is
+  // currently executing, so conservatively return false if they are in
+  // different basic blocks.
+  if (LHSI->isConvergent() && LHSI->getParent() != RHSI->getParent())
+      return false;
 
   return LHSI->isIdenticalTo(RHSI);
 }

llvm/test/Transforms/EarlyCSE/AMDGPU/convergent-call.ll

@@ -0,0 +1,84 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt < %s -S -mtriple=amdgcn-- -passes=early-cse -earlycse-debug-hash | FileCheck %s
+
+; Should not CSE calls marked as convergent, even if the callee is not convergent.
+
+define i32 @test_read_register(i32 %cond) {
+; CHECK-LABEL: define i32 @test_read_register
+; CHECK-SAME: (i32 [[COND:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X1:%.*]] = call i32 @llvm.read_register.i32(metadata [[META0:![0-9]+]]) #[[ATTR2:[0-9]+]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[COND]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF:%.*]], label [[END:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    [[Y1:%.*]] = call i32 @llvm.read_register.i32(metadata [[META0]]) #[[ATTR2]]
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[Y2:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[Y1]], [[IF]] ]
+; CHECK-NEXT:    [[RET:%.*]] = add i32 [[X1]], [[Y2]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+entry:
+  ; %x = ballot operation over all lanes.
+  %x1 = call i32 @llvm.read_register.i32(metadata !{!"exec_lo"}) convergent
+  %cmp = icmp eq i32 %cond, 0
+  br i1 %cmp, label %if, label %end
+
+if:
+  ; %y = ballot operation over lanes satisfying %cond.
+  %y1 = call i32 @llvm.read_register.i32(metadata !{!"exec_lo"}) convergent
+  br label %end
+
+end:
+  %y2 = phi i32 [0, %entry], [%y1, %if]
+  %ret = add i32 %x1, %y2
+  ret i32 %ret
+}
+
+define i32 @test_read_register_samebb(i32 %cond) {
+; CHECK-LABEL: define i32 @test_read_register_samebb
+; CHECK-SAME: (i32 [[COND:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X:%.*]] = call i32 @llvm.read_register.i32(metadata [[META0]]) #[[ATTR2]]
+; CHECK-NEXT:    [[RET:%.*]] = add i32 [[X]], [[X]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+entry:
+  %x = call i32 @llvm.read_register.i32(metadata !{!"exec_lo"}) convergent
+  %y = call i32 @llvm.read_register.i32(metadata !{!"exec_lo"}) convergent
+  %ret = add i32 %x, %y
+  ret i32 %ret
+}
+
+define i1 @test_live_mask(i32 %cond) {
+; CHECK-LABEL: define i1 @test_live_mask
+; CHECK-SAME: (i32 [[COND:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X1:%.*]] = call i1 @llvm.amdgcn.live.mask() #[[ATTR2]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[COND]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF:%.*]], label [[END:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    [[Y1:%.*]] = call i1 @llvm.amdgcn.live.mask() #[[ATTR2]]
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[Y2:%.*]] = phi i1 [ false, [[ENTRY:%.*]] ], [ [[Y1]], [[IF]] ]
+; CHECK-NEXT:    [[RET:%.*]] = add i1 [[X1]], [[Y2]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+entry:
+  %x1 = call i1 @llvm.amdgcn.live.mask() convergent
+  %cmp = icmp eq i32 %cond, 0
+  br i1 %cmp, label %if, label %end
+
+if:
+  %y1 = call i1 @llvm.amdgcn.live.mask() convergent
+  br label %end
+
+end:
+  %y2 = phi i1 [0, %entry], [%y1, %if]
+  %ret = add i1 %x1, %y2
+  ret i1 %ret
+}
+
+declare i32 @llvm.read_register.i32(metadata)
+declare i1 @llvm.amdgcn.live.mask()