8261235: C1 compilation fails with assert(res->vreg_number() == index) failed: conversion check

hotspot/src/share/vm/c1/c1_LIR.hpp

@@ -237,7 +237,7 @@ class LIR_OprDesc: public CompilationResourceObj {
     , is_xmm_bits    = 1
     , last_use_bits  = 1
     , is_fpu_stack_offset_bits = 1        // used in assertion checking on x86 for FPU stack slot allocation
-    , non_data_bits  = kind_bits + type_bits + size_bits + destroys_bits + last_use_bits +
+    , non_data_bits  = pointer_bits + kind_bits + type_bits + size_bits + destroys_bits + last_use_bits +
                        is_fpu_stack_offset_bits + virtual_bits + is_xmm_bits
     , data_bits      = BitsPerInt - non_data_bits
     , reg_bits       = data_bits / 2      // for two registers in one value encoding
@@ -661,6 +661,11 @@ class LIR_OprFact: public AllStatic {
 #endif // PPC
 
   static LIR_Opr virtual_register(int index, BasicType type) {
+    if (index > LIR_OprDesc::vreg_max) {
+      // Running out of virtual registers. Caller should bailout.
+      return illegalOpr;
+    }
+
     LIR_Opr res;
     switch (type) {
       case T_OBJECT: // fall through

hotspot/src/share/vm/c1/c1_LIRGenerator.cpp

@@ -1040,20 +1040,21 @@ void LIRGenerator::move_to_phi(ValueStack* cur_state) {
 
 
 LIR_Opr LIRGenerator::new_register(BasicType type) {
-  int vreg = _virtual_register_number;
-  // add a little fudge factor for the bailout, since the bailout is
-  // only checked periodically.  This gives a few extra registers to
-  // hand out before we really run out, which helps us keep from
-  // tripping over assertions.
-  if (vreg + 20 >= LIR_OprDesc::vreg_max) {
-    bailout("out of virtual registers");
-    if (vreg + 2 >= LIR_OprDesc::vreg_max) {
-      // wrap it around
+  int vreg_num = _virtual_register_number;
+  // Add a little fudge factor for the bailout since the bailout is only checked periodically. This allows us to hand out
+  // a few extra registers before we really run out which helps to avoid to trip over assertions.
+  if (vreg_num + 20 >= LIR_OprDesc::vreg_max) {
+    bailout("out of virtual registers in LIR generator");
+    if (vreg_num + 2 >= LIR_OprDesc::vreg_max) {
+    // Wrap it around and continue until bailout really happens to avoid hitting assertions.
       _virtual_register_number = LIR_OprDesc::vreg_base;
+      vreg_num = LIR_OprDesc::vreg_base;
     }
   }
   _virtual_register_number += 1;
-  return LIR_OprFact::virtual_register(vreg, type);
+  LIR_Opr vreg = LIR_OprFact::virtual_register(vreg_num, type);
+  assert(vreg != LIR_OprFact::illegal(), "ran out of virtual registers");
+  return vreg;
 }
 
 

hotspot/src/share/vm/c1/c1_LinearScan.cpp

@@ -3838,8 +3838,8 @@ void MoveResolver::insert_move(Interval* from_interval, Interval* to_interval) {
   assert(_insert_list != NULL && _insert_idx != -1, "must setup insert position first");
   assert(_insertion_buffer.lir_list() == _insert_list, "wrong insertion buffer");
 
-  LIR_Opr from_opr = LIR_OprFact::virtual_register(from_interval->reg_num(), from_interval->type());
-  LIR_Opr to_opr = LIR_OprFact::virtual_register(to_interval->reg_num(), to_interval->type());
+  LIR_Opr from_opr = get_virtual_register(from_interval);
+  LIR_Opr to_opr = get_virtual_register(to_interval);
 
   if (!_multiple_reads_allowed) {
     // the last_use flag is an optimization for FPU stack allocation. When the same
@@ -3857,12 +3857,27 @@ void MoveResolver::insert_move(LIR_Opr from_opr, Interval* to_interval) {
   assert(_insert_list != NULL && _insert_idx != -1, "must setup insert position first");
   assert(_insertion_buffer.lir_list() == _insert_list, "wrong insertion buffer");
 
-  LIR_Opr to_opr = LIR_OprFact::virtual_register(to_interval->reg_num(), to_interval->type());
+  LIR_Opr to_opr = get_virtual_register(to_interval);
   _insertion_buffer.move(_insert_idx, from_opr, to_opr);
 
   TRACE_LINEAR_SCAN(4, tty->print("MoveResolver: inserted move from constant "); from_opr->print(); tty->print_cr("  to %d (%d, %d)", to_interval->reg_num(), to_interval->assigned_reg(), to_interval->assigned_regHi()));
 }
 
+LIR_Opr MoveResolver::get_virtual_register(Interval* interval) {
+  // Add a little fudge factor for the bailout since the bailout is only checked periodically. This allows us to hand out
+  // a few extra registers before we really run out which helps to avoid to trip over assertions.
+  int reg_num = interval->reg_num();
+  if (reg_num + 20 >= LIR_OprDesc::vreg_max) {
+    _allocator->bailout("out of virtual registers in linear scan");
+    if (reg_num + 2 >= LIR_OprDesc::vreg_max) {
+      // Wrap it around and continue until bailout really happens to avoid hitting assertions.
+      reg_num = LIR_OprDesc::vreg_base;
+    }
+  }
+  LIR_Opr vreg = LIR_OprFact::virtual_register(reg_num, interval->type());
+  assert(vreg != LIR_OprFact::illegal(), "ran out of virtual registers");
+  return vreg;
+}
 
 void MoveResolver::resolve_mappings() {
   TRACE_LINEAR_SCAN(4, tty->print_cr("MoveResolver: resolving mappings for Block B%d, index %d", _insert_list->block() != NULL ? _insert_list->block()->block_id() : -1, _insert_idx));

hotspot/src/share/vm/c1/c1_LinearScan.hpp

@@ -444,6 +444,7 @@ class MoveResolver: public StackObj {
   void append_insertion_buffer();
   void insert_move(Interval* from_interval, Interval* to_interval);
   void insert_move(LIR_Opr from_opr, Interval* to_interval);
+  LIR_Opr get_virtual_register(Interval* interval);
 
   DEBUG_ONLY(void verify_before_resolve();)
   void resolve_mappings();