8372700: [lworld] compiler/c2/irTests/stable/* fail with --enable-preview

src/hotspot/share/ci/ciArray.hpp

@@ -68,7 +68,7 @@ class ciArray : public ciObject {
 
   // Current value of an element at the specified offset.
   // Returns T_ILLEGAL if there is no element at the given offset.
-  ciConstant element_value_by_offset(intptr_t element_offset);
+  virtual ciConstant element_value_by_offset(intptr_t element_offset);
 
   // What kind of ciObject is this?
   bool is_array()        { return true; }

src/hotspot/share/ci/ciArrayKlass.hpp

@@ -49,8 +49,8 @@ class ciArrayKlass : public ciKlass {
 
 public:
   jint dimension() { return _dimension; }
-  ciType* element_type();       // JLS calls this the "component type"
-  ciType* base_element_type();  // JLS calls this the "element type"
+  ciType* element_type();       // JLS calls this the "component type", (T[] for T[][])
+  ciType* base_element_type();  // JLS calls this the "element type", (T for T[][])
   bool is_leaf_type();          // No subtypes of this array type.
 
   bool is_refined() const { return !is_type_array_klass() && properties() != ArrayKlass::INVALID; }

src/hotspot/share/ci/ciClassList.hpp

@@ -53,6 +53,7 @@ class     ciMethodType;
 class     ciArray;
 class       ciObjArray;
 class       ciTypeArray;
+class       ciFlatArray;
 class   ciSymbol;
 class   ciMetadata;
 class   ciMethod;
@@ -108,6 +109,7 @@ friend class ciTypeEntries;            \
 friend class ciSpeculativeTrapData;    \
 friend class ciSymbol;                 \
 friend class ciArray;                  \
+friend class ciFlatArray;              \
 friend class ciObjArray;               \
 friend class ciMetadata;               \
 friend class ciReplay;                 \

src/hotspot/share/ci/ciConstant.cpp

@@ -33,6 +33,7 @@
 
 // ------------------------------------------------------------------
 // ciConstant::is_null_or_zero
+// This assumes `this->is_valid()`, otherwise, `as_object` will assert.
 bool ciConstant::is_null_or_zero() const {
   if (!is_java_primitive(basic_type())) {
     return as_object()->is_null_object();

src/hotspot/share/ci/ciField.cpp

@@ -238,7 +238,7 @@ ciField::ciField(ciField* declared_field, ciField* subfield) {
 
   _signature = subfield->_signature;
   _type = subfield->_type;
-  _is_constant = declared_field->is_strict() && declared_field->is_final();
+  _is_constant = (declared_field->is_strict() && declared_field->is_final()) || declared_field->is_constant();
   _known_to_link_with_put = subfield->_known_to_link_with_put;
   _known_to_link_with_get = subfield->_known_to_link_with_get;
   _constant_value = ciConstant();
@@ -265,7 +265,7 @@ ciField::ciField(ciField* declared_field) {
   _signature = ciSymbols::bool_signature();
   _type = ciType::make(T_BOOLEAN);
 
-  _is_constant = declared_field->is_strict() && declared_field->is_final();
+  _is_constant = (declared_field->is_strict() && declared_field->is_final()) || declared_field->is_constant();
   _known_to_link_with_put = nullptr;
   _known_to_link_with_get = nullptr;
   _constant_value = ciConstant();

src/hotspot/share/ci/ciFlatArray.cpp

@@ -0,0 +1,155 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "ci/ciArray.hpp"
+#include "ci/ciConstant.hpp"
+#include "ci/ciField.hpp"
+#include "ci/ciFlatArray.hpp"
+#include "ci/ciInlineKlass.hpp"
+#include "ci/ciUtilities.inline.hpp"
+#include "oops/oop.inline.hpp"
+
+ciConstant ciFlatArray::null_marker_of_element_by_offset_impl(arrayOop ary, int index) {
+  if (ary == nullptr) {
+    return ciConstant();
+  }
+  assert(ary->is_array(), "");
+  if (index < 0 || index >= ary->length()) {
+    return ciConstant();
+  }
+    assert(ary->is_objArray(), "");
+    flatArrayOop objary = (flatArrayOop) ary;
+    jboolean elem = objary->null_marker_of_obj_at(index);
+    return ciConstant(T_BOOLEAN, elem);
+}
+
+ciConstant ciFlatArray::check_constant_null_marker_cache(int off) {
+  if (_constant_null_markers != nullptr) {
+    for (int i = 0; i < _constant_null_markers->length(); ++i) {
+      ConstantValue cached_val = _constant_null_markers->at(i);
+      if (cached_val.off() == off) {
+        return cached_val.value();
+      }
+    }
+  }
+  return ciConstant();
+}
+
+void ciFlatArray::add_to_constant_null_marker_cache(int off, ciConstant val) {
+  assert(val.is_valid(), "value must be valid");
+  assert(!check_constant_value_cache(off, val.basic_type()).is_valid(), "duplicate");
+  if (_constant_null_markers == nullptr) {
+    Arena* arena = CURRENT_ENV->arena();
+    _constant_null_markers = new (arena) GrowableArray<ConstantValue>(arena, 1, 0, ConstantValue());
+  }
+  _constant_null_markers->append(ConstantValue(off, val));
+}
+
+// Current value of an element.
+// Returns T_ILLEGAL if there is no element at the given index.
+ciConstant ciFlatArray::null_marker_of_element_by_index(int index) {
+  ciConstant value = check_constant_null_marker_cache(index);
+  if (value.is_valid()) {
+    return value;
+  }
+  GUARDED_VM_ENTRY(
+      value = null_marker_of_element_by_offset_impl(get_arrayOop(), index);)
+  add_to_constant_null_marker_cache(index, value);
+  return value;
+}
+
+ciConstant ciFlatArray::null_marker_of_element_by_offset(intptr_t element_offset) {
+  FlatArrayKlass* faklass;
+  GUARDED_VM_ENTRY(faklass = FlatArrayKlass::cast(get_arrayOop()->klass());)
+  int lh = faklass->layout_helper();
+  int shift = Klass::layout_helper_log2_element_size(lh);
+  intptr_t header = arrayOopDesc::base_offset_in_bytes(T_FLAT_ELEMENT);
+  intptr_t index = (element_offset - header) >> shift;
+  intptr_t offset = header + (index << shift);
+  if (offset != element_offset || index != (jint) index || index < 0 || index >= length()) {
+    return ciConstant();
+  }
+  return null_marker_of_element_by_index((jint) index);
+}
+
+ciConstant ciFlatArray::element_value_by_offset(intptr_t element_offset) {
+  FlatArrayKlass* faklass;
+  GUARDED_VM_ENTRY(faklass = FlatArrayKlass::cast(get_arrayOop()->klass());)
+  int lh = faklass->layout_helper();
+  int shift = Klass::layout_helper_log2_element_size(lh);
+  intptr_t header = arrayOopDesc::base_offset_in_bytes(T_FLAT_ELEMENT);
+  intptr_t index = (element_offset - header) >> shift;
+  intptr_t offset = header + (index << shift);
+  if (offset != element_offset || index != (jint) index || index < 0 || index >= length()) {
+    return ciConstant();
+  }
+  return element_value((jint) index);
+}
+
+ciConstant ciFlatArray::field_value_by_offset(intptr_t field_offset) {
+  ciInlineKlass* elt_type = element_type()->as_inline_klass();
+  FlatArrayKlass* faklass;
+  GUARDED_VM_ENTRY(faklass = FlatArrayKlass::cast(get_arrayOop()->klass());)
+  int lh = faklass->layout_helper();
+  int shift = Klass::layout_helper_log2_element_size(lh);
+  intptr_t header = arrayOopDesc::base_offset_in_bytes(T_FLAT_ELEMENT);
+  intptr_t index = (field_offset - header) >> shift;
+  intptr_t element_offset = header + (index << shift);
+  int field_offset_in_element = (int)(field_offset - element_offset);
+  ciField* field = elt_type->get_field_by_offset(elt_type->payload_offset() + field_offset_in_element, false);
+  if (field == nullptr) {
+    if (field_offset_in_element != elt_type->null_marker_offset_in_payload()) {
+      return ciConstant();
+    }
+  }
+
+  if (index != (jint) index || index < 0 || index >= length()) {
+    return ciConstant();
+  }
+  ciConstant elt = field_value((jint) index, field);
+
+  return elt;
+}
+
+ciConstant ciFlatArray::field_value(int index, ciField* field) {
+  BasicType elembt = element_basic_type();
+  ciConstant value = check_constant_value_cache(index, elembt);
+  if (value.is_valid()) {
+    if (field == nullptr) {
+      return value.as_object()->as_instance()->null_marker_value();
+    }
+    return value.as_object()->as_instance()->field_value(field);
+  }
+  GUARDED_VM_ENTRY(
+    value = element_value_impl(T_OBJECT, get_arrayOop(), index);
+  )
+
+  add_to_constant_value_cache(index, value);
+
+  if (field == nullptr) {
+    return value.as_object()->as_instance()->null_marker_value();
+  }
+  return value.as_object()->as_instance()->field_value(field);
+}
+

src/hotspot/share/ci/ciFlatArray.hpp

@@ -38,10 +38,26 @@ class ciFlatArray : public ciArray {
 protected:
   ciFlatArray(flatArrayHandle h_o) : ciArray(h_o) {}
 
-  const char* type_string() { return "ciFlatArray"; }
+  const char* type_string() override { return "ciFlatArray"; }
 
 public:
-  bool is_flat()      { return true; }
+  bool is_flat_array() const override { return true; }
+  bool is_flat() override { return true; }
+
+  // Current value of an element at the specified offset.
+  // Returns T_ILLEGAL if there is no element at the given offset.
+  ciConstant element_value_by_offset(intptr_t element_offset) override;
+  ciConstant field_value_by_offset(intptr_t field_offset);
+  ciConstant field_value(int index, ciField* field);
+  ciConstant null_marker_of_element_by_offset(intptr_t element_offset);
+  ciConstant null_marker_of_element_by_index(int index);
+
+private:
+  ciConstant null_marker_of_element_by_offset_impl(arrayOop ary, int index);
+  ciConstant check_constant_null_marker_cache(int off);
+  void add_to_constant_null_marker_cache(int off, ciConstant val);
+
+  GrowableArray<ConstantValue>* _constant_null_markers = nullptr;
 };
 
 #endif // SHARE_VM_CI_CIFLATARRAY_HPP

src/hotspot/share/ci/ciInstance.cpp

@@ -24,6 +24,7 @@
 
 #include "ci/ciConstant.hpp"
 #include "ci/ciField.hpp"
+#include "ci/ciInlineKlass.hpp"
 #include "ci/ciInstance.hpp"
 #include "ci/ciInstanceKlass.hpp"
 #include "ci/ciNullObject.hpp"
@@ -100,6 +101,15 @@ ciConstant ciInstance::field_value_impl(BasicType field_btype, int offset) {
   return value;
 }
 
+// Constant value of the null marker.
+ciConstant ciInstance::null_marker_value() {
+  if (!klass()->is_inlinetype()) {
+    return ciConstant();
+  }
+  ciInlineKlass* ik = klass()->as_inline_klass();
+  return field_value_impl(T_BOOLEAN, ik->null_marker_offset_in_payload() + ik->payload_offset());
+}
+
 // ------------------------------------------------------------------
 // ciInstance::field_value
 //

src/hotspot/share/ci/ciInstance.hpp

@@ -65,6 +65,7 @@ class ciInstance : public ciObject {
 
   // Constant value of a field at the specified offset.
   ciConstant field_value_by_offset(int field_offset);
+  ciConstant null_marker_value();
 
   ciKlass* java_lang_Class_klass();
   char* java_lang_String_str(char* buf, size_t buflen);

src/hotspot/share/ci/ciObject.hpp

@@ -59,6 +59,7 @@ class ciObject : public ciBaseObject {
   jobject  _handle;
   ciKlass* _klass;
 
+protected:
   // Cache constant value lookups to ensure that consistent values are observed during compilation.
   class ConstantValue {
     private:
@@ -73,6 +74,7 @@ class ciObject : public ciBaseObject {
       ciConstant value() const { return _value; }
   };
 
+private:
   GrowableArray<ConstantValue>* _constant_values = nullptr;
 
 protected:
@@ -128,6 +130,7 @@ class ciObject : public ciBaseObject {
   virtual bool is_array()                   { return false; }
   virtual bool is_obj_array()               { return false; }
   virtual bool is_type_array()              { return false; }
+  virtual bool is_flat_array()        const { return false; }
   virtual bool is_native_entry_point()const { return false; }
 
   // Is this a type or value which has no associated class?
@@ -182,6 +185,10 @@ class ciObject : public ciBaseObject {
     assert(is_type_array(), "bad cast");
     return (ciTypeArray*)this;
   }
+  ciFlatArray* as_flat_array() {
+    assert(is_flat_array(), "bad cast");
+    return (ciFlatArray*)this;
+  }
 
   // Print debugging output about this ciObject.
   void print(outputStream* st);

src/hotspot/share/oops/flatArrayOop.hpp

@@ -42,6 +42,8 @@ class flatArrayOopDesc : public objArrayOopDesc {
 
   inline oop obj_at(int index) const;
   inline oop obj_at(int index, TRAPS) const;
+  inline jboolean null_marker_of_obj_at(int index) const;
+  inline jboolean null_marker_of_obj_at(int index, TRAPS) const;
   inline void obj_at_put(int index, oop value);
   inline void obj_at_put(int index, oop value, TRAPS);
 

src/hotspot/share/oops/flatArrayOop.inline.hpp

@@ -66,6 +66,21 @@ inline oop flatArrayOopDesc::obj_at(int index, TRAPS) const {
   return res;
 }
 
+inline jboolean flatArrayOopDesc::null_marker_of_obj_at(int index) const {
+  EXCEPTION_MARK;
+  return null_marker_of_obj_at(index, THREAD);
+}
+
+inline jboolean flatArrayOopDesc::null_marker_of_obj_at(int index, TRAPS) const {
+  assert(is_within_bounds(index), "index %d out of bounds %d", index, length());
+  FlatArrayKlass* faklass = FlatArrayKlass::cast(klass());
+  InlineKlass* vk = InlineKlass::cast(faklass->element_klass());
+  char* this_oop = (char*) (oopDesc*) this;
+  char* val = (char*) value_at_addr(index, faklass->layout_helper());
+  ptrdiff_t offset = val - this_oop + (ptrdiff_t)vk->null_marker_offset_in_payload();
+  return bool_field(offset);
+}
+
 inline void flatArrayOopDesc::obj_at_put(int index, oop value) {
   EXCEPTION_MARK;                                 // What if the caller is not a Java Thread?
   obj_at_put(index, value, THREAD);

src/hotspot/share/opto/compile.cpp

@@ -24,6 +24,7 @@
 
 #include "asm/macroAssembler.hpp"
 #include "asm/macroAssembler.inline.hpp"
+#include "ci/ciFlatArray.hpp"
 #include "ci/ciInlineKlass.hpp"
 #include "ci/ciReplay.hpp"
 #include "classfile/javaClasses.hpp"
@@ -2091,7 +2092,42 @@ void Compile::process_flat_accesses(PhaseIterGVN& igvn) {
     Node* n = _flat_access_nodes.at(i);
     assert(n != nullptr, "unexpected nullptr");
     if (n->is_LoadFlat()) {
-      n->as_LoadFlat()->expand_atomic(igvn);
+      LoadFlatNode* loadn = n->as_LoadFlat();
+      // Expending a flat load atomically means that we get a chunk of memory spanning multiple fields
+      // that we chop with bitwise operations. That is too subtle for some optimizations, especially
+      // constant folding when fields are constant. But if the flattened field being accessed is read-only
+      // then no concurrent writes can happen and non-atomic loads are fine, allowing better optimizations.
+      // A way for fields to be read-only is to be stable and already initialized. Here, we check if the
+      // field being accessed is stable, and if the null marker of the field/array element is non-zero.
+      // If so, we know that the stable value was initialized away from the default value (null), and
+      // that we can assume it's read-only, so can the load can be performed non-atomically.
+      bool non_atomic_is_fine = false;
+      if (FoldStableValues) {
+        const TypeOopPtr* base_type = igvn.type(loadn->base())->isa_oopptr();
+        ciObject* oop = base_type->const_oop();
+        ciInstance* holder = oop != nullptr && oop->is_instance() ? oop->as_instance() : nullptr;
+        ciArray* array = oop != nullptr && oop->is_array() ? oop->as_array() : nullptr;
+        int off = igvn.type(loadn->ptr())->isa_ptr()->offset();
+
+        if (holder != nullptr) {
+          ciKlass* klass = holder->klass();
+          ciInstanceKlass* iklass = klass->as_instance_klass();
+          const ciField* field = iklass->get_non_flat_field_by_offset(off);
+          ciField* nm_field = iklass->get_field_by_offset(field->null_marker_offset(), false);
+          ciConstant cst = nm_field != nullptr ? holder->field_value(nm_field) : ciConstant() /* invalid */;
+          non_atomic_is_fine = field->is_stable() && cst.is_valid() && cst.as_boolean();
+        } else if (array != nullptr) {
+          const TypeAryPtr* aryptr = base_type->is_aryptr();
+          ciConstant elt = ((ciFlatArray*)array)->null_marker_of_element_by_offset(off);
+          non_atomic_is_fine = aryptr->is_stable() && elt.is_valid() && !elt.is_null_or_zero();
+        }
+      }
+
+      if (non_atomic_is_fine) {
+        loadn->expand_non_atomic(igvn);
+      } else {
+        loadn->expand_atomic(igvn);
+      }
     } else {
       n->as_StoreFlat()->expand_atomic(igvn);
     }