Adding flattening of strict final fields

src/hotspot/share/classfile/classFileParser.cpp

@@ -5518,7 +5518,8 @@ void ClassFileParser::fill_instance_klass(InstanceKlass* ik,
     vk->set_non_atomic_size_in_bytes(_layout_info->_non_atomic_size_in_bytes);
     vk->set_non_atomic_alignment(_layout_info->_non_atomic_alignment);
     vk->set_atomic_size_in_bytes(_layout_info->_atomic_layout_size_in_bytes);
-    vk->set_nullable_size_in_bytes(_layout_info->_nullable_layout_size_in_bytes);
+    vk->set_nullable_atomic_size_in_bytes(_layout_info->_nullable_atomic_layout_size_in_bytes);
+    vk->set_nullable_non_atomic_size_in_bytes(_layout_info->_nullable_non_atomic_layout_size_in_bytes);
     vk->set_null_marker_offset(_layout_info->_null_marker_offset);
     vk->set_null_reset_value_offset(_layout_info->_null_reset_value_offset);
     if (_layout_info->_is_empty_inline_klass) vk->set_is_empty_inline_type();

src/hotspot/share/classfile/classFileParser.hpp

@@ -83,6 +83,8 @@ class FieldLayoutInfo : public ResourceObj {
   int _non_atomic_size_in_bytes;
   int _non_atomic_alignment;
   int _atomic_layout_size_in_bytes;
+  int _nullable_atomic_layout_size_in_bytes;
+  int _nullable_non_atomic_layout_size_in_bytes;
   int _nullable_layout_size_in_bytes;
   int _null_marker_offset;
   int _null_reset_value_offset;

src/hotspot/share/classfile/fieldLayoutBuilder.cpp

@@ -38,7 +38,12 @@
 #include "utilities/powerOfTwo.hpp"
 
 static LayoutKind field_layout_selection(FieldInfo field_info, Array<InlineLayoutInfo>* inline_layout_info_array,
-                                         bool use_atomic_flat) {
+                                         bool can_use_atomic_flat) {
+
+  // The can_use_atomic_flat argument indicates if an atomic flat layout can be used for this field.
+  // This argument will be false if the container is a loosely consistent value class. Using an atomic layout
+  // in a container that has no atomicity guarantee creates a risk to see this field's value be subject to
+  // tearing even if the field's class was declared atomic (non loosely consistent).
 
   if (!UseFieldFlattening) {
     return LayoutKind::REFERENCE;
@@ -66,13 +71,23 @@ static LayoutKind field_layout_selection(FieldInfo field_info, Array<InlineLayou
     assert(field_info.access_flags().is_strict(), "null-free fields must be strict");
     if (vk->must_be_atomic() || AlwaysAtomicAccesses) {
       if (vk->is_naturally_atomic() && vk->has_non_atomic_layout()) return LayoutKind::NULL_FREE_NON_ATOMIC_FLAT;
-      return (vk->has_atomic_layout() && use_atomic_flat) ? LayoutKind::NULL_FREE_ATOMIC_FLAT : LayoutKind::REFERENCE;
+      return (vk->has_atomic_layout() && can_use_atomic_flat) ? LayoutKind::NULL_FREE_ATOMIC_FLAT : LayoutKind::REFERENCE;
     } else {
       return vk->has_non_atomic_layout() ? LayoutKind::NULL_FREE_NON_ATOMIC_FLAT : LayoutKind::REFERENCE;
     }
   } else {
+    // To preserve the consistency between the null-marker and the field content, the NULLABLE_NON_ATOMIC_FLAT
+    // can only be used in containers that have atomicity quarantees (can_use_atomic_flat argument set to true)
+    if (field_info.access_flags().is_strict() && field_info.access_flags().is_final() && can_use_atomic_flat) {
+      if (vk->has_nullable_non_atomic_layout()) return LayoutKind::NULLABLE_NON_ATOMIC_FLAT;
+    }
+    // Another special case where NULLABLE_NON_ATOMIC_FLAT can be used: nullable empty values, because the
+    // payload of those values contains only the null-marker
+    if (vk->is_empty_inline_type() && vk->has_nullable_non_atomic_layout()) {
+      return LayoutKind::NULLABLE_NON_ATOMIC_FLAT;
+    }
     if (UseNullableValueFlattening && vk->has_nullable_atomic_layout()) {
-      return use_atomic_flat ? LayoutKind::NULLABLE_ATOMIC_FLAT : LayoutKind::REFERENCE;
+      return can_use_atomic_flat ? LayoutKind::NULLABLE_ATOMIC_FLAT : LayoutKind::REFERENCE;
     } else {
       return LayoutKind::REFERENCE;
     }
@@ -92,7 +107,11 @@ static void get_size_and_alignment(InlineKlass* vk, LayoutKind kind, int* size,
     case LayoutKind::NULLABLE_ATOMIC_FLAT:
       *size = vk->nullable_atomic_size_in_bytes();
       *alignment = *size;
-    break;
+      break;
+    case LayoutKind::NULLABLE_NON_ATOMIC_FLAT:
+      *size = vk->nullable_non_atomic_size_in_bytes();
+      *alignment = vk->non_atomic_alignment();
+      break;
     default:
       ShouldNotReachHere();
   }
@@ -736,7 +755,8 @@ FieldLayoutBuilder::FieldLayoutBuilder(const Symbol* classname, ClassLoaderData*
   _non_atomic_layout_size_in_bytes(-1),
   _non_atomic_layout_alignment(-1),
   _atomic_layout_size_in_bytes(-1),
-  _nullable_layout_size_in_bytes(-1),
+  _nullable_atomic_layout_size_in_bytes(-1),
+  _nullable_non_atomic_layout_size_in_bytes(-1),
   _fields_size_sum(0),
   _declared_non_static_fields_count(0),
   _has_non_naturally_atomic_fields(false),
@@ -1139,10 +1159,9 @@ void FieldLayoutBuilder::compute_inline_class_layout() {
       }
     }
 
-    // Next step is the nullable layout: the layout must include a null marker and must also be atomic
-    if (UseNullableValueFlattening) {
+    // Next step is the nullable layouts: they must include a null marker
+    if (UseNullableValueFlattening || UseNullableNonAtomicValueFlattening) {
       // Looking if there's an empty slot inside the layout that could be used to store a null marker
-      // FIXME: could it be possible to re-use the .empty field as a null marker for empty values?
       LayoutRawBlock* b = _layout->first_field_block();
       assert(b != nullptr, "A concrete value class must have at least one (possible dummy) field");
       int null_marker_offset = -1;
@@ -1171,20 +1190,28 @@ void FieldLayoutBuilder::compute_inline_class_layout() {
           null_marker_offset = marker->offset();
         }
       }
-
-      // Now that the null marker is there, the size of the nullable layout must computed (remember, must be atomic too)
+      assert(null_marker_offset != -1, "Sanity check");
+      // Now that the null marker is there, the size of the nullable layout must computed
       int new_raw_size = _layout->last_block()->offset() - _layout->first_field_block()->offset();
-      int nullable_size = round_up_power_of_2(new_raw_size);
-      if (nullable_size <= (int)MAX_ATOMIC_OP_SIZE) {
-        _nullable_layout_size_in_bytes = nullable_size;
+      if (UseNullableNonAtomicValueFlattening) {
+        _nullable_non_atomic_layout_size_in_bytes = new_raw_size;
         _null_marker_offset = null_marker_offset;
-      } else {
+        _non_atomic_layout_alignment = _payload_alignment;
+      }
+      if (UseNullableValueFlattening) {
+        // For the nullable atomic layout, the size mut be compatible with the platform capabilities
+        int nullable_atomic_size = round_up_power_of_2(new_raw_size);
+        if (nullable_atomic_size <= (int)MAX_ATOMIC_OP_SIZE) {
+          _nullable_atomic_layout_size_in_bytes = nullable_atomic_size;
+          _null_marker_offset = null_marker_offset;
+        }
+      }
+      if (_null_marker_offset == -1) { // No nullable layout has been accepted
         // If the nullable layout is rejected, the NULL_MARKER block should be removed
         // from the layout, otherwise it will appear anyway if the layout is printer
         if (!_is_empty_inline_class) {  // empty values don't have a dedicated NULL_MARKER block
           _layout->remove_null_marker();
         }
-        _null_marker_offset = -1;
       }
     }
     // If the inline class has an atomic or nullable (which is also atomic) layout,
@@ -1207,18 +1234,18 @@ void FieldLayoutBuilder::compute_inline_class_layout() {
         assert(_layout->first_field_block() != nullptr, "A concrete value class must have at least one (possible dummy) field");
         _layout->shift_fields(shift);
         _payload_offset = _layout->first_field_block()->offset();
-        if (has_nullable_atomic_layout()) {
+        if (has_nullable_atomic_layout() || has_nullable_non_atomic_layout()) {
           assert(!_is_empty_inline_class, "Should not get here with empty values");
           _null_marker_offset = _layout->find_null_marker()->offset();
         }
         _payload_alignment = required_alignment;
       } else {
         _atomic_layout_size_in_bytes = -1;
-        if (has_nullable_atomic_layout() && !_is_empty_inline_class) {  // empty values don't have a dedicated NULL_MARKER block
+        if (has_nullable_atomic_layout() && !has_nullable_non_atomic_layout() && !_is_empty_inline_class) {  // empty values don't have a dedicated NULL_MARKER block
           _layout->remove_null_marker();
+          _null_marker_offset = -1;
         }
-        _nullable_layout_size_in_bytes = -1;
-        _null_marker_offset = -1;
+        _nullable_atomic_layout_size_in_bytes = -1;
       }
     } else {
       _payload_alignment = required_alignment;
@@ -1472,7 +1499,8 @@ void FieldLayoutBuilder::epilogue() {
     _info->_non_atomic_size_in_bytes = _non_atomic_layout_size_in_bytes;
     _info->_non_atomic_alignment = _non_atomic_layout_alignment;
     _info->_atomic_layout_size_in_bytes = _atomic_layout_size_in_bytes;
-    _info->_nullable_layout_size_in_bytes = _nullable_layout_size_in_bytes;
+    _info->_nullable_atomic_layout_size_in_bytes = _nullable_atomic_layout_size_in_bytes;
+    _info->_nullable_non_atomic_layout_size_in_bytes = _nullable_non_atomic_layout_size_in_bytes;
     _info->_null_marker_offset = _null_marker_offset;
     _info->_null_reset_value_offset = _static_layout->null_reset_value_offset();
     _info->_is_empty_inline_klass = _is_empty_inline_class;
@@ -1560,11 +1588,19 @@ void FieldLayoutBuilder::epilogue() {
       if (has_nullable_atomic_layout()) {
         st.print_cr("%s layout: %d/%d",
                     LayoutKindHelper::layout_kind_as_string(LayoutKind::NULLABLE_ATOMIC_FLAT),
-                    _nullable_layout_size_in_bytes, _nullable_layout_size_in_bytes);
+                    _nullable_atomic_layout_size_in_bytes, _nullable_atomic_layout_size_in_bytes);
       } else {
         st.print_cr("%s layout: -/-",
                     LayoutKindHelper::layout_kind_as_string(LayoutKind::NULLABLE_ATOMIC_FLAT));
       }
+      if (has_nullable_non_atomic_layout()) {
+        st.print_cr("%s layout: %d/%d",
+                    LayoutKindHelper::layout_kind_as_string(LayoutKind::NULLABLE_NON_ATOMIC_FLAT),
+                    _nullable_non_atomic_layout_size_in_bytes, _non_atomic_layout_alignment);
+      } else {
+        st.print_cr("%s layout: -/-",
+                    LayoutKindHelper::layout_kind_as_string(LayoutKind::NULLABLE_NON_ATOMIC_FLAT));
+      }
       if (_null_marker_offset != -1) {
         st.print_cr("Null marker offset = %d", _null_marker_offset);
       }

src/hotspot/share/classfile/fieldLayoutBuilder.hpp

@@ -296,7 +296,8 @@ class FieldLayoutBuilder : public ResourceObj {
   int _non_atomic_layout_size_in_bytes;
   int _non_atomic_layout_alignment;
   int _atomic_layout_size_in_bytes;
-  int _nullable_layout_size_in_bytes;
+  int _nullable_atomic_layout_size_in_bytes;
+  int _nullable_non_atomic_layout_size_in_bytes;
   int _fields_size_sum;
   int _declared_non_static_fields_count;
   bool _has_non_naturally_atomic_fields;
@@ -318,16 +319,18 @@ class FieldLayoutBuilder : public ResourceObj {
                      GrowableArray<FieldInfo>* field_info, bool is_contended, bool is_inline_type, bool is_abstract_value,
                      bool must_be_atomic, FieldLayoutInfo* info, Array<InlineLayoutInfo>* inline_layout_info_array);
 
-  int payload_offset() const               { assert(_payload_offset != -1, "Uninitialized"); return _payload_offset; }
+  int  payload_offset() const                  { assert(_payload_offset != -1, "Uninitialized"); return _payload_offset; }
   int  payload_layout_size_in_bytes() const    { return _payload_size_in_bytes; }
   int  payload_layout_alignment() const        { assert(_payload_alignment != -1, "Uninitialized"); return _payload_alignment; }
   bool has_non_atomic_flat_layout() const      { return _non_atomic_layout_size_in_bytes != -1; }
   int  non_atomic_layout_size_in_bytes() const { return _non_atomic_layout_size_in_bytes; }
   int  non_atomic_layout_alignment() const     { return _non_atomic_layout_alignment; }
   bool has_atomic_layout() const               { return _atomic_layout_size_in_bytes != -1; }
   int  atomic_layout_size_in_bytes() const     { return _atomic_layout_size_in_bytes; }
-  bool has_nullable_atomic_layout() const      { return _nullable_layout_size_in_bytes != -1; }
-  int  nullable_layout_size_in_bytes() const   { return _nullable_layout_size_in_bytes; }
+  bool has_nullable_atomic_layout() const      { return _nullable_atomic_layout_size_in_bytes != -1; }
+  int  nullable_layout_size_in_bytes() const   { return _nullable_atomic_layout_size_in_bytes; }
+  bool has_nullable_non_atomic_layout() const  { return _nullable_non_atomic_layout_size_in_bytes != -1; }
+  int  nullable_non_atomic_layout_size_in_bytes() const { return _nullable_non_atomic_layout_size_in_bytes; }
   int  null_marker_offset() const              { return _null_marker_offset; }
   bool is_empty_inline_class() const           { return _is_empty_inline_class; }
 

src/hotspot/share/oops/flatArrayKlass.cpp

@@ -57,6 +57,7 @@
 FlatArrayKlass::FlatArrayKlass(Klass* element_klass, Symbol* name, ArrayProperties props, LayoutKind lk) :
                 ObjArrayKlass(1, element_klass, name, Kind, props, markWord::flat_array_prototype(lk)) {
   assert(element_klass->is_inline_klass(), "Expected Inline");
+  assert(lk != LayoutKind::NULLABLE_NON_ATOMIC_FLAT, "Layout not supported by arrays yet (needs frozen arrays)");
   assert(LayoutKindHelper::is_flat(lk), "Must be a flat layout");
 
   set_element_klass(InlineKlass::cast(element_klass));
@@ -82,6 +83,8 @@ FlatArrayKlass::FlatArrayKlass(Klass* element_klass, Symbol* name, ArrayProperti
       assert(!layout_helper_is_null_free(layout_helper()), "Must be");
       assert(!prototype_header().is_null_free_array(), "Must be");
     break;
+    case LayoutKind::NULLABLE_NON_ATOMIC_FLAT:
+      ShouldNotReachHere();
     default:
       ShouldNotReachHere();
     break;

src/hotspot/share/oops/inlineKlass.cpp

@@ -80,7 +80,7 @@ void InlineKlass::init_fixed_block() {
   set_non_atomic_size_in_bytes(-1);
   set_non_atomic_alignment(-1);
   set_atomic_size_in_bytes(-1);
-  set_nullable_size_in_bytes(-1);
+  set_nullable_atomic_size_in_bytes(-1);
   set_null_marker_offset(-1);
 }
 
@@ -134,6 +134,10 @@ int InlineKlass::layout_size_in_bytes(LayoutKind kind) const {
       assert(has_nullable_atomic_layout(), "Layout not available");
       return nullable_atomic_size_in_bytes();
       break;
+    case LayoutKind::NULLABLE_NON_ATOMIC_FLAT:
+      assert(has_nullable_non_atomic_layout(), "Layout not available");
+      return nullable_non_atomic_size_in_bytes();
+      break;
     case LayoutKind::BUFFERED:
       return payload_size_in_bytes();
       break;
@@ -156,6 +160,10 @@ int InlineKlass::layout_alignment(LayoutKind kind) const {
       assert(has_nullable_atomic_layout(), "Layout not available");
       return nullable_atomic_size_in_bytes();
       break;
+    case LayoutKind::NULLABLE_NON_ATOMIC_FLAT:
+      assert(has_nullable_non_atomic_layout(), "Layout not available");
+      return non_atomic_alignment();
+    break;
     case LayoutKind::BUFFERED:
       return payload_alignment();
       break;
@@ -175,6 +183,9 @@ bool InlineKlass::is_layout_supported(LayoutKind lk) {
     case LayoutKind::NULLABLE_ATOMIC_FLAT:
       return has_nullable_atomic_layout();
       break;
+    case LayoutKind::NULLABLE_NON_ATOMIC_FLAT:
+      return has_nullable_non_atomic_layout();
+      break;
     case LayoutKind::BUFFERED:
       return true;
       break;
@@ -187,8 +198,9 @@ void InlineKlass::copy_payload_to_addr(void* src, void* dst, LayoutKind lk, bool
   assert(is_layout_supported(lk), "Unsupported layout");
   assert(lk != LayoutKind::REFERENCE && lk != LayoutKind::UNKNOWN, "Sanity check");
   switch(lk) {
+    case LayoutKind::NULLABLE_NON_ATOMIC_FLAT:
     case LayoutKind::NULLABLE_ATOMIC_FLAT: {
-    if (is_payload_marked_as_null((address)src)) {
+     if (is_payload_marked_as_null((address)src)) {
         if (!contains_oops()) {
           mark_payload_as_null((address)dst);
           return;
@@ -230,6 +242,7 @@ oop InlineKlass::read_payload_from_addr(const oop src, size_t offset, LayoutKind
   assert(src != nullptr, "Must be");
   assert(is_layout_supported(lk), "Unsupported layout");
   switch(lk) {
+    case LayoutKind::NULLABLE_NON_ATOMIC_FLAT:
     case LayoutKind::NULLABLE_ATOMIC_FLAT: {
       if (is_payload_marked_as_null((address)((char*)(oopDesc*)src + offset))) {
         return nullptr;

src/hotspot/share/oops/inlineKlass.hpp

@@ -113,7 +113,12 @@ class InlineKlass: public InstanceKlass {
 
   address adr_nullable_atomic_size_in_bytes() const {
     assert(_adr_inlineklass_fixed_block != nullptr, "Should have been initialized");
-    return ((address)_adr_inlineklass_fixed_block) + in_bytes(byte_offset_of(InlineKlassFixedBlock, _nullable_size_in_bytes));
+    return ((address)_adr_inlineklass_fixed_block) + in_bytes(byte_offset_of(InlineKlassFixedBlock, _nullable_atomic_size_in_bytes));
+  }
+
+  address adr_nullable_non_atomic_size_in_bytes() const {
+    assert(_adr_inlineklass_fixed_block != nullptr, "Should have been initialized");
+    return ((address)_adr_inlineklass_fixed_block) + in_bytes(byte_offset_of(InlineKlassFixedBlock, _nullable_non_atomic_size_in_bytes));
   }
 
   address adr_null_marker_offset() const {
@@ -152,23 +157,26 @@ class InlineKlass: public InstanceKlass {
 
   bool has_nullable_atomic_layout() const { return nullable_atomic_size_in_bytes() != -1; }
   int nullable_atomic_size_in_bytes() const { return *(int*)adr_nullable_atomic_size_in_bytes(); }
-  void set_nullable_size_in_bytes(int size) { *(int*)adr_nullable_atomic_size_in_bytes() = size; }
+  void set_nullable_atomic_size_in_bytes(int size) { *(int*)adr_nullable_atomic_size_in_bytes() = size; }
+  bool has_nullable_non_atomic_layout() const { return nullable_non_atomic_size_in_bytes() != -1; }
+  int nullable_non_atomic_size_in_bytes() const { return *(int*)adr_nullable_non_atomic_size_in_bytes(); }
+  void set_nullable_non_atomic_size_in_bytes(int size) { *(int*)adr_nullable_non_atomic_size_in_bytes() = size; }
   int null_marker_offset() const { return *(int*)adr_null_marker_offset(); }
   int null_marker_offset_in_payload() const { return null_marker_offset() - payload_offset(); }
   void set_null_marker_offset(int offset) { *(int*)adr_null_marker_offset() = offset; }
 
   bool is_payload_marked_as_null(address payload) {
-    assert(has_nullable_atomic_layout(), " Must have");
+    assert(has_nullable_atomic_layout() || has_nullable_non_atomic_layout(), " Must have");
     return *((jbyte*)payload + null_marker_offset_in_payload()) == 0;
   }
 
   void mark_payload_as_non_null(address payload) {
-    assert(has_nullable_atomic_layout(), " Must have");
+    assert(has_nullable_atomic_layout() || has_nullable_non_atomic_layout(), " Must have");
     *((jbyte*)payload + null_marker_offset_in_payload()) = 1;
   }
 
   void mark_payload_as_null(address payload) {
-    assert(has_nullable_atomic_layout(), " Must have");
+    assert(has_nullable_atomic_layout() || has_nullable_non_atomic_layout(), " Must have");
     *((jbyte*)payload + null_marker_offset_in_payload()) = 0;
   }
 

src/hotspot/share/oops/instanceKlass.hpp

@@ -153,7 +153,8 @@ class InlineKlassFixedBlock {
   int _non_atomic_size_in_bytes; // size of null-free non-atomic flat layout
   int _non_atomic_alignment;    // alignment requirement for null-free non-atomic layout
   int _atomic_size_in_bytes;    // size and alignment requirement for a null-free atomic layout, -1 if no atomic flat layout is possible
-  int _nullable_size_in_bytes;  // size and alignment requirement for a nullable layout (always atomic), -1 if no nullable flat layout is possible
+  int _nullable_atomic_size_in_bytes;  // size and alignment requirement for a nullable atomic layout, -1 if not available
+  int _nullable_non_atomic_size_in_bytes; // size and alignment requirement for a nullable non-atomic layout, -1 if not available
   int _null_marker_offset;      // expressed as an offset from the beginning of the object for a heap buffered value
                                 // payload_offset must be subtracted to get the offset from the beginning of the payload