[release/10.0] Fix nullable complex property with PropertyValues.ToObject() and SetValues() creating instance instead of null

src/EFCore/ChangeTracking/EntityEntry.cs

@@ -603,7 +603,7 @@ public virtual PropertyValues OriginalValues
     {
         var values = Finder.GetDatabaseValues(InternalEntry);
 
-        return values == null ? null : new ArrayPropertyValues(InternalEntry, values);
+        return values == null ? null : new ArrayPropertyValues(InternalEntry, values, null);
     }
 
     /// <summary>
@@ -634,7 +634,7 @@ public virtual PropertyValues OriginalValues
     {
         var values = await Finder.GetDatabaseValuesAsync(InternalEntry, cancellationToken).ConfigureAwait(false);
 
-        return values == null ? null : new ArrayPropertyValues(InternalEntry, values);
+        return values == null ? null : new ArrayPropertyValues(InternalEntry, values, null);
     }
 
     /// <summary>

src/EFCore/ChangeTracking/Internal/ArrayPropertyValues.cs

@@ -16,18 +16,20 @@ public class ArrayPropertyValues : PropertyValues
 {
     private readonly object?[] _values;
     private readonly List<ArrayPropertyValues?>?[] _complexCollectionValues;
+    private readonly bool[]? _nullComplexPropertyFlags;
 
     /// <summary>
     ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
     ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
     ///     any release. You should only use it directly in your code with extreme caution and knowing that
     ///     doing so can result in application failures when updating to a new Entity Framework Core release.
     /// </summary>
-    public ArrayPropertyValues(InternalEntryBase internalEntry, object?[] values)
+    public ArrayPropertyValues(InternalEntryBase internalEntry, object?[] values, bool[]? nullComplexPropertyFlags)
         : base(internalEntry)
     {
         _values = values;
         _complexCollectionValues = new List<ArrayPropertyValues?>?[ComplexCollectionProperties.Count];
+        _nullComplexPropertyFlags = nullComplexPropertyFlags;
     }
 
     /// <summary>
@@ -41,6 +43,18 @@ public override object ToObject()
         var structuralObject = StructuralType.GetOrCreateMaterializer(MaterializerSource)(
             new MaterializationContext(new ValueBuffer(_values), InternalEntry.Context));
 
+        if (_nullComplexPropertyFlags != null && NullableComplexProperties != null)
+        {
+            for (var i = 0; i < _nullComplexPropertyFlags.Length; i++)
+            {
+                if (_nullComplexPropertyFlags[i])
+                {
+                    var complexProperty = NullableComplexProperties[i];
+                    structuralObject = ((IRuntimeComplexProperty)complexProperty).GetSetter().SetClrValue(structuralObject, null);
+                }
+            }
+        }
+
         for (var i = 0; i < _complexCollectionValues.Length; i++)
         {
             var propertyValuesList = _complexCollectionValues[i];
@@ -65,6 +79,15 @@ public override object ToObject()
         return structuralObject;
     }
 
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    public override bool IsNullableComplexPropertyNull(int index)
+        => _nullComplexPropertyFlags != null && _nullComplexPropertyFlags[index];
+
     /// <summary>
     ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
     ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
@@ -132,7 +155,15 @@ public override PropertyValues Clone()
         var copies = new object[_values.Length];
         Array.Copy(_values, copies, _values.Length);
 
-        var clone = new ArrayPropertyValues(InternalEntry, copies);
+        bool[]? flagsCopy = null;
+        if (_nullComplexPropertyFlags != null)
+        {
+            flagsCopy = new bool[_nullComplexPropertyFlags.Length];
+            Array.Copy(_nullComplexPropertyFlags, flagsCopy, _nullComplexPropertyFlags.Length);
+        }
+
+        var clone = new ArrayPropertyValues(InternalEntry, copies, flagsCopy);
+
         for (var i = 0; i < _complexCollectionValues.Length; i++)
         {
             var list = _complexCollectionValues[i];
@@ -354,7 +385,7 @@ private void SetValuesFromDictionary<TProperty>(IRuntimeTypeBase structuralType,
                 var complexEntry = new InternalComplexEntry((IRuntimeComplexType)complexProperty.ComplexType, InternalEntry, i);
                 var complexType = complexEntry.StructuralType;
                 var values = new object?[complexType.GetFlattenedProperties().Count()];
-                var complexPropertyValues = new ArrayPropertyValues(complexEntry, values);
+                var complexPropertyValues = new ArrayPropertyValues(complexEntry, values, null);
                 complexPropertyValues.SetValues(itemDict);
 
                 propertyValuesList.Add(complexPropertyValues);
@@ -468,7 +499,7 @@ ArrayPropertyValues CreateComplexPropertyValues(object complexObject, InternalCo
                 values[i] = getter.GetClrValue(complexObject);
             }
 
-            var complexPropertyValues = new ArrayPropertyValues(entry, values);
+            var complexPropertyValues = new ArrayPropertyValues(entry, values, null);
 
             foreach (var nestedComplexProperty in complexPropertyValues.ComplexCollectionProperties)
             {

src/EFCore/ChangeTracking/Internal/EntryPropertyValues.cs

@@ -282,7 +282,19 @@ public override PropertyValues Clone()
             values[i] = GetValueInternal(InternalEntry, Properties[i]);
         }
 
-        var cloned = new ArrayPropertyValues(InternalEntry, values);
+        bool[]? flags = null;
+        var nullableComplexProperties = NullableComplexProperties;
+        if (nullableComplexProperties != null && nullableComplexProperties.Count > 0)
+        {
+            flags = new bool[nullableComplexProperties.Count];
+            for (var i = 0; i < nullableComplexProperties.Count; i++)
+            {
+                flags[i] = GetValueInternal(InternalEntry, nullableComplexProperties[i]) == null;
+            }
+        }
+
+        var cloned = new ArrayPropertyValues(InternalEntry, values, flags);
+
         foreach (var complexProperty in ComplexCollectionProperties)
         {
             var collection = (IList?)GetValueInternal(InternalEntry, complexProperty);
@@ -305,15 +317,58 @@ public override void SetValues(PropertyValues propertyValues)
     {
         Check.NotNull(propertyValues);
 
+        var nullableComplexProperties = NullableComplexProperties;
+        HashSet<IComplexProperty>? nullComplexProperties = null;
+        if (nullableComplexProperties != null)
+        {
+            for (var i = 0; i < nullableComplexProperties.Count; i++)
+            {
+                if (propertyValues.IsNullableComplexPropertyNull(i))
+                {
+                    nullComplexProperties ??= [];
+                    nullComplexProperties.Add(nullableComplexProperties[i]);
+                }
+            }
+        }
+
         foreach (var property in Properties)
         {
+            if (nullComplexProperties != null && IsPropertyInNullComplexType(property, nullComplexProperties))
+            {
+                continue;
+            }
+
             SetValueInternal(InternalEntry, property, propertyValues[property]);
         }
 
         foreach (var complexProperty in ComplexCollectionProperties)
         {
             SetValueInternal(InternalEntry, complexProperty, propertyValues[complexProperty]);
         }
+
+        if (nullComplexProperties != null)
+        {
+            foreach (var complexProperty in nullComplexProperties)
+            {
+                SetValueInternal(InternalEntry, complexProperty, null);
+            }
+        }
+    }
+
+    private static bool IsPropertyInNullComplexType(IProperty property, HashSet<IComplexProperty> nullComplexProperties)
+    {
+        var declaringType = property.DeclaringType;
+        while (declaringType is IComplexType complexType)
+        {
+            if (nullComplexProperties.Contains(complexType.ComplexProperty))
+            {
+                return true;
+            }
+
+            declaringType = complexType.ComplexProperty.DeclaringType;
+        }
+
+        return false;
     }
 
     /// <inheritdoc />

src/EFCore/ChangeTracking/Internal/OriginalPropertyValues.cs

@@ -97,7 +97,7 @@ private PropertyValues GetPropertyValues(InternalEntryBase entry)
             values[i] = entry.GetOriginalValue(properties[i]);
         }
 
-        var cloned = new ArrayPropertyValues(entry, values);
+        var cloned = new ArrayPropertyValues(entry, values, null);
 
         foreach (var nestedComplexProperty in cloned.ComplexCollectionProperties)
         {

src/EFCore/ChangeTracking/Internal/SnapshotFactoryFactory.cs

@@ -145,6 +145,13 @@ protected virtual Expression CreateSnapshotExpression(
                 case var _ when propertyBase.IsShadowProperty():
                     arguments[i] = CreateSnapshotValueExpression(CreateReadShadowValueExpression(parameter, propertyBase), propertyBase);
                     continue;
+
+                case IComplexProperty { IsCollection: false, IsNullable: true }:
+                    // For nullable non-collection complex properties, convert to object to store the null reference
+                    arguments[i] = Expression.Convert(
+                        CreateSnapshotValueExpression(CreateReadValueExpression(parameter, propertyBase), propertyBase),
+                        typeof(object));
+                    continue;
             }
 
             arguments[i] = CreateSnapshotValueExpression(CreateReadValueExpression(parameter, propertyBase), propertyBase);

src/EFCore/ChangeTracking/PropertyValues.cs

@@ -26,7 +26,11 @@ namespace Microsoft.EntityFrameworkCore.ChangeTracking;
 /// </remarks>
 public abstract class PropertyValues
 {
+    private static readonly bool UseOldBehavior37249 =
+        AppContext.TryGetSwitch("Microsoft.EntityFrameworkCore.Issue37249", out var enabled) && enabled;
+
     private readonly IReadOnlyList<IComplexProperty> _complexCollectionProperties;
+    private readonly IReadOnlyList<IComplexProperty>? _nullableComplexProperties;
 
     /// <summary>
     ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
@@ -38,10 +42,27 @@ public abstract class PropertyValues
     protected PropertyValues(InternalEntryBase internalEntry)
     {
         InternalEntry = internalEntry;
-        _complexCollectionProperties = [.. internalEntry.StructuralType.GetFlattenedComplexProperties().Where(p => p.IsCollection)];
+
+        var complexCollectionProperties = new List<IComplexProperty>();
+        var nullableComplexProperties = UseOldBehavior37249 ? null : new List<IComplexProperty>();
+
+        foreach (var complexProperty in internalEntry.StructuralType.GetFlattenedComplexProperties())
+        {
+            if (complexProperty.IsCollection)
+            {
+                complexCollectionProperties.Add(complexProperty);
+            }
+            else if (!UseOldBehavior37249 && complexProperty.IsNullable && !complexProperty.IsShadowProperty())
+            {
+                nullableComplexProperties!.Add(complexProperty);
+            }
+        }
+
+        _complexCollectionProperties = complexCollectionProperties;
         Check.DebugAssert(
             _complexCollectionProperties.Select((p, i) => p.GetIndex() == i).All(e => e),
             "Complex collection properties indices are not sequential.");
+        _nullableComplexProperties = nullableComplexProperties?.Count > 0 ? nullableComplexProperties : null;
     }
 
     /// <summary>
@@ -154,6 +175,29 @@ public virtual IReadOnlyList<IComplexProperty> ComplexCollectionProperties
         get => _complexCollectionProperties;
     }
 
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    [EntityFrameworkInternal]
+    protected virtual IReadOnlyList<IComplexProperty>? NullableComplexProperties
+    {
+        [DebuggerStepThrough]
+        get => _nullableComplexProperties;
+    }
+
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    [EntityFrameworkInternal]
+    public virtual bool IsNullableComplexPropertyNull(int index)
+        => false;
+
     /// <summary>
     ///     Gets the underlying structural type for which this object is storing values.
     /// </summary>

src/EFCore/Metadata/Internal/EntityTypeExtensions.cs

@@ -244,7 +244,10 @@ private static void CalculateCounts(
     {
         var indexes = new PropertyIndexes(
             index: complexProperty.IsCollection ? complexCollectionIndex++ : complexPropertyIndex++,
-            originalValueIndex: complexProperty.IsCollection ? originalValueIndex++ : -1,
+            originalValueIndex: complexProperty.IsCollection
+                || (complexProperty.IsNullable && !complexProperty.IsShadowProperty())
+                    ? originalValueIndex++
+                    : -1,
             shadowIndex: complexProperty.IsShadowProperty() ? shadowIndex++ : -1,
             relationshipIndex: -1,
             storeGenerationIndex: -1);