diff --git a/pkgs/sdk/server/src/Interfaces/DataSourceStatus.cs b/pkgs/sdk/server/src/Interfaces/DataSourceStatus.cs index affd59ae..a7cd22e1 100644 --- a/pkgs/sdk/server/src/Interfaces/DataSourceStatus.cs +++ b/pkgs/sdk/server/src/Interfaces/DataSourceStatus.cs @@ -62,6 +62,12 @@ public struct ErrorInfo /// public ErrorKind Kind { get; set; } + /// + /// Whether the error is recoverable. Recoverable errors are those that can be retried, such as network errors. Unrecoverable + /// errors are those that cannot be retried, such as invalid SDK key errors. + /// + public bool Recoverable { get; set; } + /// /// The HTTP status code if the error was , or zero otherwise. /// @@ -90,24 +96,28 @@ public struct ErrorInfo /// Constructs an instance based on an exception. /// /// the exception + /// whether the error is recoverable /// an ErrorInfo - public static ErrorInfo FromException(Exception e) => new ErrorInfo + public static ErrorInfo FromException(Exception e, bool recoverable) => new ErrorInfo { Kind = e is IOException ? ErrorKind.NetworkError : ErrorKind.Unknown, Message = e.Message, - Time = DateTime.Now + Time = DateTime.Now, + Recoverable = recoverable }; /// /// Constructs an instance based on an HTTP error status. /// /// the status code + /// whether the error is recoverable /// an ErrorInfo - public static ErrorInfo FromHttpError(int statusCode) => new ErrorInfo + public static ErrorInfo FromHttpError(int statusCode, bool recoverable) => new ErrorInfo { Kind = ErrorKind.ErrorResponse, StatusCode = statusCode, - Time = DateTime.Now + Time = DateTime.Now, + Recoverable = recoverable }; /// diff --git a/pkgs/sdk/server/src/Internal/DataSources/PollingDataSource.cs b/pkgs/sdk/server/src/Internal/DataSources/PollingDataSource.cs index 2540a763..9048c9fa 100644 --- a/pkgs/sdk/server/src/Internal/DataSources/PollingDataSource.cs +++ b/pkgs/sdk/server/src/Internal/DataSources/PollingDataSource.cs @@ -23,6 +23,9 @@ internal sealed class PollingDataSource : IDataSource private readonly Logger _log; private CancellationTokenSource _canceller; + private bool _disposed = false; + private readonly AtomicBoolean _shuttingDown = new AtomicBoolean(false); + internal PollingDataSource( LdClientContext context, IFeatureRequestor featureRequestor, @@ -81,9 +84,10 @@ private async Task UpdateTaskAsync() } catch (UnsuccessfulResponseException ex) { - var errorInfo = DataSourceStatus.ErrorInfo.FromHttpError(ex.StatusCode); + var recoverable = HttpErrors.IsRecoverable(ex.StatusCode); + var errorInfo = DataSourceStatus.ErrorInfo.FromHttpError(ex.StatusCode, recoverable); - if (HttpErrors.IsRecoverable(ex.StatusCode)) + if (errorInfo.Recoverable) { _log.Warn(HttpErrors.ErrorMessage(ex.StatusCode, "polling request", "will retry")); _dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, errorInfo); @@ -91,52 +95,69 @@ private async Task UpdateTaskAsync() else { _log.Error(HttpErrors.ErrorMessage(ex.StatusCode, "polling request", "")); - _dataSourceUpdates.UpdateStatus(DataSourceState.Off, errorInfo); try { // if client is initializing, make it stop waiting - _initTask.SetResult(true); + _initTask.SetResult(false); } catch (InvalidOperationException) { // the task was already set - nothing more to do } - ((IDisposable)this).Dispose(); + Shutdown(errorInfo); } } catch (JsonException ex) { _log.Error("Polling request received malformed data: {0}", LogValues.ExceptionSummary(ex)); - _dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, - new DataSourceStatus.ErrorInfo - { - Kind = DataSourceStatus.ErrorKind.InvalidData, - Time = DateTime.Now - }); + var errorInfo = new DataSourceStatus.ErrorInfo + { + Kind = DataSourceStatus.ErrorKind.InvalidData, + Message = ex.Message, + Time = DateTime.Now, + Recoverable = true + }; + _dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, errorInfo); } catch (Exception ex) { Exception realEx = (ex is AggregateException ae) ? ae.Flatten() : ex; _log.Warn("Polling for feature flag updates failed: {0}", LogValues.ExceptionSummary(ex)); _log.Debug(LogValues.ExceptionTrace(ex)); - _dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, - DataSourceStatus.ErrorInfo.FromException(realEx)); + var errorInfo = DataSourceStatus.ErrorInfo.FromException(realEx, true); // default to recoverable + _dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, errorInfo); } } void IDisposable.Dispose() { + // dispose is currently overloaded with shutdown responsibility, we handle this first + Shutdown(null); + Dispose(true); GC.SuppressFinalize(this); } private void Dispose(bool disposing) { - if (disposing) - { - _canceller?.Cancel(); + if (_disposed) return; + + if (disposing) { + // dispose managed resources if any _featureRequestor.Dispose(); } + + _disposed = true; + } + + private void Shutdown(DataSourceStatus.ErrorInfo? errorInfo) + { + // Prevent concurrent shutdown calls - only allow the first call to proceed + // GetAndSet returns the OLD value, so if it was already true, we return early + if (_shuttingDown.GetAndSet(true)) return; + + _canceller?.Cancel(); + _dataSourceUpdates.UpdateStatus(DataSourceState.Off, errorInfo); } private bool InitWithHeaders(DataStoreTypes.FullDataSet allData, diff --git a/pkgs/sdk/server/src/Internal/DataSources/StreamingDataSource.cs b/pkgs/sdk/server/src/Internal/DataSources/StreamingDataSource.cs index bd0c5f20..b68c6cdc 100644 --- a/pkgs/sdk/server/src/Internal/DataSources/StreamingDataSource.cs +++ b/pkgs/sdk/server/src/Internal/DataSources/StreamingDataSource.cs @@ -52,6 +52,9 @@ internal class StreamingDataSource : IDataSource private IEnumerable>> _headers; + private bool _disposed = false; + private readonly AtomicBoolean _shuttingDown = new AtomicBoolean(false); + internal delegate IEventSource EventSourceCreator(Uri streamUri, HttpConfiguration httpConfig); @@ -104,20 +107,36 @@ public Task Start() public void Dispose() { + // dispose is currently overloaded with shutdown responsibility, we handle this first + Shutdown(null); + Dispose(true); GC.SuppressFinalize(this); } private void Dispose(bool disposing) { - if (disposing) + if (_disposed) return; + + if (disposing) { + // dispose managed resources if any + } + + _disposed = true; + } + + private void Shutdown(DataSourceStatus.ErrorInfo? errorInfo) + { + // Prevent concurrent shutdown calls - only allow the first call to proceed + // GetAndSet returns the OLD value, so if it was already true, we return early + if (_shuttingDown.GetAndSet(true)) return; + + _es.Close(); + if (_storeStatusMonitoringEnabled) { - _es.Close(); - if (_storeStatusMonitoringEnabled) - { - _dataSourceUpdates.DataStoreStatusProvider.StatusChanged -= OnDataStoreStatusChanged; - } + _dataSourceUpdates.DataStoreStatusProvider.StatusChanged -= OnDataStoreStatusChanged; } + _dataSourceUpdates.UpdateStatus(DataSourceState.Off, errorInfo); } #endregion @@ -175,7 +194,8 @@ private void OnMessage(object sender, EventSource.MessageReceivedEventArgs e) { Kind = DataSourceStatus.ErrorKind.InvalidData, Message = ex.Message, - Time = DateTime.Now + Time = DateTime.Now, + Recoverable = true }; _dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, errorInfo); @@ -187,7 +207,8 @@ private void OnMessage(object sender, EventSource.MessageReceivedEventArgs e) { Kind = DataSourceStatus.ErrorKind.StoreError, Message = (ex.InnerException ?? ex).Message, - Time = DateTime.Now + Time = DateTime.Now, + Recoverable = true }; _dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, errorInfo); if (!_storeStatusMonitoringEnabled) @@ -210,17 +231,16 @@ private void OnMessage(object sender, EventSource.MessageReceivedEventArgs e) private void OnError(object sender, EventSource.ExceptionEventArgs e) { var ex = e.Exception; - var recoverable = true; DataSourceStatus.ErrorInfo errorInfo; if (ex is EventSourceServiceUnsuccessfulResponseException respEx) { int status = respEx.StatusCode; - errorInfo = DataSourceStatus.ErrorInfo.FromHttpError(status); + var recoverable = HttpErrors.IsRecoverable(status); + errorInfo = DataSourceStatus.ErrorInfo.FromHttpError(status, recoverable); RecordStreamInit(true); - if (!HttpErrors.IsRecoverable(status)) + if (!recoverable) { - recoverable = false; _log.Error(HttpErrors.ErrorMessage(status, "streaming connection", "")); } else @@ -230,21 +250,23 @@ private void OnError(object sender, EventSource.ExceptionEventArgs e) } else { - errorInfo = DataSourceStatus.ErrorInfo.FromException(ex); + errorInfo = DataSourceStatus.ErrorInfo.FromException(ex, true); // default to recoverable _log.Warn("Encountered EventSource error: {0}", LogValues.ExceptionSummary(ex)); _log.Debug(LogValues.ExceptionTrace(ex)); } - _dataSourceUpdates.UpdateStatus(recoverable ? DataSourceState.Interrupted : DataSourceState.Off, - errorInfo); - - if (!recoverable) + if (errorInfo.Recoverable) + { + _dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, errorInfo); + return; + } + else { // Make _initTask complete to tell the client to stop waiting for initialization. We use // TrySetResult rather than SetResult here because it might have already been completed // (if for instance the stream started successfully, then restarted and got a 401). _initTask.TrySetResult(false); - ((IDisposable)this).Dispose(); + Shutdown(errorInfo); } } diff --git a/pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs b/pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs index ca8d18bd..d8ed6600 100644 --- a/pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs +++ b/pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs @@ -194,7 +194,10 @@ public void UpdateStatus(DataSourceState newState, DataSourceStatus.ErrorInfo? n // When a synchronizer reports it is off, fall back immediately if (newState == DataSourceState.Off) { - _actionable.BlacklistCurrent(); + if (newError != null && !newError.Value.Recoverable) + { + _actionable.BlacklistCurrent(); + } _actionable.DisposeCurrent(); _actionable.GoToNext(); _actionable.StartCurrent(); diff --git a/pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2PollingDataSource.cs b/pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2PollingDataSource.cs index e223e62c..703e986c 100644 --- a/pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2PollingDataSource.cs +++ b/pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2PollingDataSource.cs @@ -30,6 +30,9 @@ internal sealed class FDv2PollingDataSource : IDataSource private CancellationTokenSource _canceler; private string _environmentId; + private bool _disposed = false; + private readonly AtomicBoolean _shuttingDown = new AtomicBoolean(false); + internal FDv2PollingDataSource( LdClientContext context, IDataSourceUpdates dataSourceUpdates, @@ -95,7 +98,8 @@ private async Task UpdateTaskAsync() } catch (UnsuccessfulResponseException ex) { - var errorInfo = DataSourceStatus.ErrorInfo.FromHttpError(ex.StatusCode); + var recoverable = HttpErrors.IsRecoverable(ex.StatusCode); + var errorInfo = DataSourceStatus.ErrorInfo.FromHttpError(ex.StatusCode, recoverable); // Check for LD fallback header if (ex.Headers != null) @@ -106,7 +110,7 @@ private async Task UpdateTaskAsync() .Any(v => string.Equals(v, "true", StringComparison.OrdinalIgnoreCase)); } - if (HttpErrors.IsRecoverable(ex.StatusCode)) + if (errorInfo.Recoverable) { _log.Warn(HttpErrors.ErrorMessage(ex.StatusCode, "polling request", "will retry")); _dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, errorInfo); @@ -114,7 +118,6 @@ private async Task UpdateTaskAsync() else { _log.Error(HttpErrors.ErrorMessage(ex.StatusCode, "polling request", "")); - _dataSourceUpdates.UpdateStatus(DataSourceState.Off, errorInfo); try { _initTask.SetResult(false); @@ -124,26 +127,28 @@ private async Task UpdateTaskAsync() // the task was already set - nothing more to do } - ((IDisposable)this).Dispose(); + Shutdown(errorInfo); } } catch (JsonException ex) { _log.Error("Polling request received malformed data: {0}", LogValues.ExceptionSummary(ex)); - _dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, - new DataSourceStatus.ErrorInfo - { - Kind = DataSourceStatus.ErrorKind.InvalidData, - Time = DateTime.Now - }); + var errorInfo = new DataSourceStatus.ErrorInfo + { + Kind = DataSourceStatus.ErrorKind.InvalidData, + Message = ex.Message, + Time = DateTime.Now, + Recoverable = true + }; + _dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, errorInfo); } catch (Exception ex) { var realEx = (ex is AggregateException ae) ? ae.Flatten() : ex; _log.Warn("Polling for feature flag updates failed: {0}", LogValues.ExceptionSummary(ex)); _log.Debug(LogValues.ExceptionTrace(ex)); - _dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, - DataSourceStatus.ErrorInfo.FromException(realEx)); + var errorInfo = DataSourceStatus.ErrorInfo.FromException(realEx, true); // default to recoverable + _dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, errorInfo); } } @@ -169,7 +174,8 @@ private void HandleJsonError(string message) { Kind = DataSourceStatus.ErrorKind.InvalidData, Message = message, - Time = DateTime.Now + Time = DateTime.Now, + Recoverable = true }; _dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, errorInfo); } @@ -227,21 +233,33 @@ private void ProcessChangeSet(FDv2ChangeSet fdv2ChangeSet) void IDisposable.Dispose() { + // dispose is currently overloaded with shutdown responsibility, we handle this first + Shutdown(null); + Dispose(true); GC.SuppressFinalize(this); } private void Dispose(bool disposing) { - if (!disposing) return; + if (_disposed) return; - Shutdown(); + if (disposing) { + // dispose managed resources if any + _requestor.Dispose(); + } + + _disposed = true; } - private void Shutdown() + private void Shutdown(DataSourceStatus.ErrorInfo? errorInfo) { + // Prevent concurrent shutdown calls - only allow the first call to proceed + // GetAndSet returns the OLD value, so if it was already true, we return early + if (_shuttingDown.GetAndSet(true)) return; + _canceler?.Cancel(); - _requestor.Dispose(); + _dataSourceUpdates.UpdateStatus(DataSourceState.Off, errorInfo); } } } diff --git a/pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2StreamingDataSource.cs b/pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2StreamingDataSource.cs index eba876f2..5110ad93 100644 --- a/pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2StreamingDataSource.cs +++ b/pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2StreamingDataSource.cs @@ -56,6 +56,9 @@ internal delegate IEventSource EventSourceCreator(Uri streamUri, /// private readonly AtomicBoolean _lastStoreUpdateFailed = new AtomicBoolean(false); + private bool _disposed = false; + private readonly AtomicBoolean _shuttingDown = new AtomicBoolean(false); + internal FDv2StreamingDataSource( LdClientContext context, IDataSourceUpdates dataSourceUpdates, @@ -102,24 +105,36 @@ internal FDv2StreamingDataSource( public void Dispose() { + // dispose is currently overloaded with shutdown responsibility, we handle this first + Shutdown(null); + Dispose(true); GC.SuppressFinalize(this); } private void Dispose(bool disposing) { - if (!disposing) return; + if (_disposed) return; + + if (disposing) { + // dispose managed resources if any + } - Shutdown(); + _disposed = true; } - private void Shutdown() + private void Shutdown(DataSourceStatus.ErrorInfo? errorInfo) { + // Prevent concurrent shutdown calls - only allow the first call to proceed + // GetAndSet returns the OLD value, so if it was already true, we return early + if (_shuttingDown.GetAndSet(true)) return; + _es.Close(); if (_storeStatusMonitoringEnabled) { _dataSourceUpdates.DataStoreStatusProvider.StatusChanged -= OnDataStoreStatusChanged; } + _dataSourceUpdates.UpdateStatus(DataSourceState.Off, errorInfo); } public Task Start() @@ -144,7 +159,8 @@ private void HandleJsonError(string message) { Kind = DataSourceStatus.ErrorKind.InvalidData, Message = message, - Time = DateTime.Now + Time = DateTime.Now, + Recoverable = true }; _dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, errorInfo); @@ -157,7 +173,8 @@ private void HandleStoreError(string message) { Kind = DataSourceStatus.ErrorKind.StoreError, Message = message, - Time = DateTime.Now + Time = DateTime.Now, + Recoverable = true }; _dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, errorInfo); if (_storeStatusMonitoringEnabled) return; @@ -268,13 +285,13 @@ void MaybeMarkInitialized() private void OnError(object sender, ExceptionEventArgs e) { var ex = e.Exception; - var recoverable = true; DataSourceStatus.ErrorInfo errorInfo; if (ex is EventSourceServiceUnsuccessfulResponseException respEx) { var status = respEx.StatusCode; - errorInfo = DataSourceStatus.ErrorInfo.FromHttpError(status); + var recoverable = HttpErrors.IsRecoverable(status); + errorInfo = DataSourceStatus.ErrorInfo.FromHttpError(status, recoverable); // Check for LD fallback header if (respEx.Headers != null) @@ -286,9 +303,8 @@ private void OnError(object sender, ExceptionEventArgs e) } RecordStreamInit(true); - if (!HttpErrors.IsRecoverable(status)) + if (!recoverable) { - recoverable = false; _log.Error(HttpErrors.ErrorMessage(status, "streaming connection", "")); } else @@ -298,20 +314,21 @@ private void OnError(object sender, ExceptionEventArgs e) } else { - errorInfo = DataSourceStatus.ErrorInfo.FromException(ex); + errorInfo = DataSourceStatus.ErrorInfo.FromException(ex, true); // default to recoverable _log.Warn("Encountered EventSource error: {0}", LogValues.ExceptionSummary(ex)); _log.Debug(LogValues.ExceptionTrace(ex)); } - _dataSourceUpdates.UpdateStatus(recoverable ? DataSourceState.Interrupted : DataSourceState.Off, - errorInfo); - - if (recoverable) return; - // Make _initTask complete to tell the client to stop waiting for initialization. We use - // TrySetResult rather than SetResult here because it might have already been completed - // (if, for instance, the stream started successfully, then restarted and got a 401). - _initTask.TrySetResult(false); - Shutdown(); + if (errorInfo.Recoverable) { + _dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, errorInfo); + return; + } else { + // Make _initTask complete to tell the client to stop waiting for initialization. We use + // TrySetResult rather than SetResult here because it might have already been completed + // (if, for instance, the stream started successfully, then restarted and got a 401). + _initTask.TrySetResult(false); + Shutdown(errorInfo); + } } private void OnOpen(object sender, StateChangedEventArgs e) diff --git a/pkgs/sdk/server/test/Integrations/TestDataWithClientTest.cs b/pkgs/sdk/server/test/Integrations/TestDataWithClientTest.cs index abab4c7e..cc54fe28 100644 --- a/pkgs/sdk/server/test/Integrations/TestDataWithClientTest.cs +++ b/pkgs/sdk/server/test/Integrations/TestDataWithClientTest.cs @@ -105,7 +105,7 @@ public void CanUpdateStatus() { Assert.Equal(DataSourceState.Valid, client.DataSourceStatusProvider.Status.State); - var ei = DataSourceStatus.ErrorInfo.FromHttpError(500); + var ei = DataSourceStatus.ErrorInfo.FromHttpError(500, false); _td.UpdateStatus(DataSourceState.Interrupted, ei); Assert.Equal(DataSourceState.Interrupted, client.DataSourceStatusProvider.Status.State); diff --git a/pkgs/sdk/server/test/Internal/DataSources/DataSourceStatusProviderImplTest.cs b/pkgs/sdk/server/test/Internal/DataSources/DataSourceStatusProviderImplTest.cs index e94bae13..9e6d58be 100644 --- a/pkgs/sdk/server/test/Internal/DataSources/DataSourceStatusProviderImplTest.cs +++ b/pkgs/sdk/server/test/Internal/DataSources/DataSourceStatusProviderImplTest.cs @@ -32,7 +32,7 @@ public void Status() Assert.Equal(DataSourceState.Initializing, statusProvider.Status.State); var timeBefore = DateTime.Now; - var errorInfo = DataSourceStatus.ErrorInfo.FromHttpError(500); + var errorInfo = DataSourceStatus.ErrorInfo.FromHttpError(500, false); updates.UpdateStatus(DataSourceState.Valid, errorInfo); diff --git a/pkgs/sdk/server/test/Internal/DataSources/DataSourceUpdatesImplTest.cs b/pkgs/sdk/server/test/Internal/DataSources/DataSourceUpdatesImplTest.cs index 0ae2427c..e65a9dbd 100644 --- a/pkgs/sdk/server/test/Internal/DataSources/DataSourceUpdatesImplTest.cs +++ b/pkgs/sdk/server/test/Internal/DataSources/DataSourceUpdatesImplTest.cs @@ -290,7 +290,7 @@ public void UpdateStatusBroadcastsNewStatus() updates.StatusChanged += statuses.Add; var timeBeforeUpdate = DateTime.Now; - var errorInfo = DataSourceStatus.ErrorInfo.FromHttpError(401); + var errorInfo = DataSourceStatus.ErrorInfo.FromHttpError(401, false); updates.UpdateStatus(DataSourceState.Off, errorInfo); var status = statuses.ExpectValue(); @@ -310,7 +310,7 @@ public void UpdateStatusKeepsStateUnchangedIfStateWasInitializingAndNewStateIsIn var statuses = new EventSink(); updates.StatusChanged += statuses.Add; - var errorInfo = DataSourceStatus.ErrorInfo.FromHttpError(401); + var errorInfo = DataSourceStatus.ErrorInfo.FromHttpError(401, false); updates.UpdateStatus(DataSourceState.Interrupted, errorInfo); var status = statuses.ExpectValue(); @@ -346,7 +346,7 @@ public void OutageTimeoutLogging() ); // simulate an outage - updates.UpdateStatus(DataSourceState.Interrupted, DataSourceStatus.ErrorInfo.FromHttpError(500)); + updates.UpdateStatus(DataSourceState.Interrupted, DataSourceStatus.ErrorInfo.FromHttpError(500, true)); // but recover from it immediately updates.UpdateStatus(DataSourceState.Valid, null); @@ -355,11 +355,11 @@ public void OutageTimeoutLogging() Thread.Sleep(outageTimeout.Add(TimeSpan.FromMilliseconds(50))); // simulate another outage - updates.UpdateStatus(DataSourceState.Interrupted, DataSourceStatus.ErrorInfo.FromHttpError(501)); - updates.UpdateStatus(DataSourceState.Interrupted, DataSourceStatus.ErrorInfo.FromHttpError(502)); + updates.UpdateStatus(DataSourceState.Interrupted, DataSourceStatus.ErrorInfo.FromHttpError(501, true)); + updates.UpdateStatus(DataSourceState.Interrupted, DataSourceStatus.ErrorInfo.FromHttpError(502, true)); updates.UpdateStatus(DataSourceState.Interrupted, - DataSourceStatus.ErrorInfo.FromException(new IOException("x"))); - updates.UpdateStatus(DataSourceState.Interrupted, DataSourceStatus.ErrorInfo.FromHttpError(501)); + DataSourceStatus.ErrorInfo.FromException(new IOException("x"), true)); + updates.UpdateStatus(DataSourceState.Interrupted, DataSourceStatus.ErrorInfo.FromHttpError(501, true)); Thread.Sleep(outageTimeout); AssertEventually(TimeSpan.FromSeconds(1), TimeSpan.FromMilliseconds(50), () => diff --git a/pkgs/sdk/server/test/Internal/DataSources/PollingDataSourceTest.cs b/pkgs/sdk/server/test/Internal/DataSources/PollingDataSourceTest.cs index 517a7a1b..af92f2b7 100644 --- a/pkgs/sdk/server/test/Internal/DataSources/PollingDataSourceTest.cs +++ b/pkgs/sdk/server/test/Internal/DataSources/PollingDataSourceTest.cs @@ -129,6 +129,7 @@ public void VerifyUnrecoverableHttpError(int errorStatus) var status = _updateSink.StatusUpdates.ExpectValue(); errorCondition.VerifyDataSourceStatusError(status); + Assert.False(status.LastError.Value.Recoverable, "Recoverable should be false for unrecoverable errors"); recorder.RequireRequest(); recorder.RequireNoRequests(TimeSpan.FromMilliseconds(100)); // did not retry @@ -160,6 +161,7 @@ public void VerifyRecoverableError(int errorStatus) var status = _updateSink.StatusUpdates.ExpectValue(); errorCondition.VerifyDataSourceStatusError(status); + Assert.True(status.LastError.Value.Recoverable, "Recoverable should be true for recoverable errors"); recorder.RequireRequest(); recorder.RequireNoRequests(TimeSpan.FromMilliseconds(100)); @@ -176,13 +178,20 @@ public void VerifyRecoverableError(int errorStatus) c => c.DataSource(Components.PollingDataSource().PollIntervalNoMinimum(BriefInterval)) .Http(httpConfig))) { + // Get the count of status updates before starting, so we can check the new one + var initialStatusCount = _updateSink.GetAllStatusUpdates().Count; var initTask = dataSource.Start(); + bool completed = initTask.Wait(TimeSpan.FromSeconds(1)); Assert.True(completed); Assert.True(dataSource.Initialized); - var status = _updateSink.StatusUpdates.ExpectValue(); + // Check the status update that occurred during initialization + var allUpdates = _updateSink.GetAllStatusUpdates(); + Assert.True(allUpdates.Count > initialStatusCount, "Expected at least one new status update"); + var status = allUpdates[initialStatusCount]; errorCondition.VerifyDataSourceStatusError(status); + Assert.True(status.LastError.Value.Recoverable, "Recoverable should be true for recoverable errors"); // We don't check here for a second status update to the Valid state, because that was // done by DataSourceUpdatesImpl when Init was called - our test fixture doesn't do it. diff --git a/pkgs/sdk/server/test/Internal/DataSources/StreamingDataSourceTest.cs b/pkgs/sdk/server/test/Internal/DataSources/StreamingDataSourceTest.cs index 90e2e49c..0a737dd3 100644 --- a/pkgs/sdk/server/test/Internal/DataSources/StreamingDataSourceTest.cs +++ b/pkgs/sdk/server/test/Internal/DataSources/StreamingDataSourceTest.cs @@ -198,6 +198,7 @@ public void VerifyRecoverableHttpError(int errorStatus) var status = _updateSink.StatusUpdates.ExpectValue(); errorCondition.VerifyDataSourceStatusError(status); + Assert.True(status.LastError.Value.Recoverable, "Recoverable should be true for recoverable errors"); // We don't check here for a second status update to the Valid state, because that was // done by DataSourceUpdatesImpl when Init was called - our test fixture doesn't do it. @@ -230,6 +231,7 @@ public void VerifyUnrecoverableHttpError(int errorStatus) var initTask = dataSource.Start(); var status = _updateSink.StatusUpdates.ExpectValue(); errorCondition.VerifyDataSourceStatusError(status); + Assert.False(status.LastError.Value.Recoverable, "Recoverable should be false for unrecoverable errors"); _updateSink.Inits.ExpectNoValue(); @@ -289,7 +291,11 @@ public async void StreamInitDiagnosticRecordedOnError() public void EventWithMalformedJsonCausesStreamRestart(string eventName, string data) { VerifyEventCausesStreamRestart(eventName, data, - err => Assert.Equal(DataSourceStatus.ErrorKind.InvalidData, err.Kind)); + err => + { + Assert.Equal(DataSourceStatus.ErrorKind.InvalidData, err.Kind); + Assert.True(err.Recoverable, "Recoverable should be true for invalid data errors"); + }); } [Theory] @@ -300,7 +306,11 @@ public void EventWithMalformedJsonCausesStreamRestart(string eventName, string d public void EventWithWellFormedJsonButInvalidDataCausesStreamRestart(string eventName, string data) { VerifyEventCausesStreamRestart(eventName, data, - err => Assert.Equal(DataSourceStatus.ErrorKind.InvalidData, err.Kind)); + err => + { + Assert.Equal(DataSourceStatus.ErrorKind.InvalidData, err.Kind); + Assert.True(err.Recoverable, "Recoverable should be true for invalid data errors"); + }); } [Theory] diff --git a/pkgs/sdk/server/test/Internal/FDv2DataSources/FDv2PollingDataSourceTest.cs b/pkgs/sdk/server/test/Internal/FDv2DataSources/FDv2PollingDataSourceTest.cs index dc06a4e8..12f16dbf 100644 --- a/pkgs/sdk/server/test/Internal/FDv2DataSources/FDv2PollingDataSourceTest.cs +++ b/pkgs/sdk/server/test/Internal/FDv2DataSources/FDv2PollingDataSourceTest.cs @@ -356,6 +356,7 @@ public void HttpErrorRecoverableUpdatesStatusToInterrupted() Assert.NotNull(status.LastError); Assert.Equal(DataSourceStatus.ErrorKind.ErrorResponse, status.LastError.Value.Kind); Assert.Equal(503, status.LastError.Value.StatusCode); + Assert.True(status.LastError.Value.Recoverable, "Recoverable should be true for recoverable errors"); Assert.False(dataSource.Initialized); } @@ -379,6 +380,7 @@ public async Task HttpErrorUnrecoverableUpdatesStatusToOffAndShutsDownDataSource Assert.Equal(DataSourceState.Off, status.State); Assert.NotNull(status.LastError); Assert.Equal(DataSourceStatus.ErrorKind.ErrorResponse, status.LastError.Value.Kind); + Assert.False(status.LastError.Value.Recoverable, "Recoverable should be false for unrecoverable errors"); Assert.False(dataSource.Initialized); } @@ -756,6 +758,7 @@ public async Task JsonErrorInEventUpdatesStatusToInterrupted() Assert.Equal(DataSourceState.Interrupted, status.State); Assert.NotNull(status.LastError); Assert.Equal(DataSourceStatus.ErrorKind.InvalidData, status.LastError.Value.Kind); + Assert.True(status.LastError.Value.Recoverable, "Recoverable should be true for invalid data errors"); Assert.Contains("Failed to deserialize", status.LastError.Value.Message); // Data source should not be initialized due to the error @@ -816,6 +819,7 @@ public async Task JsonErrorInEventHasInvalidDataErrorKind() Assert.Equal(DataSourceState.Interrupted, status.State); Assert.NotNull(status.LastError); Assert.Equal(DataSourceStatus.ErrorKind.InvalidData, status.LastError.Value.Kind); + Assert.True(status.LastError.Value.Recoverable, "Recoverable should be true for invalid data errors"); Assert.Contains("Failed to deserialize", status.LastError.Value.Message); // Data source should remain initialized @@ -845,6 +849,7 @@ public void RecoverableHttpErrorWithFallbackHeaderSetsFDv1Fallback() Assert.Equal(DataSourceStatus.ErrorKind.ErrorResponse, status.LastError.Value.Kind); Assert.Equal(503, status.LastError.Value.StatusCode); Assert.True(status.LastError.Value.FDv1Fallback, "FDv1Fallback should be true when fallback header is present"); + Assert.True(status.LastError.Value.Recoverable, "Recoverable should be true for recoverable errors"); Assert.False(dataSource.Initialized); } @@ -876,6 +881,7 @@ public async Task UnrecoverableHttpErrorWithFallbackHeaderSetsFDv1Fallback() Assert.Equal(DataSourceStatus.ErrorKind.ErrorResponse, status.LastError.Value.Kind); Assert.Equal(401, status.LastError.Value.StatusCode); Assert.True(status.LastError.Value.FDv1Fallback, "FDv1Fallback should be true when fallback header is present"); + Assert.False(status.LastError.Value.Recoverable, "Recoverable should be false for unrecoverable errors"); Assert.False(dataSource.Initialized); } @@ -903,6 +909,7 @@ public void RecoverableHttpErrorWithoutFallbackHeaderDoesNotSetFDv1Fallback() Assert.Equal(DataSourceStatus.ErrorKind.ErrorResponse, status.LastError.Value.Kind); Assert.Equal(503, status.LastError.Value.StatusCode); Assert.False(status.LastError.Value.FDv1Fallback, "FDv1Fallback should be false when fallback header is not present"); + Assert.True(status.LastError.Value.Recoverable, "Recoverable should be true for recoverable errors"); Assert.False(dataSource.Initialized); } @@ -931,6 +938,7 @@ public void RecoverableHttpErrorWithFallbackHeaderFalseDoesNotSetFDv1Fallback() Assert.Equal(DataSourceStatus.ErrorKind.ErrorResponse, status.LastError.Value.Kind); Assert.Equal(503, status.LastError.Value.StatusCode); Assert.False(status.LastError.Value.FDv1Fallback, "FDv1Fallback should be false when fallback header value is false"); + Assert.True(status.LastError.Value.Recoverable, "Recoverable should be true for recoverable errors"); Assert.False(dataSource.Initialized); } @@ -962,10 +970,102 @@ public async Task UnrecoverableHttpErrorWithFallbackHeaderFalseDoesNotSetFDv1Fal Assert.Equal(DataSourceStatus.ErrorKind.ErrorResponse, status.LastError.Value.Kind); Assert.Equal(401, status.LastError.Value.StatusCode); Assert.False(status.LastError.Value.FDv1Fallback, "FDv1Fallback should be false when fallback header value is false"); + Assert.False(status.LastError.Value.Recoverable, "Recoverable should be false for unrecoverable errors"); Assert.False(dataSource.Initialized); } } } + + [Fact] + public async Task UnrecoverableHttpError403ReportsOffStatusWithRecoverableFalse() + { + _mockRequestor.Setup(r => r.PollingRequestAsync(It.IsAny())) + .ThrowsAsync(new UnsuccessfulResponseException(403)); + + using (var dataSource = MakeDataSource()) + { + var startTask = dataSource.Start(); + + var result = await startTask; + Assert.False(result); // Init task completes even on error + + var status = _updateSink.StatusUpdates.ExpectValue(); + Assert.Equal(DataSourceState.Off, status.State); + Assert.NotNull(status.LastError); + Assert.Equal(DataSourceStatus.ErrorKind.ErrorResponse, status.LastError.Value.Kind); + Assert.Equal(403, status.LastError.Value.StatusCode); + Assert.False(status.LastError.Value.Recoverable, "Recoverable should be false for unrecoverable errors"); + + Assert.False(dataSource.Initialized); + } + } + + [Fact] + public async Task UnrecoverableHttpError401ReportsOffStatusWithRecoverableFalse() + { + _mockRequestor.Setup(r => r.PollingRequestAsync(It.IsAny())) + .ThrowsAsync(new UnsuccessfulResponseException(401)); + + using (var dataSource = MakeDataSource()) + { + var startTask = dataSource.Start(); + + var result = await startTask; + Assert.False(result); // Init task completes even on error + + var status = _updateSink.StatusUpdates.ExpectValue(); + Assert.Equal(DataSourceState.Off, status.State); + Assert.NotNull(status.LastError); + Assert.Equal(DataSourceStatus.ErrorKind.ErrorResponse, status.LastError.Value.Kind); + Assert.Equal(401, status.LastError.Value.StatusCode); + Assert.False(status.LastError.Value.Recoverable, "Recoverable should be false for unrecoverable errors"); + + Assert.False(dataSource.Initialized); + } + } + + [Fact] + public async Task UnrecoverableHttpErrorAfterInitializationReportsOffStatusWithRecoverableFalse() + { + var callCount = 0; + _mockRequestor.Setup(r => r.PollingRequestAsync(It.IsAny())) + .ReturnsAsync(() => + { + callCount++; + if (callCount == 1) + { + // First poll: return server-intent "none" to initialize + return CreatePollingResponse( + CreateServerIntentEvent("none", "test-payload", 1) + ); + } + else + { + // Second poll: throw unrecoverable error + throw new UnsuccessfulResponseException(403); + } + }); + + using (var dataSource = MakeDataSource()) + { + var startTask = dataSource.Start(); + + var result = await startTask; + Assert.True(result); + Assert.True(dataSource.Initialized); + + // Wait for second poll to happen and trigger the error + await Task.Delay(TimeSpan.FromMilliseconds(100)); + + // Now trigger an unrecoverable error after initialization + var status = _updateSink.StatusUpdates.ExpectValue(); + Assert.Equal(DataSourceState.Off, status.State); + Assert.NotNull(status.LastError); + Assert.Equal(DataSourceStatus.ErrorKind.ErrorResponse, status.LastError.Value.Kind); + Assert.Equal(403, status.LastError.Value.StatusCode); + Assert.False(status.LastError.Value.Recoverable, "Recoverable should be false for unrecoverable errors"); + } + } } } diff --git a/pkgs/sdk/server/test/Internal/FDv2DataSources/FDv2StreamingDataSourceTest.cs b/pkgs/sdk/server/test/Internal/FDv2DataSources/FDv2StreamingDataSourceTest.cs index 4d319ccf..53dd537f 100644 --- a/pkgs/sdk/server/test/Internal/FDv2DataSources/FDv2StreamingDataSourceTest.cs +++ b/pkgs/sdk/server/test/Internal/FDv2DataSources/FDv2StreamingDataSourceTest.cs @@ -432,6 +432,7 @@ public void MalformedJsonTriggersInvalidDataErrorAndStreamRestart() Assert.Equal(DataSourceState.Interrupted, status.State); Assert.True(status.LastError.HasValue); Assert.Equal(DataSourceStatus.ErrorKind.InvalidData, status.LastError.Value.Kind); + Assert.True(status.LastError.Value.Recoverable, "Recoverable should be true for invalid data errors"); } } @@ -502,6 +503,7 @@ public void StoreFailureWithStatusMonitoringUpdatesStatusWithoutRestart() Assert.Equal(DataSourceState.Interrupted, status.State); Assert.NotNull(status.LastError); Assert.Equal(DataSourceStatus.ErrorKind.StoreError, status.LastError.Value.Kind); + Assert.True(status.LastError.Value.Recoverable, "Recoverable should be true for store errors"); } } @@ -565,6 +567,7 @@ public void RecoverableHttpErrorUpdatesStatusAndLogsWarning() Assert.NotNull(status.LastError); Assert.Equal(DataSourceStatus.ErrorKind.ErrorResponse, status.LastError.Value.Kind); Assert.Equal(503, status.LastError.Value.StatusCode); + Assert.True(status.LastError.Value.Recoverable, "Recoverable should be true for recoverable errors"); AssertLogMessageRegex(true, LogLevel.Warn, ".*will retry.*"); } @@ -590,6 +593,7 @@ public void RecoverableHttpErrorWithFallbackHeaderSetsFDv1Fallback() Assert.Equal(DataSourceStatus.ErrorKind.ErrorResponse, status.LastError.Value.Kind); Assert.Equal(503, status.LastError.Value.StatusCode); Assert.True(status.LastError.Value.FDv1Fallback, "FDv1Fallback should be true when fallback header is present"); + Assert.True(status.LastError.Value.Recoverable, "Recoverable should be true for recoverable errors"); AssertLogMessageRegex(true, LogLevel.Warn, ".*will retry.*"); } @@ -610,6 +614,7 @@ public async Task UnrecoverableHttpErrorStopsInitializationAndShutsDown() Assert.NotNull(status.LastError); Assert.Equal(DataSourceStatus.ErrorKind.ErrorResponse, status.LastError.Value.Kind); Assert.Equal(403, status.LastError.Value.StatusCode); + Assert.False(status.LastError.Value.Recoverable, "Recoverable should be false for unrecoverable errors"); var result = await startTask; Assert.False(result); @@ -639,6 +644,7 @@ public async Task UnrecoverableHttpErrorWithFallbackHeaderSetsFDv1Fallback() Assert.Equal(DataSourceStatus.ErrorKind.ErrorResponse, status.LastError.Value.Kind); Assert.Equal(401, status.LastError.Value.StatusCode); Assert.True(status.LastError.Value.FDv1Fallback, "FDv1Fallback should be true when fallback header is present"); + Assert.False(status.LastError.Value.Recoverable, "Recoverable should be false for unrecoverable errors"); var result = await startTask; Assert.False(result); @@ -665,6 +671,7 @@ public void RecoverableHttpErrorWithoutFallbackHeaderDoesNotSetFDv1Fallback() Assert.Equal(DataSourceStatus.ErrorKind.ErrorResponse, status.LastError.Value.Kind); Assert.Equal(503, status.LastError.Value.StatusCode); Assert.False(status.LastError.Value.FDv1Fallback, "FDv1Fallback should be false when fallback header is not present"); + Assert.True(status.LastError.Value.Recoverable, "Recoverable should be true for recoverable errors"); AssertLogMessageRegex(true, LogLevel.Warn, ".*will retry.*"); } @@ -690,6 +697,7 @@ public void RecoverableHttpErrorWithFallbackHeaderFalseDoesNotSetFDv1Fallback() Assert.Equal(DataSourceStatus.ErrorKind.ErrorResponse, status.LastError.Value.Kind); Assert.Equal(503, status.LastError.Value.StatusCode); Assert.False(status.LastError.Value.FDv1Fallback, "FDv1Fallback should be false when fallback header value is false"); + Assert.True(status.LastError.Value.Recoverable, "Recoverable should be true for recoverable errors"); AssertLogMessageRegex(true, LogLevel.Warn, ".*will retry.*"); } @@ -715,6 +723,7 @@ public async Task UnrecoverableHttpErrorWithFallbackHeaderFalseDoesNotSetFDv1Fal Assert.Equal(DataSourceStatus.ErrorKind.ErrorResponse, status.LastError.Value.Kind); Assert.Equal(401, status.LastError.Value.StatusCode); Assert.False(status.LastError.Value.FDv1Fallback, "FDv1Fallback should be false when fallback header value is false"); + Assert.False(status.LastError.Value.Recoverable, "Recoverable should be false for unrecoverable errors"); var result = await startTask; Assert.False(result); @@ -724,6 +733,79 @@ public async Task UnrecoverableHttpErrorWithFallbackHeaderFalseDoesNotSetFDv1Fal } } + [Fact] + public async Task UnrecoverableHttpError403ReportsOffStatusWithRecoverableFalse() + { + using (var dataSource = MakeDataSource()) + { + var startTask = dataSource.Start(); + + var exception = new EventSourceServiceUnsuccessfulResponseException(403); + _mockEventSource.TriggerError(exception); + + var status = _updateSink.StatusUpdates.ExpectValue(); + Assert.Equal(DataSourceState.Off, status.State); + Assert.NotNull(status.LastError); + Assert.Equal(DataSourceStatus.ErrorKind.ErrorResponse, status.LastError.Value.Kind); + Assert.Equal(403, status.LastError.Value.StatusCode); + Assert.False(status.LastError.Value.Recoverable, "Recoverable should be false for unrecoverable errors"); + + var result = await startTask; + Assert.False(result); + Assert.False(dataSource.Initialized); + } + } + + [Fact] + public async Task UnrecoverableHttpError401ReportsOffStatusWithRecoverableFalse() + { + using (var dataSource = MakeDataSource()) + { + var startTask = dataSource.Start(); + + var exception = new EventSourceServiceUnsuccessfulResponseException(401); + _mockEventSource.TriggerError(exception); + + var status = _updateSink.StatusUpdates.ExpectValue(); + Assert.Equal(DataSourceState.Off, status.State); + Assert.NotNull(status.LastError); + Assert.Equal(DataSourceStatus.ErrorKind.ErrorResponse, status.LastError.Value.Kind); + Assert.Equal(401, status.LastError.Value.StatusCode); + Assert.False(status.LastError.Value.Recoverable, "Recoverable should be false for unrecoverable errors"); + + var result = await startTask; + Assert.False(result); + Assert.False(dataSource.Initialized); + } + } + + [Fact] + public async Task UnrecoverableHttpErrorAfterInitializationReportsOffStatusWithRecoverableFalse() + { + using (var dataSource = MakeDataSource()) + { + var startTask = dataSource.Start(); + + _mockEventSource.TriggerOpen(); + _mockEventSource.TriggerMessage(CreateMessageEvent("server-intent", + CreateServerIntentJson("none", "test-payload", 1))); + + await startTask; + Assert.True(dataSource.Initialized); + + // Now trigger an unrecoverable error after initialization + var exception = new EventSourceServiceUnsuccessfulResponseException(403); + _mockEventSource.TriggerError(exception); + + var status = _updateSink.StatusUpdates.ExpectValue(); + Assert.Equal(DataSourceState.Off, status.State); + Assert.NotNull(status.LastError); + Assert.Equal(DataSourceStatus.ErrorKind.ErrorResponse, status.LastError.Value.Kind); + Assert.Equal(403, status.LastError.Value.StatusCode); + Assert.False(status.LastError.Value.Recoverable, "Recoverable should be false for unrecoverable errors"); + } + } + [Fact] public void NetworkErrorUpdatesStatusToInterrupted() { @@ -738,6 +820,7 @@ public void NetworkErrorUpdatesStatusToInterrupted() Assert.Equal(DataSourceState.Interrupted, status.State); Assert.NotNull(status.LastError); Assert.Equal(DataSourceStatus.ErrorKind.NetworkError, status.LastError.Value.Kind); + Assert.True(status.LastError.Value.Recoverable, "Recoverable should be true for network errors"); AssertLogMessageRegex(true, LogLevel.Warn, ".*EventSource error.*"); } @@ -816,6 +899,9 @@ public void DisposeShutsDownEventSource() dataSource.Dispose(); Assert.True(_mockEventSource.IsClosed); + + var status = _updateSink.StatusUpdates.ExpectValue(); + Assert.Equal(DataSourceState.Off, status.State); } [Fact] @@ -829,6 +915,9 @@ public void DisposeUnsubscribesFromDataStoreStatusEvents() dataSource.Dispose(); + var status = _updateSink.StatusUpdates.ExpectValue(); + Assert.Equal(DataSourceState.Off, status.State); + _updateSink.MockDataStoreStatusProvider.FireStatusChanged( new DataStoreStatus { Available = true, RefreshNeeded = true }); @@ -943,6 +1032,7 @@ public void PartialTransferStopsOnFirstApplyFailure() Assert.Equal(DataSourceState.Interrupted, status.State); Assert.NotNull(status.LastError); Assert.Equal(DataSourceStatus.ErrorKind.StoreError, status.LastError.Value.Kind); + Assert.True(status.LastError.Value.Recoverable, "Recoverable should be true for store errors"); } } diff --git a/pkgs/sdk/server/test/LdClientEndToEndTest.cs b/pkgs/sdk/server/test/LdClientEndToEndTest.cs index 2af22f5a..4c50b15d 100644 --- a/pkgs/sdk/server/test/LdClientEndToEndTest.cs +++ b/pkgs/sdk/server/test/LdClientEndToEndTest.cs @@ -57,6 +57,9 @@ public void ClientFailsToStartInStreamingModeWith401Error() using (var client = new LdClient(config)) { Assert.False(client.Initialized); + // Wait for the status to be updated to Off, as the status update happens asynchronously + var statusUpdated = client.DataSourceStatusProvider.WaitFor(DataSourceState.Off, TimeSpan.FromSeconds(5)); + Assert.True(statusUpdated, "Status should have been updated to Off"); Assert.Equal(DataSourceState.Off, client.DataSourceStatusProvider.Status.State); var value = client.BoolVariation(AlwaysTrueFlag.Key, BasicUser, false); @@ -132,6 +135,9 @@ public void ClientRetriesConnectionInStreamingModeWithNonFatalError() using (var client = new LdClient(config)) { Assert.True(client.Initialized); + // Wait for the status to be updated to Valid, as the status update happens asynchronously + var statusUpdated = client.DataSourceStatusProvider.WaitFor(DataSourceState.Valid, TimeSpan.FromSeconds(5)); + Assert.True(statusUpdated, "Status should have been updated to Valid"); Assert.Equal(DataSourceState.Valid, client.DataSourceStatusProvider.Status.State); var value = client.BoolVariation(AlwaysTrueFlag.Key, BasicUser, false); @@ -187,6 +193,9 @@ public void ClientFailsToStartInPollingModeWith401Error() using (var client = new LdClient(config)) { Assert.False(client.Initialized); + // Wait for the status to be updated to Off, as the status update happens asynchronously + var statusUpdated = client.DataSourceStatusProvider.WaitFor(DataSourceState.Off, TimeSpan.FromSeconds(5)); + Assert.True(statusUpdated, "Status should have been updated to Off"); Assert.Equal(DataSourceState.Off, client.DataSourceStatusProvider.Status.State); var value = client.BoolVariation(AlwaysTrueFlag.Key, BasicUser, false); @@ -219,6 +228,9 @@ public void ClientRetriesConnectionInPollingModeWithNonFatalError() using (var client = new LdClient(config)) { Assert.True(client.Initialized); + // Wait for the status to be updated to Valid, as the status update happens asynchronously + var statusUpdated = client.DataSourceStatusProvider.WaitFor(DataSourceState.Valid, TimeSpan.FromSeconds(5)); + Assert.True(statusUpdated, "Status should have been updated to Valid"); Assert.Equal(DataSourceState.Valid, client.DataSourceStatusProvider.Status.State); var value = client.BoolVariation(AlwaysTrueFlag.Key, BasicUser, false); @@ -354,6 +366,9 @@ private static FullDataSet MakeExpectedData() => private static void VerifyClientStartedAndHasExpectedData(LdClient client, HttpServer server) { + // Wait for the status to be updated to Valid, as the status update happens asynchronously + var statusUpdated = client.DataSourceStatusProvider.WaitFor(DataSourceState.Valid, TimeSpan.FromSeconds(5)); + Assert.True(statusUpdated, "Status should have been updated to Valid"); Assert.Equal(DataSourceState.Valid, client.DataSourceStatusProvider.Status.State); Assert.True(client.Initialized); diff --git a/pkgs/sdk/server/test/LdClientListenersTest.cs b/pkgs/sdk/server/test/LdClientListenersTest.cs index 7ff243f4..c1756de9 100644 --- a/pkgs/sdk/server/test/LdClientListenersTest.cs +++ b/pkgs/sdk/server/test/LdClientListenersTest.cs @@ -119,7 +119,7 @@ public void DataSourceStatusProviderReturnsLatestStatus() Assert.True(initialStatus.StateSince >= timeBeforeStarting); Assert.Null(initialStatus.LastError); - var errorInfo = DataSourceStatus.ErrorInfo.FromHttpError(401); + var errorInfo = DataSourceStatus.ErrorInfo.FromHttpError(401, false); testData.UpdateStatus(DataSourceState.Off, errorInfo); var newStatus = client.DataSourceStatusProvider.Status; @@ -142,7 +142,7 @@ public void DataSourceStatusProviderSendsStatusUpdates() var statuses = new EventSink(); client.DataSourceStatusProvider.StatusChanged += statuses.Add; - var errorInfo = DataSourceStatus.ErrorInfo.FromHttpError(401); + var errorInfo = DataSourceStatus.ErrorInfo.FromHttpError(401, false); testData.UpdateStatus(DataSourceState.Off, errorInfo); var newStatus = statuses.ExpectValue();