From c7ef6f083a0500d1f2fa2e3fae66914c53704872 Mon Sep 17 00:00:00 2001 From: Ari Sulistiono Date: Wed, 19 Aug 2026 11:09:10 +0700 Subject: [PATCH 01/13] chore(p6.2-c): stage bounded reconnect patch applicator --- .github/workflows/p6-2-c-apply-patch.yml | 35 ++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/p6-2-c-apply-patch.yml diff --git a/.github/workflows/p6-2-c-apply-patch.yml b/.github/workflows/p6-2-c-apply-patch.yml new file mode 100644 index 000000000..b1f9ad659 --- /dev/null +++ b/.github/workflows/p6-2-c-apply-patch.yml @@ -0,0 +1,35 @@ +name: P6.2-C apply smart reconnect patch + +on: + push: + branches: + - fix/p6-2-c-smart-reconnect + paths: + - .p6-2-c/smart-reconnect.patch + +permissions: + contents: write + +jobs: + apply-patch: + runs-on: ubuntu-latest + steps: + - name: Checkout branch + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Apply reviewed P6.2-C patch + shell: bash + run: | + set -euo pipefail + git apply --check .p6-2-c/smart-reconnect.patch + git apply .p6-2-c/smart-reconnect.patch + rm -f .p6-2-c/smart-reconnect.patch + rmdir .p6-2-c 2>/dev/null || true + rm -f .github/workflows/p6-2-c-apply-patch.yml + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git commit -m "fix(p6.2-c): make MMS recovery bounded and staged" + git push origin HEAD:fix/p6-2-c-smart-reconnect From 496c9f26d646c6f973a880c164f68b229ee72ba7 Mon Sep 17 00:00:00 2001 From: Ari Sulistiono Date: Wed, 19 Aug 2026 11:09:45 +0700 Subject: [PATCH 02/13] chore(p6.2-c): stage patch parts before applying --- .github/workflows/p6-2-c-apply-patch.yml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/p6-2-c-apply-patch.yml b/.github/workflows/p6-2-c-apply-patch.yml index b1f9ad659..b40c6163e 100644 --- a/.github/workflows/p6-2-c-apply-patch.yml +++ b/.github/workflows/p6-2-c-apply-patch.yml @@ -5,7 +5,7 @@ on: branches: - fix/p6-2-c-smart-reconnect paths: - - .p6-2-c/smart-reconnect.patch + - .p6-2-c/apply.ready permissions: contents: write @@ -19,14 +19,22 @@ jobs: with: fetch-depth: 0 - - name: Apply reviewed P6.2-C patch + - name: Apply reviewed P6.2-C patch parts shell: bash run: | set -euo pipefail - git apply --check .p6-2-c/smart-reconnect.patch - git apply .p6-2-c/smart-reconnect.patch - rm -f .p6-2-c/smart-reconnect.patch - rmdir .p6-2-c 2>/dev/null || true + shopt -s nullglob + patches=(.p6-2-c/*.patch) + if [ ${#patches[@]} -eq 0 ]; then + echo "No staged patch files were found." + exit 1 + fi + for patch in "${patches[@]}"; do + echo "Checking $patch" + git apply --check "$patch" + git apply "$patch" + done + rm -rf .p6-2-c rm -f .github/workflows/p6-2-c-apply-patch.yml git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" From bac26ea858e7d6847ff4390c4ddf3f8638e0ac53 Mon Sep 17 00:00:00 2001 From: Ari Sulistiono Date: Wed, 19 Aug 2026 11:11:59 +0700 Subject: [PATCH 03/13] chore(p6.2-c): stage reconnect runtime patch part 1 --- .p6-2-c/01-runtime-early.patch | 115 +++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 .p6-2-c/01-runtime-early.patch diff --git a/.p6-2-c/01-runtime-early.patch b/.p6-2-c/01-runtime-early.patch new file mode 100644 index 000000000..e53b7ed06 --- /dev/null +++ b/.p6-2-c/01-runtime-early.patch @@ -0,0 +1,115 @@ +diff --git a/Services/Iec61850MonitorRuntime.cs b/Services/Iec61850MonitorRuntime.cs +index 34eda3f..5c219d0 100644 +--- a/Services/Iec61850MonitorRuntime.cs ++++ b/Services/Iec61850MonitorRuntime.cs +@@ -39,6 +39,7 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable + public bool CommandReportMissLogged { get; set; } + public bool StaleReportSuppressedLogged { get; set; } + public bool ReportValueRejectedLogged { get; set; } ++ public string LastLoggedDegradedQuality { get; set; } = string.Empty; + public int ConsecutiveErrors { get; set; } + } + +@@ -73,6 +74,8 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable + public DateTime ReportSetupNotBeforeUtc { get; set; } = DateTime.MinValue; + public DateTime ReportSetupDeadlineUtc { get; set; } = DateTime.MinValue; + public DateTime NextReconnectUtc { get; set; } = DateTime.MinValue; ++ public int ConsecutiveReconnectFailures { get; set; } ++ public DateTime RecoveryWarmupUntilUtc { get; set; } = DateTime.MinValue; + public int ConsecutiveSessionErrors { get; set; } + public DateTime LastSuccessfulIoUtc { get; set; } = DateTime.UtcNow; + public DateTime NextHealthProbeUtc { get; set; } = DateTime.MinValue; +@@ -225,7 +228,11 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable + { + device.IsConnected = false; + _sessions.TryRemove(device.DeviceId, out _); +- await session.Client.DisposeAsync().ConfigureAwait(false); ++ await DisposeClientForReconnectAsync( ++ session.Client, ++ device.Name, ++ SmartReconnectPolicy.ClientCleanupBudget, ++ CancellationToken.None).ConfigureAwait(false); + } + throw; + } +@@ -283,7 +290,7 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable + 2, + 4)); + +- await session.Client.ConnectAsync(device.IpAddress, device.Port, cancellationToken).ConfigureAwait(false); ++ await ConnectCachedAssociationWithRetryAsync(session, cancellationToken).ConfigureAwait(false); + if (!session.Client.IsConnected) + { + device.Status = "Connection failed"; +@@ -326,7 +333,11 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable + { + device.IsConnected = false; + _sessions.TryRemove(device.DeviceId, out _); +- await session.Client.DisposeAsync().ConfigureAwait(false); ++ await DisposeClientForReconnectAsync( ++ session.Client, ++ device.Name, ++ SmartReconnectPolicy.ClientCleanupBudget, ++ CancellationToken.None).ConfigureAwait(false); + } + throw; + } +@@ -374,6 +385,8 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable + session.ReportSetupPending = false; + session.ReportSetupNotBeforeUtc = DateTime.MinValue; + session.ReportSetupDeadlineUtc = DateTime.MinValue; ++ session.ConsecutiveReconnectFailures = 0; ++ session.RecoveryWarmupUntilUtc = DateTime.MinValue; + session.ConsecutiveSessionErrors = 0; + session.LastSuccessfulIoUtc = DateTime.UtcNow; + session.NextHealthProbeUtc = DateTime.UtcNow.AddSeconds(1); +@@ -789,7 +802,9 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable + cancellationToken).ConfigureAwait(false); + + await StartReportPlansAsync(session, plans, cancellationToken).ConfigureAwait(false); +- ResetPollQueue(session); ++ ResetPollQueue( ++ session, ++ staggerForRecovery: DateTime.UtcNow < session.RecoveryWarmupUntilUtc); + UpdateDeviceAcquisitionSummary(session); + } + +@@ -1185,7 +1200,9 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable + var reportAssigned = session.PointPlanIds.ContainsKey(point.PointKey); + cancellationToken.ThrowIfCancellationRequested(); + processed++; +- var nextIntervalMs = GetVerificationPollIntervalMs(point, state, reportAssigned); ++ var nextIntervalMs = SmartReconnectPolicy.ApplyRecoveryPollFloor( ++ GetVerificationPollIntervalMs(point, state, reportAssigned), ++ nowUtc < session.RecoveryWarmupUntilUtc); + state.NextPollUtc = nowUtc.AddMilliseconds(nextIntervalMs); + session.PollQueue.Enqueue(point.PointKey, state.NextPollUtc.Ticks); + +@@ -1228,6 +1245,7 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable + var deviceTimestamp = rich?.HasDeviceTimestamp == true ? rich.DeviceTimestamp : state.DeviceTimestamp; + + if ((rich?.HasQuality != true || rich?.HasDeviceTimestamp != true) && ++ nowUtc >= session.RecoveryWarmupUntilUtc && + nowUtc >= state.NextCompanionPollUtc) + { + state.NextCompanionPollUtc = nowUtc.AddMilliseconds(GetCompanionPollIntervalMs(point)); +@@ -1345,6 +1363,19 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable + var changed = hasProcessValue && hadValue && (trustReportEdge + ? HasExactSemanticEdge(point, oldValue, display) + : HasMeaningfulEdge(point, oldValue, display)); ++ ++ if (qualityChangedForUi && IsDegradedIecQuality(quality) && ++ !state.LastLoggedDegradedQuality.Equals(quality, StringComparison.OrdinalIgnoreCase)) ++ { ++ state.LastLoggedDegradedQuality = quality; ++ Log("INFO", session.Device.Name, ++ $"QUALITY_EVIDENCE: {point.SignalName} ({point.IecReference}) quality={quality}; acquisition={sourceMode}; qRef={(string.IsNullOrWhiteSpace(point.QualityReference) ? "derived companion/report q" : point.QualityReference)}; timestamp={(string.IsNullOrWhiteSpace(deviceTimestamp) ? "-" : deviceTimestamp)}. Quality is preserved from IED evidence and is not converted to Good."); ++ } ++ else if (!IsDegradedIecQuality(quality)) ++ { ++ state.LastLoggedDegradedQuality = string.Empty; ++ } ++ + if (hasProcessValue) + { + state.HasValue = true; From e84395dc5e094ed453dd6ec6e299ebe9bee7ae08 Mon Sep 17 00:00:00 2001 From: Ari Sulistiono Date: Wed, 19 Aug 2026 11:12:42 +0700 Subject: [PATCH 04/13] chore(p6.2-c): stage reconnect runtime patch part 2 --- .p6-2-c/02-runtime-reconnect.patch | 277 +++++++++++++++++++++++++++++ 1 file changed, 277 insertions(+) create mode 100644 .p6-2-c/02-runtime-reconnect.patch diff --git a/.p6-2-c/02-runtime-reconnect.patch b/.p6-2-c/02-runtime-reconnect.patch new file mode 100644 index 000000000..8025ee12a --- /dev/null +++ b/.p6-2-c/02-runtime-reconnect.patch @@ -0,0 +1,277 @@ +diff --git a/Services/Iec61850MonitorRuntime.cs b/Services/Iec61850MonitorRuntime.cs +index 34eda3f..5c219d0 100644 +--- a/Services/Iec61850MonitorRuntime.cs ++++ b/Services/Iec61850MonitorRuntime.cs +@@ -1426,58 +1457,243 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable + }); + } + ++ private async Task ConnectCachedAssociationWithRetryAsync( ++ DeviceSession session, ++ CancellationToken cancellationToken) ++ { ++ const int maxAttempts = 2; ++ for (var attempt = 1; attempt <= maxAttempts; attempt++) ++ { ++ using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); ++ timeout.CancelAfter(SmartReconnectPolicy.ConnectBudget); ++ try ++ { ++ await session.Client.ConnectAsync( ++ session.Device.IpAddress, ++ session.Device.Port, ++ timeout.Token).ConfigureAwait(false); ++ } ++ catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) ++ { ++ session.Device.LastDiagnosticSnapshot = session.Client.CaptureDiagnosticSnapshot( ++ $"Fast saved-model connect attempt #{attempt} timed out"); ++ } ++ ++ if (session.Client.IsConnected) ++ { ++ if (attempt > 1) ++ { ++ Log("INFO", session.Device.Name, ++ $"Fast saved-model connection recovered automatically on attempt #{attempt}; no manual Play retry was required."); ++ } ++ return; ++ } ++ ++ if (attempt >= maxAttempts) ++ return; ++ ++ var failure = string.IsNullOrWhiteSpace(session.Client.LastErrorMessage) ++ ? $"native state={session.Client.NativeState}" ++ : session.Client.LastErrorMessage; ++ Log("WARN", session.Device.Name, ++ $"Fast saved-model connection attempt #{attempt} did not establish MMS ({failure}). Retrying once after {SmartReconnectPolicy.InitialAssociationRetryDelay.TotalMilliseconds:0} ms."); ++ ++ var staleClient = session.Client; ++ session.Client = new NativeIec61850Client(); ++ await DisposeClientForReconnectAsync( ++ staleClient, ++ session.Device.Name, ++ SmartReconnectPolicy.ClientCleanupBudget, ++ cancellationToken).ConfigureAwait(false); ++ await Task.Delay(SmartReconnectPolicy.InitialAssociationRetryDelay, cancellationToken).ConfigureAwait(false); ++ } ++ } ++ + private async Task TryReconnectAsync(DeviceSession session, CancellationToken cancellationToken) + { +- if (DateTime.UtcNow < session.NextReconnectUtc) return; +- session.NextReconnectUtc = DateTime.UtcNow.AddSeconds(2); ++ var nowUtc = DateTime.UtcNow; ++ if (nowUtc < session.NextReconnectUtc) ++ return; ++ ++ var attempt = session.ConsecutiveReconnectFailures + 1; ++ var reconnectStopwatch = Stopwatch.StartNew(); + MarkSessionOffline(session, $"Reconnecting MMS association to {session.Device.EndpointText}."); + session.Device.Status = "Reconnecting"; +- Log("WARN", session.Device.Name, "IEC 61850 session is offline. Smart reconnect started."); +- try { await session.Client.DisposeAsync().ConfigureAwait(false); } catch { } +- session.Client = new NativeIec61850Client(); +- try { await session.Client.ConnectAsync(session.Device.IpAddress, session.Device.Port, cancellationToken).ConfigureAwait(false); } +- catch (OperationCanceledException) { throw; } ++ session.Device.Detail = $"Smart reconnect attempt #{attempt}: opening a fresh MMS association."; ++ session.Device.RefreshComputed(); ++ Log("WARN", session.Device.Name, ++ $"Smart reconnect attempt #{attempt} started. Transport recovery is bounded independently from report re-arming."); ++ ++ var staleClient = session.Client; ++ await DisposeClientForReconnectAsync( ++ staleClient, ++ session.Device.Name, ++ SmartReconnectPolicy.ClientCleanupBudget, ++ cancellationToken).ConfigureAwait(false); ++ ++ var replacement = new NativeIec61850Client(); ++ session.Client = replacement; ++ ++ using var connectTimeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); ++ connectTimeout.CancelAfter(SmartReconnectPolicy.ConnectBudget); ++ try ++ { ++ await replacement.ConnectAsync( ++ session.Device.IpAddress, ++ session.Device.Port, ++ connectTimeout.Token).ConfigureAwait(false); ++ } ++ catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) ++ { ++ session.Device.LastDiagnosticSnapshot = replacement.CaptureDiagnosticSnapshot( ++ "Smart reconnect association timed out"); ++ ScheduleReconnectRetry( ++ session, ++ attempt, ++ reconnectStopwatch.Elapsed, ++ $"MMS association exceeded the {SmartReconnectPolicy.ConnectBudget.TotalSeconds:0.#} s reconnect budget."); ++ return; ++ } ++ catch (OperationCanceledException) ++ { ++ throw; ++ } + catch (Exception ex) + { +- session.Device.Status = "Reconnect pending"; +- session.Device.Detail = ex.Message; +- session.Device.RefreshComputed(); ++ session.Device.LastDiagnosticSnapshot = replacement.CaptureDiagnosticSnapshot( ++ "Smart reconnect association failed", ++ ex); ++ ScheduleReconnectRetry( ++ session, ++ attempt, ++ reconnectStopwatch.Elapsed, ++ $"{ex.GetType().Name}: {ex.Message}"); + return; + } +- if (!session.Client.IsConnected) ++ ++ if (!replacement.IsConnected) + { +- session.Device.Status = "Reconnect pending"; +- session.Device.Detail = session.Client.LastErrorMessage; +- session.Device.RefreshComputed(); ++ session.Device.LastDiagnosticSnapshot = replacement.CaptureDiagnosticSnapshot( ++ "Smart reconnect association not established"); ++ ScheduleReconnectRetry( ++ session, ++ attempt, ++ reconnectStopwatch.Elapsed, ++ string.IsNullOrWhiteSpace(replacement.LastErrorMessage) ++ ? $"Native state={replacement.NativeState}; MMS association was not established." ++ : replacement.LastErrorMessage); + return; + } ++ ++ // Connection recovery and report recovery are intentionally separate stages. ++ // Once ACSE/MMS is healthy, resume bounded MMS reads immediately. Static RCB ++ // discovery/re-arming returns to the normal background report pipeline so a ++ // slow vendor RCB read/write can never hold the reconnect state machine hostage. + session.ActiveReportPlans.Clear(); + session.ActiveReportPlanOrder.Clear(); + session.PointPlanIds.Clear(); + session.ReportStreams.Clear(); + session.LastUnroutedReportCount = 0; +- session.PendingReportPlans = Array.Empty(); +- session.ReportSetupPending = false; +- session.ReportSetupNotBeforeUtc = DateTime.MinValue; +- session.ReportSetupDeadlineUtc = DateTime.MinValue; + ResetAssociationReportEvidence(session); ++ + var legacyPlans = Iec61850ReportPlanner.BuildPlans(session.Device, session.Points.Values); +- var plans = await BuildReportPlansForCurrentAssociationAsync( +- session, +- legacyPlans, +- cancellationToken).ConfigureAwait(false); +- await StartReportPlansAsync(session, plans, cancellationToken).ConfigureAwait(false); +- ResetPollQueue(session); +- UpdateDeviceAcquisitionSummary(session); ++ session.PendingReportPlans = legacyPlans; ++ session.ReportSetupPending = ++ replacement.CanUseHybridReportPlanner(session.Device) || ++ legacyPlans.Count > 0; ++ var associatedUtc = DateTime.UtcNow; ++ session.ReportSetupNotBeforeUtc = associatedUtc.Add(SmartReconnectPolicy.ReportRearmDelay); ++ session.ReportSetupDeadlineUtc = associatedUtc.Add(SmartReconnectPolicy.ReportRearmDeadline); ++ session.RecoveryWarmupUntilUtc = associatedUtc.Add(SmartReconnectPolicy.RecoveryWarmupDuration); ++ ++ ResetPollQueue(session, staggerForRecovery: true); ++ session.ConsecutiveReconnectFailures = 0; ++ session.NextReconnectUtc = DateTime.MinValue; + session.ConsecutiveSessionErrors = 0; + session.ConsecutiveHealthProbeFailures = 0; +- session.LastSuccessfulIoUtc = DateTime.UtcNow; +- session.NextHealthProbeUtc = DateTime.UtcNow.AddSeconds(1); ++ session.LastSuccessfulIoUtc = associatedUtc; ++ session.NextHealthProbeUtc = session.RecoveryWarmupUntilUtc; + session.Device.IsConnected = true; + session.Device.Status = "Monitoring"; +- session.Device.Detail = $"MMS reconnected. {session.Points.Count} point(s) resumed."; ++ session.Device.AcquisitionMode = "MMS recovered • static report re-arm pending"; ++ session.Device.Detail = ++ $"MMS reconnected in {reconnectStopwatch.Elapsed.TotalMilliseconds:0} ms. " + ++ $"{session.Points.Count} point(s) resumed with staggered MMS recovery; report re-arm continues in the background."; ++ session.Device.LastDiagnosticSnapshot = replacement.CaptureDiagnosticSnapshot( ++ "Smart reconnect MMS associated; report re-arm deferred"); + session.Device.RefreshComputed(); +- Log("INFO", session.Device.Name, "MMS reconnect successful. Monitoring resumed automatically."); ++ ++ Log("INFO", session.Device.Name, ++ $"MMS reconnect successful on attempt #{attempt} in {reconnectStopwatch.Elapsed.TotalMilliseconds:0} ms. " + ++ $"Polling resumed immediately with a {SmartReconnectPolicy.RecoveryWarmupDuration.TotalSeconds:0} s recovery warm-up; static report re-arm is deferred to the background pipeline."); ++ } ++ ++ private void ScheduleReconnectRetry( ++ DeviceSession session, ++ int attempt, ++ TimeSpan elapsed, ++ string detail) ++ { ++ session.ConsecutiveReconnectFailures = attempt; ++ var retryDelay = SmartReconnectPolicy.GetRetryDelay(attempt); ++ session.NextReconnectUtc = DateTime.UtcNow.Add(retryDelay); ++ session.Device.IsConnected = false; ++ session.Device.Status = "Reconnect pending"; ++ session.Device.AcquisitionMode = "Connection lost • reconnect pending"; ++ session.Device.Detail = ++ $"Smart reconnect attempt #{attempt} failed after {elapsed.TotalMilliseconds:0} ms: {detail} " + ++ $"Retry in {retryDelay.TotalSeconds:0.#} s."; ++ session.Device.RefreshComputed(); ++ Log("WARN", session.Device.Name, ++ $"Smart reconnect attempt #{attempt} failed after {elapsed.TotalMilliseconds:0} ms; retry in {retryDelay.TotalSeconds:0.#} s. {detail}"); ++ } ++ ++ private async Task DisposeClientForReconnectAsync( ++ NativeIec61850Client client, ++ string deviceName, ++ TimeSpan budget, ++ CancellationToken cancellationToken) ++ { ++ Task disposeTask; ++ try ++ { ++ disposeTask = client.DisposeAsync().AsTask(); ++ } ++ catch (Exception ex) ++ { ++ Log("WARN", deviceName, ++ $"Stale MMS client cleanup could not start during reconnect: {ex.GetType().Name}: {ex.Message}"); ++ return; ++ } ++ ++ var delayTask = Task.Delay(budget, cancellationToken); ++ var completed = await Task.WhenAny(disposeTask, delayTask).ConfigureAwait(false); ++ if (completed == disposeTask) ++ { ++ try ++ { ++ await disposeTask.ConfigureAwait(false); ++ } ++ catch (Exception ex) ++ { ++ Log("WARN", deviceName, ++ $"Stale MMS client cleanup completed with {ex.GetType().Name}: {ex.Message}. Reconnect will continue."); ++ } ++ return; ++ } ++ ++ cancellationToken.ThrowIfCancellationRequested(); ++ ++ // Never let a vendor/session cleanup stall the monitor loop. Observe any later ++ // fault while allowing the replacement association to proceed independently. ++ _ = disposeTask.ContinueWith( ++ task => _ = task.Exception, ++ CancellationToken.None, ++ TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously, ++ TaskScheduler.Default); ++ ++ Log("WARN", deviceName, ++ $"Stale MMS client cleanup exceeded the {budget.TotalMilliseconds:0} ms reconnect budget. A fresh association will proceed without waiting for cleanup."); + } + + private static void ResetAssociationReportEvidence(DeviceSession session) From ce427e5e1e78a473b12364a7bde96054538aac58 Mon Sep 17 00:00:00 2001 From: Ari Sulistiono Date: Wed, 19 Aug 2026 11:13:08 +0700 Subject: [PATCH 05/13] chore(p6.2-c): stage reconnect runtime patch part 3 --- .p6-2-c/03-runtime-tail.patch | 79 +++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 .p6-2-c/03-runtime-tail.patch diff --git a/.p6-2-c/03-runtime-tail.patch b/.p6-2-c/03-runtime-tail.patch new file mode 100644 index 000000000..9080f6564 --- /dev/null +++ b/.p6-2-c/03-runtime-tail.patch @@ -0,0 +1,79 @@ +diff --git a/Services/Iec61850MonitorRuntime.cs b/Services/Iec61850MonitorRuntime.cs +index 34eda3f..5c219d0 100644 +--- a/Services/Iec61850MonitorRuntime.cs ++++ b/Services/Iec61850MonitorRuntime.cs +@@ -1508,7 +1724,14 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable + session.ConsecutiveSessionErrors = 0; + session.ConsecutiveHealthProbeFailures = 0; + session.NextReconnectUtc = DateTime.MinValue; +- try { await session.Client.DisposeAsync().ConfigureAwait(false); } catch { } ++ ++ var staleClient = session.Client; ++ session.Client = new NativeIec61850Client(); ++ await DisposeClientForReconnectAsync( ++ staleClient, ++ session.Device.Name, ++ SmartReconnectPolicy.ClientCleanupBudget, ++ CancellationToken.None).ConfigureAwait(false); + } + + private void RecordSuccessfulIo(DeviceSession session) +@@ -1534,6 +1757,8 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable + if (Volatile.Read(ref session.ControlCommandActive) > 0) + return; + var now = DateTime.UtcNow; ++ if (now < session.RecoveryWarmupUntilUtc) ++ return; + if (now < session.NextHealthProbeUtc || now - session.LastSuccessfulIoUtc < TimeSpan.FromMilliseconds(900)) return; + session.NextHealthProbeUtc = now.AddSeconds(1); + if (string.IsNullOrWhiteSpace(session.HealthProbePointKey) || !session.Points.TryGetValue(session.HealthProbePointKey, out var point)) +@@ -1950,6 +2175,17 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable + return quality; + } + ++ private static bool IsDegradedIecQuality(string? quality) ++ { ++ if (string.IsNullOrWhiteSpace(quality)) ++ return false; ++ ++ var normalized = quality.Trim().ToLowerInvariant(); ++ return normalized.Contains("questionable", StringComparison.Ordinal) || ++ normalized.Contains("invalid", StringComparison.Ordinal) || ++ normalized.Contains("reserved", StringComparison.Ordinal); ++ } ++ + private static void IndexPointReference(DeviceSession session, Iec61850MonitorPoint point) + { + foreach (var key in GetReferenceKeys(point.IecReference)) +@@ -2073,18 +2309,26 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable + .ToList(); + } + +- private static void ResetPollQueue(DeviceSession session) ++ private static void ResetPollQueue( ++ DeviceSession session, ++ bool staggerForRecovery = false) + { + session.PollQueue.Clear(); + var nowUtc = DateTime.UtcNow; ++ var index = 0; + foreach (var point in session.Points.Values) + { +- // Every selected point gets an immediate initial read. For report-assigned +- // points this supplies value/q/t and verifies that the RCB is not silently +- // frozen; after dchg is proven, validation automatically slows down. ++ // Initial startup keeps the legacy immediate-read behavior. After an actual ++ // reconnect, spread the first recovery reads over a small bounded window so ++ // a large selected signal set does not hit a recovering IED with a request burst. ++ var dueUtc = staggerForRecovery ++ ? nowUtc.AddMilliseconds(SmartReconnectPolicy.GetRecoveryStaggerDelayMs(index++)) ++ : nowUtc; + var state = session.States[point.PointKey]; +- state.NextPollUtc = nowUtc; +- session.PollQueue.Enqueue(point.PointKey, nowUtc.Ticks); ++ state.NextPollUtc = dueUtc; ++ if (staggerForRecovery) ++ state.NextCompanionPollUtc = session.RecoveryWarmupUntilUtc; ++ session.PollQueue.Enqueue(point.PointKey, dueUtc.Ticks); + } + } From f7b64c45340d0969677c6bd54759c33e6866475f Mon Sep 17 00:00:00 2001 From: Ari Sulistiono Date: Wed, 19 Aug 2026 11:13:20 +0700 Subject: [PATCH 06/13] chore(p6.2-c): stage reconnect regression patch --- .p6-2-c/04-hybrid-test.patch | 40 ++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .p6-2-c/04-hybrid-test.patch diff --git a/.p6-2-c/04-hybrid-test.patch b/.p6-2-c/04-hybrid-test.patch new file mode 100644 index 000000000..1ffd480b4 --- /dev/null +++ b/.p6-2-c/04-hybrid-test.patch @@ -0,0 +1,40 @@ +diff --git a/tests/ARSAS.Tests/HybridReportPhysicalValidationTests.cs b/tests/ARSAS.Tests/HybridReportPhysicalValidationTests.cs +index 79f3d20..2dc16d6 100644 +--- a/tests/ARSAS.Tests/HybridReportPhysicalValidationTests.cs ++++ b/tests/ARSAS.Tests/HybridReportPhysicalValidationTests.cs +@@ -243,21 +243,28 @@ public sealed class HybridReportPhysicalValidationTests + } + + [Fact] +- public void Reconnect_ReusesTheSameHybridPlanningPipelineAsInitialSetup() ++ public void Reconnect_RecoversMmsFirst_AndDefersReportRearmToBackgroundPipeline() + { + var source = File.ReadAllText(FindRepoFile("Services/Iec61850MonitorRuntime.cs")) + .Replace("\r\n", "\n", StringComparison.Ordinal); + var reconnectStart = source.IndexOf("private async Task TryReconnectAsync", StringComparison.Ordinal); +- var reconnectEnd = source.IndexOf("private async Task ForceReconnectAsync", reconnectStart, StringComparison.Ordinal); ++ var reconnectEnd = source.IndexOf("private static void ResetAssociationReportEvidence", reconnectStart, StringComparison.Ordinal); + Assert.True(reconnectStart >= 0 && reconnectEnd > reconnectStart); + + var reconnect = source[reconnectStart..reconnectEnd]; +- Assert.Contains("BuildReportPlansForCurrentAssociationAsync(", reconnect, StringComparison.Ordinal); ++ Assert.Contains("SmartReconnectPolicy.ConnectBudget", reconnect, StringComparison.Ordinal); + Assert.Contains("ResetAssociationReportEvidence(session);", reconnect, StringComparison.Ordinal); +- Assert.Contains("await StartReportPlansAsync(session, plans", reconnect, StringComparison.Ordinal); +- Assert.True( +- source.Split("BuildReportPlansForCurrentAssociationAsync(", StringSplitOptions.None).Length - 1 >= 3, +- "Initial setup and reconnect must both call the shared hybrid planning pipeline."); ++ Assert.Contains("session.ReportSetupPending =", reconnect, StringComparison.Ordinal); ++ Assert.Contains("ResetPollQueue(session, staggerForRecovery: true);", reconnect, StringComparison.Ordinal); ++ Assert.DoesNotContain("BuildReportPlansForCurrentAssociationAsync(", reconnect, StringComparison.Ordinal); ++ Assert.DoesNotContain("StartReportPlansAsync(session", reconnect, StringComparison.Ordinal); ++ ++ var pendingStart = source.IndexOf("private async Task TryStartPendingReportSetupAsync", StringComparison.Ordinal); ++ var pendingEnd = source.IndexOf("private async Task> BuildReportPlansForCurrentAssociationAsync", pendingStart, StringComparison.Ordinal); ++ Assert.True(pendingStart >= 0 && pendingEnd > pendingStart); ++ var pending = source[pendingStart..pendingEnd]; ++ Assert.Contains("BuildReportPlansForCurrentAssociationAsync(", pending, StringComparison.Ordinal); ++ Assert.Contains("StartReportPlansAsync(session, plans", pending, StringComparison.Ordinal); + + var resetStart = source.IndexOf("private static void ResetAssociationReportEvidence", StringComparison.Ordinal); + var resetEnd = source.IndexOf("private async Task ForceReconnectAsync", resetStart, StringComparison.Ordinal); From bc27cc2cf9fbe730f1e3d739269f673ffc72a322 Mon Sep 17 00:00:00 2001 From: Ari Sulistiono Date: Wed, 19 Aug 2026 11:13:33 +0700 Subject: [PATCH 07/13] feat(p6.2-c): add bounded smart reconnect policy --- Services/SmartReconnectPolicy.cs | 44 ++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 Services/SmartReconnectPolicy.cs diff --git a/Services/SmartReconnectPolicy.cs b/Services/SmartReconnectPolicy.cs new file mode 100644 index 000000000..2434d1f87 --- /dev/null +++ b/Services/SmartReconnectPolicy.cs @@ -0,0 +1,44 @@ +namespace ArIED61850Tester.Services; + +/// +/// Bounded recovery policy for long-running IEC 61850 monitoring sessions. +/// Transport recovery is intentionally independent from Report Control Block re-arming: +/// ACSE/MMS must become usable first, then reporting is restored by the normal background +/// acquisition pipeline. +/// +public static class SmartReconnectPolicy +{ + public static TimeSpan ClientCleanupBudget => TimeSpan.FromMilliseconds(750); + public static TimeSpan ConnectBudget => TimeSpan.FromSeconds(10); + public static TimeSpan InitialAssociationRetryDelay => TimeSpan.FromMilliseconds(750); + public static TimeSpan ReportRearmDelay => TimeSpan.FromMilliseconds(750); + public static TimeSpan ReportRearmDeadline => TimeSpan.FromSeconds(3); + public static TimeSpan RecoveryWarmupDuration => TimeSpan.FromSeconds(10); + + public static TimeSpan GetRetryDelay(int consecutiveFailureCount) + { + var attempt = Math.Max(1, consecutiveFailureCount); + var seconds = attempt switch + { + 1 => 1, + 2 => 2, + 3 => 4, + 4 => 8, + 5 => 15, + _ => 30 + }; + return TimeSpan.FromSeconds(seconds); + } + + public static int ApplyRecoveryPollFloor(int intervalMs, bool recoveryWarmup) + { + var bounded = Math.Clamp(intervalMs, 50, 600000); + return recoveryWarmup ? Math.Max(bounded, 2000) : bounded; + } + + public static int GetRecoveryStaggerDelayMs(int zeroBasedIndex) + { + var index = Math.Max(0, zeroBasedIndex); + return Math.Min(2000, index * 20); + } +} From dc60bbecd84c74001cc7f7a55f96c62237fea101 Mon Sep 17 00:00:00 2001 From: Ari Sulistiono Date: Wed, 19 Aug 2026 11:13:44 +0700 Subject: [PATCH 08/13] test(p6.2-c): lock bounded recovery and quality evidence --- .../P62CSmartReconnectRegressionTests.cs | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 tests/ARSAS.Tests/P62CSmartReconnectRegressionTests.cs diff --git a/tests/ARSAS.Tests/P62CSmartReconnectRegressionTests.cs b/tests/ARSAS.Tests/P62CSmartReconnectRegressionTests.cs new file mode 100644 index 000000000..8dd7385c9 --- /dev/null +++ b/tests/ARSAS.Tests/P62CSmartReconnectRegressionTests.cs @@ -0,0 +1,64 @@ +using ArIED61850Tester.Services; + +namespace ARSAS.Tests; + +public sealed class P62CSmartReconnectRegressionTests +{ + [Theory] + [InlineData(1, 1)] + [InlineData(2, 2)] + [InlineData(3, 4)] + [InlineData(4, 8)] + [InlineData(5, 15)] + [InlineData(6, 30)] + [InlineData(20, 30)] + public void RetryBackoff_IsBoundedAndDeterministic(int failureCount, int expectedSeconds) + => Assert.Equal( + TimeSpan.FromSeconds(expectedSeconds), + SmartReconnectPolicy.GetRetryDelay(failureCount)); + + [Fact] + public void RecoveryWarmup_ReducesImmediateMmsPressureWithoutChangingSteadyState() + { + Assert.Equal(2000, SmartReconnectPolicy.ApplyRecoveryPollFloor(1000, recoveryWarmup: true)); + Assert.Equal(10000, SmartReconnectPolicy.ApplyRecoveryPollFloor(10000, recoveryWarmup: true)); + Assert.Equal(1000, SmartReconnectPolicy.ApplyRecoveryPollFloor(1000, recoveryWarmup: false)); + Assert.Equal(0, SmartReconnectPolicy.GetRecoveryStaggerDelayMs(0)); + Assert.Equal(2000, SmartReconnectPolicy.GetRecoveryStaggerDelayMs(100)); + Assert.Equal(2000, SmartReconnectPolicy.GetRecoveryStaggerDelayMs(1000)); + } + + [Fact] + public void ReconnectBudgets_AreShorterThanAFieldVisibleStall() + { + Assert.True(SmartReconnectPolicy.ClientCleanupBudget <= TimeSpan.FromSeconds(1)); + Assert.True(SmartReconnectPolicy.ConnectBudget <= TimeSpan.FromSeconds(10)); + Assert.True(SmartReconnectPolicy.ReportRearmDelay < SmartReconnectPolicy.RecoveryWarmupDuration); + Assert.True(SmartReconnectPolicy.ReportRearmDeadline <= SmartReconnectPolicy.RecoveryWarmupDuration); + } + + [Fact] + public void Runtime_LogsDegradedQualityAsEvidenceWithoutForcingGood() + { + var source = ReadRepoFile("Services/Iec61850MonitorRuntime.cs"); + + Assert.Contains("QUALITY_EVIDENCE:", source, StringComparison.Ordinal); + Assert.Contains("Quality is preserved from IED evidence and is not converted to Good.", source, StringComparison.Ordinal); + Assert.DoesNotContain("quality = \"Good\";", source, StringComparison.Ordinal); + Assert.DoesNotContain("state.Quality = \"Good\";", source, StringComparison.Ordinal); + } + + private static string ReadRepoFile(string relativePath) + { + DirectoryInfo? directory = new(AppContext.BaseDirectory); + while (directory != null) + { + var candidate = Path.Combine(directory.FullName, relativePath); + if (File.Exists(candidate)) + return File.ReadAllText(candidate).Replace("\r\n", "\n", StringComparison.Ordinal); + directory = directory.Parent; + } + + throw new FileNotFoundException(relativePath); + } +} From c5e925cb1a613ca73adc65ee719b13e6b800b347 Mon Sep 17 00:00:00 2001 From: Ari Sulistiono Date: Wed, 19 Aug 2026 11:14:01 +0700 Subject: [PATCH 09/13] chore(p6.2-c): apply staged smart reconnect patch --- .p6-2-c/apply.ready | 1 + 1 file changed, 1 insertion(+) create mode 100644 .p6-2-c/apply.ready diff --git a/.p6-2-c/apply.ready b/.p6-2-c/apply.ready new file mode 100644 index 000000000..9197fbbd2 --- /dev/null +++ b/.p6-2-c/apply.ready @@ -0,0 +1 @@ +P6.2-C smart reconnect patch staged and ready. From 1c153abdfccc427a92b29445e0ac96ca952c534c Mon Sep 17 00:00:00 2001 From: Ari Sulistiono Date: Wed, 19 Aug 2026 11:15:52 +0700 Subject: [PATCH 10/13] chore(p6.2-c): allow PR-triggered patch application --- .github/workflows/p6-2-c-apply-patch.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/p6-2-c-apply-patch.yml b/.github/workflows/p6-2-c-apply-patch.yml index b40c6163e..2114ce767 100644 --- a/.github/workflows/p6-2-c-apply-patch.yml +++ b/.github/workflows/p6-2-c-apply-patch.yml @@ -6,17 +6,26 @@ on: - fix/p6-2-c-smart-reconnect paths: - .p6-2-c/apply.ready + pull_request: + branches: + - main + types: + - opened + - synchronize + - reopened permissions: contents: write jobs: apply-patch: + if: github.event_name == 'push' || github.head_ref == 'fix/p6-2-c-smart-reconnect' runs-on: ubuntu-latest steps: - - name: Checkout branch + - name: Checkout source branch uses: actions/checkout@v4 with: + ref: fix/p6-2-c-smart-reconnect fetch-depth: 0 - name: Apply reviewed P6.2-C patch parts @@ -26,8 +35,8 @@ jobs: shopt -s nullglob patches=(.p6-2-c/*.patch) if [ ${#patches[@]} -eq 0 ]; then - echo "No staged patch files were found." - exit 1 + echo "No staged patch files were found; source branch is already materialized." + exit 0 fi for patch in "${patches[@]}"; do echo "Checking $patch" From 868fc3d89153421fd768d31a23cd5d8c6cb4e8ce Mon Sep 17 00:00:00 2001 From: Ari Sulistiono Date: Wed, 19 Aug 2026 11:17:26 +0700 Subject: [PATCH 11/13] chore(p6.2-c): fix staged tail patch hunk length --- .p6-2-c/03-runtime-tail.patch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.p6-2-c/03-runtime-tail.patch b/.p6-2-c/03-runtime-tail.patch index 9080f6564..0341a3c2e 100644 --- a/.p6-2-c/03-runtime-tail.patch +++ b/.p6-2-c/03-runtime-tail.patch @@ -45,7 +45,7 @@ index 34eda3f..5c219d0 100644 private static void IndexPointReference(DeviceSession session, Iec61850MonitorPoint point) { foreach (var key in GetReferenceKeys(point.IecReference)) -@@ -2073,18 +2309,26 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable +@@ -2073,17 +2309,25 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable .ToList(); } From ca084f7dca5d0b43134760c9a0ff6192f3ae1e38 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 19 Aug 2026 04:17:37 +0000 Subject: [PATCH 12/13] fix(p6.2-c): make MMS recovery bounded and staged --- .github/workflows/p6-2-c-apply-patch.yml | 52 --- .p6-2-c/01-runtime-early.patch | 115 ------ .p6-2-c/02-runtime-reconnect.patch | 277 --------------- .p6-2-c/03-runtime-tail.patch | 79 ----- .p6-2-c/04-hybrid-test.patch | 40 --- .p6-2-c/apply.ready | 1 - Services/Iec61850MonitorRuntime.cs | 326 +++++++++++++++--- .../HybridReportPhysicalValidationTests.cs | 21 +- 8 files changed, 299 insertions(+), 612 deletions(-) delete mode 100644 .github/workflows/p6-2-c-apply-patch.yml delete mode 100644 .p6-2-c/01-runtime-early.patch delete mode 100644 .p6-2-c/02-runtime-reconnect.patch delete mode 100644 .p6-2-c/03-runtime-tail.patch delete mode 100644 .p6-2-c/04-hybrid-test.patch delete mode 100644 .p6-2-c/apply.ready diff --git a/.github/workflows/p6-2-c-apply-patch.yml b/.github/workflows/p6-2-c-apply-patch.yml deleted file mode 100644 index 2114ce767..000000000 --- a/.github/workflows/p6-2-c-apply-patch.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: P6.2-C apply smart reconnect patch - -on: - push: - branches: - - fix/p6-2-c-smart-reconnect - paths: - - .p6-2-c/apply.ready - pull_request: - branches: - - main - types: - - opened - - synchronize - - reopened - -permissions: - contents: write - -jobs: - apply-patch: - if: github.event_name == 'push' || github.head_ref == 'fix/p6-2-c-smart-reconnect' - runs-on: ubuntu-latest - steps: - - name: Checkout source branch - uses: actions/checkout@v4 - with: - ref: fix/p6-2-c-smart-reconnect - fetch-depth: 0 - - - name: Apply reviewed P6.2-C patch parts - shell: bash - run: | - set -euo pipefail - shopt -s nullglob - patches=(.p6-2-c/*.patch) - if [ ${#patches[@]} -eq 0 ]; then - echo "No staged patch files were found; source branch is already materialized." - exit 0 - fi - for patch in "${patches[@]}"; do - echo "Checking $patch" - git apply --check "$patch" - git apply "$patch" - done - rm -rf .p6-2-c - rm -f .github/workflows/p6-2-c-apply-patch.yml - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add -A - git commit -m "fix(p6.2-c): make MMS recovery bounded and staged" - git push origin HEAD:fix/p6-2-c-smart-reconnect diff --git a/.p6-2-c/01-runtime-early.patch b/.p6-2-c/01-runtime-early.patch deleted file mode 100644 index e53b7ed06..000000000 --- a/.p6-2-c/01-runtime-early.patch +++ /dev/null @@ -1,115 +0,0 @@ -diff --git a/Services/Iec61850MonitorRuntime.cs b/Services/Iec61850MonitorRuntime.cs -index 34eda3f..5c219d0 100644 ---- a/Services/Iec61850MonitorRuntime.cs -+++ b/Services/Iec61850MonitorRuntime.cs -@@ -39,6 +39,7 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable - public bool CommandReportMissLogged { get; set; } - public bool StaleReportSuppressedLogged { get; set; } - public bool ReportValueRejectedLogged { get; set; } -+ public string LastLoggedDegradedQuality { get; set; } = string.Empty; - public int ConsecutiveErrors { get; set; } - } - -@@ -73,6 +74,8 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable - public DateTime ReportSetupNotBeforeUtc { get; set; } = DateTime.MinValue; - public DateTime ReportSetupDeadlineUtc { get; set; } = DateTime.MinValue; - public DateTime NextReconnectUtc { get; set; } = DateTime.MinValue; -+ public int ConsecutiveReconnectFailures { get; set; } -+ public DateTime RecoveryWarmupUntilUtc { get; set; } = DateTime.MinValue; - public int ConsecutiveSessionErrors { get; set; } - public DateTime LastSuccessfulIoUtc { get; set; } = DateTime.UtcNow; - public DateTime NextHealthProbeUtc { get; set; } = DateTime.MinValue; -@@ -225,7 +228,11 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable - { - device.IsConnected = false; - _sessions.TryRemove(device.DeviceId, out _); -- await session.Client.DisposeAsync().ConfigureAwait(false); -+ await DisposeClientForReconnectAsync( -+ session.Client, -+ device.Name, -+ SmartReconnectPolicy.ClientCleanupBudget, -+ CancellationToken.None).ConfigureAwait(false); - } - throw; - } -@@ -283,7 +290,7 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable - 2, - 4)); - -- await session.Client.ConnectAsync(device.IpAddress, device.Port, cancellationToken).ConfigureAwait(false); -+ await ConnectCachedAssociationWithRetryAsync(session, cancellationToken).ConfigureAwait(false); - if (!session.Client.IsConnected) - { - device.Status = "Connection failed"; -@@ -326,7 +333,11 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable - { - device.IsConnected = false; - _sessions.TryRemove(device.DeviceId, out _); -- await session.Client.DisposeAsync().ConfigureAwait(false); -+ await DisposeClientForReconnectAsync( -+ session.Client, -+ device.Name, -+ SmartReconnectPolicy.ClientCleanupBudget, -+ CancellationToken.None).ConfigureAwait(false); - } - throw; - } -@@ -374,6 +385,8 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable - session.ReportSetupPending = false; - session.ReportSetupNotBeforeUtc = DateTime.MinValue; - session.ReportSetupDeadlineUtc = DateTime.MinValue; -+ session.ConsecutiveReconnectFailures = 0; -+ session.RecoveryWarmupUntilUtc = DateTime.MinValue; - session.ConsecutiveSessionErrors = 0; - session.LastSuccessfulIoUtc = DateTime.UtcNow; - session.NextHealthProbeUtc = DateTime.UtcNow.AddSeconds(1); -@@ -789,7 +802,9 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable - cancellationToken).ConfigureAwait(false); - - await StartReportPlansAsync(session, plans, cancellationToken).ConfigureAwait(false); -- ResetPollQueue(session); -+ ResetPollQueue( -+ session, -+ staggerForRecovery: DateTime.UtcNow < session.RecoveryWarmupUntilUtc); - UpdateDeviceAcquisitionSummary(session); - } - -@@ -1185,7 +1200,9 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable - var reportAssigned = session.PointPlanIds.ContainsKey(point.PointKey); - cancellationToken.ThrowIfCancellationRequested(); - processed++; -- var nextIntervalMs = GetVerificationPollIntervalMs(point, state, reportAssigned); -+ var nextIntervalMs = SmartReconnectPolicy.ApplyRecoveryPollFloor( -+ GetVerificationPollIntervalMs(point, state, reportAssigned), -+ nowUtc < session.RecoveryWarmupUntilUtc); - state.NextPollUtc = nowUtc.AddMilliseconds(nextIntervalMs); - session.PollQueue.Enqueue(point.PointKey, state.NextPollUtc.Ticks); - -@@ -1228,6 +1245,7 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable - var deviceTimestamp = rich?.HasDeviceTimestamp == true ? rich.DeviceTimestamp : state.DeviceTimestamp; - - if ((rich?.HasQuality != true || rich?.HasDeviceTimestamp != true) && -+ nowUtc >= session.RecoveryWarmupUntilUtc && - nowUtc >= state.NextCompanionPollUtc) - { - state.NextCompanionPollUtc = nowUtc.AddMilliseconds(GetCompanionPollIntervalMs(point)); -@@ -1345,6 +1363,19 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable - var changed = hasProcessValue && hadValue && (trustReportEdge - ? HasExactSemanticEdge(point, oldValue, display) - : HasMeaningfulEdge(point, oldValue, display)); -+ -+ if (qualityChangedForUi && IsDegradedIecQuality(quality) && -+ !state.LastLoggedDegradedQuality.Equals(quality, StringComparison.OrdinalIgnoreCase)) -+ { -+ state.LastLoggedDegradedQuality = quality; -+ Log("INFO", session.Device.Name, -+ $"QUALITY_EVIDENCE: {point.SignalName} ({point.IecReference}) quality={quality}; acquisition={sourceMode}; qRef={(string.IsNullOrWhiteSpace(point.QualityReference) ? "derived companion/report q" : point.QualityReference)}; timestamp={(string.IsNullOrWhiteSpace(deviceTimestamp) ? "-" : deviceTimestamp)}. Quality is preserved from IED evidence and is not converted to Good."); -+ } -+ else if (!IsDegradedIecQuality(quality)) -+ { -+ state.LastLoggedDegradedQuality = string.Empty; -+ } -+ - if (hasProcessValue) - { - state.HasValue = true; diff --git a/.p6-2-c/02-runtime-reconnect.patch b/.p6-2-c/02-runtime-reconnect.patch deleted file mode 100644 index 8025ee12a..000000000 --- a/.p6-2-c/02-runtime-reconnect.patch +++ /dev/null @@ -1,277 +0,0 @@ -diff --git a/Services/Iec61850MonitorRuntime.cs b/Services/Iec61850MonitorRuntime.cs -index 34eda3f..5c219d0 100644 ---- a/Services/Iec61850MonitorRuntime.cs -+++ b/Services/Iec61850MonitorRuntime.cs -@@ -1426,58 +1457,243 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable - }); - } - -+ private async Task ConnectCachedAssociationWithRetryAsync( -+ DeviceSession session, -+ CancellationToken cancellationToken) -+ { -+ const int maxAttempts = 2; -+ for (var attempt = 1; attempt <= maxAttempts; attempt++) -+ { -+ using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); -+ timeout.CancelAfter(SmartReconnectPolicy.ConnectBudget); -+ try -+ { -+ await session.Client.ConnectAsync( -+ session.Device.IpAddress, -+ session.Device.Port, -+ timeout.Token).ConfigureAwait(false); -+ } -+ catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) -+ { -+ session.Device.LastDiagnosticSnapshot = session.Client.CaptureDiagnosticSnapshot( -+ $"Fast saved-model connect attempt #{attempt} timed out"); -+ } -+ -+ if (session.Client.IsConnected) -+ { -+ if (attempt > 1) -+ { -+ Log("INFO", session.Device.Name, -+ $"Fast saved-model connection recovered automatically on attempt #{attempt}; no manual Play retry was required."); -+ } -+ return; -+ } -+ -+ if (attempt >= maxAttempts) -+ return; -+ -+ var failure = string.IsNullOrWhiteSpace(session.Client.LastErrorMessage) -+ ? $"native state={session.Client.NativeState}" -+ : session.Client.LastErrorMessage; -+ Log("WARN", session.Device.Name, -+ $"Fast saved-model connection attempt #{attempt} did not establish MMS ({failure}). Retrying once after {SmartReconnectPolicy.InitialAssociationRetryDelay.TotalMilliseconds:0} ms."); -+ -+ var staleClient = session.Client; -+ session.Client = new NativeIec61850Client(); -+ await DisposeClientForReconnectAsync( -+ staleClient, -+ session.Device.Name, -+ SmartReconnectPolicy.ClientCleanupBudget, -+ cancellationToken).ConfigureAwait(false); -+ await Task.Delay(SmartReconnectPolicy.InitialAssociationRetryDelay, cancellationToken).ConfigureAwait(false); -+ } -+ } -+ - private async Task TryReconnectAsync(DeviceSession session, CancellationToken cancellationToken) - { -- if (DateTime.UtcNow < session.NextReconnectUtc) return; -- session.NextReconnectUtc = DateTime.UtcNow.AddSeconds(2); -+ var nowUtc = DateTime.UtcNow; -+ if (nowUtc < session.NextReconnectUtc) -+ return; -+ -+ var attempt = session.ConsecutiveReconnectFailures + 1; -+ var reconnectStopwatch = Stopwatch.StartNew(); - MarkSessionOffline(session, $"Reconnecting MMS association to {session.Device.EndpointText}."); - session.Device.Status = "Reconnecting"; -- Log("WARN", session.Device.Name, "IEC 61850 session is offline. Smart reconnect started."); -- try { await session.Client.DisposeAsync().ConfigureAwait(false); } catch { } -- session.Client = new NativeIec61850Client(); -- try { await session.Client.ConnectAsync(session.Device.IpAddress, session.Device.Port, cancellationToken).ConfigureAwait(false); } -- catch (OperationCanceledException) { throw; } -+ session.Device.Detail = $"Smart reconnect attempt #{attempt}: opening a fresh MMS association."; -+ session.Device.RefreshComputed(); -+ Log("WARN", session.Device.Name, -+ $"Smart reconnect attempt #{attempt} started. Transport recovery is bounded independently from report re-arming."); -+ -+ var staleClient = session.Client; -+ await DisposeClientForReconnectAsync( -+ staleClient, -+ session.Device.Name, -+ SmartReconnectPolicy.ClientCleanupBudget, -+ cancellationToken).ConfigureAwait(false); -+ -+ var replacement = new NativeIec61850Client(); -+ session.Client = replacement; -+ -+ using var connectTimeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); -+ connectTimeout.CancelAfter(SmartReconnectPolicy.ConnectBudget); -+ try -+ { -+ await replacement.ConnectAsync( -+ session.Device.IpAddress, -+ session.Device.Port, -+ connectTimeout.Token).ConfigureAwait(false); -+ } -+ catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) -+ { -+ session.Device.LastDiagnosticSnapshot = replacement.CaptureDiagnosticSnapshot( -+ "Smart reconnect association timed out"); -+ ScheduleReconnectRetry( -+ session, -+ attempt, -+ reconnectStopwatch.Elapsed, -+ $"MMS association exceeded the {SmartReconnectPolicy.ConnectBudget.TotalSeconds:0.#} s reconnect budget."); -+ return; -+ } -+ catch (OperationCanceledException) -+ { -+ throw; -+ } - catch (Exception ex) - { -- session.Device.Status = "Reconnect pending"; -- session.Device.Detail = ex.Message; -- session.Device.RefreshComputed(); -+ session.Device.LastDiagnosticSnapshot = replacement.CaptureDiagnosticSnapshot( -+ "Smart reconnect association failed", -+ ex); -+ ScheduleReconnectRetry( -+ session, -+ attempt, -+ reconnectStopwatch.Elapsed, -+ $"{ex.GetType().Name}: {ex.Message}"); - return; - } -- if (!session.Client.IsConnected) -+ -+ if (!replacement.IsConnected) - { -- session.Device.Status = "Reconnect pending"; -- session.Device.Detail = session.Client.LastErrorMessage; -- session.Device.RefreshComputed(); -+ session.Device.LastDiagnosticSnapshot = replacement.CaptureDiagnosticSnapshot( -+ "Smart reconnect association not established"); -+ ScheduleReconnectRetry( -+ session, -+ attempt, -+ reconnectStopwatch.Elapsed, -+ string.IsNullOrWhiteSpace(replacement.LastErrorMessage) -+ ? $"Native state={replacement.NativeState}; MMS association was not established." -+ : replacement.LastErrorMessage); - return; - } -+ -+ // Connection recovery and report recovery are intentionally separate stages. -+ // Once ACSE/MMS is healthy, resume bounded MMS reads immediately. Static RCB -+ // discovery/re-arming returns to the normal background report pipeline so a -+ // slow vendor RCB read/write can never hold the reconnect state machine hostage. - session.ActiveReportPlans.Clear(); - session.ActiveReportPlanOrder.Clear(); - session.PointPlanIds.Clear(); - session.ReportStreams.Clear(); - session.LastUnroutedReportCount = 0; -- session.PendingReportPlans = Array.Empty(); -- session.ReportSetupPending = false; -- session.ReportSetupNotBeforeUtc = DateTime.MinValue; -- session.ReportSetupDeadlineUtc = DateTime.MinValue; - ResetAssociationReportEvidence(session); -+ - var legacyPlans = Iec61850ReportPlanner.BuildPlans(session.Device, session.Points.Values); -- var plans = await BuildReportPlansForCurrentAssociationAsync( -- session, -- legacyPlans, -- cancellationToken).ConfigureAwait(false); -- await StartReportPlansAsync(session, plans, cancellationToken).ConfigureAwait(false); -- ResetPollQueue(session); -- UpdateDeviceAcquisitionSummary(session); -+ session.PendingReportPlans = legacyPlans; -+ session.ReportSetupPending = -+ replacement.CanUseHybridReportPlanner(session.Device) || -+ legacyPlans.Count > 0; -+ var associatedUtc = DateTime.UtcNow; -+ session.ReportSetupNotBeforeUtc = associatedUtc.Add(SmartReconnectPolicy.ReportRearmDelay); -+ session.ReportSetupDeadlineUtc = associatedUtc.Add(SmartReconnectPolicy.ReportRearmDeadline); -+ session.RecoveryWarmupUntilUtc = associatedUtc.Add(SmartReconnectPolicy.RecoveryWarmupDuration); -+ -+ ResetPollQueue(session, staggerForRecovery: true); -+ session.ConsecutiveReconnectFailures = 0; -+ session.NextReconnectUtc = DateTime.MinValue; - session.ConsecutiveSessionErrors = 0; - session.ConsecutiveHealthProbeFailures = 0; -- session.LastSuccessfulIoUtc = DateTime.UtcNow; -- session.NextHealthProbeUtc = DateTime.UtcNow.AddSeconds(1); -+ session.LastSuccessfulIoUtc = associatedUtc; -+ session.NextHealthProbeUtc = session.RecoveryWarmupUntilUtc; - session.Device.IsConnected = true; - session.Device.Status = "Monitoring"; -- session.Device.Detail = $"MMS reconnected. {session.Points.Count} point(s) resumed."; -+ session.Device.AcquisitionMode = "MMS recovered • static report re-arm pending"; -+ session.Device.Detail = -+ $"MMS reconnected in {reconnectStopwatch.Elapsed.TotalMilliseconds:0} ms. " + -+ $"{session.Points.Count} point(s) resumed with staggered MMS recovery; report re-arm continues in the background."; -+ session.Device.LastDiagnosticSnapshot = replacement.CaptureDiagnosticSnapshot( -+ "Smart reconnect MMS associated; report re-arm deferred"); - session.Device.RefreshComputed(); -- Log("INFO", session.Device.Name, "MMS reconnect successful. Monitoring resumed automatically."); -+ -+ Log("INFO", session.Device.Name, -+ $"MMS reconnect successful on attempt #{attempt} in {reconnectStopwatch.Elapsed.TotalMilliseconds:0} ms. " + -+ $"Polling resumed immediately with a {SmartReconnectPolicy.RecoveryWarmupDuration.TotalSeconds:0} s recovery warm-up; static report re-arm is deferred to the background pipeline."); -+ } -+ -+ private void ScheduleReconnectRetry( -+ DeviceSession session, -+ int attempt, -+ TimeSpan elapsed, -+ string detail) -+ { -+ session.ConsecutiveReconnectFailures = attempt; -+ var retryDelay = SmartReconnectPolicy.GetRetryDelay(attempt); -+ session.NextReconnectUtc = DateTime.UtcNow.Add(retryDelay); -+ session.Device.IsConnected = false; -+ session.Device.Status = "Reconnect pending"; -+ session.Device.AcquisitionMode = "Connection lost • reconnect pending"; -+ session.Device.Detail = -+ $"Smart reconnect attempt #{attempt} failed after {elapsed.TotalMilliseconds:0} ms: {detail} " + -+ $"Retry in {retryDelay.TotalSeconds:0.#} s."; -+ session.Device.RefreshComputed(); -+ Log("WARN", session.Device.Name, -+ $"Smart reconnect attempt #{attempt} failed after {elapsed.TotalMilliseconds:0} ms; retry in {retryDelay.TotalSeconds:0.#} s. {detail}"); -+ } -+ -+ private async Task DisposeClientForReconnectAsync( -+ NativeIec61850Client client, -+ string deviceName, -+ TimeSpan budget, -+ CancellationToken cancellationToken) -+ { -+ Task disposeTask; -+ try -+ { -+ disposeTask = client.DisposeAsync().AsTask(); -+ } -+ catch (Exception ex) -+ { -+ Log("WARN", deviceName, -+ $"Stale MMS client cleanup could not start during reconnect: {ex.GetType().Name}: {ex.Message}"); -+ return; -+ } -+ -+ var delayTask = Task.Delay(budget, cancellationToken); -+ var completed = await Task.WhenAny(disposeTask, delayTask).ConfigureAwait(false); -+ if (completed == disposeTask) -+ { -+ try -+ { -+ await disposeTask.ConfigureAwait(false); -+ } -+ catch (Exception ex) -+ { -+ Log("WARN", deviceName, -+ $"Stale MMS client cleanup completed with {ex.GetType().Name}: {ex.Message}. Reconnect will continue."); -+ } -+ return; -+ } -+ -+ cancellationToken.ThrowIfCancellationRequested(); -+ -+ // Never let a vendor/session cleanup stall the monitor loop. Observe any later -+ // fault while allowing the replacement association to proceed independently. -+ _ = disposeTask.ContinueWith( -+ task => _ = task.Exception, -+ CancellationToken.None, -+ TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously, -+ TaskScheduler.Default); -+ -+ Log("WARN", deviceName, -+ $"Stale MMS client cleanup exceeded the {budget.TotalMilliseconds:0} ms reconnect budget. A fresh association will proceed without waiting for cleanup."); - } - - private static void ResetAssociationReportEvidence(DeviceSession session) diff --git a/.p6-2-c/03-runtime-tail.patch b/.p6-2-c/03-runtime-tail.patch deleted file mode 100644 index 0341a3c2e..000000000 --- a/.p6-2-c/03-runtime-tail.patch +++ /dev/null @@ -1,79 +0,0 @@ -diff --git a/Services/Iec61850MonitorRuntime.cs b/Services/Iec61850MonitorRuntime.cs -index 34eda3f..5c219d0 100644 ---- a/Services/Iec61850MonitorRuntime.cs -+++ b/Services/Iec61850MonitorRuntime.cs -@@ -1508,7 +1724,14 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable - session.ConsecutiveSessionErrors = 0; - session.ConsecutiveHealthProbeFailures = 0; - session.NextReconnectUtc = DateTime.MinValue; -- try { await session.Client.DisposeAsync().ConfigureAwait(false); } catch { } -+ -+ var staleClient = session.Client; -+ session.Client = new NativeIec61850Client(); -+ await DisposeClientForReconnectAsync( -+ staleClient, -+ session.Device.Name, -+ SmartReconnectPolicy.ClientCleanupBudget, -+ CancellationToken.None).ConfigureAwait(false); - } - - private void RecordSuccessfulIo(DeviceSession session) -@@ -1534,6 +1757,8 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable - if (Volatile.Read(ref session.ControlCommandActive) > 0) - return; - var now = DateTime.UtcNow; -+ if (now < session.RecoveryWarmupUntilUtc) -+ return; - if (now < session.NextHealthProbeUtc || now - session.LastSuccessfulIoUtc < TimeSpan.FromMilliseconds(900)) return; - session.NextHealthProbeUtc = now.AddSeconds(1); - if (string.IsNullOrWhiteSpace(session.HealthProbePointKey) || !session.Points.TryGetValue(session.HealthProbePointKey, out var point)) -@@ -1950,6 +2175,17 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable - return quality; - } - -+ private static bool IsDegradedIecQuality(string? quality) -+ { -+ if (string.IsNullOrWhiteSpace(quality)) -+ return false; -+ -+ var normalized = quality.Trim().ToLowerInvariant(); -+ return normalized.Contains("questionable", StringComparison.Ordinal) || -+ normalized.Contains("invalid", StringComparison.Ordinal) || -+ normalized.Contains("reserved", StringComparison.Ordinal); -+ } -+ - private static void IndexPointReference(DeviceSession session, Iec61850MonitorPoint point) - { - foreach (var key in GetReferenceKeys(point.IecReference)) -@@ -2073,17 +2309,25 @@ public sealed class Iec61850MonitorRuntime : IAsyncDisposable - .ToList(); - } - -- private static void ResetPollQueue(DeviceSession session) -+ private static void ResetPollQueue( -+ DeviceSession session, -+ bool staggerForRecovery = false) - { - session.PollQueue.Clear(); - var nowUtc = DateTime.UtcNow; -+ var index = 0; - foreach (var point in session.Points.Values) - { -- // Every selected point gets an immediate initial read. For report-assigned -- // points this supplies value/q/t and verifies that the RCB is not silently -- // frozen; after dchg is proven, validation automatically slows down. -+ // Initial startup keeps the legacy immediate-read behavior. After an actual -+ // reconnect, spread the first recovery reads over a small bounded window so -+ // a large selected signal set does not hit a recovering IED with a request burst. -+ var dueUtc = staggerForRecovery -+ ? nowUtc.AddMilliseconds(SmartReconnectPolicy.GetRecoveryStaggerDelayMs(index++)) -+ : nowUtc; - var state = session.States[point.PointKey]; -- state.NextPollUtc = nowUtc; -- session.PollQueue.Enqueue(point.PointKey, nowUtc.Ticks); -+ state.NextPollUtc = dueUtc; -+ if (staggerForRecovery) -+ state.NextCompanionPollUtc = session.RecoveryWarmupUntilUtc; -+ session.PollQueue.Enqueue(point.PointKey, dueUtc.Ticks); - } - } diff --git a/.p6-2-c/04-hybrid-test.patch b/.p6-2-c/04-hybrid-test.patch deleted file mode 100644 index 1ffd480b4..000000000 --- a/.p6-2-c/04-hybrid-test.patch +++ /dev/null @@ -1,40 +0,0 @@ -diff --git a/tests/ARSAS.Tests/HybridReportPhysicalValidationTests.cs b/tests/ARSAS.Tests/HybridReportPhysicalValidationTests.cs -index 79f3d20..2dc16d6 100644 ---- a/tests/ARSAS.Tests/HybridReportPhysicalValidationTests.cs -+++ b/tests/ARSAS.Tests/HybridReportPhysicalValidationTests.cs -@@ -243,21 +243,28 @@ public sealed class HybridReportPhysicalValidationTests - } - - [Fact] -- public void Reconnect_ReusesTheSameHybridPlanningPipelineAsInitialSetup() -+ public void Reconnect_RecoversMmsFirst_AndDefersReportRearmToBackgroundPipeline() - { - var source = File.ReadAllText(FindRepoFile("Services/Iec61850MonitorRuntime.cs")) - .Replace("\r\n", "\n", StringComparison.Ordinal); - var reconnectStart = source.IndexOf("private async Task TryReconnectAsync", StringComparison.Ordinal); -- var reconnectEnd = source.IndexOf("private async Task ForceReconnectAsync", reconnectStart, StringComparison.Ordinal); -+ var reconnectEnd = source.IndexOf("private static void ResetAssociationReportEvidence", reconnectStart, StringComparison.Ordinal); - Assert.True(reconnectStart >= 0 && reconnectEnd > reconnectStart); - - var reconnect = source[reconnectStart..reconnectEnd]; -- Assert.Contains("BuildReportPlansForCurrentAssociationAsync(", reconnect, StringComparison.Ordinal); -+ Assert.Contains("SmartReconnectPolicy.ConnectBudget", reconnect, StringComparison.Ordinal); - Assert.Contains("ResetAssociationReportEvidence(session);", reconnect, StringComparison.Ordinal); -- Assert.Contains("await StartReportPlansAsync(session, plans", reconnect, StringComparison.Ordinal); -- Assert.True( -- source.Split("BuildReportPlansForCurrentAssociationAsync(", StringSplitOptions.None).Length - 1 >= 3, -- "Initial setup and reconnect must both call the shared hybrid planning pipeline."); -+ Assert.Contains("session.ReportSetupPending =", reconnect, StringComparison.Ordinal); -+ Assert.Contains("ResetPollQueue(session, staggerForRecovery: true);", reconnect, StringComparison.Ordinal); -+ Assert.DoesNotContain("BuildReportPlansForCurrentAssociationAsync(", reconnect, StringComparison.Ordinal); -+ Assert.DoesNotContain("StartReportPlansAsync(session", reconnect, StringComparison.Ordinal); -+ -+ var pendingStart = source.IndexOf("private async Task TryStartPendingReportSetupAsync", StringComparison.Ordinal); -+ var pendingEnd = source.IndexOf("private async Task> BuildReportPlansForCurrentAssociationAsync", pendingStart, StringComparison.Ordinal); -+ Assert.True(pendingStart >= 0 && pendingEnd > pendingStart); -+ var pending = source[pendingStart..pendingEnd]; -+ Assert.Contains("BuildReportPlansForCurrentAssociationAsync(", pending, StringComparison.Ordinal); -+ Assert.Contains("StartReportPlansAsync(session, plans", pending, StringComparison.Ordinal); - - var resetStart = source.IndexOf("private static void ResetAssociationReportEvidence", StringComparison.Ordinal); - var resetEnd = source.IndexOf("private async Task ForceReconnectAsync", resetStart, StringComparison.Ordinal); diff --git a/.p6-2-c/apply.ready b/.p6-2-c/apply.ready deleted file mode 100644 index 9197fbbd2..000000000 --- a/.p6-2-c/apply.ready +++ /dev/null @@ -1 +0,0 @@ -P6.2-C smart reconnect patch staged and ready. diff --git a/Services/Iec61850MonitorRuntime.cs b/Services/Iec61850MonitorRuntime.cs index 34eda3fd2..5c219d068 100644 --- a/Services/Iec61850MonitorRuntime.cs +++ b/Services/Iec61850MonitorRuntime.cs @@ -39,6 +39,7 @@ private sealed class RuntimePointState public bool CommandReportMissLogged { get; set; } public bool StaleReportSuppressedLogged { get; set; } public bool ReportValueRejectedLogged { get; set; } + public string LastLoggedDegradedQuality { get; set; } = string.Empty; public int ConsecutiveErrors { get; set; } } @@ -73,6 +74,8 @@ private sealed class DeviceSession public DateTime ReportSetupNotBeforeUtc { get; set; } = DateTime.MinValue; public DateTime ReportSetupDeadlineUtc { get; set; } = DateTime.MinValue; public DateTime NextReconnectUtc { get; set; } = DateTime.MinValue; + public int ConsecutiveReconnectFailures { get; set; } + public DateTime RecoveryWarmupUntilUtc { get; set; } = DateTime.MinValue; public int ConsecutiveSessionErrors { get; set; } public DateTime LastSuccessfulIoUtc { get; set; } = DateTime.UtcNow; public DateTime NextHealthProbeUtc { get; set; } = DateTime.MinValue; @@ -225,7 +228,11 @@ public async Task> ConnectAndDiscoverAsync( { device.IsConnected = false; _sessions.TryRemove(device.DeviceId, out _); - await session.Client.DisposeAsync().ConfigureAwait(false); + await DisposeClientForReconnectAsync( + session.Client, + device.Name, + SmartReconnectPolicy.ClientCleanupBudget, + CancellationToken.None).ConfigureAwait(false); } throw; } @@ -283,7 +290,7 @@ public async Task ConnectUsingCachedModelAsync( 2, 4)); - await session.Client.ConnectAsync(device.IpAddress, device.Port, cancellationToken).ConfigureAwait(false); + await ConnectCachedAssociationWithRetryAsync(session, cancellationToken).ConfigureAwait(false); if (!session.Client.IsConnected) { device.Status = "Connection failed"; @@ -326,7 +333,11 @@ public async Task ConnectUsingCachedModelAsync( { device.IsConnected = false; _sessions.TryRemove(device.DeviceId, out _); - await session.Client.DisposeAsync().ConfigureAwait(false); + await DisposeClientForReconnectAsync( + session.Client, + device.Name, + SmartReconnectPolicy.ClientCleanupBudget, + CancellationToken.None).ConfigureAwait(false); } throw; } @@ -374,6 +385,8 @@ public async Task> StartMonitoringAsync( session.ReportSetupPending = false; session.ReportSetupNotBeforeUtc = DateTime.MinValue; session.ReportSetupDeadlineUtc = DateTime.MinValue; + session.ConsecutiveReconnectFailures = 0; + session.RecoveryWarmupUntilUtc = DateTime.MinValue; session.ConsecutiveSessionErrors = 0; session.LastSuccessfulIoUtc = DateTime.UtcNow; session.NextHealthProbeUtc = DateTime.UtcNow.AddSeconds(1); @@ -789,7 +802,9 @@ private async Task TryStartPendingReportSetupAsync( cancellationToken).ConfigureAwait(false); await StartReportPlansAsync(session, plans, cancellationToken).ConfigureAwait(false); - ResetPollQueue(session); + ResetPollQueue( + session, + staggerForRecovery: DateTime.UtcNow < session.RecoveryWarmupUntilUtc); UpdateDeviceAcquisitionSummary(session); } @@ -1185,7 +1200,9 @@ private async Task PollDuePointsAsync(DeviceSession session, CancellationToken c var reportAssigned = session.PointPlanIds.ContainsKey(point.PointKey); cancellationToken.ThrowIfCancellationRequested(); processed++; - var nextIntervalMs = GetVerificationPollIntervalMs(point, state, reportAssigned); + var nextIntervalMs = SmartReconnectPolicy.ApplyRecoveryPollFloor( + GetVerificationPollIntervalMs(point, state, reportAssigned), + nowUtc < session.RecoveryWarmupUntilUtc); state.NextPollUtc = nowUtc.AddMilliseconds(nextIntervalMs); session.PollQueue.Enqueue(point.PointKey, state.NextPollUtc.Ticks); @@ -1228,6 +1245,7 @@ private async Task PollDuePointsAsync(DeviceSession session, CancellationToken c var deviceTimestamp = rich?.HasDeviceTimestamp == true ? rich.DeviceTimestamp : state.DeviceTimestamp; if ((rich?.HasQuality != true || rich?.HasDeviceTimestamp != true) && + nowUtc >= session.RecoveryWarmupUntilUtc && nowUtc >= state.NextCompanionPollUtc) { state.NextCompanionPollUtc = nowUtc.AddMilliseconds(GetCompanionPollIntervalMs(point)); @@ -1345,6 +1363,19 @@ private void ApplyValueUpdate( var changed = hasProcessValue && hadValue && (trustReportEdge ? HasExactSemanticEdge(point, oldValue, display) : HasMeaningfulEdge(point, oldValue, display)); + + if (qualityChangedForUi && IsDegradedIecQuality(quality) && + !state.LastLoggedDegradedQuality.Equals(quality, StringComparison.OrdinalIgnoreCase)) + { + state.LastLoggedDegradedQuality = quality; + Log("INFO", session.Device.Name, + $"QUALITY_EVIDENCE: {point.SignalName} ({point.IecReference}) quality={quality}; acquisition={sourceMode}; qRef={(string.IsNullOrWhiteSpace(point.QualityReference) ? "derived companion/report q" : point.QualityReference)}; timestamp={(string.IsNullOrWhiteSpace(deviceTimestamp) ? "-" : deviceTimestamp)}. Quality is preserved from IED evidence and is not converted to Good."); + } + else if (!IsDegradedIecQuality(quality)) + { + state.LastLoggedDegradedQuality = string.Empty; + } + if (hasProcessValue) { state.HasValue = true; @@ -1426,58 +1457,243 @@ private void EmitStatusSnapshot( }); } + private async Task ConnectCachedAssociationWithRetryAsync( + DeviceSession session, + CancellationToken cancellationToken) + { + const int maxAttempts = 2; + for (var attempt = 1; attempt <= maxAttempts; attempt++) + { + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(SmartReconnectPolicy.ConnectBudget); + try + { + await session.Client.ConnectAsync( + session.Device.IpAddress, + session.Device.Port, + timeout.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + session.Device.LastDiagnosticSnapshot = session.Client.CaptureDiagnosticSnapshot( + $"Fast saved-model connect attempt #{attempt} timed out"); + } + + if (session.Client.IsConnected) + { + if (attempt > 1) + { + Log("INFO", session.Device.Name, + $"Fast saved-model connection recovered automatically on attempt #{attempt}; no manual Play retry was required."); + } + return; + } + + if (attempt >= maxAttempts) + return; + + var failure = string.IsNullOrWhiteSpace(session.Client.LastErrorMessage) + ? $"native state={session.Client.NativeState}" + : session.Client.LastErrorMessage; + Log("WARN", session.Device.Name, + $"Fast saved-model connection attempt #{attempt} did not establish MMS ({failure}). Retrying once after {SmartReconnectPolicy.InitialAssociationRetryDelay.TotalMilliseconds:0} ms."); + + var staleClient = session.Client; + session.Client = new NativeIec61850Client(); + await DisposeClientForReconnectAsync( + staleClient, + session.Device.Name, + SmartReconnectPolicy.ClientCleanupBudget, + cancellationToken).ConfigureAwait(false); + await Task.Delay(SmartReconnectPolicy.InitialAssociationRetryDelay, cancellationToken).ConfigureAwait(false); + } + } + private async Task TryReconnectAsync(DeviceSession session, CancellationToken cancellationToken) { - if (DateTime.UtcNow < session.NextReconnectUtc) return; - session.NextReconnectUtc = DateTime.UtcNow.AddSeconds(2); + var nowUtc = DateTime.UtcNow; + if (nowUtc < session.NextReconnectUtc) + return; + + var attempt = session.ConsecutiveReconnectFailures + 1; + var reconnectStopwatch = Stopwatch.StartNew(); MarkSessionOffline(session, $"Reconnecting MMS association to {session.Device.EndpointText}."); session.Device.Status = "Reconnecting"; - Log("WARN", session.Device.Name, "IEC 61850 session is offline. Smart reconnect started."); - try { await session.Client.DisposeAsync().ConfigureAwait(false); } catch { } - session.Client = new NativeIec61850Client(); - try { await session.Client.ConnectAsync(session.Device.IpAddress, session.Device.Port, cancellationToken).ConfigureAwait(false); } - catch (OperationCanceledException) { throw; } + session.Device.Detail = $"Smart reconnect attempt #{attempt}: opening a fresh MMS association."; + session.Device.RefreshComputed(); + Log("WARN", session.Device.Name, + $"Smart reconnect attempt #{attempt} started. Transport recovery is bounded independently from report re-arming."); + + var staleClient = session.Client; + await DisposeClientForReconnectAsync( + staleClient, + session.Device.Name, + SmartReconnectPolicy.ClientCleanupBudget, + cancellationToken).ConfigureAwait(false); + + var replacement = new NativeIec61850Client(); + session.Client = replacement; + + using var connectTimeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + connectTimeout.CancelAfter(SmartReconnectPolicy.ConnectBudget); + try + { + await replacement.ConnectAsync( + session.Device.IpAddress, + session.Device.Port, + connectTimeout.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + session.Device.LastDiagnosticSnapshot = replacement.CaptureDiagnosticSnapshot( + "Smart reconnect association timed out"); + ScheduleReconnectRetry( + session, + attempt, + reconnectStopwatch.Elapsed, + $"MMS association exceeded the {SmartReconnectPolicy.ConnectBudget.TotalSeconds:0.#} s reconnect budget."); + return; + } + catch (OperationCanceledException) + { + throw; + } catch (Exception ex) { - session.Device.Status = "Reconnect pending"; - session.Device.Detail = ex.Message; - session.Device.RefreshComputed(); + session.Device.LastDiagnosticSnapshot = replacement.CaptureDiagnosticSnapshot( + "Smart reconnect association failed", + ex); + ScheduleReconnectRetry( + session, + attempt, + reconnectStopwatch.Elapsed, + $"{ex.GetType().Name}: {ex.Message}"); return; } - if (!session.Client.IsConnected) + + if (!replacement.IsConnected) { - session.Device.Status = "Reconnect pending"; - session.Device.Detail = session.Client.LastErrorMessage; - session.Device.RefreshComputed(); + session.Device.LastDiagnosticSnapshot = replacement.CaptureDiagnosticSnapshot( + "Smart reconnect association not established"); + ScheduleReconnectRetry( + session, + attempt, + reconnectStopwatch.Elapsed, + string.IsNullOrWhiteSpace(replacement.LastErrorMessage) + ? $"Native state={replacement.NativeState}; MMS association was not established." + : replacement.LastErrorMessage); return; } + + // Connection recovery and report recovery are intentionally separate stages. + // Once ACSE/MMS is healthy, resume bounded MMS reads immediately. Static RCB + // discovery/re-arming returns to the normal background report pipeline so a + // slow vendor RCB read/write can never hold the reconnect state machine hostage. session.ActiveReportPlans.Clear(); session.ActiveReportPlanOrder.Clear(); session.PointPlanIds.Clear(); session.ReportStreams.Clear(); session.LastUnroutedReportCount = 0; - session.PendingReportPlans = Array.Empty(); - session.ReportSetupPending = false; - session.ReportSetupNotBeforeUtc = DateTime.MinValue; - session.ReportSetupDeadlineUtc = DateTime.MinValue; ResetAssociationReportEvidence(session); + var legacyPlans = Iec61850ReportPlanner.BuildPlans(session.Device, session.Points.Values); - var plans = await BuildReportPlansForCurrentAssociationAsync( - session, - legacyPlans, - cancellationToken).ConfigureAwait(false); - await StartReportPlansAsync(session, plans, cancellationToken).ConfigureAwait(false); - ResetPollQueue(session); - UpdateDeviceAcquisitionSummary(session); + session.PendingReportPlans = legacyPlans; + session.ReportSetupPending = + replacement.CanUseHybridReportPlanner(session.Device) || + legacyPlans.Count > 0; + var associatedUtc = DateTime.UtcNow; + session.ReportSetupNotBeforeUtc = associatedUtc.Add(SmartReconnectPolicy.ReportRearmDelay); + session.ReportSetupDeadlineUtc = associatedUtc.Add(SmartReconnectPolicy.ReportRearmDeadline); + session.RecoveryWarmupUntilUtc = associatedUtc.Add(SmartReconnectPolicy.RecoveryWarmupDuration); + + ResetPollQueue(session, staggerForRecovery: true); + session.ConsecutiveReconnectFailures = 0; + session.NextReconnectUtc = DateTime.MinValue; session.ConsecutiveSessionErrors = 0; session.ConsecutiveHealthProbeFailures = 0; - session.LastSuccessfulIoUtc = DateTime.UtcNow; - session.NextHealthProbeUtc = DateTime.UtcNow.AddSeconds(1); + session.LastSuccessfulIoUtc = associatedUtc; + session.NextHealthProbeUtc = session.RecoveryWarmupUntilUtc; session.Device.IsConnected = true; session.Device.Status = "Monitoring"; - session.Device.Detail = $"MMS reconnected. {session.Points.Count} point(s) resumed."; + session.Device.AcquisitionMode = "MMS recovered • static report re-arm pending"; + session.Device.Detail = + $"MMS reconnected in {reconnectStopwatch.Elapsed.TotalMilliseconds:0} ms. " + + $"{session.Points.Count} point(s) resumed with staggered MMS recovery; report re-arm continues in the background."; + session.Device.LastDiagnosticSnapshot = replacement.CaptureDiagnosticSnapshot( + "Smart reconnect MMS associated; report re-arm deferred"); session.Device.RefreshComputed(); - Log("INFO", session.Device.Name, "MMS reconnect successful. Monitoring resumed automatically."); + + Log("INFO", session.Device.Name, + $"MMS reconnect successful on attempt #{attempt} in {reconnectStopwatch.Elapsed.TotalMilliseconds:0} ms. " + + $"Polling resumed immediately with a {SmartReconnectPolicy.RecoveryWarmupDuration.TotalSeconds:0} s recovery warm-up; static report re-arm is deferred to the background pipeline."); + } + + private void ScheduleReconnectRetry( + DeviceSession session, + int attempt, + TimeSpan elapsed, + string detail) + { + session.ConsecutiveReconnectFailures = attempt; + var retryDelay = SmartReconnectPolicy.GetRetryDelay(attempt); + session.NextReconnectUtc = DateTime.UtcNow.Add(retryDelay); + session.Device.IsConnected = false; + session.Device.Status = "Reconnect pending"; + session.Device.AcquisitionMode = "Connection lost • reconnect pending"; + session.Device.Detail = + $"Smart reconnect attempt #{attempt} failed after {elapsed.TotalMilliseconds:0} ms: {detail} " + + $"Retry in {retryDelay.TotalSeconds:0.#} s."; + session.Device.RefreshComputed(); + Log("WARN", session.Device.Name, + $"Smart reconnect attempt #{attempt} failed after {elapsed.TotalMilliseconds:0} ms; retry in {retryDelay.TotalSeconds:0.#} s. {detail}"); + } + + private async Task DisposeClientForReconnectAsync( + NativeIec61850Client client, + string deviceName, + TimeSpan budget, + CancellationToken cancellationToken) + { + Task disposeTask; + try + { + disposeTask = client.DisposeAsync().AsTask(); + } + catch (Exception ex) + { + Log("WARN", deviceName, + $"Stale MMS client cleanup could not start during reconnect: {ex.GetType().Name}: {ex.Message}"); + return; + } + + var delayTask = Task.Delay(budget, cancellationToken); + var completed = await Task.WhenAny(disposeTask, delayTask).ConfigureAwait(false); + if (completed == disposeTask) + { + try + { + await disposeTask.ConfigureAwait(false); + } + catch (Exception ex) + { + Log("WARN", deviceName, + $"Stale MMS client cleanup completed with {ex.GetType().Name}: {ex.Message}. Reconnect will continue."); + } + return; + } + + cancellationToken.ThrowIfCancellationRequested(); + + // Never let a vendor/session cleanup stall the monitor loop. Observe any later + // fault while allowing the replacement association to proceed independently. + _ = disposeTask.ContinueWith( + task => _ = task.Exception, + CancellationToken.None, + TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); + + Log("WARN", deviceName, + $"Stale MMS client cleanup exceeded the {budget.TotalMilliseconds:0} ms reconnect budget. A fresh association will proceed without waiting for cleanup."); } private static void ResetAssociationReportEvidence(DeviceSession session) @@ -1508,7 +1724,14 @@ private async Task ForceReconnectAsync(DeviceSession session, string reason) session.ConsecutiveSessionErrors = 0; session.ConsecutiveHealthProbeFailures = 0; session.NextReconnectUtc = DateTime.MinValue; - try { await session.Client.DisposeAsync().ConfigureAwait(false); } catch { } + + var staleClient = session.Client; + session.Client = new NativeIec61850Client(); + await DisposeClientForReconnectAsync( + staleClient, + session.Device.Name, + SmartReconnectPolicy.ClientCleanupBudget, + CancellationToken.None).ConfigureAwait(false); } private void RecordSuccessfulIo(DeviceSession session) @@ -1534,6 +1757,8 @@ private async Task ProbeSessionHealthAsync(DeviceSession session, CancellationTo if (Volatile.Read(ref session.ControlCommandActive) > 0) return; var now = DateTime.UtcNow; + if (now < session.RecoveryWarmupUntilUtc) + return; if (now < session.NextHealthProbeUtc || now - session.LastSuccessfulIoUtc < TimeSpan.FromMilliseconds(900)) return; session.NextHealthProbeUtc = now.AddSeconds(1); if (string.IsNullOrWhiteSpace(session.HealthProbePointKey) || !session.Points.TryGetValue(session.HealthProbePointKey, out var point)) @@ -1950,6 +2175,17 @@ private static string NormalizeQuality(string? quality) return quality; } + private static bool IsDegradedIecQuality(string? quality) + { + if (string.IsNullOrWhiteSpace(quality)) + return false; + + var normalized = quality.Trim().ToLowerInvariant(); + return normalized.Contains("questionable", StringComparison.Ordinal) || + normalized.Contains("invalid", StringComparison.Ordinal) || + normalized.Contains("reserved", StringComparison.Ordinal); + } + private static void IndexPointReference(DeviceSession session, Iec61850MonitorPoint point) { foreach (var key in GetReferenceKeys(point.IecReference)) @@ -2073,18 +2309,26 @@ private static IReadOnlyList ResolveCoveredPoints( .ToList(); } - private static void ResetPollQueue(DeviceSession session) + private static void ResetPollQueue( + DeviceSession session, + bool staggerForRecovery = false) { session.PollQueue.Clear(); var nowUtc = DateTime.UtcNow; + var index = 0; foreach (var point in session.Points.Values) { - // Every selected point gets an immediate initial read. For report-assigned - // points this supplies value/q/t and verifies that the RCB is not silently - // frozen; after dchg is proven, validation automatically slows down. + // Initial startup keeps the legacy immediate-read behavior. After an actual + // reconnect, spread the first recovery reads over a small bounded window so + // a large selected signal set does not hit a recovering IED with a request burst. + var dueUtc = staggerForRecovery + ? nowUtc.AddMilliseconds(SmartReconnectPolicy.GetRecoveryStaggerDelayMs(index++)) + : nowUtc; var state = session.States[point.PointKey]; - state.NextPollUtc = nowUtc; - session.PollQueue.Enqueue(point.PointKey, nowUtc.Ticks); + state.NextPollUtc = dueUtc; + if (staggerForRecovery) + state.NextCompanionPollUtc = session.RecoveryWarmupUntilUtc; + session.PollQueue.Enqueue(point.PointKey, dueUtc.Ticks); } } diff --git a/tests/ARSAS.Tests/HybridReportPhysicalValidationTests.cs b/tests/ARSAS.Tests/HybridReportPhysicalValidationTests.cs index 79f3d20e4..2dc16d68a 100644 --- a/tests/ARSAS.Tests/HybridReportPhysicalValidationTests.cs +++ b/tests/ARSAS.Tests/HybridReportPhysicalValidationTests.cs @@ -243,21 +243,28 @@ public void ConcurrentMonitorUpdatesAndSnapshots_RemainAtomicAndConsistent() } [Fact] - public void Reconnect_ReusesTheSameHybridPlanningPipelineAsInitialSetup() + public void Reconnect_RecoversMmsFirst_AndDefersReportRearmToBackgroundPipeline() { var source = File.ReadAllText(FindRepoFile("Services/Iec61850MonitorRuntime.cs")) .Replace("\r\n", "\n", StringComparison.Ordinal); var reconnectStart = source.IndexOf("private async Task TryReconnectAsync", StringComparison.Ordinal); - var reconnectEnd = source.IndexOf("private async Task ForceReconnectAsync", reconnectStart, StringComparison.Ordinal); + var reconnectEnd = source.IndexOf("private static void ResetAssociationReportEvidence", reconnectStart, StringComparison.Ordinal); Assert.True(reconnectStart >= 0 && reconnectEnd > reconnectStart); var reconnect = source[reconnectStart..reconnectEnd]; - Assert.Contains("BuildReportPlansForCurrentAssociationAsync(", reconnect, StringComparison.Ordinal); + Assert.Contains("SmartReconnectPolicy.ConnectBudget", reconnect, StringComparison.Ordinal); Assert.Contains("ResetAssociationReportEvidence(session);", reconnect, StringComparison.Ordinal); - Assert.Contains("await StartReportPlansAsync(session, plans", reconnect, StringComparison.Ordinal); - Assert.True( - source.Split("BuildReportPlansForCurrentAssociationAsync(", StringSplitOptions.None).Length - 1 >= 3, - "Initial setup and reconnect must both call the shared hybrid planning pipeline."); + Assert.Contains("session.ReportSetupPending =", reconnect, StringComparison.Ordinal); + Assert.Contains("ResetPollQueue(session, staggerForRecovery: true);", reconnect, StringComparison.Ordinal); + Assert.DoesNotContain("BuildReportPlansForCurrentAssociationAsync(", reconnect, StringComparison.Ordinal); + Assert.DoesNotContain("StartReportPlansAsync(session", reconnect, StringComparison.Ordinal); + + var pendingStart = source.IndexOf("private async Task TryStartPendingReportSetupAsync", StringComparison.Ordinal); + var pendingEnd = source.IndexOf("private async Task> BuildReportPlansForCurrentAssociationAsync", pendingStart, StringComparison.Ordinal); + Assert.True(pendingStart >= 0 && pendingEnd > pendingStart); + var pending = source[pendingStart..pendingEnd]; + Assert.Contains("BuildReportPlansForCurrentAssociationAsync(", pending, StringComparison.Ordinal); + Assert.Contains("StartReportPlansAsync(session, plans", pending, StringComparison.Ordinal); var resetStart = source.IndexOf("private static void ResetAssociationReportEvidence", StringComparison.Ordinal); var resetEnd = source.IndexOf("private async Task ForceReconnectAsync", resetStart, StringComparison.Ordinal); From c5cd6e537a31967e55aeba77da7e02f028770976 Mon Sep 17 00:00:00 2001 From: Ari Sulistiono Date: Wed, 19 Aug 2026 11:18:51 +0700 Subject: [PATCH 13/13] tune(p6.2-c): let recovered MMS settle before static RCB re-arm --- Services/SmartReconnectPolicy.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Services/SmartReconnectPolicy.cs b/Services/SmartReconnectPolicy.cs index 2434d1f87..6c5187895 100644 --- a/Services/SmartReconnectPolicy.cs +++ b/Services/SmartReconnectPolicy.cs @@ -11,8 +11,8 @@ public static class SmartReconnectPolicy public static TimeSpan ClientCleanupBudget => TimeSpan.FromMilliseconds(750); public static TimeSpan ConnectBudget => TimeSpan.FromSeconds(10); public static TimeSpan InitialAssociationRetryDelay => TimeSpan.FromMilliseconds(750); - public static TimeSpan ReportRearmDelay => TimeSpan.FromMilliseconds(750); - public static TimeSpan ReportRearmDeadline => TimeSpan.FromSeconds(3); + public static TimeSpan ReportRearmDelay => TimeSpan.FromSeconds(3); + public static TimeSpan ReportRearmDeadline => TimeSpan.FromSeconds(5); public static TimeSpan RecoveryWarmupDuration => TimeSpan.FromSeconds(10); public static TimeSpan GetRetryDelay(int consecutiveFailureCount)