diff --git a/lib/services/rustpush/profile_retry_policy.dart b/lib/services/rustpush/profile_retry_policy.dart new file mode 100644 index 0000000000..f0f440ec11 --- /dev/null +++ b/lib/services/rustpush/profile_retry_policy.dart @@ -0,0 +1,58 @@ +enum ProfileFailureKind { transient, permanent } + +class ProfileRetryPolicy { + ProfileRetryPolicy({ + this.delays = const [ + Duration(seconds: 5), + Duration(seconds: 30), + Duration(minutes: 2), + ], + }); + + final List delays; + final Map _attempts = {}; + final Set _scheduled = {}; + + ProfileFailureKind classify(Object error) { + final description = error.toString().toLowerCase(); + if (description.contains("profile service unavailable") || + description.contains("timeout") || + description.contains("connection") || + description.contains("network") || + description.contains("socket") || + description.contains("dns")) { + return ProfileFailureKind.transient; + } + return ProfileFailureKind.permanent; + } + + bool get isEmpty => _attempts.isEmpty && _scheduled.isEmpty; + + bool isScheduled(String profileKey) => _scheduled.contains(profileKey); + + Duration? schedule(String profileKey) { + if (!_scheduled.add(profileKey)) return null; + final attempt = _attempts[profileKey] ?? 0; + if (attempt >= delays.length) { + _scheduled.remove(profileKey); + _attempts.remove(profileKey); + return null; + } + _attempts[profileKey] = attempt + 1; + return delays[attempt]; + } + + void timerFired(String profileKey) { + _scheduled.remove(profileKey); + } + + void complete(String profileKey) { + _scheduled.remove(profileKey); + _attempts.remove(profileKey); + } + + void clear() { + _scheduled.clear(); + _attempts.clear(); + } +} diff --git a/lib/services/rustpush/rustpush_service.dart b/lib/services/rustpush/rustpush_service.dart index 6deb3c1e42..ed42a0e9f7 100644 --- a/lib/services/rustpush/rustpush_service.dart +++ b/lib/services/rustpush/rustpush_service.dart @@ -19,6 +19,8 @@ import 'package:bluebubbles/src/rust/lib.dart' as lib; import 'package:bluebubbles/helpers/helpers.dart'; import 'package:bluebubbles/database/models.dart'; import 'package:bluebubbles/services/services.dart'; +import 'package:bluebubbles/services/rustpush/send_retry_policy.dart'; +import 'package:bluebubbles/services/rustpush/profile_retry_policy.dart'; import 'package:bluebubbles/utils/crypto_utils.dart'; import 'package:bluebubbles/utils/logger/logger.dart'; import 'package:collection/collection.dart'; @@ -342,6 +344,8 @@ class RustPushBBUtils { } class RustPushBackend implements BackendService { + final InFlightSendRegistry _inFlightSends = InFlightSendRegistry(); + Future getDefaultHandle() async { var myHandles = await api.getHandles(state: pushService.state!.client); var setHandle = ss.settings.defaultHandle.value; @@ -398,22 +402,33 @@ class RustPushBackend implements BackendService { return const api.MessageType.iMessage(); } - Future sendMsg(api.MessageInst msg) async { + Future sendMsg(api.MessageInst msg, {bool waitForResource = true}) { + return _inFlightSends.run(msg.id, () => _sendMsgWithRetry(msg, waitForResource: waitForResource)); + } + + Future _sendMsgWithRetry(api.MessageInst msg, {required bool waitForResource}) async { var message = Message.findOne(guid: msg.id); if (message != null) { message.sendingServiceId = pushService.serviceId; message.save(updateSendingServiceId: true); } var stillRunning = false; + final retryPolicy = SendRetryPolicy(); try { - stillRunning = await api.send(state: pushService.state!.client, local: pushService.state!.localBroadcast, msg: msg); - } catch (e) { - if (e is AnyhowException) { - if (e.message.contains("Failed to generate resource") && e.message.contains("not retrying")) { - pushService.markFailedToLogin(); + while (true) { + try { + stillRunning = await api.send(state: pushService.state!.client, local: pushService.state!.localBroadcast, msg: msg); + break; + } catch (e) { + final decision = retryPolicy.next(e, waitForResource: waitForResource); + if (decision.markFailedLogin) { + pushService.markFailedToLogin(); + } + if (!decision.retry) rethrow; + Logger.warn("Retrying send ${msg.id} in ${decision.delay.inSeconds}s (${decision.kind.name})"); + await Future.delayed(decision.delay); } } - rethrow; } finally { if (!stillRunning) { message = Message.findOne(guid: msg.id); @@ -1279,7 +1294,7 @@ class RustPushBackend implements BackendService { icon: base64Decode(appdata.appIcon!), ) : null) ); - await sendMsg(msg); + await sendMsg(msg, waitForResource: false); // typing state is stale after a reconnect } @override @@ -1290,7 +1305,7 @@ class RustPushBackend implements BackendService { sender: await c.ensureHandle(), message: const api.Message.typing(false) ); - await sendMsg(msg); + await sendMsg(msg, waitForResource: false); } @override @@ -2827,8 +2842,86 @@ class RustPushService extends GetxService { return null; } - List profilesDownloading = []; - Future handleSharedProfile(api.ShareProfileMessage shared, String sender, List targets) async { + final Set profilesDownloading = {}; + final Map _profileRetryTimers = {}; + final ProfileRetryPolicy _profileRetryPolicy = ProfileRetryPolicy(); + + String _profileFailureCategory(Object error) { + final description = error.toString().toLowerCase(); + if (description.contains("profile service unavailable")) return "service_unavailable"; + if (description.contains("timeout")) return "timeout"; + if (description.contains("connection") || + description.contains("network") || + description.contains("socket") || + description.contains("dns")) { + return "network"; + } + if (description.contains("record") && description.contains("not found")) return "record_not_found"; + if (description.contains("plist") || description.contains("serde")) return "plist"; + if (description.contains("decrypt") || + description.contains("hmac") || + description.contains("crypto")) { + return "crypto"; + } + if (description.contains("asset")) return "asset"; + if (description.contains("panic")) return "panic"; + return error.runtimeType.toString(); + } + + void _clearProfileRetry(String profileKey) { + _profileRetryTimers.remove(profileKey)?.cancel(); + _profileRetryPolicy.complete(profileKey); + } + + void _scheduleProfileRetry( + api.ShareProfileMessage shared, + String sender, + List targets, + String category, + ) { + final profileKey = shared.cloudKitRecordKey; + final delay = _profileRetryPolicy.schedule(profileKey); + if (delay == null) { + Logger.warn("Shared profile fetch exhausted or already scheduled category=$category"); + return; + } + + Logger.warn( + "Shared profile fetch deferred category=$category " + "retry_in_seconds=${delay.inSeconds}", + ); + _profileRetryTimers[profileKey] = Timer(delay, () { + _profileRetryTimers.remove(profileKey); + _profileRetryPolicy.timerFired(profileKey); + unawaited(handleSharedProfile(shared, sender, targets)); + }); + } + + Future handleSharedProfile(api.ShareProfileMessage shared, String sender, List targets) async { + final profileKey = shared.cloudKitRecordKey; + if (_profileRetryPolicy.isScheduled(profileKey) || !profilesDownloading.add(profileKey)) return; + + try { + await _handleSharedProfile(shared, sender, targets); + _clearProfileRetry(profileKey); + } catch (error) { + // Shared profile payloads are optional message metadata. A malformed + // CloudKit plist must not escape an unawaited profile task and disturb + // message delivery. Retry independently so the contact image can recover + // after transient CloudKit, network, or service-initialization failures. + final category = _profileFailureCategory(error); + if (_profileRetryPolicy.classify(error) == ProfileFailureKind.transient) { + _scheduleProfileRetry(shared, sender, targets, category); + } else { + Logger.warn("Shared profile fetch permanently failed category=$category"); + _profileRetryPolicy.complete(profileKey); + } + } finally { + profilesDownloading.remove(profileKey); + } + } + + Future _handleSharedProfile(api.ShareProfileMessage shared, String sender, List targets) async { var myHandles = await api.getHandles(state: pushService.state!.client); if (myHandles.contains(sender)) { for (var target in targets) { @@ -5036,6 +5129,11 @@ class RustPushService extends GetxService { @override void onClose() { + for (final timer in _profileRetryTimers.values) { + timer.cancel(); + } + _profileRetryTimers.clear(); + _profileRetryPolicy.clear(); if (state != null) disposeState(state!, true, false); super.onClose(); } diff --git a/lib/services/rustpush/send_retry_policy.dart b/lib/services/rustpush/send_retry_policy.dart new file mode 100644 index 0000000000..a1a812f4b2 --- /dev/null +++ b/lib/services/rustpush/send_retry_policy.dart @@ -0,0 +1,116 @@ +import 'dart:async'; + +enum SendFailureKind { + resourceUnavailable, + confirmationTimeout, + permanent, + unknown, +} + +class SendRetryDecision { + const SendRetryDecision({ + required this.kind, + required this.retry, + this.delay = Duration.zero, + this.markFailedLogin = false, + }); + + final SendFailureKind kind; + final bool retry; + final Duration delay; + final bool markFailedLogin; +} + +class SendRetryPolicy { + SendRetryPolicy({ + this.budget = const Duration(minutes: 3), + this.maxResourceWait = const Duration(seconds: 35), + this.timeoutRetryWait = const Duration(seconds: 2), + this.maxTimeoutRetries = 1, + }); + + static final RegExp resourceRetryRegex = RegExp(r"retrying in (\d+)s"); + + final Duration budget; + final Duration maxResourceWait; + final Duration timeoutRetryWait; + final int maxTimeoutRetries; + Duration elapsed = Duration.zero; + int timeoutRetries = 0; + + SendFailureKind classify(Object error) { + final description = error.toString(); + if (description.contains("Failed to generate resource")) { + return description.contains("not retrying") + ? SendFailureKind.permanent + : SendFailureKind.resourceUnavailable; + } + if (description.contains("Send timeout; try again")) { + return SendFailureKind.confirmationTimeout; + } + return SendFailureKind.unknown; + } + + Duration? resourceRetryWait(Object error) { + final seconds = int.tryParse( + resourceRetryRegex.firstMatch(error.toString())?.group(1) ?? "", + ); + if (seconds == null) return null; + final wait = Duration(seconds: seconds) + const Duration(seconds: 1); + return wait > maxResourceWait ? maxResourceWait : wait; + } + + SendRetryDecision next(Object error, {bool waitForResource = true}) { + final kind = classify(error); + if (kind == SendFailureKind.permanent) { + return const SendRetryDecision( + kind: SendFailureKind.permanent, + retry: false, + markFailedLogin: true, + ); + } + + Duration? delay; + if (kind == SendFailureKind.confirmationTimeout && + timeoutRetries < maxTimeoutRetries) { + timeoutRetries++; + delay = timeoutRetryWait; + } else if (kind == SendFailureKind.resourceUnavailable && waitForResource) { + delay = resourceRetryWait(error); + } + + if (delay == null || elapsed + delay > budget) { + return SendRetryDecision(kind: kind, retry: false); + } + + elapsed += delay; + return SendRetryDecision(kind: kind, retry: true, delay: delay); + } +} + +class InFlightSendRegistry { + final Map> _active = >{}; + + Future run(String messageId, Future Function() operation) { + final existing = _active[messageId]; + if (existing != null) return existing; + + final completer = Completer(); + _active[messageId] = completer.future; + Future.sync(operation).then( + (_) { + completer.complete(); + if (identical(_active[messageId], completer.future)) { + _active.remove(messageId); + } + }, + onError: (Object error, StackTrace stackTrace) { + completer.completeError(error, stackTrace); + if (identical(_active[messageId], completer.future)) { + _active.remove(messageId); + } + }, + ); + return completer.future; + } +} diff --git a/test/services/rustpush/profile_retry_policy_test.dart b/test/services/rustpush/profile_retry_policy_test.dart new file mode 100644 index 0000000000..0b98080a52 --- /dev/null +++ b/test/services/rustpush/profile_retry_policy_test.dart @@ -0,0 +1,65 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:bluebubbles/services/rustpush/profile_retry_policy.dart'; + +void main() { + group("ProfileRetryPolicy", () { + test("classifies service and network failures as transient", () { + final policy = ProfileRetryPolicy(); + + expect( + policy.classify(StateError("Profile service unavailable")), + ProfileFailureKind.transient, + ); + expect( + policy.classify(TimeoutExceptionForTest()), + ProfileFailureKind.transient, + ); + expect( + policy.classify(Exception("socket reset by peer")), + ProfileFailureKind.transient, + ); + }); + + test("does not retry permanent payload failures", () { + final policy = ProfileRetryPolicy(); + + expect( + policy.classify(Exception("invalid plist signature")), + ProfileFailureKind.permanent, + ); + expect( + policy.classify(Exception("record not found")), + ProfileFailureKind.permanent, + ); + }); + + test("suppresses duplicate schedules and exhausts the retry budget", () { + final policy = ProfileRetryPolicy( + delays: const [Duration(seconds: 1), Duration(seconds: 2)], + ); + + expect(policy.schedule("profile-1"), const Duration(seconds: 1)); + expect(policy.schedule("profile-1"), isNull); + policy.timerFired("profile-1"); + expect(policy.schedule("profile-1"), const Duration(seconds: 2)); + policy.timerFired("profile-1"); + expect(policy.schedule("profile-1"), isNull); + expect(policy.isEmpty, isTrue); + }); + + test("completion resets attempts for a later independent fetch", () { + final policy = ProfileRetryPolicy( + delays: const [Duration(seconds: 1)], + ); + + expect(policy.schedule("profile-1"), const Duration(seconds: 1)); + policy.complete("profile-1"); + expect(policy.schedule("profile-1"), const Duration(seconds: 1)); + }); + }); +} + +class TimeoutExceptionForTest implements Exception { + @override + String toString() => "timeout while fetching profile"; +} diff --git a/test/services/rustpush/send_retry_policy_test.dart b/test/services/rustpush/send_retry_policy_test.dart new file mode 100644 index 0000000000..77bbc2c881 --- /dev/null +++ b/test/services/rustpush/send_retry_policy_test.dart @@ -0,0 +1,76 @@ +import 'dart:async'; + +import 'package:bluebubbles/services/rustpush/send_retry_policy.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + group("SendRetryPolicy", () { + test("retries a transient resource failure", () { + final policy = SendRetryPolicy(); + + final decision = policy.next( + "Failed to generate resource: retrying in 4s", + ); + + expect(decision.kind, SendFailureKind.resourceUnavailable); + expect(decision.retry, isTrue); + expect(decision.delay, const Duration(seconds: 5)); + }); + + test("does not retry permanent resource failure", () { + final policy = SendRetryPolicy(); + + final decision = policy.next("Failed to generate resource: not retrying"); + + expect(decision.kind, SendFailureKind.permanent); + expect(decision.retry, isFalse); + expect(decision.markFailedLogin, isTrue); + }); + + test("stops when the reconnect budget is exhausted", () { + final policy = SendRetryPolicy(budget: const Duration(seconds: 6)); + + expect( + policy.next("Failed to generate resource: retrying in 4s").retry, + isTrue, + ); + final exhausted = policy.next( + "Failed to generate resource: retrying in 4s", + ); + + expect(exhausted.retry, isFalse); + expect(policy.elapsed, const Duration(seconds: 5)); + }); + + test("allows only one confirmation-timeout retry", () { + final policy = SendRetryPolicy(); + + expect(policy.next("Send timeout; try again").retry, isTrue); + expect(policy.next("Send timeout; try again").retry, isFalse); + }); + }); + + test("coalesces duplicate sends by message id", () async { + final registry = InFlightSendRegistry(); + final gate = Completer(); + var calls = 0; + + final first = registry.run("message-1", () async { + calls++; + await gate.future; + }); + final second = registry.run("message-1", () async { + calls++; + }); + + expect(identical(first, second), isTrue); + expect(calls, 1); + gate.complete(); + await first; + + await registry.run("message-1", () async { + calls++; + }); + expect(calls, 2); + }); +}