diff --git a/dogfooding/lib/screens/call_screen.dart b/dogfooding/lib/screens/call_screen.dart index 06443ae83..a74190755 100644 --- a/dogfooding/lib/screens/call_screen.dart +++ b/dogfooding/lib/screens/call_screen.dart @@ -312,6 +312,9 @@ class _CallScreenState extends State { call: call, disabledMicrophoneBackgroundColor: AppColorPalette.appRed, + // Keep the track alive on mute so speaking-while- + // muted detection also works on iOS/macOS. + stopTrackOnMute: false, ), ToggleCameraOption( call: call, diff --git a/packages/stream_video/CHANGELOG.md b/packages/stream_video/CHANGELOG.md index aa9c26323..25b9daab6 100644 --- a/packages/stream_video/CHANGELOG.md +++ b/packages/stream_video/CHANGELOG.md @@ -1,3 +1,10 @@ +## Upcoming + +### ✅ Added + +- Speaking-while-muted detection (`SpeakingWhileMutedRecognition`) now works on iOS, macOS and web (previously Android-only). On iOS/macOS it requires muting with `stopTrackOnMute: false`. Check the [cookbook](https://getstream.io/video/docs/flutter/ui-cookbook/speaking-while-muted/) for details and per-platform requirements. +- Added an optional `stopTrackOnMute` parameter to `Call.setMicrophoneEnabled`. The default (`true`, unchanged) stops and releases the audio track on mute; `false` keeps the track alive and sends silence instead. See the [documentation](https://getstream.io/video/docs/flutter/guides/camera-and-microphone/microphone-and-audio/) for the trade-offs. + ## 1.4.2 ### ✅ Added diff --git a/packages/stream_video/lib/src/audio_processing/audio_recognition_factory_io.dart b/packages/stream_video/lib/src/audio_processing/audio_recognition_factory_io.dart new file mode 100644 index 000000000..4a1d9c56e --- /dev/null +++ b/packages/stream_video/lib/src/audio_processing/audio_recognition_factory_io.dart @@ -0,0 +1,9 @@ +import 'audio_recognition.dart'; +import 'audio_recognition_webrtc.dart'; + +/// Native platforms receive speech-activity events from the audio device +/// module through the webrtc plugin. The audio device module already captures +/// the selected input device, so [audioInputDeviceIdProvider] is unused here. +AudioRecognition createPlatformAudioRecognition({ + String? Function()? audioInputDeviceIdProvider, +}) => AudioRecognitionWebRTC(); diff --git a/packages/stream_video/lib/src/audio_processing/audio_recognition_factory_web.dart b/packages/stream_video/lib/src/audio_processing/audio_recognition_factory_web.dart new file mode 100644 index 000000000..c62ee20dc --- /dev/null +++ b/packages/stream_video/lib/src/audio_processing/audio_recognition_factory_web.dart @@ -0,0 +1,10 @@ +import 'audio_recognition.dart'; +import 'audio_recognition_web_audio.dart'; + +/// Browsers deliver no audio-device-module speech-activity events, so web +/// falls back to a Web Audio API analyser over a dedicated microphone stream. +/// [audioInputDeviceIdProvider] resolves the microphone to monitor at each +/// detection start. +AudioRecognition createPlatformAudioRecognition({ + String? Function()? audioInputDeviceIdProvider, +}) => AudioRecognitionWebAudio(deviceIdProvider: audioInputDeviceIdProvider); diff --git a/packages/stream_video/lib/src/audio_processing/audio_recognition_web_audio.dart b/packages/stream_video/lib/src/audio_processing/audio_recognition_web_audio.dart new file mode 100644 index 000000000..ba7a77652 --- /dev/null +++ b/packages/stream_video/lib/src/audio_processing/audio_recognition_web_audio.dart @@ -0,0 +1,184 @@ +import 'dart:async'; +import 'dart:js_interop'; +import 'dart:typed_data'; + +import 'package:web/web.dart' as web; + +import '../logger/impl/tagged_logger.dart'; +import 'audio_recognition.dart'; + +/// [AudioRecognition] implementation for web backed by the Web Audio API. +/// +/// This implementation opens its own microphone stream and samples an +/// [web.AnalyserNode] on a timer, mirroring the JS SDK's `createSoundDetector`. +/// +/// Note: while active it holds a microphone capture, so the browser's +/// "microphone in use" indicator stays on even though the user is muted. This +/// is a limitation of the Web Audio API. +class AudioRecognitionWebAudio implements AudioRecognition { + AudioRecognitionWebAudio({ + this.config = const WebAudioRecognitionConfig(), + this.deviceIdProvider, + }); + + final WebAudioRecognitionConfig config; + + /// Resolves the microphone to monitor at each [start]. Used when + /// [WebAudioRecognitionConfig.deviceId] is null. Falls back to the + /// browser's default microphone when both are null. + final String? Function()? deviceIdProvider; + + web.MediaStream? _stream; + web.AudioContext? _audioContext; + Timer? _detectionTimer; + Timer? _trailingSilenceTimer; + bool _isSpeaking = false; + bool _started = false; + + /// Bumped on every [stop] so an in-flight [start] can detect it was + /// cancelled while awaiting `getUserMedia`. + int _generation = 0; + + @override + Future start({ + required SoundStateChangedCallback onSoundStateChanged, + }) async { + if (_started) return; + _started = true; + final generation = _generation; + + final web.MediaStream stream; + try { + final deviceId = config.deviceId ?? deviceIdProvider?.call(); + stream = await web.window.navigator.mediaDevices + .getUserMedia( + web.MediaStreamConstraints( + audio: deviceId == null + ? true.toJS + : { + 'deviceId': {'exact': deviceId}, + }.jsify()!, + ), + ) + .toDart; + } catch (_) { + _started = false; + rethrow; + } + + if (!_started || generation != _generation) { + // Stopped while acquiring the microphone. + _stopStream(stream); + return; + } + + _stream = stream; + final audioContext = web.AudioContext(); + _audioContext = audioContext; + + final source = audioContext.createMediaStreamSource(stream); + final analyser = audioContext.createAnalyser()..fftSize = config.fftSize; + source.connect(analyser); + + final frequencyData = Uint8List(analyser.frequencyBinCount).toJS; + _detectionTimer = Timer.periodic(config.detectionInterval, (_) { + analyser.getByteFrequencyData(frequencyData); + + var maxByte = 0; + for (final value in frequencyData.toDart) { + if (value > maxByte) maxByte = value; + } + + if (maxByte >= config.audioLevelThreshold) { + _trailingSilenceTimer?.cancel(); + _trailingSilenceTimer = null; + if (_isSpeaking) return; + _isSpeaking = true; + onSoundStateChanged( + SoundState(isSpeaking: true, audioLevel: maxByte / 255), + ); + } else if (_isSpeaking) { + // Debounce brief pauses so consumers don't flap. + _trailingSilenceTimer ??= Timer(config.speechTimeout, () { + _trailingSilenceTimer = null; + if (!_isSpeaking) return; + _isSpeaking = false; + onSoundStateChanged( + const SoundState(isSpeaking: false, audioLevel: 0), + ); + }); + } + }); + } + + @override + Future stop() async { + if (!_started) return; + _started = false; + _generation++; + + _detectionTimer?.cancel(); + _detectionTimer = null; + _trailingSilenceTimer?.cancel(); + _trailingSilenceTimer = null; + _isSpeaking = false; + + final audioContext = _audioContext; + _audioContext = null; + if (audioContext != null) { + try { + await audioContext.close().toDart; + } catch (e) { + _logger.w(() => 'Failed to close audio context: $e'); + } + } + + final stream = _stream; + _stream = null; + if (stream != null) _stopStream(stream); + } + + @override + Future dispose() async { + await stop(); + } + + void _stopStream(web.MediaStream stream) { + for (final track in stream.getTracks().toDart) { + track.stop(); + } + } +} + +class WebAudioRecognitionConfig { + const WebAudioRecognitionConfig({ + this.detectionInterval = const Duration(milliseconds: 500), + this.speechTimeout = const Duration(milliseconds: 500), + this.fftSize = 128, + this.audioLevelThreshold = 150, + this.deviceId, + }); + + /// How often the analyser is sampled. Mirrors the JS SDK's + /// `detectionFrequencyInMs`. + final Duration detectionInterval; + + /// Trailing silence duration before a speaking→silent transition is + /// surfaced to the consumer. Mirrors the debounce of the native + /// event-based implementation. + final Duration speechTimeout; + + /// FFT size of the analyser node; the sampled frequency-bin count is half + /// of this. + final int fftSize; + + /// A frequency-bin byte value (0–255) at or above this threshold counts as + /// sound. Mirrors the JS SDK's `audioLevelThreshold`. + final int audioLevelThreshold; + + /// Microphone device to monitor. Defaults to the browser's default + /// microphone. + final String? deviceId; +} + +final _logger = taggedLogger(tag: 'SV:AudioRecognitionWebAudio'); diff --git a/packages/stream_video/lib/src/audio_processing/speaking_while_muted_recognition.dart b/packages/stream_video/lib/src/audio_processing/speaking_while_muted_recognition.dart index 59c552604..fde70d510 100644 --- a/packages/stream_video/lib/src/audio_processing/speaking_while_muted_recognition.dart +++ b/packages/stream_video/lib/src/audio_processing/speaking_while_muted_recognition.dart @@ -4,7 +4,8 @@ import 'package:equatable/equatable.dart'; import 'package:state_notifier/state_notifier.dart'; import '../../stream_video.dart'; -import 'audio_recognition_webrtc.dart'; +import 'audio_recognition_factory_io.dart' + if (dart.library.js_interop) 'audio_recognition_factory_web.dart'; /// The [SpeakingWhileMutedRecognition.stream] emits state changes when an increase in audio volume /// is detected while the user is muted. @@ -18,6 +19,21 @@ import 'audio_recognition_webrtc.dart'; /// - If no audio is detected for a period of time, or if the user unmutes, /// the state reverts to `false`. /// +/// Platform behavior: +/// - Android/iOS/macOS: detection is driven by speech-activity events from the +/// native audio device module, no additional microphone capture is opened. +/// - iOS/macOS: events are only delivered while the audio engine keeps +/// capturing, so the microphone must be muted without stopping the track: +/// `call.setMicrophoneEnabled(enabled: false, stopTrackOnMute: false)`. +/// With the default mute (track stopped and released) no speech events +/// arrive on these platforms. Note that keeping the track alive leaves the +/// OS microphone-in-use indicator on while muted. +/// - Web: browsers expose no such events, so a dedicated microphone stream is +/// analysed with the Web Audio API while detection is active, which keeps +/// the browser's "microphone in use" indicator on while muted. +/// - Windows/Linux: not supported by the default implementation; supply a +/// custom [AudioRecognition] to enable it there. +/// /// Note: /// - Audio detection begins only after the user mutes themselves or is muted by someone else. /// - If the user joins a call already muted, audio detection won't start automatically. @@ -60,16 +76,29 @@ class SpeakingWhileMutedRecognition SpeakingWhileMutedRecognition({ required this.call, AudioRecognition? audioRecognition, - }) : _audioRecognition = audioRecognition ?? AudioRecognitionWebRTC(), - super(const SpeakingWhileMutedState._(isSpeakingWhileMuted: false)) { + }) : super(const SpeakingWhileMutedState._(isSpeakingWhileMuted: false)) { + _audioRecognition = + audioRecognition ?? + createPlatformAudioRecognition( + // Resolved lazily at each detection start so the web recognizer + // always monitors the currently selected microphone. + audioInputDeviceIdProvider: () => + call.state.value.audioInputDevice?.id, + ); _init(); } final Call call; - final AudioRecognition _audioRecognition; + late final AudioRecognition _audioRecognition; StreamSubscription? _callStateSubscription; bool _isActive = false; + /// The selected audio input reported by the most recent call-state event. + String? _lastSeenAudioInputDeviceId; + + /// The selected audio input at the time detection was started. + String? _activeAudioInputDeviceId; + void _init() { _callStateSubscription = call .partialState( @@ -77,16 +106,26 @@ class SpeakingWhileMutedRecognition isAudioEnabled: state.isAudioEnabled, canSendAudio: state.canSendAudio, status: state.status, + audioInputDeviceId: state.audioInputDevice?.id, ), ) .listen((state) { + _lastSeenAudioInputDeviceId = state.audioInputDeviceId; + if (state.status.isDisconnected) _stop(); if (!(state.status.isJoined || state.status.isConnected)) return; if (state.isAudioEnabled) { _stop(); } else if (state.canSendAudio) { - start(); + if (_isActive && + state.audioInputDeviceId != _activeAudioInputDeviceId) { + // The selected microphone changed while detecting — restart so + // detection follows the new device. + _restart(); + } else { + start(); + } } }); } @@ -100,6 +139,7 @@ class SpeakingWhileMutedRecognition Future start() async { if (_isActive) return; _isActive = true; + _activeAudioInputDeviceId = _lastSeenAudioInputDeviceId; try { await _audioRecognition.start( onSoundStateChanged: (soundState) { @@ -121,6 +161,11 @@ class SpeakingWhileMutedRecognition await _audioRecognition.stop(); } + Future _restart() async { + await _stop(); + await start(); + } + @override Future dispose() async { await _callStateSubscription?.cancel(); diff --git a/packages/stream_video/lib/src/call/call.dart b/packages/stream_video/lib/src/call/call.dart index 04a740a54..ef886e783 100644 --- a/packages/stream_video/lib/src/call/call.dart +++ b/packages/stream_video/lib/src/call/call.dart @@ -362,6 +362,11 @@ class Call { await _session?.resumeSuspendedAudioTracks(_suspendedTrackStates); _suspendedTrackStates.clear(); + // Resuming restarts recording, which clears the native ADM's microphone + // mute, and re-enables tracks from a snapshot taken before the suspension. + // Reconcile once the tracks have settled. + await _session?.rtcManager?.reconcileAppleAdmMicrophoneMute(); + _stateManager.state = _stateManager.callState.copyWith( isAudioSuspended: false, ); @@ -3656,9 +3661,14 @@ class Call { } /// Enables or disables the microphone for this call. + /// + /// [stopTrackOnMute] controls whether muting disables and stops (default: `true`) + /// or keeps the audio track alive but silent (`false`). On iOS/macOS, `false` keeps + /// muted-talker detection active but leaves the mic indicator on. When null, keeps default behavior. Future> setMicrophoneEnabled({ required bool enabled, AudioConstraints? constraints, + bool? stopTrackOnMute, }) async { if (enabled && state.value.isVideoModerated && @@ -3674,6 +3684,7 @@ class Call { await _session?.setMicrophoneEnabled( enabled, constraints: constraints, + stopTrackOnMute: stopTrackOnMute, ) ?? Result.error('Session is null'); diff --git a/packages/stream_video/lib/src/call/session/call_session.dart b/packages/stream_video/lib/src/call/session/call_session.dart index 031bb9999..af1c94bf5 100644 --- a/packages/stream_video/lib/src/call/session/call_session.dart +++ b/packages/stream_video/lib/src/call/session/call_session.dart @@ -1291,6 +1291,7 @@ class CallSession extends Disposable { Future> setMicrophoneEnabled( bool enabled, { AudioConstraints? constraints, + bool? stopTrackOnMute, }) async { final rtcManager = this.rtcManager; if (rtcManager == null) { @@ -1301,6 +1302,7 @@ class CallSession extends Disposable { return rtcManager.setMicrophoneEnabled( enabled: enabled, constraints: constraints, + stopTrackOnMute: stopTrackOnMute, ); }); diff --git a/packages/stream_video/lib/src/webrtc/peer_connection_factory.dart b/packages/stream_video/lib/src/webrtc/peer_connection_factory.dart index a10865f05..5f7e1443d 100644 --- a/packages/stream_video/lib/src/webrtc/peer_connection_factory.dart +++ b/packages/stream_video/lib/src/webrtc/peer_connection_factory.dart @@ -123,6 +123,55 @@ class StreamPeerConnectionFactory { await native.resumeAudio(); } + /// Whether ADM-level microphone mute is available on this platform. + static bool get isAppleAdmMicrophoneMuteSupported => + rtc.NativePeerConnectionFactory.isAdmMicrophoneMuteSupported; + + /// Mutes / unmutes microphone capture at the audio-device-module level + /// while the audio engine keeps running, arming Apple's muted-talker + /// detection, which powers speaking-while-muted events. + /// + /// No-op unless [isAppleAdmMicrophoneMuteSupported]. + Future setAppleAdmMicrophoneMuted(bool muted) async { + if (!isAppleAdmMicrophoneMuteSupported) { + _logger.w( + () => '[setAppleAdmMicrophoneMuted] unsupported platform, skipping', + ); + return; + } + + final native = _nativeFactory; + if (native == null) { + _logger.w(() => '[setAppleAdmMicrophoneMuted] no native factory'); + return; + } + _logger.i( + () => + '[setAppleAdmMicrophoneMuted] muted: $muted, ' + 'factoryId: ${native.factoryId}', + ); + await native.setMicrophoneMuted(muted); + } + + /// Reads back the ADM microphone mute state, or `null` when it cannot be + /// asked: no native factory, or a platform without ADM-level mute. + Future isAppleAdmMicrophoneMuted() async { + if (!isAppleAdmMicrophoneMuteSupported) { + _logger.w( + () => '[isAppleAdmMicrophoneMuted] unsupported platform, skipping', + ); + return null; + } + + final native = _nativeFactory; + if (native == null) { + _logger.w(() => '[isAppleAdmMicrophoneMuted] no native factory'); + return null; + } + + return native.isMicrophoneMuted(); + } + Future makeSubscriber({ required String sessionId, required SdpEditor sdpEditor, diff --git a/packages/stream_video/lib/src/webrtc/rtc_manager.dart b/packages/stream_video/lib/src/webrtc/rtc_manager.dart index 4531d4bcb..91c5eb064 100644 --- a/packages/stream_video/lib/src/webrtc/rtc_manager.dart +++ b/packages/stream_video/lib/src/webrtc/rtc_manager.dart @@ -788,7 +788,12 @@ extension PublisherRtcManager on RtcManager { } Future> publishTrack(RtcLocalTrack track) async { - if (track is RtcLocalAudioTrack) return publishAudioTrack(track: track); + if (track is RtcLocalAudioTrack) { + return publishAudioTrack( + track: track, + stopTrackOnMute: track.stopTrackOnMute, + ); + } if (track is RtcLocalVideoTrack) return publishVideoTrack(track: track); return Result.error('Unsupported track type: ${track.runtimeType}'); @@ -811,6 +816,12 @@ extension PublisherRtcManager on RtcManager { tracks[audioTrack.trackId] = audioTrack; var updatedTrack = audioTrack.copyWith(stopTrackOnMute: stopTrackOnMute); + // Ensure a new live audio track isn't still muted by an existing ADM-level mute. + // ADM mute persists beyond sessions (per-call factory). + if (_isAppleAdmMuteSupported(audioTrack) && audioTrack.mediaTrack.enabled) { + await _setAppleAdmMicrophoneMuted(false); + } + for (final option in publishOptions) { if (option.trackType != audioTrack.trackType) continue; @@ -868,6 +879,10 @@ extension PublisherRtcManager on RtcManager { onLocalTrackPublished?.call(updatedTrack); tracks[updatedTrack.trackId] = updatedTrack; + // Republishing restarts capture, which drops the ADM mute. No-op when the + // ADM already matches the track being published. + await reconcileAppleAdmMicrophoneMute(); + return Result.success(updatedTrack); } @@ -1131,11 +1146,18 @@ extension PublisherRtcManager on RtcManager { return Result.error('Track is not local'); } + // If [stopTrackOnMute] is true or unset, the track is stopped and released (default). + // If false, the track stays alive and is muted at the device level (iOS/macOS only). final track = originalTrack.copyWith(stopTrackOnMute: stopTrackOnMute); tracks[trackId] = track; + final useAdmLevelMute = + !track.stopTrackOnMute && _isAppleAdmMuteSupported(track); + track.disable(); - if (track.stopTrackOnMute) { + if (useAdmLevelMute) { + await _setAppleAdmMicrophoneMuted(true); + } else if (track.stopTrackOnMute) { // Releases the track and stops the permission indicator. await track.stop(); } @@ -1156,6 +1178,12 @@ extension PublisherRtcManager on RtcManager { return Result.error('Track is not local'); } + // Lift the ADM-level mute applied for soft-muted (not stopped) audio + // tracks on iOS/macOS. + if (!track.stopTrackOnMute && _isAppleAdmMuteSupported(track)) { + await _setAppleAdmMicrophoneMuted(false); + } + // If the track was released before, restart it. if (track.stopTrackOnMute) { final transceivers = transceiversManager @@ -1191,6 +1219,63 @@ extension PublisherRtcManager on RtcManager { return Result.success(track); } + /// Returns true if [track] should use ADM-level mute. + /// Audio tracks on iOS / macOS only. + bool _isAppleAdmMuteSupported(RtcLocalTrack track) => + StreamPeerConnectionFactory.isAppleAdmMicrophoneMuteSupported && + track.trackType == SfuTrackType.audio; + + Future _setAppleAdmMicrophoneMuted(bool muted) async { + try { + await pcFactory.setAppleAdmMicrophoneMuted(muted); + } catch (e, stk) { + _logger.w(() => '[setAppleAdmMicrophoneMuted] failed: $e\n$stk'); + } + } + + /// Reads the ADM's own mute state, or `null` when it cannot be determined. + Future _getAppleAdmMicrophoneMuted() async { + try { + return await pcFactory.isAppleAdmMicrophoneMuted(); + } catch (e, stk) { + _logger.w(() => '[getAppleAdmMicrophoneMuted] failed: $e\n$stk'); + return null; + } + } + + /// Syncs the ADM mute state with the local audio track, to avoid mismatches + /// after audio restarts or suspends. Idempotent: only updates when needed. + /// + /// No-op on platforms without ADM-level mute. + Future reconcileAppleAdmMicrophoneMute() async { + final track = getPublisherTrackByType(SfuTrackType.audio); + if (track == null) return; + if (!_isAppleAdmMuteSupported(track)) return; + + final bool desiredMuted; + if (track.mediaTrack.enabled) { + // A live track must never stay silenced by a leftover ADM mute. + desiredMuted = false; + } else if (!track.stopTrackOnMute) { + // A disabled track is the SDK's muted state when using ADM-level mute. + desiredMuted = true; + } else { + // When the track is muted by stopping it, ADM mute is not used. + return; + } + + final actual = await _getAppleAdmMicrophoneMuted(); + if (actual == desiredMuted) return; + + _logger.w( + () => + '[reconcileAppleAdmMicrophoneMute] ADM mute out of sync ' + '(adm reported: $actual, expected: $desiredMuted), re-applying for ' + 'track ${track.trackId}', + ); + await _setAppleAdmMicrophoneMuted(desiredMuted); + } + Future> createAudioTrack({ AudioConstraints? constraints, }) async { @@ -1495,11 +1580,13 @@ extension RtcManagerTrackHelper on RtcManager { Future> setMicrophoneEnabled({ bool enabled = true, AudioConstraints? constraints, + bool? stopTrackOnMute, }) { return _setTrackEnabled( trackType: SfuTrackType.audio, enabled: enabled, constraints: constraints, + stopTrackOnMute: stopTrackOnMute, ); } @@ -1544,6 +1631,7 @@ extension RtcManagerTrackHelper on RtcManager { required SfuTrackType trackType, required bool enabled, MediaConstraints? constraints, + bool? stopTrackOnMute, }) async { final track = getPublisherTrackByType(trackType); @@ -1561,6 +1649,7 @@ extension RtcManagerTrackHelper on RtcManager { final toggledTrack = await _toggleTrackMuteState( track: track, muted: !enabled, + stopTrackOnMute: stopTrackOnMute, ); return Result.success(toggledTrack); @@ -1581,9 +1670,10 @@ extension RtcManagerTrackHelper on RtcManager { Future _toggleTrackMuteState({ required RtcLocalTrack track, required bool muted, + bool? stopTrackOnMute, }) async { if (muted) { - await muteTrack(trackId: track.trackId); + await muteTrack(trackId: track.trackId, stopTrackOnMute: stopTrackOnMute); // If the track is a screen share track, mute the audio track as well. if (track.trackType == SfuTrackType.screenShare) { diff --git a/packages/stream_video/test/src/audio_processing/speaking_while_muted_recognition_test.dart b/packages/stream_video/test/src/audio_processing/speaking_while_muted_recognition_test.dart index 0020499d8..2c41efe02 100644 --- a/packages/stream_video/test/src/audio_processing/speaking_while_muted_recognition_test.dart +++ b/packages/stream_video/test/src/audio_processing/speaking_while_muted_recognition_test.dart @@ -10,6 +10,7 @@ typedef FilteredCallState = ({ bool isAudioEnabled, bool canSendAudio, CallStatus status, + String? audioInputDeviceId, }); void main() { @@ -137,6 +138,60 @@ void main() { await sut.dispose(); }); + test( + 'Test audio input device change while active restarts detection', + () async { + final sut = SpeakingWhileMutedRecognition( + call: call, + audioRecognition: audioRecognition, + ); + + callStateStreamController.add( + createCallState(isAudioEnabled: false, audioInputDeviceId: 'mic-a'), + ); + await Future.delayed(Duration.zero); + + verify( + () => audioRecognition.start( + onSoundStateChanged: any(named: 'onSoundStateChanged'), + ), + ).called(1); + + // Switching the microphone while muted restarts detection so it + // follows the newly selected device. + callStateStreamController.add( + createCallState(isAudioEnabled: false, audioInputDeviceId: 'mic-b'), + ); + await Future.delayed(Duration.zero); + + verify(() => audioRecognition.stop()).called(1); + verify( + () => audioRecognition.start( + onSoundStateChanged: any(named: 'onSoundStateChanged'), + ), + ).called(1); + + // Same device again — no restart. + callStateStreamController.add( + createCallState( + isAudioEnabled: false, + canSendAudio: true, + audioInputDeviceId: 'mic-b', + ), + ); + await Future.delayed(Duration.zero); + + verifyNever(() => audioRecognition.stop()); + verifyNever( + () => audioRecognition.start( + onSoundStateChanged: any(named: 'onSoundStateChanged'), + ), + ); + + await sut.dispose(); + }, + ); + test('Test disconnecting from call', () async { final sut = SpeakingWhileMutedRecognition( call: call, @@ -171,10 +226,12 @@ FilteredCallState createCallState({ bool? isAudioEnabled, bool? canSendAudio, CallStatus? status, + String? audioInputDeviceId, }) { return ( isAudioEnabled: isAudioEnabled ?? true, canSendAudio: canSendAudio ?? true, status: status ?? CallStatus.joined(), + audioInputDeviceId: audioInputDeviceId, ); } diff --git a/packages/stream_video/test/src/webrtc/rtc_manager_mute_test.dart b/packages/stream_video/test/src/webrtc/rtc_manager_mute_test.dart new file mode 100644 index 000000000..c5fcea509 --- /dev/null +++ b/packages/stream_video/test/src/webrtc/rtc_manager_mute_test.dart @@ -0,0 +1,406 @@ +import 'dart:io' show Platform; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:mocktail/mocktail.dart'; +import 'package:stream_video/src/webrtc/peer_connection_factory.dart'; +import 'package:stream_video/src/webrtc/rtc_manager.dart'; +import 'package:stream_video/src/webrtc/traced_peer_connection.dart'; +import 'package:stream_video/stream_video.dart'; +import 'package:stream_webrtc_flutter/stream_webrtc_flutter.dart' as rtc; + +import '../../test_helpers.dart'; +import '../call/fixtures/data.dart'; + +class _MockTracedPeerConnection extends Mock + implements TracedStreamPeerConnection {} + +class _MockStreamPeerConnectionFactory extends Mock + implements StreamPeerConnectionFactory {} + +class _FakeMediaStreamTrack extends Fake implements rtc.MediaStreamTrack { + _FakeMediaStreamTrack({required this.kind}); + + @override + final String? kind; + + int stopCallCount = 0; + + @override + bool enabled = true; + + @override + Future stop() async { + stopCallCount++; + } +} + +class _FakeMediaStream extends Fake implements rtc.MediaStream { + int disposeCallCount = 0; + + @override + Future dispose() async { + disposeCallCount++; + } +} + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + + late _MockStreamPeerConnectionFactory pcFactory; + late RtcManager rtcManager; + + setUp(() { + pcFactory = _MockStreamPeerConnectionFactory(); + when( + () => pcFactory.setAppleAdmMicrophoneMuted(any()), + ).thenAnswer((_) async {}); + when( + () => pcFactory.isAppleAdmMicrophoneMuted(), + ).thenAnswer((_) async => false); + + final streamVideo = MockStreamVideo(); + when(() => streamVideo.options).thenReturn( + StreamVideoOptions(clientEventsReportingEnabled: false), + ); + + rtcManager = RtcManager( + sessionId: 'test-session', + callCid: SampleCallData.defaultCid, + publisherId: 'test-publisher', + publisher: null, + subscriber: _MockTracedPeerConnection(), + publishOptions: [], + stateManager: MockCallStateNotifier(), + streamVideo: streamVideo, + pcFactory: pcFactory, + ); + }); + + RtcLocalAudioTrack addAudioTrack({ + required _FakeMediaStreamTrack mediaTrack, + required _FakeMediaStream mediaStream, + }) { + final track = RtcLocalAudioTrack( + trackIdPrefix: 'test-publisher', + trackType: SfuTrackType.audio, + mediaStream: mediaStream, + mediaTrack: mediaTrack, + mediaConstraints: const AudioConstraints(), + ); + rtcManager.tracks[track.trackId] = track; + return track; + } + + group( + 'RtcManager audio mute on Apple platforms (ADM-level mute)', + // The ADM-level mute path is gated on CurrentPlatform.isIos/isMacOS; + // running the suite on a macOS host exercises the same branch iOS takes. + skip: !Platform.isMacOS + ? 'ADM-level mute branch only runs on Apple platforms' + : false, + () { + test('default mute keeps the legacy stop-and-release behavior', () async { + final mediaTrack = _FakeMediaStreamTrack(kind: 'audio'); + final mediaStream = _FakeMediaStream(); + final track = addAudioTrack( + mediaTrack: mediaTrack, + mediaStream: mediaStream, + ); + + final result = await rtcManager.muteTrack(trackId: track.trackId); + + expect(result.isSuccess, isTrue); + verifyNever(() => pcFactory.setAppleAdmMicrophoneMuted(any())); + expect(mediaTrack.stopCallCount, greaterThan(0)); + expect(mediaStream.disposeCallCount, greaterThan(0)); + }); + + test('soft mute (stopTrackOnMute: false) mutes via the ADM ' + 'and keeps the track alive', () async { + final mediaTrack = _FakeMediaStreamTrack(kind: 'audio'); + final mediaStream = _FakeMediaStream(); + final track = addAudioTrack( + mediaTrack: mediaTrack, + mediaStream: mediaStream, + ); + + final result = await rtcManager.muteTrack( + trackId: track.trackId, + stopTrackOnMute: false, + ); + + expect(result.isSuccess, isTrue); + verify(() => pcFactory.setAppleAdmMicrophoneMuted(true)).called(1); + // The capture keeps running — muted inside the ADM, not stopped. + expect(mediaTrack.stopCallCount, 0); + expect(mediaStream.disposeCallCount, 0); + expect(mediaTrack.enabled, isFalse); + + // The stored track must not be flagged for recreation on unmute. + final storedTrack = rtcManager.tracks[track.trackId]; + expect((storedTrack! as RtcLocalTrack).stopTrackOnMute, isFalse); + }); + + test('unmute lifts the ADM mute and re-enables the track', () async { + final mediaTrack = _FakeMediaStreamTrack(kind: 'audio'); + final track = addAudioTrack( + mediaTrack: mediaTrack, + mediaStream: _FakeMediaStream(), + ); + + await rtcManager.muteTrack( + trackId: track.trackId, + stopTrackOnMute: false, + ); + final result = await rtcManager.unmuteTrack(trackId: track.trackId); + + expect(result.isSuccess, isTrue); + verify(() => pcFactory.setAppleAdmMicrophoneMuted(false)).called(1); + // No recreation happened — the original track was simply re-enabled. + expect(mediaTrack.enabled, isTrue); + expect(mediaTrack.stopCallCount, 0); + }); + + test('soft-mute behavior sticks for subsequent default mutes', () async { + final mediaTrack = _FakeMediaStreamTrack(kind: 'audio'); + final track = addAudioTrack( + mediaTrack: mediaTrack, + mediaStream: _FakeMediaStream(), + ); + + await rtcManager.muteTrack( + trackId: track.trackId, + stopTrackOnMute: false, + ); + await rtcManager.unmuteTrack(trackId: track.trackId); + + // No explicit argument — the track keeps its soft-mute flag. + await rtcManager.muteTrack(trackId: track.trackId); + + verify(() => pcFactory.setAppleAdmMicrophoneMuted(true)).called(2); + expect(mediaTrack.stopCallCount, 0); + }); + + test('explicit stopTrackOnMute: true stops the track', () async { + final mediaTrack = _FakeMediaStreamTrack(kind: 'audio'); + final mediaStream = _FakeMediaStream(); + final track = addAudioTrack( + mediaTrack: mediaTrack, + mediaStream: mediaStream, + ); + + final result = await rtcManager.muteTrack( + trackId: track.trackId, + stopTrackOnMute: true, + ); + + expect(result.isSuccess, isTrue); + verifyNever(() => pcFactory.setAppleAdmMicrophoneMuted(any())); + expect(mediaTrack.stopCallCount, greaterThan(0)); + expect(mediaStream.disposeCallCount, greaterThan(0)); + }); + + test( + 'setMicrophoneEnabled threads stopTrackOnMute down to the mute', + () async { + final mediaTrack = _FakeMediaStreamTrack(kind: 'audio'); + final track = addAudioTrack( + mediaTrack: mediaTrack, + mediaStream: _FakeMediaStream(), + ); + + final result = await rtcManager.setMicrophoneEnabled( + enabled: false, + stopTrackOnMute: false, + ); + + expect(result.isSuccess, isTrue); + verify(() => pcFactory.setAppleAdmMicrophoneMuted(true)).called(1); + expect(mediaTrack.stopCallCount, 0); + expect(rtcManager.tracks[track.trackId], isNotNull); + }, + ); + + test('video tracks are not muted through the ADM', () async { + final mediaTrack = _FakeMediaStreamTrack(kind: 'video'); + final mediaStream = _FakeMediaStream(); + final track = RtcLocalCameraTrack( + trackIdPrefix: 'test-publisher', + trackType: SfuTrackType.video, + mediaStream: mediaStream, + mediaTrack: mediaTrack, + mediaConstraints: const CameraConstraints(), + ); + rtcManager.tracks[track.trackId] = track; + + final result = await rtcManager.muteTrack(trackId: track.trackId); + + expect(result.isSuccess, isTrue); + verifyNever(() => pcFactory.setAppleAdmMicrophoneMuted(any())); + // Default video behavior is unchanged: stop-and-release. + expect(mediaTrack.stopCallCount, greaterThan(0)); + }); + + test('publishing a live audio track lifts a leftover ADM mute', () async { + // The ADM belongs to the per-call factory and outlives sessions. + // After mute → rejoin → unmute, the new session publishes a fresh + // track instead of running unmuteTrack — the leftover ADM mute must + // be lifted or the microphone stays silent. + final track = RtcLocalAudioTrack( + trackIdPrefix: 'test-publisher', + trackType: SfuTrackType.audio, + mediaStream: _FakeMediaStream(), + mediaTrack: _FakeMediaStreamTrack(kind: 'audio'), + mediaConstraints: const AudioConstraints(), + ); + + final result = await rtcManager.publishAudioTrack(track: track); + + expect(result.isSuccess, isTrue); + verify(() => pcFactory.setAppleAdmMicrophoneMuted(false)).called(1); + }); + + test('publishing a disabled audio track keeps the ADM mute', () async { + final mediaTrack = _FakeMediaStreamTrack(kind: 'audio') + ..enabled = false; + final track = RtcLocalAudioTrack( + trackIdPrefix: 'test-publisher', + trackType: SfuTrackType.audio, + mediaStream: _FakeMediaStream(), + mediaTrack: mediaTrack, + mediaConstraints: const AudioConstraints(), + ); + + final result = await rtcManager.publishAudioTrack(track: track); + + expect(result.isSuccess, isTrue); + verifyNever(() => pcFactory.setAppleAdmMicrophoneMuted(any())); + }); + + test('reconcile restores a mute the ADM dropped', () async { + // Anything that restarts capture (resumeAudio, an internal + // StartRecording) clears the ADM mute without telling the SDK. + final mediaTrack = _FakeMediaStreamTrack(kind: 'audio'); + final track = addAudioTrack( + mediaTrack: mediaTrack, + mediaStream: _FakeMediaStream(), + ); + await rtcManager.muteTrack( + trackId: track.trackId, + stopTrackOnMute: false, + ); + + // ADM reports unmuted while the SDK considers the track muted. + await rtcManager.reconcileAppleAdmMicrophoneMute(); + + // Once for the mute itself, once for the re-assert. + verify(() => pcFactory.setAppleAdmMicrophoneMuted(true)).called(2); + }); + + test('reconcile lifts an ADM mute left on a live track', () async { + // Muting during an audio suspension and then resuming re-enables the + // track from the pre-suspension snapshot, leaving the ADM muted while + // the SDK reports the mic live — the direction where the user talks and + // nobody hears them. + when( + () => pcFactory.isAppleAdmMicrophoneMuted(), + ).thenAnswer((_) async => true); + + final mediaTrack = _FakeMediaStreamTrack(kind: 'audio'); + final track = addAudioTrack( + mediaTrack: mediaTrack, + mediaStream: _FakeMediaStream(), + ); + await rtcManager.muteTrack( + trackId: track.trackId, + stopTrackOnMute: false, + ); + // Re-enabled outside of unmuteTrack, as the resume path does. + mediaTrack.enabled = true; + + await rtcManager.reconcileAppleAdmMicrophoneMute(); + + verify(() => pcFactory.setAppleAdmMicrophoneMuted(false)).called(1); + }); + + test('reconcile is a no-op when the ADM already matches', () async { + when( + () => pcFactory.isAppleAdmMicrophoneMuted(), + ).thenAnswer((_) async => true); + + final track = addAudioTrack( + mediaTrack: _FakeMediaStreamTrack(kind: 'audio'), + mediaStream: _FakeMediaStream(), + ); + await rtcManager.muteTrack( + trackId: track.trackId, + stopTrackOnMute: false, + ); + + await rtcManager.reconcileAppleAdmMicrophoneMute(); + + verify(() => pcFactory.setAppleAdmMicrophoneMuted(true)).called(1); + verifyNever(() => pcFactory.setAppleAdmMicrophoneMuted(false)); + }); + + test('reconcile leaves a stopped-on-mute track alone', () async { + // The mute is expressed by stopping the track, so the ADM mute is not + // the mechanism and has no desired value to enforce. + when( + () => pcFactory.isAppleAdmMicrophoneMuted(), + ).thenAnswer((_) async => true); + + final mediaTrack = _FakeMediaStreamTrack(kind: 'audio') + ..enabled = false; + addAudioTrack( + mediaTrack: mediaTrack, + mediaStream: _FakeMediaStream(), + ); + + await rtcManager.reconcileAppleAdmMicrophoneMute(); + + verifyNever(() => pcFactory.isAppleAdmMicrophoneMuted()); + verifyNever(() => pcFactory.setAppleAdmMicrophoneMuted(any())); + }); + + test('republishing preserves the ADM-level mute opt-in', () async { + // publishAudioTrack defaults stopTrackOnMute to true; republishing an + // existing track must not silently downgrade its mute strategy, or the + // next mute would stop the track and kill speech detection. + final track = RtcLocalAudioTrack( + trackIdPrefix: 'test-publisher', + trackType: SfuTrackType.audio, + mediaStream: _FakeMediaStream(), + mediaTrack: _FakeMediaStreamTrack(kind: 'audio'), + mediaConstraints: const AudioConstraints(), + stopTrackOnMute: false, + ); + + final result = await rtcManager.publishTrack(track); + + expect(result.isSuccess, isTrue); + expect(result.getDataOrNull()!.stopTrackOnMute, isFalse); + }); + + test('ADM mute failure still leaves the track muted', () async { + when( + () => pcFactory.setAppleAdmMicrophoneMuted(any()), + ).thenThrow(Exception('method channel unavailable')); + + final mediaTrack = _FakeMediaStreamTrack(kind: 'audio'); + final track = addAudioTrack( + mediaTrack: mediaTrack, + mediaStream: _FakeMediaStream(), + ); + + final result = await rtcManager.muteTrack( + trackId: track.trackId, + stopTrackOnMute: false, + ); + + expect(result.isSuccess, isTrue); + expect(mediaTrack.enabled, isFalse); + }); + }, + ); +} diff --git a/packages/stream_video_flutter/CHANGELOG.md b/packages/stream_video_flutter/CHANGELOG.md index fc3398106..2b7be6f06 100644 --- a/packages/stream_video_flutter/CHANGELOG.md +++ b/packages/stream_video_flutter/CHANGELOG.md @@ -1,3 +1,9 @@ +## Upcoming + +### ✅ Added + +- Added an optional `stopTrackOnMute` parameter to `ToggleMicrophoneOption`, passed through to `Call.setMicrophoneEnabled`. Set it to `false` to keep the audio track alive on mute — required on iOS/macOS for [speaking-while-muted detection](https://getstream.io/video/docs/flutter/ui-cookbook/speaking-while-muted/). + ## 1.4.2 ### ✅ Added diff --git a/packages/stream_video_flutter/lib/src/call_controls/controls/toggle_microphone_option.dart b/packages/stream_video_flutter/lib/src/call_controls/controls/toggle_microphone_option.dart index 7771d1a50..c784e889c 100644 --- a/packages/stream_video_flutter/lib/src/call_controls/controls/toggle_microphone_option.dart +++ b/packages/stream_video_flutter/lib/src/call_controls/controls/toggle_microphone_option.dart @@ -18,6 +18,7 @@ class ToggleMicrophoneOption extends StatelessWidget { this.disabledMicrophoneIconColor, this.enabledMicrophoneBackgroundColor, this.disabledMicrophoneBackgroundColor, + this.stopTrackOnMute, }); /// Represents a call. @@ -45,6 +46,10 @@ class ToggleMicrophoneOption extends StatelessWidget { /// Color of the background when microphone is disabled final Color? disabledMicrophoneBackgroundColor; + /// Determines if muting the microphone stops and releases (`true`) or keeps and silences (`false`) the audio track. + /// Setting to `false` is necessary for "speaking-while-muted" detection on iOS and macOS. + final bool? stopTrackOnMute; + @override Widget build(BuildContext context) { Widget buildContent(bool enabled) { @@ -59,7 +64,10 @@ class ToggleMicrophoneOption extends StatelessWidget { ? enabledMicrophoneBackgroundColor : disabledMicrophoneBackgroundColor, onPressed: () { - call.setMicrophoneEnabled(enabled: !enabled); + call.setMicrophoneEnabled( + enabled: !enabled, + stopTrackOnMute: stopTrackOnMute, + ); }, ); }