feat(llc): speaking while muted (android, ios, web)

dogfooding/lib/screens/call_screen.dart

@@ -312,6 +312,9 @@ class _CallScreenState extends State<CallScreen> {
                           call: call,
                           disabledMicrophoneBackgroundColor:
                               AppColorPalette.appRed,
+                          // Keep the track alive on mute so speaking-while-
+                          // muted detection also works on iOS/macOS.
+                          stopTrackOnMute: false,
                         ),
                         ToggleCameraOption(
                           call: call,

packages/stream_video/CHANGELOG.md

@@ -1,3 +1,10 @@
+## Upcoming
+
+### ✅ Added
+
+- Speaking-while-muted detection (`SpeakingWhileMutedRecognition`) now works on iOS, macOS and web (previously Android-only). On iOS/macOS it requires muting with `stopTrackOnMute: false`. Check the [cookbook](https://getstream.io/video/docs/flutter/ui-cookbook/speaking-while-muted/) for details and per-platform requirements.
+- Added an optional `stopTrackOnMute` parameter to `Call.setMicrophoneEnabled`. The default (`true`, unchanged) stops and releases the audio track on mute; `false` keeps the track alive and sends silence instead. See the [documentation](https://getstream.io/video/docs/flutter/guides/camera-and-microphone/microphone-and-audio/) for the trade-offs.
+
 ## 1.4.2
 
 ### ✅ Added

packages/stream_video/lib/src/audio_processing/audio_recognition_factory_io.dart

@@ -0,0 +1,9 @@
+import 'audio_recognition.dart';
+import 'audio_recognition_webrtc.dart';
+
+/// Native platforms receive speech-activity events from the audio device
+/// module through the webrtc plugin. The audio device module already captures
+/// the selected input device, so [audioInputDeviceIdProvider] is unused here.
+AudioRecognition createPlatformAudioRecognition({
+  String? Function()? audioInputDeviceIdProvider,
+}) => AudioRecognitionWebRTC();

packages/stream_video/lib/src/audio_processing/audio_recognition_factory_web.dart

@@ -0,0 +1,10 @@
+import 'audio_recognition.dart';
+import 'audio_recognition_web_audio.dart';
+
+/// Browsers deliver no audio-device-module speech-activity events, so web
+/// falls back to a Web Audio API analyser over a dedicated microphone stream.
+/// [audioInputDeviceIdProvider] resolves the microphone to monitor at each
+/// detection start.
+AudioRecognition createPlatformAudioRecognition({
+  String? Function()? audioInputDeviceIdProvider,
+}) => AudioRecognitionWebAudio(deviceIdProvider: audioInputDeviceIdProvider);

packages/stream_video/lib/src/audio_processing/audio_recognition_web_audio.dart

@@ -0,0 +1,184 @@
+import 'dart:async';
+import 'dart:js_interop';
+import 'dart:typed_data';
+
+import 'package:web/web.dart' as web;
+
+import '../logger/impl/tagged_logger.dart';
+import 'audio_recognition.dart';
+
+/// [AudioRecognition] implementation for web backed by the Web Audio API.
+///
+/// This implementation opens its own microphone stream and samples an
+/// [web.AnalyserNode] on a timer, mirroring the JS SDK's `createSoundDetector`.
+///
+/// Note: while active it holds a microphone capture, so the browser's
+/// "microphone in use" indicator stays on even though the user is muted. This
+/// is a limitation of the Web Audio API.
+class AudioRecognitionWebAudio implements AudioRecognition {
+  AudioRecognitionWebAudio({
+    this.config = const WebAudioRecognitionConfig(),
+    this.deviceIdProvider,
+  });
+
+  final WebAudioRecognitionConfig config;
+
+  /// Resolves the microphone to monitor at each [start]. Used when
+  /// [WebAudioRecognitionConfig.deviceId] is null. Falls back to the
+  /// browser's default microphone when both are null.
+  final String? Function()? deviceIdProvider;
+
+  web.MediaStream? _stream;
+  web.AudioContext? _audioContext;
+  Timer? _detectionTimer;
+  Timer? _trailingSilenceTimer;
+  bool _isSpeaking = false;
+  bool _started = false;
+
+  /// Bumped on every [stop] so an in-flight [start] can detect it was
+  /// cancelled while awaiting `getUserMedia`.
+  int _generation = 0;
+
+  @override
+  Future<void> start({
+    required SoundStateChangedCallback onSoundStateChanged,
+  }) async {
+    if (_started) return;
+    _started = true;
+    final generation = _generation;
+
+    final web.MediaStream stream;
+    try {
+      final deviceId = config.deviceId ?? deviceIdProvider?.call();
+      stream = await web.window.navigator.mediaDevices
+          .getUserMedia(
+            web.MediaStreamConstraints(
+              audio: deviceId == null
+                  ? true.toJS
+                  : <String, Object>{
+                      'deviceId': {'exact': deviceId},
+                    }.jsify()!,
+            ),
+          )
+          .toDart;
+    } catch (_) {
+      _started = false;
+      rethrow;
+    }
+
+    if (!_started || generation != _generation) {
+      // Stopped while acquiring the microphone.
+      _stopStream(stream);
+      return;
+    }
+
+    _stream = stream;
+    final audioContext = web.AudioContext();
+    _audioContext = audioContext;
+
+    final source = audioContext.createMediaStreamSource(stream);
+    final analyser = audioContext.createAnalyser()..fftSize = config.fftSize;
+    source.connect(analyser);
+
+    final frequencyData = Uint8List(analyser.frequencyBinCount).toJS;
+    _detectionTimer = Timer.periodic(config.detectionInterval, (_) {
+      analyser.getByteFrequencyData(frequencyData);
+
+      var maxByte = 0;
+      for (final value in frequencyData.toDart) {
+        if (value > maxByte) maxByte = value;
+      }
+
+      if (maxByte >= config.audioLevelThreshold) {
+        _trailingSilenceTimer?.cancel();
+        _trailingSilenceTimer = null;
+        if (_isSpeaking) return;
+        _isSpeaking = true;
+        onSoundStateChanged(
+          SoundState(isSpeaking: true, audioLevel: maxByte / 255),
+        );
+      } else if (_isSpeaking) {
+        // Debounce brief pauses so consumers don't flap.
+        _trailingSilenceTimer ??= Timer(config.speechTimeout, () {
+          _trailingSilenceTimer = null;
+          if (!_isSpeaking) return;
+          _isSpeaking = false;
+          onSoundStateChanged(
+            const SoundState(isSpeaking: false, audioLevel: 0),
+          );
+        });
+      }
+    });
+  }
+
+  @override
+  Future<void> stop() async {
+    if (!_started) return;
+    _started = false;
+    _generation++;
+
+    _detectionTimer?.cancel();
+    _detectionTimer = null;
+    _trailingSilenceTimer?.cancel();
+    _trailingSilenceTimer = null;
+    _isSpeaking = false;
+
+    final audioContext = _audioContext;
+    _audioContext = null;
+    if (audioContext != null) {
+      try {
+        await audioContext.close().toDart;
+      } catch (e) {
+        _logger.w(() => 'Failed to close audio context: $e');
+      }
+    }
+
+    final stream = _stream;
+    _stream = null;
+    if (stream != null) _stopStream(stream);
+  }
+
+  @override
+  Future<void> dispose() async {
+    await stop();
+  }
+
+  void _stopStream(web.MediaStream stream) {
+    for (final track in stream.getTracks().toDart) {
+      track.stop();
+    }
+  }
+}
+
+class WebAudioRecognitionConfig {
+  const WebAudioRecognitionConfig({
+    this.detectionInterval = const Duration(milliseconds: 500),
+    this.speechTimeout = const Duration(milliseconds: 500),
+    this.fftSize = 128,
+    this.audioLevelThreshold = 150,
+    this.deviceId,
+  });
+
+  /// How often the analyser is sampled. Mirrors the JS SDK's
+  /// `detectionFrequencyInMs`.
+  final Duration detectionInterval;
+
+  /// Trailing silence duration before a speaking→silent transition is
+  /// surfaced to the consumer. Mirrors the debounce of the native
+  /// event-based implementation.
+  final Duration speechTimeout;
+
+  /// FFT size of the analyser node; the sampled frequency-bin count is half
+  /// of this.
+  final int fftSize;
+
+  /// A frequency-bin byte value (0–255) at or above this threshold counts as
+  /// sound. Mirrors the JS SDK's `audioLevelThreshold`.
+  final int audioLevelThreshold;
+
+  /// Microphone device to monitor. Defaults to the browser's default
+  /// microphone.
+  final String? deviceId;
+}
+
+final _logger = taggedLogger(tag: 'SV:AudioRecognitionWebAudio');

packages/stream_video/lib/src/audio_processing/speaking_while_muted_recognition.dart

@@ -4,7 +4,8 @@ import 'package:equatable/equatable.dart';
 import 'package:state_notifier/state_notifier.dart';
 
 import '../../stream_video.dart';
-import 'audio_recognition_webrtc.dart';
+import 'audio_recognition_factory_io.dart'
+    if (dart.library.js_interop) 'audio_recognition_factory_web.dart';
 
 /// The [SpeakingWhileMutedRecognition.stream] emits state changes when an increase in audio volume
 /// is detected while the user is muted.
@@ -18,6 +19,21 @@ import 'audio_recognition_webrtc.dart';
 /// - If no audio is detected for a period of time, or if the user unmutes,
 ///   the state reverts to `false`.
 ///
+/// Platform behavior:
+/// - Android/iOS/macOS: detection is driven by speech-activity events from the
+///   native audio device module, no additional microphone capture is opened.
+/// - iOS/macOS: events are only delivered while the audio engine keeps
+///   capturing, so the microphone must be muted without stopping the track:
+///   `call.setMicrophoneEnabled(enabled: false, stopTrackOnMute: false)`.
+///   With the default mute (track stopped and released) no speech events
+///   arrive on these platforms. Note that keeping the track alive leaves the
+///   OS microphone-in-use indicator on while muted.
+/// - Web: browsers expose no such events, so a dedicated microphone stream is
+///   analysed with the Web Audio API while detection is active, which keeps
+///   the browser's "microphone in use" indicator on while muted.
+/// - Windows/Linux: not supported by the default implementation; supply a
+///   custom [AudioRecognition] to enable it there.
+///
 /// Note:
 /// - Audio detection begins only after the user mutes themselves or is muted by someone else.
 /// - If the user joins a call already muted, audio detection won't start automatically.
@@ -60,33 +76,56 @@ class SpeakingWhileMutedRecognition
   SpeakingWhileMutedRecognition({
     required this.call,
     AudioRecognition? audioRecognition,
-  }) : _audioRecognition = audioRecognition ?? AudioRecognitionWebRTC(),
-       super(const SpeakingWhileMutedState._(isSpeakingWhileMuted: false)) {
+  }) : super(const SpeakingWhileMutedState._(isSpeakingWhileMuted: false)) {
+    _audioRecognition =
+        audioRecognition ??
+        createPlatformAudioRecognition(
+          // Resolved lazily at each detection start so the web recognizer
+          // always monitors the currently selected microphone.
+          audioInputDeviceIdProvider: () =>
+              call.state.value.audioInputDevice?.id,
+        );
     _init();
   }
 
   final Call call;
-  final AudioRecognition _audioRecognition;
+  late final AudioRecognition _audioRecognition;
   StreamSubscription<void>? _callStateSubscription;
   bool _isActive = false;
 
+  /// The selected audio input reported by the most recent call-state event.
+  String? _lastSeenAudioInputDeviceId;
+
+  /// The selected audio input at the time detection was started.
+  String? _activeAudioInputDeviceId;
+
   void _init() {
     _callStateSubscription = call
         .partialState(
           (state) => (
             isAudioEnabled: state.isAudioEnabled,
             canSendAudio: state.canSendAudio,
             status: state.status,
+            audioInputDeviceId: state.audioInputDevice?.id,
           ),
         )
         .listen((state) {
+          _lastSeenAudioInputDeviceId = state.audioInputDeviceId;
+
           if (state.status.isDisconnected) _stop();
           if (!(state.status.isJoined || state.status.isConnected)) return;
 
           if (state.isAudioEnabled) {
             _stop();
           } else if (state.canSendAudio) {
-            start();
+            if (_isActive &&
+                state.audioInputDeviceId != _activeAudioInputDeviceId) {
+              // The selected microphone changed while detecting — restart so
+              // detection follows the new device.
+              _restart();
+            } else {
+              start();
+            }
           }
         });
   }
@@ -100,6 +139,7 @@ class SpeakingWhileMutedRecognition
   Future<void> start() async {
     if (_isActive) return;
     _isActive = true;
+    _activeAudioInputDeviceId = _lastSeenAudioInputDeviceId;
     try {
       await _audioRecognition.start(
         onSoundStateChanged: (soundState) {
@@ -121,6 +161,11 @@ class SpeakingWhileMutedRecognition
     await _audioRecognition.stop();
   }
 
+  Future<void> _restart() async {
+    await _stop();
+    await start();
+  }
+
   @override
   Future<void> dispose() async {
     await _callStateSubscription?.cancel();

packages/stream_video/lib/src/call/call.dart

@@ -362,6 +362,11 @@ class Call {
     await _session?.resumeSuspendedAudioTracks(_suspendedTrackStates);
     _suspendedTrackStates.clear();
 
+    // Resuming restarts recording, which clears the native ADM's microphone
+    // mute, and re-enables tracks from a snapshot taken before the suspension.
+    // Reconcile once the tracks have settled.
+    await _session?.rtcManager?.reconcileAppleAdmMicrophoneMute();
+
     _stateManager.state = _stateManager.callState.copyWith(
       isAudioSuspended: false,
     );
@@ -3656,9 +3661,14 @@ class Call {
   }
 
   /// Enables or disables the microphone for this call.
+  ///
+  /// [stopTrackOnMute] controls whether muting disables and stops (default: `true`)
+  /// or keeps the audio track alive but silent (`false`). On iOS/macOS, `false` keeps
+  /// muted-talker detection active but leaves the mic indicator on. When null, keeps default behavior.
   Future<Result<None>> setMicrophoneEnabled({
     required bool enabled,
     AudioConstraints? constraints,
+    bool? stopTrackOnMute,
   }) async {
     if (enabled &&
         state.value.isVideoModerated &&
@@ -3674,6 +3684,7 @@ class Call {
         await _session?.setMicrophoneEnabled(
           enabled,
           constraints: constraints,
+          stopTrackOnMute: stopTrackOnMute,
         ) ??
         Result.error('Session is null');
 

packages/stream_video/lib/src/call/session/call_session.dart

@@ -1291,6 +1291,7 @@ class CallSession extends Disposable {
   Future<Result<RtcLocalTrack>> setMicrophoneEnabled(
     bool enabled, {
     AudioConstraints? constraints,
+    bool? stopTrackOnMute,
   }) async {
     final rtcManager = this.rtcManager;
     if (rtcManager == null) {
@@ -1301,6 +1302,7 @@ class CallSession extends Disposable {
       return rtcManager.setMicrophoneEnabled(
         enabled: enabled,
         constraints: constraints,
+        stopTrackOnMute: stopTrackOnMute,
       );
     });