feat: Visualizer for web. (#718)

* feat: Visualizer for web. * wip. * done. * update. * dart run import_sorter:main --no-comments. * dart format. * export Visualizer and add VisualizerOptions. * support barCount configuration. * Fixed bug for visualizer restart, removed setting enableVisualizer from RoomOptions. * update. * cleanup. * rename. * fix. * fix. * fix.
livekit · Mar 5, 2025 · 6365ec6 · 6365ec6
1 parent a503e95
commit 6365ec6
Show file tree

Hide file tree

Showing 23 changed files with 472 additions and 116 deletions.
diff --git a/android/src/main/kotlin/io/livekit/plugin/LiveKitPlugin.kt b/android/src/main/kotlin/io/livekit/plugin/LiveKitPlugin.kt
@@ -50,8 +50,9 @@ class LiveKitPlugin: FlutterPlugin, MethodCallHandler {
   @SuppressLint("SuspiciousIndentation")
   private fun handleStartVisualizer(@NonNull call: MethodCall, @NonNull result: Result) {
     val trackId = call.argument<String>("trackId")
-    if (trackId == null) {
-      result.error("INVALID_ARGUMENT", "trackId is required", null)
+    val visualizerId = call.argument<String>("visualizerId")
+    if (trackId == null || visualizerId == null) {
+      result.error("INVALID_ARGUMENT", "trackId and visualizerId is required", null)
       return
     }
     var audioTrack: LKAudioTrack? = null
@@ -75,19 +76,21 @@ class LiveKitPlugin: FlutterPlugin, MethodCallHandler {
 
     val visualizer = Visualizer(
       barCount = barCount, isCentered = isCentered,
-      audioTrack = audioTrack, binaryMessenger = binaryMessenger!!)
+      audioTrack = audioTrack, binaryMessenger = binaryMessenger!!,
+      visualizerId = visualizerId)
 
-    processors[trackId] = visualizer
+    processors[visualizerId] = visualizer
     result.success(null)
   }
 
   private fun handleStopVisualizer(@NonNull call: MethodCall, @NonNull result: Result) {
     val trackId = call.argument<String>("trackId")
-    if (trackId == null) {
-      result.error("INVALID_ARGUMENT", "trackId is required", null)
+    val visualizerId = call.argument<String>("visualizerId")
+    if (trackId == null || visualizerId == null) {
+      result.error("INVALID_ARGUMENT", "trackId and visualizerId is required", null)
       return
     }
-    processors.entries.removeAll { (k, v) -> k == trackId }
+    processors.entries.removeAll { (k, v) -> k == visualizerId }
     result.success(null)
   }
 

diff --git a/android/src/main/kotlin/io/livekit/plugin/Visualizer.kt b/android/src/main/kotlin/io/livekit/plugin/Visualizer.kt
@@ -29,16 +29,18 @@ class Visualizer(
     private var barCount: Int,
     private var isCentered: Boolean,
     audioTrack: LKAudioTrack,
-    binaryMessenger: BinaryMessenger
+    binaryMessenger: BinaryMessenger,
+    visualizerId: String
 ) : EventChannel.StreamHandler, AudioTrackSink {
     private var eventChannel: EventChannel? = null
     private var eventSink: EventChannel.EventSink? = null
     private var ffiAudioAnalyzer = FFTAudioAnalyzer()
     private var audioTrack: LKAudioTrack? = audioTrack
     private var amplitudes: FloatArray = FloatArray(0)
-    private var bands: FloatArray = FloatArray(0)
+    private var bands: FloatArray
     private var loPass: Int = 0
     private var hiPass: Int = 80
+
     private var audioFormat = AudioFormat(16, 48000, 1)
 
     fun stop() {
@@ -99,8 +101,9 @@ class Visualizer(
     }
 
     init {
-        eventChannel = EventChannel(binaryMessenger, "io.livekit.audio.visualizer/eventChannel-" + audioTrack.id())
+        eventChannel = EventChannel(binaryMessenger, "io.livekit.audio.visualizer/eventChannel-" + audioTrack.id() + "-" + visualizerId)
         eventChannel?.setStreamHandler(this)
+        bands = FloatArray(barCount)
         ffiAudioAnalyzer.configure(audioFormat)
         audioTrack.addSink(this)
     }

diff --git a/example/lib/pages/prejoin.dart b/example/lib/pages/prejoin.dart
@@ -131,7 +131,6 @@ class _PreJoinPageState extends State<PreJoinPage> {
         AudioCaptureOptions(
           deviceId: _selectedAudioDevice!.deviceId,
         ),
-        true, // enableVisualizer
       );
       await _audioTrack!.start();
     }
@@ -212,7 +211,6 @@ class _PreJoinPageState extends State<PreJoinPage> {
             screenShareEncoding: screenEncoding,
           ),
           e2eeOptions: e2eeOptions,
-          enableVisualizer: true,
         ),
       );
       // Create a Listener before connecting

diff --git a/example/lib/widgets/participant_stats.dart b/example/lib/widgets/participant_stats.dart
@@ -55,7 +55,9 @@ class _ParticipantStatsWidgetState extends State<ParticipantStatsWidget> {
       listener.on<VideoReceiverStatsEvent>((event) {
         Map<String, String> stats = {};
         setState(() {
-          stats['rx'] = '${event.currentBitrate.toInt()} kpbs';
+          if (!event.currentBitrate.isFinite && !event.currentBitrate.isNaN) {
+            stats['rx'] = '${event.currentBitrate.toInt()} kpbs';
+          }
           if (event.stats.mimeType != null) {
             stats['codec'] =
                 '${event.stats.mimeType!.split('/')[1]}/${event.stats.clockRate}';

diff --git a/example/lib/widgets/sound_waveform.dart b/example/lib/widgets/sound_waveform.dart
@@ -42,15 +42,15 @@ import 'package:flutter/material.dart';
 import 'package:livekit_client/livekit_client.dart';
 
 class SoundWaveformWidget extends StatefulWidget {
-  final int count;
+  final int barCount;
   final double width;
   final double minHeight;
   final double maxHeight;
   final int durationInMilliseconds;
   const SoundWaveformWidget({
     super.key,
     required this.audioTrack,
-    this.count = 7,
+    this.barCount = 5,
     this.width = 5,
     this.minHeight = 8,
     this.maxHeight = 100,
@@ -64,23 +64,32 @@ class SoundWaveformWidget extends StatefulWidget {
 class _SoundWaveformWidgetState extends State<SoundWaveformWidget>
     with TickerProviderStateMixin {
   late AnimationController controller;
-  List<double> samples = [0, 0, 0, 0, 0, 0, 0];
-  EventsListener<TrackEvent>? _listener;
+  late List<double> samples;
+  AudioVisualizer? _visualizer;
+  EventsListener<AudioVisualizerEvent>? _listener;
 
   void _startVisualizer(AudioTrack track) async {
-    await _listener?.dispose();
-    _listener = track.createListener();
+    samples = List.filled(widget.barCount, 0);
+    _visualizer ??= createVisualizer(track,
+        options: AudioVisualizerOptions(barCount: widget.barCount));
+    _listener ??= _visualizer?.createListener();
     _listener?.on<AudioVisualizerEvent>((e) {
       if (mounted) {
         setState(() {
           samples = e.event.map((e) => ((e as num) * 100).toDouble()).toList();
         });
       }
     });
+
+    await _visualizer!.start();
   }
 
   void _stopVisualizer(AudioTrack track) async {
+    await _visualizer?.stop();
+    await _visualizer?.dispose();
+    _visualizer = null;
     await _listener?.dispose();
+    _listener = null;
   }
 
   @override
@@ -106,7 +115,7 @@ class _SoundWaveformWidgetState extends State<SoundWaveformWidget>
 
   @override
   Widget build(BuildContext context) {
-    final count = widget.count;
+    final count = widget.barCount;
     final minHeight = widget.minHeight;
     final maxHeight = widget.maxHeight;
     return AnimatedBuilder(

diff --git a/lib/livekit_client.dart b/lib/livekit_client.dart
@@ -45,6 +45,7 @@ export 'src/track/remote/remote.dart';
 export 'src/track/remote/video.dart';
 export 'src/track/track.dart';
 export 'src/track/processor.dart';
+export 'src/track/audio_visualizer.dart';
 export 'src/types/other.dart';
 export 'src/types/participant_permissions.dart';
 export 'src/types/video_dimensions.dart';

diff --git a/lib/src/core/room.dart b/lib/src/core/room.dart
@@ -585,7 +585,6 @@ class Room extends DisposableChangeNotifier with EventsEmittable<RoomEvent> {
           trackSid,
           receiver: event.receiver,
           audioOutputOptions: roomOptions.defaultAudioOutputOptions,
-          enableVisualizer: roomOptions.enableVisualizer,
         );
       } on TrackSubscriptionExceptionEvent catch (event) {
         logger.severe('addSubscribedMediaTrack() throwed ${event}');

diff --git a/lib/src/options.dart b/lib/src/options.dart
@@ -115,13 +115,10 @@ class RoomOptions {
   /// Options for end-to-end encryption.
   final E2EEOptions? e2eeOptions;
 
-  /// audio visualizer is disabled by default
-  /// When enabled, the native layer will register an FFI audio analyzer
-  /// and will emit AudioVisualizerEvent events from AudioTrack.
-  /// You can use SoundWaveformWidget (example/lib/widgets/sound_waveform.dart)
-  /// to display the audio wave. Or write a custom widget to visualize the audio
-  /// wave.
-  final bool enableVisualizer;
+  /// deprecated, use [createVisualizer] instead
+  /// please refer to example/lib/widgets/sound_waveform.dart
+  @Deprecated('Use createVisualizer instead')
+  final bool? enableVisualizer;
 
   const RoomOptions({
     this.defaultCameraCaptureOptions = const CameraCaptureOptions(),
@@ -134,7 +131,7 @@ class RoomOptions {
     this.dynacast = false,
     this.stopLocalTrackOnUnpublish = true,
     this.e2eeOptions,
-    this.enableVisualizer = false,
+    this.enableVisualizer,
   });
 
   RoomOptions copyWith({

diff --git a/lib/src/participant/local.dart b/lib/src/participant/local.dart
@@ -637,8 +637,7 @@ class LocalParticipant extends Participant<LocalTrackPublication> {
       } else if (source == TrackSource.microphone) {
         AudioCaptureOptions captureOptions =
             audioCaptureOptions ?? room.roomOptions.defaultAudioCaptureOptions;
-        final track = await LocalAudioTrack.create(
-            captureOptions, room.roomOptions.enableVisualizer);
+        final track = await LocalAudioTrack.create(captureOptions);
         return await publishAudioTrack(track);
       } else if (source == TrackSource.screenShareVideo) {
         ScreenShareCaptureOptions captureOptions = screenShareCaptureOptions ??

diff --git a/lib/src/participant/remote.dart b/lib/src/participant/remote.dart
@@ -112,7 +112,6 @@ class RemoteParticipant extends Participant<RemoteTrackPublication> {
     String trackSid, {
     rtc.RTCRtpReceiver? receiver,
     AudioOutputOptions audioOutputOptions = const AudioOutputOptions(),
-    bool? enableVisualizer,
   }) async {
     logger.fine('addSubscribedMediaTrack()');
 
@@ -154,8 +153,8 @@ class RemoteParticipant extends Participant<RemoteTrackPublication> {
           RemoteVideoTrack(pub.source, stream, mediaTrack, receiver: receiver);
     } else if (pub.kind == TrackType.AUDIO) {
       // audio track
-      track = RemoteAudioTrack(pub.source, stream, mediaTrack,
-          receiver: receiver, enableVisualizer: enableVisualizer);
+      track =
+          RemoteAudioTrack(pub.source, stream, mediaTrack, receiver: receiver);
 
       var listener = track.createListener();
       listener.on<AudioPlaybackStarted>((event) {

diff --git a/lib/src/support/native.dart b/lib/src/support/native.dart
@@ -54,6 +54,7 @@ class Native {
     String trackId, {
     bool isCentered = true,
     int barCount = 7,
+    String visualizerId = '',
   }) async {
     try {
       final result = await channel.invokeMethod<bool>(
@@ -62,6 +63,7 @@ class Native {
           'trackId': trackId,
           'isCentered': isCentered,
           'barCount': barCount,
+          'visualizerId': visualizerId,
         },
       );
       return result == true;
@@ -72,12 +74,14 @@ class Native {
   }
 
   @internal
-  static Future<void> stopVisualizer(String trackId) async {
+  static Future<void> stopVisualizer(String trackId,
+      {required String visualizerId}) async {
     try {
       await channel.invokeMethod<void>(
         'stopVisualizer',
         <String, dynamic>{
           'trackId': trackId,
+          'visualizerId': visualizerId,
         },
       );
     } catch (error) {

diff --git a/lib/src/track/audio_visualizer.dart b/lib/src/track/audio_visualizer.dart
@@ -0,0 +1,26 @@
+import 'package:livekit_client/src/support/disposable.dart';
+import '../events.dart' show AudioVisualizerEvent;
+import '../managers/event.dart' show EventsEmittable;
+import 'local/local.dart' show AudioTrack;
+
+import 'audio_visualizer_native.dart'
+    if (dart.library.js_interop) 'audio_visualizer_web.dart';
+
+class AudioVisualizerOptions {
+  final bool centeredBands;
+  final int barCount;
+  const AudioVisualizerOptions({
+    this.centeredBands = true,
+    this.barCount = 7,
+  });
+}
+
+abstract class AudioVisualizer extends DisposableChangeNotifier
+    with EventsEmittable<AudioVisualizerEvent> {
+  Future<void> start();
+  Future<void> stop();
+}
+
+AudioVisualizer createVisualizer(AudioTrack track,
+        {AudioVisualizerOptions? options}) =>
+    createVisualizerImpl(track, options: options);
diff --git a/lib/src/track/audio_visualizer_native.dart b/lib/src/track/audio_visualizer_native.dart
@@ -0,0 +1,72 @@
+import 'dart:async';
+
+import 'package:flutter/services.dart';
+
+import 'package:flutter_webrtc/flutter_webrtc.dart';
+
+import 'package:livekit_client/src/events.dart' show AudioVisualizerEvent;
+import 'package:livekit_client/src/track/local/local.dart';
+import '../support/native.dart' show Native;
+import 'audio_visualizer.dart';
+
+class AudioVisualizerNative extends AudioVisualizer {
+  final String visualizerId = '${DateTime.now().millisecondsSinceEpoch}';
+  EventChannel? _eventChannel;
+  StreamSubscription? _streamSubscription;
+  final AudioTrack? _audioTrack;
+  MediaStreamTrack get mediaStreamTrack => _audioTrack!.mediaStreamTrack;
+  final AudioVisualizerOptions visualizerOptions;
+  AudioVisualizerNative(this._audioTrack, {required this.visualizerOptions}) {
+    onDispose(() async {
+      await events.dispose();
+    });
+  }
+
+  @override
+  Future<void> start() async {
+    if (_eventChannel != null) {
+      return;
+    }
+
+    await Native.startVisualizer(
+      mediaStreamTrack.id!,
+      isCentered: visualizerOptions.centeredBands,
+      barCount: visualizerOptions.barCount,
+      visualizerId: visualizerId,
+    );
+
+    _eventChannel = EventChannel(
+        'io.livekit.audio.visualizer/eventChannel-${mediaStreamTrack.id}-$visualizerId');
+    _streamSubscription =
+        _eventChannel?.receiveBroadcastStream().listen((event) {
+      events.emit(AudioVisualizerEvent(
+        track: _audioTrack!,
+        event: event,
+      ));
+    });
+  }
+
+  @override
+  Future<void> stop() async {
+    if (_eventChannel == null) {
+      return;
+    }
+
+    await Native.stopVisualizer(mediaStreamTrack.id!,
+        visualizerId: visualizerId);
+
+    events.emit(AudioVisualizerEvent(
+      track: _audioTrack!,
+      event: [],
+    ));
+
+    await _streamSubscription?.cancel();
+    _streamSubscription = null;
+    _eventChannel = null;
+  }
+}
+
+AudioVisualizer createVisualizerImpl(AudioTrack track,
+        {AudioVisualizerOptions? options}) =>
+    AudioVisualizerNative(track,
+        visualizerOptions: options ?? AudioVisualizerOptions());