[WIP] feat: Try to enable software APM(aec,ns,agc) for rust-core.

.gitignore

@@ -2,4 +2,4 @@ target
 /.idea
 soxr-sys/test-input.wav
 soxr-sys/test-output.wav
-.DS_Store
+.DS_Store

examples/wgpu_room/src/app.rs

@@ -5,7 +5,9 @@ use crate::{
 };
 use egui::{Rounding, Stroke};
 use livekit::{e2ee::EncryptionType, prelude::*, SimulateScenario};
+use livekit::webrtc::{audio_stream::native::NativeAudioStream};
 use std::collections::HashMap;
+use futures::StreamExt;
 
 /// The state of the application are saved on app exit and restored on app start.
 #[derive(serde::Deserialize, serde::Serialize)]
@@ -88,8 +90,15 @@ impl LkApp {
                             );
                             self.video_renderers
                                 .insert((participant.identity(), track.sid()), video_renderer);
-                        } else if let RemoteTrack::Audio(_) = track {
-                            // TODO(theomonnom): Once we support media devices, we can play audio tracks here
+                        } else if let RemoteTrack::Audio(ref audio_track) = track {
+                            let rtc_track = audio_track.rtc_track();
+                            let mut audio_stream = NativeAudioStream::new(rtc_track, 48000, 2);
+                            // Receive the audio frames in a new task
+                            self.async_runtime.spawn(async move {
+                                while let Some(frame) = audio_stream.next().await {
+                                    println!("Received audio frame {:?}", frame);
+                                }
+                            });
                         }
                     }
                     RoomEvent::TrackUnsubscribed {

webrtc-sys/build.rs

@@ -54,6 +54,7 @@ fn main() {
         "src/peer_connection_factory.cpp",
         "src/media_stream.cpp",
         "src/media_stream_track.cpp",
+        "src/audio_context.cpp",
         "src/audio_track.cpp",
         "src/video_track.cpp",
         "src/data_channel.cpp",

webrtc-sys/include/livekit/audio_context.h

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2023 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the “License”);
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an “AS IS” BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <memory>
+
+#include "api/scoped_refptr.h"
+#include "livekit/audio_device.h"
+#include "livekit/webrtc.h"
+#include "modules/audio_mixer/audio_mixer_impl.h"
+
+namespace webrtc {
+class TaskQueueFactory;
+class AudioDeviceBuffer;
+class AudioTransport;
+}  // namespace webrtc
+
+namespace livekit {
+
+class AudioContext {
+ public:
+  AudioContext(std::shared_ptr<RtcRuntime> rtc_runtime);
+  virtual ~AudioContext();
+
+  rtc::scoped_refptr<AudioDevice> audio_device(
+      webrtc::TaskQueueFactory* task_queue_factory);
+
+  rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer();
+
+  webrtc::AudioDeviceBuffer* audio_device_buffer();
+
+  webrtc::AudioTransport* audio_transport();
+
+ private:
+  std::shared_ptr<RtcRuntime> rtc_runtime_;
+  rtc::scoped_refptr<livekit::AudioDevice> audio_device_;
+  rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer_;
+};
+
+}  // namespace livekit

webrtc-sys/include/livekit/audio_device.h

@@ -20,6 +20,7 @@
 
 #include "api/task_queue/task_queue_factory.h"
 #include "modules/audio_device/include/audio_device.h"
+#include "modules/audio_device/audio_device_buffer.h"
 #include "rtc_base/synchronization/mutex.h"
 #include "rtc_base/task_queue.h"
 #include "rtc_base/task_utils/repeating_task.h"
@@ -116,13 +117,22 @@ class AudioDevice : public webrtc::AudioDeviceModule {
 
   int32_t SetAudioDeviceSink(webrtc::AudioDeviceSink* sink) const override;
 
+  webrtc::AudioDeviceBuffer *audio_device_buffer() {
+    return &audio_device_buffer_;
+  }
+
+  webrtc::AudioTransport* audio_transport() {
+    return audio_transport_;
+  }
+
  private:
   mutable webrtc::Mutex mutex_;
   std::vector<int16_t> data_;
   std::unique_ptr<rtc::TaskQueue> audio_queue_;
   webrtc::RepeatingTaskHandle audio_task_;
   webrtc::AudioTransport* audio_transport_;
   webrtc::TaskQueueFactory* task_queue_factory_;
+  webrtc::AudioDeviceBuffer audio_device_buffer_;
   bool playing_{false};
   bool initialized_{false};
 };

webrtc-sys/include/livekit/peer_connection_factory.h

@@ -20,6 +20,7 @@
 #include "api/scoped_refptr.h"
 #include "api/task_queue/task_queue_factory.h"
 #include "livekit/audio_device.h"
+#include "livekit/audio_context.h"
 #include "media_stream.h"
 #include "rtp_parameters.h"
 #include "rust/cxx.h"
@@ -64,7 +65,7 @@ class PeerConnectionFactory {
 
  private:
   std::shared_ptr<RtcRuntime> rtc_runtime_;
-  rtc::scoped_refptr<AudioDevice> audio_device_;
+  AudioContext audio_context_;
   rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> peer_factory_;
   webrtc::TaskQueueFactory* task_queue_factory_;
 };

webrtc-sys/src/audio_context.cpp

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2023 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the “License”);
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an “AS IS” BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "livekit/audio_context.h"
+
+#include "modules/audio_mixer/audio_mixer_impl.h"
+#include "rtc_base/thread.h"
+
+namespace livekit {
+
+AudioContext::AudioContext(std::shared_ptr<RtcRuntime> rtc_runtime)
+    : rtc_runtime_(rtc_runtime) {}
+
+AudioContext::~AudioContext() {
+  if (audio_device_) {
+    rtc_runtime_->worker_thread()->BlockingCall(
+        [&] { audio_device_ = nullptr; });
+  }
+  if (audio_mixer_) {
+    rtc_runtime_->worker_thread()->BlockingCall(
+        [&] { audio_mixer_ = nullptr; });
+  }
+}
+
+rtc::scoped_refptr<AudioDevice> AudioContext::audio_device(
+    webrtc::TaskQueueFactory* task_queue_factory) {
+  if (!audio_device_) {
+    audio_device_ = rtc_runtime_->worker_thread()->BlockingCall([&] {
+      return rtc::make_ref_counted<livekit::AudioDevice>(task_queue_factory);
+    });
+  }
+
+  return audio_device_;
+}
+
+rtc::scoped_refptr<webrtc::AudioMixer> AudioContext::audio_mixer() {
+  if (!audio_mixer_) {
+    audio_mixer_ = rtc_runtime_->worker_thread()->BlockingCall(
+        [&] { return webrtc::AudioMixerImpl::Create(); });
+  }
+  return audio_mixer_;
+}
+
+webrtc::AudioDeviceBuffer* AudioContext::audio_device_buffer() {
+  if (audio_device_) {
+    return audio_device_->audio_device_buffer();
+  }
+  return nullptr;
+}
+
+webrtc::AudioTransport* AudioContext::audio_transport() {
+  if (audio_device_) {
+    return audio_device_->audio_transport();
+  }
+  return nullptr;
+}
+
+}  // namespace livekit

webrtc-sys/src/audio_device.cpp

@@ -25,7 +25,8 @@ namespace livekit {
 
 AudioDevice::AudioDevice(webrtc::TaskQueueFactory* task_queue_factory)
     : task_queue_factory_(task_queue_factory),
-      data_(kSamplesPer10Ms * kChannels) {}
+      data_(kSamplesPer10Ms * kChannels),
+      audio_device_buffer_(task_queue_factory) {}
 
 AudioDevice::~AudioDevice() {
   Terminate();
@@ -39,6 +40,7 @@ int32_t AudioDevice::ActiveAudioLayer(AudioLayer* audioLayer) const {
 int32_t AudioDevice::RegisterAudioCallback(webrtc::AudioTransport* transport) {
   webrtc::MutexLock lock(&mutex_);
   audio_transport_ = transport;
+  audio_device_buffer_.RegisterAudioCallback(transport);
   return 0;
 }
 
@@ -47,6 +49,11 @@ int32_t AudioDevice::Init() {
   if (initialized_)
     return 0;
 
+  audio_device_buffer_.SetRecordingSampleRate(kSampleRate);
+  audio_device_buffer_.SetPlayoutSampleRate(kSampleRate);
+  audio_device_buffer_.SetRecordingChannels(kChannels);
+  audio_device_buffer_.SetPlayoutChannels(kChannels);
+
   audio_queue_ =
       std::make_unique<rtc::TaskQueue>(task_queue_factory_->CreateTaskQueue(
           "AudioDevice", webrtc::TaskQueueFactory::Priority::NORMAL));
@@ -63,9 +70,13 @@ int32_t AudioDevice::Init() {
 
           // Request the AudioData, otherwise WebRTC will ignore the packets.
           // 10ms of audio data.
+          audio_device_buffer_.RequestPlayoutData(kSamplesPer10Ms);
+          audio_device_buffer_.GetPlayoutData(data);
+          /*
           audio_transport_->NeedMorePlayData(
               kSamplesPer10Ms, kBytesPerSample, kChannels, kSampleRate, data,
               n_samples_out, &elapsed_time_ms, &ntp_time_ms);
+          */
         }
 
         return webrtc::TimeDelta::Millis(10);
@@ -156,12 +167,14 @@ bool AudioDevice::RecordingIsInitialized() const {
 int32_t AudioDevice::StartPlayout() {
   webrtc::MutexLock lock(&mutex_);
   playing_ = true;
+  audio_device_buffer_.StartPlayout();
   return 0;
 }
 
 int32_t AudioDevice::StopPlayout() {
   webrtc::MutexLock lock(&mutex_);
   playing_ = false;
+  audio_device_buffer_.StopPlayout();
   return 0;
 }
 
@@ -171,10 +184,12 @@ bool AudioDevice::Playing() const {
 }
 
 int32_t AudioDevice::StartRecording() {
+  audio_device_buffer_.StartRecording();
   return 0;
 }
 
 int32_t AudioDevice::StopRecording() {
+  audio_device_buffer_.StopRecording();
   return 0;
 }
 

webrtc-sys/src/peer_connection_factory.cpp

@@ -21,6 +21,8 @@
 
 #include "api/audio_codecs/builtin_audio_decoder_factory.h"
 #include "api/audio_codecs/builtin_audio_encoder_factory.h"
+#include "api/audio/echo_canceller3_factory.h"
+#include "api/audio/echo_canceller3_config.h"
 #include "api/peer_connection_interface.h"
 #include "api/rtc_error.h"
 #include "api/rtc_event_log/rtc_event_log_factory.h"
@@ -46,7 +48,8 @@ class PeerConnectionObserver;
 
 PeerConnectionFactory::PeerConnectionFactory(
     std::shared_ptr<RtcRuntime> rtc_runtime)
-    : rtc_runtime_(rtc_runtime) {
+    : rtc_runtime_(rtc_runtime), 
+    audio_context_(rtc_runtime) {
   RTC_LOG(LS_VERBOSE) << "PeerConnectionFactory::PeerConnectionFactory()";
 
   webrtc::PeerConnectionFactoryDependencies dependencies;
@@ -63,20 +66,24 @@ PeerConnectionFactory::PeerConnectionFactory(
   cricket::MediaEngineDependencies media_deps;
   media_deps.task_queue_factory = dependencies.task_queue_factory.get();
 
-  audio_device_ = rtc_runtime_->worker_thread()->BlockingCall([&] {
-    return rtc::make_ref_counted<livekit::AudioDevice>(
-        media_deps.task_queue_factory);
-  });
-
-  media_deps.adm = audio_device_;
+  media_deps.adm = audio_context_.audio_device(media_deps.task_queue_factory);
 
   media_deps.video_encoder_factory =
       std::move(std::make_unique<livekit::VideoEncoderFactory>());
   media_deps.video_decoder_factory =
       std::move(std::make_unique<livekit::VideoDecoderFactory>());
   media_deps.audio_encoder_factory = webrtc::CreateBuiltinAudioEncoderFactory();
   media_deps.audio_decoder_factory = webrtc::CreateBuiltinAudioDecoderFactory();
-  media_deps.audio_processing = webrtc::AudioProcessingBuilder().Create();
+
+  auto apm = webrtc::AudioProcessingBuilder();
+  auto cfg = webrtc::EchoCanceller3Config();
+  auto echo_control = std::make_unique<webrtc::EchoCanceller3Factory>(cfg);
+
+  apm.SetEchoControlFactory(std::move(echo_control));
+  media_deps.audio_processing = apm.Create();
+
+  media_deps.audio_mixer = audio_context_.audio_mixer();
+
   media_deps.trials = dependencies.trials.get();
 
   dependencies.media_engine = cricket::CreateMediaEngine(std::move(media_deps));
@@ -97,8 +104,6 @@ PeerConnectionFactory::~PeerConnectionFactory() {
   RTC_LOG(LS_VERBOSE) << "PeerConnectionFactory::~PeerConnectionFactory()";
 
   peer_factory_ = nullptr;
-  rtc_runtime_->worker_thread()->BlockingCall(
-      [this] { audio_device_ = nullptr; });
 }
 
 std::shared_ptr<PeerConnection> PeerConnectionFactory::create_peer_connection(