[media] Move StarboardRenderer to MojoRenderer

Move StarboardRenderer to Gpu thread, and each of StarboardRenderer is ran on PooledSingleThreadTaskRunner. This allows Starboard to access Chrome_InProcGpuThread for graphical tasks.

b/394368542

Author: borongc

cobalt/build/configs/common.gn

@@ -22,3 +22,11 @@ enable_platform_ac3_eac3_audio = true
 # We use Angle+EGL based graphics pipeline, not Dawn path
 # see //ui/gl/features.gni
 use_dawn = false
+
+# Use MojoRenderer for StarboardRenderer.
+# Use MojoCdm for StarboardCdm.
+mojo_media_host = "gpu"
+mojo_media_services = [
+  "renderer",
+  "cdm",
+]

cobalt/renderer/BUILD.gn

@@ -31,6 +31,7 @@ source_set("renderer") {
     "//content/test:content_test_mojo_bindings",
     "//media",
     "//media/mojo:buildflags",
+    "//media/mojo/clients",
     "//starboard:starboard_headers_only",
     "//v8",
   ]

cobalt/renderer/cobalt_content_renderer_client.cc

@@ -13,12 +13,10 @@
 #include "components/js_injection/renderer/js_communication.h"
 #include "content/public/renderer/render_frame.h"
 #include "content/public/renderer/render_thread.h"
-#include "media/base/decoder_factory.h"
 #include "media/base/media_log.h"
 #include "media/base/renderer_factory.h"
+#include "media/mojo/clients/starboard/starboard_renderer_client_factory.h"
 #include "media/starboard/bind_host_receiver_callback.h"
-#include "media/starboard/starboard_renderer_factory.h"
-#include "media/video/gpu_video_accelerator_factories.h"
 #include "mojo/public/cpp/bindings/generic_pending_receiver.h"
 #include "starboard/media.h"
 #include "starboard/player.h"
@@ -180,26 +178,19 @@ void CobaltContentRendererClient::BindHostReceiver(
   BindHostReceiverWithValuation(std::move(receiver));
 }
 
-std::unique_ptr<::media::RendererFactory>
-CobaltContentRendererClient::GetBaseRendererFactory(
-    content::RenderFrame* /* render_frame */,
-    ::media::MediaLog* media_log,
-    ::media::DecoderFactory* /* decoder_factory */,
-    base::RepeatingCallback<::media::GpuVideoAcceleratorFactories*()>
-    /* get_gpu_factories_cb */) {
+void CobaltContentRendererClient::GetStarboardRendererFactoryTraits(
+    media::RendererFactoryTraits* renderer_factory_traits) {
   DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
-  // TODO(b/394368542): Add Content API to create StarboardRenderer.
-  base::TimeDelta audio_write_duration_local =
+  // TODO(b/383327725) - Cobalt: Inject these values from the web app.
+  renderer_factory_traits->audio_write_duration_local =
       base::Microseconds(kSbPlayerWriteDurationLocal);
-  base::TimeDelta audio_write_duration_remote =
+  renderer_factory_traits->audio_write_duration_remote =
       base::Microseconds(kSbPlayerWriteDurationRemote);
-  return std::make_unique<::media::StarboardRendererFactory>(
-      media_log,
-      // TODO(b/383327725) - Cobalt: Inject these values from the web app.
-      audio_write_duration_local, audio_write_duration_remote,
+  // TODO(b/405424096) - Cobalt: Move VideoGeometrySetterService to Gpu thread.
+  renderer_factory_traits->bind_host_receiver_callback =
       base::BindPostTaskToCurrentDefault(
           base::BindRepeating(&CobaltContentRendererClient::BindHostReceiver,
-                              weak_factory_.GetWeakPtr())));
+                              weak_factory_.GetWeakPtr()));
 }
 
 }  // namespace cobalt

cobalt/renderer/cobalt_content_renderer_client.h

@@ -11,16 +11,14 @@
 #include "base/threading/thread_checker.h"
 #include "cobalt/media/audio/cobalt_audio_device_factory.h"
 #include "content/public/renderer/content_renderer_client.h"
+#include "media/base/starboard/renderer_factory_traits.h"
 
 namespace content {
 class RenderFrame;
-class RenderThread;
 }  // namespace content
 
 namespace media {
 class MediaLog;
-class DecoderFactory;
-class GpuVideoAcceleratorFactories;
 class RendererFactory;
 }  // namespace media
 
@@ -48,13 +46,8 @@ class CobaltContentRendererClient : public content::ContentRendererClient {
   bool IsSupportedVideoType(const ::media::VideoType& type) override;
   // JS Injection hook
   void RunScriptsAtDocumentStart(content::RenderFrame* render_frame) override;
-  // TODO(b/394368542): Add Content API to create StarboardRenderer.
-  std::unique_ptr<::media::RendererFactory> GetBaseRendererFactory(
-      content::RenderFrame* render_frame,
-      ::media::MediaLog* media_log,
-      ::media::DecoderFactory* decoder_factory,
-      base::RepeatingCallback<::media::GpuVideoAcceleratorFactories*()>
-          get_gpu_factories_cb) override;
+  void GetStarboardRendererFactoryTraits(
+      media::RendererFactoryTraits* traits) override;
 
   // Bind Host Receiver to VideoGeometryChangeSubscriber on Browser thread.
   // This is called from StarboardRenderer with |BindPostTaskToCurrentDefault|

content/browser/media/frameless_media_interface_proxy.cc

@@ -137,6 +137,19 @@ void FramelessMediaInterfaceProxy::CreateMediaFoundationRenderer(
         client_extension_remote) {}
 #endif  // BUILDFLAG(IS_WIN)
 
+#if BUILDFLAG(USE_STARBOARD_MEDIA)
+void FramelessMediaInterfaceProxy::CreateStarboardRenderer(
+    mojo::PendingRemote<media::mojom::MediaLog> media_log_remote,
+    const base::UnguessableToken& overlay_plane_id,
+    base::TimeDelta audio_write_duration_local,
+    base::TimeDelta video_write_duration_remote,
+    mojo::PendingReceiver<media::mojom::Renderer> receiver,
+    mojo::PendingReceiver<media::mojom::StarboardRendererExtension>
+        renderer_extension_receiver,
+    mojo::PendingRemote<media::mojom::StarboardRendererClientExtension>
+        client_extension_remote) {}
+#endif  // BUILDFLAG(USE_STARBOARD_MEDIA)
+
 void FramelessMediaInterfaceProxy::CreateCdm(const media::CdmConfig& cdm_config,
                                              CreateCdmCallback callback) {
   std::move(callback).Run(mojo::NullRemote(), nullptr, "CDM not supported");

content/browser/media/frameless_media_interface_proxy.h

@@ -86,6 +86,18 @@ class FramelessMediaInterfaceProxy final
       mojo::PendingRemote<media::mojom::MediaFoundationRendererClientExtension>
           client_extension_remote) final;
 #endif  // BUILDFLAG(IS_WIN)
+#if BUILDFLAG(USE_STARBOARD_MEDIA)
+  void CreateStarboardRenderer(
+      mojo::PendingRemote<media::mojom::MediaLog> media_log_remote,
+      const base::UnguessableToken& overlay_plane_id,
+      base::TimeDelta audio_write_duration_local,
+      base::TimeDelta video_write_duration_remote,
+      mojo::PendingReceiver<media::mojom::Renderer> receiver,
+      mojo::PendingReceiver<media::mojom::StarboardRendererExtension>
+        renderer_extension_receiver,
+      mojo::PendingRemote<media::mojom::StarboardRendererClientExtension>
+          client_extension_remote) final;
+#endif  // BUILDFLAG(USE_STARBOARD_MEDIA)
   void CreateCdm(const media::CdmConfig& cdm_config,
                  CreateCdmCallback callback) final;
 

content/browser/media/media_interface_proxy.cc

@@ -391,6 +391,31 @@ void MediaInterfaceProxy::CreateMediaFoundationRenderer(
 }
 #endif  // BUILDFLAG(IS_WIN)
 
+#if BUILDFLAG(USE_STARBOARD_MEDIA)
+void MediaInterfaceProxy::CreateStarboardRenderer(
+    mojo::PendingRemote<media::mojom::MediaLog> media_log_remote,
+    const base::UnguessableToken& overlay_plane_id,
+    base::TimeDelta audio_write_duration_local,
+    base::TimeDelta video_write_duration_remote,
+    mojo::PendingReceiver<media::mojom::Renderer> receiver,
+    mojo::PendingReceiver<media::mojom::StarboardRendererExtension>
+        renderer_extension_receiver,
+    mojo::PendingRemote<media::mojom::StarboardRendererClientExtension>
+        client_extension_remote) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DVLOG(1) << __func__ << ": this=" << this;
+
+  InterfaceFactory* factory = media_interface_factory_ptr_->Get();
+  if (factory) {
+    factory->CreateStarboardRenderer(
+        std::move(media_log_remote), overlay_plane_id,
+        audio_write_duration_local, video_write_duration_remote,
+        std::move(receiver), std::move(renderer_extension_receiver),
+        std::move(client_extension_remote));
+  }
+}
+#endif  // BUILDFLAG(USE_STARBOARD_MEDIA)
+
 void MediaInterfaceProxy::CreateCdm(const media::CdmConfig& cdm_config,
                                     CreateCdmCallback create_cdm_cb) {
   DCHECK(thread_checker_.CalledOnValidThread());

content/browser/media/media_interface_proxy.h

@@ -90,6 +90,18 @@ class MediaInterfaceProxy final : public DocumentUserData<MediaInterfaceProxy>,
       mojo::PendingRemote<media::mojom::MediaFoundationRendererClientExtension>
           client_extension_remote) final;
 #endif  // BUILDFLAG(IS_WIN)
+#if BUILDFLAG(USE_STARBOARD_MEDIA)
+  void CreateStarboardRenderer(
+      mojo::PendingRemote<media::mojom::MediaLog> media_log_remote,
+      const base::UnguessableToken& overlay_plane_id,
+      base::TimeDelta audio_write_duration_local,
+      base::TimeDelta video_write_duration_remote,
+      mojo::PendingReceiver<media::mojom::Renderer> receiver,
+      mojo::PendingReceiver<media::mojom::StarboardRendererExtension>
+        renderer_extension_receiver,
+      mojo::PendingRemote<media::mojom::StarboardRendererClientExtension>
+          client_extension_remote) final;
+#endif  // BUILDFLAG(USE_STARBOARD_MEDIA)
   void CreateCdm(const media::CdmConfig& cdm_config,
                  CreateCdmCallback create_cdm_cb) final;
 

content/public/renderer/BUILD.gn

@@ -6,6 +6,10 @@ import("//build/config/features.gni")
 import("//media/media_options.gni")
 import("//ppapi/buildflags/buildflags.gni")
 
+if (is_cobalt) {
+  import("//starboard/build/buildflags.gni")
+}
+
 # See //content/BUILD.gn for how this works.
 group("renderer") {
   if (is_component_build) {

content/public/renderer/content_renderer_client.cc

@@ -274,4 +274,8 @@ ContentRendererClient::CreateCastStreamingResourceProvider() {
 }
 #endif
 
+#if BUILDFLAG(USE_STARBOARD_MEDIA)
+void ContentRendererClient::GetStarboardRendererFactoryTraits(
+  media::RendererFactoryTraits* renderer_factory_traits) {}
+#endif // BUILDFLAG(USE_STARBOARD_MEDIA)
 }  // namespace content

content/public/renderer/content_renderer_client.h

@@ -34,6 +34,10 @@
 #include "ui/base/page_transition_types.h"
 #include "v8/include/v8-forward.h"
 
+#if BUILDFLAG(USE_STARBOARD_MEDIA)
+#include "media/base/starboard/renderer_factory_traits.h"
+#endif // BUILDFLAG(USE_STARBOARD_MEDIA)
+
 #if !BUILDFLAG(IS_ANDROID)
 #include "media/base/speech_recognition_client.h"
 #endif
@@ -429,6 +433,13 @@ class CONTENT_EXPORT ContentRendererClient {
   virtual std::unique_ptr<cast_streaming::ResourceProvider>
   CreateCastStreamingResourceProvider();
 #endif
+
+#if BUILDFLAG(USE_STARBOARD_MEDIA)
+  // This is used for StarboardRenderer to acquire customizations
+  // as MojoRenderer from cobalt_content_renderer_client.cc.
+  virtual void GetStarboardRendererFactoryTraits(
+      media::RendererFactoryTraits* renderer_factory_traits);
+#endif // BUILDFLAG(USE_STARBOARD_MEDIA)
 };
 
 }  // namespace content

content/renderer/media/media_factory.cc

@@ -92,8 +92,6 @@
 #include "media/cdm/fuchsia/fuchsia_cdm_factory.h"
 #include "media/fuchsia/video/fuchsia_decoder_factory.h"
 #include "media/mojo/clients/mojo_fuchsia_cdm_provider.h"
-#elif BUILDFLAG(USE_STARBOARD_MEDIA)
-#include "media/starboard/starboard_cdm_factory.h"
 #elif BUILDFLAG(ENABLE_MOJO_CDM)
 #include "media/mojo/clients/mojo_cdm_factory.h"  // nogncheck
 #else
@@ -136,6 +134,11 @@
 #include "media/mojo/mojom/speech_recognition_service.mojom.h"
 #endif  // BUILDFLAG(IS_WIN)
 
+#if BUILDFLAG(USE_STARBOARD_MEDIA)
+#include "media/base/starboard/renderer_factory_traits.h"
+#include "media/mojo/clients/starboard/starboard_renderer_client_factory.h"
+#endif // BUILDFLAG(USE_STARBOARD_MEDIA)
+
 namespace {
 
 // This limit is much higher than it needs to be right now, because the logic
@@ -570,15 +573,8 @@ MediaFactory::CreateRendererFactorySelector(
                           base::Unretained(render_thread)));
   if (factory) {
     is_base_renderer_factory_set = true;
-#if BUILDFLAG(USE_STARBOARD_MEDIA)
-    // TODO(b/326827007): Revisit renderer to support secondary videos.
-    // TODO(b/394368542): Add Content API to create StarboardRenderer.
-    factory_selector->AddBaseFactory(RendererType::kStarboard,
-                                     std::move(factory));
-#else // BUILDFLAG(USE_STARBOARD_MEDIA)
     factory_selector->AddBaseFactory(RendererType::kContentEmbedderDefined,
                                      std::move(factory));
-#endif // BUILDFLAG(USE_STARBOARD_MEDIA)
   }
 
 #if BUILDFLAG(IS_ANDROID)
@@ -619,6 +615,15 @@ MediaFactory::CreateRendererFactorySelector(
 #endif  // BUILDFLAG(IS_ANDROID)
 
 #if BUILDFLAG(ENABLE_MOJO_RENDERER)
+#if BUILDFLAG(USE_STARBOARD_MEDIA)
+  // TODO(b/326827007): Revisit renderer to support secondary videos.
+  media::RendererFactoryTraits renderer_factory_traits;
+  GetContentClient()->renderer()->GetStarboardRendererFactoryTraits(&renderer_factory_traits);
+  is_base_renderer_factory_set = true;
+  factory_selector->AddBaseFactory(RendererType::kStarboard,
+    std::make_unique<media::StarboardRendererClientFactory>(media_log,
+        CreateMojoRendererFactory(), &renderer_factory_traits));
+#endif // BUILDFLAG(USE_STARBOARD_MEDIA)
   if (!is_base_renderer_factory_set &&
       renderer_media_playback_options.is_mojo_renderer_enabled()) {
     is_base_renderer_factory_set = true;
@@ -864,8 +869,6 @@ media::CdmFactory* MediaFactory::GetCdmFactory() {
   DCHECK(interface_broker_);
   cdm_factory_ = std::make_unique<media::FuchsiaCdmFactory>(
       std::make_unique<media::MojoFuchsiaCdmProvider>(interface_broker_));
-#elif BUILDFLAG(USE_STARBOARD_MEDIA)
-  cdm_factory_ = std::make_unique<media::StarboardCdmFactory>();
 #elif BUILDFLAG(ENABLE_MOJO_CDM)
   cdm_factory_ =
       std::make_unique<media::MojoCdmFactory>(GetMediaInterfaceFactory());

content/renderer/media/media_interface_factory.cc

@@ -189,6 +189,39 @@ void MediaInterfaceFactory::CreateMediaFoundationRenderer(
 }
 #endif  // BUILDFLAG(IS_WIN)
 
+#if BUILDFLAG(USE_STARBOARD_MEDIA)
+void MediaInterfaceFactory::CreateStarboardRenderer(
+    mojo::PendingRemote<media::mojom::MediaLog> media_log_remote,
+    const base::UnguessableToken& overlay_plane_id,
+    base::TimeDelta audio_write_duration_local,
+    base::TimeDelta video_write_duration_remote,
+    mojo::PendingReceiver<media::mojom::Renderer> receiver,
+    mojo::PendingReceiver<media::mojom::StarboardRendererExtension>
+        renderer_extension_receiver,
+    mojo::PendingRemote<media::mojom::StarboardRendererClientExtension>
+          client_extension_remote) {
+  if (!task_runner_->BelongsToCurrentThread()) {
+    task_runner_->PostTask(
+        FROM_HERE,
+        base::BindOnce(&MediaInterfaceFactory::CreateStarboardRenderer,
+                       weak_this_, std::move(media_log_remote),
+                       overlay_plane_id, audio_write_duration_local,
+                       video_write_duration_remote,
+                       std::move(receiver),
+                       std::move(renderer_extension_receiver),
+                       std::move(client_extension_remote)));
+    return;
+  }
+
+  DVLOG(1) << __func__;
+  GetMediaInterfaceFactory()->CreateStarboardRenderer(
+      std::move(media_log_remote), overlay_plane_id,
+      audio_write_duration_local, video_write_duration_remote,
+      std::move(receiver), std::move(renderer_extension_receiver),
+      std::move(client_extension_remote));
+}
+#endif  // BUILDFLAG(USE_STARBOARD_MEDIA)
+
 void MediaInterfaceFactory::CreateCdm(const media::CdmConfig& cdm_config,
                                       CreateCdmCallback callback) {
   if (!task_runner_->BelongsToCurrentThread()) {

content/renderer/media/media_interface_factory.h

@@ -83,6 +83,18 @@ class MediaInterfaceFactory final : public media::mojom::InterfaceFactory {
       mojo::PendingRemote<media::mojom::MediaFoundationRendererClientExtension>
           client_extension_remote) final;
 #endif  // BUILDFLAG(IS_WIN)
+#if BUILDFLAG(USE_STARBOARD_MEDIA)
+  void CreateStarboardRenderer(
+      mojo::PendingRemote<media::mojom::MediaLog> media_log_remote,
+      const base::UnguessableToken& overlay_plane_id,
+      base::TimeDelta audio_write_duration_local,
+      base::TimeDelta video_write_duration_remote,
+      mojo::PendingReceiver<media::mojom::Renderer> receiver,
+      mojo::PendingReceiver<media::mojom::StarboardRendererExtension>
+        renderer_extension_receiver,
+      mojo::PendingRemote<media::mojom::StarboardRendererClientExtension>
+          client_extension_remote) final;
+#endif  // BUILDFLAG(USE_STARBOARD_MEDIA)
   void CreateCdm(const media::CdmConfig& cdm_config,
                  CreateCdmCallback callback) final;
 

media/base/BUILD.gn

@@ -463,7 +463,10 @@ source_set("base") {
   }
 
   if (is_cobalt && use_starboard_media) {
-    sources += [ "starboard/demuxer_memory_limit_starboard.cc" ]
+    sources += [
+        "starboard/demuxer_memory_limit_starboard.cc",
+        "starboard/renderer_factory_traits.h",
+    ]
   } else if (is_android) {
     sources += [ "demuxer_memory_limit_android.cc" ]
   } else if (is_castos) {