Add PiP support

Package.resolved

@@ -5,8 +5,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/millicast/millicast-sdk-swift-package",
       "state" : {
-        "revision" : "f5e0abd63564f591d4fef0ab82d9e0cdee858579",
-        "version" : "1.5.0"
+        "revision" : "001f8654ba31461ecf805ae8a15e4d92efa8064e",
+        "version" : "1.7.0"
       }
     }
   ],

Package.swift

@@ -19,7 +19,7 @@ let package = Package(
             targets: ["DolbyIORTSUIKit"])
     ],
     dependencies: [
-        .package(url: "https://github.com/millicast/millicast-sdk-swift-package", from: "1.5.1")
+        .package(url: "https://github.com/millicast/millicast-sdk-swift-package", exact: "1.7.0")
     ],
     targets: [
         // Targets are the basic building blocks of a package. A target can define a module or a test suite.

Sources/DolbyIORTSCore/Manager/SubscriptionManager.swift

@@ -22,7 +22,7 @@ protocol SubscriptionManagerDelegate: AnyObject {
 
     func onStopped()
 
-    func onLayers(_ mid: String, activeLayers: [MCLayerData], inactiveLayers: [MCLayerData])
+    func onLayers(_ mid: String, activeLayers: [MCLayerData], inactiveLayers: [String])
 
     func onConnected()
 
@@ -41,7 +41,7 @@ protocol SubscriptionManagerProtocol: AnyObject {
     var delegate: SubscriptionManagerDelegate? { get set }
 
     func connect(streamName: String, accountID: String, configuration: SubscriptionConfiguration) async -> Bool
-    func startSubscribe() async -> Bool
+    func startSubscribe(configuration: SubscriptionConfiguration) async -> Bool
     func stopSubscribe() async -> Bool
     func addRemoteTrack(_ sourceBuilder: StreamSourceBuilder)
     func projectVideo(for source: StreamSource, withQuality quality: VideoQuality)
@@ -93,7 +93,10 @@ final class SubscriptionManager: SubscriptionManagerProtocol {
 
             self.subscriber.setCredentials(credentials)
 
-            guard self.subscriber.connect() else {
+            let connectionOptions = MCConnectionOptions()
+            connectionOptions.autoReconnect = configuration.autoReconnect
+
+            guard self.subscriber.connect(with: connectionOptions) else {
                 Self.logger.error("💼 Subscriber has failed to connect")
                 return false
             }
@@ -104,7 +107,7 @@ final class SubscriptionManager: SubscriptionManagerProtocol {
         return await task.value
     }
 
-    func startSubscribe() async -> Bool {
+    func startSubscribe(configuration: SubscriptionConfiguration) async -> Bool {
         let task = Task { [weak self] () -> Bool in
             Self.logger.debug("💼 Start subscribe")
 
@@ -122,7 +125,16 @@ final class SubscriptionManager: SubscriptionManagerProtocol {
                 return false
             }
 
-            guard self.subscriber.subscribe() else {
+            let options = MCClientOptions()
+            options.videoJitterMinimumDelayMs = Int32(configuration.videoJitterMinimumDelayInMs)
+            options.statsDelayMs = Int32(configuration.statsDelayMs)
+            if let rtcEventLogOutputPath = configuration.rtcEventLogPath {
+                options.rtcEventLogOutputPath = rtcEventLogOutputPath
+            }
+            options.disableAudio = configuration.disableAudio
+            options.forcePlayoutDelay = configuration.noPlayoutDelay
+
+            guard self.subscriber.subscribe(with: options) else {
                 Self.logger.error("💼 Subscribe call has failed")
                 return false
             }
@@ -219,18 +231,6 @@ private extension SubscriptionManager {
 
     func makeSubscriber(with configuration: SubscriptionConfiguration) -> MCSubscriber? {
         let subscriber = MCSubscriber.create()
-
-        let options = MCClientOptions()
-        options.autoReconnect = configuration.autoReconnect
-        options.videoJitterMinimumDelayMs = Int32(configuration.videoJitterMinimumDelayInMs)
-        options.statsDelayMs = Int32(configuration.statsDelayMs)
-        if let rtcEventLogOutputPath = configuration.rtcEventLogPath {
-            options.rtcEventLogOutputPath = rtcEventLogOutputPath
-        }
-        options.disableAudio = configuration.disableAudio
-        options.forcePlayoutDelay = configuration.noPlayoutDelay
-
-        subscriber?.setOptions(options)
         subscriber?.enableStats(configuration.enableStats)
 
         return subscriber
@@ -294,7 +294,7 @@ extension SubscriptionManager: MCSubscriberListener {
         Self.logger.debug("💼 Delegate - onVad with mid \(mid), sourceId \(sourceId)")
     }
 
-    func onLayers(_ mid: String, activeLayers: [MCLayerData], inactiveLayers: [MCLayerData]) {
+    func onLayers(_ mid: String, activeLayers: [MCLayerData], inactiveLayers: [String]) {
         Self.logger.debug("💼 Delegate - onLayers for mid - \(mid) with activeLayers \(activeLayers), inactiveLayers \(inactiveLayers)")
         delegate?.onLayers(mid, activeLayers: activeLayers, inactiveLayers: inactiveLayers)
     }

Sources/DolbyIORTSCore/Model/StreamSourceViewRenderer.swift

@@ -12,16 +12,29 @@ public class StreamSourceViewRenderer: Identifiable {
         static let defaultVideoTileSize = CGSize(width: 533, height: 300)
     }
 
-    private let renderer: MCIosVideoRenderer
-
-    let videoTrack: MCVideoTrack
-
+    public let streamSource: StreamSource
+    public let videoTrack: MCVideoTrack
+    public let playbackView: MCSampleBufferVideoUIView
+    public let pipView: MCSampleBufferVideoUIView
     public let id = UUID()
 
+    private let renderer: MCIosVideoRenderer
+
     public init(_ streamSource: StreamSource) {
+        self.streamSource = streamSource
         let videoTrack = streamSource.videoTrack.track
         self.renderer = MCIosVideoRenderer()
         self.videoTrack = videoTrack
+
+        let playbackView = MCSampleBufferVideoUIView()
+        playbackView.scalingMode = .aspectFit
+        playbackView.attach(videoTrack: videoTrack, mirrored: false)
+        self.playbackView = playbackView
+
+        let pipView = MCSampleBufferVideoUIView()
+        pipView.scalingMode = .aspectFit
+        pipView.attach(videoTrack: videoTrack, mirrored: false)
+        self.pipView = pipView
 
         Task {
             await MainActor.run {
@@ -37,10 +50,6 @@ public class StreamSourceViewRenderer: Identifiable {
     public var frameHeight: CGFloat {
         hasValidDimensions ? CGFloat(renderer.getHeight()) : Constants.defaultVideoTileSize.height
     }
-
-    public var playbackView: UIView {
-        renderer.getView()
-    }
 }
 
 // MARK: Helper functions

Sources/DolbyIORTSCore/State/StateMachine.swift

@@ -146,28 +146,50 @@ final class StateMachine {
         }
     }
 
-    func onLayers(_ mid: String, activeLayers: [MCLayerData], inactiveLayers: [MCLayerData]) {
+    func onLayers(_ mid: String, activeLayers: [MCLayerData], inactiveLayers: [String]) {
         switch currentState {
         case let .subscribed(state):
             let streamTypes: [StreamSource.LowLevelVideoQuality]
-            let filteredActiveLayers = activeLayers.filter({ layer in
-                // For H.264 there are no temporal layers and the id is set to 255. For VP8 use the first temporal layer.
-                return layer.temporalLayerId == 0 || layer.temporalLayerId == 255
-            })
+            var layersForSelection: [MCLayerData] = []
+
+            // Simulcast active layers
+            let simulcastLayers = activeLayers.filter { !$0.encodingId.isEmpty }
+            if !simulcastLayers.isEmpty {
+                // Select the max (best) temporal layer Id from a specific encodingId
+                let dictionaryOfLayersMatchingEncodingId = Dictionary(grouping: simulcastLayers, by: { $0.encodingId })
+                dictionaryOfLayersMatchingEncodingId.forEach { (encodingId: String, layers: [MCLayerData]) in
+                    // Picking the layer matching the max temporal layer id - represents the layer with the best FPS
+                    if let layerWithBestFrameRate = layers.first { $0.temporalLayerId == $0.maxTemporalLayerId } ?? layers.last {
+                        layersForSelection.append(layerWithBestFrameRate)
+                    }
+                }
+                layersForSelection.sort(by: >)
+            }
+            // Using SVC layer selection logic
+            else {
+                let simulcastLayers = activeLayers.filter { $0.spatialLayerId != nil }
+                let dictionaryOfLayersMatchingSpatialLayerId = Dictionary(grouping: simulcastLayers, by: { $0.spatialLayerId! })
+                dictionaryOfLayersMatchingSpatialLayerId.forEach { (spatialLayerId: NSNumber, layers: [MCLayerData]) in
+                    // Picking the layer matching the max temporal layer id - represents the layer with the best FPS
+                    if let layerWithBestFrameRate = layers.first { $0.spatialLayerId == $0.maxSpatialLayerId } ?? layers.last {
+                        layersForSelection.append(layerWithBestFrameRate)
+                    }
+                }
+            }
 
-            switch filteredActiveLayers.count {
+            switch layersForSelection.count {
             case 2:
                 streamTypes = [
                     .auto,
-                    .high(layer: filteredActiveLayers[0]),
-                    .low(layer: filteredActiveLayers[1])
+                    .high(layer: layersForSelection[0]),
+                    .low(layer: layersForSelection[1])
                 ]
             case 3:
                 streamTypes = [
                     .auto,
-                    .high(layer: filteredActiveLayers[0]),
-                    .medium(layer: filteredActiveLayers[1]),
-                    .low(layer: filteredActiveLayers[2])
+                    .high(layer: layersForSelection[0]),
+                    .medium(layer: layersForSelection[1]),
+                    .low(layer: layersForSelection[2])
                 ]
             default:
                 streamTypes = [.auto]
@@ -214,3 +236,14 @@ final class StateMachine {
         currentState = .stopped
     }
 }
+
+extension MCLayerData: Comparable {
+    public static func < (lhs: MCLayerData, rhs: MCLayerData) -> Bool {
+        switch (lhs.encodingId.lowercased(), rhs.encodingId.lowercased()) {
+        case ("h", "m"), ("l", "m"), ("h", "s"), ("l", "s"), ("m", "s"):
+            return false
+        default:
+            return true
+        }
+    }
+}

Sources/DolbyIORTSCore/StreamOrchestrator.swift

@@ -29,6 +29,8 @@ public final actor StreamOrchestrator {
         .eraseToAnyPublisher()
     private var activeStreamDetail: StreamDetail?
     private let logHandler: MillicastLoggerHandler = .init()
+
+    private var subscriptionConfiguration: SubscriptionConfiguration = .init()
 
     private init() {
         self.init(
@@ -57,6 +59,7 @@ public final actor StreamOrchestrator {
     public func connect(streamName: String, accountID: String, configuration: SubscriptionConfiguration = .init()) async -> Bool {
         Self.logger.debug("👮‍♂️ Start subscribe")
         logHandler.setLogFilePath(filePath: configuration.sdkLogPath)
+        self.subscriptionConfiguration = configuration
 
         async let startConnectionStateUpdate: Void = stateMachine.startConnection(
             streamName: streamName,
@@ -220,7 +223,7 @@ private extension StreamOrchestrator {
 
     func startSubscribe() async -> Bool {
         stateMachine.startSubscribe()
-        return await subscriptionManager.startSubscribe()
+        return await subscriptionManager.startSubscribe(configuration: subscriptionConfiguration)
     }
 
     func stopAudio(for sourceId: String?) {
@@ -236,6 +239,7 @@ private extension StreamOrchestrator {
     func reset() {
         activeStreamDetail = nil
         logHandler.setLogFilePath(filePath: nil)
+        subscriptionConfiguration = .init()
     }
 }
 
@@ -324,10 +328,10 @@ extension StreamOrchestrator: SubscriptionManagerDelegate {
         }
     }
 
-    nonisolated func onLayers(_ mid_: String, activeLayers: [MCLayerData], inactiveLayers: [MCLayerData]) {
+    nonisolated func onLayers(_ mid: String, activeLayers: [MCLayerData], inactiveLayers: [String]) {
         Task { @StreamOrchestrator [weak self] in
             guard let self = self else { return }
-            self.stateMachine.onLayers(mid_, activeLayers: activeLayers, inactiveLayers: inactiveLayers)
+            self.stateMachine.onLayers(mid, activeLayers: activeLayers, inactiveLayers: inactiveLayers)
         }
     }
 

Sources/DolbyIORTSUIKit/Private/Managers/PiPManager.swift

@@ -0,0 +1,74 @@
+//
+//  PiPManager.swift
+//
+
+import AVFoundation
+import AVKit
+import Foundation
+import MillicastSDK
+import UIKit
+
+final class PiPManager: NSObject {
+    static let shared: PiPManager = PiPManager()
+
+    private override init() {}
+
+    private(set) var pipController: AVPictureInPictureController?
+    private(set) var pipView: MCSampleBufferVideoUIView?
+
+    var isPiPActive: Bool {
+        pipController?.isPictureInPictureActive ?? false
+    }
+
+    func set(pipView: MCSampleBufferVideoUIView, with targetView: UIView) {
+        pipController?.stopPictureInPicture()
+
+        guard AVPictureInPictureController.isPictureInPictureSupported() else {
+            return
+        }
+
+        let pipVideoCallViewController = AVPictureInPictureVideoCallViewController()
+        pipVideoCallViewController.view.addSubview(pipView)
+        pipView.translatesAutoresizingMaskIntoConstraints = false
+
+        NSLayoutConstraint.activate([
+            pipVideoCallViewController.view.topAnchor.constraint(equalTo: pipView.topAnchor),
+            pipVideoCallViewController.view.leadingAnchor.constraint(equalTo: pipView.leadingAnchor),
+            pipView.bottomAnchor.constraint(equalTo: pipVideoCallViewController.view.bottomAnchor),
+            pipView.trailingAnchor.constraint(equalTo: pipVideoCallViewController.view.trailingAnchor)
+        ])
+        pipVideoCallViewController.preferredContentSize = targetView.frame.size
+
+        let pipContentSource = AVPictureInPictureController.ContentSource(
+            activeVideoCallSourceView: targetView,
+            contentViewController: pipVideoCallViewController
+        )
+
+        let pipController = AVPictureInPictureController(contentSource: pipContentSource)
+        pipController.canStartPictureInPictureAutomaticallyFromInline = true
+        pipController.delegate = self
+
+        NotificationCenter.default
+            .addObserver(forName: UIApplication.didBecomeActiveNotification, object: nil, queue: .main) { [weak self] _ in
+                self?.stopPiP()
+            }
+
+        self.pipView = pipView
+        self.pipController = pipController
+    }
+
+    func stopPiP() {
+        pipController?.stopPictureInPicture()
+    }
+}
+
+extension PiPManager: AVPictureInPictureControllerDelegate {
+    func pictureInPictureControllerDidStartPictureInPicture(_ pictureInPictureController: AVPictureInPictureController) {
+    }
+
+    func pictureInPictureControllerDidStopPictureInPicture(_ pictureInPictureController: AVPictureInPictureController) {
+    }
+
+    func pictureInPictureController(_ pictureInPictureController: AVPictureInPictureController, failedToStartPictureInPictureWithError error: Error) {
+    }
+}