Implementation of call feature streams Support

packages/calling-component-bindings/src/handlers/createCommonHandlers.ts

@@ -35,6 +35,8 @@ import { Features } from '@azure/communication-calling';
 import { TeamsCaptions } from '@azure/communication-calling';
 import { Reaction } from '@azure/communication-calling';
 import { _ComponentCallingHandlers } from './createHandlers';
+/* @conditional-compile-remove(together-mode) */
+import { TogetherModeStreamViewResult } from '@internal/react-components/dist/dist-esm/types/TogetherModeTypes';
 
 /**
  * Object containing all the handlers required for calling components.
@@ -106,6 +108,20 @@ export interface CommonCallingHandlers {
   onMuteParticipant: (userId: string) => Promise<void>;
   /* @conditional-compile-remove(soft-mute) */
   onMuteAllRemoteParticipants: () => Promise<void>;
+  /* @conditional-compile-remove(together-mode) */
+  /**
+   * Call back to create a view for together mode
+   *
+   * @beta
+   */
+  onCreateTogetherModeStreamView: (options?: VideoStreamOptions) => Promise<void | TogetherModeStreamViewResult>;
+  /* @conditional-compile-remove(together-mode) */
+  /**
+   * Call back to dispose together mode views
+   *
+   * @beta
+   */
+  onDisposeTogetherModeStreamViews: () => Promise<void>;
 }
 
 /**
@@ -715,6 +731,59 @@ export const createDefaultCommonCallingHandlers = memoizeOne(
         }
       : undefined;
 
+    /* @conditional-compile-remove(together-mode) */
+    const onCreateTogetherModeStreamView = async (
+      options = { scalingMode: 'Fit', isMirrored: true } as VideoStreamOptions
+    ): Promise<void | TogetherModeStreamViewResult> => {
+      if (!call) {
+        return;
+      }
+      const callState = callClient.getState().calls[call.id];
+      if (!callState) {
+        return;
+      }
+      const togetherModeStreams = callState.togetherMode.streams;
+      const togetherModeCreateViewResult: TogetherModeStreamViewResult = {};
+      if (!togetherModeStreams.size) {
+        const togetherModeFeature = call?.feature(Features.TogetherMode);
+        await togetherModeFeature?.start();
+      } else {
+        for (const stream of togetherModeStreams) {
+          if (!stream[1].view) {
+            const createViewResult = await callClient.createCallFeatureView(call.id, stream[1], options);
+            // SDK currently only supports 1 Video media stream type
+            if (stream[1].mediaStreamType === 'Video') {
+              togetherModeCreateViewResult.mainVideoView = createViewResult?.view
+                ? { view: createViewResult?.view }
+                : undefined;
+            }
+          }
+        }
+      }
+      return togetherModeCreateViewResult;
+    };
+    /* @conditional-compile-remove(together-mode) */
+    const onDisposeTogetherModeStreamViews = async (): Promise<void> => {
+      if (!call) {
+        return;
+      }
+      const callState = callClient.getState().calls[call.id];
+      if (!callState) {
+        throw new Error(`Call Not Found: ${call.id}`);
+      }
+
+      const togetherModeStreams = callState.togetherMode.streams;
+
+      if (!togetherModeStreams.size) {
+        return;
+      }
+
+      for (const stream of togetherModeStreams) {
+        if (stream[1].view) {
+          callClient.disposeCallFeatureView(call.id, stream[1]);
+        }
+      }
+    };
     return {
       onHangUp,
       onToggleHold,
@@ -768,7 +837,11 @@ export const createDefaultCommonCallingHandlers = memoizeOne(
       /* @conditional-compile-remove(soft-mute) */
       onMuteAllRemoteParticipants,
       onAcceptCall: notImplemented,
-      onRejectCall: notImplemented
+      onRejectCall: notImplemented,
+      /* @conditional-compile-remove(together-mode) */
+      onCreateTogetherModeStreamView,
+      /* @conditional-compile-remove(together-mode) */
+      onDisposeTogetherModeStreamViews
     };
   }
 );

packages/calling-stateful-client/src/CallClientState.ts

@@ -278,21 +278,39 @@ export interface RaiseHandCallFeatureState {
 /**
  * State only version of {@link @azure/communication-calling#TogetherModeCallFeature}. {@link StatefulCallClient} will
  * automatically listen for raised hands on the call and update the state exposed by {@link StatefulCallClient} accordingly.
- * @alpha
+ * @beta
  */
 export interface TogetherModeCallFeatureState {
   /**
    * Proxy of {@link @azure/communication-calling#TogetherModeCallFeature.togetherModeStream}.
    */
-  stream: TogetherModeStreamState[];
+  streams: Map<string, TogetherModeStreamState>;
+  /**
+   * Proxy of {@link @azure/communication-calling#TogetherModeCallFeature.TogetherModeSeatingMap}.
+   */
+  seatingCoordinates: Map<string, TogetherModeSeatingCoordinatesState>;
+}
+
+/* @conditional-compile-remove(together-mode) */
+/**
+ * @beta
+ */
+export type CallFeatureStreamName = 'togetherMode';
+
+/* @conditional-compile-remove(together-mode) */
+/**
+ * @beta
+ */
+export interface CallFeatureStreamState {
+  feature: CallFeatureStreamName;
 }
 
 /* @conditional-compile-remove(together-mode) */
 /**
  * State only version of {@link @azure/communication-calling#TogetherModeVideoStream}.
- * @alpha
+ * @beta
  */
-export interface TogetherModeStreamState {
+export interface TogetherModeStreamState extends CallFeatureStreamState {
   /**
    * Proxy of {@link @azure/communication-calling#TogetherModeVideoStream.id}.
    */
@@ -317,6 +335,22 @@ export interface TogetherModeStreamState {
   streamSize?: { width: number; height: number };
 }
 
+/* @conditional-compile-remove(together-mode) */
+/**
+ * State only version of {@link @azure/communication-calling#TogetherModeSeatingMap}.
+ * @beta
+ */
+export interface TogetherModeSeatingCoordinatesState {
+  // the y coordinate of the participant seating position in the together mode stream
+  top: number;
+  // the x coordinate of the participant seating position in the together mode stream
+  left: number;
+  // the width of the participant in the together mode stream
+  width: number;
+  // the height of the participant in the together mode stream
+  height: number;
+}
+
 /**
  * State only version of {@link @azure/communication-calling#PPTLiveCallFeature}. {@link StatefulCallClient} will
  * automatically listen for pptLive on the call and update the state exposed by {@link StatefulCallClient} accordingly.
@@ -625,6 +659,7 @@ export interface CallState {
   /* @conditional-compile-remove(together-mode) */
   /**
    * Proxy of {@link @azure/communication-calling#TogetherModeCallFeature}.
+   * @beta
    */
   togetherMode: TogetherModeCallFeatureState;
   /**

packages/calling-stateful-client/src/CallContext.ts

@@ -24,7 +24,7 @@ import { CaptionsKind, CaptionsInfo as AcsCaptionsInfo } from '@azure/communicat
 /* @conditional-compile-remove(unsupported-browser) */
 import { EnvironmentInfo } from '@azure/communication-calling';
 /* @conditional-compile-remove(together-mode) */
-import { TogetherModeVideoStream } from '@azure/communication-calling';
+import { TogetherModeVideoStream, TogetherModeSeatingMap } from '@azure/communication-calling';
 import { AzureLogger, createClientLogger, getLogLevel } from '@azure/logger';
 import { EventEmitter } from 'events';
 import { enableMapSet, enablePatches, Patch, produce } from 'immer';
@@ -47,6 +47,8 @@ import {
   CallErrorTarget,
   CallError
 } from './CallClientState';
+/* @conditional-compile-remove(together-mode) */
+import { TogetherModeStreamState, TogetherModeSeatingCoordinatesState } from './CallClientState';
 /* @conditional-compile-remove(breakout-rooms) */
 import { NotificationTarget, CallNotification, CallNotifications } from './CallClientState';
 import { TeamsIncomingCallState } from './CallClientState';
@@ -457,11 +459,25 @@ export class CallContext {
   }
 
   /* @conditional-compile-remove(together-mode) */
-  public setTogetherModeVideoStream(callId: string, addedStream: TogetherModeVideoStream[]): void {
+  public setTogetherModeVideoStream(callId: string, addedStreams: TogetherModeVideoStream[]): void {
     this.modifyState((draft: CallClientState) => {
       const call = draft.calls[this._callIdHistory.latestCallId(callId)];
       if (call) {
-        call.togetherMode = { stream: addedStream };
+        const streamsToAdd: Map<string, TogetherModeStreamState> = new Map();
+        for (const stream of addedStreams) {
+          const streamToAdd: TogetherModeStreamState = {
+            feature: 'togetherMode',
+            id: stream.id,
+            view: undefined,
+            mediaStreamType: stream.mediaStreamType,
+            isReceiving: stream.isReceiving
+          };
+          streamsToAdd.set(stream.mediaStreamType, streamToAdd);
+        }
+        call.togetherMode.streams = streamsToAdd;
+        if (!call.togetherMode.seatingCoordinates) {
+          call.togetherMode.seatingCoordinates = new Map<string, TogetherModeSeatingCoordinatesState>();
+        }
       }
     });
   }
@@ -472,15 +488,24 @@ export class CallContext {
       const call = draft.calls[this._callIdHistory.latestCallId(callId)];
       if (call) {
         for (const stream of removedStream) {
-          if (stream.mediaStreamType in call.togetherMode.stream) {
-            // Temporary lint fix: Remove the stream from the list
-            call.togetherMode.stream = [];
+          if (stream.mediaStreamType in call.togetherMode.streams) {
+            call.togetherMode.streams.delete(stream.mediaStreamType);
           }
         }
       }
     });
   }
 
+  /* @conditional-compile-remove(together-mode) */
+  public setTogetherModeSeatingCoordinatesState(callId: string, seatingMap: TogetherModeSeatingMap): void {
+    this.modifyState((draft: CallClientState) => {
+      const call = draft.calls[this._callIdHistory.latestCallId(callId)];
+      if (call) {
+        call.togetherMode.seatingCoordinates = seatingMap;
+      }
+    });
+  }
+
   public setCallRaisedHands(callId: string, raisedHands: RaisedHand[]): void {
     this.modifyState((draft: CallClientState) => {
       const call = draft.calls[this._callIdHistory.latestCallId(callId)];
@@ -720,6 +745,24 @@ export class CallContext {
     });
   }
 
+  /* @conditional-compile-remove(together-mode) */
+  public setTogetherModeVideoStreamRendererView(
+    callId: string,
+    togetherModeStreamType: string,
+    view: VideoStreamRendererViewState | undefined
+  ): void {
+    this.modifyState((draft: CallClientState) => {
+      const call = draft.calls[this._callIdHistory.latestCallId(callId)];
+      /* @conditional-compile-remove(together-mode) */
+      if (call) {
+        const togetherModeStream = call.togetherMode.streams.get(togetherModeStreamType);
+        if (togetherModeStream) {
+          togetherModeStream.view = view;
+        }
+      }
+    });
+  }
+
   public setParticipantState(callId: string, participantKey: string, state: RemoteParticipantStatus): void {
     this.modifyState((draft: CallClientState) => {
       const call = draft.calls[this._callIdHistory.latestCallId(callId)];

packages/calling-stateful-client/src/Converter.ts

@@ -157,7 +157,7 @@ export function convertSdkCallToDeclarativeCall(call: CallCommon): CallState {
     pptLive: { isActive: false },
     raiseHand: { raisedHands: [] },
     /* @conditional-compile-remove(together-mode) */
-    togetherMode: { stream: [] },
+    togetherMode: { streams: new Map(), seatingCoordinates: new Map() },
     localParticipantReaction: undefined,
     transcription: { isTranscriptionActive: false },
     screenShareRemoteParticipant: undefined,

packages/calling-stateful-client/src/InternalCallContext.ts

@@ -7,6 +7,8 @@ import {
   RemoteVideoStream,
   VideoStreamRenderer
 } from '@azure/communication-calling';
+/* @conditional-compile-remove(together-mode) */
+import { RemoteVideoStreamCommon } from '@azure/communication-calling';
 import { LocalVideoStreamState } from './CallClientState';
 import type { CallContext } from './CallContext';
 import { CallIdHistory } from './CallIdHistory';
@@ -45,6 +47,12 @@ export type LocalRenderInfo = RenderInfo<LocalVideoStream>;
  */
 export type RemoteRenderInfo = RenderInfo<RemoteVideoStream>;
 
+/* @conditional-compile-remove(together-mode) */
+/**
+ * Internally used to keep track of the status, renderer, and awaiting promise, associated with a CallFeatureVideoStream.
+ */
+export type CallFeatureRenderInfo = RenderInfo<RemoteVideoStreamCommon>;
+
 /**
  * Contains internal data used between different Declarative components to share data.
  */
@@ -55,6 +63,9 @@ export class InternalCallContext {
   // <CallId, <MediaStreamType, LocalRenderInfo>>.
   private _localRenderInfos = new Map<string, Map<MediaStreamType, LocalRenderInfo>>();
 
+  /* @conditional-compile-remove(together-mode) */
+  // <CallId, <featureName, <MediaStreamType, CallFeatureRenderInfo>>>.
+  private _callFeatureRenderInfos = new Map<string, Map<string, Map<MediaStreamType, CallFeatureRenderInfo>>>();
   // Used for keeping track of rendered LocalVideoStreams that are not part of a Call.
   private _unparentedRenderInfos = new Map<MediaStreamType, LocalRenderInfo>();
   private _callIdHistory = new CallIdHistory();
@@ -77,6 +88,13 @@ export class InternalCallContext {
       this._localRenderInfos.delete(oldCallId);
       this._localRenderInfos.set(newCallId, localRenderInfos);
     }
+    /* @conditional-compile-remove(together-mode) */
+    const callFeatureRenderInfos = this._callFeatureRenderInfos.get(oldCallId);
+    /* @conditional-compile-remove(together-mode) */
+    if (callFeatureRenderInfos) {
+      this._callFeatureRenderInfos.delete(oldCallId);
+      this._callFeatureRenderInfos.set(newCallId, callFeatureRenderInfos);
+    }
   }
 
   public getCallIds(): IterableIterator<string> {
@@ -221,5 +239,63 @@ export class InternalCallContext {
   public clearCallRelatedState(): void {
     this._remoteRenderInfos.clear();
     this._localRenderInfos.clear();
+    /* @conditional-compile-remove(together-mode) */
+    this._callFeatureRenderInfos.clear();
+  }
+
+  /* @conditional-compile-remove(together-mode) */
+  public getCallFeatureRenderInfosForCall(
+    callId: string
+  ): Map<string, Map<MediaStreamType, CallFeatureRenderInfo>> | undefined {
+    return this._callFeatureRenderInfos.get(this._callIdHistory.latestCallId(callId));
+  }
+
+  /* @conditional-compile-remove(together-mode) */
+  public getCallFeatureRenderInfo(
+    callId: string,
+    featureNameKey: string,
+    streamKey: MediaStreamType
+  ): CallFeatureRenderInfo | undefined {
+    const callFeatureRenderInfosForCall = this._callFeatureRenderInfos
+      .get(this._callIdHistory.latestCallId(callId))
+      ?.get(featureNameKey)
+      ?.get(streamKey);
+    if (!callFeatureRenderInfosForCall) {
+      return undefined;
+    }
+    return callFeatureRenderInfosForCall;
+  }
+
+  /* @conditional-compile-remove(together-mode) */
+  public setCallFeatureRenderInfo(
+    callId: string,
+    featureNameKey: string,
+    streamKey: MediaStreamType,
+    stream: RemoteVideoStreamCommon,
+    status: RenderStatus,
+    renderer: VideoStreamRenderer | undefined
+  ): void {
+    let callRenderInfos = this._callFeatureRenderInfos.get(this._callIdHistory.latestCallId(callId));
+    if (!callRenderInfos) {
+      callRenderInfos = new Map<string, Map<MediaStreamType, CallFeatureRenderInfo>>();
+      // If the callId is not found, create a new map for the callId.
+      this._callFeatureRenderInfos.set(this._callIdHistory.latestCallId(callId), callRenderInfos);
+    }
+    let featureRenderInfos = callRenderInfos.get(featureNameKey);
+    if (!featureRenderInfos) {
+      featureRenderInfos = new Map<MediaStreamType, CallFeatureRenderInfo>();
+      callRenderInfos.set(featureNameKey, featureRenderInfos);
+    }
+    featureRenderInfos.set(streamKey, { stream, status, renderer });
+  }
+
+  /* @conditional-compile-remove(together-mode) */
+  public deleteCallFeatureRenderInfo(callId: string, featureName: string, streamKey: MediaStreamType): void {
+    const callFeatureRenderInfoForCall = this._callFeatureRenderInfos.get(this._callIdHistory.latestCallId(callId));
+    if (!callFeatureRenderInfoForCall || !callFeatureRenderInfoForCall.get(featureName)) {
+      return;
+    }
+
+    callFeatureRenderInfoForCall.get(featureName)?.delete(streamKey);
   }
 }