From b163f8ce8f8379c5b838c81d3ed66ebf1d38dda1 Mon Sep 17 00:00:00 2001
From: Yassir Bisteni <yassirb@microsoft.com>
Date: Tue, 23 Apr 2024 19:45:25 -0600
Subject: [PATCH] Enabled video resolution picker

---
 RawVideo/CameraCaptureService.cs | 141 ++++++++++++++++++++++++++-----
 RawVideo/CaptureService.cs       |  28 +++---
 RawVideo/MainPage.xaml           |  41 ++++++---
 RawVideo/MainPage.xaml.cs        |  63 ++++++++++----
 4 files changed, 210 insertions(+), 63 deletions(-)

diff --git a/RawVideo/CameraCaptureService.cs b/RawVideo/CameraCaptureService.cs
index 64426c7a..6a3f1df0 100644
--- a/RawVideo/CameraCaptureService.cs
+++ b/RawVideo/CameraCaptureService.cs
@@ -3,59 +3,81 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
+using Windows.Graphics;
 using Windows.Graphics.Imaging;
 using Windows.Media.Capture;
 using Windows.Media.Capture.Frames;
 
 namespace RawVideo
 {
+    internal class MediaFrameSourceBundle
+    {
+        public MediaFrameSourceGroup Group { get; set; }
+        public MediaFrameSourceInfo Info { get; set; }
+        public MediaFrameFormat Format { get; set; }
+    }
+
+    internal class VideoFormatBundle
+    {
+        public SizeInt32 Size { get; set; }
+        public MediaFrameFormat Format { get; set; }
+    }
+
+    internal class VideoFormatBundleComparer : IEqualityComparer<VideoFormatBundle>
+    {
+        public bool Equals(VideoFormatBundle s1, VideoFormatBundle s2)
+        {
+            return s1.Size.Width == s2.Size.Width && s1.Size.Height == s2.Size.Height;
+        }
+
+        public int GetHashCode(VideoFormatBundle obj)
+        {
+            return obj.Size.Width.GetHashCode() ^ obj.Size.Height.GetHashCode();
+        }
+    }
+
     internal class CameraCaptureService : CaptureService
     {
+        private readonly MediaFrameSourceBundle bundle;
         private MediaCapture mediaCapture;
         private MediaFrameReader mediaFrameReader;
-        private MediaFrameSourceGroup sourceGroup;
-        private MediaFrameSourceInfo sourceInfo;
 
         public CameraCaptureService(RawOutgoingVideoStream rawOutgoingVideoStream,
-            RawVideoFrameKind rawVideoFrameKind,
-            Tuple<MediaFrameSourceGroup, MediaFrameSourceInfo> mediaFrameSource) :
+            RawVideoFrameKind rawVideoFrameKind, MediaFrameSourceBundle bundle) :
             base(rawOutgoingVideoStream, rawVideoFrameKind)
         {
-            sourceGroup = mediaFrameSource.Item1;
-            sourceInfo = mediaFrameSource.Item2;
+            this.bundle = bundle;
         }
 
         public async Task StartAsync()
         {
             var settings = new MediaCaptureInitializationSettings
             {
-                SourceGroup = sourceGroup,
-                SharingMode = MediaCaptureSharingMode.SharedReadOnly,
+                SourceGroup = bundle.Group,
+                SharingMode = MediaCaptureSharingMode.ExclusiveControl,
                 StreamingCaptureMode = StreamingCaptureMode.Video,
                 MemoryPreference = MediaCaptureMemoryPreference.Cpu
             };
 
             mediaCapture = new MediaCapture();
-            MediaFrameReaderStartStatus mediaFrameReaderStatus;
+            MediaFrameReaderStartStatus mediaFrameReaderStatus = 
+                MediaFrameReaderStartStatus.UnknownFailure;
 
             try
             {
                 await mediaCapture.InitializeAsync(settings);
-                MediaFrameSource selectedSource = mediaCapture.FrameSources[sourceInfo.Id];
-                mediaFrameReader = await mediaCapture.CreateFrameReaderAsync(selectedSource);
+                MediaFrameSource source = mediaCapture.FrameSources[bundle.Info.Id];
+
+                await source.SetFormatAsync(bundle.Format);
+
+                mediaFrameReader = await mediaCapture.CreateFrameReaderAsync(source);
+                mediaFrameReader.FrameArrived += FrameArrived;
                 mediaFrameReaderStatus = await mediaFrameReader.StartAsync();
             }
             catch (Exception ex)
             {
-                mediaFrameReaderStatus = MediaFrameReaderStartStatus.UnknownFailure;
-
                 Console.WriteLine(ex.Message);
             }
-
-            if (mediaFrameReaderStatus == MediaFrameReaderStartStatus.Success)
-            {
-                mediaFrameReader.FrameArrived += FrameArrived;
-            }
         }
 
         public async Task StopAsync()
@@ -88,10 +110,10 @@ public async Task StopAsync()
             }
         }
 
-        public static async Task<List<Tuple<MediaFrameSourceGroup, MediaFrameSourceInfo>>> GetCameraListAsync()
+        public static async Task<List<MediaFrameSourceBundle>> GetCameraListAsync()
         {
             IReadOnlyList<MediaFrameSourceGroup> groups = await MediaFrameSourceGroup.FindAllAsync();
-            var cameraList = new List<Tuple<MediaFrameSourceGroup, MediaFrameSourceInfo>>();
+            var cameraList = new List<MediaFrameSourceBundle>();
 
             foreach (MediaFrameSourceGroup sourceGroup in groups)
             {
@@ -99,12 +121,87 @@ public static async Task<List<Tuple<MediaFrameSourceGroup, MediaFrameSourceInfo>
                 {
                     if (sourceInfo.SourceKind == MediaFrameSourceKind.Color)
                     {
-                        cameraList.Add(new Tuple<MediaFrameSourceGroup, MediaFrameSourceInfo>(sourceGroup, sourceInfo));
+                        cameraList.Add(new MediaFrameSourceBundle()
+                        {
+                            Group = sourceGroup,
+                            Info = sourceInfo
+                        });
                     }
                 }
             }
 
-            return cameraList.OrderBy(item => item.Item2.DeviceInformation.Name).ToList();
+            return cameraList
+                .OrderBy(x => x.Info.DeviceInformation.Name)
+                .ToList();
+        }
+
+        public static async Task<List<VideoFormatBundle>> GetSupportedVideoFormats(MediaFrameSourceBundle bundle)
+        {
+            var settings = new MediaCaptureInitializationSettings
+            {
+                SourceGroup = bundle.Group,
+                SharingMode = MediaCaptureSharingMode.SharedReadOnly,
+                StreamingCaptureMode = StreamingCaptureMode.Video,
+                MemoryPreference = MediaCaptureMemoryPreference.Cpu
+            };
+
+            IReadOnlyList<MediaFrameFormat> sourceFormatList = null;
+            var videoFormatList= new List<VideoFormatBundle>();
+
+            using (var mediaCapture = new MediaCapture())
+            {
+                try
+                {
+                    await mediaCapture.InitializeAsync(settings);
+                    MediaFrameSource source = mediaCapture.FrameSources[bundle.Info.Id];
+                    sourceFormatList = source.SupportedFormats;
+                }
+                catch (Exception ex)
+                {
+                    Console.WriteLine(ex.Message);
+                }
+            }
+
+            if (sourceFormatList != null)
+            {
+                var resolutionList = Enum.GetValues(typeof(VideoStreamResolution))
+                    .Cast<VideoStreamResolution>()
+                    .ToList();
+
+                var acsFormatList = resolutionList
+                    .Select(x =>
+                    {
+                        var format = new VideoStreamFormat
+                        {
+                            Resolution = x
+                        };
+
+                        return new SizeInt32()
+                        {
+                            Width = format.Width,
+                            Height = format.Height
+                        };
+                    })
+                    .Distinct()
+                    .ToList();
+
+                videoFormatList = sourceFormatList
+                    .Select(x => new VideoFormatBundle()
+                    {
+                        Size = new SizeInt32()
+                        { 
+                            Width = (int) x.VideoFormat.Width,
+                            Height = (int) x.VideoFormat.Height
+                        },
+                        Format = x
+                    })
+                    .Distinct(new VideoFormatBundleComparer())
+                    .Where(x => acsFormatList.Contains(x.Size))
+                    .OrderByDescending(x => x.Size.Width)
+                    .ToList();
+            }
+
+            return videoFormatList;
         }
     }
 }
diff --git a/RawVideo/CaptureService.cs b/RawVideo/CaptureService.cs
index a45903a8..88211529 100644
--- a/RawVideo/CaptureService.cs
+++ b/RawVideo/CaptureService.cs
@@ -11,24 +11,25 @@ namespace RawVideo
     internal abstract class CaptureService
     {
         public event EventHandler<RawVideoFrame> FrameArrived;
-        protected readonly RawOutgoingVideoStream rawOutgoingVideoStream;
+        protected readonly RawOutgoingVideoStream stream;
         private readonly RawVideoFrameKind rawVideoFrameKind;
 
-        protected CaptureService(RawOutgoingVideoStream rawOutgoingVideoStream, RawVideoFrameKind rawVideoFrameKind)
+        protected CaptureService(RawOutgoingVideoStream stream, RawVideoFrameKind rawVideoFrameKind)
         {
-            this.rawOutgoingVideoStream = rawOutgoingVideoStream;
+            this.stream = stream;
             this.rawVideoFrameKind = rawVideoFrameKind;
         }
 
         protected async Task SendRawVideoFrame(SoftwareBitmap bitmap)
         {
-            if (bitmap != null && CanSendRawVideoFrames())
+            var format = stream.Format;
+            if (bitmap != null && format != null && CanSendRawVideoFrames())
             {
                 try
                 {
-                    RawVideoFrame frame = ConvertSoftwareBitmapToRawVideoFrame(bitmap);
+                    RawVideoFrame frame = ConvertSoftwareBitmapToRawVideoFrame(bitmap, format);
 
-                    await rawOutgoingVideoStream.SendRawVideoFrameAsync(frame);
+                    await stream.SendRawVideoFrameAsync(frame);
 
                     FrameArrived?.Invoke(this, frame);
                 }
@@ -39,7 +40,9 @@ protected async Task SendRawVideoFrame(SoftwareBitmap bitmap)
             }
         }
 
-        private unsafe RawVideoFrame ConvertSoftwareBitmapToRawVideoFrame(SoftwareBitmap bitmap)
+        private unsafe RawVideoFrame ConvertSoftwareBitmapToRawVideoFrame(
+            SoftwareBitmap bitmap, 
+            VideoStreamFormat format)
         {
             bitmap = SoftwareBitmap.Convert(bitmap, BitmapPixelFormat.Rgba8);
 
@@ -59,21 +62,22 @@ private unsafe RawVideoFrame ConvertSoftwareBitmapToRawVideoFrame(SoftwareBitmap
             {
                 case RawVideoFrameKind.Buffer:
                     MemoryBuffer buffer = Buffer.CreateMemoryBufferOverIBuffer(bitmapBuffer);
+
                     frame = new RawVideoFrameBuffer()
                     {
                         Buffers = new MemoryBuffer[] { buffer },
-                        StreamFormat = rawOutgoingVideoStream.Format
+                        StreamFormat = format
                     };
 
                     break;
                 case RawVideoFrameKind.Texture:
-                    var timeSpan = new TimeSpan(rawOutgoingVideoStream.TimestampInTicks);
+                    var timeSpan = new TimeSpan(stream.TimestampInTicks);
                     var sample = MediaStreamSample.CreateFromBuffer(bitmapBuffer, timeSpan);
 
                     frame = new RawVideoFrameTexture()
                     {
                         Texture = sample,
-                        StreamFormat = rawOutgoingVideoStream.Format
+                        StreamFormat = stream.Format
                     };
 
                     break;
@@ -84,9 +88,7 @@ private unsafe RawVideoFrame ConvertSoftwareBitmapToRawVideoFrame(SoftwareBitmap
 
         private bool CanSendRawVideoFrames()
         {
-            return rawOutgoingVideoStream != null &&
-                    rawOutgoingVideoStream.Format != null &&
-                    rawOutgoingVideoStream.State == VideoStreamState.Started;
+            return stream != null && stream.State == VideoStreamState.Started;
         }
     }
 }
diff --git a/RawVideo/MainPage.xaml b/RawVideo/MainPage.xaml
index 656d243f..df92c356 100644
--- a/RawVideo/MainPage.xaml
+++ b/RawVideo/MainPage.xaml
@@ -23,11 +23,12 @@
                                 </VisualStateGroup>
                             </VisualStateManager.VisualStateGroups>
                             <ContentPresenter
-                                Width="500"
+                                Width="Auto"
                                 Height="50"
                                 FontSize="25"
                                 Background="White"
                                 Foreground="Black"
+                                Padding="20,0,20,0"
                                 />
                         </Grid>
                     </ControlTemplate>
@@ -215,9 +216,8 @@
                     </ComboBox>
                 </StackPanel>
 
-
                 <StackPanel
-                    Width="500"
+                    Width="Auto"
                     Orientation="Horizontal"
                     >
                     <ComboBox
@@ -250,6 +250,22 @@
                             <SolidColorBrush x:Key="ComboBoxBorderBrushPointerOver" Color="Black"/>
                         </ComboBox.Resources>
                     </ComboBox>
+
+                    <ComboBox
+                        Width="200"
+                        x:Name="outgoingVideoFormatComboBox"
+                        PlaceholderText="OutgoingVideoFormat"
+                        SelectionChanged="OutgoingVideoFormatSelected"
+                        Style="{StaticResource ComboBoxStyle}"
+                        ItemContainerStyle="{StaticResource ComboBoxItemStyle}"
+                        Margin="10,10,0,0"
+                        >
+                        <ComboBox.Resources>
+                            <SolidColorBrush x:Key="ComboBoxBackgroundPointerOver" Color="white"/>
+                            <SolidColorBrush x:Key="ComboBoxForegroundPointerOver" Color="Black"/>
+                            <SolidColorBrush x:Key="ComboBoxBorderBrushPointerOver" Color="Black"/>
+                        </ComboBox.Resources>
+                    </ComboBox>
                 </StackPanel>
 
                 <ComboBox
@@ -298,34 +314,33 @@
                 </ComboBox>
             </StackPanel>
 
-            <RelativePanel
+            <StackPanel
                 x:Name="videoContainer"
-                Width="640"
-                Height="360"
                 Margin="0,10,0,0"
                 Visibility="Collapsed"
+                Orientation="Vertical"
                 >
                 <RelativePanel
                     x:Name="incomingVideoContainer"
-                    Width="640"
-                    Height="360"
+                    Width="480"
+                    Height="270"
                     BorderBrush="Black"
                     BorderThickness="2"
                     CornerRadius="8"
+                    Margin="0,0,0,10"
                     >
                 </RelativePanel>
 
                 <RelativePanel
                     x:Name="outgoingVideoContainer"
-                    Width="224"
-                    Height="108"
-                    Margin="10,10,0,0"
+                    Width="480"
+                    Height="270"
                     BorderBrush="Black"
                     BorderThickness="2"
                     CornerRadius="8"
                     >
                 </RelativePanel>
-            </RelativePanel>
+            </StackPanel>
 
             <StackPanel
                 Width="Auto"
@@ -334,7 +349,7 @@
                 Spacing="10"
                 Background="White"
                 HorizontalAlignment="Center"
-                    Margin="0,10,0,0"
+                Margin="0,10,0,0"
                 >
                 <Button
                     x:Name="startCallButton"
diff --git a/RawVideo/MainPage.xaml.cs b/RawVideo/MainPage.xaml.cs
index e1a50343..55194c60 100644
--- a/RawVideo/MainPage.xaml.cs
+++ b/RawVideo/MainPage.xaml.cs
@@ -22,11 +22,13 @@ public sealed partial class MainPage : Page
         private int videoDeviceInfoListIndex = -1;
         private int cameraListIndex = -1;
         private int displayListIndex = -1;
+        private int videoFormatListIndex = -1;
 
         // App
         private IReadOnlyList<VideoDeviceDetails> videoDeviceInfoList;
-        private List<Tuple<MediaFrameSourceGroup, MediaFrameSourceInfo>> cameraList;
+        private List<MediaFrameSourceBundle> cameraList;
         private List<GraphicsCaptureItem> displayList;
+        private List<VideoFormatBundle> videoFormatList;
         private List<VideoStreamKind> outgoingVideoStreamKindList;
         private List<VideoStreamKind> incomingVideoStreamKindList;
         private List<RawVideoFrameKind> rawVideoFrameKindList;
@@ -110,7 +112,10 @@ private async void InitResources()
                 return;
             }
 
-            videoDeviceInfoList = deviceManager.Cameras.OrderBy(item => item.Name).ToList();
+            videoDeviceInfoList = deviceManager.Cameras
+                .OrderBy(item => item.Name)
+                .ToList();
+
             foreach (VideoDeviceDetails item in videoDeviceInfoList)
             {
                 videoDeviceInfoComboBox.Items.Add(item.Name);
@@ -122,9 +127,9 @@ private async void InitResources()
             }
 
             cameraList = await CameraCaptureService.GetCameraListAsync();
-            foreach (Tuple<MediaFrameSourceGroup, MediaFrameSourceInfo> item in cameraList)
+            foreach (MediaFrameSourceBundle item in cameraList)
             {
-                cameraComboBox.Items.Add(item.Item2.DeviceInformation.Name);
+                cameraComboBox.Items.Add(item.Info.DeviceInformation.Name);
             }
 
             if (cameraList.Count > 0)
@@ -322,17 +327,14 @@ private VideoStreamFormat CreateVideoStreamFormat()
             switch (outgoingVideoStreamKind)
             {
                 case VideoStreamKind.VirtualOutgoing:
-                    format.Resolution = VideoStreamResolution.P360;
-                    w = format.Width;
-                    h = format.Height;
+                    VideoFormatBundle bundle = videoFormatList[videoFormatListIndex];
+                    format.Width = w = bundle.Size.Width;
+                    format.Height = h = bundle.Size.Height;
                     break;
                 case VideoStreamKind.ScreenShareOutgoing:
                     GraphicsCaptureItem display = displayList[displayListIndex];
-                    w = display.Size.Width;
-                    h = display.Size.Height;
-
-                    format.Width = w;
-                    format.Height = h;
+                    format.Width = w = display.Size.Width;
+                    format.Height = h = display.Size.Height;
                     break;
             }
 
@@ -580,6 +582,8 @@ private async Task StartCameraCaptureService()
         {
             if (cameraCaptureService == null)
             {
+                cameraList[cameraListIndex].Format = videoFormatList[videoFormatListIndex].Format;
+
                 cameraCaptureService = new CameraCaptureService(virtualOutgoingVideoStream,
                     outgoingVideoFrameKind,
                     cameraList[cameraListIndex]);
@@ -776,6 +780,8 @@ private void OutgoingVideoStreamKindSelected(object sender, SelectionChangedEven
                     cameraComboBox.Visibility = Visibility.Collapsed;
                     displayComboBox.Visibility = Visibility.Collapsed;
 
+                    outgoingVideoFormatComboBox.Visibility = Visibility.Collapsed;
+
                     outgoingVideoStreamKindComboBox.Width = 500;
                     outgoingRawVideoFrameKindComboBox.Visibility = Visibility.Collapsed;
                     break;
@@ -784,6 +790,8 @@ private void OutgoingVideoStreamKindSelected(object sender, SelectionChangedEven
                     cameraComboBox.Visibility = Visibility.Visible;
                     displayComboBox.Visibility = Visibility.Collapsed;
 
+                    outgoingVideoFormatComboBox.Visibility = Visibility.Visible;
+
                     outgoingVideoStreamKindComboBox.Width = 290;
                     outgoingRawVideoFrameKindComboBox.Visibility = Visibility.Visible;
                     break;
@@ -792,6 +800,8 @@ private void OutgoingVideoStreamKindSelected(object sender, SelectionChangedEven
                     cameraComboBox.Visibility = Visibility.Collapsed;
                     displayComboBox.Visibility = Visibility.Visible;
 
+                    outgoingVideoFormatComboBox.Visibility = Visibility.Collapsed;
+
                     outgoingVideoStreamKindComboBox.Width = 290;
                     outgoingRawVideoFrameKindComboBox.Visibility = Visibility.Visible;
                     break;
@@ -803,9 +813,25 @@ private void VideoDeviceInfoSelected(object sender, SelectionChangedEventArgs ar
             videoDeviceInfoListIndex = videoDeviceInfoComboBox.SelectedIndex;
         }
 
-        private void CameraSelected(object sender, SelectionChangedEventArgs args)
+        private async void CameraSelected(object sender, SelectionChangedEventArgs args)
         {
             cameraListIndex = cameraComboBox.SelectedIndex;
+            videoFormatListIndex = -1;
+            outgoingVideoFormatComboBox.Items.Clear();
+
+            videoFormatList = await CameraCaptureService.GetSupportedVideoFormats(
+                cameraList[cameraListIndex]);
+
+            foreach (VideoFormatBundle bundle in videoFormatList)
+            {
+                outgoingVideoFormatComboBox.Items.Add(
+                    string.Format("{0}x{1}", bundle.Size.Width, bundle.Size.Height));
+            }
+
+            if (videoFormatList.Count > 0)
+            {
+                outgoingVideoFormatComboBox.SelectedIndex = 0;
+            }
         }
 
         private void DisplaySelected(object sender, SelectionChangedEventArgs args)
@@ -815,12 +841,19 @@ private void DisplaySelected(object sender, SelectionChangedEventArgs args)
 
         private void IncomingRawVideoFrameKindSelected(object sender, SelectionChangedEventArgs args)
         {
-            incomingVideoFrameKind = rawVideoFrameKindList[incomingRawVideoFrameKindComboBox.SelectedIndex];
+            incomingVideoFrameKind = 
+                rawVideoFrameKindList[incomingRawVideoFrameKindComboBox.SelectedIndex];
         }
 
         private void OutgoingRawVideoFrameKindSelected(object sender, SelectionChangedEventArgs args)
         {
-            outgoingVideoFrameKind = rawVideoFrameKindList[outgoingRawVideoFrameKindComboBox.SelectedIndex];
+            outgoingVideoFrameKind = 
+                rawVideoFrameKindList[outgoingRawVideoFrameKindComboBox.SelectedIndex];
+        }
+
+        private void OutgoingVideoFormatSelected(object sender, SelectionChangedEventArgs e)
+        {
+            videoFormatListIndex = outgoingVideoFormatComboBox.SelectedIndex;
         }
     }
 }