:sprakles: Tool command to extract frames and convert to RGB

src/Directory.Packages.props

@@ -2,6 +2,7 @@
   <!-- Centralize dependency management -->
   <ItemGroup>
     <PackageVersion Include="BenchmarkDotNet" Version="0.13.9" />
+    <PackageVersion Include="Texim" Version="0.1.0-preview.195" />
     <PackageVersion Include="Yarhl" Version="4.0.0-preview.221" />
     <PackageVersion Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
     <PackageVersion Include="nunit" Version="3.13.3" />

src/PlayMobic.Tool/PlayMobic.Tool.csproj

@@ -14,6 +14,7 @@
 
   <ItemGroup>
     <PackageReference Include="System.CommandLine" />
+    <PackageReference Include="Texim" />
   </ItemGroup>
 
   <ItemGroup>

src/PlayMobic.Tool/Program.cs

@@ -1,15 +1,17 @@
-#pragma warning disable SA1200 // False positive on top-level without namespace
-using System.CommandLine;
+using System.CommandLine;
 using PlayMobic.Containers;
 using PlayMobic.Containers.Mods;
 using PlayMobic.Video;
 using PlayMobic.Video.Mobiclip;
+using Texim.Colors;
+using Texim.Formats;
+using Texim.Images;
 using Yarhl.FileSystem;
 using Yarhl.IO;
-#pragma warning restore SA1200
 
 return await new RootCommand("Tool for MODS videos") {
     SetupInfoCommand(),
+    SetupExtraFramesCommand(),
     SetupDemuxCommand(),
 }.InvokeAsync(args);
 
@@ -24,6 +26,19 @@ Command SetupInfoCommand()
     return infoCommand;
 }
 
+Command SetupExtraFramesCommand()
+{
+    var inputArg = new Option<FileInfo>("--input", "Path to the .mods file") { IsRequired = true };
+    var outputArg = new Option<string>("--output", "Path to the folder to write the frames") { IsRequired = true };
+    var command = new Command("extract-frames", "Extract each video frame into PNG images") {
+        inputArg,
+        outputArg,
+    };
+    command.SetHandler(ExtractFrames, inputArg, outputArg);
+
+    return command;
+}
+
 Command SetupDemuxCommand()
 {
     var inputArg = new Option<FileInfo>("--input", "Path to the .mods file") { IsRequired = true };
@@ -62,6 +77,45 @@ void PrintInfo(FileInfo videoFile)
     }
 }
 
+void ExtractFrames(FileInfo videoFile, string outputPath)
+{
+    Console.WriteLine("Video: {0}", videoFile.FullName);
+    Console.WriteLine("Output: {0}", outputPath);
+
+    using Node videoNode = NodeFactory.FromFile(videoFile.FullName, FileOpenMode.Read)
+        .TransformWith<Binary2Mods>();
+
+    ModsVideo video = videoNode.GetFormatAs<ModsVideo>()!;
+    ModsInfo info = video.Info;
+
+    var demuxer = new ModsDemuxer(video);
+    var videoDecoder = new MobiclipDecoder(info.Width, info.Height);
+    var image2BinaryBitmap = new FullImage2Bitmap();
+
+    // This work because video is always the first stream in the packets
+    // and we don't need to decode audio to advance to next frame.
+    foreach (MediaPacket framePacket in demuxer.ReadFrames().OfType<VideoPacket>()) {
+        if (!framePacket.IsKeyFrame) {
+            // not supported yet
+            continue;
+        }
+
+        FrameYuv420 frame = videoDecoder.DecodeFrame(framePacket.Data);
+
+        byte[] rgbFrame = ColorSpaceConverter.YCoCg2Rgb32(frame);
+        var frameImage = new FullImage(frame.Width, frame.Height) {
+            Pixels = Rgb32.Instance.Decode(rgbFrame),
+        };
+        image2BinaryBitmap.Convert(frameImage)
+            .Stream.WriteTo(Path.Combine(outputPath, $"frame{framePacket.FrameCount}.png"));
+
+        Console.Write('+');
+    }
+
+    Console.WriteLine();
+    Console.WriteLine("Done");
+}
+
 void Demux(FileInfo videoFile, string outputPath)
 {
     string videoPath = Path.Combine(outputPath, videoFile.Name + ".rawvideo");
@@ -80,13 +134,12 @@ void Demux(FileInfo videoFile, string outputPath)
 
     var demuxer = new ModsDemuxer(video);
     foreach (MediaPacket framePacket in demuxer.ReadFrames()) {
-        if (framePacket.IsKeyFrame && framePacket is VideoPacket) {
+        if (framePacket is VideoPacket) {
             FrameYuv420 frame = videoDecoder.DecodeFrame(framePacket.Data);
             videoStream.Write(frame.PackedData);
-
             Console.Write('+');
         } else {
-            Console.Write('.');
+            // TODO: decode audio
         }
     }
 

src/PlayMobic/Video/ColorSpaceConverter.cs

@@ -0,0 +1,36 @@
+namespace PlayMobic.Video;
+using System;
+using System.Runtime.CompilerServices;
+
+public static class ColorSpaceConverter
+{
+    public static byte[] YCoCg2Rgb32(FrameYuv420 source)
+    {
+        byte[] rgb = new byte[source.Width * source.Height * 4];
+
+        for (int y = 0; y < source.Height; y++) {
+            for (int x = 0; x < source.Width; x++) {
+                // luma is in range 0-255 but chroma is centered at 128, center at 0
+                byte luma = source.Luma[x, y];
+                int co = source.ChromaU[x / 2, y / 2] - 128;
+                int cg = source.ChromaV[x / 2, y / 2] - 128;
+
+                int tmp = luma - cg;
+                int g = luma + cg;
+                int b = tmp - co;
+                int r = tmp + co;
+
+                int index = ((y * source.Width) + x) * 4;
+                rgb[index + 0] = ClampByte(r);
+                rgb[index + 1] = ClampByte(g);
+                rgb[index + 2] = ClampByte(b);
+            }
+        }
+
+        return rgb;
+    }
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    private static byte ClampByte(double value) =>
+            (byte)Math.Clamp(value, byte.MinValue, byte.MaxValue);
+}