enha: replay: use rawRGBA for replay native integration on ios (#2573)

* capture screenshots in sequence until they're stable * improve timestamps * fix tests & future handling * example: debug log only in debug mode * formatting * min-version compat * linter issue * fix scheduler tests * cleanup * fix: scheduler test * improve performance * improve screenshot comparison performance * comments * screenshot list-equal fix & tests * min-version failure * rename retrier to stabilizer * chore: changelog * capture stable issue screenshots when masking is enabled * fixes and tests * better logging * cocoa replay fix * fix dart2js tests * fix screenshot capture hanging * fix integration test * time out if we can't take a screenshot for events * enha: use rgba for cocoa replay * chore: improve docs * chore: update changelog
getsentry · Jan 14, 2025 · a43c400 · a43c400
1 parent d1488a1
commit a43c400
Show file tree

Hide file tree

Showing 9 changed files with 88 additions and 37 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -14,7 +14,7 @@
 
 ### Enhancements
 
-- Replay: improve iOS native interop performance ([#2530](https://github.com/getsentry/sentry-dart/pull/2530))
+- Replay: improve iOS native interop performance ([#2530](https://github.com/getsentry/sentry-dart/pull/2530), [#2573](https://github.com/getsentry/sentry-dart/pull/2573))
 - Replay: improve orientation change tracking accuracy on Android ([#2540](https://github.com/getsentry/sentry-dart/pull/2540))
 
 ### Fixes

diff --git a/flutter/ios/Classes/SentryFlutterReplayScreenshotProvider.m b/flutter/ios/Classes/SentryFlutterReplayScreenshotProvider.m
@@ -33,11 +33,45 @@ - (void)imageWithView:(UIView *_Nonnull)view
                 NSDictionary *dict = (NSDictionary *)value;
                 long address = ((NSNumber *)dict[@"address"]).longValue;
                 NSNumber *length = ((NSNumber *)dict[@"length"]);
+                NSNumber *width = ((NSNumber *)dict[@"width"]);
+                NSNumber *height = ((NSNumber *)dict[@"height"]);
                 NSData *data =
                     [NSData dataWithBytesNoCopy:(void *)address
                                          length:length.unsignedLongValue
                                    freeWhenDone:TRUE];
-                UIImage *image = [UIImage imageWithData:data];
+
+                // We expect rawRGBA, see docs for ImageByteFormat:
+                // https://api.flutter.dev/flutter/dart-ui/ImageByteFormat.html
+                // Unencoded bytes, in RGBA row-primary form with premultiplied
+                // alpha, 8 bits per channel.
+                static const int kBitsPerChannel = 8;
+                static const int kBytesPerPixel = 4;
+                assert(length.unsignedLongValue % kBytesPerPixel == 0);
+
+                // Let's create an UIImage from the raw data.
+                // We need to provide it the width & height and
+                // the info how the data is encoded.
+                CGDataProviderRef provider =
+                    CGDataProviderCreateWithCFData((CFDataRef)data);
+                CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();
+                CGBitmapInfo bitmapInfo =
+                    kCGBitmapByteOrderDefault | kCGImageAlphaPremultipliedLast;
+                CGImageRef cgImage = CGImageCreate(
+                    width.unsignedLongValue,          // width
+                    height.unsignedLongValue,         // height
+                    kBitsPerChannel,                  // bits per component
+                    kBitsPerChannel * kBytesPerPixel, // bits per pixel
+                    width.unsignedLongValue * kBytesPerPixel, // bytes per row
+                    colorSpace, bitmapInfo, provider, NULL, false,
+                    kCGRenderingIntentDefault);
+
+                UIImage *image = [UIImage imageWithCGImage:cgImage];
+
+                // UIImage takes its own refs, we need to release these here.
+                CGImageRelease(cgImage);
+                CGColorSpaceRelease(colorSpace);
+                CGDataProviderRelease(provider);
+
                 onComplete(image);
                 return;
               } else if ([value isKindOfClass:[FlutterError class]]) {

diff --git a/flutter/lib/src/native/cocoa/cocoa_replay_recorder.dart b/flutter/lib/src/native/cocoa/cocoa_replay_recorder.dart
@@ -1,5 +1,4 @@
 import 'dart:async';
-import 'dart:typed_data';
 
 import 'package:meta/meta.dart';
 
@@ -8,30 +7,36 @@ import '../../replay/replay_recorder.dart';
 import '../../screenshot/recorder.dart';
 import '../../screenshot/recorder_config.dart';
 import '../../screenshot/stabilizer.dart';
+import '../native_memory.dart';
 
 @internal
 class CocoaReplayRecorder {
   final SentryFlutterOptions _options;
   final ScreenshotRecorder _recorder;
   late final ScreenshotStabilizer<void> _stabilizer;
-  var _completer = Completer<Uint8List?>();
+  var _completer = Completer<Map<String, int>?>();
 
   CocoaReplayRecorder(this._options)
       : _recorder =
             ReplayScreenshotRecorder(ScreenshotRecorderConfig(), _options) {
     _stabilizer = ScreenshotStabilizer(_recorder, _options, (screenshot) async {
-      final pngData = await screenshot.pngData;
+      final data = await screenshot.rawRgbaData;
       _options.logger(
           SentryLevel.debug,
           'Replay: captured screenshot ('
           '${screenshot.width}x${screenshot.height} pixels, '
-          '${pngData.lengthInBytes} bytes)');
-      _completer.complete(pngData.buffer.asUint8List());
+          '${data.lengthInBytes} bytes)');
+
+      // Malloc memory and copy the data. Native must free it.
+      final json = data.toNativeMemory().toJson();
+      json['width'] = screenshot.width;
+      json['height'] = screenshot.height;
+      _completer.complete(json);
     });
   }
 
-  Future<Uint8List?> captureScreenshot() async {
-    _completer = Completer<Uint8List?>();
+  Future<Map<String, int>?> captureScreenshot() async {
+    _completer = Completer();
     _stabilizer.ensureFrameAndAddCallback((msSinceEpoch) {
       _stabilizer.capture(msSinceEpoch).onError(_completer.completeError);
     });

diff --git a/flutter/lib/src/native/cocoa/sentry_native_cocoa.dart b/flutter/lib/src/native/cocoa/sentry_native_cocoa.dart
@@ -5,7 +5,6 @@ import 'package:meta/meta.dart';
 
 import '../../../sentry_flutter.dart';
 import '../../replay/replay_config.dart';
-import '../native_memory.dart';
 import '../sentry_native_channel.dart';
 import 'binding.dart' as cocoa;
 import 'cocoa_replay_recorder.dart';
@@ -43,9 +42,7 @@ class SentryNativeCocoa extends SentryNativeChannel {
               });
             }
 
-            final uint8List = await _replayRecorder!.captureScreenshot();
-            // Malloc memory and copy the data. Native must free it.
-            return uint8List?.toNativeMemory().toJson();
+            return _replayRecorder!.captureScreenshot();
           default:
             throw UnimplementedError('Method ${call.method} not implemented');
         }

diff --git a/flutter/lib/src/native/native_memory.dart b/flutter/lib/src/native/native_memory.dart
@@ -12,13 +12,24 @@ class NativeMemory {
 
   const NativeMemory._(this.pointer, this.length);
 
-  factory NativeMemory.fromUint8List(Uint8List source) {
-    final length = source.length;
-    final ptr = pkg_ffi.malloc.allocate<Uint8>(length);
-    if (length > 0) {
-      ptr.asTypedList(length).setAll(0, source);
+  factory NativeMemory.fromByteData(ByteData source) {
+    final lengthInBytes = source.lengthInBytes;
+    final ptr = pkg_ffi.malloc.allocate<Uint8>(lengthInBytes);
+
+    // TODO memcpy() from source.buffer.asUint8List().address
+    //      once we can depend on Dart SDK 3.5+
+    final numWords = lengthInBytes ~/ 8;
+    final words = ptr.cast<Uint64>().asTypedList(numWords);
+    if (numWords > 0) {
+      words.setAll(0, source.buffer.asUint64List(0, numWords));
     }
-    return NativeMemory._(ptr, length);
+
+    final bytes = ptr.asTypedList(lengthInBytes);
+    for (var i = words.lengthInBytes; i < source.lengthInBytes; i++) {
+      bytes[i] = source.getUint8(i);
+    }
+
+    return NativeMemory._(ptr, lengthInBytes);
   }
 
   factory NativeMemory.fromJson(Map<dynamic, dynamic> json) {
@@ -44,6 +55,6 @@ class NativeMemory {
 }
 
 @internal
-extension Uint8ListNativeMemory on Uint8List {
-  NativeMemory toNativeMemory() => NativeMemory.fromUint8List(this);
+extension ByteDataNativeMemory on ByteData {
+  NativeMemory toNativeMemory() => NativeMemory.fromByteData(this);
 }
diff --git a/flutter/lib/src/screenshot/screenshot.dart b/flutter/lib/src/screenshot/screenshot.dart
@@ -12,20 +12,20 @@ class Screenshot {
   final Image _image;
   final DateTime timestamp;
   final Flow flow;
-  Future<ByteData>? _rawData;
+  Future<ByteData>? _rawRgbaData;
   Future<ByteData>? _pngData;
   bool _disposed = false;
 
   Screenshot(this._image, this.timestamp, this.flow);
   Screenshot._cloned(
-      this._image, this.timestamp, this.flow, this._rawData, this._pngData);
+      this._image, this.timestamp, this.flow, this._rawRgbaData, this._pngData);
 
   int get width => _image.width;
   int get height => _image.height;
 
-  Future<ByteData> get rawData {
-    _rawData ??= _encode(ImageByteFormat.rawUnmodified);
-    return _rawData!;
+  Future<ByteData> get rawRgbaData {
+    _rawRgbaData ??= _encode(ImageByteFormat.rawRgba);
+    return _rawRgbaData!;
   }
 
   Future<ByteData> get pngData {
@@ -46,20 +46,20 @@ class Screenshot {
       return false;
     }
 
-    return listEquals(await rawData, await other.rawData);
+    return listEquals(await rawRgbaData, await other.rawRgbaData);
   }
 
   Screenshot clone() {
     assert(!_disposed, 'Cannot clone a disposed screenshot');
     return Screenshot._cloned(
-        _image.clone(), timestamp, flow, _rawData, _pngData);
+        _image.clone(), timestamp, flow, _rawRgbaData, _pngData);
   }
 
   void dispose() {
     if (!_disposed) {
       _disposed = true;
       _image.dispose();
-      _rawData = null;
+      _rawRgbaData = null;
       _pngData = null;
     }
   }

diff --git a/flutter/test/native_memory_test.dart b/flutter/test/native_memory_test.dart
@@ -8,29 +8,30 @@ import 'native_memory_web_mock.dart'
     if (dart.library.io) 'package:sentry_flutter/src/native/native_memory.dart';
 
 void main() {
-  final testSrcList = Uint8List.fromList([1, 2, 3]);
+  final testSrcList = Uint8List.fromList([1, 2, 3, 4, 5, 6, 7, 8, 9, 0]);
+  final testSrcData = testSrcList.buffer.asByteData();
 
   test('empty list', () async {
-    final sut = NativeMemory.fromUint8List(Uint8List.fromList([]));
+    final sut = NativeMemory.fromByteData(ByteData(0));
     expect(sut.length, 0);
     expect(sut.pointer.address, greaterThan(0));
     expect(sut.asTypedList(), isEmpty);
     sut.free();
   });
 
   test('non-empty list', () async {
-    final sut = NativeMemory.fromUint8List(testSrcList);
-    expect(sut.length, 3);
+    final sut = NativeMemory.fromByteData(testSrcData);
+    expect(sut.length, 10);
     expect(sut.pointer.address, greaterThan(0));
     expect(sut.asTypedList(), testSrcList);
     sut.free();
   });
 
   test('json', () async {
-    final sut = NativeMemory.fromUint8List(testSrcList);
+    final sut = NativeMemory.fromByteData(testSrcData);
     final json = sut.toJson();
     expect(json['address'], greaterThan(0));
-    expect(json['length'], 3);
+    expect(json['length'], 10);
     expect(json.entries, hasLength(2));
 
     final sut2 = NativeMemory.fromJson(json);

diff --git a/flutter/test/native_memory_web_mock.dart b/flutter/test/native_memory_web_mock.dart
@@ -9,8 +9,9 @@ class NativeMemory {
 
   const NativeMemory._(this.pointer, this.length);
 
-  factory NativeMemory.fromUint8List(Uint8List source) {
-    return NativeMemory._(Pointer<Uint8>._store(source), source.length);
+  factory NativeMemory.fromByteData(ByteData source) {
+    return NativeMemory._(Pointer<Uint8>._store(source.buffer.asUint8List()),
+        source.lengthInBytes);
   }
 
   factory NativeMemory.fromJson(Map<dynamic, dynamic> json) {

diff --git a/flutter/test/replay/replay_native_test.dart b/flutter/test/replay/replay_native_test.dart
@@ -209,6 +209,8 @@ void main() {
 
                 expect(json['length'], greaterThan(3000));
                 expect(json['address'], greaterThan(0));
+                expect(json['width'], greaterThan(0));
+                expect(json['height'], greaterThan(0));
                 NativeMemory.fromJson(json).free();
               }