diff --git a/packages/example/lib/vision_detector_views/camera_view.dart b/packages/example/lib/vision_detector_views/camera_view.dart index d6023adc..8231e88e 100644 --- a/packages/example/lib/vision_detector_views/camera_view.dart +++ b/packages/example/lib/vision_detector_views/camera_view.dart @@ -358,27 +358,52 @@ class _CameraViewState extends State { // get image format final format = InputImageFormatValue.fromRawValue(image.format.raw); - // validate format depending on platform - // only supported formats: - // * nv21 for Android - // * bgra8888 for iOS - if (format == null || - (Platform.isAndroid && format != InputImageFormat.nv21) || - (Platform.isIOS && format != InputImageFormat.bgra8888)) return null; + if (format == null) { + print('could not find format from raw value: $image.format.raw'); + return null; + } + // Validate format depending on platform + final androidSupportedFormats = [ + InputImageFormat.nv21, + InputImageFormat.yv12, + InputImageFormat.yuv_420_888 + ]; + if ((Platform.isAndroid && androidSupportedFormats.contains(format)) || + (Platform.isIOS && format != InputImageFormat.bgra8888)) { + print('image format is not supported: $format'); + return null; + } - // since format is constraint to nv21 or bgra8888, both only have one plane - if (image.planes.length != 1) return null; - final plane = image.planes.first; + // Compile a flat list of all image data. For image formats with multiple planes, + // takes some copying. + final Uint8List bytes = image.planes.length == 1 + ? image.planes.first.bytes + : _concatenatePlanes(image); // compose InputImage using bytes return InputImage.fromBytes( - bytes: plane.bytes, + bytes: bytes, metadata: InputImageMetadata( size: Size(image.width.toDouble(), image.height.toDouble()), rotation: rotation, // used only in Android - format: format, // used only in iOS - bytesPerRow: plane.bytesPerRow, // used only in iOS + format: format, + bytesPerRow: image.planes.first.bytesPerRow, // used only in iOS ), ); } + + Uint8List _concatenatePlanes(CameraImage image) { + int length = 0; + for (final Plane p in image.planes) { + length += p.bytes.length; + } + + final Uint8List bytes = Uint8List(length); + int offset = 0; + for (final Plane p in image.planes) { + bytes.setRange(offset, offset + p.bytes.length, p.bytes); + offset += p.bytes.length; + } + return bytes; + } } diff --git a/packages/example/pubspec.lock b/packages/example/pubspec.lock index 606bc631..69da6443 100644 --- a/packages/example/pubspec.lock +++ b/packages/example/pubspec.lock @@ -25,38 +25,30 @@ packages: url: "https://pub.dev" source: hosted version: "0.11.0+2" - camera_android: - dependency: "direct main" - description: - name: camera_android - sha256: "19b7226387218864cb2388e1ad5db7db50d065222f5511254b03fc397dd21a5e" - url: "https://pub.dev" - source: hosted - version: "0.10.9+17" camera_android_camerax: dependency: transitive description: name: camera_android_camerax - sha256: "2bb0724371bae3c0889d7e0b1665357e4aa6ba6c8d32ffa3e178098ba81ed3df" + sha256: ecadc214daed34d8503540525d26577731c066f1993c254aa5272da7629e8f10 url: "https://pub.dev" source: hosted - version: "0.6.11" + version: "0.6.12" camera_avfoundation: dependency: transitive description: name: camera_avfoundation - sha256: "7c28969a975a7eb2349bc2cb2dfe3ad218a33dba9968ecfb181ce08c87486655" + sha256: c3038e6e72e284b14ad246a419f26908c08f8886d114cb8a2e351988439bfa68 url: "https://pub.dev" source: hosted - version: "0.9.17+3" + version: "0.9.17+6" camera_platform_interface: dependency: transitive description: name: camera_platform_interface - sha256: b3ede1f171532e0d83111fe0980b46d17f1aa9788a07a2fbed07366bbdbb9061 + sha256: "953e7baed3a7c8fae92f7200afeb2be503ff1a17c3b4e4ed7b76f008c2810a31" url: "https://pub.dev" source: hosted - version: "2.8.0" + version: "2.9.0" camera_web: dependency: transitive description: @@ -117,10 +109,10 @@ packages: dependency: transitive description: name: file_selector_linux - sha256: "712ce7fab537ba532c8febdb1a8f167b32441e74acd68c3ccb2e36dcb52c4ab2" + sha256: "54cbbd957e1156d29548c7d9b9ec0c0ebb6de0a90452198683a7d23aed617a33" url: "https://pub.dev" source: hosted - version: "0.9.3" + version: "0.9.3+2" file_selector_macos: dependency: transitive description: @@ -162,18 +154,18 @@ packages: dependency: "direct main" description: name: flutter_pdfview - sha256: "6b625b32a9102780236554dff42f2d798b4627704ab4a3153c07f2134a52b697" + sha256: "2e3fa359524e9865ec25a64593b65092b4a9974c5871228c1a771300a003d150" url: "https://pub.dev" source: hosted - version: "1.3.3" + version: "1.4.0" flutter_plugin_android_lifecycle: dependency: transitive description: name: flutter_plugin_android_lifecycle - sha256: "9b78450b89f059e96c9ebb355fa6b3df1d6b330436e0b885fb49594c41721398" + sha256: "615a505aef59b151b46bbeef55b36ce2b6ed299d160c51d84281946f0aa0ce0e" url: "https://pub.dev" source: hosted - version: "2.0.23" + version: "2.0.24" flutter_test: dependency: "direct dev" description: flutter @@ -308,10 +300,10 @@ packages: dependency: transitive description: name: http_parser - sha256: "2aa08ce0341cc9b354a498388e30986515406668dbcc4f7c950c3e715496693b" + sha256: "178d74305e7866013777bab2c3d8726205dc5a4dd935297175b19a23a2e66571" url: "https://pub.dev" source: hosted - version: "4.0.2" + version: "4.1.2" image_picker: dependency: "direct main" description: @@ -324,26 +316,26 @@ packages: dependency: transitive description: name: image_picker_android - sha256: d3e5e00fdfeca8fd4ffb3227001264d449cc8950414c2ff70b0e06b9c628e643 + sha256: b62d34a506e12bb965e824b6db4fbf709ee4589cf5d3e99b45ab2287b008ee0c url: "https://pub.dev" source: hosted - version: "0.8.12+15" + version: "0.8.12+20" image_picker_for_web: dependency: transitive description: name: image_picker_for_web - sha256: "65d94623e15372c5c51bebbcb820848d7bcb323836e12dfdba60b5d3a8b39e50" + sha256: "717eb042ab08c40767684327be06a5d8dbb341fe791d514e4b92c7bbe1b7bb83" url: "https://pub.dev" source: hosted - version: "3.0.5" + version: "3.0.6" image_picker_ios: dependency: transitive description: name: image_picker_ios - sha256: "6703696ad49f5c3c8356d576d7ace84d1faf459afb07accbb0fae780753ff447" + sha256: "05da758e67bc7839e886b3959848aa6b44ff123ab4b28f67891008afe8ef9100" url: "https://pub.dev" source: hosted - version: "0.8.12" + version: "0.8.12+2" image_picker_linux: dependency: transitive description: @@ -364,10 +356,10 @@ packages: dependency: transitive description: name: image_picker_platform_interface - sha256: "9ec26d410ff46f483c5519c29c02ef0e02e13a543f882b152d4bfd2f06802f80" + sha256: "886d57f0be73c4b140004e78b9f28a8914a09e50c2d816bdd0520051a71236a0" url: "https://pub.dev" source: hosted - version: "2.10.0" + version: "2.10.1" image_picker_windows: dependency: transitive description: @@ -404,10 +396,10 @@ packages: dependency: transitive description: name: lints - sha256: "3315600f3fb3b135be672bf4a178c55f274bebe368325ae18462c89ac1e3b413" + sha256: c35bb79562d980e9a453fc715854e1ed39e24e7d0297a880ef54e17f9874a9d7 url: "https://pub.dev" source: hosted - version: "5.0.0" + version: "5.1.1" matcher: dependency: transitive description: @@ -436,10 +428,10 @@ packages: dependency: transitive description: name: mime - sha256: "801fd0b26f14a4a58ccb09d5892c3fbdeff209594300a542492cf13fba9d247a" + sha256: "41a20518f0cb1256669420fdba0cd90d21561e560ac240f26ef8322e45bb7ed6" url: "https://pub.dev" source: hosted - version: "1.0.6" + version: "2.0.0" path: dependency: "direct main" description: @@ -452,26 +444,26 @@ packages: dependency: "direct main" description: name: path_provider - sha256: fec0d61223fba3154d87759e3cc27fe2c8dc498f6386c6d6fc80d1afdd1bf378 + sha256: "50c5dd5b6e1aaf6fb3a78b33f6aa3afca52bf903a8a5298f53101fdaee55bbcd" url: "https://pub.dev" source: hosted - version: "2.1.4" + version: "2.1.5" path_provider_android: dependency: transitive description: name: path_provider_android - sha256: c464428172cb986b758c6d1724c603097febb8fb855aa265aeecc9280c294d4a + sha256: "4adf4fd5423ec60a29506c76581bc05854c55e3a0b72d35bb28d661c9686edf2" url: "https://pub.dev" source: hosted - version: "2.2.12" + version: "2.2.15" path_provider_foundation: dependency: transitive description: name: path_provider_foundation - sha256: f234384a3fdd67f989b4d54a5d73ca2a6c422fa55ae694381ae0f4375cd1ea16 + sha256: "4843174df4d288f5e29185bd6e72a6fbdf5a4a4602717eed565497429f179942" url: "https://pub.dev" source: hosted - version: "2.4.0" + version: "2.4.1" path_provider_linux: dependency: transitive description: @@ -500,10 +492,10 @@ packages: dependency: transitive description: name: platform - sha256: "9b71283fc13df574056616011fb138fd3b793ea47cc509c189a6c3fa5f8a1a65" + sha256: "5d6b1b0036a5f331ebc77c850ebc8506cbc1e9416c27e59b439f917a902a4984" url: "https://pub.dev" source: hosted - version: "3.1.5" + version: "3.1.6" plugin_platform_interface: dependency: transitive description: @@ -545,10 +537,10 @@ packages: dependency: transitive description: name: stream_transform - sha256: "14a00e794c7c11aa145a170587321aedce29769c08d7f58b1d141da75e3b1c6f" + sha256: ad47125e588cfd37a9a7f86c7d6356dde8dfe89d071d293f80ca9e9273a33871 url: "https://pub.dev" source: hosted - version: "2.1.0" + version: "2.1.1" string_scanner: dependency: transitive description: @@ -577,10 +569,10 @@ packages: dependency: transitive description: name: typed_data - sha256: facc8d6582f16042dd49f2463ff1bd6e2c9ef9f3d5da3d9b087e244a7b564b3c + sha256: f9049c039ebfeb4cf7a7104a675823cd72dba8297f264b6637062516699fa006 url: "https://pub.dev" source: hosted - version: "1.3.2" + version: "1.4.0" vector_math: dependency: transitive description: diff --git a/packages/example/pubspec.yaml b/packages/example/pubspec.yaml index 2e39599a..545248bd 100644 --- a/packages/example/pubspec.yaml +++ b/packages/example/pubspec.yaml @@ -16,8 +16,6 @@ dependencies: flutter_pdfview: ^1.3.3 image_picker: ^1.1.2 camera: ^0.11.0+2 - # The default Android implementation from camera_android_camerax doesn't support the required image format. - camera_android: ^0.10.9+17 path: ^1.9.0 path_provider: ^2.1.4 diff --git a/packages/google_mlkit_commons/android/src/main/java/com/google_mlkit_commons/InputImageConverter.java b/packages/google_mlkit_commons/android/src/main/java/com/google_mlkit_commons/InputImageConverter.java index 33405ab4..2cdd961b 100644 --- a/packages/google_mlkit_commons/android/src/main/java/com/google_mlkit_commons/InputImageConverter.java +++ b/packages/google_mlkit_commons/android/src/main/java/com/google_mlkit_commons/InputImageConverter.java @@ -1,14 +1,20 @@ package com.google_mlkit_commons; import android.content.Context; +import android.graphics.Canvas; import android.graphics.ImageFormat; +import android.graphics.SurfaceTexture; +import android.media.Image; +import android.media.ImageWriter; import android.net.Uri; import android.util.Log; +import android.view.Surface; import com.google.mlkit.vision.common.InputImage; import java.io.File; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Map; import java.util.Objects; @@ -53,9 +59,36 @@ public static InputImage getInputImageFromData(Map imageData, rotationDegrees, imageFormat); } + if (imageFormat == ImageFormat.YUV_420_888) { + // This image format is only supported in InputImage.fromMediaImage, which requires to transform the data to the right java type. + ImageWriter writer = new ImageWriter.Builder(new Surface(new SurfaceTexture(true))).setWidthAndHeight(width, height).setImageFormat(imageFormat).build(); + // TODO(panmari): Does this need any cleanup by calling close() somewhere? + // Currently, this causes some logging like + // A resource failed to call Surface.release + // Calling writer.close() likely will do the trick (after) processing). + Image image = writer.dequeueInputImage(); + if (image == null) { + result.error("InputImageConverterError", "failed to allocate space for input image", null); + return null; + } + // Deconstruct individual planes again from flattened array. + Image.Plane[] planes = image.getPlanes(); + // Y plane + ByteBuffer yBuffer = planes[0].getBuffer(); + yBuffer.put(data, 0, width * height); + + // U plane + ByteBuffer uBuffer = planes[1].getBuffer(); + int uOffset = width * height; + uBuffer.put(data, uOffset, (width * height) / 4); + + // V plane + ByteBuffer vBuffer = planes[2].getBuffer(); + int vOffset = uOffset + (width * height) / 4; + vBuffer.put(data, vOffset, (width * height) / 4); + return InputImage.fromMediaImage(image, rotationDegrees); + } result.error("InputImageConverterError", "ImageFormat is not supported.", null); - // TODO: Use InputImage.fromMediaImage, which supports more types, e.g. IMAGE_FORMAT_YUV_420_888. - // See https://developers.google.com/android/reference/com/google/mlkit/vision/common/InputImage#fromMediaImage(android.media.Image,%20int) return null; } catch (Exception e) { Log.e("ImageError", "Getting Image failed");