From f90755e17d39a69a645c298ae4c6f2b3de9d49a4 Mon Sep 17 00:00:00 2001 From: Maurice Parrish Date: Thu, 28 Jun 2018 09:33:53 -0700 Subject: [PATCH] Code cleanup and added metadata for ML Vision (#631) --- FlutterFire.md | 47 ++++++------ README.md | 9 +-- packages/firebase_ml_vision/CHANGELOG.md | 2 +- packages/firebase_ml_vision/LICENSE | 28 +++++++- packages/firebase_ml_vision/README.md | 72 +++++++++++++++++-- .../FirebaseMlVisionPlugin.java | 47 ++++++++---- packages/firebase_ml_vision/example/README.md | 6 +- .../firebase_ml_vision/example/lib/main.dart | 34 +++++---- .../ios/Classes/FirebaseMlVisionPlugin.m | 24 ++++--- .../lib/src/firebase_vision.dart | 7 ++ .../lib/src/text_detector.dart | 48 ++++++++----- packages/firebase_ml_vision/pubspec.yaml | 11 ++- .../test/firebase_ml_vision_test.dart | 69 +++++++++++++++++- 13 files changed, 312 insertions(+), 92 deletions(-) diff --git a/FlutterFire.md b/FlutterFire.md index 5bb3ccc99a90..387ad5de0e27 100644 --- a/FlutterFire.md +++ b/FlutterFire.md @@ -14,18 +14,19 @@ The plugins are still under development, and some APIs might not be available ye | Plugin | Version | Firebase feature | Source code | |---|---|---|---| +| [cloud_firestore][firestore_pub] | ![pub package][firestore_badge] | [Cloud Firestore][firestore_product] | [`packages/cloud_firestore`][firestore_code] | +| [cloud_functions][functions_pub] | ![pub package][functions_badge] | [Cloud Functions][functions_product] | [`packages/cloud_functions`][functions_code] | | [firebase_admob][admob_pub] | ![pub package][admob_badge] | [Firebase AdMob][admob_product] | [`packages/firebase_admob`][admob_code] | | [firebase_analytics][analytics_pub] | ![pub package][analytics_badge] | [Firebase Analytics][analytics_product] | [`packages/firebase_analytics`][analytics_code] | | [firebase_auth][auth_pub] | ![pub package][auth_badge] | [Firebase Authentication][auth_product] | [`packages/firebase_auth`][auth_code] | | [firebase_core][core_pub] | ![pub package][core_badge] | [Firebase Core][core_product] | [`packages/firebase_core`][core_code] | | [firebase_database][database_pub] | ![pub package][database_badge] | [Firebase Realtime Database][database_product] | [`packages/firebase_database`][database_code] | | [firebase_dynamic_links][dynamic_links_pub] | ![pub package][dynamic_links_badge] | [Firebase Dynamic Links][dynamic_links_product] | [`packages/firebase_dynamic_links`][dynamic_links_code] | -| [cloud_firestore][firestore_pub] | ![pub package][firestore_badge] | [Cloud Firestore][firestore_product] | [`packages/cloud_firestore`][firestore_code] | | [firebase_messaging][messaging_pub] | ![pub package][messaging_badge] | [Firebase Cloud Messaging][messaging_product] | [`packages/firebase_messaging`][messaging_code] | -| [firebase_storage][storage_pub] | ![pub package][storage_badge] | [Firebase Cloud Storage][storage_product] | [`packages/firebase_storage`][storage_code] | +| [firebase_ml_vision][ml_vision_pub] | ![pub package][ml_vision_badge] | [Firebase ML Kit][ml_vision_product] | [`packages/firebase_ml_vision`][ml_vision_code] | | [firebase_performance][performance_pub] | ![pub package][performance_badge] | [Firebase Performance Monitoring][performance_product] | [`packages/firebase_performance`][performance_code] | | [firebase_remote_config][remote_config_pub] | ![pub package][remote_config_badge] | [Firebase Remote Config][remote_config_product] | [`packages/firebase_remote_config`][remote_config_code] | -| [cloud_functions][functions_pub] | ![pub package][functions_badge] | [Cloud Functions][functions_product] | [`packages/cloud_functions`][functions_code] | +| [firebase_storage][storage_pub] | ![pub package][storage_badge] | [Firebase Cloud Storage][storage_product] | [`packages/firebase_storage`][storage_code] | [admob_pub]: https://pub.dartlang.org/packages/firebase_admob [admob_product]: https://firebase.google.com/docs/admob/ @@ -42,11 +43,6 @@ The plugins are still under development, and some APIs might not be available ye [auth_code]: https://github.com/flutter/plugins/tree/master/packages/firebase_auth [auth_badge]: https://img.shields.io/pub/v/firebase_auth.svg -[firestore_pub]: https://pub.dartlang.org/packages/cloud_firestore -[firestore_product]: https://firebase.google.com/products/firestore/ -[firestore_code]: https://github.com/flutter/plugins/tree/master/packages/cloud_firestore -[firestore_badge]: https://img.shields.io/pub/v/cloud_firestore.svg - [core_pub]: https://pub.dartlang.org/packages/firebase_core [core_product]: https://firebase.google.com/ [core_code]: https://github.com/flutter/plugins/tree/master/packages/firebase_core @@ -57,15 +53,30 @@ The plugins are still under development, and some APIs might not be available ye [database_code]: https://github.com/flutter/plugins/tree/master/packages/firebase_database [database_badge]: https://img.shields.io/pub/v/firebase_database.svg +[dynamic_links_pub]: https://pub.dartlang.org/packages/firebase_dynamic_links +[dynamic_links_product]: https://firebase.google.com/products/dynamic-links/ +[dynamic_links_code]: https://github.com/flutter/plugins/tree/master/packages/firebase_dynamic_links +[dynamic_links_badge]: https://img.shields.io/pub/v/firebase_dynamic_links.svg + +[firestore_pub]: https://pub.dartlang.org/packages/cloud_firestore +[firestore_product]: https://firebase.google.com/products/firestore/ +[firestore_code]: https://github.com/flutter/plugins/tree/master/packages/cloud_firestore +[firestore_badge]: https://img.shields.io/pub/v/cloud_firestore.svg + +[functions_pub]: https://pub.dartlang.org/packages/cloud_functions +[functions_product]: https://firebase.google.com/products/functions/ +[functions_code]: https://github.com/flutter/plugins/tree/master/packages/cloud_functions +[functions_badge]: https://img.shields.io/pub/v/cloud_functions.svg + [messaging_pub]: https://pub.dartlang.org/packages/firebase_messaging [messaging_product]: https://firebase.google.com/products/cloud-messaging/ [messaging_code]: https://github.com/flutter/plugins/tree/master/packages/firebase_messaging [messaging_badge]: https://img.shields.io/pub/v/firebase_messaging.svg -[storage_pub]: https://pub.dartlang.org/packages/firebase_storage -[storage_product]: https://firebase.google.com/products/storage/ -[storage_code]: https://github.com/flutter/plugins/tree/master/packages/firebase_storage -[storage_badge]: https://img.shields.io/pub/v/firebase_storage.svg +[ml_vision_pub]: https://pub.dartlang.org/packages/firebase_ml_vision +[ml_vision_product]: https://firebase.google.com/products/ml-kit/ +[ml_vision_code]: https://github.com/flutter/plugins/tree/master/packages/firebase_ml_vision +[ml_vision_badge]: https://img.shields.io/pub/v/firebase_ml_vision.svg [performance_pub]: https://pub.dartlang.org/packages/firebase_performance [performance_product]: https://firebase.google.com/products/performance/ @@ -77,12 +88,8 @@ The plugins are still under development, and some APIs might not be available ye [remote_config_code]: https://github.com/flutter/plugins/tree/master/packages/firebase_remote_config [remote_config_badge]: https://img.shields.io/pub/v/firebase_remote_config.svg -[dynamic_links_pub]: https://pub.dartlang.org/packages/firebase_dynamic_links -[dynamic_links_product]: https://firebase.google.com/products/dynamic-links/ -[dynamic_links_code]: https://github.com/flutter/plugins/tree/master/packages/firebase_dynamic_links -[dynamic_links_badge]: https://img.shields.io/pub/v/firebase_dynamic_links.svg +[storage_pub]: https://pub.dartlang.org/packages/firebase_storage +[storage_product]: https://firebase.google.com/products/storage/ +[storage_code]: https://github.com/flutter/plugins/tree/master/packages/firebase_storage +[storage_badge]: https://img.shields.io/pub/v/firebase_storage.svg -[functions_pub]: https://pub.dartlang.org/packages/cloud_functions -[functions_product]: https://firebase.google.com/products/functions/ -[functions_code]: https://github.com/flutter/plugins/tree/master/packages/cloud_functions -[functions_badge]: https://img.shields.io/pub/v/cloud_functions.svg diff --git a/README.md b/README.md index fe03e6c9a38e..92d79c959759 100644 --- a/README.md +++ b/README.md @@ -58,17 +58,18 @@ These are the available plugins in this repository. | [video_player](./packages/video_player/) | [![pub package](https://img.shields.io/pub/v/video_player.svg)](https://pub.dartlang.org/packages/video_player) | | | | | **FlutterFire Plugins** | | +| [cloud_firestore](./packages/cloud_firestore/) | [![pub package](https://img.shields.io/pub/v/cloud_firestore.svg)](https://pub.dartlang.org/packages/cloud_firestore) +| [cloud_functions](./packages/cloud_functions/) | [![pub package](https://img.shields.io/pub/v/cloud_functions.svg)](https://pub.dartlang.org/packages/cloud_functions) | | [firebase_admob](./packages/firebase_admob/) | [![pub package](https://img.shields.io/pub/v/firebase_admob.svg)](https://pub.dartlang.org/packages/firebase_admob) | | [firebase_analytics](./packages/firebase_analytics/) | [![pub package](https://img.shields.io/pub/v/firebase_analytics.svg)](https://pub.dartlang.org/packages/firebase_analytics) | | [firebase_auth](./packages/firebase_auth/) | [![pub package](https://img.shields.io/pub/v/firebase_auth.svg)](https://pub.dartlang.org/packages/firebase_auth) | -| [cloud_firestore](./packages/cloud_firestore/) | [![pub package](https://img.shields.io/pub/v/cloud_firestore.svg)](https://pub.dartlang.org/packages/cloud_firestore) | [firebase_core](./packages/firebase_core/) | [![pub package](https://img.shields.io/pub/v/firebase_core.svg)](https://pub.dartlang.org/packages/firebase_core) | | [firebase_database](./packages/firebase_database/) | [![pub package](https://img.shields.io/pub/v/firebase_database.svg)](https://pub.dartlang.org/packages/firebase_database) | +| [firebase_dynamic_links](./packages/firebase_dynamic_links/) | [![pub package](https://img.shields.io/pub/v/firebase_dynamic_links.svg)](https://pub.dartlang.org/packages/firebase_dynamic_links) | | [firebase_messaging](./packages/firebase_messaging/) | [![pub package](https://img.shields.io/pub/v/firebase_messaging.svg)](https://pub.dartlang.org/packages/firebase_messaging) | -| [firebase_storage](./packages/firebase_storage/) | [![pub package](https://img.shields.io/pub/v/firebase_storage.svg)](https://pub.dartlang.org/packages/firebase_storage) | +| [firebase_ml_vision](./packages/firebase_ml_vision/) | [![pub package](https://img.shields.io/pub/v/firebase_ml_vision.svg)](https://pub.dartlang.org/packages/firebase_ml_vision) | | [firebase_performance](./packages/firebase_performance/) | [![pub package](https://img.shields.io/pub/v/firebase_performance.svg)](https://pub.dartlang.org/packages/firebase_performance) | | [firebase_remote_config](./packages/firebase_remote_config/) | [![pub package](https://img.shields.io/pub/v/firebase_remote_config.svg)](https://pub.dartlang.org/packages/firebase_remote_config) | -| [firebase_dynamic_links](./packages/firebase_dynamic_links/) | [![pub package](https://img.shields.io/pub/v/firebase_dynamic_links.svg)](https://pub.dartlang.org/packages/firebase_dynamic_links) | -| [cloud_functions](./packages/cloud_functions/) | [![pub package](https://img.shields.io/pub/v/cloud_functions.svg)](https://pub.dartlang.org/packages/cloud_functions) | +| [firebase_storage](./packages/firebase_storage/) | [![pub package](https://img.shields.io/pub/v/firebase_storage.svg)](https://pub.dartlang.org/packages/firebase_storage) | Learn more about [FlutterFire](https://github.com/flutter/plugins/blob/master/FlutterFire.md). diff --git a/packages/firebase_ml_vision/CHANGELOG.md b/packages/firebase_ml_vision/CHANGELOG.md index 41cc7d8192ec..57dbea109e21 100644 --- a/packages/firebase_ml_vision/CHANGELOG.md +++ b/packages/firebase_ml_vision/CHANGELOG.md @@ -1,3 +1,3 @@ ## 0.0.1 -* TODO: Describe initial release. +* Initial release with text detector. diff --git a/packages/firebase_ml_vision/LICENSE b/packages/firebase_ml_vision/LICENSE index ba75c69f7f21..8940a4be1b58 100644 --- a/packages/firebase_ml_vision/LICENSE +++ b/packages/firebase_ml_vision/LICENSE @@ -1 +1,27 @@ -TODO: Add your license here. +// Copyright 2018 The Chromium Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/packages/firebase_ml_vision/README.md b/packages/firebase_ml_vision/README.md index 8f088e9ea1d9..be6b6f08c737 100644 --- a/packages/firebase_ml_vision/README.md +++ b/packages/firebase_ml_vision/README.md @@ -1,10 +1,70 @@ -# firebase_ml_vision +# Google ML Kit for Firebase -A new Flutter plugin. +[![pub package](https://img.shields.io/pub/v/firebase_ml_vision.svg)](https://pub.dartlang.org/packages/firebase_ml_vision) -## Getting Started +A Flutter plugin to use the [Google ML Kit for Firebase API](https://firebase.google.com/docs/ml-kit/). + +For Flutter plugins for other Firebase products, see [FlutterFire.md](https://github.com/flutter/plugins/blob/master/FlutterFire.md). + +*Note*: This plugin is still under development, and some APIs might not be available yet. [Feedback](https://github.com/flutter/flutter/issues) and [Pull Requests](https://github.com/flutter/plugins/pulls) are most welcome! + +## Usage + +To use this plugin, add `firebase_ml_vision` as a [dependency in your pubspec.yaml file](https://flutter.io/platform-plugins/). You must also configure Firebase for each platform project: Android and iOS (see the example folder or https://codelabs.developers.google.com/codelabs/flutter-firebase/#4 for step by step details). + +### Android +Optional but recommended: If you use the on-device API, configure your app to automatically download the ML model to the device after your app is installed from the Play Store. To do so, add the following declaration to your app's AndroidManifest.xml file: + +```manifest + + ... + + + +``` + +## On-device Text Recognition + +To use the on-device text recognition model, run the text detector as described below: + +1. Create a `FirebaseVisionImage` object from your image. -For help getting started with Flutter, view our online -[documentation](https://flutter.io/). +To create a `FirebaseVisionImage` from an image `File` object: + +```dart +final File imageFile = getImageFile(); +final FirebaseVisionImage visionImage = FirebaseVisionImage.fromFile(imageFile); +``` + +2. Get an instance of `TextDetector` and pass `visionImage` to `detectInImage().` + +```dart +final TextDetector detector = FirebaseVision.instance.getTextDetector(); +final List blocks = await detector.detectInImage(visionImage); + +detector.close(); +``` + +3. Extract text and text locations from blocks of recognized text. + +```dart +for (TextBlock block in textLocations) { + final Rectangle boundingBox = block.boundingBox; + final List> cornerPoints = block.cornerPoints; + final String text = block.text; + + for (TextLine line in block.lines) { + // ... + + for (TextElement element in line.elements) { + // ... + } + } +} +``` + +## Getting Started -For help on editing plugin code, view the [documentation](https://flutter.io/platform-plugins/#edit-code). \ No newline at end of file +See the `example` directory for a complete sample app using Google ML Kit for Firebase. diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java index bb71e8f41f04..5a58c7ba5a6c 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FirebaseMlVisionPlugin.java @@ -25,6 +25,7 @@ /** FirebaseMlVisionPlugin */ public class FirebaseMlVisionPlugin implements MethodCallHandler { private Registrar registrar; + private FirebaseVisionTextDetector textDetector; private FirebaseMlVisionPlugin(Registrar registrar) { this.registrar = registrar; @@ -39,10 +40,24 @@ public static void registerWith(Registrar registrar) { @Override public void onMethodCall(MethodCall call, Result result) { - if (call.method.equals("TextDetector#detectInImage")) { - handleTextDetectionResult(call, result); - } else { - result.notImplemented(); + switch (call.method) { + case "TextDetector#detectInImage": + handleTextDetectionResult(call, result); + break; + case "TextDetector#close": + if (textDetector != null) { + try { + textDetector.close(); + result.success(null); + } catch (IOException exception) { + result.error("textDetectorError", exception.getLocalizedMessage(), null); + } + + textDetector = null; + } + break; + default: + result.notImplemented(); } } @@ -57,8 +72,8 @@ private void handleTextDetectionResult(MethodCall call, final Result result) { return; } - FirebaseVisionTextDetector detector = FirebaseVision.getInstance().getVisionTextDetector(); - detector + if (textDetector == null) textDetector = FirebaseVision.getInstance().getVisionTextDetector(); + textDetector .detectInImage(image) .addOnSuccessListener( new OnSuccessListener() { @@ -98,8 +113,8 @@ public void onSuccess(FirebaseVisionText firebaseVisionText) { .addOnFailureListener( new OnFailureListener() { @Override - public void onFailure(@NonNull Exception e) { - result.error("textDetectorError", e.getLocalizedMessage(), null); + public void onFailure(@NonNull Exception exception) { + result.error("textDetectorError", exception.getLocalizedMessage(), null); } }); } @@ -108,14 +123,18 @@ private void addTextData( Map addTo, Rect boundingBox, Point[] cornerPoints, String text) { addTo.put("text", text); - addTo.put("left", boundingBox.left); - addTo.put("top", boundingBox.top); - addTo.put("width", boundingBox.width()); - addTo.put("height", boundingBox.height()); + if (boundingBox != null) { + addTo.put("left", boundingBox.left); + addTo.put("top", boundingBox.top); + addTo.put("width", boundingBox.width()); + addTo.put("height", boundingBox.height()); + } List points = new ArrayList<>(); - for (Point point : cornerPoints) { - points.add(new int[] {point.x, point.y}); + if (cornerPoints != null) { + for (Point point : cornerPoints) { + points.add(new int[] {point.x, point.y}); + } } addTo.put("points", points); } diff --git a/packages/firebase_ml_vision/example/README.md b/packages/firebase_ml_vision/example/README.md index fcb429d16af2..13f4934d76ff 100644 --- a/packages/firebase_ml_vision/example/README.md +++ b/packages/firebase_ml_vision/example/README.md @@ -2,7 +2,11 @@ Demonstrates how to use the firebase_ml_vision plugin. +## Usage + +This example uses the *image_picker* plugin to get images from the device gallery. If using an iOS device you will have to configure you project with the correct permissions seen under iOS configuration [here.](https://pub.dartlang.org/packages/image_picker) + ## Getting Started For help getting started with Flutter, view our online -[documentation](https://flutter.io/). +[documentation.](https://flutter.io/) diff --git a/packages/firebase_ml_vision/example/lib/main.dart b/packages/firebase_ml_vision/example/lib/main.dart index f734813bc2e1..f5c002851426 100644 --- a/packages/firebase_ml_vision/example/lib/main.dart +++ b/packages/firebase_ml_vision/example/lib/main.dart @@ -42,24 +42,28 @@ class _MyHomePageState extends State<_MyHomePage> { final TextDetector detector = FirebaseVision.instance.getTextDetector(); final List blocks = await detector.detectInImage(visionImage); - final Image image = Image.file(_imageFile); - final Size imageSize = await _getImageSize(image); - setState(() { _textLocations = blocks; - _imageSize = imageSize; }); + + detector.close(); } - Future _getImageSize(Image image) { + Future _getImageSize(Image image) async { final Completer completer = new Completer(); - image.image - .resolve(const ImageConfiguration()) - .addListener((ImageInfo info, bool _) => completer.complete(Size( - info.image.width.toDouble(), - info.image.height.toDouble(), - ))); - return completer.future; + image.image.resolve(const ImageConfiguration()).addListener( + (ImageInfo info, bool _) { + completer.complete(Size( + info.image.width.toDouble(), + info.image.height.toDouble(), + )); + }, + ); + + final Size imageSize = await completer.future; + setState(() { + _imageSize = imageSize; + }); } Widget _buildImage() { @@ -98,7 +102,10 @@ class _MyHomePageState extends State<_MyHomePage> { floatingActionButton: new FloatingActionButton( onPressed: () async { await _getImage(); - if (_imageFile != null) _scanImage(); + if (_imageFile != null) { + _getImageSize(Image.file(_imageFile)); + _scanImage(); + } }, tooltip: 'Pick Image', child: const Icon(Icons.add_a_photo), @@ -107,6 +114,7 @@ class _MyHomePageState extends State<_MyHomePage> { } } +// Paints rectangles around all the text in the image. class ScannedTextPainter extends CustomPainter { ScannedTextPainter(this.absoluteImageSize, this.textLocations); diff --git a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m index d7168dff9f4f..e227aab5fd79 100644 --- a/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m +++ b/packages/firebase_ml_vision/ios/Classes/FirebaseMlVisionPlugin.m @@ -38,23 +38,25 @@ - (instancetype)init { } - (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result { - NSString *filePath = call.arguments; - UIImage *image = [UIImage imageWithContentsOfFile:filePath]; - FIRVisionImage *visionImage = [[FIRVisionImage alloc] initWithImage:image]; - if ([@"TextDetector#detectInImage" isEqualToString:call.method]) { - [self handleTextDetectionResult:visionImage result:result]; + [self handleTextDetectionResult:call result:result]; + } else if ([@"TextDetector#close" isEqualToString:call.method]) { + result(_textDetector = nil); } else { result(FlutterMethodNotImplemented); } } -- (void)handleTextDetectionResult:(FIRVisionImage *)image result:(FlutterResult)result { +- (void)handleTextDetectionResult:(FlutterMethodCall *)call result:(FlutterResult)result { + NSString *filePath = call.arguments; + UIImage *image = [UIImage imageWithContentsOfFile:filePath]; + FIRVisionImage *visionImage = [[FIRVisionImage alloc] initWithImage:image]; + FIRVision *vision = [FIRVision vision]; if (_textDetector == nil) _textDetector = [vision textDetector]; [_textDetector - detectInImage:image + detectInImage:visionImage completion:^(NSArray> *_Nullable features, NSError *_Nullable error) { if (error) { result([error flutterError]); @@ -121,10 +123,10 @@ - (NSDictionary *)getTextData:(CGRect)frame return @{ @"text" : text, - @"left" : @(frame.origin.x), - @"top" : @(frame.origin.y), - @"width" : @(frame.size.width), - @"height" : @(frame.size.height), + @"left" : @((int)frame.origin.x), + @"top" : @((int)frame.origin.y), + @"width" : @((int)frame.size.width), + @"height" : @((int)frame.size.height), @"points" : points, }; } diff --git a/packages/firebase_ml_vision/lib/src/firebase_vision.dart b/packages/firebase_ml_vision/lib/src/firebase_vision.dart index b15d9c1a04bf..1e45d4f9c0f2 100644 --- a/packages/firebase_ml_vision/lib/src/firebase_vision.dart +++ b/packages/firebase_ml_vision/lib/src/firebase_vision.dart @@ -38,18 +38,25 @@ class FirebaseVision { class FirebaseVisionImage { FirebaseVisionImage._(this.imageFile); + /// Construct a [FirebaseVisionImage] from a file. factory FirebaseVisionImage.fromFile(File imageFile) { return FirebaseVisionImage._(imageFile); } + /// Construct a [FirebaseVisionImage] from a file path. factory FirebaseVisionImage.fromFilePath(String imagePath) { return FirebaseVisionImage._(new File(imagePath)); } + /// The file location of the image. final File imageFile; } /// Abstract class for detectors in [FirebaseVision] API. abstract class FirebaseVisionDetector { + /// Uses machine learning model to detect objects of interest in an image. Future detectInImage(FirebaseVisionImage visionImage); + + /// Release model resources for the detector. + Future close(); } diff --git a/packages/firebase_ml_vision/lib/src/text_detector.dart b/packages/firebase_ml_vision/lib/src/text_detector.dart index 4df383273e25..c6b37d53dc23 100644 --- a/packages/firebase_ml_vision/lib/src/text_detector.dart +++ b/packages/firebase_ml_vision/lib/src/text_detector.dart @@ -31,39 +31,53 @@ class TextDetector implements FirebaseVisionDetector { return blocks; } + + /// Closes the text detector and release its model resources. + @override + Future close() { + return FirebaseVision.channel.invokeMethod('TextDetector#close'); + } } /// Abstract class representing dimensions of recognized text in an image. abstract class TextContainer { TextContainer._(Map data) - : boundingBox = Rectangle( - data['left'], - data['top'], - data['width'], - data['height'], - ), - cornerPoints = data['points'] - .map>((dynamic item) => Point( + : boundingBox = data['left'] != null + ? Rectangle( + data['left'], + data['top'], + data['width'], + data['height'], + ) + : null, + _cornerPoints = data['points'] + .map>((dynamic item) => Point( item[0], item[1], )) .toList(), text = data['text']; - /// Axis-aligned bounding rectangle of the detected text. - final Rectangle boundingBox; + final List> _cornerPoints; - /// The four corner points in clockwise direction starting with top-left. + /// Axis-aligned bounding rectangle of the detected text. /// - /// Due to the possible perspective distortions, this is not necessarily a - /// rectangle. Parts of the region could be outside of the image. - final List> cornerPoints; + /// Could be null even if text is found. + final Rectangle boundingBox; /// The recognized text as a string. /// /// Returned in reading order for the language. For Latin, this is top to /// bottom within a Block, and left-to-right within a Line. final String text; + + /// The four corner points in clockwise direction starting with top-left. + /// + /// Due to the possible perspective distortions, this is not necessarily a + /// rectangle. Parts of the region could be outside of the image. + /// + /// Could be empty even if text is found. + List> get cornerPoints => List>.from(_cornerPoints); } /// A block of text (think of it as a paragraph) as deemed by the OCR engine. @@ -76,6 +90,7 @@ class TextBlock extends TextContainer { final List _lines; + /// The contents of the text block, broken down into individual lines. List get lines => List.from(_lines); } @@ -89,13 +104,14 @@ class TextLine extends TextContainer { final List _elements; + /// The contents of this line, broken down into individual elements. List get elements => List.from(_elements); } /// Roughly equivalent to a space-separated "word." /// -/// Separates elements into words in most Latin languages, but could separate -/// by characters in others. +/// The API separates elements into words in most Latin languages, but could +/// separate by characters in others. /// /// If a word is split between two lines by a hyphen, each part is encoded as a /// separate element. diff --git a/packages/firebase_ml_vision/pubspec.yaml b/packages/firebase_ml_vision/pubspec.yaml index d6613aaf62ce..66cadd06c2e4 100644 --- a/packages/firebase_ml_vision/pubspec.yaml +++ b/packages/firebase_ml_vision/pubspec.yaml @@ -1,8 +1,9 @@ name: firebase_ml_vision -description: A new Flutter plugin. +description: Flutter plugin for Google ML Vision for Firebase, an SDK that brings Google's machine + learning expertise to Android and iOS apps in a powerful yet easy-to-use package. version: 0.0.1 -author: -homepage: +author: Flutter Team +homepage: https://github.com/flutter/plugins/tree/master/packages/firebase_ml_vision dependencies: flutter: @@ -17,3 +18,7 @@ flutter: androidPackage: io.flutter.plugins.firebasemlvision iosPrefix: FLT pluginClass: FirebaseMlVisionPlugin + +environment: + sdk: ">=2.0.0-dev.28.0 <3.0.0" + flutter: ">=0.2.4 <2.0.0" diff --git a/packages/firebase_ml_vision/test/firebase_ml_vision_test.dart b/packages/firebase_ml_vision/test/firebase_ml_vision_test.dart index 486154a622a6..8a2acb12c643 100644 --- a/packages/firebase_ml_vision/test/firebase_ml_vision_test.dart +++ b/packages/firebase_ml_vision/test/firebase_ml_vision_test.dart @@ -50,8 +50,8 @@ void main() { 'width': 7, 'height': 8, 'points': [ - [9.0, 10.0], - [11.0, 12.0], + [9, 10], + [11, 12], ], 'elements': [ textElement, @@ -114,6 +114,71 @@ void main() { const Point(7, 8), ]); }); + + test('detectInImage no blocks', () async { + returnValue = []; + + final TextDetector detector = FirebaseVision.instance.getTextDetector(); + final FirebaseVisionImage image = + new FirebaseVisionImage.fromFilePath('empty'); + + final List blocks = await detector.detectInImage(image); + + expect(log, [ + isMethodCall( + 'TextDetector#detectInImage', + arguments: 'empty', + ), + ]); + + expect(blocks, isEmpty); + }); + + test('close', () async { + final TextDetector detector = FirebaseVision.instance.getTextDetector(); + await detector.close(); + + expect(log, [ + isMethodCall( + 'TextDetector#close', + arguments: null, + ), + ]); + }); + + test('detectInImage no bounding box', () async { + returnValue = [ + { + 'text': 'potato', + 'points': [ + [17, 18], + [19, 20], + ], + 'lines': [], + }, + ]; + + final TextDetector detector = FirebaseVision.instance.getTextDetector(); + final FirebaseVisionImage image = + new FirebaseVisionImage.fromFilePath('empty'); + + final List blocks = await detector.detectInImage(image); + + expect(log, [ + isMethodCall( + 'TextDetector#detectInImage', + arguments: 'empty', + ), + ]); + + final TextBlock block = blocks[0]; + expect(block.boundingBox, null); + expect(block.text, 'potato'); + expect(block.cornerPoints, >[ + const Point(17, 18), + const Point(19, 20), + ]); + }); }); }); }