[camera] add heif support image iOS (flutter#4586)

Add support to heic format on IOS on take picture.

Based on flutter/flutter#119795

packages/camera/camera_avfoundation/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.9.14
+
+* Adds support to HEIF format.
+
 ## 0.9.13+11
 
 * Fixes a memory leak of sample buffer when pause and resume the video recording.

packages/camera/camera_avfoundation/example/integration_test/camera_test.dart

@@ -319,4 +319,25 @@ void main() {
 
     expect(await completer.future, isNotNull);
   });
+
+  // Test fileFormat is respected when taking a picture.
+  testWidgets('Capture specific image output formats',
+      (WidgetTester tester) async {
+    final List<CameraDescription> cameras =
+        await CameraPlatform.instance.availableCameras();
+    if (cameras.isEmpty) {
+      return;
+    }
+    for (final CameraDescription cameraDescription in cameras) {
+      for (final ImageFileFormat fileFormat in ImageFileFormat.values) {
+        final CameraController controller =
+            CameraController(cameraDescription, ResolutionPreset.low);
+        await controller.initialize();
+        await controller.setImageFileFormat(fileFormat);
+        final XFile file = await controller.takePicture();
+        await controller.dispose();
+        expect(file.path.endsWith(fileFormat.name), true);
+      }
+    }
+  });
 }

packages/camera/camera_avfoundation/example/ios/RunnerTests/CameraPropertiesTests.m

@@ -104,4 +104,12 @@ - (void)testFLTGetStringForUIDeviceOrientation {
   XCTAssertEqualObjects(@"portraitUp", FLTGetStringForUIDeviceOrientation(-1));
 }
 
+#pragma mark - file format tests
+
+- (void)testFLTGetFileFormatForString {
+  XCTAssertEqual(FCPFileFormatJPEG, FCPGetFileFormatFromString(@"jpg"));
+  XCTAssertEqual(FCPFileFormatHEIF, FCPGetFileFormatFromString(@"heif"));
+  XCTAssertEqual(FCPFileFormatInvalid, FCPGetFileFormatFromString(@"unknown"));
+}
+
 @end

packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamPhotoCaptureTests.m

@@ -94,4 +94,90 @@ - (void)testCaptureToFile_mustReportPathToResultIfSavePhotoDelegateCompletionsWi
   [self waitForExpectationsWithTimeout:1 handler:nil];
 }
 
+- (void)testCaptureToFile_mustReportFileExtensionWithHeifWhenHEVCIsAvailableAndFileFormatIsHEIF {
+  XCTestExpectation *expectation =
+      [self expectationWithDescription:
+                @"Test must set extension to heif if availablePhotoCodecTypes contains HEVC."];
+  dispatch_queue_t captureSessionQueue = dispatch_queue_create("capture_session_queue", NULL);
+  dispatch_queue_set_specific(captureSessionQueue, FLTCaptureSessionQueueSpecific,
+                              (void *)FLTCaptureSessionQueueSpecific, NULL);
+  FLTCam *cam = FLTCreateCamWithCaptureSessionQueue(captureSessionQueue);
+  [cam setImageFileFormat:FCPFileFormatHEIF];
+
+  AVCapturePhotoSettings *settings =
+      [AVCapturePhotoSettings photoSettingsWithFormat:@{AVVideoCodecKey : AVVideoCodecTypeHEVC}];
+
+  id mockSettings = OCMClassMock([AVCapturePhotoSettings class]);
+  OCMStub([mockSettings photoSettingsWithFormat:OCMOCK_ANY]).andReturn(settings);
+
+  id mockResult = OCMClassMock([FLTThreadSafeFlutterResult class]);
+  OCMStub([mockResult sendSuccessWithData:OCMOCK_ANY]).andDo(^(NSInvocation *invocation) {
+    NSString *filePath;
+    [invocation getArgument:&filePath atIndex:2];
+    XCTAssertEqualObjects([filePath pathExtension], @"heif");
+    [expectation fulfill];
+  });
+
+  id mockOutput = OCMClassMock([AVCapturePhotoOutput class]);
+  // Set availablePhotoCodecTypes to HEVC
+  NSArray *codecTypes = @[ AVVideoCodecTypeHEVC ];
+  OCMStub([mockOutput availablePhotoCodecTypes]).andReturn(codecTypes);
+
+  OCMStub([mockOutput capturePhotoWithSettings:OCMOCK_ANY delegate:OCMOCK_ANY])
+      .andDo(^(NSInvocation *invocation) {
+        FLTSavePhotoDelegate *delegate = cam.inProgressSavePhotoDelegates[@(settings.uniqueID)];
+        // Completion runs on IO queue.
+        dispatch_queue_t ioQueue = dispatch_queue_create("io_queue", NULL);
+        dispatch_async(ioQueue, ^{
+          delegate.completionHandler(delegate.filePath, nil);
+        });
+      });
+  cam.capturePhotoOutput = mockOutput;
+  // `FLTCam::captureToFile` runs on capture session queue.
+  dispatch_async(captureSessionQueue, ^{
+    [cam captureToFile:mockResult];
+  });
+  [self waitForExpectationsWithTimeout:1 handler:nil];
+}
+
+- (void)testCaptureToFile_mustReportFileExtensionWithJpgWhenHEVCNotAvailableAndFileFormatIsHEIF {
+  XCTestExpectation *expectation = [self
+      expectationWithDescription:
+          @"Test must set extension to jpg if availablePhotoCodecTypes does not contain HEVC."];
+  dispatch_queue_t captureSessionQueue = dispatch_queue_create("capture_session_queue", NULL);
+  dispatch_queue_set_specific(captureSessionQueue, FLTCaptureSessionQueueSpecific,
+                              (void *)FLTCaptureSessionQueueSpecific, NULL);
+  FLTCam *cam = FLTCreateCamWithCaptureSessionQueue(captureSessionQueue);
+  [cam setImageFileFormat:FCPFileFormatHEIF];
+
+  AVCapturePhotoSettings *settings = [AVCapturePhotoSettings photoSettings];
+  id mockSettings = OCMClassMock([AVCapturePhotoSettings class]);
+  OCMStub([mockSettings photoSettings]).andReturn(settings);
+
+  id mockResult = OCMClassMock([FLTThreadSafeFlutterResult class]);
+  OCMStub([mockResult sendSuccessWithData:OCMOCK_ANY]).andDo(^(NSInvocation *invocation) {
+    NSString *filePath;
+    [invocation getArgument:&filePath atIndex:2];
+    XCTAssertEqualObjects([filePath pathExtension], @"jpg");
+    [expectation fulfill];
+  });
+
+  id mockOutput = OCMClassMock([AVCapturePhotoOutput class]);
+
+  OCMStub([mockOutput capturePhotoWithSettings:OCMOCK_ANY delegate:OCMOCK_ANY])
+      .andDo(^(NSInvocation *invocation) {
+        FLTSavePhotoDelegate *delegate = cam.inProgressSavePhotoDelegates[@(settings.uniqueID)];
+        // Completion runs on IO queue.
+        dispatch_queue_t ioQueue = dispatch_queue_create("io_queue", NULL);
+        dispatch_async(ioQueue, ^{
+          delegate.completionHandler(delegate.filePath, nil);
+        });
+      });
+  cam.capturePhotoOutput = mockOutput;
+  // `FLTCam::captureToFile` runs on capture session queue.
+  dispatch_async(captureSessionQueue, ^{
+    [cam captureToFile:mockResult];
+  });
+  [self waitForExpectationsWithTimeout:1 handler:nil];
+}
 @end

packages/camera/camera_avfoundation/example/lib/camera_controller.dart

@@ -437,6 +437,11 @@ class CameraController extends ValueNotifier<CameraValue> {
     value = value.copyWith(focusMode: mode);
   }
 
+  /// Sets the output format for taking pictures.
+  Future<void> setImageFileFormat(ImageFileFormat format) async {
+    await CameraPlatform.instance.setImageFileFormat(_cameraId, format);
+  }
+
   /// Releases the resources of this camera.
   @override
   Future<void> dispose() async {

packages/camera/camera_avfoundation/example/pubspec.yaml

@@ -14,7 +14,7 @@ dependencies:
     # The example app is bundled with the plugin so we use a path dependency on
     # the parent directory to use the current plugin's version.
     path: ../
-  camera_platform_interface: ^2.4.0
+  camera_platform_interface: ^2.7.0
   flutter:
     sdk: flutter
   path_provider: ^2.0.0

packages/camera/camera_avfoundation/ios/Classes/CameraPlugin.m

@@ -153,6 +153,7 @@ - (void)handleMethodCallAsync:(FlutterMethodCall *)call
     NSUInteger cameraId = ((NSNumber *)argsMap[@"cameraId"]).unsignedIntegerValue;
     if ([@"initialize" isEqualToString:call.method]) {
       NSString *videoFormatValue = ((NSString *)argsMap[@"imageFormatGroup"]);
+
       [_camera setVideoFormat:FLTGetVideoFormatFromString(videoFormatValue)];
 
       __weak CameraPlugin *weakSelf = self;
@@ -255,6 +256,9 @@ - (void)handleMethodCallAsync:(FlutterMethodCall *)call
       [_camera resumePreviewWithResult:result];
     } else if ([@"setDescriptionWhileRecording" isEqualToString:call.method]) {
       [_camera setDescriptionWhileRecording:(call.arguments[@"cameraName"]) result:result];
+    } else if ([@"setImageFileFormat" isEqualToString:call.method]) {
+      NSString *fileFormat = call.arguments[@"fileFormat"];
+      [_camera setImageFileFormat:FCPGetFileFormatFromString(fileFormat)];
     } else {
       [result sendNotImplemented];
     }

packages/camera/camera_avfoundation/ios/Classes/CameraProperties.h

@@ -127,4 +127,20 @@ extern FLTResolutionPreset FLTGetFLTResolutionPresetForString(NSString *preset);
  */
 extern OSType FLTGetVideoFormatFromString(NSString *videoFormatString);
 
+/**
+ * Represents image format. Mirrors ImageFileFormat in camera.dart.
+ */
+typedef NS_ENUM(NSInteger, FCPFileFormat) {
+  FCPFileFormatJPEG,
+  FCPFileFormatHEIF,
+  FCPFileFormatInvalid,
+};
+
+#pragma mark - image extension
+
+/**
+ * Gets a string representation of ImageFileFormat.
+ */
+extern FCPFileFormat FCPGetFileFormatFromString(NSString *fileFormatString);
+
 NS_ASSUME_NONNULL_END

packages/camera/camera_avfoundation/ios/Classes/CameraProperties.m

@@ -148,3 +148,15 @@ OSType FLTGetVideoFormatFromString(NSString *videoFormatString) {
     return kCVPixelFormatType_32BGRA;
   }
 }
+
+#pragma mark - file format
+
+FCPFileFormat FCPGetFileFormatFromString(NSString *fileFormatString) {
+  if ([fileFormatString isEqualToString:@"jpg"]) {
+    return FCPFileFormatJPEG;
+  } else if ([fileFormatString isEqualToString:@"heif"]) {
+    return FCPFileFormatHEIF;
+  } else {
+    return FCPFileFormatInvalid;
+  }
+}

packages/camera/camera_avfoundation/ios/Classes/FLTCam.h

@@ -30,6 +30,7 @@ NS_ASSUME_NONNULL_BEGIN
 @property(assign, nonatomic) FLTFlashMode flashMode;
 // Format used for video and image streaming.
 @property(assign, nonatomic) FourCharCode videoFormat;
+@property(assign, nonatomic) FCPFileFormat fileFormat;
 
 /// Initializes an `FLTCam` instance.
 /// @param cameraName a name used to uniquely identify the camera.
@@ -50,6 +51,7 @@ NS_ASSUME_NONNULL_BEGIN
 - (void)captureToFile:(FLTThreadSafeFlutterResult *)result;
 - (void)close;
 - (void)startVideoRecordingWithResult:(FLTThreadSafeFlutterResult *)result;
+- (void)setImageFileFormat:(FCPFileFormat)fileFormat;
 /**
  * Starts recording a video with an optional streaming messenger.
  * If the messenger is non-null then it will be called for each

packages/camera/camera_avfoundation/ios/Classes/FLTCam.m

@@ -144,6 +144,7 @@ - (instancetype)initWithCameraName:(NSString *)cameraName
   _deviceOrientation = orientation;
   _videoFormat = kCVPixelFormatType_32BGRA;
   _inProgressSavePhotoDelegates = [NSMutableDictionary dictionary];
+  _fileFormat = FCPFileFormatJPEG;
 
   // To limit memory consumption, limit the number of frames pending processing.
   // After some testing, 4 was determined to be the best maximum value.
@@ -218,6 +219,10 @@ - (void)setVideoFormat:(OSType)videoFormat {
       @{(NSString *)kCVPixelBufferPixelFormatTypeKey : @(videoFormat)};
 }
 
+- (void)setImageFileFormat:(FCPFileFormat)fileFormat {
+  _fileFormat = fileFormat;
+}
+
 - (void)setDeviceOrientation:(UIDeviceOrientation)orientation {
   if (_deviceOrientation == orientation) {
     return;
@@ -254,16 +259,30 @@ - (void)updateOrientation:(UIDeviceOrientation)orientation
 
 - (void)captureToFile:(FLTThreadSafeFlutterResult *)result {
   AVCapturePhotoSettings *settings = [AVCapturePhotoSettings photoSettings];
+
   if (_resolutionPreset == FLTResolutionPresetMax) {
     [settings setHighResolutionPhotoEnabled:YES];
   }
 
+  NSString *extension;
+
+  BOOL isHEVCCodecAvailable =
+      [self.capturePhotoOutput.availablePhotoCodecTypes containsObject:AVVideoCodecTypeHEVC];
+
+  if (_fileFormat == FCPFileFormatHEIF && isHEVCCodecAvailable) {
+    settings =
+        [AVCapturePhotoSettings photoSettingsWithFormat:@{AVVideoCodecKey : AVVideoCodecTypeHEVC}];
+    extension = @"heif";
+  } else {
+    extension = @"jpg";
+  }
+
   AVCaptureFlashMode avFlashMode = FLTGetAVCaptureFlashModeForFLTFlashMode(_flashMode);
   if (avFlashMode != -1) {
     [settings setFlashMode:avFlashMode];
   }
   NSError *error;
-  NSString *path = [self getTemporaryFilePathWithExtension:@"jpg"
+  NSString *path = [self getTemporaryFilePathWithExtension:extension
                                                  subfolder:@"pictures"
                                                     prefix:@"CAP_"
                                                      error:error];

packages/camera/camera_avfoundation/ios/Classes/FLTSavePhotoDelegate.m

@@ -55,4 +55,7 @@ - (void)captureOutput:(AVCapturePhotoOutput *)output
                         }];
 }
 
+- (NSString *)filePath {
+  return self.path;
+}
 @end

packages/camera/camera_avfoundation/ios/Classes/FLTSavePhotoDelegate_Test.h

@@ -14,6 +14,10 @@
 /// Exposed for unit tests to manually trigger the completion.
 @property(readonly, nonatomic) FLTSavePhotoDelegateCompletionHandler completionHandler;
 
+/// The path for captured photo file.
+/// Exposed for unit tests to verify the captured photo file path.
+@property(readwrite, nonatomic) NSString *filePath;
+
 /// Handler to write captured photo data into a file.
 /// @param error the capture error.
 /// @param photoDataProvider a closure that provides photo data.

packages/camera/camera_avfoundation/lib/src/avfoundation_camera.dart

@@ -522,6 +522,17 @@ class AVFoundationCamera extends CameraPlatform {
     );
   }
 
+  @override
+  Future<void> setImageFileFormat(int cameraId, ImageFileFormat format) {
+    return _channel.invokeMethod<void>(
+      'setImageFileFormat',
+      <String, dynamic>{
+        'cameraId': cameraId,
+        'fileFormat': format.name,
+      },
+    );
+  }
+
   @override
   Widget buildPreview(int cameraId) {
     return Texture(textureId: cameraId);

packages/camera/camera_avfoundation/pubspec.yaml

@@ -2,7 +2,7 @@ name: camera_avfoundation
 description: iOS implementation of the camera plugin.
 repository: https://github.com/flutter/packages/tree/main/packages/camera/camera_avfoundation
 issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22
-version: 0.9.13+11
+version: 0.9.14
 
 environment:
   sdk: ^3.2.3
@@ -17,7 +17,7 @@ flutter:
         dartPluginClass: AVFoundationCamera
 
 dependencies:
-  camera_platform_interface: ^2.4.0
+  camera_platform_interface: ^2.7.0
   flutter:
     sdk: flutter
   stream_transform: ^2.0.0
@@ -29,3 +29,4 @@ dev_dependencies:
 
 topics:
   - camera
+

packages/camera/camera_avfoundation/test/avfoundation_camera_test.dart

@@ -1145,6 +1145,46 @@ void main() {
         isMethodCall('stopImageStream', arguments: null),
       ]);
     });
+
+    test('Should set the ImageFileFormat to heif', () async {
+      // Arrange
+      final MethodChannelMock channel = MethodChannelMock(
+        channelName: _channelName,
+        methods: <String, dynamic>{'setImageFileFormat': 'heif'},
+      );
+
+      // Act
+      await camera.setImageFileFormat(cameraId, ImageFileFormat.heif);
+
+      // Assert
+      expect(channel.log, <Matcher>[
+        isMethodCall('setImageFileFormat', arguments: <String, Object?>{
+          'cameraId': cameraId,
+          'fileFormat': 'heif',
+        }),
+      ]);
+    });
+
+    test('Should set the ImageFileFormat to jpeg', () async {
+      // Arrange
+      final MethodChannelMock channel = MethodChannelMock(
+        channelName: _channelName,
+        methods: <String, dynamic>{
+          'setImageFileFormat': 'jpeg',
+        },
+      );
+
+      // Act
+      await camera.setImageFileFormat(cameraId, ImageFileFormat.jpeg);
+
+      // Assert
+      expect(channel.log, <Matcher>[
+        isMethodCall('setImageFileFormat', arguments: <String, Object?>{
+          'cameraId': cameraId,
+          'fileFormat': 'jpeg',
+        }),
+      ]);
+    });
   });
 }