[camera_avfoundation] enable more than 30 fps

packages/camera/camera_avfoundation/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.9.17+5
+
+* Adds ability to use any supported FPS and fixes crash when using unsupported FPS.
+
 ## 0.9.17+4
 
 * Updates Pigeon for non-nullable collection type support.
@@ -13,7 +17,7 @@
 
 ## 0.9.17+1
 
-* Fixes a crash due to appending sample buffers when readyForMoreMediaData is NO
+* Fixes a crash due to appending sample buffers when readyForMoreMediaData is NO.
 
 ## 0.9.17
 

packages/camera/camera_avfoundation/example/ios/RunnerTests/CameraSettingsTests.m

@@ -202,4 +202,20 @@ - (void)testSettings_ShouldBeSupportedByMethodCall {
   XCTAssertNotNil(resultValue);
 }
 
+- (void)testSettings_ShouldSelectFormatWhichSupports60FPS {
+  FCPPlatformMediaSettings *settings =
+      [FCPPlatformMediaSettings makeWithResolutionPreset:gTestResolutionPreset
+                                         framesPerSecond:@(60)
+                                            videoBitrate:@(gTestVideoBitrate)
+                                            audioBitrate:@(gTestAudioBitrate)
+                                             enableAudio:gTestEnableAudio];
+
+  FLTCam *camera = FLTCreateCamWithCaptureSessionQueueAndMediaSettings(
+      dispatch_queue_create("test", NULL), settings, nil, nil);
+
+  AVFrameRateRange *range = camera.captureDevice.activeFormat.videoSupportedFrameRateRanges[0];
+  XCTAssertLessThanOrEqual(range.minFrameRate, 60);
+  XCTAssertGreaterThanOrEqual(range.maxFrameRate, 60);
+}
+
 @end

packages/camera/camera_avfoundation/example/ios/RunnerTests/CameraTestUtils.m

@@ -52,14 +52,44 @@
   OCMStub([audioSessionMock addInputWithNoConnections:[OCMArg any]]);
   OCMStub([audioSessionMock canSetSessionPreset:[OCMArg any]]).andReturn(YES);
 
+  id frameRateRangeMock1 = OCMClassMock([AVFrameRateRange class]);
+  OCMStub([frameRateRangeMock1 minFrameRate]).andReturn(3);
+  OCMStub([frameRateRangeMock1 maxFrameRate]).andReturn(30);
+  id captureDeviceFormatMock1 = OCMClassMock([AVCaptureDeviceFormat class]);
+  OCMStub([captureDeviceFormatMock1 videoSupportedFrameRateRanges]).andReturn(@[
+    frameRateRangeMock1
+  ]);
+
+  id frameRateRangeMock2 = OCMClassMock([AVFrameRateRange class]);
+  OCMStub([frameRateRangeMock2 minFrameRate]).andReturn(3);
+  OCMStub([frameRateRangeMock2 maxFrameRate]).andReturn(60);
+  id captureDeviceFormatMock2 = OCMClassMock([AVCaptureDeviceFormat class]);
+  OCMStub([captureDeviceFormatMock2 videoSupportedFrameRateRanges]).andReturn(@[
+    frameRateRangeMock2
+  ]);
+
+  id captureDeviceMock = OCMClassMock([AVCaptureDevice class]);
+  OCMStub([captureDeviceMock lockForConfiguration:[OCMArg setTo:nil]]).andReturn(YES);
+  OCMStub([captureDeviceMock formats]).andReturn((@[
+    captureDeviceFormatMock1, captureDeviceFormatMock2
+  ]));
+  __block AVCaptureDeviceFormat *format = captureDeviceFormatMock1;
+  OCMStub([captureDeviceMock setActiveFormat:[OCMArg any]]).andDo(^(NSInvocation *invocation) {
+    [invocation retainArguments];
+    [invocation getArgument:&format atIndex:2];
+  });
+  OCMStub([captureDeviceMock activeFormat]).andDo(^(NSInvocation *invocation) {
+    [invocation setReturnValue:&format];
+  });
+
   id fltCam = [[FLTCam alloc] initWithMediaSettings:mediaSettings
                              mediaSettingsAVWrapper:mediaSettingsAVWrapper
                              orientation:UIDeviceOrientationPortrait
                              videoCaptureSession:videoSessionMock
                              audioCaptureSession:audioSessionMock
                              captureSessionQueue:captureSessionQueue
                                captureDeviceFactory:captureDeviceFactory ?: ^AVCaptureDevice *(void) {
-                               return [AVCaptureDevice deviceWithUniqueID:@"camera"];
+                               return captureDeviceMock;
                              }
                              videoDimensionsForFormat:^CMVideoDimensions(AVCaptureDeviceFormat *format) {
                                return CMVideoFormatDescriptionGetDimensions(format.formatDescription);

packages/camera/camera_avfoundation/ios/camera_avfoundation/Sources/camera_avfoundation/FLTCam.m

@@ -153,6 +153,59 @@ - (instancetype)initWithCameraName:(NSString *)cameraName
       error:error];
 }
 
+// Returns frame rate supported by format closest to targetFrameRate.
+static double bestFrameRateForFormat(AVCaptureDeviceFormat *format, double targetFrameRate) {
+  double bestFrameRate = 0;
+  double minDistance = DBL_MAX;
+  for (AVFrameRateRange *range in format.videoSupportedFrameRateRanges) {
+    double frameRate = MIN(MAX(targetFrameRate, range.minFrameRate), range.maxFrameRate);
+    double distance = fabs(frameRate - targetFrameRate);
+    if (distance < minDistance) {
+      bestFrameRate = frameRate;
+      minDistance = distance;
+    }
+  }
+  return bestFrameRate;
+}
+
+// Finds format with same resolution as current activeFormat in captureDevice for which
+// bestFrameRateForFormat returned frame rate closest to mediaSettings.framesPerSecond.
+// Preferred are formats with the same subtype as current activeFormat. Sets this format
+// as activeFormat and also updates mediaSettings.framesPerSecond to value which
+// bestFrameRateForFormat returned for that format.
+static void selectBestFormatForRequestedFrameRate(
+    AVCaptureDevice *captureDevice, FCPPlatformMediaSettings *mediaSettings,
+    VideoDimensionsForFormat videoDimensionsForFormat) {
+  CMVideoDimensions targetResolution = videoDimensionsForFormat(captureDevice.activeFormat);
+  double targetFrameRate = mediaSettings.framesPerSecond.doubleValue;
+  FourCharCode preferredSubType =
+      CMFormatDescriptionGetMediaSubType(captureDevice.activeFormat.formatDescription);
+  AVCaptureDeviceFormat *bestFormat = captureDevice.activeFormat;
+  double bestFrameRate = bestFrameRateForFormat(bestFormat, targetFrameRate);
+  double minDistance = fabs(bestFrameRate - targetFrameRate);
+  BOOL isBestSubTypePreferred = YES;
+  for (AVCaptureDeviceFormat *format in captureDevice.formats) {
+    CMVideoDimensions resolution = videoDimensionsForFormat(format);
+    if (resolution.width != targetResolution.width ||
+        resolution.height != targetResolution.height) {
+      continue;
+    }
+    double frameRate = bestFrameRateForFormat(format, targetFrameRate);
+    double distance = fabs(frameRate - targetFrameRate);
+    FourCharCode subType = CMFormatDescriptionGetMediaSubType(format.formatDescription);
+    BOOL isSubTypePreferred = subType == preferredSubType;
+    if (distance < minDistance ||
+        (distance == minDistance && isSubTypePreferred && !isBestSubTypePreferred)) {
+      bestFormat = format;
+      bestFrameRate = frameRate;
+      minDistance = distance;
+      isBestSubTypePreferred = isSubTypePreferred;
+    }
+  }
+  captureDevice.activeFormat = bestFormat;
+  mediaSettings.framesPerSecond = @(bestFrameRate);
+}
+
 - (instancetype)initWithMediaSettings:(FCPPlatformMediaSettings *)mediaSettings
                mediaSettingsAVWrapper:(FLTCamMediaSettingsAVWrapper *)mediaSettingsAVWrapper
                           orientation:(UIDeviceOrientation)orientation
@@ -226,6 +279,9 @@ - (instancetype)initWithMediaSettings:(FCPPlatformMediaSettings *)mediaSettings
         return nil;
       }
 
+      selectBestFormatForRequestedFrameRate(_captureDevice, _mediaSettings,
+                                            _videoDimensionsForFormat);
+
       // Set frame rate with 1/10 precision allowing not integral values.
       int fpsNominator = floor([_mediaSettings.framesPerSecond doubleValue] * 10.0);
       CMTime duration = CMTimeMake(10, fpsNominator);
@@ -474,56 +530,42 @@ - (BOOL)setCaptureSessionPreset:(FCPPlatformResolutionPreset)resolutionPreset
           // Set the best device format found and finish the device configuration.
           _captureDevice.activeFormat = bestFormat;
           [_captureDevice unlockForConfiguration];
-
-          // Set the preview size based on values from the current capture device.
-          _previewSize =
-              CGSizeMake(_captureDevice.activeFormat.highResolutionStillImageDimensions.width,
-                         _captureDevice.activeFormat.highResolutionStillImageDimensions.height);
           break;
         }
       }
     }
     case FCPPlatformResolutionPresetUltraHigh:
       if ([_videoCaptureSession canSetSessionPreset:AVCaptureSessionPreset3840x2160]) {
         _videoCaptureSession.sessionPreset = AVCaptureSessionPreset3840x2160;
-        _previewSize = CGSizeMake(3840, 2160);
         break;
       }
       if ([_videoCaptureSession canSetSessionPreset:AVCaptureSessionPresetHigh]) {
         _videoCaptureSession.sessionPreset = AVCaptureSessionPresetHigh;
-        _previewSize =
-            CGSizeMake(_captureDevice.activeFormat.highResolutionStillImageDimensions.width,
-                       _captureDevice.activeFormat.highResolutionStillImageDimensions.height);
         break;
       }
     case FCPPlatformResolutionPresetVeryHigh:
       if ([_videoCaptureSession canSetSessionPreset:AVCaptureSessionPreset1920x1080]) {
         _videoCaptureSession.sessionPreset = AVCaptureSessionPreset1920x1080;
-        _previewSize = CGSizeMake(1920, 1080);
         break;
       }
     case FCPPlatformResolutionPresetHigh:
       if ([_videoCaptureSession canSetSessionPreset:AVCaptureSessionPreset1280x720]) {
         _videoCaptureSession.sessionPreset = AVCaptureSessionPreset1280x720;
-        _previewSize = CGSizeMake(1280, 720);
         break;
       }
     case FCPPlatformResolutionPresetMedium:
       if ([_videoCaptureSession canSetSessionPreset:AVCaptureSessionPreset640x480]) {
         _videoCaptureSession.sessionPreset = AVCaptureSessionPreset640x480;
-        _previewSize = CGSizeMake(640, 480);
         break;
       }
     case FCPPlatformResolutionPresetLow:
       if ([_videoCaptureSession canSetSessionPreset:AVCaptureSessionPreset352x288]) {
         _videoCaptureSession.sessionPreset = AVCaptureSessionPreset352x288;
-        _previewSize = CGSizeMake(352, 288);
         break;
       }
     default:
       if ([_videoCaptureSession canSetSessionPreset:AVCaptureSessionPresetLow]) {
         _videoCaptureSession.sessionPreset = AVCaptureSessionPresetLow;
-        _previewSize = CGSizeMake(352, 288);
       } else {
         if (error != nil) {
           *error =
@@ -537,23 +579,33 @@ - (BOOL)setCaptureSessionPreset:(FCPPlatformResolutionPreset)resolutionPreset
         return NO;
       }
   }
+  CMVideoDimensions size = self.videoDimensionsForFormat(_captureDevice.activeFormat);
+  _previewSize = CGSizeMake(size.width, size.height);
   _audioCaptureSession.sessionPreset = _videoCaptureSession.sessionPreset;
   return YES;
 }
 
 /// Finds the highest available resolution in terms of pixel count for the given device.
+/// Preferred are formats with the same subtype as current activeFormat.
 - (AVCaptureDeviceFormat *)highestResolutionFormatForCaptureDevice:
     (AVCaptureDevice *)captureDevice {
+  FourCharCode preferredSubType =
+      CMFormatDescriptionGetMediaSubType(_captureDevice.activeFormat.formatDescription);
   AVCaptureDeviceFormat *bestFormat = nil;
   NSUInteger maxPixelCount = 0;
+  BOOL isBestSubTypePreferred = NO;
   for (AVCaptureDeviceFormat *format in _captureDevice.formats) {
     CMVideoDimensions res = self.videoDimensionsForFormat(format);
     NSUInteger height = res.height;
     NSUInteger width = res.width;
     NSUInteger pixelCount = height * width;
-    if (pixelCount > maxPixelCount) {
-      maxPixelCount = pixelCount;
+    FourCharCode subType = CMFormatDescriptionGetMediaSubType(format.formatDescription);
+    BOOL isSubTypePreferred = subType == preferredSubType;
+    if (pixelCount > maxPixelCount ||
+        (pixelCount == maxPixelCount && isSubTypePreferred && !isBestSubTypePreferred)) {
       bestFormat = format;
+      maxPixelCount = pixelCount;
+      isBestSubTypePreferred = isSubTypePreferred;
     }
   }
   return bestFormat;

packages/camera/camera_avfoundation/pubspec.yaml

@@ -2,7 +2,7 @@ name: camera_avfoundation
 description: iOS implementation of the camera plugin.
 repository: https://github.com/flutter/packages/tree/main/packages/camera/camera_avfoundation
 issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22
-version: 0.9.17+4
+version: 0.9.17+5
 
 environment:
   sdk: ^3.3.0