Skip to content
This repository was archived by the owner on Feb 22, 2023. It is now read-only.

Commit 431f7d2

Browse files
stuartmorganstuartmorgan
authored
[camera] Move camera streaming to platform interface (#5783)
1 parent bdab120 commit 431f7d2

File tree

10 files changed

+429
-2
lines changed

10 files changed

+429
-2
lines changed

‎packages/camera/camera_platform_interface/CHANGELOG.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
1-
## NEXT
1+
## 2.2.0
22

3+
* Adds image streaming to the platform interface.
34
* Removes unnecessary imports.
45

56
## 2.1.6

‎packages/camera/camera_platform_interface/lib/src/method_channel/method_channel_camera.dart

+54
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ import 'package:flutter/services.dart';
1212
import 'package:flutter/widgets.dart';
1313
import 'package:stream_transform/stream_transform.dart';
1414

15+
import 'type_conversion.dart';
16+
1517
const MethodChannel _channel = MethodChannel('plugins.flutter.io/camera');
1618

1719
/// An implementation of [CameraPlatform] that uses method channels.
@@ -48,6 +50,12 @@ class MethodChannelCamera extends CameraPlatform {
4850
final StreamController<DeviceEvent> deviceEventStreamController =
4951
StreamController<DeviceEvent>.broadcast();
5052

53+
// The stream to receive frames from the native code.
54+
StreamSubscription<dynamic>? _platformImageStreamSubscription;
55+
56+
// The stream for vending frames to platform interface clients.
57+
StreamController<CameraImageData>? _frameStreamController;
58+
5159
Stream<CameraEvent> _cameraEvents(int cameraId) =>
5260
cameraEventStreamController.stream
5361
.where((CameraEvent event) => event.cameraId == cameraId);
@@ -267,6 +275,52 @@ class MethodChannelCamera extends CameraPlatform {
267275
<String, dynamic>{'cameraId': cameraId},
268276
);
269277

278+
@override
279+
Stream<CameraImageData> onStreamedFrameAvailable(int cameraId,
280+
{CameraImageStreamOptions? options}) {
281+
_frameStreamController = StreamController<CameraImageData>(
282+
onListen: _onFrameStreamListen,
283+
onPause: _onFrameStreamPauseResume,
284+
onResume: _onFrameStreamPauseResume,
285+
onCancel: _onFrameStreamCancel,
286+
);
287+
return _frameStreamController!.stream;
288+
}
289+
290+
void _onFrameStreamListen() {
291+
_startPlatformStream();
292+
}
293+
294+
Future<void> _startPlatformStream() async {
295+
await _channel.invokeMethod<void>('startImageStream');
296+
const EventChannel cameraEventChannel =
297+
EventChannel('plugins.flutter.io/camera/imageStream');
298+
_platformImageStreamSubscription =
299+
cameraEventChannel.receiveBroadcastStream().listen((dynamic imageData) {
300+
if (defaultTargetPlatform == TargetPlatform.iOS) {
301+
try {
302+
_channel.invokeMethod<void>('receivedImageStreamData');
303+
} on PlatformException catch (e) {
304+
throw CameraException(e.code, e.message);
305+
}
306+
}
307+
_frameStreamController!
308+
.add(cameraImageFromPlatformData(imageData as Map<dynamic, dynamic>));
309+
});
310+
}
311+
312+
FutureOr<void> _onFrameStreamCancel() async {
313+
await _channel.invokeMethod<void>('stopImageStream');
314+
await _platformImageStreamSubscription?.cancel();
315+
_platformImageStreamSubscription = null;
316+
_frameStreamController = null;
317+
}
318+
319+
void _onFrameStreamPauseResume() {
320+
throw CameraException('InvalidCall',
321+
'Pause and resume are not supported for onStreamedFrameAvailable');
322+
}
323+
270324
@override
271325
Future<void> setFlashMode(int cameraId, FlashMode mode) =>
272326
_channel.invokeMethod<void>(

‎packages/camera/camera_platform_interface/lib/src/method_channel/type_conversion.dart

+61
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
// Copyright 2013 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
import 'dart:typed_data';
6+
7+
import 'package:flutter/foundation.dart';
8+
9+
import '../types/types.dart';
10+
11+
/// Converts method channel call [data] for `receivedImageStreamData` to a
12+
/// [CameraImageData].
13+
CameraImageData cameraImageFromPlatformData(Map<dynamic, dynamic> data) {
14+
return CameraImageData(
15+
format: _cameraImageFormatFromPlatformData(data['format']),
16+
height: data['height'] as int,
17+
width: data['width'] as int,
18+
lensAperture: data['lensAperture'] as double?,
19+
sensorExposureTime: data['sensorExposureTime'] as int?,
20+
sensorSensitivity: data['sensorSensitivity'] as double?,
21+
planes: List<CameraImagePlane>.unmodifiable(
22+
(data['planes'] as List<dynamic>).map<CameraImagePlane>(
23+
(dynamic planeData) => _cameraImagePlaneFromPlatformData(
24+
planeData as Map<dynamic, dynamic>))));
25+
}
26+
27+
CameraImageFormat _cameraImageFormatFromPlatformData(dynamic data) {
28+
return CameraImageFormat(_imageFormatGroupFromPlatformData(data), raw: data);
29+
}
30+
31+
ImageFormatGroup _imageFormatGroupFromPlatformData(dynamic data) {
32+
if (defaultTargetPlatform == TargetPlatform.android) {
33+
switch (data) {
34+
case 35: // android.graphics.ImageFormat.YUV_420_888
35+
return ImageFormatGroup.yuv420;
36+
case 256: // android.graphics.ImageFormat.JPEG
37+
return ImageFormatGroup.jpeg;
38+
}
39+
}
40+
41+
if (defaultTargetPlatform == TargetPlatform.iOS) {
42+
switch (data) {
43+
case 875704438: // kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
44+
return ImageFormatGroup.yuv420;
45+
46+
case 1111970369: // kCVPixelFormatType_32BGRA
47+
return ImageFormatGroup.bgra8888;
48+
}
49+
}
50+
51+
return ImageFormatGroup.unknown;
52+
}
53+
54+
CameraImagePlane _cameraImagePlaneFromPlatformData(Map<dynamic, dynamic> data) {
55+
return CameraImagePlane(
56+
bytes: data['bytes'] as Uint8List,
57+
bytesPerPixel: data['bytesPerPixel'] as int?,
58+
bytesPerRow: data['bytesPerRow'] as int,
59+
height: data['height'] as int?,
60+
width: data['width'] as int?);
61+
}

‎packages/camera/camera_platform_interface/lib/src/platform_interface/camera_platform.dart

+15
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,21 @@ abstract class CameraPlatform extends PlatformInterface {
149149
throw UnimplementedError('resumeVideoRecording() is not implemented.');
150150
}
151151

152+
/// A new streamed frame is available.
153+
///
154+
/// Listening to this stream will start streaming, and canceling will stop.
155+
/// Pausing will throw a [CameraException], as pausing the stream would cause
156+
/// very high memory usage; to temporarily stop receiving frames, cancel, then
157+
/// listen again later.
158+
///
159+
///
160+
// TODO(bmparr): Add options to control streaming settings (e.g.,
161+
// resolution and FPS).
162+
Stream<CameraImageData> onStreamedFrameAvailable(int cameraId,
163+
{CameraImageStreamOptions? options}) {
164+
throw UnimplementedError('onStreamedFrameAvailable() is not implemented.');
165+
}
166+
152167
/// Sets the flash mode for the selected camera.
153168
/// On Web [FlashMode.auto] corresponds to [FlashMode.always].
154169
Future<void> setFlashMode(int cameraId, FlashMode mode) {

‎packages/camera/camera_platform_interface/lib/src/types/camera_image_data.dart

+126
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
// Copyright 2013 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
import 'dart:typed_data';
6+
7+
import 'package:flutter/foundation.dart';
8+
9+
import '../../camera_platform_interface.dart';
10+
11+
/// Options for configuring camera streaming.
12+
///
13+
/// Currently unused; this exists for future-proofing of the platform interface
14+
/// API.
15+
@immutable
16+
class CameraImageStreamOptions {}
17+
18+
/// A single color plane of image data.
19+
///
20+
/// The number and meaning of the planes in an image are determined by its
21+
/// format.
22+
@immutable
23+
class CameraImagePlane {
24+
/// Creates a new instance with the given bytes and optional metadata.
25+
const CameraImagePlane({
26+
required this.bytes,
27+
required this.bytesPerRow,
28+
this.bytesPerPixel,
29+
this.height,
30+
this.width,
31+
});
32+
33+
/// Bytes representing this plane.
34+
final Uint8List bytes;
35+
36+
/// The row stride for this color plane, in bytes.
37+
final int bytesPerRow;
38+
39+
/// The distance between adjacent pixel samples in bytes, when available.
40+
final int? bytesPerPixel;
41+
42+
/// Height of the pixel buffer, when available.
43+
final int? height;
44+
45+
/// Width of the pixel buffer, when available.
46+
final int? width;
47+
}
48+
49+
/// Describes how pixels are represented in an image.
50+
@immutable
51+
class CameraImageFormat {
52+
/// Create a new format with the given cross-platform group and raw underyling
53+
/// platform identifier.
54+
const CameraImageFormat(this.group, {required this.raw});
55+
56+
/// Describes the format group the raw image format falls into.
57+
final ImageFormatGroup group;
58+
59+
/// Raw version of the format from the underlying platform.
60+
///
61+
/// On Android, this should be an `int` from class
62+
/// `android.graphics.ImageFormat`. See
63+
/// https://developer.android.com/reference/android/graphics/ImageFormat
64+
///
65+
/// On iOS, this should be a `FourCharCode` constant from Pixel Format
66+
/// Identifiers. See
67+
/// https://developer.apple.com/documentation/corevideo/1563591-pixel_format_identifiers
68+
final dynamic raw;
69+
}
70+
71+
/// A single complete image buffer from the platform camera.
72+
///
73+
/// This class allows for direct application access to the pixel data of an
74+
/// Image through one or more [Uint8List]. Each buffer is encapsulated in a
75+
/// [CameraImagePlane] that describes the layout of the pixel data in that
76+
/// plane. [CameraImageData] is not directly usable as a UI resource.
77+
///
78+
/// Although not all image formats are planar on all platforms, this class
79+
/// treats 1-dimensional images as single planar images.
80+
@immutable
81+
class CameraImageData {
82+
/// Creates a new instance with the given format, planes, and metadata.
83+
const CameraImageData({
84+
required this.format,
85+
required this.planes,
86+
required this.height,
87+
required this.width,
88+
this.lensAperture,
89+
this.sensorExposureTime,
90+
this.sensorSensitivity,
91+
});
92+
93+
/// Format of the image provided.
94+
///
95+
/// Determines the number of planes needed to represent the image, and
96+
/// the general layout of the pixel data in each [Uint8List].
97+
final CameraImageFormat format;
98+
99+
/// Height of the image in pixels.
100+
///
101+
/// For formats where some color channels are subsampled, this is the height
102+
/// of the largest-resolution plane.
103+
final int height;
104+
105+
/// Width of the image in pixels.
106+
///
107+
/// For formats where some color channels are subsampled, this is the width
108+
/// of the largest-resolution plane.
109+
final int width;
110+
111+
/// The pixels planes for this image.
112+
///
113+
/// The number of planes is determined by the format of the image.
114+
final List<CameraImagePlane> planes;
115+
116+
/// The aperture settings for this image.
117+
///
118+
/// Represented as an f-stop value.
119+
final double? lensAperture;
120+
121+
/// The sensor exposure time for this image in nanoseconds.
122+
final int? sensorExposureTime;
123+
124+
/// The sensor sensitivity in standard ISO arithmetic units.
125+
final double? sensorSensitivity;
126+
}

‎packages/camera/camera_platform_interface/lib/src/types/types.dart

+1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
export 'camera_description.dart';
66
export 'camera_exception.dart';
7+
export 'camera_image_data.dart';
78
export 'exposure_mode.dart';
89
export 'flash_mode.dart';
910
export 'focus_mode.dart';

‎packages/camera/camera_platform_interface/pubspec.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ repository: https://github.com/flutter/plugins/tree/main/packages/camera/camera_
44
issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22
55
# NOTE: We strongly prefer non-breaking changes, even at the expense of a
66
# less-clean API. See https://flutter.dev/go/platform-interface-breaking-changes
7-
version: 2.1.6
7+
version: 2.2.0
88

99
environment:
1010
sdk: '>=2.12.0 <3.0.0'

‎packages/camera/camera_platform_interface/test/method_channel/method_channel_camera_test.dart

+46
Original file line numberDiff line numberDiff line change
@@ -1038,6 +1038,52 @@ void main() {
10381038
arguments: <String, Object?>{'cameraId': cameraId}),
10391039
]);
10401040
});
1041+
1042+
test('Should start streaming', () async {
1043+
// Arrange
1044+
final MethodChannelMock channel = MethodChannelMock(
1045+
channelName: 'plugins.flutter.io/camera',
1046+
methods: <String, dynamic>{
1047+
'startImageStream': null,
1048+
'stopImageStream': null,
1049+
},
1050+
);
1051+
1052+
// Act
1053+
final StreamSubscription<CameraImageData> subscription = camera
1054+
.onStreamedFrameAvailable(cameraId)
1055+
.listen((CameraImageData imageData) {});
1056+
1057+
// Assert
1058+
expect(channel.log, <Matcher>[
1059+
isMethodCall('startImageStream', arguments: null),
1060+
]);
1061+
1062+
subscription.cancel();
1063+
});
1064+
1065+
test('Should stop streaming', () async {
1066+
// Arrange
1067+
final MethodChannelMock channel = MethodChannelMock(
1068+
channelName: 'plugins.flutter.io/camera',
1069+
methods: <String, dynamic>{
1070+
'startImageStream': null,
1071+
'stopImageStream': null,
1072+
},
1073+
);
1074+
1075+
// Act
1076+
final StreamSubscription<CameraImageData> subscription = camera
1077+
.onStreamedFrameAvailable(cameraId)
1078+
.listen((CameraImageData imageData) {});
1079+
subscription.cancel();
1080+
1081+
// Assert
1082+
expect(channel.log, <Matcher>[
1083+
isMethodCall('startImageStream', arguments: null),
1084+
isMethodCall('stopImageStream', arguments: null),
1085+
]);
1086+
});
10411087
});
10421088
});
10431089
}

0 commit comments

Comments
 (0)