Piping frames to ffmpeg

example/android/build.gradle

@@ -26,6 +26,6 @@ subprojects {
     project.evaluationDependsOn(':app')
 }
 
-task clean(type: Delete) {
+tasks.register("clean", Delete) {
     delete rootProject.buildDir
 }

example/lib/animated_example_controller.dart

@@ -37,7 +37,7 @@ class ExampleAnimationController extends ChangeNotifier {
     VideoPlayerController? videoController;
     if (!Platform.isMacOS) {
        videoController = VideoPlayerController.network(
-        'https://www.fluttercampus.com/video.mp4',
+        'https://flutter.github.io/assets-for-api-docs/assets/videos/bee.mp4',
         // 1 min: https://storage.googleapis.com/gtv-videos-bucket/sample/ForBiggerFun.mp4
         // 4 sec: 'https://www.fluttercampus.com/video.mp4'
       );

example/lib/main.dart

@@ -76,7 +76,6 @@ class _MyHomePageState extends State<MyHomePage>
                         settings: const MotionSettings(
                           pixelRatio: 5,
                           frameRate: 30,
-                          simultaneousCaptureHandlers: 6,
                         ),
                         logInConsole: true,
                         format: Mp4Format(audio: [
@@ -131,6 +130,7 @@ class _MyHomePageState extends State<MyHomePage>
 
   Future<void> displayResult(RenderResult result,
       [bool saveToGallery = false]) async {
+    print("file path: ${result.output.path}");
     print("file exits: ${await result.output.exists()}");
     if (mounted) {
       showDialog(

lib/src/capturer.dart

@@ -1,8 +1,9 @@
 import 'dart:async';
+import 'dart:io';
 import 'dart:typed_data';
 import 'dart:ui' as ui;
 
-import 'package:ffmpeg_kit_flutter_https_gpl/ffmpeg_kit.dart';
+import 'package:ffmpeg_kit_flutter_https_gpl/ffmpeg_kit_config.dart';
 import 'package:flutter/material.dart';
 import 'package:flutter/rendering.dart';
 import 'package:render/src/service/notifier.dart';
@@ -23,11 +24,6 @@ class RenderCapturer<K extends RenderFormat> {
 
   RenderCapturer(this.session, [this.context]);
 
-  int _activeHandlers = 0;
-
-  /// Captures that are yet to be handled. Handled images will be disposed.
-  final List<ui.Image> _unhandledCaptures = [];
-
   /// Current image handling process. Handlers are being handles asynchronous
   /// as conversion and file writing is involved.
   final List<Future<void>> _handlers = [];
@@ -47,6 +43,9 @@ class RenderCapturer<K extends RenderFormat> {
   /// will be seen as first frame.
   Size? firstFrameSize;
 
+  /// The writer to write the captured images to.
+  IOSink? _frameWriter;
+
   /// Runs a capturing process for a defined time. Returns capturing time duration.
   Future<RenderSession<K, RealRenderSettings>> run(Duration duration) async {
     start(duration);
@@ -60,11 +59,7 @@ class RenderCapturer<K extends RenderFormat> {
   Future<RenderSession<K, RealRenderSettings>> single() async {
     startTime = DateTime.now();
     _captureFrame(0, 1);
-    await Future.doWhile(() async {
-      //await all active capture handlers
-      await Future.wait(_handlers);
-      return _handlers.length < _unhandledCaptures.length;
-    });
+    await Future.wait(_handlers);
     final capturingDuration = Duration(
         milliseconds: DateTime.now().millisecondsSinceEpoch -
             startTime!.millisecondsSinceEpoch);
@@ -76,6 +71,7 @@ class RenderCapturer<K extends RenderFormat> {
     assert(!_rendering, "Cannot start new process, during an active one.");
     _rendering = true;
     startTime = DateTime.now();
+
     session.binding.addPostFrameCallback((binderTimeStamp) {
       startingDuration = session.binding.currentFrameTimeStamp;
       _postFrameCallback(
@@ -95,15 +91,10 @@ class RenderCapturer<K extends RenderFormat> {
     _rendering = false;
     startingDuration = null;
     // * wait for handlers
-    await Future.doWhile(() async {
-      //await all active capture handlers
-      await Future.wait(_handlers);
-      return _handlers.length < _unhandledCaptures.length;
-    });
+    await Future.wait(_handlers);
     // * finish capturing, notify session
-    final frameAmount = _unhandledCaptures.length;
+    final frameAmount = _handlers.length;
     _handlers.clear();
-    _unhandledCaptures.clear();
     return session.upgrade(capturingDuration, frameAmount);
   }
 
@@ -149,41 +140,23 @@ class RenderCapturer<K extends RenderFormat> {
 
   /// Converting the raw image data to a png file and writing the capture.
   Future<void> _handleCapture(
+    ui.Image capture,
     int captureNumber, [
     int? totalFrameTarget,
   ]) async {
-    _activeHandlers++;
     try {
-      final ui.Image capture = _unhandledCaptures.elementAt(captureNumber);
       // * retrieve bytes
       // toByteData(format: ui.ImageByteFormat.png) takes way longer than raw
       // and then converting to png with ffmpeg
       final ByteData? byteData =
           await capture.toByteData(format: ui.ImageByteFormat.rawRgba);
       final rawIntList = byteData!.buffer.asInt8List();
-      // * write raw file for processing
-      final rawFile = session
-          .createProcessFile("frameHandling/frame_raw$captureNumber.bmp");
-      await rawFile.writeAsBytes(rawIntList);
-      // * write & convert file (to save storage)
-      final file = session.createInputFile("frame$captureNumber.png");
-      final saveSize = Size(
-        // adjust frame size, so that it can be divided by 2
-        (capture.width / 2).ceil() * 2,
-        (capture.height / 2).ceil() * 2,
-      );
-      await FFmpegKit.executeWithArguments([
-        "-y",
-        "-f", "rawvideo", // specify input format
-        "-pixel_format", "rgba", // maintain transparency
-        "-video_size", "${capture.width}x${capture.height}", // set capture size
-        "-i", rawFile.path, // input the raw frame
-        "-vf", "scale=${saveSize.width}:${saveSize.height}", // scale to save
-        file.path, //out put png
-      ]);
+
+      // * write image to pipe
+      _writeToPipe(rawIntList);
+
       // * finish
       capture.dispose();
-      rawFile.deleteSync();
       if (!_rendering) {
         //only record next state, when rendering is done not to mix up notification
         _recordActivity(RenderState.handleCaptures, captureNumber,
@@ -197,19 +170,6 @@ class RenderCapturer<K extends RenderFormat> {
         ),
       );
     }
-    _activeHandlers--;
-    _triggerHandler(totalFrameTarget);
-  }
-
-  /// Triggers the next handler, if within allowed simultaneous handlers
-  /// and images still available.
-  void _triggerHandler([int? totalFrameTarget]) {
-    final nextCaptureIndex = _handlers.length;
-    if (_activeHandlers <
-            (session.settings.asMotion?.simultaneousCaptureHandlers ?? 1) &&
-        nextCaptureIndex < _unhandledCaptures.length) {
-      _handlers.add(_handleCapture(nextCaptureIndex, totalFrameTarget));
-    }
   }
 
   /// Captures associated task of this frame
@@ -238,8 +198,7 @@ class RenderCapturer<K extends RenderFormat> {
       );
     }
     // * initiate handler
-    _unhandledCaptures.add(image);
-    _triggerHandler(totalFrameTarget);
+    _handlers.add(_handleCapture(image, frameNumber, totalFrameTarget));
     _recordActivity(RenderState.capturing, frameNumber, totalFrameTarget,
         "Captured frame $frameNumber");
   }
@@ -346,4 +305,21 @@ class RenderCapturer<K extends RenderFormat> {
       session.recordActivity(state, null, message: message);
     }
   }
+
+  /// Opens the pipe to the ffmpeg process
+  void openPipe() {
+    var f = File(session.inputPipe);
+    _frameWriter = f.openWrite();
+  }
+
+  /// Closes the pipe to the ffmpeg process
+  Future<void> closePipe() async {
+    await _frameWriter?.close();
+    await FFmpegKitConfig.closeFFmpegPipe(session.inputPipe);
+  }
+
+  /// Writes data to the pipe to the ffmpeg process
+  void _writeToPipe(List<int> data) {
+    return _frameWriter?.add(data);
+  }
 }

lib/src/core.dart

@@ -220,8 +220,12 @@ class RenderController {
         .then((detachedSession) async {
       final session = _createRenderSessionFrom(detachedSession, notifier);
       final capturer = RenderCapturer(session);
-      final realSession = await capturer.single();
-      final processor = ImageProcessor(realSession);
+      capturer.openPipe();
+      capturer.single().then((value) => capturer.closePipe());
+      final processor = ImageProcessor(
+          session,
+          capturer.firstFrameSize!.width.toInt(),
+          capturer.firstFrameSize!.height.toInt());
       await processor.process();
       await session.dispose();
     });
@@ -251,9 +255,16 @@ class RenderController {
         .then((detachedSession) async {
       final session = _createRenderSessionFrom(detachedSession, notifier);
       final capturer = RenderCapturer(session);
-      final realSession = await capturer.run(duration);
-      final processor = MotionProcessor(realSession);
-      await processor.process();
+      capturer.openPipe();
+      await capturer
+          .single(); // wait for first frame to be captured to get the size of the frame
+      capturer.run(duration).then(
+          (value) => capturer.closePipe()); // run the capturer for the duration
+      final processor = MotionProcessor(
+          session,
+          capturer.firstFrameSize!.width.toInt(),
+          capturer.firstFrameSize!.height.toInt());
+      await processor.process(duration: duration);
       await session.dispose();
     });
     if (logInConsole) {
@@ -291,8 +302,12 @@ class RenderController {
         widgetTask,
       );
       final capturer = RenderCapturer(session, context);
-      final realSession = await capturer.single();
-      final processor = ImageProcessor(realSession);
+      capturer.openPipe();
+      capturer.single().then((value) => capturer.closePipe());
+      final processor = ImageProcessor(
+          session,
+          capturer.firstFrameSize!.width.toInt(),
+          capturer.firstFrameSize!.height.toInt());
       await processor.process();
       await session.dispose();
     });
@@ -337,9 +352,14 @@ class RenderController {
         widgetTask,
       );
       final capturer = RenderCapturer(session, context);
-      final realSession = await capturer.run(duration);
-      final processor = MotionProcessor(realSession);
-      await processor.process();
+      capturer.openPipe();
+      await capturer.single();
+      capturer.run(duration).then((value) => capturer.closePipe());
+      final processor = MotionProcessor(
+          session,
+          capturer.firstFrameSize!.width.toInt(),
+          capturer.firstFrameSize!.height.toInt());
+      await processor.process(duration: duration);
       await session.dispose();
     });
     if (logInConsole) {
@@ -440,6 +460,7 @@ class MotionRecorder<T extends MotionFormat> {
       );
       _capturer = RenderCapturer(_session, context);
       _capturer.start();
+      _capturer.openPipe();
     });
     if (logInConsole) {
       _controller._debugPrintOnStream(
@@ -454,8 +475,12 @@ class MotionRecorder<T extends MotionFormat> {
 
   /// Stops the recording and returns the result of the recording.
   Future<RenderResult> stop() async {
-    final realSession = await _capturer.finish();
-    final processor = MotionProcessor(realSession);
+    await _capturer.finish();
+    await _capturer.closePipe();
+    final processor = MotionProcessor(
+        _session,
+        _capturer.firstFrameSize!.width.toInt(),
+        _capturer.firstFrameSize!.height.toInt());
     processor.process(); // wait for result instead of process
     final out = await stream
         .firstWhere((event) => event.isResult || event.isFatalError);

lib/src/formats/abstract.dart

@@ -74,6 +74,8 @@ abstract class RenderFormat {
     required String inputPath,
     required String outputPath,
     required double frameRate,
+    required int width,
+    required int height,
   });
 
   /// Scaling ffmpeg filter with appropriate interpolation integration
@@ -118,10 +120,13 @@ abstract class MotionFormat extends RenderFormat {
   /// Default motion processor. This can be override, if more/other settings are
   /// needed.
   @override
-  FFmpegRenderOperation processor(
-      {required String inputPath,
-      required String outputPath,
-      required double frameRate}) {
+  FFmpegRenderOperation processor({
+    required String inputPath,
+    required String outputPath,
+    required double frameRate,
+    required int width,
+    required int height,
+  }) {
     final audioInput = audio != null && audio!.isNotEmpty
         ? audio!.map((e) => "-i??${e.path}").join('??')
         : null;
@@ -138,6 +143,10 @@ abstract class MotionFormat extends RenderFormat {
                 "aac??-shortest??-pix_fmt??yuv420p??-vsync??2"
             : "-map??[v]??-pix_fmt??yuv420p";
     return FFmpegRenderOperation([
+      "-f", "rawvideo", // input format
+      "-pixel_format", "rgba", // input pixel format
+      "-s", "${width}x${height}", // input size
+      "-r", "$frameRate", // input frame rate
       "-i", inputPath, // retrieve  captures
       audioInput,
       "-filter_complex",
@@ -176,12 +185,19 @@ abstract class ImageFormat extends RenderFormat {
   /// Default image processor. This can be override, if more settings are
   /// needed.
   @override
-  FFmpegRenderOperation processor(
-      {required String inputPath,
-      required String outputPath,
-      required double frameRate}) {
+  FFmpegRenderOperation processor({
+    required String inputPath,
+    required String outputPath,
+    required double frameRate,
+    required int width,
+    required int height,
+  }) {
     return FFmpegRenderOperation([
       "-y",
+      "-f", "rawvideo",
+      "-pixel_format", "rgba",
+      "-s", "${width}x${height}",
+      "-r", "$frameRate",
       "-i", inputPath, // input image
       scalingFilter != null ? "-vf??$scalingFilter" : null,
       "-vframes", "1", // indicate that there is only one frame

lib/src/formats/image.dart

@@ -94,10 +94,13 @@ class BmpFormat extends ImageFormat {
   }
 
   @override
-  FFmpegRenderOperation processor(
-      {required String inputPath,
-      required String outputPath,
-      required double frameRate}) {
+  FFmpegRenderOperation processor({
+    required String inputPath,
+    required String outputPath,
+    required double frameRate,
+    required int width,
+    required int height,
+  }) {
     return FFmpegRenderOperation([
       "-y",
       "-i", inputPath, // input image

lib/src/formats/motion.dart

@@ -91,9 +91,15 @@ class GifFormat extends MotionFormat {
     required String inputPath,
     required String outputPath,
     required double frameRate,
+    required int width,
+    required int height,
   }) {
     return FFmpegRenderOperation([
       "-y",
+      "-f", "rawvideo", // input format
+      "-pixel_format", "rgba", // input pixel format
+      "-s", "${width}x${height}", // input size
+      "-r", "$frameRate", // input frame rate
       "-i", inputPath, // retrieve  captures
       transparency
           ? "-filter_complex??[0:v] setpts=N/($frameRate*TB),"