Implement reading of other data streams into Frame using FFmpegFrameGrabber #1377

CHANGELOG.md

@@ -1,4 +1,5 @@
 
+ * Let `FFmpegFrameGrabber.grab()` return non-audio/video streams as new `Frame.DATA` type ([pull #1378](https://github.com/bytedeco/javacv/pull/1378))
  * Fix crash in `FFmpegFrameRecorder.flush()` for HLS format and possibly others ([pull #1374](https://github.com/bytedeco/javacv/pull/1374))
  * Fix "Resetting to invalid mark" `IOException` thrown on `FFmpegFrameGrabber.release()` ([issue #911](https://github.com/bytedeco/javacv/issues/911))
  * Upgrade dependencies for OpenCV 4.2.0, FFmpeg 4.2.2, Leptonica 1.79.0, and Tesseract 4.1.1

src/main/java/org/bytedeco/javacv/FFmpegFrameGrabber.java

@@ -68,7 +68,6 @@
 import org.bytedeco.javacpp.PointerPointer;
 
 import org.bytedeco.ffmpeg.avcodec.*;
-import org.bytedeco.ffmpeg.avdevice.*;
 import org.bytedeco.ffmpeg.avformat.*;
 import org.bytedeco.ffmpeg.avutil.*;
 import org.bytedeco.ffmpeg.swresample.*;
@@ -1208,18 +1207,21 @@ private void processSamples() throws Exception {
     }
 
     public Frame grab() throws Exception {
-        return grabFrame(true, true, true, false);
+        return grabFrame(true, true, true, false, true);
     }
     public Frame grabImage() throws Exception {
-        return grabFrame(false, true, true, false);
+        return grabFrame(false, true, true, false, false);
     }
     public Frame grabSamples() throws Exception {
-        return grabFrame(true, false, true, false);
+        return grabFrame(true, false, true, false, false);
     }
     public Frame grabKeyFrame() throws Exception {
-        return grabFrame(false, true, true, true);
+        return grabFrame(false, true, true, true, false);
     }
     public Frame grabFrame(boolean doAudio, boolean doVideo, boolean doProcessing, boolean keyFrames) throws Exception {
+        return grabFrame(doAudio, doVideo, doProcessing, keyFrames, true);
+    }
+    public Frame grabFrame(boolean doAudio, boolean doVideo, boolean doProcessing, boolean keyFrames, boolean doData) throws Exception {
         if (oc == null || oc.isNull()) {
             throw new Exception("Could not grab: No AVFormatContext. (Has start() been called?)");
         } else if ((!doVideo || video_st == null) && (!doAudio || audio_st == null)) {
@@ -1242,6 +1244,7 @@ public Frame grabFrame(boolean doAudio, boolean doVideo, boolean doProcessing, b
         frame.sampleRate = 0;
         frame.audioChannels = 0;
         frame.samples = null;
+        frame.data = null;
         frame.opaque = null;
         if (doVideo && videoFrameGrabbed) {
             if (doProcessing) {
@@ -1272,6 +1275,8 @@ public Frame grabFrame(boolean doAudio, boolean doVideo, boolean doProcessing, b
                 }
             }
 
+            frame.streamIndex = pkt.stream_index();
+
             // Is this a packet from the video stream?
             if (doVideo && video_st != null && pkt.stream_index() == video_st.index()
                     && (!keyFrames || pkt.flags() == AV_PKT_FLAG_KEY)) {
@@ -1325,6 +1330,10 @@ public Frame grabFrame(boolean doAudio, boolean doVideo, boolean doProcessing, b
                         frame.keyFrame = samples_frame.key_frame() != 0;
                     }
                 }
+            } else if (doData) {
+                // Export the stream byte data for non audio / video frames
+                frame.data = pkt.data().position(0).capacity(pkt.size()).asByteBuffer();
+                done = true;
             }
 
             if (pkt2.size() <= 0) {

src/main/java/org/bytedeco/javacv/Frame.java

@@ -71,11 +71,12 @@ public class Frame implements Indexable {
             DEPTH_LONG   = -64,
             DEPTH_FLOAT  =  32,
             DEPTH_DOUBLE =  64;
-    
+
     /** Constants defining data type in the frame*/
     public static enum Type {
         VIDEO,
         AUDIO,
+        DATA
     }
 
     /** Information associated with the {@link #image} field. */
@@ -94,6 +95,12 @@ public static enum Type {
     /** Buffers to hold audio samples from multiple channels for an audio frame. */
     public Buffer[] samples;
 
+    /** Buffer to hold a data stream associated with a frame. */
+    public ByteBuffer data;
+
+    /** Stream number the audio|video|other data is associated with. */
+    public int streamIndex;
+
     /** The underlying data object, for example, Pointer, AVFrame, IplImage, or Mat. */
     public Object opaque;
 
@@ -119,6 +126,8 @@ public Frame(int width, int height, int depth, int channels, int imageStride) {
         this.imageChannels = channels;
         this.imageStride = imageStride;
         this.image = new Buffer[1];
+        this.data = null;
+        this.streamIndex = -1;
 
         Pointer pointer = new BytePointer(imageHeight * imageStride * pixelSize(depth));
         ByteBuffer buffer = pointer.asByteBuffer();
@@ -207,7 +216,8 @@ public Frame clone() {
         newFrame.imageChannels = imageChannels;
         newFrame.imageStride = imageStride;
         newFrame.keyFrame = keyFrame;
-        newFrame.opaque = new Pointer[2];
+        newFrame.streamIndex = streamIndex;
+        newFrame.opaque = new Pointer[3];
         if (image != null) {
             newFrame.image = new Buffer[image.length];
             ((Pointer[])newFrame.opaque)[0] = cloneBufferArray(image, newFrame.image);
@@ -221,6 +231,13 @@ public Frame clone() {
             ((Pointer[])newFrame.opaque)[1] = cloneBufferArray(samples, newFrame.samples);
         }
 
+        // Other data streams
+        if (data != null) {
+            ByteBuffer[] dst = new ByteBuffer[1];
+            ((Pointer[])newFrame.opaque)[2] = cloneBufferArray(new ByteBuffer[]{data}, dst);
+            newFrame.data = dst[0];
+        }
+
         // Add timestamp
         newFrame.timestamp = timestamp;
 
@@ -330,6 +347,7 @@ public EnumSet<Type> getTypes() {
         EnumSet<Type> type = EnumSet.noneOf(Type.class);
         if (image != null) type.add(Type.VIDEO);
         if (samples != null) type.add(Type.AUDIO);
+        if (data != null) type.add(Type.DATA);
         return type;
     }
 }