Audio frame.samples buffer type as FloatBuffer nok for replay.

[taskin224]

FFmpegFrameGrabber returns frame.samples as ShortBuffer if the file type is *.mp3.
This is Ok. It is clear played back.

Both video and audio (recorded from *.mp3 or from microphone) are written into *.mp4 where audio.samples as ShortBuffer. Replay via mediaplayer Ok.

However, despite written as ShortBuffer, if the fileType read is *.mp4, frame.samples come in FloatFormat. And handling of the FloatFormat is the problem. The extracted byteArray plays with a lot of noise.

Only by using the first byte of the extracted byteArray, the noise was a lot reduced, but it is still there.

Please see the code extract below. Also the strange filter is given as a dirty workaround.
Am I missing a better way of converting this strange FloatFormat to ByteArray?
I suspect this is a bug. Also often by recording the sampleRate needs to be doubled and by playing halved, indicate there is a problem with audio sampleRate too.

Any tipp is very much appreciated.

private void me_PlayFrameAudio(Buffer[] audioSample,SourceDataLine playLine){
        Buffer buff = audioSample[0];
        if(buff instanceof ShortBuffer){
            ShortBuffer sBuff = (ShortBuffer)buff;
            int len = sBuff.capacity();
            ByteBuffer bBuff = ByteBuffer.allocate(len*2);
            bBuff.asShortBuffer().put(sBuff);
            byte[] byteArr = bBuff.array();
            playLine.write(byteArr, 0, byteArr.length);
            sBuff.clear();
        }
        else if(buff instanceof FloatBuffer){
            FloatBuffer fBuff = (FloatBuffer)buff;
            int len = fBuff.capacity();
            ShortBuffer sBuff = ShortBuffer.allocate(len*2);
            ByteBuffer bBuff = ByteBuffer.allocate(len*4);
            bBuff.asFloatBuffer().put(fBuff);
            byte[] byteArr = bBuff.array();
            byteArr = me_Filter(byteArr);   // filtering trials to see if the noise can be eliminated...
            playLine.write(byteArr, 0, byteArr.length);
            fBuff.clear();
        }
    }

    private byte[] me_Filter(byte[] byteArr){
        int len = byteArr.length;
        // byte[] newByteArr = new byte[len];
        byte[] newByteArr = new byte[len/4];
        int countIn = 0;
        int countOut = 0;
        for(int i=0; i<len/4; i++){
            newByteArr[countOut+0] = byteArr[countIn+0];
//          newByteArr[countOut+0] = byteArr[countIn+1];
//          newByteArr[countOut+3] = byteArr[countIn+2];
//          newByteArr[countOut+2] = byteArr[countIn+3];
            countOut += 1;
            countIn += 4;
        }
        return newByteArr;
    }

[saudet]

It's probably a better idea to convert the float samples to short, and not byte. Then it's going to work the same way as if you had short samples in the first place.

BTW. we can modify FFmpegFrameGrabber to support that through FFmpeg:
https://code.google.com/p/javacv/issues/detail?id=316

[d-a-gerashenko]

For me works this code:

AudioFormat audioFormat = new AudioFormat(44100, 16, 1, true, true);
...
double SC16 = (double) 0x7FFF + 0.4999999999999999;
short val = (short) ((double) channelSamplesFloatBuffer.get(i) * SC16);
...

I found it here: https://www.kvraudio.com/forum/viewtopic.php?f=33&t=414666

Full JavaFx player example:

package javacvtest;

import java.nio.ByteBuffer;
import java.nio.FloatBuffer;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
import javafx.application.Application;
import javafx.application.Platform;
import javafx.embed.swing.SwingFXUtils;
import javafx.scene.Scene;
import javafx.scene.image.Image;
import javafx.scene.image.ImageView;
import javafx.scene.layout.StackPane;
import javafx.stage.Stage;
import javax.sound.sampled.AudioFormat;
import javax.sound.sampled.AudioSystem;
import javax.sound.sampled.DataLine;
import javax.sound.sampled.SourceDataLine;
import org.bytedeco.javacv.FFmpegFrameGrabber;
import org.bytedeco.javacv.Frame;
import org.bytedeco.javacv.Java2DFrameConverter;
import org.bytedeco.javacv.OpenCVFrameConverter;

public class PlayerTest1 extends Application {

    private static final Logger LOG = Logger.getLogger(PlayerTest1.class.getName());
    private static final double SC16 = (double) 0x7FFF + 0.4999999999999999;

    private static volatile Thread playThread;

    public static void main(String[] args) {
        launch(args);
    }

    @Override
    public void start(Stage primaryStage) throws Exception {
        StackPane root = new StackPane();
        ImageView imageView = new ImageView();

        root.getChildren().add(imageView);

        Scene scene = new Scene(root, 640, 480);

        primaryStage.setTitle("Video + audio");
        primaryStage.setScene(scene);
        primaryStage.show();

        playThread = new Thread(() -> {
            try {
                FFmpegFrameGrabber grabber = new FFmpegFrameGrabber("C:\\Users\\gda\\Desktop\\bunny_move\\1486430724718.mp4");
                grabber.start();
                AudioFormat audioFormat = new AudioFormat(44100, 16, 1, true, true);

                DataLine.Info info = new DataLine.Info(SourceDataLine.class, audioFormat);
                SourceDataLine soundLine = (SourceDataLine) AudioSystem.getLine(info);
                soundLine.open(audioFormat);
                soundLine.start();

                OpenCVFrameConverter converter = new OpenCVFrameConverter.ToIplImage();
                Java2DFrameConverter paintConverter = new Java2DFrameConverter();

                ExecutorService executor = Executors.newSingleThreadExecutor();

                while (!Thread.interrupted()) {
                    Frame frame = grabber.grab();
                    if (frame == null) {
                        break;
                    }
                    if (frame.image != null) {
                        Image image = SwingFXUtils.toFXImage(paintConverter.convert(frame), null);
                        Platform.runLater(() -> {
                            imageView.setImage(image);
                        });
                    } else if (frame.samples != null) {
                        FloatBuffer channelSamplesFloatBuffer = (FloatBuffer) frame.samples[0];
                        channelSamplesFloatBuffer.rewind();

                        ByteBuffer outBuffer = ByteBuffer.allocate(channelSamplesFloatBuffer.capacity() * 2);

                        for (int i = 0; i < channelSamplesFloatBuffer.capacity(); i++) {
                            short val = (short) ((double) channelSamplesFloatBuffer.get(i) * SC16);
                            outBuffer.putShort(val);
                        }

                        /**
                         * We need this because soundLine.write ignores
                         * interruptions during writing.
                         */
                        try {
                            executor.submit(() -> {
                                soundLine.write(outBuffer.array(), 0, outBuffer.capacity());
                                outBuffer.clear();
                            }).get();
                        } catch (InterruptedException interruptedException) {
                            Thread.currentThread().interrupt();
                        }
                    }
                }
                executor.shutdownNow();
                executor.awaitTermination(10, TimeUnit.SECONDS);
                soundLine.stop();
                grabber.stop();
                grabber.release();
                Platform.exit();
            } catch (Exception exception) {
                LOG.log(Level.SEVERE, null, exception);
                System.exit(1);
            }
        });
        playThread.start();
    }

    @Override
    public void stop() throws Exception {
        playThread.interrupt();
    }

}

[saudet]

@d-a-gerashenko Awesome, could you send a pull request to add it to the samples?

[d-a-gerashenko]

#618

[saudet]

The fix for this has been released as part of JavaCV 1.3.3. Enjoy!