diff --git a/CHANGELOG.md b/CHANGELOG.md index b794b8c6..07f055ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ + * Fix `FFmpegFrameRecorder` dropped frame issues with audio samples ([pull #2307](https://github.com/bytedeco/javacv/pull/2307)) * Add `FrameFilter.videoFilterArgs/audioFilterArgs` properties to support multiple different inputs ([pull #2304](https://github.com/bytedeco/javacv/pull/2304)) * Ensure `FFmpegFrameGrabber.start()` skips over streams with no codecs ([issue #2299](https://github.com/bytedeco/javacv/issues/2299)) * Add `FFmpegLogCallback.logRejectedOptions()` for debugging purposes ([pull #2301](https://github.com/bytedeco/javacv/pull/2301)) diff --git a/platform/src/test/java/org/bytedeco/javacv/FrameFilterTest.java b/platform/src/test/java/org/bytedeco/javacv/FrameFilterTest.java index 89fbce7c..07733118 100644 --- a/platform/src/test/java/org/bytedeco/javacv/FrameFilterTest.java +++ b/platform/src/test/java/org/bytedeco/javacv/FrameFilterTest.java @@ -197,11 +197,11 @@ public void testFFmpegFrameFilterMultipleInputs() { } if (frame3.samples != null) { d++; - assertEquals(2, frame3.audioChannels); + assertEquals(4, frame3.audioChannels); assertEquals(1, frame3.samples.length); - assertTrue(frame3.samples[0] instanceof ByteBuffer); + assertTrue(frame3.samples[0] instanceof ShortBuffer); assertEquals(frame2.samples.length, frame3.samples.length); - assertEquals(frame2.samples[0].limit(), frame3.samples[0].limit()); + assertEquals(2 * frame2.samples[0].limit(), frame3.samples[0].limit()); } } } diff --git a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java index 47f0fa31..be8ce262 100644 --- a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java +++ b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java @@ -77,7 +77,6 @@ import org.bytedeco.javacpp.ShortPointer; import org.bytedeco.ffmpeg.avcodec.*; -import org.bytedeco.ffmpeg.avdevice.*; import org.bytedeco.ffmpeg.avformat.*; import org.bytedeco.ffmpeg.avutil.*; import org.bytedeco.ffmpeg.swresample.*; @@ -397,6 +396,7 @@ static class SeekCallback extends Seek_Pointer_long_int { private PointerPointer plane_ptr, plane_ptr2; private AVPacket video_pkt, audio_pkt; private int[] got_video_packet, got_audio_packet; + private boolean wrote_samples = false; private AVFormatContext ifmt_ctx; private IntPointer display_matrix; private AVChannelLayout default_layout; @@ -468,6 +468,7 @@ public synchronized void startUnsafe() throws Exception { plane_ptr2 = new PointerPointer(AVFrame.AV_NUM_DATA_POINTERS).retainReference(); video_pkt = new AVPacket().retainReference(); audio_pkt = new AVPacket().retainReference(); + wrote_samples = false; got_video_packet = new int[1]; got_audio_packet = new int[1]; default_layout = new AVChannelLayout().retainReference(); @@ -1169,11 +1170,9 @@ public synchronized boolean recordSamples(int sampleRate, int audioChannels, Buf throw new Exception("start() was not called successfully!"); } - if (samples == null && samples_out[0].position() > 0) { - // Typically samples_out[0].limit() is double the audio_input_frame_size --> sampleDivisor = 2 - double sampleDivisor = Math.floor((int)Math.min(samples_out[0].limit(), Integer.MAX_VALUE) / audio_input_frame_size); - writeSamples((int)Math.floor((int)samples_out[0].position() / sampleDivisor)); - return writeFrame((AVFrame)null); + if (samples == null && samples_convert_ctx == null) { + // We haven't tried to record any samples yet so we don't need to flush. + return false; } int ret; @@ -1265,7 +1264,8 @@ public synchronized boolean recordSamples(int sampleRate, int audioChannels, Buf throw new Exception("Audio samples Buffer has unsupported type: " + samples); } - if (samples_convert_ctx == null || samples_channels != audioChannels || samples_format != inputFormat || samples_rate != sampleRate) { + boolean formatChanged = samples_channels != audioChannels || samples_format != inputFormat || samples_rate != sampleRate; + if (samples != null && (samples_convert_ctx == null || formatChanged)) { if (samples_convert_ctx == null) { samples_convert_ctx = new SwrContext().retainReference(); } @@ -1294,9 +1294,14 @@ public synchronized boolean recordSamples(int sampleRate, int audioChannels, Buf for (int i = 0; i < samples_out.length; i++) { plane_ptr2.put(i, samples_out[i]); } + if (samples == null && inputCount == 0 && plane_ptr != null) { + plane_ptr.releaseReference(); + // needs to be null to flush swr context. + plane_ptr = null; + } if ((ret = swr_convert(samples_convert_ctx, plane_ptr2, outputCount, plane_ptr, inputCount)) < 0) { throw new Exception("swr_convert() error " + ret + ": Cannot convert audio samples."); - } else if (ret == 0) { + } else if (ret == 0 && inputCount == 0) { break; } for (int i = 0; samples != null && i < samples.length; i++) { @@ -1310,6 +1315,14 @@ public synchronized boolean recordSamples(int sampleRate, int audioChannels, Buf writeSamples(audio_input_frame_size); } } + + if (samples == null && samples_out[0].position() > 0) { + // Typically samples_out[0].limit() is double the audio_input_frame_size --> sampleDivisor = 2 + double sampleDivisor = Math.floor((int)Math.min(samples_out[0].limit(), Integer.MAX_VALUE) / audio_input_frame_size); + writeSamples((int)Math.floor((int)samples_out[0].position() / sampleDivisor)); + return writeFrame((AVFrame)null); + } + return samples != null ? frame.key_frame() != 0 : writeFrame((AVFrame)null); } @@ -1338,6 +1351,7 @@ private void writeSamples(int nb_samples) throws Exception { frame.format(audio_c.sample_fmt()); frame.quality(audio_c.global_quality()); writeFrame(frame); + wrote_samples = true; } private boolean writeFrame(AVFrame frame) throws Exception { @@ -1376,8 +1390,7 @@ private boolean writeFrame(AVFrame frame) throws Exception { /* write the compressed frame in the media file */ writePacket(AVMEDIA_TYPE_AUDIO, audio_pkt); - if (frame == null) { - // avoid infinite loop with buggy codecs on flush + if (frame == null && !wrote_samples) { break; } }