From 2eac6daa7c7ff10aac99d3a066b373484dd00300 Mon Sep 17 00:00:00 2001 From: Josh Bultman Date: Mon, 2 Dec 2024 11:50:27 -0600 Subject: [PATCH 01/10] Break out of the FFmpegFrameRecorder writePacket loop after maxRetries iterations instead of a single iteration. --- src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java index 47f0fa31..c32986ef 100644 --- a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java +++ b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java @@ -1350,6 +1350,8 @@ private boolean writeFrame(AVFrame frame) throws Exception { frame.pts(frame.pts() + frame.nb_samples()); // magic required by libvorbis and webm } + int retries = 0; + int maxRetries = 1000; /* if zero size, it means the image was buffered */ got_audio_packet[0] = 0; while (ret >= 0) { @@ -1376,7 +1378,7 @@ private boolean writeFrame(AVFrame frame) throws Exception { /* write the compressed frame in the media file */ writePacket(AVMEDIA_TYPE_AUDIO, audio_pkt); - if (frame == null) { + if (frame == null && retries++ > maxRetries) { // avoid infinite loop with buggy codecs on flush break; } From bc8af5551767f4da800c078fa0eac6e5c910ced2 Mon Sep 17 00:00:00 2001 From: Josh Bultman Date: Mon, 2 Dec 2024 12:16:02 -0600 Subject: [PATCH 02/10] Revert "Break out of the FFmpegFrameRecorder writePacket loop after maxRetries iterations instead of a single iteration." This reverts commit 2eac6daa7c7ff10aac99d3a066b373484dd00300. --- src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java index c32986ef..47f0fa31 100644 --- a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java +++ b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java @@ -1350,8 +1350,6 @@ private boolean writeFrame(AVFrame frame) throws Exception { frame.pts(frame.pts() + frame.nb_samples()); // magic required by libvorbis and webm } - int retries = 0; - int maxRetries = 1000; /* if zero size, it means the image was buffered */ got_audio_packet[0] = 0; while (ret >= 0) { @@ -1378,7 +1376,7 @@ private boolean writeFrame(AVFrame frame) throws Exception { /* write the compressed frame in the media file */ writePacket(AVMEDIA_TYPE_AUDIO, audio_pkt); - if (frame == null && retries++ > maxRetries) { + if (frame == null) { // avoid infinite loop with buggy codecs on flush break; } From fce1be3683a2aea97af1f41ccfb9205df5387cd5 Mon Sep 17 00:00:00 2001 From: Josh Bultman Date: Mon, 2 Dec 2024 13:47:19 -0600 Subject: [PATCH 03/10] Reapply "Break out of the FFmpegFrameRecorder writePacket loop after maxRetries iterations instead of a single iteration." This reverts commit bc8af5551767f4da800c078fa0eac6e5c910ced2. --- src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java index 47f0fa31..c32986ef 100644 --- a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java +++ b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java @@ -1350,6 +1350,8 @@ private boolean writeFrame(AVFrame frame) throws Exception { frame.pts(frame.pts() + frame.nb_samples()); // magic required by libvorbis and webm } + int retries = 0; + int maxRetries = 1000; /* if zero size, it means the image was buffered */ got_audio_packet[0] = 0; while (ret >= 0) { @@ -1376,7 +1378,7 @@ private boolean writeFrame(AVFrame frame) throws Exception { /* write the compressed frame in the media file */ writePacket(AVMEDIA_TYPE_AUDIO, audio_pkt); - if (frame == null) { + if (frame == null && retries++ > maxRetries) { // avoid infinite loop with buggy codecs on flush break; } From e5c7b1530f42bd0423c0d1249efe270d25c0cf31 Mon Sep 17 00:00:00 2001 From: Josh Bultman Date: Mon, 2 Dec 2024 13:48:16 -0600 Subject: [PATCH 04/10] Fix testFFmpegFrameFilterMultipleInputs expecting incorrect number of audio channels. --- platform/src/test/java/org/bytedeco/javacv/FrameFilterTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/src/test/java/org/bytedeco/javacv/FrameFilterTest.java b/platform/src/test/java/org/bytedeco/javacv/FrameFilterTest.java index 89fbce7c..1bec4cfb 100644 --- a/platform/src/test/java/org/bytedeco/javacv/FrameFilterTest.java +++ b/platform/src/test/java/org/bytedeco/javacv/FrameFilterTest.java @@ -197,7 +197,7 @@ public void testFFmpegFrameFilterMultipleInputs() { } if (frame3.samples != null) { d++; - assertEquals(2, frame3.audioChannels); + assertEquals(4, frame3.audioChannels); assertEquals(1, frame3.samples.length); assertTrue(frame3.samples[0] instanceof ByteBuffer); assertEquals(frame2.samples.length, frame3.samples.length); From 4135b463f8f613329b95722972c02dd64a522b9e Mon Sep 17 00:00:00 2001 From: Josh Bultman Date: Mon, 2 Dec 2024 14:15:07 -0600 Subject: [PATCH 05/10] Fix testFFmpegFrameFilterMultipleInputs expecting incorrect number of audio channels. --- .../src/test/java/org/bytedeco/javacv/FrameFilterTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platform/src/test/java/org/bytedeco/javacv/FrameFilterTest.java b/platform/src/test/java/org/bytedeco/javacv/FrameFilterTest.java index 1bec4cfb..07733118 100644 --- a/platform/src/test/java/org/bytedeco/javacv/FrameFilterTest.java +++ b/platform/src/test/java/org/bytedeco/javacv/FrameFilterTest.java @@ -199,9 +199,9 @@ public void testFFmpegFrameFilterMultipleInputs() { d++; assertEquals(4, frame3.audioChannels); assertEquals(1, frame3.samples.length); - assertTrue(frame3.samples[0] instanceof ByteBuffer); + assertTrue(frame3.samples[0] instanceof ShortBuffer); assertEquals(frame2.samples.length, frame3.samples.length); - assertEquals(frame2.samples[0].limit(), frame3.samples[0].limit()); + assertEquals(2 * frame2.samples[0].limit(), frame3.samples[0].limit()); } } } From 1f066029c8bd87624123d7d38db167dc22470251 Mon Sep 17 00:00:00 2001 From: Josh Bultman Date: Tue, 3 Dec 2024 15:16:45 -0600 Subject: [PATCH 06/10] Fix bug that was breaking out of the swr_convert loop early before all samples_in were sent. --- .../java/org/bytedeco/javacv/FFmpegFrameRecorder.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java index c32986ef..723df36c 100644 --- a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java +++ b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java @@ -77,7 +77,6 @@ import org.bytedeco.javacpp.ShortPointer; import org.bytedeco.ffmpeg.avcodec.*; -import org.bytedeco.ffmpeg.avdevice.*; import org.bytedeco.ffmpeg.avformat.*; import org.bytedeco.ffmpeg.avutil.*; import org.bytedeco.ffmpeg.swresample.*; @@ -1285,6 +1284,7 @@ public synchronized boolean recordSamples(int sampleRate, int audioChannels, Buf limit((samples_in[i].position() + inputSize) * inputDepth); } while (true) { + // TODO: We're dropping any samples that don't fit into the sample size. We might want to pad instead. int inputCount = (int)Math.min(samples != null ? (samples_in[0].limit() - samples_in[0].position()) / (inputChannels * inputDepth) : 0, Integer.MAX_VALUE); int outputCount = (int)Math.min((samples_out[0].limit() - samples_out[0].position()) / (outputChannels * outputDepth), Integer.MAX_VALUE); inputCount = Math.min(inputCount, (outputCount * sampleRate + audio_c.sample_rate() - 1) / audio_c.sample_rate()); @@ -1296,7 +1296,7 @@ public synchronized boolean recordSamples(int sampleRate, int audioChannels, Buf } if ((ret = swr_convert(samples_convert_ctx, plane_ptr2, outputCount, plane_ptr, inputCount)) < 0) { throw new Exception("swr_convert() error " + ret + ": Cannot convert audio samples."); - } else if (ret == 0) { + } else if (ret == 0 && inputCount == 0) { break; } for (int i = 0; samples != null && i < samples.length; i++) { @@ -1350,8 +1350,6 @@ private boolean writeFrame(AVFrame frame) throws Exception { frame.pts(frame.pts() + frame.nb_samples()); // magic required by libvorbis and webm } - int retries = 0; - int maxRetries = 1000; /* if zero size, it means the image was buffered */ got_audio_packet[0] = 0; while (ret >= 0) { @@ -1377,11 +1375,6 @@ private boolean writeFrame(AVFrame frame) throws Exception { /* write the compressed frame in the media file */ writePacket(AVMEDIA_TYPE_AUDIO, audio_pkt); - - if (frame == null && retries++ > maxRetries) { - // avoid infinite loop with buggy codecs on flush - break; - } } return got_audio_packet[0] != 0; From 5950f49e7596296a1899c4b17e295109ea4a1836 Mon Sep 17 00:00:00 2001 From: Josh Bultman Date: Tue, 3 Dec 2024 16:35:00 -0600 Subject: [PATCH 07/10] Remove TODO --- src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java index 723df36c..aeb89260 100644 --- a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java +++ b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java @@ -1284,7 +1284,6 @@ public synchronized boolean recordSamples(int sampleRate, int audioChannels, Buf limit((samples_in[i].position() + inputSize) * inputDepth); } while (true) { - // TODO: We're dropping any samples that don't fit into the sample size. We might want to pad instead. int inputCount = (int)Math.min(samples != null ? (samples_in[0].limit() - samples_in[0].position()) / (inputChannels * inputDepth) : 0, Integer.MAX_VALUE); int outputCount = (int)Math.min((samples_out[0].limit() - samples_out[0].position()) / (outputChannels * outputDepth), Integer.MAX_VALUE); inputCount = Math.min(inputCount, (outputCount * sampleRate + audio_c.sample_rate() - 1) / audio_c.sample_rate()); From 8b85ac1a6c96d1ec89fe5aee98c8040dcb3f2793 Mon Sep 17 00:00:00 2001 From: Josh Bultman Date: Wed, 4 Dec 2024 15:25:04 -0600 Subject: [PATCH 08/10] Fix bug where swr_context wasn't getting flushed. Add wrote samples check to avoid infinite loop when Vorbis attempts to flush encoder before writing any samples. --- .../bytedeco/javacv/FFmpegFrameRecorder.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java index aeb89260..aee3cd5c 100644 --- a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java +++ b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java @@ -396,6 +396,7 @@ static class SeekCallback extends Seek_Pointer_long_int { private PointerPointer plane_ptr, plane_ptr2; private AVPacket video_pkt, audio_pkt; private int[] got_video_packet, got_audio_packet; + private boolean wrote_samples = false; private AVFormatContext ifmt_ctx; private IntPointer display_matrix; private AVChannelLayout default_layout; @@ -467,6 +468,7 @@ public synchronized void startUnsafe() throws Exception { plane_ptr2 = new PointerPointer(AVFrame.AV_NUM_DATA_POINTERS).retainReference(); video_pkt = new AVPacket().retainReference(); audio_pkt = new AVPacket().retainReference(); + wrote_samples = false; got_video_packet = new int[1]; got_audio_packet = new int[1]; default_layout = new AVChannelLayout().retainReference(); @@ -1175,6 +1177,11 @@ public synchronized boolean recordSamples(int sampleRate, int audioChannels, Buf return writeFrame((AVFrame)null); } + if (samples == null && samples_convert_ctx == null) { + // We haven't tried to record any samples yet so we don't need to flush. + return false; + } + int ret; if (sampleRate <= 0) { @@ -1264,7 +1271,8 @@ public synchronized boolean recordSamples(int sampleRate, int audioChannels, Buf throw new Exception("Audio samples Buffer has unsupported type: " + samples); } - if (samples_convert_ctx == null || samples_channels != audioChannels || samples_format != inputFormat || samples_rate != sampleRate) { + boolean formatChanged = samples_channels != audioChannels || samples_format != inputFormat || samples_rate != sampleRate; + if (samples != null && (samples_convert_ctx == null || formatChanged)) { if (samples_convert_ctx == null) { samples_convert_ctx = new SwrContext().retainReference(); } @@ -1293,6 +1301,11 @@ public synchronized boolean recordSamples(int sampleRate, int audioChannels, Buf for (int i = 0; i < samples_out.length; i++) { plane_ptr2.put(i, samples_out[i]); } + if (samples == null && inputCount == 0 && plane_ptr != null) { + plane_ptr.releaseReference(); + // needs to be null to flush swr context. + plane_ptr = null; + } if ((ret = swr_convert(samples_convert_ctx, plane_ptr2, outputCount, plane_ptr, inputCount)) < 0) { throw new Exception("swr_convert() error " + ret + ": Cannot convert audio samples."); } else if (ret == 0 && inputCount == 0) { @@ -1337,6 +1350,7 @@ private void writeSamples(int nb_samples) throws Exception { frame.format(audio_c.sample_fmt()); frame.quality(audio_c.global_quality()); writeFrame(frame); + wrote_samples = true; } private boolean writeFrame(AVFrame frame) throws Exception { @@ -1374,6 +1388,10 @@ private boolean writeFrame(AVFrame frame) throws Exception { /* write the compressed frame in the media file */ writePacket(AVMEDIA_TYPE_AUDIO, audio_pkt); + + if (frame == null && !wrote_samples) { + break; + } } return got_audio_packet[0] != 0; From d3fe0b7814abe66bfdde7f78788ec251301fb58a Mon Sep 17 00:00:00 2001 From: Josh Bultman Date: Thu, 5 Dec 2024 13:45:22 -0600 Subject: [PATCH 09/10] Fix bug where samples_out was being flushed before flushing the swr_context. --- .../org/bytedeco/javacv/FFmpegFrameRecorder.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java index aee3cd5c..be8ce262 100644 --- a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java +++ b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java @@ -1170,13 +1170,6 @@ public synchronized boolean recordSamples(int sampleRate, int audioChannels, Buf throw new Exception("start() was not called successfully!"); } - if (samples == null && samples_out[0].position() > 0) { - // Typically samples_out[0].limit() is double the audio_input_frame_size --> sampleDivisor = 2 - double sampleDivisor = Math.floor((int)Math.min(samples_out[0].limit(), Integer.MAX_VALUE) / audio_input_frame_size); - writeSamples((int)Math.floor((int)samples_out[0].position() / sampleDivisor)); - return writeFrame((AVFrame)null); - } - if (samples == null && samples_convert_ctx == null) { // We haven't tried to record any samples yet so we don't need to flush. return false; @@ -1322,6 +1315,14 @@ public synchronized boolean recordSamples(int sampleRate, int audioChannels, Buf writeSamples(audio_input_frame_size); } } + + if (samples == null && samples_out[0].position() > 0) { + // Typically samples_out[0].limit() is double the audio_input_frame_size --> sampleDivisor = 2 + double sampleDivisor = Math.floor((int)Math.min(samples_out[0].limit(), Integer.MAX_VALUE) / audio_input_frame_size); + writeSamples((int)Math.floor((int)samples_out[0].position() / sampleDivisor)); + return writeFrame((AVFrame)null); + } + return samples != null ? frame.key_frame() != 0 : writeFrame((AVFrame)null); } From b48ec2e41d006ecdcf3328c498bdfbcf74aa2a3c Mon Sep 17 00:00:00 2001 From: Samuel Audet Date: Sun, 15 Dec 2024 19:56:00 +0900 Subject: [PATCH 10/10] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b794b8c6..07f055ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ + * Fix `FFmpegFrameRecorder` dropped frame issues with audio samples ([pull #2307](https://github.com/bytedeco/javacv/pull/2307)) * Add `FrameFilter.videoFilterArgs/audioFilterArgs` properties to support multiple different inputs ([pull #2304](https://github.com/bytedeco/javacv/pull/2304)) * Ensure `FFmpegFrameGrabber.start()` skips over streams with no codecs ([issue #2299](https://github.com/bytedeco/javacv/issues/2299)) * Add `FFmpegLogCallback.logRejectedOptions()` for debugging purposes ([pull #2301](https://github.com/bytedeco/javacv/pull/2301))