From ead9d723ae480c275b85a2d620c85e3d9e2b9a45 Mon Sep 17 00:00:00 2001 From: Samuel Audet Date: Wed, 19 Jan 2022 14:09:04 +0900 Subject: [PATCH] * Upgrade dependencies for FFmpeg 5.0 --- CHANGELOG.md | 2 +- platform/pom.xml | 2 +- pom.xml | 3 +- .../bytedeco/javacv/FFmpegFrameFilter.java | 6 +- .../bytedeco/javacv/FFmpegFrameGrabber.java | 20 +-- .../bytedeco/javacv/FFmpegFrameRecorder.java | 145 ++++++++++-------- 6 files changed, 95 insertions(+), 83 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index afc4e04c..069791a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ * Add `charset` property to `FrameGrabber` and `FrameRecorder` to use for metadata from FFmpeg ([pull #1720](https://github.com/bytedeco/javacv/pull/1720)) * Call `Frame.close()` on temporary clones in `Java2DFrameUtils` to prevent premature deallocations ([issue #1716](https://github.com/bytedeco/javacv/issues/1716)) * Ignore errors from `avcodec_send_packet()` and `avcodec_receive_frame()` to emulate old API in `FFmpegFrameGrabber` ([issue #1679](https://github.com/bytedeco/javacv/issues/1679)) - * Upgrade dependencies for OpenBLAS 0.3.19, OpenCV 4.5.5, FFmpeg 4.4.1, librealsense2 2.50.0, Leptonica 1.82.0, Tesseract 5.0.1 + * Upgrade dependencies for OpenBLAS 0.3.19, OpenCV 4.5.5, FFmpeg 5.0, librealsense2 2.50.0, Leptonica 1.82.0, Tesseract 5.0.1 ### August 2, 2021 version 1.5.6 * Enhance audio and video synchronization of `JavaFxPlayVideoAndAudio` sample ([pull #1662](https://github.com/bytedeco/javacv/pull/1662)) diff --git a/platform/pom.xml b/platform/pom.xml index 33022286..d50ff596 100644 --- a/platform/pom.xml +++ b/platform/pom.xml @@ -38,7 +38,7 @@ org.bytedeco ffmpeg-platform - 4.4.1-${javacpp.version} + 5.0-${javacpp.version} org.bytedeco diff --git a/pom.xml b/pom.xml index 8f4ba0d7..2b86b251 100644 --- a/pom.xml +++ b/pom.xml @@ -80,7 +80,7 @@ org.bytedeco ffmpeg - 4.4.1-${javacpp.version} + 5.0-${javacpp.version} org.bytedeco @@ -195,6 +195,7 @@ 1.7 1.7 + org/bytedeco/javacv/FFmpegLockCallback.java org/bytedeco/javacv/FlyCaptureFrameGrabber.java diff --git a/src/main/java/org/bytedeco/javacv/FFmpegFrameFilter.java b/src/main/java/org/bytedeco/javacv/FFmpegFrameFilter.java index 4e59e3b5..725cfec2 100644 --- a/src/main/java/org/bytedeco/javacv/FFmpegFrameFilter.java +++ b/src/main/java/org/bytedeco/javacv/FFmpegFrameFilter.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2015-2020 Samuel Audet + * Copyright (C) 2015-2022 Samuel Audet * * Licensed either under the Apache License, Version 2.0, or (at your option) * under the terms of the GNU General Public License as published by @@ -104,8 +104,8 @@ public static void tryLoad() throws Exception { Loader.load(org.bytedeco.ffmpeg.global.swscale.class); Loader.load(org.bytedeco.ffmpeg.global.avfilter.class); - av_register_all(); - avfilter_register_all(); +// av_register_all(); +// avfilter_register_all(); } catch (Throwable t) { if (t instanceof Exception) { throw loadingException = (Exception)t; diff --git a/src/main/java/org/bytedeco/javacv/FFmpegFrameGrabber.java b/src/main/java/org/bytedeco/javacv/FFmpegFrameGrabber.java index 03065294..234c2fc4 100644 --- a/src/main/java/org/bytedeco/javacv/FFmpegFrameGrabber.java +++ b/src/main/java/org/bytedeco/javacv/FFmpegFrameGrabber.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2009-2021 Samuel Audet + * Copyright (C) 2009-2022 Samuel Audet * * Licensed either under the Apache License, Version 2.0, or (at your option) * under the terms of the GNU General Public License as published by @@ -115,8 +115,8 @@ public static void tryLoad() throws Exception { // Register all formats and codecs av_jni_set_java_vm(Loader.getJavaVM(), null); - avcodec_register_all(); - av_register_all(); +// avcodec_register_all(); +// av_register_all(); avformat_network_init(); Loader.load(org.bytedeco.ffmpeg.global.avdevice.class); @@ -131,12 +131,12 @@ public static void tryLoad() throws Exception { } } - static { - try { - tryLoad(); - FFmpegLockCallback.init(); - } catch (Exception ex) { } - } +// static { +// try { +// tryLoad(); +// FFmpegLockCallback.init(); +// } catch (Exception ex) { } +// } public FFmpegFrameGrabber(File file) { this(file.getAbsolutePath()); @@ -296,7 +296,7 @@ static class ReadCallback extends Read_packet_Pointer_BytePointer_int { InputStream is = inputStreams.get(opaque); int size = is.read(b, 0, buf_size); if (size < 0) { - return 0; + return AVERROR_EOF(); } else { buf.put(b, 0, size); return size; diff --git a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java index 656b995f..1d62aaaa 100644 --- a/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java +++ b/src/main/java/org/bytedeco/javacv/FFmpegFrameRecorder.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2009-2021 Samuel Audet + * Copyright (C) 2009-2022 Samuel Audet * * Licensed either under the Apache License, Version 2.0, or (at your option) * under the terms of the GNU General Public License as published by @@ -116,8 +116,8 @@ public static void tryLoad() throws Exception { /* initialize libavcodec, and register all codecs and formats */ av_jni_set_java_vm(Loader.getJavaVM(), null); - avcodec_register_all(); - av_register_all(); +// avcodec_register_all(); +// av_register_all(); avformat_network_init(); Loader.load(org.bytedeco.ffmpeg.global.avdevice.class); @@ -132,12 +132,12 @@ public static void tryLoad() throws Exception { } } - static { - try { - tryLoad(); - FFmpegLockCallback.init(); - } catch (Exception ex) { } - } +// static { +// try { +// tryLoad(); +// FFmpegLockCallback.init(); +// } catch (Exception ex) { } +// } public FFmpegFrameRecorder(File file, int audioChannels) { this(file, 0, 0, audioChannels); @@ -466,7 +466,7 @@ public synchronized void startUnsafe() throws Exception { outputStreams.put(oc, outputStream); } oc.oformat(oformat); - oc.filename().putString(filename); + oc.url(new BytePointer(av_malloc(filename.getBytes().length + 1)).putString(filename)); oc.max_delay(maxDelay); /* add the audio and video streams using the format codecs @@ -476,40 +476,43 @@ public synchronized void startUnsafe() throws Exception { // get input video and audio stream indices from ifmt_ctx for (int idx = 0; idx < ifmt_ctx.nb_streams(); idx++) { AVStream inputStream = ifmt_ctx.streams(idx); - if (inputStream.codec().codec_type() == AVMEDIA_TYPE_VIDEO) { + if (inputStream.codecpar().codec_type() == AVMEDIA_TYPE_VIDEO) { inpVideoStream = inputStream; - videoCodec = inpVideoStream.codec().codec_id(); + videoCodec = inpVideoStream.codecpar().codec_id(); if (inpVideoStream.r_frame_rate().num() != AV_NOPTS_VALUE && inpVideoStream.r_frame_rate().den() != 0) { frameRate = (inpVideoStream.r_frame_rate().num())*1.0d / (inpVideoStream.r_frame_rate().den()); } - } else if (inputStream.codec().codec_type() == AVMEDIA_TYPE_AUDIO) { + } else if (inputStream.codecpar().codec_type() == AVMEDIA_TYPE_AUDIO) { inpAudioStream = inputStream; - audioCodec = inpAudioStream.codec().codec_id(); + audioCodec = inpAudioStream.codecpar().codec_id(); } } } if (imageWidth > 0 && imageHeight > 0) { - if (videoCodec != AV_CODEC_ID_NONE) { - oformat.video_codec(videoCodec); - } else if ("flv".equals(format_name)) { - oformat.video_codec(AV_CODEC_ID_FLV1); - } else if ("mp4".equals(format_name)) { - oformat.video_codec(AV_CODEC_ID_MPEG4); - } else if ("3gp".equals(format_name)) { - oformat.video_codec(AV_CODEC_ID_H263); - } else if ("avi".equals(format_name)) { - oformat.video_codec(AV_CODEC_ID_HUFFYUV); - } + if (videoCodec == AV_CODEC_ID_NONE) { + videoCodec = oformat.video_codec(); + } +// if (videoCodec != AV_CODEC_ID_NONE) { +// oformat.video_codec(videoCodec); +// } else if ("flv".equals(format_name)) { +// oformat.video_codec(AV_CODEC_ID_FLV1); +// } else if ("mp4".equals(format_name)) { +// oformat.video_codec(AV_CODEC_ID_MPEG4); +// } else if ("3gp".equals(format_name)) { +// oformat.video_codec(AV_CODEC_ID_H263); +// } else if ("avi".equals(format_name)) { +// oformat.video_codec(AV_CODEC_ID_HUFFYUV); +// } /* find the video encoder */ if ((video_codec = avcodec_find_encoder_by_name(videoCodecName)) == null && - (video_codec = avcodec_find_encoder(oformat.video_codec())) == null) { + (video_codec = avcodec_find_encoder(videoCodec)) == null) { releaseUnsafe(); throw new Exception("avcodec_find_encoder() error: Video codec not found."); } - oformat.video_codec(video_codec.id()); +// oformat.video_codec(video_codec.id()); AVRational frame_rate = av_d2q(frameRate, 1001000); AVRational supported_framerates = video_codec.supported_framerates(); @@ -530,19 +533,19 @@ public synchronized void startUnsafe() throws Exception { } if (inpVideoStream != null) { - if ((ret = avcodec_copy_context(video_st.codec(), inpVideoStream.codec())) < 0) { + if ((ret = avcodec_parameters_copy(video_st.codecpar(), inpVideoStream.codecpar())) < 0) { releaseUnsafe(); - throw new Exception("avcodec_copy_context() error:\tFailed to copy context from input to output stream codec context"); + throw new Exception("avcodec_parameters_copy() error " + ret + ": Failed to copy video stream codec parameters from input to output"); } - videoBitrate = (int) inpVideoStream.codec().bit_rate(); - pixelFormat = inpVideoStream.codec().pix_fmt(); - aspectRatio = inpVideoStream.codec().sample_aspect_ratio().num()*1.0d/ inpVideoStream.codec().sample_aspect_ratio().den(); - videoQuality = inpVideoStream.codec().global_quality(); + videoBitrate = (int)inpVideoStream.codecpar().bit_rate(); + pixelFormat = inpVideoStream.codecpar().format(); + aspectRatio = inpVideoStream.codecpar().sample_aspect_ratio().num()*1.0d/ inpVideoStream.codecpar().sample_aspect_ratio().den(); +// videoQuality = inpVideoStream.codecpar().global_quality(); video_c.codec_tag(0); } - video_c.codec_id(oformat.video_codec()); + video_c.codec_id(video_codec.id()); video_c.codec_type(AVMEDIA_TYPE_VIDEO); @@ -569,7 +572,7 @@ public synchronized void startUnsafe() throws Exception { video_c.time_base(time_base); video_st.time_base(time_base); video_st.avg_frame_rate(frame_rate); - video_st.codec().time_base(time_base); // "deprecated", but this is actually required +// video_st.codec().time_base(time_base); // "deprecated", but this is actually required if (gopSize >= 0) { video_c.gop_size(gopSize); /* emit one intra frame every gopSize frames at most */ } @@ -641,21 +644,24 @@ public synchronized void startUnsafe() throws Exception { * add an audio output stream */ if (audioChannels > 0 && audioBitrate > 0 && sampleRate > 0) { - if (audioCodec != AV_CODEC_ID_NONE) { - oformat.audio_codec(audioCodec); - } else if ("flv".equals(format_name) || "mp4".equals(format_name) || "3gp".equals(format_name)) { - oformat.audio_codec(AV_CODEC_ID_AAC); - } else if ("avi".equals(format_name)) { - oformat.audio_codec(AV_CODEC_ID_PCM_S16LE); - } + if (audioCodec == AV_CODEC_ID_NONE) { + audioCodec = oformat.audio_codec(); + } +// if (audioCodec != AV_CODEC_ID_NONE) { +// oformat.audio_codec(audioCodec); +// } else if ("flv".equals(format_name) || "mp4".equals(format_name) || "3gp".equals(format_name)) { +// oformat.audio_codec(AV_CODEC_ID_AAC); +// } else if ("avi".equals(format_name)) { +// oformat.audio_codec(AV_CODEC_ID_PCM_S16LE); +// } /* find the audio encoder */ if ((audio_codec = avcodec_find_encoder_by_name(audioCodecName)) == null && - (audio_codec = avcodec_find_encoder(oformat.audio_codec())) == null) { + (audio_codec = avcodec_find_encoder(audioCodec)) == null) { releaseUnsafe(); throw new Exception("avcodec_find_encoder() error: Audio codec not found."); } - oformat.audio_codec(audio_codec.id()); +// oformat.audio_codec(audio_codec.id()); AVRational sample_rate = av_d2q(sampleRate, 1001000); @@ -670,15 +676,15 @@ public synchronized void startUnsafe() throws Exception { } if (inpAudioStream != null && audioChannels > 0) { - if ((ret = avcodec_copy_context(audio_st.codec(), inpAudioStream.codec())) < 0) { - throw new Exception("avcodec_copy_context() error " + ret + ": Failed to copy context from input audio to output audio stream codec context\n"); + if ((ret = avcodec_parameters_copy(audio_st.codecpar(), inpAudioStream.codecpar())) < 0) { + throw new Exception("avcodec_parameters_copy() error " + ret + ": Failed to copy audio stream codec parameters from input to output"); } - audioBitrate = (int) inpAudioStream.codec().bit_rate(); - sampleRate = inpAudioStream.codec().sample_rate(); - audioChannels = inpAudioStream.codec().channels(); - sampleFormat = inpAudioStream.codec().sample_fmt(); - audioQuality = inpAudioStream.codec().global_quality(); + audioBitrate = (int) inpAudioStream.codecpar().bit_rate(); + sampleRate = inpAudioStream.codecpar().sample_rate(); + audioChannels = inpAudioStream.codecpar().channels(); + sampleFormat = inpAudioStream.codecpar().format(); +// audioQuality = inpAudioStream.codecpar().global_quality(); audio_c.codec_tag(0); // audio_st.pts(inpAudioStream.pts()); audio_st.duration(inpAudioStream.duration()); @@ -686,7 +692,7 @@ public synchronized void startUnsafe() throws Exception { audio_st.time_base().den(inpAudioStream.time_base().den()); } - audio_c.codec_id(oformat.audio_codec()); + audio_c.codec_id(audio_codec.id()); audio_c.codec_type(AVMEDIA_TYPE_AUDIO); @@ -711,7 +717,7 @@ public synchronized void startUnsafe() throws Exception { AVRational time_base = av_inv_q(sample_rate); audio_c.time_base(time_base); audio_st.time_base(time_base); - audio_st.codec().time_base(time_base); // "deprecated", but this is actually required +// audio_st.codec().time_base(time_base); // "deprecated", but this is actually required switch (audio_c.sample_fmt()) { case AV_SAMPLE_FMT_U8: case AV_SAMPLE_FMT_U8P: audio_c.bits_per_raw_sample(8); break; @@ -1057,8 +1063,10 @@ public synchronized boolean recordImage(int width, int height, int depth, int ch av_new_packet(video_pkt, video_outbuf_size); ret = avcodec_receive_packet(video_c, video_pkt); if (ret == AVERROR_EAGAIN() || ret == AVERROR_EOF()) { + av_packet_unref(video_pkt); break; } else if (ret < 0) { + av_packet_unref(video_pkt); throw new Exception("avcodec_receive_packet() error " + ret + ": Error during video encoding."); } got_video_packet[0] = 1; @@ -1097,7 +1105,7 @@ public synchronized boolean recordSamples(int sampleRate, int audioChannels, Buf // Typically samples_out[0].limit() is double the audio_input_frame_size --> sampleDivisor = 2 double sampleDivisor = Math.floor((int)Math.min(samples_out[0].limit(), Integer.MAX_VALUE) / audio_input_frame_size); writeSamples((int)Math.floor((int)samples_out[0].position() / sampleDivisor)); - return record((AVFrame)null); + return writeFrame((AVFrame)null); } int ret; @@ -1232,7 +1240,7 @@ public synchronized boolean recordSamples(int sampleRate, int audioChannels, Buf writeSamples(audio_input_frame_size); } } - return samples != null ? frame.key_frame() != 0 : record((AVFrame)null); + return samples != null ? frame.key_frame() != 0 : writeFrame((AVFrame)null); } } @@ -1259,10 +1267,10 @@ private void writeSamples(int nb_samples) throws Exception { frame.channels(audio_c.channels()); frame.format(audio_c.sample_fmt()); frame.quality(audio_c.global_quality()); - record(frame); + writeFrame(frame); } - private boolean record(AVFrame frame) throws Exception { + private boolean writeFrame(AVFrame frame) throws Exception { int ret; if ((ret = avcodec_send_frame(audio_c, frame)) < 0 && frame != null) { @@ -1278,8 +1286,10 @@ private boolean record(AVFrame frame) throws Exception { av_new_packet(audio_pkt, audio_outbuf_size); ret = avcodec_receive_packet(audio_c, audio_pkt); if (ret == AVERROR_EAGAIN() || ret == AVERROR_EOF()) { + av_packet_unref(audio_pkt); break; } else if (ret < 0) { + av_packet_unref(audio_pkt); throw new Exception("avcodec_receive_packet() error " + ret + ": Error during audio encoding."); } got_audio_packet[0] = 1; @@ -1295,13 +1305,17 @@ private boolean record(AVFrame frame) throws Exception { /* write the compressed frame in the media file */ writePacket(AVMEDIA_TYPE_AUDIO, audio_pkt); + + if (frame == null) { + // avoid infinite loop with buggy codecs on flush + break; + } } return got_audio_packet[0] != 0; } private void writePacket(int mediaType, AVPacket avPacket) throws Exception { - AVStream avStream = (mediaType == AVMEDIA_TYPE_VIDEO) ? video_st : (mediaType == AVMEDIA_TYPE_AUDIO) ? audio_st : null; String mediaTypeStr = (mediaType == AVMEDIA_TYPE_VIDEO) ? "video" : (mediaType == AVMEDIA_TYPE_AUDIO) ? "audio" : "unsupported media stream type"; @@ -1309,15 +1323,16 @@ private void writePacket(int mediaType, AVPacket avPacket) throws Exception { int ret; if (interleaved && avStream != null) { if ((ret = av_interleaved_write_frame(oc, avPacket)) < 0) { + av_packet_unref(avPacket); throw new Exception("av_interleaved_write_frame() error " + ret + " while writing interleaved " + mediaTypeStr + " packet."); } } else { if ((ret = av_write_frame(oc, avPacket)) < 0) { + av_packet_unref(avPacket); throw new Exception("av_write_frame() error " + ret + " while writing " + mediaTypeStr + " packet."); } } } - av_packet_unref(avPacket); } @@ -1342,18 +1357,15 @@ public synchronized boolean recordPacket(AVPacket pkt) throws Exception { // pkt.dts(AV_NOPTS_VALUE); // pkt.pts(AV_NOPTS_VALUE); pkt.pos(-1); - if (in_stream.codec().codec_type() == AVMEDIA_TYPE_VIDEO && video_st != null) { - + if (in_stream.codecpar().codec_type() == AVMEDIA_TYPE_VIDEO && video_st != null) { pkt.stream_index(video_st.index()); - pkt.duration((int) av_rescale_q(pkt.duration(), in_stream.codec().time_base(), video_st.codec().time_base())); + pkt.duration((int) av_rescale_q(pkt.duration(), in_stream.time_base(), video_st.time_base())); pkt.pts(av_rescale_q_rnd(pkt.pts(), in_stream.time_base(), video_st.time_base(),(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX)));//Increase pts calculation pkt.dts(av_rescale_q_rnd(pkt.dts(), in_stream.time_base(), video_st.time_base(),(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX)));//Increase dts calculation writePacket(AVMEDIA_TYPE_VIDEO, pkt); - - } else if (in_stream.codec().codec_type() == AVMEDIA_TYPE_AUDIO && audio_st != null && (audioChannels > 0)) { - + } else if (in_stream.codecpar().codec_type() == AVMEDIA_TYPE_AUDIO && audio_st != null && (audioChannels > 0)) { pkt.stream_index(audio_st.index()); - pkt.duration((int) av_rescale_q(pkt.duration(), in_stream.codec().time_base(), audio_st.codec().time_base())); + pkt.duration((int) av_rescale_q(pkt.duration(), in_stream.time_base(), audio_st.time_base())); pkt.pts(av_rescale_q_rnd(pkt.pts(), in_stream.time_base(), audio_st.time_base(),(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX)));//Increase pts calculation pkt.dts(av_rescale_q_rnd(pkt.dts(), in_stream.time_base(), audio_st.time_base(),(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX)));//Increase dts calculation writePacket(AVMEDIA_TYPE_AUDIO, pkt); @@ -1361,5 +1373,4 @@ public synchronized boolean recordPacket(AVPacket pkt) throws Exception { return true; } - }