From 449371f36ace0df96058dc6469f78457b1f05093 Mon Sep 17 00:00:00 2001 From: thegobot Date: Tue, 16 Nov 2021 16:44:56 +0800 Subject: [PATCH] SrsEncoder video reconfigure on fly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If you reset MediaCodec in the middle of the stream, then the video freezes (artifacts) in the players. After the experiments, I seem to understand what the reason is. Not all predicred frames have time to be sent to the stream (the encoder stops). Therefore, first we complete the sending of all P-frames (this is signaled by the MediaCodec.BUFFER_FLAG_KEY_FRAME), then we reconfigure the encoder with new parameters. Tested in latest Chrome and FF. The video now changes format and continues to play Perhaps you will have other thoughts about this. Please give comments ChunkDemuxer Selected FFmpegAudioDecoder for audio decoding, config: codec: aac, profile: unknown, bytes_per_channel: 2, channel_layout: STEREO, channels: 2, samples_per_second: 44100, sample_format: Signed 16-bit, bytes_per_frame: 4, seek_preroll: 0us, codec_delay: 0, has extra data: false, encryption scheme: Unencrypted, discard decoder delay: false, target_output_channel_layout: STEREO, has aac extra data: true Video rendering in low delay mode. Failed to initialize DecryptingVideoDecoder Effective playback rate changed from 0 to 1 Starting Initialization of DXVAVDA Using D3D9 device for DXVA Selected VDAVideoDecoder for video decoding, config: codec: h264, profile: h264 baseline, level: not available, alpha_mode: is_opaque, coded size: [480,640], visible rect: [0,0,480,640], natural size: [480,640], has extra data: false, encryption scheme: Unencrypted, rotation: 0°, flipped: 0, color space: {primaries:BT709, transfer:BT709, matrix:BT709, range:LIMITED} video decoder config changed midstream, new config: codec: h264, profile: h264 baseline, level: not available, alpha_mode: is_opaque, coded size: [1080,1920], visible rect: [0,0,1080,1920], natural size: [1080,1920], has extra data: false, encryption scheme: Unencrypted, rotation: 0°, flipped: 0, color space: {primaries:BT709, transfer:BT709, matrix:BT709, range:LIMITED} Failed to initialize VDAVideoDecoder Failed to initialize DecryptingVideoDecoder Failed to initialize VDAVideoDecoder Failed to initialize VpxVideoDecoder Failed to initialize Dav1dVideoDecoder Selected FFmpegVideoDecoder for video decoding, config: codec: h264, profile: h264 baseline, level: not available, alpha_mode: is_opaque, coded size: [1080,1920], visible rect: [0,0,1080,1920], natural size: [1080,1920], has extra data: false, encryption scheme: Unencrypted, rotation: 0°, flipped: 0, color space: {primaries:BT709, transfer:BT709, matrix:BT709, range:LIMITED} --- .../main/java/net/ossrs/yasea/SrsEncoder.java | 85 ++++++++++++++++--- 1 file changed, 72 insertions(+), 13 deletions(-) diff --git a/library/src/main/java/net/ossrs/yasea/SrsEncoder.java b/library/src/main/java/net/ossrs/yasea/SrsEncoder.java index 39501282..3e8008ae 100644 --- a/library/src/main/java/net/ossrs/yasea/SrsEncoder.java +++ b/library/src/main/java/net/ossrs/yasea/SrsEncoder.java @@ -62,6 +62,9 @@ public class SrsEncoder { private int videoMp4Track; private int audioFlvTrack; private int audioMp4Track; + + private boolean reconfiguringVideo = false; + private int requestReconfigure = 0; // Y, U (Cb) and V (Cr) // yuv420 yuv yuv yuv yuv @@ -142,18 +145,44 @@ public boolean start() { audioFlvTrack = flvMuxer.addTrack(audioFormat); audioMp4Track = mp4Muxer.addTrack(audioFormat); + MediaFormat videoFormat = videoCodecConfigure(); + + if(videoFormat == null){ + return false; + } + + + // add the video tracker to muxer. + videoFlvTrack = flvMuxer.addTrack(videoFormat); + videoMp4Track = mp4Muxer.addTrack(videoFormat); + + // start device and encoder. + vencoder.start(); + aencoder.start(); + return true; + } + + private MediaFormat videoCodecConfigure(){ // vencoder yuv to 264 es stream. // requires sdk level 16+, Android 4.1, 4.1.1, the JELLY_BEAN - try { - vencoder = MediaCodec.createByCodecName(vmci.getName()); - } catch (IOException e) { - Log.e(TAG, "create vencoder failed."); - e.printStackTrace(); - return false; + Log.d(TAG, "VideoCodec configure"); + + setEncoderResolution(vOutWidth, vOutHeight); + + + if(!reconfiguringVideo || vencoder == null) { + try { + vencoder = MediaCodec.createByCodecName(vmci.getName()); + } catch (IOException e) { + Log.e(TAG, "create vencoder failed."); + e.printStackTrace(); + return null; + } } // setup the vencoder. // Note: landscape to portrait, 90 degree rotation, so we need to switch width and height in configuration + MediaFormat videoFormat = MediaFormat.createVideoFormat(VCODEC, vOutWidth, vOutHeight); videoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, mVideoColorFormat); videoFormat.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 0); @@ -161,15 +190,35 @@ public boolean start() { videoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, VFPS); videoFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, VGOP / VFPS); vencoder.configure(videoFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); - // add the video tracker to muxer. - videoFlvTrack = flvMuxer.addTrack(videoFormat); - videoMp4Track = mp4Muxer.addTrack(videoFormat); - // start device and encoder. + flvMuxer.addTrack(videoFormat); + + return videoFormat; + } + + + public void videoCodecReConfigure(){ + + + if(requestReconfigure == 0){ + Log.e(TAG, "Request reconfigure"); + requestReconfigure = 1; + return; + } + + if(requestReconfigure == 1){ + throw new IllegalStateException("reconfiguring"); + } + + Log.d(TAG, "VideoCodec reconfigure: " + vOutWidth +"x"+ vOutHeight); + reconfiguringVideo = true; + + vencoder.reset(); + videoCodecConfigure(); vencoder.start(); - aencoder.start(); - return true; - } + requestReconfigure = 0; + + } public void pause(){ mPausetime = System.nanoTime() / 1000; @@ -328,6 +377,16 @@ private void onProcessedYuvFrame(byte[] yuvFrame, long pts) { ByteBuffer bb = outBuffers[outBufferIndex]; onEncodedAnnexbFrame(bb, vebi); vencoder.releaseOutputBuffer(outBufferIndex, false); + + if(vebi.flags == MediaCodec.BUFFER_FLAG_KEY_FRAME){ + if(requestReconfigure == 1){ + //reconfigure when all P (predicred) frames have been sent + requestReconfigure = 2; + videoCodecReConfigure(); + } + + } + } else { break; }