From 8632d7ebda028d8a229c31c2683d6e5e4a2ca0b7 Mon Sep 17 00:00:00 2001 From: Austin Borger Date: Tue, 8 Aug 2023 16:26:39 -0700 Subject: [PATCH 01/22] Camera2Video: Fix MediaRecorder issues. MediaRecorder recordings were sideways in portrait orientation, and would not correctly record the HardwarePipeline. The former was due to the lack of an orientation hint to MediaRecorder and an incorrectly sized Surface. The latter was due to the presentation time being set, which is incompatible with MediaRecorder. Passing the Surface to a temporary MediaRecorder will set its size correctly, and passing an orientation hint will rotate the output video. Removing presentation time will ensure that frames' presentation time in the output video start at zero and increment by 1/FPS seconds for each frame. onReady would also be signaled at the beginning of recording for the software path, which has been replaced by onClose. Change-Id: Ib6d578a408413440c2071245944fc161db783cd3 --- .../android/camera2/video/EncoderWrapper.kt | 75 +++++++++++-------- .../android/camera2/video/HardwarePipeline.kt | 2 - .../video/fragments/PreviewFragment.kt | 14 ++-- 3 files changed, 53 insertions(+), 38 deletions(-) diff --git a/Camera2Video/app/src/main/java/com/example/android/camera2/video/EncoderWrapper.kt b/Camera2Video/app/src/main/java/com/example/android/camera2/video/EncoderWrapper.kt index 418beef5..d973d62d 100644 --- a/Camera2Video/app/src/main/java/com/example/android/camera2/video/EncoderWrapper.kt +++ b/Camera2Video/app/src/main/java/com/example/android/camera2/video/EncoderWrapper.kt @@ -50,8 +50,13 @@ class EncoderWrapper(width: Int, val IFRAME_INTERVAL = 1 // sync one frame every second } + private val mWidth = width + private val mHeight = height + private val mBitRate = bitRate + private val mFrameRate = frameRate + private val mDynamicRange = dynamicRange private val mOrientationHint = orientationHint - + private val mOutputFile = outputFile private val mUseMediaRecorder = useMediaRecorder private val mMimeType = when { @@ -79,17 +84,48 @@ class EncoderWrapper(width: Int, private val mInputSurface: Surface by lazy { if (useMediaRecorder) { // Get a persistent Surface from MediaCodec, don't forget to release when done - MediaCodec.createPersistentInputSurface() + val surface = MediaCodec.createPersistentInputSurface() + + // Prepare and release a dummy MediaRecorder with our new surface + // Required to allocate an appropriately sized buffer before passing the Surface as the + // output target to the capture session + createRecorder(surface).apply { + prepare() + release() + } + + surface } else { mEncoder!!.createInputSurface() } } - private val mMediaRecorder: MediaRecorder? by lazy { - if (useMediaRecorder) { - MediaRecorder() - } else { - null + private var mMediaRecorder: MediaRecorder? = null + + private fun createRecorder(surface: Surface): MediaRecorder { + return MediaRecorder().apply { + setAudioSource(MediaRecorder.AudioSource.MIC) + setVideoSource(MediaRecorder.VideoSource.SURFACE) + setOutputFormat(MediaRecorder.OutputFormat.MPEG_4) + setOutputFile(mOutputFile.absolutePath) + setVideoEncodingBitRate(mBitRate) + if (mFrameRate > 0) setVideoFrameRate(mFrameRate) + setVideoSize(mWidth, mHeight) + + val videoEncoder = when { + mDynamicRange == DynamicRangeProfiles.STANDARD -> + MediaRecorder.VideoEncoder.H264 + mDynamicRange < DynamicRangeProfiles.PUBLIC_MAX -> + MediaRecorder.VideoEncoder.HEVC + else -> throw IllegalArgumentException("Unknown dynamic range format") + } + + setVideoEncoder(videoEncoder) + setAudioEncoder(MediaRecorder.AudioEncoder.AAC) + setAudioEncodingBitRate(16) + setAudioSamplingRate(44100) + setInputSurface(surface) + setOrientationHint(mOrientationHint) } } @@ -98,30 +134,7 @@ class EncoderWrapper(width: Int, */ init { if (useMediaRecorder) { - mMediaRecorder!!.apply { - Log.i(TAG, "useMediaRecorder"); - setAudioSource(MediaRecorder.AudioSource.MIC) - setVideoSource(MediaRecorder.VideoSource.SURFACE) - setOutputFormat(MediaRecorder.OutputFormat.MPEG_4) - setOutputFile(outputFile.absolutePath) - setVideoEncodingBitRate(bitRate) - if (frameRate > 0) setVideoFrameRate(frameRate) - setVideoSize(width, height) - - val videoEncoder = when { - dynamicRange == DynamicRangeProfiles.STANDARD -> - MediaRecorder.VideoEncoder.H264 - dynamicRange < DynamicRangeProfiles.PUBLIC_MAX -> - MediaRecorder.VideoEncoder.HEVC - else -> throw IllegalArgumentException("Unknown dynamic range format") - } - - setVideoEncoder(videoEncoder) - setAudioEncoder(MediaRecorder.AudioEncoder.AAC) - setAudioEncodingBitRate(16) - setAudioSamplingRate(44100) - setInputSurface(mInputSurface) - } + mMediaRecorder = createRecorder(mInputSurface) } else { val codecProfile = when { dynamicRange == DynamicRangeProfiles.HLG10 -> diff --git a/Camera2Video/app/src/main/java/com/example/android/camera2/video/HardwarePipeline.kt b/Camera2Video/app/src/main/java/com/example/android/camera2/video/HardwarePipeline.kt index 5169630b..d4f24e20 100644 --- a/Camera2Video/app/src/main/java/com/example/android/camera2/video/HardwarePipeline.kt +++ b/Camera2Video/app/src/main/java/com/example/android/camera2/video/HardwarePipeline.kt @@ -948,8 +948,6 @@ class HardwarePipeline(width: Int, height: Int, fps: Int, filterOn: Boolean, tra encoder.frameAvailable() - EGLExt.eglPresentationTimeANDROID(eglDisplay, eglEncoderSurface, - cameraTexture.getTimestamp()) EGL14.eglSwapBuffers(eglDisplay, eglEncoderSurface) } diff --git a/Camera2Video/app/src/main/java/com/example/android/camera2/video/fragments/PreviewFragment.kt b/Camera2Video/app/src/main/java/com/example/android/camera2/video/fragments/PreviewFragment.kt index aa7f9dcd..4957db9b 100644 --- a/Camera2Video/app/src/main/java/com/example/android/camera2/video/fragments/PreviewFragment.kt +++ b/Camera2Video/app/src/main/java/com/example/android/camera2/video/fragments/PreviewFragment.kt @@ -287,7 +287,8 @@ class PreviewFragment : Fragment() { val previewTargets = pipeline.getPreviewTargets() // Start a capture session using our open camera and list of Surfaces where frames will go - session = createCaptureSession(camera, previewTargets, cameraHandler) + session = createCaptureSession(camera, previewTargets, cameraHandler, + recordingCompleteOnClose = (pipeline !is SoftwarePipeline)) // Sends the capture request as frequently as possible until the session is torn down or // session.stopRepeating() is called @@ -323,7 +324,8 @@ class PreviewFragment : Fragment() { val recordTargets = pipeline.getRecordTargets() session.close() - session = createCaptureSession(camera, recordTargets, cameraHandler) + session = createCaptureSession(camera, recordTargets, cameraHandler, + recordingCompleteOnClose = true) session.setRepeatingRequest(recordRequest, object : CameraCaptureSession.CaptureCallback() { @@ -352,6 +354,7 @@ class PreviewFragment : Fragment() { encoder.waitForFirstFrame() session.stopRepeating() + session.close() pipeline.clearFrameListener() fragmentBinding.captureButton.setOnTouchListener(null) @@ -464,7 +467,8 @@ class PreviewFragment : Fragment() { private suspend fun createCaptureSession( device: CameraDevice, targets: List, - handler: Handler + handler: Handler, + recordingCompleteOnClose: Boolean ): CameraCaptureSession = suspendCoroutine { cont -> val stateCallback = object: CameraCaptureSession.StateCallback() { override fun onConfigured(session: CameraCaptureSession) = cont.resume(session) @@ -476,8 +480,8 @@ class PreviewFragment : Fragment() { } /** Called after all captures have completed - shut down the encoder */ - override fun onReady(session: CameraCaptureSession) { - if (!isCurrentlyRecording()) { + override fun onClosed(session: CameraCaptureSession) { + if (!recordingCompleteOnClose or !isCurrentlyRecording()) { return } From 9f7b654af1c36fe296845000331a1f7cac922bd6 Mon Sep 17 00:00:00 2001 From: Austin Borger Date: Tue, 26 Sep 2023 16:54:53 -0700 Subject: [PATCH 02/22] Add support for color space selection. On devices that support ColorSpaceProfiles and DynamicRangeProfiles, a new fragment for selecting the color space will appear allowing the user to select P3 for capture. Change-Id: Ice8e9529fb982585985667299e1c92650ba2e451 --- Camera2Video/app/build.gradle | 8 +- Camera2Video/app/src/main/AndroidManifest.xml | 4 +- .../video/fragments/ColorSpaceFragment.kt | 151 ++++++++++++++++++ .../video/fragments/DynamicRangeFragment.kt | 29 +++- .../video/fragments/EncodeApiFragment.kt | 2 +- .../camera2/video/fragments/FilterFragment.kt | 6 +- .../video/fragments/PreviewFragment.kt | 9 +- .../fragments/PreviewStabilizationFragment.kt | 4 +- .../video/fragments/RecordModeFragment.kt | 9 +- .../video/fragments/SelectorFragment.kt | 22 ++- .../video/fragments/TransferFragment.kt | 2 +- .../app/src/main/res/navigation/nav_graph.xml | 73 ++++++++- 12 files changed, 290 insertions(+), 29 deletions(-) create mode 100644 Camera2Video/app/src/main/java/com/example/android/camera2/video/fragments/ColorSpaceFragment.kt diff --git a/Camera2Video/app/build.gradle b/Camera2Video/app/build.gradle index f48b4583..163c662b 100644 --- a/Camera2Video/app/build.gradle +++ b/Camera2Video/app/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright 2020 The Android Open Source Project + * Copyright 2023 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,12 +20,12 @@ apply plugin: 'kotlin-kapt' apply plugin: "androidx.navigation.safeargs" android { - compileSdkVersion 33 + compileSdkVersion 34 defaultConfig { testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" applicationId "com.android.example.camera2.video" - minSdkVersion 23 - targetSdkVersion 33 + minSdkVersion 26 + targetSdkVersion 34 versionCode 1 versionName "1.0.0" } diff --git a/Camera2Video/app/src/main/AndroidManifest.xml b/Camera2Video/app/src/main/AndroidManifest.xml index db50613a..c9e30d9f 100644 --- a/Camera2Video/app/src/main/AndroidManifest.xml +++ b/Camera2Video/app/src/main/AndroidManifest.xml @@ -1,6 +1,6 @@ -