From 270fd2b1cde8b9c5d4e65df3e301299c17674b96 Mon Sep 17 00:00:00 2001 From: Austin Borger Date: Wed, 25 Oct 2023 11:39:28 -0700 Subject: [PATCH] Camera2Video: Fix stability issues due to race conditions The app is not waiting for rendering / encoding operations to complete before the preview fragment ends. This leads to exceptions in render / encode, and an app crash. Bug: 307767427 Test: Ran Camera2Video down various paths, observed no crashes. Change-Id: I1896fa7753ddf8c1e9022fffb1719fdd8ed7ee43 --- .../android/camera2/video/HardwarePipeline.kt | 47 +++++++++++++++---- .../video/fragments/PreviewFragment.kt | 6 ++- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/Camera2Video/app/src/main/java/com/example/android/camera2/video/HardwarePipeline.kt b/Camera2Video/app/src/main/java/com/example/android/camera2/video/HardwarePipeline.kt index 5d523930..86f1dd3f 100644 --- a/Camera2Video/app/src/main/java/com/example/android/camera2/video/HardwarePipeline.kt +++ b/Camera2Video/app/src/main/java/com/example/android/camera2/video/HardwarePipeline.kt @@ -388,6 +388,7 @@ class HardwarePipeline(width: Int, height: Int, fps: Int, filterOn: Boolean, tra override fun destroyWindowSurface() { renderHandler.sendMessage(renderHandler.obtainMessage( RenderHandler.MSG_DESTROY_WINDOW_SURFACE)) + renderHandler.waitDestroyWindowSurface() } override fun setPreviewSize(previewSize: Size) { @@ -415,11 +416,13 @@ class HardwarePipeline(width: Int, height: Int, fps: Int, filterOn: Boolean, tra override fun clearFrameListener() { renderHandler.sendMessage(renderHandler.obtainMessage( RenderHandler.MSG_CLEAR_FRAME_LISTENER)) + renderHandler.waitClearFrameListener() } override fun cleanup() { renderHandler.sendMessage(renderHandler.obtainMessage( RenderHandler.MSG_CLEANUP)) + renderHandler.waitCleanup() } private class ShaderProgram(id: Int, @@ -513,9 +516,9 @@ class HardwarePipeline(width: Int, height: Int, fps: Int, filterOn: Boolean, tra private var eglDisplay = EGL14.EGL_NO_DISPLAY private var eglContext = EGL14.EGL_NO_CONTEXT private var eglConfig: EGLConfig? = null - private var eglRenderSurface: EGLSurface? = null - private var eglEncoderSurface: EGLSurface? = null - private var eglWindowSurface: EGLSurface? = null + private var eglRenderSurface: EGLSurface = EGL14.EGL_NO_SURFACE + private var eglEncoderSurface: EGLSurface = EGL14.EGL_NO_SURFACE + private var eglWindowSurface: EGLSurface = EGL14.EGL_NO_SURFACE private var vertexShader = 0 private var cameraToRenderFragmentShader = 0 private var renderToPreviewFragmentShader = 0 @@ -526,6 +529,9 @@ class HardwarePipeline(width: Int, height: Int, fps: Int, filterOn: Boolean, tra private var renderToEncodeShaderProgram: ShaderProgram? = null private val cvResourcesCreated = ConditionVariable(false) + private val cvDestroyWindowSurface = ConditionVariable(false) + private val cvClearFrameListener = ConditionVariable(false) + private val cvCleanup = ConditionVariable(false) public fun startRecording() { currentlyRecording = true @@ -865,7 +871,12 @@ class HardwarePipeline(width: Int, height: Int, fps: Int, filterOn: Boolean, tra private fun destroyWindowSurface() { EGL14.eglDestroySurface(eglDisplay, eglWindowSurface) - eglWindowSurface = null + eglWindowSurface = EGL14.EGL_NO_SURFACE + cvDestroyWindowSurface.open() + } + + public fun waitDestroyWindowSurface() { + cvDestroyWindowSurface.block() } private fun copyTexture(texId: Int, texture: SurfaceTexture, viewportRect: Rect, @@ -964,36 +975,54 @@ class HardwarePipeline(width: Int, height: Int, fps: Int, filterOn: Boolean, tra private fun clearFrameListener() { cameraTexture.setOnFrameAvailableListener(null) + cvClearFrameListener.open() + } + + public fun waitClearFrameListener() { + cvClearFrameListener.block() } private fun cleanup() { EGL14.eglDestroySurface(eglDisplay, eglEncoderSurface) - eglEncoderSurface = null + eglEncoderSurface = EGL14.EGL_NO_SURFACE EGL14.eglDestroySurface(eglDisplay, eglRenderSurface) - eglRenderSurface = null + eglRenderSurface = EGL14.EGL_NO_SURFACE cameraTexture.release() EGL14.eglDestroyContext(eglDisplay, eglContext) + + eglDisplay = EGL14.EGL_NO_DISPLAY + eglContext = EGL14.EGL_NO_CONTEXT + + cvCleanup.open() + } + + public fun waitCleanup() { + cvCleanup.block() } @Suppress("UNUSED_PARAMETER") private fun onFrameAvailableImpl(surfaceTexture: SurfaceTexture) { + if (eglContext == EGL14.EGL_NO_CONTEXT) { + return + } + /** The camera API does not update the tex image. Do so here. */ cameraTexture.updateTexImage() /** Copy from the camera texture to the render texture */ - if (eglRenderSurface != null) { + if (eglRenderSurface != EGL14.EGL_NO_SURFACE) { copyCameraToRender() } /** Copy from the render texture to the TextureView */ - if (eglWindowSurface != null) { + if (eglWindowSurface != EGL14.EGL_NO_SURFACE) { copyRenderToPreview() } /** Copy to the encoder surface if we're currently recording. */ - if (eglEncoderSurface != null && currentlyRecording) { + if (eglEncoderSurface != EGL14.EGL_NO_SURFACE && currentlyRecording) { copyRenderToEncode() } } diff --git a/Camera2Video/app/src/main/java/com/example/android/camera2/video/fragments/PreviewFragment.kt b/Camera2Video/app/src/main/java/com/example/android/camera2/video/fragments/PreviewFragment.kt index 4e97380b..de63697b 100644 --- a/Camera2Video/app/src/main/java/com/example/android/camera2/video/fragments/PreviewFragment.kt +++ b/Camera2Video/app/src/main/java/com/example/android/camera2/video/fragments/PreviewFragment.kt @@ -376,11 +376,11 @@ class PreviewFragment : Fragment() { delay(CameraActivity.ANIMATION_SLOW_MILLIS) + pipeline.cleanup() + Log.d(TAG, "Recording stopped. Output file: $outputFile") encoder.shutdown() - pipeline.cleanup() - // Broadcasts the media file to the rest of the system MediaScannerConnection.scanFile( requireView().context, arrayOf(outputFile.absolutePath), null, null) @@ -512,6 +512,8 @@ class PreviewFragment : Fragment() { override fun onDestroy() { super.onDestroy() + pipeline.clearFrameListener() + pipeline.cleanup() cameraThread.quitSafely() encoderSurface.release() }