我正在使用 MediaCodec 来解码视频，由于某种原因，我无法使用 Surface 配置编解码器。所以我使用 EGL 渲染视频。这是我的渲染逻辑。
videoDecoder.setStateListener(object : DefDecoderStateListener {
override fun decodeOneFrame(
codec: MediaCodec, // already set the format MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible
bufferInfo: MediaCodec.BufferInfo?,
outputBuffers: Array<ByteBuffer>,
index: Int
) {
// renderer draw
val image = codec.getOutputImage(index)!!
yuvDrawer.setImage(image)
mRenderer.notifySwap(bufferInfo?.presentationTimeUs ?: 0)
}
})
我的想法是从输出图像中获取 YUV 缓冲区并通过 YuvDrawer 渲染它。 但是当我运行代码时，我得到了错误的结果。我的YuvDrawer有问题吗？
package com.cxp.learningvideo.opengl.drawer
import android.graphics.SurfaceTexture
import android.media.Image
import android.opengl.GLES11Ext
import android.opengl.GLES20
import android.opengl.Matrix
import android.util.Log
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.nio.FloatBuffer
class YuvDrawer: IDrawer {
private val mVertexCoors = floatArrayOf(
-1f, -1f,
1f, -1f,
-1f, 1f,
1f, 1f
)
private val mTextureCoors = floatArrayOf(
0f, 1f,
1f, 1f,
0f, 0f,
1f, 0f
)
private var mWorldWidth: Int = -1
private var mWorldHeight: Int = -1
private var mVideoWidth: Int = -1
private var mVideoHeight: Int = -1
private var mTextureId: Int = -1
private var mSurfaceTexture: SurfaceTexture? = null
private var mSftCb: (() -> Unit)? = null
private var mProgram: Int = -1
private var mVertexMatrixHandler: Int = -1
private var mVertexPosHandler: Int = -1
private var mTexturePosHandler: Int = -1
private var mTextureYHandler: Int = -1
private var mTextureUHandler: Int = -1
private var mTextureVHandler: Int = -1
private var mAlphaHandler: Int = -1
private lateinit var mVertexBuffer: FloatBuffer
private lateinit var mTextureBuffer: FloatBuffer
private var mMatrix: FloatArray? = null
private var mAlpha = 1f
private val textures = IntArray(3)
private var bufferY: ByteBuffer? = null
private var bufferU: ByteBuffer? = null
private var bufferV: ByteBuffer? = null
private var image: Image? = null
init {
initPos()
}
private fun initPos() {
val bb = ByteBuffer.allocateDirect(mVertexCoors.size * 4)
bb.order(ByteOrder.nativeOrder())
mVertexBuffer = bb.asFloatBuffer()
mVertexBuffer.put(mVertexCoors)
mVertexBuffer.position(0)
val cc = ByteBuffer.allocateDirect(mTextureCoors.size * 4)
cc.order(ByteOrder.nativeOrder())
mTextureBuffer = cc.asFloatBuffer()
mTextureBuffer.put(mTextureCoors)
mTextureBuffer.position(0)
}
private var mWidthRatio = 1f
private var mHeightRatio = 1f
private fun initDefMatrix() {
if (mMatrix != null) return
if (mVideoWidth != -1 && mVideoHeight != -1 &&
mWorldWidth != -1 && mWorldHeight != -1) {
mMatrix = FloatArray(16)
var prjMatrix = FloatArray(16)
val originRatio = mVideoWidth / mVideoHeight.toFloat()
val worldRatio = mWorldWidth / mWorldHeight.toFloat()
if (mWorldWidth > mWorldHeight) {
if (originRatio > worldRatio) {
mHeightRatio = originRatio / worldRatio
Matrix.orthoM(
prjMatrix, 0,
-mWidthRatio, mWidthRatio,
-mHeightRatio, mHeightRatio,
3f, 5f
)
} else {
mWidthRatio = worldRatio / originRatio
Matrix.orthoM(
prjMatrix, 0,
-mWidthRatio, mWidthRatio,
-mHeightRatio, mHeightRatio,
3f, 5f
)
}
} else {
if (originRatio > worldRatio) {
mHeightRatio = originRatio / worldRatio
Matrix.orthoM(
prjMatrix, 0,
-mWidthRatio, mWidthRatio,
-mHeightRatio, mHeightRatio,
3f, 5f
)
} else {
mWidthRatio = worldRatio / originRatio
Matrix.orthoM(
prjMatrix, 0,
-mWidthRatio, mWidthRatio,
-mHeightRatio, mHeightRatio,
3f, 5f
)
}
}
val viewMatrix = FloatArray(16)
Matrix.setLookAtM(
viewMatrix, 0,
0f, 0f, 5.0f,
0f, 0f, 0f,
0f, 1.0f, 0f
)
Matrix.multiplyMM(mMatrix, 0, prjMatrix, 0, viewMatrix, 0)
}
}
fun setImage(image: Image) {
this.image = image
try {
bufferY = this.image!!.planes[0].buffer
Log.d("testt", "setImage: buffer y $bufferY")
bufferU = this.image!!.planes[1].buffer
bufferV = this.image!!.planes[2].buffer
} catch (e: Exception) {
e.printStackTrace()
Log.d("YuvDrawer", "setImage: exception")
}
}
override fun setVideoSize(videoW: Int, videoH: Int) {
mVideoWidth = videoW
mVideoHeight = videoH
}
override fun setWorldSize(worldW: Int, worldH: Int) {
mWorldWidth = worldW
mWorldHeight = worldH
}
override fun setAlpha(alpha: Float) {
mAlpha = alpha
}
fun setSurfaceCreateListener(listener: () -> Unit) {
mSftCb = listener
}
private fun createGLPrg() {
if (mProgram == -1) {
val vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, getVertexShader())
val fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, getFragmentShader())
mProgram = GLES20.glCreateProgram()
GLES20.glAttachShader(mProgram, vertexShader)
GLES20.glAttachShader(mProgram, fragmentShader)
GLES20.glLinkProgram(mProgram)
mVertexMatrixHandler = GLES20.glGetUniformLocation(mProgram, "uMatrix")
mVertexPosHandler = GLES20.glGetAttribLocation(mProgram, "a_Position")
mTextureYHandler = GLES20.glGetUniformLocation(mProgram, "textureY")
mTextureUHandler = GLES20.glGetUniformLocation(mProgram, "textureU")
mTextureVHandler = GLES20.glGetUniformLocation(mProgram, "textureV")
mTexturePosHandler = GLES20.glGetAttribLocation(mProgram, "aCoordinate")
mAlphaHandler = GLES20.glGetAttribLocation(mProgram, "alpha")
}
GLES20.glUseProgram(mProgram)
}
private fun activateTexture() {
GLES20.glActiveTexture(GLES20.GL_TEXTURE0)
// y
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textures[0])
GLES20.glTexImage2D(
GLES20.GL_TEXTURE_2D,
0,
GLES20.GL_LUMINANCE,
mVideoWidth,
mVideoHeight,
0,
GLES20.GL_LUMINANCE,
GLES20.GL_UNSIGNED_BYTE,
bufferY
)
// u
GLES20.glActiveTexture(GLES20.GL_TEXTURE1)
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textures[1])
GLES20.glTexImage2D(
GLES20.GL_TEXTURE_2D,
0,
GLES20.GL_LUMINANCE,
mVideoWidth / 2,
mVideoHeight / 2,
0,
GLES20.GL_LUMINANCE,
GLES20.GL_UNSIGNED_BYTE,
bufferU
)
//v
GLES20.glActiveTexture(GLES20.GL_TEXTURE2)
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textures[2])
GLES20.glTexImage2D(
GLES20.GL_TEXTURE_2D,
0,
GLES20.GL_LUMINANCE,
mVideoWidth / 2,
mVideoHeight / 2,
0,
GLES20.GL_LUMINANCE,
GLES20.GL_UNSIGNED_BYTE,
bufferV
)
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR.toFloat())
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR.toFloat())
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE)
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE)
// bufferY?.clear()
// bufferU?.clear()
// bufferV?.clear()
}
override fun draw() {
initDefMatrix()
activateTexture()
doDraw()
bufferY = null
bufferU = null
bufferV = null
image?.close()
}
private fun doDraw() {
GLES20.glEnableVertexAttribArray(mVertexPosHandler)
GLES20.glEnableVertexAttribArray(mTexturePosHandler)
GLES20.glUniformMatrix4fv(mVertexMatrixHandler, 1, false, mMatrix, 0)
GLES20.glVertexAttribPointer(mVertexPosHandler, 2, GLES20.GL_FLOAT, false, 0, mVertexBuffer)
GLES20.glVertexAttribPointer(mTexturePosHandler, 2, GLES20.GL_FLOAT, false, 0, mTextureBuffer)
GLES20.glVertexAttrib1f(mAlphaHandler, mAlpha)
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4)
}
override fun setTextureID(id: Int) {
createGLPrg()
GLES20.glUniform1i(mTextureYHandler, 0)
GLES20.glUniform1i(mTextureUHandler, 1)
GLES20.glUniform1i(mTextureVHandler, 2)
GLES20.glGenTextures(3, textures, 0)
for(i in 0..2) {
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textures[i])
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_REPEAT)
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_REPEAT)
GLES20.glTexParameteri(
GLES20.GL_TEXTURE_2D,
GLES20.GL_TEXTURE_MIN_FILTER,
GLES20.GL_NEAREST
)
GLES20.glTexParameteri(
GLES20.GL_TEXTURE_2D,
GLES20.GL_TEXTURE_MAG_FILTER,
GLES20.GL_LINEAR
)
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0)
}
mSftCb?.invoke()
}
override fun release() {
GLES20.glDisableVertexAttribArray(mVertexPosHandler)
GLES20.glDisableVertexAttribArray(mTexturePosHandler)
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0)
GLES20.glDeleteTextures(1, intArrayOf(mTextureId), 0)
GLES20.glDeleteProgram(mProgram)
}
private fun loadShader(type: Int, shaderCode: String): Int {
val shader = GLES20.glCreateShader(type)
GLES20.glShaderSource(shader, shaderCode)
GLES20.glCompileShader(shader)
return shader
}
fun translate(dx: Float, dy: Float) {
Matrix.translateM(mMatrix, 0, dx*mWidthRatio*2, -dy*mHeightRatio*2, 0f)
}
fun scale(sx: Float, sy: Float) {
Matrix.scaleM(mMatrix, 0, sx, sy, 1f)
mWidthRatio /= sx
mHeightRatio /= sy
}
private fun getVertexShader(): String {
return """#version 300 es
layout(location = 0) in vec4 a_Position;
layout(location = 1) in vec2 aTexture;
out vec2 vTexture;
void main()
{
gl_Position = a_Position;
vTexture = aTexture;
}
"""
}
private fun getFragmentShader(): String {
return """#version 300 es
precision mediump float;
out vec4 FragColor;
in vec2 vTexture;
uniform sampler2D textureY;
uniform sampler2D textureU;
uniform sampler2D textureV;
void main() {
float y,u,v;
vec3 rgb;
y = texture(textureY, vTexture).r;
u = texture(textureU, vTexture).g - 0.5;
v = texture(textureV, vTexture).b - 0.5;
rgb.r = y + 1.540 * v;
rgb.g = y - 0.183*u - 0.459*v;
rgb.b = y + 1.818*u;
FragColor = vec4(rgb, 1.0);
}
"""
}
}
这个渲染逻辑有意义吗还是我应该考虑另一种方式。
mTexturePosHandler = GLES20.glGetAttribLocation(mProgram, "aCoordinate") -> mTexturePosHandler = GLES20.glGetAttribLocation(mProgram, "aTexture")