我从相机预览中拍摄一张照片，然后将获得的字节数组保存到jpeg文件中。
现在我想将该图像文件（jpeg）保存/编码为时长 2 秒的视频文件（mp4）。
我了解 Android 4.3 中的 MediaMuxer，并且我尝试使用 https://android.googlesource.com/platform/cts/+/jb-mr2-release/tests/tests/media/src/android/media/ 中的示例cts/EncodeDecodeTest.java,
但没有成功，即我得到一个空白的视频 mp4 文件。
我使用适用于 Android 的 ffmpeg 库将图像转换为 mp4 视频，但需要很长时间，如果可能的话，我想避免使用第三方库。 请帮助我解决我的问题。
谢谢你。
基于存储库此处，请参阅我编写的代码：
纹理渲染器.kt
class TextureRenderer {
private val vertexShaderCode =
"precision highp float;\n" +
"attribute vec3 vertexPosition;\n" +
"attribute vec2 uvs;\n" +
"varying vec2 varUvs;\n" +
"uniform mat4 mvp;\n" +
"\n" +
"void main()\n" +
"{\n" +
"\tvarUvs = uvs;\n" +
"\tgl_Position = mvp * vec4(vertexPosition, 1.0);\n" +
"}"
private val fragmentShaderCode =
"precision mediump float;\n" +
"\n" +
"varying vec2 varUvs;\n" +
"uniform sampler2D texSampler;\n" +
"\n" +
"void main()\n" +
"{\t\n" +
"\tgl_FragColor = texture2D(texSampler, varUvs);\n" +
"}"
private var vertices = floatArrayOf(
// x, y, z, u, v
-1.0f, -1.0f, 0.0f, 0f, 0f,
-1.0f, 1.0f, 0.0f, 0f, 1f,
1.0f, 1.0f, 0.0f, 1f, 1f,
1.0f, -1.0f, 0.0f, 1f, 0f
)
private var indices = intArrayOf(
2, 1, 0, 0, 3, 2
)
private var program: Int
private var vertexHandle: Int = 0
private var bufferHandles = IntArray(2)
private var uvsHandle: Int = 0
private var mvpHandle: Int = 0
private var samplerHandle: Int = 0
private val textureHandle = IntArray(1)
private var vertexBuffer: FloatBuffer = ByteBuffer.allocateDirect(vertices.size * 4).run {
order(ByteOrder.nativeOrder())
asFloatBuffer().apply {
put(vertices)
position(0)
}
}
private var indexBuffer: IntBuffer = ByteBuffer.allocateDirect(indices.size * 4).run {
order(ByteOrder.nativeOrder())
asIntBuffer().apply {
put(indices)
position(0)
}
}
init {
// Create program
val vertexShader: Int = loadShader(GLES20.GL_VERTEX_SHADER, vertexShaderCode)
val fragmentShader: Int = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentShaderCode)
program = GLES20.glCreateProgram().also {
GLES20.glAttachShader(it, vertexShader)
GLES20.glAttachShader(it, fragmentShader)
GLES20.glLinkProgram(it)
vertexHandle = GLES20.glGetAttribLocation(it, "vertexPosition")
uvsHandle = GLES20.glGetAttribLocation(it, "uvs")
mvpHandle = GLES20.glGetUniformLocation(it, "mvp")
samplerHandle = GLES20.glGetUniformLocation(it, "texSampler")
}
// Initialize buffers
GLES20.glGenBuffers(2, bufferHandles, 0)
GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, bufferHandles[0])
GLES20.glBufferData(GLES20.GL_ARRAY_BUFFER, vertices.size * 4, vertexBuffer, GLES20.GL_DYNAMIC_DRAW)
GLES20.glBindBuffer(GLES20.GL_ELEMENT_ARRAY_BUFFER, bufferHandles[1])
GLES20.glBufferData(GLES20.GL_ELEMENT_ARRAY_BUFFER, indices.size * 4, indexBuffer, GLES20.GL_DYNAMIC_DRAW)
// Init texture handle
GLES20.glGenTextures(1, textureHandle, 0)
// Ensure I can draw transparent stuff that overlaps properly
GLES20.glEnable(GLES20.GL_BLEND)
GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA)
}
private fun loadShader(type: Int, shaderCode: String): Int {
return GLES20.glCreateShader(type).also { shader ->
GLES20.glShaderSource(shader, shaderCode)
GLES20.glCompileShader(shader)
}
}
fun draw(viewportWidth: Int, viewportHeight: Int, bitmap: Bitmap, mvpMatrix: FloatArray) {
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT or GLES20.GL_DEPTH_BUFFER_BIT)
GLES20.glClearColor(0f, 0f, 0f, 0f)
GLES20.glViewport(0, 0, viewportWidth, viewportHeight)
GLES20.glUseProgram(program)
// Pass transformations to shader
GLES20.glUniformMatrix4fv(mvpHandle, 1, false, mvpMatrix, 0)
// Prepare texture for drawing
GLES20.glActiveTexture(GLES20.GL_TEXTURE0)
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureHandle[0])
GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1)
GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, bitmap, 0)
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST)
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_NEAREST)
// Prepare buffers with vertices and indices & draw
GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, bufferHandles[0])
GLES20.glBindBuffer(GLES20.GL_ELEMENT_ARRAY_BUFFER, bufferHandles[1])
GLES20.glEnableVertexAttribArray(vertexHandle)
GLES20.glVertexAttribPointer(vertexHandle, 3, GLES20.GL_FLOAT, false, 4 * 5, 0)
GLES20.glEnableVertexAttribArray(uvsHandle)
GLES20.glVertexAttribPointer(uvsHandle, 2, GLES20.GL_FLOAT, false, 4 * 5, 3 * 4)
GLES20.glDrawElements(GLES20.GL_TRIANGLES, 6, GLES20.GL_UNSIGNED_INT, 0)
}
}
TimeLapseEncoder.kt
class TimeLapseEncoder {
private var renderer: TextureRenderer? = null
// MediaCodec and encoding configuration
private var encoder: MediaCodec? = null
private var muxer: MediaMuxer? = null
private var mime = "video/avc"
private var trackIndex = -1
private var presentationTimeUs = 0L
private var frameRate = 30.0
private val timeoutUs = 10000L
private val bufferInfo = MediaCodec.BufferInfo()
private var size: Size? = null
// EGL
private var eglDisplay: EGLDisplay? = null
private var eglContext: EGLContext? = null
private var eglSurface: EGLSurface? = null
// Surface provided by MediaCodec and used to get data produced by OpenGL
private var surface: Surface? = null
fun prepareForEncoding(outVideoFilePath: String, bitmapWidth: Int, bitmapHeight: Int): Boolean {
try {
encoder = MediaCodec.createEncoderByType(mime)
// Try to find supported size by checking the resolution of first supplied image
// This could also be set manually as parameter to TimeLapseEncoder
size = getBestSupportedResolution(encoder!!, mime, Size(bitmapWidth, bitmapHeight))
val format = getFormat(size!!)
encoder!!.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
// Prepare surface
initEgl()
// Switch to executing state - we're ready to encode
encoder!!.start()
// Prepare muxer
muxer = MediaMuxer(outVideoFilePath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4)
renderer = TextureRenderer()
return true
} catch (e: Exception) {
releaseEncoder()
return false
}
}
fun encodeFrame(bitmap: Bitmap, delay: Int): Boolean {
return try {
frameRate = 1000.0 / delay
drainEncoder(false)
renderer!!.draw(size!!.width, size!!.height, bitmap, getMvp())
EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, presentationTimeUs * 1000)
EGL14.eglSwapBuffers(eglDisplay, eglSurface)
true
} catch (e: Exception) {
releaseEncoder()
false
}
}
fun finishEncoding(): Boolean {
return try {
drainEncoder(true)
true
} catch (e: Exception) {
false
} finally {
releaseEncoder()
}
}
private fun getBestSupportedResolution(mediaCodec: MediaCodec, mime: String, preferredResolution: Size): Size? {
// First check if exact combination supported
if (mediaCodec.codecInfo.getCapabilitiesForType(mime)
.videoCapabilities.isSizeSupported(preferredResolution.width, preferredResolution.height))
return preferredResolution
// I prefer similar resolution with similar aspect
val pix = preferredResolution.width * preferredResolution.height
val preferredAspect = preferredResolution.width.toFloat() / preferredResolution.height.toFloat()
// I try the resolutions suggested by docs for H.264 and VP8
// https://developer.android.com/guide/topics/media/media-formats#video-encoding
// TODO: find more supported resolutions
val resolutions = arrayListOf(
Size(176, 144), Size(320, 240), Size(320, 180),
Size(640, 360), Size(720, 480), Size(1280, 720),
Size(1920, 1080)
)
resolutions.sortWith(compareBy({ pix - it.width * it.height },
// First compare by aspect
{
val aspect = if (it.width < it.height) it.width.toFloat() / it.height.toFloat()
else it.height.toFloat() / it.width.toFloat()
(preferredAspect - aspect).absoluteValue
}))
for (size in resolutions) {
if (mediaCodec.codecInfo.getCapabilitiesForType(mime)
.videoCapabilities.isSizeSupported(size.width, size.height)
)
return size
}
return null
}
private fun getFormat(size: Size): MediaFormat {
val format = MediaFormat.createVideoFormat(mime, size.width, size.height)
format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface)
format.setInteger(MediaFormat.KEY_BIT_RATE, 2000000)
format.setInteger(MediaFormat.KEY_FRAME_RATE, 60)
format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 15)
return format
}
private fun initEgl() {
surface = encoder!!.createInputSurface()
eglDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY)
if (eglDisplay == EGL14.EGL_NO_DISPLAY)
throw RuntimeException("eglDisplay == EGL14.EGL_NO_DISPLAY: " + GLUtils.getEGLErrorString(EGL14.eglGetError()))
val version = IntArray(2)
if (!EGL14.eglInitialize(eglDisplay, version, 0, version, 1))
throw RuntimeException("eglInitialize(): " + GLUtils.getEGLErrorString(EGL14.eglGetError()))
val attribList = intArrayOf(
EGL14.EGL_RED_SIZE, 8,
EGL14.EGL_GREEN_SIZE, 8,
EGL14.EGL_BLUE_SIZE, 8,
EGL14.EGL_ALPHA_SIZE, 8,
EGL14.EGL_RENDERABLE_TYPE, EGL14.EGL_OPENGL_ES2_BIT,
EGLExt.EGL_RECORDABLE_ANDROID, 1,
EGL14.EGL_NONE
)
val configs = arrayOfNulls<EGLConfig>(1)
val nConfigs = IntArray(1)
EGL14.eglChooseConfig(eglDisplay, attribList, 0, configs, 0, configs.size, nConfigs, 0)
var err = EGL14.eglGetError()
if (err != EGL14.EGL_SUCCESS)
throw RuntimeException(GLUtils.getEGLErrorString(err))
val ctxAttribs = intArrayOf(EGL14.EGL_CONTEXT_CLIENT_VERSION, 2, EGL14.EGL_NONE)
eglContext = EGL14.eglCreateContext(eglDisplay, configs[0], EGL14.EGL_NO_CONTEXT, ctxAttribs, 0)
err = EGL14.eglGetError()
if (err != EGL14.EGL_SUCCESS)
throw RuntimeException(GLUtils.getEGLErrorString(err))
val surfaceAttribs = intArrayOf(EGL14.EGL_NONE)
eglSurface = EGL14.eglCreateWindowSurface(eglDisplay, configs[0], surface, surfaceAttribs, 0)
err = EGL14.eglGetError()
if (err != EGL14.EGL_SUCCESS)
throw RuntimeException(GLUtils.getEGLErrorString(err))
if (!EGL14.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext))
throw RuntimeException("eglMakeCurrent(): " + GLUtils.getEGLErrorString(EGL14.eglGetError()))
}
private fun drainEncoder(endOfStream: Boolean) {
if (endOfStream)
encoder!!.signalEndOfInputStream()
while (true) {
val outBufferId = encoder!!.dequeueOutputBuffer(bufferInfo, timeoutUs)
if (outBufferId >= 0) {
val encodedBuffer = encoder!!.getOutputBuffer(outBufferId)!!
// MediaMuxer is ignoring KEY_FRAMERATE, so I set it manually here
// to achieve the desired frame rate
bufferInfo.presentationTimeUs = presentationTimeUs
muxer!!.writeSampleData(trackIndex, encodedBuffer, bufferInfo)
presentationTimeUs += (1000000.0 / frameRate).toLong()
encoder!!.releaseOutputBuffer(outBufferId, false)
// Are we finished here?
if ((bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0)
break
} else if (outBufferId == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (!endOfStream)
break
// End of stream, but still no output available. Try again.
} else if (outBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
trackIndex = muxer!!.addTrack(encoder!!.outputFormat)
muxer!!.start()
}
}
}
private fun getMvp(): FloatArray {
val mvp = FloatArray(16)
Matrix.setIdentityM(mvp, 0)
Matrix.scaleM(mvp, 0, 1f, -1f, 1f)
return mvp
}
private fun releaseEncoder() {
encoder?.stop()
encoder?.release()
encoder = null
releaseEgl()
muxer?.stop()
muxer?.release()
muxer = null
size = null
trackIndex = -1
presentationTimeUs = 0L
}
private fun releaseEgl() {
if (eglDisplay != EGL14.EGL_NO_DISPLAY) {
EGL14.eglDestroySurface(eglDisplay, eglSurface)
EGL14.eglDestroyContext(eglDisplay, eglContext)
EGL14.eglReleaseThread()
EGL14.eglTerminate(eglDisplay)
}
surface?.release()
surface = null
eglDisplay = EGL14.EGL_NO_DISPLAY
eglContext = EGL14.EGL_NO_CONTEXT
eglSurface = EGL14.EGL_NO_SURFACE
}
}
用途：
val outputPath = ...
val videoFile = File(outputPath)
if (videoFile.exists())
videoFile.delete()
videoFile.parentFile!!.mkdirs()
val timeLapseEncoder = TimeLapseEncoder()
val width=...
val height=...
timeLapseEncoder.prepareForEncoding(outputPath, width, height))
val bitmap=...
val delay=... //in ms, of this specific frame
timeLapseEncoder.encodeFrame(bitmap, delay)
timeLapseEncoder.finishEncoding()
编辑：更新的解决方案，因为上面有一些问题，使用更简化的方法来复用到 MP4：
class BitmapToVideoEncoder(outputPath: String?, width: Int, height: Int, bitRate: Int, frameRate: Int) {
private var encoder: MediaCodec?
private val inputSurface: Surface
private var mediaMuxer: MediaMuxer?
private var videoTrackIndex = 0
private var isMuxerStarted: Boolean
private var presentationTimeUs: Long
init {
val format = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, width, height)
format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface)
format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate)
format.setInteger(MediaFormat.KEY_FRAME_RATE, frameRate)
format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1)
encoder = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_VIDEO_AVC)
encoder!!.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
inputSurface = encoder!!.createInputSurface()
encoder!!.start()
mediaMuxer = MediaMuxer(outputPath!!, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4)
isMuxerStarted = false
presentationTimeUs = 0
}
@Throws(IOException::class)
fun encodeFrame(bitmap: Bitmap, durationInMs: Long) {
val frameDurationUs = durationInMs * 1000
drawBitmapToSurface(bitmap)
drainEncoder(false)
presentationTimeUs += frameDurationUs
}
@Throws(IOException::class)
fun finishEncoding() {
drainEncoder(true)
release()
}
private fun drawBitmapToSurface(bitmap: Bitmap) {
val canvas = inputSurface.lockCanvas(null)
canvas.drawBitmap(bitmap, 0f, 0f, null)
inputSurface.unlockCanvasAndPost(canvas)
}
@Throws(IOException::class)
private fun drainEncoder(endOfStream: Boolean) {
if (endOfStream) {
//Sending end of stream signal to encoder
encoder!!.signalEndOfInputStream()
}
val bufferInfo = MediaCodec.BufferInfo()
while (true) {
val encoderStatus = encoder!!.dequeueOutputBuffer(bufferInfo, 10000)
@Suppress("DEPRECATION")
when {
encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER -> {
if (!endOfStream) {
break
}
}
encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED -> {
//Output buffers changed
}
encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> {
if (isMuxerStarted) {
throw RuntimeException("format changed twice")
}
val newFormat = encoder!!.outputFormat
videoTrackIndex = mediaMuxer!!.addTrack(newFormat)
mediaMuxer!!.start()
isMuxerStarted = true
}
encoderStatus < 0 -> {
// Unexpected result from encoder
}
else -> {
val encodedData = encoder!!.getOutputBuffer(encoderStatus)
?: throw RuntimeException("encoderOutputBuffer $encoderStatus was null")
if (bufferInfo.size != 0) {
if (!isMuxerStarted) {
throw RuntimeException("muxer hasn't started")
}
// Adjust the bufferInfo to have the correct presentation time
bufferInfo.presentationTimeUs = presentationTimeUs
encodedData.position(bufferInfo.offset)
encodedData.limit(bufferInfo.offset + bufferInfo.size)
mediaMuxer!!.writeSampleData(videoTrackIndex, encodedData, bufferInfo)
}
encoder!!.releaseOutputBuffer(encoderStatus, false)
if ((bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
//End of stream reached
break
}
}
}
}
}
private fun release() {
if (encoder != null) {
encoder!!.stop()
encoder!!.release()
encoder = null
}
if (mediaMuxer != null) {
mediaMuxer!!.stop()
mediaMuxer!!.release()
mediaMuxer = null
}
}
}
这里处理的是所有输入位图的分辨率与 CTOR 参数相同且不透明的情况。此外，输入分辨率应与设备可以处理编码的分辨率相匹配。
为了解决这个问题，有两种方法：
至于你可以处理的分辨率，我需要检查一下这样的东西支持什么：
MediaCodec codec = MediaCodec.createEncoderByType(mimeType);
MediaCodecInfo codecInfo = codec.getCodecInfo();
MediaCodecInfo.CodecCapabilities capabilities = codecInfo.getCapabilitiesForType(mimeType);
MediaCodecInfo.VideoCapabilities videoCapabilities = capabilities.getVideoCapabilities();
codec.release();
我没有在这里添加这个，因为它变得更复杂。我可能会将它添加到存储库中，或者准备拥有它。
根据我的测试，这应该可以正常工作。