我正在尝试使用MediaCodec和MediaMuxer对来自摄像头的视频和来自麦克风的音频进行编码。我在录制时使用OpenGL覆盖图像上的文本。
我把这些课作为例子：
我写了一个执行编码的主类。它产生2个线程用于录制音频和视频。它不起作用（生成的文件无效），但如果我评论其中一个线程（音频或视频），它可以正常工作。另外，我需要将TRACK_COUNT设置为1.这是主类的代码：
import android.graphics.SurfaceTexture;
import android.media.AudioFormat;
import android.media.AudioRecord;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaFormat;
import android.media.MediaMuxer;
import android.media.MediaRecorder;
import com.google.common.base.Throwables;
import java.io.IOException;
import java.nio.ByteBuffer;
import static com.google.common.base.Preconditions.checkNotNull;
/**
* Class for recording a reply including a text message.
*/
public class ReplyRecorder {
// Encoding state
private boolean encoding;
long startWhen;
// Muxer
private static final int TRACK_COUNT = 2;
private Muxer mMuxer;
// Video
private static final String VIDEO_MIME_TYPE = "video/avc"; // H.264 Advanced Video Coding
private static final int FRAME_RATE = 15; // 30fps
private static final int IFRAME_INTERVAL = 10; // 5 seconds between I-frames
private static final int BIT_RATE = 2000000;
private Encoder mVideoEncoder;
private CodecInputSurface mInputSurface;
private SurfaceTextureManager mStManager;
// Audio
private static final String AUDIO_MIME_TYPE = "audio/mp4a-latm";
private static final int SAMPLE_RATE = 44100;
private static final int SAMPLES_PER_FRAME = 1024;
private static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_MONO;
private static final int AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT;
private Encoder mAudioEncoder;
private AudioRecord audioRecord;
public void start(final CameraManager cameraManager, final String messageText, final String filePath) {
checkNotNull(cameraManager);
checkNotNull(messageText);
checkNotNull(filePath);
try {
// Create a MediaMuxer. We can't add the video track and start() the muxer here,
// because our MediaFormat doesn't have the Magic Goodies. These can only be
// obtained from the encoder after it has started processing data.
mMuxer = new Muxer(new MediaMuxer(filePath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4), TRACK_COUNT);
startWhen = System.nanoTime();
encoding = true;
new Thread(new Runnable() {
@Override
public void run() {
initVideoComponents(cameraManager, messageText);
encodeVideo(cameraManager);
}
}).start();
new Thread(new Runnable() {
@Override
public void run() {
initAudioComponents();
encodeAudio();
}
}).start();
} catch (IOException e) {
release();
throw Throwables.propagate(e);
}
}
private void initVideoComponents(CameraManager cameraManager,
String messageText) {
try {
MediaFormat format = MediaFormat.createVideoFormat(VIDEO_MIME_TYPE, cameraManager.getEncWidth(), cameraManager.getEncHeight());
// Set some properties. Failing to specify some of these can cause the MediaCodec
// configure() call to throw an unhelpful exception.
format.setInteger(MediaFormat.KEY_COLOR_FORMAT,
MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
format.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE);
format.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE);
format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL);
// Create a MediaCodec encoder, and configure it with our format. Get a Surface
// we can use for input and wrap it with a class that handles the EGL work.
//
// If you want to have two EGL contexts -- one for display, one for recording --
// you will likely want to defer instantiation of CodecInputSurface until after the
// "display" EGL context is created, then modify the eglCreateContext call to
// take eglGetCurrentContext() as the share_context argument.
mVideoEncoder = new Encoder(VIDEO_MIME_TYPE, format, mMuxer);
mInputSurface = new CodecInputSurface(mVideoEncoder.getEncoder().createInputSurface());
mVideoEncoder.getEncoder().start();
mInputSurface.makeCurrent();
mStManager = new SurfaceTextureManager(messageText, cameraManager.getEncWidth(), cameraManager.getEncHeight());
} catch (RuntimeException e) {
releaseVideo();
throw e;
}
}
private void encodeVideo(CameraManager cameraManager) {
try {
SurfaceTexture st = mStManager.getSurfaceTexture();
cameraManager.record(st);
while (encoding) {
// Feed any pending encoder output into the muxer.
mVideoEncoder.drain(false);
// Acquire a new frame of input, and render it to the Surface. If we had a
// GLSurfaceView we could switch EGL contexts and call drawImage() a second
// time to render it on screen. The texture can be shared between contexts by
// passing the GLSurfaceView's EGLContext as eglCreateContext()'s share_context
// argument.
mStManager.awaitNewImage();
mStManager.drawImage();
// Set the presentation time stamp from the SurfaceTexture's time stamp. This
// will be used by MediaMuxer to set the PTS in the video.
mInputSurface.setPresentationTime(st.getTimestamp() - startWhen);
// Submit it to the encoder. The eglSwapBuffers call will block if the input
// is full, which would be bad if it stayed full until we dequeued an output
// buffer (which we can't do, since we're stuck here). So long as we fully drain
// the encoder before supplying additional input, the system guarantees that we
// can supply another frame without blocking.
mInputSurface.swapBuffers();
}
// send end-of-stream to encoder, and drain remaining output
mVideoEncoder.drain(true);
} finally {
releaseVideo();
}
}
private void initAudioComponents() {
try {
int min_buffer_size = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT);
int buffer_size = SAMPLES_PER_FRAME * 10;
if (buffer_size < min_buffer_size)
buffer_size = ((min_buffer_size / SAMPLES_PER_FRAME) + 1) * SAMPLES_PER_FRAME * 2;
audioRecord = new AudioRecord(
MediaRecorder.AudioSource.MIC, // source
SAMPLE_RATE, // sample rate, hz
CHANNEL_CONFIG, // channels
AUDIO_FORMAT, // audio format
buffer_size); // buffer size (bytes)
/////////////////
MediaFormat format = new MediaFormat();
format.setString(MediaFormat.KEY_MIME, AUDIO_MIME_TYPE);
format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
format.setInteger(MediaFormat.KEY_SAMPLE_RATE, 44100);
format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, 1);
format.setInteger(MediaFormat.KEY_BIT_RATE, 128000);
format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 16384);
mAudioEncoder = new Encoder(AUDIO_MIME_TYPE, format, mMuxer);
mAudioEncoder.getEncoder().start();
} catch (RuntimeException e) {
releaseAudio();
throw e;
}
}
private void encodeAudio() {
try {
audioRecord.startRecording();
while (encoding) {
mAudioEncoder.drain(false);
sendAudioToEncoder(false);
}
//TODO: Sending "false" because calling signalEndOfInputStream fails on this encoder
mAudioEncoder.drain(false);
} finally {
releaseAudio();
}
}
public void sendAudioToEncoder(boolean endOfStream) {
// send current frame data to encoder
ByteBuffer[] inputBuffers = mAudioEncoder.getEncoder().getInputBuffers();
int inputBufferIndex = mAudioEncoder.getEncoder().dequeueInputBuffer(-1);
if (inputBufferIndex >= 0) {
ByteBuffer inputBuffer = inputBuffers[inputBufferIndex];
inputBuffer.clear();
long presentationTimeNs = System.nanoTime();
int inputLength = audioRecord.read(inputBuffer, SAMPLES_PER_FRAME);
presentationTimeNs -= (inputLength / SAMPLE_RATE) / 1000000000;
long presentationTimeUs = (presentationTimeNs - startWhen) / 1000;
if (endOfStream) {
mAudioEncoder.getEncoder().queueInputBuffer(inputBufferIndex, 0, inputLength, presentationTimeUs, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
} else {
mAudioEncoder.getEncoder().queueInputBuffer(inputBufferIndex, 0, inputLength, presentationTimeUs, 0);
}
}
}
public void stop() {
encoding = false;
}
/**
* Releases encoder resources.
*/
public void release() {
releaseVideo();
releaseAudio();
}
private void releaseVideo() {
if (mVideoEncoder != null) {
mVideoEncoder.release();
mVideoEncoder = null;
}
if (mInputSurface != null) {
mInputSurface.release();
mInputSurface = null;
}
if (mStManager != null) {
mStManager.release();
mStManager = null;
}
releaseMuxer();
}
private void releaseAudio() {
if (audioRecord != null) {
audioRecord.stop();
audioRecord = null;
}
if (mAudioEncoder != null) {
mAudioEncoder.release();
mAudioEncoder = null;
}
releaseMuxer();
}
private void releaseMuxer() {
if (mMuxer != null && mVideoEncoder == null && mAudioEncoder == null) {
mMuxer.release();
mMuxer = null;
}
}
public boolean isRecording() {
return mMuxer != null;
}
}
包装复用器并在开始之前等待轨道完成的类是以下（我添加了一些仅用于测试的同步）：
import android.media.MediaCodec;
import android.media.MediaFormat;
import android.media.MediaMuxer;
import com.google.common.base.Throwables;
import java.nio.ByteBuffer;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
/**
* Class responsible for muxing. Wraps a MediaMuxer.
*/
public class Muxer {
private final MediaMuxer muxer;
private final int totalTracks;
private int trackCounter;
public Muxer(MediaMuxer muxer, int totalTracks) {
this.muxer = checkNotNull(muxer);
this.totalTracks = totalTracks;
}
synchronized public int addTrack(MediaFormat format) {
checkState(!isStarted(), "Muxer already started");
int trackIndex = muxer.addTrack(format);
trackCounter++;
if (isStarted()) {
muxer.start();
notifyAll();
} else {
while (!isStarted()) {
try {
wait();
} catch (InterruptedException e) {
Throwables.propagate(e);
}
}
}
return trackIndex;
}
synchronized public void writeSampleData(int trackIndex, ByteBuffer byteBuf,
MediaCodec.BufferInfo bufferInfo) {
checkState(isStarted(), "Muxer not started");
muxer.writeSampleData(trackIndex, byteBuf, bufferInfo);
}
public void release() {
if (muxer != null) {
try {
muxer.stop();
} catch (Exception e) {
}
muxer.release();
}
}
private boolean isStarted() {
return trackCounter == totalTracks;
}
}
负责写入MediaCodec编码器的类如下：
import android.media.MediaCodec;
import android.media.MediaFormat;
import com.google.common.base.Throwables;
import java.io.IOException;
import java.nio.ByteBuffer;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
/**
* Class responsible for encoding.
*/
public class Encoder {
private final MediaCodec encoder;
private final Muxer muxer;
private final MediaCodec.BufferInfo bufferInfo;
private int trackIndex;
public Encoder(String mimeType, MediaFormat format, Muxer muxer) {
checkNotNull(mimeType);
checkNotNull(format);
checkNotNull(muxer);
try {
encoder = MediaCodec.createEncoderByType(mimeType);
encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
this.muxer = muxer;
bufferInfo = new MediaCodec.BufferInfo();
} catch (IOException e) {
throw Throwables.propagate(e);
}
}
public MediaCodec getEncoder() {
return encoder;
}
/**
* Extracts all pending data from the encoder and forwards it to the muxer.
* <p/>
* If endOfStream is not set, this returns when there is no more data to drain. If it
* is set, we send EOS to the encoder, and then iterate until we see EOS on the output.
* Calling this with endOfStream set should be done once, right before stopping the muxer.
* <p/>
* We're just using the muxer to get a .mp4 file (instead of a raw H.264 stream).
*/
public void drain(boolean endOfStream) {
final int TIMEOUT_USEC = 10000;
if (endOfStream) {
encoder.signalEndOfInputStream();
}
ByteBuffer[] encoderOutputBuffers = encoder.getOutputBuffers();
while (true) {
int encoderStatus = encoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_USEC);
if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
// no output available yet
if (!endOfStream) {
break; // out of while
}
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
// not expected for an encoder
encoderOutputBuffers = encoder.getOutputBuffers();
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
// now that we have the Magic Goodies, start the muxer
trackIndex = muxer.addTrack(encoder.getOutputFormat());
} else if (encoderStatus < 0) {
// let's ignore it
} else {
ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
checkState(encodedData != null, "encoderOutputBuffer %s was null", encoderStatus);
if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
// The codec config data was pulled out and fed to the muxer when we got
// the INFO_OUTPUT_FORMAT_CHANGED status. Ignore it.
bufferInfo.size = 0;
}
if (bufferInfo.size != 0) {
// adjust the ByteBuffer values to match BufferInfo (not needed?)
encodedData.position(bufferInfo.offset);
encodedData.limit(bufferInfo.offset + bufferInfo.size);
muxer.writeSampleData(trackIndex, encodedData, bufferInfo);
}
encoder.releaseOutputBuffer(encoderStatus, false);
if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
break; // out of while
}
}
}
}
public void release() {
if (encoder != null) {
try {
encoder.stop();
} catch (Exception e) {
}
encoder.release();
}
}
}
知道为什么在并发运行时可能会失败？
提前致谢。
好的，我终于实现了原始海报的最终目标。问题就像我预期的那样，它必须使用为音频轨道生成的时间戳来完全不匹配视频轨道给我们的内容。
我的解决方案是将我们的VideoEncoder用于存储在其BufferInfo中的Surface时间戳传递给AudioEncoder。而不是根据原始海报正在进行的线程的运行时间计算该时间戳。我刚从表面获取了时间戳，并将其用作我的AudioEncoder BufferInfo时间戳。您必须确保您的录音机的缓冲区限制设置得足够大，以便处理，因为我们不会以采样率接收音频帧，而是接收视频的帧速率。这是微不足道的。
要清楚，音频和视频编码仍然在单独的线程上进行，但每当我调用mVideoEncoder.onFrameAvailable向视频编码器线程发送带有表面时间戳的消息时。我对AudioEncoder线程使用表面纹理的TimeStamp做同样的事情，我们用它来进行视频编码。这有一个完整功能的MP4视频的理想结果，包括音频和视频轨道，没有最初发生的口吃。我希望这可以帮助目前遇到类似问题或过去的人。
当其他线程已经开始编写样本数据（muxer.addTrack(encoder.getOutputFormat())）时，问题必须是你调用..muxer.writeSampleData(trackIndex, encodedData, bufferInfo)。这会在MediaMuxer中导致IllegalStateException，但是你没有捕获它，只是在releaseAudio()部分调用finally。
muxer.addTrack(encoder.getOutputFormat())，然后允许线程通过muxer.writeSampleData(trackIndex, encodedData, bufferInfo)编写样本。