对麦克风数据进行编码并解码以提供音频编解码器会导致白噪声

Question

我想使用 Opus 对来自麦克风的音频流进行编码，然后对其进行解码并以流模式播放音频输出。我使用的参数：每秒 8000 个样本的麦克风单声道音频必须以 1200 bps 进行编码，并解码为音频输出的相同采样率。完整的源代码可以在公共 GitHub 存储库https://github.com/Dmitry-Nikishov/RdmWalkieTalkie 中找到。以下是一些评论和我看到的实际问题。

这是我应用的音频格式：

var audioFormat = new AudioFormat(OpusSettings.OPUS_SAMPLE_RATE/*8000*/,
        16,
        1,
        true,
        false);

为了对 PCM 数据进行编码，我有以下类：

package audio.opus;

import com.sun.jna.ptr.PointerByReference;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import tomp2p.opuswrapper.Opus;

import java.nio.Buffer;
import java.nio.ByteBuffer;
import java.nio.IntBuffer;
import java.nio.ShortBuffer;

public class OpusEncoder {
    private static Logger logger = LogManager.getRootLogger();
    private PointerByReference opusEncoder;

    public OpusEncoder() {
        IntBuffer error = IntBuffer.allocate(1);
        opusEncoder = Opus.INSTANCE.opus_encoder_create(
                OpusSettings.OPUS_SAMPLE_RATE,
                OpusSettings.OPUS_CHANNEL_COUNT,
                Opus.OPUS_APPLICATION_AUDIO,
                error
        );
        if (error.get() != Opus.OPUS_OK && opusEncoder == null) {
            logger.debug(
                    String.format("Received error status from opus_encoder_create(...): {%s}", error.get())
            );
        }

        Opus.INSTANCE.opus_encoder_ctl(opusEncoder, Opus.OPUS_SET_BITRATE_REQUEST, OpusSettings.OPUS_BITRATE_IN_BPS);
        Opus.INSTANCE.opus_encoder_ctl(opusEncoder, Opus.OPUS_SET_VBR_REQUEST, 0);
    }

    public ByteBuffer encodeToOpus(ByteBuffer rawAudio)
    {
        ShortBuffer nonEncodedBuffer = ShortBuffer.allocate(rawAudio.remaining() / 2);
        ByteBuffer encoded = ByteBuffer.allocate(4096);
        for (int i = rawAudio.position(); i < rawAudio.limit(); i += 2)
        {
            int firstByte =  (0x000000FF & rawAudio.get(i));      //Promotes to int and handles the fact that it was unsigned.
            int secondByte = (0x000000FF & rawAudio.get(i + 1));

            //Combines the 2 bytes into a short. Opus deals with unsigned shorts, not bytes.
            short toShort = (short) ((firstByte << 8) | secondByte);

            nonEncodedBuffer.put(toShort);
        }
        ((Buffer) nonEncodedBuffer).flip();

        int result = Opus.INSTANCE.opus_encode(
                opusEncoder,
                nonEncodedBuffer,
                OpusSettings.OPUS_FRAME_SIZE_IN_SAMPLES,
                encoded,
                encoded.capacity()
        );

        if (result <= 0)
        {
            logger.debug(String.format("Received error code from opus_encode(...): {%d}", result));
            return null;
        }

        ((Buffer) encoded).position(0).limit(result);
        return encoded;
    }

    public void shutdown() {
        if (opusEncoder != null) {
            Opus.INSTANCE.opus_encoder_destroy(opusEncoder);
            opusEncoder = null;
        }
    }
}

对于每个 PCM 音频数据块，我执行以下代码（每个 20 ms 帧都转换为 byte[]，然后推送到循环缓冲区以供另一个线程读取、解码并写入音频输出）：

private void audioInputCb( byte[] audioData )
{
    final int numOf20msFrames = audioData.length/OpusSettings.OPUS_FRAME_SIZE_IN_BYTES;

    for (int frameIdx = 0; frameIdx < numOf20msFrames; frameIdx++) {
        final var frameOffset = frameIdx*OpusSettings.OPUS_FRAME_SIZE_IN_BYTES;
        final var frameToBeEncoded = ByteBuffer.wrap(audioData, frameOffset, OpusSettings.OPUS_FRAME_SIZE_IN_BYTES);
        final var encodedFrame = encoder.encodeToOpus(frameToBeEncoded);
        final var encodedArray = Arrays.copyOf(encodedFrame.array(), encodedFrame.limit());
        m_audioBuffer.push(Arrays.asList(ArrayUtils.toObject(encodedArray)));
    }
}

这是我使用的解码器类：

package audio.opus;

import com.sun.jna.ptr.PointerByReference;
import tomp2p.opuswrapper.Opus;

import java.nio.ByteBuffer;
import java.nio.IntBuffer;
import java.nio.ShortBuffer;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

public class OpusDecoder {
    private static Logger logger = LogManager.getRootLogger();
    private PointerByReference opusDecoder;

    public OpusDecoder() {
        IntBuffer error = IntBuffer.allocate(1);
        opusDecoder = Opus.INSTANCE.opus_decoder_create(
                OpusSettings.OPUS_SAMPLE_RATE,
                OpusSettings.OPUS_CHANNEL_COUNT,
                error);

        if (error.get() != Opus.OPUS_OK && opusDecoder == null) {
            logger.debug(
                    String.format("Received error code from opus_decoder_create(): %s", error.get())
            );
        }
    }

    public short[] decodeFromOpus(ByteBuffer encodedAudio) {
        int length = encodedAudio.remaining();
        int offset = encodedAudio.arrayOffset() + encodedAudio.position();
        byte[] buf = new byte[length];
        byte[] data = encodedAudio.array();
        System.arraycopy(data, offset, buf, 0, length);

        int result;
        ShortBuffer decoded = ShortBuffer.allocate(4096);

        result = Opus.INSTANCE.opus_decode(
                opusDecoder,
                buf,
                buf.length,
                decoded,
                OpusSettings.OPUS_FRAME_SIZE_IN_SAMPLES,
                0
        );

        if (result < 0) {
            logger.debug(String.format("Opus decode -> result < 0 : %d", result));
            return null;
        }

        short[] audio = new short[result];
        decoded.get(audio);
        return audio;
    }

    public void shutdown() {
        if (opusDecoder != null) {
            Opus.INSTANCE.opus_decoder_destroy(opusDecoder);
            opusDecoder = null;
        }
    }
}

执行以下可运行程序来播放此音频流（我提取尽可能多的帧，对其进行解码并馈送到音频输出）：

Runnable task = () -> {
    while (m_taskRunFlag.get()) {
        if (m_audioBuffer.getSize() >= OpusSettings.OPUS_ENCODED_1_SEC_FRAME_LENGTH_IN_BYTES) {
            final var numOfFrames = m_audioBuffer.getSize()/OpusSettings.OPUS_ENCODED_20_MS_FRAME_LENGTH_IN_BYTES;
            for (int frameId = 0; frameId < numOfFrames; frameId++) {
                final var audioFrame = ArrayUtils.toPrimitive(
                        m_audioBuffer.pop(OpusSettings.OPUS_ENCODED_20_MS_FRAME_LENGTH_IN_BYTES).toArray(Byte[]::new)
                );
                final var decodedFrame = opusDecoder.decodeFromOpus(ByteBuffer.wrap(audioFrame));
                writeDataToAudioOutput(shortToByteArray(decodedFrame));
            }
        } else {
            try {
                Thread.sleep(1_000);
            }catch (Exception e) {
                e.printStackTrace();
            }
        }
    }
};

结果我只得到白噪声。有谁知道这里可能出了什么问题吗？我处理音频设备的方式是 100% 正确的 - 如果我去掉 Opus 的东西，我就能清楚地听到自己的声音。

Answer 1

好的，可能涉及到的人：问题与字节排序有关，在

opus_decode(...)

的帮助下字节交换

Short.reverseBytes

的短项之后，我得到了正确的声音！