MediaCodec 即时解码 && 编码答案

【问题标题】：MediaCodec decode && encode on-the-flyMediaCodec 即时解码 && 编码
【发布时间】：2020-05-27 18:58:59
【问题描述】：

我想将视频文件中的一些音轨从 AAC 5.1（不仅是为了启动）转换为 AAC 2（只是因为 MediaMuxer 和 vorbis 和 opus 编码器不支持 ac3 在 Pixel 2 中没有硬件支持）并使用回调媒体编解码器。我写了这样的代码：

mExtractor = new MediaExtractor();
mExtractor.setDataSource(source.getPath());
mExtractor.selectTrack(trackNumber);

MediaFormat sourceMf = mExtractor.getTrackFormat(trackNumber);
mDecoder = MediaCodec.createDecoderByType(sourceMf.getString(MediaFormat.KEY_MIME));
mDecoder.setCallback(createCallbackDecoder());
mDecoder.configure(sourceMf, null, null, 0);

MediaFormat wantedMediaFormat = MediaFormat.createAudioFormat(MediaFormat.MIMETYPE_AUDIO_AAC, 44100, 2);
mEncoder = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC);
mEncoder.setCallback(createCallbackEncoder());
mEncoder.configure(wantedMediaFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);

mMuxer = new MediaMuxer(saveTo.getPath(), MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);

public void start() {
        mDecoder.start();

还有回调

解码器：

@Override
public void onInputBufferAvailable(@NonNull MediaCodec codec, int index) {
    ByteBuffer byteBuffer = codec.getInputBuffer(index);
    Log.i(TAG, "onInputBufferAvailable: byteBuffer b/f readSampleData (decoder): " + byteBuffer);
    if (byteBuffer != null) {
        int offset = 0;
        long presentationTimeUs = 0;
        int flags;
        int size;
        if ((size = mExtractor.readSampleData(byteBuffer, offset)) > -1) {
            presentationTimeUs = mExtractor.getSampleTime();
            flags = mExtractor.getSampleFlags();
            mExtractor.advance();
        } else {
            flags = MediaCodec.BUFFER_FLAG_END_OF_STREAM;
        }
        try {
            codec.queueInputBuffer(index, offset, size, presentationTimeUs, flags);
            Log.i(TAG, "onInputBufferAvailable (decoder): SUCCESS");
        } catch (Exception e) {
            Log.e(TAG, "EXCEPTION (decoder)!\nonInputBufferAvailable (decoder): ", e);
            throw e;
        }
    } else {
        Log.e(TAG, "onInputBufferAvailable = null");
    }
}

@Override
public void onOutputBufferAvailable(@NonNull MediaCodec codec, int index, @NonNull MediaCodec.BufferInfo info) {
    ByteBuffer byteBuffer = codec.getOutputBuffer(index);
    Log.i(TAG, "onOutputBufferAvailable: byteBuffer with data (decoder): " + byteBuffer);
    if (byteBuffer != null) {
        ByteBuffer buffer2 = ByteBuffer.allocate(info.size);
        Log.i(TAG, "onOutputBufferAvailable: allocated byteBuffer (decoder): " + buffer2);
        buffer2.put(byteBuffer);
        MediaCodec.BufferInfo info2 = new MediaCodec.BufferInfo();
        info2.flags = info.flags;
        info2.size = info.size;
        info2.presentationTimeUs = info.presentationTimeUs;
        info2.offset = info.offset;
        if (mQueue.add(new Pair<>(buffer2, info2))) {
            Log.i(TAG, String.format("onOutputBufferAvailable (decoder): added in queue: %s\n%s %s %s %s", buffer2,
                      info2.offset, info2.size, info2.presentationTimeUs, info2.flags));
            codec.releaseOutputBuffer(index, false);
        }
    } else {
        Log.e(TAG, "onOutputBufferAvailable = null");
    }
}

@Override
public void onOutputFormatChanged(@NonNull MediaCodec codec, @NonNull MediaFormat format) {
    Log.i(TAG, String.format("onOutputFormatChanged (decoder): OLD=%s NEW=%s", codec.getInputFormat(), format));
    mEncoder.start();
}

编码器：

@Override
public void onInputBufferAvailable(@NonNull MediaCodec codec, int index) {
    Log.i(TAG, "onInputBufferAvailable (encoder): index=" + index);
    Pair<ByteBuffer, MediaCodec.BufferInfo> mediaChunk;
    if ((mediaChunk = mQueue.poll()) != null) {
        Log.i(TAG, "onInputBufferAvailable (encoder): queue poll != null");
        ByteBuffer byteBuffer = codec.getInputBuffer(index);
        Log.i(TAG, "onInputBufferAvailable: byteBuffer b/f queue (encoder): " + byteBuffer);
        if (byteBuffer != null) {
            int offset = mediaChunk.second.offset;
            int flags = mediaChunk.second.flags;
            long presentationTimeUs = mediaChunk.second.presentationTimeUs;
            int size = mediaChunk.second.size;
            byteBuffer.put(mediaChunk.first);
            try {
                Log.i(TAG, String.format("onInputBufferAvailable (encoder): %s\n%s %s %s %s", mediaChunk.first,
                    mediaChunk.second.offset,
                    mediaChunk.second.size,
                    mediaChunk.second.presentationTimeUs,
                    mediaChunk.second.flags));
                codec.queueInputBuffer(index, offset, size, presentationTimeUs, flags);
                Log.i(TAG, "queueInputBuffer (encoder): SUCCESS");
            } catch (Exception e) {
                Log.e(TAG, "EXCEPTION (encoder)!\nonInputBufferAvailable (encoder): ", e);
                throw e;
            }
        }
    } else {
        Log.e(TAG, "onInputBufferAvailable (encoder): empty queue");
    }
}

@Override
public void onOutputBufferAvailable(@NonNull MediaCodec codec, int index, @NonNull MediaCodec.BufferInfo info) {
    ByteBuffer byteBuffer = codec.getOutputBuffer(index);
    Log.i(TAG, "onOutputBufferAvailable: byteBuffer from codec with data (encoder): " + byteBuffer);
    if (byteBuffer != null) {
        mMuxer.writeSampleData(mTrackNumber, byteBuffer, info);
        Log.i(TAG, "onOutputBufferAvailable (encoder): muxer written");
        if (info.flags == MediaCodec.BUFFER_FLAG_END_OF_STREAM) {
            finish();
        } else {
            codec.releaseOutputBuffer(index, false);
        }
    } else {
        Log.e(TAG, "onOutputBufferAvailable (encoder): buffer = null");
    }
}

@Override
public void onOutputFormatChanged(@NonNull MediaCodec codec, @NonNull MediaFormat format) {
    Log.i(TAG, String.format("onOutputFormatChanged (encoder): OLD=%s NEW=%s", codec.getInputFormat(), format));
    mTrackNumber = mMuxer.addTrack(format);
    mMuxer.start();
}

但在执行时出现异常：

I/MediaCodec: MediaCodec will operate in async mode
I/MediaCodec: MediaCodec will operate in async mode
I/Codec: onInputBufferAvailable: byteBuffer b/f readSampleData (decoder): java.nio.DirectByteBuffer[pos=0 lim=8192 cap=8192]
I/Codec: onInputBufferAvailable (decoder): SUCCESS
I/Codec: onInputBufferAvailable: byteBuffer b/f readSampleData (decoder): java.nio.DirectByteBuffer[pos=0 lim=8192 cap=8192]
I/Codec: onInputBufferAvailable (decoder): SUCCESS
I/Codec: onInputBufferAvailable: byteBuffer b/f readSampleData (decoder): java.nio.DirectByteBuffer[pos=0 lim=8192 cap=8192]
I/Codec: onInputBufferAvailable (decoder): SUCCESS
I/Codec: onInputBufferAvailable: byteBuffer b/f readSampleData (decoder): java.nio.DirectByteBuffer[pos=0 lim=8192 cap=8192]
I/Codec: onInputBufferAvailable (decoder): SUCCESS
I/Codec: onInputBufferAvailable: byteBuffer b/f readSampleData (decoder): java.nio.DirectByteBuffer[pos=0 lim=8192 cap=8192]
I/Codec: onInputBufferAvailable (decoder): SUCCESS
I/Codec: onInputBufferAvailable: byteBuffer b/f readSampleData (decoder): java.nio.DirectByteBuffer[pos=0 lim=8192 cap=8192]
I/Codec: onInputBufferAvailable (decoder): SUCCESS
I/Codec: onInputBufferAvailable: byteBuffer b/f readSampleData (decoder): java.nio.DirectByteBuffer[pos=0 lim=8192 cap=8192]
I/Codec: onInputBufferAvailable (decoder): SUCCESS
I/Codec: onOutputFormatChanged (decoder): OLD={sample-rate=44100, mime=audio/mp4a-latm, channel-count=1, bitrate=0} NEW={sample-rate=48000, pcm-encoding=2, mime=audio/raw, channel-count=6}
I/Codec: onOutputBufferAvailable: byteBuffer with data (decoder): java.nio.DirectByteBuffer[pos=0 lim=0 cap=32768]
I/Codec: onOutputBufferAvailable: allocated byteBuffer (decoder): java.nio.HeapByteBuffer[pos=0 lim=0 cap=0]
I/Codec: onOutputBufferAvailable (decoder): added in queue: java.nio.HeapByteBuffer[pos=0 lim=0 cap=0]
    0 0 0 0
I/Codec: onInputBufferAvailable: byteBuffer b/f readSampleData (decoder): java.nio.DirectByteBuffer[pos=0 lim=8192 cap=8192]
I/Codec: onInputBufferAvailable (decoder): SUCCESS
I/Codec: onOutputBufferAvailable: byteBuffer with data (decoder): java.nio.DirectByteBuffer[pos=0 lim=0 cap=32768]
I/Codec: onOutputBufferAvailable: allocated byteBuffer (decoder): java.nio.HeapByteBuffer[pos=0 lim=0 cap=0]
I/Codec: onOutputBufferAvailable (decoder): added in queue: java.nio.HeapByteBuffer[pos=0 lim=0 cap=0]
    0 0 21333 0
I/Codec: onInputBufferAvailable: byteBuffer b/f readSampleData (decoder): java.nio.DirectByteBuffer[pos=0 lim=8192 cap=8192]
I/Codec: onInputBufferAvailable (decoder): SUCCESS
I/Codec: onOutputBufferAvailable: byteBuffer with data (decoder): java.nio.DirectByteBuffer[pos=0 lim=12288 cap=32768]
I/Codec: onOutputBufferAvailable: allocated byteBuffer (decoder): java.nio.HeapByteBuffer[pos=0 lim=12288 cap=12288]
I/Codec: onOutputBufferAvailable (decoder): added in queue: java.nio.HeapByteBuffer[pos=12288 lim=12288 cap=12288]
    0 12288 42666 0
I/Codec: onInputBufferAvailable: byteBuffer b/f readSampleData (decoder): java.nio.DirectByteBuffer[pos=0 lim=8192 cap=8192]
I/Codec: onInputBufferAvailable (decoder): SUCCESS
I/Codec: onOutputBufferAvailable: byteBuffer with data (decoder): java.nio.DirectByteBuffer[pos=0 lim=12288 cap=32768]
I/Codec: onOutputBufferAvailable: allocated byteBuffer (decoder): java.nio.HeapByteBuffer[pos=0 lim=12288 cap=12288]
I/Codec: onOutputBufferAvailable (decoder): added in queue: java.nio.HeapByteBuffer[pos=12288 lim=12288 cap=12288]
    0 12288 64000 0
I/Codec: onInputBufferAvailable (encoder): index=0
I/Codec: onInputBufferAvailable (encoder): queue poll != null
I/Codec: onInputBufferAvailable: byteBuffer b/f queue (encoder): java.nio.DirectByteBuffer[pos=0 lim=4096 cap=4096]
I/Codec: onInputBufferAvailable (encoder): java.nio.HeapByteBuffer[pos=0 lim=0 cap=0]
    0 0 0 0
I/Codec: queueInputBuffer (encoder): SUCCESS
I/Codec: onInputBufferAvailable (encoder): index=1
I/Codec: onInputBufferAvailable (encoder): queue poll != null
I/Codec: onInputBufferAvailable: byteBuffer b/f queue (encoder): java.nio.DirectByteBuffer[pos=0 lim=4096 cap=4096]
I/Codec: onInputBufferAvailable (encoder): java.nio.HeapByteBuffer[pos=0 lim=0 cap=0]
    0 0 21333 0
I/Codec: queueInputBuffer (encoder): SUCCESS
I/Codec: onInputBufferAvailable (encoder): index=2
I/Codec: onInputBufferAvailable (encoder): queue poll != null
I/Codec: onInputBufferAvailable: byteBuffer b/f queue (encoder): java.nio.DirectByteBuffer[pos=0 lim=4096 cap=4096]
I/Codec: onInputBufferAvailable (encoder): java.nio.HeapByteBuffer[pos=12288 lim=12288 cap=12288]
    0 12288 42666 0
E/Codec: EXCEPTION (encoder)!
    onInputBufferAvailable (encoder): 
    java.lang.IllegalArgumentException
        at android.media.MediaCodec.native_queueInputBuffer(Native Method)
        at android.media.MediaCodec.queueInputBuffer(MediaCodec.java:2450)
        at opensource.umnik.media2media.codec.SyncCodec$2.onInputBufferAvailable(SyncCodec.java:149)
        at android.media.MediaCodec$EventHandler.handleCallback(MediaCodec.java:1738)
        at android.media.MediaCodec$EventHandler.handleMessage(MediaCodec.java:1696)
        at android.os.Handler.dispatchMessage(Handler.java:107)
        at android.os.Looper.loop(Looper.java:214)
        at android.app.ActivityThread.main(ActivityThread.java:7356)
        at java.lang.reflect.Method.invoke(Native Method)
        at com.android.internal.os.RuntimeInit$MethodAndArgsCaller.run(RuntimeInit.java:492)
        at com.android.internal.os.ZygoteInit.main(ZygoteInit.java:930)
D/AndroidRuntime: Shutting down VM

at android.media.MediaCodec.queueInputBuffer(MediaCodec.java:2450) ← 这是编码器的缓冲区

我做错了什么？

UPD：代码更新了几次。 UPD2：更多日志

【问题讨论】：

请在异常发生时将所有参数转储到queueInputBuffer()，并添加到您的帖子中。
``` Log.i(TAG, String.format("onInputBufferAvailable: %s\n%s %s %s %s", mediaChunk.first, mediaChunk.second.offset, mediaChunk. second.size, mediaChunk.second.presentationTimeUs, mediaChunk.second.flags)); codec.queueInputBuffer（索引，偏移量，大小，presentationTimeUs，标志）； ``` 并在 2020-02-10 10:39:20.461 26534-26534/? I/Codec: onInputBufferAvailable: индекс=0 行的 logcat 中跟随。还是你问别的？
好的，我在这里遗漏了一些东西，所以让我们再试一次：从你的问题中，我们怎么知道onInputBufferAvailable() 扔的是哪个？你有两个，不是很明显是哪一个。
我重写日志并更新异常信息。感谢 cmets。
这很有帮助，但我仍然不能 100% 确定是编码器在扔东西。由于使用了匿名内部类，堆栈跟踪不明确。尝试catch-ing 异常，或者在调用之后放置一个显式的“queueInputBuffer SUCCESS！（编码器）”。

标签： android android-mediacodec mediaextractor

【解决方案1】：

IllegalArgumentException 的原因最终是您为输入 ByteBuffer 指定了 12288 字节的大小，而该输入的容量仅为 4096 字节。

请注意，对于queueInputBuffer()（编码器）的前两次调用，您没有输入任何数据；它们都是 0 字节写入，因此它们是微不足道的，并且它们成功了。（就我个人而言，我会跳过 0 字节写入）。

第三次迭代是您第一次真正有一些数据要编码。但是您的 byteBuffer.put(mediaChunk.first) 实际上并没有做任何事情，因为 mediaChunk.first 已经“播放完毕”。它剩余 0 个字节。在执行put() 之前，您需要“倒带”它：

mediaChunk.first.position(0)

当然，现在的问题是，你有 12288 字节要写入，但目标只能容纳 4096 字节，所以put() 无疑会抛出。你必须写一个较小的数量。这将避免IllegalArgumentException。

正如我在评论中提到的，您的解码音频包含 6 个 5.1 音频通道。（`1024 个样本 x 每个样本 2 个字节 x 6 个通道 == 12288）。您的编码器需要 2048 个音频样本（每个样本 2 个字节；我假设您已将编码器配置为每个样本 1 个通道/2 个字节）。这可能是决定如何进行“5.1 -> 单声道”音频转换的好时机；这将有助于减少需要复制到编码器中的解码音频量。

【讨论】：