MediaCodec 在高质量流上崩溃

Question

我正在使用以下代码 (original guide) 解码 h264 视频流：

public void configure(Surface surface, int width, int height, ByteBuffer csd0) {
        String VIDEO_FORMAT = "video/avc";
        if (mConfigured) {
            throw new IllegalStateException("Decoder is already configured");
        }
        MediaFormat format = MediaFormat.createVideoFormat(VIDEO_FORMAT, width, height);
        // little tricky here, csd-0 is required in order to configure the codec properly
        // it is basically the first sample from encoder with flag: BUFFER_FLAG_CODEC_CONFIG
        format.setByteBuffer("csd-0", csd0);
        try {
            mCodec = MediaCodec.createDecoderByType(VIDEO_FORMAT);
        } catch (IOException e) {
            throw new RuntimeException("Failed to create codec", e);
        }
        mCodec.configure(format, surface, null, 0);
        mCodec.start();
        mConfigured = true;
    }

    @SuppressWarnings("deprecation")
    public void decodeSample(byte[] data, int offset, int size, long presentationTimeUs, int flags) {
        if (mConfigured && mRunning) {
            int index = mCodec.dequeueInputBuffer(mTimeoutUs);
            if (index >= 0) {
                ByteBuffer buffer;
                // since API 21 we have new API to use
                if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) {
                    buffer = mCodec.getInputBuffers()[index];
                    buffer.clear();
                } else {
                    buffer = mCodec.getInputBuffer(index);
                }
                if (buffer != null) {
                    buffer.put(data, offset, size);
                    mCodec.queueInputBuffer(index, 0, size, presentationTimeUs, flags);
                }
            }
        }
    }

    @Override
    public void run() {
        try {
            MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
            while (mRunning) {
                if (mConfigured) {
                    int index = mCodec.dequeueOutputBuffer(info, mTimeoutUs);
                    if (index >= 0) {
                        // setting true is telling system to render frame onto Surface
                        mCodec.releaseOutputBuffer(index, true);
                        if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) == MediaCodec.BUFFER_FLAG_END_OF_STREAM) {
                            break;
                        }
                    }
                } else {
                    // just waiting to be configured, then decode and render
                    try {
                        Thread.sleep(10);
                    } catch (InterruptedException ignore) {
                    }
                }
            }
        } finally {
            if (mConfigured) {
                mCodec.stop();
                mCodec.release();
            }
        }
    }

我可以在我的 Nexus 6 (api 22) 和 Samsung Galaxy Core (api 16) 上以低质量和中等质量运行它。但是，当我切换到高质量 (720p) 时，它会在大约 30 帧后在三星上崩溃（但没有任何内容呈现到屏幕上）。

E/ACodec﹕ [OMX.qcom.video.decoder.avc] ERROR(0x8000100a)
E/MediaCodec﹕ Codec reported an error. (omx error 0x8000100a, internalError -2147483648)
[...]
W/System.err﹕ java.lang.IllegalStateException
W/System.err﹕ at android.media.MediaCodec.dequeueInputBuffer(Native Method)
W/System.err﹕ at com.test.stream.VideoDecoder$Worker.decodeSample(VideoDecoder.java:95)
W/System.err﹕ at com.test.stream.VideoDecoder.decodeSample(VideoDecoder.java:24)
W/System.err﹕ at com.test.stream.VideoThread.run(VideoThread.java:160)

上面的错误是第一个出现的错误，之后每一帧都会抛出IllegalStateException。

我的问题是，这是特定于设备的问题（因为：较旧的 api/设备、功能较弱等）还是实际上有问题？ 我该如何处理？

Answer 1

对于我的 Android h.264 解码器，我的设置与您的设置略有不同。我认为您使用的是更现代的 api 级别。但对我来说，它看起来更像这样：

public void startDecoder() {
    // Initilize codec
    mediaCodec = MediaCodec.createDecoderByType("video/avc");
    mediaFormat = MediaFormat.createVideoFormat("video/avc", 0, 0);
    bufferInfo = new MediaCodec.BufferInfo();

    // STOPS unit-tests from crashing here from mocked out android
    if (mediaCodec != null) {
        mediaCodec.configure(mediaFormat, targetSurface, null, 0);
        mediaCodec.start();
        decoderThread = new Thread(this);
        decoderThread.start();
    }
}

// 解码器线程是指这个类，它执行解码器/渲染循环：

public void run() {
    //mediaCodec input + output dequeue timeouts
    long kInputBufferTimeoutMs = 50;
    long kOutputBufferTimeoutMs = 50;

    while (running && mediaCodec != null) {
        synchronized (mediaCodec) {
            // stop if not running.
            if (!running || mediaCodec == null)
                break;

            // Only push in new data if there is data available in the queue
            if (naluSegmentQueue.size() > 0) {
                int inputBufferIndex = mediaCodec.dequeueInputBuffer(kInputBufferTimeoutMs);
                if (inputBufferIndex >= 0) {
                    NaluSegment segment = naluSegmentQueue.poll();
                    codecInputBufferAvailable(segment, mediaCodec, inputBufferIndex);
                }
            }

            // always check if output is available.
            int outputBufferIndex = mediaCodec.dequeueOutputBuffer(bufferInfo, kOutputBufferTimeoutMs);
            if (outputBufferIndex >= 0) {
                // Try and render first
                codecOuputBufferAvailable(mediaCodec, outputBufferIndex, bufferInfo);
            } else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
                // Subsequent data will conform to new format.
                // Can ignore if using getOutputFormat(outputBufferId)
                mediaFormat = mediaCodec.getOutputFormat();
            }
        }
    }
}

将数据放入解码器，包括参数。我不介意尝试使用 csd-0/1 网络流可以具有不断变化的格式描述，并且更容易让它被动态拾取。

private void codecInputBufferAvailable(NaluSegment segment, MediaCodec codec, int index) {
    int flags = (segment.getType() == NaluType.SPS
            || segment.getType() == NaluType.PPS
            || segment.getType() == NaluType.SUPP_ENHANCEMENT) ?
            MediaCodec.BUFFER_FLAG_CODEC_CONFIG : MediaCodec.BUFFER_FLAG_SYNC_FRAME;

    ByteBuffer[] buffers = codec.getInputBuffers();
    ByteBuffer buffer = buffers[index];
    // Can throw buffer overflow exception when buffer sizes are too small.
    try {
        buffer.put(segment.getBuffer());
        codec.queueInputBuffer(index, 0, segment.getBufferSize(), 0, flags);
    } catch(Exception e) {
        Log.e(TAG, "Failed to push buffer to decoder");
    }
}

重要提示：buffer.put(segment.getBuffer()); 这里的 getBuffer() 总是返回一个 4 字节的 annexb 缓冲区。 android 解码器不理解 3 字节最终单元。因此，如果您有一个 3 字节的最终单元，请将其转换为长度为 + 1 且 0x00, 0x00, 0x00, 0x01 作为起始魔术序列的 4 字节魔术序列，则缓冲区的其余部分应为 &buffer[headerLength ]。

注意这里的 try-catch 不会给出编译器警告，但如果您的负载非常大并且字节缓冲区太小，它会在此处抛出缓冲区溢出异常。

只要你正确解析出你的 NAL 单元，这应该对你有用。但就我而言，我注意到 NAL 单元可以是魔术头的 3 或 4 个字节。

/**
 * H264 is comprised of NALU segments.
 *
 * XXXX Y ZZZZZZZZ -> XXXX Y ZZZZZZZZ -> XXXX Y ZZZZZZZZ
 *
 * Each segment is comprised of:
 *
 * XXXX   -> Magic byte header (0x00, 0x00, 0x00, 0x01) NOTE: this can be either 3 of 4 bytes
 * Y      -> The Nalu Type
 * ZZZ... -> The Payload
 *
 * Notice there is no nalu length specified. To parse an nalu, you must
 * read until the next magic-byte-sequence AKA the next segment to figure
 * out the full nalu length
 **/
public static List<NaluSegment> parseNaluSegments(byte[] buffer) throws NaluBufferException {
    List<NaluSegment> segmentList = new ArrayList<>();
    if (buffer.length < 6) {
        return segmentList;
    }

    int lastStartingOffset = -1;
    for (int i = 0; i < buffer.length - 10; ++i) {
        **if (buffer[i] == 0x00 && buffer[i+1] == 0x00 && buffer[i+2] == 0x01)** {
            int naluType = (buffer[i+3] & 0x1F);
            NaluSegment segment = new NaluSegment(naluType, 3, i);

            **if (i > 0 && buffer[i-1] == 0x00)** {
                // This is actually a 4 byte segment
                int currentSegmentOffset = segment.getOffset();
                segment.setHeaderSize(4);
                segment.setOffset(currentSegmentOffset - 1);
            }
...

创建您自己的 nalu-segment 对象，不要忘记尾随的 NAL。

我希望这会有所帮助。