在 Android 5+ 中使用 MediaCodec 对视频的解码损坏

Question

只需尝试解码视频中的帧。 在使用 Android 4+ (http://bigflake.com/mediacodec/ “ExtractMpegFramesTest.java（需要 4.1，API 16）”

问题是 - 它提取了一个帧，但结果位图如下所示（解码后立即保存图像）：

真实的视频当然有“真实”的帧，而不是“拉伸”的 1 列。

我在代码行之后保存了这张图片：

bmp.copyPixelsFromBuffer(mPixelBuf);
// <-- here I saved the above image

解决此问题的解码器是否有一些重大更新（我找不到）？

Answer 1

在 API 级别 21 及更高级别上，解码器在渲染到表面时应用旋转。因此，您从 SurfaceTexture 获得的 transformMatrix 包含旋转信息，这意味着示例中用于反转 SurfaceTexture 的方式不起作用。为了正确反转纹理，我将其旋转 z 轴并进行 x、y 轴变换。以下是我的工作：

修改

st.getTransformMatrix(mSTMatrix);
if (invert) {
     mSTMatrix[5] = -mSTMatrix[5];
     mSTMatrix[13] = 1.0f - mSTMatrix[13];
}

到

st.getTransformMatrix(mMatrix);
if(invert){
    Matrix.setIdentityM(identifyMatrix, 0); 
    Matrix.translateM(identifyMatrix, 0, 1, 1, 0);      
    Matrix.rotateM(identifyMatrix, 0, 180, 0, 0, 1); 
    Matrix.multiplyMM(mSTMatrix, 0, identifyMatrix, 0, mMatrix,0);
}

其中 mMatrix 和 identifyMatrix 都是

new float[16];

Answer 2

yst 的回答创建了一个镜像

st.getTransformMatrix(mIntermediateMatrix);
if(invert){
   Matrix.setIdentityM(identityMatrix, 0);
   Matrix.translateM(identityMatrix, 0, 1, 1, 0);
   Matrix.rotateM(identityMatrix, 0, 180, 0, 0, 1);
   //fixes mirror image
   Matrix.translateM(identityMatrix, 0, 0, 1, 0);
   Matrix.rotateM(identityMatrix, 0, 180, 1, 0, 0);
   Matrix.multiplyMM(mSTMatrix, 0, identityMatrix, 0,mIntermediateMatrix,0);
} else {
   identityMatrix = mIntermediateMatrix;
}

这是我从 bigflake 的 ExtractMpegFramesTest.java (requires 4.2, API 17) 中稍微修改的 STextureRender 类

/**
* Code for rendering a texture onto a surface using OpenGL ES 2.0.
*/
private static class STextureRender {


private static final int FLOAT_SIZE_BYTES = 4;
private static final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES;
private static final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0;
private static final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3;
private final float[] mTriangleVerticesData = {
        // X, Y, Z, U, V
        -1.0f, -1.0f, 0, 0.f, 0.f,
        1.0f, -1.0f, 0, 1.f, 0.f,
        -1.0f,  1.0f, 0, 0.f, 1.f,
        1.0f,  1.0f, 0, 1.f, 1.f,
};

private FloatBuffer mTriangleVertices;

private static final String VERTEX_SHADER =
        "uniform mat4 uMVPMatrix;\n" +
                "uniform mat4 uSTMatrix;\n" +
                "attribute vec4 aPosition;\n" +
                "attribute vec4 aTextureCoord;\n" +
                "varying vec2 vTextureCoord;\n" +
                "void main() {\n" +
                "    gl_Position = uMVPMatrix * aPosition;\n" +
                "    vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" +
                "}\n";

private static final String FRAGMENT_SHADER =
        "#extension GL_OES_EGL_image_external : require\n" +
                "precision mediump float;\n" +      // highp here doesn't seem to matter
                "varying vec2 vTextureCoord;\n" +
                "uniform samplerExternalOES sTexture;\n" +
                "void main() {\n" +
                "    gl_FragColor = texture2D(sTexture, vTextureCoord);\n" +
                "}\n";

private float[] mMVPMatrix = new float[16];
private float[] mSTMatrix = new float[16];
private float[] identityMatrix = new float[16];
private float[] mIntermediateMatrix = new float[16];

private int mProgram;
private int mTextureID = -12345;
private int muMVPMatrixHandle;
private int muSTMatrixHandle;
private int maPositionHandle;
private int maTextureHandle;

public STextureRender() {
    mTriangleVertices = ByteBuffer.allocateDirect(
            mTriangleVerticesData.length * FLOAT_SIZE_BYTES)
            .order(ByteOrder.nativeOrder()).asFloatBuffer();
    mTriangleVertices.put(mTriangleVerticesData).position(0);

    Matrix.setIdentityM(mSTMatrix, 0);
}

public int getTextureId() {
    return mTextureID;
}

/**
 * Draws the external texture in SurfaceTexture onto the current EGL surface.
 */
public void drawFrame(SurfaceTexture st, boolean invert) {
    checkGlError("onDrawFrame start");
    st.getTransformMatrix(mIntermediateMatrix);


    if(invert){
        Matrix.setIdentityM(identityMatrix, 0);
        Matrix.translateM(identityMatrix, 0, 1, 1, 0);
        Matrix.rotateM(identityMatrix, 0, 180, 0, 0, 1);
        //fixes mirror image
        Matrix.translateM(identityMatrix, 0, 0, 1, 0);
        Matrix.rotateM(identityMatrix, 0, 180, 1, 0, 0);
        Matrix.multiplyMM(mSTMatrix, 0, identityMatrix, 0, mIntermediateMatrix,0);
    } else {
        mSTMatrix = mIntermediateMatrix;
    }

/*
    if (invert) {
        mSTMatrix[5] = -mSTMatrix[5];
        mSTMatrix[13] = 1.0f - mSTMatrix[13];
    }
*/
    // (optional) clear to green so we can see if we're failing to set pixels
    GLES20.glClearColor(0.0f, 1.0f, 0.0f, 1.0f);
    GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);

    GLES20.glUseProgram(mProgram);
    checkGlError("glUseProgram");

    GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
    GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureID);

    mTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET);
    GLES20.glVertexAttribPointer(maPositionHandle, 3, GLES20.GL_FLOAT, false,
            TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
    checkGlError("glVertexAttribPointer maPosition");
    GLES20.glEnableVertexAttribArray(maPositionHandle);
    checkGlError("glEnableVertexAttribArray maPositionHandle");

    mTriangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET);
    GLES20.glVertexAttribPointer(maTextureHandle, 2, GLES20.GL_FLOAT, false,
            TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
    checkGlError("glVertexAttribPointer maTextureHandle");
    GLES20.glEnableVertexAttribArray(maTextureHandle);
    checkGlError("glEnableVertexAttribArray maTextureHandle");

    Matrix.setIdentityM(mMVPMatrix, 0);
    GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0);
    GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0);

    GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
    checkGlError("glDrawArrays");

    GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0);
}

/**
 * Initializes GL state.  Call this after the EGL surface has been created and made current.
 */
public void surfaceCreated() {
    mProgram = createProgram(VERTEX_SHADER, FRAGMENT_SHADER);
    if (mProgram == 0) {
        throw new RuntimeException("failed creating program");
    }

    maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition");
    checkLocation(maPositionHandle, "aPosition");
    maTextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord");
    checkLocation(maTextureHandle, "aTextureCoord");

    muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
    checkLocation(muMVPMatrixHandle, "uMVPMatrix");
    muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix");
    checkLocation(muSTMatrixHandle, "uSTMatrix");

    int[] textures = new int[1];
    GLES20.glGenTextures(1, textures, 0);

    mTextureID = textures[0];
    GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureID);
    checkGlError("glBindTexture mTextureID");

    GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER,
            GLES20.GL_NEAREST);
    GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER,
            GLES20.GL_LINEAR);
    GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S,
            GLES20.GL_CLAMP_TO_EDGE);
    GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T,
            GLES20.GL_CLAMP_TO_EDGE);
    checkGlError("glTexParameter");
}

/**
 * Replaces the fragment shader.  Pass in null to reset to default.
 */
public void changeFragmentShader(String fragmentShader) {
    if (fragmentShader == null) {
        fragmentShader = FRAGMENT_SHADER;
    }
    GLES20.glDeleteProgram(mProgram);
    mProgram = createProgram(VERTEX_SHADER, fragmentShader);
    if (mProgram == 0) {
        throw new RuntimeException("failed creating program");
    }
}

private int loadShader(int shaderType, String source) {
    int shader = GLES20.glCreateShader(shaderType);
    checkGlError("glCreateShader type=" + shaderType);
    GLES20.glShaderSource(shader, source);
    GLES20.glCompileShader(shader);
    int[] compiled = new int[1];
    GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0);
    if (compiled[0] == 0) {
        Log.e(TAG, "Could not compile shader " + shaderType + ":");
        Log.e(TAG, " " + GLES20.glGetShaderInfoLog(shader));
        GLES20.glDeleteShader(shader);
        shader = 0;
    }
    return shader;
}

private int createProgram(String vertexSource, String fragmentSource) {
    int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource);
    if (vertexShader == 0) {
        return 0;
    }
    int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource);
    if (pixelShader == 0) {
        return 0;
    }

    int program = GLES20.glCreateProgram();
    if (program == 0) {
        Log.e(TAG, "Could not create program");
    }
    GLES20.glAttachShader(program, vertexShader);
    checkGlError("glAttachShader");
    GLES20.glAttachShader(program, pixelShader);
    checkGlError("glAttachShader");
    GLES20.glLinkProgram(program);
    int[] linkStatus = new int[1];
    GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0);
    if (linkStatus[0] != GLES20.GL_TRUE) {
        Log.e(TAG, "Could not link program: ");
        Log.e(TAG, GLES20.glGetProgramInfoLog(program));
        GLES20.glDeleteProgram(program);
        program = 0;
    }
    return program;
}

public void checkGlError(String op) {
    int error;
    while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
        Log.e(TAG, op + ": glError " + error);
        throw new RuntimeException(op + ": glError " + error);
    }
}

public static void checkLocation(int location, String label) {
    if (location < 0) {
        throw new RuntimeException("Unable to locate '" + label + "' in program");
    }
}

}