操纵android相机的深度图导致应用程序崩溃

Question

试图通过android的camera2 api访问TOF相机生成的深度图，但是复制深度图时应用总是崩溃。

• Android 操作系统：10
• Android api : 28
• 拱门：armv8a
• Qt 版本：Felgo 3.6.0
• 开发：windows 10 64 位
• ndk : r18b
• 电话：mate30

错误消息：F libc：致命信号 11 (SIGSEGV)，代码 2 (SEGV_ACCERR)，tid 31512 (ImageReader-480) 中的故障地址 0x741c259980，pid 31412 (P.Androidndkcam)

找到后置摄像头支持depth16

std::tuple<std::string, bool> get_camera_depth_id(ACameraManager *cam_manager, int camera_facing)
{
    auto camera_ids = get_camera_id_list(cam_manager);
    if(camera_ids){
        qInfo()<<__func__<<": found camera count "<<camera_ids->numCameras;
        for(int i = 0; i < camera_ids->numCameras; ++i){
            const char *id = camera_ids->cameraIds[i];
            camera_status_t ret = ACAMERA_OK;
            auto chars = get_camera_characteristics(cam_manager, id, &ret);
            if(ret != ACAMERA_OK){
                qInfo()<<__func__<<": cannot obtain characteristics of camera id = "<<id;
                continue;
            }
            auto const entry = get_camera_capabilities(chars.get(), &ret);
            if(ret != ACAMERA_OK){
                qInfo()<<__func__<<": cannot obtain capabilities of camera id = "<<id;
                continue;
            }

            ACameraMetadata_const_entry lens_info;
            ACameraMetadata_getConstEntry(chars.get(), ACAMERA_LENS_FACING, &lens_info);
            auto const facing = static_cast<acamera_metadata_enum_android_lens_facing_t>(lens_info.data.u8[i]);
            bool is_right_face = facing == camera_facing;
            bool support_bc = false, support_depth = false;
            for(uint32_t i = 0; i < entry.count; i++) {
                if(entry.data.u8[i] == ACAMERA_REQUEST_AVAILABLE_CAPABILITIES_BACKWARD_COMPATIBLE){
                    support_bc = true;
                }
                if(entry.data.u8[i] == ACAMERA_REQUEST_AVAILABLE_CAPABILITIES_DEPTH_OUTPUT){
                    support_depth = true;
                }
            }
            qInfo()<<__func__<<" support bc = "<<support_bc<<", support depth = "<<support_depth
                  <<", is right face = "<<is_right_face;
            if(is_right_face && support_depth){
                qInfo()<<__func__<<": obtain depth camera id = "<<id;
                return {id, support_bc};
            }
        }
    }else{
        qInfo()<<__func__<<": cannot get depth cam";
    }

    return {};
}

打开相机

    void initCam()
    {
        qDebug()<<__func__<<": init camera manager";
        cameraManager = ACameraManager_create();

        qDebug()<<__func__<<": get back facing camera id";
        auto [id, support_bc] = get_camera_depth_id(cameraManager, ACAMERA_LENS_FACING_BACK);
        //auto const id = get_camera_id(cameraManager, ACAMERA_LENS_FACING_BACK);
        qInfo()<<__func__<<": back camera id = "<<id.c_str();
        if(!id.empty()){
            auto const cam_status =
                    ACameraManager_openCamera(cameraManager, id.c_str(), &cameraDeviceCallbacks, &cameraDevice);
            qInfo()<<__func__<<" cam status = "<<cam_status;
            qDebug()<<__func__<<": open camera";

            android_cam_info cam_info(*cameraManager, id.c_str());
            qInfo()<<__func__<<" print depth stream configuration info";
            //print the format, width, height, is input information
            cam_info.stream_config(ACAMERA_DEPTH_AVAILABLE_DEPTH_STREAM_CONFIGURATIONS).print();
            //obtain minimum widh and height for the depth map
            std::tie(width_, height_) =
                    cam_info.stream_config(ACAMERA_DEPTH_AVAILABLE_DEPTH_STREAM_CONFIGURATIONS).get_minimum_dimension();

            imageReader = createJpegReader();
            if(imageReader){
                imageWindow = createSurface(imageReader);
                ANativeWindow_acquire(imageWindow);
                ACameraDevice_createCaptureRequest(cameraDevice, TEMPLATE_PREVIEW, &request);
                ACameraOutputTarget_create(imageWindow, &imageTarget);
                ACaptureRequest_addTarget(request, imageTarget);
                ACaptureSessionOutput_create(imageWindow, &imageOutput);
                ACaptureSessionOutputContainer_create(&outputs);
                ACaptureSessionOutputContainer_add(outputs, imageOutput);

                ACameraDevice_createCaptureSession(cameraDevice, outputs, &sessionStateCallbacks, &textureSession);
                // Start capturing continuously
                ACameraCaptureSession_setRepeatingRequest(textureSession, &captureCallbacks, 1, &request, nullptr);
            }
        }
    }

我创建 imageReader 的方式

    AImageReader* createJpegReader()
    {
        AImageReader* reader = nullptr;
        media_status_t status = AImageReader_new(width_, height_, AIMAGE_FORMAT_DEPTH16, 1, &reader);

        if(status != AMEDIA_OK){
            qInfo()<<__func__<<": cannot create AImageReader, error code is = "<<status;
            return nullptr;
        }

        AImageReader_ImageListener listener;
        listener.context = this;
        listener.onImageAvailable = imageCallback;

        AImageReader_setImageListener(reader, &listener);

        return reader;
    }

创建表面

ANativeWindow* createSurface(AImageReader* reader)
{
    ANativeWindow *nativeWindow;
    AImageReader_getWindow(reader, &nativeWindow);

    return nativeWindow;
}

AImageReader 的回调

static void process_depth_16(void* context, AImage *image)
{
    uint16_t *data = nullptr;
    int len = 0;
    auto const status = AImage_getPlaneData(image, 0, reinterpret_cast<uint8_t**>(&data), &len);
    if(status != AMEDIA_OK){
        qInfo()<<__func__<<": AImage_getPlaneData fail, error code = "<<status;
        return;
    }
    qInfo()<<__func__<<": image len = "<<len;

    auto *impl = static_cast<pimpl*>(context);
    convert_depth_16_to_cvmat(data, impl->width_, impl->height_);
}

static void imageCallback(void* context, AImageReader* reader)
{
    qDebug()<<__func__;
    int status = -1;
    auto image = get_next_image(reader, &status);
    if(status != AMEDIA_OK){
        qInfo()<<__func__<<": cannot acquire next image, error code = "<<status;
        return;
    }

    int32_t format = -1;
    AImage_getFormat(image.get(), &format);
    if(format == AIMAGE_FORMAT_DEPTH16){
        process_depth_16(context, image.get());
    }else{
        qInfo()<<__func__<<": do not support format = "<<format;
    }
}

复制深度图（此功能会导致应用崩溃）

cv::Mat convert_depth_16_to_cvmat(uint16_t *data, int32_t width, int32_t height)
{
    auto *depth_ptr = data;
    cv::Mat output(height, width, CV_16U);
    for(int32_t row = 0; row != height; ++row){
        depth_ptr += width * row;
        auto *output_ptr = output.ptr<ushort>(row);
        qInfo()<<__func__<<": row = "<<row; //crash when row equal to 27,sometimes over 50 
        for(int32_t col = 0; col != width; ++col){
            output_ptr[col] = depth_ptr[col];
        }
    }

    return output;
}

我从 ACAMERA_DEPTH_AVAILABLE_DEPTH_STREAM_CONFIGURATIONS 获得的最小宽度和高度为 480x360。代码有什么问题吗？

其他实用功能放在pastebin。

编辑：数据长度为 345600，这意味着应该有 172800(480x360) 像素，数据类型为 uint16_t

Answer 1

我找到答案了，mate30不支持深度图，但是原生api没有告诉我们，可能原生api相关深度图的功能还没有那么成熟。