在 QML VideoOutput 中将 cv::Mat 显示为 QVideoFrame

Question

我有一个 OpenCV 后端，它通过 cv::VideoCapture 从摄像头设备中检索视频帧，进行一些处理，然后将 cv::Mat 帧传递给 Qt5 应用程序以在 QML VideoOutput 中显示。

问题是绘制的框架是空的/白色的。

CameraService 类正在接收来自后端对象的cv::Mat，该对象使用Qt::QueuedConnection 信号在自己的线程上运行。然后我将其转换为QImage，我用它来初始化QVideoFrame，并在设置像素格式后将其传递给从QML VideoOutput 接收的QAbstractVideoSurface。

我在转换为QVideoFrame之前检查了cv::Mat是否有有效内容，所以不是这样。

还是我做错了，应该画一张图片？

相关代码：

CameraService.cpp

CameraService::CameraService(Video::Backend *backend) 
  : QObject(), 
  surface(nullptr),
  isFormatSet(false) {

  this->backend = backend;

  connect(
    backend, &Video::Backend::onFrameReady,
    this, &CameraService::onVideoFrameReady, 
    Qt::QueuedConnection);
}

CameraService::~CameraService() {

  backend->deleteLater();
}

QAbstractVideoSurface *CameraService::getVideoSurface() const {

  return surface;
}

void CameraService::setVideoSurface(QAbstractVideoSurface *surface) {

  if (!this->surface && surface)
    backend->start();

  if (this->surface && this->surface != surface && this->surface->isActive())      
    this->surface->stop();

  this->surface = surface;

  if (this->surface && format.isValid()) {

    format = this->surface->nearestFormat(format);
    this->surface->start(format);
  }
}

void CameraService::setFormat(
  int width, 
  int height, 
  QVideoFrame::PixelFormat frameFormat
){
    QSize size(width, height);
    QVideoSurfaceFormat format(size, frameFormat);

    this->format = format;

    if (surface) {

        if (surface->isActive())
            surface->stop();

        this->format = surface->nearestFormat(this->format);

        surface->start(this->format);
    }
}

void CameraService::onVideoFrameReady(cv::Mat currentFrame) {

  if (!surface || currentFrame.empty())
    return;

  cv::Mat continuousFrame;

  if (!currentFrame.isContinuous())
    continuousFrame = currentFrame.clone();
  else
    continuousFrame = currentFrame;

  if (!isFormatSet) {

    setFormat(
      continuousFrame.cols, 
      continuousFrame.rows, 
      QVideoFrame::PixelFormat::Format_BGR32);
    isFormatSet = true;
  }

  frame = QImage(
    (uchar *)continuousFrame.data,
    continuousFrame.cols,
    continuousFrame.rows,
    continuousFrame.step,
    QVideoFrame::imageFormatFromPixelFormat(
      QVideoFrame::PixelFormat::Format_BGR32));

  surface->present(QVideoFrame(frame));
}

QML 对象：

    VideoOutput {
        objectName: "videoOutput";
        anchors.fill: parent;
        fillMode: VideoOutput.PreserveAspectCrop;
        source: CameraService;
    }

CameraService 对象使用以下语句作为单例提供给 QML：

qmlRegisterSingletonInstance<Application::CameraService>("Application.CameraService", 1, 0, "CameraService", service);

Answer 1

分析代码我注意到不支持转换（我建议你检查格式是否有效）。为此我做了一些改变：...

#ifndef CAMERASERVICE_H
#define CAMERASERVICE_H

#include "backend.h"

#include <QObject>
#include <QPointer>
#include <QVideoFrame>
#include <QVideoSurfaceFormat>
#include <opencv2/core/mat.hpp>

class QAbstractVideoSurface;

class CameraService : public QObject
{
    Q_OBJECT
    Q_PROPERTY(QAbstractVideoSurface* videoSurface READ videoSurface WRITE setVideoSurface NOTIFY surfaceChanged)
public:
    explicit CameraService(Backend *backend, QObject *parent = nullptr);
    QAbstractVideoSurface* videoSurface() const;

public Q_SLOTS:
    void setVideoSurface(QAbstractVideoSurface* surface);
Q_SIGNALS:
    void surfaceChanged(QAbstractVideoSurface* surface);
private Q_SLOTS:
    void onVideoFrameReady(cv::Mat currentFrame);
private:
    void setFormat(int width, int height, QVideoFrame::PixelFormat frameFormat);

    QPointer<QAbstractVideoSurface> m_surface;
    QScopedPointer<Backend> m_backend;
    QVideoSurfaceFormat m_format;
    bool m_isFormatSet;
    QImage m_image;
};

#endif // CAMERASERVICE_H

#include "backend.h"
#include "cameraservice.h"

#include <QAbstractVideoSurface>
#include <iostream>

CameraService::CameraService(Backend *backend, QObject *parent)
    : QObject(parent), m_backend(backend), m_isFormatSet(false)
{
    connect(m_backend.data(), &Backend::frameReady, this, &CameraService::onVideoFrameReady);
}

QAbstractVideoSurface *CameraService::videoSurface() const
{
    return m_surface;
}

void CameraService::setVideoSurface(QAbstractVideoSurface *surface){
    if (m_surface == surface)
        return;
    if(m_surface && m_surface != surface && m_surface->isActive())
        m_surface->stop();
    m_surface = surface;
    Q_EMIT surfaceChanged(m_surface);
    m_backend->start();
    if (m_surface && m_format.isValid()) {
        m_format = m_surface->nearestFormat(m_format);
        m_surface->start(m_format);
    }
}

void CameraService::setFormat(
        int width,
        int height,
        QVideoFrame::PixelFormat frameFormat
        ){
    QSize size(width, height);
    QVideoSurfaceFormat format(size, frameFormat);
    m_format = format;
    if (m_surface) {
        if (m_surface->isActive())
            m_surface->stop();
        m_format = m_surface->nearestFormat(m_format);
        m_surface->start(m_format);
    }
}

void CameraService::onVideoFrameReady(cv::Mat currentFrame){
    if (!m_surface || currentFrame.empty())
        return;
    cv::Mat continuousFrame;
    if (!currentFrame.isContinuous())
        continuousFrame = currentFrame.clone();
    else
        continuousFrame = currentFrame;
    if (!m_isFormatSet) {
        setFormat(continuousFrame.cols,
                  continuousFrame.rows,
                  QVideoFrame::Format_RGB32);
        m_isFormatSet = true;
    }
    m_image = QImage(continuousFrame.data,
                     continuousFrame.cols,
                     continuousFrame.rows,
                     continuousFrame.step,
                     QImage::Format_RGB888);
    m_image = m_image.rgbSwapped();
    m_image.convertTo(QVideoFrame::imageFormatFromPixelFormat(QVideoFrame::Format_RGB32));
    m_surface->present(QVideoFrame(m_image));
}

_{你可以找到完整的例子here。}