c++ - 在QML VideoOutput中将cv::Mat显示为QVideoFrame

Question

我有一个OpenCV后端，该后端通过cv::VideoCapture从摄像机设备检索视频帧，进行一些处理，然后将cv::Mat帧传递给Qt5应用程序，以在QML VideoOutput中显示。

问题在于绘制的框架为空/白色。
CameraService类正在从后端对象接收cv::Mat，该后端对象使用Qt::QueuedConnection信号在自己的线程上运行。然后，将其转换为QImage，用于初始化QVideoFrame并将其设置为从QML QAbstractVideoSurface接收到的VideoOutput，然后再设置像素格式。

在转换为cv::Mat之前，我已经检查了QVideoFrame是否具有有效的内容，因此情况并非如此。

还是我做的完全错误，应该绘制图像？

相关代码:

CameraService.cpp

CameraService::CameraService(Video::Backend *backend) 
  : QObject(), 
  surface(nullptr),
  isFormatSet(false) {

  this->backend = backend;

  connect(
    backend, &Video::Backend::onFrameReady,
    this, &CameraService::onVideoFrameReady, 
    Qt::QueuedConnection);
}

CameraService::~CameraService() {

  backend->deleteLater();
}

QAbstractVideoSurface *CameraService::getVideoSurface() const {

  return surface;
}

void CameraService::setVideoSurface(QAbstractVideoSurface *surface) {

  if (!this->surface && surface)
    backend->start();

  if (this->surface && this->surface != surface && this->surface->isActive())      
    this->surface->stop();

  this->surface = surface;

  if (this->surface && format.isValid()) {

    format = this->surface->nearestFormat(format);
    this->surface->start(format);
  }
}

void CameraService::setFormat(
  int width, 
  int height, 
  QVideoFrame::PixelFormat frameFormat
){
    QSize size(width, height);
    QVideoSurfaceFormat format(size, frameFormat);

    this->format = format;

    if (surface) {

        if (surface->isActive())
            surface->stop();

        this->format = surface->nearestFormat(this->format);

        surface->start(this->format);
    }
}

void CameraService::onVideoFrameReady(cv::Mat currentFrame) {

  if (!surface || currentFrame.empty())
    return;

  cv::Mat continuousFrame;

  if (!currentFrame.isContinuous())
    continuousFrame = currentFrame.clone();
  else
    continuousFrame = currentFrame;

  if (!isFormatSet) {

    setFormat(
      continuousFrame.cols, 
      continuousFrame.rows, 
      QVideoFrame::PixelFormat::Format_BGR32);
    isFormatSet = true;
  }

  frame = QImage(
    (uchar *)continuousFrame.data,
    continuousFrame.cols,
    continuousFrame.rows,
    continuousFrame.step,
    QVideoFrame::imageFormatFromPixelFormat(
      QVideoFrame::PixelFormat::Format_BGR32));

  surface->present(QVideoFrame(frame));
}

QML对象:

    VideoOutput {
        objectName: "videoOutput";
        anchors.fill: parent;
        fillMode: VideoOutput.PreserveAspectCrop;
        source: CameraService;
    }

使用以下语句，可以将CameraService对象作为单例提供给QML:
qmlRegisterSingletonInstance<Application::CameraService>("Application.CameraService", 1, 0, "CameraService", service);

Answer 1

分析代码后，我注意到不支持该转换(建议您检查格式是否有效)。为此，我进行了一些更改:...

#ifndef CAMERASERVICE_H
#define CAMERASERVICE_H

#include "backend.h"

#include <QObject>
#include <QPointer>
#include <QVideoFrame>
#include <QVideoSurfaceFormat>
#include <opencv2/core/mat.hpp>

class QAbstractVideoSurface;

class CameraService : public QObject
{
    Q_OBJECT
    Q_PROPERTY(QAbstractVideoSurface* videoSurface READ videoSurface WRITE setVideoSurface NOTIFY surfaceChanged)
public:
    explicit CameraService(Backend *backend, QObject *parent = nullptr);
    QAbstractVideoSurface* videoSurface() const;

public Q_SLOTS:
    void setVideoSurface(QAbstractVideoSurface* surface);
Q_SIGNALS:
    void surfaceChanged(QAbstractVideoSurface* surface);
private Q_SLOTS:
    void onVideoFrameReady(cv::Mat currentFrame);
private:
    void setFormat(int width, int height, QVideoFrame::PixelFormat frameFormat);

    QPointer<QAbstractVideoSurface> m_surface;
    QScopedPointer<Backend> m_backend;
    QVideoSurfaceFormat m_format;
    bool m_isFormatSet;
    QImage m_image;
};

#endif // CAMERASERVICE_H

#include "backend.h"
#include "cameraservice.h"

#include <QAbstractVideoSurface>
#include <iostream>

CameraService::CameraService(Backend *backend, QObject *parent)
    : QObject(parent), m_backend(backend), m_isFormatSet(false)
{
    connect(m_backend.data(), &Backend::frameReady, this, &CameraService::onVideoFrameReady);
}

QAbstractVideoSurface *CameraService::videoSurface() const
{
    return m_surface;
}

void CameraService::setVideoSurface(QAbstractVideoSurface *surface){
    if (m_surface == surface)
        return;
    if(m_surface && m_surface != surface && m_surface->isActive())
        m_surface->stop();
    m_surface = surface;
    Q_EMIT surfaceChanged(m_surface);
    m_backend->start();
    if (m_surface && m_format.isValid()) {
        m_format = m_surface->nearestFormat(m_format);
        m_surface->start(m_format);
    }
}

void CameraService::setFormat(
        int width,
        int height,
        QVideoFrame::PixelFormat frameFormat
        ){
    QSize size(width, height);
    QVideoSurfaceFormat format(size, frameFormat);
    m_format = format;
    if (m_surface) {
        if (m_surface->isActive())
            m_surface->stop();
        m_format = m_surface->nearestFormat(m_format);
        m_surface->start(m_format);
    }
}

void CameraService::onVideoFrameReady(cv::Mat currentFrame){
    if (!m_surface || currentFrame.empty())
        return;
    cv::Mat continuousFrame;
    if (!currentFrame.isContinuous())
        continuousFrame = currentFrame.clone();
    else
        continuousFrame = currentFrame;
    if (!m_isFormatSet) {
        setFormat(continuousFrame.cols,
                  continuousFrame.rows,
                  QVideoFrame::Format_RGB32);
        m_isFormatSet = true;
    }
    m_image = QImage(continuousFrame.data,
                     continuousFrame.cols,
                     continuousFrame.rows,
                     continuousFrame.step,
                     QImage::Format_RGB888);
    m_image = m_image.rgbSwapped();
    m_image.convertTo(QVideoFrame::imageFormatFromPixelFormat(QVideoFrame::Format_RGB32));
    m_surface->present(QVideoFrame(m_image));
}

您可以找到完整的示例here。