libvlc 在内存中流式传输图像时的主要输入错误和延迟

Question

大家！

我正在开发一个使用 libvlc 在内存中流式传输图像的项目。为了测试，我流式传输相机帧。我在这里遇到了麻烦：首先有巨大的延迟（大约7s），并且流非常不稳定。

如果您能在我的代码中发现一些错误，将会很有帮助！

这 3 个错误重复了很多次。

主输入错误：ES_OUT_SET_PCR 调用太晚（pts_delay 增加到 692 毫秒）
主输入错误：调用了 ES_OUT_RESET_PCR
avcodec 解码器错误：延迟视频超过 5 秒 -> 丢帧（电脑太慢？）

我特别好奇最后一个错误：为什么我只想编码一些图像时会出现解码器错误？

这是我的代码：

#include <Windows.h>
#include <vlc.h>
#include <vlc_common.h>
#include <vlc_threads.h>
//#include <vlc/plugins/vlc_threads.h>
using namespace std;

#include <opencv2/opencv.hpp>
using namespace cv;

#define CAMERA_WIDTH 640
#define CAMERA_HEIGHT 480



vlc_mutex_t imem_get_mutex;
VideoCapture *g_camera;
int g_transport_number = 8080;



static int vlc_imem_get_callback(void *data, const char *cookie,
    int64_t * dts, int64_t * pts,
    unsigned *flags, size_t * size,
    void **output)
{
    //usleep(20000);
    Mat frame;
    vlc_mutex_lock(&imem_get_mutex);


    (*g_camera) >> frame;

    *output = malloc(frame.rows * frame.cols * 3);
    memcpy(*output,frame.data,frame.rows * frame.cols * 3);

    if (pts)
        *pts = 1;
    if (dts)
        *dts = 1;
    //  *size=(size_t)300;
    *size=(size_t)(frame.rows * frame.cols * 3);

    vlc_mutex_unlock(&imem_get_mutex);

    frame.release();

    return 0;
}

static void vlc_imem_release_callback(void *data, const char *cookie,
    size_t size, void *unknown)
{
    //  printf("release\n\n");

    free(unknown);
}


int main()
{
    vlc_mutex_init(&imem_get_mutex);

    g_camera = new VideoCapture(0);
    g_camera->set(CV_CAP_PROP_FRAME_WIDTH, CAMERA_WIDTH);
    g_camera->set(CV_CAP_PROP_FRAME_WIDTH, CAMERA_HEIGHT);

    libvlc_instance_t * inst;
    libvlc_media_player_t *mp;
    libvlc_media_t *m;

    char smem_options1[2000];
    char venc_options[1000];
//  sprintf(venc_options,"profile=baseline,level=3,keyint=50,bframes=3,no-cabac,ref=3,no-interlaced,vbv-maxrate=512,vbv-bufsize=256,aq-mode=0,no-mbtree,partitions=none,no-weightb,weightp=0,me=dia,subme=0,no-mixed-refs,no-8x8dct,trellis=0");
    sprintf(venc_options,"bframes=6,ref=6");

    sprintf(smem_options1,"#transcode{venc=x264{%s},vcodec=h264,vb=1000,fps=30,scale=0,width=640,height=480,channels=1,samplerate=44100}:duplicate{dst=http{mux=ts,dst=:%d/test}",venc_options,g_transport_number);
//  sprintf(smem_options1,"#transcode{vcodec=h264,vb=1000,fps=30,scale=0,width=640,height=480,channels=1,samplerate=44100}:duplicate{dst=http{mux=ts,dst=:%d/test},dst=display",venc_options,g_transport_number);

    char str_imem_get[100], str_imem_release[100],str_imem_data[100];
    sprintf(str_imem_get, "--imem-get=%ld", vlc_imem_get_callback);
    sprintf(str_imem_release, "--imem-release=%ld", vlc_imem_release_callback);
    //  sprintf(str_imem_data,"--imem-data=%ld",(long int)test_buffer);


    const char * const vlc_args[] = {
        "-I","dummy",
        "--ignore-config",

        "--demux","rawvideo",
        "--rawvid-fps","30",
        "--rawvid-width","640",
        "--rawvid-height","480",
        "--rawvid-chroma","RV24",

        "--imem-channels=1",
        "--imem-data=0",
        "--imem-cat=4",
        "--imem-fps=30",
        "--imem-codec=none",


        str_imem_get,
        str_imem_release,

        "--sout",
        smem_options1
    };

    inst = libvlc_new (sizeof (vlc_args) / sizeof (vlc_args[0]), vlc_args);
    m = libvlc_media_new_location(inst, "imem://");
    mp = libvlc_media_player_new_from_media (m);
    libvlc_media_release (m);

    libvlc_media_player_play (mp);

    Sleep (200000); 

    libvlc_media_player_stop (mp);
    libvlc_media_player_release (mp);
    libvlc_release (inst);


    return 0;
}

感谢您的帮助，我很抱歉我的英语不好......

Answer 1

我遇到了同样的问题，并发现该问题与我使用 OpenCV 从实时源捕获时使用的 DTS 和 PTS 值有关。我正在实时计算 DTS 和 PTS 值以避免 pts_delay 增加，但是像你一样大约 5 秒后，imem get 函数回调之间的时间不断增加。然后我使用了一个固定的帧速率值间隔值，例如每次添加的 33333。这解决了滞后问题，但导致时钟重置的第一个错误。我找到的解决方案是将DTS设置为-1（未使用），并将PTS的值设置为libvlc_clock()。例如：

int MyImemGetCallback (void *data, 
    const char *cookie, 
    int64_t *dts, 
    int64_t *pts, 
    unsigned *flags, 
    size_t * bufferSize, 
    void ** buffer) 
{
    MyImemData* imem = (MyImemData*)data;

    if(imem == NULL)
        return 0;

    if(imem->mQuitFlag)
        return 1; // Exit

    if(imem->mFrameNumber == imem->mPrevFrameNumber)
    {
        return 0; // No new image data
    }

    // Update frame count information...
    imem->mPrevFrameNumber = imem->mFrameNumber;
    *dts = -1;
    // You can use libvlc_clock to avoid PCR reset and delays
    // on realtime data...
    *pts = libvlc_clock();

    *bufferSize = 
        imem->mOpenCvImage->rows * 
        imem->mOpenCvImage->cols * 
        imem->mOpenCvImage->channels();
    *buffer = imem->mOpenCvImage->data;

    return 0; // Success.
}

我知道这是 6 个月前发布的，但我今天遇到了同样的问题，并认为可能还有其他问题。