FFMPEG C api h.264 编码/MPEG2 ts 流问题

Question

类原型如下：

#ifndef _FULL_MOTION_VIDEO_STREAM_H_
#define _FULL_MOTION_VIDEO_STREAM_H_

#include <memory>
#include <string>

#ifndef INT64_C
# define INT64_C(c) (c ## LL)
# define UINT64_C(c) (c ## ULL)
#endif

extern "C" 
{
    #include "libavutil/opt.h"
    #include "libavcodec/avcodec.h"
    #include "libavutil/channel_layout.h"
    #include "libavutil/common.h"
    #include "libavutil/imgutils.h"
    #include "libavutil/mathematics.h"
    #include "libavutil/samplefmt.h"
    #include "libavformat/avformat.h"

    #include <libavutil/timestamp.h>
    #include <libswscale/swscale.h>
    #include <libswresample/swresample.h>
}

class FMVStream 
{
    public:
        struct OutputStream 
        {
            OutputStream() :
            st(0),
            next_pts(0),
            samples_count(0),
            frame(0),
            tmpFrame(0),
            sws_ctx(0)
            {
            }

            AVStream *st;

            /* pts of the next frame that will be generated */
            int64_t next_pts;
            int samples_count;

            AVFrame *frame;
            AVFrame *tmpFrame;

            struct SwsContext *sws_ctx;
        };

        ///
        /// Constructor
        ///
        FMVStream();

        ///
        /// Destructor
        ///
        ~FMVStream();

        ///
        /// Frame encoder helper function
        ///
        /// Encodes a raw RGB frame into the transport stream
        ///
        int EncodeFrame(uint8_t* frame);

        ///
        /// Frame width setter
        ///
        void setFrameWidth(int width);

        ///
        /// Frame width getter
        ///
        int getFrameWidth() const;

        ///
        /// Frame height setter
        ///
        void setFrameHeight(int height);

        ///
        /// Frame height getter
        ///
        int getFrameHeight() const;

        ///
        /// Stream address setter
        ///
        void setStreamAddress(const std::string& address);

        ///
        /// Stream address getter
        ///
        std::string getStreamAddress() const;

    private:

        ///
        /// Video Stream creation
        ///
        AVStream* initVideoStream(AVFormatContext* oc);

        ///
        /// Raw frame transcoder
        ///
        /// This will convert the raw RGB frame to a raw YUV frame necessary for h.264 encoding
        ///
        void CopyFrameData(uint8_t* src_frame);

        ///
        /// Video frame allocator
        ///
        AVFrame* AllocPicture(PixelFormat pix_fmt, int width, int height);

        ///
        /// Debug print helper function
        ///
        void print_sdp(AVFormatContext **avc, int n);

        ///
        /// Write the frame to the stream
        ///
        int write_frame(AVFormatContext *fmt_ctx, const AVRational *time_base, AVStream *st, AVPacket *pkt);

        ///
        /// initialize the frame data
        ///
        void initFrame();

        // formatting data needed for output streaming and the output container (MPEG 2 TS)
        AVOutputFormat* format;
        AVFormatContext* format_ctx;

        // structure container for our video stream
        OutputStream stream;

        AVIOContext* io_ctx;

        std::string streamFilename;

        int frameWidth;
        int frameHeight;
};

#endif

这个块开始类声明。

#include "FullMotionVideoStream.h"

#include <stdexcept>
#include <iostream>

FMVStream::FMVStream()
    : format(0),
    format_ctx(0),
    stream(),
    io_ctx(0),
    streamFilename("test.mpeg"),
    frameWidth(640),
    frameHeight(480)
{
    // Register all formats and codecs
    av_register_all();
    avcodec_register_all();

    // Init networking
    avformat_network_init();

    // Find format
    this->format = av_guess_format("mpegts", NULL, NULL);

    // allocate the AVFormatContext
    this->format_ctx = avformat_alloc_context();

    if (!this->format_ctx) 
    {
        throw std::runtime_error("avformat_alloc_context failed");
    }

    this->format_ctx->oformat = this->format;
    //sprintf_s(this->format_ctx->filename, sizeof(this->format_ctx->filename), "%s", this->streamFilename.c_str());

    this->stream.st = initVideoStream(this->format_ctx);

    this->initFrame();

    // Allocate AVIOContext
    int ret = avio_open(&this->io_ctx, this->streamFilename.c_str(), AVIO_FLAG_WRITE);

    if (ret != 0) 
    {
        throw std::runtime_error("avio_open failed");
    }

    this->format_ctx->pb = this->io_ctx;

    // Print some debug info about the format
    av_dump_format(this->format_ctx, 0, NULL, 1);

    // Begin the output by writing the container header
    avformat_write_header(this->format_ctx, NULL);

    AVFormatContext* ac[] = { this->format_ctx };
    print_sdp(ac, 1);
}

FMVStream::~FMVStream()
{
    av_write_trailer(this->format_ctx);
    avcodec_close(this->stream.st->codec);

    avio_close(io_ctx);

    avformat_free_context(this->format_ctx);

    av_frame_free(&this->stream.frame);
    av_free(this->format);
}

AVFrame* FMVStream::AllocPicture(PixelFormat pix_fmt, int width, int height)
{
    // Allocate a frame
    AVFrame* frame = av_frame_alloc();

    if (frame == nullptr)
    {
        throw std::runtime_error("avcodec_alloc_frame failed");
    }

    if (av_image_alloc(frame->data, frame->linesize, width, height, pix_fmt, 1) < 0)
    {
        throw std::runtime_error("av_image_alloc failed");
    }

    frame->width = width;
    frame->height = height;
    frame->format = pix_fmt;

    return frame;
}

void FMVStream::print_sdp(AVFormatContext **avc, int n)
{
    char sdp[2048];
    av_sdp_create(avc, n, sdp, sizeof(sdp));
    printf("SDP:\n%s\n", sdp);
    fflush(stdout);
}

AVStream* FMVStream::initVideoStream(AVFormatContext *oc)
{
    AVStream* st = avformat_new_stream(oc, NULL);

    if (st == nullptr) 
    {
        std::runtime_error("Could not alloc stream");
    }

    AVCodec* codec = avcodec_find_encoder(AV_CODEC_ID_H264);

    if (codec == nullptr) 
    {
        throw std::runtime_error("couldn't find mpeg2 encoder");
    }

    st->codec = avcodec_alloc_context3(codec);

    st->codec->codec_id = AV_CODEC_ID_H264;
    st->codec->codec_type = AVMEDIA_TYPE_VIDEO;
    st->codec->bit_rate = 400000;

    st->codec->width = this->frameWidth;
    st->codec->height = this->frameHeight;

    st->time_base.num = 1;
    st->time_base.den = 30;

    st->codec->framerate.num = 1;
    st->codec->framerate.den = 30;

    st->codec->max_b_frames = 2;
    st->codec->gop_size = 12;
    st->codec->pix_fmt = PIX_FMT_YUV420P;

    st->id = oc->nb_streams - 1;

    if (oc->oformat->flags & AVFMT_GLOBALHEADER)
    {
        st->codec->flags |= CODEC_FLAG_GLOBAL_HEADER;
    }

    // option setup for the codec
    av_opt_set(st->codec->priv_data, "profile", "baseline", AV_OPT_SEARCH_CHILDREN);

    if (avcodec_open2(st->codec, codec, NULL) < 0)
    {
        throw std::runtime_error("avcodec_open failed");
    }

    return st;
}

void FMVStream::initFrame()
{
    // Allocate a tmp frame for converting our raw RGB data to YUV for encoding
    this->stream.tmpFrame = this->AllocPicture(PIX_FMT_RGB24, this->frameWidth, this->frameHeight);

    // Allocate a main frame
    this->stream.frame = this->AllocPicture(PIX_FMT_YUV420P, this->frameWidth, this->frameHeight);
}

此块正在尝试将原始 RGB 转换为我们所需的 YUV 格式以进行 h.264 编码。

void FMVStream::CopyFrameData(uint8_t* data)
{
    // fill image with our raw RGB data
    //avpicture_alloc((AVPicture*)this->stream.tmpFrame, PIX_FMT_RGB24, this->stream.st->codec->width, this->stream.st->codec->height);

    int numBytes = avpicture_get_size(PIX_FMT_RGB24, this->stream.st->codec->width, this->stream.st->codec->height);

    uint8_t* buffer = (uint8_t*) av_malloc(numBytes * sizeof(uint8_t));

    avpicture_fill((AVPicture*)this->stream.tmpFrame, buffer, PIX_FMT_RGB24, this->stream.st->codec->width, this->stream.st->codec->height);

    for (int y = 0; y < this->stream.st->codec->height; y++) 
    {
        for (int x = 0; x < this->stream.st->codec->width; x++) 
        {
            int offset = 3 * (x + y * this->stream.st->codec->width);
            this->stream.tmpFrame->data[0][offset + 0] = data[x + y * this->stream.st->codec->width]; // R
            this->stream.tmpFrame->data[0][offset + 1] = data[x + y * this->stream.st->codec->width + 1]; // G
            this->stream.tmpFrame->data[0][offset + 2] = data[x + y * this->stream.st->codec->width + 2]; // B
        }
    }

    // convert the RGB frame to a YUV frame using the sws Context
    this->stream.sws_ctx = sws_getContext(this->stream.st->codec->width, this->stream.st->codec->height, PIX_FMT_RGB32, this->stream.st->codec->width, this->stream.st->codec->height, PIX_FMT_YUV420P, SWS_FAST_BILINEAR, NULL, NULL, NULL);

    // use the scale function to transcode this raw frame to the correct type
    sws_scale(this->stream.sws_ctx, this->stream.tmpFrame->data, this->stream.tmpFrame->linesize, 0, this->stream.st->codec->height, this->stream.frame->data, this->stream.frame->linesize);
}

这是将原始数据编码为 h.264，然后将其发送到 Mpeg2 ts 的块。我相信问题出在这个街区。我可以在我的写入帧块中放置一个断点并查看正在写入的帧，但是，在 VLC 中打开生成的文件会导致一个空白视频。该文件约为 2Mb。

int FMVStream::EncodeFrame(uint8_t* data)
{
    AVCodecContext* c = this->stream.st->codec;

    AVRational one;
    one.den = one.num = 1;

    // check to see if we want to keep writing frames we can probably change this to a toggle switch
    if (av_compare_ts(this->stream.next_pts, this->stream.st->codec->time_base, 10, one) >= 0)
    {
        this->stream.frame = nullptr;
    }
    else
    {
        // Convert and load the frame data into the AVFrame struct
        CopyFrameData(data);
    }

    // setup the timestamp stepping
    AVPacket pkt = { 0 };
    av_init_packet(&pkt);
    this->stream.frame->pts = (int64_t)((1.0 / this->stream.st->codec->framerate.den) * 90000.0 * this->stream.next_pts++);

    int gotPacket, out_size, ret;

    out_size = avcodec_encode_video2(c, &pkt, this->stream.frame, &gotPacket);


    if (gotPacket == 1)
    {
        ret = write_frame(this->format_ctx, &c->time_base, this->stream.st, &pkt);
    }
    else
    {
        ret = 0;
    }

    if (ret < 0)
    {
        std::cerr << "Error writing video frame" << std::endl;
    }

    av_free_packet(&pkt);

    return ((this->stream.frame != nullptr) || gotPacket) ? 0 : 1;
}

int FMVStream::write_frame(AVFormatContext *fmt_ctx, const AVRational *time_base, AVStream *st, AVPacket *pkt)
{
    /* rescale output packet timestamp values from codec to stream timebase */
    av_packet_rescale_ts(pkt, *time_base, st->time_base);
    pkt->stream_index = st->index;

    return av_interleaved_write_frame(fmt_ctx, pkt);
}

void FMVStream::setFrameWidth(const int width)
{
    this->frameWidth = width;
}

int FMVStream::getFrameWidth() const
{
    return this->frameWidth;
}

void FMVStream::setFrameHeight(const int height)
{
    this->frameHeight = height;
}

int FMVStream::getFrameHeight() const
{
    return this->frameHeight;
}

void FMVStream::setStreamAddress(const std::string& address)
{
    this->streamFilename = address;
}

std::string FMVStream::getStreamAddress() const
{
    return this->streamFilename;
}

这里是主函数。

#include "FullMotionVideoStream.h"

#include <iostream>
#include <thread>
#include <chrono>

int main(int argc, char** argv)
{
    FMVStream* fmv = new FMVStream;

    fmv->setFrameWidth(640);
    fmv->setFrameHeight(480);

    std::cout << "Streaming Address: " << fmv->getStreamAddress() << std::endl;

    // create our alternating frame of black and white to test the streaming functionality
    uint8_t white[640 * 480 * sizeof(uint8_t) * 3];
    uint8_t black[640 * 480 * sizeof(uint8_t) * 3];

    std::memset(white, 255, 640 * 480 * sizeof(uint8_t) * 3);
    std::memset(black, 0, 640 * 480 * sizeof(uint8_t)* 3);

    for (auto i = 0; i < 100; i++)
    {
        auto ret = fmv->EncodeFrame(white);

        if (ret != 0)
        {
            std::cerr << "There was a problem encoding the frame: " << i << std::endl;
        }

        std::this_thread::sleep_for(std::chrono::milliseconds(10));
    }

    for (auto i = 0; i < 100; i++)
    {
        auto ret = fmv->EncodeFrame(black);

        if (ret != 0)
        {
            std::cerr << "There was a problem encoding the frame: " << i << std::endl;
        }

        std::this_thread::sleep_for(std::chrono::milliseconds(10));
    }

    delete fmv;
}

这是通过控制台/我的打印 SDP 功能产生的输出。

[libx264 @ 000000ac95f58440] using cpu capabilities: MMX2 SSE2Fast SSSE3 SSE4.2
AVX FMA3 AVX2 LZCNT BMI2
[libx264 @ 000000ac95f58440] profile Constrained Baseline, level 3.0
Output #0, mpegts, to '(null)':
    Stream #0:0: Video: h264 (libx264), yuv420p, 640x480, q=-1--1, 400 kb/s, 30
tbn
SDP:
v=0
o=- 0 0 IN IP4 127.0.0.1
s=No Name
t=0 0
a=tool:libavformat 56.23.104
m=video 0 RTP/AVP 96
b=AS:400
a=rtpmap:96 H264/90000
a=fmtp:96 packetization-mode=1
a=control:streamid=0

Streaming Address: test.mpeg
[libx264 @ 000000ac95f58440] frame I:45    Avg QP: 0.51  size:  1315
[libx264 @ 000000ac95f58440] frame P:136   Avg QP: 0.29  size:   182
[libx264 @ 000000ac95f58440] mb I  I16..4: 99.7%  0.0%  0.3%
[libx264 @ 000000ac95f58440] mb P  I16..4:  0.1%  0.0%  0.1%  P16..4:  0.1%  0.0
%  0.0%  0.0%  0.0%    skip:99.7%
[libx264 @ 000000ac95f58440] final ratefactor: -68.99
[libx264 @ 000000ac95f58440] coded y,uvDC,uvAC intra: 0.5% 0.5% 0.5% inter: 0.0%
 0.1% 0.1%
[libx264 @ 000000ac95f58440] i16 v,h,dc,p: 96%  0%  3%  0%
[libx264 @ 000000ac95f58440] i4 v,h,dc,ddl,ddr,vr,hd,vl,hu:  1% 10% 85%  0%  3%
 0%  1%  0%  0%
[libx264 @ 000000ac95f58440] i8c dc,h,v,p: 100%  0%  0%  0%
[libx264 @ 000000ac95f58440] ref P L0: 46.8% 25.2% 28.0%
[libx264 @ 000000ac95f58440] kb/s:0.03

我知道这个程序可能有很多问题，我对 FFMPEG 和多媒体编程很陌生。我已经使用了许多通过搜索 google/stack overflow 找到的代码来达到这一点。该文件的大小很好，但长度为 0.04 告诉我必须在帧/pkts 之间破坏我的时间戳，但我不确定如何解决此问题。

我尝试使用 ffmpeg -i 使用 ffmpeg.exe 检查文件并输出到常规 TS。看来我的代码比我最初的预期更有效，但我只是想输出一堆全白帧。

ffmpeg -i test.mpeg test.ts
ffmpeg version N-70125-g6c9537b Copyright (c) 2000-2015 the FFmpeg developers
  built with gcc 4.9.2 (GCC)
  configuration: --disable-static --enable-shared --enable-gpl --enable-version3
 --disable-w32threads --enable-avisynth --enable-bzlib --enable-fontconfig --ena
ble-frei0r --enable-gnutls --enable-iconv --enable-libass --enable-libbluray --e
nable-libbs2b --enable-libcaca --enable-libfreetype --enable-libgme --enable-lib
gsm --enable-libilbc --enable-libmodplug --enable-libmp3lame --enable-libopencor
e-amrnb --enable-libopencore-amrwb --enable-libopenjpeg --enable-libopus --enabl
e-librtmp --enable-libschroedinger --enable-libsoxr --enable-libspeex --enable-l
ibtheora --enable-libtwolame --enable-libvidstab --enable-libvo-aacenc --enable-
libvo-amrwbenc --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-l
ibwebp --enable-libx264 --enable-libx265 --enable-libxavs --enable-libxvid --ena
ble-lzma --enable-decklink --enable-zlib
  libavutil      54. 19.100 / 54. 19.100
  libavcodec     56. 26.100 / 56. 26.100
  libavformat    56. 23.104 / 56. 23.104
  libavdevice    56.  4.100 / 56.  4.100
  libavfilter     5. 11.101 /  5. 11.101
  libswscale      3.  1.101 /  3.  1.101
  libswresample   1.  1.100 /  1.  1.100
  libpostproc    53.  3.100 / 53.  3.100
Input #0, mpegts, from 'test.mpeg':
  Duration: 00:00:00.04, start: 0.000000, bitrate: 24026 kb/s
  Program 1
    Metadata:
      service_name    : Service01
      service_provider: FFmpeg
    Stream #0:0[0x100]: Video: h264 (Constrained Baseline) ([27][0][0][0] / 0x00
1B), yuv420p, 640x480, 25 fps, 25 tbr, 90k tbn, 50 tbc
File 'test.ts' already exists. Overwrite ? [y/N] y
Output #0, mpegts, to 'test.ts':
  Metadata:
    encoder         : Lavf56.23.104
    Stream #0:0: Video: mpeg2video, yuv420p, 640x480, q=2-31, 200 kb/s, 25 fps,
90k tbn, 25 tbc
    Metadata:
      encoder         : Lavc56.26.100 mpeg2video
Stream mapping:
  Stream #0:0 -> #0:0 (h264 (native) -> mpeg2video (native))
Press [q] to stop, [?] for help
frame=    3 fps=0.0 q=2.0 Lsize=       9kB time=00:00:00.08 bitrate= 883.6kbits/
s dup=0 drop=178
video:7kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxing ove
rhead: 22.450111%

Answer 1

avpicture_fill 并没有按照你的想法去做。它不使用来自ptr的数据作为源填充图片，它使用ptr作为目标填充图片。所以基本上，你在编码之前清除图像。

Answer 2

在你的 av_packet_rescale_ts(pkt, *time_base, st->time_base);

您正在使用 AvCodecContext::time_base 而你设置了 AvCodecContext::framerate。

st->time_base.num = 1; st->time_base.den = 30;

st->codec->framerate.num = 1; st->codec->framerate.den = 30;

改为：

st->time_base.num = 1; st->time_base.den = 30;

st->codec->time_base = st->time_base;