Goal
Today it is common to have a digital video recording system at your disposal. Therefore, you will eventually come to the situation that you no longer process a batch of images, but video streams.因此，您最终会遇到不再处理一批图像，而是处理视频流的情况。 These may be of two kinds: real-time image feed (in the case of a webcam) or prerecorded and hard disk drive stored files.这些可能有两种：实时图像馈送（在网络摄像头的情况下）或预先记录的和硬盘驱动器存储的文件。 Luckily OpenCV treats these two in the same manner, with the same C++ class. So here's what you'll learn in this tutorial:幸运的是OpenCV以相同的方式处理这两个，使用相同的C ++类。所以这里是你将在本教程中学到的东西：
- How to open and read video streams
- 打开和阅读视频流
- Two ways for checking image similarity: PSNR and SSIM
- 两种检查图像相似性的方法：
Image similarity - PSNR and SSIM
We want to check just how imperceptible our video converting operation went, therefore we need a system to check frame by frame the similarity or differences. 需要一个系统检查帧与帧之间的相似性。The most common algorithm used for this is the PSNR (aka Peak signal-to-noise ratio). 最常用的算法是PSNR。The simplest definition of this starts out from the mean squad error.平均队误差。 Let there be two images: I1 and I2; with a two dimensional size i and j, composed of c number of channels.两个图像，二维大小（i，j），通道数为c。
Then the PSNR is expressed as:
Here the MAXI is the maximum valid value for a pixel.像素中的最大值。 In case of the simple single byte image per pixel per channel this is 255.单字节单通道图像，这个值是255。 When two images are the same the MSE will give zero, resulting in an invalid divide by zero operation in the PSNR formula. In this case the PSNR is undefined and as we'll need to handle this case separately. 但两个图像相同时，其MSE为零，则PSNR没有办法计算，对于这种情况，这里单独处理。The transition to a logarithmic scale is made because the pixel values have a very wide dynamic range. 由于像素具有较大的范围，因而转化为对数刻度。
The source code
As a test case where to show off these using OpenCV I've created a small program that reads in two video files and performs a similarity check between them. 创建了一个小程序，它读入两个视频文件并执行它们之间的相似性检查。This is something you could use to check just how well a new video compressing algorithms works. 这可以用来检查新视频压缩算法的工作效果。Let there be a reference (original) video like this small Megamind clip and a compressed version of it. You may also find the source code and these video file in the samples/data folder of the OpenCV source library.这里有一个视频this small Megamind clip和它的压缩视频。您还可以samples/data在OpenCV源库的文件夹中找到源代码和这些视频文件。
#include <iostream> // for standard I/O
#include <string> // for strings
#include <iomanip> // for controlling float print precision
#include <sstream> // string to number conversion
#include <opencv2/core.hpp> // Basic OpenCV structures (cv::Mat, Scalar)
#include <opencv2/imgproc.hpp> // Gaussian Blur
#include <opencv2/videoio.hpp>
#include <opencv2/highgui.hpp> // OpenCV window I/O
using namespace std;
using namespace cv;
double getPSNR ( const Mat& I1, const Mat& I2);
Scalar getMSSIM( const Mat& I1, const Mat& I2);
static void help()
{
cout
<< "------------------------------------------------------------------------------" << endl
<< "This program shows how to read a video file with OpenCV. In addition, it "
<< "tests the similarity of two input videos first with PSNR, and for the frames "
<< "below a PSNR trigger value, also with MSSIM." << endl
<< "Usage:" << endl
<< "./video-input-psnr-ssim <referenceVideo> <useCaseTestVideo> <PSNR_Trigger_Value> <Wait_Between_Frames> " << endl
<< "--------------------------------------------------------------------------" << endl
<< endl;
}
int main()
{
help();
// if (argc != 5)
// {
// cout << "Not enough parameters" << endl;
// return -1;
// }
stringstream conv;
const string sourceReference = "Megamind.avi", sourceCompareWith = "Megamind_bugy.avi";
int psnrTriggerValue, delay;
//conv << argv[3] << endl << argv[4]; // put in the strings
//conv >> psnrTriggerValue >> delay; // take out the numbers
psnrTriggerValue = 40;
delay = 100;
int frameNum = -1; // Frame counter
VideoCapture captRefrnc(sourceReference), captUndTst(sourceCompareWith);
//打开视频文件
if (!captRefrnc.isOpened())
{
cout << "Could not open reference " << sourceReference << endl;
return -1;
}
if (!captUndTst.isOpened())
{
cout << "Could not open case test " << sourceCompareWith << endl;
return -1;
}
Size refS = Size((int) captRefrnc.get(CAP_PROP_FRAME_WIDTH),
(int) captRefrnc.get(CAP_PROP_FRAME_HEIGHT)),
uTSi = Size((int) captUndTst.get(CAP_PROP_FRAME_WIDTH),
(int) captUndTst.get(CAP_PROP_FRAME_HEIGHT));
if (refS != uTSi)
{
cout << "Inputs have different size!!! Closing." << endl;
return -1;
}
const char* WIN_UT = "Under Test";
const char* WIN_RF = "Reference";
// Windows
namedWindow(WIN_RF, WINDOW_AUTOSIZE);
namedWindow(WIN_UT, WINDOW_AUTOSIZE);
moveWindow(WIN_RF, 400 , 0); //750, 2 (bernat =0)
moveWindow(WIN_UT, refS.width, 0); //1500, 2
cout << "Reference frame resolution: Width=" << refS.width << " Height=" << refS.height
<< " of nr#: " << captRefrnc.get(CAP_PROP_FRAME_COUNT) << endl;
cout << "PSNR trigger value " << setiosflags(ios::fixed) << setprecision(3)
<< psnrTriggerValue << endl;
Mat frameReference, frameUnderTest;
double psnrV;
Scalar mssimV;
for(;;) //Show the image captured in the window and repeat
{
captRefrnc >> frameReference;
captUndTst >> frameUnderTest;
if (frameReference.empty() || frameUnderTest.empty())
{
cout << " < < < Game over! > > > ";
break;
}
++frameNum;
cout << "Frame: " << frameNum << "# ";
psnrV = getPSNR(frameReference,frameUnderTest);
cout << setiosflags(ios::fixed) << setprecision(3) << psnrV << "dB";
if (psnrV < psnrTriggerValue && psnrV)
{
mssimV = getMSSIM(frameReference, frameUnderTest);
cout << " MSSIM: "
<< " R " << setiosflags(ios::fixed) << setprecision(2) << mssimV.val[2] * 100 << "%"
<< " G " << setiosflags(ios::fixed) << setprecision(2) << mssimV.val[1] * 100 << "%"
<< " B " << setiosflags(ios::fixed) << setprecision(2) << mssimV.val[0] * 100 << "%";
}
cout << endl;
imshow(WIN_RF, frameReference);
imshow(WIN_UT, frameUnderTest);
char c = (char)waitKey(delay);
if (c == 27) break;
}
return 0;
}
double getPSNR(const Mat& I1, const Mat& I2)
{
Mat s1;
absdiff(I1, I2, s1); // |I1 - I2|
s1.convertTo(s1, CV_32F); // cannot make a square on 8 bits
s1 = s1.mul(s1); // |I1 - I2|^2
Scalar s = sum(s1); // sum elements per channel
double sse = s.val[0] + s.val[1] + s.val[2]; // sum channels
if( sse <= 1e-10) // for small values return zero
return 0;
else
{
double mse = sse / (double)(I1.channels() * I1.total());
double psnr = 10.0 * log10((255 * 255) / mse);
return psnr;
}
}
Scalar getMSSIM( const Mat& i1, const Mat& i2)
{
const double C1 = 6.5025, C2 = 58.5225;
/***************************** INITS **********************************/
int d = CV_32F;
Mat I1, I2;
i1.convertTo(I1, d); // cannot calculate on one byte large values
i2.convertTo(I2, d);
Mat I2_2 = I2.mul(I2); // I2^2
Mat I1_2 = I1.mul(I1); // I1^2
Mat I1_I2 = I1.mul(I2); // I1 * I2
/*************************** END INITS **********************************/
Mat mu1, mu2; // PRELIMINARY COMPUTING
GaussianBlur(I1, mu1, Size(11, 11), 1.5);
GaussianBlur(I2, mu2, Size(11, 11), 1.5);
Mat mu1_2 = mu1.mul(mu1);
Mat mu2_2 = mu2.mul(mu2);
Mat mu1_mu2 = mu1.mul(mu2);
Mat sigma1_2, sigma2_2, sigma12;
GaussianBlur(I1_2, sigma1_2, Size(11, 11), 1.5);
sigma1_2 -= mu1_2;
GaussianBlur(I2_2, sigma2_2, Size(11, 11), 1.5);
sigma2_2 -= mu2_2;
GaussianBlur(I1_I2, sigma12, Size(11, 11), 1.5);
sigma12 -= mu1_mu2;
Mat t1, t2, t3;
t1 = 2 * mu1_mu2 + C1;
t2 = 2 * sigma12 + C2;
t3 = t1.mul(t2); // t3 = ((2*mu1_mu2 + C1).*(2*sigma12 + C2))
t1 = mu1_2 + mu2_2 + C1;
t2 = sigma1_2 + sigma2_2 + C2;
t1 = t1.mul(t2); // t1 =((mu1_2 + mu2_2 + C1).*(sigma1_2 + sigma2_2 + C2))
Mat ssim_map;
divide(t3, t1, ssim_map); // ssim_map = t3./t1;
Scalar mssim = mean(ssim_map); // mssim = average of ssim map
return mssim;
}
Results
开始时提供的源代码将针对每个帧执行PSNR测量，而SSIM仅针对PSNR降至输入值以下的帧进行。
编程的思路主要是围绕这上面的内容进行的。