OpenCV，为什么我的鱼眼相机校准这么差？

Question

我正在尝试设置鱼眼相机，用于相对于特定参考框架进行对象定位。

我尝试了 OpenCV 鱼眼模块和来自 calibrateCamera() 的有理模型进行校准。我获得了this result。

我收集了 2 个不同的datasets，一个是在大部分靠近相机的地方拍摄的校准图像 (ds1)，另一个是从远处拍摄的图像 (ds2)。 ds12是两者合并得到的数据集。

nok4表示k4固定为0的鱼眼模型。
rm是来自cv2.calibrateCamera()的有理模型

相机具有 178° 水平 FOV 和 101° 垂直 FOV，畸变主要在图像的中心进行了校正，而在图像的最外侧部分的结果令人失望。

我做错了吗？我可以做些什么来改善结果？

编辑

这是我用于两个校准过程的代码：

import cv2 as cv
import os
import numpy as np

cwd = os.path.dirname(os.path.realpath(__file__))
os.chdir(cwd)

number = None
folder_name = "merged_images"

if number is not None:
    folder_name += "_" + str(number)

points_path = os.path.join(folder_name, "dataset", "good_detections")

npz = np.load(os.path.join(points_path, "points.npz"))

square_size = 0.02435

imgpoints = npz["imgpoints"]
objpoints = npz["objpoints"] * square_size
file_names = npz["file_names"]

shuffle = True
if shuffle:
    if "indices.npz" in os.listdir(points_path):
        p = np.load(os.path.join(points_path, "indices.npz"))["indices"]
    else:
        print("Random indices assigned")
        p = np.random.permutation(len(imgpoints))
    
    imgpoints = imgpoints[p]
    objpoints = objpoints[p]
    file_names = file_names[p]

img = cv.imread(os.path.join(points_path, file_names[2].replace("detected_", "")))
    
flag_list = [
    cv.CALIB_RATIONAL_MODEL,
#    cv.CALIB_ZERO_TANGENT_DIST,
    ]

calibration_flags = 0
for flag in flag_list:
    calibration_flags += flag

gray = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
shape = gray.shape[::-1]

ret, mtx, dist, rvecs, tvecs = cv.calibrateCamera(objpoints,
                                                  imgpoints,
                                                  shape,
                                                  None,
                                                  None,
                                                  flags = calibration_flags
                                                  )

h, w = img.shape[:2]
newcameramtx, roi = cv.getOptimalNewCameraMatrix(mtx, dist, (w,h), 1, (w,h))

dst = cv.undistort(img, mtx, dist, None, newcameramtx)

# fisheye model
flag_list = [
    cv.fisheye.CALIB_RECOMPUTE_EXTRINSIC,
    cv.fisheye.CALIB_CHECK_COND,
    cv.fisheye.CALIB_FIX_SKEW,
#    cv.fisheye.CALIB_FIX_K4,
#    cv.fisheye.CALIB_FIX_K3,
#    cv.fisheye.CALIB_FIX_K2,
#    cv.fisheye.CALIB_FIX_K1,
    ]

calibration_flags = 0
for flag in flag_list:
    calibration_flags += flag

N_OK = len(objpoints)
K = np.zeros((3, 3))
D = np.zeros((4, 1))
rvecs = [np.zeros((1, 1, 3), dtype=np.float64) for i in range(N_OK)]
tvecs = [np.zeros((1, 1, 3), dtype=np.float64) for i in range(N_OK)]

n_objpoints = [np.expand_dims(objp, 0) for objp in objpoints]

all_true_points = list(n_objpoints)
all_image_points = list(imgpoints)
all_frames = list(file_names)
rejected = []

counter = 0
while True:
    try:
        rms, mtx, dist, rvecs, tvecs = \
            cv.fisheye.calibrate(
                all_true_points,
                all_image_points,
                gray.shape[::-1],
                K,
                D,
                rvecs,
                tvecs,
                calibration_flags,
                (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 30, 1e-3)
            )
        print('Found a calibration based on {} well-conditioned images.'.format(len(all_true_points)))  
        break
    
    except cv.error as err:
        try:
            idx = int(str(err).split('array ')[1][0])  # Parse index of invalid image from error message
            all_true_points.pop(idx)
            all_image_points.pop(idx)
            rejected.append(all_frames.pop(idx))
            print(f"{counter}. Removed ill-conditioned image {idx} from the data.  Trying again...".format(idx))
            counter += 1
        except IndexError:
            raise err

h,w = img.shape[:2]
DIM = (w, h)
dim1 = img.shape[:2][::-1]  # dim1 is the dimension of input image to un-distort
dim2 = None
dim3 = None

balance = 1

assert dim1[0]/dim1[1] == DIM[0]/DIM[1], "Image to undistort needs to have same aspect ratio as the ones used in calibration"
if not dim2:
    dim2 = dim1
if not dim3:
    dim3 = dim1
    
scaled_K = K * dim1[0] / DIM[0]  # The values of K is to scale with image dimension.
scaled_K[2][2] = 1.0  # Except that K[2][2] is always 1.0
# This is how scaled_K, dim2 and balance are used to determine the final K used to un-distort image. OpenCV document failed to make this clear!
new_K = cv.fisheye.estimateNewCameraMatrixForUndistortRectify(scaled_K, D, dim2, np.eye(3), balance=balance)
map1, map2 = cv.fisheye.initUndistortRectifyMap(scaled_K, D, np.eye(3), new_K, dim3, cv.CV_16SC2)

undistorted_img = cv.remap(img, map1, map2, interpolation=cv.INTER_LINEAR, borderMode=cv.BORDER_CONSTANT)

new_img = cv.hconcat([undistorted_img, img])

使用以下代码执行角点提取：

objpoints = [] # 3d point in real world space
imgpoints = [] # 2d points in image plane.
file_names = [] # files analyzed

for file in os.listdir(detections_path):
    if file.startswith("hd_frame"):
        
        frame = cv.imread(os.path.join(detections_path, file))
        
        gray = cv.cvtColor(frame, cv.COLOR_BGR2GRAY)
        
        found, corners = cv.findChessboardCorners(gray, (9,6), None)
        
        if found:
            file_name = "detected_" + file

            objpoints.append(objp)
            imgpoints.append(corners)

            # save file_name
            file_names.append(file_name)

            corners2 = cv.cornerSubPix(gray, corners, (11,11), (-1,-1), criteria)
            # Draw and display the corners
            cv.drawChessboardCorners(frame, (9,6), corners2, found)
            
            # save image and points
            cv.imwrite(os.path.join(detections_path, file_name), frame)

Answer 1

您可以使用单个图像进行校准，如下所述：https://discorpy.readthedocs.io/en/latest/usage/demo_06.html。这个包中使用的校正模型可能会给你比opencv使用的模型更好的结果：https://discorpy.readthedocs.io/en/latest/tutorials/methods.html