How to draw eplines on stereo images using `cv.computeCorrespondEpilines` with Fundamental Matrix

Question

I am following this tutorial and trying to draw eplines on a stereo image pair using a fundamental matrix (Fmat) I obtained with cv2.stereoCalibrate. I am trying to use my imported Fmat.npy instead of cv.findFundamentalMat and cv.FM_RANSAC. However, both attempts at the code produce similar value errors.

Here's the code:

# 1. Detect keypoints and their descriptors
# Based on: https://docs.opencv.org/master/dc/dc3/tutorial_py_matcher.html

# Initiate SIFT detector
sift = cv.SIFT_create()
# find the keypoints and descriptors with SIFT
kp1, des1 = sift.detectAndCompute(img1, None)
kp2, des2 = sift.detectAndCompute(img2, None)


# Visualize keypoints
imgSift = cv.drawKeypoints(
    img1, kp1, None, flags=cv.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)
cv.imshow("SIFT Keypoints", imgSift)


# Match keypoints in both images
# Based on: https://docs.opencv.org/master/dc/dc3/tutorial_py_matcher.html
FLANN_INDEX_KDTREE = 1
index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=5)
search_params = dict(checks=50)   # or pass empty dictionary
flann = cv.FlannBasedMatcher(index_params, search_params)
matches = flann.knnMatch(des1, des2, k=2)

# Keep good matches: calculate distinctive image features
# Lowe, D.G. Distinctive Image Features from Scale-Invariant Keypoints. International Journal of Computer Vision 60, 91–110 (2004). https://doi.org/10.1023/B:VISI.0000029664.99615.94
# https://www.cs.ubc.ca/~lowe/papers/ijcv04.pdf
matchesMask = [[0, 0] for i in range(len(matches))]
good = []
pts1 = []
pts2 = []

for i, (m, n) in enumerate(matches):
    if m.distance < 0.7*n.distance:
        # Keep this keypoint pair
        matchesMask[i] = [1, 0]
        good.append(m)
        pts2.append(kp2[m.trainIdx].pt)
        pts1.append(kp1[m.queryIdx].pt)


# Draw the keypoint matches between both pictures
# Still based on: https://docs.opencv.org/master/dc/dc3/tutorial_py_matcher.html
draw_params = dict(matchColor=(0, 255, 0),
                   singlePointColor=(255, 0, 0),
                   matchesMask=matchesMask[300:500],
                   flags=cv.DrawMatchesFlags_DEFAULT)

keypoint_matches = cv.drawMatchesKnn(
    img1, kp1, img2, kp2, matches[300:500], None, **draw_params)
cv.imshow("Keypoint matches", keypoint_matches)



# ------------------------------------------------------------
# STEREO RECTIFICATION

Fmat = np.load('Fmat.npy') # Load fundamental matrix

# Calculate the fundamental matrix for the cameras
# https://docs.opencv.org/master/da/de9/tutorial_py_epipolar_geometry.html
pts1 = np.int32(pts1)
pts2 = np.int32(pts2)
fundamental_matrix, inliers = cv.findFundamentalMat(pts1, pts2, cv.FM_RANSAC)  #Fmat

# We select only inlier points
pts1 = pts1[inliers.ravel() == 1]
pts2 = pts2[inliers.ravel() == 1]


# Visualize epilines
# Adapted from: https://docs.opencv.org/master/da/de9/tutorial_py_epipolar_geometry.html
def drawlines(img1src, img2src, lines, pts1src, pts2src):
    ''' img1 - image on which we draw the epilines for the points in img2
        lines - corresponding epilines '''
    r, c = img1src.shape
    img1color = cv.cvtColor(img1src, cv.COLOR_GRAY2BGR)
    img2color = cv.cvtColor(img2src, cv.COLOR_GRAY2BGR)
    # Edit: use the same random seed so that two images are comparable!
    np.random.seed(0)
    for r, pt1, pt2 in zip(lines, pts1src, pts2src):
        color = tuple(np.random.randint(0, 255, 3).tolist())
        x0, y0 = map(int, [0, -r[2]/r[1]])
        x1, y1 = map(int, [c, -(r[2]+r[0]*c)/r[1]])
        img1color = cv.line(img1color, (x0, y0), (x1, y1), color, 1)
        img1color = cv.circle(img1color, tuple(pt1), 5, color, -1)
        img2color = cv.circle(img2color, tuple(pt2), 5, color, -1)
    return img1color, img2color


# Find epilines corresponding to points in right image (second image) and
# drawing its lines on left image
lines1 = cv.computeCorrespondEpilines(
    pts2.reshape(-1, 1, 2), 2, fundamental_matrix)
lines1 = lines1.reshape(-1, 3)
img5, img6 = drawlines(img1, img2, lines1, pts1, pts2)

# Find epilines corresponding to points in left image (first image) and
# drawing its lines on right image
lines2 = cv.computeCorrespondEpilines(
    pts1.reshape(-1, 1, 2), 1, fundamental_matrix)
lines2 = lines2.reshape(-1, 3)
img3, img4 = drawlines(img2, img1, lines2, pts2, pts1)

plt.subplot(121), plt.imshow(img5)
plt.subplot(122), plt.imshow(img3)
plt.suptitle("Epilines in both images")
plt.show()

When I run the above code with the fundamental_matrix, inliers = cv.findFundamentalMat(pts1, pts2, cv.FM_RANSAC) line, as is provided in the tutorial, the follwoing error is returned:

Traceback (most recent call last):
  File "C:\Users\xxx\stereo-camera\featureMatching.py", line 103, in <module>
    img5, img6 = drawlines(img1, img2, lines1, pts1, pts2)
  File "C:\Users\xxx\stereo-camera\featureMatching.py", line 83, in drawlines
    r, c = img1src.shape
ValueError: too many values to unpack (expected 2)

When I replace the r, c = img1src.shape line with r, c, *_ = img1src.shape I get the following new error:

Traceback (most recent call last):
  File "C:\Users\joini\OneDrive\Documents\code\DCE\stereoVision\ObstacleAvoidSystem\stereo-camera\featureMatching.py", line 105, in <module>
    img5, img6 = drawlines(img1, img2, lines1, pts1, pts2)
  File "C:\Users\joini\OneDrive\Documents\code\DCE\stereoVision\ObstacleAvoidSystem\stereo-camera\featureMatching.py", line 86, in drawlines
    img1color = cv.cvtColor(img1src, cv.COLOR_GRAY2BGR)
cv2.error: OpenCV(4.6.0) d:\a\opencv-python\opencv-python\opencv\modules\imgproc\src\color.simd_helpers.hpp:92: error: (-2:Unspecified error) in function '__cdecl cv::impl::`a
nonymous-namespace'::CvtHelper<struct cv::impl::`anonymous namespace'::Set<1,-1,-1>,struct cv::impl::A0xf2302844::Set<3,4,-1>,struct cv::impl::A0xf2302844::Set<0,2,5>,2>::CvtH
elper(const class cv::_InputArray &,const class cv::_OutputArray &,int)'
> Invalid number of channels in input image:
>     'VScn::contains(scn)'
> where
>     'scn' is 3

When I run the code with the line changed to fundamental_matrix, inliers = Fmat in order to take my fundamental matrix, I get the following error:

Traceback (most recent call last):
  File "C:\Users\xxx\stereo-camera\featureMatching.py", line 71, in <module>
    fundamental_matrix, inliers = Fmat
ValueError: too many values to unpack (expected 2)

The shape of the images is (480, 640, 3).

What is going wrong in each case, and what do I need to do to produce the desired eplines result using the Fmat.npy file?

Answer 1

The line throwing the error is

r, c = img1src.shape

Most likely, the tutorial code was only tested with grayscale images, which just have a (row x col) shape. And I guess you are using an RGB image, having a (row x col x rgb) shape. Thus the shape has a third value in the tuple, which can't be unpacked into just (r, c).

Try replacing the line with

r, c, *_ = img1src.shape

This ignores the third (color channel) dimension if present.