I tried creating a program to stitch together multiple drone captured images. However, the program can only stitch two images at a time. So, I decided to stitch them in pairs, using the previously stitched result as the query image for the next stitching.
Unfortunately, the program only manages to stitch the right and bottom parts of the images. The top and left parts remain unchanged, even when the images to be stitched have additional areas that are not present in the base image.
Initially, I used image data from this source: https://www.kaggle.com/code/phsophea101/stitch-image-from-drone-capture-farm-area/input . However, the results were not as expected. The stitched image only included the right part of the train image without the top part. see the train image, the building on the top of the train image have more "image" than the query image (blue water tank?)
Later, I tried using cropped images from the internet, assuming that the previous image data had different sizes and alignment issues. I divided the images into 12 sections and stitched them individually. The stitching worked well for the first row and even when combining the first row with the first image in the second row. However, when I tried to continue stitching with the second image in the second row, it didn't produce any results despite having feature matches. I suspect this is due to the black area in the query image used for stitching. Notice that the image have black background, because of the incomplete stitching Try to stitch more
Here is my code:
feature_extraction_algo = 'surf'
feature_to_match = 'knn'
train_photo = cv2.imread('./' + 'train.jpg')
# OpenCV defines the color channel in the order BGR
# Hence converting to RGB for Matplotlib
train_photo = cv2.cvtColor(train_photo,cv2.COLOR_BGR2RGB)
# converting to grayscale
train_photo_gray = cv2.cvtColor(train_photo, cv2.COLOR_RGB2GRAY)
# Do the same for the query image
query_photo = cv2.imread('./' + 'query.jpg')
query_photo = cv2.cvtColor(query_photo,cv2.COLOR_BGR2RGB)
query_photo_gray = cv2.cvtColor(query_photo, cv2.COLOR_RGB2GRAY)
def select_descriptor_methods(image, method=None):
assert method is not None, "Please define a feature descriptor method. accepted Values are: 'sift', 'surf', 'brisk', 'orb'"
if method == 'sift':
descriptor = cv2.SIFT_create()
elif method == 'surf':
descriptor = cv2.xfeatures2d.SURF_create()
elif method == 'brisk':
descriptor = cv2.BRISK_create()
elif method == 'orb':
descriptor = cv2.ORB_create()
(keypoints, features) = descriptor.detectAndCompute(image, None)
return (keypoints, features)
keypoints_train_img, features_train_img = select_descriptor_methods(train_photo_gray, method=feature_extraction_algo)
keypoints_query_img, features_query_img = select_descriptor_methods(query_photo_gray, method=feature_extraction_algo)
def create_matching_object(method,crossCheck):
# For BF matcher, first we have to create the BFMatcher object using cv2.BFMatcher().
# It takes two optional params.
# normType - It specifies the distance measurement
# crossCheck - which is false by default. If it is true, Matcher returns only those matches
# with value (i,j) such that i-th descriptor in set A has j-th descriptor in set B as the best match
# and vice-versa.
if method == 'sift' or method == 'surf':
bf = cv2.BFMatcher(cv2.NORM_L2, crossCheck=crossCheck)
elif method == 'orb' or method == 'brisk':
bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=crossCheck)
return bf
def key_points_matching(features_train_img, features_query_img, method):
bf = create_matching_object(method, crossCheck=True)
# Match descriptors.
best_matches = bf.match(features_train_img,features_query_img)
# Sort the features in order of distance.
# The points with small distance (more similarity) are ordered first in the vector
rawMatches = sorted(best_matches, key = lambda x:x.distance)
print("Raw matches with Brute force):", len(rawMatches))
return rawMatches
def key_points_matching_KNN(features_train_img, features_query_img, ratio, method):
bf = create_matching_object(method, crossCheck=False)
# compute the raw matches and initialize the list of actual matches
rawMatches = bf.knnMatch(features_train_img, features_query_img, k=2)
print("Raw matches (knn):", len(rawMatches))
matches = []
# loop over the raw matches
for m,n in rawMatches:
# ensure the distance is within a certain ratio of each
# other (i.e. Lowe's ratio test)
if m.distance < n.distance * ratio:
matches.append(m)
return matches
if feature_to_match == 'bf':
matches = key_points_matching(features_train_img, features_query_img, method=feature_extraction_algo)
mapped_features_image = cv2.drawMatches(train_photo,keypoints_train_img,query_photo,keypoints_query_img,matches[:100],
None,flags=cv2.DrawMatchesFlags_NOT_DRAW_SINGLE_POINTS)
# Now for cross checking draw the feature-mapping lines also with KNN
elif feature_to_match == 'knn':
matches = key_points_matching_KNN(features_train_img, features_query_img, ratio=0.75, method=feature_extraction_algo)
mapped_features_image = cv2.drawMatches(train_photo, keypoints_train_img, query_photo, keypoints_query_img, np.random.choice(matches,100),
None,flags=cv2.DrawMatchesFlags_NOT_DRAW_SINGLE_POINTS)
def homography_stitching(keypoints_train_img, keypoints_query_img, matches, reprojThresh):
""" converting the keypoints to numpy arrays before passing them for calculating Homography Matrix.
Because we are supposed to pass 2 arrays of coordinates to cv2.findHomography, as in I have these points in image-1, and I have points in image-2, so now what is the homography matrix to transform the points from image 1 to image 2
"""
keypoints_train_img = np.float32([keypoint.pt for keypoint in keypoints_train_img])
keypoints_query_img = np.float32([keypoint.pt for keypoint in keypoints_query_img])
''' For findHomography() - I need to have an assumption of a minimum of correspondence points that are present between the 2 images. Here, I am assuming that Minimum Match Count to be 4 '''
if len(matches) > 4:
# construct the two sets of points
points_train = np.float32([keypoints_train_img[m.queryIdx] for m in matches])
points_query = np.float32([keypoints_query_img[m.trainIdx] for m in matches])
# Calculate the homography between the sets of points
(H, status) = cv2.findHomography(points_train, points_query, cv2.RANSAC, reprojThresh)
return (matches, H, status)
else:
return None
M = homography_stitching(keypoints_train_img, keypoints_query_img, matches, reprojThresh=4)
if M is None:
print("Error!")
(matches, Homography_Matrix, status) = M
width = query_photo.shape[1] + train_photo.shape[1]
height = query_photo.shape[0] + train_photo.shape[0]
result = cv2.warpPerspective(train_photo, Homography_Matrix, (width, height))
result[0:query_photo.shape[0], 0:query_photo.shape[1]] = query_photo
img = result
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# threshold
thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU)[1]
hh, ww = thresh.shape
print(thresh.shape)
# make bottom 2 rows black where they are white the full width of the image
thresh[hh-3:hh, 0:ww] = 0
# # get bounds of white pixels
white = np.where(thresh==255)
xmin, ymin, xmax, ymax = np.min(white[1]), np.min(white[0]), np.max(white[1]), np.max(white[0])
print(xmin,xmax,ymin,ymax)
# crop the image at the bounds adding back the two blackened rows at the bottom
crop = img[ymin:ymax, xmin:xmax]
I tried using the solution discussed in the following link: OpenCV, Python: How to stitch two images of different sizes and transparent backgrounds, which suggests using masking to ignore the black/translucent areas.
Instead, just take a normal approach with SIFT or SURF or ORB or BRISK. Note that all of those allow to add a mask for their keypoint detection step, so that you can create a mask from the alpha channel to ignore keypoints in. See the OpenCV docs for SIFT and SURF and for ORB and BRISK for more.
I attempted to apply masking, but the results remained the same.
def select_descriptor_methods(image, method=None):
assert method is not None, "Please define a feature descriptor method. accepted Values are: 'sift', 'surf', 'brisk', 'orb'"
if method == 'sift':
descriptor = cv2.SIFT_create()
elif method == 'surf':
descriptor = cv2.xfeatures2d.SURF_create()
elif method == 'brisk':
descriptor = cv2.BRISK_create()
elif method == 'orb':
descriptor = cv2.ORB_create()
# I tried to add this for masking
_, mask = cv2.threshold(image, thresh=0, maxval=255, type=cv2.THRESH_BINARY)
# and add the mask to this code (previously its "detectAndCompute(image, None)")
(keypoints, features) = descriptor.detectAndCompute(image, mask)
return (keypoints, features)
Could it be that my masking approach is incorrect or not properly implemented, or are there other factors that may be affecting this issue? I would appreciate your assistance.
