IndexError: index 3 is out of bounds for axis 0 with size 3 while giving original image (color image)?

Question

I am trying to apply the perspective information for unshearing the image using open cv . I had converted the image to black and white and then passes it to the model which is working fine . It is displaying the white mask which is actually the output . How can I display its original object of image instead of mask , it is throwing the index error.

**Code: **

import cv2
import numpy as np


def find_corners(im):

    # Find contours in img.
    cnts = cv2.findContours(im, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)[-2]  # [-2] indexing takes return value before last (due to OpenCV compatibility issues).

    # Find the contour with the maximum area (required if there is more than one contour).
    c = max(cnts, key=cv2.contourArea)

    epsilon = 0.1*cv2.arcLength(c, True)
    box = cv2.approxPolyDP(c, epsilon, True)

    tmp_im = cv2.cvtColor(im, cv2.COLOR_GRAY2BGR)
    cv2.drawContours(tmp_im, [box], 0, (0, 255, 0), 2)
    cv2.imshow("tmp_im", tmp_im)

    box = np.squeeze(box).astype(np.float32)  


    # Sorting the points order is top-left, top-right, bottom-right, bottom-left.
    # Find the center of the contour
    M = cv2.moments(c)
    cx = M['m10']/M['m00']
    cy = M['m01']/M['m00']
    center_xy = np.array([cx, cy])

    cbox = box - center_xy  # Subtract the center from each corner
    
    ang = np.arctan2(cbox[:,1], cbox[:,0]) * 180 / np.pi  # Compute the angles from the center to each corner

    # Sort the corners of box counterclockwise (sort box elements according the order of ang).
    box = box[ang.argsort()]
    
    # Reorder points: top-left, top-right, bottom-left, bottom-right
    print('bbox',box)
    coor = np.float32([box[0], box[1], box[3], box[2]])
  

    return coor


input_image2 = cv2.imread("/home/hamza/Desktop/cards/card2.jpeg", cv2.IMREAD_GRAYSCALE)  # Read image as Grayscale
input_image2 = cv2.threshold(input_image2, 0, 255, cv2.THRESH_OTSU)[1]  # Convert to binary image (just in case...)

# Find the corners of the card, and sort them
orig_im_coor = find_corners(input_image2)

height, width = 450, 350
new_image_coor =  np.float32([[0, 0], [width, 0], [0, height], [width, height]])

P = cv2.getPerspectiveTransform(orig_im_coor, new_image_coor)

perspective = cv2.warpPerspective(input_image2, P, (width, height))
cv2.imshow("Perspective transformation", perspective)
cv2.waitKey(0)
cv2.destroyAllWindows()

Original Image

Black and white image

Expected Output: The output I am getting is white mask from code which is fine but I want to take the image of that particular mask . How can I access it ?

If it is not possible using this approach , How can I do it from some other approach ?

Edit 1: Code returning the output after unshearing the image as shown below in the image , I have to put that particular mask on that output image which I give it as a input of mask from that specified image

Code returning output now . Below Output image is white please press on the image it will display you otherwise background is also white and it will not proper display you :

Answer 1

All you need to do is applying the transformation cv2.warpPerspective on the original image, instead of the black and white image.

• Read the "original image":

    original_image = cv2.imread("original_image.jpg")

• Compute orig_im_coor and P as before.
• Apply the transformation on original_image:

    perspective = cv2.warpPerspective(original_image, P, (width, height))

---

There is one more recommended stage:
Instead of manually setting height and width: height, width = 450, 350, it's recommended to compute the height and width.

Assuming the orig_im_coor coordinates are accurate enough, we can compute height, width according to the coordinates:

• Width is approximately the distance from top-left to top-right.
• Height is approximately the distance from top-left to bottom-left.

Computing the Euclidian distance is simple: sqrt(x² + y²).
As a shortcut we can use np.linalg.norm(a-b) as described in the following post:

width = round(np.linalg.norm(orig_im_coor[0] - orig_im_coor[1]))  # Assume width is the distance from top-left to top-right
height = round(np.linalg.norm(orig_im_coor[0] - orig_im_coor[2]))  # Assume height is the distance from top-left to bottom-left

---

Here is the complete code:

import cv2
import numpy as np


def find_corners(im):
    """ 
    Find "card" corners in a binary image.
    Return a list of points in the following format: [[640, 184], [1002, 409], [211, 625], [589, 940]] 
    The points order is top-left, top-right, bottom-left, bottom-right.
    """

    # Better approach: https://stackoverflow.com/questions/44127342/detect-card-minarea-quadrilateral-from-contour-opencv

    # Find contours in img.
    cnts = cv2.findContours(im, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)[-2]  # [-2] indexing takes return value before last (due to OpenCV compatibility issues).

    # Find the contour with the maximum area (required if there is more than one contour).
    c = max(cnts, key=cv2.contourArea)

    # https://stackoverflow.com/questions/41138000/fit-quadrilateral-tetragon-to-a-blob
    epsilon = 0.1*cv2.arcLength(c, True)
    box = cv2.approxPolyDP(c, epsilon, True)

    # Draw box for testing
    #tmp_im = cv2.cvtColor(im, cv2.COLOR_GRAY2BGR)
    #cv2.drawContours(tmp_im, [box], 0, (0, 255, 0), 2)
    #cv2.imshow("tmp_im", tmp_im)

    box = np.squeeze(box).astype(np.float32)  # Remove redundant dimensions


    # Sorting the points order is top-left, top-right, bottom-right, bottom-left.
    # Note: 
    # The method I am using is a bit of an "overkill".
    # I am not sure if the implementation is correct.
    # You may sort the corners using simple logic - find top left, bottom right, and match the other two points.
    ############################################################################
    # Find the center of the contour
    # https://docs.opencv.org/3.4/dd/d49/tutorial_py_contour_features.html
    M = cv2.moments(c)
    cx = M['m10']/M['m00']
    cy = M['m01']/M['m00']
    center_xy = np.array([cx, cy])

    cbox = box - center_xy  # Subtract the center from each corner

    # For a square the angles of the corners are:
    # -135   -45
    #
    #
    # 135     45
    ang = np.arctan2(cbox[:,1], cbox[:,0]) * 180 / np.pi  # Compute the angles from the center to each corner

    # Sort the corners of box counterclockwise (sort box elements according the order of ang).
    box = box[ang.argsort()]
    ############################################################################

    # Reorder points: top-left, top-right, bottom-left, bottom-right
    coor = np.float32([box[0], box[1], box[3], box[2]])

    return coor

original_image = cv2.imread("original_image.jpg")  # Read the original image

bw_mask = cv2.imread("black_and_white_mask.jpg", cv2.IMREAD_GRAYSCALE)  # Read image as Grayscale
bw_mask = cv2.threshold(bw_mask, 0, 255, cv2.THRESH_OTSU)[1]  # Convert to binary image (just in case...)

# Find the corners of the mask, and sort them
orig_im_coor = find_corners(bw_mask)

# Compute width and height of the destination image according to orig_im_coor
# https://stackoverflow.com/questions/1401712/how-can-the-euclidean-distance-be-calculated-with-numpy
width = round(np.linalg.norm(orig_im_coor[0] - orig_im_coor[1]))  # Assume width is the distance from top-left to top-right
height = round(np.linalg.norm(orig_im_coor[0] - orig_im_coor[2]))  # Assume height is the distance from top-left to bottom-left

#height, width = 450, 350
new_image_coor =  np.float32([[0, 0], [width, 0], [0, height], [width, height]])

P = cv2.getPerspectiveTransform(orig_im_coor, new_image_coor)

#perspective = cv2.warpPerspective(input_image2, P, (width, height))

# Apply the perspective transform on the original_image
perspective = cv2.warpPerspective(original_image, P, (width, height))

cv2.imshow("Perspective transformation", perspective)
cv2.waitKey(0)
cv2.destroyAllWindows()

---

Result:

---

Update:

Cropping the masked region without warping.

Here is an example for cropping without warping:

original_image = cv2.imread("original_image.jpg")  # Read the original image

bw_mask = cv2.imread("black_and_white_mask.jpg", cv2.IMREAD_GRAYSCALE)  # Read image as Grayscale
bw_mask = cv2.threshold(bw_mask, 0, 255, cv2.THRESH_OTSU)[1]  # Convert to binary image (just in case...)

# Find the corners of the mask, and sort them
orig_im_coor = find_corners(bw_mask)

# Mask the background - place zeros where mask is zero
masked_image = cv2.bitwise_and(original_image, original_image, mask=bw_mask)

# The corners are the minimum and maximum of orig_im_coor
x0 = int(round(orig_im_coor[:, 0].min()))
y0 = int(round(orig_im_coor[:, 1].min()))
x1 = int(round(orig_im_coor[:, 0].max()))
y1 = int(round(orig_im_coor[:, 1].max()))
cropped_masked_image = masked_image[y0:y1+1, x0:x1+1, :].copy()  # Crop area from top-left to bottom-right

# Show the results:
cv2.imshow('masked_image', masked_image)
cv2.imshow('cropped_masked_image', cropped_masked_image)
cv2.waitKey()
cv2.destroyAllWindows()

---

Results:

masked_image:

cropped_masked_image: