get coordinates of 4 corners of display screen on image

Question

I am trying to get 4 corners of screen (display) which is on image. I have two images taken from the same position (so I think good starting point will be extracting differences between two images /first and second image/). Just image on the screen has changed. So I would like to get top/bottom left/right (X,Y) coordinates of display screen corners.

I am using python 2.7 with cv2 and numpy (if possible not to use another modules). Unfotunately, I do not have knowledge how to get these coordinates.

Any idea please? P.S. sample code would be great and highly appretiated, many many thanks

Final result:

Answer 1

I have created a new solution using the difference between images and finding contours from that. I have left the old solution using hough line processing at the bottom.

import numpy as np
import cv2


def main():
    im1 = cv2.imread('s123/ss1.jpg')
    im2 = cv2.imread('s123/ss2.jpg')

    gray1 = cv2.cvtColor(im1, cv2.COLOR_BGR2GRAY)
    gray2 = cv2.cvtColor(im2, cv2.COLOR_BGR2GRAY)

    # Try to match the two image's exposures
    gray1 = cv2.equalizeHist(gray1)
    gray2 = cv2.equalizeHist(gray2)

    # Find the difference and threshold it
    diff = cv2.absdiff(gray1, gray2)
    _, thresh = cv2.threshold(diff, 50, 255, cv2.THRESH_BINARY)

    # Filtering to improve the thresholded image
    thresh = cv2.medianBlur(thresh, 5)
    thresh = cv2.dilate(thresh, None, iterations=2)

    # Calculate contours and find the largest one
    _, cnts, hierachy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
    cnt = max([c for c in cnts], key=lambda x: cv2.contourArea(x))

    cv2.drawContours(im1, [cnt], 0, (0, 255, 0), 3)

    # Remove the concavities
    hull = cv2.convexHull(cnt)
    cv2.drawContours(im1, [hull], 0, (255, 0, 0), 2)
    hull = [tuple(p[0]) for p in hull]

    # Find all the corners
    tr = max(hull, key=lambda x: x[0] - x[1])
    cv2.circle(im1, tr, 3, (0, 0, 255), -1)

    tl = min(hull, key=lambda x: x[0] + x[1])
    cv2.circle(im1, tl, 3, (0, 0, 255), -1)

    br = max(hull, key=lambda x: x[0] + x[1])
    cv2.circle(im1, br, 3, (0, 0, 255), -1)

    bl = min(hull, key=lambda x: x[0] - x[1])
    cv2.circle(im1, bl, 3, (0, 0, 255), -1)

    cv2.imshow('im1', im1)
    cv2.imshow('diff', thresh)

    cv2.waitKey(0)


if __name__ == '__main__':
    main()

This method has the disadvantage of requiring a large difference in the screens (ie. 1&2 work but 1&3 don't work however 2&3 work since 2 is mostly white). If you want a more robust method try a background subtractor which will need many more images.

---

I averaged the two images and then used hough line processing to find the lines. Then I filtered those and then found the intersection points:

import numpy as np
import cv2


# Code to find line intersections. From https://stackoverflow.com/a/20677983
def line_intersection(line1, line2):
    xdiff = (line1[0][0] - line1[1][0], line2[0][0] - line2[1][0])
    ydiff = (line1[0][1] - line1[1][1], line2[0][1] - line2[1][1])

    def det(a, b):
        return a[0] * b[1] - a[1] * b[0]

    div = det(xdiff, ydiff)
    if div == 0:
        return -1, -1

    d = (det(*line1), det(*line2))
    x = det(d, xdiff) / div
    y = det(d, ydiff) / div
    return x, y


def main():
    im1 = cv2.imread('GaJrr.jpg')
    im2 = cv2.imread('kR2pl.jpg')

    gray1 = cv2.cvtColor(im1, cv2.COLOR_BGR2GRAY)
    gray2 = cv2.cvtColor(im2, cv2.COLOR_BGR2GRAY)

    # Average the images
    diff = cv2.addWeighted(gray1, 0.5, gray2, 0.5, 0)

    # Canny and Hough lines
    c = cv2.Canny(diff, 89, 200)
    lines = cv2.HoughLines(c, 1, np.pi / 180, 100, None, 0, 0)

    pts = []

    # Create segments for each line
    if lines is not None:
        for i in range(len(lines)):
            rho = lines[i][0][0]
            theta = lines[i][0][1]
            a = np.cos(theta)
            b = np.sin(theta)
            x0 = a * rho
            y0 = b * rho
            pt1 = np.array([int(x0 + 1000 * (-b)), int(y0 + 1000 * a)])
            pt2 = np.array([int(x0 - 1000 * (-b)), int(y0 - 1000 * a)])

            if not any([np.linalg.norm(pt1 - p[0]) < 100 for p in pts]):    # Filter out lines too close to each other
                pts.append(np.array([pt1, pt2]))

                cv2.line(im1, tuple(pt1), tuple(pt2), (0, 0, 255), 1, cv2.LINE_AA)

    for pt in pts:
        for comp in pts:
            intersect = np.array(line_intersection(pt, comp))
            if any(intersect < 0) or intersect[0] > im1.shape[1] or intersect[1] > im1.shape[0]:    # Filter out off-screen intersections
                continue

            intersect = np.asarray(intersect, dtype=int)
            print(intersect)
            cv2.circle(im1, tuple(intersect), 3, (0, 255, 0), -1)

    cv2.imshow('im1', im1)

    cv2.waitKey(0)


if __name__ == '__main__':
    main()

This can definitely be optimized a bunch.

Answer 2

You should take a look at the opencv python tutorial in the feature detection section, there are a few algorithms that may provide some help(and the explanations of the math behind are great), but from my understanding probably the Shi-Tomasi algorithm is the best for your situation, here is an example:

def get_corners(img):
    img_m = np.float32(img)
    img_m = cv2.cvtColor(img_m,cv2.COLOR_BGR2GRAY)
    corners = cv2.goodFeaturesToTrack(img_m,4,0.01,15)
    corners = np.int0(corners)
    return corners

img = cv2.imread("...") #you should make sure that the image is colored or else it's going to give an error
img_g = get_corners(img)
for i in img_g:
    x,y = i.ravel()
    cv.circle(img,(x,y),3,255,-1)
cv2.imshow('img', img)
cv2.waitKey(0)

but you should always remember: computer vision is a research type field, that means that you try different algorithms until one works best, it's okay to use prewritten ones, but you should try them by yourself, usually there isn't a magic solution that works for all things in this types of fields, usually every solution is very specific to the problem and the process of tuning the different algorithms to your need is long and hard, it depends solely on your research and level of commitment.

Hope that was helpful!