Extract artwork from table game card image with OpenCV

Question

I wrote a small script in python where I'm trying to extract or crop the part of the playing card that represents the artwork only, removing all the rest. I've been trying various methods of thresholding but couldn't get there. Also note that I can't simply record manually the position of the artwork because it's not always in the same position or size, but always in a rectangular shape where everything else is just text and borders.

from matplotlib import pyplot as plt
import cv2

img = cv2.imread(filename)
gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)

ret,binary = cv2.threshold(gray, 0, 255, cv2.THRESH_OTSU | cv2.THRESH_BINARY)

binary = cv2.bitwise_not(binary)
kernel = np.ones((15, 15), np.uint8)

closing = cv2.morphologyEx(binary, cv2.MORPH_OPEN, kernel)

plt.imshow(closing),plt.show()

The current output is the closest thing I could get. I could be on the right way and try some further wrangling to draw a rectangle around the white parts, but I don't think it's a sustainable method :

As a last note, see the cards below, not all frames are exactly the same sizes or positions, but there's always a piece of artwork with only text and borders around it. It doesn't have to be super precisely cut, but clearly the art is a "region" of the card, surrounded by other regions containing some text. My goal is to try to capture the region of the artwork as well as I can.

Answer 1

I used Hough line transform to detect linear parts of the image. The crossings of all lines were used to construct all possible rectangles, which do not contain other crossing points. Since the part of the card you are looking for is always the biggest of those rectangles (at least in the samples you provided), i simply chose the biggest of those rectangles as winner. The script works without user interaction.

import cv2
import numpy as np
from collections import defaultdict

def segment_by_angle_kmeans(lines, k=2, **kwargs):
    #Groups lines based on angle with k-means.
    #Uses k-means on the coordinates of the angle on the unit circle 
    #to segment `k` angles inside `lines`.

    # Define criteria = (type, max_iter, epsilon)
    default_criteria_type = cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER
    criteria = kwargs.get('criteria', (default_criteria_type, 10, 1.0))
    flags = kwargs.get('flags', cv2.KMEANS_RANDOM_CENTERS)
    attempts = kwargs.get('attempts', 10)

    # returns angles in [0, pi] in radians
    angles = np.array([line[0][1] for line in lines])
    # multiply the angles by two and find coordinates of that angle
    pts = np.array([[np.cos(2*angle), np.sin(2*angle)]
                    for angle in angles], dtype=np.float32)

    # run kmeans on the coords
    labels, centers = cv2.kmeans(pts, k, None, criteria, attempts, flags)[1:]
    labels = labels.reshape(-1)  # transpose to row vec

    # segment lines based on their kmeans label
    segmented = defaultdict(list)
    for i, line in zip(range(len(lines)), lines):
        segmented[labels[i]].append(line)
    segmented = list(segmented.values())
    return segmented

def intersection(line1, line2):
    #Finds the intersection of two lines given in Hesse normal form.
    #Returns closest integer pixel locations.
    #See https://stackoverflow.com/a/383527/5087436

    rho1, theta1 = line1[0]
    rho2, theta2 = line2[0]

    A = np.array([
        [np.cos(theta1), np.sin(theta1)],
        [np.cos(theta2), np.sin(theta2)]
    ])
    b = np.array([[rho1], [rho2]])
    x0, y0 = np.linalg.solve(A, b)
    x0, y0 = int(np.round(x0)), int(np.round(y0))
    return [[x0, y0]]


def segmented_intersections(lines):
    #Finds the intersections between groups of lines.

    intersections = []
    for i, group in enumerate(lines[:-1]):
        for next_group in lines[i+1:]:
            for line1 in group:
                for line2 in next_group:
                    intersections.append(intersection(line1, line2)) 
    return intersections

def rect_from_crossings(crossings):
    #find all rectangles without other points inside
    rectangles = []

    # Search all possible rectangles
    for i in range(len(crossings)):
        x1= int(crossings[i][0][0])
        y1= int(crossings[i][0][1])

        for j in range(len(crossings)):
            x2= int(crossings[j][0][0])
            y2= int(crossings[j][0][1])

            #Search all points
            flag = 1
            for k in range(len(crossings)):
                x3= int(crossings[k][0][0])
                y3= int(crossings[k][0][1])

                #Dont count double (reverse rectangles)
                if (x1 > x2 or y1 > y2):
                    flag = 0
                #Dont count rectangles with points inside   
                elif ((((x3 >= x1) and (x2 >= x3))and (y3 > y1) and (y2 > y3) or ((x3 > x1) and (x2 > x3))and (y3 >= y1) and (y2 >= y3))):    
                    if(i!=k and j!=k):    
                        flag = 0

            if flag:
                rectangles.append([[x1,y1],[x2,y2]])

    return rectangles

if __name__ == '__main__':
    #img = cv2.imread('TAJFp.jpg')
    #img = cv2.imread('Bj2uu.jpg')
    img = cv2.imread('yi8db.png')

    width = int(img.shape[1])
    height = int(img.shape[0])

    scale = 380/width
    dim = (int(width*scale), int(height*scale))
    # resize image
    img = cv2.resize(img, dim, interpolation = cv2.INTER_AREA) 

    img2 = img.copy()
    gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
    gray = cv2.GaussianBlur(gray,(5,5),cv2.BORDER_DEFAULT)

    # Parameters of Canny and Hough may have to be tweaked to work for as many cards as possible
    edges = cv2.Canny(gray,10,45,apertureSize = 7)
    lines = cv2.HoughLines(edges,1,np.pi/90,160)

    segmented = segment_by_angle_kmeans(lines)
    crossings = segmented_intersections(segmented)
    rectangles = rect_from_crossings(crossings)

    #Find biggest remaining rectangle
    size = 0
    for i in range(len(rectangles)):
        x1 = rectangles[i][0][0]
        x2 = rectangles[i][1][0]
        y1 = rectangles[i][0][1]
        y2 = rectangles[i][1][1]

        if(size < (abs(x1-x2)*abs(y1-y2))):
            size = abs(x1-x2)*abs(y1-y2)
            x1_rect = x1
            x2_rect = x2
            y1_rect = y1
            y2_rect = y2

    cv2.rectangle(img2, (x1_rect,y1_rect), (x2_rect,y2_rect), (0,0,255), 2)
    roi = img[y1_rect:y2_rect, x1_rect:x2_rect]

    cv2.imshow("Output",roi)
    cv2.imwrite("Output.png", roi)
    cv2.waitKey()

These are the results with the samples you provided:

The code for finding line crossings can be found here: find intersection point of two lines drawn using houghlines opencv

You can read more about Hough Lines here.

Answer 2

We know that cards have straight boundaries along the x and y axes. We can use this to extract parts of the image. The following code implements detecting horizontal and vertical lines in the image.

import cv2
import numpy as np

def mouse_callback(event, x, y, flags, params):
    global num_click
    if num_click < 2 and event == cv2.EVENT_LBUTTONDOWN:
        num_click = num_click + 1
        print(num_click)
        global upper_bound, lower_bound, left_bound, right_bound
        upper_bound.append(max(i for i in hor if i < y) + 1)
        lower_bound.append(min(i for i in hor if i > y) - 1)
        left_bound.append(max(i for i in ver if i < x) + 1)
        right_bound.append(min(i for i in ver if i > x) - 1)

filename = 'image.png'
thr = 100  # edge detection threshold
lined = 50  # number of consequtive True pixels required an axis to be counted as line
num_click = 0  # select only twice
upper_bound, lower_bound, left_bound, right_bound = [], [], [], []
winname = 'img'

cv2.namedWindow(winname)
cv2.setMouseCallback(winname, mouse_callback)

img = cv2.imread(filename, 1)
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
bw = cv2.Canny(gray, thr, 3*thr)

height, width, _ = img.shape

# find horizontal lines
hor = []
for i in range (0, height-1):
    count = 0
    for j in range (0, width-1):
        if bw[i,j]:
            count = count + 1
        else:
            count = 0
        if count >= lined:
            hor.append(i)
            break

# find vertical lines
ver = []
for j in range (0, width-1):
    count = 0
    for i in range (0, height-1):
        if bw[i,j]:
            count = count + 1
        else:
            count = 0
        if count >= lined:
            ver.append(j)
            break

# draw lines
disp_img = np.copy(img)
for i in hor:
    cv2.line(disp_img, (0, i), (width-1, i), (0,0,255), 1)
for i in ver:
    cv2.line(disp_img, (i, 0), (i, height-1), (0,0,255), 1)

while num_click < 2:
    cv2.imshow(winname, disp_img)
    cv2.waitKey(10)
disp_img = img[min(upper_bound):max(lower_bound), min(left_bound):max(right_bound)]
cv2.imshow(winname, disp_img)
cv2.waitKey()   # Press any key to exit
cv2.destroyAllWindows()

You just need to click two areas to include. A sample click area and the corresponding result are as follows:

Results from other images:

Answer 3

I don't think it is possible to automatically crop the artwork ROI using traditional image processing techniques due to the dynamic nature of the colors, dimensions, locations, and textures for each card. You would have to look into machine/deep learning and train your own classifier if you want to do it automatically. Instead, here's a manual approach to select and crop a static ROI from an image.

The idea is to use cv2.setMouseCallback() and event handlers to detect if the mouse has been clicked or released. For this implementation, you can extract the artwork ROI by holding down the left mouse button and dragging to select the desired ROI. Once you have selected the desired ROI, press c to crop and save the ROI. You can reset the ROI using the right mouse button.

Saved artwork ROIs

Code

import cv2

class ExtractArtworkROI(object):
    def __init__(self):
        # Load image
        self.original_image = cv2.imread('1.png')
        self.clone = self.original_image.copy()
        cv2.namedWindow('image')
        cv2.setMouseCallback('image', self.extractROI)
        self.selected_ROI = False

        # ROI bounding box reference points
        self.image_coordinates = []

    def extractROI(self, event, x, y, flags, parameters):
        # Record starting (x,y) coordinates on left mouse button click
        if event == cv2.EVENT_LBUTTONDOWN:
            self.image_coordinates = [(x,y)]

        # Record ending (x,y) coordintes on left mouse button release
        elif event == cv2.EVENT_LBUTTONUP:
            # Remove old bounding box
            if self.selected_ROI:
                self.clone = self.original_image.copy()

            # Draw rectangle 
            self.selected_ROI = True
            self.image_coordinates.append((x,y))
            cv2.rectangle(self.clone, self.image_coordinates[0], self.image_coordinates[1], (36,255,12), 2)

            print('top left: {}, bottom right: {}'.format(self.image_coordinates[0], self.image_coordinates[1]))
            print('x,y,w,h : ({}, {}, {}, {})'.format(self.image_coordinates[0][0], self.image_coordinates[0][1], self.image_coordinates[1][0] - self.image_coordinates[0][0], self.image_coordinates[1][1] - self.image_coordinates[0][1]))

        # Clear drawing boxes on right mouse button click
        elif event == cv2.EVENT_RBUTTONDOWN:
            self.selected_ROI = False
            self.clone = self.original_image.copy()

    def show_image(self):
        return self.clone

    def crop_ROI(self):
        if self.selected_ROI:
            x1 = self.image_coordinates[0][0]
            y1 = self.image_coordinates[0][1]
            x2 = self.image_coordinates[1][0]
            y2 = self.image_coordinates[1][1]

            # Extract ROI
            self.cropped_image = self.original_image.copy()[y1:y2, x1:x2]

            # Display and save image
            cv2.imshow('Cropped Image', self.cropped_image)
            cv2.imwrite('ROI.png', self.cropped_image)
        else:
            print('Select ROI before cropping!')

if __name__ == '__main__':
    extractArtworkROI = ExtractArtworkROI()
    while True:
        cv2.imshow('image', extractArtworkROI.show_image())
        key = cv2.waitKey(1)

        # Close program with keyboard 'q'
        if key == ord('q'):
            cv2.destroyAllWindows()
            exit(1)

        # Crop ROI
        if key == ord('c'):
            extractArtworkROI.crop_ROI()