Crop all characters from an image

Question

I put together some code to extract all characters from an image. I sort the characters from left to right and I try to crop each character into a separate image. Not all characters are properly cropped, some of them end up having size zero.

The only characters that do not have one dimension zero are BCDEF. Here is an image of the output.

import cv2
import numpy as np

def crop_minAreaRect(img, rect):
# https://stackoverflow.com/questions/37177811/crop-rectangle-returned-by-minarearect-opencv-python
    # rotate img
    center = rect[0]
    size = rect[1]
    print("size[0]: " + str(int(size[0])) + ", size[1]: " + str(int(size[1])))
    angle = rect[2]
    print("angle: " + str(angle))
    rows,cols = img.shape[0], img.shape[1]
    M = cv2.getRotationMatrix2D((cols/2,rows/2),angle,1)
    img_rot = cv2.warpAffine(img,M,(cols,rows))

    # rotate bounding box
    rect0 = (rect[0], rect[1], angle) 
    box = cv2.boxPoints(rect0)
    pts = np.int0(cv2.transform(np.array([box]), M))[0]    
    pts[pts < 0] = 0

    # crop
    img_crop = img_rot[pts[1][1]:pts[0][1], pts[1][0]:pts[2][0]]

    w, h = img_crop.shape[0], img_crop.shape[1]
    print("w_cropped: " + str(w) + ", h_cropped: " + str(h))
    return img_crop

def sort_contours(cnts, method="left-to-right"):
# from https://pyimagesearch.com/2015/04/20/sorting-contours-using-python-and-opencv/    
    reverse = False
    i = 0
    if method == "right-to-left" or method == "bottom-to-top":
        reverse = True
    if method == "top-to-bottom" or method == "bottom-to-top":
        i = 1
    boundingBoxes = [cv2.boundingRect(c) for c in cnts]
    (cnts, boundingBoxes) = zip(*sorted(zip(cnts, boundingBoxes),   key=lambda b:b[1][i], reverse=reverse))
    return (cnts, boundingBoxes)    
    
im_name = 'letters.png'
im = cv2.imread(im_name)
im_copy = im.copy()

imgray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(imgray, 127, 255, 0)
contours, hierarchy = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)

#cv2.drawContours(im_copy, contours, -1, (0,255,0), 2)
#cv2.imshow("contours", im_copy)

print("num contours: " + str(len(contours)))
i = 0

sorted_cnts, bounding_boxes = sort_contours(contours, method="left-to-right")

for cnt in sorted_cnts:
  size = cv2.contourArea(cnt)
  x,y,w,h = cv2.boundingRect(cnt)
  rect = cv2.minAreaRect(cnt)
#  print(str(rect))
#  if rect[1][0] > 0 and rect[1][1]>0:
  im_cropped = crop_minAreaRect(im, rect)
    
  h,w = im_cropped.shape[0], im_cropped.shape[1]
  if w > h:
    im_cropped = cv2.rotate(im_cropped, cv2.ROTATE_90_CLOCKWISE)  
    
  print("w: " + str(w) + ", h: " + str(h))
  if w>0 and h>0:
    cv2.imshow("cropped" + str(i), im_cropped)
  i += 1
#    cv2.waitKey(0)

cv2.waitKey(0)

show us what you see, any and all program output, and *also* intermediate results (you did debug the program?). nobody's gonna run that code until you've demonstrated the issue. please review [mre]. — Christoph Rackwitz, Jun 25 '22 at 14:26
Do you need the cropped characters for something other than identifying them? If you don't, pytesseract should be able to read them with the right arguments. — fmw42, Jun 25 '22 at 16:57
Hi Cristoph, I will add the output I see. I did debug the program, I know the function crop_minAreaRect(img, rect) is not always working right, but I could not figure out what is wrong with it. — Nick_F, Jun 25 '22 at 22:48
Hi fmw42. In fact, I started with a different task, a captcha where the digits are a bit distorted and rotated at random angles. I used tesseract to identify the digits and (I suppose) because of the slight rotation, it does a very poor job. See https://stackoverflow.com/questions/72583448/improving-tesseract-digit-recognition — Nick_F, Jun 25 '22 at 22:52

code-lukas · Accepted Answer · 2022-06-27T05:58:59.407

There appears to be an error in your crop_minAreaRect function. I haven't debugged your code any further than the return of crop_minAreaRect, so the letters may or may not be correctly rotated according following your approach, but this change fixes the underlying problem.
The proposed function is taken from the following question and modified: How to straighten a rotated rectangle area of an image using OpenCV in Python?

import cv2
import numpy as np


def subimage(image, center, theta, width, height):
    '''
   Rotates OpenCV image around center with angle theta (in deg)
   then crops the image according to width and height.
   '''
    width = int(width)
    height = int(height)
    # Uncomment for theta in radians
    # theta *= 180/np.pi

    shape = (image.shape[1], image.shape[0])  # cv2.warpAffine expects shape in (length, height)

    matrix = cv2.getRotationMatrix2D(center=center, angle=theta, scale=1)
    image = cv2.warpAffine(src=image, M=matrix, dsize=shape)

    x = int(center[0] - width / 2)
    y = int(center[1] - height / 2)

    image = image[y:y + height, x:x + width]

    return image


def sort_contours(cnts, method="left-to-right"):
    # from https://pyimagesearch.com/2015/04/20/sorting-contours-using-python-and-opencv/
    reverse = False
    i = 0
    if method == "right-to-left" or method == "bottom-to-top":
        reverse = True
    if method == "top-to-bottom" or method == "bottom-to-top":
        i = 1
    boundingBoxes = [cv2.boundingRect(c) for c in cnts]
    (cnts, boundingBoxes) = zip(*sorted(zip(cnts, boundingBoxes), key=lambda b: b[1][i], reverse=reverse))
    return (cnts, boundingBoxes)


im_name = 'letters.png'
im = cv2.imread(im_name)
im_copy = im.copy()

imgray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(imgray, 127, 255, 0)
contours, hierarchy = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)

cv2.drawContours(im_copy, contours, -1, (0, 255, 0), 2)
cv2.imshow("contours", im_copy)

# print("num contours: " + str(len(contours)))
i = 0

sorted_cnts, bounding_boxes = sort_contours(contours, method="left-to-right")

for cnt in sorted_cnts:
    size = cv2.contourArea(cnt)
    x, y, w, h = cv2.boundingRect(cnt)
    rect = cv2.minAreaRect(cnt)


    im_cropped = subimage(im, center=rect[0], theta=rect[2], width=rect[1][0], height=rect[1][1])

    h, w = im_cropped.shape[0], im_cropped.shape[1]
    if w > h:
        im_cropped = cv2.rotate(im_cropped, cv2.ROTATE_90_CLOCKWISE)

    # print("w: " + str(w) + ", h: " + str(h))
    if w > 0 and h > 0:
        cv2.imshow("cropped" + str(i), im_cropped)
    i += 1
#    cv2.waitKey(0)

cv2.waitKey(0)

Thanks code-lukas. You are right, I also noticed the function crop_minAreaRect(img, rect) fails to always return a correct rectangle. The function you suggested seems to crop some of the characters too much, but it is a good start. — Nick_F, Jun 25 '22 at 23:14
Hi code-lukas, I found out there is a tiny mistake in your code. im_cropped = subimage(im, center=rect[0], theta=rect[2], width=rect[1][0], height=rect[1][0]). The height should be rect[1][1]. Now it's all ok. I will accept your solution. Can you please edit that line, I can't edit it myself. — Nick_F, Jun 25 '22 at 23:21

Crop all characters from an image

1 Answers1