Need to identify the "tray is empty or not" using opencv

Question

I want to recognize the tray is empty or not in the given image using OpenCV in python.

below is what I have tried

1. detect the biggest rectangle and cropped by using the below code

kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3,3))
thresh = cv2.erode(thresh, kernel, iterations=4)
thresh = cv2.dilate(thresh, kernel, iterations=4)
cnts, hier = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

# Find the biggest contour
max_area = -1
max_c = 0
for i in range(len(cnts)):
    contour = cnts[i]
    area = cv2.contourArea(contour)
    if (area > max_area):
        max_area = area
        max_c = i
contour = cnts[max_c]

areas = [cv2.contourArea(c) for c in cnts]
max_index = np.argmax(areas)
cnt=cnts[max_index]

x,y,w,h = cv2.boundingRect(cnt)

crop_img = img[y:y+h, x:x+w]

2. and then trying to find black dots by getting contours after applying the canny edge detection and applying mask and noise removal methods

#Create empty mask and flood fill
mask = np.zeros(edges.shape)
#Create 3-channel alpha mask
mask_stack = np.dstack([mask]*3)

3. and again threshold the image and find contours.

It works for medium and large objects but when I put the small objects like coins, this method is not working because the tray has some scratches and dust too. This is my first project using OpenCV in python

please help to find the solution to achieve this requirement.

Input:

Output:

Empty Tray:

Answer 1

I recommend you to:

1. do camera calibration to make the lines in your image straight, and
2. align your camera well to your production line so you can simplify the image processing and make it more robust, and
3. use better illumination conditions if possible.

Given the images above, here is a brute force solution:

• you can first extract the region of interest (ROI) through: HSV segmentation followed by contour detection, then you can extract the rectangular shape of the tray using this SO answer.

#!/usr/bin/python3
# -*- coding: utf-8 -*-

import cv2
import numpy as np

# Ref: https://stackoverflow.com/questions/37177811
def crop_minAreaRect(img, rect):

    box = cv2.boxPoints(rect) 
    box = np.int0(box)

    W = rect[1][0]
    H = rect[1][1]

    Xs = [i[0] for i in box]
    Ys = [i[1] for i in box]
    x1 = min(Xs)
    x2 = max(Xs)
    y1 = min(Ys)
    y2 = max(Ys)

    angle = rect[2]
    
    rotated = False

    if angle < -45:
        angle += 90
        rotated = True
    # Center of rectangle in source image
    center = ((x1+x2)/2,(y1+y2)/2)
    # Size of the upright rectangle bounding the rotated rectangle
    size = (x2-x1, y2-y1)
    M = cv2.getRotationMatrix2D((size[0]/2, size[1]/2), angle, 1.0)
    # Cropped upright rectangle
    cropped = cv2.getRectSubPix(img, size, center)
    cropped = cv2.warpAffine(cropped, M, size)
    croppedW = W if not rotated else H 
    croppedH = H if not rotated else W
    # Final cropped & rotated rectangle
    croppedRotated = cv2.getRectSubPix(cropped, (int(croppedW),int(croppedH)), (size[0]/2, size[1]/2))
    return croppedRotated

def ROI(img):
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)

    lower_hsv = np.array([0, 0, 135]) # 0, 0, 135
    higher_hsv = np.array([180, 20, 240]) # 180, 20, 240

    mask = cv2.inRange(hsv, lower_hsv, higher_hsv)

    seg = cv2.bitwise_and(img, img, mask=mask)

    seg_gray = cv2.cvtColor(seg, cv2.COLOR_BGR2GRAY)

    k1 = 51
    seg_gauss = cv2.GaussianBlur(seg_gray, (k1, k1), 0)
    seg_th = cv2.threshold(seg_gauss, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)[1]

    contours, hierarchy = cv2.findContours(image=seg_th, mode=cv2.RETR_EXTERNAL, method=cv2.CHAIN_APPROX_NONE)
    for i, cnt in enumerate(contours):
        if  cv2.contourArea(cnt)>1000000:
            x,y,w,h = cv2.boundingRect(cnt)     
            rect = cv2.minAreaRect(cnt)

    roi = crop_minAreaRect(img, rect)
    return roi

• As you reduced your problem to a specific ROI, you can do use edge detection, Morphology, and Filtering as follows:

def ImProc(roi):

    gray = cv2.cvtColor(roi, cv2.COLOR_BGR2GRAY)

    k_smooth = 5
    gauss = cv2.GaussianBlur(gray, (k_smooth, k_smooth), 0)

    # # Canny
    edges = cv2.Canny(gauss,0,255)

    k = 31 #300
    kernel = np.ones((k,k),np.uint8)
    mph = cv2.morphologyEx(edges, cv2.MORPH_CLOSE, kernel)

    res = cv2.medianBlur(mph, 21)

    return res

I have applied this approach to the given images:

empty = cv2.imread("empty.jpg")
full = cv2.imread("full.jpg")

roi_empty = ROI(empty)
roi_full = ROI(full)

res_empty = ImProc(roi_empty)
res_full = ImProc(roi_full)

cv2.namedWindow("res_full", cv2.WINDOW_NORMAL)
cv2.imshow("res_full", res_full)

cv2.waitKey(0)

and I have got the following result for the full tray (rotated):

Note that this solution is subject to different parameters, to better control them and get stable results I advise you to consider the recommendations above.