Using OpenCV to find the bounding box of numbers on an image

Question

I'm trying to find the bounding box of the numbers in the middle of the below 3 images.

Here's 3 example cards I'm trying to work with.

The code I'm using is based (almost a complete copy) of the code provided in the first answer here, although converted to Java (answers in C++ are fine) and added parameters for the size of the contours to merge (defined as sizeHorizonal and sizeVertical in my code), which are the two parameters I'm playing with in the images below.

MatOfPoint2f approxCurve = new MatOfPoint2f();

Mat imgMAT = new Mat();
Utils.bitmapToMat(bmp32, imgMAT);

Mat grad = new Mat();

Imgproc.cvtColor(imgMAT, grad, Imgproc.COLOR_BGR2GRAY);

Mat img_sobel = new Mat();
Mat img_threshold = new Mat();

Imgproc.Sobel(grad, img_sobel, CvType.CV_8U, 1, 0, 3, 1, 0, Core.BORDER_DEFAULT);

Imgproc.threshold(img_sobel, img_threshold, 0, 255, Imgproc.THRESH_OTSU + Imgproc.THRESH_BINARY);

Mat element = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(sizeHorizontal, sizeVertical));

Imgproc.morphologyEx(img_threshold, img_threshold, Imgproc.MORPH_CLOSE, element);

Imgproc.cvtColor(imgMAT, imgMAT, Imgproc.COLOR_BGR2GRAY);


List<MatOfPoint> contours = new ArrayList<>();
Imgproc.findContours(img_threshold, contours, new Mat(), Imgproc.RETR_CCOMP, Imgproc.CHAIN_APPROX_SIMPLE, new org.opencv.core.Point(0, 0));

for (int i = 0; i < contours.size(); i++) {

    //Convert contours(i) from MatOfPoint to MatOfPoint2f
    MatOfPoint2f contour2f = new MatOfPoint2f( contours.get(i).toArray() );

    //Processing on mMOP2f1 which is in type MatOfPoint2f
    double approxDistance = Imgproc.arcLength(contour2f, true)*0.02;
    Imgproc.approxPolyDP(contour2f, approxCurve, approxDistance, true);

    //Convert back to MatOfPoint
    MatOfPoint points = new MatOfPoint( approxCurve.toArray() );

    // Get bounding rect of contour
    org.opencv.core.Rect rect = Imgproc.boundingRect(points);

    Imgproc.rectangle(imgMAT, rect.tl(), rect.br(), new Scalar(0, 255, 0), 2);
}

I've got the separate number sections contoured, but I can't find a way to isolate the contours I want. Here's the areas contoured with the parameters for the size input. As you can see for the second image, this is working exactly as I want and has contoured the whole number, rather than each section.

1: Size param input: 17, 5

2: Size param input: 23, 7

3: Size param input: 23, 13

So, things I need help with:

1. Isolating the four contours in the middle and finding a way to merge these contours together

I've thought about taking the contours that match a given aspect ratio and cropping to a bounding box encompassing all of them, but there are other surrounding contours with similar ratios.

2. Finding a way to choose the correct size parameters automatically (as each card type requires different parameters to isolate the numbers)

Short of trying all 3 size inputs and seeing what gives the expected contours, I could use the prevailing colour as an indicator of the card type and then use the parameters for this card type. But, any other suggestions would be helpful, as I feel there's a better way to do this.

Many thanks!

Answer 1

Out of my busy schedule i could help you to some extinct. Please find the code below which will help you for first two images. Fine tune it for the third image. Just play with morphological operations to get the required output.

//#include "stdafx.h"

#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <iostream>
#include "tchar.h"
using namespace cv;
using namespace std;

#define INPUT_FILE              "p.jpg"
#define OUTPUT_FOLDER_PATH      string("")

int _tmain(int argc, _TCHAR* argv[])
{
    Mat large = imread(INPUT_FILE);
    Mat rgb;
    // downsample and use it for processing
    pyrDown(large, rgb);
    Mat small;
    cvtColor(rgb, small, CV_BGR2GRAY);
    // morphological gradient
    Mat grad;
    Mat morphKernel = getStructuringElement(MORPH_ELLIPSE, Size(2, 2));
    Mat morphKernel1 = getStructuringElement(MORPH_ELLIPSE, Size(1, 1));
    morphologyEx(small, grad, MORPH_GRADIENT, morphKernel);
    // binarize
    Mat bw;
    threshold(grad, bw, 5.0, 50.0, THRESH_BINARY | THRESH_OTSU);
    // connect horizontally oriented regions
    Mat connected;
    morphKernel = getStructuringElement(MORPH_RECT, Size(9, 1));
    morphologyEx(bw, connected, MORPH_CLOSE, morphKernel);
    morphologyEx(bw, connected, MORPH_OPEN, morphKernel1);
    morphologyEx(connected, connected, MORPH_CLOSE, morphKernel);
    morphologyEx(connected, connected, MORPH_CLOSE, morphKernel);
     morphologyEx(connected, connected, MORPH_CLOSE, morphKernel);
    // find contours
    Mat mask = Mat::zeros(bw.size(), CV_8UC1);
    vector<vector<Point>> contours;
    vector<Vec4i> hierarchy;
    findContours(connected, contours, hierarchy, CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE, Point(0, 0));
    // filter contours
    int y=0;
    for(int idx = 0; idx >= 0; idx = hierarchy[idx][0])
    {
        Rect rect = boundingRect(contours[idx]);
        Mat maskROI(mask, rect);
        maskROI = Scalar(0, 0, 0);
        // fill the contour
        drawContours(mask, contours, idx, Scalar(255, 255, 255), CV_FILLED);

        double a=contourArea( contours[idx],false);
                    if(a> 575)

        {
            rectangle(rgb, rect, Scalar(0, 255, 0), 2);
            y++;
        }
        imshow("Result1",rgb);
    }
    cout<<" The number of elements"<<y<< endl; 
    imshow("Result",mask);
    imwrite(OUTPUT_FOLDER_PATH + string("rgb.jpg"), rgb);
    waitKey(0);
    return 0;
}