How to convert words from png file to string in openCV/C++?

Question

So my project is to create a program that will detect words on a Scrabble board based on the position where the user clicks on the image. My question is the following:

1. How to store the detected letters into strings/words? (i.e. check each contour in sequence and depending on whether it is close to another contour then add it as the next element of a string)

The program draws the contours around each letter based on moments and contour areas. Any corrections in the current code would be much appreciated. It works but obviously it could be written better. Here are the obtained images:

binary:

color:

#include <opencv2\core.hpp> //cv::Mat etc
#include <opencv2\imgproc.hpp> // algorithms 
#include <opencv2\highgui.hpp> // input, output 
#include <iostream>
#include <fstream>

using namespace std;
using namespace cv;

Mat image, labimg, perspective, gaussian, binary, gray, erodedimg, dilatedimg, newmat, newmat2, transformed, transformed2;
Mat_<Point> capturePoint; // point coordinates, global variable;

vector< vector <Point> > contours; // Vector for storing contour
vector< Vec4i > hierarchy;
int largest_contour_index = 0;
double largest_area = 0;

void on_mouse(int e, int x, int y, int d, void *ptr)
{
    if (e == EVENT_LBUTTONDOWN)
    {
        Point*p = (Point*)ptr;
        p->x = x;
        p->y = y;

        //Point center = *p;
        cout << *p << endl;

        //destroyWindow("My Window");
    }
}

void myperspective()
{
    Mat dst = image.clone(); //(image.rows, image.cols, CV_8UC1, Scalar::all(0)); //create destination image
    findContours(binary.clone(), contours, hierarchy, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE, Point(0,0)); // Find the contours in the image
    for (int i = 0; i < contours.size(); i++) 
    {
        double a = contourArea(contours[i], false);  //  Find the area of contour
        if (a > largest_area) 
        {
            largest_area = a;
            largest_contour_index = i;                //Store the index of largest contour
        }
    }

    drawContours(dst, contours, largest_contour_index, Scalar(0, 255, 0), 2, 8, hierarchy);
    //imshow("Display window", dst);
    //waitKey(0);

    vector<vector<Point> > contours_poly(1);
    approxPolyDP(Mat(contours[largest_contour_index]), contours_poly[0], 10, true);
    Point2f output_points[4];
    Point2f input_points[4];

    cout << contours_poly[0].size();

    int min_x = image.cols;
    int min_y = image.rows;
    int max_x = 0;
    int max_y = 0;
    int myxmin = image.cols;
    int myxmin2 = image.cols;
    int myxmax = 0;
    int myymin = image.rows;

    for (int k = 0; k < contours_poly[0].size(); k++)
    {
        if (contours_poly[0][k].x < min_x && contours_poly[0][k].y < min_y)
        {
            min_x = contours_poly[0][k].x;
            min_y = contours_poly[0][k].y;
        }
        else if (contours_poly[0][k].x <= min_x && contours_poly[0][k].y > max_y)
        {
            //min_x = contours_poly[0][k].x;
            max_y = contours_poly[0][k].y;
            myxmin2 = contours_poly[0][k].x;
        }
        else if (contours_poly[0][k].x > max_x && contours_poly[0][k].y > max_y)
        {
            max_x = contours_poly[0][k].x;
            //max_y = contours_poly[0][k].y;
        }
        else if (contours_poly[0][k].x < max_x && contours_poly[0][k].y <= (min_y+20) && contours_poly[0][k].x > myxmax)
        {
            myxmax = contours_poly[0][k].x;
        }
        else if (contours_poly[0][k].x > min_x && contours_poly[0][k].y <= min_y && contours_poly[0][k].x < myxmin)
        {
            myxmin = contours_poly[0][k].x;
            myymin = contours_poly[0][k].y;
        }
        else
        {
            cout << "NA" << endl;
        }
    }

    input_points[0] = Point(myxmin, myymin);
    input_points[1] = Point(myxmax, min_y);
    input_points[2] = Point(myxmin2, max_y);
    input_points[3] = Point(max_x-30, max_y+5);

    output_points[0] = Point(0, 0);
    output_points[1] = Point(image.cols, 0);
    output_points[2] = Point(0, image.rows);
    output_points[3] = Point(image.cols, image.rows);

    Mat transmtx = getPerspectiveTransform(input_points, output_points);
    transformed = Mat::zeros(image.rows, image.cols, CV_8UC3);
    transformed2 = Mat::zeros(image.rows, image.cols, CV_8UC3);
    warpPerspective(binary, transformed, transmtx, image.size());
    warpPerspective(image, transformed2, transmtx, image.size());
}

int main(int argc, char** argv)
{
    image = cv::imread("ideal.png"); // Read image from file
    namedWindow("Display window", WINDOW_NORMAL);

    cv::GaussianBlur(image, gaussian, Size(3, 3), 0);
    cvtColor(gaussian, gray, CV_BGR2GRAY);

    cv::adaptiveThreshold(gray, binary, 255, ADAPTIVE_THRESH_GAUSSIAN_C, THRESH_BINARY_INV, 7, 2);
    //createTrackbar("Threshold:", "Display window2", &thresh_elem, max_thresh_size, Threshold);
    //createTrackbar("Kernel:", "Display window2", &kernel_size, max_kernel_size, Threshold);
    //imshow("Display window", binary);
    //waitKey(0);

    myperspective();

    vector<vector<Point>>lettercontours;
    vector<Vec4i> hierarchyoflet;

    findContours(transformed, lettercontours, hierarchyoflet, CV_RETR_LIST, CV_CHAIN_APPROX_TC89_L1);
    int idx = 0;
    vector<Point>contour;

    //double table[100][7];
    fstream humom("humombase.txt", ios::out);

    double hu[7];

    for (int i = 0; idx >= 0; idx = hierarchyoflet[idx][0], i++)
    {
        double area = contourArea(lettercontours[i]);
        Moments lettermoments = moments(lettercontours[i]);
        HuMoments(lettermoments, hu);
        humom << hu[0] << " " << hu[1] << " " << hu[2] << " " << hu[3] << " " << hu[4] << " " << hu[5] << " " << hu[6] << "\n";
        if (area > 500 && area < 1800 && hu[0] > 0.16292 && hu[0] < 0.18292 && hu[1]> 0.000534 && hu[1] < 0.000665 && hu[2]> 0.000162 && hu[2] < 0.000339)
        { 
            cout << "D"; 
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else if (area > 500 && area < 1800 && hu[0] > 0.257 && hu[0] < 0.283 && hu[1]> 0.0423 && hu[1] < 0.0473 && hu[2]> 1.044e-05 && hu[2] < 8.475e-05)
        {
            cout << "I";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
         else if (area > 500 && area < 1800 && hu[0] > 0.243 && hu[0] < 0.263 && hu[1]> 0.000837 && hu[1] < 0.000857 && hu[2]> 0.000136 && hu[2] < 0.000156)
        {
            cout << "S";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else if (area > 500 && area < 1800 && hu[0] > 0.287 && hu[0] < 0.3 && hu[1]> 0.009 && hu[1] < 0.015 && hu[2]> 0.019 && hu[2] < 0.021)
        {
            cout << "T";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else if (area > 500 && area < 1800 && hu[0] > 0.28 && hu[0] < 0.3 && hu[1]> 0.002 && hu[1] < 0.003 && hu[2]> 0.0005 && hu[2] < 0.0007)
        {
            cout << "U";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else if (area > 500 && area < 1800 && hu[0] > 0.191 && hu[0] < 0.197 && hu[1]> 0.0001 && hu[1] < 0.005 && hu[2]> 4.39e-05 && hu[2] < 0.0003)
        {
            cout << "R";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else if (area > 500 && area < 1800 && hu[0] > 0.156 && hu[0] < 0.176 && hu[1]> 4.606e-05 && hu[1] < 4.806e-05 && hu[2]> 4.89e-05 && hu[2] < 5e-05)
        {
            cout << "B";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else if (area > 500 && area < 1800 && hu[0] > 0.24 && hu[0] < 0.28 && hu[1]> 0.001 && hu[1] < 0.004 && hu[2]> 0 && hu[2] < 0.0002)
        {
            cout << "E";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else if (area > 500 && area < 1800 && hu[0] > 0.15 && hu[0] < 0.25 && hu[1]> 0.0004 && hu[1] < 0.0006 && hu[2]> 0.004 && hu[2] < 0.006)
        {
            cout << "A";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else if (area > 500 && area < 1800 && hu[0] > 0.33 && hu[0] < 0.35 && hu[1]> 0.02 && hu[1] < 0.04 && hu[2]> 0.008 && hu[2] < 0.01)
        {
            cout << "M";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else if (area > 500 && area < 1800 && hu[0] > 0.29 && hu[0] < 0.32 && hu[1]> 0.027 && hu[1] < 0.034 && hu[2]> 0.009 && hu[2] < 0.015)
        {
            cout << "L";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else if (area > 500 && area < 1800 && hu[0] > 0.23 && hu[0] < 0.25 && hu[1]> 0.0009 && hu[1] < 0.0015 && hu[2]> 0.0005 && hu[2] < 0.0007)
        {
            cout << "K";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else if (area > 500 && area < 1800 && hu[0] > 0.27 && hu[0] < 0.29 && hu[1]> 0.0002 && hu[1] < 0.0003 && hu[2]> 0.019 && hu[2] < 0.021)
        {
            cout << "Y";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else if (area > 500 && area < 1800 && hu[0] > 0.15 && hu[0] < 0.17 && hu[1]> 0.001 && hu[1] < 0.002 && hu[2]> 2.2e-07 && hu[2] < 2.5e-07)
        {
            cout << "O";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else if (area > 500 && area < 1800 && hu[0] > 0.28 && hu[0] < 0.30 && hu[1]> 0.022 && hu[1] < 0.024 && hu[2]> 0.002 && hu[2] < 0.0025)
        {
            cout << "W";
            drawContours(transformed2, lettercontours, idx, Scalar(0, 255, 255), 2, 8, hierarchyoflet, 3);
        }
        else
        {
            drawContours(transformed, lettercontours, idx, Scalar(0,0,0), CV_FILLED, 8, hierarchyoflet);
        }
    }

    Mat element = getStructuringElement(MORPH_CROSS, Size(3, 3), Point(1, 1));
    for (int o = 0; o < 1; o++)
    {
        erode(transformed, erodedimg, element);
        dilate(erodedimg, dilatedimg, element);
        transformed = dilatedimg;
    }

    imshow("Display window", transformed);
    waitKey(0);
    imshow("Display window", transformed2);

    namedWindow("Display window2", WINDOW_NORMAL);
    Point p;
    setMouseCallback("Display window2", on_mouse, &p); //set the callback function for any mouse event
    imshow("Display window2", transformed2);
    waitKey(0); //wait till any key is pressed 
    return 0;
}

Answer 1

I would start with detecting the grid.

1. find 4 whole corner cells

   

   so let (x1,y1)...(x4,y4) be the center points of found full edges forming rectangle. and nx=12 , ny=12 are distance between these points in cells. Let assume (x1,y1) has grid position (1,1) and (x2,y2) (1+nx,1) so:

   screen(x,y) grid(i,j)
   x1,y1        1, 1
   x2,y2       13, 1
   x3,y3        1,13
   x4,y4       13,13
   

2. interpolate/fit the grid lines

   So for cell (i,j) to screen (x,y) conversion we can write:

   a=x1+(x3-x1)*(j-j1); // x position of (i1,j)
   b=x2+(x4-x2)*(j-j2); // x position of (i2,j)
   x=a +(b - a)*(i-i1)/(i2-i1) // x position of (i,j)
   a=y1+(y2-y1)*(i-i1); // y position of (i,j1)
   b=y3+(y4-y3)*(i-i3); // y position of (i,j3)
   y=a +(b - a)*(j-j1)/(j3-j1) // x position of (i,j)
   

   hope I did not make any mistake (as I derived this directly in SO editor)... Now you know each cell center position. The corners of cell (i,j) are computed with this conversion by using (i+/-0.5,j+/-0.5) coordinates ...

   This is simple bi-linear interpolation, you can use also bi-cubic or any other if the distortion is too big.

   In case you do not have full cells forming corners of rectangle you can use 4 lines covering as large area as possible instead. The equations would change a bit but it is doable.

3. create function to convert between map and screen position

   beware empty cell is not the same as filled cell which is offset above the ground board. If you need also the reverse conversion (x,y)->(i,j) then you can do it in the same manner as #2 just swap i,j and x,y ...

You can use this to look for contours only in the desired cell instead of whole image eliminating the problem with contour grouping.

You can also cut of the bottom right part of each cell to ignore the score of numbers from contours. I do not use OpenCV so if you got problems to include this into OpenCV code you can always generate ROI map from this which should be easily usable in OpenCV to select specific area for any operation.

Of coarse for the detection of characters you need some kind of OCR which is not part of OpenCV (at least to my knowledge). There are few open source OCR's out there like Tesseract. You can also try something much simpler like this related QA:

• OCR and character similarity