Open CV tic tac toe X / O detection

Question

I made a program to make graphic interface while someone playing tic tac toe ( on video). A Problem is how to determine X or O? I tried to use contour area but sometimes it isn't a closed line so it gives something small as maximum contour area. Another idea I tried is to use hierarchy and if it has more than 3 it is O, but it gives some bad results. I can't use erosionas it eats my object too. My code:

struct myclass {
    bool operator() (Vec4i l1, Vec4i l2) { return (l1[0] < l2[0]); }
} myobjectv;

struct myclass1 {
    bool operator() (Vec4i l1, Vec4i l2) { return (l1[1] < l2[1]); }
} myobjecth;


using Contour = std::vector<cv::Point>;


void filtriranje(vector<Vec4i> lines, vector<Vec4i> &v_lines, vector<Vec4i> &h_lines){
for (size_t i = 0; i < lines.size(); i++)
    {
        Vec4i l = lines[i];
        double Angle = atan2(l[3] - l[1], l[2] - l[0]) * 180.0 / CV_PI;

        //vertikalne
        if ((abs(Angle) > 85) && (abs(Angle) <=92)){
            v_lines.push_back(lines[i]);

    }
    //horizontalne
    else if ((abs(Angle) >= 0) && (abs(Angle) <= 2)){
        h_lines.push_back(lines[i]);

    }

}



void uklanjanje_duplikata(vector<Vec4i> v_lines, vector<Vec4i> h_lines, vector<Vec4i> &vv_lines, vector<Vec4i> &hh_lines,Mat &cdst){
    int broj[10] = { 0 };
    if (v_lines.size() == 1){ vv_lines.push_back(v_lines[0]); line(cdst, Point(vv_lines[0][0], vv_lines[0][1]), Point(vv_lines[0][2], vv_lines[0][3]), Scalar(0, 255, 0), 3, CV_AA); }
    if (h_lines.size() == 1){ hh_lines.push_back(h_lines[0]); line(cdst, Point(hh_lines[0][0], hh_lines[0][1]), Point(hh_lines[0][2], hh_lines[0][3]), Scalar(0, 255, 0), 3, CV_AA); }
    sort(v_lines.begin(), v_lines.end(), myobjectv);
    if (v_lines.size() > 1){
        for (size_t i = 0; i < v_lines.size() - 1; i++)
        {
            for (size_t j = i + 1; j < v_lines.size(); j++)
            {
                Vec4i l1 = v_lines[i];
                Vec4i l2 = v_lines[j];
                if ((abs(l1[2] - l2[2]) < 47)){
                    if ((broj[i] == 0) && (broj[j] == 0))
                    {
                        Vec4i lnew;
                        lnew[0] =( l1[0] + l2[0])/2;
                        lnew[1] = (l1[1] + l2[1]) / 2;
                        lnew[2] = (l1[2] + l2[2]) / 2;
                        lnew[3] = (l1[3] + l2[3]) / 2;
                        vv_lines.push_back(lnew);
                        broj[i] = broj[i] + 1;
                        broj[j] = broj[j] + 1;
                    }

                }
                else{
                    if ((broj[i] == 0)){ vv_lines.push_back(l1); broj[i] = broj[i] + 1; }
                    if ((broj[j] == 0)){ vv_lines.push_back(l2); broj[j] = broj[j] + 1; }

                }
            }
        }
        for (size_t i = 0; i < vv_lines.size(); i++)
        {
            Vec4i lcr = vv_lines[i];
            line(cdst, Point(lcr[0], lcr[1]), Point(lcr[2], lcr[3]), Scalar(0, 255, 0), 3, CV_AA);
        }
    }
    int brojb[10] = { 0 };
    sort(h_lines.begin(), h_lines.end(), myobjecth);
    if (h_lines.size()>1){
        for (size_t i = 0; i < h_lines.size(); i++)
        {
            for (size_t j = i+1; j < h_lines.size() - 1; j++)
            {
                Vec4i l1 = h_lines[i];
                Vec4i l2 = h_lines[j];
                if ((abs(l1[1] - l2[1]) < 47)){
                    if ((brojb[i] == 0) && (brojb[j] == 0))
                    {
                        Vec4i lhnew;

                        hh_lines.push_back(l1);
                        brojb[i] = brojb[i]+1;
                        brojb[j] = brojb[j] + 1;
                    }

                }
                else{
                    if ((brojb[i] == 0)){ hh_lines.push_back(l1); brojb[i] = brojb[i] + 1; }
                    if ((brojb[j] == 0)){ hh_lines.push_back(l2); brojb[j] = brojb[j] + 1; }

                }
            }
        }
        for (size_t i = 0; i < hh_lines.size(); i++)
        {
            Vec4i lcr = hh_lines[i];
            line(cdst, Point(lcr[0], lcr[1]), Point(lcr[2], lcr[3]), Scalar(0, 255, 0), 3, CV_AA);
        }
    }


  }

void presek(vector<Vec4i> vv_lines, vector<Vec4i> hh_lines, vector<Point> &grid){
    Point P;
    int s = 0;
    for (size_t i = 0; i < hh_lines.size(); i++)
    {
        for (size_t j = 0; j < vv_lines.size(); j++)
        {
            Vec4i lb1 = hh_lines[i];
            Vec4i lb2 = vv_lines[j];
            float p1startx = lb1[0];
            float p1starty = lb1[1];
            float p1endx = lb1[2];
            float p1endy = lb1[3];
            float p2startx = lb2[0];
            float p2starty = lb2[1];
            float p2endx = lb2[2];
            float p2endy = lb2[3];

            if ((p2startx>=p1startx)&(p2startx <= p1endx)){
                P.x = p2startx;
                P.y = p1starty;
                grid.push_back(P);
                s++;

            }
        }
    }
}
void find_cross(vector<Vec4i> hh_lines, vector<Vec4i> vv_lines, vector<Point> grid, Point &cross1, Point &cross2, Point &cross3, Point &cross4){

    if ((hh_lines.size() == 2)&(vv_lines.size() == 2)){//kompletna mreza
        cross1 = grid[0];
        cross2 = grid[1];
        cross3 = grid[2];
        cross4 = grid[3];
        Vec4i lin = vv_lines[0];
        Vec4i linh = hh_lines[0];
        Vec4i linh2 = hh_lines[1];
        Vec4i linv2 = vv_lines[1];
        int g = linh[0];
        int s1 = lin[1];
        int p = cross1.x;
        int h = cross2.x;
        int cell_size = abs(p - h);

        //circle(cdst, cross1, 8, Scalar(0, 0, 255), 1, 8, 0);
        //circle(cdst, Point((cross4.x - cell_size), (cross4.y)), 8, Scalar(0, 0, 255), 1, 8, 0);
    }
    else if ((hh_lines.size() == 2)&(vv_lines.size() == 1)){//dve horizontalne i jedna vertikalna
        Vec4i lin = vv_lines[0];
        Vec4i linh = hh_lines[0];
        Vec4i linh2 = hh_lines[1];
        int flag[5] = { 0 };
        int p = linh[1];
        int h = linh2[1];
        int cell_size = abs(p - h);
        Point presek = grid[0];
        if (abs(presek.x - linh[0]) < abs(presek.x - linh[2])){ cross1 = grid[0]; flag[1] = 1; cross3 = grid[1]; flag[3] = 1; cross2.x = cross1.x + cell_size; cross2.y = cross1.y; cross4.x = cross2.x; cross4.y = cross3.y; }
        else { cross2 = grid[0]; flag[2] = 1; cross4 = grid[1]; flag[4] = 1; cross1.x = cross2.x - cell_size; cross1.y = cross2.y; cross3.x = cross4.x - cell_size; cross3.y = cross4.y; }


    }

    else if ((hh_lines.size() == 1)&(vv_lines.size() == 2)){//horizontalna i dve vertikalne
        Vec4i linv = vv_lines[0];
        Vec4i linv2 = vv_lines[1];
        Vec4i linh = hh_lines[0];
        int flag[5] = { 0 };
        int cell_size = abs(linv[0] - linv2[0]);
        Point presek = grid[0];
        int pocetak = linv[1];
        int kraj = linv[3];
        if (linv[3] > linv[1]){
            if (abs(presek.y - linv[3]) > abs(presek.y - linv[1])){ cross1 = grid[0]; flag[1] = 1; flag[2] = 1; cross2 = grid[1]; cross3.x = cross1.x; cross3.y = cross1.y + cell_size; cross4.x = cross2.x; cross4.y = cross2.y + cell_size; }
            else{ cross3 = grid[0]; flag[3] = 1; flag[4] = 1; cross4 = grid[1]; cross1.x = cross3.x; cross1.y = cross3.y - cell_size; cross2.x = cross4.x; cross2.y = cross4.y - cell_size; }
        }
        else{
            if (abs(presek.y - linv[3]) < abs(presek.y - linv[1])){ cross1 = grid[0]; flag[1] = 1; flag[2] = 1; cross2 = grid[1]; cross3.x = cross1.x; cross3.y = cross1.y + cell_size; cross4.x = cross2.x; cross4.y = cross2.y + cell_size; }
            else{ cross3 = grid[0]; flag[3] = 1; flag[4] = 1; cross4 = grid[1]; cross1.x = cross3.x; cross1.y = cross3.y - cell_size; cross2.x = cross4.x; cross2.y = cross4.y - cell_size; }
        }

        }
    else if ((hh_lines.size() == 1)&(vv_lines.size() == 1)){//jedna vertikalna jedna horizontalna

        Vec4i linv = vv_lines[0];
        Vec4i linh = hh_lines[0];
        Point presek = grid[0];
        int flag[5] = { 0 };


        int cell_size = 47;
        if (abs(linh[0] - presek.x) < abs(linh[2] - presek.x)){//leva linija vertikalna
            if (abs(linv[3] - presek.y) < abs(linv[1] - presek.y)){ cross1 = grid[0]; cross3.x = cross1.x; cross3.y = cross1.y + cell_size; cross2.x = cross1.x + cell_size; cross2.y = cross1.y; cross4.x = cross3.x + cell_size; cross4.y = cross3.y; }//gornja horizontala
            else { cross3 = presek; cross1.x = cross3.x; cross1.y = cross3.y - cell_size; cross2.x = cross1.x + cell_size; cross2.y = cross1.y; cross4.x = cross3.x + cell_size; cross4.y = cross3.y; }
        }

        else {//desna vertikalna
            int cell_size = 47;
            if (linv[1] > linv[3]){
                if (abs(linv[3] - presek.y) < abs(linv[1] - presek.y))
                {
                    cross2 = grid[0]; cross1.x = cross2.x - cell_size; cross1.y = cross2.y; cross3.x = cross1.x; cross3.y = cross1.y + cell_size; cross4.x = cross3.x + cell_size; cross4.y = cross3.y;
                }
                else{ cross4 = grid[0]; cross3.x = cross4.x - cell_size; cross3.y = cross4.y; cross2.x = cross4.x; cross2.y = cross4.y - cell_size; cross1.x = cross2.x - cell_size; cross1.y = cross2.y; }
            }
            else{
                if (abs(linv[3] - presek.y) > abs(linv[1] - presek.y))
                {
                    cross2 = grid[0]; cross1.x = cross2.x - cell_size; cross1.y = cross2.y; cross3.x = cross1.x; cross3.y = cross1.y + cell_size; cross4.x = cross3.x + cell_size; cross4.y = cross3.y;
                }
                else{ cross4 = grid[0]; cross3.x = cross4.x - cell_size; cross3.y = cross4.y; cross2.x = cross4.x; cross2.y = cross4.y - cell_size; cross1.x = cross2.x - cell_size; cross1.y = cross2.y; }


            }
        }
    }
}

void set_ROI(Point cross1, Point cross2, Point cross3, Point cross4, Mat dst, Mat cdst, int cell_size,Mat &roi1, Mat &roi2, Mat &roi3, Mat &roi4, Mat &roi5, Mat &roi6, Mat &roi7, Mat &roi8, Mat &roi9){

    roi1 = dst(Rect((cross1.x - cell_size), (cross1.y - cell_size), cell_size, cell_size));
    rectangle(cdst, Point((cross1.x - cell_size), (cross1.y - cell_size)), cross1, Scalar(0, 0, 255), 3, CV_AA);
    roi4 = dst(Rect((cross1.x - cell_size), cross1.y, cell_size, cell_size));
    rectangle(cdst, Point((cross1.x - cell_size), cross1.y), cross3, Scalar(0, 0, 255), 3, CV_AA);
    roi7 = dst(Rect((cross3.x - cell_size), cross3.y, cell_size, cell_size));
    rectangle(cdst, Point((cross3.x - cell_size), cross3.y), Point(cross3.x, cross3.y + cell_size), Scalar(0, 0, 255), 3, CV_AA);
    roi2 = dst(Rect(cross1.x, (cross1.y - cell_size), cell_size, cell_size));
    rectangle(cdst, Point(cross1.x, (cross1.y - cell_size)), cross2, Scalar(0, 0, 255), 3, CV_AA);
    roi5 = dst(Rect(cross1.x, cross1.y, cell_size, cell_size));
    rectangle(cdst, Point(cross1.x, cross1.y), cross4, Scalar(0, 0, 255), 3, CV_AA);
    roi8 = dst(Rect(cross3.x, cross3.y, cell_size, cell_size));
    rectangle(cdst, cross3, Point(cross4.x, cross4.y + cell_size), Scalar(0, 0, 255), 3, CV_AA);
    roi3 = dst(Rect(cross2.x, (cross2.y - cell_size), cell_size, cell_size));
    rectangle(cdst, Point(cross2.x, (cross2.y - cell_size)), Point(cross2.x + cell_size, cross2.y), Scalar(0, 0, 255), 3, CV_AA);
    roi6 = dst(Rect(cross2.x, cross2.y, cell_size, cell_size));
    rectangle(cdst, cross2, Point(cross4.x + cell_size, cross4.y), Scalar(0, 0, 255), 3, CV_AA);
    roi9 = dst(Rect(cross4.x, cross4.y, cell_size, cell_size));
    rectangle(cdst, cross4, Point(cross4.x + cell_size, cross4.y + cell_size), Scalar(0, 0, 255), 3, CV_AA);


}


int main(int argc, char** argv)
{

    //cvNamedWindow("Prvi Video");
    vector<Rect*> components(250, (Rect *)NULL);
    VideoCapture video1;
    int vektor, cell_size;

    int width, height, frames1, fps1;
    int popunjeno[9] = { 0 };
    int ukupno_komponenti[3][3] = { 0 };
    video1.open(argv[1]);
    fps1 = video1.get(CV_CAP_PROP_FPS);
    width = video1.get(CAP_PROP_FRAME_WIDTH);
    height = video1.get(CAP_PROP_FRAME_HEIGHT);
    frames1 = video1.get(CAP_PROP_FRAME_COUNT);

    vector<Point> preseci;
    vector<Vec4i> vertikale;
    vector<Vec4i> horizontale;

    cout << "Video1 " << argv[1] <<
        ": width=" << width <<
        ", height=" << height <<
        ", frames=" << frames1 <<
        ", fps1=" << fps1 << endl;


    int i = 0;
    int j = 0;




    Mat src, tmp, dst, cdst, krug;
    Mat frameTime1(height, width, CV_8UC3, Scalar(0, 0, 0));
    int fvd = 0;
    int r;
    while (1)
    {
        fvd++;

        cout << fvd;
        video1 >> src;
        bool bSuccess = video1.read(src);
        if (!bSuccess) //if not success, break loop
        {
            cout << "ERROR: Cannot read a frame from video file" << endl;
            break;
        }
        Mat roi_w1;
        roi_w1 = src(Rect(150, 50, 320, 320));
        GaussianBlur(roi_w1, roi_w1, Size(11, 11), 0);
        Canny(roi_w1, dst, 50, 200, 3);
        cvtColor(dst, cdst, CV_GRAY2BGR);
        double povrsina = 0;
        vector<Vec4i> lines;
        vector<Vec4i> h_lines;
        vector<Vec4i> v_lines;
        vector<Vec4i> hh_lines;
        vector<Vec4i> vv_lines;
        vector<Mat> ROI;
        Point cross1, cross2, cross3, cross4;

        Mat roi1, roi2, roi3, roi4, roi5, roi6, roi7, roi8, roi9;
        vector<Point> grid;
        if (fvd == 1){//prvi frejm
            HoughLinesP(dst, lines, 1, CV_PI / 220, 50, 150, 50);
            filtriranje(lines, v_lines, h_lines);
            uklanjanje_duplikata(v_lines, h_lines, vv_lines, hh_lines, cdst);
            if ((hh_lines.size() == 2)&(vv_lines.size() == 2)){

                presek(vv_lines, hh_lines, grid);//u grid su tacke preseka
                cell_size = abs(grid[0].x - grid[1].x);
                find_cross(hh_lines, vv_lines, grid, cross1, cross2, cross3, cross4);//potpuna rekonstrukcija mreze,resen problem ako nisu nadjene sve cetiri linije
                set_ROI(cross1, cross2, cross3, cross4, dst, cdst, cell_size, roi1, roi2, roi3, roi4, roi5, roi6, roi7, roi8, roi9);
                vertikale = vv_lines;
                horizontale = hh_lines;
                preseci.push_back(cross1);
                preseci.push_back(cross2);
                preseci.push_back(cross3);
                preseci.push_back(cross4);

            }
            else if ((hh_lines.size() == 1) & (vv_lines.size() == 2)) {
                presek(vv_lines, hh_lines, grid);//u grid su tacke preseka
                find_cross(hh_lines, vv_lines, grid, cross1, cross2, cross3, cross4);
                vertikale = vv_lines;
                horizontale = hh_lines;
                preseci.push_back(cross1);
                preseci.push_back(cross2);
                preseci.push_back(cross3);
                preseci.push_back(cross4);
                cell_size = abs(vv_lines[0][0] - vv_lines[1][0]);//potpuna rekonstrukcija mreze,resen problem ako nisu nadjene sve cetiri linije
                set_ROI(cross1, cross2, cross3, cross4, dst, cdst, cell_size, roi1, roi2, roi3, roi4, roi5, roi6, roi7, roi8, roi9);

            }
            else if ((hh_lines.size() == 2)& (vv_lines.size() == 1)){
                presek(vv_lines, hh_lines, grid);//u grid su tacke preseka
                find_cross(hh_lines, vv_lines, grid, cross1, cross2, cross3, cross4);
                cell_size = abs(hh_lines[0][1] - hh_lines[1][1]);//potpuna rekonstrukcija mreze,resen problem ako nisu nadjene sve cetiri linije
                set_ROI(cross1, cross2, cross3, cross4, dst, cdst, cell_size, roi1, roi2, roi3, roi4, roi5, roi6, roi7, roi8, roi9);
                vertikale = vv_lines;
                horizontale = hh_lines;
                preseci.push_back(cross1);
                preseci.push_back(cross2);
                preseci.push_back(cross3);
                preseci.push_back(cross4);

            }

            ROI.push_back(roi1);
            ROI.push_back(roi2);
            ROI.push_back(roi3);
            ROI.push_back(roi4);
            ROI.push_back(roi5);
            ROI.push_back(roi6);
            ROI.push_back(roi7);
            ROI.push_back(roi8);
            ROI.push_back(roi9);

        }
        vector<Rect*> components(250, (Rect *)NULL);
        Mat motion, motion2[3], f1, f2, f3, fout1, maska;
        absdiff(frameTime1, src, motion);
        split(motion, motion2);
        threshold(motion2[0], f1, 20, 255, CV_THRESH_BINARY);
        threshold(motion2[1], f2, 20, 255, CV_THRESH_BINARY);
        threshold(motion2[2], f3, 20, 255, CV_THRESH_BINARY);
        bitwise_or(f1, f2, fout1);
        bitwise_or(fout1, f3, maska);
        //morphologyEx(maska, maska, MORPH_OPEN, Mat());
        dilate(maska, maska, Mat());
        dilate(maska, maska, Mat());
        dilate(maska, maska, Mat());
        dilate(maska, maska, Mat());
        dilate(maska, maska, Mat());
        dilate(maska, maska, Mat());

        int c = countNonZero(maska);
        src.copyTo(frameTime1);
        //if (fvd == 138){
        if (c < 2000){//ruka nije u kadru
            vector<Vec4i>().swap(lines);
            vector<Vec4i>().swap(h_lines);
            vector<Vec4i>().swap(hh_lines);
            vector<Vec4i>().swap(v_lines);
            vector<Vec4i>().swap(vv_lines);
            vector<Mat>().swap(ROI);
            HoughLinesP(dst, lines, 1, CV_PI / 200, 50, 150, 30);
            filtriranje(lines, v_lines, h_lines);
            uklanjanje_duplikata(v_lines, h_lines, vv_lines, hh_lines, cdst);
            Mat roi1, roi2, roi3, roi4, roi5, roi6, roi7, roi8, roi9;
            vector<Point> grid;

            if ((hh_lines.size() == 0)&(vv_lines.size() == 2)){
                Vec4i ver1 = vv_lines[0];
                Vec4i ver2 = vv_lines[1];
                cell_size = 48;
                if (ver1[1] > ver1[3]){
                    cross1.x = ver1[2];
                    cross1.y = ver1[3] + cell_size;
                    cross2.x = ver2[2];
                    cross2.y = cross1.y;
                    cross3.x = cross1.x;
                    cross3.y = cross1.y + cell_size;
                    cross4.x = cross2.x;
                    cross4.y = cross1.y + cell_size;
                }
                else{
                    cross1.x = ver1[0];
                    cross1.y = ver1[1] + cell_size;
                    cross2.x = ver2[0];
                    cross2.y = cross1.y;
                    cross3.x = cross1.x;
                    cross3.y = cross1.y + cell_size;
                    cross4.x = cross2.x;
                    cross4.y = cross1.y + cell_size;

                }

                set_ROI(cross1, cross2, cross3, cross4, dst, cdst, cell_size, roi1, roi2, roi3, roi4, roi5, roi6, roi7, roi8, roi9);
            }
            else if ((vv_lines.size() == 0)&(hh_lines.size() == 2)){
                Vec4i h1 = hh_lines[0];
                Vec4i h2 = hh_lines[1];
                cross1.x = h1[0] + cell_size;
                cross1.y = h1[1];
                cross2.x = h1[0] + 2 * cell_size;
                cross2.y = h1[1];
                cross3.x = h1[0] + cell_size;
                cross3.y = h2[1];
                cross4.x = h2[0] + 2 * cell_size;
                cross4.y = h2[1];
                set_ROI(cross1, cross2, cross3, cross4, dst, cdst, cell_size, roi1, roi2, roi3, roi4, roi5, roi6, roi7, roi8, roi9);
            }
            else if ((hh_lines.size() == 2)&(vv_lines.size() == 2)){
                presek(vv_lines, hh_lines, grid);//u grid su tacke preseka
                find_cross(hh_lines, vv_lines, grid, cross1, cross2, cross3, cross4);//potpuna rekonstrukcija mreze,resen problem ako nisu nadjene sve cetiri linije
                set_ROI(cross1, cross2, cross3, cross4, dst, cdst, cell_size, roi1, roi2, roi3, roi4, roi5, roi6, roi7, roi8, roi9);
            }
            else if ((hh_lines.size() == 1) & (vv_lines.size() == 2)) {
                presek(vv_lines, hh_lines, grid);//u grid su tacke preseka
                find_cross(hh_lines, vv_lines, grid, cross1, cross2, cross3, cross4);//potpuna rekonstrukcija mreze,resen problem ako nisu nadjene sve cetiri linije
                set_ROI(cross1, cross2, cross3, cross4, dst, cdst, cell_size, roi1, roi2, roi3, roi4, roi5, roi6, roi7, roi8, roi9);
            }
            else if ((hh_lines.size() == 2)& (vv_lines.size() == 1)){
                presek(vv_lines, hh_lines, grid);//u grid su tacke preseka
                find_cross(hh_lines, vv_lines, grid, cross1, cross2, cross3, cross4);//potpuna rekonstrukcija mreze,resen problem ako nisu nadjene sve cetiri linije
                set_ROI(cross1, cross2, cross3, cross4, dst, cdst, cell_size, roi1, roi2, roi3, roi4, roi5, roi6, roi7, roi8, roi9);
            }
            else if ((hh_lines.size() == 2)&(vv_lines.size() > 2)){
                presek(vertikale, hh_lines, grid);//u grid su tacke preseka
                find_cross(hh_lines, vertikale, grid, cross1, cross2, cross3, cross4);//potpuna rekonstrukcija mreze,resen problem ako nisu nadjene sve cetiri linije
                set_ROI(cross1, cross2, cross3, cross4, dst, cdst, cell_size, roi1, roi2, roi3, roi4, roi5, roi6, roi7, roi8, roi9);
            }
            else if ((hh_lines.size() > 2) &(vv_lines.size() == 2)){
                presek(vv_lines, horizontale, grid);//u grid su tacke preseka
                find_cross(horizontale, vv_lines, grid, cross1, cross2, cross3, cross4);//potpuna rekonstrukcija mreze,resen problem ako nisu nadjene sve cetiri linije
                set_ROI(cross1, cross2, cross3, cross4, dst, cdst, cell_size, roi1, roi2, roi3, roi4, roi5, roi6, roi7, roi8, roi9);
            }
            else if ((hh_lines.size() == 1)& (vv_lines.size() == 1)){
                presek(vv_lines, hh_lines, grid);//u grid su tacke preseka
                find_cross(hh_lines, vv_lines, grid, cross1, cross2, cross3, cross4);//potpuna rekonstrukcija mreze,resen problem ako nisu nadjene sve cetiri linije
                set_ROI(cross1, cross2, cross3, cross4, dst, cdst, cell_size, roi1, roi2, roi3, roi4, roi5, roi6, roi7, roi8, roi9);
            }
            else{ set_ROI(preseci[0], preseci[1], preseci[2], preseci[3], dst, cdst, cell_size, roi1, roi2, roi3, roi4, roi5, roi6, roi7, roi8, roi9); }



            roi1 = roi1 > 128;
            roi2 = roi2 > 128;
            roi3 = roi3 > 128;
            roi4 = roi4 > 128;
            roi5 = roi5 > 128;
            roi6 = roi6 > 128;
            roi7 = roi7 > 128;
            roi8 = roi8 > 128;
            roi9 = roi9 > 128;

            int m = 0;
            ROI.push_back(roi1);
            ROI.push_back(roi2);
            ROI.push_back(roi3);
            ROI.push_back(roi4);
            ROI.push_back(roi5);
            ROI.push_back(roi6);
            ROI.push_back(roi7);
            ROI.push_back(roi8);
            ROI.push_back(roi9);
            int ks;
            int simbol = 0;
            if (r == 1){
                for (size_t k = 0; k < 9; k++)
                {
                    if (popunjeno[k] == 0){
                        vector<Contour> contour_vec;
                        vector<Vec4i> hierarchy;
                        vector<Vec4i> lines_junk;
                        Mat tmp, tmp1;

                        //tmp = ROI[k](Rect(15, 10, 30, 30));
                        //dilate(tmp, tmp, Mat());
                        ROI[k].copyTo(tmp);
                        ROI[k].copyTo(tmp1);
                        HoughLinesP(tmp, lines_junk, 1, CV_PI / 220, 30, 15, 10);
                        for (size_t i = 0; i < lines_junk.size(); i++)
                        {
                            Vec4i l = lines_junk[i];
                            line(tmp1, Point(l[0], l[1]), Point(l[2], l[3]), Scalar(0, 0, 0), 3, CV_AA);
                        }
                        int ukupno_komponenti;
                        tmp = tmp1(Rect(5, 5, 35, 35));
                        int largest_area = 0;
                        int largest_contour_index = 0;
                        findContours(tmp1, contour_vec, hierarchy, CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE);
                        if ((hierarchy.size() == 3)){
                            popunjeno[k] = 1;

                        }

                        else if (((hierarchy.size() > 3))){
                            popunjeno[k] = -1;

                        }


                    }



                }
                r = -1;
                //imshow("detected lines", cdst);
                //waitKey();
            }


        }
    //}
        else{ r = 1;    continue; }




    }

    return 0;



}

These are my cells:

Answer 1

The best will be to calculate a bottom projection for one of the images and by that determine if it is X or O: A bottom projection is simply an array of ints where each cell in index i contains the number of pixels in column i in the image from bottom to top until the first non black pixel.

for the X image you will get a projection that contains an ascending sequence and then a descending sequence, for the O image you will get a descending sequence and then ascending sequence.

to calculate the projection simply sum the number of pixels in each column untill you reach a non black pixel, because of the noise in the images you will need to omit from the projection array a value in cell i if that is greater or smaller by 2 or more from the value in cell i-1, also you need to omit the higher values at the start and at the end of the projection array because the are probably the sum of the full columns (sum from bottom to top).

remember to remove all first and last values from the projection - just leave ones lower then height/2 Check sequences:

    private bool IsDescending(int[] proj,int len)
    {
            for (int i = 0; i < len-1; i++)
            {
                if (proj[i] < proj[i + 1])
                    return false;
            }
            return true;
    }
    private bool IsAscending(int[] proj,int len)
    {
            for (int i = 0; i < len-1; i++)
            {
                if (proj[i] > proj[i + 1])
                    return false;
            }
            return true;
    }

Good luck.

Answer 2

I used to do that by comparing the contours detected, with contours that I already have to see which one is the closest, in shape (best match).

How? There is an OpenCV function which does that for us. It takes couple contours and it returns a metric showing their similarities (double value [0 to 1] I think). Therefore, if you have two images of X and O already, and you detected a shape, then you would just call that function on each of the images that you already have, and it will return two values. Then, you just compare these two values to see if it was O or X (the smaller the value is, the more similar the contours are).

Here: Check section 3. Match Shapes http://docs.opencv.org/3.1.0/d5/d45/tutorial_py_contours_more_functions.html#gsc.tab=0