Template Matching with template update

Question

I am trying to implement real-time tracking using templates in OpenCV/C++. I am facing problem to update the template with every frame.

Following is the code:

#include <iostream>
#include "opencv2/opencv.hpp"
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/objdetect/objdetect.hpp>

#include <sstream>


using namespace cv;
using namespace std;

Point point1, point2; /* vertical points of the bounding box */
int drag = 0;
Rect rect; /* bounding box */
Mat img, roiImg; /* roiImg - the part of the image in the bounding box */
int select_flag = 0;
bool go_fast = false;

Mat mytemplate;


///------- template matching -----------------------------------------------------------------------------------------------

Mat TplMatch( Mat &img, Mat &mytemplate )
{
  Mat result;

  matchTemplate( img, mytemplate, result, CV_TM_SQDIFF_NORMED );
  normalize( result, result, 0, 1, NORM_MINMAX, -1, Mat() );

  return result;
}


///------- Localizing the best match with minMaxLoc ------------------------------------------------------------------------

Point minmax( Mat &result )
{
  double minVal, maxVal;
  Point  minLoc, maxLoc, matchLoc;

  minMaxLoc( result, &minVal, &maxVal, &minLoc, &maxLoc, Mat() );
  matchLoc = minLoc;

  return matchLoc;
}


///------- tracking --------------------------------------------------------------------------------------------------------

void track()
{
    if (select_flag)
    {
        roiImg.copyTo(mytemplate);
//         select_flag = false;   //select_flag is kept false so that new template can
        go_fast = true;           //copied to 'mytemplate' for each iteration
    }

//     imshow( "mytemplate", mytemplate ); waitKey(0);

    Mat result  =  TplMatch( img, mytemplate );
    Point match =  minmax( result );  //PROBLEM: "match" always returning same value!!!

    rectangle( img, match, Point( match.x + mytemplate.cols , match.y + mytemplate.rows ), CV_RGB(255, 255, 255), 0.5 );

    std::cout << "match: " << match << endl;

    /// template update step
    Rect ROI = cv::Rect( match.x, match.y, mytemplate.cols, mytemplate.rows );
    roiImg = img( ROI );
    imshow( "roiImg", roiImg ); //waitKey(0);
}


///------- MouseCallback function ------------------------------------------------------------------------------------------

void mouseHandler(int event, int x, int y, int flags, void *param)
{
    if (event == CV_EVENT_LBUTTONDOWN && !drag)
    {
        /// left button clicked. ROI selection begins
        point1 = Point(x, y);
        drag = 1;
    }

    if (event == CV_EVENT_MOUSEMOVE && drag)
    {
        /// mouse dragged. ROI being selected
        Mat img1 = img.clone();
        point2 = Point(x, y);
        rectangle(img1, point1, point2, CV_RGB(255, 0, 0), 3, 8, 0);
        imshow("image", img1);
    }

    if (event == CV_EVENT_LBUTTONUP && drag)
    {
        point2 = Point(x, y);
        rect = Rect(point1.x, point1.y, x - point1.x, y - point1.y);
        drag = 0;
        roiImg = img(rect);
    }

    if (event == CV_EVENT_LBUTTONUP)
    {
        /// ROI selected
        select_flag = 1;
        drag = 0;
    }

}



///------- Main() ----------------------------------------------------------------------------------------------------------

int main()
{
    int k;

    ///open video file
    VideoCapture cap;
    cap.open( "Megamind.avi" );
    if ( !cap.isOpened() )
    {   cout << "Unable to open video file" << endl;    return -1;    }

    cap >> img;
    GaussianBlur( img, img, Size(7,7), 3.0 );
    imshow( "image", img );

    while (1)
    {
        cap >> img;
        if ( img.empty() )
            break;

        // Flip the frame horizontally and add blur
        cv::flip( img, img, 1 );
        GaussianBlur( img, img, Size(7,7), 3.0 );

        if ( rect.width == 0 && rect.height == 0 )
            cvSetMouseCallback( "image", mouseHandler, NULL );
        else
            track();

        imshow("image", img);
        k = waitKey(go_fast ? 30 : 10000);
        if (k == 27)
            break;
    }

    return 0;
}

The updated template is not being tracked. I am not able to figure out why this is happening since I am updating my template (roiImg) with each iteration. The match value from minmax() function is returning the same value (coordinates) every-time. Test video is availbale at: Megamind Please look into it and guide ahead...thanks a lot!

EDIT: if you run the code (with the video) you will see the white bounding-box is always at the same position. This is because, the minmax() is returning same "match" value all the time. This value should change with every update. Try running code with select_flag = false; (uncommitted). The bounding-box is moving according to template. But in this case no template update takes place.

Answer 1

The problem is in this section:

if (select_flag)
{
    roiImg.copyTo(mytemplate);
    // select_flag = false;   //select_flag is kept false so that new template can
    // ^^^^^^^^^^^ WRONG
    go_fast = true;           //copied to 'mytemplate' for each iteration
}

You actually need to set select_flag to be false on the first iteration. Otherwise, you're just copying what's in the current image on that frame into your template, and of course you find it in exactly the same place!

Once that's done, make sure you move your template update to after the tracking is done on that frame. I'd also recommend not drawing on the original image (your rectangle) until after all image accesses are done. You were actually drawing the rectangle into your image before copying out of it. Here's my adjusted function with template update:

void track()
{
  std::cout << select_flag << std::endl;

    if (select_flag)
    {
        roiImg.copyTo(mytemplate);
        select_flag = false;   //select_flag is kept false so that new template can
        go_fast = true;           //copied to 'mytemplate' for each iteration

    }

//     imshow( "mytemplate", mytemplate ); waitKey(0);

    Mat result  =  TplMatch( img, mytemplate );

    imshow("match", result);

    Point match =  minmax( result );  //PROBLEM: "match" always returning same value!!!

    std::cout << "match: " << match << endl;

    /// template update step
    Rect ROI = cv::Rect( match.x, match.y, mytemplate.cols, mytemplate.rows );

    std::cout << ROI << std::endl;

    roiImg = img( ROI );
    imshow( "roiImg", roiImg ); //waitKey(0);

    // Update the template AFTER tracking has occurred to carry it over to the next frame
    roiImg.copyTo(mytemplate);
    imshow("mytemplate", mytemplate);

    // Draw onto the image AFTER all accesses are performed
    rectangle( img, match, Point( match.x + mytemplate.cols , match.y + mytemplate.rows ), CV_RGB(255, 255, 255), 0.5 );
}