Template matching from a screenshot of a window

Question

What I've done

I have a small template image which is meant to be used to find coordinates of matching subimages within a larger screenshot image. The screenshot itself is captured into a memory DC with the help of BitBlt, then converted into a cv::Mat via GetDIBits, like so:

HDC windowDc = GetWindowDC(hwndTarget);
HDC memDc = CreateCompatibleDC(windowDc);

// ...

HBITMAP hbmp = CreateCompatibleBitmap(windowDc, width, height);
SelectObject(memDc, hbmp);
BITMAPINFOHEADER bi =
{
    sizeof(BITMAPINFOHEADER), // biSize
    width,                    // biWidth
    -height,                  // biHeight
    1,                        // biPlanes
    32,                       // biBitCount
    BI_RGB,                   // biCompression
    0,                        // biSizeImage
    0,                        // biXPelsPerMeter
    0,                        // biYPelsPerMeter
    0,                        // biClrUser
    0                         // biClrImportant
};

// ...

BitBlt(memDc, 0, 0, width, height, windowDc, offsetX, offsetY, SRCCOPY);
matScreen.create(height, width, CV_8UC4);
GetDIBits(memDc, hbmp, 0, (UINT)height, matScreen.data, (BITMAPINFO*)&bi, DIB_RGB_COLORS);

This appears to work fine, and a quick imshow("Source", matScreen) displays the image correctly.

However...

Since this screenshot image has been created as a 32-bit RGB memory bitmap, and placed inside a cv::Mat using the CV_8UC4 flag, it fails several assertions in OpenCV (as well as outputs whacky results when utilizing several OpenCV methods). Most notably, matchTemplate always fails on the following line:

CV_Assert( (depth == CV_8U || depth == CV_32F) && type == _templ.type() && _img.dims() <= 2 );

I've tried matching up the depth flags of the screenshot and the template/result Mats, and the best case scenario is that I am able to display all 3 images, but the template matching doesn't work because I'm assuming the source (screenshot) image is displayed in color whilst the others are in grayscale. Other conversions either fail, or display strange results and fail the template matching (or simply raise an error)... And changing the screenshot image's Mat to use any other depth flag ends up displaying the image incorrectly.

Question

What can I do to utilize OpenCV's template matching with a screenshot taken in C++? Is there some sort of manual conversion I should be doing to the screenshot image, or something?

Code:

Screenshot code taken from: github.com/acdx/opencv-screen-capture
My main code: codeshare.io/vLio1

Answer 1

I have included my code for finding a Template image from your Desktop image. Hope this solves your problem!

#include <fstream>
#include <memory>
#include <string>
#include <iostream>
#include <strstream>
#include <functional>
#include <Windows.h>
#include <iostream>
#include <string>

using namespace std;
using namespace cv;

Mat hwnd2mat(HWND hwnd){

    HDC hwindowDC,hwindowCompatibleDC;

    int height,width,srcheight,srcwidth;
    HBITMAP hbwindow;
    Mat src;
    BITMAPINFOHEADER  bi;

    hwindowDC=GetDC(hwnd);
    hwindowCompatibleDC=CreateCompatibleDC(hwindowDC);
    SetStretchBltMode(hwindowCompatibleDC,COLORONCOLOR);  

    RECT windowsize;    // get the height and width of the screen
    GetClientRect(hwnd, &windowsize);

    srcheight = windowsize.bottom;
    srcwidth = windowsize.right;
    height = windowsize.bottom;  //change this to whatever size you want to resize to
    width = windowsize.right;

    src.create(height,width,CV_8UC4);

    // create a bitmap
    hbwindow = CreateCompatibleBitmap( hwindowDC, width, height);
    bi.biSize = sizeof(BITMAPINFOHEADER);    //http://msdn.microsoft.com/en-us/library/windows/window/dd183402%28v=vs.85%29.aspx
    bi.biWidth = width;    
    bi.biHeight = -height;  //this is the line that makes it draw upside down or not
    bi.biPlanes = 1;    
    bi.biBitCount = 32;    
    bi.biCompression = BI_RGB;    
    bi.biSizeImage = 0;  
    bi.biXPelsPerMeter = 0;    
    bi.biYPelsPerMeter = 0;    
    bi.biClrUsed = 0;    
    bi.biClrImportant = 0;

    // use the previously created device context with the bitmap
    SelectObject(hwindowCompatibleDC, hbwindow);
    // copy from the window device context to the bitmap device context
    StretchBlt( hwindowCompatibleDC, 0,0, width, height, hwindowDC, 0, 0,srcwidth,srcheight, SRCCOPY); //change SRCCOPY to NOTSRCCOPY for wacky colors !
    GetDIBits(hwindowCompatibleDC,hbwindow,0,height,src.data,(BITMAPINFO *)&bi,DIB_RGB_COLORS);  //copy from hwindowCompatibleDC to hbwindow

    // avoid memory leak
    DeleteObject (hbwindow); DeleteDC(hwindowCompatibleDC); ReleaseDC(hwnd, hwindowDC);

    return src;
}

bool NMultipleTemplateMatching(Mat mInput,Mat mTemplate,float Threshold,float Closeness,vector<Point2f> &List_Matches)
{
    Mat mResult;
    Size szTemplate= mTemplate.size();
    Size szTemplateCloseRadius((szTemplate.width/2)* Closeness,(szTemplate.height/2)* Closeness);

    matchTemplate(mInput, mTemplate, mResult, TM_CCOEFF_NORMED);
    threshold(mResult, mResult, Threshold, 1.0, THRESH_TOZERO);
    while (true) 
    {
        double minval, maxval ;
        Point minloc, maxloc;
        minMaxLoc(mResult, &minval, &maxval, &minloc, &maxloc);

        if (maxval >= Threshold)
        {
            List_Matches.push_back(maxloc);
            rectangle(mResult,Point2f(maxloc.x-szTemplateCloseRadius.width,maxloc.y-szTemplateCloseRadius.height),Point2f(maxloc.x+szTemplateCloseRadius.width,maxloc.y+szTemplateCloseRadius.height),Scalar(0),-1);
        }
        else
            break;
    }
    //imshow("reference", mDebug_Bgr);
    return true;
}


int main(int argc, char** argv)
{
    Mat mTemplate_Bgr,mTemplate_Gray;
    mTemplate_Bgr= imread("Template.png",1);
    imshow("mTemplate_Bgr",mTemplate_Bgr);

    HWND hDesktopWnd;
    hDesktopWnd=GetDesktopWindow();
    Mat mScreenShot= hwnd2mat(hDesktopWnd);
    Mat mSource_Gray,mResult_Bgr= mScreenShot.clone();


    float Threshold= 0.9;
    float Closeness= 0.9;
    vector<Point2f> List_Matches;

    cvtColor(mScreenShot,mSource_Gray,COLOR_BGR2GRAY);
    cvtColor(mTemplate_Bgr,mTemplate_Gray,COLOR_BGR2GRAY);

    namedWindow("Screen Shot",WINDOW_AUTOSIZE);
    imshow("Screen Shot",mSource_Gray);

    NMultipleTemplateMatching(mSource_Gray,mTemplate_Gray,Threshold,Closeness,List_Matches);

    for (int i = 0; i < List_Matches.size(); i++)
    {
        rectangle(mResult_Bgr,List_Matches[i],Point(List_Matches[i].x + mTemplate_Bgr.cols, List_Matches[i].y + mTemplate_Bgr.rows),Scalar(0,255,0), 2);
    }

    imshow("Final Results",mResult_Bgr);

    waitKey(0);

    return 0;
}   

Answer 2

To expand on Balaji's very useful answer, make sure that all the Mats being passed to the function are of the same type with the .type() function. You can change src.create(height,width,CV_8UC4); to the same type as template (or vice versa) and the error should be gone. Here's another version of the solution.