C++ Capture Full screen in real time

Question

I'm developing a software that should capture what is happening on the screen to make some processing. One of the requirements is that the software has to run at least 30FPS.

I've tried several options and I'm going to shopw you two, But none of them fully works to my needs:

(1) Using Direct X - The problem with this approach is that I cannot run it at 30 FPS (I get between 15 and 20):

DirectXScreenCapturer.h:

#include "IScreenCapturer.h"

#include <Wincodec.h>             // we use WIC for saving images
#include <d3d9.h>                 // DirectX 9 header
#pragma comment(lib, "d3d9.lib")  // link to DirectX 9 library

class DirectXScreenCapturer : public IScreenCapturer
{
public:
    DirectXScreenCapturer();
    ~DirectXScreenCapturer();

    virtual bool CaptureScreen(cv::Mat&) override;

private:
    IDirect3D9* _d3d;
    IDirect3DDevice9* _device;
    IDirect3DSurface9* _surface;

    cv::Mat _screen;
};

DirectXScreenCapturer.cpp:

#include "DirectXScreenCapturer.h"


DirectXScreenCapturer::DirectXScreenCapturer() : _d3d(NULL), _device(NULL), _surface(NULL)
{
    HRESULT hr = S_OK;
    D3DPRESENT_PARAMETERS parameters = { 0 };
    D3DDISPLAYMODE mode;
    D3DLOCKED_RECT rc;
    LPBYTE *shots = nullptr;

    // init D3D and get screen size
    _d3d = Direct3DCreate9(D3D_SDK_VERSION);
    _d3d->GetAdapterDisplayMode(D3DADAPTER_DEFAULT, &mode);

    parameters.Windowed = TRUE;
    parameters.BackBufferCount = 1;
    parameters.BackBufferHeight = mode.Height;
    parameters.BackBufferWidth = mode.Width;
    parameters.SwapEffect = D3DSWAPEFFECT_DISCARD;
    parameters.hDeviceWindow = NULL;

    // create device & capture surface
    hr = _d3d->CreateDevice(D3DADAPTER_DEFAULT, _D3DDEVTYPE::D3DDEVTYPE_HAL, NULL, D3DCREATE_HARDWARE_VERTEXPROCESSING | D3DCREATE_DISABLE_PSGP_THREADING | D3DCREATE_PUREDEVICE, &parameters, &_device);
    hr = _device->CreateOffscreenPlainSurface(mode.Width, mode.Height, D3DFMT_A8R8G8B8, D3DPOOL_SYSTEMMEM, &_surface, nullptr);

    // compute the required buffer size
    hr = _surface->LockRect(&rc, NULL, 0);
    hr = _surface->UnlockRect();

    // allocate screenshots buffers
    _screen = cv::Mat(mode.Height, mode.Width, CV_8UC4, rc.pBits);
}

DirectXScreenCapturer::~DirectXScreenCapturer()
{
}



bool DirectXScreenCapturer::CaptureScreen(cv::Mat& result)
{
    _device->GetFrontBufferData(0, _surface);
    result = _screen;
    return true;
}

(2) I've tried DXGI with Direct X. This solution works really well in Real Time, but fails to capture the screen when another application is running in full screen, so the software doesn't work if I'm watching movies, playign games, etc:

DXGIScreenCapturer.h

#include "IScreenCapturer.h"
#include <DXGI1_2.h>
#pragma comment(lib, "Dxgi.lib")
#include <D3D11.h>
#pragma comment(lib, "D3D11.lib")


class DXGIScreenCapturer : public IScreenCapturer
{
public:
    DXGIScreenCapturer();
    ~DXGIScreenCapturer();

    bool Init();
    virtual bool CaptureScreen(cv::Mat&) override;

private:
    ID3D11Device* _lDevice;
    ID3D11DeviceContext* _lImmediateContext;
    IDXGIOutputDuplication* _lDeskDupl;
    ID3D11Texture2D* _lAcquiredDesktopImage;
    DXGI_OUTPUT_DESC _lOutputDesc;
    DXGI_OUTDUPL_DESC _lOutputDuplDesc;
    D3D11_MAPPED_SUBRESOURCE _resource;
    ID3D11Texture2D* currTexture;

    cv::Mat _result;
};

DXGIScreenCapturer.cpp

#include "DXGIScreenCapturer.h"


DXGIScreenCapturer::DXGIScreenCapturer()
{
    Init();
}

DXGIScreenCapturer::~DXGIScreenCapturer()
{
}



bool DXGIScreenCapturer::Init() {
    // Feature levels supported
    D3D_FEATURE_LEVEL gFeatureLevels[] = {
        D3D_FEATURE_LEVEL_11_0,
        D3D_FEATURE_LEVEL_10_1,
        D3D_FEATURE_LEVEL_10_0,
        D3D_FEATURE_LEVEL_9_1
    };
    UINT gNumFeatureLevels = ARRAYSIZE(gFeatureLevels);
    D3D_FEATURE_LEVEL lFeatureLevel;

    HRESULT hr(E_FAIL);
    hr = D3D11CreateDevice(nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, D3D11_CREATE_DEVICE_FLAG::D3D11_CREATE_DEVICE_SINGLETHREADED, gFeatureLevels, gNumFeatureLevels, D3D11_SDK_VERSION, &_lDevice, &lFeatureLevel, &_lImmediateContext);

    if (FAILED(hr))
        return false;

    if (!_lDevice)
        return false;

    // Get DXGI device
    IDXGIDevice* lDxgiDevice;
    hr = _lDevice->QueryInterface(__uuidof(IDXGIDevice), reinterpret_cast<void**>(&lDxgiDevice));
    if (FAILED(hr))
        return false;

    // Get DXGI adapter
    IDXGIAdapter* lDxgiAdapter;
    hr = lDxgiDevice->GetParent(__uuidof(IDXGIAdapter), reinterpret_cast<void**>(&lDxgiAdapter));
    lDxgiDevice->Release();
    lDxgiDevice = nullptr;
    if (FAILED(hr))
        return false;

    UINT Output = 0;
    // Get output
    IDXGIOutput* lDxgiOutput;
    hr = lDxgiAdapter->EnumOutputs(Output, &lDxgiOutput);

    if (FAILED(hr))
        return false;
    lDxgiAdapter->Release();
    lDxgiAdapter = nullptr;

    hr = lDxgiOutput->GetDesc(&_lOutputDesc);

    if (FAILED(hr))
        return false;

    // QI for Output 1
    IDXGIOutput1* lDxgiOutput1;
    hr = lDxgiOutput->QueryInterface(__uuidof(lDxgiOutput1), reinterpret_cast<void**>(&lDxgiOutput1));
    lDxgiOutput->Release();
    lDxgiOutput = nullptr;
    if (FAILED(hr))
        return false;

    // Create desktop duplication
    hr = lDxgiOutput1->DuplicateOutput(_lDevice, &_lDeskDupl);

    if (FAILED(hr))
        return false;

    lDxgiOutput1->Release();
    lDxgiOutput1 = nullptr;

    // Create GUI drawing texture
    _lDeskDupl->GetDesc(&_lOutputDuplDesc);
    // Create CPU access texture
    D3D11_TEXTURE2D_DESC desc;
    desc.Width = _lOutputDuplDesc.ModeDesc.Width;
    desc.Height = _lOutputDuplDesc.ModeDesc.Height;
    desc.Format = _lOutputDuplDesc.ModeDesc.Format;
    desc.ArraySize = 1;
    desc.BindFlags = 0;
    desc.MiscFlags = 0;
    desc.SampleDesc.Count = 1;
    desc.SampleDesc.Quality = 0;
    desc.MipLevels = 1;
    desc.CPUAccessFlags = D3D11_CPU_ACCESS_FLAG::D3D11_CPU_ACCESS_READ;
    desc.Usage = D3D11_USAGE::D3D11_USAGE_STAGING;

    hr = _lDevice->CreateTexture2D(&desc, NULL, &currTexture);
    if (!currTexture)
    {
        hr = _lDeskDupl->ReleaseFrame();
        return false;
    }

    while (!CaptureScreen(_result));
    _result = cv::Mat(desc.Height, desc.Width, CV_8UC4, _resource.pData);
    return true;
}

bool DXGIScreenCapturer::CaptureScreen(cv::Mat& output)
{
    HRESULT hr(E_FAIL);
    IDXGIResource* lDesktopResource = nullptr;
    DXGI_OUTDUPL_FRAME_INFO lFrameInfo;

    hr = _lDeskDupl->AcquireNextFrame(999, &lFrameInfo, &lDesktopResource);

    if (FAILED(hr))
        return false;

    if (lFrameInfo.LastPresentTime.HighPart == 0) // not interested in just mouse updates, which can happen much faster than 60fps if you really shake the mouse
    {
        hr = _lDeskDupl->ReleaseFrame();
        return false;
    }

    // QI for ID3D11Texture2D
    hr = lDesktopResource->QueryInterface(__uuidof(ID3D11Texture2D), reinterpret_cast<void **>(&_lAcquiredDesktopImage));
    lDesktopResource->Release();
    lDesktopResource = nullptr;
    if (FAILED(hr))
    {
        hr = _lDeskDupl->ReleaseFrame();
        return false;
    }

    _lImmediateContext->CopyResource(currTexture, _lAcquiredDesktopImage);
    UINT subresource = D3D11CalcSubresource(0, 0, 0);
    _lImmediateContext->Map(currTexture, subresource, D3D11_MAP_READ, 0, &_resource);
    _lImmediateContext->Unmap(currTexture, 0);
    hr = _lDeskDupl->ReleaseFrame();

    output = _result;
    return true;
}

Please note that IScreenCapturer is just and interface to quickly swap implementations and also note that the return result is a cv::Mat object (OpenCV to do the rest of processing).

Any help on this issue? I'm still trying to figure out a solution that can capture the screen at least 30 FPS and that can capture the whole screen even when an app is running at full screen.

Check this https://stackoverflow.com/questions/14148758/how-to-capture-the-desktop-in-opencv-ie-turn-a-bitmap-into-a-mat — Victor Gubin, Apr 07 '18 at 11:56
You *are* testing your code with an optimized build - right? Not a debug build. It can make quite a significant performance difference in some cases. — Jesper Juhl, Apr 07 '18 at 12:01
OBS Studio is an app for: screen-recording, streaming, etc; and is open source, check how they do it: https://obsproject.com/ Download > GitHub — Richard Critten, Apr 07 '18 at 12:22

C++ Capture Full screen in real time

0 Answers0