adjust corners and crop the image openCV

Question

I am using open CV,in IOS. I already detected the boundary of the paper sheet in an Image as show in image and , Now I have to drag these boundary line on touch for adjusting the crop frame. how we can adjust boundary line and how we can crop image inside the boundary?

This is possible in openCV or I use openGL for this?

@moosgummi : I call your method in below method

- (cv::Mat)finshWork:(cv::Mat &)image
{

Mat img0 =image;

Mat img1;
cvtColor(img0, img1, CV_RGB2GRAY);

// apply your filter
Canny(img1, img1, 100, 200);

// find the contours
vector< vector<cv::Point> > contours;
findContours(img1, contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_NONE);


// you could also reuse img1 here
Mat mask = Mat::zeros(img1.rows, img1.cols, CV_8UC1);

// CV_FILLED fills the connected components found
drawContours(mask, contours, -1, Scalar(255), CV_FILLED);


// let's create a new image now
Mat crop(img0.rows, img0.cols, CV_8UC3);

// set background to green
crop.setTo(Scalar(0,255,0));

// and copy the magic apple
img0.copyTo(crop, mask);

// normalize so imwrite(...)/imshow(...) shows the mask correctly!
normalize(mask.clone(), mask, 0.0, 255.0, CV_MINMAX, CV_8UC1);



std::vector<cv::Point> biggestContour = contours[contours.size()-1];

NSLog(@"%d",biggestContour[0].x);
NSLog(@"%d",biggestContour[0].y);

cv::Mat paperImage =[self getPaperAreaFromImage:image:biggestContour];


//return crop;
return paperImage;

}

Thanks All

Answer 1

After you got the corners you have to deskewing the paper and "extract" it to a new image.

You should do the following:

1. Sort corner points (the order matters; they must be in the same order in both vectors)
2. cv::getAffineTransform
3. cv::warpAffine

I wrote myself a helper function, which takes a std::vector with four cv::Point in it and sorts them in clockwise order beginning in the upper left. For more information on this topic take a look at these thread:

• Sort points in clockwise order?
• Sort Four Points in Clockwise Order
• Sorting of Points in 2D space

Another thing you should take into account is the size of the paper you want to extract. In my example I assume you're extracting a DIN A4 paper (210x297mm). Feel free to edit paperWidth and paperHeight inside my code.

Combining everything looks like this:

// Helper
cv::Point getCenter( std::vector<cv::Point> points ) {

    cv::Point center = cv::Point( 0.0, 0.0 );

    for( size_t i = 0; i < points.size(); i++ ) {
        center.x += points[ i ].x;
        center.y += points[ i ].y;
    }

    center.x = center.x / points.size();
    center.y = center.y / points.size();

    return center;

}

// Helper;
// 0----1
// |    |
// |    |
// 3----2
std::vector<cv::Point> sortSquarePointsClockwise( std::vector<cv::Point> square ) {

    cv::Point center = getCenter( square );

    std::vector<cv::Point> sorted_square;
    for( size_t i = 0; i < square.size(); i++ ) {
        if ( (square[i].x - center.x) < 0 && (square[i].y - center.y) < 0 ) {
            switch( i ) {
                case 0:
                    sorted_square = square;
                    break;
                case 1:
                    sorted_square.push_back( square[1] );
                    sorted_square.push_back( square[2] );
                    sorted_square.push_back( square[3] );
                    sorted_square.push_back( square[0] );
                    break;
                case 2:
                    sorted_square.push_back( square[2] );
                    sorted_square.push_back( square[3] );
                    sorted_square.push_back( square[0] );
                    sorted_square.push_back( square[1] );
                    break;
                case 3:
                    sorted_square.push_back( square[3] );
                    sorted_square.push_back( square[0] );
                    sorted_square.push_back( square[1] );
                    sorted_square.push_back( square[2] );
                    break;
            }
            break;
        }
    }

    return sorted_square;

}

// Helper
float distanceBetweenPoints( cv::Point p1, cv::Point p2 ) {

    if( p1.x == p2.x ) {
        return abs( p2.y - p1.y );
    }
    else if( p1.y == p2.y ) {
        return abs( p2.x - p1.x );
    }
    else {
        float dx = p2.x - p1.x;
        float dy = p2.y - p1.y;
        return sqrt( (dx*dx)+(dy*dy) );
    }
}

cv::Mat getPaperAreaFromImage( cv::Mat image, std::vector<cv::Point> square )
{

    // declare used vars
    int paperWidth  = 210; // in mm, because scale factor is taken into account
    int paperHeight = 297; // in mm, because scale factor is taken into account
    cv::Point2f imageVertices[4];
    float distanceP1P2;
    float distanceP1P3;
    BOOL isLandscape = true;
    int scaleFactor;
    cv::Mat paperImage;
    cv::Mat paperImageCorrected;
    cv::Point2f paperVertices[4];

    // sort square corners for further operations
    square = sortSquarePointsClockwise( square );

    // rearrange to get proper order for getPerspectiveTransform()
    imageVertices[0] = square[0];
    imageVertices[1] = square[1];
    imageVertices[2] = square[3];
    imageVertices[3] = square[2];

    // get distance between corner points for further operations
    distanceP1P2 = distanceBetweenPoints( imageVertices[0], imageVertices[1] );
    distanceP1P3 = distanceBetweenPoints( imageVertices[0], imageVertices[2] );

    // calc paper, paperVertices; take orientation into account
    if ( distanceP1P2 > distanceP1P3 ) {
        scaleFactor =  ceil( lroundf(distanceP1P2/paperHeight) ); // we always want to scale the image down to maintain the best quality possible
        paperImage = cv::Mat( paperWidth*scaleFactor, paperHeight*scaleFactor, CV_8UC3 );
        paperVertices[0] = cv::Point( 0, 0 );
        paperVertices[1] = cv::Point( paperHeight*scaleFactor, 0 );
        paperVertices[2] = cv::Point( 0, paperWidth*scaleFactor );
        paperVertices[3] = cv::Point( paperHeight*scaleFactor, paperWidth*scaleFactor );
    }
    else {
        isLandscape = false;
        scaleFactor =  ceil( lroundf(distanceP1P3/paperHeight) ); // we always want to scale the image down to maintain the best quality possible
        paperImage = cv::Mat( paperHeight*scaleFactor, paperWidth*scaleFactor, CV_8UC3 );
        paperVertices[0] = cv::Point( 0, 0 );
        paperVertices[1] = cv::Point( paperWidth*scaleFactor, 0 );
        paperVertices[2] = cv::Point( 0, paperHeight*scaleFactor );
        paperVertices[3] = cv::Point( paperWidth*scaleFactor, paperHeight*scaleFactor );
    }

    cv::Mat warpMatrix = getPerspectiveTransform( imageVertices, paperVertices );
    cv::warpPerspective(_image, paperImage, warpMatrix, paperImage.size(), cv::INTER_LINEAR, cv::BORDER_CONSTANT );

    // we want portrait output
    if ( isLandscape ) {
        cv::transpose(paperImage, paperImageCorrected);
        cv::flip(paperImageCorrected, paperImageCorrected, 1);
        return paperImageCorrected;
    }

    return paperImage;

}

Usage:

// ... get paper square ...

cv::Mat paperImage = getPaperAreaFromImage( srcImage, paperSquare );

Answer 2

What you should do is :

1. Feed the 4 corners that you have found and the 4 real corners of the image to cv::getPerspectiveTransform. It will give you a matrix of the perspective transformation that will warp the quadrangle to the whole image.

2. Use cv::WarpPerspective to create the image you want.

The links will take you to the documentation.

EDIT : You could use cv::findHomography to do step 1. But this is more about having a lot of corresponding points and outliers.

EDIT : Here is an example. It is with the C interface but you could easily make it to work with the c++

#include <stdio.h>
#include "highgui.h"
#include "cv.h"

int main( int argc, char** argv ) {
    // cvLoadImage determines an image type and creates datastructure with appropriate size
    IplImage* img = cvLoadImage( argv[1], CV_LOAD_IMAGE_COLOR);
    IplImage* img1 = cvCreateImage(
            cvSize(img->width, img->height),
            img->depth,
            img->nChannels
            );

    cvNamedWindow( "out", CV_WINDOW_AUTOSIZE );
    cvShowImage( "out", img1 );
    // create a window. Window name is determined by a supplied argument
    cvNamedWindow( argv[1], CV_WINDOW_AUTOSIZE );
    // Display an image inside and window. Window name is determined by a supplied argument
    cvShowImage( argv[1], img );

    // The part you need
    // Here is the points that you take the image from (the small quadrangle)
    CvPoint2D32f first[4] = {
      {0,0},
      {(img->width /4)* 3, img->height /4    },
      { img->width /4    ,(img->height /4) *3},
      {(img->width /4)* 3,(img->height /4) *3},
    };
    // Here are the points that you draw the quadrangle into (the four corners)
    CvPoint2D32f second[4] = {
      {0,0},
      {img->width,0},
      {0,img->height},
      {img->width,img->height}
    };
    // The part you need
    CvMat *transform = cvCreateMat(3,3, CV_32F);
    cvGetPerspectiveTransform(first,second, transform);
    cvWarpPerspective(img, img1, transform, CV_INTER_LINEAR+CV_WARP_FILL_OUTLIERS,
        cvScalarAll(0));
    // End of part you need

    cvShowImage( "out", img1 );


    // wait indefinitely for keystroke
    cvWaitKey(0);
    // release pointer to an object
    cvReleaseImage( &img );
    // Destroy a window
    cvDestroyWindow( argv[1] );
}

You should replace the array first with end points of the quadrangle you have found.

EDIT : Here are some samples. I haven't looked them very well.

Geometric Image Transformations

cvGetPerspectiveTransform