OpenCV Java : Card Extraction from Image

Question

I am trying to implement some image processing using OpenCV and Java to extract a card out of an image.

Following is my approach:

1. Convert to BGR image
2. Convert to GRAY image
3. Apply GaussianBlur
4. Apply Canny Edge detection
5. Dilate
6. Find contours
7. Find the largest contour
8. Find corners of the largest contour using approxPolyDP
9. Getting a top-down view of the cropped image along the largest contour

At step no 8, I am facing some issues, as I am not getting the appropriate corners/vertices. Following sample images shows the scenario :

The original Image

After edge detection and dilation. (What is to be done to get appropriate edges?? Here I've got broken edges. Could not get Hough transform working)

After finding vertices. (shown in green)

Following is the code :

System.loadLibrary( Core.NATIVE_LIBRARY_NAME );

         //load Image
         File input = new File("card4.png");
         BufferedImage image = ImageIO.read(input); 
         byte[] data = ((DataBufferByte) image.getRaster().getDataBuffer()).getData();

         //put read image to Mat
         mat = new Mat(image.getHeight(), image.getWidth(), CvType.CV_8UC3); //original Mat
         mat.put(0, 0, data);
         mat_f = new Mat(image.getHeight(), image.getWidth(), CvType.CV_8UC3); //for storing manipulated Mat

         //conversion to grayscale, blurring and edge detection
         Imgproc.cvtColor(mat, mat_f, Imgproc.COLOR_RGB2BGR);
         Imgproc.cvtColor(mat_f, mat_f, Imgproc.COLOR_RGB2GRAY);
         Imgproc.GaussianBlur(mat_f, mat_f, new Size(13,13), 0);             
         Imgproc.Canny(mat_f, mat_f, 300, 600, 5, true);
         Imgproc.dilate(mat_f, mat_f, new Mat(), new Point(-1, -1), 2);
         Imgcodecs.imwrite("D:\\JAVA\\Image_Proc\\CVTest1.jpg",mat_f);

         //finding contours
         List<MatOfPoint> contours = new ArrayList<MatOfPoint>();    
         Mat hierarchy = new Mat();
         Imgproc.findContours(mat_f, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);
         double maxArea=0;
         int maxAreaIdx=0;

         //finding largest contour
         for (int idx = 0; idx != contours.size(); ++idx)
         {
               Mat contour = contours.get(idx);
               double contourarea = Imgproc.contourArea(contour);
               if (contourarea > maxArea)
               {
                   maxArea = contourarea;
                   maxAreaIdx = idx;
               }

          }

            //Rect rect = Imgproc.boundingRect(contours.get(maxAreaIdx));
            //Imgproc.rectangle(mat, new Point(rect.x,rect.y), new Point(rect.x+rect.width,rect.y+rect.height),new Scalar(0,0,255),7);
           // mat = mat.submat(rect.y, rect.y + rect.height, rect.x, rect.x + rect.width);


          //Polygon approximation
          MatOfPoint2f approxCurve = new MatOfPoint2f();
          MatOfPoint2f oriCurve = new MatOfPoint2f(contours.get(maxAreaIdx).toArray());
          Imgproc.approxPolyDP(oriCurve, approxCurve, 6.0, true);

          //drawing red markers at vertices
          Point [] array = approxCurve.toArray();
          for(int i=0; i < array.length;i++) {
             Imgproc.circle(mat, array[i], 2, new Scalar(0, 255, 0), 5);
          }
          Imgcodecs.imwrite("D:\\JAVA\\Image_Proc\\CVTest.jpg",mat);

Seeking help in getting the appropriate corner vertices... Thanks in advance..

Answer 1

In order to archive the good result using your approach then your cards have to contain 4 corners. But i prefer to use the HoughLine approach for this task.

Step 1: Resize image for higher performance

Step 2: Edges detection

• Transform the image into gray scale
• Blur image to clear noises
• Edge detection using Canny filters

You can use the dilation for make the white bigger for the next step

Step 3: Find card's corners

• Find contour of image
• From the list of contour get the largest contour
• Get the convexHull of it
• Use approxPolyDP to simplify the convex hull (this should give a quadrilateral)
• From now you can draw contour to get the rectangle after restore scale
• From the quadrilateral you can get the 4 corners.
• Find Homography
• Warp the input image using the computed homography matrix

Here is sample code in Java

    // STEP 1: Resize input image to img_proc to reduce computation
    double ratio = DOWNSCALE_IMAGE_SIZE / Math.max(frame.width(), frame.height());
    Size downscaledSize = new Size(frame.width() * ratio, frame.height() * ratio);
    Mat dst = new Mat(downscaledSize, frame.type());
    Imgproc.resize(frame, dst, downscaledSize);
    Mat grayImage = new Mat();
    Mat detectedEdges = new Mat();
    // STEP 2: convert to grayscale
    Imgproc.cvtColor(dst, grayImage, Imgproc.COLOR_BGR2GRAY);
    // STEP 3: try to filter text inside document
    Imgproc.medianBlur(grayImage, detectedEdges, 9);
    // STEP 4: Edge detection
    Mat edges = new Mat();
    // Imgproc.erode(edges, edges, new Mat());
    // Imgproc.dilate(edges, edges, new Mat(), new Point(-1, -1), 1); // 1
    // canny detector, with ratio of lower:upper threshold of 3:1
    Imgproc.Canny(detectedEdges, edges, this.threshold.getValue(), this.threshold.getValue() * 3, 3, true);
    // STEP 5: makes the object in white bigger to join nearby lines
    Imgproc.dilate(edges, edges, new Mat(), new Point(-1, -1), 1); // 1
    Image imageToShow = Utils.mat2Image(edges);
    updateImageView(cannyFrame, imageToShow);
    // STEP 6: Compute the contours
    List<MatOfPoint> contours = new ArrayList<>();
    Imgproc.findContours(edges, contours, new Mat(), Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_SIMPLE);
    // STEP 7: Sort the contours by length and only keep the largest one
    MatOfPoint largestContour = getMaxContour(contours);
    // STEP 8: Generate the convex hull of this contour
    Mat convexHullMask = Mat.zeros(frame.rows(), frame.cols(), frame.type());
    MatOfInt hullInt = new MatOfInt();
    Imgproc.convexHull(largestContour, hullInt);
    MatOfPoint hullPoint = OpenCVUtil.getNewContourFromIndices(largestContour, hullInt);
    // STEP 9: Use approxPolyDP to simplify the convex hull (this should give a quadrilateral)
    MatOfPoint2f polygon = new MatOfPoint2f();
    Imgproc.approxPolyDP(OpenCVUtil.convert(hullPoint), polygon, 20, true);
    List<MatOfPoint> tmp = new ArrayList<>();
    tmp.add(OpenCVUtil.convert(polygon));
    restoreScaleMatOfPoint(tmp, ratio);
    Imgproc.drawContours(convexHullMask, tmp, 0, new Scalar(25, 25, 255), 2);
    // Image extractImageToShow = Utils.mat2Image(convexHullMask);
    // updateImageView(extractFrame, extractImageToShow);
    MatOfPoint2f finalCorners = new MatOfPoint2f();
    Point[] tmpPoints = polygon.toArray();
    for (Point point : tmpPoints) {
        point.x = point.x / ratio;
        point.y = point.y / ratio;
    }
    finalCorners.fromArray(tmpPoints);
    boolean clockwise = true;
    double currentThreshold = this.threshold.getValue();
    if (finalCorners.toArray().length == 4) {
        Size size = getRectangleSize(finalCorners);
        Mat result = Mat.zeros(size, frame.type());
        // STEP 10: Homography: Use findHomography to find the affine transformation of your paper sheet
        Mat homography = new Mat();
        MatOfPoint2f dstPoints = new MatOfPoint2f();
        Point[] arrDstPoints = { new Point(result.cols(), result.rows()), new Point(0, result.rows()), new Point(0, 0), new Point(result.cols(), 0) };
        dstPoints.fromArray(arrDstPoints);
        homography = Calib3d.findHomography(finalCorners, dstPoints);

        // STEP 11: Warp the input image using the computed homography matrix
        Imgproc.warpPerspective(frame, result, homography, size);
    }