Handle OCR / Computer Vision result to match a receipt structure

Question

I'm using Microsoft Computer Vision to read receipts.

The result I get is ordered into regions that are grouped by columns, e.g. quantities, product names, amount are in three different regions.

I would to prefer if the whole list of products is one region and that each line is a product.

Is there any way to configure the Computer Vision to accomplish this, or maybe more likely are there any good techniques or libraries that one can use in a post-processing of the result since the positions of all the words are available.

Bellow is the image of the receipt and the result from the computer vision.

{
  "language": "sv",
  "textAngle": 2.0999999999999632,
  "orientation": "Up",
  "regions": [
    {
      "boundingBox": "1012,450,660,326",
      "lines": [
        {
          "boundingBox": "1362,450,76,30",
          "words": [
            {
              "boundingBox": "1362,450,76,30",
              "text": "JULA"
            }
          ]
        },
        {
          "boundingBox": "1207,486,465,49",
          "words": [
            {
              "boundingBox": "1207,502,172,33",
              "text": "Ekslinsan"
            },
            {
              "boundingBox": "1400,497,51,30",
              "text": "3B,"
            },
            {
              "boundingBox": "1479,491,95,33",
              "text": "25467"
            },
            {
              "boundingBox": "1595,486,77,32",
              "text": "VALA"
            }
          ]
        },
        {
          "boundingBox": "1304,539,265,38",
          "words": [
            {
              "boundingBox": "1304,539,265,38",
              "text": "SE5S6944785601"
            }
          ]
        },
        {
          "boundingBox": "1245,584,369,44",
          "words": [
            {
              "boundingBox": "1245,594,148,34",
              "text": "Telefon:"
            },
            {
              "boundingBox": "1421,584,193,37",
              "text": "042-324040"
            }
          ]
        },
        {
          "boundingBox": "1012,695,269,35",
          "words": [
            {
              "boundingBox": "1012,702,75,28",
              "text": "Kund"
            },
            {
              "boundingBox": "1109,695,172,33",
              "text": "072202787"
            }
          ]
        },
        {
          "boundingBox": "1109,738,289,38",
          "words": [
            {
              "boundingBox": "1109,744,133,32",
              "text": "LILLVIK"
            },
            {
              "boundingBox": "1265,738,133,32",
              "text": "ANDREAS"
            }
          ]
        }
      ]
    },
    {
      "boundingBox": "1085,845,14,516",
      "lines": [
        {
          "boundingBox": "1090,845,9,29",
          "words": [
            {
              "boundingBox": "1090,845,9,29",
              "text": "1"
            }
          ]
        },
        {
          "boundingBox": "1087,1037,9,28",
          "words": [
            {
              "boundingBox": "1087,1037,9,28",
              "text": "1"
            }
          ]
        },
        {
          "boundingBox": "1086,1133,9,27",
          "words": [
            {
              "boundingBox": "1086,1133,9,27",
              "text": "I"
            }
          ]
        },
        {
          "boundingBox": "1085,1332,9,29",
          "words": [
            {
              "boundingBox": "1085,1332,9,29",
              "text": "1"
            }
          ]
        }
      ]
    },
    {
      "boundingBox": "1122,839,454,573",
      "lines": [
        {
          "boundingBox": "1128,839,173,33",
          "words": [
            {
              "boundingBox": "1128,843,36,29",
              "text": "ST"
            },
            {
              "boundingBox": "1186,839,115,30",
              "text": "661107"
            }
          ]
        },
        {
          "boundingBox": "1127,879,389,41",
          "words": [
            {
              "boundingBox": "1127,887,232,33",
              "text": "VERKTYGSLÅDR"
            },
            {
              "boundingBox": "1382,883,36,28",
              "text": "JC"
            },
            {
              "boundingBox": "1441,882,16,26",
              "text": "5"
            },
            {
              "boundingBox": "1481,879,35,28",
              "text": "ÅR"
            }
          ]
        },
        {
          "boundingBox": "1126,935,173,34",
          "words": [
            {
              "boundingBox": "1126,940,36,29",
              "text": "ST"
            },
            {
              "boundingBox": "1187,935,112,32",
              "text": "181460"
            }
          ]
        },
        {
          "boundingBox": "1126,967,450,50",
          "words": [
            {
              "boundingBox": "1126,987,75,30",
              "text": "BORR"
            },
            {
              "boundingBox": "1224,977,193,35",
              "text": "GLAS/KRKEL"
            },
            {
              "boundingBox": "1440,974,16,27",
              "text": "ø"
            },
            {
              "boundingBox": "1482,971,34,27",
              "text": "10"
            },
            {
              "boundingBox": "1539,967,37,28",
              "text": "MM"
            }
          ]
        },
        {
          "boundingBox": "1125,1027,173,37",
          "words": [
            {
              "boundingBox": "1125,1036,36,28",
              "text": "ST"
            },
            {
              "boundingBox": "1185,1027,113,34",
              "text": "181740"
            }
          ]
        },
        {
          "boundingBox": "1124,1062,432,49",
          "words": [
            {
              "boundingBox": "1124,1071,252,40",
              "text": "UNIVERSALBORR"
            },
            {
              "boundingBox": "1400,1066,96,32",
              "text": "8X120"
            },
            {
              "boundingBox": "1519,1062,37,30",
              "text": "MM"
            }
          ]
        },
        {
          "boundingBox": "1123,1125,175,34",
          "words": [
            {
              "boundingBox": "1123,1129,36,30",
              "text": "ST"
            },
            {
              "boundingBox": "1183,1125,115,32",
              "text": "181738"
            }
          ]
        },
        {
          "boundingBox": "1122,1164,416,44",
          "words": [
            {
              "boundingBox": "1122,1170,255,38",
              "text": "UNIVERSRLBORR"
            },
            {
              "boundingBox": "1501,1164,37,31",
              "text": "MM"
            }
          ]
        },
        {
          "boundingBox": "1123,1225,170,33",
          "words": [
            {
              "boundingBox": "1123,1228,36,30",
              "text": "ST"
            },
            {
              "boundingBox": "1183,1225,110,32",
              "text": "316401"
            }
          ]
        },
        {
          "boundingBox": "1123,1270,355,39",
          "words": [
            {
              "boundingBox": "1123,1275,216,34",
              "text": "LÅSCYLINDER"
            },
            {
              "boundingBox": "1362,1270,116,33",
              "text": "2-PACK"
            }
          ]
        },
        {
          "boundingBox": "1123,1327,177,34",
          "words": [
            {
              "boundingBox": "1123,1330,37,31",
              "text": "ST"
            },
            {
              "boundingBox": "1183,1327,117,32",
              "text": "396026"
            }
          ]
        },
        {
          "boundingBox": "1124,1373,356,39",
          "words": [
            {
              "boundingBox": "1124,1377,216,35",
              "text": "LÅSCYLINDER"
            },
            {
              "boundingBox": "1363,1373,117,33",
              "text": "2-PRCK"
            }
          ]
        }
      ]
    },
    {
      "boundingBox": "1644,820,118,524",
      "lines": [
        {
          "boundingBox": "1658,820,96,31",
          "words": [
            {
              "boundingBox": "1658,820,96,31",
              "text": "79,00"
            }
          ]
        },
        {
          "boundingBox": "1659,912,97,31",
          "words": [
            {
              "boundingBox": "1659,916,50,27",
              "text": "44,"
            },
            {
              "boundingBox": "1719,912,37,28",
              "text": "90"
            }
          ]
        },
        {
          "boundingBox": "1659,1004,98,32",
          "words": [
            {
              "boundingBox": "1659,1007,51,29",
              "text": "69,"
            },
            {
              "boundingBox": "1720,1004,37,28",
              "text": "90"
            }
          ]
        },
        {
          "boundingBox": "1661,1103,97,35",
          "words": [
            {
              "boundingBox": "1661,1103,97,35",
              "text": "49,90"
            }
          ]
        },
        {
          "boundingBox": "1644,1309,118,35",
          "words": [
            {
              "boundingBox": "1644,1309,118,35",
              "text": "299,00"
            }
          ]
        }
      ]
    },
    {
      "boundingBox": "1064,1469,620,45",
      "lines": [
        {
          "boundingBox": "1064,1469,620,45",
          "words": [
            {
              "boundingBox": "1064,1481,237,33",
              "text": "-Rabattcheck"
            },
            {
              "boundingBox": "1324,1486,51,24",
              "text": "nr:"
            },
            {
              "boundingBox": "1384,1469,300,38",
              "text": "935011035567095"
            }
          ]
        }
      ]
    },
    {
      "boundingBox": "1123,1584,159,82",
      "lines": [
        {
          "boundingBox": "1123,1584,159,33",
          "words": [
            {
              "boundingBox": "1123,1584,159,33",
              "text": "DELSUMMA"
            }
          ]
        },
        {
          "boundingBox": "1143,1635,116,31",
          "words": [
            {
              "boundingBox": "1143,1635,116,31",
              "text": "Rabatt"
            }
          ]
        }
      ]
    },
    {
      "boundingBox": "1609,1570,180,189",
      "lines": [
        {
          "boundingBox": "1609,1570,160,36",
          "words": [
            {
              "boundingBox": "1609,1575,11,31",
              "text": "|"
            },
            {
              "boundingBox": "1648,1570,121,34",
              "text": "041,70"
            }
          ]
        },
        {
          "boundingBox": "1690,1621,99,34",
          "words": [
            {
              "boundingBox": "1690,1621,99,34",
              "text": "50,00"
            }
          ]
        },
        {
          "boundingBox": "1651,1725,120,34",
          "words": [
            {
              "boundingBox": "1651,1727,53,32",
              "text": "991"
            },
            {
              "boundingBox": "1715,1746,9,13",
              "text": ","
            },
            {
              "boundingBox": "1732,1725,39,32",
              "text": "70"
            }
          ]
        }
      ]
    },
    {
      "boundingBox": "992,1737,310,1226",
      "lines": [
        {
          "boundingBox": "1123,1737,179,35",
          "words": [
            {
              "boundingBox": "1123,1737,179,35",
              "text": "SLUTSUMMA"
            }
          ]
        },
        {
          "boundingBox": "1036,2756,227,35",
          "words": [
            {
              "boundingBox": "1036,2756,227,35",
              "text": "Totalbelopp"
            }
          ]
        },
        {
          "boundingBox": "1140,2811,124,37",
          "words": [
            {
              "boundingBox": "1140,2811,53,35",
              "text": "991"
            },
            {
              "boundingBox": "1207,2833,8,15",
              "text": "/"
            },
            {
              "boundingBox": "1225,2811,39,34",
              "text": "70"
            }
          ]
        },
        {
          "boundingBox": "992,2927,271,36",
          "words": [
            {
              "boundingBox": "992,2928,159,35",
              "text": "Säljare:"
            },
            {
              "boundingBox": "1182,2927,81,33",
              "text": "7688"
            }
          ]
        }
      ]
    },
    {
      "boundingBox": "1330,2754,145,92",
      "lines": [
        {
          "boundingBox": "1330,2754,144,34",
          "words": [
            {
              "boundingBox": "1330,2754,39,33",
              "text": "Ex"
            },
            {
              "boundingBox": "1394,2754,80,34",
              "text": "Moms"
            }
          ]
        },
        {
          "boundingBox": "1352,2809,123,37",
          "words": [
            {
              "boundingBox": "1352,2809,123,37",
              "text": "793,36"
            }
          ]
        }
      ]
    },
    {
      "boundingBox": "1563,2752,126,92",
      "lines": [
        {
          "boundingBox": "1563,2752,125,33",
          "words": [
            {
              "boundingBox": "1563,2752,82,33",
              "text": "Moms"
            },
            {
              "boundingBox": "1670,2755,18,27",
              "text": "%"
            }
          ]
        },
        {
          "boundingBox": "1586,2808,103,36",
          "words": [
            {
              "boundingBox": "1586,2808,103,36",
              "text": "25,00"
            }
          ]
        }
      ]
    },
    {
      "boundingBox": "1780,2751,123,93",
      "lines": [
        {
          "boundingBox": "1820,2751,83,33",
          "words": [
            {
              "boundingBox": "1820,2751,83,33",
              "text": "Moms"
            }
          ]
        },
        {
          "boundingBox": "1780,2807,123,37",
          "words": [
            {
              "boundingBox": "1780,2807,123,37",
              "text": "198,34"
            }
          ]
        }
      ]
    },
    {
      "boundingBox": "985,2924,966,573",
      "lines": [
        {
          "boundingBox": "1523,2924,83,33",
          "words": [
            {
              "boundingBox": "1523,2924,83,33",
              "text": "7618"
            }
          ]
        },
        {
          "boundingBox": "1288,2926,167,33",
          "words": [
            {
              "boundingBox": "1288,2939,17,7",
              "text": "-"
            },
            {
              "boundingBox": "1330,2926,125,33",
              "text": "Sabina"
            }
          ]
        },
        {
          "boundingBox": "1182,2981,468,36",
          "words": [
            {
              "boundingBox": "1182,2983,38,34",
              "text": "24"
            },
            {
              "boundingBox": "1245,2982,146,34",
              "text": "oktober"
            },
            {
              "boundingBox": "1416,2982,82,34",
              "text": "2016"
            },
            {
              "boundingBox": "1547,2982,10,33",
              "text": "1"
            },
            {
              "boundingBox": "1571,2981,79,34",
              "text": "7:20"
            }
          ]
        },
        {
          "boundingBox": "991,2985,103,33",
          "words": [
            {
              "boundingBox": "991,2985,103,33",
              "text": "Datum"
            }
          ]
        },
        {
          "boundingBox": "1161,3040,403,34",
          "words": [
            {
              "boundingBox": "1161,3040,96,34",
              "text": "44601"
            },
            {
              "boundingBox": "1288,3040,140,34",
              "text": "Kvitto:"
            },
            {
              "boundingBox": "1460,3040,104,34",
              "text": "51756"
            }
          ]
        },
        {
          "boundingBox": "990,3042,103,33",
          "words": [
            {
              "boundingBox": "990,3042,103,33",
              "text": "Kassa"
            }
          ]
        },
        {
          "boundingBox": "1096,3157,728,40",
          "words": [
            {
              "boundingBox": "1096,3159,105,38",
              "text": "Spara"
            },
            {
              "boundingBox": "1225,3157,163,39",
              "text": "kvittot,"
            },
            {
              "boundingBox": "1418,3157,127,39",
              "text": "gäller"
            },
            {
              "boundingBox": "1570,3169,63,26",
              "text": "som"
            },
            {
              "boundingBox": "1657,3158,167,39",
              "text": "garanti."
            }
          ]
        },
        {
          "boundingBox": "1268,3217,388,39",
          "words": [
            {
              "boundingBox": "1268,3217,103,39",
              "text": "Öppet"
            },
            {
              "boundingBox": "1397,3218,62,38",
              "text": "köp"
            },
            {
              "boundingBox": "1484,3218,41,37",
              "text": "30"
            },
            {
              "boundingBox": "1550,3218,106,38",
              "text": "dager"
            }
          ]
        },
        {
          "boundingBox": "1290,3276,317,39",
          "words": [
            {
              "boundingBox": "1290,3276,192,38",
              "text": "VÄLKOMMEN"
            },
            {
              "boundingBox": "1506,3278,101,37",
              "text": "ÅTER!"
            }
          ]
        },
        {
          "boundingBox": "1116,3335,719,42",
          "words": [
            {
              "boundingBox": "1116,3337,41,36",
              "text": "Om"
            },
            {
              "boundingBox": "1182,3335,82,38",
              "text": "ditt"
            },
            {
              "boundingBox": "1290,3346,84,28",
              "text": "namn"
            },
            {
              "boundingBox": "1398,3337,63,38",
              "text": "och"
            },
            {
              "boundingBox": "1485,3349,261,28",
              "text": "personnummer"
            },
            {
              "boundingBox": "1771,3338,64,37",
              "text": "har"
            }
          ]
        },
        {
          "boundingBox": "1032,3395,894,42",
          "words": [
            {
              "boundingBox": "1032,3397,146,36",
              "text": "lämnats"
            },
            {
              "boundingBox": "1204,3395,62,38",
              "text": "för"
            },
            {
              "boundingBox": "1290,3395,61,38",
              "text": "att"
            },
            {
              "boundingBox": "1377,3399,194,36",
              "text": "genomföra"
            },
            {
              "boundingBox": "1596,3399,61,36",
              "text": "ett"
            },
            {
              "boundingBox": "1685,3399,241,38",
              "text": "JulaPro-köp"
            }
          ]
        },
        {
          "boundingBox": "985,3455,966,42",
          "words": [
            {
              "boundingBox": "985,3456,193,37",
              "text": "behandlar"
            },
            {
              "boundingBox": "1203,3455,85,37",
              "text": "Jula"
            },
            {
              "boundingBox": "1312,3456,84,37",
              "text": "dina"
            },
            {
              "boundingBox": "1421,3458,195,39",
              "text": "uppgifter"
            },
            {
              "boundingBox": "1645,3462,12,33",
              "text": "i"
            },
            {
              "boundingBox": "1686,3458,173,38",
              "text": "enlighet"
            },
            {
              "boundingBox": "1886,3461,65,36",
              "text": "med"
            }
          ]
        }
      ]
    }
  ]
}

We extract all the words putting them in a single list. We will later use Azure machine learning to determine what belongs together and what kind of property it is. But for now we just use the bounding boxes for the separate words to create clickable areas where the user needs to select the right boxes for different properties. — Lillvik, Feb 28 '17 at 07:40

Shital Shah · Answer 1 · 2018-10-17T23:01:09.630

5

This is not computer vision problem, its NLP/text pattern recognition problem. In other words, none of the OCRs would do what you want to do; they only extracts text from images.

The usual approach is to collect many different types of receipts, study their structure and then either use rule based approach or machine learning based approach to classify each piece of information. The classifier can have categories like { ItemName, ItemPrice, Subtotal, Total, Heading, Other }. You can use the bounding boxes to form grid cells and use the neighbouring cells as your feature.

Tutorial

Applying OCR Technology for Receipt Recognition.

Cloud Services

Open source project

edited Oct 17 '18 at 23:01

answered Dec 03 '16 at 06:29

Shital Shah

63,284
17
238
185

We are planning to use Microsofts Computer Vision together with Azure Machine Learning. Do you if there are any modules within Azure ML to accomplish this? – Lillvik Dec 04 '16 at 17:49
1

@ShitalShah It would seem the link to the tutorial is not existant anymore. Can you update it if it still exists? – Félix Adriyel Gagnon-Grenier Oct 17 '18 at 14:56
Updated tutorial link, also added bunch of available cloud services – Shital Shah Oct 17 '18 at 23:01

janagan · Answer 2 · 2018-06-28T06:32:33.910

Create model

public class Region
{
    public string BoundingBox { get; set; }
    public List<Line> Lines { get; set; }
}

public class Line
{
    public string BoundingBox { get; set; }
    public List<Word> Words { get; set; }
}

public class Word
{
    public string BoundingBox { get; set; }
    public string Text { get; set; }
}

If you want C# types for the returned response, you can use the official client SDK in github. It's also available in NuGet.

Once you have the OcrResults, and you just want the text, you could write some hacky C# code with Linq like this:

string OcrResultsToString(OcrResult result)
{
    return string.Join("\n",
        result.Regions.ToList().Select(region =>
            string.Join(" ", region.Lines.ToList().Select(line =>
                 string.Join(" ", line.Words.ToList().Select(word =>
                     word.Text).ToArray())).ToArray())).ToArray());
}

Or, if that hurts your eyes, you could use conventional loops like this:

 string OcrResultsToString(OcrResults results)
 {
    StringBuilder stringBuilder = new StringBuilder();

    if (results != null && results.Regions != null)
    {
        foreach (var item in results.Regions)
        {
            foreach (var line in item.Lines)
            {
                foreach (var word in line.Words)
                {
                    stringBuilder.Append(word.Text);
                    stringBuilder.Append(" ");
                }
                stringBuilder.AppendLine();
            }
            stringBuilder.AppendLine();
        }
    }
    return stringBuilder.ToString();
}

This is just a way to create a string of the result and not a solution to our problem. The problem we have is that the data we get is not ordered such that we can use i as receipt data. But this is as earler mentioned a ML problem — Lillvik, Dec 14 '18 at 06:34

Handle OCR / Computer Vision result to match a receipt structure

2 Answers2