How to retain table structure when it is read from a PDF and stored in a dataframe?

Question

I used the read_pdf from tabula to read the table present in a PDF by specifying the area parameters. I wish to retain the table structure as is (including the lines in between the columns and rows (if applicable)). I read that matplotlib can be used to do, but when I try to put the read table into a CSV, the table structure vanishes and there are only spaces between rows of column. My code-

from tabula import read_pdf
import csv
path = "---"
df = read_pdf(path, stream=True , encoding="utf-8", guess = False, nospreadsheet = True, area = (112.37, 35.34, 153.36, 212.43))
print(df)
df.to_csv("path to destination csv file")

import matplotlib.pyplot as plt
fig, ax = plt.subplots()
ax.xaxis.set_visible(False) 
ax.yaxis.set_visible(False)

ax.table(cellText=df.values, colLabels=df.columns, loc='center')
fig.tight_layout()
plt.show()

When I look into the content of the destination csv file, the lines between columns are not retained. For example, in the below given PDF, i wish to read data from the table and put it into a csv file by retaining the lines between columns, whereas my code doesn't retain the lines.

whereas i want my code to produce a csv file that bifurcates or draws lines between the columns like this-

The pdf included here is sample. My original pdf is displaying the following output when i used matplotlib. , whereas i want it to look like this-> (only the part inside the black lines with the bifurcation)

Answer 1

The csv-format (c[haracter/omma] separated values) are values seperated by (the same) seperator character. csv is text-based - there are no "lines" in it.

There are unicode characters that can be used to "form lines":

    U+2500    ─   e2 94 80    ─ ─   BOX DRAWINGS LIGHT HORIZONTAL
    U+2501    ━   e2 94 81    ━ ━   BOX DRAWINGS HEAVY HORIZONTAL
    U+2502    │   e2 94 82    │ │   BOX DRAWINGS LIGHT VERTICAL
    U+2503    ┃   e2 94 83    ┃ ┃   BOX DRAWINGS HEAVY VERTICAL
... snipp ...
    U+250C    ┌   e2 94 8c    ┌ ┌   BOX DRAWINGS LIGHT DOWN AND RIGHT
    U+250D    ┍   e2 94 8d    ┍ ┍   BOX DRAWINGS DOWN LIGHT AND RIGHT HEAVY
    U+250E    ┎   e2 94 8e    ┎ ┎   BOX DRAWINGS DOWN HEAVY AND RIGHT LIGHT
... snipp ...
    U+2533    ┳   e2 94 b3    ┳ ┳   BOX DRAWINGS HEAVY DOWN AND HORIZONTAL
    U+2534    ┴   e2 94 b4    ┴ ┴   BOX DRAWINGS LIGHT UP AND HORIZONTAL
    U+2535    ┵   e2 94 b5    ┵ ┵   BOX DRAWINGS LEFT HEAVY AND RIGHT UP LIGHT
... snipp ...
    U+2548    ╈   e2 95 88    ╈ ╈   BOX DRAWINGS UP LIGHT AND DOWN HORIZONTAL HEAVY
    U+2549    ╉   e2 95 89    ╉ ╉   BOX DRAWINGS RIGHT LIGHT AND LEFT VERTICAL HEAVY
    U+254A    ╊   e2 95 8a    ╊ ╊   BOX DRAWINGS LEFT LIGHT AND RIGHT VERTICAL HEAVY
    U+254B    ╋   e2 95 8b    ╋ ╋   BOX DRAWINGS HEAVY VERTICAL AND HORIZONTAL

source

But you would have to "typeset" each line piece on its own to mimic a table in unicode:

┍━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┓
┃   tata         ┃   ta    ┃    fo   ┃  ka     ┃
┝━━━━━━━━━━━━━━━╈━━━━━━━━╈━━━━━━━━╈━━━━━━━━┧
┃   1234         ┃   45    ┃   79    ┃  45     ┃
┝━━━━━━━━━━━━━━━╈━━━━━━━━╈━━━━━━━━╈━━━━━━━━┧
┃   1234         ┃   45    ┃   79    ┃  45     ┃
┕━━━━━━━━━━━━━━━┻━━━━━━━━┻━━━━━━━━┻━━━━━━━━┛

However this is no csv - it is more like human readable ascii-art. The corresponding csv would be:

tata,ta,fo,ka
1234,45,79,45
1234,45,79,45

(if using , as seperator char - replace with char you like better: [" ","|",";",\t] )

---

Disclaimer:

I am purposefully bad at ascii-art and choose not to match exactly corresponding unicode lines (LIGHT, HEAVY) to get my point across. This is on purpose - call me lazy.

Answer 2

Seperate answer - this code formats a "well-defined" csv

tata,ta,for,kattatom
1234,45,79,45
1234,45,79,45

into a "utf8-art":

┌────┬──┬───┬────────┐
│tata│ta│for│kattatom│
├────┼──┼───┼────────┤
│1234│45│79 │45      │
├────┼──┼───┼────────┤
│1234│45│79 │45      │
└────┴──┴───┴────────┘

The utf8-art is appended to the file output.txt.

---

import csv 

def create_table(file_name):
    """Takes a file_name to a csv. Produces utf8-art of the data. 
    Missing columns will be assumed to miss at end and replaced 
    by empty columns."""
    # mostly untested code - works for the 2 examples mentioned here
    with open(file_name,"r") as f:
        reader = csv.reader(f) 
        w = get_widths(reader)
        row_count = w["last"] 
        del w["last"]
        f.seek(0)
        return create_table_string(reader, w, row_count)

def get_widths(csv_reader):
    widths = {}
    row_count = 0
    for row in csv_reader:
        if row: # ignore empties
            row_count += 1
            for idx,data in enumerate(row):
                widths[idx] = max(widths.get(idx,0),len(data))

    widths["last"] = row_count
    return widths

# supply other set of lines if you like
deco = {k:v for k,v in zip("hv012345678","─│┌┬┐├┼┤└┴┘")} 


def base_row(widths, row, max_key, _v, _h, _l, _m, _r):
    decoration = []
    text_data = []

    decoration.append(_l + _h*widths[0])
    for i in range(1,max_key):
        decoration.append(_m + _h*widths[i])
    decoration.append(_m + _h*widths[max_key] + _r)

    if row:
        for i,data in enumerate(row): 
            text_data.append(_v + "{:<{}}".format(data, widths[i]))

        for empty in range(i+1,max_key+1):
            text_data.append(_v + " "*widths[empty])
        text_data[-1]+=_v

    return [decoration, text_data]

def get_first_row(widths,row):
    return base_row(widths, row, max(widths.keys()), deco["v"], deco["h"], 
                    deco["0"], deco["1"], deco["2"])

def get_middle_row(widths,row):
    return base_row(widths, row, max(widths.keys()),  deco["v"], deco["h"],
                    deco["3"], deco["4"], deco["5"])

def get_last_row(widths):
    decoration, _ = base_row(widths, [], max(widths.keys()), deco["v"], 
                             deco["h"], deco["6"], deco["7"], deco["8"])

    return [decoration]


def create_table_string(reader, widths, row_count): 
    output = []
    r = 0 
    for row in reader:
        if row:
            r += 1
            if r==1:
                output.extend(get_first_row(widths, row))
            else:
                output.extend(get_middle_row(widths, row))

    output.extend( get_last_row(widths))
    return output

Usage:

#create sample csv
with open("data.csv","w") as f:
    f.write("""tata,ta,for,kattatom
1234,45,79,45
1234,45,79,45""")

# open outputfile for append
with open("output.txt", "a", encoding="UTF8") as output:
    output.write("\n" + "-" * 40 + "\n\n")

    # get utf8 art
    for line in create_table("data.csv"):
        output.write(''.join(line)+"\n")

Input csv:

tata,ta,for,kattatom
1234,45,79,45
1234,45,79,45

then:

tata,ta,for,kattatom
1234,45,79,45,8,0
1234,45,79,45

Output:

┌────┬──┬───┬────────┐
│tata│ta│for│kattatom│
├────┼──┼───┼────────┤
│1234│45│79 │45      │
├────┼──┼───┼────────┤
│1234│45│79 │45      │
└────┴──┴───┴────────┘

----------------------------------------

┌────┬──┬───┬────────┬─┬─┐
│tata│ta│for│kattatom│ │ │
├────┼──┼───┼────────┼─┼─┤
│1234│45│79 │45      │8│0│
├────┼──┼───┼────────┼─┼─┤
│1234│45│79 │45      │ │ │
└────┴──┴───┴────────┴─┴─┘