I am having a .dat file and a structure file in.xlsx where the start values and end values of the column are there . I want to fetch the data accordingly from the dat file using the excel file. the structure file has around 13000 rows
I try to did the same taking around 300 rows and my code works ,And when i try the same code on file having 13000 rows i'm having a memory error
import numpy as np
import pandas as pd
import xlrd
from pyexcel.cookbook import merge_all_to_a_book
import glob
loc = ("excelstruct.xlsx")
wb = xlrd.open_workbook(loc)
sheet = wb.sheet_by_index(0)
df=pd.DataFrame()
file_txt= open("Maybach_WK0102.dat", "r+")
file_txt1=file_txt.readlines()
ColumnData = []
for line in file_txt1:
ColumnData.append(line[int(sheet.cell_value(1, 1))*2:int(sheet.cell_value(1, 2))*2])
df[sheet.cell_value(1,0)]=ColumnData
for k in range(2,sheet.nrows):
ColumnData= []
for line in file_txt1:
ColumnData.append(line[int(sheet.cell_value(k, 1))*2-1:int(sheet.cell_value(k, 2))*2])
df[sheet.cell_value(k,0)]=ColumnData