I am using iter_loadtxt as seen on stack overflow to load > 600MB text files. Occasionally the program generating the files make a bad write to the text file and iter_loadtxt barfs:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "jpk_outfile_to_LOG_timeV4.py", line 296, in makeLOGtimefile
a=readjpk(filename,lateral,vertical,xpos)
File "jpk_outfile_to_LOG_timeV4.py", line 107, in readjpk
a=iter_loadtxt2( filename,delimiter=" ",skiprows=7)
File "jpk_outfile_to_LOG_timeV4.py", line 58, in iter_loadtxt2
data = np.fromiter(iter_func(), dtype=dtype)
File "jpk_outfile_to_LOG_timeV4.py", line 55, in iter_func
yield dtype(item)
*605.6998r: invalid literal for float(): -1.5005492E-5
My solution is to try and add an exception that replaces the dtype, but this doesn't work. any suggestions?
#==============================================================================
# credit to joe kington on stackoverflow:
# https://stackoverflow.com/questions/8956832
#==============================================================================
def iter_loadtxt2(filename, delimiter=' ', skiprows=6, dtype=float):
def iter_func():
with open(filename, 'r') as infile:
for _ in range(skiprows):
next(infile)
for line in infile:
line = line.rstrip().split(delimiter)
for item in line:
# added try/except/else to this loop, originally only had yiled dtype(item)
try:
yield dtype(item)
a=dtype(item)
except:
print "fail"
else:
yield a
iter_loadtxt.rowlength = len(line)
data = np.fromiter(iter_func(), dtype=dtype)
data = data.reshape((-1, iter_loadtxt.rowlength))
return data