How to make this task much faster to be finished? The 3 calls of generate_ngrams_from_file() can be done in parallel? Just get into python and don't know how to make it faster. I think multiprocessing or threading should be doing the job, but no idea of how to do it. This looks like a typical task can be done concurrently to use multiple cores on my Mac machine.
def tokenize(text):
return [token for token in text.split(' ')]
def generate_ngrams(text, n):
tokens = tokenize(text)
ngrams = zip(*[tokens[i:] for i in range(n)])
return [''.join(ngram) for ngram in ngrams]
def generate_ngrams_from_file(input, out, n):
count = 0
with open(input, 'r') as f:
for line in f:
count += 1
if line:
ngrams = generate_ngrams(line, n)
if n == 2:
bigrams.update(ngrams)
elif n == 3:
trigrams.update(ngrams)
elif n == 4:
fourgrams.update(ngrams)
elif n == 5:
fourgrams.update(ngrams)
print("Ngram done!")
if __name__ == "__main__":
start = time.time()
input_file = 'bigfile.txt'
output_3_tram = '3gram.txt'
output_4_tram = '4ngram.txt'
output_5_tram = '5ngram.txt'
print('Generate trigram: ')
generate_ngrams_from_file(input_file, output_3_tram, 3)
print("Generate fourgrams: ")
generate_ngrams_from_file(input_file, output_4_tram, 4)
print("Generate fivegrams: ")
generate_ngrams_from_file(input_file, output_5_tram, 5)
end = time.time()
mytime(start, end)
