javascript FileReader - parsing long file in chunks

Question

I have long file I need to parse. Because it's very long I need to do it chunk by chunk. I tried this:

function parseFile(file){
    var chunkSize = 2000;
    var fileSize = (file.size - 1);

    var foo = function(e){
        console.log(e.target.result);
    };

    for(var i =0; i < fileSize; i += chunkSize)
    {
        (function( fil, start ) {
            var reader = new FileReader();
            var blob = fil.slice(start, chunkSize + 1);
            reader.onload = foo;
            reader.readAsText(blob);
        })( file, i );
    }
}

After running it I see only the first chunk in the console. If I change 'console.log' to jquery append to some div I see only first chunk in that div. What about other chunks? How to make it work?

Answer 1

FileReader API is asynchronous so you should handle it with block calls. A for loop wouldn't do the trick since it wouldn't wait for each read to complete before reading the next chunk. Here's a working approach.

function parseFile(file, callback) {
    var fileSize   = file.size;
    var chunkSize  = 64 * 1024; // bytes
    var offset     = 0;
    var self       = this; // we need a reference to the current object
    var chunkReaderBlock = null;

    var readEventHandler = function(evt) {
        if (evt.target.error == null) {
            offset += evt.target.result.length;
            callback(evt.target.result); // callback for handling read chunk
        } else {
            console.log("Read error: " + evt.target.error);
            return;
        }
        if (offset >= fileSize) {
            console.log("Done reading file");
            return;
        }

        // of to the next chunk
        chunkReaderBlock(offset, chunkSize, file);
    }

    chunkReaderBlock = function(_offset, length, _file) {
        var r = new FileReader();
        var blob = _file.slice(_offset, length + _offset);
        r.onload = readEventHandler;
        r.readAsText(blob);
    }

    // now let's start the read with the first block
    chunkReaderBlock(offset, chunkSize, file);
}

Answer 2

You can take advantage of Response (part of fetch) to convert most things to anything else blob, text, json and also get a ReadableStream that can help you read the blob in chunks

var dest = new WritableStream({
  write (str) {
    console.log(str)
  }
})

var blob = new Blob(['bloby']);

(blob.stream ? blob.stream() : new Response(blob).body)
  // Decode the binary-encoded response to string
  .pipeThrough(new TextDecoderStream())
  .pipeTo(dest)
  .then(() => {
    console.log('done')
  })

Old answer (WritableStreams pipeTo and pipeThrough was not implemented before)

I came up with a interesting idéa that is probably very fast since it will convert the blob to a ReadableByteStreamReader probably much easier too since you don't need to handle stuff like chunk size and offset and then doing it all recursive in a loop

function streamBlob(blob) {
  const reader = new Response(blob).body.getReader()
  const pump = reader => reader.read()
  .then(({ value, done }) => {
    if (done) return
    // uint8array chunk (use TextDecoder to read as text)
    console.log(value)
    return pump(reader)
  })
  return pump(reader)
}

streamBlob(new Blob(['bloby'])).then(() => {
  console.log('done')
})

Answer 3

The second argument of slice is actually the end byte. Your code should look something like:

 function parseFile(file){
    var chunkSize = 2000;
    var fileSize = (file.size - 1);

    var foo = function(e){
        console.log(e.target.result);
    };

    for(var i =0; i < fileSize; i += chunkSize) {
        (function( fil, start ) {
            var reader = new FileReader();
            var blob = fil.slice(start, chunkSize + start);
            reader.onload = foo;
            reader.readAsText(blob);
        })(file, i);
    }
}

Or you can use this BlobReader for easier interface:

BlobReader(blob)
.readText(function (text) {
  console.log('The text in the blob is', text);
});

More information:

• README.md
• Docs

Answer 4

Here is my FileStreamer typescript version here

class FileStreamer {
    constructor(file, encoding = 'utf-8') {
        this.file = file;
        this.offset = 0;
        this.defaultChunkSize = 64 * 1024; // bytes
        this.textDecoder = new TextDecoder(encoding);
        this.rewind();
    }
    rewind() {
        this.offset = 0;
    }
    isEndOfFile() {
        return this.offset >= this.getFileSize();
    }
    async eventPromise(target, eventName) {
        return new Promise((resolve) => {
            const handleEvent = (event) => {
                resolve(event);
            };
            target.addEventListener(eventName, handleEvent);
        });
    }
    async readFile(blob) {
        const fileReader = new FileReader();
        fileReader.readAsArrayBuffer(blob);
        const event = await this.eventPromise(fileReader, 'loadend');
        const target = event.target;
        if (target.error) {
            throw target.error;
        }
        return target.result;
    }
    async readBlockAsText(length = this.defaultChunkSize) {
        const blob = this.file.slice(this.offset, this.offset + length);
        const buffer = await this.readFile(blob);
        const decodedText = this.textDecoder.decode(buffer, { stream: true });
        this.offset += blob.size;

        if (this.isEndOfFile()) {
            const finalText = this.textDecoder.decode();
            if (finalText) {
                return decodedText + finalText;
            }
        }
        return decodedText;
    }
    getFileSize() {
        return this.file.size;
    }
}

Example printing a whole file in the console (within an async context)

    const fileStreamer = new FileStreamer(aFile);
    while (!fileStreamer.isEndOfFile()) {
      const data = await fileStreamer.readBlockAsText();
      console.log(data);
    }

Answer 5

Parsing the large file into small chunk by using the simple method:

                //Parse large file in to small chunks
                var parseFile = function (file) {

                        var chunkSize = 1024 * 1024 * 16; //16MB Chunk size
                        var fileSize = file.size;
                        var currentChunk = 1;
                        var totalChunks = Math.ceil((fileSize/chunkSize), chunkSize);

                        while (currentChunk <= totalChunks) {

                            var offset = (currentChunk-1) * chunkSize;
                            var currentFilePart = file.slice(offset, (offset+chunkSize));

                            console.log('Current chunk number is ', currentChunk);
                            console.log('Current chunk data', currentFilePart);

                            currentChunk++;
                        }
                };

Answer 6

Getting chunks by blindly slicing a blob by bytes, can cut a multi-byte character in half and break the encoding.

Since handling character boundaries by yourself is a nightmare, you can use TextDecoderStream to help with that.

Here is a solution implemented as an async generator function:

async function* read_chunks(file, chunk_size=1000000, encoding=undefined) {
  let offset = 0;
  const stream = new ReadableStream({
    async pull(controller) {
      let chunk = file.slice(offset, offset + chunk_size);
      chunk = await chunk.arrayBuffer();
      chunk = new Uint8Array(chunk);
      controller.enqueue(chunk);
      if (offset >= file.size) {
        controller.close()
      }
      offset += chunk.length;
    }
  }).pipeThrough(new TextDecoderStream(encoding));
  const reader = stream.getReader();
  for (;;) {
    const { done, value } = await reader.read();
    if (done) return;
    yield value;
  }
}

const file = new Blob(["000001002003"]);
(async() => {
  for await (const chunk of read_chunks(file, 4)) {
    console.log(`Chunk: [${chunk.length}] "${chunk}"`);
  }
})();

You can get rid of the custom ReadableStream, by replacing it with file.stream(), if you don't care about the size of the chunks.