Parse streamed chunk data into JSON

Question

Hi I'm trying to display data in chunk since I'm getting data in chunk.

for example let us assume that data is something like this.

data: {
user: [
    {
        name: 'a',
        bankAccounts: ['123', '234', '567'],
        address: ['some address', 'some other address', 'some more addres']
    },
    {
        name: 'b',
        bankAccounts: ['1233', '2334', '5637'],
        address: ['some address1', 'some other address1', 'some more addres1']
    },
    {
        name: 'c',
        bankAccounts: ['123355', '233455', '563700'],
        address: ['some address12', 'some other address12', 'some more addres12']
    },
]    
}

but the chunk I'm receiving is something like this

1st chunk: "data: user: [ {name: a"
2nd chunk: "bankAccounts: ['123', '234', '567'],"
3rd chunk: "address: ['some address', 'some other address', 'some more addres']"

and so on..

I'm receiving chunked data in such a way which can't be converted into json since it is incomplete.

How can I stream this data in UI?

Any Idea !!!

My code for fetching streaming data

fetch('some url which stream data')
// Retrieve its body as ReadableStream
    .then(response => {
        const reader = response.body.getReader();
        let decoder = new TextDecoder();
        return new ReadableStream({
        start(controller) {
            return pump();
            function pump() {
                return reader.read().then(({ done, value }) => {
                    // When no more data needs to be consumed, close the stream
                    let newData = decoder.decode(value, {stream: !done});
                    console.log(newData);
                    if (done) {
                        controller.close();
                        return;
                    }
                    // Enqueue the next data chunk into our target stream
                    controller.enqueue(value);
                    return pump();
                });
            }
        }
    })
})
.then(stream => new Response(stream))
.then(response => {
    console.log('response', response)
})

Krzysztof Krzeszewski · Answer 1 · 2019-08-16T08:08:40.247

I know that generators are not very commonly used, but i feel like they would be perfect for streaming the data in this task,

async function* streamAsyncIterator(stream) {
  const reader = stream.getReader();
  const decoder = new TextDecoder();
  while (true) {
    const {done,value} = await reader.read();
    if (done) break;
    yield decoder.decode(value, { stream: !done });
  }
  reader.releaseLock();
}

fetch('https://httpbin.org/stream/1')
    .then(async response => {
        let str="";
        for await (const value of streamAsyncIterator(response.body))
            str+=value;
        return JSON.parse(str);
    })
    .then(response => {
        console.log('response', response)
    })

however it seems what you want is to parse partially complete JSON, which can be achieved in variety of ways, for instance by using an npm library partial-json-parser

import partialParse from 'partial-json-parser';

fetch('https://httpbin.org/stream/1')
    .then(async response => {
        let str="";
        for await (const value of streamAsyncIterator(response.body)){
            str+=value;
            functionUpdatingYourUi(partialParse(str));
        }
        return JSON.parse(str);
    })
    .then(response => {
        console.log('response', response)
    })

score 1 · Answer 2 · answered Aug 13 '19 at 10:26

You can accept a string(start with an empty string) to your function pump and keep appending it until chunk is there. at the end when terminating the recursion, return the parsed data.

const manager = require('./manager');

// manager.UpdateEC2Instances().then(console.log);
manager.UpdateRDSInstances().then(console.log);

fetch('some url which stream data')
    // Retrieve its body as ReadableStream
    .then(response => {
        const reader = response.body.getReader();
        let decoder = new TextDecoder();
        return new ReadableStream({
            start(controller) {
                return pump('');
                function pump(str) {
                    return reader.read().then(({ done, value }) => {
                        // When no more data needs to be consumed, close the stream
                        str += decoder.decode(value, { stream: !done });
                        console.log(str);
                        if (done) {
                            controller.close();
                            return JSON.parse(str);
                        }
                        // Enqueue the next data chunk into our target stream
                        controller.enqueue(value);
                        return pump(str);
                    });
                }
            }
        })
    })
    .then(stream => new Response(stream))
    .then(response => {
        console.log('response', response)
    })

You are parsing the string if done is true, this is not streaming , this is same as when entire data is received process it, but I'm looking for processing chunked data — shaan, Aug 13 '19 at 10:55
what do you mean by processing the chunk data? what do you want to do with that? you cant parse something which is not a valid JSON. even if some chunk is valid JSON still should not be doing that. — AZ_, Aug 13 '19 at 11:00
processing means , display chunked data in UI so that user don't have to wait for a longer period, but I cannot display it because I need to parse it in a json — shaan, Aug 13 '19 at 11:03
oh ok, no you can't, user will have to wait or you can change the API only to send the user data with pagination having each element in a single request and then call the API multiple times. — AZ_, Aug 13 '19 at 11:12

score 0 · Answer 3 · answered Mar 23 '20 at 19:35

See this thread for a more complete discussion & more complete examples from @Damian Nadales. If you are expecting your chunks to be complete JSON, which is not at all guarantee, you may decode your chunked value (of type Uint8Array) into UTF-8 using TextDecoder.decode, then parse the JSON using JSON.parse. E.g.,

var num = JSON.parse(
      new TextDecoder("utf-8").decode(result.value)
);

Parse streamed chunk data into JSON

3 Answers3