1

I am parsing a large JSON file using JSON stream. This works but it returns the file line by line. So when I try and restructure the data I can only get the data that is not repeating.

For example, this is the structure:

{
    "Test": {
        "id": 3454534344334554345434,
        "details": {
            "text": "78679786787"
        },
        "content": {
            "text": 567566767656776
        },
        "content": {
            "text": 567566767656776
        },
        "content": {
            "text": 567566767656776
        }
    }
}

I'm able to get Test.id or Test.details.id but I can only get the First Test.content per line.

I've tried to set in an array but this still gets only the first line of Test.content.

Is there another way to transform a large file other than using JSONStream?

Parsing code:

var getStream = function () {
    let stream = fs.createReadStream(path.join(__dirname, '../test.json'), {
            encoding: 'utf8'
        }),
        parser = JSONStream.parse('*.Test')
    return stream.pipe(parser);
};

getStream()
    .pipe(es.mapSync(function (data) {
        let dataObj = []
        dataObj.push(data)
        processData(dataObj)
    }))


function processData(d) {
    let js = JSON.parse(JSON.stringify(d))
    console.log(js)
    // js.forEach(function (value, index) {
    //     Object.keys(value).forEach(function (v, i) {});

    // })
}


BluLotus
  • 139
  • 1
  • 11
  • Comments are not for extended discussion; this conversation has been [moved to chat](https://chat.stackoverflow.com/rooms/200371/discussion-on-question-by-blulotus-how-can-i-parse-a-large-json-file-with-repeat). – Samuel Liew Oct 04 '19 at 00:48

0 Answers0