I have an upload in my application, which receives an encrypted file. When doing the decryption I'm writing json in another file. After completing this operation I have a 500MB json file.
Note: The properties of my json are dynamic.
How can I parse this json, I read about stream-json/Parser but could not implement it.
const {parser} = require('stream-json/Parser');
static async _recoverObj(pathAnalise) {
let readStream = fs.createReadStream(pathAnalise, {encoding: 'utf8', highWaterMark: 1024});
function createJson(stream) {
let decipher = crypto.createDecipher('aes-128-ecb', 'AmbientalRuralBR');
return new Promise((resolve) => {
stream.on('data', chunk => {
let newDecrypt = decipher.update(chunk, 'base64', 'utf8');
fs.appendFile('./data.json', newDecrypt, function() {
console.log('Saved!');
});
});
stream.on('end', () => {
fs.appendFile('./data.json', '"}', function() {
resolve();
});
});
});
}
const result = await createJson(readStream);
function getParse() {
return new Promise((resolve) => {
let jsonData = './data.json';
let pipeline = fs.createReadStream(jsonData).pipe(parser());
let objectCounter = 0;
pipeline.on('data', data => {
data.name === 'startObject' && ++objectCounter
});
pipeline.on('end', () => {
console.log(`Found ${objectCounter} objects.`);
resolve();
});
});
}
let dataAnalyze = await getParse();
return dataAnalyze;
}
Anyone have any idea how to do this parse?
MY SOLUTION:
I used https://www.npmjs.com/package/big-json
const json = require('big-json');
static async _recoverObj(pathAnalise) {
var readStream = fs.createReadStream(pathAnalise, {encoding: 'utf8', highWaterMark: 1024 * 2});
function createJson(stream) {
var decipher = crypto.createDecipher('aes-128-ecb', 'AmbientalRuralBR');
return new Promise((resolve) => {
stream.on('data', chunk => {
var newDecrypt = decipher.update(chunk, 'base64', 'utf8');
fs.appendFile('./data.json', newDecrypt, function() {
console.log('Saved!');
});
});
stream.on('end', () => {
fs.appendFile('./data.json', '"}', function() {
resolve();
});
});
});
}
await streamToString(readStream);
function getParse() {
return new Promise((resolve) => {
var jsonData = './max.json';
var myObj = [];
const readStream = fs.createReadStream(jsonData);
const parseStream = json.createParseStream();
parseStream.on('data', function(pojo) {
myObj = pojo;
});
parseStream.on('end', function() {
resolve(myObj);
});
readStream.pipe(parseStream);
});
}
let dataAnalyze = await getParse();
return dataAnalyze;
}
