Nodejs https Repsponse ghost characters

Question

I'm trying to get data from an API, but it somehow gets weird data, I've tried many things and i don't know what else I could do.

const https = require("https");
https.get(`https://drednot.io/api/scoreboard?count=1000&offset_score=-1`, (resp) => {
    let data = '';
    resp.on('data', (chunk) => {
        data += chunk;
    });
    resp.on('end', async () => {
        try {
            //console.log(data)
            let result = JSON.parse(data.toString())["ships"];
            console.log(result)
        } catch (err) {
            console.log("Error: " + err.message);
        }
    });
}).on("error", async (err) => {
    console.log("Error: " + err.message);
});

both the raw data and the parsed data have some weird things inside them, for example, a substring in the raw data

{"sh         ip_name":"⚜️ Full Farm Barn","hex_code":"20DE1","color":9079434,"score":2395790}

or a part in the parsed data (different because above isn't inside the parsed for some reason)

{
    ship_name: '�������� '        ,
    hex_code: '78AD5F',
    color: 815352,
    score: 25915190
}

Edit: the � characters are parsed correctly, they just cant be displayed by the terminal which isn't the problem.

Full code to see what i mean, it should sometimes print a change but it prints changes all the time which is incorrect:

const https = require("https");


function sleep(ms) {
    if (ms <= 0) {

    } else {
        return new Promise(resolve => setTimeout(resolve, ms));
    }
}

var shipData = {};
class Syncer {
    finished;
    round_fetch;
    round_error;

    constructor() {
        this.finished = true;
        this.round_fetch = 0;
        this.round_error = 0;
    }

    fetch_data(offset_score, next_request, recursion_deep) {
        //console.log(offset_score)
        this.round_fetch += 1;

        https.get(`https://drednot.io/api/scoreboard?count=1000&offset_score=${offset_score.toString()}`, (resp) => {
            let chunks = [];
            resp.on('data', (chunk) => {
                chunks.push(chunk);
            });

            resp.on('end', async () => {
                let end = Date.now();
                try {
                    //console.log(data)
                    let data = Buffer.concat(chunks).toString('utf8');
                    //console.log(data)
                    console.log(`Index of hexcode: ${data.search("32FF2B")}`)
                    let result = JSON.parse(data)["ships"];
                    data = null;
                    let next_offset = result[result.length - 1].score;
                    if (next_offset === offset_score) {
                        next_offset -= 1;
                    }
                    if (next_offset < 10000000) {
                        next_offset = -1;
                    }
                    if (next_offset !== -1) {
                        if (end < next_request) {
                            await sleep(next_request - end);
                        }
                        this.fetch_data(next_offset, Date.now() + 100, recursion_deep + 1);
                    } else {
                        this.finished = true;
                    }
                    next_offset = null;
                    let rank = (recursion_deep * 1000) + 1;

                    for (let ship of result) {
                        await sleep(1);
                        ship["rank"] = rank;
                        if (ship.hex_code === "32FF2B") {
                            console.log(`Parsed ship: ${ship}`)
                        }
                        if (shipData[ship.hex_code] !== undefined) {
                            this.checkChange(shipData[ship.hex_code], ship);
                        }
                        shipData[ship.hex_code] = ship;
                    }
                    result = null;
                    offset_score = null;
                } catch (err) {
                    console.log("Error: " + err.message);
                    await sleep(1000);
                    this.round_error += 1;
                    await this.fetch_data(offset_score, Date.now() + 100, recursion_deep);
                }
            });
        }).on("error", async (err) => {
            console.log("Error: " + err.message);
            await sleep(1000);
            this.round_error += 1;
            await this.fetch_data(offset_score, Date.now() + 100, recursion_deep);
        });
    }
    checkChange(oldShip, newShip) {

        // CHECK IF OLD NAME ISNT THE SAME AS THE NEW ONE, IF SO PRINT CHANGE
        if (oldShip.ship_name !== newShip.ship_name) {
            console.log(`${oldShip.ship_name} | ${newShip.ship_name}`)
        }
    }


    async launch() {
        //this.postChanges();
        while (true) {
            try {
                if (this.finished) {
                    this.finished = false;
                    console.log(`Fetch all ships, fetch ${this.round_fetch}, error ${this.round_error}`);
                    this.round_error = 0;
                    this.round_fetch = 0;
                    await sleep(100);
                    this.fetch_data(-1, Date.now() + 100, 0);
                }
                await sleep(1000);
            } catch (e) {
                console.log(e);
            }
        }
    }
}

function launch() {
    // Start the servers
    var synchro = new Syncer();
    synchro.launch()
}

launch()

Answer 1

The problem could be that multi-byte sequences have arrived split into different chunks, and it doesn't make sense to turn any single chunk into a string by itself because some of the bytes of a character is in one chunk and the other bytes of that character are in the next chunk.

To get around this, just accumulate the incoming data chunks by keeping the the buffers they originally arrived in, and concatenate all the buffers (chunks) into a single buffer before turning them into a string.

Instead of data = '' do chunks = [], and instead of data += chunk; do chunks.push(chunk) and then in your 'end' handler, do data = Buffers.concat(chunks).toString('utf8').

    let chunks = [];
    resp.on('data', (chunk) => {
        chunks.push(chunk);
    });
    resp.on('end', async () => {
        let data = Buffer.concat(chunks).toString('utf8');
        try {
            //console.log(data)
            let result = JSON.parse(data.toString())["ships"];
            console.log(result)
        } catch (err) {
            console.log("Error: " + err.message);
        }
    });