How to convert Bigtable HBase API returning byte array to integer or float in nodeJS?

Question

I am writing data from dataflow into bigtable, and need to retrieve data from NodeJS, but realised that the data is in byte array. How do I convert back to integer or float?

The key "\u0000\u0000\u0000\u0000" was originally a 0, but I could never get it to output correctly in my nodeJS code.

I have tried the below methods using Buffer, bin2string, byteArrayToLong, but none of them worked correctly. The following is the code to query data.

async function query(table, start, end) {
  return new Promise((resolve, reject) => {
    table.createReadStream({
      start: start,
      end: end
    }).on('data', function(row) {
        for(var key in row.data.ch){
            console.log(JSON.stringify(key));               // Output: "\u0000\u0000\u0000\u0000"
            console.log(`bin2string: ${bin2string(key)}`);  // Output: bin2string:

            let keybuf = Buffer.from(key);
            console.log(keybuf);                            // Output: <Buffer 00 00 00 00>
            console.log(keybuf.toString('utf8'));           // Output:
            const utf16Buffer = Buffer.from(key,'utf16le'); // Output: <Buffer 00 00 00 00 00 00 00 00>
            console.log(utf16Buffer);
            console.log(utf16Buffer.toString());            // Output:
            console.log(byteArrayToLong(key));              // Output: NaN
        }
      // Nothing to do with data
      // We can measure the time needed to get the first row
    }).on('end', function(){
      resolve();
    });  
  });
}

function bin2string(array){
    var result = "";
    for(var i = 0; i < array.length; ++i){
        result+= (String.fromCharCode(array[i]));
    }
    return result;
}

function byteArrayToLong (byteArray){
    var value =0;
    for(var i=byteArray.length-1; i>=0; i--){
        value = value*256 + byteArray[i];
    }
    return value;
}

score 3 · Answer 1 · answered Oct 18 '18 at 18:39

I don't have a full answer to your question, but I have some pointers.

Cloud Bigtable numbers are all "64-bit integer encoded as an 8-byte big-endian value" (see here). Node.js longs "endian-ness" are system specific (see here). PHP has a similar issue with Cloud Bigtable (see here).

In Java, all numeric values are big-endian. The HBase Bytes class does all of the conversions between numbers and bytes (source code), and may provide some clues.

It may be wroth posting an issue on https://github.com/googleapis/nodejs-bigtable/issues to get a better solution.

score 0 · Answer 2 · answered Nov 08 '18 at 07:40

For Integers, the function below would work.

function byteToInt(x){
    let val=0;
    for (let i=0; i<x.length; ++i) {        
        val+=x[i];        
        if(i<x.length-1) val = val << 8;
    }
    return val;
}

For float, NodeJS already provides a method to read from Buffer:

let buf = Buffer.from(value, 'binary');
let num = buf.readFloatBE(0);

It is also be possible to use the below, depending on the endianness and no of bits:

buf.readInt8(offset)

buf.readInt16BE(offset)

buf.readInt16LE(offset)

buf.readInt32BE(offset)

buf.readInt32LE(offset)

How to convert Bigtable HBase API returning byte array to integer or float in nodeJS?

2 Answers2