RSA encryption with Java and decryption with JavaScript

Question

I am trying to encrypt with Java (using javax.crypto.Cipher) and decrypt with JavaScript (using crypto.subtle). What I am doing is, I make the JavaScript side generate the key pair, then send the public key to the Java side by the following:

$(window).on("load", function () {

    const enc = new TextEncoder();
    const dec = new TextDecoder();

    crypto.subtle.generateKey({
      name: "RSA-OAEP",
      modulusLength: 1024,
      publicExponent: new Uint8Array([1, 0, 1]),
      hash: "SHA-256"
    },
        true,
        ["encrypt", "decrypt"]
    ).then(function ({ privateKey, publicKey }) {

        crypto.subtle.exportKey("spki", publicKey).then(function (spki) {

            const strPublicKey = spkiToString(spki);
            // SEND THE PUBLIC KEY TO THE SERVER (JAVA)
            
        });


    });


});

function spkiToString(keydata) {
    var keydataS = arrayBufferToString(keydata);
    return window.btoa(keydataS);
}

function arrayBufferToString(buffer) {
    var binary = '';
    var bytes = new Uint8Array(buffer);
    var len = bytes.byteLength;
    for (var i = 0; i < len; i++) {
        binary += String.fromCharCode(bytes[i]);
    }
    return binary;
}

The server uses the public key for encryption:

 try {
    String publicKey = ""// this will come from the JS side
    String message = "encrypt me"//  
    byte[] publicBytes = Base64.getDecoder().decode(publicKey).getBytes());
    X509EncodedKeySpec keySpec = new X509EncodedKeySpec(publicBytes);
    KeyFactory keyFactory = KeyFactory.getInstance("RSA");
    PublicKey pubKey = keyFactory.generatePublic(keySpec);

    Cipher cipher = Cipher.getInstance("RSA/ECB/OAEPPadding");
    OAEPParameterSpec oaepParams = new OAEPParameterSpec("SHA-256", "MGF1", new MGF1ParameterSpec("SHA-256"),
            PSource.PSpecified.DEFAULT);
    cipher.init(Cipher.ENCRYPT_MODE, pubKey, oaepParams);

    return Base64.getEncoder().encode(cipher.doFinal(message));

} catch (Exception e) {
    e.printStacktrace();
}

return new byte[0];

The resulted value is then returned to the Javascript side, so it can decrypt the message:

const encryptedToken = "" // this will be obtained from the server
crypto.subtle.decrypt({
    name: "RSA-OAEP",
    hash: { name: "SHA-256" }
},
    privateKey,
    enc.encode(atob(encryptedToken))
).then(function (result) {
    console.log("decrypted", dec.decode(result))
}).catch(function (e) {
  console.log(e);
})

When the Javascript tries to decrypt, it throws a DOMException with no message (check the attached image).

What I am doing wrong? Thank you in advanced.

score 2 · Accepted Answer · answered Feb 03 '21 at 23:09

The problem is the enc.encode(atob(encryptedToken)) line in the decrypt() method of the last JavaScript code snippet. The UTF-8 encoding corrupts the data, preventing successful decryption. If this is changed, decryption works as shown in the following.
The JavaScript code below is essentially the same as the first code snippet from the question, with the addition of exporting the private key in PKCS8 format:

crypto.subtle.generateKey(
    {
        name: "RSA-OAEP",
        modulusLength: 1024,
        publicExponent: new Uint8Array([1, 0, 1]),
        hash: "SHA-256"
    },
    true,
    ["encrypt", "decrypt"]
).then(function ({ privateKey, publicKey }) {
    crypto.subtle.exportKey("spki", publicKey).then(function (spki) {
        const strPublicKey = spkiToString(spki);  
        console.log(strPublicKey.replace(/(.{56})/g,'$1\n'));
        crypto.subtle.exportKey("pkcs8", privateKey).then(function (pkcs8) {
            const strPrivateKey = spkiToString(pkcs8);  
            console.log(strPrivateKey.replace(/(.{56})/g,'$1\n'));
        });
    });
});             

function spkiToString(keydata) {
    var keydataS = arrayBufferToString(keydata);
    return window.btoa(keydataS);
}

function arrayBufferToString(buffer) {
    var binary = '';
    var bytes = new Uint8Array(buffer);
    var len = bytes.byteLength;
    for (var i = 0; i < len; i++) {
        binary += String.fromCharCode(bytes[i]);
    }
    return binary;
}

This code was used to generate e.g. the following public key in X.509/SPKI format:

MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC1F8+EvG9XP8jSXItV89QtlYy/5Z+arMegvMwsasS5IIUdr4b4eE2FGoDalaqyAxWOg/pBkzfBWAQkhuKz3i14OsBYQl1QDDm3yfmI498SsE7tZyrENCfTGrPwoCrQmEwTWOCfIBCh+mGRAUTgcsQO/g8pIFglF3QTTzlu3n0KhQIDAQAB

and private key in PKCS8 format:

MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBALUXz4S8b1c/yNJci1Xz1C2VjL/ln5qsx6C8zCxqxLkghR2vhvh4TYUagNqVqrIDFY6D+kGTN8FYBCSG4rPeLXg6wFhCXVAMObfJ+Yjj3xKwTu1nKsQ0J9Mas/CgKtCYTBNY4J8gEKH6YZEBROByxA7+DykgWCUXdBNPOW7efQqFAgMBAAECgYAK6oUFVNCHW15xI8f4ZerH1qh11tMgoUKlU0whb0wtdqLfj7mcl6/gkqDqzDPOaDYv8Y+vzT6CppoVU5YtznpCF4YRLuOfeAkY0kT9C7w62lu1C1aFMDS1Eydv0a10t001sp0W5U8J0LMgPpevPlksv2t9gZa08yGsBnVX9BIXjwJBAOrlsV6LsxNBnSKqXhZf1+uQe1vpPPzF3IXTvJzd4LhamcnImYayrg4Zjgj71+/0BFdWT9qGxtKGwJJGIjrMDG8CQQDFXLIrFMHVpdjrsAaQXvPWTSVIfVayi6Uib1HpXKiLJ53snebsBrBiShbAsJjrgWXzdurky6nGIlp5NV7i//pLAkEA4XaxRfe/XhdXtWNjxgQe41ueHH2GbXWZktbGrqcFwM4t2RHz0ueEy8HZpGPfQ9GrrQ0Kvs0o4AA5rO0mg9tBfwJBAJMWuaaH6spatyc4Yjv4uEuv5Sh4WUPp9WGLi4WbS/Whyf4N1It1lME8LGbhdqaWIrBnoTpxWw9SjREmqJgPZK8CQBAAI6IUkCeE+Lwub+akoBFuqyyIdIpIfXu4ntyxnZemmCNdotEfNL3yp0J3Rw6TpXyPDN/4uOrxt/aY2heXAKM=

The public key was applied in the Java code to generate the ciphertext below:

jCt9rD/6Q6OsjH+bd1XKB2FhDYTwzupQsFnwjKkrxulC3ztZx0j9/Zr6hBeCbFrdYFtxZi+j8lyyLJCHv0hpN0S5F/O6v/mhMIgCTWCmpWqcLqKC2zDWo180uL+dMysZm2JaBHzWA9VjnVTdVY3aRTWfu1fpEpTK6W9ESTVSS8Y=

The following JavaScript code is essentially the same as the 3rd code snippet extended by the import of the private key generated above. In addition, the ciphertext generated above is applied:

var pkcs8B64 = "MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBALUXz4S8b1c/yNJci1Xz1C2VjL/ln5qsx6C8zCxqxLkghR2vhvh4TYUagNqVqrIDFY6D+kGTN8FYBCSG4rPeLXg6wFhCXVAMObfJ+Yjj3xKwTu1nKsQ0J9Mas/CgKtCYTBNY4J8gEKH6YZEBROByxA7+DykgWCUXdBNPOW7efQqFAgMBAAECgYAK6oUFVNCHW15xI8f4ZerH1qh11tMgoUKlU0whb0wtdqLfj7mcl6/gkqDqzDPOaDYv8Y+vzT6CppoVU5YtznpCF4YRLuOfeAkY0kT9C7w62lu1C1aFMDS1Eydv0a10t001sp0W5U8J0LMgPpevPlksv2t9gZa08yGsBnVX9BIXjwJBAOrlsV6LsxNBnSKqXhZf1+uQe1vpPPzF3IXTvJzd4LhamcnImYayrg4Zjgj71+/0BFdWT9qGxtKGwJJGIjrMDG8CQQDFXLIrFMHVpdjrsAaQXvPWTSVIfVayi6Uib1HpXKiLJ53snebsBrBiShbAsJjrgWXzdurky6nGIlp5NV7i//pLAkEA4XaxRfe/XhdXtWNjxgQe41ueHH2GbXWZktbGrqcFwM4t2RHz0ueEy8HZpGPfQ9GrrQ0Kvs0o4AA5rO0mg9tBfwJBAJMWuaaH6spatyc4Yjv4uEuv5Sh4WUPp9WGLi4WbS/Whyf4N1It1lME8LGbhdqaWIrBnoTpxWw9SjREmqJgPZK8CQBAAI6IUkCeE+Lwub+akoBFuqyyIdIpIfXu4ntyxnZemmCNdotEfNL3yp0J3Rw6TpXyPDN/4uOrxt/aY2heXAKM=";
const pkcs8StrDER = atob(pkcs8B64);
const pkcs8DER = str2ab(pkcs8StrDER);
crypto.subtle.importKey(
    "pkcs8",
    pkcs8DER,
    {
        name: "RSA-OAEP",
        modulusLength: 1024,
        publicExponent: new Uint8Array([1, 0, 1]),
        hash: "SHA-256",
    },
    true,
    ["decrypt"]
).then(function(privateKey){
    const dec = new TextDecoder();
    const encryptedToken = "jCt9rD/6Q6OsjH+bd1XKB2FhDYTwzupQsFnwjKkrxulC3ztZx0j9/Zr6hBeCbFrdYFtxZi+j8lyyLJCHv0hpN0S5F/O6v/mhMIgCTWCmpWqcLqKC2zDWo180uL+dMysZm2JaBHzWA9VjnVTdVY3aRTWfu1fpEpTK6W9ESTVSS8Y="; 
    crypto.subtle.decrypt(
        {
            name: "RSA-OAEP",
        },
        privateKey,
        str2ab(atob(encryptedToken))
    ).then(function (result) {
        console.log("decrypted:", dec.decode(result))
    }).catch(function (e) {
        console.log(e);
    });             
});

function str2ab(str) {
    const buf = new ArrayBuffer(str.length);
    const bufView = new Uint8Array(buf);
    for (let i = 0, strLen = str.length; i < strLen; i++) {
        bufView[i] = str.charCodeAt(i);
    }
    return buf;
}

Instead of the UTF-8 encoding which corrupts the data, the function str2ab() is used which converts the Base64 decoded data into an ArrayBuffer.
The substitution _base64ToArrayBuffer(encryptedToken) suggested in my comment is equally possible. Unlike str2ab(), _base64ToArrayBuffer() also performs the base64 decoding.
Running the code results in the plaintext encrypt me of the Java code.

RSA encryption with Java and decryption with JavaScript

1 Answers1

Linked