3

I'm getting the error The data to be decrypted exceeds the maximum for this modulus of 256 bytes when decrypting the RSA encrypted string in C#.

What i'm trying to achieve:

  1. Generate public/private key pairs in C# (RSA)
  2. Save private key in session/temp storage for use during decryption
  3. Send/return public key to client/JS for use with encryption
  4. Encrypt string (max 20 chars) in JS with public key and send to server
  5. Decrypt encrypted string in server using the private key saved in step 2

What works:

  1. Generation of public/private key pairs
  2. Encryption in JS using the library jsencrypt

Code written so far:

C#

    ///Source: https://stackoverflow.com/questions/17128038/c-sharp-rsa-encryption-decryption-with-transmission
    public static void GenerateKeys()
    {
        //CSP with a new 2048 bit rsa key pair
        var csp = new RSACryptoServiceProvider(2048);

        //Private key
        var privKey = csp.ExportParameters(true);

        //Public key
        var pubKey = csp.ExportParameters(false);

        string privKeyString = string.Empty;

        var sw = new System.IO.StringWriter();
        var xs = new System.Xml.Serialization.XmlSerializer(typeof(RSAParameters));
        xs.Serialize(sw, privKey);
        privKeyString = sw.ToString();

        //This will give public key in the following format which is required by the JS library
        //-----BEGIN PUBLIC KEY-----
        //XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
        //-----END PUBLIC KEY-----
        string publicKeyBase64 = ConvertPublicKeyForJS(pubKey);

        //Will be saved in sesssion for later use during decryption
        string privateKeyBase64 = Convert.ToBase64String(System.Text.Encoding.ASCII.GetBytes(privKeyString));

        //Save in session/temp location
        //Return publicKeyBase64 to JS
    }

    /// <summary>
    /// Source: https://stackoverflow.com/questions/28406888/c-sharp-rsa-public-key-output-not-correct/28407693#28407693
    /// </summary>
    /// <param name="publicKey"></param>
    /// <returns></returns>
    public static string ConvertPublicKeyForJS(RSAParameters publicKey)
    {
        string output = string.Empty;

        using (var stream = new MemoryStream())
        {
            var writer = new BinaryWriter(stream);
            writer.Write((byte)0x30); // SEQUENCE
            using (var innerStream = new MemoryStream())
            {
                var innerWriter = new BinaryWriter(innerStream);
                innerWriter.Write((byte)0x30); // SEQUENCE
                EncodeLength(innerWriter, 13);
                innerWriter.Write((byte)0x06); // OBJECT IDENTIFIER
                var rsaEncryptionOid = new byte[] { 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01 };
                EncodeLength(innerWriter, rsaEncryptionOid.Length);
                innerWriter.Write(rsaEncryptionOid);
                innerWriter.Write((byte)0x05); // NULL
                EncodeLength(innerWriter, 0);
                innerWriter.Write((byte)0x03); // BIT STRING
                using (var bitStringStream = new MemoryStream())
                {
                    var bitStringWriter = new BinaryWriter(bitStringStream);
                    bitStringWriter.Write((byte)0x00); // # of unused bits
                    bitStringWriter.Write((byte)0x30); // SEQUENCE
                    using (var paramsStream = new MemoryStream())
                    {
                        var paramsWriter = new BinaryWriter(paramsStream);
                        EncodeIntegerBigEndian(paramsWriter, publicKey.Modulus); // Modulus
                        EncodeIntegerBigEndian(paramsWriter, publicKey.Exponent); // Exponent
                        var paramsLength = (int)paramsStream.Length;
                        EncodeLength(bitStringWriter, paramsLength);
                        bitStringWriter.Write(paramsStream.GetBuffer(), 0, paramsLength);
                    }
                    var bitStringLength = (int)bitStringStream.Length;
                    EncodeLength(innerWriter, bitStringLength);
                    innerWriter.Write(bitStringStream.GetBuffer(), 0, bitStringLength);
                }
                var length = (int)innerStream.Length;
                EncodeLength(writer, length);
                writer.Write(innerStream.GetBuffer(), 0, length);
            }

            var base64 = Convert.ToBase64String(stream.GetBuffer(), 0, (int)stream.Length);

            StringBuilder sb = new StringBuilder();
            sb.AppendLine("-----BEGIN PUBLIC KEY-----");
            sb.AppendLine(base64);
            sb.AppendLine("-----END PUBLIC KEY-----");

            output = sb.ToString();
        }

        return output;
    }

    private static void EncodeLength(BinaryWriter stream, int length)
    {
        if (length < 0) throw new ArgumentOutOfRangeException("length", "Length must be non-negative");
        if (length < 0x80)
        {
            // Short form
            stream.Write((byte)length);
        }
        else
        {
            // Long form
            var temp = length;
            var bytesRequired = 0;
            while (temp > 0)
            {
                temp >>= 8;
                bytesRequired++;
            }
            stream.Write((byte)(bytesRequired | 0x80));
            for (var i = bytesRequired - 1; i >= 0; i--)
            {
                stream.Write((byte)(length >> (8 * i) & 0xff));
            }
        }
    }

    private static void EncodeIntegerBigEndian(BinaryWriter stream, byte[] value, bool forceUnsigned = true)
    {
        stream.Write((byte)0x02); // INTEGER
        var prefixZeros = 0;
        for (var i = 0; i < value.Length; i++)
        {
            if (value[i] != 0) break;
            prefixZeros++;
        }
        if (value.Length - prefixZeros == 0)
        {
            EncodeLength(stream, 1);
            stream.Write((byte)0);
        }
        else
        {
            if (forceUnsigned && value[prefixZeros] > 0x7f)
            {
                // Add a prefix zero to force unsigned if the MSB is 1
                EncodeLength(stream, value.Length - prefixZeros + 1);
                stream.Write((byte)0);
            }
            else
            {
                EncodeLength(stream, value.Length - prefixZeros);
            }
            for (var i = prefixZeros; i < value.Length; i++)
            {
                stream.Write(value[i]);
            }
        }
    }

    public static string DecryptValue(string cypherText)
    {
        string plainTextData = string.Empty;

        string privKeyBase64 = ""; //get value from session/temp storage

        string privKeyString = System.Text.Encoding.ASCII.GetString(Convert.FromBase64String(privKeyBase64));

        var sr = new System.IO.StringReader(privKeyString);
        var xs = new System.Xml.Serialization.XmlSerializer(typeof(RSAParameters));
        var privKey = (RSAParameters)xs.Deserialize(sr);

        var csp = new RSACryptoServiceProvider();
        csp.ImportParameters(privKey);

        var bytesCypherText = Convert.FromBase64String(cypherText);

        //Problematic line
        var bytesPlainTextData = csp.Decrypt(bytesCypherText, false);

        plainTextData = System.Text.Encoding.Unicode.GetString(bytesPlainTextData);

        return plainTextData;
    }

JS

   function encrypt(msg, publicKey) {
        var jsEncrypt = new JSEncrypt();
        jsEncrypt.setPublicKey(publicKey);
        var encrypted = jsEncrypt.encrypt(msg);

        var base64result = btoa(encrypted);

        return base64result;            
    }

Update/Variation:

If i make the key size 1024 instead of 2048, the error i'm getting is The data to be decrypted exceeds the maximum for this modulus of 128 bytes.. Total bytes of the encrypted string is 172 based on what i can see in the debugger.

Update | Solution:

  1. Remove this line in the JS var base64result = btoa(encrypted);. Just return the encrypted value directly.
  2. In C# decrypt method, change this line var bytesPlainTextData = csp.Decrypt(bytesCypherText, false); to var bytesPlainTextData = csp.Decrypt(bytesCypherText, RSAEncryptionPadding.Pkcs1);.

Notes:

  • Please ignore the incorrect method return types, this is still a POC and a lot of values are being filled manually.
  • I'm not an expert in security matters so a lot of the codes have been picked from different sources.
Sang Suantak
  • 5,213
  • 1
  • 27
  • 46
  • It is highly unusual to encrypt a *message* with public-private key cryptography, due to the length limitations ([RSA tops out at 245 bytes](https://security.stackexchange.com/a/33445/34115)) and performance. It is much more common to encrypt a *key*. If you need to send a message, you can use a symmetric key (e.g. AES) to encrypt the message then use RSA to encrypt the key. – John Wu Dec 09 '19 at 07:06
  • Thanks @JohnWu, i'll look into it. Also, i've updated my question with a variation, please check and let me know what u think. – Sang Suantak Dec 09 '19 at 07:13
  • I don't have a specific answer, but here is what I would do, OP, if I were stuck on this problem. I would start out by writing both the decryption and encryption steps in the *same language*, and get a known-good code base working with known good inputs and outputs for each step. Then I would implement in the other language, comparing inputs and outputs step by step and ensuring they are all identical. – John Wu Dec 09 '19 at 08:04
  • Thanks for the suggestion @JohnWu, but given the limited time i have and my limited knowledge on the subject, it would not be feasible. – Sang Suantak Dec 09 '19 at 08:22

1 Answers1

2

You are facing a limitation of asymmetric encryption. It is very slow for large chunks of data and the encryption string size is limited by the RSA key size you are using.

RSA is usually used to exchange symmetric keys and handle large part of data. If you must use asymmetric for a big load of data, then you need to break your payload to smaller ones and reconstruct on the other side.

RSA is only able to encrypt data to a maximum amount of your key size (2048 bits = 256 bytes) minus padding / header data (11 bytes for PKCS#1 v1.5 padding).

If as you say, you are only sending 20 chars, then check indeed with a breakpoint, that your decrypt function gets indeed a small enough cypher text. If not you need to backtrack and check where you send the wrong thing.

It might also be that the standards of RSA are not the same in JSEncrypt and RSA in C# as it can be seen in this SO answer here

Athanasios Kataras
  • 25,191
  • 4
  • 32
  • 61
  • Please check my updated question. Also, the max string to be encrypted won't be more than 20 chars as described. – Sang Suantak Dec 09 '19 at 07:15
  • Log the js Encrypt output and the cypher text in your debugger. Are they the same side by side? – Athanasios Kataras Dec 09 '19 at 07:22
  • I'm directly copying the console output and feeding it to the decrypt method. – Sang Suantak Dec 09 '19 at 07:35
  • Does jsencrypt, decrypt function work if you call it right after the encryption? – Athanasios Kataras Dec 09 '19 at 07:44
  • Ok then, your problem is most probably solved by this: https://stackoverflow.com/questions/47685688/rsa-encryption-and-decryption-successed-in-js-but-decryption-fails-in-c-sharp – Athanasios Kataras Dec 09 '19 at 08:33
  • The linked question seems to be related but not exactly the same approach. Moreover, the whole approach the OP took to generate the public/private keys is not there so it's difficult to derive any solution from that. I placed the JS encryptor error correction code but it doesn't fix the issue. – Sang Suantak Dec 09 '19 at 09:48
  • Found my mistakes!!!!! The first issue is with the base 64 encoding done in JS (`btoa`), it's not required. This is the reason why the bytes were exceeding the expected limit. Second is the one as u suggested, the padding, changing it to `Pkcs1` does the trick. Will check with more values to be sure. Thanks for pointing me towards the right direction. – Sang Suantak Dec 09 '19 at 10:58
  • Should i mark this answer as the accepted one or should i create another answer with the proper solution and mark that as the accepted answer? – Sang Suantak Dec 09 '19 at 10:59