Encrypt/decrypt string and file using rsa and windows crypto api

Question

I am working on a program for secure file encryption and data exchange.

As I am not a professional in encryption, I get stucked... Maybe someone can help me to complete and correct some of my functions.

I think my encryptFileWithPublicKey function works, but I can't test it because the decryptFileWithPrivateKey function does not. The same for the encryptStringWithPublicKey function prints out some garbage which seems to me it does work. But again I can't test it because I don't know how to write to decrypt function correctly.

I have this code in my main-function:

LPCWSTR szPemPrivateKey = 
    L"-----BEGIN CERTIFICATE-----"
    "MIIEowIBAAKCAQEAqRLhZXK29Xo5YdSoMdAeMHwDYAmThPSJzbQaBhVLCY1DTQr0"
    "JRkvd+0xfdwih97bWUXVpxuOgYH9hofIzZGPZSErmCkBbzs8Q344ECFraG/1zNmH"
    "M2QaMLewztbZtpt9whIgaxwCRuoJWSmFzlp2ny0PtOzHeWf9c+iqEAnYxqrocGED"
    "Rdf08/8siZLQDZugSjsmg9C0oLhGvpTruh0S4Gg9IPjemDD6IWhOESbZC239f1++"
    "hsBLRIADWzRFt4Ka4p+VcWpJXSjkifUYLsvu9MceJgUDvGbViB1sFOhCc6C3B3e5"
    "y2WPfi+p56T+lPWy8BjmYqpWBlYWp8YTruUdxQIDAQABAoIBAEukl9U0iqivI0ci"
    "xpFqxGb6xOPNxQNiCNlQb6gzFJxUaDzq3qpOZ09nfpIJ0CUrzsVVGi6mhI5qowvy"
    "aiOWsSoUuhVcmzixrtefe+OzoANcvAMHe5bBfXo4ZNFVjJhHXu9Z/skNAfPKN559"
    "llDpZlL27SUMHn6xyNJ30MZWMXdIOkBa6FCuu35ygpUt9gKQ6vZ6xUNvODnoTjVs"
    "A/+d3G+3NMkZwkFy2ygJtpfrPhhby4AxUtW4/Ms6FImB7R0XvNJ4+x43PCFE61N/"
    "c77BXyIE7DRfoh6+sbaQnzZIvYEucTXdroZUlfNaDryGRClue8c9Mc9KCfje0Pk9"
    "0SRrCiECgYEA6eETNhjIKN1q3De99UujMLpy46JxyulDvQj8wsvYmozAubngL4fQ"
    "1DcOSsUzpM7u0FHQW0mDeZXn3/qXIT2bQHquIgo3DiOHXeE4iiKwQbtFl2t4klwo"
    "l4lpUBvZrqGxXRyicOsOwZ4dWUUxnsaCShtjMzxdSxs8hs282E7c1RkCgYEAuRCr"
    "Pi8oHr4S4JM0Os1dbkQYlpLJvTANFX6ACpXOiC+Uyi6Klanxc7ENM4VZQoqEd/nf"
    "3EsYWqjwD5RHjm0P0mKDpTGorU7WhsBj/qp6SUB9f9ZgI0QACfcI7HFD1oiskJi1"
    "+NtsLw9ZeunX9Ru0jtbQbL9vLajZ7qJjCPpil40CgYEAqsG7Dg8v7K4kfB5yoaqU"
    "vloS0eJjpJuE6nqqWOsN/WDwwWlVpj5KGPcNh0fNufGf6RciwdO9m0L4pc5HE2zw"
    "cjZf9thwc/E6vy9V4X0MJSG+4JwFzw5cUK9x5Zw/5cGGRo5Gh50HDBQOJ4Regyc6"
    "LzMpzVmwX8rF+UIzM9b4YtkCgYBzgMLP+dCZd1oCzf0QIjKI6nlKKO0+ZaoU2zIA"
    "6UTQ3JmJooieJkXrRfqOSFDQZfGNi+e67yeRFdVWc95BNzJjxVaSmK7FE+pyicsf"
    "G7lc1KI/UkYIXJ2Fmqf4Ii0LzZueV6FjY1de+76GJ0LA2PzVFZ9Z8BTss3DpLnKP"
    "HO7B1QKBgCqY+fzJO5JlZgQ7++70VjPVMH8sbpAYMUhadLb27IYdnfCMfMc0aos8"
    "9W3cUPJEJ6+HExKiBs+bjIIppPz+jlb20K445O6SwPRkg38yFEd0EjRc+PkxZXM4"
    "2i0rEOzV/2RqvJdMJkWW2SSz1TBuvW5sVgKPtyx6lCi1QWQ+h3Gj"
    "-----END CERTIFICATE-----";


LPCWSTR szPemPublicKey =
    L"-----BEGIN CERTIFICATE-----"
    "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqRLhZXK29Xo5YdSoMdAe"
    "MHwDYAmThPSJzbQaBhVLCY1DTQr0JRkvd+0xfdwih97bWUXVpxuOgYH9hofIzZGP"
    "ZSErmCkBbzs8Q344ECFraG/1zNmHM2QaMLewztbZtpt9whIgaxwCRuoJWSmFzlp2"
    "ny0PtOzHeWf9c+iqEAnYxqrocGEDRdf08/8siZLQDZugSjsmg9C0oLhGvpTruh0S"
    "4Gg9IPjemDD6IWhOESbZC239f1++hsBLRIADWzRFt4Ka4p+VcWpJXSjkifUYLsvu"
    "9MceJgUDvGbViB1sFOhCc6C3B3e5y2WPfi+p56T+lPWy8BjmYqpWBlYWp8YTruUd"
    "xQIDAQAB"
    "-----END CERTIFICATE-----";


int main()
{
    RSA rsa = RSA();

    _TCHAR infile[] = L"c:\\test.txt";
    _TCHAR outfile[] = L"c:\\test.enc";
    _TCHAR decfile[] = L"c:\\test.dec.txt";

    rsa.EncryptFileWithPublicKey(szPemPublicKey, infile, outfile);
    rsa.DecryptFileWithPrivateKey(szPemPrivateKey, outfile, decfile);

    BYTE ciphertext = NULL;
    BYTE cleartext[] = "test";

    rsa.EncryptStringWithPublicKey(szPemPublicKey, cleartext, &ciphertext);
    printf("CIPHER: %s\n", ciphertext);
}

And the RSA class:

#include "pch.h"
#include "RSA.h"


RSA::RSA()
{
}


RSA::~RSA()
{
}

// Keys
/*
int Keys(_TCHAR* strPublicKeyFile, _TCHAR* strPrivateKeyFile)
{
    // Variables
    HCRYPTPROV hCryptProv = NULL;
    HCRYPTKEY hKey = NULL;
    DWORD dwPublicKeyLen = 0;
    DWORD dwPrivateKeyLen = 0;
    BYTE* pbPublicKey = NULL;
    BYTE* pbPrivateKey = NULL;
    HANDLE hPublicKeyFile = NULL;
    HANDLE hPrivateKeyFile = NULL;
    DWORD lpNumberOfBytesWritten = 0;

    __try
    {
        // Acquire access to key container
        _tprintf(_T("CryptAcquireContext...\n"));
        if (!CryptAcquireContext(&hCryptProv, _T("AlejaCMa.EncryptDecrypt"), NULL, PROV_RSA_FULL, 0))
        {
            // Error
            _tprintf(_T("CryptAcquireContext error 0x%x\n"), GetLastError());

            // Try to create a new key container
            if (!CryptAcquireContext(&hCryptProv, _T("AlejaCMa.EncryptDecrypt"), NULL, PROV_RSA_FULL, CRYPT_NEWKEYSET))
            {
                // Error
                _tprintf(_T("CryptAcquireContext error 0x%x\n"), GetLastError());
                return 1;
            }
        }

        // Generate new key pair
        _tprintf(_T("CryptGenKey...\n"));
        if (!CryptGenKey(hCryptProv, AT_KEYEXCHANGE, CRYPT_ARCHIVABLE, &hKey))
        {
            // Error
            _tprintf(_T("CryptGenKey error 0x%x\n"), GetLastError());
            return 1;
        }

        // Get public key size
        _tprintf(_T("CryptExportKey...\n"));
        if (!CryptExportKey(hKey, NULL, PUBLICKEYBLOB, 0, NULL, &dwPublicKeyLen))
        {
            // Error
            _tprintf(_T("CryptExportKey error 0x%x\n"), GetLastError());
            return 1;
        }

        // Create a buffer for the public key
        _tprintf(_T("malloc...\n"));
        if (!(pbPublicKey = (BYTE *)malloc(dwPublicKeyLen)))
        {
            // Error
            _tprintf(_T("malloc error 0x%x\n"), GetLastError());
            return 1;
        }

        // Get public key
        _tprintf(_T("CryptExportKey...\n"));
        if (!CryptExportKey(hKey, NULL, PUBLICKEYBLOB, 0, pbPublicKey, &dwPublicKeyLen))
        {
            // Error
            _tprintf(_T("CryptExportKey error 0x%x\n"), GetLastError());
            return 1;
        }

        // Get private key size
        _tprintf(_T("CryptExportKey...\n"));
        if (!CryptExportKey(hKey, NULL, PRIVATEKEYBLOB, 0, NULL, &dwPrivateKeyLen))
        {
            // Error
            _tprintf(_T("CryptExportKey error 0x%x\n"), GetLastError());
            return 1;
        }

        // Create a buffer for the private key
        _tprintf(_T("malloc...\n"));
        if (!(pbPrivateKey = (BYTE *)malloc(dwPrivateKeyLen)))
        {
            // Error
            _tprintf(_T("malloc error 0x%x\n"), GetLastError());
            return 1;
        }


        // Get private key
        _tprintf(_T("CryptExportKey...\n"));
        if (!CryptExportKey(hKey, NULL, PRIVATEKEYBLOB, 0, pbPrivateKey, &dwPrivateKeyLen))
        {
            // Error
            _tprintf(_T("CryptExportKey error 0x%x\n"), GetLastError());
            return 1;
        }

        // Create a file to save the public key
        _tprintf(_T("CreateFile...\n"));
        if ((hPublicKeyFile = CreateFile(
            strPublicKeyFile,
            GENERIC_WRITE,
            0,
            NULL,
            CREATE_ALWAYS,
            FILE_ATTRIBUTE_NORMAL,
            NULL
        )) == INVALID_HANDLE_VALUE)
        {
            // Error
            _tprintf(_T("CreateFile error 0x%x\n"), GetLastError());
            return 1;
        }


        // Write the public key to the file
        _tprintf(_T("WriteFile...\n"));
        if (!WriteFile(
            hPublicKeyFile,
            (LPCVOID)pbPublicKey,
            dwPublicKeyLen,
            &lpNumberOfBytesWritten,
            NULL
        ))
        {
            // Error
            _tprintf(_T("WriteFile error 0x%x\n"), GetLastError());
            return 1;
        }

        // Create a file to save the private key
        _tprintf(_T("CreateFile...\n"));
        if ((hPrivateKeyFile = CreateFile(
            strPrivateKeyFile,
            GENERIC_WRITE,
            0,
            NULL,
            CREATE_ALWAYS,
            FILE_ATTRIBUTE_NORMAL,
            NULL
        )) == INVALID_HANDLE_VALUE)
        {
            // Error
            _tprintf(_T("CreateFile error 0x%x\n"), GetLastError());
            return 1;
        }

        // Write the private key to the file
        _tprintf(_T("WriteFile...\n"));
        if (!WriteFile(
            hPrivateKeyFile,
            (LPCVOID)pbPrivateKey,
            dwPrivateKeyLen,
            &lpNumberOfBytesWritten,
            NULL
        ))
        {
            // Error
            _tprintf(_T("WriteFile error 0x%x\n"), GetLastError());
            return 1;
        }
        return 0;
    }

    __finally
    {
        // Clean up
        if (!pbPublicKey) {
            _tprintf(_T("free...\n"));
            free(pbPublicKey);
        }

        if (!pbPrivateKey) {
            _tprintf(_T("free...\n"));
            free(pbPrivateKey);
        }

        if (hPublicKeyFile) {
            _tprintf(_T("CloseHandle...\n"));
            CloseHandle(hPublicKeyFile);
        }

        if (hPrivateKeyFile) {
            _tprintf(_T("CloseHandle...\n"));
            CloseHandle(hPrivateKeyFile);
        }

        if (hKey) {
            _tprintf(_T("CryptDestroyKey...\n"));
            CryptDestroyKey(hKey);
        }

        if (hCryptProv) {
            _tprintf(_T("CryptReleaseContext...\n"));
            CryptReleaseContext(hCryptProv, 0);
        }
    }
}
*/
// End of Keys

// Encrypt file
int RSA::EncryptFileWithPublicKey(LPCWSTR szPemPublicKey, _TCHAR* strPlainFile, _TCHAR* strEncryptedFile)
{
    // Variables
    HCRYPTPROV hCryptProv = NULL;
    HCRYPTKEY hKey = NULL;
    DWORD dwPublicKeyLen = 0;
    DWORD dwDataLen = 0;
    DWORD dwEncryptedLen = 0;
    BYTE* pbPublicKey = NULL;
    BYTE* pbData = NULL;
    HANDLE hPublicKeyFile = NULL;
    HANDLE hEncryptedFile = NULL;
    HANDLE hPlainFile = NULL;
    DWORD lpNumberOfBytesWritten = 0;

    BYTE derPubKey[2048];
    DWORD derPubKeyLen = 2048;
    CERT_PUBLIC_KEY_INFO *publicKeyInfo = NULL;
    DWORD publicKeyInfoLen = 0;
    HANDLE hFile = NULL;

    __try
    {
        /*
        // Acquire access to key container
        _tprintf(_T("CryptAcquireContext...\n"));
        if (!CryptAcquireContext(&hCryptProv, NULL, NULL, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT))
        {
            // Error
            _tprintf(_T("CryptAcquireContext error 0x%x\n"), GetLastError());
            return 1;
        }
        */

        /*
         * Read the public key cert from the file
         */
        /*
        hFile = CreateFileA("c:\\pub.pem", GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
        if(hFile == INVALID_HANDLE_VALUE)
        {
            fprintf(stderr, "Failed to open file. error: %d\n", GetLastError());
        }

        if(!ReadFile(hFile, pemPubKey, 2048, &readLen, NULL))
        {
            fprintf(stderr, "Failed to read file. error: %d\n", GetLastError());
        }
        */
        /*
         * Convert from PEM format to DER format - removes header and footer and decodes from base64
         */
        if(!CryptStringToBinary(szPemPublicKey, 0, CRYPT_STRING_BASE64HEADER, derPubKey, &derPubKeyLen, NULL, NULL))
        {
            //fprintf(stderr, "CryptStringToBinary failed. Err: %d\n", GetLastError());
            return -1;
        }

        /*
         * Decode from DER format to CERT_PUBLIC_KEY_INFO
         */
        if(!CryptDecodeObjectEx(X509_ASN_ENCODING, X509_PUBLIC_KEY_INFO, derPubKey, derPubKeyLen, CRYPT_ENCODE_ALLOC_FLAG, NULL, &publicKeyInfo, &publicKeyInfoLen))
        {
            //fprintf(stderr, "CryptDecodeObjectEx 1 failed. Err: %p\n", GetLastError());
            return -1;
        }

        /*
         * Acquire context
         */
        if(!CryptAcquireContext(&hCryptProv, NULL, NULL, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT))
        {
            {
                //printf("CryptAcquireContext failed - err=0x%x.\n", GetLastError());
                return -1;
            }
        }

        /*
         * Import the public key using the context
         */
        if(!CryptImportPublicKeyInfo(hCryptProv, X509_ASN_ENCODING, publicKeyInfo, &hKey))
        {
            //fprintf(stderr, "CryptImportPublicKeyInfo failed. error: %d\n", GetLastError());
            return -1;
        }
        LocalFree(publicKeyInfo);

        /*
         * Now use hKey to encrypt whatever you need.
         */

        // Open plain text file
        //_tprintf(_T("CreateFile...\n"));
        if ((hPlainFile = CreateFile(strPlainFile, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, NULL)) == INVALID_HANDLE_VALUE)
        {
            // Error
            //_tprintf(_T("CreateFile error 0x%x\n"), GetLastError());
            return 1;
        }

        // Get file size
        //_tprintf(_T("GetFileSize...\n"));
        if ((dwDataLen = GetFileSize(hPlainFile, NULL)) == INVALID_FILE_SIZE)
        {
            // Error
            //_tprintf(_T("GetFileSize error 0x%x\n"), GetLastError());
            return 1;
        }

        // Create a buffer for the plain text
        //_tprintf(_T("malloc...\n"));
        if (!(pbData = (BYTE *)malloc(dwDataLen)))
        {
            // Error
            //_tprintf(_T("malloc error 0x%x\n"), GetLastError());
            return 1;
        }

        // Read plain text
        //_tprintf(_T("ReadFile...\n"));
        if (!ReadFile(hPlainFile, pbData, dwDataLen, &dwDataLen, NULL))
        {
            // Error
            //_tprintf(_T("ReadFile error 0x%x\n"), GetLastError());
            return 1;
        }

        // Get lenght for encrypted data
        if (!CryptEncrypt(hKey, NULL, TRUE, 0, NULL, &dwEncryptedLen, 0))
        {
            // Error
            //_tprintf(_T("CryptEncrypt error 0x%x\n"), GetLastError());
            return 1;
        }

        // Create a buffer for encrypted data
        //_tprintf(_T("malloc...\n"));
        if (!(pbData = (BYTE *)realloc(pbData, dwEncryptedLen)))
        {
            // Error
            //_tprintf(_T("malloc error 0x%x\n"), GetLastError());
            return 1;
        }

        // Encrypt data
        if (!CryptEncrypt(hKey, NULL, TRUE, 0, pbData, &dwDataLen, dwEncryptedLen))
        {
            // Error
            //_tprintf(_T("CryptEncrypt error 0x%x\n"), GetLastError());
            return 1;
        }

        // Create a file to save the encrypted data
        //_tprintf(_T("CreateFile...\n"));
        if ((hEncryptedFile = CreateFile(strEncryptedFile, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL)) == INVALID_HANDLE_VALUE)
        {
            // Error
            //_tprintf(_T("CreateFile error 0x%x\n"), GetLastError());
            return 1;
        }

        // Write the public key to the file
        //_tprintf(_T("WriteFile...\n"));
        if (!WriteFile(hEncryptedFile, (LPCVOID)pbData, dwDataLen, &lpNumberOfBytesWritten, NULL))
        {
            // Error
            //_tprintf(_T("WriteFile error 0x%x\n"), GetLastError());
            return 1;
        }
        return 0;
    }

    __finally
    {
        // Clean up
        if (!pbPublicKey) {
            //_tprintf(_T("free...\n"));
            free(pbPublicKey);
        }

        if (!pbData) {
            //_tprintf(_T("free...\n"));
            free(pbData);
        }

        if (hPublicKeyFile) {
            //_tprintf(_T("CloseHandle...\n"));
            CloseHandle(hPublicKeyFile);
        }

        if (hPlainFile) {
            //_tprintf(_T("CloseHandle...\n"));
            CloseHandle(hPlainFile);
        }

        if (hEncryptedFile) {
            //_tprintf(_T("CloseHandle...\n"));
            CloseHandle(hEncryptedFile);
        }

        if (hKey) {
            //_tprintf(_T("CryptDestroyKey...\n"));
            CryptDestroyKey(hKey);
        }

        if (hCryptProv) {
            //_tprintf(_T("CryptReleaseContext...\n"));
            CryptReleaseContext(hCryptProv, 0);
        }
    }
}
// End of Encrypt

// Encrypt string
int RSA::EncryptStringWithPublicKey(LPCWSTR szPemPublicKey, BYTE* pbDataPlaintext, BYTE* pbDataCiphertext)
{
    // Variables
    HCRYPTPROV hCryptProv = NULL;
    HCRYPTKEY hKey = NULL;
    DWORD dwPublicKeyLen = 0;
    DWORD dwDataLen = 0;
    DWORD dwEncryptedLen = 0;
    BYTE* pbPublicKey = NULL;
    //BYTE* pbData = NULL;
    HANDLE hPublicKeyFile = NULL;
    HANDLE hEncryptedFile = NULL;
    HANDLE hPlainFile = NULL;
    DWORD lpNumberOfBytesWritten = 0;

    int readLen;
    BYTE derPubKey[2048];
    DWORD derPubKeyLen = 2048;
    CERT_PUBLIC_KEY_INFO *publicKeyInfo;
    DWORD publicKeyInfoLen;
    HANDLE hFile;

    if (!CryptStringToBinary(szPemPublicKey, 0, CRYPT_STRING_BASE64HEADER, derPubKey, &derPubKeyLen, NULL, NULL))
    {
        //fprintf(stderr, "CryptStringToBinary failed. Err: %d\n", GetLastError());
        return -1;
    }

    /*
    * Decode from DER format to CERT_PUBLIC_KEY_INFO
    */
    if (!CryptDecodeObjectEx(X509_ASN_ENCODING, X509_PUBLIC_KEY_INFO, derPubKey, derPubKeyLen, CRYPT_ENCODE_ALLOC_FLAG, NULL, &publicKeyInfo, &publicKeyInfoLen))
    {
        //fprintf(stderr, "CryptDecodeObjectEx 1 failed. Err: %p\n", GetLastError());
        return -1;
    }

    /*
    * Acquire context
    */
    if (!CryptAcquireContext(&hCryptProv, NULL, NULL, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT))
    {
        {
            //printf("CryptAcquireContext failed - err=0x%x.\n", GetLastError());
            return -1;
        }
    }

    /*
    * Import the public key using the context
    */
    if (!CryptImportPublicKeyInfo(hCryptProv, X509_ASN_ENCODING, publicKeyInfo, &hKey))
    {
        //fprintf(stderr, "CryptImportPublicKeyInfo failed. error: %d\n", GetLastError());
        return -1;
    }
    LocalFree(publicKeyInfo);

    /*
    * Now use hKey to encrypt whatever you need.
    */

    // Create a buffer for the plain text
        //_tprintf(_T("malloc...\n"));
    if (!(pbDataPlaintext = (BYTE *)malloc(dwDataLen)))
    {
        // Error
        _tprintf(_T("malloc error 0x%x\n"), GetLastError());
        return 1;
    }



    // Get lenght for encrypted data
    if (!CryptEncrypt(hKey, NULL, TRUE, 0, NULL, &dwEncryptedLen, 0))
    {
        // Error
        _tprintf(_T("CryptEncrypt error 0x%x\n"), GetLastError());
        return 1;
    }

    // Create a buffer for encrypted data
    //_tprintf(_T("malloc...\n"));
    if (!(pbDataPlaintext = (BYTE *)realloc(pbDataPlaintext, dwEncryptedLen)))
    {
        // Error
        _tprintf(_T("malloc error 0x%x\n"), GetLastError());
        return 1;
    }

    // Encrypt data
    if (!CryptEncrypt(hKey, NULL, TRUE, 0, pbDataPlaintext, &dwDataLen, dwEncryptedLen))
    {
        // Error
        _tprintf(_T("CryptEncrypt error 0x%x\n"), GetLastError());
        return 1;
    }

    printf("S: %s\n", pbDataPlaintext);

    // Clean up
    if (!pbPublicKey) {
        //_tprintf(_T("free...\n"));
        free(pbPublicKey);
    }

    if (!pbDataPlaintext) {
        //_tprintf(_T("free...\n"));
        free(pbDataPlaintext);
    }

    if (hPublicKeyFile) {
        //_tprintf(_T("CloseHandle...\n"));
        CloseHandle(hPublicKeyFile);
    }

    if (hPlainFile) {
        //_tprintf(_T("CloseHandle...\n"));
        CloseHandle(hPlainFile);
    }

    if (hEncryptedFile) {
        //_tprintf(_T("CloseHandle...\n"));
        CloseHandle(hEncryptedFile);
    }

    if (hKey) {
        //_tprintf(_T("CryptDestroyKey...\n"));
        CryptDestroyKey(hKey);
    }

    if (hCryptProv) {
        //_tprintf(_T("CryptReleaseContext...\n"));
        CryptReleaseContext(hCryptProv, 0);
    }
}
// End of Encrypt

// Decrypt file
int RSA::DecryptFileWithPrivateKey(LPCWSTR szPemPrivateKey, _TCHAR* strEncryptedFile, _TCHAR* strPlainFile)
{
    // Variables
    HCRYPTPROV hCryptProv = NULL;
    HCRYPTKEY hKey = NULL;
    DWORD dwPrivateKeyLen = 0;
    DWORD dwDataLen = 0;
    BYTE* pbPrivateKey = NULL;
    BYTE* pbData = NULL;
    HANDLE hPrivateKeyFile = NULL;
    HANDLE hEncryptedFile = NULL;
    HANDLE hPlainFile = NULL;
    DWORD lpNumberOfBytesWritten = 0;

    BYTE derPrivateKey[2048];
    DWORD derPrivateKeyLen = 2048;
    CRYPT_PRIVATE_KEY_INFO *privateKeyInfo = NULL;
    DWORD privateKeyInfoLen = 0;
    HANDLE hFile = NULL;

    __try
    {
        // Acquire access to key container
        /*
         * Convert from PEM format to DER format - removes header and footer and decodes from base64
         */
        if (!CryptStringToBinary(szPemPrivateKey, 0, CRYPT_STRING_BASE64HEADER, derPrivateKey, &derPrivateKeyLen, NULL, NULL))
        {
            //fprintf(stderr, "CryptStringToBinary failed. Err: %d\n", GetLastError());
            return -1;
        }

        /*
         * Decode from DER format to CERT_PUBLIC_KEY_INFO
         */
        if (!CryptDecodeObjectEx(X509_ASN_ENCODING, X509_PUBLIC_KEY_INFO, derPrivateKey, derPrivateKeyLen, CRYPT_ENCODE_ALLOC_FLAG, NULL, &privateKeyInfo, &privateKeyInfoLen))
        {
            //fprintf(stderr, "CryptDecodeObjectEx 1 failed. Err: %p\n", GetLastError());
            return -1;
        }

        /*
         * Acquire context
         */
        if (!CryptAcquireContext(&hCryptProv, NULL, NULL, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT))
        {
            {
                //printf("CryptAcquireContext failed - err=0x%x.\n", GetLastError());
                return -1;
            }
        }

        /*
         * Import the public key using the context
         */
        if (!CryptImportKey(hCryptProv, X509_ASN_ENCODING, privateKey, &hKey))
        {
            //fprintf(stderr, "CryptImportPublicKeyInfo failed. error: %d\n", GetLastError());
            return -1;
        }
        LocalFree(privateKeyInfo);
        // Import private key
        _tprintf(_T("CryptImportKey...\n"));
        if (!CryptImportKey(hCryptProv, pbPrivateKey, dwPrivateKeyLen, 0, 0, &hKey))
        {
            // Error
            _tprintf(_T("CryptImportKey error 0x%x\n"), GetLastError());
            return 1;
        }

        // Open encrypted file
        _tprintf(_T("CreateFile...\n"));
        if ((hEncryptedFile = CreateFile(
            strEncryptedFile,
            GENERIC_READ,
            FILE_SHARE_READ,
            NULL,
            OPEN_EXISTING,
            FILE_FLAG_SEQUENTIAL_SCAN,
            NULL
        )) == INVALID_HANDLE_VALUE)
        {
            // Error
            _tprintf(_T("CreateFile error 0x%x\n"), GetLastError());
            return 1;
        }

        // Get file size
        _tprintf(_T("GetFileSize...\n"));
        if ((dwDataLen = GetFileSize(hEncryptedFile, NULL)) == INVALID_FILE_SIZE)
        {
            // Error
            _tprintf(_T("GetFileSize error 0x%x\n"), GetLastError());
            return 1;
        }

        // Create a buffer for the encrypted data
        _tprintf(_T("malloc...\n"));
        if (!(pbData = (BYTE *)malloc(dwDataLen)))
        {   // Error
            _tprintf(_T("malloc error 0x%x\n"), GetLastError());
            return 1;
        }

        // Read encrypted data
        _tprintf(_T("ReadFile...\n"));
        if (!ReadFile(hEncryptedFile, pbData, dwDataLen, &dwDataLen, NULL))
        {
            // Error
            _tprintf(_T("ReadFile error 0x%x\n"), GetLastError());
            return 1;
        }

        // Get lenght for plain text
        if (!CryptDecrypt(hKey, NULL, TRUE, 0, pbData, &dwDataLen))
        {
            // Error
            _tprintf(_T("CryptDecrypt error 0x%x\n"), GetLastError());
            return 1;
        }

        // Create a file to save the plain text
        _tprintf(_T("CreateFile...\n"));
        if ((hPlainFile = CreateFile(
            strPlainFile,
            GENERIC_WRITE,
            0,
            NULL,
            CREATE_ALWAYS,
            FILE_ATTRIBUTE_NORMAL,
            NULL
        )) == INVALID_HANDLE_VALUE)
        {
            // Error
            _tprintf(_T("CreateFile error 0x%x\n"), GetLastError());
            return 1;
        }

        // Write the plain text the file
        _tprintf(_T("WriteFile...\n"));
        if (!WriteFile(
            hPlainFile,
            (LPCVOID)pbData,
            dwDataLen,
            &lpNumberOfBytesWritten,
            NULL
        ))
        {
            // Error
            _tprintf(_T("WriteFile error 0x%x\n"), GetLastError());
            return 1;
        }

        return 0;
    }

    __finally
    {
        // Clean up
        if (!pbPrivateKey) {
            _tprintf(_T("free...\n"));
            free(pbPrivateKey);
        }

        if (!pbData) {
            _tprintf(_T("free...\n"));
            free(pbData);
        }

        if (hPrivateKeyFile) {
            _tprintf(_T("CloseHandle...\n"));
            CloseHandle(hPrivateKeyFile);
        }

        if (hEncryptedFile) {
            _tprintf(_T("CloseHandle...\n"));
            CloseHandle(hEncryptedFile);
        }

        if (hPlainFile) {
            _tprintf(_T("CloseHandle...\n"));
            CloseHandle(hPlainFile);
        }

        if (hKey) {
            _tprintf(_T("CryptDestroyKey...\n"));
            CryptDestroyKey(hKey);
        }

        if (hCryptProv) {
            _tprintf(_T("CryptReleaseContext...\n"));
            CryptReleaseContext(hCryptProv, 0);
        }
    }
}
// End of Decrypt

// Decrypt string
int RSA::DecryptStringWithPrivateKey(LPCWSTR szPemPrivateKey, BYTE* pbDataCiphertext, BYTE* pbDataPlaintext)
{

}

*As i am not a professional in encryption* -- Then do not try to do this yourself. This is one aspect where you leave this to the experts and get a ready-made library that has examples, has been tested, and works. Don't waste your time debugging someone else's code if it doesn't work out-of-the-box. If there is an issue, report it to the author of the library, or if you're the author (this isn't clear), my advice is to drop trying to do this yourself. Encryption routines again, are best done by libraries whose business is to make enryption/decryption work correctly. — PaulMcKenzie, Feb 02 '19 at 13:39
Possible duplicate of [Window C/C++ Crypto API Examples and tips](https://stackoverflow.com/q/4796590/608639). It should provide you with some inspiration. Also see friends like [How to generate and use public key cryptography in WinCrypt](https://stackoverflow.com/q/4869454/608639), [RSA ncryption with C++ and WinCrypt Library and Decryption with C#](https://stackoverflow.com/q/44303924/608639), [How to import private key in PEM format using WinCrypt and C++?](https://stackoverflow.com/q/8412838/608639), etc. — jww, Feb 02 '19 at 18:48
i saw many of these examples while googling... but none answering my question or giving me a hint or working example function to fix my code — user8092240, Feb 02 '19 at 19:35

score 0 · Answer 1 · edited Oct 02 '19 at 03:46

I could fix the decryptFileWithPrivateKey() function myself, so I hope it will help anyone else fighting with the same problems. Updated code follows.

Now I'm just stuck with the string en/decryption function... Maybe someone would help...

// Decrypt file
int RSA::DecryptFileWithPrivateKey(LPCWSTR szPemPrivateKey, _TCHAR* strEncryptedFile, _TCHAR* strPlainFile)
{
    // Variables
    HCRYPTPROV hCryptProv = NULL;
    HCRYPTKEY hKey = NULL;
    DWORD dwPrivateKeyLen = 0;
    DWORD dwDataLen = 0;
    BYTE* pbPrivateKey = NULL;
    BYTE* pbData = NULL;
    HANDLE hPrivateKeyFile = NULL;
    HANDLE hEncryptedFile = NULL;
    HANDLE hPlainFile = NULL;
    DWORD lpNumberOfBytesWritten = 0;

    BYTE derPrivateKey[2048];
    DWORD derPrivateKeyLen = 2048;

    DWORD dwBufferLen = 0, cbKeyBlob = 0, cbSignature = 0, i;
    LPBYTE pbBuffer = NULL, pbKeyBlob = NULL, pbSignature = NULL;
    HCRYPTPROV hProv = NULL;
    HCRYPTHASH hHash = NULL;

    __try
    {
        if (!CryptStringToBinary(szPemPrivateKey, 0, CRYPT_STRING_BASE64HEADER, NULL, &dwBufferLen, NULL, NULL))
        {
            printf("Failed to convert BASE64 private key. Error 0x%.8X\n", GetLastError());
            return -1;
        }

        pbBuffer = (LPBYTE)LocalAlloc(0, dwBufferLen);
        if (!CryptStringToBinary(szPemPrivateKey, 0, CRYPT_STRING_BASE64HEADER, pbBuffer, &dwBufferLen, NULL, NULL))
        {
            printf("Failed to convert BASE64 private key. Error 0x%.8X\n", GetLastError());
            return -1;
        }

        if (!CryptDecodeObjectEx(X509_ASN_ENCODING | PKCS_7_ASN_ENCODING, PKCS_RSA_PRIVATE_KEY, pbBuffer, dwBufferLen, 0, NULL, NULL, &cbKeyBlob))
        {
            printf("Failed to parse private key. Error 0x%.8X\n", GetLastError());
            return -1;
        }

        pbKeyBlob = (LPBYTE)LocalAlloc(0, cbKeyBlob);
        if (!CryptDecodeObjectEx(X509_ASN_ENCODING | PKCS_7_ASN_ENCODING, PKCS_RSA_PRIVATE_KEY, pbBuffer, dwBufferLen, 0, NULL, pbKeyBlob, &cbKeyBlob))
        {
            printf("Failed to parse private key. Error 0x%.8X\n", GetLastError());
            return -1;
        }

        // Create a temporary and volatile CSP context in order to import
        // the key and use for signing
        if (!CryptAcquireContext(&hProv, NULL, MS_ENHANCED_PROV, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT))
        {
            printf("CryptAcquireContext failed with error 0x%.8X\n", GetLastError());
            return -1;
        }

        if (!CryptImportKey(hProv, pbKeyBlob, cbKeyBlob, NULL, 0, &hKey))
        {
            printf("CryptImportKey for private key failed with error 0x%.8X\n", GetLastError());
            return -1;
        }


        // Open encrypted file
        _tprintf(_T("CreateFile...\n"));
        if ((hEncryptedFile = CreateFile(
            strEncryptedFile,
            GENERIC_READ,
            FILE_SHARE_READ,
            NULL,
            OPEN_EXISTING,
            FILE_FLAG_SEQUENTIAL_SCAN,
            NULL
        )) == INVALID_HANDLE_VALUE)
        {
            // Error
            _tprintf(_T("CreateFile error 0x%x\n"), GetLastError());
            return 1;
        }
        // Get file size
        _tprintf(_T("GetFileSize...\n"));
        if ((dwDataLen = GetFileSize(hEncryptedFile, NULL)) == INVALID_FILE_SIZE)
        {
            // Error
            _tprintf(_T("GetFileSize error 0x%x\n"), GetLastError());
            return 1;
        }
        // Create a buffer for the encrypted data
        _tprintf(_T("malloc...\n"));
        if (!(pbData = (BYTE *)malloc(dwDataLen)))
        {
            // Error
            _tprintf(_T("malloc error 0x%x\n"), GetLastError());
            return 1;
        }
        // Read encrypted data
        _tprintf(_T("ReadFile...\n"));
        if (!ReadFile(hEncryptedFile, pbData, dwDataLen, &dwDataLen, NULL))
        {
            // Error
            _tprintf(_T("ReadFile error 0x%x\n"), GetLastError());
            return 1;
        }
        // Get lenght for plain text
        if (!CryptDecrypt(hKey, NULL, TRUE, 0, pbData, &dwDataLen))
        {
            // Error
            _tprintf(_T("CryptDecrypt error 0x%x\n"), GetLastError());
            return 1;
        }
        // Create a file to save the plain text
        _tprintf(_T("CreateFile...\n"));
        if ((hPlainFile = CreateFile(
            strPlainFile,
            GENERIC_WRITE,
            0,
            NULL,
            CREATE_ALWAYS,
            FILE_ATTRIBUTE_NORMAL,
            NULL
        )) == INVALID_HANDLE_VALUE)
        {
            // Error
            _tprintf(_T("CreateFile error 0x%x\n"), GetLastError());
            return 1;
        }
        // Write the plain text the file
        _tprintf(_T("WriteFile...\n"));
        if (!WriteFile(
            hPlainFile,
            (LPCVOID)pbData,
            dwDataLen,
            &lpNumberOfBytesWritten,
            NULL
        ))
        {
            // Error
            _tprintf(_T("WriteFile error 0x%x\n"), GetLastError());
            return 1;
        }
        return 0;
    }
    __finally
    {
        // Clean up
        if (!pbPrivateKey) {
            _tprintf(_T("free...\n"));
            free(pbPrivateKey);
        }
        if (!pbData) {
            _tprintf(_T("free...\n"));
            free(pbData);
        }
        if (hPrivateKeyFile) {
            _tprintf(_T("CloseHandle...\n"));
            CloseHandle(hPrivateKeyFile);
        }
        if (hEncryptedFile) {
            _tprintf(_T("CloseHandle...\n"));
            CloseHandle(hEncryptedFile);
        }
        if (hPlainFile) {
            _tprintf(_T("CloseHandle...\n"));
            CloseHandle(hPlainFile);
        }
        if (hKey) {
            _tprintf(_T("CryptDestroyKey...\n"));
            CryptDestroyKey(hKey);
        }
        if (hCryptProv) {
            _tprintf(_T("CryptReleaseContext...\n"));
            CryptReleaseContext(hCryptProv, 0);
        }
    }
}
// End of Decrypt

Encrypt/decrypt string and file using rsa and windows crypto api

1 Answers1