Openssl and Windows CryptoAPI compatibility issue

Question

I have problem in encrypted communication between windows application and server app. Client app is running at Windows, written in C and uses CryptoAPI. Server application uses PHP and Openssl extension. As cipher algorithm AES-256-CBC was chosen. Using the same algorithm Openssl and CryptoAPI produces different results. The same problem I had with RC2-CBC algorithm. This online tool http://asecuritysite.com/Encryption/openssl generates the same result as openssl, so I conclude that bug in C code.

PHP code:

<?php
//$flag = OPENSSL_RAW_DATA;
$flag = false;
//this string will encode
$dataString = 'some data string';
$pass = "1234567812345678";
$method = "aes-256-cbc";

$iv = "Zievrs8NZievrs8N";
echo "original:\n";
var_dump($dataString);
$encryptedMessage = openssl_encrypt($dataString, $method, $pass, $flag, $iv);
echo "after encrypt:\n";
var_dump($encryptedMessage);
echo "vector:\n";
var_dump($iv);
$decryptedMessage = openssl_decrypt($encryptedMessage, $method, $pass, $flag, $iv);
echo "after decrypt:\n";
var_dump($decryptedMessage);

Output:

original:
string(16) "some data string"
after encrypt:
string(44) "9O8UAaRRCfneeRbRCeiYi9nOM8F2KA6gtkAsvPliUdA="
vector:
string(16) "Zievrs8NZievrs8N"
after decrypt:
string(16) "some data string"

C code:

BOOL SetKey(BYTE* szKey, DWORD dwKeySize, HCRYPTPROV* m_hProv, HCRYPTHASH* m_hHash, HCRYPTKEY* m_hKey)
{
    BOOL m_fOK= TRUE;
    if (*m_hProv == 0) {
        m_fOK = CryptAcquireContextA(m_hProv, NULL, 
            NULL, //MS_DEF_PROV_A, 
            PROV_RSA_AES, 
            CRYPT_VERIFYCONTEXT 
        );
    }
    if (m_fOK && (*m_hHash != 0)) {
        m_fOK = CryptDestroyHash(*m_hHash); 
        m_hHash = 0;
    }
    if (m_fOK && (*m_hHash == 0)) {
        m_fOK = CryptCreateHash(*m_hProv, CALG_SHA_256, 0, 0, m_hHash);
    }
    if (m_fOK) {
        m_fOK = CryptHashData(*m_hHash, (BYTE*)szKey, dwKeySize, 0);
    }
    if (m_fOK) {
        m_fOK = CryptDeriveKey(*m_hProv, CALG_AES_256, *m_hHash, CRYPT_EXPORTABLE | CRYPT_NO_SALT, m_hKey);
    }
    if (m_fOK) {
        DWORD mode = CRYPT_MODE_CBC;
        m_fOK = CryptSetKeyParam(*m_hKey, KP_MODE, (BYTE*)&mode, 0);
    }
    if (m_fOK) {
        BYTE iv[] = {'Z','i','e','v','r','s','8','N','Z','i','e','v','r','s','8','N',0};
        m_fOK = CryptSetKeyParam(*m_hKey, KP_IV, (BYTE*)iv, 0);
    }

    return m_fOK;
}

BOOL EncryptDecrypt(BYTE* pData, BYTE** pRes, DWORD* dwDataLen, BYTE* pKey, DWORD dwKeySize, BOOL fEncrypt)
{
    HCRYPTPROV m_hProv = 0;
    HCRYPTHASH m_hHash = 0;
    HCRYPTKEY  m_hKey  = 0;

    BOOL m_fOK= TRUE;
    m_fOK = SetKey(pKey, dwKeySize, &m_hProv, &m_hHash, &m_hKey);
    if (fEncrypt) {
        DWORD dwTotalBufferSize = 0;
        DWORD dwNewLen = *dwDataLen;
        if((m_fOK = CryptEncrypt(m_hKey, 0, TRUE, 0, NULL, &dwNewLen, dwTotalBufferSize))) {
            *pRes = (BYTE*)malloc(dwNewLen);
            memcpy(*pRes, pData, *dwDataLen);
            dwTotalBufferSize = dwNewLen;
            if(!(m_fOK = CryptEncrypt(m_hKey, 0, TRUE, 0, *pRes, dwDataLen, dwTotalBufferSize))) {
                free(*pRes);
                *pRes = NULL;
                *dwDataLen = 0;
            }
        }
    }
    else  {
        *pRes = (BYTE*)malloc(*dwDataLen);
        memcpy(*pRes, pData, *dwDataLen);
        if(!(m_fOK = CryptDecrypt(m_hKey, 0, TRUE, 0, *pRes, dwDataLen))) {
            DWORD err = GetLastError();
            char msg[100];
            wsprintfA(msg, "err = %d\n", err);
            OutputDebugStringA(msg);
            free(*pRes);
            *pRes = NULL;
            *dwDataLen = 0;
        }
    }

    if (m_hKey)  CryptDestroyKey(m_hKey); 
    if (m_hHash) CryptDestroyHash(m_hHash); 
    if (m_hProv) CryptReleaseContext(m_hProv, 0); 

    return m_fOK;
}

void main() {
    const char* data = "some data string";
    BYTE* res = NULL;
    DWORD len = strlen(data);
    EncryptDecrypt((BYTE*)data, &res, &len, (BYTE*)"1234567812345678", 16, TRUE);
    size_t len_en = 0;
    char* base64 = base64_encode(res, len, &len_en);
    printf("base64 = %s\n", base64);
}

Output:

base64 = miFMwk4/ZwjMLsnV4Po9UdWxix32TrK5BcSgSKYr384=

I think you've forgotten to pad your data using pkcs7; see also [this answer](http://stackoverflow.com/questions/10916284/how-to-encrypt-decrypt-data-in-php/10945097#10945097). — Ja͢ck, Mar 26 '15 at 06:15
Thanks for nice answer, it shed some light, but unfortunately did not help. According to MSDN CryptoAPI uses PKCS#5, but manually disabling it and implementing PKCS#7 padding makes no difference to result(also observing values of unallocated bytes in the end of buffer after decryption gives the nice fingerprint of PKCS#7). I tried to use AES-128 and check encoding results by RFC3602 example and get the right output(with builtin and manual padding) in PHP. But not in C program. I suggest that crypto-context misconfigured(provider name, type, hash algorithm, maybe something else) — Cargo Tan, Mar 26 '15 at 10:35
Forgot to add the link [RFC3602](https://www.ietf.org/rfc/rfc3602.txt) — Cargo Tan, Mar 26 '15 at 11:42

score 0 · Accepted Answer · answered Mar 26 '15 at 06:12

0

Encrypted output is different. It means that key which is ultimately used is different or data is different. But data is same, hence key must be different.

It means in the process of key generation, something is different. It might be possible that OpenSSL may be using some other key deriving function which is not visible here. Try to use some standard algorithm for key generation. Instead of hashing, try not to use hash.

answered Mar 26 '15 at 06:12

doptimusprime

9,115
6
52
90

Thanks for answer, but I don't quite understand how to do that without hash. Maybe example or link? – Cargo Tan Mar 26 '15 at 11:39
In CryptDeriveKey, directly pass the password. – doptimusprime Mar 26 '15 at 13:21
I don't think it supposed to work that way. Here is function [declaration](https://msdn.microsoft.com/en-us/library/windows/desktop/aa379916%28v=vs.85%29.aspx) from MSDN and there are no mention of direct password input in description. `BOOL WINAPI CryptDeriveKey(HCRYPTPROV hProv, ALG_ID Algid, HCRYPTHASH hBaseData, DWORD dwFlags, HCRYPTKEY *phKey);` – Cargo Tan Mar 26 '15 at 13:58
In the end you was right. Code fixes I will post below. – Cargo Tan Mar 27 '15 at 16:17

score 0 · Answer 2 · answered Mar 27 '15 at 16:20

I finally got proper results from CryptoAPI by importing key as PLAINTEXTBLOB.

Here is not final, but workable fixes to SetKey function.

#include <WinCrypt.h>

typedef struct {
  BLOBHEADER hdr;
  DWORD      dwKeySize;
  BYTE       rgbKeyData[16];
} PLAINTEXTKEYBLOB_t;

BOOL SetKey(BYTE* szKey, DWORD dwKeySize, HCRYPTPROV* m_hProv, HCRYPTHASH* m_hHash, HCRYPTKEY* m_hKey)
{
    BOOL m_fOK= TRUE;
    if (*m_hProv == 0) {
        m_fOK = CryptAcquireContextW(m_hProv, NULL, 
            MS_ENH_RSA_AES_PROV, //MS_DEF_PROV_A, 
            PROV_RSA_AES, 
            CRYPT_NEWKEYSET | CRYPT_VERIFYCONTEXT 
        );
    }
    if (m_fOK && (*m_hHash != 0)) {
        m_fOK = CryptDestroyHash(*m_hHash); 
        m_hHash = 0;
    }

    if(m_fOK) {
        BYTE key[] = {0x06, 0xa9, 0x21, 0x40, 0x36, 0xb8, 0xa1, 0x5b, 0x51, 0x2e, 0x03, 0xd5, 0x34, 0x12, 0x00, 0x06};

        PLAINTEXTKEYBLOB_t blob;
        blob.hdr.bType = PLAINTEXTKEYBLOB;
        blob.hdr.bVersion = 2;
        blob.hdr.reserved = 0;
        blob.hdr.aiKeyAlg = CALG_AES_128;
        blob.dwKeySize = 16;
        for(int i=0; i<16; i++) {
            //blob.rgbKeyData[16-1-i] = key[i];
            blob.rgbKeyData[i] = key[i];
        }
        m_fOK = CryptImportKey(*m_hProv, (BYTE*)&blob, sizeof(PLAINTEXTKEYBLOB_t), 0, NULL, m_hKey);
    }

    if (m_fOK) {
        DWORD mode = CRYPT_MODE_CBC;
        m_fOK = CryptSetKeyParam(*m_hKey, KP_MODE, (BYTE*)&mode, 0);
    }
    if (m_fOK) {
        DWORD mode = 0;
        DWORD dwDataLen = sizeof(mode);
        m_fOK = CryptGetKeyParam(*m_hKey, KP_PADDING, (BYTE*)&mode, &dwDataLen, 0);
        mode = 0;
        //m_fOK = CryptSetKeyParam(*m_hKey, KP_PADDING, (BYTE*)&mode, 0);
    }
    if (m_fOK) {
        BYTE iv[] = {0x3d, 0xaf, 0xba, 0x42, 0x9d, 0x9e, 0xb4, 0x30, 0xb4, 0x22, 0xda, 0x80, 0x2c, 0x9f, 0xac, 0x41};
        m_fOK = CryptSetKeyParam(*m_hKey, KP_IV, (BYTE*)iv, 0);
    }

    return m_fOK;
}

Openssl and Windows CryptoAPI compatibility issue

2 Answers2