Words breaking into stange symbols in decrypted text when performing authenticated encryption/decrpytion using OpenSSL

Question

I am working on using OpenSSL in Linux Ubuntu to perform authenticated encryption/decryption. I am currently using C for the code, AES-CTR Encryption method for ciphering the text, HMAC-SHA256 for the tag. My code works by using Makefile for command "make"; after that, using the two instructions shown below and original.txt for original text, shared.key for the key to encrypt/decrypt. My problem here is that although no error occurs(I tried to handle as many errors as possible, but there can be more...) during encryption and decryption, when I open the decryption.txt file, generated after the decrpytion, the words are broken into stange symbols...

Here is the C code that I've written:

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <openssl/sha.h>
#include <openssl/evp.h>
#include <openssl/hmac.h>

char* readFile(char* fileName){
    int size;
    int count;
    FILE *fp = fopen(fileName, "r");
    fseek(fp, 0, SEEK_END);
    size = ftell(fp);
    char* buffer = malloc(size+1);
    char* result = malloc(size+1);
    memset(buffer, 0, size+1);
    memset(result, 0, size+1);
    fseek(fp, 0, SEEK_SET);
    while(1){
        char* pStr = fgets(buffer, size+1, fp);
        if(pStr==NULL)break;
        strcat(result, pStr);
    }
    fclose(fp);
    free(buffer);
    return result;
}

int main(int argc, char* argv[]){

    if(argc != 10){
        printf("ERROR\n");
        exit(2);
    }
    int ikey, iin, iout, itag;
    for(int i=2; i < 9;i+=2){
        if(!strcmp(argv[i], "-key")) ikey = i+1;
        else if(!strcmp(argv[i], "-in")) iin = i+1;
        else if(!strcmp(argv[i], "-out")) iout = i+1;
        else if(!strcmp(argv[i], "-tag")) itag = i+1;
    }
    if(ikey+iin+iout+itag != 24){
        printf("ERROR\n");
        exit(2);
    }
    char* key = readFile(argv[ikey]);
    char *iv = "0123456789012345";



    if(!strcmp(argv[1], "enc")){
        FILE *inFp = fopen(argv[iin],"rb");
        FILE *outFp = fopen(argv[iout], "wb");
        int inLen, outLen;
        char inBuf[BUFSIZ], outBuf[BUFSIZ+EVP_MAX_BLOCK_LENGTH];
        EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new();
        EVP_EncryptInit_ex(ctx, EVP_aes_256_ctr(), NULL, key, iv);
        while((inLen=fread(inBuf, 1, sizeof(inBuf), inFp))>0){

            if(!EVP_EncryptUpdate(ctx, outBuf, &outLen, inBuf, inLen)){
                printf("ERROR\n");
                EVP_CIPHER_CTX_cleanup(ctx);
                exit(2);
            }
            fwrite(outBuf, 1, outLen, outFp);
        }
        if(!EVP_EncryptFinal_ex(ctx, outBuf, &outLen)){
            printf("ERROR\n");
            EVP_CIPHER_CTX_cleanup(ctx);
            exit(2);
        }
        fwrite(outBuf, 1, outLen, outFp);
        EVP_CIPHER_CTX_cleanup(ctx);

        fclose(inFp);
        fclose(outFp);
        char* cipher = readFile(argv[iout]);
        char* hashVal;
        hashVal = HMAC(EVP_sha256(), key,strlen((char*)key), cipher, strlen((char*)cipher), NULL, NULL);
        FILE *tagFp = fopen(argv[itag], "w");
        fwrite(hashVal, 1, strlen(hashVal), tagFp);
        exit(0);
    }
    else if(!strcmp(argv[1], "dec")){
        FILE *inFp = fopen(argv[iin],"rb");
        char* cipher = readFile(argv[iin]);
        char* hashVal;
        hashVal = HMAC(EVP_sha256(), key,strlen(key), cipher, strlen(cipher), NULL, NULL);
        char* compareVal = readFile(argv[itag]);
        if(strcmp(compareVal, hashVal) != 0){
            printf("VERIFICATION FAILURE\n");
            exit(1);
        }

        FILE *outFp = fopen(argv[iout], "wb");
        int inLen, outLen;
        char inBuf[BUFSIZ], outBuf[BUFSIZ+EVP_MAX_BLOCK_LENGTH];
        EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new();
        EVP_DecryptInit_ex(ctx, EVP_aes_256_ctr(), NULL, key, iv);
        //printf("First Buf %s\n", outBuf);
        //printf("First Buf %s\n", inBuf);
        while((inLen=fread(inBuf, 1, sizeof(inBuf), inFp))>0){
            //printf("Second Buf %s\n", outBuf);
            //printf("Second Buf %s\n", inBuf);
            if(!EVP_DecryptUpdate(ctx, outBuf, &outLen, inBuf, inLen)){
                printf("ERROR\n");
                EVP_CIPHER_CTX_cleanup(ctx);
                exit(2);
            }
            //printf("Third Buf %s\n", outBuf);
            //printf("Third Buf %s\n", inBuf);
            //printf("First number %d\n", inLen);
            fwrite(outBuf, 1, outLen, outFp);
        }
        if(!EVP_DecryptFinal_ex(ctx, outBuf, &outLen)){
            printf("ERROR\n");
            EVP_CIPHER_CTX_cleanup(ctx);
            exit(2);
        }
        fwrite(outBuf, 1, outLen, outFp);
        EVP_CIPHER_CTX_cleanup(ctx);
        fclose(inFp);
        fclose(outFp);
        exit(0);
    }
    exit(0);
}

And below is the Makefile that I have used to compile the upper code with gcc, using "make" command:

cryp: cryp.o
    gcc -o $@ cryp.o -lssl -lcrypto

cryp.o: cryp.c

For the original.txt and shared.key, I just typed a simple sentence like "This is original text" and "This is shared key". With these elements, I used ./cryp enc -key shared.key -in original.txt -out encrypted.txt -tag encrypted.tag instruction in Linux terminal(Within the file storage that contains the elements) for encryption, and ./cryp dec -key shared.key -in encrypted.txt -tag encrypted.tag -out decrypted.txt instuction for decryption. After these actions, I expected a decrypted.txt that contains same content as original.txt, but the generated decrypted.txt showed words that are broken into stange symbols...What can I do to fix this situation? PLEASE help...

------------------------------------------------------------------------------------------------------------

Thanks for the help!

Answer 1

After the kind comments and answers, I examined the code, and fixed the readFile() function part, and it worked! I am posting this to save others who may suffer from same problem... The changed readFile() function looks like this:

char* readFile(const char* filename) {
    FILE* fp;
    char* buffer = NULL;
    size_t size = 0;
    size_t read_size;

    fp = fopen(filename, "rb");
    if (fp == NULL) {
        perror("Error opening file");
        return NULL;
    }

    while (!feof(fp)) {
        buffer = realloc(buffer, size + 1024);  // increase buffer size by 1024 bytes
        if (buffer == NULL) {
            perror("Error allocating memory");
            fclose(fp);
            return NULL;
        }
        read_size = fread(buffer + size, 1, 1024, fp);  // read 1024 bytes from file
        size += read_size;
    }

    buffer = realloc(buffer, size + 1);  // increase buffer size by 1 byte for null terminator
    if (buffer == NULL) {
        perror("Error allocating memory");
        fclose(fp);
        return NULL;
    }
    buffer[size] = '\0';  // add null terminator to end of buffer

    fclose(fp);
    return buffer;
}

Thanks for the help, AGAIN!

Answer 2

One problem that could be causing your issue if you're running on Windows is this code

FILE *fp = fopen(fileName, "r");
fseek(fp, 0, SEEK_END);
size = ftell(fp);

does not tell you the size of the file. (Nevermind even if it does "work" it will still fail anyway for files larger than 2 GB...)

fp is a text stream, and per 7.21.9.4 The ftell function, paragraph 2 of the (draft) C11 standard (bolding mine):

For a text stream, its file position indicator contains unspecified information, usable by the fseek function for returning the file position indicator for the stream to its position at the time of the ftell call; the difference between two such return values is not necessarily a meaningful measure of the number of characters written or read.

Whoever taught you to use fseek()/ftell() to get the size of a file taught you a fundamentally flawed method. It's not portable - it won't work on text streams on Windows, for just one example.

And you can't fix that by using a binary stream, either:

A binary stream need not meaningfully support fseek calls with a whence value of SEEK_END.

In fact:

Setting the file position indicator to end-of-file, as with fseek(file, 0, SEEK_END), has undefined behavior for a binary stream...

See How do you determine the size of a file in C?.