Difference between openssl rc4 command line encrypt and cpp file implementing rc4

Question

I am working on trying to encrypt a text file via RC4 with a cpp file that I wrote with openssl/rc4 headers, and then decrypt via the command line to show that my implementation is correct.

My terminal command for the file is below, and the cpp file is below it, along with the terminal compile command I used for it.

There barely seems to be any information about this anywhere online, outside of some vague youtube videos that explain how the RC4 cypher works(which I already know). I can't find anything in the man pages to explain the details of the openssl implementation.

Any pointers on why my cpp file isn't decrypting to the original content would be much appreciated. I am tearing my hair out over here trying to figure this out. Thanks in advance.

(and yes, I understand there are vulnerabilities that make RC4 less of a good option, but right now, I just want to understand how these work)

command line encrypt:

openssl rc4-40 -k PASSword123 -in /home/chris/Desktop/test.txt -out /home/chris/Desktop/ssloutput.txt -p -nosalt

cpp file compilation:

g++ rc4.cpp -o rc4 -lssl -lcrypto

cpp file:

#include <openssl/rc4.h>
#include <string>
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>

int main()
{
    int fd = open("/home/chris/Desktop/ssloutput.txt", O_RDWR);
    unsigned char keygen[12] = "PASSword123";
    RC4_KEY key;

    struct stat st;
    fstat(fd, &st);
    int size = st.st_size;

    unsigned char* fileIn;
    fileIn = (unsigned char*) calloc(size, sizeof(char));
    pread(fd, fileIn, size, 0);
    unsigned char *fileOut = (unsigned char*)malloc(size);

    RC4_set_key(&key, 16, keygen);
    RC4(&key, size, fileIn, fileOut);
    close(fd);

    int fd2 = open("/home/chris/Desktop/rc4output.txt", O_RDWR | O_CREAT);
    pwrite(fd2, fileOut, size, 0);
    close(fd2);

    free(fileIn);
    free(fileOut);

    return 0;
}

Answer 1

So, here's a version of your code with a lot of error checking added, bugs fixed, odd stuff (Using O_RDWR with open() when you're only reading or writing? pread()? pwrite()?) cleaned up, and using EVP_BytesToKey() like the -k option to openssl rc4 uses (That was the key (heh) factor):

#include <fcntl.h>
#include <openssl/evp.h>
#include <openssl/rc4.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>

int main() {
  int fd = open("ssloutput.txt", O_RDONLY);
  if (fd < 0) {
    perror("open ssloutput.txt");
    return 1;
  }

  struct stat st;
  if (fstat(fd, &st) < 0) {
    perror("fstat");
    return 1;
  }
  size_t size = st.st_size;

  unsigned char *fileIn = calloc(size, 1);
  if (!fileIn) {
    perror("calloc");
    return 1;
  }
  if (read(fd, fileIn, size) != (ssize_t)size) {
    perror("read");
    return 1;
  }
  close(fd);

  unsigned char *fileOut = malloc(size);
  if (!fileOut) {
    perror("malloc");
    return 1;
  }

  // Prepare the key according to the same rules as openssl rc4 -k foo
  char keygen[] = "PASSword123";
  RC4_KEY key;
  unsigned char computed_key[16];
  if (EVP_BytesToKey(EVP_rc4(), EVP_sha256(), NULL,
                     (const unsigned char *)keygen, strlen(keygen), 1,
                     computed_key, NULL) != 16) {

    fputs("Error calculating rc4 key!\n", stderr);
    return 1;
  }
  // Should match the one printed out by openssl rc4 -p
  fputs("key=", stdout);
  for (size_t n = 0; n < sizeof computed_key; n += 1) {
    printf("%02hhx", computed_key[n]);
  }
  putchar('\n');

  RC4_set_key(&key, sizeof computed_key, computed_key);
  RC4(&key, size, fileIn, fileOut);

  int fd2 = open("rc4output.txt", O_WRONLY | O_TRUNC | O_CREAT, 0644);
  if (fd2 < 0) {
    perror("open rc4output.txt");
    return 1;
  }
  if (write(fd2, fileOut, size) != (ssize_t)size) {
    perror("write");
    return 1;
  }
  close(fd2);

  free(fileIn);
  free(fileOut);

  return 0;
}

Demonstration:

$ cat input.txt
the quick brown dog jumped over the lazy red fox.
$ gcc -o myrc4 -O -Wall -Wextra myrc4.c -lcrypto
$ openssl rc4 -k PASSword123 -md sha256 -p -nosalt -in input.txt -out ssloutput.txt
key=B554C1D224D8EF1738ED4EE238317463
$ ./myrc4
key=B554C1D224D8EF1738ED4EE238317463
$ cat rc4output.txt
the quick brown dog jumped over the lazy red fox.