4

I am new to making engines in openssl. Basically I want to implement an OpenSSL RSA engine that uses the functions I mentioned while encrypting and decrypting. My engine compiles and loads but it seems that it is not using the functions I want it to use for encryption and decryption.

#include <openssl/opensslconf.h>
#include <stdio.h>
#include <string.h>
#include <openssl/crypto.h>
#include <openssl/buffer.h>
#include <openssl/engine.h>
#include <openssl/rsa.h>
#include <openssl/bn.h>
#include <openssl/err.h>

static int
eng_rsa_pub_enc (int flen, const unsigned char *from,
     unsigned char *to, RSA * rsa, int padding)
{

    printf ("Engine is encrypting using pub key \n");
    RSA_public_encrypt (flen, from, to, rsa, RSA_PKCS1_PADDING);
}

static int
eng_rsa_pub_dec (int flen, const unsigned char *from,
     unsigned char *to, RSA * rsa, int padding)
{

   printf ("Engine is decrypting using pub key \n");
   RSA_public_decrypt (flen, from, to, rsa, RSA_PKCS1_PADDING);
}

static int
eng_rsa_priv_enc (int flen, const unsigned char *from, unsigned char *to,
      RSA * rsa, int padding __attribute__ ((unused)))
{
   printf ("Engine is encrypting using priv key \n");
   RSA_private_encrypt (flen, from, to, rsa, RSA_PKCS1_PADDING);
}


static int
eng_rsa_priv_dec (int flen, const unsigned char *from, unsigned char *to,
      RSA * rsa, int padding __attribute__ ((unused)))
{
   printf ("Engine is decrypting using priv key \n");
   RSA_private_decrypt (flen, from, to, rsa, RSA_PKCS1_PADDING);
}

/* Constants used when creating the ENGINE */
static const char *engine_rsa_id = "rsa-engine 1";
static const char *engine_rsa_name = "engine for testing 1";



static RSA_METHOD struct_rsa = {
    "RSA engine for demo",
    eng_rsa_pub_enc,
    eng_rsa_pub_dec,
    eng_rsa_priv_enc,
    eng_rsa_priv_dec,
    NULL,
    NULL,
    NULL,
    NULL,
    RSA_FLAG_CACHE_PUBLIC | RSA_FLAG_CACHE_PRIVATE,
    NULL,
    NULL,
    NULL
};

static int bind (ENGINE * e, const char *id)
{
  printf ("%s\n", id);

  if (!ENGINE_set_id (e, engine_rsa_id) ||
      !ENGINE_set_name (e, engine_rsa_name) ||
      !ENGINE_set_RSA (e, &struct_rsa))
  return 0;

  return 1;
}

IMPLEMENT_DYNAMIC_BIND_FN (bind) 
IMPLEMENT_DYNAMIC_CHECK_FN ()

I am compiling the code using following command.

gcc -fPIC -c rsa-engine.c
gcc -shared -o librsa_engine.so -lcrypto rsa-engine.o
openssl engine -t -c rsa_engine

Here the engine loads but when i try to encrypt a text file by using following command

openssl pkeyutl -encrypt -in message.txt -pubin -inkey pubkey-B.pem -engine rsa_engine -out cipher.bin
openssl pkeyutl -decrypt -in cipher.bin -inkey privkey-B.pem -engine rsa_engine -out rec.txt

It seems that it is not using the functions which I defined in the struct_rsa. It is also not giving the output from the printf in the function.

Khurram
  • 71
  • 3
  • Feel free to add then appropriate tag for your programing language! – Daniel W. Feb 13 '17 at 11:19
  • 1
    I think you need to add your engine to `openssl.cnf` so it gets loaded. I think there's a way to load it through environmental variables, but I'm not going to hazard a guess. See the `engine(1)` , `engine(3)` and `conf(5)` man pages. Also see [openssl does not load engine from config file](http://unix.stackexchange.com/q/279415/56041) on [Unix & Linux Stack Exchange](http://unix.stackexchange.com/), and [dynamic engines in openssl.cnf](http://openssl.6102.n7.nabble.com/dynamic-engines-in-openssl-cnf-td2454.html) on the OpenSSL mailing list. – jww Feb 14 '17 at 01:42

0 Answers0