Why do I get different signatures when using OpenSSL API from C and from Python?

Question

I'm trying to use the OpenSSL API through pyopenssl• in Python to compute an RSA signature with what I believe is PKCS#1 1.5 padding and SHA1 digest.

It gives wrong result. Here's the minimized sample.

#!/usr/bin/env python3

from binascii import hexlify, unhexlify
from pprint import pprint

key_pem = (
    b"-----BEGIN PRIVATE KEY-----\n"
    b"MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDRgJ0gr3uHa/UB\n"
    b"0CQiPu2/YQByMLYvko3SbI/seMNvkq+iawYTVl62Q1SoZ9rZldNFlEvqaePiKYaJ\n"
    b"PB4ZXMcEqnonD5YATq92SpxkvPngu4t2FvPPS86C6FvKbnIxfgHbR64vWQTrEOj4\n"
    b"s48GFFJ+5QSzl6J7/HB0WpEHdGjTaMM5kp54DvO1VX3mbOEQyoq4itm2m+eyREic\n"
    b"Ye3KHGykGZJJTcrTCf+qJXOWecvNpuVIny9IGyWy1CDnGNX5Qj7ckA91/WVPm7d0\n"
    b"qsbrKWuDKxwp0dV8p64+wWGkDJcj87HQcroU3tCsDJN4iF8KpCGQ097UyA9xdXTw\n"
    b"kk5riQZdAgMBAAECggEABiLVt1Dcdd1wGiL+A/DC5uGQ8UdC9sa8l6atWng5BSoP\n"
    b"MdsfnO5hLMZxEtOj4c6VFwseZBnA3m1n7toPaZ/Bhn31wPIgaxbi5byOsxaj3PSx\n"
    b"Q36tmms2e7gRhC7S8mcl58XEMMfTMI1YvXwI2t06g1Py3M26qRX+NzI14DmFHnf7\n"
    b"+EdiI5vfms/Whu6r3ekCkjIcPo67icP3RdaKaXzW7r9TwBPcyvriREjxCq+/sav8\n"
    b"ZwCUc9Zo91eh11G7tT8QrTXdxZHuVRTOxWEdgSAo5ZZ77nPHIWvNOvnI5ZEkMNG3\n"
    b"IFEuqtvwEHoTSQCsgaIQXhe/zOG65Cs5mQZWn7h9kQKBgQDwy0L6SFJisPrYUFyJ\n"
    b"U+IdnxfgVltOEVT/Oaiq7it7BoxMAuJbm03acmkFRuran7ZJz6D3NK3YKVX6HFAB\n"
    b"R9JJx4BjSfXL1By7gZ9ZUkbTVHWSqsXztKYGsxC7cOC0VuUvW40Xn6c7+3mJae/0\n"
    b"TLoFfoMda8M8cTHAQISuODtiuQKBgQDeu3vuNO+vRE/Mfl1SqLgVLrJ3HqNpv/E3\n"
    b"I+6kR957o8kXJGVRGPhUAeTuZMq0+W9/RFClg/D2nTfXk/XWw1bC1RsC3qYv64Gr\n"
    b"1kCXPhs0ykyB1gqJiCHXuKIP+wS1QtVNThzIy7E5ArXhTXvf/4Wu9KAfrxmGy29q\n"
    b"PCo2obd+xQKBgQCCTeyT1llG8PD96Bb7dbpSP0rDatgEGhr99qzQuwwqijOX2qO1\n"
    b"4QgzY2Bzq5nh7zXNIZ/AxvAgntXZAENHPh+NL3nJwTdTMxjNW2rpAj4zlGv/j4yJ\n"
    b"wkNqMrKmTII89R0XEJr8orf0HLT7aKmicXblDD5VyIAhkDvVBtUGFoYEeQKBgEZ0\n"
    b"shhBAIzFpCSA2I58Nnbk5alOtMyP3gLeR/AJl/QudD7w0Wfc6TjRvJQ4p/KlcMKm\n"
    b"Xohs+z1XsEFuWXbNJdXNyZSXz6Qa8FLmHFp7V+nUEG2FwqGMwX/WtNUvR2b7NDQX\n"
    b"AH34CSCKnfQeKZBK6QPV+Aztu7prAdxuGcBcWYotAoGAfoZYElXiuzytY9XALgrr\n"
    b"mkn5hBglH3w7PgRpPXiEz9JtRKsPkd+LGOHnb61iS0c7d/ZqInKd2EvXHW0G50g1\n"
    b"1A6iCWIZ1BB/EeTm1PXpfkHO+nS996vJsDAgpER+XiVI0ZYZ7wO3fUbAmawJIv6m\n"
    b"NhoqVkkXl5bpYs7UiMxwZKA=\n"
    b"-----END PRIVATE KEY-----\n"
)

challenge = unhexlify("f78b704e 49176ec7 8c53265f 626cd69d 5bd07b9a".replace(' ', ''))
assert len(challenge) == 20


import OpenSSL.crypto as openssl
k = openssl.load_privatekey(openssl.FILETYPE_PEM, key_pem)

signature = openssl.sign(k, challenge, 'sha1')

pprint(hexlify(signature))
# --->
# (b'308a810a90a30513c93167ed2674f97cef1c4bc6df8160a1ea5480763d34ac3043c576fa2fa1'
#  b'71e48bdd0bbb13bcf38bbd4483b3cba215f347439e3e169bb02e49b5b47679ec2dad328174cd'
#  b'f893b2c71465b3eba858b00cc92aa536af2f5c85307cc331a19c4acd54923f23e0b9bf5009b2'
#  b'6d2a4469378e352eaf29f7ce333b8cabca39d9b8858b73e93b745b30ee74264623ef790e6a61'
#  b'a1e7ffb360aa9505f7fc868881d8440ff6765f233dac259a11d49c221ab7549e16df07bdc99e'
#  b'dbbe953ca7e9b1164a115932a9e4c4e3c3509008127298f9d5baae405d97e179c949b013b983'
#  b'76c04d92b7fdf0056f60c9c6df2932c122a7bce0afcf334c13e982b3')

#-- wrong! should be:

# 21 B3 AD 47 7A 3F 83 A6  CA 00 E7 D9 89 DB 11 C0
# 90 6F 5D 27 9C 43 BB BD  5D 4A 02 4E 1C 11 F3 6C
# 3A DC D8 25 B1 8D 16 AC  64 8D 34 7B 9F 1F 77 AC
# 29 F6 4F B3 C8 A7 42 78  D3 1E 2C 9E E8 09 9F 58
# A4 65 0F 45 9F 33 CC B5  01 38 7E A7 D7 31 B9 C7
# 46 D4 82 8C 47 68 B1 F5  86 BD 1E 01 7D 03 3B 88
# 57 0C F6 80 FC 7D 47 88  24 D9 EF F8 19 2D B3 73
# 31 B0 9D 5E 8F 9F 77 9E  33 2B E7 EE AA 51 90 05
# 29 75 A2 88 08 25 7A 9E  31 9E 5B ED 28 14 3E 54
# EC 63 AB 08 3B 61 8C 60  93 83 74 63 1F CD E7 10
# E2 B5 1D EC 61 15 40 83  A2 1E E2 80 B8 90 B3 A2
# 7D 10 BB 8E D8 1D 42 DD  F2 52 E2 08 C1 7C 27 FB
# 97 C0 DF BD 28 20 C7 B9  94 D3 71 85 2D 1E 3F 75
# FF 33 EE 8F 44 D7 1A A3  A7 37 2F BD 84 4B D4 2D
# C6 72 75 C7 F4 CE 56 3C  98 6F C4 F8 1B DD 37 13
# B0 E2 BA AA 69 75 25 4D  2B A4 3C A1 D2 C3 88 69
#

I know it's wrong by having [what looks like] exact same code in C. This C version does replicate the "correct" signature (the one which I see in my protocol dump).

#include <openssl/bio.h>
#include <openssl/err.h>
#include <openssl/rsa.h>
#include <openssl/pem.h>

#include <stdio.h>
#include <stdint.h>

static const char key_pem[] = {
    "-----BEGIN PRIVATE KEY-----\n"
    "MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDRgJ0gr3uHa/UB\n"
    "0CQiPu2/YQByMLYvko3SbI/seMNvkq+iawYTVl62Q1SoZ9rZldNFlEvqaePiKYaJ\n"
    "PB4ZXMcEqnonD5YATq92SpxkvPngu4t2FvPPS86C6FvKbnIxfgHbR64vWQTrEOj4\n"
    "s48GFFJ+5QSzl6J7/HB0WpEHdGjTaMM5kp54DvO1VX3mbOEQyoq4itm2m+eyREic\n"
    "Ye3KHGykGZJJTcrTCf+qJXOWecvNpuVIny9IGyWy1CDnGNX5Qj7ckA91/WVPm7d0\n"
    "qsbrKWuDKxwp0dV8p64+wWGkDJcj87HQcroU3tCsDJN4iF8KpCGQ097UyA9xdXTw\n"
    "kk5riQZdAgMBAAECggEABiLVt1Dcdd1wGiL+A/DC5uGQ8UdC9sa8l6atWng5BSoP\n"
    "MdsfnO5hLMZxEtOj4c6VFwseZBnA3m1n7toPaZ/Bhn31wPIgaxbi5byOsxaj3PSx\n"
    "Q36tmms2e7gRhC7S8mcl58XEMMfTMI1YvXwI2t06g1Py3M26qRX+NzI14DmFHnf7\n"
    "+EdiI5vfms/Whu6r3ekCkjIcPo67icP3RdaKaXzW7r9TwBPcyvriREjxCq+/sav8\n"
    "ZwCUc9Zo91eh11G7tT8QrTXdxZHuVRTOxWEdgSAo5ZZ77nPHIWvNOvnI5ZEkMNG3\n"
    "IFEuqtvwEHoTSQCsgaIQXhe/zOG65Cs5mQZWn7h9kQKBgQDwy0L6SFJisPrYUFyJ\n"
    "U+IdnxfgVltOEVT/Oaiq7it7BoxMAuJbm03acmkFRuran7ZJz6D3NK3YKVX6HFAB\n"
    "R9JJx4BjSfXL1By7gZ9ZUkbTVHWSqsXztKYGsxC7cOC0VuUvW40Xn6c7+3mJae/0\n"
    "TLoFfoMda8M8cTHAQISuODtiuQKBgQDeu3vuNO+vRE/Mfl1SqLgVLrJ3HqNpv/E3\n"
    "I+6kR957o8kXJGVRGPhUAeTuZMq0+W9/RFClg/D2nTfXk/XWw1bC1RsC3qYv64Gr\n"
    "1kCXPhs0ykyB1gqJiCHXuKIP+wS1QtVNThzIy7E5ArXhTXvf/4Wu9KAfrxmGy29q\n"
    "PCo2obd+xQKBgQCCTeyT1llG8PD96Bb7dbpSP0rDatgEGhr99qzQuwwqijOX2qO1\n"
    "4QgzY2Bzq5nh7zXNIZ/AxvAgntXZAENHPh+NL3nJwTdTMxjNW2rpAj4zlGv/j4yJ\n"
    "wkNqMrKmTII89R0XEJr8orf0HLT7aKmicXblDD5VyIAhkDvVBtUGFoYEeQKBgEZ0\n"
    "shhBAIzFpCSA2I58Nnbk5alOtMyP3gLeR/AJl/QudD7w0Wfc6TjRvJQ4p/KlcMKm\n"
    "Xohs+z1XsEFuWXbNJdXNyZSXz6Qa8FLmHFp7V+nUEG2FwqGMwX/WtNUvR2b7NDQX\n"
    "AH34CSCKnfQeKZBK6QPV+Aztu7prAdxuGcBcWYotAoGAfoZYElXiuzytY9XALgrr\n"
    "mkn5hBglH3w7PgRpPXiEz9JtRKsPkd+LGOHnb61iS0c7d/ZqInKd2EvXHW0G50g1\n"
    "1A6iCWIZ1BB/EeTm1PXpfkHO+nS996vJsDAgpER+XiVI0ZYZ7wO3fUbAmawJIv6m\n"
    "NhoqVkkXl5bpYs7UiMxwZKA=\n"
    "-----END PRIVATE KEY-----\n"
};

static const uint8_t challenge[20] = {
0xf7,0x8b,0x70,0x4e, 0x49,0x17,0x6e,0xc7, 0x8c,0x53,0x26,0x5f,
0x62,0x6c,0xd6,0x9d, 0x5b,0xd0,0x7b,0x9a
};

void report_openssl_errors() {
    uint32_t errcode;
    while (0 != (errcode = ERR_get_error())) {
        printf("! %s\n", ERR_error_string(errcode, 0));
    }
}

int main() {
    BIO* biomemfile = BIO_new_mem_buf(key_pem, sizeof(key_pem));
    RSA* pkey = PEM_read_bio_RSAPrivateKey(biomemfile, 0, 0, 0);
    if (!pkey) { report_openssl_errors(); }

    uint8_t sig[256];
    int sigsize;

    RSA_sign(NID_sha1,
            challenge, sizeof(challenge),
            sig, &sigsize,
            pkey);
    if (sigsize == -1) { report_openssl_errors(); }

    // hex dump
    for(int i = 0; i < sigsize; ++i) {
        printf("%02X ", sig[i]);
        if (i % 16 == 7)  printf(" ");
        if (i % 16 == 15) printf("\n");
    }
    printf("\n");
}

// --->
// 21 B3 AD 47 7A 3F 83 A6  CA 00 E7 D9 89 DB 11 C0
// 90 6F 5D 27 9C 43 BB BD  5D 4A 02 4E 1C 11 F3 6C
// 3A DC D8 25 B1 8D 16 AC  64 8D 34 7B 9F 1F 77 AC
// 29 F6 4F B3 C8 A7 42 78  D3 1E 2C 9E E8 09 9F 58
// A4 65 0F 45 9F 33 CC B5  01 38 7E A7 D7 31 B9 C7
// 46 D4 82 8C 47 68 B1 F5  86 BD 1E 01 7D 03 3B 88
// 57 0C F6 80 FC 7D 47 88  24 D9 EF F8 19 2D B3 73
// 31 B0 9D 5E 8F 9F 77 9E  33 2B E7 EE AA 51 90 05
// 29 75 A2 88 08 25 7A 9E  31 9E 5B ED 28 14 3E 54
// EC 63 AB 08 3B 61 8C 60  93 83 74 63 1F CD E7 10
// E2 B5 1D EC 61 15 40 83  A2 1E E2 80 B8 90 B3 A2
// 7D 10 BB 8E D8 1D 42 DD  F2 52 E2 08 C1 7C 27 FB
// 97 C0 DF BD 28 20 C7 B9  94 D3 71 85 2D 1E 3F 75
// FF 33 EE 8F 44 D7 1A A3  A7 37 2F BD 84 4B D4 2D
// C6 72 75 C7 F4 CE 56 3C  98 6F C4 F8 1B DD 37 13
// B0 E2 BA AA 69 75 25 4D  2B A4 3C A1 D2 C3 88 69
//

// -- that's correct!

---

With reproducers in place, The Question is easy... What am I missing?

The two outputs are supposed to match. The challenge strings and the pkeys are the same.

---

• I tried cryptography API too, its output is no different from pyopenssl's

Answer 1

Okay, turns out I can also reproduce the two signatures with just openssl(1) pkeyutl/dgst:

printf "$challenge" \
  | openssl pkeyutl -sign -pkeyopt digest:sha1 -inkey key.pem \
  | hexdump -C
00000000  21 b3 ad 47 7a 3f 83 a6  ca 00 e7 d9 89 db 11 c0  |!..Gz?..........|
00000010  90 6f 5d 27 9c 43 bb bd  5d 4a 02 4e 1c 11 f3 6c  |.o]'.C..]J.N...l|
...

And (what I called "wrong" signature):

printf "$challenge" \
  | openssl dgst -sha1 -sign key.pem \
  | hexdump -C
00000000  30 8a 81 0a 90 a3 05 13  c9 31 67 ed 26 74 f9 7c  |0........1g.&t.||
00000010  ef 1c 4b c6 df 81 60 a1  ea 54 80 76 3d 34 ac 30  |..K...`..T.v=4.0|

Interestringly, openssl rsautl gives a third signature value (!); and this matches pkeyutl output if you omit -pkeyopt digest:sha1:

printf "$challenge" \
  | openssl rsautl -inkey key.pem -sign -pkcs \
  | hexdump -C
00000000  54 4b b4 27 af 0e 0f 1f  d6 a4 31 10 ae 45 7c 77  |TK.'......1..E|w|
00000010  ec 0a 6b cd ae a6 bf 4e  c1 88 a9 3f d7 db f2 91  |..k....N...?....|

And I found... many explanations of this madness:

• https://superuser.com/questions/943972/what-is-the-difference-between-openssl-pkeyutl-sign-and-openssl-rsautl-sign
• Difference between openSSL rsautl and dgst
• How to compute RSA-SHA1(sha1WithRSAEncryption) value with OpenSSL

Answer 2

This figuring-it-out experience was full of grief.

And I abandoned Python.

Because no one smartass will tell me what I don't want to compute.

... textbook RSA signatures, which we don't (and won't) support.

(illegible mocking)

---

Ruby solution

require 'openssl'

def asshole_signature(key, token)
  asn1obj = OpenSSL::ASN1::Sequence [
      OpenSSL::ASN1::Sequence([
        OpenSSL::ASN1::ObjectId("SHA1"),
        OpenSSL::ASN1::Null(nil)
      ]),
      OpenSSL::ASN1::OctetString(token)
    ]
  key.private_encrypt(asn1obj.to_der)
end

key = OpenSSL::PKey::RSA::new(
    "-----BEGIN PRIVATE KEY-----\n"\
    #-- <SNIP> exactly the previous (test) key <SNIP>
    "-----END PRIVATE KEY-----\n"
    )

challenge = "\xf7\x8b\x70\x4e\x49\x17\x6e\xc7\x8c\x53\x26\x5f\x62\x6c\xd6\x9d\x5b\xd0\x7b\x9a"

def bin2hex(s) s.each_byte.map { |b| b.to_s(16) }.join end

print bin2hex(asshole_signature(key, challenge))

This produces the bytes that I need (21b3ad...8869).

These match with what openssl pkeyutl -sign -pkeyopt digest:sha1 spews out; the RSA_sign() OpenSSL API computes the same, I probably don't want to know why.

---

Edit A couple of things to note.

• Porting the above ASN1-wrapping to Python is left as an exercise to the dear reader.

• You shouldn't be using PKCS 1.5 signatures in new code.
  I guess you probably already know this, but just in case.

• Don't use OpenSSL's RSA_sign()/RSA_verify(), they're a crypto legacy artifact. Instead, EVP_DigestSignInit, EVP_DigestSignUpdate, EVP_DigestSignFinal, EVP_DigestVerifyInit, EVP_DigestVerifyUpdate, EVP_DigestVerifyFinal.