The way ECDSA signatures encode the r and s values is not well-defined: While e.g. OpenSSL exclusively uses a DER encoded ASN.1 SEQUENCE, Windows uses IEEE P1363 encoding (see this excellent SO answer for details).
In order to enable ECDSA signature verification with OpenSSL I patched ossl_ecdsa_verify(..) in ec/ecdsa_assl.c (in OpenSSL 3 that is; in 1.0.2 it's ECDSA_verify(..) in ecdsa/ecs_vrf.c). (See this question of mine for a related question on debugging the patch.)
Stock code does this:
/*-
* returns
* 1: correct signature
* 0: incorrect signature
* -1: error
*/
int ossl_ecdsa_verify(int type, const unsigned char *dgst, int dgst_len,
const unsigned char *sigbuf, int sig_len, EC_KEY *eckey)
{
ECDSA_SIG *s;
const unsigned char *p = sigbuf;
unsigned char *der = NULL;
int derlen = -1;
int ret = -1;
s = ECDSA_SIG_new();
if (s == NULL)
return ret;
if (d2i_ECDSA_SIG(&s, &p, sig_len) == NULL)
goto err;
/* Ensure signature uses DER and doesn't have trailing garbage */
derlen = i2d_ECDSA_SIG(s, &der);
if (derlen != sig_len || memcmp(sigbuf, der, derlen) != 0)
goto err;
ret = ECDSA_do_verify(dgst, dgst_len, s, eckey);
err:
OPENSSL_free(der);
ECDSA_SIG_free(s);
return ret;
}
My patch does this:
int ossl_ecdsa_verify(int type, const unsigned char *dgst, int dgst_len,
const unsigned char *sigbuf, int sig_len, EC_KEY *eckey)
{
ECDSA_SIG *s;
const unsigned char *p = sigbuf;
unsigned char *der = NULL;
int derlen = -1;
int ret = -1;
s = ECDSA_SIG_new();
if (s == NULL)
return ret;
#ifdef P1363_PATCH
if (d2i_ECDSA_SIG(&s, &p, sig_len) == NULL) {
/*
* ASN.1 decoding failed, see crypto/asn1/tasn_dec.c line 515ff.
* Assume s is encoded as IEEE P1363. for a comprehensive description see
* ttps://stackoverflow.com/questions/36542645/does-openssl-sign-for-ecdsa-apply-asn1-encoding-to-the-hash-before-signing
* Fill the ECDSA_SIG from the P1363.
*/
if ((sig_len % 2) != 0)
return (ret);
if (s == NULL)
s = ECDSA_SIG_new();
if (s == NULL)
return (ret);
/*
* Prepare a buffer large enough to hold either r or s part of the P1363.
* Add 1 to the size to allow for padding if needed.
* Define some variables for pointer arithmetic.
*/
int buf_size = sig_len / 2 + 1;
void *buf = malloc(buf_size);
const unsigned char *sigbuf_half = sigbuf + sig_len / 2;
const unsigned char *sigbuf_full = sigbuf + sig_len;
/*
* Skip possible padding of the r part of the P1363.
* I /think/ only the s part may be padded, but it does no harm to skip them
* for the r part, too.
*/
const unsigned char *q = sigbuf;
while (*q == '\0' && q < sigbuf_half)
q++;
int buf_len = sigbuf_half - q;
/*
* Prepare buf for BIGNUM creation.
*/
memcpy(buf, q, buf_len);
if (*(char*)buf & 0x80) {
/* Add padding if needed to assert positive integer. */
memmove((char*)buf + 1, buf, buf_len);
memset(buf, '\0', 1);
buf_len++;
}
/*
* Finally create the BIGNUM and put it in the r part of the ECDSA_SIG.
*/
s->r = BN_bin2bn((const unsigned char *)buf, buf_len, NULL);
/*
* Now do the same for the s part...
*/
q = sigbuf_half;
while (*q == '\0' && q < sigbuf_full)
q++;
buf_len = sigbuf_full - q;
memcpy(buf, q, buf_len);
if (*(char*)buf & 0x80) {
/*Add padding if needed to assert positive integer. */
memmove((char*)buf + 1, buf, buf_len);
memset(buf, '\0', 1);
buf_len++;
}
s->s = BN_bin2bn((const unsigned char *)buf, buf_len, NULL);
free(buf);
}
else {
/* Ensure signature uses DER and doesn't have trailing garbage */
derlen = i2d_ECDSA_SIG(s, &der);
if (derlen != sig_len || memcmp(sigbuf, der, derlen))
goto err;
}
ret = ECDSA_do_verify(dgst, dgst_len, s, eckey);
err:
if (derlen > 0) {
OPENSSL_cleanse(der, derlen);
OPENSSL_free(der);
}
#else
if (d2i_ECDSA_SIG(&s, &p, sig_len) == NULL)
goto err;
/* Ensure signature uses DER and doesn't have trailing garbage */
derlen = i2d_ECDSA_SIG(s, &der);
if (derlen != sig_len || memcmp(sigbuf, der, derlen) != 0)
goto err;
ret = ECDSA_do_verify(dgst, dgst_len, s, eckey);
err:
OPENSSL_free(der);
#endif /* P1363_PATCH */
ECDSA_SIG_free(s);
return ret;
}
This allows me to verify P1363 encoded ECDSA signatures. Garbage in sigbuf is still caught by ECDSA_do_verify(..).
However OpenSSL's provided ecdsa test fails with the patch:
15-test_ecdsa.t ....................
# INFO:
# testing ECDSA for curve secp112r1 as EC key type
# ERROR: (int) 'EVP_DigestVerify(mctx, sig, sig_len - 1, tbs, sizeof(tbs)) == -1' failed @ ..\..\..\3rdparty\openssl-3.0.5-RIB\test\ecdsatest.c:262
# [0] compared to [-1]
# 442C0000:error:0800009C:elliptic curve routines:ossl_ecdsa_simple_verify_sig:bad signature:..\..\..\3rdparty\openssl-3.0.5-RIB\crypto\ec\ecdsa_ossl.c:482:
# OPENSSL_TEST_RAND_ORDER=1666859286
not ok 1 - iteration 1
and so on for every single curve. I haven't been able to fully analyse, but I assume it's due to the negative tests fail. This would be because garbage provided by the test now runs through my code and doesn't bypass the call to ECDSA_do_verify(..). Is that correct?
In order for the test to succeed, and as a general improvement to my code, where stock OpenSSL only distinguishes 2 cases (ASN.1 or garbage) I now need to distinguish 3 cases (ASN.1, P1363 or garbage). Is there any way to distinguish P1363 from garbage once ASN.1 decode failed?
