dev-libs/libp11/files/libp11-0.2.8-Add-OpenSSL-1.1-support.patch

From 340221b78e7af95b703dcefa9afa4ffe891db920 Mon Sep 17 00:00:00 2001
From: Mattias Nissler <mnissler@chromium.org>
Date: Mon, 20 Jan 2020 14:07:12 +0100
Subject: [PATCH] Add OpenSSL 1.1 support

Add support for OpenSSL 1.1 API while maintaining support for OpenSSL
1.0.2.

Considerations:
 * RSA & EVP_PKEY are now opaque and require use of simple accessors
 * OpenSSL-1.1 does not define or use the RSA_FLAG_SIGN_VER
 * RSA_METHOD has become opaque, requiring a heap allocation
 * X509_SIG has become opaque and requires heap allocation and
   X509_SIG_getm() to manipulate the algorithm and digest.

Most of the fixes in this patch are copied directly from, or inspired by
the equivalent upstream libp11 fixes, backported onto 0.2.8. See in
particular these upstream commits:

https://github.com/OpenSC/libp11/commit/491c6b150d31d3126059556b757002f9228cc38f
https://github.com/OpenSC/libp11/commit/932e8e036d8f8a0ef74446cf65c2c11129a36c03
https://github.com/OpenSC/libp11/commit/3f248533bbad437a103626ad3c6e5a37726dbf8f

The X509_SIG fix is unique to this patch, since PKCS11_sign has been
significantly re-implemented to avoid direct X509_SIG manipulation.

BUG=chromium:739688
TEST=FEATURES=test emerge-sarien libp11
---
 src/p11_key.c |  46 +++++++++++++++----
 src/p11_ops.c |  29 ++++++++----
 src/p11_rsa.c | 119 +++++++++++++++++++++++++++++++++++++++++++++-----
 3 files changed, 165 insertions(+), 29 deletions(-)

--- a/src/p11_key.c
+++ b/src/p11_key.c
@@ -339,6 +339,7 @@ static int pkcs11_store_private_key(PKCS11_TOKEN * token, EVP_PKEY * pk,
 	CK_ATTRIBUTE attrs[32];
 	unsigned int n = 0;
 	int rv;
+	const BIGNUM *rsa_n, *rsa_e, *rsa_d, *rsa_p, *rsa_q;
 
 	/* First, make sure we have a session */
 	if (!PRIVSLOT(slot)->haveSession && PKCS11_open_session(slot, 1))
@@ -346,8 +347,13 @@ static int pkcs11_store_private_key(PKCS11_TOKEN * token, EVP_PKEY * pk,
 	session = PRIVSLOT(slot)->session;
 
 	/* Now build the key attrs */
-	if (pk->type == EVP_PKEY_RSA) {
+#if OPENSSL_VERSION_NUMBER >= 0x10100003L
+	if (EVP_PKEY_base_id(pk) == EVP_PKEY_RSA) {
 		RSA *rsa = EVP_PKEY_get1_RSA(pk);
+#else
+	if (pk->type == EVP_PKEY_RSA) {
+		RSA *rsa = pk->pkey.rsa;
+#endif
 
 		pkcs11_addattr_int(attrs + n++, CKA_CLASS, CKO_PRIVATE_KEY);
 		pkcs11_addattr_int(attrs + n++, CKA_KEY_TYPE, CKK_RSA);
@@ -359,11 +365,21 @@ static int pkcs11_store_private_key(PKCS11_TOKEN * token, EVP_PKEY * pk,
 		pkcs11_addattr_bool(attrs + n++, CKA_SIGN, TRUE);
 		pkcs11_addattr_bool(attrs + n++, CKA_UNWRAP, TRUE);
 
-		pkcs11_addattr_bn(attrs + n++, CKA_MODULUS, rsa->n);
-		pkcs11_addattr_bn(attrs + n++, CKA_PUBLIC_EXPONENT, rsa->e);
-		pkcs11_addattr_bn(attrs + n++, CKA_PRIVATE_EXPONENT, rsa->d);
-		pkcs11_addattr_bn(attrs + n++, CKA_PRIME_1, rsa->p);
-		pkcs11_addattr_bn(attrs + n++, CKA_PRIME_2, rsa->q);
+#if OPENSSL_VERSION_NUMBER >= 0x10100005L
+		RSA_get0_key(rsa, &rsa_n, &rsa_e, &rsa_d);
+		RSA_get0_factors(rsa, &rsa_p, &rsa_q);
+#else
+		rsa_n=rsa->n;
+		rsa_e=rsa->e;
+		rsa_d=rsa->d;
+		rsa_p=rsa->p;
+		rsa_q=rsa->q;
+#endif
+		pkcs11_addattr_bn(attrs + n++, CKA_MODULUS, rsa_n);
+		pkcs11_addattr_bn(attrs + n++, CKA_PUBLIC_EXPONENT, rsa_e);
+		pkcs11_addattr_bn(attrs + n++, CKA_PRIVATE_EXPONENT, rsa_d);
+		pkcs11_addattr_bn(attrs + n++, CKA_PRIME_1, rsa_p);
+		pkcs11_addattr_bn(attrs + n++, CKA_PRIME_2, rsa_q);
 
 		if (label)
 			pkcs11_addattr_s(attrs + n++, CKA_LABEL, label);
@@ -402,6 +418,7 @@ static int pkcs11_store_public_key(PKCS11_TOKEN * token, EVP_PKEY * pk,
 	CK_ATTRIBUTE attrs[32];
 	unsigned int n = 0;
 	int rv;
+	const BIGNUM *rsa_n, *rsa_e;
 
 	/* First, make sure we have a session */
 	if (!PRIVSLOT(slot)->haveSession && PKCS11_open_session(slot, 1))
@@ -409,8 +426,13 @@ static int pkcs11_store_public_key(PKCS11_TOKEN * token, EVP_PKEY * pk,
 	session = PRIVSLOT(slot)->session;
 
 	/* Now build the key attrs */
-	if (pk->type == EVP_PKEY_RSA) {
+#if OPENSSL_VERSION_NUMBER >= 0x10100003L
+	if (EVP_PKEY_base_id(pk) == EVP_PKEY_RSA) {
 		RSA *rsa = EVP_PKEY_get1_RSA(pk);
+#else
+	if (pk->type == EVP_PKEY_RSA) {
+		RSA *rsa = pk->pkey.rsa;
+#endif
 
 		pkcs11_addattr_int(attrs + n++, CKA_CLASS, CKO_PUBLIC_KEY);
 		pkcs11_addattr_int(attrs + n++, CKA_KEY_TYPE, CKK_RSA);
@@ -420,8 +442,14 @@ static int pkcs11_store_public_key(PKCS11_TOKEN * token, EVP_PKEY * pk,
 		pkcs11_addattr_bool(attrs + n++, CKA_VERIFY, TRUE);
 		pkcs11_addattr_bool(attrs + n++, CKA_WRAP, TRUE);
 
-		pkcs11_addattr_bn(attrs + n++, CKA_MODULUS, rsa->n);
-		pkcs11_addattr_bn(attrs + n++, CKA_PUBLIC_EXPONENT, rsa->e);
+#if OPENSSL_VERSION_NUMBER >= 0x10100005L
+		RSA_get0_key(rsa, &rsa_n, &rsa_e, NULL);
+#else
+		rsa_n=rsa->n;
+		rsa_e=rsa->e;
+#endif
+		pkcs11_addattr_bn(attrs + n++, CKA_MODULUS, rsa_n);
+		pkcs11_addattr_bn(attrs + n++, CKA_PUBLIC_EXPONENT, rsa_e);
 		if (label)
 			pkcs11_addattr_s(attrs + n++, CKA_LABEL, label);
 		if (id && id_len)
--- a/src/p11_ops.c
+++ b/src/p11_ops.c
@@ -43,25 +43,38 @@ PKCS11_sign(int type, const unsigned char *m, unsigned int m_len,
 			return(0); /* the size is wrong */
 		}
 	} else {
-		ASN1_TYPE parameter = { V_ASN1_NULL, { NULL } };
- 		ASN1_STRING digest = { m_len, V_ASN1_OCTET_STRING, (unsigned char *)m };
-		X509_ALGOR algor = { NULL, &parameter };
-		X509_SIG digest_info = { &algor, &digest };
+		ASN1_STRING *digest;
+		X509_ALGOR *algor;
+		X509_SIG *digest_info;
 		int size;
+
+		digest_info = X509_SIG_new();
+		if (digest_info == NULL)
+			return 0;
+
+#if OPENSSL_VERSION_NUMBER >= 0x10100001L
+		X509_SIG_getm(digest_info, &algor, &digest);
+#else
+		digest = digest_info->digest;
+		algor = digest_info->algor;
+#endif
+
 		/* Fetch the OID of the algorithm used */
-		if((algor.algorithm = OBJ_nid2obj(type)) && 
-		   (algor.algorithm->length) &&
+		if(X509_ALGOR_set0(algor, OBJ_nid2obj(type), V_ASN1_NULL, NULL) &&
+		   ASN1_OCTET_STRING_set(digest, m, m_len) &&
 		   /* Get the size of the encoded DigestInfo */
-		   (size = i2d_X509_SIG(&digest_info, NULL)) &&
+		   (size = i2d_X509_SIG(digest_info, NULL)) &&
 		   /* Check that size is compatible with PKCS#11 padding */
 		   (size + RSA_PKCS1_PADDING_SIZE <= sigsize) &&
 		   (encoded = (unsigned char *) malloc(sigsize))) {
 			unsigned char *tmp = encoded;
 			/* Actually do the encoding */
-			i2d_X509_SIG(&digest_info,&tmp);
+			i2d_X509_SIG(digest_info, &tmp);
 			m = encoded;
 			m_len = size;
+			X509_SIG_free(digest_info);
 		} else {
+			X509_SIG_free(digest_info);
 			return(0);
 		}
 	}
--- a/src/p11_rsa.c
+++ b/src/p11_rsa.c
@@ -38,6 +38,7 @@ static int pkcs11_get_rsa_private(PKCS11_KEY * key, EVP_PKEY * pk)
 {
 	CK_BBOOL sensitive, extractable;
 	RSA *rsa;
+	BIGNUM *rsa_n = NULL, *rsa_e = NULL;
 
 	if (!(rsa = EVP_PKEY_get1_RSA(pk))) {
 		ERR_clear_error();	/* the above flags an error */
@@ -51,17 +52,30 @@ static int pkcs11_get_rsa_private(PKCS11_KEY * key, EVP_PKEY * pk)
 		return -1;
 	}
 
-	if (key_getattr_bn(key, CKA_MODULUS, &rsa->n) ||
-	    key_getattr_bn(key, CKA_PUBLIC_EXPONENT, &rsa->e)) {
+	if (key_getattr_bn(key, CKA_MODULUS, &rsa_n) ||
+	    key_getattr_bn(key, CKA_PUBLIC_EXPONENT, &rsa_e)) {
 		RSA_free(rsa);
+		BN_clear_free(rsa_n);
+		BN_clear_free(rsa_e);
 		return -1;
 	}
 
+#if OPENSSL_VERSION_NUMBER >= 0x10100005L
+	RSA_set0_key(rsa, rsa_n, rsa_e, NULL);
+#else
+	rsa->n = rsa_n;
+	rsa->e = rsa_e;
+#endif
+
 	/* If the key is not extractable, create a key object
 	 * that will use the card's functions to sign & decrypt */
 	if (sensitive || !extractable) {
 		RSA_set_method(rsa, PKCS11_get_rsa_method());
+
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+		/* RSA_FLAG_SIGN_VER no longer  in OpenSSL 1.1 */
 		rsa->flags |= RSA_FLAG_SIGN_VER;
+#endif
 		RSA_set_app_data(rsa, key);
 
 		RSA_free(rsa);
@@ -71,7 +85,11 @@ static int pkcs11_get_rsa_private(PKCS11_KEY * key, EVP_PKEY * pk)
 	/* TBD - extract RSA private key. */
 	/* In the mean time let's use the card anyway */
 	RSA_set_method(rsa, PKCS11_get_rsa_method());
+
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+	/* RSA_FLAG_SIGN_VER no longer  in OpenSSL 1.1 */
 	rsa->flags |= RSA_FLAG_SIGN_VER;
+#endif
 	RSA_set_app_data(rsa, key);
 
 	RSA_free(rsa);
@@ -110,6 +128,8 @@ static int pkcs11_rsa_sign(int type, const unsigned char *m, unsigned int m_len,
 	
 	return PKCS11_sign(type,m,m_len,sigret,siglen,(PKCS11_KEY *) RSA_get_app_data(rsa));
 }
+
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
 /* Lousy hack alert. If RSA_verify detects that the key has the
  * RSA_FLAG_SIGN_VER flags set, it will assume that verification
  * is implemented externally as well.
@@ -118,7 +138,7 @@ static int pkcs11_rsa_sign(int type, const unsigned char *m, unsigned int m_len,
  */
 static int
 pkcs11_rsa_verify(int type, const unsigned char *m, unsigned int m_len,
-		  unsigned char *signature, unsigned int siglen, const RSA * rsa)
+		  const unsigned char *signature, unsigned int siglen, const RSA * rsa)
 {
 	RSA *r = (RSA *) rsa;	/* Ugly hack to get rid of compiler warning */
 	int res;
@@ -131,24 +151,99 @@ pkcs11_rsa_verify(int type, const unsigned char *m, unsigned int m_len,
 		PKCS11err(PKCS11_F_PKCS11_RSA_VERIFY, PKCS11_NOT_SUPPORTED);
 		res = 0;
 	}
+
 	return res;
 }
+#endif
+
+#if OPENSSL_VERSION_NUMBER < 0x10100005L
+
+static RSA_METHOD *RSA_meth_dup(const RSA_METHOD *meth)
+{
+	RSA_METHOD *ret = OPENSSL_malloc(sizeof(RSA_METHOD));
+	if (ret == NULL)
+		return NULL;
+	memcpy(ret, meth, sizeof(RSA_METHOD));
+	ret->name = OPENSSL_strdup(meth->name);
+	if (ret->name == NULL) {
+		OPENSSL_free(ret);
+		return NULL;
+	}
+	return ret;
+}
+
+static int RSA_meth_set1_name(RSA_METHOD *meth, const char *name)
+{
+	char *tmp = OPENSSL_strdup(name);
+	if (tmp == NULL)
+		return 0;
+	OPENSSL_free((char *)meth->name);
+	meth->name = tmp;
+	return 1;
+}
+
+static int RSA_meth_set_flags(RSA_METHOD *meth, int flags)
+{
+	meth->flags = flags;
+	return 1;
+}
+
+static int RSA_meth_set_priv_enc(RSA_METHOD *meth,
+		int (*priv_enc) (int flen, const unsigned char *from,
+		unsigned char *to, RSA *rsa, int padding))
+{
+	meth->rsa_priv_enc = priv_enc;
+	return 1;
+}
+
+static int RSA_meth_set_priv_dec(RSA_METHOD *meth,
+		int (*priv_dec) (int flen, const unsigned char *from,
+		unsigned char *to, RSA *rsa, int padding))
+{
+	meth->rsa_priv_dec = priv_dec;
+	return 1;
+}
+
+static int RSA_meth_set_sign(RSA_METHOD *meth,
+		int (*sign) (int type, const unsigned char *m,
+		unsigned int m_len, unsigned char *sigret, unsigned int *siglen,
+		const RSA * rsa))
+{
+	meth->rsa_sign = sign;
+	return 1;
+}
+
+static int RSA_meth_set_verify(RSA_METHOD *meth,
+		int (*verify) (int type, const unsigned char *m,
+        unsigned int m_len, const unsigned char *sigbuf,
+        unsigned int siglen, const RSA *rsa))
+{
+	meth->rsa_verify = verify;
+	return 1;
+}
+#endif
 
 /*
  * Overload the default OpenSSL methods for RSA
  */
 RSA_METHOD *PKCS11_get_rsa_method(void)
 {
-	static RSA_METHOD ops;
-
-	if (!ops.rsa_priv_enc) {
-		ops = *RSA_get_default_method();
-		ops.rsa_priv_enc = pkcs11_rsa_encrypt;
-		ops.rsa_priv_dec = pkcs11_rsa_decrypt;
-		ops.rsa_sign = pkcs11_rsa_sign;
-		ops.rsa_verify = pkcs11_rsa_verify;
+	static RSA_METHOD *ops = NULL;
+
+	if (ops == NULL) {
+		ops = RSA_meth_dup(RSA_get_default_method());
+		if (ops == NULL)
+			return NULL;
+		RSA_meth_set1_name(ops, "libp11 RSA method");
+		RSA_meth_set_flags(ops, 0);
+		RSA_meth_set_priv_enc(ops, pkcs11_rsa_encrypt);
+		RSA_meth_set_priv_dec(ops, pkcs11_rsa_decrypt);
+		RSA_meth_set_sign(ops, pkcs11_rsa_sign);
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+		RSA_meth_set_verify(ops, pkcs11_rsa_verify);
+#endif
 	}
-	return &ops;
+	return ops;
 }
 
 PKCS11_KEY_ops pkcs11_rsa_ops = {
-- 
2.24.1