crypto: split out RSA specific functions

In order to support more key types, first split out all the mechanism
specific key handling into its own separate file.

Signed-off-by: James Bottomley <James.Bottomley@HansenPartnership.com>

4 files changed, 261 insertions, 185 deletions

diff --git a/Makefile.am b/Makefile.am
index 3bb0d70..783bf94 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -7,7 +7,7 @@ pkcs11_configsdir=@pkcs11_configs@
 pkcs11_configs_DATA = openssl-pkcs11-export.module
 
 openssl_pkcs11_export_la_LDFLAGS= -module -no-undefined -avoid-version -shared
-openssl_pkcs11_export_la_SOURCES= pkcs11.c ini.c openssl-pkcs11.h cache.c crypto.c
+openssl_pkcs11_export_la_SOURCES= pkcs11.c ini.c openssl-pkcs11.h cache.c crypto.c crypto.h crypto-rsa.c
 openssl_pkcs11_export_la_CFLAGS = $(CFLAGS) $(CRYPTO_CFLAGS) $(P11KIT_CFLAGS) -Werror -Wall
 openssl_pkcs11_export_la_LIBADD = $(CRYPTO_LIBS)
 
diff --git a/crypto-rsa.c b/crypto-rsa.c
new file mode 100644
index 0000000..5dec3c9
--- /dev/null
+++ b/crypto-rsa.c
@@ -0,0 +1,209 @@
+/*
+ * Handle the RSA specific openssl crypto functions
+ *
+ * Copyright (C) 2019 James.Bottomley@HansenPartnership.com
+ *
+ * SPDX-License-Identifier: LGPL-2.1-only
+ */
+
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <openssl/evp.h>
+#include <openssl/rsa.h>
+#include <openssl/buffer.h>
+#include <openssl/err.h>
+
+#include "openssl-pkcs11.h"
+#include "crypto.h"
+
+void
+crypto_rsa_populate(int sec_num, EVP_PKEY *pkey)
+{
+	RSA *rsa = EVP_PKEY_get1_RSA(pkey);
+	unsigned long size = RSA_size(rsa);
+	const BIGNUM *n, *e;
+	EVP_MD_CTX *ctx;
+
+#if OPENSSL_VERSION_NUMBER < 0x10100000
+	n = rsa->n;
+	e = rsa->e;
+#else
+	RSA_get0_key(rsa, &n, &e, NULL);
+#endif
+	ctx = EVP_MD_CTX_create();
+	EVP_DigestInit(ctx, EVP_sha256());
+
+	crypto_add_BN(sec_num, "CKA_PUBLIC_EXPONENT", e, ctx);
+	crypto_add_BN(sec_num, "CKA_MODULUS", n, ctx);
+	crypto_add_serial(sec_num, ctx);
+	cache_add_by_secnum(sec_num, "CKA_MODULUS_BITS",
+			    (const char *)size, CACHE_INT);
+	cache_add_by_secnum(sec_num, "CKA_KEY_TYPE",
+			    (const char *)CKK_RSA, CACHE_INT);
+	/* booleans: only need to add true ones */
+	cache_add_by_secnum(sec_num, "CKA_ENCRYPT",
+			    (const char *)BOOL_FOR_PUBLIC, CACHE_INT);
+	cache_add_by_secnum(sec_num, "CKA_DECRYPT",
+			    (const char *)BOOL_FOR_PRIVATE, CACHE_INT);
+	cache_add_by_secnum(sec_num, "CKA_VERIFY_RECOVER",
+			    (const char *)BOOL_FOR_PUBLIC, CACHE_INT);
+
+	RSA_free(rsa);
+}
+
+const EVP_MD *get_mgf1(unsigned long mgf1)
+{
+	switch (mgf1) {
+	case CKG_MGF1_SHA1:
+		return EVP_sha1();
+	case CKG_MGF1_SHA224:
+		return EVP_sha224();
+	case CKG_MGF1_SHA256:
+		return EVP_sha256();
+	case CKG_MGF1_SHA384:
+		return EVP_sha384();
+	case CKG_MGF1_SHA512:
+		return EVP_sha512();
+	default:
+		fprintf(stderr, "Unknown mgf1 type %ld\n", mgf1);
+		return NULL;
+	}
+}
+
+const EVP_MD *get_hash(unsigned long hash)
+{
+	switch (hash) {
+	case CKM_SHA_1:
+		return EVP_sha1();
+	case CKM_SHA224:
+		return EVP_sha224();
+	case CKM_SHA256:
+		return EVP_sha256();
+	case CKM_SHA384:
+		return EVP_sha384();
+	case CKM_SHA512:
+		return EVP_sha512();
+	default:
+		fprintf(stderr, "Unknown hash type %ld\n", hash);
+		return NULL;
+	}
+}
+
+EVP_PKEY_CTX *crypto_rsa_add_padding(EVP_PKEY_CTX *ctx, CK_MECHANISM_PTR mech)
+{
+	if (mech->mechanism == CKM_RSA_PKCS) {
+		EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PADDING);
+	} else if (mech->mechanism == CKM_RSA_X_509) {
+		EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_NO_PADDING);
+	} else if (mech->mechanism == CKM_RSA_PKCS_PSS) {
+		CK_RSA_PKCS_PSS_PARAMS *p = mech->pParameter;
+
+		if (mech->ulParameterLen != sizeof(*p)) {
+			fprintf(stderr, "PSS mechanism parameter length %ld != %ld\n",
+				mech->ulParameterLen, (long)sizeof(*p));
+			goto err;
+		}
+		EVP_PKEY_CTX_set_signature_md(ctx, get_hash(p->hashAlg));
+		EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PSS_PADDING);
+		EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, get_mgf1(p->mgf));
+		EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, p->sLen);
+	} else if (mech->mechanism == CKM_RSA_PKCS_OAEP) {
+		CK_RSA_PKCS_OAEP_PARAMS *p = mech->pParameter;
+
+		if (mech->ulParameterLen != sizeof(*p)) {
+			fprintf(stderr, "OAEP mechanism parameter length %ld != %ld\n",
+				mech->ulParameterLen, (long)sizeof(*p));
+			goto err;
+		}
+		EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING);
+		EVP_PKEY_CTX_set_rsa_oaep_md(ctx, get_hash(p->hashAlg));
+		EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, get_mgf1(p->mgf));
+		if (p->source & CKZ_DATA_SPECIFIED) {
+			void *l = BUF_memdup(p->pSourceData,
+					     p->ulSourceDataLen);
+
+			EVP_PKEY_CTX_set0_rsa_oaep_label(ctx, l,
+							 p->ulSourceDataLen);
+		}
+	} else {
+		fprintf(stderr, "unknown mechanism %ld\n", mech->mechanism);
+		goto err;
+	}
+
+	return ctx;
+ err:
+	EVP_PKEY_CTX_free(ctx);
+	return NULL;
+}
+
+/* decrypt is a RSA only operation */
+void *crypto_decrypt_init(int sec_num, CK_MECHANISM_PTR mech)
+{
+	EVP_PKEY_CTX *ctx;
+
+	ctx = crypto_get_key(sec_num);
+	EVP_PKEY_decrypt_init(ctx);
+	return crypto_rsa_add_padding(ctx, mech);
+}
+
+int crypto_decrypt(void *opdata, void *enc_data, unsigned long enc_len,
+		   void *data, unsigned long *data_len)
+{
+	EVP_PKEY_CTX *ctx = opdata;
+	int ret = 0;
+	size_t len;
+
+	if (data_len)
+		len = *data_len;
+
+	if (!data) {
+		*data_len = EVP_PKEY_size(EVP_PKEY_CTX_get0_pkey(ctx));
+		return 0;
+	}
+
+	if (EVP_PKEY_decrypt(ctx, data, &len, enc_data, enc_len) <= 0) {
+		ERR_print_errors_fp(stderr);
+		ret = -1;
+	}
+	if (data_len)
+		*data_len = len;
+	EVP_PKEY_CTX_free(ctx);
+
+	return ret;
+}
+
+static CK_MECHANISM_TYPE mechanism_types[] = {
+	CKM_RSA_PKCS,
+	CKM_RSA_X_509,
+	CKM_RSA_PKCS_PSS,
+	CKM_RSA_PKCS_OAEP,
+};
+
+void crypto_rsa_fill_mechanism_list(int sec_num, unsigned long *mechs,
+				    unsigned long *count)
+{
+	*count = ARRAY_SIZE(mechanism_types);
+	if (mechs)
+		memcpy(mechs, mechanism_types, sizeof(mechanism_types));
+}
+
+int crypto_rsa_check_mechanism(int sec_num, CK_MECHANISM_TYPE mech,
+			       CK_MECHANISM_INFO_PTR info)
+{
+	int i, found = 0;
+
+	for (i = 0; i < ARRAY_SIZE(mechanism_types); i++)
+		if (mech == mechanism_types[i])
+			found = 1;
+
+	if (!found)
+		return 0;
+
+	info->ulMinKeySize = 1024;
+	info->ulMaxKeySize = 4096;
+	info->flags = CKF_HW|CKF_ENCRYPT|CKF_DECRYPT;
+
+	return 1;
+}
diff --git a/crypto.c b/crypto.c
index 9f6afbc..0f47dcc 100644
--- a/crypto.c
+++ b/crypto.c
@@ -15,18 +15,16 @@
 #include <openssl/evp.h>
 #include <openssl/pem.h>
 #include <openssl/err.h>
-#include <openssl/rsa.h>
 #include <openssl/ui.h>
 
 #include "openssl-pkcs11.h"
-
-#define ARRAY_SIZE(a) (sizeof (a) / sizeof ((a)[0]))
+#include "crypto.h"
 
 /* number of bytes in the serial number */
 #define SERIAL_COUNT	10
 
-static void crypto_add_BN(int sec_num, const char *key, const BIGNUM *value,
-			  EVP_MD_CTX *ctx)
+void crypto_add_BN(int sec_num, const char *key, const BIGNUM *value,
+		   EVP_MD_CTX *ctx)
 {
 	int len = BN_num_bytes(value);
 	char *buf;
@@ -40,8 +38,7 @@ static void crypto_add_BN(int sec_num, const char *key, const BIGNUM *value,
 	cache_add_by_secnum(sec_num, key, buf, len);
 }
 
-static void
-crypto_add_serial(int sec_num, EVP_MD_CTX *ctx)
+void crypto_add_serial(int sec_num, EVP_MD_CTX *ctx)
 {
 	unsigned char hash[SHA256_DIGEST_LENGTH];
 	char *serial;
@@ -61,49 +58,17 @@ crypto_add_serial(int sec_num, EVP_MD_CTX *ctx)
 }
 
 static void
-populate_rsa(int sec_num, EVP_PKEY *pkey)
+populate_global(int sec_num)
 {
-	RSA *rsa = EVP_PKEY_get1_RSA(pkey);
-	unsigned long size = RSA_size(rsa);
-	const BIGNUM *n, *e;
-	EVP_MD_CTX *ctx;
-
-#if OPENSSL_VERSION_NUMBER < 0x10100000
-	n = rsa->n;
-	e = rsa->e;
-#else
-	RSA_get0_key(rsa, &n, &e, NULL);
-#endif
-	ctx = EVP_MD_CTX_create();
-	EVP_DigestInit(ctx, EVP_sha256());
-
-	crypto_add_BN(sec_num, "CKA_PUBLIC_EXPONENT", e, ctx);
-	crypto_add_BN(sec_num, "CKA_MODULUS", n, ctx);
-	crypto_add_serial(sec_num, ctx);
-	cache_add_by_secnum(sec_num, "CKA_MODULUS_BITS",
-			    (const char *)size, CACHE_INT);
-	cache_add_by_secnum(sec_num, "CKA_KEY_TYPE",
-			    (const char *)CKK_RSA, CACHE_INT);
-	/* booleans: only need to add true ones */
 	cache_add_by_secnum(sec_num, "CKA_TOKEN",
 			    (const char *)(BOOL_FOR_PRIVATE | BOOL_FOR_PUBLIC),
 			    CACHE_INT);
 	cache_add_by_secnum(sec_num, "CKA_PRIVATE",
 			    (const char *)BOOL_FOR_PRIVATE, CACHE_INT);
-	cache_add_by_secnum(sec_num, "CKA_ENCRYPT",
-			    (const char *)BOOL_FOR_PUBLIC, CACHE_INT);
-	cache_add_by_secnum(sec_num, "CKA_DECRYPT",
-			    (const char *)BOOL_FOR_PRIVATE, CACHE_INT);
 	cache_add_by_secnum(sec_num, "CKA_SIGN",
 			    (const char *)BOOL_FOR_PRIVATE, CACHE_INT);
-	cache_add_by_secnum(sec_num, "CKA_VERIFY_RECOVER",
+	cache_add_by_secnum(sec_num, "CKA_VERIFY",
 			    (const char *)BOOL_FOR_PUBLIC, CACHE_INT);
-	cache_add_by_secnum(sec_num, "CKA_SENSITIVE",
-			    (const char *)BOOL_FOR_PRIVATE, CACHE_INT);
-	cache_add_by_secnum(sec_num, "CKA_NEVER_EXTRACTABLE",
-			    (const char *)BOOL_FOR_PRIVATE, CACHE_INT);
-
-	RSA_free(rsa);
 }
 
 int crypto_load_public_key(int sec_num, const char *pub)
@@ -128,8 +93,9 @@ int crypto_load_public_key(int sec_num, const char *pub)
 		return -1;
 	}
 
+	populate_global(sec_num);
 	if (EVP_PKEY_type(EVP_PKEY_id(pkey)) == EVP_PKEY_RSA) {
-		populate_rsa(sec_num, pkey);
+		crypto_rsa_populate(sec_num, pkey);
 	} else {
 		fprintf(stderr, "Unknown key type\n");
 		return -1;
@@ -233,7 +199,7 @@ void crypto_cache_free_pkey(void *pkey)
 	EVP_PKEY_free(pkey);
 }
 
-static EVP_PKEY_CTX *get_key(int sec_num)
+EVP_PKEY_CTX *crypto_get_key(int sec_num)
 {
 	EVP_PKEY_CTX *ctx;
 	EVP_PKEY *pkey;
@@ -248,98 +214,27 @@ static EVP_PKEY_CTX *get_key(int sec_num)
 	return ctx;
 }
 
-const EVP_MD *get_mgf1(unsigned long mgf1)
-{
-	switch (mgf1) {
-	case CKG_MGF1_SHA1:
-		return EVP_sha1();
-	case CKG_MGF1_SHA224:
-		return EVP_sha224();
-	case CKG_MGF1_SHA256:
-		return EVP_sha256();
-	case CKG_MGF1_SHA384:
-		return EVP_sha384();
-	case CKG_MGF1_SHA512:
-		return EVP_sha512();
-	default:
-		fprintf(stderr, "Unknown mgf1 type %ld\n", mgf1);
-		return NULL;
-	}
-}
-
-const EVP_MD *get_hash(unsigned long hash)
-{
-	switch (hash) {
-	case CKM_SHA_1:
-		return EVP_sha1();
-	case CKM_SHA224:
-		return EVP_sha224();
-	case CKM_SHA256:
-		return EVP_sha256();
-	case CKM_SHA384:
-		return EVP_sha384();
-	case CKM_SHA512:
-		return EVP_sha512();
-	default:
-		fprintf(stderr, "Unknown hash type %ld\n", hash);
-		return NULL;
-	}
-}
-
-static EVP_PKEY_CTX *add_padding(EVP_PKEY_CTX *ctx, CK_MECHANISM_PTR mech)
+static CK_KEY_TYPE get_key_type(int sec_num)
 {
-	if (mech->mechanism == CKM_RSA_PKCS) {
-		EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PADDING);
-	} else if (mech->mechanism == CKM_RSA_X_509) {
-		EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_NO_PADDING);
-	} else if (mech->mechanism == CKM_RSA_PKCS_PSS) {
-		CK_RSA_PKCS_PSS_PARAMS *p = mech->pParameter;
-
-		if (mech->ulParameterLen != sizeof(*p)) {
-			fprintf(stderr, "PSS mechanism parameter length %ld != %ld\n",
-				mech->ulParameterLen, (long)sizeof(*p));
-			goto err;
-		}
-		EVP_PKEY_CTX_set_signature_md(ctx, get_hash(p->hashAlg));
-		EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PSS_PADDING);
-		EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, get_mgf1(p->mgf));
-		EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, p->sLen);
-	} else if (mech->mechanism == CKM_RSA_PKCS_OAEP) {
-		CK_RSA_PKCS_OAEP_PARAMS *p = mech->pParameter;
-
-		if (mech->ulParameterLen != sizeof(*p)) {
-			fprintf(stderr, "OAEP mechanism parameter length %ld != %ld\n",
-				mech->ulParameterLen, (long)sizeof(*p));
-			goto err;
-		}
-		EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING);
-		EVP_PKEY_CTX_set_rsa_oaep_md(ctx, get_hash(p->hashAlg));
-		EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, get_mgf1(p->mgf));
-		if (p->source & CKZ_DATA_SPECIFIED) {
-			void *l = BUF_memdup(p->pSourceData,
-					     p->ulSourceDataLen);
-
-			EVP_PKEY_CTX_set0_rsa_oaep_label(ctx, l,
-							 p->ulSourceDataLen);
-		}
-	} else {
-		fprintf(stderr, "unknown mechanism %ld\n", mech->mechanism);
-		goto err;
-	}
+	int type;
+	const char *val;
 
-	return ctx;
- err:
-	EVP_PKEY_CTX_free(ctx);
-	return NULL;
+	val = cache_get_by_secnum(sec_num, "CKA_KEY_TYPE", &type);
+	if (type != CACHE_INT)
+		return -1;
+	return (CK_KEY_TYPE)val;
 }
 
 void *crypto_sign_init(int sec_num, CK_MECHANISM_PTR mech)
 {
 	EVP_PKEY_CTX *ctx;
 
-	ctx = get_key(sec_num);
+	ctx = crypto_get_key(sec_num);
 	EVP_PKEY_sign_init(ctx);
-	return add_padding(ctx, mech);
+	if (get_key_type(sec_num) == CKK_RSA)
+		ctx = crypto_rsa_add_padding(ctx, mech);
+
+	return ctx;
 }
 
 int crypto_sign(void *opdata, void *data, unsigned long data_len,
@@ -368,71 +263,23 @@ int crypto_sign(void *opdata, void *data, unsigned long data_len,
 	return ret;
 }
 
-void *crypto_decrypt_init(int sec_num, CK_MECHANISM_PTR mech)
-{
-	EVP_PKEY_CTX *ctx;
-
-	ctx = get_key(sec_num);
-	EVP_PKEY_decrypt_init(ctx);
-	return add_padding(ctx, mech);
-}
-
-int crypto_decrypt(void *opdata, void *enc_data, unsigned long enc_len,
-		   void *data, unsigned long *data_len)
-{
-	EVP_PKEY_CTX *ctx = opdata;
-	int ret = 0;
-	size_t len;
-
-	if (data_len)
-		len = *data_len;
-
-	if (!data) {
-		*data_len = EVP_PKEY_size(EVP_PKEY_CTX_get0_pkey(ctx));
-		return 0;
-	}
-
-	if (EVP_PKEY_decrypt(ctx, data, &len, enc_data, enc_len) <= 0) {
-		ERR_print_errors_fp(stderr);
-		ret = -1;
-	}
-	if (data_len)
-		*data_len = len;
-	EVP_PKEY_CTX_free(ctx);
-
-	return ret;
-}
-
-static CK_MECHANISM_TYPE mechanism_types[] = {
-	CKM_RSA_PKCS,
-	CKM_RSA_X_509,
-	CKM_RSA_PKCS_PSS,
-	CKM_RSA_PKCS_OAEP,
-};
-
 void crypto_fill_mechanism_list(int sec_num, unsigned long *mechs,
 				unsigned long *count)
 {
-	*count = ARRAY_SIZE(mechanism_types);
-	if (mechs)
-		memcpy(mechs, mechanism_types, sizeof(mechanism_types));
+	switch (get_key_type(sec_num)) {
+	case CKK_RSA:
+		crypto_rsa_fill_mechanism_list(sec_num, mechs, count);
+		break;
+	}
 }
 
 int crypto_check_mechanism(int sec_num, CK_MECHANISM_TYPE mech,
 			   CK_MECHANISM_INFO_PTR info)
 {
-	int i, found = 0;
-
-	for (i = 0; i < ARRAY_SIZE(mechanism_types); i++)
-		if (mech == mechanism_types[i])
-			found = 1;
-
-	if (!found)
+	switch (get_key_type(sec_num)) {
+	case CKK_RSA:
+		return crypto_rsa_check_mechanism(sec_num, mech, info);
+	default:
 		return 0;
-
-	info->ulMinKeySize = 2048;
-	info->ulMaxKeySize = 2048;
-	info->flags = CKF_HW|CKF_ENCRYPT|CKF_DECRYPT;
-
-	return 1;
+	}
 }
diff --git a/crypto.h b/crypto.h
new file mode 100644
index 0000000..ed42006
--- /dev/null
+++ b/crypto.h
@@ -0,0 +1,20 @@
+#ifndef _OPENSSL_CRYPTO_H
+#define _OPENSSL_CRYPTO_H
+
+#define ARRAY_SIZE(a) (sizeof (a) / sizeof ((a)[0]))
+
+/* crypto.c functions */
+void crypto_add_BN(int sec_num, const char *key, const BIGNUM *value,
+		   EVP_MD_CTX *ctx);
+void crypto_add_serial(int sec_num, EVP_MD_CTX *ctx);
+EVP_PKEY_CTX *crypto_get_key(int sec_num);
+
+/* crypto-rsa.c functions */
+void crypto_rsa_populate(int sec_num, EVP_PKEY *pkey);
+EVP_PKEY_CTX *crypto_rsa_add_padding(EVP_PKEY_CTX *ctx, CK_MECHANISM_PTR mech);
+void crypto_rsa_fill_mechanism_list(int sec_num, unsigned long *mechs,
+				    unsigned long *count);
+int crypto_rsa_check_mechanism(int sec_num, CK_MECHANISM_TYPE mech,
+			       CK_MECHANISM_INFO_PTR info);
+
+#endif