Add raw (x509) padding type

also redo signatures so we only return the size for a null destination
argument.

Signed-off-by: James Bottomley <James.Bottomley@HansenPartnership.com>

3 files changed, 88 insertions, 31 deletions

diff --git a/crypto.c b/crypto.c
index 027248a..7baab5d 100644
--- a/crypto.c
+++ b/crypto.c
@@ -10,6 +10,7 @@
 #include <stdio.h>
 #include <string.h>
 #include <wordexp.h>
+#include <libiberty.h>
 
 #include <openssl/evp.h>
 #include <openssl/pem.h>
@@ -145,8 +146,7 @@ int crypto_load_private_key(int sec_num, const unsigned char *pin, int pin_len)
 	return 0;
 }
 
-int crypto_sign(int sec_num, void *data, unsigned long data_len,
-		void *sig, unsigned long *sig_len)
+static EVP_PKEY_CTX *get_key(int sec_num)
 {
 	EVP_PKEY_CTX *ctx;
 	EVP_PKEY *pkey;
@@ -154,33 +154,86 @@ int crypto_sign(int sec_num, void *data, unsigned long data_len,
 	pkey = (EVP_PKEY *)cache_get_by_secnum(sec_num, "pkey", NULL);
 	if (!pkey) {
 		fprintf(stderr, "crypto_encrypt internal error: no PKEY\n");
-		return -1;
+		return NULL;
 	}
 	ctx = EVP_PKEY_CTX_new(pkey, NULL);
+
+	return ctx;
+}
+
+static EVP_PKEY_CTX *add_padding(EVP_PKEY_CTX *ctx, CK_MECHANISM_PTR mech)
+{
+	if (mech->mechanism == CKM_RSA_PKCS) {
+		EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PADDING);
+	} else if (mech->mechanism == CKM_RSA_X_509) {
+		EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_NO_PADDING);
+	} else {
+		fprintf(stderr, "unknown mechanism %ld\n", mech->mechanism);
+		return NULL;
+	}
+
+	return ctx;
+}
+
+void *crypto_sign_init(int sec_num, CK_MECHANISM_PTR mech)
+{
+	EVP_PKEY_CTX *ctx;
+
+	ctx = get_key(sec_num);
 	EVP_PKEY_sign_init(ctx);
-	EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PADDING);
+	return add_padding(ctx, mech);
+}
+
+int crypto_sign(void *opdata, void *data, unsigned long data_len,
+		void *sig, unsigned long *sig_len)
+{
+	EVP_PKEY_CTX *ctx = opdata;
+
+	if (!sig) {
+		*sig_len = EVP_PKEY_size(EVP_PKEY_CTX_get0_pkey(ctx));
+		return 0;
+	}
+
 	EVP_PKEY_sign(ctx, sig, sig_len, data, data_len);
 	EVP_PKEY_CTX_free(ctx);
 
 	return 0;
 }
 
-int crypto_decrypt(int sec_num, void *enc_data, unsigned long enc_len,
-		   void *data, unsigned long *data_len)
+void *crypto_decrypt_init(int sec_num, CK_MECHANISM_PTR mech)
 {
 	EVP_PKEY_CTX *ctx;
-	EVP_PKEY *pkey;
 
-	pkey = (EVP_PKEY *)cache_get_by_secnum(sec_num, "pkey", NULL);
-	if (!pkey) {
-		fprintf(stderr, "crypto_encrypt internal error: no PKEY\n");
-		return -1;
-	}
-	ctx = EVP_PKEY_CTX_new(pkey, NULL);
+	ctx = get_key(sec_num);
 	EVP_PKEY_decrypt_init(ctx);
-	EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PADDING);
+	return add_padding(ctx, mech);
+}
+
+int crypto_decrypt(void *opdata, void *enc_data, unsigned long enc_len,
+		   void *data, unsigned long *data_len)
+{
+	EVP_PKEY_CTX *ctx = opdata;
+
+	if (!data) {
+		*data_len = EVP_PKEY_size(EVP_PKEY_CTX_get0_pkey(ctx));
+		return 0;
+	}
+
 	EVP_PKEY_decrypt(ctx, data, data_len, enc_data, enc_len);
 	EVP_PKEY_CTX_free(ctx);
 
 	return 0;
 }
+
+void crypto_fill_mechanism_list(int sec_num, unsigned long *mechs,
+				unsigned long *count)
+{
+	CK_MECHANISM_TYPE types[] = {
+		CKM_RSA_PKCS,
+		CKM_RSA_X_509,
+	};
+
+	*count = ARRAY_SIZE(types);
+	if (mechs)
+		memcpy(mechs, types, sizeof(types));
+}
diff --git a/openssl-pkcs11.h b/openssl-pkcs11.h
index 10252d3..f225d85 100644
--- a/openssl-pkcs11.h
+++ b/openssl-pkcs11.h
@@ -1,6 +1,8 @@
 #ifndef _OPENSSL_PKCS11_H
 #define _OPENSSL_PKCS11_H
 
+#include <opencryptoki/pkcs11.h>
+
 #define ENV_CONFIG "OPENSSL_PKCS11_CONF"
 #define CONFIG_FILE ".config/openssl-pkcs11/openssl-pkcs11.conf"
 #define GLOBAL_SECTION "global"
@@ -16,10 +18,14 @@
 /* crypto.c exported functions */
 int crypto_load_public_key(int sec_num, const char *pub);
 int crypto_load_private_key(int sec_num, const unsigned char *pin, int pin_len);
-int crypto_sign(int sec_num, void *data, unsigned long data_len,
+void *crypto_sign_init(int sec_num, CK_MECHANISM_PTR mech);
+int crypto_sign(void *opdata, void *data, unsigned long data_len,
 		void *sig, unsigned long *sig_len);
-int crypto_decrypt(int sec_num, void *enc_data, unsigned long enc_len,
+void *crypto_decrypt_init(int sec_num, CK_MECHANISM_PTR mech);
+int crypto_decrypt(void *opdata, void *enc_data, unsigned long enc_len,
 		   void *data, unsigned long *data_len);
+void crypto_fill_mechanism_list(int sec_num, unsigned long *mechs,
+				unsigned long *count);
 
 /* ini.c exported functions */
 void parse_ini_file(void);
diff --git a/pkcs11.c b/pkcs11.c
index 529e957..55f6785 100644
--- a/pkcs11.c
+++ b/pkcs11.c
@@ -406,9 +406,7 @@ CK_RV
 C_GetMechanismList(CK_SLOT_ID slot, CK_MECHANISM_TYPE_PTR mechs,
 		   CK_ULONG_PTR count)
 {
-	if (mechs)
-		mechs[0] = CKM_RSA_PKCS;
-	*count = 1;
+	crypto_fill_mechanism_list(slot, mechs, count);
 
 	return CKR_OK;
 }
@@ -428,24 +426,25 @@ C_GetMechanismInfo(CK_SLOT_ID slot, CK_MECHANISM_TYPE type,
 	return CKR_OK;
 }
 
+void *opstate;
+
 CK_RV
 C_SignInit(CK_SESSION_HANDLE handle, CK_MECHANISM_PTR mech,
 	   CK_OBJECT_HANDLE key)
 {
-	if (mech->mechanism != CKM_RSA_PKCS)
-		return CKR_ARGUMENTS_BAD;
 	if ((key & 1) != 1 || key >> 1 != handle)
 		return CKR_ARGUMENTS_BAD;
-
-	return CKR_OK;
+	opstate = crypto_sign_init(handle, mech);
+	if (opstate)
+		return CKR_OK;
+	return CKR_ARGUMENTS_BAD;
 }
 
 CK_RV
 C_Sign(CK_SESSION_HANDLE handle, CK_BYTE_PTR data, CK_ULONG data_len,
        CK_BYTE_PTR sig, CK_ULONG_PTR sig_len)
 {
-	int sec_num = handle;
-	if (crypto_sign(sec_num, data, data_len, sig, sig_len))
+	if (crypto_sign(opstate, data, data_len, sig, sig_len))
 		return CKR_ARGUMENTS_BAD;
 	return CKR_OK;
 }
@@ -454,20 +453,19 @@ CK_RV
 C_DecryptInit(CK_SESSION_HANDLE handle, CK_MECHANISM_PTR mech,
 	      CK_OBJECT_HANDLE key)
 {
-	if (mech->mechanism != CKM_RSA_PKCS)
-		return CKR_ARGUMENTS_BAD;
 	if ((key & 1) != 1 || key >> 1 != handle)
 		return CKR_ARGUMENTS_BAD;
-
-	return CKR_OK;
+	opstate = crypto_decrypt_init(handle, mech);
+	if (opstate)
+		return CKR_OK;
+	return CKR_ARGUMENTS_BAD;
 }
 
 CK_RV
 C_Decrypt(CK_SESSION_HANDLE handle, CK_BYTE_PTR enc_data, CK_ULONG enc_len,
 	  CK_BYTE_PTR data, CK_ULONG_PTR data_len)
 {
-	int sec_num = handle;
-	if (crypto_decrypt(sec_num, enc_data, enc_len, data, data_len))
+	if (crypto_decrypt(opstate, enc_data, enc_len, data, data_len))
 		return CKR_ARGUMENTS_BAD;
 	return CKR_OK;
 }