From 1d27e1f5c8f7ade40f0e85ddecbe9158393265e5 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 12 Apr 2024 08:45:59 -0700 Subject: crypto: x86/aes-xts - handle CTS encryption more efficiently When encrypting a message whose length isn't a multiple of 16 bytes, encrypt the last full block in the main loop. This works because only decryption uses the last two tweaks in reverse order, not encryption. This improves the performance of decrypting messages whose length isn't a multiple of the AES block length, shrinks the size of aes-xts-avx-x86_64.o by 5.0%, and eliminates two instructions (a test and a not-taken conditional jump) when encrypting a message whose length *is* a multiple of the AES block length. While it's not super useful to optimize for ciphertext stealing given that it's rarely needed in practice, the other two benefits mentioned above make this optimization worthwhile. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/aes-xts-avx-x86_64.S | 53 ++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/arch/x86/crypto/aes-xts-avx-x86_64.S b/arch/x86/crypto/aes-xts-avx-x86_64.S index 95e412e7601df..52f1997ed05a1 100644 --- a/arch/x86/crypto/aes-xts-avx-x86_64.S +++ b/arch/x86/crypto/aes-xts-avx-x86_64.S @@ -537,12 +537,16 @@ add $7*16, KEY .else add $(15+7)*16, KEY -.endif - // Check whether the data length is a multiple of the AES block length. + // When decrypting a message whose length isn't a multiple of the AES + // block length, exclude the last full block from the main loop by + // subtracting 16 from LEN. This is needed because ciphertext stealing + // decryption uses the last two tweaks in reverse order. We'll handle + // the last full block and the partial block specially at the end. test $15, LEN - jnz .Lneed_cts\@ + jnz .Lneed_cts_dec\@ .Lxts_init\@: +.endif // Cache as many round keys as possible. _load_round_keys @@ -685,31 +689,31 @@ _vaes_4x \enc, 1, 12 jmp .Lencrypt_4x_done\@ -.Lneed_cts\@: - // The data length isn't a multiple of the AES block length, so - // ciphertext stealing (CTS) will be needed. Subtract one block from - // LEN so that the main loop doesn't process the last full block. The - // CTS step will process it specially along with the partial block. +.if !\enc +.Lneed_cts_dec\@: sub $16, LEN jmp .Lxts_init\@ +.endif .Lcts\@: // Do ciphertext stealing (CTS) to en/decrypt the last full block and - // the partial block. CTS needs two tweaks. TWEAK0_XMM contains the - // next tweak; compute the one after that. Decryption uses these two - // tweaks in reverse order, so also define aliases to handle that. - _next_tweak TWEAK0_XMM, %xmm0, TWEAK1_XMM + // the partial block. TWEAK0_XMM contains the next tweak. + .if \enc - .set CTS_TWEAK0, TWEAK0_XMM - .set CTS_TWEAK1, TWEAK1_XMM + // If encrypting, the main loop already encrypted the last full block to + // create the CTS intermediate ciphertext. Prepare for the rest of CTS + // by rewinding the pointers and loading the intermediate ciphertext. + sub $16, SRC + sub $16, DST + vmovdqu (DST), %xmm0 .else - .set CTS_TWEAK0, TWEAK1_XMM - .set CTS_TWEAK1, TWEAK0_XMM -.endif - - // En/decrypt the last full block. + // If decrypting, the main loop didn't decrypt the last full block + // because CTS decryption uses the last two tweaks in reverse order. + // Do it now by advancing the tweak and decrypting the last full block. + _next_tweak TWEAK0_XMM, %xmm0, TWEAK1_XMM vmovdqu (SRC), %xmm0 - _aes_crypt \enc, _XMM, CTS_TWEAK0, %xmm0 + _aes_crypt \enc, _XMM, TWEAK1_XMM, %xmm0 +.endif .if USE_AVX10 // Create a mask that has the first LEN bits set. @@ -717,9 +721,10 @@ bzhi LEN, %rax, %rax kmovq %rax, %k1 - // Swap the first LEN bytes of the above result with the partial block. - // Note that to support in-place en/decryption, the load from the src - // partial block must happen before the store to the dst partial block. + // Swap the first LEN bytes of the en/decryption of the last full block + // with the partial block. Note that to support in-place en/decryption, + // the load from the src partial block must happen before the store to + // the dst partial block. vmovdqa %xmm0, %xmm1 vmovdqu8 16(SRC), %xmm0{%k1} vmovdqu8 %xmm1, 16(DST){%k1} @@ -750,7 +755,7 @@ vpblendvb %xmm3, %xmm0, %xmm1, %xmm0 .endif // En/decrypt again and store the last full block. - _aes_crypt \enc, _XMM, CTS_TWEAK1, %xmm0 + _aes_crypt \enc, _XMM, TWEAK0_XMM, %xmm0 vmovdqu %xmm0, (DST) jmp .Ldone\@ .endm -- cgit 1.2.3-korg