Skip to content

Commit a9a8b1a

Browse files
ardbiesheuvelEric Biggers
ardbiesheuvel
authored and
Eric Biggers
committed
crypto/arm64: aes/xts - Use single ksimd scope to reduce stack bloat
The ciphertext stealing logic in the AES-XTS implementation creates a separate ksimd scope to call into the FP/SIMD core routines, and in some cases (CONFIG_KASAN_STACK is one, but there might be others), the 528 byte kernel mode FP/SIMD buffer that is allocated inside this scope is not shared with the preceding ksimd scope, resulting in unnecessary stack bloat. Considering that a) the XTS ciphertext stealing logic is never called for block encryption use cases, and XTS is rarely used for anything else, b) in the vast majority of cases, the entire input block is processed during the first iteration of the loop, we can combine both ksimd scopes into a single one with no practical impact on how often/how long FP/SIMD is en/disabled, allowing us to reuse the same stack slot for both FP/SIMD routine calls. Fixes: ba3c1b3 ("crypto/arm64: aes-blk - Switch to 'ksimd' scoped guard API") Signed-off-by: Ard Biesheuvel <ardb@kernel.org> Tested-by: Arnd Bergmann <arnd@arndb.de> Link: https://lore.kernel.org/r/20251203163803.157541-5-ardb@kernel.org Signed-off-by: Eric Biggers <ebiggers@kernel.org>
1 parent 68b233b commit a9a8b1a

2 files changed

Lines changed: 57 additions & 62 deletions

File tree

‎arch/arm64/crypto/aes-glue.c‎

Lines changed: 36 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -549,38 +549,37 @@ static int __maybe_unused xts_encrypt(struct skcipher_request *req)
549549
tail = 0;
550550
}
551551

552-
for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) {
553-
int nbytes = walk.nbytes;
552+
scoped_ksimd() {
553+
for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) {
554+
int nbytes = walk.nbytes;
554555

555-
if (walk.nbytes < walk.total)
556-
nbytes &= ~(AES_BLOCK_SIZE - 1);
556+
if (walk.nbytes < walk.total)
557+
nbytes &= ~(AES_BLOCK_SIZE - 1);
557558

558-
scoped_ksimd()
559559
aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
560560
ctx->key1.key_enc, rounds, nbytes,
561561
ctx->key2.key_enc, walk.iv, first);
562-
err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
563-
}
562+
err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
563+
}
564564

565-
if (err || likely(!tail))
566-
return err;
565+
if (err || likely(!tail))
566+
return err;
567567

568-
dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
569-
if (req->dst != req->src)
570-
dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
568+
dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
569+
if (req->dst != req->src)
570+
dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
571571

572-
skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
573-
req->iv);
572+
skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
573+
req->iv);
574574

575-
err = skcipher_walk_virt(&walk, &subreq, false);
576-
if (err)
577-
return err;
575+
err = skcipher_walk_virt(&walk, &subreq, false);
576+
if (err)
577+
return err;
578578

579-
scoped_ksimd()
580579
aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
581580
ctx->key1.key_enc, rounds, walk.nbytes,
582581
ctx->key2.key_enc, walk.iv, first);
583-
582+
}
584583
return skcipher_walk_done(&walk, 0);
585584
}
586585

@@ -619,39 +618,37 @@ static int __maybe_unused xts_decrypt(struct skcipher_request *req)
619618
tail = 0;
620619
}
621620

622-
for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) {
623-
int nbytes = walk.nbytes;
621+
scoped_ksimd() {
622+
for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) {
623+
int nbytes = walk.nbytes;
624624

625-
if (walk.nbytes < walk.total)
626-
nbytes &= ~(AES_BLOCK_SIZE - 1);
625+
if (walk.nbytes < walk.total)
626+
nbytes &= ~(AES_BLOCK_SIZE - 1);
627627

628-
scoped_ksimd()
629628
aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
630629
ctx->key1.key_dec, rounds, nbytes,
631630
ctx->key2.key_enc, walk.iv, first);
632-
err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
633-
}
631+
err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
632+
}
634633

635-
if (err || likely(!tail))
636-
return err;
637-
638-
dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
639-
if (req->dst != req->src)
640-
dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
634+
if (err || likely(!tail))
635+
return err;
641636

642-
skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
643-
req->iv);
637+
dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
638+
if (req->dst != req->src)
639+
dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
644640

645-
err = skcipher_walk_virt(&walk, &subreq, false);
646-
if (err)
647-
return err;
641+
skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
642+
req->iv);
648643

644+
err = skcipher_walk_virt(&walk, &subreq, false);
645+
if (err)
646+
return err;
649647

650-
scoped_ksimd()
651648
aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
652649
ctx->key1.key_dec, rounds, walk.nbytes,
653650
ctx->key2.key_enc, walk.iv, first);
654-
651+
}
655652
return skcipher_walk_done(&walk, 0);
656653
}
657654

‎arch/arm64/crypto/aes-neonbs-glue.c‎

Lines changed: 21 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -312,13 +312,13 @@ static int __xts_crypt(struct skcipher_request *req, bool encrypt,
312312
if (err)
313313
return err;
314314

315-
while (walk.nbytes >= AES_BLOCK_SIZE) {
316-
int blocks = (walk.nbytes / AES_BLOCK_SIZE) & ~7;
317-
out = walk.dst.virt.addr;
318-
in = walk.src.virt.addr;
319-
nbytes = walk.nbytes;
315+
scoped_ksimd() {
316+
while (walk.nbytes >= AES_BLOCK_SIZE) {
317+
int blocks = (walk.nbytes / AES_BLOCK_SIZE) & ~7;
318+
out = walk.dst.virt.addr;
319+
in = walk.src.virt.addr;
320+
nbytes = walk.nbytes;
320321

321-
scoped_ksimd() {
322322
if (blocks >= 8) {
323323
if (first == 1)
324324
neon_aes_ecb_encrypt(walk.iv, walk.iv,
@@ -344,30 +344,28 @@ static int __xts_crypt(struct skcipher_request *req, bool encrypt,
344344
ctx->twkey, walk.iv, first);
345345
nbytes = first = 0;
346346
}
347+
err = skcipher_walk_done(&walk, nbytes);
347348
}
348-
err = skcipher_walk_done(&walk, nbytes);
349-
}
350349

351-
if (err || likely(!tail))
352-
return err;
350+
if (err || likely(!tail))
351+
return err;
353352

354-
/* handle ciphertext stealing */
355-
dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
356-
if (req->dst != req->src)
357-
dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
353+
/* handle ciphertext stealing */
354+
dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
355+
if (req->dst != req->src)
356+
dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
358357

359-
skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
360-
req->iv);
358+
skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
359+
req->iv);
361360

362-
err = skcipher_walk_virt(&walk, req, false);
363-
if (err)
364-
return err;
361+
err = skcipher_walk_virt(&walk, req, false);
362+
if (err)
363+
return err;
365364

366-
out = walk.dst.virt.addr;
367-
in = walk.src.virt.addr;
368-
nbytes = walk.nbytes;
365+
out = walk.dst.virt.addr;
366+
in = walk.src.virt.addr;
367+
nbytes = walk.nbytes;
369368

370-
scoped_ksimd() {
371369
if (encrypt)
372370
neon_aes_xts_encrypt(out, in, ctx->cts.key_enc,
373371
ctx->key.rounds, nbytes, ctx->twkey,

0 commit comments

Comments
 (0)