|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
| 2 | +/* |
| 3 | + * Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions |
| 4 | + * |
| 5 | + * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> |
| 6 | + */ |
| 7 | + |
| 8 | +#include <linux/cpufeature.h> |
| 9 | +#include <linux/crc-t10dif.h> |
| 10 | +#include <linux/init.h> |
| 11 | +#include <linux/kernel.h> |
| 12 | +#include <linux/module.h> |
| 13 | +#include <linux/string.h> |
| 14 | + |
| 15 | +#include <crypto/internal/simd.h> |
| 16 | + |
| 17 | +#include <asm/neon.h> |
| 18 | +#include <asm/simd.h> |
| 19 | + |
| 20 | +static DEFINE_STATIC_KEY_FALSE(have_asimd); |
| 21 | +static DEFINE_STATIC_KEY_FALSE(have_pmull); |
| 22 | + |
| 23 | +#define CRC_T10DIF_PMULL_CHUNK_SIZE 16U |
| 24 | + |
| 25 | +asmlinkage void crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len, |
| 26 | + u8 out[16]); |
| 27 | +asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len); |
| 28 | + |
| 29 | +u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length) |
| 30 | +{ |
| 31 | + if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) { |
| 32 | + if (static_branch_likely(&have_pmull)) { |
| 33 | + if (crypto_simd_usable()) { |
| 34 | + kernel_neon_begin(); |
| 35 | + crc = crc_t10dif_pmull_p64(crc, data, length); |
| 36 | + kernel_neon_end(); |
| 37 | + return crc; |
| 38 | + } |
| 39 | + } else if (length > CRC_T10DIF_PMULL_CHUNK_SIZE && |
| 40 | + static_branch_likely(&have_asimd) && |
| 41 | + crypto_simd_usable()) { |
| 42 | + u8 buf[16]; |
| 43 | + |
| 44 | + kernel_neon_begin(); |
| 45 | + crc_t10dif_pmull_p8(crc, data, length, buf); |
| 46 | + kernel_neon_end(); |
| 47 | + |
| 48 | + crc = 0; |
| 49 | + data = buf; |
| 50 | + length = sizeof(buf); |
| 51 | + } |
| 52 | + } |
| 53 | + return crc_t10dif_generic(crc, data, length); |
| 54 | +} |
| 55 | +EXPORT_SYMBOL(crc_t10dif_arch); |
| 56 | + |
| 57 | +static int __init crc_t10dif_arm64_init(void) |
| 58 | +{ |
| 59 | + if (cpu_have_named_feature(ASIMD)) { |
| 60 | + static_branch_enable(&have_asimd); |
| 61 | + if (cpu_have_named_feature(PMULL)) |
| 62 | + static_branch_enable(&have_pmull); |
| 63 | + } |
| 64 | + return 0; |
| 65 | +} |
| 66 | +arch_initcall(crc_t10dif_arm64_init); |
| 67 | + |
| 68 | +static void __exit crc_t10dif_arm64_exit(void) |
| 69 | +{ |
| 70 | +} |
| 71 | +module_exit(crc_t10dif_arm64_exit); |
| 72 | + |
| 73 | +bool crc_t10dif_is_optimized(void) |
| 74 | +{ |
| 75 | + return static_key_enabled(&have_asimd); |
| 76 | +} |
| 77 | +EXPORT_SYMBOL(crc_t10dif_is_optimized); |
| 78 | + |
| 79 | +MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); |
| 80 | +MODULE_DESCRIPTION("CRC-T10DIF using arm64 NEON and Crypto Extensions"); |
| 81 | +MODULE_LICENSE("GPL v2"); |
0 commit comments