crc-t10dif.h

 1// SPDX-License-Identifier: GPL-2.0-or-later
 2/*
 3 * Calculate a CRC T10-DIF with vpmsum acceleration
 4 *
 5 * Copyright 2017, Daniel Axtens, IBM Corporation.
 6 * [based on crc32c-vpmsum_glue.c]
 7 */
 8
 9#include <asm/simd.h>
10#include <asm/switch_to.h>
11#include <linux/cpufeature.h>
12#include <linux/jump_label.h>
13#include <linux/preempt.h>
14#include <linux/uaccess.h>
15
16#define VMX_ALIGN		16
17#define VMX_ALIGN_MASK		(VMX_ALIGN-1)
18
19#define VECTOR_BREAKPOINT	64
20
21static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vec_crypto);
22
23u32 __crct10dif_vpmsum(u32 crc, unsigned char const *p, size_t len);
24
25static inline u16 crc_t10dif_arch(u16 crci, const u8 *p, size_t len)
26{
27	unsigned int prealign;
28	unsigned int tail;
29	u32 crc = crci;
30
31	if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) ||
32	    !static_branch_likely(&have_vec_crypto) ||
33	    unlikely(!may_use_simd()))
34		return crc_t10dif_generic(crc, p, len);
35
36	if ((unsigned long)p & VMX_ALIGN_MASK) {
37		prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK);
38		crc = crc_t10dif_generic(crc, p, prealign);
39		len -= prealign;
40		p += prealign;
41	}
42
43	if (len & ~VMX_ALIGN_MASK) {
44		crc <<= 16;
45		preempt_disable();
46		pagefault_disable();
47		enable_kernel_altivec();
48		crc = __crct10dif_vpmsum(crc, p, len & ~VMX_ALIGN_MASK);
49		disable_kernel_altivec();
50		pagefault_enable();
51		preempt_enable();
52		crc >>= 16;
53	}
54
55	tail = len & VMX_ALIGN_MASK;
56	if (tail) {
57		p += len & ~VMX_ALIGN_MASK;
58		crc = crc_t10dif_generic(crc, p, tail);
59	}
60
61	return crc & 0xffff;
62}
63
64#define crc_t10dif_mod_init_arch crc_t10dif_mod_init_arch
65static void crc_t10dif_mod_init_arch(void)
66{
67	if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
68	    (cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO))
69		static_branch_enable(&have_vec_crypto);
70}