crc32.h

 1// SPDX-License-Identifier: GPL-2.0
 2/*
 3 * CRC-32 implemented with the z/Architecture Vector Extension Facility.
 4 *
 5 * Copyright IBM Corp. 2015
 6 * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
 7 */
 8
 9#include <linux/cpufeature.h>
10#include <asm/fpu.h>
11#include "crc32-vx.h"
12
13#define VX_MIN_LEN		64
14#define VX_ALIGNMENT		16L
15#define VX_ALIGN_MASK		(VX_ALIGNMENT - 1)
16
17/*
18 * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension
19 *
20 * Creates a function to perform a particular CRC-32 computation. Depending
21 * on the message buffer, the hardware-accelerated or software implementation
22 * is used.   Note that the message buffer is aligned to improve fetch
23 * operations of VECTOR LOAD MULTIPLE instructions.
24 */
25#define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw)		    \
26	static inline u32 ___fname(u32 crc, const u8 *data, size_t datalen) \
27	{								    \
28		unsigned long prealign, aligned, remaining;		    \
29		DECLARE_KERNEL_FPU_ONSTACK16(vxstate);			    \
30									    \
31		if (datalen < VX_MIN_LEN + VX_ALIGN_MASK || !cpu_has_vx())  \
32			return ___crc32_sw(crc, data, datalen);		    \
33									    \
34		if ((unsigned long)data & VX_ALIGN_MASK) {		    \
35			prealign = VX_ALIGNMENT -			    \
36				  ((unsigned long)data & VX_ALIGN_MASK);    \
37			datalen -= prealign;				    \
38			crc = ___crc32_sw(crc, data, prealign);		    \
39			data = (void *)((unsigned long)data + prealign);    \
40		}							    \
41									    \
42		aligned = datalen & ~VX_ALIGN_MASK;			    \
43		remaining = datalen & VX_ALIGN_MASK;			    \
44									    \
45		kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW);		    \
46		crc = ___crc32_vx(crc, data, aligned);			    \
47		kernel_fpu_end(&vxstate, KERNEL_VXR_LOW);		    \
48									    \
49		if (remaining)						    \
50			crc = ___crc32_sw(crc, data + aligned, remaining);  \
51									    \
52		return crc;						    \
53	}
54
55DEFINE_CRC32_VX(crc32_le_arch, crc32_le_vgfm_16, crc32_le_base)
56DEFINE_CRC32_VX(crc32_be_arch, crc32_be_vgfm_16, crc32_be_base)
57DEFINE_CRC32_VX(crc32c_arch, crc32c_le_vgfm_16, crc32c_base)
58
59static inline u32 crc32_optimizations_arch(void)
60{
61	if (cpu_has_vx()) {
62		return CRC32_LE_OPTIMIZATION |
63		       CRC32_BE_OPTIMIZATION |
64		       CRC32C_OPTIMIZATION;
65	}
66	return 0;
67}