crc32.h

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * crc32-mips.c - CRC32 and CRC32C using optional MIPSr6 instructions
  4 *
  5 * Module based on arm64/crypto/crc32-arm.c
  6 *
  7 * Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org>
  8 * Copyright (C) 2018 MIPS Tech, LLC
  9 */
 10
 11#include <linux/cpufeature.h>
 12#include <asm/mipsregs.h>
 13#include <linux/unaligned.h>
 14
 15#ifndef TOOLCHAIN_SUPPORTS_CRC
 16#define _ASM_SET_CRC(OP, SZ, TYPE)					  \
 17_ASM_MACRO_3R(OP, rt, rs, rt2,						  \
 18	".ifnc	\\rt, \\rt2\n\t"					  \
 19	".error	\"invalid operands \\\"" #OP " \\rt,\\rs,\\rt2\\\"\"\n\t" \
 20	".endif\n\t"							  \
 21	_ASM_INSN_IF_MIPS(0x7c00000f | (__rt << 16) | (__rs << 21) |	  \
 22			  ((SZ) <<  6) | ((TYPE) << 8))			  \
 23	_ASM_INSN32_IF_MM(0x00000030 | (__rs << 16) | (__rt << 21) |	  \
 24			  ((SZ) << 14) | ((TYPE) << 3)))
 25#define _ASM_UNSET_CRC(op, SZ, TYPE) ".purgem " #op "\n\t"
 26#else /* !TOOLCHAIN_SUPPORTS_CRC */
 27#define _ASM_SET_CRC(op, SZ, TYPE) ".set\tcrc\n\t"
 28#define _ASM_UNSET_CRC(op, SZ, TYPE)
 29#endif
 30
 31#define __CRC32(crc, value, op, SZ, TYPE)		\
 32do {							\
 33	__asm__ __volatile__(				\
 34		".set	push\n\t"			\
 35		_ASM_SET_CRC(op, SZ, TYPE)		\
 36		#op "	%0, %1, %0\n\t"			\
 37		_ASM_UNSET_CRC(op, SZ, TYPE)		\
 38		".set	pop"				\
 39		: "+r" (crc)				\
 40		: "r" (value));				\
 41} while (0)
 42
 43#define _CRC32_crc32b(crc, value)	__CRC32(crc, value, crc32b, 0, 0)
 44#define _CRC32_crc32h(crc, value)	__CRC32(crc, value, crc32h, 1, 0)
 45#define _CRC32_crc32w(crc, value)	__CRC32(crc, value, crc32w, 2, 0)
 46#define _CRC32_crc32d(crc, value)	__CRC32(crc, value, crc32d, 3, 0)
 47#define _CRC32_crc32cb(crc, value)	__CRC32(crc, value, crc32cb, 0, 1)
 48#define _CRC32_crc32ch(crc, value)	__CRC32(crc, value, crc32ch, 1, 1)
 49#define _CRC32_crc32cw(crc, value)	__CRC32(crc, value, crc32cw, 2, 1)
 50#define _CRC32_crc32cd(crc, value)	__CRC32(crc, value, crc32cd, 3, 1)
 51
 52#define _CRC32(crc, value, size, op) \
 53	_CRC32_##op##size(crc, value)
 54
 55#define CRC32(crc, value, size) \
 56	_CRC32(crc, value, size, crc32)
 57
 58#define CRC32C(crc, value, size) \
 59	_CRC32(crc, value, size, crc32c)
 60
 61static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32);
 62
 63static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
 64{
 65	if (!static_branch_likely(&have_crc32))
 66		return crc32_le_base(crc, p, len);
 67
 68	if (IS_ENABLED(CONFIG_64BIT)) {
 69		for (; len >= sizeof(u64); p += sizeof(u64), len -= sizeof(u64)) {
 70			u64 value = get_unaligned_le64(p);
 71
 72			CRC32(crc, value, d);
 73		}
 74
 75		if (len & sizeof(u32)) {
 76			u32 value = get_unaligned_le32(p);
 77
 78			CRC32(crc, value, w);
 79			p += sizeof(u32);
 80		}
 81	} else {
 82		for (; len >= sizeof(u32); len -= sizeof(u32)) {
 83			u32 value = get_unaligned_le32(p);
 84
 85			CRC32(crc, value, w);
 86			p += sizeof(u32);
 87		}
 88	}
 89
 90	if (len & sizeof(u16)) {
 91		u16 value = get_unaligned_le16(p);
 92
 93		CRC32(crc, value, h);
 94		p += sizeof(u16);
 95	}
 96
 97	if (len & sizeof(u8)) {
 98		u8 value = *p++;
 99
100		CRC32(crc, value, b);
101	}
102
103	return crc;
104}
105
106static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
107{
108	if (!static_branch_likely(&have_crc32))
109		return crc32c_base(crc, p, len);
110
111	if (IS_ENABLED(CONFIG_64BIT)) {
112		for (; len >= sizeof(u64); p += sizeof(u64), len -= sizeof(u64)) {
113			u64 value = get_unaligned_le64(p);
114
115			CRC32C(crc, value, d);
116		}
117
118		if (len & sizeof(u32)) {
119			u32 value = get_unaligned_le32(p);
120
121			CRC32C(crc, value, w);
122			p += sizeof(u32);
123		}
124	} else {
125		for (; len >= sizeof(u32); len -= sizeof(u32)) {
126			u32 value = get_unaligned_le32(p);
127
128			CRC32C(crc, value, w);
129			p += sizeof(u32);
130		}
131	}
132
133	if (len & sizeof(u16)) {
134		u16 value = get_unaligned_le16(p);
135
136		CRC32C(crc, value, h);
137		p += sizeof(u16);
138	}
139
140	if (len & sizeof(u8)) {
141		u8 value = *p++;
142
143		CRC32C(crc, value, b);
144	}
145	return crc;
146}
147
148#define crc32_be_arch crc32_be_base /* not implemented on this arch */
149
150#define crc32_mod_init_arch crc32_mod_init_arch
151static void crc32_mod_init_arch(void)
152{
153	if (cpu_have_feature(cpu_feature(MIPS_CRC32)))
154		static_branch_enable(&have_crc32);
155}
156
157static inline u32 crc32_optimizations_arch(void)
158{
159	if (static_key_enabled(&have_crc32))
160		return CRC32_LE_OPTIMIZATION | CRC32C_OPTIMIZATION;
161	return 0;
162}