crc32.h

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * CRC32 and CRC32C using LoongArch crc* instructions
  4 *
  5 * Module based on mips/crypto/crc32-mips.c
  6 *
  7 * Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org>
  8 * Copyright (C) 2018 MIPS Tech, LLC
  9 * Copyright (C) 2020-2023 Loongson Technology Corporation Limited
 10 */
 11
 12#include <asm/cpu-features.h>
 13#include <linux/unaligned.h>
 14
 15#define _CRC32(crc, value, size, type)			\
 16do {							\
 17	__asm__ __volatile__(				\
 18		#type ".w." #size ".w" " %0, %1, %0\n\t"\
 19		: "+r" (crc)				\
 20		: "r" (value)				\
 21		: "memory");				\
 22} while (0)
 23
 24#define CRC32(crc, value, size)		_CRC32(crc, value, size, crc)
 25#define CRC32C(crc, value, size)	_CRC32(crc, value, size, crcc)
 26
 27static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32);
 28
 29static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
 30{
 31	if (!static_branch_likely(&have_crc32))
 32		return crc32_le_base(crc, p, len);
 33
 34	while (len >= sizeof(u64)) {
 35		u64 value = get_unaligned_le64(p);
 36
 37		CRC32(crc, value, d);
 38		p += sizeof(u64);
 39		len -= sizeof(u64);
 40	}
 41
 42	if (len & sizeof(u32)) {
 43		u32 value = get_unaligned_le32(p);
 44
 45		CRC32(crc, value, w);
 46		p += sizeof(u32);
 47	}
 48
 49	if (len & sizeof(u16)) {
 50		u16 value = get_unaligned_le16(p);
 51
 52		CRC32(crc, value, h);
 53		p += sizeof(u16);
 54	}
 55
 56	if (len & sizeof(u8)) {
 57		u8 value = *p++;
 58
 59		CRC32(crc, value, b);
 60	}
 61
 62	return crc;
 63}
 64
 65static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
 66{
 67	if (!static_branch_likely(&have_crc32))
 68		return crc32c_base(crc, p, len);
 69
 70	while (len >= sizeof(u64)) {
 71		u64 value = get_unaligned_le64(p);
 72
 73		CRC32C(crc, value, d);
 74		p += sizeof(u64);
 75		len -= sizeof(u64);
 76	}
 77
 78	if (len & sizeof(u32)) {
 79		u32 value = get_unaligned_le32(p);
 80
 81		CRC32C(crc, value, w);
 82		p += sizeof(u32);
 83	}
 84
 85	if (len & sizeof(u16)) {
 86		u16 value = get_unaligned_le16(p);
 87
 88		CRC32C(crc, value, h);
 89		p += sizeof(u16);
 90	}
 91
 92	if (len & sizeof(u8)) {
 93		u8 value = *p++;
 94
 95		CRC32C(crc, value, b);
 96	}
 97
 98	return crc;
 99}
100
101#define crc32_be_arch crc32_be_base /* not implemented on this arch */
102
103#define crc32_mod_init_arch crc32_mod_init_arch
104static void crc32_mod_init_arch(void)
105{
106	if (cpu_has_crc32)
107		static_branch_enable(&have_crc32);
108}
109
110static inline u32 crc32_optimizations_arch(void)
111{
112	if (static_key_enabled(&have_crc32))
113		return CRC32_LE_OPTIMIZATION | CRC32C_OPTIMIZATION;
114	return 0;
115}