recov_rvv.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright 2024 Institute of Software, CAS.
  4 * Author: Chunyan Zhang <zhangchunyan@iscas.ac.cn>
  5 */
  6
  7#include <linux/raid/pq.h>
  8#include "rvv.h"
  9
 10static void __raid6_2data_recov_rvv(int bytes, u8 *p, u8 *q, u8 *dp,
 11				    u8 *dq, const u8 *pbmul,
 12				    const u8 *qmul)
 13{
 14	asm volatile (".option	push\n"
 15		      ".option	arch,+v\n"
 16		      "vsetvli	x0, %[avl], e8, m1, ta, ma\n"
 17		      ".option	pop\n"
 18		      : :
 19		      [avl]"r"(16)
 20	);
 21
 22	/*
 23	 * while ( bytes-- ) {
 24	 *	uint8_t px, qx, db;
 25	 *
 26	 *	px	  = *p ^ *dp;
 27	 *	qx	  = qmul[*q ^ *dq];
 28	 *	*dq++ = db = pbmul[px] ^ qx;
 29	 *	*dp++ = db ^ px;
 30	 *	p++; q++;
 31	 * }
 32	 */
 33	while (bytes) {
 34		/*
 35		 * v0:px, v1:dp,
 36		 * v2:qx, v3:dq,
 37		 * v4:vx, v5:vy,
 38		 * v6:qm0, v7:qm1,
 39		 * v8:pm0, v9:pm1,
 40		 * v14:p/qm[vx], v15:p/qm[vy]
 41		 */
 42		asm volatile (".option		push\n"
 43			      ".option		arch,+v\n"
 44			      "vle8.v		v0, (%[px])\n"
 45			      "vle8.v		v1, (%[dp])\n"
 46			      "vxor.vv		v0, v0, v1\n"
 47			      "vle8.v		v2, (%[qx])\n"
 48			      "vle8.v		v3, (%[dq])\n"
 49			      "vxor.vv		v4, v2, v3\n"
 50			      "vsrl.vi		v5, v4, 4\n"
 51			      "vand.vi		v4, v4, 0xf\n"
 52			      "vle8.v		v6, (%[qm0])\n"
 53			      "vle8.v		v7, (%[qm1])\n"
 54			      "vrgather.vv	v14, v6, v4\n" /* v14 = qm[vx] */
 55			      "vrgather.vv	v15, v7, v5\n" /* v15 = qm[vy] */
 56			      "vxor.vv		v2, v14, v15\n" /* v2 = qmul[*q ^ *dq] */
 57
 58			      "vsrl.vi		v5, v0, 4\n"
 59			      "vand.vi		v4, v0, 0xf\n"
 60			      "vle8.v		v8, (%[pm0])\n"
 61			      "vle8.v		v9, (%[pm1])\n"
 62			      "vrgather.vv	v14, v8, v4\n" /* v14 = pm[vx] */
 63			      "vrgather.vv	v15, v9, v5\n" /* v15 = pm[vy] */
 64			      "vxor.vv		v4, v14, v15\n" /* v4 = pbmul[px] */
 65			      "vxor.vv		v3, v4, v2\n" /* v3 = db = pbmul[px] ^ qx */
 66			      "vxor.vv		v1, v3, v0\n" /* v1 = db ^ px; */
 67			      "vse8.v		v3, (%[dq])\n"
 68			      "vse8.v		v1, (%[dp])\n"
 69			      ".option		pop\n"
 70			      : :
 71			      [px]"r"(p),
 72			      [dp]"r"(dp),
 73			      [qx]"r"(q),
 74			      [dq]"r"(dq),
 75			      [qm0]"r"(qmul),
 76			      [qm1]"r"(qmul + 16),
 77			      [pm0]"r"(pbmul),
 78			      [pm1]"r"(pbmul + 16)
 79			      :);
 80
 81		bytes -= 16;
 82		p += 16;
 83		q += 16;
 84		dp += 16;
 85		dq += 16;
 86	}
 87}
 88
 89static void __raid6_datap_recov_rvv(int bytes, u8 *p, u8 *q,
 90				    u8 *dq, const u8 *qmul)
 91{
 92	asm volatile (".option	push\n"
 93		      ".option	arch,+v\n"
 94		      "vsetvli	x0, %[avl], e8, m1, ta, ma\n"
 95		      ".option	pop\n"
 96		      : :
 97		      [avl]"r"(16)
 98	);
 99
100	/*
101	 * while (bytes--) {
102	 *  *p++ ^= *dq = qmul[*q ^ *dq];
103	 *  q++; dq++;
104	 * }
105	 */
106	while (bytes) {
107		/*
108		 * v0:vx, v1:vy,
109		 * v2:dq, v3:p,
110		 * v4:qm0, v5:qm1,
111		 * v10:m[vx], v11:m[vy]
112		 */
113		asm volatile (".option		push\n"
114			      ".option		arch,+v\n"
115			      "vle8.v		v0, (%[vx])\n"
116			      "vle8.v		v2, (%[dq])\n"
117			      "vxor.vv		v0, v0, v2\n"
118			      "vsrl.vi		v1, v0, 4\n"
119			      "vand.vi		v0, v0, 0xf\n"
120			      "vle8.v		v4, (%[qm0])\n"
121			      "vle8.v		v5, (%[qm1])\n"
122			      "vrgather.vv	v10, v4, v0\n"
123			      "vrgather.vv	v11, v5, v1\n"
124			      "vxor.vv		v0, v10, v11\n"
125			      "vle8.v		v1, (%[vy])\n"
126			      "vxor.vv		v1, v0, v1\n"
127			      "vse8.v		v0, (%[dq])\n"
128			      "vse8.v		v1, (%[vy])\n"
129			      ".option		pop\n"
130			      : :
131			      [vx]"r"(q),
132			      [vy]"r"(p),
133			      [dq]"r"(dq),
134			      [qm0]"r"(qmul),
135			      [qm1]"r"(qmul + 16)
136			      :);
137
138		bytes -= 16;
139		p += 16;
140		q += 16;
141		dq += 16;
142	}
143}
144
145static void raid6_2data_recov_rvv(int disks, size_t bytes, int faila,
146				  int failb, void **ptrs)
147{
148	u8 *p, *q, *dp, *dq;
149	const u8 *pbmul;	/* P multiplier table for B data */
150	const u8 *qmul;		/* Q multiplier table (for both) */
151
152	p = (u8 *)ptrs[disks - 2];
153	q = (u8 *)ptrs[disks - 1];
154
155	/*
156	 * Compute syndrome with zero for the missing data pages
157	 * Use the dead data pages as temporary storage for
158	 * delta p and delta q
159	 */
160	dp = (u8 *)ptrs[faila];
161	ptrs[faila] = raid6_get_zero_page();
162	ptrs[disks - 2] = dp;
163	dq = (u8 *)ptrs[failb];
164	ptrs[failb] = raid6_get_zero_page();
165	ptrs[disks - 1] = dq;
166
167	raid6_call.gen_syndrome(disks, bytes, ptrs);
168
169	/* Restore pointer table */
170	ptrs[faila]     = dp;
171	ptrs[failb]     = dq;
172	ptrs[disks - 2] = p;
173	ptrs[disks - 1] = q;
174
175	/* Now, pick the proper data tables */
176	pbmul = raid6_vgfmul[raid6_gfexi[failb - faila]];
177	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
178					 raid6_gfexp[failb]]];
179
180	kernel_vector_begin();
181	__raid6_2data_recov_rvv(bytes, p, q, dp, dq, pbmul, qmul);
182	kernel_vector_end();
183}
184
185static void raid6_datap_recov_rvv(int disks, size_t bytes, int faila,
186				  void **ptrs)
187{
188	u8 *p, *q, *dq;
189	const u8 *qmul;		/* Q multiplier table */
190
191	p = (u8 *)ptrs[disks - 2];
192	q = (u8 *)ptrs[disks - 1];
193
194	/*
195	 * Compute syndrome with zero for the missing data page
196	 * Use the dead data page as temporary storage for delta q
197	 */
198	dq = (u8 *)ptrs[faila];
199	ptrs[faila] = raid6_get_zero_page();
200	ptrs[disks - 1] = dq;
201
202	raid6_call.gen_syndrome(disks, bytes, ptrs);
203
204	/* Restore pointer table */
205	ptrs[faila]     = dq;
206	ptrs[disks - 1] = q;
207
208	/* Now, pick the proper data tables */
209	qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
210
211	kernel_vector_begin();
212	__raid6_datap_recov_rvv(bytes, p, q, dq, qmul);
213	kernel_vector_end();
214}
215
216const struct raid6_recov_calls raid6_recov_rvv = {
217	.data2		= raid6_2data_recov_rvv,
218	.datap		= raid6_datap_recov_rvv,
219	.valid		= rvv_has_vector,
220	.name		= "rvv",
221	.priority	= 1,
222};