xor.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Optimized xor_block operation for RAID4/5
  4 *
  5 * Copyright IBM Corp. 2016
  6 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
  7 */
  8
  9#include <linux/types.h>
 10#include <linux/export.h>
 11#include <linux/raid/xor.h>
 12#include <asm/xor.h>
 13
 14static void xor_xc_2(unsigned long bytes, unsigned long * __restrict p1,
 15		     const unsigned long * __restrict p2)
 16{
 17	asm volatile(
 18		"	aghi	%0,-1\n"
 19		"	jm	3f\n"
 20		"	srlg	0,%0,8\n"
 21		"	ltgr	0,0\n"
 22		"	jz	1f\n"
 23		"0:	xc	0(256,%1),0(%2)\n"
 24		"	la	%1,256(%1)\n"
 25		"	la	%2,256(%2)\n"
 26		"	brctg	0,0b\n"
 27		"1:	exrl	%0,2f\n"
 28		"	j	3f\n"
 29		"2:	xc	0(1,%1),0(%2)\n"
 30		"3:"
 31		: "+d" (bytes), "+a" (p1), "+a" (p2)
 32		: : "0", "cc", "memory");
 33}
 34
 35static void xor_xc_3(unsigned long bytes, unsigned long * __restrict p1,
 36		     const unsigned long * __restrict p2,
 37		     const unsigned long * __restrict p3)
 38{
 39	asm volatile(
 40		"	aghi	%0,-1\n"
 41		"	jm	4f\n"
 42		"	srlg	0,%0,8\n"
 43		"	ltgr	0,0\n"
 44		"	jz	1f\n"
 45		"0:	xc	0(256,%1),0(%2)\n"
 46		"	xc	0(256,%1),0(%3)\n"
 47		"	la	%1,256(%1)\n"
 48		"	la	%2,256(%2)\n"
 49		"	la	%3,256(%3)\n"
 50		"	brctg	0,0b\n"
 51		"1:	exrl	%0,2f\n"
 52		"	exrl	%0,3f\n"
 53		"	j	4f\n"
 54		"2:	xc	0(1,%1),0(%2)\n"
 55		"3:	xc	0(1,%1),0(%3)\n"
 56		"4:"
 57		: "+d" (bytes), "+a" (p1), "+a" (p2), "+a" (p3)
 58		: : "0", "cc", "memory");
 59}
 60
 61static void xor_xc_4(unsigned long bytes, unsigned long * __restrict p1,
 62		     const unsigned long * __restrict p2,
 63		     const unsigned long * __restrict p3,
 64		     const unsigned long * __restrict p4)
 65{
 66	asm volatile(
 67		"	aghi	%0,-1\n"
 68		"	jm	5f\n"
 69		"	srlg	0,%0,8\n"
 70		"	ltgr	0,0\n"
 71		"	jz	1f\n"
 72		"0:	xc	0(256,%1),0(%2)\n"
 73		"	xc	0(256,%1),0(%3)\n"
 74		"	xc	0(256,%1),0(%4)\n"
 75		"	la	%1,256(%1)\n"
 76		"	la	%2,256(%2)\n"
 77		"	la	%3,256(%3)\n"
 78		"	la	%4,256(%4)\n"
 79		"	brctg	0,0b\n"
 80		"1:	exrl	%0,2f\n"
 81		"	exrl	%0,3f\n"
 82		"	exrl	%0,4f\n"
 83		"	j	5f\n"
 84		"2:	xc	0(1,%1),0(%2)\n"
 85		"3:	xc	0(1,%1),0(%3)\n"
 86		"4:	xc	0(1,%1),0(%4)\n"
 87		"5:"
 88		: "+d" (bytes), "+a" (p1), "+a" (p2), "+a" (p3), "+a" (p4)
 89		: : "0", "cc", "memory");
 90}
 91
 92static void xor_xc_5(unsigned long bytes, unsigned long * __restrict p1,
 93		     const unsigned long * __restrict p2,
 94		     const unsigned long * __restrict p3,
 95		     const unsigned long * __restrict p4,
 96		     const unsigned long * __restrict p5)
 97{
 98	asm volatile(
 99		"	aghi	%0,-1\n"
100		"	jm	6f\n"
101		"	srlg	0,%0,8\n"
102		"	ltgr	0,0\n"
103		"	jz	1f\n"
104		"0:	xc	0(256,%1),0(%2)\n"
105		"	xc	0(256,%1),0(%3)\n"
106		"	xc	0(256,%1),0(%4)\n"
107		"	xc	0(256,%1),0(%5)\n"
108		"	la	%1,256(%1)\n"
109		"	la	%2,256(%2)\n"
110		"	la	%3,256(%3)\n"
111		"	la	%4,256(%4)\n"
112		"	la	%5,256(%5)\n"
113		"	brctg	0,0b\n"
114		"1:	exrl	%0,2f\n"
115		"	exrl	%0,3f\n"
116		"	exrl	%0,4f\n"
117		"	exrl	%0,5f\n"
118		"	j	6f\n"
119		"2:	xc	0(1,%1),0(%2)\n"
120		"3:	xc	0(1,%1),0(%3)\n"
121		"4:	xc	0(1,%1),0(%4)\n"
122		"5:	xc	0(1,%1),0(%5)\n"
123		"6:"
124		: "+d" (bytes), "+a" (p1), "+a" (p2), "+a" (p3), "+a" (p4),
125		  "+a" (p5)
126		: : "0", "cc", "memory");
127}
128
129struct xor_block_template xor_block_xc = {
130	.name = "xc",
131	.do_2 = xor_xc_2,
132	.do_3 = xor_xc_3,
133	.do_4 = xor_xc_4,
134	.do_5 = xor_xc_5,
135};
136EXPORT_SYMBOL(xor_block_xc);