ev6-clear_user.S

  1/* SPDX-License-Identifier: GPL-2.0 */
  2/*
  3 * arch/alpha/lib/ev6-clear_user.S
  4 * 21264 version contributed by Rick Gorton <rick.gorton@alpha-processor.com>
  5 *
  6 * Zero user space, handling exceptions as we go.
  7 *
  8 * We have to make sure that $0 is always up-to-date and contains the
  9 * right "bytes left to zero" value (and that it is updated only _after_
 10 * a successful copy).  There is also some rather minor exception setup
 11 * stuff.
 12 *
 13 * Much of the information about 21264 scheduling/coding comes from:
 14 *	Compiler Writer's Guide for the Alpha 21264
 15 *	abbreviated as 'CWG' in other comments here
 16 *	ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html
 17 * Scheduling notation:
 18 *	E	- either cluster
 19 *	U	- upper subcluster; U0 - subcluster U0; U1 - subcluster U1
 20 *	L	- lower subcluster; L0 - subcluster L0; L1 - subcluster L1
 21 * Try not to change the actual algorithm if possible for consistency.
 22 * Determining actual stalls (other than slotting) doesn't appear to be easy to do.
 23 * From perusing the source code context where this routine is called, it is
 24 * a fair assumption that significant fractions of entire pages are zeroed, so
 25 * it's going to be worth the effort to hand-unroll a big loop, and use wh64.
 26 * ASSUMPTION:
 27 *	The believed purpose of only updating $0 after a store is that a signal
 28 *	may come along during the execution of this chunk of code, and we don't
 29 *	want to leave a hole (and we also want to avoid repeating lots of work)
 30 */
 31
 32#include <linux/export.h>
 33/* Allow an exception for an insn; exit if we get one.  */
 34#define EX(x,y...)			\
 35	99: x,##y;			\
 36	.section __ex_table,"a";	\
 37	.long 99b - .;			\
 38	lda $31, $exception-99b($31); 	\
 39	.previous
 40
 41	.set noat
 42	.set noreorder
 43	.align 4
 44
 45	.globl __clear_user
 46	.ent __clear_user
 47	.frame	$30, 0, $26
 48	.prologue 0
 49
 50				# Pipeline info : Slotting & Comments
 51__clear_user:
 52	and	$17, $17, $0
 53	and	$16, 7, $4	# .. E  .. ..	: find dest head misalignment
 54	beq	$0, $zerolength # U  .. .. ..	:  U L U L
 55
 56	addq	$0, $4, $1	# .. .. .. E	: bias counter
 57	and	$1, 7, $2	# .. .. E  ..	: number of misaligned bytes in tail
 58# Note - we never actually use $2, so this is a moot computation
 59# and we can rewrite this later...
 60	srl	$1, 3, $1	# .. E  .. ..	: number of quadwords to clear
 61	beq	$4, $headalign	# U  .. .. ..	: U L U L
 62
 63/*
 64 * Head is not aligned.  Write (8 - $4) bytes to head of destination
 65 * This means $16 is known to be misaligned
 66 */
 67	EX( ldq_u $5, 0($16) )	# .. .. .. L	: load dst word to mask back in
 68	beq	$1, $onebyte	# .. .. U  ..	: sub-word store?
 69	mskql	$5, $16, $5	# .. U  .. ..	: take care of misaligned head
 70	addq	$16, 8, $16	# E  .. .. .. 	: L U U L
 71
 72	EX( stq_u $5, -8($16) )	# .. .. .. L	:
 73	subq	$1, 1, $1	# .. .. E  ..	:
 74	addq	$0, $4, $0	# .. E  .. ..	: bytes left -= 8 - misalignment
 75	subq	$0, 8, $0	# E  .. .. ..	: U L U L
 76
 77	.align	4
 78/*
 79 * (The .align directive ought to be a moot point)
 80 * values upon initial entry to the loop
 81 * $1 is number of quadwords to clear (zero is a valid value)
 82 * $2 is number of trailing bytes (0..7) ($2 never used...)
 83 * $16 is known to be aligned 0mod8
 84 */
 85$headalign:
 86	subq	$1, 16, $4	# .. .. .. E	: If < 16, we can not use the huge loop
 87	and	$16, 0x3f, $2	# .. .. E  ..	: Forward work for huge loop
 88	subq	$2, 0x40, $3	# .. E  .. ..	: bias counter (huge loop)
 89	blt	$4, $trailquad	# U  .. .. ..	: U L U L
 90
 91/*
 92 * We know that we're going to do at least 16 quads, which means we are
 93 * going to be able to use the large block clear loop at least once.
 94 * Figure out how many quads we need to clear before we are 0mod64 aligned
 95 * so we can use the wh64 instruction.
 96 */
 97
 98	nop			# .. .. .. E
 99	nop			# .. .. E  ..
100	nop			# .. E  .. ..
101	beq	$3, $bigalign	# U  .. .. ..	: U L U L : Aligned 0mod64
102
103$alignmod64:
104	EX( stq_u $31, 0($16) )	# .. .. .. L
105	addq	$3, 8, $3	# .. .. E  ..
106	subq	$0, 8, $0	# .. E  .. ..
107	nop			# E  .. .. ..	: U L U L
108
109	nop			# .. .. .. E
110	subq	$1, 1, $1	# .. .. E  ..
111	addq	$16, 8, $16	# .. E  .. ..
112	blt	$3, $alignmod64	# U  .. .. ..	: U L U L
113
114$bigalign:
115/*
116 * $0 is the number of bytes left
117 * $1 is the number of quads left
118 * $16 is aligned 0mod64
119 * we know that we'll be taking a minimum of one trip through
120 * CWG Section 3.7.6: do not expect a sustained store rate of > 1/cycle
121 * We are _not_ going to update $0 after every single store.  That
122 * would be silly, because there will be cross-cluster dependencies
123 * no matter how the code is scheduled.  By doing it in slightly
124 * staggered fashion, we can still do this loop in 5 fetches
125 * The worse case will be doing two extra quads in some future execution,
126 * in the event of an interrupted clear.
127 * Assumes the wh64 needs to be for 2 trips through the loop in the future
128 * The wh64 is issued on for the starting destination address for trip +2
129 * through the loop, and if there are less than two trips left, the target
130 * address will be for the current trip.
131 */
132	nop			# E :
133	nop			# E :
134	nop			# E :
135	bis	$16,$16,$3	# E : U L U L : Initial wh64 address is dest
136	/* This might actually help for the current trip... */
137
138$do_wh64:
139	wh64	($3)		# .. .. .. L1	: memory subsystem hint
140	subq	$1, 16, $4	# .. .. E  ..	: Forward calculation - repeat the loop?
141	EX( stq_u $31, 0($16) )	# .. L  .. ..
142	subq	$0, 8, $0	# E  .. .. ..	: U L U L
143
144	addq	$16, 128, $3	# E : Target address of wh64
145	EX( stq_u $31, 8($16) )	# L :
146	EX( stq_u $31, 16($16) )	# L :
147	subq	$0, 16, $0	# E : U L L U
148
149	nop			# E :
150	EX( stq_u $31, 24($16) )	# L :
151	EX( stq_u $31, 32($16) )	# L :
152	subq	$0, 168, $5	# E : U L L U : two trips through the loop left?
153	/* 168 = 192 - 24, since we've already completed some stores */
154
155	subq	$0, 16, $0	# E :
156	EX( stq_u $31, 40($16) )	# L :
157	EX( stq_u $31, 48($16) )	# L :
158	cmovlt	$5, $16, $3	# E : U L L U : Latency 2, extra mapping cycle
159
160	subq	$1, 8, $1	# E :
161	subq	$0, 16, $0	# E :
162	EX( stq_u $31, 56($16) )	# L :
163	nop			# E : U L U L
164
165	nop			# E :
166	subq	$0, 8, $0	# E :
167	addq	$16, 64, $16	# E :
168	bge	$4, $do_wh64	# U : U L U L
169
170$trailquad:
171	# zero to 16 quadwords left to store, plus any trailing bytes
172	# $1 is the number of quadwords left to go.
173	# 
174	nop			# .. .. .. E
175	nop			# .. .. E  ..
176	nop			# .. E  .. ..
177	beq	$1, $trailbytes	# U  .. .. ..	: U L U L : Only 0..7 bytes to go
178
179$onequad:
180	EX( stq_u $31, 0($16) )	# .. .. .. L
181	subq	$1, 1, $1	# .. .. E  ..
182	subq	$0, 8, $0	# .. E  .. ..
183	nop			# E  .. .. ..	: U L U L
184
185	nop			# .. .. .. E
186	nop			# .. .. E  ..
187	addq	$16, 8, $16	# .. E  .. ..
188	bgt	$1, $onequad	# U  .. .. ..	: U L U L
189
190	# We have an unknown number of bytes left to go.
191$trailbytes:
192	nop			# .. .. .. E
193	nop			# .. .. E  ..
194	nop			# .. E  .. ..
195	beq	$0, $zerolength	# U  .. .. ..	: U L U L
196
197	# $0 contains the number of bytes left to copy (0..31)
198	# so we will use $0 as the loop counter
199	# We know for a fact that $0 > 0 zero due to previous context
200$onebyte:
201	EX( stb $31, 0($16) )	# .. .. .. L
202	subq	$0, 1, $0	# .. .. E  ..	:
203	addq	$16, 1, $16	# .. E  .. ..	:
204	bgt	$0, $onebyte	# U  .. .. ..	: U L U L
205
206$zerolength:
207$exception:			# Destination for exception recovery(?)
208	nop			# .. .. .. E	:
209	nop			# .. .. E  ..	:
210	nop			# .. E  .. ..	:
211	ret	$31, ($26), 1	# L0 .. .. ..	: L U L U
212	.end __clear_user
213	EXPORT_SYMBOL(__clear_user)