asmmacro.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2/*
  3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
  4 */
  5#ifndef _ASM_ASMMACRO_H
  6#define _ASM_ASMMACRO_H
  7
  8#include <linux/sizes.h>
  9#include <asm/asm-offsets.h>
 10#include <asm/regdef.h>
 11#include <asm/fpregdef.h>
 12#include <asm/loongarch.h>
 13
 14#ifdef CONFIG_64BIT
 15#define TASK_STRUCT_OFFSET 0
 16#else
 17#define TASK_STRUCT_OFFSET 2000
 18#endif
 19
 20	.macro	cpu_save_nonscratch thread
 21	LONG_SPTR	s0, \thread, (THREAD_REG23 - TASK_STRUCT_OFFSET)
 22	LONG_SPTR	s1, \thread, (THREAD_REG24 - TASK_STRUCT_OFFSET)
 23	LONG_SPTR	s2, \thread, (THREAD_REG25 - TASK_STRUCT_OFFSET)
 24	LONG_SPTR	s3, \thread, (THREAD_REG26 - TASK_STRUCT_OFFSET)
 25	LONG_SPTR	s4, \thread, (THREAD_REG27 - TASK_STRUCT_OFFSET)
 26	LONG_SPTR	s5, \thread, (THREAD_REG28 - TASK_STRUCT_OFFSET)
 27	LONG_SPTR	s6, \thread, (THREAD_REG29 - TASK_STRUCT_OFFSET)
 28	LONG_SPTR	s7, \thread, (THREAD_REG30 - TASK_STRUCT_OFFSET)
 29	LONG_SPTR	s8, \thread, (THREAD_REG31 - TASK_STRUCT_OFFSET)
 30	LONG_SPTR	ra, \thread, (THREAD_REG01 - TASK_STRUCT_OFFSET)
 31	LONG_SPTR	sp, \thread, (THREAD_REG03 - TASK_STRUCT_OFFSET)
 32	LONG_SPTR	fp, \thread, (THREAD_REG22 - TASK_STRUCT_OFFSET)
 33	.endm
 34
 35	.macro	cpu_restore_nonscratch thread
 36	LONG_LPTR	s0, \thread, (THREAD_REG23 - TASK_STRUCT_OFFSET)
 37	LONG_LPTR	s1, \thread, (THREAD_REG24 - TASK_STRUCT_OFFSET)
 38	LONG_LPTR	s2, \thread, (THREAD_REG25 - TASK_STRUCT_OFFSET)
 39	LONG_LPTR	s3, \thread, (THREAD_REG26 - TASK_STRUCT_OFFSET)
 40	LONG_LPTR	s4, \thread, (THREAD_REG27 - TASK_STRUCT_OFFSET)
 41	LONG_LPTR	s5, \thread, (THREAD_REG28 - TASK_STRUCT_OFFSET)
 42	LONG_LPTR	s6, \thread, (THREAD_REG29 - TASK_STRUCT_OFFSET)
 43	LONG_LPTR	s7, \thread, (THREAD_REG30 - TASK_STRUCT_OFFSET)
 44	LONG_LPTR	s8, \thread, (THREAD_REG31 - TASK_STRUCT_OFFSET)
 45	LONG_LPTR	ra, \thread, (THREAD_REG01 - TASK_STRUCT_OFFSET)
 46	LONG_LPTR	sp, \thread, (THREAD_REG03 - TASK_STRUCT_OFFSET)
 47	LONG_LPTR	fp, \thread, (THREAD_REG22 - TASK_STRUCT_OFFSET)
 48	.endm
 49
 50	.macro fpu_save_csr thread tmp
 51	movfcsr2gr	\tmp, fcsr0
 52#ifdef CONFIG_32BIT
 53	st.w		\tmp, \thread, THREAD_FCSR
 54#else
 55	stptr.w		\tmp, \thread, THREAD_FCSR
 56#endif
 57#ifdef CONFIG_CPU_HAS_LBT
 58	/* TM bit is always 0 if LBT not supported */
 59	andi		\tmp, \tmp, FPU_CSR_TM
 60	beqz		\tmp, 1f
 61	/* Save FTOP */
 62	x86mftop	\tmp
 63	stptr.w		\tmp, \thread, THREAD_FTOP
 64	/* Turn off TM to ensure the order of FPR in memory independent of TM */
 65	x86clrtm
 661:
 67#endif
 68	.endm
 69
 70	.macro fpu_restore_csr thread tmp0 tmp1
 71#ifdef CONFIG_32BIT
 72	ld.w		\tmp0, \thread, THREAD_FCSR
 73#else
 74	ldptr.w		\tmp0, \thread, THREAD_FCSR
 75#endif
 76	movgr2fcsr	fcsr0, \tmp0
 77#ifdef CONFIG_CPU_HAS_LBT
 78	/* TM bit is always 0 if LBT not supported */
 79	andi		\tmp0, \tmp0, FPU_CSR_TM
 80	beqz		\tmp0, 2f
 81	/* Restore FTOP */
 82	ldptr.w		\tmp0, \thread, THREAD_FTOP
 83	andi		\tmp0, \tmp0, 0x7
 84	la.pcrel	\tmp1, 1f
 85	alsl.d		\tmp1, \tmp0, \tmp1, 3
 86	jr		\tmp1
 871:
 88	x86mttop	0
 89	b	2f
 90	x86mttop	1
 91	b	2f
 92	x86mttop	2
 93	b	2f
 94	x86mttop	3
 95	b	2f
 96	x86mttop	4
 97	b	2f
 98	x86mttop	5
 99	b	2f
100	x86mttop	6
101	b	2f
102	x86mttop	7
1032:
104#endif
105	.endm
106
107#ifdef CONFIG_32BIT
108	.macro fpu_save_cc thread tmp0 tmp1
109	movcf2gr	\tmp0, $fcc0
110	move		\tmp1, \tmp0
111	movcf2gr	\tmp0, $fcc1
112	bstrins.w	\tmp1, \tmp0, 15, 8
113	movcf2gr	\tmp0, $fcc2
114	bstrins.w	\tmp1, \tmp0, 23, 16
115	movcf2gr	\tmp0, $fcc3
116	bstrins.w	\tmp1, \tmp0, 31, 24
117	st.w		\tmp1, \thread, THREAD_FCC
118	movcf2gr	\tmp0, $fcc4
119	move		\tmp1, \tmp0
120	movcf2gr	\tmp0, $fcc5
121	bstrins.w	\tmp1, \tmp0, 15, 8
122	movcf2gr	\tmp0, $fcc6
123	bstrins.w	\tmp1, \tmp0, 23, 16
124	movcf2gr	\tmp0, $fcc7
125	bstrins.w	\tmp1, \tmp0, 31, 24
126	st.w		\tmp1, \thread, (THREAD_FCC + 4)
127	.endm
128
129	.macro fpu_restore_cc thread tmp0 tmp1
130	ld.w		\tmp0, \thread, THREAD_FCC
131	bstrpick.w	\tmp1, \tmp0, 7, 0
132	movgr2cf	$fcc0, \tmp1
133	bstrpick.w	\tmp1, \tmp0, 15, 8
134	movgr2cf	$fcc1, \tmp1
135	bstrpick.w	\tmp1, \tmp0, 23, 16
136	movgr2cf	$fcc2, \tmp1
137	bstrpick.w	\tmp1, \tmp0, 31, 24
138	movgr2cf	$fcc3, \tmp1
139	ld.w		\tmp0, \thread, (THREAD_FCC + 4)
140	bstrpick.w	\tmp1, \tmp0, 7, 0
141	movgr2cf	$fcc4, \tmp1
142	bstrpick.w	\tmp1, \tmp0, 15, 8
143	movgr2cf	$fcc5, \tmp1
144	bstrpick.w	\tmp1, \tmp0, 23, 16
145	movgr2cf	$fcc6, \tmp1
146	bstrpick.w	\tmp1, \tmp0, 31, 24
147	movgr2cf	$fcc7, \tmp1
148	.endm
149#else
150	.macro fpu_save_cc thread tmp0 tmp1
151	movcf2gr	\tmp0, $fcc0
152	move		\tmp1, \tmp0
153	movcf2gr	\tmp0, $fcc1
154	bstrins.d	\tmp1, \tmp0, 15, 8
155	movcf2gr	\tmp0, $fcc2
156	bstrins.d	\tmp1, \tmp0, 23, 16
157	movcf2gr	\tmp0, $fcc3
158	bstrins.d	\tmp1, \tmp0, 31, 24
159	movcf2gr	\tmp0, $fcc4
160	bstrins.d	\tmp1, \tmp0, 39, 32
161	movcf2gr	\tmp0, $fcc5
162	bstrins.d	\tmp1, \tmp0, 47, 40
163	movcf2gr	\tmp0, $fcc6
164	bstrins.d	\tmp1, \tmp0, 55, 48
165	movcf2gr	\tmp0, $fcc7
166	bstrins.d	\tmp1, \tmp0, 63, 56
167	stptr.d		\tmp1, \thread, THREAD_FCC
168	.endm
169
170	.macro fpu_restore_cc thread tmp0 tmp1
171	ldptr.d		\tmp0, \thread, THREAD_FCC
172	bstrpick.d	\tmp1, \tmp0, 7, 0
173	movgr2cf	$fcc0, \tmp1
174	bstrpick.d	\tmp1, \tmp0, 15, 8
175	movgr2cf	$fcc1, \tmp1
176	bstrpick.d	\tmp1, \tmp0, 23, 16
177	movgr2cf	$fcc2, \tmp1
178	bstrpick.d	\tmp1, \tmp0, 31, 24
179	movgr2cf	$fcc3, \tmp1
180	bstrpick.d	\tmp1, \tmp0, 39, 32
181	movgr2cf	$fcc4, \tmp1
182	bstrpick.d	\tmp1, \tmp0, 47, 40
183	movgr2cf	$fcc5, \tmp1
184	bstrpick.d	\tmp1, \tmp0, 55, 48
185	movgr2cf	$fcc6, \tmp1
186	bstrpick.d	\tmp1, \tmp0, 63, 56
187	movgr2cf	$fcc7, \tmp1
188	.endm
189#endif
190
191	.macro	fpu_save_double thread tmp
192	li.w	\tmp, THREAD_FPR0
193	PTR_ADD \tmp, \tmp, \thread
194	fst.d	$f0, \tmp, THREAD_FPR0  - THREAD_FPR0
195	fst.d	$f1, \tmp, THREAD_FPR1  - THREAD_FPR0
196	fst.d	$f2, \tmp, THREAD_FPR2  - THREAD_FPR0
197	fst.d	$f3, \tmp, THREAD_FPR3  - THREAD_FPR0
198	fst.d	$f4, \tmp, THREAD_FPR4  - THREAD_FPR0
199	fst.d	$f5, \tmp, THREAD_FPR5  - THREAD_FPR0
200	fst.d	$f6, \tmp, THREAD_FPR6  - THREAD_FPR0
201	fst.d	$f7, \tmp, THREAD_FPR7  - THREAD_FPR0
202	fst.d	$f8, \tmp, THREAD_FPR8  - THREAD_FPR0
203	fst.d	$f9, \tmp, THREAD_FPR9  - THREAD_FPR0
204	fst.d	$f10, \tmp, THREAD_FPR10 - THREAD_FPR0
205	fst.d	$f11, \tmp, THREAD_FPR11 - THREAD_FPR0
206	fst.d	$f12, \tmp, THREAD_FPR12 - THREAD_FPR0
207	fst.d	$f13, \tmp, THREAD_FPR13 - THREAD_FPR0
208	fst.d	$f14, \tmp, THREAD_FPR14 - THREAD_FPR0
209	fst.d	$f15, \tmp, THREAD_FPR15 - THREAD_FPR0
210	fst.d	$f16, \tmp, THREAD_FPR16 - THREAD_FPR0
211	fst.d	$f17, \tmp, THREAD_FPR17 - THREAD_FPR0
212	fst.d	$f18, \tmp, THREAD_FPR18 - THREAD_FPR0
213	fst.d	$f19, \tmp, THREAD_FPR19 - THREAD_FPR0
214	fst.d	$f20, \tmp, THREAD_FPR20 - THREAD_FPR0
215	fst.d	$f21, \tmp, THREAD_FPR21 - THREAD_FPR0
216	fst.d	$f22, \tmp, THREAD_FPR22 - THREAD_FPR0
217	fst.d	$f23, \tmp, THREAD_FPR23 - THREAD_FPR0
218	fst.d	$f24, \tmp, THREAD_FPR24 - THREAD_FPR0
219	fst.d	$f25, \tmp, THREAD_FPR25 - THREAD_FPR0
220	fst.d	$f26, \tmp, THREAD_FPR26 - THREAD_FPR0
221	fst.d	$f27, \tmp, THREAD_FPR27 - THREAD_FPR0
222	fst.d	$f28, \tmp, THREAD_FPR28 - THREAD_FPR0
223	fst.d	$f29, \tmp, THREAD_FPR29 - THREAD_FPR0
224	fst.d	$f30, \tmp, THREAD_FPR30 - THREAD_FPR0
225	fst.d	$f31, \tmp, THREAD_FPR31 - THREAD_FPR0
226	.endm
227
228	.macro	fpu_restore_double thread tmp
229	li.w	\tmp, THREAD_FPR0
230	PTR_ADD \tmp, \tmp, \thread
231	fld.d	$f0, \tmp, THREAD_FPR0  - THREAD_FPR0
232	fld.d	$f1, \tmp, THREAD_FPR1  - THREAD_FPR0
233	fld.d	$f2, \tmp, THREAD_FPR2  - THREAD_FPR0
234	fld.d	$f3, \tmp, THREAD_FPR3  - THREAD_FPR0
235	fld.d	$f4, \tmp, THREAD_FPR4  - THREAD_FPR0
236	fld.d	$f5, \tmp, THREAD_FPR5  - THREAD_FPR0
237	fld.d	$f6, \tmp, THREAD_FPR6  - THREAD_FPR0
238	fld.d	$f7, \tmp, THREAD_FPR7  - THREAD_FPR0
239	fld.d	$f8, \tmp, THREAD_FPR8  - THREAD_FPR0
240	fld.d	$f9, \tmp, THREAD_FPR9  - THREAD_FPR0
241	fld.d	$f10, \tmp, THREAD_FPR10 - THREAD_FPR0
242	fld.d	$f11, \tmp, THREAD_FPR11 - THREAD_FPR0
243	fld.d	$f12, \tmp, THREAD_FPR12 - THREAD_FPR0
244	fld.d	$f13, \tmp, THREAD_FPR13 - THREAD_FPR0
245	fld.d	$f14, \tmp, THREAD_FPR14 - THREAD_FPR0
246	fld.d	$f15, \tmp, THREAD_FPR15 - THREAD_FPR0
247	fld.d	$f16, \tmp, THREAD_FPR16 - THREAD_FPR0
248	fld.d	$f17, \tmp, THREAD_FPR17 - THREAD_FPR0
249	fld.d	$f18, \tmp, THREAD_FPR18 - THREAD_FPR0
250	fld.d	$f19, \tmp, THREAD_FPR19 - THREAD_FPR0
251	fld.d	$f20, \tmp, THREAD_FPR20 - THREAD_FPR0
252	fld.d	$f21, \tmp, THREAD_FPR21 - THREAD_FPR0
253	fld.d	$f22, \tmp, THREAD_FPR22 - THREAD_FPR0
254	fld.d	$f23, \tmp, THREAD_FPR23 - THREAD_FPR0
255	fld.d	$f24, \tmp, THREAD_FPR24 - THREAD_FPR0
256	fld.d	$f25, \tmp, THREAD_FPR25 - THREAD_FPR0
257	fld.d	$f26, \tmp, THREAD_FPR26 - THREAD_FPR0
258	fld.d	$f27, \tmp, THREAD_FPR27 - THREAD_FPR0
259	fld.d	$f28, \tmp, THREAD_FPR28 - THREAD_FPR0
260	fld.d	$f29, \tmp, THREAD_FPR29 - THREAD_FPR0
261	fld.d	$f30, \tmp, THREAD_FPR30 - THREAD_FPR0
262	fld.d	$f31, \tmp, THREAD_FPR31 - THREAD_FPR0
263	.endm
264
265	.macro	lsx_save_data thread tmp
266	li.w	\tmp, THREAD_FPR0
267	PTR_ADD \tmp, \thread, \tmp
268	vst	$vr0, \tmp, THREAD_FPR0  - THREAD_FPR0
269	vst	$vr1, \tmp, THREAD_FPR1  - THREAD_FPR0
270	vst	$vr2, \tmp, THREAD_FPR2  - THREAD_FPR0
271	vst	$vr3, \tmp, THREAD_FPR3  - THREAD_FPR0
272	vst	$vr4, \tmp, THREAD_FPR4  - THREAD_FPR0
273	vst	$vr5, \tmp, THREAD_FPR5  - THREAD_FPR0
274	vst	$vr6, \tmp, THREAD_FPR6  - THREAD_FPR0
275	vst	$vr7, \tmp, THREAD_FPR7  - THREAD_FPR0
276	vst	$vr8, \tmp, THREAD_FPR8  - THREAD_FPR0
277	vst	$vr9, \tmp, THREAD_FPR9  - THREAD_FPR0
278	vst	$vr10, \tmp, THREAD_FPR10 - THREAD_FPR0
279	vst	$vr11, \tmp, THREAD_FPR11 - THREAD_FPR0
280	vst	$vr12, \tmp, THREAD_FPR12 - THREAD_FPR0
281	vst	$vr13, \tmp, THREAD_FPR13 - THREAD_FPR0
282	vst	$vr14, \tmp, THREAD_FPR14 - THREAD_FPR0
283	vst	$vr15, \tmp, THREAD_FPR15 - THREAD_FPR0
284	vst	$vr16, \tmp, THREAD_FPR16 - THREAD_FPR0
285	vst	$vr17, \tmp, THREAD_FPR17 - THREAD_FPR0
286	vst	$vr18, \tmp, THREAD_FPR18 - THREAD_FPR0
287	vst	$vr19, \tmp, THREAD_FPR19 - THREAD_FPR0
288	vst	$vr20, \tmp, THREAD_FPR20 - THREAD_FPR0
289	vst	$vr21, \tmp, THREAD_FPR21 - THREAD_FPR0
290	vst	$vr22, \tmp, THREAD_FPR22 - THREAD_FPR0
291	vst	$vr23, \tmp, THREAD_FPR23 - THREAD_FPR0
292	vst	$vr24, \tmp, THREAD_FPR24 - THREAD_FPR0
293	vst	$vr25, \tmp, THREAD_FPR25 - THREAD_FPR0
294	vst	$vr26, \tmp, THREAD_FPR26 - THREAD_FPR0
295	vst	$vr27, \tmp, THREAD_FPR27 - THREAD_FPR0
296	vst	$vr28, \tmp, THREAD_FPR28 - THREAD_FPR0
297	vst	$vr29, \tmp, THREAD_FPR29 - THREAD_FPR0
298	vst	$vr30, \tmp, THREAD_FPR30 - THREAD_FPR0
299	vst	$vr31, \tmp, THREAD_FPR31 - THREAD_FPR0
300	.endm
301
302	.macro	lsx_restore_data thread tmp
303	li.w	\tmp, THREAD_FPR0
304	PTR_ADD	\tmp, \thread, \tmp
305	vld	$vr0, \tmp, THREAD_FPR0  - THREAD_FPR0
306	vld	$vr1, \tmp, THREAD_FPR1  - THREAD_FPR0
307	vld	$vr2, \tmp, THREAD_FPR2  - THREAD_FPR0
308	vld	$vr3, \tmp, THREAD_FPR3  - THREAD_FPR0
309	vld	$vr4, \tmp, THREAD_FPR4  - THREAD_FPR0
310	vld	$vr5, \tmp, THREAD_FPR5  - THREAD_FPR0
311	vld	$vr6, \tmp, THREAD_FPR6  - THREAD_FPR0
312	vld	$vr7, \tmp, THREAD_FPR7  - THREAD_FPR0
313	vld	$vr8, \tmp, THREAD_FPR8  - THREAD_FPR0
314	vld	$vr9, \tmp, THREAD_FPR9  - THREAD_FPR0
315	vld	$vr10, \tmp, THREAD_FPR10 - THREAD_FPR0
316	vld	$vr11, \tmp, THREAD_FPR11 - THREAD_FPR0
317	vld	$vr12, \tmp, THREAD_FPR12 - THREAD_FPR0
318	vld	$vr13, \tmp, THREAD_FPR13 - THREAD_FPR0
319	vld	$vr14, \tmp, THREAD_FPR14 - THREAD_FPR0
320	vld	$vr15, \tmp, THREAD_FPR15 - THREAD_FPR0
321	vld	$vr16, \tmp, THREAD_FPR16 - THREAD_FPR0
322	vld	$vr17, \tmp, THREAD_FPR17 - THREAD_FPR0
323	vld	$vr18, \tmp, THREAD_FPR18 - THREAD_FPR0
324	vld	$vr19, \tmp, THREAD_FPR19 - THREAD_FPR0
325	vld	$vr20, \tmp, THREAD_FPR20 - THREAD_FPR0
326	vld	$vr21, \tmp, THREAD_FPR21 - THREAD_FPR0
327	vld	$vr22, \tmp, THREAD_FPR22 - THREAD_FPR0
328	vld	$vr23, \tmp, THREAD_FPR23 - THREAD_FPR0
329	vld	$vr24, \tmp, THREAD_FPR24 - THREAD_FPR0
330	vld	$vr25, \tmp, THREAD_FPR25 - THREAD_FPR0
331	vld	$vr26, \tmp, THREAD_FPR26 - THREAD_FPR0
332	vld	$vr27, \tmp, THREAD_FPR27 - THREAD_FPR0
333	vld	$vr28, \tmp, THREAD_FPR28 - THREAD_FPR0
334	vld	$vr29, \tmp, THREAD_FPR29 - THREAD_FPR0
335	vld	$vr30, \tmp, THREAD_FPR30 - THREAD_FPR0
336	vld	$vr31, \tmp, THREAD_FPR31 - THREAD_FPR0
337	.endm
338
339	.macro	lsx_save_all	thread tmp0 tmp1
340	fpu_save_cc		\thread, \tmp0, \tmp1
341	fpu_save_csr		\thread, \tmp0
342	lsx_save_data		\thread, \tmp0
343	.endm
344
345	.macro	lsx_restore_all	thread tmp0 tmp1
346	lsx_restore_data	\thread, \tmp0
347	fpu_restore_cc		\thread, \tmp0, \tmp1
348	fpu_restore_csr		\thread, \tmp0, \tmp1
349	.endm
350
351	.macro	lsx_save_upper vd base tmp off
352	vpickve2gr.d	\tmp, \vd, 1
353	st.d		\tmp, \base, (\off+8)
354	.endm
355
356	.macro	lsx_save_all_upper thread base tmp
357	li.w		\tmp, THREAD_FPR0
358	PTR_ADD		\base, \thread, \tmp
359	lsx_save_upper	$vr0,  \base, \tmp, (THREAD_FPR0-THREAD_FPR0)
360	lsx_save_upper	$vr1,  \base, \tmp, (THREAD_FPR1-THREAD_FPR0)
361	lsx_save_upper	$vr2,  \base, \tmp, (THREAD_FPR2-THREAD_FPR0)
362	lsx_save_upper	$vr3,  \base, \tmp, (THREAD_FPR3-THREAD_FPR0)
363	lsx_save_upper	$vr4,  \base, \tmp, (THREAD_FPR4-THREAD_FPR0)
364	lsx_save_upper	$vr5,  \base, \tmp, (THREAD_FPR5-THREAD_FPR0)
365	lsx_save_upper	$vr6,  \base, \tmp, (THREAD_FPR6-THREAD_FPR0)
366	lsx_save_upper	$vr7,  \base, \tmp, (THREAD_FPR7-THREAD_FPR0)
367	lsx_save_upper	$vr8,  \base, \tmp, (THREAD_FPR8-THREAD_FPR0)
368	lsx_save_upper	$vr9,  \base, \tmp, (THREAD_FPR9-THREAD_FPR0)
369	lsx_save_upper	$vr10, \base, \tmp, (THREAD_FPR10-THREAD_FPR0)
370	lsx_save_upper	$vr11, \base, \tmp, (THREAD_FPR11-THREAD_FPR0)
371	lsx_save_upper	$vr12, \base, \tmp, (THREAD_FPR12-THREAD_FPR0)
372	lsx_save_upper	$vr13, \base, \tmp, (THREAD_FPR13-THREAD_FPR0)
373	lsx_save_upper	$vr14, \base, \tmp, (THREAD_FPR14-THREAD_FPR0)
374	lsx_save_upper	$vr15, \base, \tmp, (THREAD_FPR15-THREAD_FPR0)
375	lsx_save_upper	$vr16, \base, \tmp, (THREAD_FPR16-THREAD_FPR0)
376	lsx_save_upper	$vr17, \base, \tmp, (THREAD_FPR17-THREAD_FPR0)
377	lsx_save_upper	$vr18, \base, \tmp, (THREAD_FPR18-THREAD_FPR0)
378	lsx_save_upper	$vr19, \base, \tmp, (THREAD_FPR19-THREAD_FPR0)
379	lsx_save_upper	$vr20, \base, \tmp, (THREAD_FPR20-THREAD_FPR0)
380	lsx_save_upper	$vr21, \base, \tmp, (THREAD_FPR21-THREAD_FPR0)
381	lsx_save_upper	$vr22, \base, \tmp, (THREAD_FPR22-THREAD_FPR0)
382	lsx_save_upper	$vr23, \base, \tmp, (THREAD_FPR23-THREAD_FPR0)
383	lsx_save_upper	$vr24, \base, \tmp, (THREAD_FPR24-THREAD_FPR0)
384	lsx_save_upper	$vr25, \base, \tmp, (THREAD_FPR25-THREAD_FPR0)
385	lsx_save_upper	$vr26, \base, \tmp, (THREAD_FPR26-THREAD_FPR0)
386	lsx_save_upper	$vr27, \base, \tmp, (THREAD_FPR27-THREAD_FPR0)
387	lsx_save_upper	$vr28, \base, \tmp, (THREAD_FPR28-THREAD_FPR0)
388	lsx_save_upper	$vr29, \base, \tmp, (THREAD_FPR29-THREAD_FPR0)
389	lsx_save_upper	$vr30, \base, \tmp, (THREAD_FPR30-THREAD_FPR0)
390	lsx_save_upper	$vr31, \base, \tmp, (THREAD_FPR31-THREAD_FPR0)
391	.endm
392
393	.macro	lsx_restore_upper vd base tmp off
394	ld.d		\tmp, \base, (\off+8)
395	vinsgr2vr.d	\vd,  \tmp, 1
396	.endm
397
398	.macro	lsx_restore_all_upper thread base tmp
399	li.w		  \tmp, THREAD_FPR0
400	PTR_ADD		  \base, \thread, \tmp
401	lsx_restore_upper $vr0,  \base, \tmp, (THREAD_FPR0-THREAD_FPR0)
402	lsx_restore_upper $vr1,  \base, \tmp, (THREAD_FPR1-THREAD_FPR0)
403	lsx_restore_upper $vr2,  \base, \tmp, (THREAD_FPR2-THREAD_FPR0)
404	lsx_restore_upper $vr3,  \base, \tmp, (THREAD_FPR3-THREAD_FPR0)
405	lsx_restore_upper $vr4,  \base, \tmp, (THREAD_FPR4-THREAD_FPR0)
406	lsx_restore_upper $vr5,  \base, \tmp, (THREAD_FPR5-THREAD_FPR0)
407	lsx_restore_upper $vr6,  \base, \tmp, (THREAD_FPR6-THREAD_FPR0)
408	lsx_restore_upper $vr7,  \base, \tmp, (THREAD_FPR7-THREAD_FPR0)
409	lsx_restore_upper $vr8,  \base, \tmp, (THREAD_FPR8-THREAD_FPR0)
410	lsx_restore_upper $vr9,  \base, \tmp, (THREAD_FPR9-THREAD_FPR0)
411	lsx_restore_upper $vr10, \base, \tmp, (THREAD_FPR10-THREAD_FPR0)
412	lsx_restore_upper $vr11, \base, \tmp, (THREAD_FPR11-THREAD_FPR0)
413	lsx_restore_upper $vr12, \base, \tmp, (THREAD_FPR12-THREAD_FPR0)
414	lsx_restore_upper $vr13, \base, \tmp, (THREAD_FPR13-THREAD_FPR0)
415	lsx_restore_upper $vr14, \base, \tmp, (THREAD_FPR14-THREAD_FPR0)
416	lsx_restore_upper $vr15, \base, \tmp, (THREAD_FPR15-THREAD_FPR0)
417	lsx_restore_upper $vr16, \base, \tmp, (THREAD_FPR16-THREAD_FPR0)
418	lsx_restore_upper $vr17, \base, \tmp, (THREAD_FPR17-THREAD_FPR0)
419	lsx_restore_upper $vr18, \base, \tmp, (THREAD_FPR18-THREAD_FPR0)
420	lsx_restore_upper $vr19, \base, \tmp, (THREAD_FPR19-THREAD_FPR0)
421	lsx_restore_upper $vr20, \base, \tmp, (THREAD_FPR20-THREAD_FPR0)
422	lsx_restore_upper $vr21, \base, \tmp, (THREAD_FPR21-THREAD_FPR0)
423	lsx_restore_upper $vr22, \base, \tmp, (THREAD_FPR22-THREAD_FPR0)
424	lsx_restore_upper $vr23, \base, \tmp, (THREAD_FPR23-THREAD_FPR0)
425	lsx_restore_upper $vr24, \base, \tmp, (THREAD_FPR24-THREAD_FPR0)
426	lsx_restore_upper $vr25, \base, \tmp, (THREAD_FPR25-THREAD_FPR0)
427	lsx_restore_upper $vr26, \base, \tmp, (THREAD_FPR26-THREAD_FPR0)
428	lsx_restore_upper $vr27, \base, \tmp, (THREAD_FPR27-THREAD_FPR0)
429	lsx_restore_upper $vr28, \base, \tmp, (THREAD_FPR28-THREAD_FPR0)
430	lsx_restore_upper $vr29, \base, \tmp, (THREAD_FPR29-THREAD_FPR0)
431	lsx_restore_upper $vr30, \base, \tmp, (THREAD_FPR30-THREAD_FPR0)
432	lsx_restore_upper $vr31, \base, \tmp, (THREAD_FPR31-THREAD_FPR0)
433	.endm
434
435	.macro	lsx_init_upper vd tmp
436	vinsgr2vr.d	\vd, \tmp, 1
437	.endm
438
439	.macro	lsx_init_all_upper tmp
440	not		\tmp, zero
441	lsx_init_upper	$vr0 \tmp
442	lsx_init_upper	$vr1 \tmp
443	lsx_init_upper	$vr2 \tmp
444	lsx_init_upper	$vr3 \tmp
445	lsx_init_upper	$vr4 \tmp
446	lsx_init_upper	$vr5 \tmp
447	lsx_init_upper	$vr6 \tmp
448	lsx_init_upper	$vr7 \tmp
449	lsx_init_upper	$vr8 \tmp
450	lsx_init_upper	$vr9 \tmp
451	lsx_init_upper	$vr10 \tmp
452	lsx_init_upper	$vr11 \tmp
453	lsx_init_upper	$vr12 \tmp
454	lsx_init_upper	$vr13 \tmp
455	lsx_init_upper	$vr14 \tmp
456	lsx_init_upper	$vr15 \tmp
457	lsx_init_upper	$vr16 \tmp
458	lsx_init_upper	$vr17 \tmp
459	lsx_init_upper	$vr18 \tmp
460	lsx_init_upper	$vr19 \tmp
461	lsx_init_upper	$vr20 \tmp
462	lsx_init_upper	$vr21 \tmp
463	lsx_init_upper	$vr22 \tmp
464	lsx_init_upper	$vr23 \tmp
465	lsx_init_upper	$vr24 \tmp
466	lsx_init_upper	$vr25 \tmp
467	lsx_init_upper	$vr26 \tmp
468	lsx_init_upper	$vr27 \tmp
469	lsx_init_upper	$vr28 \tmp
470	lsx_init_upper	$vr29 \tmp
471	lsx_init_upper	$vr30 \tmp
472	lsx_init_upper	$vr31 \tmp
473	.endm
474
475	.macro	lasx_save_data thread tmp
476	li.w	\tmp, THREAD_FPR0
477	PTR_ADD	\tmp, \thread, \tmp
478	xvst	$xr0, \tmp, THREAD_FPR0  - THREAD_FPR0
479	xvst	$xr1, \tmp, THREAD_FPR1  - THREAD_FPR0
480	xvst	$xr2, \tmp, THREAD_FPR2  - THREAD_FPR0
481	xvst	$xr3, \tmp, THREAD_FPR3  - THREAD_FPR0
482	xvst	$xr4, \tmp, THREAD_FPR4  - THREAD_FPR0
483	xvst	$xr5, \tmp, THREAD_FPR5  - THREAD_FPR0
484	xvst	$xr6, \tmp, THREAD_FPR6  - THREAD_FPR0
485	xvst	$xr7, \tmp, THREAD_FPR7  - THREAD_FPR0
486	xvst	$xr8, \tmp, THREAD_FPR8  - THREAD_FPR0
487	xvst	$xr9, \tmp, THREAD_FPR9  - THREAD_FPR0
488	xvst	$xr10, \tmp, THREAD_FPR10 - THREAD_FPR0
489	xvst	$xr11, \tmp, THREAD_FPR11 - THREAD_FPR0
490	xvst	$xr12, \tmp, THREAD_FPR12 - THREAD_FPR0
491	xvst	$xr13, \tmp, THREAD_FPR13 - THREAD_FPR0
492	xvst	$xr14, \tmp, THREAD_FPR14 - THREAD_FPR0
493	xvst	$xr15, \tmp, THREAD_FPR15 - THREAD_FPR0
494	xvst	$xr16, \tmp, THREAD_FPR16 - THREAD_FPR0
495	xvst	$xr17, \tmp, THREAD_FPR17 - THREAD_FPR0
496	xvst	$xr18, \tmp, THREAD_FPR18 - THREAD_FPR0
497	xvst	$xr19, \tmp, THREAD_FPR19 - THREAD_FPR0
498	xvst	$xr20, \tmp, THREAD_FPR20 - THREAD_FPR0
499	xvst	$xr21, \tmp, THREAD_FPR21 - THREAD_FPR0
500	xvst	$xr22, \tmp, THREAD_FPR22 - THREAD_FPR0
501	xvst	$xr23, \tmp, THREAD_FPR23 - THREAD_FPR0
502	xvst	$xr24, \tmp, THREAD_FPR24 - THREAD_FPR0
503	xvst	$xr25, \tmp, THREAD_FPR25 - THREAD_FPR0
504	xvst	$xr26, \tmp, THREAD_FPR26 - THREAD_FPR0
505	xvst	$xr27, \tmp, THREAD_FPR27 - THREAD_FPR0
506	xvst	$xr28, \tmp, THREAD_FPR28 - THREAD_FPR0
507	xvst	$xr29, \tmp, THREAD_FPR29 - THREAD_FPR0
508	xvst	$xr30, \tmp, THREAD_FPR30 - THREAD_FPR0
509	xvst	$xr31, \tmp, THREAD_FPR31 - THREAD_FPR0
510	.endm
511
512	.macro	lasx_restore_data thread tmp
513	li.w	\tmp, THREAD_FPR0
514	PTR_ADD	\tmp, \thread, \tmp
515	xvld	$xr0, \tmp, THREAD_FPR0  - THREAD_FPR0
516	xvld	$xr1, \tmp, THREAD_FPR1  - THREAD_FPR0
517	xvld	$xr2, \tmp, THREAD_FPR2  - THREAD_FPR0
518	xvld	$xr3, \tmp, THREAD_FPR3  - THREAD_FPR0
519	xvld	$xr4, \tmp, THREAD_FPR4  - THREAD_FPR0
520	xvld	$xr5, \tmp, THREAD_FPR5  - THREAD_FPR0
521	xvld	$xr6, \tmp, THREAD_FPR6  - THREAD_FPR0
522	xvld	$xr7, \tmp, THREAD_FPR7  - THREAD_FPR0
523	xvld	$xr8, \tmp, THREAD_FPR8  - THREAD_FPR0
524	xvld	$xr9, \tmp, THREAD_FPR9  - THREAD_FPR0
525	xvld	$xr10, \tmp, THREAD_FPR10 - THREAD_FPR0
526	xvld	$xr11, \tmp, THREAD_FPR11 - THREAD_FPR0
527	xvld	$xr12, \tmp, THREAD_FPR12 - THREAD_FPR0
528	xvld	$xr13, \tmp, THREAD_FPR13 - THREAD_FPR0
529	xvld	$xr14, \tmp, THREAD_FPR14 - THREAD_FPR0
530	xvld	$xr15, \tmp, THREAD_FPR15 - THREAD_FPR0
531	xvld	$xr16, \tmp, THREAD_FPR16 - THREAD_FPR0
532	xvld	$xr17, \tmp, THREAD_FPR17 - THREAD_FPR0
533	xvld	$xr18, \tmp, THREAD_FPR18 - THREAD_FPR0
534	xvld	$xr19, \tmp, THREAD_FPR19 - THREAD_FPR0
535	xvld	$xr20, \tmp, THREAD_FPR20 - THREAD_FPR0
536	xvld	$xr21, \tmp, THREAD_FPR21 - THREAD_FPR0
537	xvld	$xr22, \tmp, THREAD_FPR22 - THREAD_FPR0
538	xvld	$xr23, \tmp, THREAD_FPR23 - THREAD_FPR0
539	xvld	$xr24, \tmp, THREAD_FPR24 - THREAD_FPR0
540	xvld	$xr25, \tmp, THREAD_FPR25 - THREAD_FPR0
541	xvld	$xr26, \tmp, THREAD_FPR26 - THREAD_FPR0
542	xvld	$xr27, \tmp, THREAD_FPR27 - THREAD_FPR0
543	xvld	$xr28, \tmp, THREAD_FPR28 - THREAD_FPR0
544	xvld	$xr29, \tmp, THREAD_FPR29 - THREAD_FPR0
545	xvld	$xr30, \tmp, THREAD_FPR30 - THREAD_FPR0
546	xvld	$xr31, \tmp, THREAD_FPR31 - THREAD_FPR0
547	.endm
548
549	.macro	lasx_save_all	thread tmp0 tmp1
550	fpu_save_cc		\thread, \tmp0, \tmp1
551	fpu_save_csr		\thread, \tmp0
552	lasx_save_data		\thread, \tmp0
553	.endm
554
555	.macro	lasx_restore_all thread tmp0 tmp1
556	lasx_restore_data	\thread, \tmp0
557	fpu_restore_cc		\thread, \tmp0, \tmp1
558	fpu_restore_csr		\thread, \tmp0, \tmp1
559	.endm
560
561	.macro	lasx_save_upper xd base tmp off
562	/* Nothing */
563	.endm
564
565	.macro	lasx_save_all_upper thread base tmp
566	/* Nothing */
567	.endm
568
569	.macro	lasx_restore_upper xd base tmp0 tmp1 off
570	vld		\tmp0, \base, (\off+16)
571	xvpermi.q 	\xd,   \tmp1, 0x2
572	.endm
573
574	.macro	lasx_restore_all_upper thread base tmp
575	li.w		\tmp, THREAD_FPR0
576	PTR_ADD		\base, \thread, \tmp
577	/* Save $vr31 ($xr31 lower bits) with xvpickve2gr */
578	xvpickve2gr.d	$r17, $xr31, 0
579	xvpickve2gr.d	$r18, $xr31, 1
580	lasx_restore_upper $xr0, \base, $vr31, $xr31, (THREAD_FPR0-THREAD_FPR0)
581	lasx_restore_upper $xr1, \base, $vr31, $xr31, (THREAD_FPR1-THREAD_FPR0)
582	lasx_restore_upper $xr2, \base, $vr31, $xr31, (THREAD_FPR2-THREAD_FPR0)
583	lasx_restore_upper $xr3, \base, $vr31, $xr31, (THREAD_FPR3-THREAD_FPR0)
584	lasx_restore_upper $xr4, \base, $vr31, $xr31, (THREAD_FPR4-THREAD_FPR0)
585	lasx_restore_upper $xr5, \base, $vr31, $xr31, (THREAD_FPR5-THREAD_FPR0)
586	lasx_restore_upper $xr6, \base, $vr31, $xr31, (THREAD_FPR6-THREAD_FPR0)
587	lasx_restore_upper $xr7, \base, $vr31, $xr31, (THREAD_FPR7-THREAD_FPR0)
588	lasx_restore_upper $xr8, \base, $vr31, $xr31, (THREAD_FPR8-THREAD_FPR0)
589	lasx_restore_upper $xr9, \base, $vr31, $xr31, (THREAD_FPR9-THREAD_FPR0)
590	lasx_restore_upper $xr10, \base, $vr31, $xr31, (THREAD_FPR10-THREAD_FPR0)
591	lasx_restore_upper $xr11, \base, $vr31, $xr31, (THREAD_FPR11-THREAD_FPR0)
592	lasx_restore_upper $xr12, \base, $vr31, $xr31, (THREAD_FPR12-THREAD_FPR0)
593	lasx_restore_upper $xr13, \base, $vr31, $xr31, (THREAD_FPR13-THREAD_FPR0)
594	lasx_restore_upper $xr14, \base, $vr31, $xr31, (THREAD_FPR14-THREAD_FPR0)
595	lasx_restore_upper $xr15, \base, $vr31, $xr31, (THREAD_FPR15-THREAD_FPR0)
596	lasx_restore_upper $xr16, \base, $vr31, $xr31, (THREAD_FPR16-THREAD_FPR0)
597	lasx_restore_upper $xr17, \base, $vr31, $xr31, (THREAD_FPR17-THREAD_FPR0)
598	lasx_restore_upper $xr18, \base, $vr31, $xr31, (THREAD_FPR18-THREAD_FPR0)
599	lasx_restore_upper $xr19, \base, $vr31, $xr31, (THREAD_FPR19-THREAD_FPR0)
600	lasx_restore_upper $xr20, \base, $vr31, $xr31, (THREAD_FPR20-THREAD_FPR0)
601	lasx_restore_upper $xr21, \base, $vr31, $xr31, (THREAD_FPR21-THREAD_FPR0)
602	lasx_restore_upper $xr22, \base, $vr31, $xr31, (THREAD_FPR22-THREAD_FPR0)
603	lasx_restore_upper $xr23, \base, $vr31, $xr31, (THREAD_FPR23-THREAD_FPR0)
604	lasx_restore_upper $xr24, \base, $vr31, $xr31, (THREAD_FPR24-THREAD_FPR0)
605	lasx_restore_upper $xr25, \base, $vr31, $xr31, (THREAD_FPR25-THREAD_FPR0)
606	lasx_restore_upper $xr26, \base, $vr31, $xr31, (THREAD_FPR26-THREAD_FPR0)
607	lasx_restore_upper $xr27, \base, $vr31, $xr31, (THREAD_FPR27-THREAD_FPR0)
608	lasx_restore_upper $xr28, \base, $vr31, $xr31, (THREAD_FPR28-THREAD_FPR0)
609	lasx_restore_upper $xr29, \base, $vr31, $xr31, (THREAD_FPR29-THREAD_FPR0)
610	lasx_restore_upper $xr30, \base, $vr31, $xr31, (THREAD_FPR30-THREAD_FPR0)
611	lasx_restore_upper $xr31, \base, $vr31, $xr31, (THREAD_FPR31-THREAD_FPR0)
612	/* Restore $vr31 ($xr31 lower bits) with xvinsgr2vr */
613	xvinsgr2vr.d	$xr31, $r17, 0
614	xvinsgr2vr.d	$xr31, $r18, 1
615	.endm
616
617	.macro	lasx_init_upper xd tmp
618	xvinsgr2vr.d	\xd, \tmp, 2
619	xvinsgr2vr.d	\xd, \tmp, 3
620	.endm
621
622	.macro	lasx_init_all_upper tmp
623	not		\tmp, zero
624	lasx_init_upper	$xr0 \tmp
625	lasx_init_upper	$xr1 \tmp
626	lasx_init_upper	$xr2 \tmp
627	lasx_init_upper	$xr3 \tmp
628	lasx_init_upper	$xr4 \tmp
629	lasx_init_upper	$xr5 \tmp
630	lasx_init_upper	$xr6 \tmp
631	lasx_init_upper	$xr7 \tmp
632	lasx_init_upper	$xr8 \tmp
633	lasx_init_upper	$xr9 \tmp
634	lasx_init_upper	$xr10 \tmp
635	lasx_init_upper	$xr11 \tmp
636	lasx_init_upper	$xr12 \tmp
637	lasx_init_upper	$xr13 \tmp
638	lasx_init_upper	$xr14 \tmp
639	lasx_init_upper	$xr15 \tmp
640	lasx_init_upper	$xr16 \tmp
641	lasx_init_upper	$xr17 \tmp
642	lasx_init_upper	$xr18 \tmp
643	lasx_init_upper	$xr19 \tmp
644	lasx_init_upper	$xr20 \tmp
645	lasx_init_upper	$xr21 \tmp
646	lasx_init_upper	$xr22 \tmp
647	lasx_init_upper	$xr23 \tmp
648	lasx_init_upper	$xr24 \tmp
649	lasx_init_upper	$xr25 \tmp
650	lasx_init_upper	$xr26 \tmp
651	lasx_init_upper	$xr27 \tmp
652	lasx_init_upper	$xr28 \tmp
653	lasx_init_upper	$xr29 \tmp
654	lasx_init_upper	$xr30 \tmp
655	lasx_init_upper	$xr31 \tmp
656	.endm
657
658.macro not dst src
659	nor	\dst, \src, zero
660.endm
661
662.macro la_abs reg, sym
663#ifndef CONFIG_RELOCATABLE
664	la.abs	\reg, \sym
665#else
666	766:
667	lu12i.w	\reg, 0
668	ori	\reg, \reg, 0
669#ifdef CONFIG_64BIT
670	lu32i.d	\reg, 0
671	lu52i.d	\reg, \reg, 0
672#endif
673	.pushsection ".la_abs", "aw", %progbits
674	.p2align PTRLOG
675	PTR	766b
676	PTR	\sym
677	.popsection
678#endif
679.endm
680
681#endif /* _ASM_ASMMACRO_H */