aes-neon.S

  1/* SPDX-License-Identifier: GPL-2.0-only */
  2/*
  3 * linux/arch/arm64/crypto/aes-neon.S - AES cipher for ARMv8 NEON
  4 *
  5 * Copyright (C) 2013 - 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
  6 */
  7
  8#include <linux/linkage.h>
  9#include <asm/assembler.h>
 10
 11#define AES_FUNC_START(func)		SYM_FUNC_START(neon_ ## func)
 12#define AES_FUNC_END(func)		SYM_FUNC_END(neon_ ## func)
 13
 14	xtsmask		.req	v7
 15	cbciv		.req	v7
 16	vctr		.req	v4
 17
 18	.macro		xts_reload_mask, tmp
 19	xts_load_mask	\tmp
 20	.endm
 21
 22	/* special case for the neon-bs driver calling into this one for CTS */
 23	.macro		xts_cts_skip_tw, reg, lbl
 24	tbnz		\reg, #1, \lbl
 25	.endm
 26
 27	/* multiply by polynomial 'x' in GF(2^8) */
 28	.macro		mul_by_x, out, in, temp, const
 29	sshr		\temp, \in, #7
 30	shl		\out, \in, #1
 31	and		\temp, \temp, \const
 32	eor		\out, \out, \temp
 33	.endm
 34
 35	/* multiply by polynomial 'x^2' in GF(2^8) */
 36	.macro		mul_by_x2, out, in, temp, const
 37	ushr		\temp, \in, #6
 38	shl		\out, \in, #2
 39	pmul		\temp, \temp, \const
 40	eor		\out, \out, \temp
 41	.endm
 42
 43	/* preload the entire Sbox */
 44	.macro		prepare, sbox, shiftrows, temp
 45	movi		v12.16b, #0x1b
 46	ldr_l		q13, \shiftrows, \temp
 47	ldr_l		q14, .Lror32by8, \temp
 48	adr_l		\temp, \sbox
 49	ld1		{v16.16b-v19.16b}, [\temp], #64
 50	ld1		{v20.16b-v23.16b}, [\temp], #64
 51	ld1		{v24.16b-v27.16b}, [\temp], #64
 52	ld1		{v28.16b-v31.16b}, [\temp]
 53	.endm
 54
 55	/* do preload for encryption */
 56	.macro		enc_prepare, ignore0, ignore1, temp
 57	prepare		crypto_aes_sbox, .LForward_ShiftRows, \temp
 58	.endm
 59
 60	.macro		enc_switch_key, ignore0, ignore1, temp
 61	/* do nothing */
 62	.endm
 63
 64	/* do preload for decryption */
 65	.macro		dec_prepare, ignore0, ignore1, temp
 66	prepare		crypto_aes_inv_sbox, .LReverse_ShiftRows, \temp
 67	.endm
 68
 69	/* apply SubBytes transformation using the preloaded Sbox */
 70	.macro		sub_bytes, in
 71	sub		v9.16b, \in\().16b, v15.16b
 72	tbl		\in\().16b, {v16.16b-v19.16b}, \in\().16b
 73	sub		v10.16b, v9.16b, v15.16b
 74	tbx		\in\().16b, {v20.16b-v23.16b}, v9.16b
 75	sub		v11.16b, v10.16b, v15.16b
 76	tbx		\in\().16b, {v24.16b-v27.16b}, v10.16b
 77	tbx		\in\().16b, {v28.16b-v31.16b}, v11.16b
 78	.endm
 79
 80	/* apply MixColumns transformation */
 81	.macro		mix_columns, in, enc
 82	.if		\enc == 0
 83	/* Inverse MixColumns: pre-multiply by { 5, 0, 4, 0 } */
 84	mul_by_x2	v8.16b, \in\().16b, v9.16b, v12.16b
 85	eor		\in\().16b, \in\().16b, v8.16b
 86	rev32		v8.8h, v8.8h
 87	eor		\in\().16b, \in\().16b, v8.16b
 88	.endif
 89
 90	mul_by_x	v9.16b, \in\().16b, v8.16b, v12.16b
 91	rev32		v8.8h, \in\().8h
 92	eor		v8.16b, v8.16b, v9.16b
 93	eor		\in\().16b, \in\().16b, v8.16b
 94	tbl		\in\().16b, {\in\().16b}, v14.16b
 95	eor		\in\().16b, \in\().16b, v8.16b
 96	.endm
 97
 98	.macro		do_block, enc, in, rounds, rk, rkp, i
 99	ld1		{v15.4s}, [\rk]
100	add		\rkp, \rk, #16
101	mov		\i, \rounds
102.La\@:	eor		\in\().16b, \in\().16b, v15.16b		/* ^round key */
103	movi		v15.16b, #0x40
104	tbl		\in\().16b, {\in\().16b}, v13.16b	/* ShiftRows */
105	sub_bytes	\in
106	sub		\i, \i, #1
107	ld1		{v15.4s}, [\rkp], #16
108	cbz		\i, .Lb\@
109	mix_columns	\in, \enc
110	b		.La\@
111.Lb\@:	eor		\in\().16b, \in\().16b, v15.16b		/* ^round key */
112	.endm
113
114	.macro		encrypt_block, in, rounds, rk, rkp, i
115	do_block	1, \in, \rounds, \rk, \rkp, \i
116	.endm
117
118	.macro		decrypt_block, in, rounds, rk, rkp, i
119	do_block	0, \in, \rounds, \rk, \rkp, \i
120	.endm
121
122	/*
123	 * Interleaved versions: functionally equivalent to the
124	 * ones above, but applied to AES states in parallel.
125	 */
126
127	.macro		sub_bytes_4x, in0, in1, in2, in3
128	sub		v8.16b, \in0\().16b, v15.16b
129	tbl		\in0\().16b, {v16.16b-v19.16b}, \in0\().16b
130	sub		v9.16b, \in1\().16b, v15.16b
131	tbl		\in1\().16b, {v16.16b-v19.16b}, \in1\().16b
132	sub		v10.16b, \in2\().16b, v15.16b
133	tbl		\in2\().16b, {v16.16b-v19.16b}, \in2\().16b
134	sub		v11.16b, \in3\().16b, v15.16b
135	tbl		\in3\().16b, {v16.16b-v19.16b}, \in3\().16b
136	tbx		\in0\().16b, {v20.16b-v23.16b}, v8.16b
137	tbx		\in1\().16b, {v20.16b-v23.16b}, v9.16b
138	sub		v8.16b, v8.16b, v15.16b
139	tbx		\in2\().16b, {v20.16b-v23.16b}, v10.16b
140	sub		v9.16b, v9.16b, v15.16b
141	tbx		\in3\().16b, {v20.16b-v23.16b}, v11.16b
142	sub		v10.16b, v10.16b, v15.16b
143	tbx		\in0\().16b, {v24.16b-v27.16b}, v8.16b
144	sub		v11.16b, v11.16b, v15.16b
145	tbx		\in1\().16b, {v24.16b-v27.16b}, v9.16b
146	sub		v8.16b, v8.16b, v15.16b
147	tbx		\in2\().16b, {v24.16b-v27.16b}, v10.16b
148	sub		v9.16b, v9.16b, v15.16b
149	tbx		\in3\().16b, {v24.16b-v27.16b}, v11.16b
150	sub		v10.16b, v10.16b, v15.16b
151	tbx		\in0\().16b, {v28.16b-v31.16b}, v8.16b
152	sub		v11.16b, v11.16b, v15.16b
153	tbx		\in1\().16b, {v28.16b-v31.16b}, v9.16b
154	tbx		\in2\().16b, {v28.16b-v31.16b}, v10.16b
155	tbx		\in3\().16b, {v28.16b-v31.16b}, v11.16b
156	.endm
157
158	.macro		mul_by_x_2x, out0, out1, in0, in1, tmp0, tmp1, const
159	sshr		\tmp0\().16b, \in0\().16b, #7
160	shl		\out0\().16b, \in0\().16b, #1
161	sshr		\tmp1\().16b, \in1\().16b, #7
162	and		\tmp0\().16b, \tmp0\().16b, \const\().16b
163	shl		\out1\().16b, \in1\().16b, #1
164	and		\tmp1\().16b, \tmp1\().16b, \const\().16b
165	eor		\out0\().16b, \out0\().16b, \tmp0\().16b
166	eor		\out1\().16b, \out1\().16b, \tmp1\().16b
167	.endm
168
169	.macro		mul_by_x2_2x, out0, out1, in0, in1, tmp0, tmp1, const
170	ushr		\tmp0\().16b, \in0\().16b, #6
171	shl		\out0\().16b, \in0\().16b, #2
172	ushr		\tmp1\().16b, \in1\().16b, #6
173	pmul		\tmp0\().16b, \tmp0\().16b, \const\().16b
174	shl		\out1\().16b, \in1\().16b, #2
175	pmul		\tmp1\().16b, \tmp1\().16b, \const\().16b
176	eor		\out0\().16b, \out0\().16b, \tmp0\().16b
177	eor		\out1\().16b, \out1\().16b, \tmp1\().16b
178	.endm
179
180	.macro		mix_columns_2x, in0, in1, enc
181	.if		\enc == 0
182	/* Inverse MixColumns: pre-multiply by { 5, 0, 4, 0 } */
183	mul_by_x2_2x	v8, v9, \in0, \in1, v10, v11, v12
184	eor		\in0\().16b, \in0\().16b, v8.16b
185	rev32		v8.8h, v8.8h
186	eor		\in1\().16b, \in1\().16b, v9.16b
187	rev32		v9.8h, v9.8h
188	eor		\in0\().16b, \in0\().16b, v8.16b
189	eor		\in1\().16b, \in1\().16b, v9.16b
190	.endif
191
192	mul_by_x_2x	v8, v9, \in0, \in1, v10, v11, v12
193	rev32		v10.8h, \in0\().8h
194	rev32		v11.8h, \in1\().8h
195	eor		v10.16b, v10.16b, v8.16b
196	eor		v11.16b, v11.16b, v9.16b
197	eor		\in0\().16b, \in0\().16b, v10.16b
198	eor		\in1\().16b, \in1\().16b, v11.16b
199	tbl		\in0\().16b, {\in0\().16b}, v14.16b
200	tbl		\in1\().16b, {\in1\().16b}, v14.16b
201	eor		\in0\().16b, \in0\().16b, v10.16b
202	eor		\in1\().16b, \in1\().16b, v11.16b
203	.endm
204
205	.macro		do_block_4x, enc, in0, in1, in2, in3, rounds, rk, rkp, i
206	ld1		{v15.4s}, [\rk]
207	add		\rkp, \rk, #16
208	mov		\i, \rounds
209.La\@:	eor		\in0\().16b, \in0\().16b, v15.16b	/* ^round key */
210	eor		\in1\().16b, \in1\().16b, v15.16b	/* ^round key */
211	eor		\in2\().16b, \in2\().16b, v15.16b	/* ^round key */
212	eor		\in3\().16b, \in3\().16b, v15.16b	/* ^round key */
213	movi		v15.16b, #0x40
214	tbl		\in0\().16b, {\in0\().16b}, v13.16b	/* ShiftRows */
215	tbl		\in1\().16b, {\in1\().16b}, v13.16b	/* ShiftRows */
216	tbl		\in2\().16b, {\in2\().16b}, v13.16b	/* ShiftRows */
217	tbl		\in3\().16b, {\in3\().16b}, v13.16b	/* ShiftRows */
218	sub_bytes_4x	\in0, \in1, \in2, \in3
219	sub		\i, \i, #1
220	ld1		{v15.4s}, [\rkp], #16
221	cbz		\i, .Lb\@
222	mix_columns_2x	\in0, \in1, \enc
223	mix_columns_2x	\in2, \in3, \enc
224	b		.La\@
225.Lb\@:	eor		\in0\().16b, \in0\().16b, v15.16b	/* ^round key */
226	eor		\in1\().16b, \in1\().16b, v15.16b	/* ^round key */
227	eor		\in2\().16b, \in2\().16b, v15.16b	/* ^round key */
228	eor		\in3\().16b, \in3\().16b, v15.16b	/* ^round key */
229	.endm
230
231	.macro		encrypt_block4x, in0, in1, in2, in3, rounds, rk, rkp, i
232	do_block_4x	1, \in0, \in1, \in2, \in3, \rounds, \rk, \rkp, \i
233	.endm
234
235	.macro		decrypt_block4x, in0, in1, in2, in3, rounds, rk, rkp, i
236	do_block_4x	0, \in0, \in1, \in2, \in3, \rounds, \rk, \rkp, \i
237	.endm
238
239#include "aes-modes.S"
240
241	.section	".rodata", "a"
242	.align		4
243.LForward_ShiftRows:
244	.octa		0x0b06010c07020d08030e09040f0a0500
245
246.LReverse_ShiftRows:
247	.octa		0x0306090c0f0205080b0e0104070a0d00
248
249.Lror32by8:
250	.octa		0x0c0f0e0d080b0a090407060500030201