vgetrandom-chacha.S

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (C) 2022-2024 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
  4 */
  5
  6#include <linux/linkage.h>
  7#include <asm/frame.h>
  8
  9.section	.rodata, "a"
 10.align 16
 11CONSTANTS:	.octa 0x6b20657479622d323320646e61707865
 12.text
 13
 14/*
 15 * Very basic SSE2 implementation of ChaCha20. Produces a given positive number
 16 * of blocks of output with a nonce of 0, taking an input key and 8-byte
 17 * counter. Importantly does not spill to the stack. Its arguments are:
 18 *
 19 *	rdi: output bytes
 20 *	rsi: 32-byte key input
 21 *	rdx: 8-byte counter input/output
 22 *	rcx: number of 64-byte blocks to write to output
 23 */
 24SYM_FUNC_START(__arch_chacha20_blocks_nostack)
 25
 26.set	output,		%rdi
 27.set	key,		%rsi
 28.set	counter,	%rdx
 29.set	nblocks,	%rcx
 30.set	i,		%al
 31/* xmm registers are *not* callee-save. */
 32.set	temp,		%xmm0
 33.set	state0,		%xmm1
 34.set	state1,		%xmm2
 35.set	state2,		%xmm3
 36.set	state3,		%xmm4
 37.set	copy0,		%xmm5
 38.set	copy1,		%xmm6
 39.set	copy2,		%xmm7
 40.set	copy3,		%xmm8
 41.set	one,		%xmm9
 42
 43	/* copy0 = "expand 32-byte k" */
 44	movaps		CONSTANTS(%rip),copy0
 45	/* copy1,copy2 = key */
 46	movups		0x00(key),copy1
 47	movups		0x10(key),copy2
 48	/* copy3 = counter || zero nonce */
 49	movq		0x00(counter),copy3
 50	/* one = 1 || 0 */
 51	movq		$1,%rax
 52	movq		%rax,one
 53
 54.Lblock:
 55	/* state0,state1,state2,state3 = copy0,copy1,copy2,copy3 */
 56	movdqa		copy0,state0
 57	movdqa		copy1,state1
 58	movdqa		copy2,state2
 59	movdqa		copy3,state3
 60
 61	movb		$10,i
 62.Lpermute:
 63	/* state0 += state1, state3 = rotl32(state3 ^ state0, 16) */
 64	paddd		state1,state0
 65	pxor		state0,state3
 66	movdqa		state3,temp
 67	pslld		$16,temp
 68	psrld		$16,state3
 69	por		temp,state3
 70
 71	/* state2 += state3, state1 = rotl32(state1 ^ state2, 12) */
 72	paddd		state3,state2
 73	pxor		state2,state1
 74	movdqa		state1,temp
 75	pslld		$12,temp
 76	psrld		$20,state1
 77	por		temp,state1
 78
 79	/* state0 += state1, state3 = rotl32(state3 ^ state0, 8) */
 80	paddd		state1,state0
 81	pxor		state0,state3
 82	movdqa		state3,temp
 83	pslld		$8,temp
 84	psrld		$24,state3
 85	por		temp,state3
 86
 87	/* state2 += state3, state1 = rotl32(state1 ^ state2, 7) */
 88	paddd		state3,state2
 89	pxor		state2,state1
 90	movdqa		state1,temp
 91	pslld		$7,temp
 92	psrld		$25,state1
 93	por		temp,state1
 94
 95	/* state1[0,1,2,3] = state1[1,2,3,0] */
 96	pshufd		$0x39,state1,state1
 97	/* state2[0,1,2,3] = state2[2,3,0,1] */
 98	pshufd		$0x4e,state2,state2
 99	/* state3[0,1,2,3] = state3[3,0,1,2] */
100	pshufd		$0x93,state3,state3
101
102	/* state0 += state1, state3 = rotl32(state3 ^ state0, 16) */
103	paddd		state1,state0
104	pxor		state0,state3
105	movdqa		state3,temp
106	pslld		$16,temp
107	psrld		$16,state3
108	por		temp,state3
109
110	/* state2 += state3, state1 = rotl32(state1 ^ state2, 12) */
111	paddd		state3,state2
112	pxor		state2,state1
113	movdqa		state1,temp
114	pslld		$12,temp
115	psrld		$20,state1
116	por		temp,state1
117
118	/* state0 += state1, state3 = rotl32(state3 ^ state0, 8) */
119	paddd		state1,state0
120	pxor		state0,state3
121	movdqa		state3,temp
122	pslld		$8,temp
123	psrld		$24,state3
124	por		temp,state3
125
126	/* state2 += state3, state1 = rotl32(state1 ^ state2, 7) */
127	paddd		state3,state2
128	pxor		state2,state1
129	movdqa		state1,temp
130	pslld		$7,temp
131	psrld		$25,state1
132	por		temp,state1
133
134	/* state1[0,1,2,3] = state1[3,0,1,2] */
135	pshufd		$0x93,state1,state1
136	/* state2[0,1,2,3] = state2[2,3,0,1] */
137	pshufd		$0x4e,state2,state2
138	/* state3[0,1,2,3] = state3[1,2,3,0] */
139	pshufd		$0x39,state3,state3
140
141	decb		i
142	jnz		.Lpermute
143
144	/* output0 = state0 + copy0 */
145	paddd		copy0,state0
146	movups		state0,0x00(output)
147	/* output1 = state1 + copy1 */
148	paddd		copy1,state1
149	movups		state1,0x10(output)
150	/* output2 = state2 + copy2 */
151	paddd		copy2,state2
152	movups		state2,0x20(output)
153	/* output3 = state3 + copy3 */
154	paddd		copy3,state3
155	movups		state3,0x30(output)
156
157	/* ++copy3.counter */
158	paddq		one,copy3
159
160	/* output += 64, --nblocks */
161	addq		$64,output
162	decq		nblocks
163	jnz		.Lblock
164
165	/* counter = copy3.counter */
166	movq		copy3,0x00(counter)
167
168	/* Zero out the potentially sensitive regs, in case nothing uses these again. */
169	pxor		state0,state0
170	pxor		state1,state1
171	pxor		state2,state2
172	pxor		state3,state3
173	pxor		copy1,copy1
174	pxor		copy2,copy2
175	pxor		temp,temp
176
177	ret
178SYM_FUNC_END(__arch_chacha20_blocks_nostack)