sha1-ce-core.S

  1/* SPDX-License-Identifier: GPL-2.0-only */
  2/*
  3 * SHA-1 secure hash using ARMv8 Crypto Extensions
  4 *
  5 * Copyright (C) 2014 Linaro Ltd <ard.biesheuvel@linaro.org>
  6 */
  7
  8#include <linux/linkage.h>
  9#include <asm/assembler.h>
 10
 11	.text
 12	.arch		armv8-a+crypto
 13
 14	k0		.req	v0
 15	k1		.req	v1
 16	k2		.req	v2
 17	k3		.req	v3
 18
 19	t0		.req	v4
 20	t1		.req	v5
 21
 22	dga		.req	q6
 23	dgav		.req	v6
 24	dgb		.req	s7
 25	dgbv		.req	v7
 26
 27	dg0q		.req	q12
 28	dg0s		.req	s12
 29	dg0v		.req	v12
 30	dg1s		.req	s13
 31	dg1v		.req	v13
 32	dg2s		.req	s14
 33
 34	.macro		add_only, op, ev, rc, s0, dg1
 35	.ifc		\ev, ev
 36	add		t1.4s, v\s0\().4s, \rc\().4s
 37	sha1h		dg2s, dg0s
 38	.ifnb		\dg1
 39	sha1\op		dg0q, \dg1, t0.4s
 40	.else
 41	sha1\op		dg0q, dg1s, t0.4s
 42	.endif
 43	.else
 44	.ifnb		\s0
 45	add		t0.4s, v\s0\().4s, \rc\().4s
 46	.endif
 47	sha1h		dg1s, dg0s
 48	sha1\op		dg0q, dg2s, t1.4s
 49	.endif
 50	.endm
 51
 52	.macro		add_update, op, ev, rc, s0, s1, s2, s3, dg1
 53	sha1su0		v\s0\().4s, v\s1\().4s, v\s2\().4s
 54	add_only	\op, \ev, \rc, \s1, \dg1
 55	sha1su1		v\s0\().4s, v\s3\().4s
 56	.endm
 57
 58	.macro		loadrc, k, val, tmp
 59	movz		\tmp, :abs_g0_nc:\val
 60	movk		\tmp, :abs_g1:\val
 61	dup		\k, \tmp
 62	.endm
 63
 64	/*
 65	 * size_t __sha1_ce_transform(struct sha1_block_state *state,
 66	 *			      const u8 *data, size_t nblocks);
 67	 */
 68SYM_FUNC_START(__sha1_ce_transform)
 69	/* load round constants */
 70	loadrc		k0.4s, 0x5a827999, w6
 71	loadrc		k1.4s, 0x6ed9eba1, w6
 72	loadrc		k2.4s, 0x8f1bbcdc, w6
 73	loadrc		k3.4s, 0xca62c1d6, w6
 74
 75	/* load state */
 76	ld1		{dgav.4s}, [x0]
 77	ldr		dgb, [x0, #16]
 78
 79	/* load input */
 800:	ld1		{v8.4s-v11.4s}, [x1], #64
 81	sub		x2, x2, #1
 82
 83CPU_LE(	rev32		v8.16b, v8.16b		)
 84CPU_LE(	rev32		v9.16b, v9.16b		)
 85CPU_LE(	rev32		v10.16b, v10.16b	)
 86CPU_LE(	rev32		v11.16b, v11.16b	)
 87
 88	add		t0.4s, v8.4s, k0.4s
 89	mov		dg0v.16b, dgav.16b
 90
 91	add_update	c, ev, k0,  8,  9, 10, 11, dgb
 92	add_update	c, od, k0,  9, 10, 11,  8
 93	add_update	c, ev, k0, 10, 11,  8,  9
 94	add_update	c, od, k0, 11,  8,  9, 10
 95	add_update	c, ev, k1,  8,  9, 10, 11
 96
 97	add_update	p, od, k1,  9, 10, 11,  8
 98	add_update	p, ev, k1, 10, 11,  8,  9
 99	add_update	p, od, k1, 11,  8,  9, 10
100	add_update	p, ev, k1,  8,  9, 10, 11
101	add_update	p, od, k2,  9, 10, 11,  8
102
103	add_update	m, ev, k2, 10, 11,  8,  9
104	add_update	m, od, k2, 11,  8,  9, 10
105	add_update	m, ev, k2,  8,  9, 10, 11
106	add_update	m, od, k2,  9, 10, 11,  8
107	add_update	m, ev, k3, 10, 11,  8,  9
108
109	add_update	p, od, k3, 11,  8,  9, 10
110	add_only	p, ev, k3,  9
111	add_only	p, od, k3, 10
112	add_only	p, ev, k3, 11
113	add_only	p, od
114
115	/* update state */
116	add		dgbv.2s, dgbv.2s, dg1v.2s
117	add		dgav.4s, dgav.4s, dg0v.4s
118
119	/* return early if voluntary preemption is needed */
120	cond_yield	1f, x5, x6
121
122	/* handled all input blocks? */
123	cbnz		x2, 0b
124
125	/* store new state */
1261:	st1		{dgav.4s}, [x0]
127	str		dgb, [x0, #16]
128	mov		x0, x2
129	ret
130SYM_FUNC_END(__sha1_ce_transform)