poly1305-armv4.pl

   1#!/usr/bin/env perl
   2# SPDX-License-Identifier: GPL-1.0+ OR BSD-3-Clause
   3#
   4# ====================================================================
   5# Written by Andy Polyakov, @dot-asm, initially for the OpenSSL
   6# project.
   7# ====================================================================
   8#
   9#			IALU(*)/gcc-4.4		NEON
  10#
  11# ARM11xx(ARMv6)	7.78/+100%		-
  12# Cortex-A5		6.35/+130%		3.00
  13# Cortex-A8		6.25/+115%		2.36
  14# Cortex-A9		5.10/+95%		2.55
  15# Cortex-A15		3.85/+85%		1.25(**)
  16# Snapdragon S4		5.70/+100%		1.48(**)
  17#
  18# (*)	this is for -march=armv6, i.e. with bunch of ldrb loading data;
  19# (**)	these are trade-off results, they can be improved by ~8% but at
  20#	the cost of 15/12% regression on Cortex-A5/A7, it's even possible
  21#	to improve Cortex-A9 result, but then A5/A7 loose more than 20%;
  22
  23$flavour = shift;
  24if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
  25else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} }
  26
  27if ($flavour && $flavour ne "void") {
  28    $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  29    ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
  30    ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
  31    die "can't locate arm-xlate.pl";
  32
  33    open STDOUT,"| \"$^X\" $xlate $flavour $output";
  34} else {
  35    open STDOUT,">$output";
  36}
  37
  38($ctx,$inp,$len,$padbit)=map("r$_",(0..3));
  39
  40$code.=<<___;
  41#ifndef	__KERNEL__
  42# include "arm_arch.h"
  43#else
  44# define __ARM_ARCH__ __LINUX_ARM_ARCH__
  45# define __ARM_MAX_ARCH__ __LINUX_ARM_ARCH__
  46# define poly1305_init   poly1305_block_init
  47# define poly1305_blocks poly1305_blocks_arm
  48#endif
  49
  50#if defined(__thumb2__)
  51.syntax	unified
  52.thumb
  53#else
  54.code	32
  55#endif
  56
  57.text
  58
  59.globl	poly1305_emit
  60.globl	poly1305_blocks
  61.globl	poly1305_init
  62.type	poly1305_init,%function
  63.align	5
  64poly1305_init:
  65.Lpoly1305_init:
  66	stmdb	sp!,{r4-r11}
  67
  68	eor	r3,r3,r3
  69	cmp	$inp,#0
  70	str	r3,[$ctx,#0]		@ zero hash value
  71	str	r3,[$ctx,#4]
  72	str	r3,[$ctx,#8]
  73	str	r3,[$ctx,#12]
  74	str	r3,[$ctx,#16]
  75	str	r3,[$ctx,#36]		@ clear is_base2_26
  76	add	$ctx,$ctx,#20
  77
  78#ifdef	__thumb2__
  79	it	eq
  80#endif
  81	moveq	r0,#0
  82	beq	.Lno_key
  83
  84#if	__ARM_MAX_ARCH__>=7
  85	mov	r3,#-1
  86	str	r3,[$ctx,#28]		@ impossible key power value
  87# ifndef __KERNEL__
  88	adr	r11,.Lpoly1305_init
  89	ldr	r12,.LOPENSSL_armcap
  90# endif
  91#endif
  92	ldrb	r4,[$inp,#0]
  93	mov	r10,#0x0fffffff
  94	ldrb	r5,[$inp,#1]
  95	and	r3,r10,#-4		@ 0x0ffffffc
  96	ldrb	r6,[$inp,#2]
  97	ldrb	r7,[$inp,#3]
  98	orr	r4,r4,r5,lsl#8
  99	ldrb	r5,[$inp,#4]
 100	orr	r4,r4,r6,lsl#16
 101	ldrb	r6,[$inp,#5]
 102	orr	r4,r4,r7,lsl#24
 103	ldrb	r7,[$inp,#6]
 104	and	r4,r4,r10
 105
 106#if	__ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
 107# if !defined(_WIN32)
 108	ldr	r12,[r11,r12]		@ OPENSSL_armcap_P
 109# endif
 110# if defined(__APPLE__) || defined(_WIN32)
 111	ldr	r12,[r12]
 112# endif
 113#endif
 114	ldrb	r8,[$inp,#7]
 115	orr	r5,r5,r6,lsl#8
 116	ldrb	r6,[$inp,#8]
 117	orr	r5,r5,r7,lsl#16
 118	ldrb	r7,[$inp,#9]
 119	orr	r5,r5,r8,lsl#24
 120	ldrb	r8,[$inp,#10]
 121	and	r5,r5,r3
 122
 123#if	__ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
 124	tst	r12,#ARMV7_NEON		@ check for NEON
 125# ifdef	__thumb2__
 126	adr	r9,.Lpoly1305_blocks_neon
 127	adr	r11,.Lpoly1305_blocks
 128	it	ne
 129	movne	r11,r9
 130	adr	r12,.Lpoly1305_emit
 131	orr	r11,r11,#1		@ thumb-ify addresses
 132	orr	r12,r12,#1
 133# else
 134	add	r12,r11,#(.Lpoly1305_emit-.Lpoly1305_init)
 135	ite	eq
 136	addeq	r11,r11,#(.Lpoly1305_blocks-.Lpoly1305_init)
 137	addne	r11,r11,#(.Lpoly1305_blocks_neon-.Lpoly1305_init)
 138# endif
 139#endif
 140	ldrb	r9,[$inp,#11]
 141	orr	r6,r6,r7,lsl#8
 142	ldrb	r7,[$inp,#12]
 143	orr	r6,r6,r8,lsl#16
 144	ldrb	r8,[$inp,#13]
 145	orr	r6,r6,r9,lsl#24
 146	ldrb	r9,[$inp,#14]
 147	and	r6,r6,r3
 148
 149	ldrb	r10,[$inp,#15]
 150	orr	r7,r7,r8,lsl#8
 151	str	r4,[$ctx,#0]
 152	orr	r7,r7,r9,lsl#16
 153	str	r5,[$ctx,#4]
 154	orr	r7,r7,r10,lsl#24
 155	str	r6,[$ctx,#8]
 156	and	r7,r7,r3
 157	str	r7,[$ctx,#12]
 158#if	__ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
 159	stmia	r2,{r11,r12}		@ fill functions table
 160	mov	r0,#1
 161#else
 162	mov	r0,#0
 163#endif
 164.Lno_key:
 165	ldmia	sp!,{r4-r11}
 166#if	__ARM_ARCH__>=5
 167	ret				@ bx	lr
 168#else
 169	tst	lr,#1
 170	moveq	pc,lr			@ be binary compatible with V4, yet
 171	bx	lr			@ interoperable with Thumb ISA:-)
 172#endif
 173.size	poly1305_init,.-poly1305_init
 174___
 175{
 176my ($h0,$h1,$h2,$h3,$h4,$r0,$r1,$r2,$r3)=map("r$_",(4..12));
 177my ($s1,$s2,$s3)=($r1,$r2,$r3);
 178
 179$code.=<<___;
 180.type	poly1305_blocks,%function
 181.align	5
 182poly1305_blocks:
 183.Lpoly1305_blocks:
 184	stmdb	sp!,{r3-r11,lr}
 185
 186	ands	$len,$len,#-16
 187	beq	.Lno_data
 188
 189	add	$len,$len,$inp		@ end pointer
 190	sub	sp,sp,#32
 191
 192#if __ARM_ARCH__<7
 193	ldmia	$ctx,{$h0-$r3}		@ load context
 194	add	$ctx,$ctx,#20
 195	str	$len,[sp,#16]		@ offload stuff
 196	str	$ctx,[sp,#12]
 197#else
 198	ldr	lr,[$ctx,#36]		@ is_base2_26
 199	ldmia	$ctx!,{$h0-$h4}		@ load hash value
 200	str	$len,[sp,#16]		@ offload stuff
 201	str	$ctx,[sp,#12]
 202
 203	adds	$r0,$h0,$h1,lsl#26	@ base 2^26 -> base 2^32
 204	mov	$r1,$h1,lsr#6
 205	adcs	$r1,$r1,$h2,lsl#20
 206	mov	$r2,$h2,lsr#12
 207	adcs	$r2,$r2,$h3,lsl#14
 208	mov	$r3,$h3,lsr#18
 209	adcs	$r3,$r3,$h4,lsl#8
 210	mov	$len,#0
 211	teq	lr,#0
 212	str	$len,[$ctx,#16]		@ clear is_base2_26
 213	adc	$len,$len,$h4,lsr#24
 214
 215	itttt	ne
 216	movne	$h0,$r0			@ choose between radixes
 217	movne	$h1,$r1
 218	movne	$h2,$r2
 219	movne	$h3,$r3
 220	ldmia	$ctx,{$r0-$r3}		@ load key
 221	it	ne
 222	movne	$h4,$len
 223#endif
 224
 225	mov	lr,$inp
 226	cmp	$padbit,#0
 227	str	$r1,[sp,#20]
 228	str	$r2,[sp,#24]
 229	str	$r3,[sp,#28]
 230	b	.Loop
 231
 232.align	4
 233.Loop:
 234#if __ARM_ARCH__<7
 235	ldrb	r0,[lr],#16		@ load input
 236# ifdef	__thumb2__
 237	it	hi
 238# endif
 239	addhi	$h4,$h4,#1		@ 1<<128
 240	ldrb	r1,[lr,#-15]
 241	ldrb	r2,[lr,#-14]
 242	ldrb	r3,[lr,#-13]
 243	orr	r1,r0,r1,lsl#8
 244	ldrb	r0,[lr,#-12]
 245	orr	r2,r1,r2,lsl#16
 246	ldrb	r1,[lr,#-11]
 247	orr	r3,r2,r3,lsl#24
 248	ldrb	r2,[lr,#-10]
 249	adds	$h0,$h0,r3		@ accumulate input
 250
 251	ldrb	r3,[lr,#-9]
 252	orr	r1,r0,r1,lsl#8
 253	ldrb	r0,[lr,#-8]
 254	orr	r2,r1,r2,lsl#16
 255	ldrb	r1,[lr,#-7]
 256	orr	r3,r2,r3,lsl#24
 257	ldrb	r2,[lr,#-6]
 258	adcs	$h1,$h1,r3
 259
 260	ldrb	r3,[lr,#-5]
 261	orr	r1,r0,r1,lsl#8
 262	ldrb	r0,[lr,#-4]
 263	orr	r2,r1,r2,lsl#16
 264	ldrb	r1,[lr,#-3]
 265	orr	r3,r2,r3,lsl#24
 266	ldrb	r2,[lr,#-2]
 267	adcs	$h2,$h2,r3
 268
 269	ldrb	r3,[lr,#-1]
 270	orr	r1,r0,r1,lsl#8
 271	str	lr,[sp,#8]		@ offload input pointer
 272	orr	r2,r1,r2,lsl#16
 273	add	$s1,$r1,$r1,lsr#2
 274	orr	r3,r2,r3,lsl#24
 275#else
 276	ldr	r0,[lr],#16		@ load input
 277	it	hi
 278	addhi	$h4,$h4,#1		@ padbit
 279	ldr	r1,[lr,#-12]
 280	ldr	r2,[lr,#-8]
 281	ldr	r3,[lr,#-4]
 282# ifdef	__ARMEB__
 283	rev	r0,r0
 284	rev	r1,r1
 285	rev	r2,r2
 286	rev	r3,r3
 287# endif
 288	adds	$h0,$h0,r0		@ accumulate input
 289	str	lr,[sp,#8]		@ offload input pointer
 290	adcs	$h1,$h1,r1
 291	add	$s1,$r1,$r1,lsr#2
 292	adcs	$h2,$h2,r2
 293#endif
 294	add	$s2,$r2,$r2,lsr#2
 295	adcs	$h3,$h3,r3
 296	add	$s3,$r3,$r3,lsr#2
 297
 298	umull	r2,r3,$h1,$r0
 299	 adc	$h4,$h4,#0
 300	umull	r0,r1,$h0,$r0
 301	umlal	r2,r3,$h4,$s1
 302	umlal	r0,r1,$h3,$s1
 303	ldr	$r1,[sp,#20]		@ reload $r1
 304	umlal	r2,r3,$h2,$s3
 305	umlal	r0,r1,$h1,$s3
 306	umlal	r2,r3,$h3,$s2
 307	umlal	r0,r1,$h2,$s2
 308	umlal	r2,r3,$h0,$r1
 309	str	r0,[sp,#0]		@ future $h0
 310	 mul	r0,$s2,$h4
 311	ldr	$r2,[sp,#24]		@ reload $r2
 312	adds	r2,r2,r1		@ d1+=d0>>32
 313	 eor	r1,r1,r1
 314	adc	lr,r3,#0		@ future $h2
 315	str	r2,[sp,#4]		@ future $h1
 316
 317	mul	r2,$s3,$h4
 318	eor	r3,r3,r3
 319	umlal	r0,r1,$h3,$s3
 320	ldr	$r3,[sp,#28]		@ reload $r3
 321	umlal	r2,r3,$h3,$r0
 322	umlal	r0,r1,$h2,$r0
 323	umlal	r2,r3,$h2,$r1
 324	umlal	r0,r1,$h1,$r1
 325	umlal	r2,r3,$h1,$r2
 326	umlal	r0,r1,$h0,$r2
 327	umlal	r2,r3,$h0,$r3
 328	ldr	$h0,[sp,#0]
 329	mul	$h4,$r0,$h4
 330	ldr	$h1,[sp,#4]
 331
 332	adds	$h2,lr,r0		@ d2+=d1>>32
 333	ldr	lr,[sp,#8]		@ reload input pointer
 334	adc	r1,r1,#0
 335	adds	$h3,r2,r1		@ d3+=d2>>32
 336	ldr	r0,[sp,#16]		@ reload end pointer
 337	adc	r3,r3,#0
 338	add	$h4,$h4,r3		@ h4+=d3>>32
 339
 340	and	r1,$h4,#-4
 341	and	$h4,$h4,#3
 342	add	r1,r1,r1,lsr#2		@ *=5
 343	adds	$h0,$h0,r1
 344	adcs	$h1,$h1,#0
 345	adcs	$h2,$h2,#0
 346	adcs	$h3,$h3,#0
 347	adc	$h4,$h4,#0
 348
 349	cmp	r0,lr			@ done yet?
 350	bhi	.Loop
 351
 352	ldr	$ctx,[sp,#12]
 353	add	sp,sp,#32
 354	stmdb	$ctx,{$h0-$h4}		@ store the result
 355
 356.Lno_data:
 357#if	__ARM_ARCH__>=5
 358	ldmia	sp!,{r3-r11,pc}
 359#else
 360	ldmia	sp!,{r3-r11,lr}
 361	tst	lr,#1
 362	moveq	pc,lr			@ be binary compatible with V4, yet
 363	bx	lr			@ interoperable with Thumb ISA:-)
 364#endif
 365.size	poly1305_blocks,.-poly1305_blocks
 366___
 367}
 368{
 369my ($ctx,$mac,$nonce)=map("r$_",(0..2));
 370my ($h0,$h1,$h2,$h3,$h4,$g0,$g1,$g2,$g3)=map("r$_",(3..11));
 371my $g4=$ctx;
 372
 373$code.=<<___;
 374.type	poly1305_emit,%function
 375.align	5
 376poly1305_emit:
 377.Lpoly1305_emit:
 378	stmdb	sp!,{r4-r11}
 379
 380	ldmia	$ctx,{$h0-$h4}
 381
 382#if __ARM_ARCH__>=7
 383	ldr	ip,[$ctx,#36]		@ is_base2_26
 384
 385	adds	$g0,$h0,$h1,lsl#26	@ base 2^26 -> base 2^32
 386	mov	$g1,$h1,lsr#6
 387	adcs	$g1,$g1,$h2,lsl#20
 388	mov	$g2,$h2,lsr#12
 389	adcs	$g2,$g2,$h3,lsl#14
 390	mov	$g3,$h3,lsr#18
 391	adcs	$g3,$g3,$h4,lsl#8
 392	mov	$g4,#0
 393	adc	$g4,$g4,$h4,lsr#24
 394
 395	tst	ip,ip
 396	itttt	ne
 397	movne	$h0,$g0
 398	movne	$h1,$g1
 399	movne	$h2,$g2
 400	movne	$h3,$g3
 401	it	ne
 402	movne	$h4,$g4
 403#endif
 404
 405	adds	$g0,$h0,#5		@ compare to modulus
 406	adcs	$g1,$h1,#0
 407	adcs	$g2,$h2,#0
 408	adcs	$g3,$h3,#0
 409	adc	$g4,$h4,#0
 410	tst	$g4,#4			@ did it carry/borrow?
 411
 412#ifdef	__thumb2__
 413	it	ne
 414#endif
 415	movne	$h0,$g0
 416	ldr	$g0,[$nonce,#0]
 417#ifdef	__thumb2__
 418	it	ne
 419#endif
 420	movne	$h1,$g1
 421	ldr	$g1,[$nonce,#4]
 422#ifdef	__thumb2__
 423	it	ne
 424#endif
 425	movne	$h2,$g2
 426	ldr	$g2,[$nonce,#8]
 427#ifdef	__thumb2__
 428	it	ne
 429#endif
 430	movne	$h3,$g3
 431	ldr	$g3,[$nonce,#12]
 432
 433	adds	$h0,$h0,$g0
 434	adcs	$h1,$h1,$g1
 435	adcs	$h2,$h2,$g2
 436	adc	$h3,$h3,$g3
 437
 438#if __ARM_ARCH__>=7
 439# ifdef __ARMEB__
 440	rev	$h0,$h0
 441	rev	$h1,$h1
 442	rev	$h2,$h2
 443	rev	$h3,$h3
 444# endif
 445	str	$h0,[$mac,#0]
 446	str	$h1,[$mac,#4]
 447	str	$h2,[$mac,#8]
 448	str	$h3,[$mac,#12]
 449#else
 450	strb	$h0,[$mac,#0]
 451	mov	$h0,$h0,lsr#8
 452	strb	$h1,[$mac,#4]
 453	mov	$h1,$h1,lsr#8
 454	strb	$h2,[$mac,#8]
 455	mov	$h2,$h2,lsr#8
 456	strb	$h3,[$mac,#12]
 457	mov	$h3,$h3,lsr#8
 458
 459	strb	$h0,[$mac,#1]
 460	mov	$h0,$h0,lsr#8
 461	strb	$h1,[$mac,#5]
 462	mov	$h1,$h1,lsr#8
 463	strb	$h2,[$mac,#9]
 464	mov	$h2,$h2,lsr#8
 465	strb	$h3,[$mac,#13]
 466	mov	$h3,$h3,lsr#8
 467
 468	strb	$h0,[$mac,#2]
 469	mov	$h0,$h0,lsr#8
 470	strb	$h1,[$mac,#6]
 471	mov	$h1,$h1,lsr#8
 472	strb	$h2,[$mac,#10]
 473	mov	$h2,$h2,lsr#8
 474	strb	$h3,[$mac,#14]
 475	mov	$h3,$h3,lsr#8
 476
 477	strb	$h0,[$mac,#3]
 478	strb	$h1,[$mac,#7]
 479	strb	$h2,[$mac,#11]
 480	strb	$h3,[$mac,#15]
 481#endif
 482	ldmia	sp!,{r4-r11}
 483#if	__ARM_ARCH__>=5
 484	ret				@ bx	lr
 485#else
 486	tst	lr,#1
 487	moveq	pc,lr			@ be binary compatible with V4, yet
 488	bx	lr			@ interoperable with Thumb ISA:-)
 489#endif
 490.size	poly1305_emit,.-poly1305_emit
 491___
 492{
 493my ($R0,$R1,$S1,$R2,$S2,$R3,$S3,$R4,$S4) = map("d$_",(0..9));
 494my ($D0,$D1,$D2,$D3,$D4, $H0,$H1,$H2,$H3,$H4) = map("q$_",(5..14));
 495my ($T0,$T1,$MASK) = map("q$_",(15,4,0));
 496
 497my ($in2,$zeros,$tbl0,$tbl1) = map("r$_",(4..7));
 498
 499$code.=<<___;
 500#if	__ARM_MAX_ARCH__>=7
 501.fpu	neon
 502
 503.type	poly1305_init_neon,%function
 504.align	5
 505poly1305_init_neon:
 506.Lpoly1305_init_neon:
 507	ldr	r3,[$ctx,#48]		@ first table element
 508	cmp	r3,#-1			@ is value impossible?
 509	bne	.Lno_init_neon
 510
 511	ldr	r4,[$ctx,#20]		@ load key base 2^32
 512	ldr	r5,[$ctx,#24]
 513	ldr	r6,[$ctx,#28]
 514	ldr	r7,[$ctx,#32]
 515
 516	and	r2,r4,#0x03ffffff	@ base 2^32 -> base 2^26
 517	mov	r3,r4,lsr#26
 518	mov	r4,r5,lsr#20
 519	orr	r3,r3,r5,lsl#6
 520	mov	r5,r6,lsr#14
 521	orr	r4,r4,r6,lsl#12
 522	mov	r6,r7,lsr#8
 523	orr	r5,r5,r7,lsl#18
 524	and	r3,r3,#0x03ffffff
 525	and	r4,r4,#0x03ffffff
 526	and	r5,r5,#0x03ffffff
 527
 528	vdup.32	$R0,r2			@ r^1 in both lanes
 529	add	r2,r3,r3,lsl#2		@ *5
 530	vdup.32	$R1,r3
 531	add	r3,r4,r4,lsl#2
 532	vdup.32	$S1,r2
 533	vdup.32	$R2,r4
 534	add	r4,r5,r5,lsl#2
 535	vdup.32	$S2,r3
 536	vdup.32	$R3,r5
 537	add	r5,r6,r6,lsl#2
 538	vdup.32	$S3,r4
 539	vdup.32	$R4,r6
 540	vdup.32	$S4,r5
 541
 542	mov	$zeros,#2		@ counter
 543
 544.Lsquare_neon:
 545	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
 546	@ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
 547	@ d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
 548	@ d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
 549	@ d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
 550	@ d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
 551
 552	vmull.u32	$D0,$R0,${R0}[1]
 553	vmull.u32	$D1,$R1,${R0}[1]
 554	vmull.u32	$D2,$R2,${R0}[1]
 555	vmull.u32	$D3,$R3,${R0}[1]
 556	vmull.u32	$D4,$R4,${R0}[1]
 557
 558	vmlal.u32	$D0,$R4,${S1}[1]
 559	vmlal.u32	$D1,$R0,${R1}[1]
 560	vmlal.u32	$D2,$R1,${R1}[1]
 561	vmlal.u32	$D3,$R2,${R1}[1]
 562	vmlal.u32	$D4,$R3,${R1}[1]
 563
 564	vmlal.u32	$D0,$R3,${S2}[1]
 565	vmlal.u32	$D1,$R4,${S2}[1]
 566	vmlal.u32	$D3,$R1,${R2}[1]
 567	vmlal.u32	$D2,$R0,${R2}[1]
 568	vmlal.u32	$D4,$R2,${R2}[1]
 569
 570	vmlal.u32	$D0,$R2,${S3}[1]
 571	vmlal.u32	$D3,$R0,${R3}[1]
 572	vmlal.u32	$D1,$R3,${S3}[1]
 573	vmlal.u32	$D2,$R4,${S3}[1]
 574	vmlal.u32	$D4,$R1,${R3}[1]
 575
 576	vmlal.u32	$D3,$R4,${S4}[1]
 577	vmlal.u32	$D0,$R1,${S4}[1]
 578	vmlal.u32	$D1,$R2,${S4}[1]
 579	vmlal.u32	$D2,$R3,${S4}[1]
 580	vmlal.u32	$D4,$R0,${R4}[1]
 581
 582	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
 583	@ lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
 584	@ and P. Schwabe
 585	@
 586	@ H0>>+H1>>+H2>>+H3>>+H4
 587	@ H3>>+H4>>*5+H0>>+H1
 588	@
 589	@ Trivia.
 590	@
 591	@ Result of multiplication of n-bit number by m-bit number is
 592	@ n+m bits wide. However! Even though 2^n is a n+1-bit number,
 593	@ m-bit number multiplied by 2^n is still n+m bits wide.
 594	@
 595	@ Sum of two n-bit numbers is n+1 bits wide, sum of three - n+2,
 596	@ and so is sum of four. Sum of 2^m n-m-bit numbers and n-bit
 597	@ one is n+1 bits wide.
 598	@
 599	@ >>+ denotes Hnext += Hn>>26, Hn &= 0x3ffffff. This means that
 600	@ H0, H2, H3 are guaranteed to be 26 bits wide, while H1 and H4
 601	@ can be 27. However! In cases when their width exceeds 26 bits
 602	@ they are limited by 2^26+2^6. This in turn means that *sum*
 603	@ of the products with these values can still be viewed as sum
 604	@ of 52-bit numbers as long as the amount of addends is not a
 605	@ power of 2. For example,
 606	@
 607	@ H4 = H4*R0 + H3*R1 + H2*R2 + H1*R3 + H0 * R4,
 608	@
 609	@ which can't be larger than 5 * (2^26 + 2^6) * (2^26 + 2^6), or
 610	@ 5 * (2^52 + 2*2^32 + 2^12), which in turn is smaller than
 611	@ 8 * (2^52) or 2^55. However, the value is then multiplied by
 612	@ by 5, so we should be looking at 5 * 5 * (2^52 + 2^33 + 2^12),
 613	@ which is less than 32 * (2^52) or 2^57. And when processing
 614	@ data we are looking at triple as many addends...
 615	@
 616	@ In key setup procedure pre-reduced H0 is limited by 5*4+1 and
 617	@ 5*H4 - by 5*5 52-bit addends, or 57 bits. But when hashing the
 618	@ input H0 is limited by (5*4+1)*3 addends, or 58 bits, while
 619	@ 5*H4 by 5*5*3, or 59[!] bits. How is this relevant? vmlal.u32
 620	@ instruction accepts 2x32-bit input and writes 2x64-bit result.
 621	@ This means that result of reduction have to be compressed upon
 622	@ loop wrap-around. This can be done in the process of reduction
 623	@ to minimize amount of instructions [as well as amount of
 624	@ 128-bit instructions, which benefits low-end processors], but
 625	@ one has to watch for H2 (which is narrower than H0) and 5*H4
 626	@ not being wider than 58 bits, so that result of right shift
 627	@ by 26 bits fits in 32 bits. This is also useful on x86,
 628	@ because it allows to use paddd in place for paddq, which
 629	@ benefits Atom, where paddq is ridiculously slow.
 630
 631	vshr.u64	$T0,$D3,#26
 632	vmovn.i64	$D3#lo,$D3
 633	 vshr.u64	$T1,$D0,#26
 634	 vmovn.i64	$D0#lo,$D0
 635	vadd.i64	$D4,$D4,$T0		@ h3 -> h4
 636	vbic.i32	$D3#lo,#0xfc000000	@ &=0x03ffffff
 637	 vadd.i64	$D1,$D1,$T1		@ h0 -> h1
 638	 vbic.i32	$D0#lo,#0xfc000000
 639
 640	vshrn.u64	$T0#lo,$D4,#26
 641	vmovn.i64	$D4#lo,$D4
 642	 vshr.u64	$T1,$D1,#26
 643	 vmovn.i64	$D1#lo,$D1
 644	 vadd.i64	$D2,$D2,$T1		@ h1 -> h2
 645	vbic.i32	$D4#lo,#0xfc000000
 646	 vbic.i32	$D1#lo,#0xfc000000
 647
 648	vadd.i32	$D0#lo,$D0#lo,$T0#lo
 649	vshl.u32	$T0#lo,$T0#lo,#2
 650	 vshrn.u64	$T1#lo,$D2,#26
 651	 vmovn.i64	$D2#lo,$D2
 652	vadd.i32	$D0#lo,$D0#lo,$T0#lo	@ h4 -> h0
 653	 vadd.i32	$D3#lo,$D3#lo,$T1#lo	@ h2 -> h3
 654	 vbic.i32	$D2#lo,#0xfc000000
 655
 656	vshr.u32	$T0#lo,$D0#lo,#26
 657	vbic.i32	$D0#lo,#0xfc000000
 658	 vshr.u32	$T1#lo,$D3#lo,#26
 659	 vbic.i32	$D3#lo,#0xfc000000
 660	vadd.i32	$D1#lo,$D1#lo,$T0#lo	@ h0 -> h1
 661	 vadd.i32	$D4#lo,$D4#lo,$T1#lo	@ h3 -> h4
 662
 663	subs		$zeros,$zeros,#1
 664	beq		.Lsquare_break_neon
 665
 666	add		$tbl0,$ctx,#(48+0*9*4)
 667	add		$tbl1,$ctx,#(48+1*9*4)
 668
 669	vtrn.32		$R0,$D0#lo		@ r^2:r^1
 670	vtrn.32		$R2,$D2#lo
 671	vtrn.32		$R3,$D3#lo
 672	vtrn.32		$R1,$D1#lo
 673	vtrn.32		$R4,$D4#lo
 674
 675	vshl.u32	$S2,$R2,#2		@ *5
 676	vshl.u32	$S3,$R3,#2
 677	vshl.u32	$S1,$R1,#2
 678	vshl.u32	$S4,$R4,#2
 679	vadd.i32	$S2,$S2,$R2
 680	vadd.i32	$S1,$S1,$R1
 681	vadd.i32	$S3,$S3,$R3
 682	vadd.i32	$S4,$S4,$R4
 683
 684	vst4.32		{${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!
 685	vst4.32		{${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!
 686	vst4.32		{${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
 687	vst4.32		{${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
 688	vst1.32		{${S4}[0]},[$tbl0,:32]
 689	vst1.32		{${S4}[1]},[$tbl1,:32]
 690
 691	b		.Lsquare_neon
 692
 693.align	4
 694.Lsquare_break_neon:
 695	add		$tbl0,$ctx,#(48+2*4*9)
 696	add		$tbl1,$ctx,#(48+3*4*9)
 697
 698	vmov		$R0,$D0#lo		@ r^4:r^3
 699	vshl.u32	$S1,$D1#lo,#2		@ *5
 700	vmov		$R1,$D1#lo
 701	vshl.u32	$S2,$D2#lo,#2
 702	vmov		$R2,$D2#lo
 703	vshl.u32	$S3,$D3#lo,#2
 704	vmov		$R3,$D3#lo
 705	vshl.u32	$S4,$D4#lo,#2
 706	vmov		$R4,$D4#lo
 707	vadd.i32	$S1,$S1,$D1#lo
 708	vadd.i32	$S2,$S2,$D2#lo
 709	vadd.i32	$S3,$S3,$D3#lo
 710	vadd.i32	$S4,$S4,$D4#lo
 711
 712	vst4.32		{${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!
 713	vst4.32		{${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!
 714	vst4.32		{${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
 715	vst4.32		{${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
 716	vst1.32		{${S4}[0]},[$tbl0]
 717	vst1.32		{${S4}[1]},[$tbl1]
 718
 719.Lno_init_neon:
 720	ret				@ bx	lr
 721.size	poly1305_init_neon,.-poly1305_init_neon
 722
 723.globl	poly1305_blocks_neon
 724.type	poly1305_blocks_neon,%function
 725.align	5
 726poly1305_blocks_neon:
 727.Lpoly1305_blocks_neon:
 728	ldr	ip,[$ctx,#36]		@ is_base2_26
 729
 730	cmp	$len,#64
 731	blo	.Lpoly1305_blocks
 732
 733	stmdb	sp!,{r4-r7}
 734	vstmdb	sp!,{d8-d15}		@ ABI specification says so
 735
 736	tst	ip,ip			@ is_base2_26?
 737	bne	.Lbase2_26_neon
 738
 739	stmdb	sp!,{r1-r3,lr}
 740	bl	.Lpoly1305_init_neon
 741
 742	ldr	r4,[$ctx,#0]		@ load hash value base 2^32
 743	ldr	r5,[$ctx,#4]
 744	ldr	r6,[$ctx,#8]
 745	ldr	r7,[$ctx,#12]
 746	ldr	ip,[$ctx,#16]
 747
 748	and	r2,r4,#0x03ffffff	@ base 2^32 -> base 2^26
 749	mov	r3,r4,lsr#26
 750	 veor	$D0#lo,$D0#lo,$D0#lo
 751	mov	r4,r5,lsr#20
 752	orr	r3,r3,r5,lsl#6
 753	 veor	$D1#lo,$D1#lo,$D1#lo
 754	mov	r5,r6,lsr#14
 755	orr	r4,r4,r6,lsl#12
 756	 veor	$D2#lo,$D2#lo,$D2#lo
 757	mov	r6,r7,lsr#8
 758	orr	r5,r5,r7,lsl#18
 759	 veor	$D3#lo,$D3#lo,$D3#lo
 760	and	r3,r3,#0x03ffffff
 761	orr	r6,r6,ip,lsl#24
 762	 veor	$D4#lo,$D4#lo,$D4#lo
 763	and	r4,r4,#0x03ffffff
 764	mov	r1,#1
 765	and	r5,r5,#0x03ffffff
 766	str	r1,[$ctx,#36]		@ set is_base2_26
 767
 768	vmov.32	$D0#lo[0],r2
 769	vmov.32	$D1#lo[0],r3
 770	vmov.32	$D2#lo[0],r4
 771	vmov.32	$D3#lo[0],r5
 772	vmov.32	$D4#lo[0],r6
 773	adr	$zeros,.Lzeros
 774
 775	ldmia	sp!,{r1-r3,lr}
 776	b	.Lhash_loaded
 777
 778.align	4
 779.Lbase2_26_neon:
 780	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
 781	@ load hash value
 782
 783	veor		$D0#lo,$D0#lo,$D0#lo
 784	veor		$D1#lo,$D1#lo,$D1#lo
 785	veor		$D2#lo,$D2#lo,$D2#lo
 786	veor		$D3#lo,$D3#lo,$D3#lo
 787	veor		$D4#lo,$D4#lo,$D4#lo
 788	vld4.32		{$D0#lo[0],$D1#lo[0],$D2#lo[0],$D3#lo[0]},[$ctx]!
 789	adr		$zeros,.Lzeros
 790	vld1.32		{$D4#lo[0]},[$ctx]
 791	sub		$ctx,$ctx,#16		@ rewind
 792
 793.Lhash_loaded:
 794	add		$in2,$inp,#32
 795	mov		$padbit,$padbit,lsl#24
 796	tst		$len,#31
 797	beq		.Leven
 798
 799	vld4.32		{$H0#lo[0],$H1#lo[0],$H2#lo[0],$H3#lo[0]},[$inp]!
 800	vmov.32		$H4#lo[0],$padbit
 801	sub		$len,$len,#16
 802	add		$in2,$inp,#32
 803
 804# ifdef	__ARMEB__
 805	vrev32.8	$H0,$H0
 806	vrev32.8	$H3,$H3
 807	vrev32.8	$H1,$H1
 808	vrev32.8	$H2,$H2
 809# endif
 810	vsri.u32	$H4#lo,$H3#lo,#8	@ base 2^32 -> base 2^26
 811	vshl.u32	$H3#lo,$H3#lo,#18
 812
 813	vsri.u32	$H3#lo,$H2#lo,#14
 814	vshl.u32	$H2#lo,$H2#lo,#12
 815	vadd.i32	$H4#hi,$H4#lo,$D4#lo	@ add hash value and move to #hi
 816
 817	vbic.i32	$H3#lo,#0xfc000000
 818	vsri.u32	$H2#lo,$H1#lo,#20
 819	vshl.u32	$H1#lo,$H1#lo,#6
 820
 821	vbic.i32	$H2#lo,#0xfc000000
 822	vsri.u32	$H1#lo,$H0#lo,#26
 823	vadd.i32	$H3#hi,$H3#lo,$D3#lo
 824
 825	vbic.i32	$H0#lo,#0xfc000000
 826	vbic.i32	$H1#lo,#0xfc000000
 827	vadd.i32	$H2#hi,$H2#lo,$D2#lo
 828
 829	vadd.i32	$H0#hi,$H0#lo,$D0#lo
 830	vadd.i32	$H1#hi,$H1#lo,$D1#lo
 831
 832	mov		$tbl1,$zeros
 833	add		$tbl0,$ctx,#48
 834
 835	cmp		$len,$len
 836	b		.Long_tail
 837
 838.align	4
 839.Leven:
 840	subs		$len,$len,#64
 841	it		lo
 842	movlo		$in2,$zeros
 843
 844	vmov.i32	$H4,#1<<24		@ padbit, yes, always
 845	vld4.32		{$H0#lo,$H1#lo,$H2#lo,$H3#lo},[$inp]	@ inp[0:1]
 846	add		$inp,$inp,#64
 847	vld4.32		{$H0#hi,$H1#hi,$H2#hi,$H3#hi},[$in2]	@ inp[2:3] (or 0)
 848	add		$in2,$in2,#64
 849	itt		hi
 850	addhi		$tbl1,$ctx,#(48+1*9*4)
 851	addhi		$tbl0,$ctx,#(48+3*9*4)
 852
 853# ifdef	__ARMEB__
 854	vrev32.8	$H0,$H0
 855	vrev32.8	$H3,$H3
 856	vrev32.8	$H1,$H1
 857	vrev32.8	$H2,$H2
 858# endif
 859	vsri.u32	$H4,$H3,#8		@ base 2^32 -> base 2^26
 860	vshl.u32	$H3,$H3,#18
 861
 862	vsri.u32	$H3,$H2,#14
 863	vshl.u32	$H2,$H2,#12
 864
 865	vbic.i32	$H3,#0xfc000000
 866	vsri.u32	$H2,$H1,#20
 867	vshl.u32	$H1,$H1,#6
 868
 869	vbic.i32	$H2,#0xfc000000
 870	vsri.u32	$H1,$H0,#26
 871
 872	vbic.i32	$H0,#0xfc000000
 873	vbic.i32	$H1,#0xfc000000
 874
 875	bls		.Lskip_loop
 876
 877	vld4.32		{${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!	@ load r^2
 878	vld4.32		{${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!	@ load r^4
 879	vld4.32		{${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
 880	vld4.32		{${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
 881	b		.Loop_neon
 882
 883.align	5
 884.Loop_neon:
 885	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
 886	@ ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
 887	@ ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
 888	@   \___________________/
 889	@ ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
 890	@ ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
 891	@   \___________________/ \____________________/
 892	@
 893	@ Note that we start with inp[2:3]*r^2. This is because it
 894	@ doesn't depend on reduction in previous iteration.
 895	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
 896	@ d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
 897	@ d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
 898	@ d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
 899	@ d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
 900	@ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
 901
 902	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
 903	@ inp[2:3]*r^2
 904
 905	vadd.i32	$H2#lo,$H2#lo,$D2#lo	@ accumulate inp[0:1]
 906	vmull.u32	$D2,$H2#hi,${R0}[1]
 907	vadd.i32	$H0#lo,$H0#lo,$D0#lo
 908	vmull.u32	$D0,$H0#hi,${R0}[1]
 909	vadd.i32	$H3#lo,$H3#lo,$D3#lo
 910	vmull.u32	$D3,$H3#hi,${R0}[1]
 911	vmlal.u32	$D2,$H1#hi,${R1}[1]
 912	vadd.i32	$H1#lo,$H1#lo,$D1#lo
 913	vmull.u32	$D1,$H1#hi,${R0}[1]
 914
 915	vadd.i32	$H4#lo,$H4#lo,$D4#lo
 916	vmull.u32	$D4,$H4#hi,${R0}[1]
 917	subs		$len,$len,#64
 918	vmlal.u32	$D0,$H4#hi,${S1}[1]
 919	it		lo
 920	movlo		$in2,$zeros
 921	vmlal.u32	$D3,$H2#hi,${R1}[1]
 922	vld1.32		${S4}[1],[$tbl1,:32]
 923	vmlal.u32	$D1,$H0#hi,${R1}[1]
 924	vmlal.u32	$D4,$H3#hi,${R1}[1]
 925
 926	vmlal.u32	$D0,$H3#hi,${S2}[1]
 927	vmlal.u32	$D3,$H1#hi,${R2}[1]
 928	vmlal.u32	$D4,$H2#hi,${R2}[1]
 929	vmlal.u32	$D1,$H4#hi,${S2}[1]
 930	vmlal.u32	$D2,$H0#hi,${R2}[1]
 931
 932	vmlal.u32	$D3,$H0#hi,${R3}[1]
 933	vmlal.u32	$D0,$H2#hi,${S3}[1]
 934	vmlal.u32	$D4,$H1#hi,${R3}[1]
 935	vmlal.u32	$D1,$H3#hi,${S3}[1]
 936	vmlal.u32	$D2,$H4#hi,${S3}[1]
 937
 938	vmlal.u32	$D3,$H4#hi,${S4}[1]
 939	vmlal.u32	$D0,$H1#hi,${S4}[1]
 940	vmlal.u32	$D4,$H0#hi,${R4}[1]
 941	vmlal.u32	$D1,$H2#hi,${S4}[1]
 942	vmlal.u32	$D2,$H3#hi,${S4}[1]
 943
 944	vld4.32		{$H0#hi,$H1#hi,$H2#hi,$H3#hi},[$in2]	@ inp[2:3] (or 0)
 945	add		$in2,$in2,#64
 946
 947	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
 948	@ (hash+inp[0:1])*r^4 and accumulate
 949
 950	vmlal.u32	$D3,$H3#lo,${R0}[0]
 951	vmlal.u32	$D0,$H0#lo,${R0}[0]
 952	vmlal.u32	$D4,$H4#lo,${R0}[0]
 953	vmlal.u32	$D1,$H1#lo,${R0}[0]
 954	vmlal.u32	$D2,$H2#lo,${R0}[0]
 955	vld1.32		${S4}[0],[$tbl0,:32]
 956
 957	vmlal.u32	$D3,$H2#lo,${R1}[0]
 958	vmlal.u32	$D0,$H4#lo,${S1}[0]
 959	vmlal.u32	$D4,$H3#lo,${R1}[0]
 960	vmlal.u32	$D1,$H0#lo,${R1}[0]
 961	vmlal.u32	$D2,$H1#lo,${R1}[0]
 962
 963	vmlal.u32	$D3,$H1#lo,${R2}[0]
 964	vmlal.u32	$D0,$H3#lo,${S2}[0]
 965	vmlal.u32	$D4,$H2#lo,${R2}[0]
 966	vmlal.u32	$D1,$H4#lo,${S2}[0]
 967	vmlal.u32	$D2,$H0#lo,${R2}[0]
 968
 969	vmlal.u32	$D3,$H0#lo,${R3}[0]
 970	vmlal.u32	$D0,$H2#lo,${S3}[0]
 971	vmlal.u32	$D4,$H1#lo,${R3}[0]
 972	vmlal.u32	$D1,$H3#lo,${S3}[0]
 973	vmlal.u32	$D3,$H4#lo,${S4}[0]
 974
 975	vmlal.u32	$D2,$H4#lo,${S3}[0]
 976	vmlal.u32	$D0,$H1#lo,${S4}[0]
 977	vmlal.u32	$D4,$H0#lo,${R4}[0]
 978	vmov.i32	$H4,#1<<24		@ padbit, yes, always
 979	vmlal.u32	$D1,$H2#lo,${S4}[0]
 980	vmlal.u32	$D2,$H3#lo,${S4}[0]
 981
 982	vld4.32		{$H0#lo,$H1#lo,$H2#lo,$H3#lo},[$inp]	@ inp[0:1]
 983	add		$inp,$inp,#64
 984# ifdef	__ARMEB__
 985	vrev32.8	$H0,$H0
 986	vrev32.8	$H1,$H1
 987	vrev32.8	$H2,$H2
 988	vrev32.8	$H3,$H3
 989# endif
 990
 991	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
 992	@ lazy reduction interleaved with base 2^32 -> base 2^26 of
 993	@ inp[0:3] previously loaded to $H0-$H3 and smashed to $H0-$H4.
 994
 995	vshr.u64	$T0,$D3,#26
 996	vmovn.i64	$D3#lo,$D3
 997	 vshr.u64	$T1,$D0,#26
 998	 vmovn.i64	$D0#lo,$D0
 999	vadd.i64	$D4,$D4,$T0		@ h3 -> h4
1000	vbic.i32	$D3#lo,#0xfc000000
1001	  vsri.u32	$H4,$H3,#8		@ base 2^32 -> base 2^26
1002	 vadd.i64	$D1,$D1,$T1		@ h0 -> h1
1003	  vshl.u32	$H3,$H3,#18
1004	 vbic.i32	$D0#lo,#0xfc000000
1005
1006	vshrn.u64	$T0#lo,$D4,#26
1007	vmovn.i64	$D4#lo,$D4
1008	 vshr.u64	$T1,$D1,#26
1009	 vmovn.i64	$D1#lo,$D1
1010	 vadd.i64	$D2,$D2,$T1		@ h1 -> h2
1011	  vsri.u32	$H3,$H2,#14
1012	vbic.i32	$D4#lo,#0xfc000000
1013	  vshl.u32	$H2,$H2,#12
1014	 vbic.i32	$D1#lo,#0xfc000000
1015
1016	vadd.i32	$D0#lo,$D0#lo,$T0#lo
1017	vshl.u32	$T0#lo,$T0#lo,#2
1018	  vbic.i32	$H3,#0xfc000000
1019	 vshrn.u64	$T1#lo,$D2,#26
1020	 vmovn.i64	$D2#lo,$D2
1021	vaddl.u32	$D0,$D0#lo,$T0#lo	@ h4 -> h0 [widen for a sec]
1022	  vsri.u32	$H2,$H1,#20
1023	 vadd.i32	$D3#lo,$D3#lo,$T1#lo	@ h2 -> h3
1024	  vshl.u32	$H1,$H1,#6
1025	 vbic.i32	$D2#lo,#0xfc000000
1026	  vbic.i32	$H2,#0xfc000000
1027
1028	vshrn.u64	$T0#lo,$D0,#26		@ re-narrow
1029	vmovn.i64	$D0#lo,$D0
1030	  vsri.u32	$H1,$H0,#26
1031	  vbic.i32	$H0,#0xfc000000
1032	 vshr.u32	$T1#lo,$D3#lo,#26
1033	 vbic.i32	$D3#lo,#0xfc000000
1034	vbic.i32	$D0#lo,#0xfc000000
1035	vadd.i32	$D1#lo,$D1#lo,$T0#lo	@ h0 -> h1
1036	 vadd.i32	$D4#lo,$D4#lo,$T1#lo	@ h3 -> h4
1037	  vbic.i32	$H1,#0xfc000000
1038
1039	bhi		.Loop_neon
1040
1041.Lskip_loop:
1042	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
1043	@ multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
1044
1045	add		$tbl1,$ctx,#(48+0*9*4)
1046	add		$tbl0,$ctx,#(48+1*9*4)
1047	adds		$len,$len,#32
1048	it		ne
1049	movne		$len,#0
1050	bne		.Long_tail
1051
1052	vadd.i32	$H2#hi,$H2#lo,$D2#lo	@ add hash value and move to #hi
1053	vadd.i32	$H0#hi,$H0#lo,$D0#lo
1054	vadd.i32	$H3#hi,$H3#lo,$D3#lo
1055	vadd.i32	$H1#hi,$H1#lo,$D1#lo
1056	vadd.i32	$H4#hi,$H4#lo,$D4#lo
1057
1058.Long_tail:
1059	vld4.32		{${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!	@ load r^1
1060	vld4.32		{${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!	@ load r^2
1061
1062	vadd.i32	$H2#lo,$H2#lo,$D2#lo	@ can be redundant
1063	vmull.u32	$D2,$H2#hi,$R0
1064	vadd.i32	$H0#lo,$H0#lo,$D0#lo
1065	vmull.u32	$D0,$H0#hi,$R0
1066	vadd.i32	$H3#lo,$H3#lo,$D3#lo
1067	vmull.u32	$D3,$H3#hi,$R0
1068	vadd.i32	$H1#lo,$H1#lo,$D1#lo
1069	vmull.u32	$D1,$H1#hi,$R0
1070	vadd.i32	$H4#lo,$H4#lo,$D4#lo
1071	vmull.u32	$D4,$H4#hi,$R0
1072
1073	vmlal.u32	$D0,$H4#hi,$S1
1074	vld4.32		{${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
1075	vmlal.u32	$D3,$H2#hi,$R1
1076	vld4.32		{${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
1077	vmlal.u32	$D1,$H0#hi,$R1
1078	vmlal.u32	$D4,$H3#hi,$R1
1079	vmlal.u32	$D2,$H1#hi,$R1
1080
1081	vmlal.u32	$D3,$H1#hi,$R2
1082	vld1.32		${S4}[1],[$tbl1,:32]
1083	vmlal.u32	$D0,$H3#hi,$S2
1084	vld1.32		${S4}[0],[$tbl0,:32]
1085	vmlal.u32	$D4,$H2#hi,$R2
1086	vmlal.u32	$D1,$H4#hi,$S2
1087	vmlal.u32	$D2,$H0#hi,$R2
1088
1089	vmlal.u32	$D3,$H0#hi,$R3
1090	 it		ne
1091	 addne		$tbl1,$ctx,#(48+2*9*4)
1092	vmlal.u32	$D0,$H2#hi,$S3
1093	 it		ne
1094	 addne		$tbl0,$ctx,#(48+3*9*4)
1095	vmlal.u32	$D4,$H1#hi,$R3
1096	vmlal.u32	$D1,$H3#hi,$S3
1097	vmlal.u32	$D2,$H4#hi,$S3
1098
1099	vmlal.u32	$D3,$H4#hi,$S4
1100	 vorn		$MASK,$MASK,$MASK	@ all-ones, can be redundant
1101	vmlal.u32	$D0,$H1#hi,$S4
1102	 vshr.u64	$MASK,$MASK,#38
1103	vmlal.u32	$D4,$H0#hi,$R4
1104	vmlal.u32	$D1,$H2#hi,$S4
1105	vmlal.u32	$D2,$H3#hi,$S4
1106
1107	beq		.Lshort_tail
1108
1109	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
1110	@ (hash+inp[0:1])*r^4:r^3 and accumulate
1111
1112	vld4.32		{${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!	@ load r^3
1113	vld4.32		{${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!	@ load r^4
1114
1115	vmlal.u32	$D2,$H2#lo,$R0
1116	vmlal.u32	$D0,$H0#lo,$R0
1117	vmlal.u32	$D3,$H3#lo,$R0
1118	vmlal.u32	$D1,$H1#lo,$R0
1119	vmlal.u32	$D4,$H4#lo,$R0
1120
1121	vmlal.u32	$D0,$H4#lo,$S1
1122	vld4.32		{${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
1123	vmlal.u32	$D3,$H2#lo,$R1
1124	vld4.32		{${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
1125	vmlal.u32	$D1,$H0#lo,$R1
1126	vmlal.u32	$D4,$H3#lo,$R1
1127	vmlal.u32	$D2,$H1#lo,$R1
1128
1129	vmlal.u32	$D3,$H1#lo,$R2
1130	vld1.32		${S4}[1],[$tbl1,:32]
1131	vmlal.u32	$D0,$H3#lo,$S2
1132	vld1.32		${S4}[0],[$tbl0,:32]
1133	vmlal.u32	$D4,$H2#lo,$R2
1134	vmlal.u32	$D1,$H4#lo,$S2
1135	vmlal.u32	$D2,$H0#lo,$R2
1136
1137	vmlal.u32	$D3,$H0#lo,$R3
1138	vmlal.u32	$D0,$H2#lo,$S3
1139	vmlal.u32	$D4,$H1#lo,$R3
1140	vmlal.u32	$D1,$H3#lo,$S3
1141	vmlal.u32	$D2,$H4#lo,$S3
1142
1143	vmlal.u32	$D3,$H4#lo,$S4
1144	 vorn		$MASK,$MASK,$MASK	@ all-ones
1145	vmlal.u32	$D0,$H1#lo,$S4
1146	 vshr.u64	$MASK,$MASK,#38
1147	vmlal.u32	$D4,$H0#lo,$R4
1148	vmlal.u32	$D1,$H2#lo,$S4
1149	vmlal.u32	$D2,$H3#lo,$S4
1150
1151.Lshort_tail:
1152	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
1153	@ horizontal addition
1154
1155	vadd.i64	$D3#lo,$D3#lo,$D3#hi
1156	vadd.i64	$D0#lo,$D0#lo,$D0#hi
1157	vadd.i64	$D4#lo,$D4#lo,$D4#hi
1158	vadd.i64	$D1#lo,$D1#lo,$D1#hi
1159	vadd.i64	$D2#lo,$D2#lo,$D2#hi
1160
1161	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
1162	@ lazy reduction, but without narrowing
1163
1164	vshr.u64	$T0,$D3,#26
1165	vand.i64	$D3,$D3,$MASK
1166	 vshr.u64	$T1,$D0,#26
1167	 vand.i64	$D0,$D0,$MASK
1168	vadd.i64	$D4,$D4,$T0		@ h3 -> h4
1169	 vadd.i64	$D1,$D1,$T1		@ h0 -> h1
1170
1171	vshr.u64	$T0,$D4,#26
1172	vand.i64	$D4,$D4,$MASK
1173	 vshr.u64	$T1,$D1,#26
1174	 vand.i64	$D1,$D1,$MASK
1175	 vadd.i64	$D2,$D2,$T1		@ h1 -> h2
1176
1177	vadd.i64	$D0,$D0,$T0
1178	vshl.u64	$T0,$T0,#2
1179	 vshr.u64	$T1,$D2,#26
1180	 vand.i64	$D2,$D2,$MASK
1181	vadd.i64	$D0,$D0,$T0		@ h4 -> h0
1182	 vadd.i64	$D3,$D3,$T1		@ h2 -> h3
1183
1184	vshr.u64	$T0,$D0,#26
1185	vand.i64	$D0,$D0,$MASK
1186	 vshr.u64	$T1,$D3,#26
1187	 vand.i64	$D3,$D3,$MASK
1188	vadd.i64	$D1,$D1,$T0		@ h0 -> h1
1189	 vadd.i64	$D4,$D4,$T1		@ h3 -> h4
1190
1191	cmp		$len,#0
1192	bne		.Leven
1193
1194	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
1195	@ store hash value
1196
1197	vst4.32		{$D0#lo[0],$D1#lo[0],$D2#lo[0],$D3#lo[0]},[$ctx]!
1198	vst1.32		{$D4#lo[0]},[$ctx]
1199
1200	vldmia	sp!,{d8-d15}			@ epilogue
1201	ldmia	sp!,{r4-r7}
1202	ret					@ bx	lr
1203.size	poly1305_blocks_neon,.-poly1305_blocks_neon
1204
1205.align	5
1206.Lzeros:
1207.long	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
1208#ifndef	__KERNEL__
1209.LOPENSSL_armcap:
1210# ifdef	_WIN32
1211.word	OPENSSL_armcap_P
1212# else
1213.word	OPENSSL_armcap_P-.Lpoly1305_init
1214# endif
1215.comm	OPENSSL_armcap_P,4,4
1216.hidden	OPENSSL_armcap_P
1217#endif
1218#endif
1219___
1220}	}
1221$code.=<<___;
1222.asciz	"Poly1305 for ARMv4/NEON, CRYPTOGAMS by \@dot-asm"
1223.align	2
1224___
1225
1226foreach (split("\n",$code)) {
1227	s/\`([^\`]*)\`/eval $1/geo;
1228
1229	s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo	or
1230	s/\bret\b/bx	lr/go						or
1231	s/\bbx\s+lr\b/.word\t0xe12fff1e/go;	# make it possible to compile with -march=armv4
1232
1233	print $_,"\n";
1234}
1235close STDOUT; # enforce flush