aes-gcm-p10.S

   1/* SPDX-License-Identifier: GPL-2.0-or-later */
   2#
   3# Accelerated AES-GCM stitched implementation for ppc64le.
   4#
   5# Copyright 2024- IBM Inc.
   6#
   7#===================================================================================
   8# Written by Danny Tsen <dtsen@us.ibm.com>
   9#
  10# GHASH is based on the Karatsuba multiplication method.
  11#
  12#    Xi xor X1
  13#
  14#    X1 * H^4 + X2 * H^3 + x3 * H^2 + X4 * H =
  15#      (X1.h * H4.h + xX.l * H4.l + X1 * H4) +
  16#      (X2.h * H3.h + X2.l * H3.l + X2 * H3) +
  17#      (X3.h * H2.h + X3.l * H2.l + X3 * H2) +
  18#      (X4.h * H.h + X4.l * H.l + X4 * H)
  19#
  20# Xi = v0
  21# H Poly = v2
  22# Hash keys = v3 - v14
  23#     ( H.l, H, H.h)
  24#     ( H^2.l, H^2, H^2.h)
  25#     ( H^3.l, H^3, H^3.h)
  26#     ( H^4.l, H^4, H^4.h)
  27#
  28# v30 is IV
  29# v31 - counter 1
  30#
  31# AES used,
  32#     vs0 - round key 0
  33#     v15, v16, v17, v18, v19, v20, v21, v22 for 8 blocks (encrypted)
  34#
  35# This implementation uses stitched AES-GCM approach to improve overall performance.
  36# AES is implemented with 8x blocks and GHASH is using 2 4x blocks.
  37#
  38# ===================================================================================
  39#
  40
  41#include <asm/ppc_asm.h>
  42#include <linux/linkage.h>
  43
  44.machine        "any"
  45.text
  46
  47.macro	SAVE_GPR GPR OFFSET FRAME
  48	std	\GPR,\OFFSET(\FRAME)
  49.endm
  50
  51.macro	SAVE_VRS VRS OFFSET FRAME
  52	stxv	\VRS+32, \OFFSET(\FRAME)
  53.endm
  54
  55.macro	RESTORE_GPR GPR OFFSET FRAME
  56	ld	\GPR,\OFFSET(\FRAME)
  57.endm
  58
  59.macro	RESTORE_VRS VRS OFFSET FRAME
  60	lxv	\VRS+32, \OFFSET(\FRAME)
  61.endm
  62
  63.macro SAVE_REGS
  64	mflr 0
  65	std 0, 16(1)
  66	stdu 1,-512(1)
  67
  68	SAVE_GPR 14, 112, 1
  69	SAVE_GPR 15, 120, 1
  70	SAVE_GPR 16, 128, 1
  71	SAVE_GPR 17, 136, 1
  72	SAVE_GPR 18, 144, 1
  73	SAVE_GPR 19, 152, 1
  74	SAVE_GPR 20, 160, 1
  75	SAVE_GPR 21, 168, 1
  76	SAVE_GPR 22, 176, 1
  77	SAVE_GPR 23, 184, 1
  78	SAVE_GPR 24, 192, 1
  79
  80	addi	9, 1, 256
  81	SAVE_VRS 20, 0, 9
  82	SAVE_VRS 21, 16, 9
  83	SAVE_VRS 22, 32, 9
  84	SAVE_VRS 23, 48, 9
  85	SAVE_VRS 24, 64, 9
  86	SAVE_VRS 25, 80, 9
  87	SAVE_VRS 26, 96, 9
  88	SAVE_VRS 27, 112, 9
  89	SAVE_VRS 28, 128, 9
  90	SAVE_VRS 29, 144, 9
  91	SAVE_VRS 30, 160, 9
  92	SAVE_VRS 31, 176, 9
  93.endm # SAVE_REGS
  94
  95.macro RESTORE_REGS
  96	addi	9, 1, 256
  97	RESTORE_VRS 20, 0, 9
  98	RESTORE_VRS 21, 16, 9
  99	RESTORE_VRS 22, 32, 9
 100	RESTORE_VRS 23, 48, 9
 101	RESTORE_VRS 24, 64, 9
 102	RESTORE_VRS 25, 80, 9
 103	RESTORE_VRS 26, 96, 9
 104	RESTORE_VRS 27, 112, 9
 105	RESTORE_VRS 28, 128, 9
 106	RESTORE_VRS 29, 144, 9
 107	RESTORE_VRS 30, 160, 9
 108	RESTORE_VRS 31, 176, 9
 109
 110	RESTORE_GPR 14, 112, 1
 111	RESTORE_GPR 15, 120, 1
 112	RESTORE_GPR 16, 128, 1
 113	RESTORE_GPR 17, 136, 1
 114	RESTORE_GPR 18, 144, 1
 115	RESTORE_GPR 19, 152, 1
 116	RESTORE_GPR 20, 160, 1
 117	RESTORE_GPR 21, 168, 1
 118	RESTORE_GPR 22, 176, 1
 119	RESTORE_GPR 23, 184, 1
 120	RESTORE_GPR 24, 192, 1
 121
 122	addi    1, 1, 512
 123	ld 0, 16(1)
 124	mtlr 0
 125.endm # RESTORE_REGS
 126
 127# 4x loops
 128.macro AES_CIPHER_4x _VCIPHER ST r
 129	\_VCIPHER	\ST, \ST, \r
 130	\_VCIPHER	\ST+1, \ST+1, \r
 131	\_VCIPHER	\ST+2, \ST+2, \r
 132	\_VCIPHER	\ST+3, \ST+3, \r
 133.endm
 134
 135# 8x loops
 136.macro AES_CIPHER_8x _VCIPHER ST r
 137	\_VCIPHER	\ST, \ST, \r
 138	\_VCIPHER	\ST+1, \ST+1, \r
 139	\_VCIPHER	\ST+2, \ST+2, \r
 140	\_VCIPHER	\ST+3, \ST+3, \r
 141	\_VCIPHER	\ST+4, \ST+4, \r
 142	\_VCIPHER	\ST+5, \ST+5, \r
 143	\_VCIPHER	\ST+6, \ST+6, \r
 144	\_VCIPHER	\ST+7, \ST+7, \r
 145.endm
 146
 147.macro LOOP_8AES_STATE
 148	xxlor	32+23, 1, 1
 149	xxlor	32+24, 2, 2
 150	xxlor	32+25, 3, 3
 151	xxlor	32+26, 4, 4
 152	AES_CIPHER_8x vcipher, 15, 23
 153	AES_CIPHER_8x vcipher, 15, 24
 154	AES_CIPHER_8x vcipher, 15, 25
 155	AES_CIPHER_8x vcipher, 15, 26
 156	xxlor	32+23, 5, 5
 157	xxlor	32+24, 6, 6
 158	xxlor	32+25, 7, 7
 159	xxlor	32+26, 8, 8
 160	AES_CIPHER_8x vcipher, 15, 23
 161	AES_CIPHER_8x vcipher, 15, 24
 162	AES_CIPHER_8x vcipher, 15, 25
 163	AES_CIPHER_8x vcipher, 15, 26
 164.endm
 165
 166#
 167# PPC_GHASH4x(H, S1, S2, S3, S4): Compute 4x hash values based on Karatsuba method.
 168# H: returning digest
 169# S#: states
 170#
 171# S1 should xor with the previous digest
 172#
 173# Xi = v0
 174# H Poly = v2
 175# Hash keys = v3 - v14
 176# Scratch: v23 - v29
 177#
 178.macro PPC_GHASH4x H S1 S2 S3 S4
 179
 180	vpmsumd	23, 12, \S1		# H4.L * X.L
 181	vpmsumd	24, 9, \S2
 182	vpmsumd	25, 6, \S3
 183	vpmsumd	26, 3, \S4
 184
 185	vpmsumd	27, 13, \S1		# H4.L * X.H + H4.H * X.L
 186	vpmsumd	28, 10, \S2		# H3.L * X1.H + H3.H * X1.L
 187
 188	vxor	23, 23, 24
 189	vxor	23, 23, 25
 190	vxor	23, 23, 26		# L
 191
 192	vxor	24, 27, 28
 193	vpmsumd	25, 7, \S3
 194	vpmsumd	26, 4, \S4
 195
 196	vxor	24, 24, 25
 197	vxor	24, 24, 26		# M
 198
 199	# sum hash and reduction with H Poly
 200	vpmsumd	28, 23, 2		# reduction
 201
 202	vxor	1, 1, 1
 203	vsldoi	25, 24, 1, 8		# mL
 204	vsldoi	1, 1, 24, 8		# mH
 205	vxor	23, 23, 25		# mL + L
 206
 207	# This performs swap and xor like,
 208	#   vsldoi	23, 23, 23, 8		# swap
 209	#   vxor	23, 23, 28
 210	xxlor	32+25, 10, 10
 211	vpermxor 23, 23, 28, 25
 212
 213	vpmsumd	26, 14, \S1		# H4.H * X.H
 214	vpmsumd	27, 11, \S2
 215	vpmsumd	28, 8, \S3
 216	vpmsumd	29, 5, \S4
 217
 218	vxor	24, 26, 27
 219	vxor	24, 24, 28
 220	vxor	24, 24, 29
 221
 222	vxor	24, 24, 1
 223
 224	# sum hash and reduction with H Poly
 225	vsldoi	25, 23, 23, 8		# swap
 226	vpmsumd	23, 23, 2
 227	vxor	27, 25, 24
 228	vxor	\H, 23, 27
 229.endm
 230
 231#
 232# Compute update single ghash
 233# scratch: v1, v22..v27
 234#
 235.macro PPC_GHASH1x H S1
 236
 237	vxor	1, 1, 1
 238
 239	vpmsumd	22, 3, \S1		# L
 240	vpmsumd	23, 4, \S1		# M
 241	vpmsumd	24, 5, \S1		# H
 242
 243	vpmsumd	27, 22, 2		# reduction
 244
 245	vsldoi	25, 23, 1, 8		# mL
 246	vsldoi	26, 1, 23, 8		# mH
 247	vxor	22, 22, 25		# LL + LL
 248	vxor	24, 24, 26		# HH + HH
 249
 250	xxlor	32+25, 10, 10
 251	vpermxor 22, 22, 27, 25
 252
 253	vsldoi	23, 22, 22, 8		# swap
 254	vpmsumd	22, 22, 2		# reduction
 255	vxor	23, 23, 24
 256	vxor	\H, 22, 23
 257.endm
 258
 259#
 260# LOAD_HASH_TABLE
 261# Xi = v0
 262# H Poly = v2
 263# Hash keys = v3 - v14
 264#
 265.macro LOAD_HASH_TABLE
 266	# Load Xi
 267	lxvb16x	32, 0, 8	# load Xi
 268
 269	# load Hash - h^4, h^3, h^2, h
 270	li	10, 32
 271	lxvd2x	2+32, 10, 8	# H Poli
 272	li	10, 48
 273	lxvd2x	3+32, 10, 8	# Hl
 274	li	10, 64
 275	lxvd2x	4+32, 10, 8	# H
 276	li	10, 80
 277	lxvd2x	5+32, 10, 8	# Hh
 278
 279	li	10, 96
 280	lxvd2x	6+32, 10, 8	# H^2l
 281	li	10, 112
 282	lxvd2x	7+32, 10, 8	# H^2
 283	li	10, 128
 284	lxvd2x	8+32, 10, 8	# H^2h
 285
 286	li	10, 144
 287	lxvd2x	9+32, 10, 8	# H^3l
 288	li	10, 160
 289	lxvd2x	10+32, 10, 8	# H^3
 290	li	10, 176
 291	lxvd2x	11+32, 10, 8	# H^3h
 292
 293	li	10, 192
 294	lxvd2x	12+32, 10, 8	# H^4l
 295	li	10, 208
 296	lxvd2x	13+32, 10, 8	# H^4
 297	li	10, 224
 298	lxvd2x	14+32, 10, 8	# H^4h
 299.endm
 300
 301################################################################################
 302# Compute AES and ghash one block at a time.
 303# r23: AES rounds
 304# v30: current IV
 305# vs0: roundkey 0
 306#
 307################################################################################
 308SYM_FUNC_START_LOCAL(aes_gcm_crypt_1x)
 309
 310	cmpdi	5, 16
 311	bge	__More_1x
 312	blr
 313__More_1x:
 314	li      10, 16
 315	divdu   12, 5, 10
 316
 317	xxlxor	32+15, 32+30, 0
 318
 319	# Pre-load 8 AES rounds to scratch vectors.
 320	xxlor	32+16, 1, 1
 321	xxlor	32+17, 2, 2
 322	xxlor	32+18, 3, 3
 323	xxlor	32+19, 4, 4
 324	xxlor	32+20, 5, 5
 325	xxlor	32+21, 6, 6
 326	xxlor	32+28, 7, 7
 327	xxlor	32+29, 8, 8
 328	lwz	23, 240(6)	# n rounds
 329	addi	22, 23, -9	# remaing AES rounds
 330
 331	cmpdi	12, 0
 332	bgt	__Loop_1x
 333	blr
 334
 335__Loop_1x:
 336	mtctr	22
 337	addi	10, 6, 144
 338	vcipher	15, 15, 16
 339	vcipher	15, 15, 17
 340	vcipher	15, 15, 18
 341	vcipher	15, 15, 19
 342	vcipher	15, 15, 20
 343	vcipher	15, 15, 21
 344	vcipher	15, 15, 28
 345	vcipher	15, 15, 29
 346
 347__Loop_aes_1state:
 348	lxv	32+1, 0(10)
 349	vcipher	15, 15, 1
 350	addi	10, 10, 16
 351	bdnz	__Loop_aes_1state
 352	lxv	32+1, 0(10)		# last round key
 353	lxvb16x 11, 0, 14		# load input block
 354	vcipherlast 15, 15, 1
 355
 356	xxlxor	32+15, 32+15, 11
 357	stxvb16x 32+15, 0, 9	# store output
 358	addi	14, 14, 16
 359	addi	9, 9, 16
 360
 361	cmpdi	24, 0	# decrypt?
 362	bne	__Encrypt_1x
 363	xxlor	15+32, 11, 11
 364__Encrypt_1x:
 365	vxor	15, 15, 0
 366	PPC_GHASH1x 0, 15
 367
 368	addi	5, 5, -16
 369	addi	11, 11, 16
 370
 371	vadduwm 30, 30, 31		# IV + counter
 372	xxlxor	32+15, 32+30, 0
 373	addi	12, 12, -1
 374	cmpdi	12, 0
 375	bgt	__Loop_1x
 376
 377	stxvb16x 32+30, 0, 7		# update IV
 378	stxvb16x 32+0, 0, 8		# update Xi
 379	blr
 380SYM_FUNC_END(aes_gcm_crypt_1x)
 381
 382################################################################################
 383# Process a normal partial block when we come here.
 384#  Compute partial mask, Load and store partial block to stack.
 385#  Update partial_len and pblock.
 386#  pblock is (encrypted ^ AES state) for encrypt
 387#        and (input ^ AES state) for decrypt.
 388#
 389################################################################################
 390SYM_FUNC_START_LOCAL(__Process_partial)
 391
 392	# create partial mask
 393	vspltisb 16, -1
 394	li	12, 16
 395	sub	12, 12, 5
 396	sldi	12, 12, 3
 397	mtvsrdd	32+17, 0, 12
 398	vslo	16, 16, 17		# partial block mask
 399
 400	lxvb16x 11, 0, 14		# load partial block
 401	xxland	11, 11, 32+16
 402
 403	# AES crypt partial
 404	xxlxor	32+15, 32+30, 0
 405	lwz	23, 240(6)		# n rounds
 406	addi	22, 23, -1		# loop - 1
 407	mtctr	22
 408	addi	10, 6, 16
 409
 410__Loop_aes_pstate:
 411	lxv	32+1, 0(10)
 412	vcipher	15, 15, 1
 413	addi	10, 10, 16
 414	bdnz	__Loop_aes_pstate
 415	lxv	32+1, 0(10)		# last round key
 416	vcipherlast 15, 15, 1
 417
 418	xxlxor	32+15, 32+15, 11
 419	vand	15, 15, 16
 420
 421	# AES crypt output v15
 422	# Write partial
 423	li	10, 224
 424	stxvb16x 15+32, 10, 1		# write v15 to stack
 425	addi	10, 1, 223
 426	addi	12, 9, -1
 427        mtctr	5			# partial block len
 428__Write_partial:
 429        lbzu	22, 1(10)
 430	stbu	22, 1(12)
 431        bdnz	__Write_partial
 432
 433	cmpdi	24, 0			# decrypt?
 434	bne	__Encrypt_partial
 435	xxlor	32+15, 11, 11		# decrypt using the input block
 436__Encrypt_partial:
 437	#vxor	15, 15, 0		# ^ previous hash
 438	#PPC_GHASH1x 0, 15
 439
 440	add	14, 14, 5
 441	add	9, 9, 5
 442	std	5, 56(7)		# update partial
 443	sub	11, 11, 5
 444	li	5, 0			# done last byte
 445
 446	#
 447	# Don't increase IV since this is the last partial.
 448	# It should get updated in gcm_update if no more data blocks.
 449	#vadduwm	30, 30, 31		# increase IV
 450	stxvb16x 32+30, 0, 7		# update IV
 451	li	10, 64
 452	stxvb16x 32+0, 0, 8		# Update X1
 453	stxvb16x 32+15, 10, 7		# Update pblock
 454	blr
 455SYM_FUNC_END(__Process_partial)
 456
 457################################################################################
 458# Combine partial blocks and ghash when we come here.
 459#
 460# The partial block has to be shifted to the right location to encrypt/decrypt
 461# and compute ghash if combing the previous partial block is needed.
 462# - Compute ghash for a full block. Clear Partial_len and pblock. Update IV.
 463#   Write Xi.
 464# - Don't compute ghash if not full block.  gcm_update will take care of it
 465#   is the last block. Update Partial_len and pblock.
 466#
 467################################################################################
 468SYM_FUNC_START_LOCAL(__Combine_partial)
 469
 470	ld	12, 56(7)
 471	mr	21, 5			# these bytes to be processed
 472
 473	li	17, 0
 474	li	16, 16
 475	sub	22, 16, 12		# bytes to complete a block
 476	sub	17, 22, 5		# remaining bytes in a block
 477	cmpdi	5, 16
 478	ble	__Inp_msg_less16
 479	li	17, 0
 480	mr	21, 22
 481	b	__Combine_continue
 482__Inp_msg_less16:
 483	cmpd	22, 5
 484	bgt	__Combine_continue
 485	li	17, 0
 486	mr	21, 22			# these bytes to be processed
 487
 488__Combine_continue:
 489	# load msg and shift to the proper location and mask
 490	vspltisb 16, -1
 491	sldi	15, 12, 3
 492	mtvsrdd	32+17, 0, 15
 493	vslo	16, 16, 17
 494	vsro	16, 16, 17
 495	sldi	15, 17, 3
 496	mtvsrdd	32+17, 0, 15
 497	vsro	16, 16, 17
 498	vslo	16, 16, 17		# mask
 499
 500	lxvb16x 32+19, 0, 14		# load partial block
 501	sldi	15, 12, 3
 502	mtvsrdd	32+17, 0, 15
 503	vsro	19, 19, 17		# 0x00..xxxx??..??
 504	sldi	15, 17, 3
 505	mtvsrdd	32+17, 0, 15
 506	vsro	19, 19, 17		# 0x00..xxxx
 507	vslo	19, 19, 17		# shift back to form 0x00..xxxx00..00
 508
 509	# AES crypt partial
 510	xxlxor	32+15, 32+30, 0
 511	lwz	23, 240(6)	# n rounds
 512	addi	22, 23, -1	# loop - 1
 513	mtctr	22
 514	addi	10, 6, 16
 515
 516__Loop_aes_cpstate:
 517	lxv	32+1, 0(10)
 518	vcipher	15, 15, 1
 519	addi	10, 10, 16
 520	bdnz	__Loop_aes_cpstate
 521	lxv	32+1, 0(10)		# last round key
 522	vcipherlast 15, 15, 1
 523
 524	vxor	15, 15, 19
 525	vand	15, 15, 16
 526
 527	# AES crypt output v15
 528	# Write partial
 529	li	10, 224
 530	stxvb16x 15+32, 10, 1		# write v15 to stack
 531	addi	10, 1, 223
 532	add	10, 10, 12		# add offset
 533	addi	15, 9, -1
 534        mtctr	21			# partial block len
 535__Write_combine_partial:
 536        lbzu	22, 1(10)
 537	stbu	22, 1(15)
 538        bdnz	__Write_combine_partial
 539
 540	add	14, 14, 21
 541	add	11, 11, 21
 542	add	9, 9, 21
 543	sub	5, 5, 21
 544
 545	# Encrypt/Decrypt?
 546	cmpdi	24, 0			# decrypt?
 547	bne	__Encrypt_combine_partial
 548	vmr	15, 19		# decrypt using the input block
 549
 550__Encrypt_combine_partial:
 551	#
 552	# Update partial flag and combine ghash.
 553__Update_partial_ghash:
 554	li	10, 64
 555	lxvb16x 32+17, 10, 7		# load previous pblock
 556	add	12, 12, 21		# combined pprocessed
 557	vxor	15, 15, 17		# combined pblock
 558
 559	cmpdi	12, 16
 560	beq	__Clear_partial_flag
 561	std	12, 56(7)		# update partial len
 562	stxvb16x 32+15, 10, 7		# Update current pblock
 563	blr
 564
 565__Clear_partial_flag:
 566	li	12, 0
 567	std	12, 56(7)
 568	# Update IV and ghash here
 569	vadduwm	30, 30, 31		# increase IV
 570	stxvb16x 32+30, 0, 7		# update IV
 571
 572	# v15 either is either (input blockor encrypted)^(AES state)
 573	vxor	15, 15, 0
 574	PPC_GHASH1x 0, 15
 575	stxvb16x 32+0, 10, 7		# update pblock for debug?
 576	stxvb16x 32+0, 0, 8		# update Xi
 577	blr
 578SYM_FUNC_END(__Combine_partial)
 579
 580################################################################################
 581# gcm_update(iv, Xi) - compute last hash
 582#
 583################################################################################
 584SYM_FUNC_START(gcm_update)
 585
 586	ld	10, 56(3)
 587	cmpdi	10, 0
 588	beq	__no_update
 589
 590	lxvb16x	32, 0, 4	# load Xi
 591	# load Hash - h^4, h^3, h^2, h
 592	li	10, 32
 593	lxvd2x	2+32, 10, 4	# H Poli
 594	li	10, 48
 595	lxvd2x	3+32, 10, 4	# Hl
 596	li	10, 64
 597	lxvd2x	4+32, 10, 4	# H
 598	li	10, 80
 599	lxvd2x	5+32, 10, 4	# Hh
 600
 601	addis	11, 2, permx@toc@ha
 602	addi	11, 11, permx@toc@l
 603	lxv	10, 0(11)	# vs10: vpermxor vector
 604
 605	li	9, 64
 606	lxvb16x 32+6, 9, 3		# load pblock
 607	vxor	6, 6, 0
 608
 609	vxor	1, 1, 1
 610	vpmsumd	12, 3, 6		# L
 611	vpmsumd	13, 4, 6		# M
 612	vpmsumd	14, 5, 6		# H
 613	vpmsumd	17, 12, 2		# reduction
 614	vsldoi	15, 13, 1, 8		# mL
 615	vsldoi	16, 1, 13, 8		# mH
 616	vxor	12, 12, 15		# LL + LL
 617	vxor	14, 14, 16		# HH + HH
 618	xxlor	32+15, 10, 10
 619	vpermxor 12, 12, 17, 15
 620	vsldoi	13, 12, 12, 8		# swap
 621	vpmsumd	12, 12, 2		# reduction
 622	vxor	13, 13, 14
 623	vxor	7, 12, 13
 624
 625	#vxor	0, 0, 0
 626	#stxvb16x 32+0, 9, 3
 627	li	10, 0
 628	std	10, 56(3)
 629	stxvb16x 32+7, 0, 4
 630
 631__no_update:
 632	blr
 633SYM_FUNC_END(gcm_update)
 634
 635################################################################################
 636# aes_p10_gcm_encrypt (const void *inp, void *out, size_t len,
 637#               const char *rk, unsigned char iv[16], void *Xip);
 638#
 639#    r3 - inp
 640#    r4 - out
 641#    r5 - len
 642#    r6 - AES round keys
 643#    r7 - iv and other data
 644#    r8 - Xi, HPoli, hash keys
 645#
 646#    rounds is at offset 240 in rk
 647#    Xi is at 0 in gcm_table (Xip).
 648#
 649################################################################################
 650SYM_FUNC_START(aes_p10_gcm_encrypt)
 651
 652	cmpdi	5, 0
 653	ble	__Invalid_msg_len
 654
 655	SAVE_REGS
 656	LOAD_HASH_TABLE
 657
 658	# initialize ICB: GHASH( IV ), IV - r7
 659	lxvb16x	30+32, 0, 7	# load IV  - v30
 660
 661	mr	14, 3
 662	mr	9, 4
 663
 664	# counter 1
 665	vxor	31, 31, 31
 666	vspltisb 22, 1
 667	vsldoi	31, 31, 22,1	# counter 1
 668
 669	addis	11, 2, permx@toc@ha
 670	addi	11, 11, permx@toc@l
 671	lxv	10, 0(11)	# vs10: vpermxor vector
 672	li	11, 0
 673
 674	# load 9 round keys to VSR
 675	lxv	0, 0(6)			# round key 0
 676	lxv	1, 16(6)		# round key 1
 677	lxv	2, 32(6)		# round key 2
 678	lxv	3, 48(6)		# round key 3
 679	lxv	4, 64(6)		# round key 4
 680	lxv	5, 80(6)		# round key 5
 681	lxv	6, 96(6)		# round key 6
 682	lxv	7, 112(6)		# round key 7
 683	lxv	8, 128(6)		# round key 8
 684
 685	# load rounds - 10 (128), 12 (192), 14 (256)
 686	lwz	23, 240(6)		# n rounds
 687	li	24, 1			# encrypt
 688
 689__Process_encrypt:
 690	#
 691	# Process different blocks
 692	#
 693	ld	12, 56(7)
 694	cmpdi	12, 0
 695	bgt	__Do_combine_enc
 696	cmpdi	5, 128
 697	blt	__Process_more_enc
 698
 699#
 700# Process 8x AES/GCM blocks
 701#
 702__Process_8x_enc:
 703	# 8x blcoks
 704	li	10, 128
 705	divdu	12, 5, 10	# n 128 bytes-blocks
 706
 707	addi	12, 12, -1	# loop - 1
 708
 709	vmr	15, 30		# first state: IV
 710	vadduwm	16, 15, 31	# state + counter
 711	vadduwm	17, 16, 31
 712	vadduwm	18, 17, 31
 713	vadduwm	19, 18, 31
 714	vadduwm	20, 19, 31
 715	vadduwm	21, 20, 31
 716	vadduwm	22, 21, 31
 717	xxlor	9, 32+22, 32+22	# save last state
 718
 719	# vxor  state, state, w # addroundkey
 720	xxlor	32+29, 0, 0
 721        vxor    15, 15, 29      # IV + round key - add round key 0
 722	vxor	16, 16, 29
 723	vxor	17, 17, 29
 724	vxor	18, 18, 29
 725	vxor	19, 19, 29
 726	vxor	20, 20, 29
 727	vxor	21, 21, 29
 728	vxor	22, 22, 29
 729
 730	li	15, 16
 731	li	16, 32
 732	li	17, 48
 733	li	18, 64
 734	li	19, 80
 735	li	20, 96
 736	li	21, 112
 737
 738	#
 739	# Pre-compute first 8 AES state and leave 1/3/5 more rounds
 740	# for the loop.
 741	#
 742	addi	22, 23, -9		# process 8 keys
 743	mtctr	22			# AES key loop
 744	addi	10, 6, 144
 745
 746	LOOP_8AES_STATE			# process 8 AES keys
 747
 748__PreLoop_aes_state:
 749	lxv	32+1, 0(10)		# round key
 750	AES_CIPHER_8x vcipher 15 1
 751	addi	10, 10, 16
 752	bdnz	__PreLoop_aes_state
 753	lxv	32+1, 0(10)		# last round key (v1)
 754
 755	cmpdi	12, 0			# Only one loop (8 block)
 756	beq	__Finish_ghash
 757
 758#
 759# Loop 8x blocks and compute ghash
 760#
 761__Loop_8x_block_enc:
 762	vcipherlast     15, 15, 1
 763	vcipherlast     16, 16, 1
 764	vcipherlast     17, 17, 1
 765	vcipherlast     18, 18, 1
 766	vcipherlast     19, 19, 1
 767	vcipherlast     20, 20, 1
 768	vcipherlast     21, 21, 1
 769	vcipherlast     22, 22, 1
 770
 771	lxvb16x	32+23, 0, 14	# load block
 772	lxvb16x	32+24, 15, 14	# load block
 773	lxvb16x	32+25, 16, 14	# load block
 774	lxvb16x	32+26, 17, 14	# load block
 775	lxvb16x	32+27, 18, 14	# load block
 776	lxvb16x	32+28, 19, 14	# load block
 777	lxvb16x	32+29, 20, 14	# load block
 778	lxvb16x	32+30, 21, 14	# load block
 779	addi	14, 14, 128
 780
 781	vxor	15, 15, 23
 782	vxor	16, 16, 24
 783	vxor	17, 17, 25
 784	vxor	18, 18, 26
 785	vxor	19, 19, 27
 786	vxor	20, 20, 28
 787	vxor	21, 21, 29
 788	vxor	22, 22, 30
 789
 790	stxvb16x 47, 0, 9	# store output
 791	stxvb16x 48, 15, 9	# store output
 792	stxvb16x 49, 16, 9	# store output
 793	stxvb16x 50, 17, 9	# store output
 794	stxvb16x 51, 18, 9	# store output
 795	stxvb16x 52, 19, 9	# store output
 796	stxvb16x 53, 20, 9	# store output
 797	stxvb16x 54, 21, 9	# store output
 798	addi	9, 9, 128
 799
 800	# ghash here
 801	vxor	15, 15, 0
 802	PPC_GHASH4x 0, 15, 16, 17, 18
 803
 804	vxor	19, 19, 0
 805	PPC_GHASH4x 0, 19, 20, 21, 22
 806
 807	xxlor	32+15, 9, 9		# last state
 808	vadduwm 15, 15, 31		# state + counter
 809	vadduwm 16, 15, 31
 810	vadduwm 17, 16, 31
 811	vadduwm 18, 17, 31
 812	vadduwm 19, 18, 31
 813	vadduwm 20, 19, 31
 814	vadduwm 21, 20, 31
 815	vadduwm 22, 21, 31
 816	xxlor	9, 32+22, 32+22		# save last state
 817
 818	xxlor	32+27, 0, 0		# restore roundkey 0
 819        vxor    15, 15, 27		# IV + round key - add round key 0
 820	vxor	16, 16, 27
 821	vxor	17, 17, 27
 822	vxor	18, 18, 27
 823	vxor	19, 19, 27
 824	vxor	20, 20, 27
 825	vxor	21, 21, 27
 826	vxor	22, 22, 27
 827
 828	addi    5, 5, -128
 829	addi    11, 11, 128
 830
 831	LOOP_8AES_STATE			# process 8 AES keys
 832	mtctr	22			# AES key loop
 833	addi	10, 6, 144
 834__LastLoop_aes_state:
 835	lxv	32+1, 0(10)		# round key
 836	AES_CIPHER_8x vcipher 15 1
 837	addi	10, 10, 16
 838	bdnz	__LastLoop_aes_state
 839	lxv	32+1, 0(10)		# last round key (v1)
 840
 841	addi	12, 12, -1
 842	cmpdi	12, 0
 843	bne	__Loop_8x_block_enc
 844
 845__Finish_ghash:
 846	vcipherlast     15, 15, 1
 847	vcipherlast     16, 16, 1
 848	vcipherlast     17, 17, 1
 849	vcipherlast     18, 18, 1
 850	vcipherlast     19, 19, 1
 851	vcipherlast     20, 20, 1
 852	vcipherlast     21, 21, 1
 853	vcipherlast     22, 22, 1
 854
 855	lxvb16x	32+23, 0, 14	# load block
 856	lxvb16x	32+24, 15, 14	# load block
 857	lxvb16x	32+25, 16, 14	# load block
 858	lxvb16x	32+26, 17, 14	# load block
 859	lxvb16x	32+27, 18, 14	# load block
 860	lxvb16x	32+28, 19, 14	# load block
 861	lxvb16x	32+29, 20, 14	# load block
 862	lxvb16x	32+30, 21, 14	# load block
 863	addi	14, 14, 128
 864
 865	vxor	15, 15, 23
 866	vxor	16, 16, 24
 867	vxor	17, 17, 25
 868	vxor	18, 18, 26
 869	vxor	19, 19, 27
 870	vxor	20, 20, 28
 871	vxor	21, 21, 29
 872	vxor	22, 22, 30
 873
 874	stxvb16x 47, 0, 9	# store output
 875	stxvb16x 48, 15, 9	# store output
 876	stxvb16x 49, 16, 9	# store output
 877	stxvb16x 50, 17, 9	# store output
 878	stxvb16x 51, 18, 9	# store output
 879	stxvb16x 52, 19, 9	# store output
 880	stxvb16x 53, 20, 9	# store output
 881	stxvb16x 54, 21, 9	# store output
 882	addi	9, 9, 128
 883
 884	vxor	15, 15, 0
 885	PPC_GHASH4x 0, 15, 16, 17, 18
 886
 887	vxor	19, 19, 0
 888	PPC_GHASH4x 0, 19, 20, 21, 22
 889
 890	xxlor	30+32, 9, 9		# last ctr
 891	vadduwm	30, 30, 31		# increase ctr
 892	stxvb16x 32+30, 0, 7		# update IV
 893	stxvb16x 32+0, 0, 8		# update Xi
 894
 895	addi    5, 5, -128
 896	addi    11, 11, 128
 897
 898	#
 899	# Done 8x blocks
 900	#
 901
 902	cmpdi   5, 0
 903	beq     aes_gcm_out
 904
 905__Process_more_enc:
 906	li	24, 1			# encrypt
 907	bl	aes_gcm_crypt_1x
 908	cmpdi   5, 0
 909	beq     aes_gcm_out
 910
 911	bl	__Process_partial
 912	cmpdi   5, 0
 913	beq     aes_gcm_out
 914__Do_combine_enc:
 915	bl	__Combine_partial
 916	cmpdi	5, 0
 917	bgt	__Process_encrypt
 918	b	aes_gcm_out
 919
 920SYM_FUNC_END(aes_p10_gcm_encrypt)
 921
 922################################################################################
 923# aes_p10_gcm_decrypt (const void *inp, void *out, size_t len,
 924#               const char *rk, unsigned char iv[16], void *Xip);
 925# 8x Decrypt
 926#
 927################################################################################
 928SYM_FUNC_START(aes_p10_gcm_decrypt)
 929
 930	cmpdi	5, 0
 931	ble	__Invalid_msg_len
 932
 933	SAVE_REGS
 934	LOAD_HASH_TABLE
 935
 936	# initialize ICB: GHASH( IV ), IV - r7
 937	lxvb16x	30+32, 0, 7	# load IV  - v30
 938
 939	mr	14, 3
 940	mr	9, 4
 941
 942	# counter 1
 943	vxor	31, 31, 31
 944	vspltisb 22, 1
 945	vsldoi	31, 31, 22,1	# counter 1
 946
 947	addis	11, 2, permx@toc@ha
 948	addi	11, 11, permx@toc@l
 949	lxv	10, 0(11)	# vs10: vpermxor vector
 950	li	11, 0
 951
 952	# load 9 round keys to VSR
 953	lxv	0, 0(6)			# round key 0
 954	lxv	1, 16(6)		# round key 1
 955	lxv	2, 32(6)		# round key 2
 956	lxv	3, 48(6)		# round key 3
 957	lxv	4, 64(6)		# round key 4
 958	lxv	5, 80(6)		# round key 5
 959	lxv	6, 96(6)		# round key 6
 960	lxv	7, 112(6)		# round key 7
 961	lxv	8, 128(6)		# round key 8
 962
 963	# load rounds - 10 (128), 12 (192), 14 (256)
 964	lwz	23, 240(6)		# n rounds
 965	li	24, 0			# decrypt
 966
 967__Process_decrypt:
 968	#
 969	# Process different blocks
 970	#
 971	ld	12, 56(7)
 972	cmpdi	12, 0
 973	bgt	__Do_combine_dec
 974	cmpdi	5, 128
 975	blt	__Process_more_dec
 976
 977#
 978# Process 8x AES/GCM blocks
 979#
 980__Process_8x_dec:
 981	# 8x blcoks
 982	li	10, 128
 983	divdu	12, 5, 10	# n 128 bytes-blocks
 984
 985	addi	12, 12, -1	# loop - 1
 986
 987	vmr	15, 30		# first state: IV
 988	vadduwm	16, 15, 31	# state + counter
 989	vadduwm	17, 16, 31
 990	vadduwm	18, 17, 31
 991	vadduwm	19, 18, 31
 992	vadduwm	20, 19, 31
 993	vadduwm	21, 20, 31
 994	vadduwm	22, 21, 31
 995	xxlor	9, 32+22, 32+22	# save last state
 996
 997	# vxor  state, state, w # addroundkey
 998	xxlor	32+29, 0, 0
 999        vxor    15, 15, 29      # IV + round key - add round key 0
1000	vxor	16, 16, 29
1001	vxor	17, 17, 29
1002	vxor	18, 18, 29
1003	vxor	19, 19, 29
1004	vxor	20, 20, 29
1005	vxor	21, 21, 29
1006	vxor	22, 22, 29
1007
1008	li	15, 16
1009	li	16, 32
1010	li	17, 48
1011	li	18, 64
1012	li	19, 80
1013	li	20, 96
1014	li	21, 112
1015
1016	#
1017	# Pre-compute first 8 AES state and leave 1/3/5 more rounds
1018	# for the loop.
1019	#
1020	addi	22, 23, -9		# process 8 keys
1021	mtctr	22			# AES key loop
1022	addi	10, 6, 144
1023
1024	LOOP_8AES_STATE			# process 8 AES keys
1025
1026__PreLoop_aes_state_dec:
1027	lxv	32+1, 0(10)		# round key
1028	AES_CIPHER_8x vcipher 15 1
1029	addi	10, 10, 16
1030	bdnz	__PreLoop_aes_state_dec
1031	lxv	32+1, 0(10)		# last round key (v1)
1032
1033	cmpdi	12, 0			# Only one loop (8 block)
1034	beq	__Finish_ghash_dec
1035
1036#
1037# Loop 8x blocks and compute ghash
1038#
1039__Loop_8x_block_dec:
1040	vcipherlast     15, 15, 1
1041	vcipherlast     16, 16, 1
1042	vcipherlast     17, 17, 1
1043	vcipherlast     18, 18, 1
1044	vcipherlast     19, 19, 1
1045	vcipherlast     20, 20, 1
1046	vcipherlast     21, 21, 1
1047	vcipherlast     22, 22, 1
1048
1049	lxvb16x	32+23, 0, 14	# load block
1050	lxvb16x	32+24, 15, 14	# load block
1051	lxvb16x	32+25, 16, 14	# load block
1052	lxvb16x	32+26, 17, 14	# load block
1053	lxvb16x	32+27, 18, 14	# load block
1054	lxvb16x	32+28, 19, 14	# load block
1055	lxvb16x	32+29, 20, 14	# load block
1056	lxvb16x	32+30, 21, 14	# load block
1057	addi	14, 14, 128
1058
1059	vxor	15, 15, 23
1060	vxor	16, 16, 24
1061	vxor	17, 17, 25
1062	vxor	18, 18, 26
1063	vxor	19, 19, 27
1064	vxor	20, 20, 28
1065	vxor	21, 21, 29
1066	vxor	22, 22, 30
1067
1068	stxvb16x 47, 0, 9	# store output
1069	stxvb16x 48, 15, 9	# store output
1070	stxvb16x 49, 16, 9	# store output
1071	stxvb16x 50, 17, 9	# store output
1072	stxvb16x 51, 18, 9	# store output
1073	stxvb16x 52, 19, 9	# store output
1074	stxvb16x 53, 20, 9	# store output
1075	stxvb16x 54, 21, 9	# store output
1076
1077	addi	9, 9, 128
1078
1079	vmr	15, 23
1080	vmr	16, 24
1081	vmr	17, 25
1082	vmr	18, 26
1083	vmr	19, 27
1084	vmr	20, 28
1085	vmr	21, 29
1086	vmr	22, 30
1087
1088	# ghash here
1089	vxor	15, 15, 0
1090	PPC_GHASH4x 0, 15, 16, 17, 18
1091
1092	vxor	19, 19, 0
1093	PPC_GHASH4x 0, 19, 20, 21, 22
1094
1095	xxlor	32+15, 9, 9		# last state
1096	vadduwm 15, 15, 31		# state + counter
1097	vadduwm 16, 15, 31
1098	vadduwm 17, 16, 31
1099	vadduwm 18, 17, 31
1100	vadduwm 19, 18, 31
1101	vadduwm 20, 19, 31
1102	vadduwm 21, 20, 31
1103	vadduwm 22, 21, 31
1104	xxlor	9, 32+22, 32+22		# save last state
1105
1106	xxlor	32+27, 0, 0		# restore roundkey 0
1107        vxor    15, 15, 27		# IV + round key - add round key 0
1108	vxor	16, 16, 27
1109	vxor	17, 17, 27
1110	vxor	18, 18, 27
1111	vxor	19, 19, 27
1112	vxor	20, 20, 27
1113	vxor	21, 21, 27
1114	vxor	22, 22, 27
1115
1116	addi    5, 5, -128
1117	addi    11, 11, 128
1118
1119	LOOP_8AES_STATE			# process 8 AES keys
1120	mtctr	22			# AES key loop
1121	addi	10, 6, 144
1122__LastLoop_aes_state_dec:
1123	lxv	32+1, 0(10)		# round key
1124	AES_CIPHER_8x vcipher 15 1
1125	addi	10, 10, 16
1126	bdnz	__LastLoop_aes_state_dec
1127	lxv	32+1, 0(10)		# last round key (v1)
1128
1129	addi	12, 12, -1
1130	cmpdi	12, 0
1131	bne	__Loop_8x_block_dec
1132
1133__Finish_ghash_dec:
1134	vcipherlast     15, 15, 1
1135	vcipherlast     16, 16, 1
1136	vcipherlast     17, 17, 1
1137	vcipherlast     18, 18, 1
1138	vcipherlast     19, 19, 1
1139	vcipherlast     20, 20, 1
1140	vcipherlast     21, 21, 1
1141	vcipherlast     22, 22, 1
1142
1143	lxvb16x	32+23, 0, 14	# load block
1144	lxvb16x	32+24, 15, 14	# load block
1145	lxvb16x	32+25, 16, 14	# load block
1146	lxvb16x	32+26, 17, 14	# load block
1147	lxvb16x	32+27, 18, 14	# load block
1148	lxvb16x	32+28, 19, 14	# load block
1149	lxvb16x	32+29, 20, 14	# load block
1150	lxvb16x	32+30, 21, 14	# load block
1151	addi	14, 14, 128
1152
1153	vxor	15, 15, 23
1154	vxor	16, 16, 24
1155	vxor	17, 17, 25
1156	vxor	18, 18, 26
1157	vxor	19, 19, 27
1158	vxor	20, 20, 28
1159	vxor	21, 21, 29
1160	vxor	22, 22, 30
1161
1162	stxvb16x 47, 0, 9	# store output
1163	stxvb16x 48, 15, 9	# store output
1164	stxvb16x 49, 16, 9	# store output
1165	stxvb16x 50, 17, 9	# store output
1166	stxvb16x 51, 18, 9	# store output
1167	stxvb16x 52, 19, 9	# store output
1168	stxvb16x 53, 20, 9	# store output
1169	stxvb16x 54, 21, 9	# store output
1170	addi	9, 9, 128
1171
1172	#vmr	15, 23
1173	vxor	15, 23, 0
1174	vmr	16, 24
1175	vmr	17, 25
1176	vmr	18, 26
1177	vmr	19, 27
1178	vmr	20, 28
1179	vmr	21, 29
1180	vmr	22, 30
1181
1182	#vxor	15, 15, 0
1183	PPC_GHASH4x 0, 15, 16, 17, 18
1184
1185	vxor	19, 19, 0
1186	PPC_GHASH4x 0, 19, 20, 21, 22
1187
1188	xxlor	30+32, 9, 9		# last ctr
1189	vadduwm	30, 30, 31		# increase ctr
1190	stxvb16x 32+30, 0, 7		# update IV
1191	stxvb16x 32+0, 0, 8		# update Xi
1192
1193	addi    5, 5, -128
1194	addi    11, 11, 128
1195
1196	#
1197	# Done 8x blocks
1198	#
1199
1200	cmpdi   5, 0
1201	beq     aes_gcm_out
1202
1203__Process_more_dec:
1204	li	24, 0			# decrypt
1205	bl	aes_gcm_crypt_1x
1206	cmpdi   5, 0
1207	beq     aes_gcm_out
1208
1209	bl	__Process_partial
1210	cmpdi   5, 0
1211	beq     aes_gcm_out
1212__Do_combine_dec:
1213	bl	__Combine_partial
1214	cmpdi	5, 0
1215	bgt	__Process_decrypt
1216	b	aes_gcm_out
1217SYM_FUNC_END(aes_p10_gcm_decrypt)
1218
1219SYM_FUNC_START_LOCAL(aes_gcm_out)
1220
1221	mr	3, 11			# return count
1222
1223	RESTORE_REGS
1224	blr
1225
1226__Invalid_msg_len:
1227	li	3, 0
1228	blr
1229SYM_FUNC_END(aes_gcm_out)
1230
1231SYM_DATA_START_LOCAL(PERMX)
1232.align 4
1233# for vector permute and xor
1234permx:
1235.long 0x4c5d6e7f, 0x08192a3b, 0xc4d5e6f7, 0x8091a2b3
1236SYM_DATA_END(permx)