copypage_64.S

  1/* SPDX-License-Identifier: GPL-2.0-or-later */
  2/*
  3 * Copyright (C) 2008 Mark Nelson, IBM Corp.
  4 */
  5#include <linux/export.h>
  6#include <asm/page.h>
  7#include <asm/processor.h>
  8#include <asm/ppc_asm.h>
  9#include <asm/asm-offsets.h>
 10#include <asm/feature-fixups.h>
 11
 12_GLOBAL_TOC(copy_page)
 13BEGIN_FTR_SECTION
 14	lis	r5,PAGE_SIZE@h
 15FTR_SECTION_ELSE
 16#ifdef CONFIG_PPC_BOOK3S_64
 17	b	copypage_power7
 18#endif
 19ALT_FTR_SECTION_END_IFCLR(CPU_FTR_VMX_COPY)
 20	ori	r5,r5,PAGE_SIZE@l
 21#ifdef CONFIG_PPC_KERNEL_PCREL
 22	/*
 23	 * Hack for toolchain - prefixed instructions cause label difference to
 24	 * be non-constant even if 8 byte alignment is known, so they can not
 25	 * be put in FTR sections.
 26	 */
 27	LOAD_REG_ADDR(r10, ppc64_caches)
 28BEGIN_FTR_SECTION
 29#else
 30BEGIN_FTR_SECTION
 31	LOAD_REG_ADDR(r10, ppc64_caches)
 32#endif
 33	lwz	r11,DCACHEL1LOGBLOCKSIZE(r10)	/* log2 of cache block size */
 34	lwz     r12,DCACHEL1BLOCKSIZE(r10)	/* get cache block size */
 35	li	r9,0
 36	srd	r8,r5,r11
 37
 38	mtctr	r8
 39.Lsetup:
 40	dcbt	r9,r4
 41	dcbz	r9,r3
 42	add	r9,r9,r12
 43	bdnz	.Lsetup
 44END_FTR_SECTION_IFSET(CPU_FTR_CP_USE_DCBTZ)
 45	addi	r3,r3,-8
 46	srdi    r8,r5,7		/* page is copied in 128 byte strides */
 47	addi	r8,r8,-1	/* one stride copied outside loop */
 48
 49	mtctr	r8
 50
 51	ld	r5,0(r4)
 52	ld	r6,8(r4)
 53	ld	r7,16(r4)
 54	ldu	r8,24(r4)
 551:	std	r5,8(r3)
 56	std	r6,16(r3)
 57	ld	r9,8(r4)
 58	ld	r10,16(r4)
 59	std	r7,24(r3)
 60	std	r8,32(r3)
 61	ld	r11,24(r4)
 62	ld	r12,32(r4)
 63	std	r9,40(r3)
 64	std	r10,48(r3)
 65	ld	r5,40(r4)
 66	ld	r6,48(r4)
 67	std	r11,56(r3)
 68	std	r12,64(r3)
 69	ld	r7,56(r4)
 70	ld	r8,64(r4)
 71	std	r5,72(r3)
 72	std	r6,80(r3)
 73	ld	r9,72(r4)
 74	ld	r10,80(r4)
 75	std	r7,88(r3)
 76	std	r8,96(r3)
 77	ld	r11,88(r4)
 78	ld	r12,96(r4)
 79	std	r9,104(r3)
 80	std	r10,112(r3)
 81	ld	r5,104(r4)
 82	ld	r6,112(r4)
 83	std	r11,120(r3)
 84	stdu	r12,128(r3)
 85	ld	r7,120(r4)
 86	ldu	r8,128(r4)
 87	bdnz	1b
 88
 89	std	r5,8(r3)
 90	std	r6,16(r3)
 91	ld	r9,8(r4)
 92	ld	r10,16(r4)
 93	std	r7,24(r3)
 94	std	r8,32(r3)
 95	ld	r11,24(r4)
 96	ld	r12,32(r4)
 97	std	r9,40(r3)
 98	std	r10,48(r3)
 99	ld	r5,40(r4)
100	ld	r6,48(r4)
101	std	r11,56(r3)
102	std	r12,64(r3)
103	ld	r7,56(r4)
104	ld	r8,64(r4)
105	std	r5,72(r3)
106	std	r6,80(r3)
107	ld	r9,72(r4)
108	ld	r10,80(r4)
109	std	r7,88(r3)
110	std	r8,96(r3)
111	ld	r11,88(r4)
112	ld	r12,96(r4)
113	std	r9,104(r3)
114	std	r10,112(r3)
115	std	r11,120(r3)
116	std	r12,128(r3)
117	blr
118EXPORT_SYMBOL(copy_page)