mte.S

  1/* SPDX-License-Identifier: GPL-2.0-only */
  2/*
  3 * Copyright (C) 2020 ARM Ltd.
  4 */
  5#include <linux/linkage.h>
  6
  7#include <asm/asm-uaccess.h>
  8#include <asm/assembler.h>
  9#include <asm/mte.h>
 10#include <asm/page.h>
 11#include <asm/sysreg.h>
 12
 13	.arch	armv8.5-a+memtag
 14
 15/*
 16 * multitag_transfer_size - set \reg to the block size that is accessed by the
 17 * LDGM/STGM instructions.
 18 */
 19	.macro	multitag_transfer_size, reg, tmp
 20	mrs_s	\reg, SYS_GMID_EL1
 21	ubfx	\reg, \reg, #GMID_EL1_BS_SHIFT, #GMID_EL1_BS_WIDTH
 22	mov	\tmp, #4
 23	lsl	\reg, \tmp, \reg
 24	.endm
 25
 26/*
 27 * Clear the tags in a page
 28 *   x0 - address of the page to be cleared
 29 */
 30SYM_FUNC_START(mte_clear_page_tags)
 31	multitag_transfer_size x1, x2
 321:	stgm	xzr, [x0]
 33	add	x0, x0, x1
 34	tst	x0, #(PAGE_SIZE - 1)
 35	b.ne	1b
 36	ret
 37SYM_FUNC_END(mte_clear_page_tags)
 38
 39/*
 40 * Zero the page and tags at the same time
 41 *
 42 * Parameters:
 43 *	x0 - address to the beginning of the page
 44 */
 45SYM_FUNC_START(mte_zero_clear_page_tags)
 46	and	x0, x0, #(1 << MTE_TAG_SHIFT) - 1	// clear the tag
 47	mrs	x1, dczid_el0
 48	tbnz	x1, #4, 2f	// Branch if DC GZVA is prohibited
 49	and	w1, w1, #0xf
 50	mov	x2, #4
 51	lsl	x1, x2, x1
 52
 531:	dc	gzva, x0
 54	add	x0, x0, x1
 55	tst	x0, #(PAGE_SIZE - 1)
 56	b.ne	1b
 57	ret
 58
 592:	stz2g	x0, [x0], #(MTE_GRANULE_SIZE * 2)
 60	tst	x0, #(PAGE_SIZE - 1)
 61	b.ne	2b
 62	ret
 63SYM_FUNC_END(mte_zero_clear_page_tags)
 64
 65/*
 66 * Copy the tags from the source page to the destination one
 67 *   x0 - address of the destination page
 68 *   x1 - address of the source page
 69 */
 70SYM_FUNC_START(mte_copy_page_tags)
 71	mov	x2, x0
 72	mov	x3, x1
 73	multitag_transfer_size x5, x6
 741:	ldgm	x4, [x3]
 75	stgm	x4, [x2]
 76	add	x2, x2, x5
 77	add	x3, x3, x5
 78	tst	x2, #(PAGE_SIZE - 1)
 79	b.ne	1b
 80	ret
 81SYM_FUNC_END(mte_copy_page_tags)
 82
 83/*
 84 * Read tags from a user buffer (one tag per byte) and set the corresponding
 85 * tags at the given kernel address. Used by PTRACE_POKEMTETAGS.
 86 *   x0 - kernel address (to)
 87 *   x1 - user buffer (from)
 88 *   x2 - number of tags/bytes (n)
 89 * Returns:
 90 *   x0 - number of tags read/set
 91 */
 92SYM_FUNC_START(mte_copy_tags_from_user)
 93	mov	x3, x1
 94	cbz	x2, 2f
 951:
 96USER(2f, ldtrb	w4, [x1])
 97	lsl	x4, x4, #MTE_TAG_SHIFT
 98	stg	x4, [x0], #MTE_GRANULE_SIZE
 99	add	x1, x1, #1
100	subs	x2, x2, #1
101	b.ne	1b
102
103	// exception handling and function return
1042:	sub	x0, x1, x3		// update the number of tags set
105	ret
106SYM_FUNC_END(mte_copy_tags_from_user)
107
108/*
109 * Get the tags from a kernel address range and write the tag values to the
110 * given user buffer (one tag per byte). Used by PTRACE_PEEKMTETAGS.
111 *   x0 - user buffer (to)
112 *   x1 - kernel address (from)
113 *   x2 - number of tags/bytes (n)
114 * Returns:
115 *   x0 - number of tags read/set
116 */
117SYM_FUNC_START(mte_copy_tags_to_user)
118	mov	x3, x0
119	cbz	x2, 2f
1201:
121	ldg	x4, [x1]
122	ubfx	x4, x4, #MTE_TAG_SHIFT, #MTE_TAG_SIZE
123USER(2f, sttrb	w4, [x0])
124	add	x0, x0, #1
125	add	x1, x1, #MTE_GRANULE_SIZE
126	subs	x2, x2, #1
127	b.ne	1b
128
129	// exception handling and function return
1302:	sub	x0, x0, x3		// update the number of tags copied
131	ret
132SYM_FUNC_END(mte_copy_tags_to_user)
133
134/*
135 * Save the tags in a page
136 *   x0 - page address
137 *   x1 - tag storage, MTE_PAGE_TAG_STORAGE bytes
138 */
139SYM_FUNC_START(mte_save_page_tags)
140	multitag_transfer_size x7, x5
1411:
142	mov	x2, #0
1432:
144	ldgm	x5, [x0]
145	orr	x2, x2, x5
146	add	x0, x0, x7
147	tst	x0, #0xFF		// 16 tag values fit in a register,
148	b.ne	2b			// which is 16*16=256 bytes
149
150	str	x2, [x1], #8
151
152	tst	x0, #(PAGE_SIZE - 1)
153	b.ne	1b
154
155	ret
156SYM_FUNC_END(mte_save_page_tags)
157
158/*
159 * Restore the tags in a page
160 *   x0 - page address
161 *   x1 - tag storage, MTE_PAGE_TAG_STORAGE bytes
162 */
163SYM_FUNC_START(mte_restore_page_tags)
164	multitag_transfer_size x7, x5
1651:
166	ldr	x2, [x1], #8
1672:
168	stgm	x2, [x0]
169	add	x0, x0, x7
170	tst	x0, #0xFF
171	b.ne	2b
172
173	tst	x0, #(PAGE_SIZE - 1)
174	b.ne	1b
175
176	ret
177SYM_FUNC_END(mte_restore_page_tags)