patch-scs.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) 2022 - Google LLC
  4 * Author: Ard Biesheuvel <ardb@google.com>
  5 */
  6
  7#include <linux/errno.h>
  8#include <linux/init.h>
  9#include <linux/linkage.h>
 10#include <linux/types.h>
 11
 12#include <asm/scs.h>
 13
 14#include "pi.h"
 15
 16bool dynamic_scs_is_enabled;
 17
 18//
 19// This minimal DWARF CFI parser is partially based on the code in
 20// arch/arc/kernel/unwind.c, and on the document below:
 21// https://refspecs.linuxbase.org/LSB_4.0.0/LSB-Core-generic/LSB-Core-generic/ehframechpt.html
 22//
 23
 24#define DW_CFA_nop                          0x00
 25#define DW_CFA_set_loc                      0x01
 26#define DW_CFA_advance_loc1                 0x02
 27#define DW_CFA_advance_loc2                 0x03
 28#define DW_CFA_advance_loc4                 0x04
 29#define DW_CFA_offset_extended              0x05
 30#define DW_CFA_restore_extended             0x06
 31#define DW_CFA_undefined                    0x07
 32#define DW_CFA_same_value                   0x08
 33#define DW_CFA_register                     0x09
 34#define DW_CFA_remember_state               0x0a
 35#define DW_CFA_restore_state                0x0b
 36#define DW_CFA_def_cfa                      0x0c
 37#define DW_CFA_def_cfa_register             0x0d
 38#define DW_CFA_def_cfa_offset               0x0e
 39#define DW_CFA_def_cfa_expression           0x0f
 40#define DW_CFA_expression                   0x10
 41#define DW_CFA_offset_extended_sf           0x11
 42#define DW_CFA_def_cfa_sf                   0x12
 43#define DW_CFA_def_cfa_offset_sf            0x13
 44#define DW_CFA_val_offset                   0x14
 45#define DW_CFA_val_offset_sf                0x15
 46#define DW_CFA_val_expression               0x16
 47#define DW_CFA_lo_user                      0x1c
 48#define DW_CFA_negate_ra_state              0x2d
 49#define DW_CFA_GNU_args_size                0x2e
 50#define DW_CFA_GNU_negative_offset_extended 0x2f
 51#define DW_CFA_hi_user                      0x3f
 52
 53#define DW_EH_PE_sdata4                     0x0b
 54#define DW_EH_PE_sdata8                     0x0c
 55#define DW_EH_PE_pcrel                      0x10
 56
 57enum {
 58	PACIASP		= 0xd503233f,
 59	AUTIASP		= 0xd50323bf,
 60	SCS_PUSH	= 0xf800865e,
 61	SCS_POP		= 0xf85f8e5e,
 62};
 63
 64static void __always_inline scs_patch_loc(u64 loc)
 65{
 66	u32 insn = le32_to_cpup((void *)loc);
 67
 68	switch (insn) {
 69	case PACIASP:
 70		*(u32 *)loc = cpu_to_le32(SCS_PUSH);
 71		break;
 72	case AUTIASP:
 73		*(u32 *)loc = cpu_to_le32(SCS_POP);
 74		break;
 75	default:
 76		/*
 77		 * While the DW_CFA_negate_ra_state directive is guaranteed to
 78		 * appear right after a PACIASP/AUTIASP instruction, it may
 79		 * also appear after a DW_CFA_restore_state directive that
 80		 * restores a state that is only partially accurate, and is
 81		 * followed by DW_CFA_negate_ra_state directive to toggle the
 82		 * PAC bit again. So we permit other instructions here, and ignore
 83		 * them.
 84		 */
 85		return;
 86	}
 87	if (IS_ENABLED(CONFIG_ARM64_WORKAROUND_CLEAN_CACHE))
 88		asm("dc civac, %0" :: "r"(loc));
 89	else
 90		asm(ALTERNATIVE("dc cvau, %0", "nop", ARM64_HAS_CACHE_IDC)
 91		    :: "r"(loc));
 92}
 93
 94/*
 95 * Skip one uleb128/sleb128 encoded quantity from the opcode stream. All bytes
 96 * except the last one have bit #7 set.
 97 */
 98static int __always_inline skip_xleb128(const u8 **opcode, int size)
 99{
100	u8 c;
101
102	do {
103		c = *(*opcode)++;
104		size--;
105	} while (c & BIT(7));
106
107	return size;
108}
109
110struct eh_frame {
111	/*
112	 * The size of this frame if 0 < size < U32_MAX, 0 terminates the list.
113	 */
114	u32	size;
115
116	/*
117	 * The first frame is a Common Information Entry (CIE) frame, followed
118	 * by one or more Frame Description Entry (FDE) frames. In the former
119	 * case, this field is 0, otherwise it is the negated offset relative
120	 * to the associated CIE frame.
121	 */
122	u32	cie_id_or_pointer;
123
124	union {
125		struct { // CIE
126			u8	version;
127			u8	augmentation_string[3];
128			u8	code_alignment_factor;
129			u8	data_alignment_factor;
130			u8	return_address_register;
131			u8	augmentation_data_size;
132			u8	fde_pointer_format;
133		};
134
135		struct { // FDE
136			s32	initial_loc;
137			s32	range;
138			u8	opcodes[];
139		};
140
141		struct { // FDE
142			s64	initial_loc64;
143			s64	range64;
144			u8	opcodes64[];
145		};
146	};
147};
148
149static int scs_handle_fde_frame(const struct eh_frame *frame,
150				int code_alignment_factor,
151				bool use_sdata8,
152				bool dry_run)
153{
154	int size = frame->size - offsetof(struct eh_frame, opcodes) + 4;
155	u64 loc = (u64)offset_to_ptr(&frame->initial_loc);
156	const u8 *opcode = frame->opcodes;
157	int l;
158
159	if (use_sdata8) {
160		loc = (u64)&frame->initial_loc64 + frame->initial_loc64;
161		opcode = frame->opcodes64;
162		size -= 8;
163	}
164
165	// assume single byte uleb128_t for augmentation data size
166	if (*opcode & BIT(7))
167		return EDYNSCS_INVALID_FDE_AUGM_DATA_SIZE;
168
169	l = *opcode++;
170	opcode += l;
171	size -= l + 1;
172
173	/*
174	 * Starting from 'loc', apply the CFA opcodes that advance the location
175	 * pointer, and identify the locations of the PAC instructions.
176	 */
177	while (size-- > 0) {
178		switch (*opcode++) {
179		case DW_CFA_nop:
180		case DW_CFA_remember_state:
181		case DW_CFA_restore_state:
182			break;
183
184		case DW_CFA_advance_loc1:
185			loc += *opcode++ * code_alignment_factor;
186			size--;
187			break;
188
189		case DW_CFA_advance_loc2:
190			loc += *opcode++ * code_alignment_factor;
191			loc += (*opcode++ << 8) * code_alignment_factor;
192			size -= 2;
193			break;
194
195		case DW_CFA_advance_loc4:
196			loc += *opcode++ * code_alignment_factor;
197			loc += (*opcode++ << 8) * code_alignment_factor;
198			loc += (*opcode++ << 16) * code_alignment_factor;
199			loc += (*opcode++ << 24) * code_alignment_factor;
200			size -= 4;
201		break;
202
203		case DW_CFA_def_cfa:
204		case DW_CFA_offset_extended:
205			size = skip_xleb128(&opcode, size);
206			fallthrough;
207		case DW_CFA_def_cfa_offset:
208		case DW_CFA_def_cfa_offset_sf:
209		case DW_CFA_def_cfa_register:
210		case DW_CFA_same_value:
211		case DW_CFA_restore_extended:
212		case 0x80 ... 0xbf:
213			size = skip_xleb128(&opcode, size);
214			break;
215
216		case DW_CFA_negate_ra_state:
217			if (!dry_run)
218				scs_patch_loc(loc - 4);
219			break;
220
221		case 0x40 ... 0x7f:
222			// advance loc
223			loc += (opcode[-1] & 0x3f) * code_alignment_factor;
224			break;
225
226		case 0xc0 ... 0xff:
227			break;
228
229		default:
230			return EDYNSCS_INVALID_CFA_OPCODE;
231		}
232	}
233	return 0;
234}
235
236int scs_patch(const u8 eh_frame[], int size, bool skip_dry_run)
237{
238	int code_alignment_factor = 1;
239	bool fde_use_sdata8 = false;
240	const u8 *p = eh_frame;
241
242	while (size > 4) {
243		const struct eh_frame *frame = (const void *)p;
244		int ret;
245
246		if (frame->size == 0 ||
247		    frame->size == U32_MAX ||
248		    frame->size > size)
249			break;
250
251		if (frame->cie_id_or_pointer == 0) {
252			/*
253			 * Require presence of augmentation data (z) with a
254			 * specifier for the size of the FDE initial_loc and
255			 * range fields (R), and nothing else.
256			 */
257			if (strcmp(frame->augmentation_string, "zR"))
258				return EDYNSCS_INVALID_CIE_HEADER;
259
260			/*
261			 * The code alignment factor is a uleb128 encoded field
262			 * but given that the only sensible values are 1 or 4,
263			 * there is no point in decoding the whole thing.  Also
264			 * sanity check the size of the data alignment factor
265			 * field, and the values of the return address register
266			 * and augmentation data size fields.
267			 */
268			if ((frame->code_alignment_factor & BIT(7)) ||
269			    (frame->data_alignment_factor & BIT(7)) ||
270			    frame->return_address_register != 30 ||
271			    frame->augmentation_data_size != 1)
272				return EDYNSCS_INVALID_CIE_HEADER;
273
274			code_alignment_factor = frame->code_alignment_factor;
275
276			switch (frame->fde_pointer_format) {
277			case DW_EH_PE_pcrel | DW_EH_PE_sdata4:
278				fde_use_sdata8 = false;
279				break;
280			case DW_EH_PE_pcrel | DW_EH_PE_sdata8:
281				fde_use_sdata8 = true;
282				break;
283			default:
284				return EDYNSCS_INVALID_CIE_SDATA_SIZE;
285			}
286		} else {
287			ret = scs_handle_fde_frame(frame, code_alignment_factor,
288						   fde_use_sdata8, !skip_dry_run);
289			if (ret)
290				return ret;
291
292			if (!skip_dry_run)
293				scs_handle_fde_frame(frame, code_alignment_factor,
294						     fde_use_sdata8, false);
295		}
296
297		p += sizeof(frame->size) + frame->size;
298		size -= sizeof(frame->size) + frame->size;
299	}
300	return 0;
301}