ftrace.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Code for replacing ftrace calls with jumps.
  4 *
  5 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
  6 *
  7 * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box.
  8 *
  9 * Added function graph tracer code, taken from x86 that was written
 10 * by Frederic Weisbecker, and ported to PPC by Steven Rostedt.
 11 *
 12 */
 13
 14#define pr_fmt(fmt) "ftrace-powerpc: " fmt
 15
 16#include <linux/spinlock.h>
 17#include <linux/hardirq.h>
 18#include <linux/uaccess.h>
 19#include <linux/module.h>
 20#include <linux/ftrace.h>
 21#include <linux/percpu.h>
 22#include <linux/init.h>
 23#include <linux/list.h>
 24
 25#include <asm/cacheflush.h>
 26#include <asm/text-patching.h>
 27#include <asm/ftrace.h>
 28#include <asm/syscall.h>
 29#include <asm/inst.h>
 30#include <asm/sections.h>
 31
 32#define	NUM_FTRACE_TRAMPS	2
 33static unsigned long ftrace_tramps[NUM_FTRACE_TRAMPS];
 34
 35unsigned long ftrace_call_adjust(unsigned long addr)
 36{
 37	if (addr >= (unsigned long)__exittext_begin && addr < (unsigned long)__exittext_end)
 38		return 0;
 39
 40	if (IS_ENABLED(CONFIG_ARCH_USING_PATCHABLE_FUNCTION_ENTRY) &&
 41	    !IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) {
 42		addr += MCOUNT_INSN_SIZE;
 43		if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS))
 44			addr += MCOUNT_INSN_SIZE;
 45	}
 46
 47	return addr;
 48}
 49
 50static ppc_inst_t ftrace_create_branch_inst(unsigned long ip, unsigned long addr, int link)
 51{
 52	ppc_inst_t op;
 53
 54	WARN_ON(!is_offset_in_branch_range(addr - ip));
 55	create_branch(&op, (u32 *)ip, addr, link ? BRANCH_SET_LINK : 0);
 56
 57	return op;
 58}
 59
 60static inline int ftrace_read_inst(unsigned long ip, ppc_inst_t *op)
 61{
 62	if (copy_inst_from_kernel_nofault(op, (void *)ip)) {
 63		pr_err("0x%lx: fetching instruction failed\n", ip);
 64		return -EFAULT;
 65	}
 66
 67	return 0;
 68}
 69
 70static inline int ftrace_validate_inst(unsigned long ip, ppc_inst_t inst)
 71{
 72	ppc_inst_t op;
 73	int ret;
 74
 75	ret = ftrace_read_inst(ip, &op);
 76	if (!ret && !ppc_inst_equal(op, inst)) {
 77		pr_err("0x%lx: expected (%08lx) != found (%08lx)\n",
 78		       ip, ppc_inst_as_ulong(inst), ppc_inst_as_ulong(op));
 79		ret = -EINVAL;
 80	}
 81
 82	return ret;
 83}
 84
 85static inline int ftrace_modify_code(unsigned long ip, ppc_inst_t old, ppc_inst_t new)
 86{
 87	int ret = ftrace_validate_inst(ip, old);
 88
 89	if (!ret && !ppc_inst_equal(old, new))
 90		ret = patch_instruction((u32 *)ip, new);
 91
 92	return ret;
 93}
 94
 95static int is_bl_op(ppc_inst_t op)
 96{
 97	return (ppc_inst_val(op) & ~PPC_LI_MASK) == PPC_RAW_BL(0);
 98}
 99
100static unsigned long find_ftrace_tramp(unsigned long ip)
101{
102	int i;
103
104	for (i = 0; i < NUM_FTRACE_TRAMPS; i++)
105		if (!ftrace_tramps[i])
106			continue;
107		else if (is_offset_in_branch_range(ftrace_tramps[i] - ip))
108			return ftrace_tramps[i];
109
110	return 0;
111}
112
113#ifdef CONFIG_MODULES
114static unsigned long ftrace_lookup_module_stub(unsigned long ip, unsigned long addr)
115{
116	struct module *mod = NULL;
117
118	scoped_guard(rcu)
119		mod = __module_text_address(ip);
120	if (!mod)
121		pr_err("No module loaded at addr=%lx\n", ip);
122
123	return (addr == (unsigned long)ftrace_caller ? mod->arch.tramp : mod->arch.tramp_regs);
124}
125#else
126static unsigned long ftrace_lookup_module_stub(unsigned long ip, unsigned long addr)
127{
128	return 0;
129}
130#endif
131
132static unsigned long ftrace_get_ool_stub(struct dyn_ftrace *rec)
133{
134#ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE
135	return rec->arch.ool_stub;
136#else
137	BUILD_BUG();
138#endif
139}
140
141static int ftrace_get_call_inst(struct dyn_ftrace *rec, unsigned long addr, ppc_inst_t *call_inst)
142{
143	unsigned long ip;
144	unsigned long stub;
145
146	if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE))
147		ip = ftrace_get_ool_stub(rec) + MCOUNT_INSN_SIZE; /* second instruction in stub */
148	else
149		ip = rec->ip;
150
151	if (!is_offset_in_branch_range(addr - ip) && addr != FTRACE_ADDR &&
152	    addr != FTRACE_REGS_ADDR) {
153		/* This can only happen with ftrace direct */
154		if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS)) {
155			pr_err("0x%lx (0x%lx): Unexpected target address 0x%lx\n",
156			       ip, rec->ip, addr);
157			return -EINVAL;
158		}
159		addr = FTRACE_ADDR;
160	}
161
162	if (is_offset_in_branch_range(addr - ip))
163		/* Within range */
164		stub = addr;
165	else if (core_kernel_text(ip))
166		/* We would be branching to one of our ftrace stubs */
167		stub = find_ftrace_tramp(ip);
168	else
169		stub = ftrace_lookup_module_stub(ip, addr);
170
171	if (!stub) {
172		pr_err("0x%lx (0x%lx): No ftrace stubs reachable\n", ip, rec->ip);
173		return -EINVAL;
174	}
175
176	*call_inst = ftrace_create_branch_inst(ip, stub, 1);
177	return 0;
178}
179
180static int ftrace_init_ool_stub(struct module *mod, struct dyn_ftrace *rec)
181{
182#ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE
183	static int ool_stub_text_index, ool_stub_text_end_index, ool_stub_inittext_index;
184	int ret = 0, ool_stub_count, *ool_stub_index;
185	ppc_inst_t inst;
186	/*
187	 * See ftrace_entry.S if changing the below instruction sequence, as we rely on
188	 * decoding the last branch instruction here to recover the correct function ip.
189	 */
190	struct ftrace_ool_stub *ool_stub, ool_stub_template = {
191		.insn = {
192			PPC_RAW_MFLR(_R0),
193			PPC_RAW_NOP(),		/* bl ftrace_caller */
194			PPC_RAW_MTLR(_R0),
195			PPC_RAW_NOP()		/* b rec->ip + 4 */
196		}
197	};
198
199	WARN_ON(rec->arch.ool_stub);
200
201	if (is_kernel_inittext(rec->ip)) {
202		ool_stub = ftrace_ool_stub_inittext;
203		ool_stub_index = &ool_stub_inittext_index;
204		ool_stub_count = ftrace_ool_stub_inittext_count;
205	} else if (is_kernel_text(rec->ip)) {
206		/*
207		 * ftrace records are sorted, so we first use up the stub area within .text
208		 * (ftrace_ool_stub_text) before using the area at the end of .text
209		 * (ftrace_ool_stub_text_end), unless the stub is out of range of the record.
210		 */
211		if (ool_stub_text_index >= ftrace_ool_stub_text_count ||
212		    !is_offset_in_branch_range((long)rec->ip -
213					       (long)&ftrace_ool_stub_text[ool_stub_text_index])) {
214			ool_stub = ftrace_ool_stub_text_end;
215			ool_stub_index = &ool_stub_text_end_index;
216			ool_stub_count = ftrace_ool_stub_text_end_count;
217		} else {
218			ool_stub = ftrace_ool_stub_text;
219			ool_stub_index = &ool_stub_text_index;
220			ool_stub_count = ftrace_ool_stub_text_count;
221		}
222#ifdef CONFIG_MODULES
223	} else if (mod) {
224		ool_stub = mod->arch.ool_stubs;
225		ool_stub_index = &mod->arch.ool_stub_index;
226		ool_stub_count = mod->arch.ool_stub_count;
227#endif
228	} else {
229		return -EINVAL;
230	}
231
232	ool_stub += (*ool_stub_index)++;
233
234	if (WARN_ON(*ool_stub_index > ool_stub_count))
235		return -EINVAL;
236
237	if (!is_offset_in_branch_range((long)rec->ip - (long)&ool_stub->insn[0]) ||
238	    !is_offset_in_branch_range((long)(rec->ip + MCOUNT_INSN_SIZE) -
239				       (long)&ool_stub->insn[3])) {
240		pr_err("%s: ftrace ool stub out of range (%p -> %p).\n",
241					__func__, (void *)rec->ip, (void *)&ool_stub->insn[0]);
242		return -EINVAL;
243	}
244
245	rec->arch.ool_stub = (unsigned long)&ool_stub->insn[0];
246
247	/* bl ftrace_caller */
248	if (!mod)
249		ret = ftrace_get_call_inst(rec, (unsigned long)ftrace_caller, &inst);
250#ifdef CONFIG_MODULES
251	else
252		/*
253		 * We can't use ftrace_get_call_inst() since that uses
254		 * __module_text_address(rec->ip) to look up the module.
255		 * But, since the module is not fully formed at this stage,
256		 * the lookup fails. We know the target though, so generate
257		 * the branch inst directly.
258		 */
259		inst = ftrace_create_branch_inst(ftrace_get_ool_stub(rec) + MCOUNT_INSN_SIZE,
260						 mod->arch.tramp, 1);
261#endif
262	ool_stub_template.insn[1] = ppc_inst_val(inst);
263
264	/* b rec->ip + 4 */
265	if (!ret && create_branch(&inst, &ool_stub->insn[3], rec->ip + MCOUNT_INSN_SIZE, 0))
266		return -EINVAL;
267	ool_stub_template.insn[3] = ppc_inst_val(inst);
268
269	if (!ret)
270		ret = patch_instructions((u32 *)ool_stub, (u32 *)&ool_stub_template,
271					 sizeof(ool_stub_template), false);
272
273	return ret;
274#else /* !CONFIG_PPC_FTRACE_OUT_OF_LINE */
275	BUILD_BUG();
276#endif
277}
278
279#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
280static const struct ftrace_ops *powerpc_rec_get_ops(struct dyn_ftrace *rec)
281{
282	const struct ftrace_ops *ops = NULL;
283
284	if (rec->flags & FTRACE_FL_CALL_OPS_EN) {
285		ops = ftrace_find_unique_ops(rec);
286		WARN_ON_ONCE(!ops);
287	}
288
289	if (!ops)
290		ops = &ftrace_list_ops;
291
292	return ops;
293}
294
295static int ftrace_rec_set_ops(struct dyn_ftrace *rec, const struct ftrace_ops *ops)
296{
297	if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE))
298		return patch_ulong((void *)(ftrace_get_ool_stub(rec) - sizeof(unsigned long)),
299				   (unsigned long)ops);
300	else
301		return patch_ulong((void *)(rec->ip - MCOUNT_INSN_SIZE - sizeof(unsigned long)),
302				   (unsigned long)ops);
303}
304
305static int ftrace_rec_set_nop_ops(struct dyn_ftrace *rec)
306{
307	return ftrace_rec_set_ops(rec, &ftrace_nop_ops);
308}
309
310static int ftrace_rec_update_ops(struct dyn_ftrace *rec)
311{
312	return ftrace_rec_set_ops(rec, powerpc_rec_get_ops(rec));
313}
314#else
315static int ftrace_rec_set_nop_ops(struct dyn_ftrace *rec) { return 0; }
316static int ftrace_rec_update_ops(struct dyn_ftrace *rec) { return 0; }
317#endif
318
319#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
320int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr)
321{
322	/* This should never be called since we override ftrace_replace_code() */
323	WARN_ON(1);
324	return -EINVAL;
325}
326#endif
327
328int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
329{
330	ppc_inst_t old, new;
331	unsigned long ip = rec->ip;
332	int ret = 0;
333
334	/* This can only ever be called during module load */
335	if (WARN_ON(!IS_ENABLED(CONFIG_MODULES) || core_kernel_text(ip)))
336		return -EINVAL;
337
338	old = ppc_inst(PPC_RAW_NOP());
339	if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) {
340		ip = ftrace_get_ool_stub(rec) + MCOUNT_INSN_SIZE; /* second instruction in stub */
341		ret = ftrace_get_call_inst(rec, (unsigned long)ftrace_caller, &old);
342	}
343
344	ret |= ftrace_get_call_inst(rec, addr, &new);
345
346	if (!ret)
347		ret = ftrace_modify_code(ip, old, new);
348
349	ret = ftrace_rec_update_ops(rec);
350	if (ret)
351		return ret;
352
353	if (!ret && IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE))
354		ret = ftrace_modify_code(rec->ip, ppc_inst(PPC_RAW_NOP()),
355			 ppc_inst(PPC_RAW_BRANCH((long)ftrace_get_ool_stub(rec) - (long)rec->ip)));
356
357	return ret;
358}
359
360int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long addr)
361{
362	/*
363	 * This should never be called since we override ftrace_replace_code(),
364	 * as well as ftrace_init_nop()
365	 */
366	WARN_ON(1);
367	return -EINVAL;
368}
369
370void ftrace_replace_code(int enable)
371{
372	ppc_inst_t old, new, call_inst, new_call_inst;
373	ppc_inst_t nop_inst = ppc_inst(PPC_RAW_NOP());
374	unsigned long ip, new_addr, addr;
375	struct ftrace_rec_iter *iter;
376	struct dyn_ftrace *rec;
377	int ret = 0, update;
378
379	for_ftrace_rec_iter(iter) {
380		rec = ftrace_rec_iter_record(iter);
381		ip = rec->ip;
382
383		if (rec->flags & FTRACE_FL_DISABLED && !(rec->flags & FTRACE_FL_ENABLED))
384			continue;
385
386		addr = ftrace_get_addr_curr(rec);
387		new_addr = ftrace_get_addr_new(rec);
388		update = ftrace_update_record(rec, enable);
389
390		if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE) && update != FTRACE_UPDATE_IGNORE) {
391			ip = ftrace_get_ool_stub(rec) + MCOUNT_INSN_SIZE;
392			ret = ftrace_get_call_inst(rec, (unsigned long)ftrace_caller, &nop_inst);
393			if (ret)
394				goto out;
395		}
396
397		switch (update) {
398		case FTRACE_UPDATE_IGNORE:
399		default:
400			continue;
401		case FTRACE_UPDATE_MODIFY_CALL:
402			ret = ftrace_get_call_inst(rec, new_addr, &new_call_inst);
403			ret |= ftrace_get_call_inst(rec, addr, &call_inst);
404			ret |= ftrace_rec_update_ops(rec);
405			old = call_inst;
406			new = new_call_inst;
407			break;
408		case FTRACE_UPDATE_MAKE_NOP:
409			ret = ftrace_get_call_inst(rec, addr, &call_inst);
410			ret |= ftrace_rec_set_nop_ops(rec);
411			old = call_inst;
412			new = nop_inst;
413			break;
414		case FTRACE_UPDATE_MAKE_CALL:
415			ret = ftrace_get_call_inst(rec, new_addr, &call_inst);
416			ret |= ftrace_rec_update_ops(rec);
417			old = nop_inst;
418			new = call_inst;
419			break;
420		}
421
422		if (!ret)
423			ret = ftrace_modify_code(ip, old, new);
424
425		if (!ret && IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE) &&
426		    (update == FTRACE_UPDATE_MAKE_NOP || update == FTRACE_UPDATE_MAKE_CALL)) {
427			/* Update the actual ftrace location */
428			call_inst = ppc_inst(PPC_RAW_BRANCH((long)ftrace_get_ool_stub(rec) -
429							    (long)rec->ip));
430			nop_inst = ppc_inst(PPC_RAW_NOP());
431			ip = rec->ip;
432
433			if (update == FTRACE_UPDATE_MAKE_NOP)
434				ret = ftrace_modify_code(ip, call_inst, nop_inst);
435			else
436				ret = ftrace_modify_code(ip, nop_inst, call_inst);
437
438			if (ret)
439				goto out;
440		}
441
442		if (ret)
443			goto out;
444	}
445
446out:
447	if (ret)
448		ftrace_bug(ret, rec);
449	return;
450}
451
452int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec)
453{
454	unsigned long addr, ip = rec->ip;
455	ppc_inst_t old, new;
456	int ret = 0;
457
458	/* Verify instructions surrounding the ftrace location */
459	if (IS_ENABLED(CONFIG_ARCH_USING_PATCHABLE_FUNCTION_ENTRY)) {
460		/* Expect nops */
461		if (!IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE))
462			ret = ftrace_validate_inst(ip - 4, ppc_inst(PPC_RAW_NOP()));
463		if (!ret)
464			ret = ftrace_validate_inst(ip, ppc_inst(PPC_RAW_NOP()));
465	} else if (IS_ENABLED(CONFIG_PPC32)) {
466		/* Expected sequence: 'mflr r0', 'stw r0,4(r1)', 'bl _mcount' */
467		ret = ftrace_validate_inst(ip - 8, ppc_inst(PPC_RAW_MFLR(_R0)));
468		if (ret)
469			return ret;
470		ret = ftrace_modify_code(ip - 4, ppc_inst(PPC_RAW_STW(_R0, _R1, 4)),
471					 ppc_inst(PPC_RAW_NOP()));
472	} else if (IS_ENABLED(CONFIG_MPROFILE_KERNEL)) {
473		/* Expected sequence: 'mflr r0', ['std r0,16(r1)'], 'bl _mcount' */
474		ret = ftrace_read_inst(ip - 4, &old);
475		if (!ret && !ppc_inst_equal(old, ppc_inst(PPC_RAW_MFLR(_R0)))) {
476			/* Gcc v5.x emit the additional 'std' instruction, gcc v6.x don't */
477			ret = ftrace_validate_inst(ip - 8, ppc_inst(PPC_RAW_MFLR(_R0)));
478			if (ret)
479				return ret;
480			ret = ftrace_modify_code(ip - 4, ppc_inst(PPC_RAW_STD(_R0, _R1, 16)),
481						 ppc_inst(PPC_RAW_NOP()));
482		}
483	} else {
484		return -EINVAL;
485	}
486
487	if (ret)
488		return ret;
489
490	/* Set up out-of-line stub */
491	if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) {
492		ret = ftrace_init_ool_stub(mod, rec);
493		goto out;
494	}
495
496	/* Nop-out the ftrace location */
497	new = ppc_inst(PPC_RAW_NOP());
498	addr = MCOUNT_ADDR;
499	if (IS_ENABLED(CONFIG_ARCH_USING_PATCHABLE_FUNCTION_ENTRY)) {
500		/* we instead patch-in the 'mflr r0' */
501		old = ppc_inst(PPC_RAW_NOP());
502		new = ppc_inst(PPC_RAW_MFLR(_R0));
503		ret = ftrace_modify_code(ip - 4, old, new);
504	} else if (is_offset_in_branch_range(addr - ip)) {
505		/* Within range */
506		old = ftrace_create_branch_inst(ip, addr, 1);
507		ret = ftrace_modify_code(ip, old, new);
508	} else if (core_kernel_text(ip) || (IS_ENABLED(CONFIG_MODULES) && mod)) {
509		/*
510		 * We would be branching to a linker-generated stub, or to the module _mcount
511		 * stub. Let's just confirm we have a 'bl' here.
512		 */
513		ret = ftrace_read_inst(ip, &old);
514		if (ret)
515			return ret;
516		if (!is_bl_op(old)) {
517			pr_err("0x%lx: expected (bl) != found (%08lx)\n", ip, ppc_inst_as_ulong(old));
518			return -EINVAL;
519		}
520		ret = patch_instruction((u32 *)ip, new);
521	} else {
522		return -EINVAL;
523	}
524
525out:
526	if (!ret)
527		ret = ftrace_rec_set_nop_ops(rec);
528
529	return ret;
530}
531
532int ftrace_update_ftrace_func(ftrace_func_t func)
533{
534	unsigned long ip = (unsigned long)(&ftrace_call);
535	ppc_inst_t old, new;
536	int ret;
537
538	/*
539	 * When using CALL_OPS, the function to call is associated with the
540	 * call site, and we don't have a global function pointer to update.
541	 */
542	if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS))
543		return 0;
544
545	old = ppc_inst_read((u32 *)&ftrace_call);
546	new = ftrace_create_branch_inst(ip, ppc_function_entry(func), 1);
547	ret = ftrace_modify_code(ip, old, new);
548
549	/* Also update the regs callback function */
550	if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS) && !ret) {
551		ip = (unsigned long)(&ftrace_regs_call);
552		old = ppc_inst_read((u32 *)&ftrace_regs_call);
553		new = ftrace_create_branch_inst(ip, ppc_function_entry(func), 1);
554		ret = ftrace_modify_code(ip, old, new);
555	}
556
557	return ret;
558}
559
560/*
561 * Use the default ftrace_modify_all_code, but without
562 * stop_machine().
563 */
564void arch_ftrace_update_code(int command)
565{
566	ftrace_modify_all_code(command);
567}
568
569void ftrace_free_init_tramp(void)
570{
571	int i;
572
573	for (i = 0; i < NUM_FTRACE_TRAMPS && ftrace_tramps[i]; i++)
574		if (ftrace_tramps[i] == (unsigned long)ftrace_tramp_init) {
575			ftrace_tramps[i] = 0;
576			return;
577		}
578}
579
580static void __init add_ftrace_tramp(unsigned long tramp)
581{
582	int i;
583
584	for (i = 0; i < NUM_FTRACE_TRAMPS; i++)
585		if (!ftrace_tramps[i]) {
586			ftrace_tramps[i] = tramp;
587			return;
588		}
589}
590
591int __init ftrace_dyn_arch_init(void)
592{
593	unsigned int *tramp[] = { ftrace_tramp_text, ftrace_tramp_init };
594	unsigned long addr = FTRACE_REGS_ADDR;
595	long reladdr;
596	int i;
597	u32 stub_insns[] = {
598#ifdef CONFIG_PPC_KERNEL_PCREL
599		/* pla r12,addr */
600		PPC_PREFIX_MLS | __PPC_PRFX_R(1),
601		PPC_INST_PADDI | ___PPC_RT(_R12),
602		PPC_RAW_MTCTR(_R12),
603		PPC_RAW_BCTR()
604#elif defined(CONFIG_PPC64)
605		PPC_RAW_LD(_R12, _R13, offsetof(struct paca_struct, kernel_toc)),
606		PPC_RAW_ADDIS(_R12, _R12, 0),
607		PPC_RAW_ADDI(_R12, _R12, 0),
608		PPC_RAW_MTCTR(_R12),
609		PPC_RAW_BCTR()
610#else
611		PPC_RAW_LIS(_R12, 0),
612		PPC_RAW_ADDI(_R12, _R12, 0),
613		PPC_RAW_MTCTR(_R12),
614		PPC_RAW_BCTR()
615#endif
616	};
617
618	if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) {
619		for (i = 0; i < 2; i++) {
620			reladdr = addr - (unsigned long)tramp[i];
621
622			if (reladdr >= (long)SZ_8G || reladdr < -(long)SZ_8G) {
623				pr_err("Address of %ps out of range of pcrel address.\n",
624					(void *)addr);
625				return -1;
626			}
627
628			memcpy(tramp[i], stub_insns, sizeof(stub_insns));
629			tramp[i][0] |= IMM_H18(reladdr);
630			tramp[i][1] |= IMM_L(reladdr);
631			add_ftrace_tramp((unsigned long)tramp[i]);
632		}
633	} else if (IS_ENABLED(CONFIG_PPC64)) {
634		reladdr = addr - kernel_toc_addr();
635
636		if (reladdr >= (long)SZ_2G || reladdr < -(long long)SZ_2G) {
637			pr_err("Address of %ps out of range of kernel_toc.\n",
638				(void *)addr);
639			return -1;
640		}
641
642		for (i = 0; i < 2; i++) {
643			memcpy(tramp[i], stub_insns, sizeof(stub_insns));
644			tramp[i][1] |= PPC_HA(reladdr);
645			tramp[i][2] |= PPC_LO(reladdr);
646			add_ftrace_tramp((unsigned long)tramp[i]);
647		}
648	} else {
649		for (i = 0; i < 2; i++) {
650			memcpy(tramp[i], stub_insns, sizeof(stub_insns));
651			tramp[i][0] |= PPC_HA(addr);
652			tramp[i][1] |= PPC_LO(addr);
653			add_ftrace_tramp((unsigned long)tramp[i]);
654		}
655	}
656
657	return 0;
658}
659
660#ifdef CONFIG_FUNCTION_GRAPH_TRACER
661void ftrace_graph_func(unsigned long ip, unsigned long parent_ip,
662		       struct ftrace_ops *op, struct ftrace_regs *fregs)
663{
664	unsigned long sp = arch_ftrace_regs(fregs)->regs.gpr[1];
665
666	if (unlikely(ftrace_graph_is_dead()))
667		goto out;
668
669	if (unlikely(atomic_read(&current->tracing_graph_pause)))
670		goto out;
671
672	if (!function_graph_enter_regs(parent_ip, ip, 0, (unsigned long *)sp, fregs))
673		parent_ip = ppc_function_entry(return_to_handler);
674
675out:
676	arch_ftrace_regs(fregs)->regs.link = parent_ip;
677}
678#endif /* CONFIG_FUNCTION_GRAPH_TRACER */