trampoline.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/* Copyright (c) 2019 Facebook */
   3#include <linux/hash.h>
   4#include <linux/bpf.h>
   5#include <linux/filter.h>
   6#include <linux/ftrace.h>
   7#include <linux/rbtree_latch.h>
   8#include <linux/perf_event.h>
   9#include <linux/btf.h>
  10#include <linux/rcupdate_trace.h>
  11#include <linux/rcupdate_wait.h>
  12#include <linux/static_call.h>
  13#include <linux/bpf_verifier.h>
  14#include <linux/bpf_lsm.h>
  15#include <linux/delay.h>
  16
  17/* dummy _ops. The verifier will operate on target program's ops. */
  18const struct bpf_verifier_ops bpf_extension_verifier_ops = {
  19};
  20const struct bpf_prog_ops bpf_extension_prog_ops = {
  21};
  22
  23/* btf_vmlinux has ~22k attachable functions. 1k htab is enough. */
  24#define TRAMPOLINE_HASH_BITS 10
  25#define TRAMPOLINE_TABLE_SIZE (1 << TRAMPOLINE_HASH_BITS)
  26
  27static struct hlist_head trampoline_table[TRAMPOLINE_TABLE_SIZE];
  28
  29/* serializes access to trampoline_table */
  30static DEFINE_MUTEX(trampoline_mutex);
  31
  32#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
  33static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex);
  34
  35static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, enum ftrace_ops_cmd cmd)
  36{
  37	struct bpf_trampoline *tr = ops->private;
  38	int ret = 0;
  39
  40	if (cmd == FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF) {
  41		/* This is called inside register_ftrace_direct_multi(), so
  42		 * tr->mutex is already locked.
  43		 */
  44		lockdep_assert_held_once(&tr->mutex);
  45
  46		/* Instead of updating the trampoline here, we propagate
  47		 * -EAGAIN to register_ftrace_direct(). Then we can
  48		 * retry register_ftrace_direct() after updating the
  49		 * trampoline.
  50		 */
  51		if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) &&
  52		    !(tr->flags & BPF_TRAMP_F_ORIG_STACK)) {
  53			if (WARN_ON_ONCE(tr->flags & BPF_TRAMP_F_SHARE_IPMODIFY))
  54				return -EBUSY;
  55
  56			tr->flags |= BPF_TRAMP_F_SHARE_IPMODIFY;
  57			return -EAGAIN;
  58		}
  59
  60		return 0;
  61	}
  62
  63	/* The normal locking order is
  64	 *    tr->mutex => direct_mutex (ftrace.c) => ftrace_lock (ftrace.c)
  65	 *
  66	 * The following two commands are called from
  67	 *
  68	 *   prepare_direct_functions_for_ipmodify
  69	 *   cleanup_direct_functions_after_ipmodify
  70	 *
  71	 * In both cases, direct_mutex is already locked. Use
  72	 * mutex_trylock(&tr->mutex) to avoid deadlock in race condition
  73	 * (something else is making changes to this same trampoline).
  74	 */
  75	if (!mutex_trylock(&tr->mutex)) {
  76		/* sleep 1 ms to make sure whatever holding tr->mutex makes
  77		 * some progress.
  78		 */
  79		msleep(1);
  80		return -EAGAIN;
  81	}
  82
  83	switch (cmd) {
  84	case FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER:
  85		tr->flags |= BPF_TRAMP_F_SHARE_IPMODIFY;
  86
  87		if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) &&
  88		    !(tr->flags & BPF_TRAMP_F_ORIG_STACK))
  89			ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */);
  90		break;
  91	case FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY_PEER:
  92		tr->flags &= ~BPF_TRAMP_F_SHARE_IPMODIFY;
  93
  94		if (tr->flags & BPF_TRAMP_F_ORIG_STACK)
  95			ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */);
  96		break;
  97	default:
  98		ret = -EINVAL;
  99		break;
 100	}
 101
 102	mutex_unlock(&tr->mutex);
 103	return ret;
 104}
 105#endif
 106
 107bool bpf_prog_has_trampoline(const struct bpf_prog *prog)
 108{
 109	enum bpf_attach_type eatype = prog->expected_attach_type;
 110	enum bpf_prog_type ptype = prog->type;
 111
 112	return (ptype == BPF_PROG_TYPE_TRACING &&
 113		(eatype == BPF_TRACE_FENTRY || eatype == BPF_TRACE_FEXIT ||
 114		 eatype == BPF_MODIFY_RETURN)) ||
 115		(ptype == BPF_PROG_TYPE_LSM && eatype == BPF_LSM_MAC);
 116}
 117
 118void bpf_image_ksym_init(void *data, unsigned int size, struct bpf_ksym *ksym)
 119{
 120	ksym->start = (unsigned long) data;
 121	ksym->end = ksym->start + size;
 122}
 123
 124void bpf_image_ksym_add(struct bpf_ksym *ksym)
 125{
 126	bpf_ksym_add(ksym);
 127	perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF, ksym->start,
 128			   PAGE_SIZE, false, ksym->name);
 129}
 130
 131void bpf_image_ksym_del(struct bpf_ksym *ksym)
 132{
 133	bpf_ksym_del(ksym);
 134	perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF, ksym->start,
 135			   PAGE_SIZE, true, ksym->name);
 136}
 137
 138static struct bpf_trampoline *bpf_trampoline_lookup(u64 key)
 139{
 140	struct bpf_trampoline *tr;
 141	struct hlist_head *head;
 142	int i;
 143
 144	mutex_lock(&trampoline_mutex);
 145	head = &trampoline_table[hash_64(key, TRAMPOLINE_HASH_BITS)];
 146	hlist_for_each_entry(tr, head, hlist) {
 147		if (tr->key == key) {
 148			refcount_inc(&tr->refcnt);
 149			goto out;
 150		}
 151	}
 152	tr = kzalloc(sizeof(*tr), GFP_KERNEL);
 153	if (!tr)
 154		goto out;
 155#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
 156	tr->fops = kzalloc(sizeof(struct ftrace_ops), GFP_KERNEL);
 157	if (!tr->fops) {
 158		kfree(tr);
 159		tr = NULL;
 160		goto out;
 161	}
 162	tr->fops->private = tr;
 163	tr->fops->ops_func = bpf_tramp_ftrace_ops_func;
 164#endif
 165
 166	tr->key = key;
 167	INIT_HLIST_NODE(&tr->hlist);
 168	hlist_add_head(&tr->hlist, head);
 169	refcount_set(&tr->refcnt, 1);
 170	mutex_init(&tr->mutex);
 171	for (i = 0; i < BPF_TRAMP_MAX; i++)
 172		INIT_HLIST_HEAD(&tr->progs_hlist[i]);
 173out:
 174	mutex_unlock(&trampoline_mutex);
 175	return tr;
 176}
 177
 178static int bpf_trampoline_update_fentry(struct bpf_trampoline *tr, u32 orig_flags,
 179					void *old_addr, void *new_addr)
 180{
 181	enum bpf_text_poke_type new_t = BPF_MOD_CALL, old_t = BPF_MOD_CALL;
 182	void *ip = tr->func.addr;
 183
 184	if (!new_addr)
 185		new_t = BPF_MOD_NOP;
 186	else if (bpf_trampoline_use_jmp(tr->flags))
 187		new_t = BPF_MOD_JUMP;
 188
 189	if (!old_addr)
 190		old_t = BPF_MOD_NOP;
 191	else if (bpf_trampoline_use_jmp(orig_flags))
 192		old_t = BPF_MOD_JUMP;
 193
 194	return bpf_arch_text_poke(ip, old_t, new_t, old_addr, new_addr);
 195}
 196
 197static int unregister_fentry(struct bpf_trampoline *tr, u32 orig_flags,
 198			     void *old_addr)
 199{
 200	int ret;
 201
 202	if (tr->func.ftrace_managed)
 203		ret = unregister_ftrace_direct(tr->fops, (long)old_addr, false);
 204	else
 205		ret = bpf_trampoline_update_fentry(tr, orig_flags, old_addr, NULL);
 206
 207	return ret;
 208}
 209
 210static int modify_fentry(struct bpf_trampoline *tr, u32 orig_flags,
 211			 void *old_addr, void *new_addr,
 212			 bool lock_direct_mutex)
 213{
 214	int ret;
 215
 216	if (tr->func.ftrace_managed) {
 217		unsigned long addr = (unsigned long) new_addr;
 218
 219		if (bpf_trampoline_use_jmp(tr->flags))
 220			addr = ftrace_jmp_set(addr);
 221
 222		if (lock_direct_mutex)
 223			ret = modify_ftrace_direct(tr->fops, addr);
 224		else
 225			ret = modify_ftrace_direct_nolock(tr->fops, addr);
 226	} else {
 227		ret = bpf_trampoline_update_fentry(tr, orig_flags, old_addr,
 228						   new_addr);
 229	}
 230	return ret;
 231}
 232
 233/* first time registering */
 234static int register_fentry(struct bpf_trampoline *tr, void *new_addr)
 235{
 236	void *ip = tr->func.addr;
 237	unsigned long faddr;
 238	int ret;
 239
 240	faddr = ftrace_location((unsigned long)ip);
 241	if (faddr) {
 242		if (!tr->fops)
 243			return -ENOTSUPP;
 244		tr->func.ftrace_managed = true;
 245	}
 246
 247	if (tr->func.ftrace_managed) {
 248		unsigned long addr = (unsigned long) new_addr;
 249
 250		if (bpf_trampoline_use_jmp(tr->flags))
 251			addr = ftrace_jmp_set(addr);
 252
 253		ret = ftrace_set_filter_ip(tr->fops, (unsigned long)ip, 0, 1);
 254		if (ret)
 255			return ret;
 256		ret = register_ftrace_direct(tr->fops, addr);
 257	} else {
 258		ret = bpf_trampoline_update_fentry(tr, 0, NULL, new_addr);
 259	}
 260
 261	return ret;
 262}
 263
 264static struct bpf_tramp_links *
 265bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_arg)
 266{
 267	struct bpf_tramp_link *link;
 268	struct bpf_tramp_links *tlinks;
 269	struct bpf_tramp_link **links;
 270	int kind;
 271
 272	*total = 0;
 273	tlinks = kcalloc(BPF_TRAMP_MAX, sizeof(*tlinks), GFP_KERNEL);
 274	if (!tlinks)
 275		return ERR_PTR(-ENOMEM);
 276
 277	for (kind = 0; kind < BPF_TRAMP_MAX; kind++) {
 278		tlinks[kind].nr_links = tr->progs_cnt[kind];
 279		*total += tr->progs_cnt[kind];
 280		links = tlinks[kind].links;
 281
 282		hlist_for_each_entry(link, &tr->progs_hlist[kind], tramp_hlist) {
 283			*ip_arg |= link->link.prog->call_get_func_ip;
 284			*links++ = link;
 285		}
 286	}
 287	return tlinks;
 288}
 289
 290static void bpf_tramp_image_free(struct bpf_tramp_image *im)
 291{
 292	bpf_image_ksym_del(&im->ksym);
 293	arch_free_bpf_trampoline(im->image, im->size);
 294	bpf_jit_uncharge_modmem(im->size);
 295	percpu_ref_exit(&im->pcref);
 296	kfree_rcu(im, rcu);
 297}
 298
 299static void __bpf_tramp_image_put_deferred(struct work_struct *work)
 300{
 301	struct bpf_tramp_image *im;
 302
 303	im = container_of(work, struct bpf_tramp_image, work);
 304	bpf_tramp_image_free(im);
 305}
 306
 307/* callback, fexit step 3 or fentry step 2 */
 308static void __bpf_tramp_image_put_rcu(struct rcu_head *rcu)
 309{
 310	struct bpf_tramp_image *im;
 311
 312	im = container_of(rcu, struct bpf_tramp_image, rcu);
 313	INIT_WORK(&im->work, __bpf_tramp_image_put_deferred);
 314	schedule_work(&im->work);
 315}
 316
 317/* callback, fexit step 2. Called after percpu_ref_kill confirms. */
 318static void __bpf_tramp_image_release(struct percpu_ref *pcref)
 319{
 320	struct bpf_tramp_image *im;
 321
 322	im = container_of(pcref, struct bpf_tramp_image, pcref);
 323	call_rcu_tasks(&im->rcu, __bpf_tramp_image_put_rcu);
 324}
 325
 326/* callback, fexit or fentry step 1 */
 327static void __bpf_tramp_image_put_rcu_tasks(struct rcu_head *rcu)
 328{
 329	struct bpf_tramp_image *im;
 330
 331	im = container_of(rcu, struct bpf_tramp_image, rcu);
 332	if (im->ip_after_call)
 333		/* the case of fmod_ret/fexit trampoline and CONFIG_PREEMPTION=y */
 334		percpu_ref_kill(&im->pcref);
 335	else
 336		/* the case of fentry trampoline */
 337		call_rcu_tasks(&im->rcu, __bpf_tramp_image_put_rcu);
 338}
 339
 340static void bpf_tramp_image_put(struct bpf_tramp_image *im)
 341{
 342	/* The trampoline image that calls original function is using:
 343	 * rcu_read_lock_trace to protect sleepable bpf progs
 344	 * rcu_read_lock to protect normal bpf progs
 345	 * percpu_ref to protect trampoline itself
 346	 * rcu tasks to protect trampoline asm not covered by percpu_ref
 347	 * (which are few asm insns before __bpf_tramp_enter and
 348	 *  after __bpf_tramp_exit)
 349	 *
 350	 * The trampoline is unreachable before bpf_tramp_image_put().
 351	 *
 352	 * First, patch the trampoline to avoid calling into fexit progs.
 353	 * The progs will be freed even if the original function is still
 354	 * executing or sleeping.
 355	 * In case of CONFIG_PREEMPT=y use call_rcu_tasks() to wait on
 356	 * first few asm instructions to execute and call into
 357	 * __bpf_tramp_enter->percpu_ref_get.
 358	 * Then use percpu_ref_kill to wait for the trampoline and the original
 359	 * function to finish.
 360	 * Then use call_rcu_tasks() to make sure few asm insns in
 361	 * the trampoline epilogue are done as well.
 362	 *
 363	 * In !PREEMPT case the task that got interrupted in the first asm
 364	 * insns won't go through an RCU quiescent state which the
 365	 * percpu_ref_kill will be waiting for. Hence the first
 366	 * call_rcu_tasks() is not necessary.
 367	 */
 368	if (im->ip_after_call) {
 369		int err = bpf_arch_text_poke(im->ip_after_call, BPF_MOD_NOP,
 370					     BPF_MOD_JUMP, NULL,
 371					     im->ip_epilogue);
 372		WARN_ON(err);
 373		if (IS_ENABLED(CONFIG_TASKS_RCU))
 374			call_rcu_tasks(&im->rcu, __bpf_tramp_image_put_rcu_tasks);
 375		else
 376			percpu_ref_kill(&im->pcref);
 377		return;
 378	}
 379
 380	/* The trampoline without fexit and fmod_ret progs doesn't call original
 381	 * function and doesn't use percpu_ref.
 382	 * Use call_rcu_tasks_trace() to wait for sleepable progs to finish.
 383	 * Then use call_rcu_tasks() to wait for the rest of trampoline asm
 384	 * and normal progs.
 385	 */
 386	call_rcu_tasks_trace(&im->rcu, __bpf_tramp_image_put_rcu_tasks);
 387}
 388
 389static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size)
 390{
 391	struct bpf_tramp_image *im;
 392	struct bpf_ksym *ksym;
 393	void *image;
 394	int err = -ENOMEM;
 395
 396	im = kzalloc(sizeof(*im), GFP_KERNEL);
 397	if (!im)
 398		goto out;
 399
 400	err = bpf_jit_charge_modmem(size);
 401	if (err)
 402		goto out_free_im;
 403	im->size = size;
 404
 405	err = -ENOMEM;
 406	im->image = image = arch_alloc_bpf_trampoline(size);
 407	if (!image)
 408		goto out_uncharge;
 409
 410	err = percpu_ref_init(&im->pcref, __bpf_tramp_image_release, 0, GFP_KERNEL);
 411	if (err)
 412		goto out_free_image;
 413
 414	ksym = &im->ksym;
 415	INIT_LIST_HEAD_RCU(&ksym->lnode);
 416	snprintf(ksym->name, KSYM_NAME_LEN, "bpf_trampoline_%llu", key);
 417	bpf_image_ksym_init(image, size, ksym);
 418	bpf_image_ksym_add(ksym);
 419	return im;
 420
 421out_free_image:
 422	arch_free_bpf_trampoline(im->image, im->size);
 423out_uncharge:
 424	bpf_jit_uncharge_modmem(size);
 425out_free_im:
 426	kfree(im);
 427out:
 428	return ERR_PTR(err);
 429}
 430
 431static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex)
 432{
 433	struct bpf_tramp_image *im;
 434	struct bpf_tramp_links *tlinks;
 435	u32 orig_flags = tr->flags;
 436	bool ip_arg = false;
 437	int err, total, size;
 438
 439	tlinks = bpf_trampoline_get_progs(tr, &total, &ip_arg);
 440	if (IS_ERR(tlinks))
 441		return PTR_ERR(tlinks);
 442
 443	if (total == 0) {
 444		err = unregister_fentry(tr, orig_flags, tr->cur_image->image);
 445		bpf_tramp_image_put(tr->cur_image);
 446		tr->cur_image = NULL;
 447		goto out;
 448	}
 449
 450	/* clear all bits except SHARE_IPMODIFY and TAIL_CALL_CTX */
 451	tr->flags &= (BPF_TRAMP_F_SHARE_IPMODIFY | BPF_TRAMP_F_TAIL_CALL_CTX);
 452
 453	if (tlinks[BPF_TRAMP_FEXIT].nr_links ||
 454	    tlinks[BPF_TRAMP_MODIFY_RETURN].nr_links) {
 455		/* NOTE: BPF_TRAMP_F_RESTORE_REGS and BPF_TRAMP_F_SKIP_FRAME
 456		 * should not be set together.
 457		 */
 458		tr->flags |= BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME;
 459	} else {
 460		tr->flags |= BPF_TRAMP_F_RESTORE_REGS;
 461	}
 462
 463	if (ip_arg)
 464		tr->flags |= BPF_TRAMP_F_IP_ARG;
 465
 466#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
 467again:
 468	if (tr->flags & BPF_TRAMP_F_CALL_ORIG) {
 469		if (tr->flags & BPF_TRAMP_F_SHARE_IPMODIFY) {
 470			/* The BPF_TRAMP_F_SKIP_FRAME can be cleared in the
 471			 * first try, reset it in the second try.
 472			 */
 473			tr->flags |= BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SKIP_FRAME;
 474		} else if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_JMP)) {
 475			/* Use "jmp" instead of "call" for the trampoline
 476			 * in the origin call case, and we don't need to
 477			 * skip the frame.
 478			 */
 479			tr->flags &= ~BPF_TRAMP_F_SKIP_FRAME;
 480		}
 481	}
 482#endif
 483
 484	size = arch_bpf_trampoline_size(&tr->func.model, tr->flags,
 485					tlinks, tr->func.addr);
 486	if (size < 0) {
 487		err = size;
 488		goto out;
 489	}
 490
 491	if (size > PAGE_SIZE) {
 492		err = -E2BIG;
 493		goto out;
 494	}
 495
 496	im = bpf_tramp_image_alloc(tr->key, size);
 497	if (IS_ERR(im)) {
 498		err = PTR_ERR(im);
 499		goto out;
 500	}
 501
 502	err = arch_prepare_bpf_trampoline(im, im->image, im->image + size,
 503					  &tr->func.model, tr->flags, tlinks,
 504					  tr->func.addr);
 505	if (err < 0)
 506		goto out_free;
 507
 508	err = arch_protect_bpf_trampoline(im->image, im->size);
 509	if (err)
 510		goto out_free;
 511
 512	WARN_ON(tr->cur_image && total == 0);
 513	if (tr->cur_image)
 514		/* progs already running at this address */
 515		err = modify_fentry(tr, orig_flags, tr->cur_image->image,
 516				    im->image, lock_direct_mutex);
 517	else
 518		/* first time registering */
 519		err = register_fentry(tr, im->image);
 520
 521#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
 522	if (err == -EAGAIN) {
 523		/* -EAGAIN from bpf_tramp_ftrace_ops_func. Now
 524		 * BPF_TRAMP_F_SHARE_IPMODIFY is set, we can generate the
 525		 * trampoline again, and retry register.
 526		 */
 527		bpf_tramp_image_free(im);
 528		goto again;
 529	}
 530#endif
 531	if (err)
 532		goto out_free;
 533
 534	if (tr->cur_image)
 535		bpf_tramp_image_put(tr->cur_image);
 536	tr->cur_image = im;
 537out:
 538	/* If any error happens, restore previous flags */
 539	if (err)
 540		tr->flags = orig_flags;
 541	kfree(tlinks);
 542	return err;
 543
 544out_free:
 545	bpf_tramp_image_free(im);
 546	goto out;
 547}
 548
 549static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog)
 550{
 551	switch (prog->expected_attach_type) {
 552	case BPF_TRACE_FENTRY:
 553		return BPF_TRAMP_FENTRY;
 554	case BPF_MODIFY_RETURN:
 555		return BPF_TRAMP_MODIFY_RETURN;
 556	case BPF_TRACE_FEXIT:
 557		return BPF_TRAMP_FEXIT;
 558	case BPF_LSM_MAC:
 559		if (!prog->aux->attach_func_proto->type)
 560			/* The function returns void, we cannot modify its
 561			 * return value.
 562			 */
 563			return BPF_TRAMP_FEXIT;
 564		else
 565			return BPF_TRAMP_MODIFY_RETURN;
 566	default:
 567		return BPF_TRAMP_REPLACE;
 568	}
 569}
 570
 571static int bpf_freplace_check_tgt_prog(struct bpf_prog *tgt_prog)
 572{
 573	struct bpf_prog_aux *aux = tgt_prog->aux;
 574
 575	guard(mutex)(&aux->ext_mutex);
 576	if (aux->prog_array_member_cnt)
 577		/* Program extensions can not extend target prog when the target
 578		 * prog has been updated to any prog_array map as tail callee.
 579		 * It's to prevent a potential infinite loop like:
 580		 * tgt prog entry -> tgt prog subprog -> freplace prog entry
 581		 * --tailcall-> tgt prog entry.
 582		 */
 583		return -EBUSY;
 584
 585	aux->is_extended = true;
 586	return 0;
 587}
 588
 589static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link,
 590				      struct bpf_trampoline *tr,
 591				      struct bpf_prog *tgt_prog)
 592{
 593	enum bpf_tramp_prog_type kind;
 594	struct bpf_tramp_link *link_exiting;
 595	int err = 0;
 596	int cnt = 0, i;
 597
 598	kind = bpf_attach_type_to_tramp(link->link.prog);
 599	if (tr->extension_prog)
 600		/* cannot attach fentry/fexit if extension prog is attached.
 601		 * cannot overwrite extension prog either.
 602		 */
 603		return -EBUSY;
 604
 605	for (i = 0; i < BPF_TRAMP_MAX; i++)
 606		cnt += tr->progs_cnt[i];
 607
 608	if (kind == BPF_TRAMP_REPLACE) {
 609		/* Cannot attach extension if fentry/fexit are in use. */
 610		if (cnt)
 611			return -EBUSY;
 612		err = bpf_freplace_check_tgt_prog(tgt_prog);
 613		if (err)
 614			return err;
 615		tr->extension_prog = link->link.prog;
 616		return bpf_arch_text_poke(tr->func.addr, BPF_MOD_NOP,
 617					  BPF_MOD_JUMP, NULL,
 618					  link->link.prog->bpf_func);
 619	}
 620	if (cnt >= BPF_MAX_TRAMP_LINKS)
 621		return -E2BIG;
 622	if (!hlist_unhashed(&link->tramp_hlist))
 623		/* prog already linked */
 624		return -EBUSY;
 625	hlist_for_each_entry(link_exiting, &tr->progs_hlist[kind], tramp_hlist) {
 626		if (link_exiting->link.prog != link->link.prog)
 627			continue;
 628		/* prog already linked */
 629		return -EBUSY;
 630	}
 631
 632	hlist_add_head(&link->tramp_hlist, &tr->progs_hlist[kind]);
 633	tr->progs_cnt[kind]++;
 634	err = bpf_trampoline_update(tr, true /* lock_direct_mutex */);
 635	if (err) {
 636		hlist_del_init(&link->tramp_hlist);
 637		tr->progs_cnt[kind]--;
 638	}
 639	return err;
 640}
 641
 642int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
 643			     struct bpf_trampoline *tr,
 644			     struct bpf_prog *tgt_prog)
 645{
 646	int err;
 647
 648	mutex_lock(&tr->mutex);
 649	err = __bpf_trampoline_link_prog(link, tr, tgt_prog);
 650	mutex_unlock(&tr->mutex);
 651	return err;
 652}
 653
 654static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
 655					struct bpf_trampoline *tr,
 656					struct bpf_prog *tgt_prog)
 657{
 658	enum bpf_tramp_prog_type kind;
 659	int err;
 660
 661	kind = bpf_attach_type_to_tramp(link->link.prog);
 662	if (kind == BPF_TRAMP_REPLACE) {
 663		WARN_ON_ONCE(!tr->extension_prog);
 664		err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP,
 665					 BPF_MOD_NOP,
 666					 tr->extension_prog->bpf_func, NULL);
 667		tr->extension_prog = NULL;
 668		guard(mutex)(&tgt_prog->aux->ext_mutex);
 669		tgt_prog->aux->is_extended = false;
 670		return err;
 671	}
 672	hlist_del_init(&link->tramp_hlist);
 673	tr->progs_cnt[kind]--;
 674	return bpf_trampoline_update(tr, true /* lock_direct_mutex */);
 675}
 676
 677/* bpf_trampoline_unlink_prog() should never fail. */
 678int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
 679			       struct bpf_trampoline *tr,
 680			       struct bpf_prog *tgt_prog)
 681{
 682	int err;
 683
 684	mutex_lock(&tr->mutex);
 685	err = __bpf_trampoline_unlink_prog(link, tr, tgt_prog);
 686	mutex_unlock(&tr->mutex);
 687	return err;
 688}
 689
 690#if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM)
 691static void bpf_shim_tramp_link_release(struct bpf_link *link)
 692{
 693	struct bpf_shim_tramp_link *shim_link =
 694		container_of(link, struct bpf_shim_tramp_link, link.link);
 695
 696	/* paired with 'shim_link->trampoline = tr' in bpf_trampoline_link_cgroup_shim */
 697	if (!shim_link->trampoline)
 698		return;
 699
 700	WARN_ON_ONCE(bpf_trampoline_unlink_prog(&shim_link->link, shim_link->trampoline, NULL));
 701	bpf_trampoline_put(shim_link->trampoline);
 702}
 703
 704static void bpf_shim_tramp_link_dealloc(struct bpf_link *link)
 705{
 706	struct bpf_shim_tramp_link *shim_link =
 707		container_of(link, struct bpf_shim_tramp_link, link.link);
 708
 709	kfree(shim_link);
 710}
 711
 712static const struct bpf_link_ops bpf_shim_tramp_link_lops = {
 713	.release = bpf_shim_tramp_link_release,
 714	.dealloc = bpf_shim_tramp_link_dealloc,
 715};
 716
 717static struct bpf_shim_tramp_link *cgroup_shim_alloc(const struct bpf_prog *prog,
 718						     bpf_func_t bpf_func,
 719						     int cgroup_atype,
 720						     enum bpf_attach_type attach_type)
 721{
 722	struct bpf_shim_tramp_link *shim_link = NULL;
 723	struct bpf_prog *p;
 724
 725	shim_link = kzalloc(sizeof(*shim_link), GFP_USER);
 726	if (!shim_link)
 727		return NULL;
 728
 729	p = bpf_prog_alloc(1, 0);
 730	if (!p) {
 731		kfree(shim_link);
 732		return NULL;
 733	}
 734
 735	p->jited = false;
 736	p->bpf_func = bpf_func;
 737
 738	p->aux->cgroup_atype = cgroup_atype;
 739	p->aux->attach_func_proto = prog->aux->attach_func_proto;
 740	p->aux->attach_btf_id = prog->aux->attach_btf_id;
 741	p->aux->attach_btf = prog->aux->attach_btf;
 742	btf_get(p->aux->attach_btf);
 743	p->type = BPF_PROG_TYPE_LSM;
 744	p->expected_attach_type = BPF_LSM_MAC;
 745	bpf_prog_inc(p);
 746	bpf_link_init(&shim_link->link.link, BPF_LINK_TYPE_UNSPEC,
 747		      &bpf_shim_tramp_link_lops, p, attach_type);
 748	bpf_cgroup_atype_get(p->aux->attach_btf_id, cgroup_atype);
 749
 750	return shim_link;
 751}
 752
 753static struct bpf_shim_tramp_link *cgroup_shim_find(struct bpf_trampoline *tr,
 754						    bpf_func_t bpf_func)
 755{
 756	struct bpf_tramp_link *link;
 757	int kind;
 758
 759	for (kind = 0; kind < BPF_TRAMP_MAX; kind++) {
 760		hlist_for_each_entry(link, &tr->progs_hlist[kind], tramp_hlist) {
 761			struct bpf_prog *p = link->link.prog;
 762
 763			if (p->bpf_func == bpf_func)
 764				return container_of(link, struct bpf_shim_tramp_link, link);
 765		}
 766	}
 767
 768	return NULL;
 769}
 770
 771int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
 772				    int cgroup_atype,
 773				    enum bpf_attach_type attach_type)
 774{
 775	struct bpf_shim_tramp_link *shim_link = NULL;
 776	struct bpf_attach_target_info tgt_info = {};
 777	struct bpf_trampoline *tr;
 778	bpf_func_t bpf_func;
 779	u64 key;
 780	int err;
 781
 782	err = bpf_check_attach_target(NULL, prog, NULL,
 783				      prog->aux->attach_btf_id,
 784				      &tgt_info);
 785	if (err)
 786		return err;
 787
 788	key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf,
 789					 prog->aux->attach_btf_id);
 790
 791	bpf_lsm_find_cgroup_shim(prog, &bpf_func);
 792	tr = bpf_trampoline_get(key, &tgt_info);
 793	if (!tr)
 794		return  -ENOMEM;
 795
 796	mutex_lock(&tr->mutex);
 797
 798	shim_link = cgroup_shim_find(tr, bpf_func);
 799	if (shim_link && !IS_ERR(bpf_link_inc_not_zero(&shim_link->link.link))) {
 800		/* Reusing existing shim attached by the other program. */
 801		mutex_unlock(&tr->mutex);
 802		bpf_trampoline_put(tr); /* bpf_trampoline_get above */
 803		return 0;
 804	}
 805
 806	/* Allocate and install new shim. */
 807
 808	shim_link = cgroup_shim_alloc(prog, bpf_func, cgroup_atype, attach_type);
 809	if (!shim_link) {
 810		err = -ENOMEM;
 811		goto err;
 812	}
 813
 814	err = __bpf_trampoline_link_prog(&shim_link->link, tr, NULL);
 815	if (err)
 816		goto err;
 817
 818	shim_link->trampoline = tr;
 819	/* note, we're still holding tr refcnt from above */
 820
 821	mutex_unlock(&tr->mutex);
 822
 823	return 0;
 824err:
 825	mutex_unlock(&tr->mutex);
 826
 827	if (shim_link)
 828		bpf_link_put(&shim_link->link.link);
 829
 830	/* have to release tr while _not_ holding its mutex */
 831	bpf_trampoline_put(tr); /* bpf_trampoline_get above */
 832
 833	return err;
 834}
 835
 836void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog)
 837{
 838	struct bpf_shim_tramp_link *shim_link = NULL;
 839	struct bpf_trampoline *tr;
 840	bpf_func_t bpf_func;
 841	u64 key;
 842
 843	key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf,
 844					 prog->aux->attach_btf_id);
 845
 846	bpf_lsm_find_cgroup_shim(prog, &bpf_func);
 847	tr = bpf_trampoline_lookup(key);
 848	if (WARN_ON_ONCE(!tr))
 849		return;
 850
 851	mutex_lock(&tr->mutex);
 852	shim_link = cgroup_shim_find(tr, bpf_func);
 853	mutex_unlock(&tr->mutex);
 854
 855	if (shim_link)
 856		bpf_link_put(&shim_link->link.link);
 857
 858	bpf_trampoline_put(tr); /* bpf_trampoline_lookup above */
 859}
 860#endif
 861
 862struct bpf_trampoline *bpf_trampoline_get(u64 key,
 863					  struct bpf_attach_target_info *tgt_info)
 864{
 865	struct bpf_trampoline *tr;
 866
 867	tr = bpf_trampoline_lookup(key);
 868	if (!tr)
 869		return NULL;
 870
 871	mutex_lock(&tr->mutex);
 872	if (tr->func.addr)
 873		goto out;
 874
 875	memcpy(&tr->func.model, &tgt_info->fmodel, sizeof(tgt_info->fmodel));
 876	tr->func.addr = (void *)tgt_info->tgt_addr;
 877out:
 878	mutex_unlock(&tr->mutex);
 879	return tr;
 880}
 881
 882void bpf_trampoline_put(struct bpf_trampoline *tr)
 883{
 884	int i;
 885
 886	if (!tr)
 887		return;
 888	mutex_lock(&trampoline_mutex);
 889	if (!refcount_dec_and_test(&tr->refcnt))
 890		goto out;
 891	WARN_ON_ONCE(mutex_is_locked(&tr->mutex));
 892
 893	for (i = 0; i < BPF_TRAMP_MAX; i++)
 894		if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[i])))
 895			goto out;
 896
 897	/* This code will be executed even when the last bpf_tramp_image
 898	 * is alive. All progs are detached from the trampoline and the
 899	 * trampoline image is patched with jmp into epilogue to skip
 900	 * fexit progs. The fentry-only trampoline will be freed via
 901	 * multiple rcu callbacks.
 902	 */
 903	hlist_del(&tr->hlist);
 904	if (tr->fops) {
 905		ftrace_free_filter(tr->fops);
 906		kfree(tr->fops);
 907	}
 908	kfree(tr);
 909out:
 910	mutex_unlock(&trampoline_mutex);
 911}
 912
 913#define NO_START_TIME 1
 914static __always_inline u64 notrace bpf_prog_start_time(void)
 915{
 916	u64 start = NO_START_TIME;
 917
 918	if (static_branch_unlikely(&bpf_stats_enabled_key)) {
 919		start = sched_clock();
 920		if (unlikely(!start))
 921			start = NO_START_TIME;
 922	}
 923	return start;
 924}
 925
 926/* The logic is similar to bpf_prog_run(), but with an explicit
 927 * rcu_read_lock() and migrate_disable() which are required
 928 * for the trampoline. The macro is split into
 929 * call __bpf_prog_enter
 930 * call prog->bpf_func
 931 * call __bpf_prog_exit
 932 *
 933 * __bpf_prog_enter returns:
 934 * 0 - skip execution of the bpf prog
 935 * 1 - execute bpf prog
 936 * [2..MAX_U64] - execute bpf prog and record execution time.
 937 *     This is start time.
 938 */
 939static u64 notrace __bpf_prog_enter_recur(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx)
 940	__acquires(RCU)
 941{
 942	rcu_read_lock_dont_migrate();
 943
 944	run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
 945
 946	if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {
 947		bpf_prog_inc_misses_counter(prog);
 948		if (prog->aux->recursion_detected)
 949			prog->aux->recursion_detected(prog);
 950		return 0;
 951	}
 952	return bpf_prog_start_time();
 953}
 954
 955static void notrace __update_prog_stats(struct bpf_prog *prog, u64 start)
 956{
 957	struct bpf_prog_stats *stats;
 958	unsigned long flags;
 959	u64 duration;
 960
 961	/*
 962	 * static_key could be enabled in __bpf_prog_enter* and disabled in
 963	 * __bpf_prog_exit*. And vice versa. Check that 'start' is valid.
 964	 */
 965	if (start <= NO_START_TIME)
 966		return;
 967
 968	duration = sched_clock() - start;
 969	stats = this_cpu_ptr(prog->stats);
 970	flags = u64_stats_update_begin_irqsave(&stats->syncp);
 971	u64_stats_inc(&stats->cnt);
 972	u64_stats_add(&stats->nsecs, duration);
 973	u64_stats_update_end_irqrestore(&stats->syncp, flags);
 974}
 975
 976static __always_inline void notrace update_prog_stats(struct bpf_prog *prog,
 977						      u64 start)
 978{
 979	if (static_branch_unlikely(&bpf_stats_enabled_key))
 980		__update_prog_stats(prog, start);
 981}
 982
 983static void notrace __bpf_prog_exit_recur(struct bpf_prog *prog, u64 start,
 984					  struct bpf_tramp_run_ctx *run_ctx)
 985	__releases(RCU)
 986{
 987	bpf_reset_run_ctx(run_ctx->saved_run_ctx);
 988
 989	update_prog_stats(prog, start);
 990	this_cpu_dec(*(prog->active));
 991	rcu_read_unlock_migrate();
 992}
 993
 994static u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog,
 995					       struct bpf_tramp_run_ctx *run_ctx)
 996	__acquires(RCU)
 997{
 998	/* Runtime stats are exported via actual BPF_LSM_CGROUP
 999	 * programs, not the shims.
1000	 */
1001	rcu_read_lock_dont_migrate();
1002
1003	run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
1004
1005	return NO_START_TIME;
1006}
1007
1008static void notrace __bpf_prog_exit_lsm_cgroup(struct bpf_prog *prog, u64 start,
1009					       struct bpf_tramp_run_ctx *run_ctx)
1010	__releases(RCU)
1011{
1012	bpf_reset_run_ctx(run_ctx->saved_run_ctx);
1013
1014	rcu_read_unlock_migrate();
1015}
1016
1017u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,
1018					     struct bpf_tramp_run_ctx *run_ctx)
1019{
1020	rcu_read_lock_trace();
1021	migrate_disable();
1022	might_fault();
1023
1024	run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
1025
1026	if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {
1027		bpf_prog_inc_misses_counter(prog);
1028		if (prog->aux->recursion_detected)
1029			prog->aux->recursion_detected(prog);
1030		return 0;
1031	}
1032	return bpf_prog_start_time();
1033}
1034
1035void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start,
1036					     struct bpf_tramp_run_ctx *run_ctx)
1037{
1038	bpf_reset_run_ctx(run_ctx->saved_run_ctx);
1039
1040	update_prog_stats(prog, start);
1041	this_cpu_dec(*(prog->active));
1042	migrate_enable();
1043	rcu_read_unlock_trace();
1044}
1045
1046static u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog,
1047					      struct bpf_tramp_run_ctx *run_ctx)
1048{
1049	rcu_read_lock_trace();
1050	migrate_disable();
1051	might_fault();
1052
1053	run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
1054
1055	return bpf_prog_start_time();
1056}
1057
1058static void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start,
1059					      struct bpf_tramp_run_ctx *run_ctx)
1060{
1061	bpf_reset_run_ctx(run_ctx->saved_run_ctx);
1062
1063	update_prog_stats(prog, start);
1064	migrate_enable();
1065	rcu_read_unlock_trace();
1066}
1067
1068static u64 notrace __bpf_prog_enter(struct bpf_prog *prog,
1069				    struct bpf_tramp_run_ctx *run_ctx)
1070	__acquires(RCU)
1071{
1072	rcu_read_lock_dont_migrate();
1073
1074	run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
1075
1076	return bpf_prog_start_time();
1077}
1078
1079static void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start,
1080				    struct bpf_tramp_run_ctx *run_ctx)
1081	__releases(RCU)
1082{
1083	bpf_reset_run_ctx(run_ctx->saved_run_ctx);
1084
1085	update_prog_stats(prog, start);
1086	rcu_read_unlock_migrate();
1087}
1088
1089void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr)
1090{
1091	percpu_ref_get(&tr->pcref);
1092}
1093
1094void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr)
1095{
1096	percpu_ref_put(&tr->pcref);
1097}
1098
1099bpf_trampoline_enter_t bpf_trampoline_enter(const struct bpf_prog *prog)
1100{
1101	bool sleepable = prog->sleepable;
1102
1103	if (bpf_prog_check_recur(prog))
1104		return sleepable ? __bpf_prog_enter_sleepable_recur :
1105			__bpf_prog_enter_recur;
1106
1107	if (resolve_prog_type(prog) == BPF_PROG_TYPE_LSM &&
1108	    prog->expected_attach_type == BPF_LSM_CGROUP)
1109		return __bpf_prog_enter_lsm_cgroup;
1110
1111	return sleepable ? __bpf_prog_enter_sleepable : __bpf_prog_enter;
1112}
1113
1114bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog)
1115{
1116	bool sleepable = prog->sleepable;
1117
1118	if (bpf_prog_check_recur(prog))
1119		return sleepable ? __bpf_prog_exit_sleepable_recur :
1120			__bpf_prog_exit_recur;
1121
1122	if (resolve_prog_type(prog) == BPF_PROG_TYPE_LSM &&
1123	    prog->expected_attach_type == BPF_LSM_CGROUP)
1124		return __bpf_prog_exit_lsm_cgroup;
1125
1126	return sleepable ? __bpf_prog_exit_sleepable : __bpf_prog_exit;
1127}
1128
1129int __weak
1130arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,
1131			    const struct btf_func_model *m, u32 flags,
1132			    struct bpf_tramp_links *tlinks,
1133			    void *func_addr)
1134{
1135	return -ENOTSUPP;
1136}
1137
1138void * __weak arch_alloc_bpf_trampoline(unsigned int size)
1139{
1140	void *image;
1141
1142	if (WARN_ON_ONCE(size > PAGE_SIZE))
1143		return NULL;
1144	image = bpf_jit_alloc_exec(PAGE_SIZE);
1145	if (image)
1146		set_vm_flush_reset_perms(image);
1147	return image;
1148}
1149
1150void __weak arch_free_bpf_trampoline(void *image, unsigned int size)
1151{
1152	WARN_ON_ONCE(size > PAGE_SIZE);
1153	/* bpf_jit_free_exec doesn't need "size", but
1154	 * bpf_prog_pack_free() needs it.
1155	 */
1156	bpf_jit_free_exec(image);
1157}
1158
1159int __weak arch_protect_bpf_trampoline(void *image, unsigned int size)
1160{
1161	WARN_ON_ONCE(size > PAGE_SIZE);
1162	return set_memory_rox((long)image, 1);
1163}
1164
1165int __weak arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
1166				    struct bpf_tramp_links *tlinks, void *func_addr)
1167{
1168	return -ENOTSUPP;
1169}
1170
1171static int __init init_trampolines(void)
1172{
1173	int i;
1174
1175	for (i = 0; i < TRAMPOLINE_TABLE_SIZE; i++)
1176		INIT_HLIST_HEAD(&trampoline_table[i]);
1177	return 0;
1178}
1179late_initcall(init_trampolines);