uprobes.c

   1// SPDX-License-Identifier: GPL-2.0+
   2/*
   3 * User-space Probes (UProbes)
   4 *
   5 * Copyright (C) IBM Corporation, 2008-2012
   6 * Authors:
   7 *	Srikar Dronamraju
   8 *	Jim Keniston
   9 * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra
  10 */
  11
  12#include <linux/kernel.h>
  13#include <linux/highmem.h>
  14#include <linux/pagemap.h>	/* read_mapping_page */
  15#include <linux/slab.h>
  16#include <linux/sched.h>
  17#include <linux/sched/mm.h>
  18#include <linux/export.h>
  19#include <linux/rmap.h>		/* anon_vma_prepare */
  20#include <linux/mmu_notifier.h>
  21#include <linux/swap.h>		/* folio_free_swap */
  22#include <linux/ptrace.h>	/* user_enable_single_step */
  23#include <linux/kdebug.h>	/* notifier mechanism */
  24#include <linux/percpu-rwsem.h>
  25#include <linux/task_work.h>
  26#include <linux/shmem_fs.h>
  27#include <linux/khugepaged.h>
  28#include <linux/rcupdate_trace.h>
  29#include <linux/workqueue.h>
  30#include <linux/srcu.h>
  31#include <linux/oom.h>          /* check_stable_address_space */
  32#include <linux/pagewalk.h>
  33
  34#include <linux/uprobes.h>
  35
  36#define UINSNS_PER_PAGE			(PAGE_SIZE/UPROBE_XOL_SLOT_BYTES)
  37#define MAX_UPROBE_XOL_SLOTS		UINSNS_PER_PAGE
  38
  39static struct rb_root uprobes_tree = RB_ROOT;
  40/*
  41 * allows us to skip the uprobe_mmap if there are no uprobe events active
  42 * at this time.  Probably a fine grained per inode count is better?
  43 */
  44#define no_uprobe_events()	RB_EMPTY_ROOT(&uprobes_tree)
  45
  46static DEFINE_RWLOCK(uprobes_treelock);	/* serialize rbtree access */
  47static seqcount_rwlock_t uprobes_seqcount = SEQCNT_RWLOCK_ZERO(uprobes_seqcount, &uprobes_treelock);
  48
  49#define UPROBES_HASH_SZ	13
  50/* serialize uprobe->pending_list */
  51static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ];
  52#define uprobes_mmap_hash(v)	(&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ])
  53
  54DEFINE_STATIC_PERCPU_RWSEM(dup_mmap_sem);
  55
  56/* Covers return_instance's uprobe lifetime. */
  57DEFINE_STATIC_SRCU(uretprobes_srcu);
  58
  59/* Have a copy of original instruction */
  60#define UPROBE_COPY_INSN	0
  61
  62struct uprobe {
  63	struct rb_node		rb_node;	/* node in the rb tree */
  64	refcount_t		ref;
  65	struct rw_semaphore	register_rwsem;
  66	struct rw_semaphore	consumer_rwsem;
  67	struct list_head	pending_list;
  68	struct list_head	consumers;
  69	struct inode		*inode;		/* Also hold a ref to inode */
  70	union {
  71		struct rcu_head		rcu;
  72		struct work_struct	work;
  73	};
  74	loff_t			offset;
  75	loff_t			ref_ctr_offset;
  76	unsigned long		flags;		/* "unsigned long" so bitops work */
  77
  78	/*
  79	 * The generic code assumes that it has two members of unknown type
  80	 * owned by the arch-specific code:
  81	 *
  82	 *	insn -	copy_insn() saves the original instruction here for
  83	 *		arch_uprobe_analyze_insn().
  84	 *
  85	 *	ixol -	potentially modified instruction to execute out of
  86	 *		line, copied to xol_area by xol_get_insn_slot().
  87	 */
  88	struct arch_uprobe	arch;
  89};
  90
  91struct delayed_uprobe {
  92	struct list_head list;
  93	struct uprobe *uprobe;
  94	struct mm_struct *mm;
  95};
  96
  97static DEFINE_MUTEX(delayed_uprobe_lock);
  98static LIST_HEAD(delayed_uprobe_list);
  99
 100/*
 101 * Execute out of line area: anonymous executable mapping installed
 102 * by the probed task to execute the copy of the original instruction
 103 * mangled by set_swbp().
 104 *
 105 * On a breakpoint hit, thread contests for a slot.  It frees the
 106 * slot after singlestep. Currently a fixed number of slots are
 107 * allocated.
 108 */
 109struct xol_area {
 110	wait_queue_head_t		wq;		/* if all slots are busy */
 111	unsigned long			*bitmap;	/* 0 = free slot */
 112
 113	struct page			*page;
 114	/*
 115	 * We keep the vma's vm_start rather than a pointer to the vma
 116	 * itself.  The probed process or a naughty kernel module could make
 117	 * the vma go away, and we must handle that reasonably gracefully.
 118	 */
 119	unsigned long			vaddr;		/* Page(s) of instruction slots */
 120};
 121
 122static void uprobe_warn(struct task_struct *t, const char *msg)
 123{
 124	pr_warn("uprobe: %s:%d failed to %s\n", t->comm, t->pid, msg);
 125}
 126
 127/*
 128 * valid_vma: Verify if the specified vma is an executable vma
 129 * Relax restrictions while unregistering: vm_flags might have
 130 * changed after breakpoint was inserted.
 131 *	- is_register: indicates if we are in register context.
 132 *	- Return 1 if the specified virtual address is in an
 133 *	  executable vma.
 134 */
 135static bool valid_vma(struct vm_area_struct *vma, bool is_register)
 136{
 137	vm_flags_t flags = VM_HUGETLB | VM_MAYEXEC | VM_MAYSHARE;
 138
 139	if (is_register)
 140		flags |= VM_WRITE;
 141
 142	return vma->vm_file && (vma->vm_flags & flags) == VM_MAYEXEC;
 143}
 144
 145static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset)
 146{
 147	return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
 148}
 149
 150static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr)
 151{
 152	return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start);
 153}
 154
 155/**
 156 * is_swbp_insn - check if instruction is breakpoint instruction.
 157 * @insn: instruction to be checked.
 158 * Default implementation of is_swbp_insn
 159 * Returns true if @insn is a breakpoint instruction.
 160 */
 161bool __weak is_swbp_insn(uprobe_opcode_t *insn)
 162{
 163	return *insn == UPROBE_SWBP_INSN;
 164}
 165
 166/**
 167 * is_trap_insn - check if instruction is breakpoint instruction.
 168 * @insn: instruction to be checked.
 169 * Default implementation of is_trap_insn
 170 * Returns true if @insn is a breakpoint instruction.
 171 *
 172 * This function is needed for the case where an architecture has multiple
 173 * trap instructions (like powerpc).
 174 */
 175bool __weak is_trap_insn(uprobe_opcode_t *insn)
 176{
 177	return is_swbp_insn(insn);
 178}
 179
 180void uprobe_copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len)
 181{
 182	void *kaddr = kmap_atomic(page);
 183	memcpy(dst, kaddr + (vaddr & ~PAGE_MASK), len);
 184	kunmap_atomic(kaddr);
 185}
 186
 187static void copy_to_page(struct page *page, unsigned long vaddr, const void *src, int len)
 188{
 189	void *kaddr = kmap_atomic(page);
 190	memcpy(kaddr + (vaddr & ~PAGE_MASK), src, len);
 191	kunmap_atomic(kaddr);
 192}
 193
 194static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t *insn,
 195			 int nbytes, void *data)
 196{
 197	uprobe_opcode_t old_opcode;
 198	bool is_swbp;
 199
 200	/*
 201	 * Note: We only check if the old_opcode is UPROBE_SWBP_INSN here.
 202	 * We do not check if it is any other 'trap variant' which could
 203	 * be conditional trap instruction such as the one powerpc supports.
 204	 *
 205	 * The logic is that we do not care if the underlying instruction
 206	 * is a trap variant; uprobes always wins over any other (gdb)
 207	 * breakpoint.
 208	 */
 209	uprobe_copy_from_page(page, vaddr, &old_opcode, UPROBE_SWBP_INSN_SIZE);
 210	is_swbp = is_swbp_insn(&old_opcode);
 211
 212	if (is_swbp_insn(insn)) {
 213		if (is_swbp)		/* register: already installed? */
 214			return 0;
 215	} else {
 216		if (!is_swbp)		/* unregister: was it changed by us? */
 217			return 0;
 218	}
 219
 220	return 1;
 221}
 222
 223static struct delayed_uprobe *
 224delayed_uprobe_check(struct uprobe *uprobe, struct mm_struct *mm)
 225{
 226	struct delayed_uprobe *du;
 227
 228	list_for_each_entry(du, &delayed_uprobe_list, list)
 229		if (du->uprobe == uprobe && du->mm == mm)
 230			return du;
 231	return NULL;
 232}
 233
 234static int delayed_uprobe_add(struct uprobe *uprobe, struct mm_struct *mm)
 235{
 236	struct delayed_uprobe *du;
 237
 238	if (delayed_uprobe_check(uprobe, mm))
 239		return 0;
 240
 241	du  = kzalloc(sizeof(*du), GFP_KERNEL);
 242	if (!du)
 243		return -ENOMEM;
 244
 245	du->uprobe = uprobe;
 246	du->mm = mm;
 247	list_add(&du->list, &delayed_uprobe_list);
 248	return 0;
 249}
 250
 251static void delayed_uprobe_delete(struct delayed_uprobe *du)
 252{
 253	if (WARN_ON(!du))
 254		return;
 255	list_del(&du->list);
 256	kfree(du);
 257}
 258
 259static void delayed_uprobe_remove(struct uprobe *uprobe, struct mm_struct *mm)
 260{
 261	struct list_head *pos, *q;
 262	struct delayed_uprobe *du;
 263
 264	if (!uprobe && !mm)
 265		return;
 266
 267	list_for_each_safe(pos, q, &delayed_uprobe_list) {
 268		du = list_entry(pos, struct delayed_uprobe, list);
 269
 270		if (uprobe && du->uprobe != uprobe)
 271			continue;
 272		if (mm && du->mm != mm)
 273			continue;
 274
 275		delayed_uprobe_delete(du);
 276	}
 277}
 278
 279static bool valid_ref_ctr_vma(struct uprobe *uprobe,
 280			      struct vm_area_struct *vma)
 281{
 282	unsigned long vaddr = offset_to_vaddr(vma, uprobe->ref_ctr_offset);
 283
 284	return uprobe->ref_ctr_offset &&
 285		vma->vm_file &&
 286		file_inode(vma->vm_file) == uprobe->inode &&
 287		(vma->vm_flags & (VM_WRITE|VM_SHARED)) == VM_WRITE &&
 288		vma->vm_start <= vaddr &&
 289		vma->vm_end > vaddr;
 290}
 291
 292static struct vm_area_struct *
 293find_ref_ctr_vma(struct uprobe *uprobe, struct mm_struct *mm)
 294{
 295	VMA_ITERATOR(vmi, mm, 0);
 296	struct vm_area_struct *tmp;
 297
 298	for_each_vma(vmi, tmp)
 299		if (valid_ref_ctr_vma(uprobe, tmp))
 300			return tmp;
 301
 302	return NULL;
 303}
 304
 305static int
 306__update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d)
 307{
 308	void *kaddr;
 309	struct page *page;
 310	int ret;
 311	short *ptr;
 312
 313	if (!vaddr || !d)
 314		return -EINVAL;
 315
 316	ret = get_user_pages_remote(mm, vaddr, 1,
 317				    FOLL_WRITE, &page, NULL);
 318	if (unlikely(ret <= 0)) {
 319		/*
 320		 * We are asking for 1 page. If get_user_pages_remote() fails,
 321		 * it may return 0, in that case we have to return error.
 322		 */
 323		return ret == 0 ? -EBUSY : ret;
 324	}
 325
 326	kaddr = kmap_atomic(page);
 327	ptr = kaddr + (vaddr & ~PAGE_MASK);
 328
 329	if (unlikely(*ptr + d < 0)) {
 330		pr_warn("ref_ctr going negative. vaddr: 0x%lx, "
 331			"curr val: %d, delta: %d\n", vaddr, *ptr, d);
 332		ret = -EINVAL;
 333		goto out;
 334	}
 335
 336	*ptr += d;
 337	ret = 0;
 338out:
 339	kunmap_atomic(kaddr);
 340	put_page(page);
 341	return ret;
 342}
 343
 344static void update_ref_ctr_warn(struct uprobe *uprobe,
 345				struct mm_struct *mm, short d)
 346{
 347	pr_warn("ref_ctr %s failed for inode: 0x%lx offset: "
 348		"0x%llx ref_ctr_offset: 0x%llx of mm: 0x%p\n",
 349		d > 0 ? "increment" : "decrement", uprobe->inode->i_ino,
 350		(unsigned long long) uprobe->offset,
 351		(unsigned long long) uprobe->ref_ctr_offset, mm);
 352}
 353
 354static int update_ref_ctr(struct uprobe *uprobe, struct mm_struct *mm,
 355			  short d)
 356{
 357	struct vm_area_struct *rc_vma;
 358	unsigned long rc_vaddr;
 359	int ret = 0;
 360
 361	rc_vma = find_ref_ctr_vma(uprobe, mm);
 362
 363	if (rc_vma) {
 364		rc_vaddr = offset_to_vaddr(rc_vma, uprobe->ref_ctr_offset);
 365		ret = __update_ref_ctr(mm, rc_vaddr, d);
 366		if (ret)
 367			update_ref_ctr_warn(uprobe, mm, d);
 368
 369		if (d > 0)
 370			return ret;
 371	}
 372
 373	mutex_lock(&delayed_uprobe_lock);
 374	if (d > 0)
 375		ret = delayed_uprobe_add(uprobe, mm);
 376	else
 377		delayed_uprobe_remove(uprobe, mm);
 378	mutex_unlock(&delayed_uprobe_lock);
 379
 380	return ret;
 381}
 382
 383static bool orig_page_is_identical(struct vm_area_struct *vma,
 384		unsigned long vaddr, struct page *page, bool *pmd_mappable)
 385{
 386	const pgoff_t index = vaddr_to_offset(vma, vaddr) >> PAGE_SHIFT;
 387	struct folio *orig_folio = filemap_get_folio(vma->vm_file->f_mapping,
 388						    index);
 389	struct page *orig_page;
 390	bool identical;
 391
 392	if (IS_ERR(orig_folio))
 393		return false;
 394	orig_page = folio_file_page(orig_folio, index);
 395
 396	*pmd_mappable = folio_test_pmd_mappable(orig_folio);
 397	identical = folio_test_uptodate(orig_folio) &&
 398		    pages_identical(page, orig_page);
 399	folio_put(orig_folio);
 400	return identical;
 401}
 402
 403static int __uprobe_write(struct vm_area_struct *vma,
 404		struct folio_walk *fw, struct folio *folio,
 405		unsigned long insn_vaddr, uprobe_opcode_t *insn, int nbytes,
 406		bool is_register)
 407{
 408	const unsigned long vaddr = insn_vaddr & PAGE_MASK;
 409	bool pmd_mappable;
 410
 411	/* For now, we'll only handle PTE-mapped folios. */
 412	if (fw->level != FW_LEVEL_PTE)
 413		return -EFAULT;
 414
 415	/*
 416	 * See can_follow_write_pte(): we'd actually prefer a writable PTE here,
 417	 * but the VMA might not be writable.
 418	 */
 419	if (!pte_write(fw->pte)) {
 420		if (!PageAnonExclusive(fw->page))
 421			return -EFAULT;
 422		if (unlikely(userfaultfd_pte_wp(vma, fw->pte)))
 423			return -EFAULT;
 424		/* SOFTDIRTY is handled via pte_mkdirty() below. */
 425	}
 426
 427	/*
 428	 * We'll temporarily unmap the page and flush the TLB, such that we can
 429	 * modify the page atomically.
 430	 */
 431	flush_cache_page(vma, vaddr, pte_pfn(fw->pte));
 432	fw->pte = ptep_clear_flush(vma, vaddr, fw->ptep);
 433	copy_to_page(fw->page, insn_vaddr, insn, nbytes);
 434
 435	/*
 436	 * When unregistering, we may only zap a PTE if uffd is disabled and
 437	 * there are no unexpected folio references ...
 438	 */
 439	if (is_register || userfaultfd_missing(vma) ||
 440	    (folio_ref_count(folio) != folio_expected_ref_count(folio) + 1))
 441		goto remap;
 442
 443	/*
 444	 * ... and the mapped page is identical to the original page that
 445	 * would get faulted in on next access.
 446	 */
 447	if (!orig_page_is_identical(vma, vaddr, fw->page, &pmd_mappable))
 448		goto remap;
 449
 450	dec_mm_counter(vma->vm_mm, MM_ANONPAGES);
 451	folio_remove_rmap_pte(folio, fw->page, vma);
 452	if (!folio_mapped(folio) && folio_test_swapcache(folio) &&
 453	     folio_trylock(folio)) {
 454		folio_free_swap(folio);
 455		folio_unlock(folio);
 456	}
 457	folio_put(folio);
 458
 459	return pmd_mappable;
 460remap:
 461	/*
 462	 * Make sure that our copy_to_page() changes become visible before the
 463	 * set_pte_at() write.
 464	 */
 465	smp_wmb();
 466	/* We modified the page. Make sure to mark the PTE dirty. */
 467	set_pte_at(vma->vm_mm, vaddr, fw->ptep, pte_mkdirty(fw->pte));
 468	return 0;
 469}
 470
 471/*
 472 * NOTE:
 473 * Expect the breakpoint instruction to be the smallest size instruction for
 474 * the architecture. If an arch has variable length instruction and the
 475 * breakpoint instruction is not of the smallest length instruction
 476 * supported by that architecture then we need to modify is_trap_at_addr and
 477 * uprobe_write_opcode accordingly. This would never be a problem for archs
 478 * that have fixed length instructions.
 479 *
 480 * uprobe_write_opcode - write the opcode at a given virtual address.
 481 * @auprobe: arch specific probepoint information.
 482 * @vma: the probed virtual memory area.
 483 * @opcode_vaddr: the virtual address to store the opcode.
 484 * @opcode: opcode to be written at @opcode_vaddr.
 485 *
 486 * Called with mm->mmap_lock held for write.
 487 * Return 0 (success) or a negative errno.
 488 */
 489int uprobe_write_opcode(struct arch_uprobe *auprobe, struct vm_area_struct *vma,
 490		const unsigned long opcode_vaddr, uprobe_opcode_t opcode,
 491		bool is_register)
 492{
 493	return uprobe_write(auprobe, vma, opcode_vaddr, &opcode, UPROBE_SWBP_INSN_SIZE,
 494			    verify_opcode, is_register, true /* do_update_ref_ctr */, NULL);
 495}
 496
 497int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma,
 498		 const unsigned long insn_vaddr, uprobe_opcode_t *insn, int nbytes,
 499		 uprobe_write_verify_t verify, bool is_register, bool do_update_ref_ctr,
 500		 void *data)
 501{
 502	const unsigned long vaddr = insn_vaddr & PAGE_MASK;
 503	struct mm_struct *mm = vma->vm_mm;
 504	struct uprobe *uprobe;
 505	int ret, ref_ctr_updated = 0;
 506	unsigned int gup_flags = FOLL_FORCE;
 507	struct mmu_notifier_range range;
 508	struct folio_walk fw;
 509	struct folio *folio;
 510	struct page *page;
 511
 512	uprobe = container_of(auprobe, struct uprobe, arch);
 513
 514	if (WARN_ON_ONCE(!is_cow_mapping(vma->vm_flags)))
 515		return -EINVAL;
 516
 517	/*
 518	 * When registering, we have to break COW to get an exclusive anonymous
 519	 * page that we can safely modify. Use FOLL_WRITE to trigger a write
 520	 * fault if required. When unregistering, we might be lucky and the
 521	 * anon page is already gone. So defer write faults until really
 522	 * required. Use FOLL_SPLIT_PMD, because __uprobe_write()
 523	 * cannot deal with PMDs yet.
 524	 */
 525	if (is_register)
 526		gup_flags |= FOLL_WRITE | FOLL_SPLIT_PMD;
 527
 528retry:
 529	ret = get_user_pages_remote(mm, vaddr, 1, gup_flags, &page, NULL);
 530	if (ret <= 0)
 531		goto out;
 532	folio = page_folio(page);
 533
 534	ret = verify(page, insn_vaddr, insn, nbytes, data);
 535	if (ret <= 0) {
 536		folio_put(folio);
 537		goto out;
 538	}
 539
 540	/* We are going to replace instruction, update ref_ctr. */
 541	if (do_update_ref_ctr && !ref_ctr_updated && uprobe->ref_ctr_offset) {
 542		ret = update_ref_ctr(uprobe, mm, is_register ? 1 : -1);
 543		if (ret) {
 544			folio_put(folio);
 545			goto out;
 546		}
 547
 548		ref_ctr_updated = 1;
 549	}
 550
 551	ret = 0;
 552	if (unlikely(!folio_test_anon(folio) || folio_is_zone_device(folio))) {
 553		VM_WARN_ON_ONCE(is_register);
 554		folio_put(folio);
 555		goto out;
 556	}
 557
 558	if (!is_register) {
 559		/*
 560		 * In the common case, we'll be able to zap the page when
 561		 * unregistering. So trigger MMU notifiers now, as we won't
 562		 * be able to do it under PTL.
 563		 */
 564		mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm,
 565					vaddr, vaddr + PAGE_SIZE);
 566		mmu_notifier_invalidate_range_start(&range);
 567	}
 568
 569	ret = -EAGAIN;
 570	/* Walk the page tables again, to perform the actual update. */
 571	if (folio_walk_start(&fw, vma, vaddr, 0)) {
 572		if (fw.page == page)
 573			ret = __uprobe_write(vma, &fw, folio, insn_vaddr, insn, nbytes, is_register);
 574		folio_walk_end(&fw, vma);
 575	}
 576
 577	if (!is_register)
 578		mmu_notifier_invalidate_range_end(&range);
 579
 580	folio_put(folio);
 581	switch (ret) {
 582	case -EFAULT:
 583		gup_flags |= FOLL_WRITE | FOLL_SPLIT_PMD;
 584		fallthrough;
 585	case -EAGAIN:
 586		goto retry;
 587	default:
 588		break;
 589	}
 590
 591out:
 592	/* Revert back reference counter if instruction update failed. */
 593	if (do_update_ref_ctr && ret < 0 && ref_ctr_updated)
 594		update_ref_ctr(uprobe, mm, is_register ? -1 : 1);
 595
 596	/* try collapse pmd for compound page */
 597	if (ret > 0)
 598		collapse_pte_mapped_thp(mm, vaddr, false);
 599
 600	return ret < 0 ? ret : 0;
 601}
 602
 603/**
 604 * set_swbp - store breakpoint at a given address.
 605 * @auprobe: arch specific probepoint information.
 606 * @vma: the probed virtual memory area.
 607 * @vaddr: the virtual address to insert the opcode.
 608 *
 609 * For mm @mm, store the breakpoint instruction at @vaddr.
 610 * Return 0 (success) or a negative errno.
 611 */
 612int __weak set_swbp(struct arch_uprobe *auprobe, struct vm_area_struct *vma,
 613		unsigned long vaddr)
 614{
 615	return uprobe_write_opcode(auprobe, vma, vaddr, UPROBE_SWBP_INSN, true);
 616}
 617
 618/**
 619 * set_orig_insn - Restore the original instruction.
 620 * @vma: the probed virtual memory area.
 621 * @auprobe: arch specific probepoint information.
 622 * @vaddr: the virtual address to insert the opcode.
 623 *
 624 * For mm @mm, restore the original opcode (opcode) at @vaddr.
 625 * Return 0 (success) or a negative errno.
 626 */
 627int __weak set_orig_insn(struct arch_uprobe *auprobe,
 628		struct vm_area_struct *vma, unsigned long vaddr)
 629{
 630	return uprobe_write_opcode(auprobe, vma, vaddr,
 631			*(uprobe_opcode_t *)&auprobe->insn, false);
 632}
 633
 634/* uprobe should have guaranteed positive refcount */
 635static struct uprobe *get_uprobe(struct uprobe *uprobe)
 636{
 637	refcount_inc(&uprobe->ref);
 638	return uprobe;
 639}
 640
 641/*
 642 * uprobe should have guaranteed lifetime, which can be either of:
 643 *   - caller already has refcount taken (and wants an extra one);
 644 *   - uprobe is RCU protected and won't be freed until after grace period;
 645 *   - we are holding uprobes_treelock (for read or write, doesn't matter).
 646 */
 647static struct uprobe *try_get_uprobe(struct uprobe *uprobe)
 648{
 649	if (refcount_inc_not_zero(&uprobe->ref))
 650		return uprobe;
 651	return NULL;
 652}
 653
 654static inline bool uprobe_is_active(struct uprobe *uprobe)
 655{
 656	return !RB_EMPTY_NODE(&uprobe->rb_node);
 657}
 658
 659static void uprobe_free_rcu_tasks_trace(struct rcu_head *rcu)
 660{
 661	struct uprobe *uprobe = container_of(rcu, struct uprobe, rcu);
 662
 663	kfree(uprobe);
 664}
 665
 666static void uprobe_free_srcu(struct rcu_head *rcu)
 667{
 668	struct uprobe *uprobe = container_of(rcu, struct uprobe, rcu);
 669
 670	call_rcu_tasks_trace(&uprobe->rcu, uprobe_free_rcu_tasks_trace);
 671}
 672
 673static void uprobe_free_deferred(struct work_struct *work)
 674{
 675	struct uprobe *uprobe = container_of(work, struct uprobe, work);
 676
 677	write_lock(&uprobes_treelock);
 678
 679	if (uprobe_is_active(uprobe)) {
 680		write_seqcount_begin(&uprobes_seqcount);
 681		rb_erase(&uprobe->rb_node, &uprobes_tree);
 682		write_seqcount_end(&uprobes_seqcount);
 683	}
 684
 685	write_unlock(&uprobes_treelock);
 686
 687	/*
 688	 * If application munmap(exec_vma) before uprobe_unregister()
 689	 * gets called, we don't get a chance to remove uprobe from
 690	 * delayed_uprobe_list from remove_breakpoint(). Do it here.
 691	 */
 692	mutex_lock(&delayed_uprobe_lock);
 693	delayed_uprobe_remove(uprobe, NULL);
 694	mutex_unlock(&delayed_uprobe_lock);
 695
 696	/* start srcu -> rcu_tasks_trace -> kfree chain */
 697	call_srcu(&uretprobes_srcu, &uprobe->rcu, uprobe_free_srcu);
 698}
 699
 700static void put_uprobe(struct uprobe *uprobe)
 701{
 702	if (!refcount_dec_and_test(&uprobe->ref))
 703		return;
 704
 705	INIT_WORK(&uprobe->work, uprobe_free_deferred);
 706	schedule_work(&uprobe->work);
 707}
 708
 709/* Initialize hprobe as SRCU-protected "leased" uprobe */
 710static void hprobe_init_leased(struct hprobe *hprobe, struct uprobe *uprobe, int srcu_idx)
 711{
 712	WARN_ON(!uprobe);
 713	hprobe->state = HPROBE_LEASED;
 714	hprobe->uprobe = uprobe;
 715	hprobe->srcu_idx = srcu_idx;
 716}
 717
 718/* Initialize hprobe as refcounted ("stable") uprobe (uprobe can be NULL). */
 719static void hprobe_init_stable(struct hprobe *hprobe, struct uprobe *uprobe)
 720{
 721	hprobe->state = uprobe ? HPROBE_STABLE : HPROBE_GONE;
 722	hprobe->uprobe = uprobe;
 723	hprobe->srcu_idx = -1;
 724}
 725
 726/*
 727 * hprobe_consume() fetches hprobe's underlying uprobe and detects whether
 728 * uprobe is SRCU protected or is refcounted. hprobe_consume() can be
 729 * used only once for a given hprobe.
 730 *
 731 * Caller has to call hprobe_finalize() and pass previous hprobe_state, so
 732 * that hprobe_finalize() can perform SRCU unlock or put uprobe, whichever
 733 * is appropriate.
 734 */
 735static inline struct uprobe *hprobe_consume(struct hprobe *hprobe, enum hprobe_state *hstate)
 736{
 737	*hstate = xchg(&hprobe->state, HPROBE_CONSUMED);
 738	switch (*hstate) {
 739	case HPROBE_LEASED:
 740	case HPROBE_STABLE:
 741		return hprobe->uprobe;
 742	case HPROBE_GONE:	/* uprobe is NULL, no SRCU */
 743	case HPROBE_CONSUMED:	/* uprobe was finalized already, do nothing */
 744		return NULL;
 745	default:
 746		WARN(1, "hprobe invalid state %d", *hstate);
 747		return NULL;
 748	}
 749}
 750
 751/*
 752 * Reset hprobe state and, if hprobe was LEASED, release SRCU lock.
 753 * hprobe_finalize() can only be used from current context after
 754 * hprobe_consume() call (which determines uprobe and hstate value).
 755 */
 756static void hprobe_finalize(struct hprobe *hprobe, enum hprobe_state hstate)
 757{
 758	switch (hstate) {
 759	case HPROBE_LEASED:
 760		__srcu_read_unlock(&uretprobes_srcu, hprobe->srcu_idx);
 761		break;
 762	case HPROBE_STABLE:
 763		put_uprobe(hprobe->uprobe);
 764		break;
 765	case HPROBE_GONE:
 766	case HPROBE_CONSUMED:
 767		break;
 768	default:
 769		WARN(1, "hprobe invalid state %d", hstate);
 770		break;
 771	}
 772}
 773
 774/*
 775 * Attempt to switch (atomically) uprobe from being SRCU protected (LEASED)
 776 * to refcounted (STABLE) state. Competes with hprobe_consume(); only one of
 777 * them can win the race to perform SRCU unlocking. Whoever wins must perform
 778 * SRCU unlock.
 779 *
 780 * Returns underlying valid uprobe or NULL, if there was no underlying uprobe
 781 * to begin with or we failed to bump its refcount and it's going away.
 782 *
 783 * Returned non-NULL uprobe can be still safely used within an ongoing SRCU
 784 * locked region. If `get` is true, it's guaranteed that non-NULL uprobe has
 785 * an extra refcount for caller to assume and use. Otherwise, it's not
 786 * guaranteed that returned uprobe has a positive refcount, so caller has to
 787 * attempt try_get_uprobe(), if it needs to preserve uprobe beyond current
 788 * SRCU lock region. See dup_utask().
 789 */
 790static struct uprobe *hprobe_expire(struct hprobe *hprobe, bool get)
 791{
 792	enum hprobe_state hstate;
 793
 794	/*
 795	 * Caller should guarantee that return_instance is not going to be
 796	 * freed from under us. This can be achieved either through holding
 797	 * rcu_read_lock() or by owning return_instance in the first place.
 798	 *
 799	 * Underlying uprobe is itself protected from reuse by SRCU, so ensure
 800	 * SRCU lock is held properly.
 801	 */
 802	lockdep_assert(srcu_read_lock_held(&uretprobes_srcu));
 803
 804	hstate = READ_ONCE(hprobe->state);
 805	switch (hstate) {
 806	case HPROBE_STABLE:
 807		/* uprobe has positive refcount, bump refcount, if necessary */
 808		return get ? get_uprobe(hprobe->uprobe) : hprobe->uprobe;
 809	case HPROBE_GONE:
 810		/*
 811		 * SRCU was unlocked earlier and we didn't manage to take
 812		 * uprobe refcnt, so it's effectively NULL
 813		 */
 814		return NULL;
 815	case HPROBE_CONSUMED:
 816		/*
 817		 * uprobe was consumed, so it's effectively NULL as far as
 818		 * uretprobe processing logic is concerned
 819		 */
 820		return NULL;
 821	case HPROBE_LEASED: {
 822		struct uprobe *uprobe = try_get_uprobe(hprobe->uprobe);
 823		/*
 824		 * Try to switch hprobe state, guarding against
 825		 * hprobe_consume() or another hprobe_expire() racing with us.
 826		 * Note, if we failed to get uprobe refcount, we use special
 827		 * HPROBE_GONE state to signal that hprobe->uprobe shouldn't
 828		 * be used as it will be freed after SRCU is unlocked.
 829		 */
 830		if (try_cmpxchg(&hprobe->state, &hstate, uprobe ? HPROBE_STABLE : HPROBE_GONE)) {
 831			/* We won the race, we are the ones to unlock SRCU */
 832			__srcu_read_unlock(&uretprobes_srcu, hprobe->srcu_idx);
 833			return get ? get_uprobe(uprobe) : uprobe;
 834		}
 835
 836		/*
 837		 * We lost the race, undo refcount bump (if it ever happened),
 838		 * unless caller would like an extra refcount anyways.
 839		 */
 840		if (uprobe && !get)
 841			put_uprobe(uprobe);
 842		/*
 843		 * Even if hprobe_consume() or another hprobe_expire() wins
 844		 * the state update race and unlocks SRCU from under us, we
 845		 * still have a guarantee that underyling uprobe won't be
 846		 * freed due to ongoing caller's SRCU lock region, so we can
 847		 * return it regardless. Also, if `get` was true, we also have
 848		 * an extra ref for the caller to own. This is used in dup_utask().
 849		 */
 850		return uprobe;
 851	}
 852	default:
 853		WARN(1, "unknown hprobe state %d", hstate);
 854		return NULL;
 855	}
 856}
 857
 858static __always_inline
 859int uprobe_cmp(const struct inode *l_inode, const loff_t l_offset,
 860	       const struct uprobe *r)
 861{
 862	if (l_inode < r->inode)
 863		return -1;
 864
 865	if (l_inode > r->inode)
 866		return 1;
 867
 868	if (l_offset < r->offset)
 869		return -1;
 870
 871	if (l_offset > r->offset)
 872		return 1;
 873
 874	return 0;
 875}
 876
 877#define __node_2_uprobe(node) \
 878	rb_entry((node), struct uprobe, rb_node)
 879
 880struct __uprobe_key {
 881	struct inode *inode;
 882	loff_t offset;
 883};
 884
 885static inline int __uprobe_cmp_key(const void *key, const struct rb_node *b)
 886{
 887	const struct __uprobe_key *a = key;
 888	return uprobe_cmp(a->inode, a->offset, __node_2_uprobe(b));
 889}
 890
 891static inline int __uprobe_cmp(struct rb_node *a, const struct rb_node *b)
 892{
 893	struct uprobe *u = __node_2_uprobe(a);
 894	return uprobe_cmp(u->inode, u->offset, __node_2_uprobe(b));
 895}
 896
 897/*
 898 * Assumes being inside RCU protected region.
 899 * No refcount is taken on returned uprobe.
 900 */
 901static struct uprobe *find_uprobe_rcu(struct inode *inode, loff_t offset)
 902{
 903	struct __uprobe_key key = {
 904		.inode = inode,
 905		.offset = offset,
 906	};
 907	struct rb_node *node;
 908	unsigned int seq;
 909
 910	lockdep_assert(rcu_read_lock_trace_held());
 911
 912	do {
 913		seq = read_seqcount_begin(&uprobes_seqcount);
 914		node = rb_find_rcu(&key, &uprobes_tree, __uprobe_cmp_key);
 915		/*
 916		 * Lockless RB-tree lookups can result only in false negatives.
 917		 * If the element is found, it is correct and can be returned
 918		 * under RCU protection. If we find nothing, we need to
 919		 * validate that seqcount didn't change. If it did, we have to
 920		 * try again as we might have missed the element (false
 921		 * negative). If seqcount is unchanged, search truly failed.
 922		 */
 923		if (node)
 924			return __node_2_uprobe(node);
 925	} while (read_seqcount_retry(&uprobes_seqcount, seq));
 926
 927	return NULL;
 928}
 929
 930/*
 931 * Attempt to insert a new uprobe into uprobes_tree.
 932 *
 933 * If uprobe already exists (for given inode+offset), we just increment
 934 * refcount of previously existing uprobe.
 935 *
 936 * If not, a provided new instance of uprobe is inserted into the tree (with
 937 * assumed initial refcount == 1).
 938 *
 939 * In any case, we return a uprobe instance that ends up being in uprobes_tree.
 940 * Caller has to clean up new uprobe instance, if it ended up not being
 941 * inserted into the tree.
 942 *
 943 * We assume that uprobes_treelock is held for writing.
 944 */
 945static struct uprobe *__insert_uprobe(struct uprobe *uprobe)
 946{
 947	struct rb_node *node;
 948again:
 949	node = rb_find_add_rcu(&uprobe->rb_node, &uprobes_tree, __uprobe_cmp);
 950	if (node) {
 951		struct uprobe *u = __node_2_uprobe(node);
 952
 953		if (!try_get_uprobe(u)) {
 954			rb_erase(node, &uprobes_tree);
 955			RB_CLEAR_NODE(&u->rb_node);
 956			goto again;
 957		}
 958
 959		return u;
 960	}
 961
 962	return uprobe;
 963}
 964
 965/*
 966 * Acquire uprobes_treelock and insert uprobe into uprobes_tree
 967 * (or reuse existing one, see __insert_uprobe() comments above).
 968 */
 969static struct uprobe *insert_uprobe(struct uprobe *uprobe)
 970{
 971	struct uprobe *u;
 972
 973	write_lock(&uprobes_treelock);
 974	write_seqcount_begin(&uprobes_seqcount);
 975	u = __insert_uprobe(uprobe);
 976	write_seqcount_end(&uprobes_seqcount);
 977	write_unlock(&uprobes_treelock);
 978
 979	return u;
 980}
 981
 982static void
 983ref_ctr_mismatch_warn(struct uprobe *cur_uprobe, struct uprobe *uprobe)
 984{
 985	pr_warn("ref_ctr_offset mismatch. inode: 0x%lx offset: 0x%llx "
 986		"ref_ctr_offset(old): 0x%llx ref_ctr_offset(new): 0x%llx\n",
 987		uprobe->inode->i_ino, (unsigned long long) uprobe->offset,
 988		(unsigned long long) cur_uprobe->ref_ctr_offset,
 989		(unsigned long long) uprobe->ref_ctr_offset);
 990}
 991
 992static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset,
 993				   loff_t ref_ctr_offset)
 994{
 995	struct uprobe *uprobe, *cur_uprobe;
 996
 997	uprobe = kzalloc(sizeof(struct uprobe), GFP_KERNEL);
 998	if (!uprobe)
 999		return ERR_PTR(-ENOMEM);
1000
1001	uprobe->inode = inode;
1002	uprobe->offset = offset;
1003	uprobe->ref_ctr_offset = ref_ctr_offset;
1004	INIT_LIST_HEAD(&uprobe->consumers);
1005	init_rwsem(&uprobe->register_rwsem);
1006	init_rwsem(&uprobe->consumer_rwsem);
1007	RB_CLEAR_NODE(&uprobe->rb_node);
1008	refcount_set(&uprobe->ref, 1);
1009
1010	/* add to uprobes_tree, sorted on inode:offset */
1011	cur_uprobe = insert_uprobe(uprobe);
1012	/* a uprobe exists for this inode:offset combination */
1013	if (cur_uprobe != uprobe) {
1014		if (cur_uprobe->ref_ctr_offset != uprobe->ref_ctr_offset) {
1015			ref_ctr_mismatch_warn(cur_uprobe, uprobe);
1016			put_uprobe(cur_uprobe);
1017			kfree(uprobe);
1018			return ERR_PTR(-EINVAL);
1019		}
1020		kfree(uprobe);
1021		uprobe = cur_uprobe;
1022	}
1023
1024	return uprobe;
1025}
1026
1027static void consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc)
1028{
1029	static atomic64_t id;
1030
1031	down_write(&uprobe->consumer_rwsem);
1032	list_add_rcu(&uc->cons_node, &uprobe->consumers);
1033	uc->id = (__u64) atomic64_inc_return(&id);
1034	up_write(&uprobe->consumer_rwsem);
1035}
1036
1037/*
1038 * For uprobe @uprobe, delete the consumer @uc.
1039 * Should never be called with consumer that's not part of @uprobe->consumers.
1040 */
1041static void consumer_del(struct uprobe *uprobe, struct uprobe_consumer *uc)
1042{
1043	down_write(&uprobe->consumer_rwsem);
1044	list_del_rcu(&uc->cons_node);
1045	up_write(&uprobe->consumer_rwsem);
1046}
1047
1048static int __copy_insn(struct address_space *mapping, struct file *filp,
1049			void *insn, int nbytes, loff_t offset)
1050{
1051	struct page *page;
1052	/*
1053	 * Ensure that the page that has the original instruction is populated
1054	 * and in page-cache. If ->read_folio == NULL it must be shmem_mapping(),
1055	 * see uprobe_register().
1056	 */
1057	if (mapping->a_ops->read_folio)
1058		page = read_mapping_page(mapping, offset >> PAGE_SHIFT, filp);
1059	else
1060		page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
1061	if (IS_ERR(page))
1062		return PTR_ERR(page);
1063
1064	uprobe_copy_from_page(page, offset, insn, nbytes);
1065	put_page(page);
1066
1067	return 0;
1068}
1069
1070static int copy_insn(struct uprobe *uprobe, struct file *filp)
1071{
1072	struct address_space *mapping = uprobe->inode->i_mapping;
1073	loff_t offs = uprobe->offset;
1074	void *insn = &uprobe->arch.insn;
1075	int size = sizeof(uprobe->arch.insn);
1076	int len, err = -EIO;
1077
1078	/* Copy only available bytes, -EIO if nothing was read */
1079	do {
1080		if (offs >= i_size_read(uprobe->inode))
1081			break;
1082
1083		len = min_t(int, size, PAGE_SIZE - (offs & ~PAGE_MASK));
1084		err = __copy_insn(mapping, filp, insn, len, offs);
1085		if (err)
1086			break;
1087
1088		insn += len;
1089		offs += len;
1090		size -= len;
1091	} while (size);
1092
1093	return err;
1094}
1095
1096static int prepare_uprobe(struct uprobe *uprobe, struct file *file,
1097				struct mm_struct *mm, unsigned long vaddr)
1098{
1099	int ret = 0;
1100
1101	if (test_bit(UPROBE_COPY_INSN, &uprobe->flags))
1102		return ret;
1103
1104	/* TODO: move this into _register, until then we abuse this sem. */
1105	down_write(&uprobe->consumer_rwsem);
1106	if (test_bit(UPROBE_COPY_INSN, &uprobe->flags))
1107		goto out;
1108
1109	ret = copy_insn(uprobe, file);
1110	if (ret)
1111		goto out;
1112
1113	ret = -ENOTSUPP;
1114	if (is_trap_insn((uprobe_opcode_t *)&uprobe->arch.insn))
1115		goto out;
1116
1117	ret = arch_uprobe_analyze_insn(&uprobe->arch, mm, vaddr);
1118	if (ret)
1119		goto out;
1120
1121	smp_wmb(); /* pairs with the smp_rmb() in handle_swbp() */
1122	set_bit(UPROBE_COPY_INSN, &uprobe->flags);
1123
1124 out:
1125	up_write(&uprobe->consumer_rwsem);
1126
1127	return ret;
1128}
1129
1130static inline bool consumer_filter(struct uprobe_consumer *uc, struct mm_struct *mm)
1131{
1132	return !uc->filter || uc->filter(uc, mm);
1133}
1134
1135static bool filter_chain(struct uprobe *uprobe, struct mm_struct *mm)
1136{
1137	struct uprobe_consumer *uc;
1138	bool ret = false;
1139
1140	down_read(&uprobe->consumer_rwsem);
1141	list_for_each_entry(uc, &uprobe->consumers, cons_node) {
1142		ret = consumer_filter(uc, mm);
1143		if (ret)
1144			break;
1145	}
1146	up_read(&uprobe->consumer_rwsem);
1147
1148	return ret;
1149}
1150
1151static int install_breakpoint(struct uprobe *uprobe, struct vm_area_struct *vma,
1152		unsigned long vaddr)
1153{
1154	struct mm_struct *mm = vma->vm_mm;
1155	bool first_uprobe;
1156	int ret;
1157
1158	ret = prepare_uprobe(uprobe, vma->vm_file, mm, vaddr);
1159	if (ret)
1160		return ret;
1161
1162	/*
1163	 * set MMF_HAS_UPROBES in advance for uprobe_pre_sstep_notifier(),
1164	 * the task can hit this breakpoint right after __replace_page().
1165	 */
1166	first_uprobe = !mm_flags_test(MMF_HAS_UPROBES, mm);
1167	if (first_uprobe)
1168		mm_flags_set(MMF_HAS_UPROBES, mm);
1169
1170	ret = set_swbp(&uprobe->arch, vma, vaddr);
1171	if (!ret)
1172		mm_flags_clear(MMF_RECALC_UPROBES, mm);
1173	else if (first_uprobe)
1174		mm_flags_clear(MMF_HAS_UPROBES, mm);
1175
1176	return ret;
1177}
1178
1179static int remove_breakpoint(struct uprobe *uprobe, struct vm_area_struct *vma,
1180		unsigned long vaddr)
1181{
1182	struct mm_struct *mm = vma->vm_mm;
1183
1184	mm_flags_set(MMF_RECALC_UPROBES, mm);
1185	return set_orig_insn(&uprobe->arch, vma, vaddr);
1186}
1187
1188struct map_info {
1189	struct map_info *next;
1190	struct mm_struct *mm;
1191	unsigned long vaddr;
1192};
1193
1194static inline struct map_info *free_map_info(struct map_info *info)
1195{
1196	struct map_info *next = info->next;
1197	kfree(info);
1198	return next;
1199}
1200
1201static struct map_info *
1202build_map_info(struct address_space *mapping, loff_t offset, bool is_register)
1203{
1204	unsigned long pgoff = offset >> PAGE_SHIFT;
1205	struct vm_area_struct *vma;
1206	struct map_info *curr = NULL;
1207	struct map_info *prev = NULL;
1208	struct map_info *info;
1209	int more = 0;
1210
1211 again:
1212	i_mmap_lock_read(mapping);
1213	vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
1214		if (!valid_vma(vma, is_register))
1215			continue;
1216
1217		if (!prev && !more) {
1218			/*
1219			 * Needs GFP_NOWAIT to avoid i_mmap_rwsem recursion through
1220			 * reclaim. This is optimistic, no harm done if it fails.
1221			 */
1222			prev = kmalloc(sizeof(struct map_info),
1223					GFP_NOWAIT | __GFP_NOMEMALLOC);
1224			if (prev)
1225				prev->next = NULL;
1226		}
1227		if (!prev) {
1228			more++;
1229			continue;
1230		}
1231
1232		if (!mmget_not_zero(vma->vm_mm))
1233			continue;
1234
1235		info = prev;
1236		prev = prev->next;
1237		info->next = curr;
1238		curr = info;
1239
1240		info->mm = vma->vm_mm;
1241		info->vaddr = offset_to_vaddr(vma, offset);
1242	}
1243	i_mmap_unlock_read(mapping);
1244
1245	if (!more)
1246		goto out;
1247
1248	prev = curr;
1249	while (curr) {
1250		mmput(curr->mm);
1251		curr = curr->next;
1252	}
1253
1254	do {
1255		info = kmalloc(sizeof(struct map_info), GFP_KERNEL);
1256		if (!info) {
1257			curr = ERR_PTR(-ENOMEM);
1258			goto out;
1259		}
1260		info->next = prev;
1261		prev = info;
1262	} while (--more);
1263
1264	goto again;
1265 out:
1266	while (prev)
1267		prev = free_map_info(prev);
1268	return curr;
1269}
1270
1271static int
1272register_for_each_vma(struct uprobe *uprobe, struct uprobe_consumer *new)
1273{
1274	bool is_register = !!new;
1275	struct map_info *info;
1276	int err = 0;
1277
1278	percpu_down_write(&dup_mmap_sem);
1279	info = build_map_info(uprobe->inode->i_mapping,
1280					uprobe->offset, is_register);
1281	if (IS_ERR(info)) {
1282		err = PTR_ERR(info);
1283		goto out;
1284	}
1285
1286	while (info) {
1287		struct mm_struct *mm = info->mm;
1288		struct vm_area_struct *vma;
1289
1290		if (err && is_register)
1291			goto free;
1292		/*
1293		 * We take mmap_lock for writing to avoid the race with
1294		 * find_active_uprobe_rcu() which takes mmap_lock for reading.
1295		 * Thus this install_breakpoint() can not make
1296		 * is_trap_at_addr() true right after find_uprobe_rcu()
1297		 * returns NULL in find_active_uprobe_rcu().
1298		 */
1299		mmap_write_lock(mm);
1300		if (check_stable_address_space(mm))
1301			goto unlock;
1302
1303		vma = find_vma(mm, info->vaddr);
1304		if (!vma || !valid_vma(vma, is_register) ||
1305		    file_inode(vma->vm_file) != uprobe->inode)
1306			goto unlock;
1307
1308		if (vma->vm_start > info->vaddr ||
1309		    vaddr_to_offset(vma, info->vaddr) != uprobe->offset)
1310			goto unlock;
1311
1312		if (is_register) {
1313			/* consult only the "caller", new consumer. */
1314			if (consumer_filter(new, mm))
1315				err = install_breakpoint(uprobe, vma, info->vaddr);
1316		} else if (mm_flags_test(MMF_HAS_UPROBES, mm)) {
1317			if (!filter_chain(uprobe, mm))
1318				err |= remove_breakpoint(uprobe, vma, info->vaddr);
1319		}
1320
1321 unlock:
1322		mmap_write_unlock(mm);
1323 free:
1324		mmput(mm);
1325		info = free_map_info(info);
1326	}
1327 out:
1328	percpu_up_write(&dup_mmap_sem);
1329	return err;
1330}
1331
1332/**
1333 * uprobe_unregister_nosync - unregister an already registered probe.
1334 * @uprobe: uprobe to remove
1335 * @uc: identify which probe if multiple probes are colocated.
1336 */
1337void uprobe_unregister_nosync(struct uprobe *uprobe, struct uprobe_consumer *uc)
1338{
1339	int err;
1340
1341	down_write(&uprobe->register_rwsem);
1342	consumer_del(uprobe, uc);
1343	err = register_for_each_vma(uprobe, NULL);
1344	up_write(&uprobe->register_rwsem);
1345
1346	/* TODO : cant unregister? schedule a worker thread */
1347	if (unlikely(err)) {
1348		uprobe_warn(current, "unregister, leaking uprobe");
1349		return;
1350	}
1351
1352	put_uprobe(uprobe);
1353}
1354EXPORT_SYMBOL_GPL(uprobe_unregister_nosync);
1355
1356void uprobe_unregister_sync(void)
1357{
1358	/*
1359	 * Now that handler_chain() and handle_uretprobe_chain() iterate over
1360	 * uprobe->consumers list under RCU protection without holding
1361	 * uprobe->register_rwsem, we need to wait for RCU grace period to
1362	 * make sure that we can't call into just unregistered
1363	 * uprobe_consumer's callbacks anymore. If we don't do that, fast and
1364	 * unlucky enough caller can free consumer's memory and cause
1365	 * handler_chain() or handle_uretprobe_chain() to do an use-after-free.
1366	 */
1367	synchronize_rcu_tasks_trace();
1368	synchronize_srcu(&uretprobes_srcu);
1369}
1370EXPORT_SYMBOL_GPL(uprobe_unregister_sync);
1371
1372/**
1373 * uprobe_register - register a probe
1374 * @inode: the file in which the probe has to be placed.
1375 * @offset: offset from the start of the file.
1376 * @ref_ctr_offset: offset of SDT marker / reference counter
1377 * @uc: information on howto handle the probe..
1378 *
1379 * Apart from the access refcount, uprobe_register() takes a creation
1380 * refcount (thro alloc_uprobe) if and only if this @uprobe is getting
1381 * inserted into the rbtree (i.e first consumer for a @inode:@offset
1382 * tuple).  Creation refcount stops uprobe_unregister from freeing the
1383 * @uprobe even before the register operation is complete. Creation
1384 * refcount is released when the last @uc for the @uprobe
1385 * unregisters. Caller of uprobe_register() is required to keep @inode
1386 * (and the containing mount) referenced.
1387 *
1388 * Return: pointer to the new uprobe on success or an ERR_PTR on failure.
1389 */
1390struct uprobe *uprobe_register(struct inode *inode,
1391				loff_t offset, loff_t ref_ctr_offset,
1392				struct uprobe_consumer *uc)
1393{
1394	struct uprobe *uprobe;
1395	int ret;
1396
1397	/* Uprobe must have at least one set consumer */
1398	if (!uc->handler && !uc->ret_handler)
1399		return ERR_PTR(-EINVAL);
1400
1401	/* copy_insn() uses read_mapping_page() or shmem_read_mapping_page() */
1402	if (!inode->i_mapping->a_ops->read_folio &&
1403	    !shmem_mapping(inode->i_mapping))
1404		return ERR_PTR(-EIO);
1405	/* Racy, just to catch the obvious mistakes */
1406	if (offset > i_size_read(inode))
1407		return ERR_PTR(-EINVAL);
1408
1409	/*
1410	 * This ensures that uprobe_copy_from_page(), copy_to_page() and
1411	 * __update_ref_ctr() can't cross page boundary.
1412	 */
1413	if (!IS_ALIGNED(offset, UPROBE_SWBP_INSN_SIZE))
1414		return ERR_PTR(-EINVAL);
1415	if (!IS_ALIGNED(ref_ctr_offset, sizeof(short)))
1416		return ERR_PTR(-EINVAL);
1417
1418	uprobe = alloc_uprobe(inode, offset, ref_ctr_offset);
1419	if (IS_ERR(uprobe))
1420		return uprobe;
1421
1422	down_write(&uprobe->register_rwsem);
1423	consumer_add(uprobe, uc);
1424	ret = register_for_each_vma(uprobe, uc);
1425	up_write(&uprobe->register_rwsem);
1426
1427	if (ret) {
1428		uprobe_unregister_nosync(uprobe, uc);
1429		/*
1430		 * Registration might have partially succeeded, so we can have
1431		 * this consumer being called right at this time. We need to
1432		 * sync here. It's ok, it's unlikely slow path.
1433		 */
1434		uprobe_unregister_sync();
1435		return ERR_PTR(ret);
1436	}
1437
1438	return uprobe;
1439}
1440EXPORT_SYMBOL_GPL(uprobe_register);
1441
1442/**
1443 * uprobe_apply - add or remove the breakpoints according to @uc->filter
1444 * @uprobe: uprobe which "owns" the breakpoint
1445 * @uc: consumer which wants to add more or remove some breakpoints
1446 * @add: add or remove the breakpoints
1447 * Return: 0 on success or negative error code.
1448 */
1449int uprobe_apply(struct uprobe *uprobe, struct uprobe_consumer *uc, bool add)
1450{
1451	struct uprobe_consumer *con;
1452	int ret = -ENOENT;
1453
1454	down_write(&uprobe->register_rwsem);
1455
1456	rcu_read_lock_trace();
1457	list_for_each_entry_rcu(con, &uprobe->consumers, cons_node, rcu_read_lock_trace_held()) {
1458		if (con == uc) {
1459			ret = register_for_each_vma(uprobe, add ? uc : NULL);
1460			break;
1461		}
1462	}
1463	rcu_read_unlock_trace();
1464
1465	up_write(&uprobe->register_rwsem);
1466
1467	return ret;
1468}
1469
1470static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm)
1471{
1472	VMA_ITERATOR(vmi, mm, 0);
1473	struct vm_area_struct *vma;
1474	int err = 0;
1475
1476	mmap_write_lock(mm);
1477	for_each_vma(vmi, vma) {
1478		unsigned long vaddr;
1479		loff_t offset;
1480
1481		if (!valid_vma(vma, false) ||
1482		    file_inode(vma->vm_file) != uprobe->inode)
1483			continue;
1484
1485		offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT;
1486		if (uprobe->offset <  offset ||
1487		    uprobe->offset >= offset + vma->vm_end - vma->vm_start)
1488			continue;
1489
1490		vaddr = offset_to_vaddr(vma, uprobe->offset);
1491		err |= remove_breakpoint(uprobe, vma, vaddr);
1492	}
1493	mmap_write_unlock(mm);
1494
1495	return err;
1496}
1497
1498static struct rb_node *
1499find_node_in_range(struct inode *inode, loff_t min, loff_t max)
1500{
1501	struct rb_node *n = uprobes_tree.rb_node;
1502
1503	while (n) {
1504		struct uprobe *u = rb_entry(n, struct uprobe, rb_node);
1505
1506		if (inode < u->inode) {
1507			n = n->rb_left;
1508		} else if (inode > u->inode) {
1509			n = n->rb_right;
1510		} else {
1511			if (max < u->offset)
1512				n = n->rb_left;
1513			else if (min > u->offset)
1514				n = n->rb_right;
1515			else
1516				break;
1517		}
1518	}
1519
1520	return n;
1521}
1522
1523/*
1524 * For a given range in vma, build a list of probes that need to be inserted.
1525 */
1526static void build_probe_list(struct inode *inode,
1527				struct vm_area_struct *vma,
1528				unsigned long start, unsigned long end,
1529				struct list_head *head)
1530{
1531	loff_t min, max;
1532	struct rb_node *n, *t;
1533	struct uprobe *u;
1534
1535	INIT_LIST_HEAD(head);
1536	min = vaddr_to_offset(vma, start);
1537	max = min + (end - start) - 1;
1538
1539	read_lock(&uprobes_treelock);
1540	n = find_node_in_range(inode, min, max);
1541	if (n) {
1542		for (t = n; t; t = rb_prev(t)) {
1543			u = rb_entry(t, struct uprobe, rb_node);
1544			if (u->inode != inode || u->offset < min)
1545				break;
1546			/* if uprobe went away, it's safe to ignore it */
1547			if (try_get_uprobe(u))
1548				list_add(&u->pending_list, head);
1549		}
1550		for (t = n; (t = rb_next(t)); ) {
1551			u = rb_entry(t, struct uprobe, rb_node);
1552			if (u->inode != inode || u->offset > max)
1553				break;
1554			/* if uprobe went away, it's safe to ignore it */
1555			if (try_get_uprobe(u))
1556				list_add(&u->pending_list, head);
1557		}
1558	}
1559	read_unlock(&uprobes_treelock);
1560}
1561
1562/* @vma contains reference counter, not the probed instruction. */
1563static int delayed_ref_ctr_inc(struct vm_area_struct *vma)
1564{
1565	struct list_head *pos, *q;
1566	struct delayed_uprobe *du;
1567	unsigned long vaddr;
1568	int ret = 0, err = 0;
1569
1570	mutex_lock(&delayed_uprobe_lock);
1571	list_for_each_safe(pos, q, &delayed_uprobe_list) {
1572		du = list_entry(pos, struct delayed_uprobe, list);
1573
1574		if (du->mm != vma->vm_mm ||
1575		    !valid_ref_ctr_vma(du->uprobe, vma))
1576			continue;
1577
1578		vaddr = offset_to_vaddr(vma, du->uprobe->ref_ctr_offset);
1579		ret = __update_ref_ctr(vma->vm_mm, vaddr, 1);
1580		if (ret) {
1581			update_ref_ctr_warn(du->uprobe, vma->vm_mm, 1);
1582			if (!err)
1583				err = ret;
1584		}
1585		delayed_uprobe_delete(du);
1586	}
1587	mutex_unlock(&delayed_uprobe_lock);
1588	return err;
1589}
1590
1591/*
1592 * Called from mmap_region/vma_merge with mm->mmap_lock acquired.
1593 *
1594 * Currently we ignore all errors and always return 0, the callers
1595 * can't handle the failure anyway.
1596 */
1597int uprobe_mmap(struct vm_area_struct *vma)
1598{
1599	struct list_head tmp_list;
1600	struct uprobe *uprobe, *u;
1601	struct inode *inode;
1602
1603	if (no_uprobe_events())
1604		return 0;
1605
1606	if (vma->vm_file &&
1607	    (vma->vm_flags & (VM_WRITE|VM_SHARED)) == VM_WRITE &&
1608	    mm_flags_test(MMF_HAS_UPROBES, vma->vm_mm))
1609		delayed_ref_ctr_inc(vma);
1610
1611	if (!valid_vma(vma, true))
1612		return 0;
1613
1614	inode = file_inode(vma->vm_file);
1615	if (!inode)
1616		return 0;
1617
1618	mutex_lock(uprobes_mmap_hash(inode));
1619	build_probe_list(inode, vma, vma->vm_start, vma->vm_end, &tmp_list);
1620	/*
1621	 * We can race with uprobe_unregister(), this uprobe can be already
1622	 * removed. But in this case filter_chain() must return false, all
1623	 * consumers have gone away.
1624	 */
1625	list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) {
1626		if (!fatal_signal_pending(current) &&
1627		    filter_chain(uprobe, vma->vm_mm)) {
1628			unsigned long vaddr = offset_to_vaddr(vma, uprobe->offset);
1629			install_breakpoint(uprobe, vma, vaddr);
1630		}
1631		put_uprobe(uprobe);
1632	}
1633	mutex_unlock(uprobes_mmap_hash(inode));
1634
1635	return 0;
1636}
1637
1638static bool
1639vma_has_uprobes(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1640{
1641	loff_t min, max;
1642	struct inode *inode;
1643	struct rb_node *n;
1644
1645	inode = file_inode(vma->vm_file);
1646
1647	min = vaddr_to_offset(vma, start);
1648	max = min + (end - start) - 1;
1649
1650	read_lock(&uprobes_treelock);
1651	n = find_node_in_range(inode, min, max);
1652	read_unlock(&uprobes_treelock);
1653
1654	return !!n;
1655}
1656
1657/*
1658 * Called in context of a munmap of a vma.
1659 */
1660void uprobe_munmap(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1661{
1662	if (no_uprobe_events() || !valid_vma(vma, false))
1663		return;
1664
1665	if (!atomic_read(&vma->vm_mm->mm_users)) /* called by mmput() ? */
1666		return;
1667
1668	if (!mm_flags_test(MMF_HAS_UPROBES, vma->vm_mm) ||
1669	     mm_flags_test(MMF_RECALC_UPROBES, vma->vm_mm))
1670		return;
1671
1672	if (vma_has_uprobes(vma, start, end))
1673		mm_flags_set(MMF_RECALC_UPROBES, vma->vm_mm);
1674}
1675
1676static vm_fault_t xol_fault(const struct vm_special_mapping *sm,
1677			    struct vm_area_struct *vma, struct vm_fault *vmf)
1678{
1679	struct xol_area *area = vma->vm_mm->uprobes_state.xol_area;
1680
1681	vmf->page = area->page;
1682	get_page(vmf->page);
1683	return 0;
1684}
1685
1686static int xol_mremap(const struct vm_special_mapping *sm, struct vm_area_struct *new_vma)
1687{
1688	return -EPERM;
1689}
1690
1691static const struct vm_special_mapping xol_mapping = {
1692	.name = "[uprobes]",
1693	.fault = xol_fault,
1694	.mremap = xol_mremap,
1695};
1696
1697unsigned long __weak arch_uprobe_get_xol_area(void)
1698{
1699	/* Try to map as high as possible, this is only a hint. */
1700	return get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE, PAGE_SIZE, 0, 0);
1701}
1702
1703/* Slot allocation for XOL */
1704static int xol_add_vma(struct mm_struct *mm, struct xol_area *area)
1705{
1706	struct vm_area_struct *vma;
1707	int ret;
1708
1709	if (mmap_write_lock_killable(mm))
1710		return -EINTR;
1711
1712	if (mm->uprobes_state.xol_area) {
1713		ret = -EALREADY;
1714		goto fail;
1715	}
1716
1717	if (!area->vaddr) {
1718		area->vaddr = arch_uprobe_get_xol_area();
1719		if (IS_ERR_VALUE(area->vaddr)) {
1720			ret = area->vaddr;
1721			goto fail;
1722		}
1723	}
1724
1725	vma = _install_special_mapping(mm, area->vaddr, PAGE_SIZE,
1726				VM_EXEC|VM_MAYEXEC|VM_DONTCOPY|VM_IO|
1727				VM_SEALED_SYSMAP,
1728				&xol_mapping);
1729	if (IS_ERR(vma)) {
1730		ret = PTR_ERR(vma);
1731		goto fail;
1732	}
1733
1734	ret = 0;
1735	/* pairs with get_xol_area() */
1736	smp_store_release(&mm->uprobes_state.xol_area, area); /* ^^^ */
1737 fail:
1738	mmap_write_unlock(mm);
1739
1740	return ret;
1741}
1742
1743void * __weak arch_uretprobe_trampoline(unsigned long *psize)
1744{
1745	static uprobe_opcode_t insn = UPROBE_SWBP_INSN;
1746
1747	*psize = UPROBE_SWBP_INSN_SIZE;
1748	return &insn;
1749}
1750
1751static struct xol_area *__create_xol_area(unsigned long vaddr)
1752{
1753	struct mm_struct *mm = current->mm;
1754	unsigned long insns_size;
1755	struct xol_area *area;
1756	void *insns;
1757
1758	area = kzalloc(sizeof(*area), GFP_KERNEL);
1759	if (unlikely(!area))
1760		goto out;
1761
1762	area->bitmap = kcalloc(BITS_TO_LONGS(UINSNS_PER_PAGE), sizeof(long),
1763			       GFP_KERNEL);
1764	if (!area->bitmap)
1765		goto free_area;
1766
1767	area->page = alloc_page(GFP_HIGHUSER | __GFP_ZERO);
1768	if (!area->page)
1769		goto free_bitmap;
1770
1771	area->vaddr = vaddr;
1772	init_waitqueue_head(&area->wq);
1773	/* Reserve the 1st slot for get_trampoline_vaddr() */
1774	set_bit(0, area->bitmap);
1775	insns = arch_uretprobe_trampoline(&insns_size);
1776	arch_uprobe_copy_ixol(area->page, 0, insns, insns_size);
1777
1778	if (!xol_add_vma(mm, area))
1779		return area;
1780
1781	__free_page(area->page);
1782 free_bitmap:
1783	kfree(area->bitmap);
1784 free_area:
1785	kfree(area);
1786 out:
1787	return NULL;
1788}
1789
1790/*
1791 * get_xol_area - Allocate process's xol_area if necessary.
1792 * This area will be used for storing instructions for execution out of line.
1793 *
1794 * Returns the allocated area or NULL.
1795 */
1796static struct xol_area *get_xol_area(void)
1797{
1798	struct mm_struct *mm = current->mm;
1799	struct xol_area *area;
1800
1801	if (!mm->uprobes_state.xol_area)
1802		__create_xol_area(0);
1803
1804	/* Pairs with xol_add_vma() smp_store_release() */
1805	area = READ_ONCE(mm->uprobes_state.xol_area); /* ^^^ */
1806	return area;
1807}
1808
1809void __weak arch_uprobe_clear_state(struct mm_struct *mm)
1810{
1811}
1812
1813void __weak arch_uprobe_init_state(struct mm_struct *mm)
1814{
1815}
1816
1817/*
1818 * uprobe_clear_state - Free the area allocated for slots.
1819 */
1820void uprobe_clear_state(struct mm_struct *mm)
1821{
1822	struct xol_area *area = mm->uprobes_state.xol_area;
1823
1824	mutex_lock(&delayed_uprobe_lock);
1825	delayed_uprobe_remove(NULL, mm);
1826	mutex_unlock(&delayed_uprobe_lock);
1827
1828	arch_uprobe_clear_state(mm);
1829
1830	if (!area)
1831		return;
1832
1833	put_page(area->page);
1834	kfree(area->bitmap);
1835	kfree(area);
1836}
1837
1838void uprobe_start_dup_mmap(void)
1839{
1840	percpu_down_read(&dup_mmap_sem);
1841}
1842
1843void uprobe_end_dup_mmap(void)
1844{
1845	percpu_up_read(&dup_mmap_sem);
1846}
1847
1848void uprobe_dup_mmap(struct mm_struct *oldmm, struct mm_struct *newmm)
1849{
1850	if (mm_flags_test(MMF_HAS_UPROBES, oldmm)) {
1851		mm_flags_set(MMF_HAS_UPROBES, newmm);
1852		/* unconditionally, dup_mmap() skips VM_DONTCOPY vmas */
1853		mm_flags_set(MMF_RECALC_UPROBES, newmm);
1854	}
1855}
1856
1857static unsigned long xol_get_slot_nr(struct xol_area *area)
1858{
1859	unsigned long slot_nr;
1860
1861	slot_nr = find_first_zero_bit(area->bitmap, UINSNS_PER_PAGE);
1862	if (slot_nr < UINSNS_PER_PAGE) {
1863		if (!test_and_set_bit(slot_nr, area->bitmap))
1864			return slot_nr;
1865	}
1866
1867	return UINSNS_PER_PAGE;
1868}
1869
1870/*
1871 * xol_get_insn_slot - allocate a slot for xol.
1872 */
1873static bool xol_get_insn_slot(struct uprobe *uprobe, struct uprobe_task *utask)
1874{
1875	struct xol_area *area = get_xol_area();
1876	unsigned long slot_nr;
1877
1878	if (!area)
1879		return false;
1880
1881	wait_event(area->wq, (slot_nr = xol_get_slot_nr(area)) < UINSNS_PER_PAGE);
1882
1883	utask->xol_vaddr = area->vaddr + slot_nr * UPROBE_XOL_SLOT_BYTES;
1884	arch_uprobe_copy_ixol(area->page, utask->xol_vaddr,
1885			      &uprobe->arch.ixol, sizeof(uprobe->arch.ixol));
1886	return true;
1887}
1888
1889/*
1890 * xol_free_insn_slot - free the slot allocated by xol_get_insn_slot()
1891 */
1892static void xol_free_insn_slot(struct uprobe_task *utask)
1893{
1894	struct xol_area *area = current->mm->uprobes_state.xol_area;
1895	unsigned long offset = utask->xol_vaddr - area->vaddr;
1896	unsigned int slot_nr;
1897
1898	utask->xol_vaddr = 0;
1899	/* xol_vaddr must fit into [area->vaddr, area->vaddr + PAGE_SIZE) */
1900	if (WARN_ON_ONCE(offset >= PAGE_SIZE))
1901		return;
1902
1903	slot_nr = offset / UPROBE_XOL_SLOT_BYTES;
1904	clear_bit(slot_nr, area->bitmap);
1905	smp_mb__after_atomic(); /* pairs with prepare_to_wait() */
1906	if (waitqueue_active(&area->wq))
1907		wake_up(&area->wq);
1908}
1909
1910void __weak arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr,
1911				  void *src, unsigned long len)
1912{
1913	/* Initialize the slot */
1914	copy_to_page(page, vaddr, src, len);
1915
1916	/*
1917	 * We probably need flush_icache_user_page() but it needs vma.
1918	 * This should work on most of architectures by default. If
1919	 * architecture needs to do something different it can define
1920	 * its own version of the function.
1921	 */
1922	flush_dcache_page(page);
1923}
1924
1925/**
1926 * uprobe_get_swbp_addr - compute address of swbp given post-swbp regs
1927 * @regs: Reflects the saved state of the task after it has hit a breakpoint
1928 * instruction.
1929 * Return the address of the breakpoint instruction.
1930 */
1931unsigned long __weak uprobe_get_swbp_addr(struct pt_regs *regs)
1932{
1933	return instruction_pointer(regs) - UPROBE_SWBP_INSN_SIZE;
1934}
1935
1936unsigned long uprobe_get_trap_addr(struct pt_regs *regs)
1937{
1938	struct uprobe_task *utask = current->utask;
1939
1940	if (unlikely(utask && utask->active_uprobe))
1941		return utask->vaddr;
1942
1943	return instruction_pointer(regs);
1944}
1945
1946static void ri_pool_push(struct uprobe_task *utask, struct return_instance *ri)
1947{
1948	ri->cons_cnt = 0;
1949	ri->next = utask->ri_pool;
1950	utask->ri_pool = ri;
1951}
1952
1953static struct return_instance *ri_pool_pop(struct uprobe_task *utask)
1954{
1955	struct return_instance *ri = utask->ri_pool;
1956
1957	if (likely(ri))
1958		utask->ri_pool = ri->next;
1959
1960	return ri;
1961}
1962
1963static void ri_free(struct return_instance *ri)
1964{
1965	kfree(ri->extra_consumers);
1966	kfree_rcu(ri, rcu);
1967}
1968
1969static void free_ret_instance(struct uprobe_task *utask,
1970			      struct return_instance *ri, bool cleanup_hprobe)
1971{
1972	unsigned seq;
1973
1974	if (cleanup_hprobe) {
1975		enum hprobe_state hstate;
1976
1977		(void)hprobe_consume(&ri->hprobe, &hstate);
1978		hprobe_finalize(&ri->hprobe, hstate);
1979	}
1980
1981	/*
1982	 * At this point return_instance is unlinked from utask's
1983	 * return_instances list and this has become visible to ri_timer().
1984	 * If seqcount now indicates that ri_timer's return instance
1985	 * processing loop isn't active, we can return ri into the pool of
1986	 * to-be-reused return instances for future uretprobes. If ri_timer()
1987	 * happens to be running right now, though, we fallback to safety and
1988	 * just perform RCU-delated freeing of ri.
1989	 * Admittedly, this is a rather simple use of seqcount, but it nicely
1990	 * abstracts away all the necessary memory barriers, so we use
1991	 * a well-supported kernel primitive here.
1992	 */
1993	if (raw_seqcount_try_begin(&utask->ri_seqcount, seq)) {
1994		/* immediate reuse of ri without RCU GP is OK */
1995		ri_pool_push(utask, ri);
1996	} else {
1997		/* we might be racing with ri_timer(), so play it safe */
1998		ri_free(ri);
1999	}
2000}
2001
2002/*
2003 * Called with no locks held.
2004 * Called in context of an exiting or an exec-ing thread.
2005 */
2006void uprobe_free_utask(struct task_struct *t)
2007{
2008	struct uprobe_task *utask = t->utask;
2009	struct return_instance *ri, *ri_next;
2010
2011	if (!utask)
2012		return;
2013
2014	t->utask = NULL;
2015	WARN_ON_ONCE(utask->active_uprobe || utask->xol_vaddr);
2016
2017	timer_delete_sync(&utask->ri_timer);
2018
2019	ri = utask->return_instances;
2020	while (ri) {
2021		ri_next = ri->next;
2022		free_ret_instance(utask, ri, true /* cleanup_hprobe */);
2023		ri = ri_next;
2024	}
2025
2026	/* free_ret_instance() above might add to ri_pool, so this loop should come last */
2027	ri = utask->ri_pool;
2028	while (ri) {
2029		ri_next = ri->next;
2030		ri_free(ri);
2031		ri = ri_next;
2032	}
2033
2034	kfree(utask);
2035}
2036
2037#define RI_TIMER_PERIOD (HZ / 10) /* 100 ms */
2038
2039#define for_each_ret_instance_rcu(pos, head) \
2040	for (pos = rcu_dereference_raw(head); pos; pos = rcu_dereference_raw(pos->next))
2041
2042static void ri_timer(struct timer_list *timer)
2043{
2044	struct uprobe_task *utask = container_of(timer, struct uprobe_task, ri_timer);
2045	struct return_instance *ri;
2046
2047	/* SRCU protects uprobe from reuse for the cmpxchg() inside hprobe_expire(). */
2048	guard(srcu)(&uretprobes_srcu);
2049	/* RCU protects return_instance from freeing. */
2050	guard(rcu)();
2051
2052	/*
2053	 * See free_ret_instance() for notes on seqcount use.
2054	 * We also employ raw API variants to avoid lockdep false-positive
2055	 * warning complaining about enabled preemption. The timer can only be
2056	 * invoked once for a uprobe_task. Therefore there can only be one
2057	 * writer. The reader does not require an even sequence count to make
2058	 * progress, so it is OK to remain preemptible on PREEMPT_RT.
2059	 */
2060	raw_write_seqcount_begin(&utask->ri_seqcount);
2061
2062	for_each_ret_instance_rcu(ri, utask->return_instances)
2063		hprobe_expire(&ri->hprobe, false);
2064
2065	raw_write_seqcount_end(&utask->ri_seqcount);
2066}
2067
2068static struct uprobe_task *alloc_utask(void)
2069{
2070	struct uprobe_task *utask;
2071
2072	utask = kzalloc(sizeof(*utask), GFP_KERNEL);
2073	if (!utask)
2074		return NULL;
2075
2076	timer_setup(&utask->ri_timer, ri_timer, 0);
2077	seqcount_init(&utask->ri_seqcount);
2078
2079	return utask;
2080}
2081
2082/*
2083 * Allocate a uprobe_task object for the task if necessary.
2084 * Called when the thread hits a breakpoint.
2085 *
2086 * Returns:
2087 * - pointer to new uprobe_task on success
2088 * - NULL otherwise
2089 */
2090static struct uprobe_task *get_utask(void)
2091{
2092	if (!current->utask)
2093		current->utask = alloc_utask();
2094	return current->utask;
2095}
2096
2097static struct return_instance *alloc_return_instance(struct uprobe_task *utask)
2098{
2099	struct return_instance *ri;
2100
2101	ri = ri_pool_pop(utask);
2102	if (ri)
2103		return ri;
2104
2105	ri = kzalloc(sizeof(*ri), GFP_KERNEL);
2106	if (!ri)
2107		return ZERO_SIZE_PTR;
2108
2109	return ri;
2110}
2111
2112static struct return_instance *dup_return_instance(struct return_instance *old)
2113{
2114	struct return_instance *ri;
2115
2116	ri = kmemdup(old, sizeof(*ri), GFP_KERNEL);
2117	if (!ri)
2118		return NULL;
2119
2120	if (unlikely(old->cons_cnt > 1)) {
2121		ri->extra_consumers = kmemdup(old->extra_consumers,
2122					      sizeof(ri->extra_consumers[0]) * (old->cons_cnt - 1),
2123					      GFP_KERNEL);
2124		if (!ri->extra_consumers) {
2125			kfree(ri);
2126			return NULL;
2127		}
2128	}
2129
2130	return ri;
2131}
2132
2133static int dup_utask(struct task_struct *t, struct uprobe_task *o_utask)
2134{
2135	struct uprobe_task *n_utask;
2136	struct return_instance **p, *o, *n;
2137	struct uprobe *uprobe;
2138
2139	n_utask = alloc_utask();
2140	if (!n_utask)
2141		return -ENOMEM;
2142	t->utask = n_utask;
2143
2144	/* protect uprobes from freeing, we'll need try_get_uprobe() them */
2145	guard(srcu)(&uretprobes_srcu);
2146
2147	p = &n_utask->return_instances;
2148	for (o = o_utask->return_instances; o; o = o->next) {
2149		n = dup_return_instance(o);
2150		if (!n)
2151			return -ENOMEM;
2152
2153		/* if uprobe is non-NULL, we'll have an extra refcount for uprobe */
2154		uprobe = hprobe_expire(&o->hprobe, true);
2155
2156		/*
2157		 * New utask will have stable properly refcounted uprobe or
2158		 * NULL. Even if we failed to get refcounted uprobe, we still
2159		 * need to preserve full set of return_instances for proper
2160		 * uretprobe handling and nesting in forked task.
2161		 */
2162		hprobe_init_stable(&n->hprobe, uprobe);
2163
2164		n->next = NULL;
2165		rcu_assign_pointer(*p, n);
2166		p = &n->next;
2167
2168		n_utask->depth++;
2169	}
2170
2171	return 0;
2172}
2173
2174static void dup_xol_work(struct callback_head *work)
2175{
2176	if (current->flags & PF_EXITING)
2177		return;
2178
2179	if (!__create_xol_area(current->utask->dup_xol_addr) &&
2180			!fatal_signal_pending(current))
2181		uprobe_warn(current, "dup xol area");
2182}
2183
2184/*
2185 * Called in context of a new clone/fork from copy_process.
2186 */
2187void uprobe_copy_process(struct task_struct *t, u64 flags)
2188{
2189	struct uprobe_task *utask = current->utask;
2190	struct mm_struct *mm = current->mm;
2191	struct xol_area *area;
2192
2193	t->utask = NULL;
2194
2195	if (!utask || !utask->return_instances)
2196		return;
2197
2198	if (mm == t->mm && !(flags & CLONE_VFORK))
2199		return;
2200
2201	if (dup_utask(t, utask))
2202		return uprobe_warn(t, "dup ret instances");
2203
2204	/* The task can fork() after dup_xol_work() fails */
2205	area = mm->uprobes_state.xol_area;
2206	if (!area)
2207		return uprobe_warn(t, "dup xol area");
2208
2209	if (mm == t->mm)
2210		return;
2211
2212	t->utask->dup_xol_addr = area->vaddr;
2213	init_task_work(&t->utask->dup_xol_work, dup_xol_work);
2214	task_work_add(t, &t->utask->dup_xol_work, TWA_RESUME);
2215}
2216
2217/*
2218 * Current area->vaddr notion assume the trampoline address is always
2219 * equal area->vaddr.
2220 *
2221 * Returns -1 in case the xol_area is not allocated.
2222 */
2223unsigned long uprobe_get_trampoline_vaddr(void)
2224{
2225	unsigned long trampoline_vaddr = UPROBE_NO_TRAMPOLINE_VADDR;
2226	struct xol_area *area;
2227
2228	/* Pairs with xol_add_vma() smp_store_release() */
2229	area = READ_ONCE(current->mm->uprobes_state.xol_area); /* ^^^ */
2230	if (area)
2231		trampoline_vaddr = area->vaddr;
2232
2233	return trampoline_vaddr;
2234}
2235
2236static void cleanup_return_instances(struct uprobe_task *utask, bool chained,
2237					struct pt_regs *regs)
2238{
2239	struct return_instance *ri = utask->return_instances, *ri_next;
2240	enum rp_check ctx = chained ? RP_CHECK_CHAIN_CALL : RP_CHECK_CALL;
2241
2242	while (ri && !arch_uretprobe_is_alive(ri, ctx, regs)) {
2243		ri_next = ri->next;
2244		rcu_assign_pointer(utask->return_instances, ri_next);
2245		utask->depth--;
2246
2247		free_ret_instance(utask, ri, true /* cleanup_hprobe */);
2248		ri = ri_next;
2249	}
2250}
2251
2252static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs,
2253			      struct return_instance *ri)
2254{
2255	struct uprobe_task *utask = current->utask;
2256	unsigned long orig_ret_vaddr, trampoline_vaddr;
2257	bool chained;
2258	int srcu_idx;
2259
2260	if (!get_xol_area())
2261		goto free;
2262
2263	if (utask->depth >= MAX_URETPROBE_DEPTH) {
2264		printk_ratelimited(KERN_INFO "uprobe: omit uretprobe due to"
2265				" nestedness limit pid/tgid=%d/%d\n",
2266				current->pid, current->tgid);
2267		goto free;
2268	}
2269
2270	trampoline_vaddr = uprobe_get_trampoline_vaddr();
2271	orig_ret_vaddr = arch_uretprobe_hijack_return_addr(trampoline_vaddr, regs);
2272	if (orig_ret_vaddr == -1)
2273		goto free;
2274
2275	/* drop the entries invalidated by longjmp() */
2276	chained = (orig_ret_vaddr == trampoline_vaddr);
2277	cleanup_return_instances(utask, chained, regs);
2278
2279	/*
2280	 * We don't want to keep trampoline address in stack, rather keep the
2281	 * original return address of first caller thru all the consequent
2282	 * instances. This also makes breakpoint unwrapping easier.
2283	 */
2284	if (chained) {
2285		if (!utask->return_instances) {
2286			/*
2287			 * This situation is not possible. Likely we have an
2288			 * attack from user-space.
2289			 */
2290			uprobe_warn(current, "handle tail call");
2291			goto free;
2292		}
2293		orig_ret_vaddr = utask->return_instances->orig_ret_vaddr;
2294	}
2295
2296	/* __srcu_read_lock() because SRCU lock survives switch to user space */
2297	srcu_idx = __srcu_read_lock(&uretprobes_srcu);
2298
2299	ri->func = instruction_pointer(regs);
2300	ri->stack = user_stack_pointer(regs);
2301	ri->orig_ret_vaddr = orig_ret_vaddr;
2302	ri->chained = chained;
2303
2304	utask->depth++;
2305
2306	hprobe_init_leased(&ri->hprobe, uprobe, srcu_idx);
2307	ri->next = utask->return_instances;
2308	rcu_assign_pointer(utask->return_instances, ri);
2309
2310	mod_timer(&utask->ri_timer, jiffies + RI_TIMER_PERIOD);
2311
2312	return;
2313free:
2314	ri_free(ri);
2315}
2316
2317/* Prepare to single-step probed instruction out of line. */
2318static int
2319pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long bp_vaddr)
2320{
2321	struct uprobe_task *utask = current->utask;
2322	int err;
2323
2324	if (!try_get_uprobe(uprobe))
2325		return -EINVAL;
2326
2327	if (!xol_get_insn_slot(uprobe, utask)) {
2328		err = -ENOMEM;
2329		goto err_out;
2330	}
2331
2332	utask->vaddr = bp_vaddr;
2333	err = arch_uprobe_pre_xol(&uprobe->arch, regs);
2334	if (unlikely(err)) {
2335		xol_free_insn_slot(utask);
2336		goto err_out;
2337	}
2338
2339	utask->active_uprobe = uprobe;
2340	utask->state = UTASK_SSTEP;
2341	return 0;
2342err_out:
2343	put_uprobe(uprobe);
2344	return err;
2345}
2346
2347/*
2348 * If we are singlestepping, then ensure this thread is not connected to
2349 * non-fatal signals until completion of singlestep.  When xol insn itself
2350 * triggers the signal,  restart the original insn even if the task is
2351 * already SIGKILL'ed (since coredump should report the correct ip).  This
2352 * is even more important if the task has a handler for SIGSEGV/etc, The
2353 * _same_ instruction should be repeated again after return from the signal
2354 * handler, and SSTEP can never finish in this case.
2355 */
2356bool uprobe_deny_signal(void)
2357{
2358	struct task_struct *t = current;
2359	struct uprobe_task *utask = t->utask;
2360
2361	if (likely(!utask || !utask->active_uprobe))
2362		return false;
2363
2364	WARN_ON_ONCE(utask->state != UTASK_SSTEP);
2365
2366	if (task_sigpending(t)) {
2367		utask->signal_denied = true;
2368		clear_tsk_thread_flag(t, TIF_SIGPENDING);
2369
2370		if (__fatal_signal_pending(t) || arch_uprobe_xol_was_trapped(t)) {
2371			utask->state = UTASK_SSTEP_TRAPPED;
2372			set_tsk_thread_flag(t, TIF_UPROBE);
2373		}
2374	}
2375
2376	return true;
2377}
2378
2379static void mmf_recalc_uprobes(struct mm_struct *mm)
2380{
2381	VMA_ITERATOR(vmi, mm, 0);
2382	struct vm_area_struct *vma;
2383
2384	for_each_vma(vmi, vma) {
2385		if (!valid_vma(vma, false))
2386			continue;
2387		/*
2388		 * This is not strictly accurate, we can race with
2389		 * uprobe_unregister() and see the already removed
2390		 * uprobe if delete_uprobe() was not yet called.
2391		 * Or this uprobe can be filtered out.
2392		 */
2393		if (vma_has_uprobes(vma, vma->vm_start, vma->vm_end))
2394			return;
2395	}
2396
2397	mm_flags_clear(MMF_HAS_UPROBES, mm);
2398}
2399
2400static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr)
2401{
2402	struct page *page;
2403	uprobe_opcode_t opcode;
2404	int result;
2405
2406	if (WARN_ON_ONCE(!IS_ALIGNED(vaddr, UPROBE_SWBP_INSN_SIZE)))
2407		return -EINVAL;
2408
2409	pagefault_disable();
2410	result = __get_user(opcode, (uprobe_opcode_t __user *)vaddr);
2411	pagefault_enable();
2412
2413	if (likely(result == 0))
2414		goto out;
2415
2416	result = get_user_pages(vaddr, 1, FOLL_FORCE, &page);
2417	if (result < 0)
2418		return result;
2419
2420	uprobe_copy_from_page(page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
2421	put_page(page);
2422 out:
2423	/* This needs to return true for any variant of the trap insn */
2424	return is_trap_insn(&opcode);
2425}
2426
2427static struct uprobe *find_active_uprobe_speculative(unsigned long bp_vaddr)
2428{
2429	struct mm_struct *mm = current->mm;
2430	struct uprobe *uprobe = NULL;
2431	struct vm_area_struct *vma;
2432	struct file *vm_file;
2433	loff_t offset;
2434	unsigned int seq;
2435
2436	guard(rcu)();
2437
2438	if (!mmap_lock_speculate_try_begin(mm, &seq))
2439		return NULL;
2440
2441	vma = vma_lookup(mm, bp_vaddr);
2442	if (!vma)
2443		return NULL;
2444
2445	/*
2446	 * vm_file memory can be reused for another instance of struct file,
2447	 * but can't be freed from under us, so it's safe to read fields from
2448	 * it, even if the values are some garbage values; ultimately
2449	 * find_uprobe_rcu() + mmap_lock_speculation_end() check will ensure
2450	 * that whatever we speculatively found is correct
2451	 */
2452	vm_file = READ_ONCE(vma->vm_file);
2453	if (!vm_file)
2454		return NULL;
2455
2456	offset = (loff_t)(vma->vm_pgoff << PAGE_SHIFT) + (bp_vaddr - vma->vm_start);
2457	uprobe = find_uprobe_rcu(vm_file->f_inode, offset);
2458	if (!uprobe)
2459		return NULL;
2460
2461	/* now double check that nothing about MM changed */
2462	if (mmap_lock_speculate_retry(mm, seq))
2463		return NULL;
2464
2465	return uprobe;
2466}
2467
2468/* assumes being inside RCU protected region */
2469static struct uprobe *find_active_uprobe_rcu(unsigned long bp_vaddr, int *is_swbp)
2470{
2471	struct mm_struct *mm = current->mm;
2472	struct uprobe *uprobe = NULL;
2473	struct vm_area_struct *vma;
2474
2475	uprobe = find_active_uprobe_speculative(bp_vaddr);
2476	if (uprobe)
2477		return uprobe;
2478
2479	mmap_read_lock(mm);
2480	vma = vma_lookup(mm, bp_vaddr);
2481	if (vma) {
2482		if (vma->vm_file) {
2483			struct inode *inode = file_inode(vma->vm_file);
2484			loff_t offset = vaddr_to_offset(vma, bp_vaddr);
2485
2486			uprobe = find_uprobe_rcu(inode, offset);
2487		}
2488
2489		if (!uprobe)
2490			*is_swbp = is_trap_at_addr(mm, bp_vaddr);
2491	} else {
2492		*is_swbp = -EFAULT;
2493	}
2494
2495	if (!uprobe && mm_flags_test_and_clear(MMF_RECALC_UPROBES, mm))
2496		mmf_recalc_uprobes(mm);
2497	mmap_read_unlock(mm);
2498
2499	return uprobe;
2500}
2501
2502static struct return_instance *push_consumer(struct return_instance *ri, __u64 id, __u64 cookie)
2503{
2504	struct return_consumer *ric;
2505
2506	if (unlikely(ri == ZERO_SIZE_PTR))
2507		return ri;
2508
2509	if (unlikely(ri->cons_cnt > 0)) {
2510		ric = krealloc(ri->extra_consumers, sizeof(*ric) * ri->cons_cnt, GFP_KERNEL);
2511		if (!ric) {
2512			ri_free(ri);
2513			return ZERO_SIZE_PTR;
2514		}
2515		ri->extra_consumers = ric;
2516	}
2517
2518	ric = likely(ri->cons_cnt == 0) ? &ri->consumer : &ri->extra_consumers[ri->cons_cnt - 1];
2519	ric->id = id;
2520	ric->cookie = cookie;
2521
2522	ri->cons_cnt++;
2523	return ri;
2524}
2525
2526static struct return_consumer *
2527return_consumer_find(struct return_instance *ri, int *iter, int id)
2528{
2529	struct return_consumer *ric;
2530	int idx;
2531
2532	for (idx = *iter; idx < ri->cons_cnt; idx++)
2533	{
2534		ric = likely(idx == 0) ? &ri->consumer : &ri->extra_consumers[idx - 1];
2535		if (ric->id == id) {
2536			*iter = idx + 1;
2537			return ric;
2538		}
2539	}
2540
2541	return NULL;
2542}
2543
2544static bool ignore_ret_handler(int rc)
2545{
2546	return rc == UPROBE_HANDLER_REMOVE || rc == UPROBE_HANDLER_IGNORE;
2547}
2548
2549static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs)
2550{
2551	struct uprobe_consumer *uc;
2552	bool has_consumers = false, remove = true;
2553	struct return_instance *ri = NULL;
2554	struct uprobe_task *utask = current->utask;
2555
2556	utask->auprobe = &uprobe->arch;
2557
2558	list_for_each_entry_rcu(uc, &uprobe->consumers, cons_node, rcu_read_lock_trace_held()) {
2559		bool session = uc->handler && uc->ret_handler;
2560		__u64 cookie = 0;
2561		int rc = 0;
2562
2563		if (uc->handler) {
2564			rc = uc->handler(uc, regs, &cookie);
2565			WARN(rc < 0 || rc > 2,
2566				"bad rc=0x%x from %ps()\n", rc, uc->handler);
2567		}
2568
2569		remove &= rc == UPROBE_HANDLER_REMOVE;
2570		has_consumers = true;
2571
2572		if (!uc->ret_handler || ignore_ret_handler(rc))
2573			continue;
2574
2575		if (!ri)
2576			ri = alloc_return_instance(utask);
2577
2578		if (session)
2579			ri = push_consumer(ri, uc->id, cookie);
2580	}
2581	utask->auprobe = NULL;
2582
2583	if (!ZERO_OR_NULL_PTR(ri))
2584		prepare_uretprobe(uprobe, regs, ri);
2585
2586	if (remove && has_consumers) {
2587		down_read(&uprobe->register_rwsem);
2588
2589		/* re-check that removal is still required, this time under lock */
2590		if (!filter_chain(uprobe, current->mm)) {
2591			WARN_ON(!uprobe_is_active(uprobe));
2592			unapply_uprobe(uprobe, current->mm);
2593		}
2594
2595		up_read(&uprobe->register_rwsem);
2596	}
2597}
2598
2599static void
2600handle_uretprobe_chain(struct return_instance *ri, struct uprobe *uprobe, struct pt_regs *regs)
2601{
2602	struct return_consumer *ric;
2603	struct uprobe_consumer *uc;
2604	int ric_idx = 0;
2605
2606	/* all consumers unsubscribed meanwhile */
2607	if (unlikely(!uprobe))
2608		return;
2609
2610	rcu_read_lock_trace();
2611	list_for_each_entry_rcu(uc, &uprobe->consumers, cons_node, rcu_read_lock_trace_held()) {
2612		bool session = uc->handler && uc->ret_handler;
2613
2614		if (uc->ret_handler) {
2615			ric = return_consumer_find(ri, &ric_idx, uc->id);
2616			if (!session || ric)
2617				uc->ret_handler(uc, ri->func, regs, ric ? &ric->cookie : NULL);
2618		}
2619	}
2620	rcu_read_unlock_trace();
2621}
2622
2623static struct return_instance *find_next_ret_chain(struct return_instance *ri)
2624{
2625	bool chained;
2626
2627	do {
2628		chained = ri->chained;
2629		ri = ri->next;	/* can't be NULL if chained */
2630	} while (chained);
2631
2632	return ri;
2633}
2634
2635void uprobe_handle_trampoline(struct pt_regs *regs)
2636{
2637	struct uprobe_task *utask;
2638	struct return_instance *ri, *ri_next, *next_chain;
2639	struct uprobe *uprobe;
2640	enum hprobe_state hstate;
2641	bool valid;
2642
2643	utask = current->utask;
2644	if (!utask)
2645		goto sigill;
2646
2647	ri = utask->return_instances;
2648	if (!ri)
2649		goto sigill;
2650
2651	do {
2652		/*
2653		 * We should throw out the frames invalidated by longjmp().
2654		 * If this chain is valid, then the next one should be alive
2655		 * or NULL; the latter case means that nobody but ri->func
2656		 * could hit this trampoline on return. TODO: sigaltstack().
2657		 */
2658		next_chain = find_next_ret_chain(ri);
2659		valid = !next_chain || arch_uretprobe_is_alive(next_chain, RP_CHECK_RET, regs);
2660
2661		instruction_pointer_set(regs, ri->orig_ret_vaddr);
2662		do {
2663			/* pop current instance from the stack of pending return instances,
2664			 * as it's not pending anymore: we just fixed up original
2665			 * instruction pointer in regs and are about to call handlers;
2666			 * this allows fixup_uretprobe_trampoline_entries() to properly fix up
2667			 * captured stack traces from uretprobe handlers, in which pending
2668			 * trampoline addresses on the stack are replaced with correct
2669			 * original return addresses
2670			 */
2671			ri_next = ri->next;
2672			rcu_assign_pointer(utask->return_instances, ri_next);
2673			utask->depth--;
2674
2675			uprobe = hprobe_consume(&ri->hprobe, &hstate);
2676			if (valid)
2677				handle_uretprobe_chain(ri, uprobe, regs);
2678			hprobe_finalize(&ri->hprobe, hstate);
2679
2680			/* We already took care of hprobe, no need to waste more time on that. */
2681			free_ret_instance(utask, ri, false /* !cleanup_hprobe */);
2682			ri = ri_next;
2683		} while (ri != next_chain);
2684	} while (!valid);
2685
2686	return;
2687
2688sigill:
2689	uprobe_warn(current, "handle uretprobe, sending SIGILL.");
2690	force_sig(SIGILL);
2691}
2692
2693bool __weak arch_uprobe_ignore(struct arch_uprobe *aup, struct pt_regs *regs)
2694{
2695	return false;
2696}
2697
2698bool __weak arch_uretprobe_is_alive(struct return_instance *ret, enum rp_check ctx,
2699					struct pt_regs *regs)
2700{
2701	return true;
2702}
2703
2704void __weak arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr)
2705{
2706}
2707
2708/*
2709 * Run handler and ask thread to singlestep.
2710 * Ensure all non-fatal signals cannot interrupt thread while it singlesteps.
2711 */
2712static void handle_swbp(struct pt_regs *regs)
2713{
2714	struct uprobe *uprobe;
2715	unsigned long bp_vaddr;
2716	int is_swbp;
2717
2718	bp_vaddr = uprobe_get_swbp_addr(regs);
2719	if (bp_vaddr == uprobe_get_trampoline_vaddr())
2720		return uprobe_handle_trampoline(regs);
2721
2722	rcu_read_lock_trace();
2723
2724	uprobe = find_active_uprobe_rcu(bp_vaddr, &is_swbp);
2725	if (!uprobe) {
2726		if (is_swbp > 0) {
2727			/* No matching uprobe; signal SIGTRAP. */
2728			force_sig(SIGTRAP);
2729		} else {
2730			/*
2731			 * Either we raced with uprobe_unregister() or we can't
2732			 * access this memory. The latter is only possible if
2733			 * another thread plays with our ->mm. In both cases
2734			 * we can simply restart. If this vma was unmapped we
2735			 * can pretend this insn was not executed yet and get
2736			 * the (correct) SIGSEGV after restart.
2737			 */
2738			instruction_pointer_set(regs, bp_vaddr);
2739		}
2740		goto out;
2741	}
2742
2743	/* change it in advance for ->handler() and restart */
2744	instruction_pointer_set(regs, bp_vaddr);
2745
2746	/*
2747	 * TODO: move copy_insn/etc into _register and remove this hack.
2748	 * After we hit the bp, _unregister + _register can install the
2749	 * new and not-yet-analyzed uprobe at the same address, restart.
2750	 */
2751	if (unlikely(!test_bit(UPROBE_COPY_INSN, &uprobe->flags)))
2752		goto out;
2753
2754	/*
2755	 * Pairs with the smp_wmb() in prepare_uprobe().
2756	 *
2757	 * Guarantees that if we see the UPROBE_COPY_INSN bit set, then
2758	 * we must also see the stores to &uprobe->arch performed by the
2759	 * prepare_uprobe() call.
2760	 */
2761	smp_rmb();
2762
2763	/* Tracing handlers use ->utask to communicate with fetch methods */
2764	if (!get_utask())
2765		goto out;
2766
2767	if (arch_uprobe_ignore(&uprobe->arch, regs))
2768		goto out;
2769
2770	handler_chain(uprobe, regs);
2771
2772	/* Try to optimize after first hit. */
2773	arch_uprobe_optimize(&uprobe->arch, bp_vaddr);
2774
2775	/*
2776	 * If user decided to take execution elsewhere, it makes little sense
2777	 * to execute the original instruction, so let's skip it.
2778	 */
2779	if (instruction_pointer(regs) != bp_vaddr)
2780		goto out;
2781
2782	if (arch_uprobe_skip_sstep(&uprobe->arch, regs))
2783		goto out;
2784
2785	if (pre_ssout(uprobe, regs, bp_vaddr))
2786		goto out;
2787
2788out:
2789	/* arch_uprobe_skip_sstep() succeeded, or restart if can't singlestep */
2790	rcu_read_unlock_trace();
2791}
2792
2793void handle_syscall_uprobe(struct pt_regs *regs, unsigned long bp_vaddr)
2794{
2795	struct uprobe *uprobe;
2796	int is_swbp;
2797
2798	guard(rcu_tasks_trace)();
2799
2800	uprobe = find_active_uprobe_rcu(bp_vaddr, &is_swbp);
2801	if (!uprobe)
2802		return;
2803	if (!get_utask())
2804		return;
2805	if (arch_uprobe_ignore(&uprobe->arch, regs))
2806		return;
2807	handler_chain(uprobe, regs);
2808}
2809
2810/*
2811 * Perform required fix-ups and disable singlestep.
2812 * Allow pending signals to take effect.
2813 */
2814static void handle_singlestep(struct uprobe_task *utask, struct pt_regs *regs)
2815{
2816	struct uprobe *uprobe;
2817	int err = 0;
2818
2819	uprobe = utask->active_uprobe;
2820	if (utask->state == UTASK_SSTEP_ACK)
2821		err = arch_uprobe_post_xol(&uprobe->arch, regs);
2822	else if (utask->state == UTASK_SSTEP_TRAPPED)
2823		arch_uprobe_abort_xol(&uprobe->arch, regs);
2824	else
2825		WARN_ON_ONCE(1);
2826
2827	put_uprobe(uprobe);
2828	utask->active_uprobe = NULL;
2829	utask->state = UTASK_RUNNING;
2830	xol_free_insn_slot(utask);
2831
2832	if (utask->signal_denied) {
2833		set_thread_flag(TIF_SIGPENDING);
2834		utask->signal_denied = false;
2835	}
2836
2837	if (unlikely(err)) {
2838		uprobe_warn(current, "execute the probed insn, sending SIGILL.");
2839		force_sig(SIGILL);
2840	}
2841}
2842
2843/*
2844 * On breakpoint hit, breakpoint notifier sets the TIF_UPROBE flag and
2845 * allows the thread to return from interrupt. After that handle_swbp()
2846 * sets utask->active_uprobe.
2847 *
2848 * On singlestep exception, singlestep notifier sets the TIF_UPROBE flag
2849 * and allows the thread to return from interrupt.
2850 *
2851 * While returning to userspace, thread notices the TIF_UPROBE flag and calls
2852 * uprobe_notify_resume().
2853 */
2854void uprobe_notify_resume(struct pt_regs *regs)
2855{
2856	struct uprobe_task *utask;
2857
2858	clear_thread_flag(TIF_UPROBE);
2859
2860	utask = current->utask;
2861	if (utask && utask->active_uprobe)
2862		handle_singlestep(utask, regs);
2863	else
2864		handle_swbp(regs);
2865}
2866
2867/*
2868 * uprobe_pre_sstep_notifier gets called from interrupt context as part of
2869 * notifier mechanism. Set TIF_UPROBE flag and indicate breakpoint hit.
2870 */
2871int uprobe_pre_sstep_notifier(struct pt_regs *regs)
2872{
2873	if (!current->mm)
2874		return 0;
2875
2876	if (!mm_flags_test(MMF_HAS_UPROBES, current->mm) &&
2877	    (!current->utask || !current->utask->return_instances))
2878		return 0;
2879
2880	set_thread_flag(TIF_UPROBE);
2881	return 1;
2882}
2883
2884/*
2885 * uprobe_post_sstep_notifier gets called in interrupt context as part of notifier
2886 * mechanism. Set TIF_UPROBE flag and indicate completion of singlestep.
2887 */
2888int uprobe_post_sstep_notifier(struct pt_regs *regs)
2889{
2890	struct uprobe_task *utask = current->utask;
2891
2892	if (!current->mm || !utask || !utask->active_uprobe)
2893		/* task is currently not uprobed */
2894		return 0;
2895
2896	utask->state = UTASK_SSTEP_ACK;
2897	set_thread_flag(TIF_UPROBE);
2898	return 1;
2899}
2900
2901static struct notifier_block uprobe_exception_nb = {
2902	.notifier_call		= arch_uprobe_exception_notify,
2903	.priority		= INT_MAX-1,	/* notified after kprobes, kgdb */
2904};
2905
2906void __init uprobes_init(void)
2907{
2908	int i;
2909
2910	for (i = 0; i < UPROBES_HASH_SZ; i++)
2911		mutex_init(&uprobes_mmap_mutex[i]);
2912
2913	BUG_ON(register_die_notifier(&uprobe_exception_nb));
2914}