syscall.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
   3 */
   4#include <crypto/sha2.h>
   5#include <linux/bpf.h>
   6#include <linux/bpf-cgroup.h>
   7#include <linux/bpf_trace.h>
   8#include <linux/bpf_lirc.h>
   9#include <linux/bpf_verifier.h>
  10#include <linux/bsearch.h>
  11#include <linux/btf.h>
  12#include <linux/syscalls.h>
  13#include <linux/slab.h>
  14#include <linux/sched/signal.h>
  15#include <linux/vmalloc.h>
  16#include <linux/mmzone.h>
  17#include <linux/anon_inodes.h>
  18#include <linux/fdtable.h>
  19#include <linux/file.h>
  20#include <linux/fs.h>
  21#include <linux/license.h>
  22#include <linux/filter.h>
  23#include <linux/kernel.h>
  24#include <linux/idr.h>
  25#include <linux/cred.h>
  26#include <linux/timekeeping.h>
  27#include <linux/ctype.h>
  28#include <linux/nospec.h>
  29#include <linux/audit.h>
  30#include <uapi/linux/btf.h>
  31#include <linux/pgtable.h>
  32#include <linux/bpf_lsm.h>
  33#include <linux/poll.h>
  34#include <linux/sort.h>
  35#include <linux/bpf-netns.h>
  36#include <linux/rcupdate_trace.h>
  37#include <linux/memcontrol.h>
  38#include <linux/trace_events.h>
  39#include <linux/tracepoint.h>
  40#include <linux/overflow.h>
  41#include <linux/cookie.h>
  42#include <linux/verification.h>
  43
  44#include <net/netfilter/nf_bpf_link.h>
  45#include <net/netkit.h>
  46#include <net/tcx.h>
  47
  48#define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
  49			  (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \
  50			  (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
  51#define IS_FD_PROG_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY)
  52#define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)
  53#define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map) || \
  54			IS_FD_HASH(map))
  55
  56#define BPF_OBJ_FLAG_MASK   (BPF_F_RDONLY | BPF_F_WRONLY)
  57
  58DEFINE_PER_CPU(int, bpf_prog_active);
  59DEFINE_COOKIE(bpf_map_cookie);
  60static DEFINE_IDR(prog_idr);
  61static DEFINE_SPINLOCK(prog_idr_lock);
  62static DEFINE_IDR(map_idr);
  63static DEFINE_SPINLOCK(map_idr_lock);
  64static DEFINE_IDR(link_idr);
  65static DEFINE_SPINLOCK(link_idr_lock);
  66
  67int sysctl_unprivileged_bpf_disabled __read_mostly =
  68	IS_BUILTIN(CONFIG_BPF_UNPRIV_DEFAULT_OFF) ? 2 : 0;
  69
  70static const struct bpf_map_ops * const bpf_map_types[] = {
  71#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type)
  72#define BPF_MAP_TYPE(_id, _ops) \
  73	[_id] = &_ops,
  74#define BPF_LINK_TYPE(_id, _name)
  75#include <linux/bpf_types.h>
  76#undef BPF_PROG_TYPE
  77#undef BPF_MAP_TYPE
  78#undef BPF_LINK_TYPE
  79};
  80
  81/*
  82 * If we're handed a bigger struct than we know of, ensure all the unknown bits
  83 * are 0 - i.e. new user-space does not rely on any kernel feature extensions
  84 * we don't know about yet.
  85 *
  86 * There is a ToCToU between this function call and the following
  87 * copy_from_user() call. However, this is not a concern since this function is
  88 * meant to be a future-proofing of bits.
  89 */
  90int bpf_check_uarg_tail_zero(bpfptr_t uaddr,
  91			     size_t expected_size,
  92			     size_t actual_size)
  93{
  94	int res;
  95
  96	if (unlikely(actual_size > PAGE_SIZE))	/* silly large */
  97		return -E2BIG;
  98
  99	if (actual_size <= expected_size)
 100		return 0;
 101
 102	if (uaddr.is_kernel)
 103		res = memchr_inv(uaddr.kernel + expected_size, 0,
 104				 actual_size - expected_size) == NULL;
 105	else
 106		res = check_zeroed_user(uaddr.user + expected_size,
 107					actual_size - expected_size);
 108	if (res < 0)
 109		return res;
 110	return res ? 0 : -E2BIG;
 111}
 112
 113const struct bpf_map_ops bpf_map_offload_ops = {
 114	.map_meta_equal = bpf_map_meta_equal,
 115	.map_alloc = bpf_map_offload_map_alloc,
 116	.map_free = bpf_map_offload_map_free,
 117	.map_check_btf = map_check_no_btf,
 118	.map_mem_usage = bpf_map_offload_map_mem_usage,
 119};
 120
 121static void bpf_map_write_active_inc(struct bpf_map *map)
 122{
 123	atomic64_inc(&map->writecnt);
 124}
 125
 126static void bpf_map_write_active_dec(struct bpf_map *map)
 127{
 128	atomic64_dec(&map->writecnt);
 129}
 130
 131bool bpf_map_write_active(const struct bpf_map *map)
 132{
 133	return atomic64_read(&map->writecnt) != 0;
 134}
 135
 136static u32 bpf_map_value_size(const struct bpf_map *map)
 137{
 138	if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 139	    map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
 140	    map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
 141	    map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
 142		return round_up(map->value_size, 8) * num_possible_cpus();
 143	else if (IS_FD_MAP(map))
 144		return sizeof(u32);
 145	else
 146		return  map->value_size;
 147}
 148
 149static void maybe_wait_bpf_programs(struct bpf_map *map)
 150{
 151	/* Wait for any running non-sleepable BPF programs to complete so that
 152	 * userspace, when we return to it, knows that all non-sleepable
 153	 * programs that could be running use the new map value. For sleepable
 154	 * BPF programs, synchronize_rcu_tasks_trace() should be used to wait
 155	 * for the completions of these programs, but considering the waiting
 156	 * time can be very long and userspace may think it will hang forever,
 157	 * so don't handle sleepable BPF programs now.
 158	 */
 159	if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS ||
 160	    map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
 161		synchronize_rcu_expedited();
 162}
 163
 164static void unpin_uptr_kaddr(void *kaddr)
 165{
 166	if (kaddr)
 167		unpin_user_page(virt_to_page(kaddr));
 168}
 169
 170static void __bpf_obj_unpin_uptrs(struct btf_record *rec, u32 cnt, void *obj)
 171{
 172	const struct btf_field *field;
 173	void **uptr_addr;
 174	int i;
 175
 176	for (i = 0, field = rec->fields; i < cnt; i++, field++) {
 177		if (field->type != BPF_UPTR)
 178			continue;
 179
 180		uptr_addr = obj + field->offset;
 181		unpin_uptr_kaddr(*uptr_addr);
 182	}
 183}
 184
 185static void bpf_obj_unpin_uptrs(struct btf_record *rec, void *obj)
 186{
 187	if (!btf_record_has_field(rec, BPF_UPTR))
 188		return;
 189
 190	__bpf_obj_unpin_uptrs(rec, rec->cnt, obj);
 191}
 192
 193static int bpf_obj_pin_uptrs(struct btf_record *rec, void *obj)
 194{
 195	const struct btf_field *field;
 196	const struct btf_type *t;
 197	unsigned long start, end;
 198	struct page *page;
 199	void **uptr_addr;
 200	int i, err;
 201
 202	if (!btf_record_has_field(rec, BPF_UPTR))
 203		return 0;
 204
 205	for (i = 0, field = rec->fields; i < rec->cnt; i++, field++) {
 206		if (field->type != BPF_UPTR)
 207			continue;
 208
 209		uptr_addr = obj + field->offset;
 210		start = *(unsigned long *)uptr_addr;
 211		if (!start)
 212			continue;
 213
 214		t = btf_type_by_id(field->kptr.btf, field->kptr.btf_id);
 215		/* t->size was checked for zero before */
 216		if (check_add_overflow(start, t->size - 1, &end)) {
 217			err = -EFAULT;
 218			goto unpin_all;
 219		}
 220
 221		/* The uptr's struct cannot span across two pages */
 222		if ((start & PAGE_MASK) != (end & PAGE_MASK)) {
 223			err = -EOPNOTSUPP;
 224			goto unpin_all;
 225		}
 226
 227		err = pin_user_pages_fast(start, 1, FOLL_LONGTERM | FOLL_WRITE, &page);
 228		if (err != 1)
 229			goto unpin_all;
 230
 231		if (PageHighMem(page)) {
 232			err = -EOPNOTSUPP;
 233			unpin_user_page(page);
 234			goto unpin_all;
 235		}
 236
 237		*uptr_addr = page_address(page) + offset_in_page(start);
 238	}
 239
 240	return 0;
 241
 242unpin_all:
 243	__bpf_obj_unpin_uptrs(rec, i, obj);
 244	return err;
 245}
 246
 247static int bpf_map_update_value(struct bpf_map *map, struct file *map_file,
 248				void *key, void *value, __u64 flags)
 249{
 250	int err;
 251
 252	/* Need to create a kthread, thus must support schedule */
 253	if (bpf_map_is_offloaded(map)) {
 254		return bpf_map_offload_update_elem(map, key, value, flags);
 255	} else if (map->map_type == BPF_MAP_TYPE_CPUMAP ||
 256		   map->map_type == BPF_MAP_TYPE_ARENA ||
 257		   map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
 258		return map->ops->map_update_elem(map, key, value, flags);
 259	} else if (map->map_type == BPF_MAP_TYPE_SOCKHASH ||
 260		   map->map_type == BPF_MAP_TYPE_SOCKMAP) {
 261		return sock_map_update_elem_sys(map, key, value, flags);
 262	} else if (IS_FD_PROG_ARRAY(map)) {
 263		return bpf_fd_array_map_update_elem(map, map_file, key, value,
 264						    flags);
 265	}
 266
 267	bpf_disable_instrumentation();
 268	if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 269	    map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
 270		err = bpf_percpu_hash_update(map, key, value, flags);
 271	} else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
 272		err = bpf_percpu_array_update(map, key, value, flags);
 273	} else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
 274		err = bpf_percpu_cgroup_storage_update(map, key, value,
 275						       flags);
 276	} else if (IS_FD_ARRAY(map)) {
 277		err = bpf_fd_array_map_update_elem(map, map_file, key, value,
 278						   flags);
 279	} else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {
 280		err = bpf_fd_htab_map_update_elem(map, map_file, key, value,
 281						  flags);
 282	} else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
 283		/* rcu_read_lock() is not needed */
 284		err = bpf_fd_reuseport_array_update_elem(map, key, value,
 285							 flags);
 286	} else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
 287		   map->map_type == BPF_MAP_TYPE_STACK ||
 288		   map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) {
 289		err = map->ops->map_push_elem(map, value, flags);
 290	} else {
 291		err = bpf_obj_pin_uptrs(map->record, value);
 292		if (!err) {
 293			rcu_read_lock();
 294			err = map->ops->map_update_elem(map, key, value, flags);
 295			rcu_read_unlock();
 296			if (err)
 297				bpf_obj_unpin_uptrs(map->record, value);
 298		}
 299	}
 300	bpf_enable_instrumentation();
 301
 302	return err;
 303}
 304
 305static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value,
 306			      __u64 flags)
 307{
 308	void *ptr;
 309	int err;
 310
 311	if (bpf_map_is_offloaded(map))
 312		return bpf_map_offload_lookup_elem(map, key, value);
 313
 314	bpf_disable_instrumentation();
 315	if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 316	    map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
 317		err = bpf_percpu_hash_copy(map, key, value);
 318	} else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
 319		err = bpf_percpu_array_copy(map, key, value);
 320	} else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
 321		err = bpf_percpu_cgroup_storage_copy(map, key, value);
 322	} else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
 323		err = bpf_stackmap_extract(map, key, value, false);
 324	} else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) {
 325		err = bpf_fd_array_map_lookup_elem(map, key, value);
 326	} else if (IS_FD_HASH(map)) {
 327		err = bpf_fd_htab_map_lookup_elem(map, key, value);
 328	} else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
 329		err = bpf_fd_reuseport_array_lookup_elem(map, key, value);
 330	} else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
 331		   map->map_type == BPF_MAP_TYPE_STACK ||
 332		   map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) {
 333		err = map->ops->map_peek_elem(map, value);
 334	} else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
 335		/* struct_ops map requires directly updating "value" */
 336		err = bpf_struct_ops_map_sys_lookup_elem(map, key, value);
 337	} else {
 338		rcu_read_lock();
 339		if (map->ops->map_lookup_elem_sys_only)
 340			ptr = map->ops->map_lookup_elem_sys_only(map, key);
 341		else
 342			ptr = map->ops->map_lookup_elem(map, key);
 343		if (IS_ERR(ptr)) {
 344			err = PTR_ERR(ptr);
 345		} else if (!ptr) {
 346			err = -ENOENT;
 347		} else {
 348			err = 0;
 349			if (flags & BPF_F_LOCK)
 350				/* lock 'ptr' and copy everything but lock */
 351				copy_map_value_locked(map, value, ptr, true);
 352			else
 353				copy_map_value(map, value, ptr);
 354			/* mask lock and timer, since value wasn't zero inited */
 355			check_and_init_map_value(map, value);
 356		}
 357		rcu_read_unlock();
 358	}
 359
 360	bpf_enable_instrumentation();
 361
 362	return err;
 363}
 364
 365/* Please, do not use this function outside from the map creation path
 366 * (e.g. in map update path) without taking care of setting the active
 367 * memory cgroup (see at bpf_map_kmalloc_node() for example).
 368 */
 369static void *__bpf_map_area_alloc(u64 size, int numa_node, bool mmapable)
 370{
 371	/* We really just want to fail instead of triggering OOM killer
 372	 * under memory pressure, therefore we set __GFP_NORETRY to kmalloc,
 373	 * which is used for lower order allocation requests.
 374	 *
 375	 * It has been observed that higher order allocation requests done by
 376	 * vmalloc with __GFP_NORETRY being set might fail due to not trying
 377	 * to reclaim memory from the page cache, thus we set
 378	 * __GFP_RETRY_MAYFAIL to avoid such situations.
 379	 */
 380
 381	gfp_t gfp = bpf_memcg_flags(__GFP_NOWARN | __GFP_ZERO);
 382	unsigned int flags = 0;
 383	unsigned long align = 1;
 384	void *area;
 385
 386	if (size >= SIZE_MAX)
 387		return NULL;
 388
 389	/* kmalloc()'ed memory can't be mmap()'ed */
 390	if (mmapable) {
 391		BUG_ON(!PAGE_ALIGNED(size));
 392		align = SHMLBA;
 393		flags = VM_USERMAP;
 394	} else if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {
 395		area = kmalloc_node(size, gfp | GFP_USER | __GFP_NORETRY,
 396				    numa_node);
 397		if (area != NULL)
 398			return area;
 399	}
 400
 401	return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END,
 402			gfp | GFP_KERNEL | __GFP_RETRY_MAYFAIL, PAGE_KERNEL,
 403			flags, numa_node, __builtin_return_address(0));
 404}
 405
 406void *bpf_map_area_alloc(u64 size, int numa_node)
 407{
 408	return __bpf_map_area_alloc(size, numa_node, false);
 409}
 410
 411void *bpf_map_area_mmapable_alloc(u64 size, int numa_node)
 412{
 413	return __bpf_map_area_alloc(size, numa_node, true);
 414}
 415
 416void bpf_map_area_free(void *area)
 417{
 418	kvfree(area);
 419}
 420
 421static u32 bpf_map_flags_retain_permanent(u32 flags)
 422{
 423	/* Some map creation flags are not tied to the map object but
 424	 * rather to the map fd instead, so they have no meaning upon
 425	 * map object inspection since multiple file descriptors with
 426	 * different (access) properties can exist here. Thus, given
 427	 * this has zero meaning for the map itself, lets clear these
 428	 * from here.
 429	 */
 430	return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY);
 431}
 432
 433void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr)
 434{
 435	map->map_type = attr->map_type;
 436	map->key_size = attr->key_size;
 437	map->value_size = attr->value_size;
 438	map->max_entries = attr->max_entries;
 439	map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags);
 440	map->numa_node = bpf_map_attr_numa_node(attr);
 441	map->map_extra = attr->map_extra;
 442}
 443
 444static int bpf_map_alloc_id(struct bpf_map *map)
 445{
 446	int id;
 447
 448	idr_preload(GFP_KERNEL);
 449	spin_lock_bh(&map_idr_lock);
 450	id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC);
 451	if (id > 0)
 452		map->id = id;
 453	spin_unlock_bh(&map_idr_lock);
 454	idr_preload_end();
 455
 456	if (WARN_ON_ONCE(!id))
 457		return -ENOSPC;
 458
 459	return id > 0 ? 0 : id;
 460}
 461
 462void bpf_map_free_id(struct bpf_map *map)
 463{
 464	unsigned long flags;
 465
 466	/* Offloaded maps are removed from the IDR store when their device
 467	 * disappears - even if someone holds an fd to them they are unusable,
 468	 * the memory is gone, all ops will fail; they are simply waiting for
 469	 * refcnt to drop to be freed.
 470	 */
 471	if (!map->id)
 472		return;
 473
 474	spin_lock_irqsave(&map_idr_lock, flags);
 475
 476	idr_remove(&map_idr, map->id);
 477	map->id = 0;
 478
 479	spin_unlock_irqrestore(&map_idr_lock, flags);
 480}
 481
 482#ifdef CONFIG_MEMCG
 483static void bpf_map_save_memcg(struct bpf_map *map)
 484{
 485	/* Currently if a map is created by a process belonging to the root
 486	 * memory cgroup, get_obj_cgroup_from_current() will return NULL.
 487	 * So we have to check map->objcg for being NULL each time it's
 488	 * being used.
 489	 */
 490	if (memcg_bpf_enabled())
 491		map->objcg = get_obj_cgroup_from_current();
 492}
 493
 494static void bpf_map_release_memcg(struct bpf_map *map)
 495{
 496	if (map->objcg)
 497		obj_cgroup_put(map->objcg);
 498}
 499
 500static struct mem_cgroup *bpf_map_get_memcg(const struct bpf_map *map)
 501{
 502	if (map->objcg)
 503		return get_mem_cgroup_from_objcg(map->objcg);
 504
 505	return root_mem_cgroup;
 506}
 507
 508void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,
 509			   int node)
 510{
 511	struct mem_cgroup *memcg, *old_memcg;
 512	void *ptr;
 513
 514	memcg = bpf_map_get_memcg(map);
 515	old_memcg = set_active_memcg(memcg);
 516	ptr = kmalloc_node(size, flags | __GFP_ACCOUNT, node);
 517	set_active_memcg(old_memcg);
 518	mem_cgroup_put(memcg);
 519
 520	return ptr;
 521}
 522
 523void *bpf_map_kmalloc_nolock(const struct bpf_map *map, size_t size, gfp_t flags,
 524			     int node)
 525{
 526	struct mem_cgroup *memcg, *old_memcg;
 527	void *ptr;
 528
 529	memcg = bpf_map_get_memcg(map);
 530	old_memcg = set_active_memcg(memcg);
 531	ptr = kmalloc_nolock(size, flags | __GFP_ACCOUNT, node);
 532	set_active_memcg(old_memcg);
 533	mem_cgroup_put(memcg);
 534
 535	return ptr;
 536}
 537
 538void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags)
 539{
 540	struct mem_cgroup *memcg, *old_memcg;
 541	void *ptr;
 542
 543	memcg = bpf_map_get_memcg(map);
 544	old_memcg = set_active_memcg(memcg);
 545	ptr = kzalloc(size, flags | __GFP_ACCOUNT);
 546	set_active_memcg(old_memcg);
 547	mem_cgroup_put(memcg);
 548
 549	return ptr;
 550}
 551
 552void *bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size,
 553		       gfp_t flags)
 554{
 555	struct mem_cgroup *memcg, *old_memcg;
 556	void *ptr;
 557
 558	memcg = bpf_map_get_memcg(map);
 559	old_memcg = set_active_memcg(memcg);
 560	ptr = kvcalloc(n, size, flags | __GFP_ACCOUNT);
 561	set_active_memcg(old_memcg);
 562	mem_cgroup_put(memcg);
 563
 564	return ptr;
 565}
 566
 567void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size,
 568				    size_t align, gfp_t flags)
 569{
 570	struct mem_cgroup *memcg, *old_memcg;
 571	void __percpu *ptr;
 572
 573	memcg = bpf_map_get_memcg(map);
 574	old_memcg = set_active_memcg(memcg);
 575	ptr = __alloc_percpu_gfp(size, align, flags | __GFP_ACCOUNT);
 576	set_active_memcg(old_memcg);
 577	mem_cgroup_put(memcg);
 578
 579	return ptr;
 580}
 581
 582#else
 583static void bpf_map_save_memcg(struct bpf_map *map)
 584{
 585}
 586
 587static void bpf_map_release_memcg(struct bpf_map *map)
 588{
 589}
 590#endif
 591
 592static bool can_alloc_pages(void)
 593{
 594	return preempt_count() == 0 && !irqs_disabled() &&
 595		!IS_ENABLED(CONFIG_PREEMPT_RT);
 596}
 597
 598static struct page *__bpf_alloc_page(int nid)
 599{
 600	if (!can_alloc_pages())
 601		return alloc_pages_nolock(__GFP_ACCOUNT, nid, 0);
 602
 603	return alloc_pages_node(nid,
 604				GFP_KERNEL | __GFP_ZERO | __GFP_ACCOUNT
 605				| __GFP_NOWARN,
 606				0);
 607}
 608
 609int bpf_map_alloc_pages(const struct bpf_map *map, int nid,
 610			unsigned long nr_pages, struct page **pages)
 611{
 612	unsigned long i, j;
 613	struct page *pg;
 614	int ret = 0;
 615#ifdef CONFIG_MEMCG
 616	struct mem_cgroup *memcg, *old_memcg;
 617
 618	memcg = bpf_map_get_memcg(map);
 619	old_memcg = set_active_memcg(memcg);
 620#endif
 621	for (i = 0; i < nr_pages; i++) {
 622		pg = __bpf_alloc_page(nid);
 623
 624		if (pg) {
 625			pages[i] = pg;
 626			continue;
 627		}
 628		for (j = 0; j < i; j++)
 629			free_pages_nolock(pages[j], 0);
 630		ret = -ENOMEM;
 631		break;
 632	}
 633
 634#ifdef CONFIG_MEMCG
 635	set_active_memcg(old_memcg);
 636	mem_cgroup_put(memcg);
 637#endif
 638	return ret;
 639}
 640
 641
 642static int btf_field_cmp(const void *a, const void *b)
 643{
 644	const struct btf_field *f1 = a, *f2 = b;
 645
 646	if (f1->offset < f2->offset)
 647		return -1;
 648	else if (f1->offset > f2->offset)
 649		return 1;
 650	return 0;
 651}
 652
 653struct btf_field *btf_record_find(const struct btf_record *rec, u32 offset,
 654				  u32 field_mask)
 655{
 656	struct btf_field *field;
 657
 658	if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & field_mask))
 659		return NULL;
 660	field = bsearch(&offset, rec->fields, rec->cnt, sizeof(rec->fields[0]), btf_field_cmp);
 661	if (!field || !(field->type & field_mask))
 662		return NULL;
 663	return field;
 664}
 665
 666void btf_record_free(struct btf_record *rec)
 667{
 668	int i;
 669
 670	if (IS_ERR_OR_NULL(rec))
 671		return;
 672	for (i = 0; i < rec->cnt; i++) {
 673		switch (rec->fields[i].type) {
 674		case BPF_KPTR_UNREF:
 675		case BPF_KPTR_REF:
 676		case BPF_KPTR_PERCPU:
 677		case BPF_UPTR:
 678			if (rec->fields[i].kptr.module)
 679				module_put(rec->fields[i].kptr.module);
 680			if (btf_is_kernel(rec->fields[i].kptr.btf))
 681				btf_put(rec->fields[i].kptr.btf);
 682			break;
 683		case BPF_LIST_HEAD:
 684		case BPF_LIST_NODE:
 685		case BPF_RB_ROOT:
 686		case BPF_RB_NODE:
 687		case BPF_SPIN_LOCK:
 688		case BPF_RES_SPIN_LOCK:
 689		case BPF_TIMER:
 690		case BPF_REFCOUNT:
 691		case BPF_WORKQUEUE:
 692		case BPF_TASK_WORK:
 693			/* Nothing to release */
 694			break;
 695		default:
 696			WARN_ON_ONCE(1);
 697			continue;
 698		}
 699	}
 700	kfree(rec);
 701}
 702
 703void bpf_map_free_record(struct bpf_map *map)
 704{
 705	btf_record_free(map->record);
 706	map->record = NULL;
 707}
 708
 709struct btf_record *btf_record_dup(const struct btf_record *rec)
 710{
 711	const struct btf_field *fields;
 712	struct btf_record *new_rec;
 713	int ret, size, i;
 714
 715	if (IS_ERR_OR_NULL(rec))
 716		return NULL;
 717	size = struct_size(rec, fields, rec->cnt);
 718	new_rec = kmemdup(rec, size, GFP_KERNEL | __GFP_NOWARN);
 719	if (!new_rec)
 720		return ERR_PTR(-ENOMEM);
 721	/* Do a deep copy of the btf_record */
 722	fields = rec->fields;
 723	new_rec->cnt = 0;
 724	for (i = 0; i < rec->cnt; i++) {
 725		switch (fields[i].type) {
 726		case BPF_KPTR_UNREF:
 727		case BPF_KPTR_REF:
 728		case BPF_KPTR_PERCPU:
 729		case BPF_UPTR:
 730			if (btf_is_kernel(fields[i].kptr.btf))
 731				btf_get(fields[i].kptr.btf);
 732			if (fields[i].kptr.module && !try_module_get(fields[i].kptr.module)) {
 733				ret = -ENXIO;
 734				goto free;
 735			}
 736			break;
 737		case BPF_LIST_HEAD:
 738		case BPF_LIST_NODE:
 739		case BPF_RB_ROOT:
 740		case BPF_RB_NODE:
 741		case BPF_SPIN_LOCK:
 742		case BPF_RES_SPIN_LOCK:
 743		case BPF_TIMER:
 744		case BPF_REFCOUNT:
 745		case BPF_WORKQUEUE:
 746		case BPF_TASK_WORK:
 747			/* Nothing to acquire */
 748			break;
 749		default:
 750			ret = -EFAULT;
 751			WARN_ON_ONCE(1);
 752			goto free;
 753		}
 754		new_rec->cnt++;
 755	}
 756	return new_rec;
 757free:
 758	btf_record_free(new_rec);
 759	return ERR_PTR(ret);
 760}
 761
 762bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *rec_b)
 763{
 764	bool a_has_fields = !IS_ERR_OR_NULL(rec_a), b_has_fields = !IS_ERR_OR_NULL(rec_b);
 765	int size;
 766
 767	if (!a_has_fields && !b_has_fields)
 768		return true;
 769	if (a_has_fields != b_has_fields)
 770		return false;
 771	if (rec_a->cnt != rec_b->cnt)
 772		return false;
 773	size = struct_size(rec_a, fields, rec_a->cnt);
 774	/* btf_parse_fields uses kzalloc to allocate a btf_record, so unused
 775	 * members are zeroed out. So memcmp is safe to do without worrying
 776	 * about padding/unused fields.
 777	 *
 778	 * While spin_lock, timer, and kptr have no relation to map BTF,
 779	 * list_head metadata is specific to map BTF, the btf and value_rec
 780	 * members in particular. btf is the map BTF, while value_rec points to
 781	 * btf_record in that map BTF.
 782	 *
 783	 * So while by default, we don't rely on the map BTF (which the records
 784	 * were parsed from) matching for both records, which is not backwards
 785	 * compatible, in case list_head is part of it, we implicitly rely on
 786	 * that by way of depending on memcmp succeeding for it.
 787	 */
 788	return !memcmp(rec_a, rec_b, size);
 789}
 790
 791void bpf_obj_free_timer(const struct btf_record *rec, void *obj)
 792{
 793	if (WARN_ON_ONCE(!btf_record_has_field(rec, BPF_TIMER)))
 794		return;
 795	bpf_timer_cancel_and_free(obj + rec->timer_off);
 796}
 797
 798void bpf_obj_free_workqueue(const struct btf_record *rec, void *obj)
 799{
 800	if (WARN_ON_ONCE(!btf_record_has_field(rec, BPF_WORKQUEUE)))
 801		return;
 802	bpf_wq_cancel_and_free(obj + rec->wq_off);
 803}
 804
 805void bpf_obj_free_task_work(const struct btf_record *rec, void *obj)
 806{
 807	if (WARN_ON_ONCE(!btf_record_has_field(rec, BPF_TASK_WORK)))
 808		return;
 809	bpf_task_work_cancel_and_free(obj + rec->task_work_off);
 810}
 811
 812void bpf_obj_free_fields(const struct btf_record *rec, void *obj)
 813{
 814	const struct btf_field *fields;
 815	int i;
 816
 817	if (IS_ERR_OR_NULL(rec))
 818		return;
 819	fields = rec->fields;
 820	for (i = 0; i < rec->cnt; i++) {
 821		struct btf_struct_meta *pointee_struct_meta;
 822		const struct btf_field *field = &fields[i];
 823		void *field_ptr = obj + field->offset;
 824		void *xchgd_field;
 825
 826		switch (fields[i].type) {
 827		case BPF_SPIN_LOCK:
 828		case BPF_RES_SPIN_LOCK:
 829			break;
 830		case BPF_TIMER:
 831			bpf_timer_cancel_and_free(field_ptr);
 832			break;
 833		case BPF_WORKQUEUE:
 834			bpf_wq_cancel_and_free(field_ptr);
 835			break;
 836		case BPF_TASK_WORK:
 837			bpf_task_work_cancel_and_free(field_ptr);
 838			break;
 839		case BPF_KPTR_UNREF:
 840			WRITE_ONCE(*(u64 *)field_ptr, 0);
 841			break;
 842		case BPF_KPTR_REF:
 843		case BPF_KPTR_PERCPU:
 844			xchgd_field = (void *)xchg((unsigned long *)field_ptr, 0);
 845			if (!xchgd_field)
 846				break;
 847
 848			if (!btf_is_kernel(field->kptr.btf)) {
 849				pointee_struct_meta = btf_find_struct_meta(field->kptr.btf,
 850									   field->kptr.btf_id);
 851				__bpf_obj_drop_impl(xchgd_field, pointee_struct_meta ?
 852								 pointee_struct_meta->record : NULL,
 853								 fields[i].type == BPF_KPTR_PERCPU);
 854			} else {
 855				field->kptr.dtor(xchgd_field);
 856			}
 857			break;
 858		case BPF_UPTR:
 859			/* The caller ensured that no one is using the uptr */
 860			unpin_uptr_kaddr(*(void **)field_ptr);
 861			break;
 862		case BPF_LIST_HEAD:
 863			if (WARN_ON_ONCE(rec->spin_lock_off < 0))
 864				continue;
 865			bpf_list_head_free(field, field_ptr, obj + rec->spin_lock_off);
 866			break;
 867		case BPF_RB_ROOT:
 868			if (WARN_ON_ONCE(rec->spin_lock_off < 0))
 869				continue;
 870			bpf_rb_root_free(field, field_ptr, obj + rec->spin_lock_off);
 871			break;
 872		case BPF_LIST_NODE:
 873		case BPF_RB_NODE:
 874		case BPF_REFCOUNT:
 875			break;
 876		default:
 877			WARN_ON_ONCE(1);
 878			continue;
 879		}
 880	}
 881}
 882
 883static void bpf_map_free(struct bpf_map *map)
 884{
 885	struct btf_record *rec = map->record;
 886	struct btf *btf = map->btf;
 887
 888	/* implementation dependent freeing. Disabling migration to simplify
 889	 * the free of values or special fields allocated from bpf memory
 890	 * allocator.
 891	 */
 892	kfree(map->excl_prog_sha);
 893	migrate_disable();
 894	map->ops->map_free(map);
 895	migrate_enable();
 896
 897	/* Delay freeing of btf_record for maps, as map_free
 898	 * callback usually needs access to them. It is better to do it here
 899	 * than require each callback to do the free itself manually.
 900	 *
 901	 * Note that the btf_record stashed in map->inner_map_meta->record was
 902	 * already freed using the map_free callback for map in map case which
 903	 * eventually calls bpf_map_free_meta, since inner_map_meta is only a
 904	 * template bpf_map struct used during verification.
 905	 */
 906	btf_record_free(rec);
 907	/* Delay freeing of btf for maps, as map_free callback may need
 908	 * struct_meta info which will be freed with btf_put().
 909	 */
 910	btf_put(btf);
 911}
 912
 913/* called from workqueue */
 914static void bpf_map_free_deferred(struct work_struct *work)
 915{
 916	struct bpf_map *map = container_of(work, struct bpf_map, work);
 917
 918	security_bpf_map_free(map);
 919	bpf_map_release_memcg(map);
 920	bpf_map_owner_free(map);
 921	bpf_map_free(map);
 922}
 923
 924static void bpf_map_put_uref(struct bpf_map *map)
 925{
 926	if (atomic64_dec_and_test(&map->usercnt)) {
 927		if (map->ops->map_release_uref)
 928			map->ops->map_release_uref(map);
 929	}
 930}
 931
 932static void bpf_map_free_in_work(struct bpf_map *map)
 933{
 934	INIT_WORK(&map->work, bpf_map_free_deferred);
 935	/* Avoid spawning kworkers, since they all might contend
 936	 * for the same mutex like slab_mutex.
 937	 */
 938	queue_work(system_dfl_wq, &map->work);
 939}
 940
 941static void bpf_map_free_rcu_gp(struct rcu_head *rcu)
 942{
 943	bpf_map_free_in_work(container_of(rcu, struct bpf_map, rcu));
 944}
 945
 946static void bpf_map_free_mult_rcu_gp(struct rcu_head *rcu)
 947{
 948	if (rcu_trace_implies_rcu_gp())
 949		bpf_map_free_rcu_gp(rcu);
 950	else
 951		call_rcu(rcu, bpf_map_free_rcu_gp);
 952}
 953
 954/* decrement map refcnt and schedule it for freeing via workqueue
 955 * (underlying map implementation ops->map_free() might sleep)
 956 */
 957void bpf_map_put(struct bpf_map *map)
 958{
 959	if (atomic64_dec_and_test(&map->refcnt)) {
 960		/* bpf_map_free_id() must be called first */
 961		bpf_map_free_id(map);
 962
 963		WARN_ON_ONCE(atomic64_read(&map->sleepable_refcnt));
 964		if (READ_ONCE(map->free_after_mult_rcu_gp))
 965			call_rcu_tasks_trace(&map->rcu, bpf_map_free_mult_rcu_gp);
 966		else if (READ_ONCE(map->free_after_rcu_gp))
 967			call_rcu(&map->rcu, bpf_map_free_rcu_gp);
 968		else
 969			bpf_map_free_in_work(map);
 970	}
 971}
 972EXPORT_SYMBOL_GPL(bpf_map_put);
 973
 974void bpf_map_put_with_uref(struct bpf_map *map)
 975{
 976	bpf_map_put_uref(map);
 977	bpf_map_put(map);
 978}
 979
 980static int bpf_map_release(struct inode *inode, struct file *filp)
 981{
 982	struct bpf_map *map = filp->private_data;
 983
 984	if (map->ops->map_release)
 985		map->ops->map_release(map, filp);
 986
 987	bpf_map_put_with_uref(map);
 988	return 0;
 989}
 990
 991static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f)
 992{
 993	fmode_t mode = fd_file(f)->f_mode;
 994
 995	/* Our file permissions may have been overridden by global
 996	 * map permissions facing syscall side.
 997	 */
 998	if (READ_ONCE(map->frozen))
 999		mode &= ~FMODE_CAN_WRITE;
1000	return mode;
1001}
1002
1003#ifdef CONFIG_PROC_FS
1004/* Show the memory usage of a bpf map */
1005static u64 bpf_map_memory_usage(const struct bpf_map *map)
1006{
1007	return map->ops->map_mem_usage(map);
1008}
1009
1010static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp)
1011{
1012	struct bpf_map *map = filp->private_data;
1013	u32 type = 0, jited = 0;
1014
1015	spin_lock(&map->owner_lock);
1016	if (map->owner) {
1017		type  = map->owner->type;
1018		jited = map->owner->jited;
1019	}
1020	spin_unlock(&map->owner_lock);
1021
1022	seq_printf(m,
1023		   "map_type:\t%u\n"
1024		   "key_size:\t%u\n"
1025		   "value_size:\t%u\n"
1026		   "max_entries:\t%u\n"
1027		   "map_flags:\t%#x\n"
1028		   "map_extra:\t%#llx\n"
1029		   "memlock:\t%llu\n"
1030		   "map_id:\t%u\n"
1031		   "frozen:\t%u\n",
1032		   map->map_type,
1033		   map->key_size,
1034		   map->value_size,
1035		   map->max_entries,
1036		   map->map_flags,
1037		   (unsigned long long)map->map_extra,
1038		   bpf_map_memory_usage(map),
1039		   map->id,
1040		   READ_ONCE(map->frozen));
1041	if (type) {
1042		seq_printf(m, "owner_prog_type:\t%u\n", type);
1043		seq_printf(m, "owner_jited:\t%u\n", jited);
1044	}
1045}
1046#endif
1047
1048static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz,
1049			      loff_t *ppos)
1050{
1051	/* We need this handler such that alloc_file() enables
1052	 * f_mode with FMODE_CAN_READ.
1053	 */
1054	return -EINVAL;
1055}
1056
1057static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf,
1058			       size_t siz, loff_t *ppos)
1059{
1060	/* We need this handler such that alloc_file() enables
1061	 * f_mode with FMODE_CAN_WRITE.
1062	 */
1063	return -EINVAL;
1064}
1065
1066/* called for any extra memory-mapped regions (except initial) */
1067static void bpf_map_mmap_open(struct vm_area_struct *vma)
1068{
1069	struct bpf_map *map = vma->vm_file->private_data;
1070
1071	if (vma->vm_flags & VM_MAYWRITE)
1072		bpf_map_write_active_inc(map);
1073}
1074
1075/* called for all unmapped memory region (including initial) */
1076static void bpf_map_mmap_close(struct vm_area_struct *vma)
1077{
1078	struct bpf_map *map = vma->vm_file->private_data;
1079
1080	if (vma->vm_flags & VM_MAYWRITE)
1081		bpf_map_write_active_dec(map);
1082}
1083
1084static const struct vm_operations_struct bpf_map_default_vmops = {
1085	.open		= bpf_map_mmap_open,
1086	.close		= bpf_map_mmap_close,
1087};
1088
1089static int bpf_map_mmap(struct file *filp, struct vm_area_struct *vma)
1090{
1091	struct bpf_map *map = filp->private_data;
1092	int err = 0;
1093
1094	if (!map->ops->map_mmap || !IS_ERR_OR_NULL(map->record))
1095		return -ENOTSUPP;
1096
1097	if (!(vma->vm_flags & VM_SHARED))
1098		return -EINVAL;
1099
1100	mutex_lock(&map->freeze_mutex);
1101
1102	if (vma->vm_flags & VM_WRITE) {
1103		if (map->frozen) {
1104			err = -EPERM;
1105			goto out;
1106		}
1107		/* map is meant to be read-only, so do not allow mapping as
1108		 * writable, because it's possible to leak a writable page
1109		 * reference and allows user-space to still modify it after
1110		 * freezing, while verifier will assume contents do not change
1111		 */
1112		if (map->map_flags & BPF_F_RDONLY_PROG) {
1113			err = -EACCES;
1114			goto out;
1115		}
1116		bpf_map_write_active_inc(map);
1117	}
1118out:
1119	mutex_unlock(&map->freeze_mutex);
1120	if (err)
1121		return err;
1122
1123	/* set default open/close callbacks */
1124	vma->vm_ops = &bpf_map_default_vmops;
1125	vma->vm_private_data = map;
1126	vm_flags_clear(vma, VM_MAYEXEC);
1127	/* If mapping is read-only, then disallow potentially re-mapping with
1128	 * PROT_WRITE by dropping VM_MAYWRITE flag. This VM_MAYWRITE clearing
1129	 * means that as far as BPF map's memory-mapped VMAs are concerned,
1130	 * VM_WRITE and VM_MAYWRITE and equivalent, if one of them is set,
1131	 * both should be set, so we can forget about VM_MAYWRITE and always
1132	 * check just VM_WRITE
1133	 */
1134	if (!(vma->vm_flags & VM_WRITE))
1135		vm_flags_clear(vma, VM_MAYWRITE);
1136
1137	err = map->ops->map_mmap(map, vma);
1138	if (err) {
1139		if (vma->vm_flags & VM_WRITE)
1140			bpf_map_write_active_dec(map);
1141	}
1142
1143	return err;
1144}
1145
1146static __poll_t bpf_map_poll(struct file *filp, struct poll_table_struct *pts)
1147{
1148	struct bpf_map *map = filp->private_data;
1149
1150	if (map->ops->map_poll)
1151		return map->ops->map_poll(map, filp, pts);
1152
1153	return EPOLLERR;
1154}
1155
1156static unsigned long bpf_get_unmapped_area(struct file *filp, unsigned long addr,
1157					   unsigned long len, unsigned long pgoff,
1158					   unsigned long flags)
1159{
1160	struct bpf_map *map = filp->private_data;
1161
1162	if (map->ops->map_get_unmapped_area)
1163		return map->ops->map_get_unmapped_area(filp, addr, len, pgoff, flags);
1164#ifdef CONFIG_MMU
1165	return mm_get_unmapped_area(filp, addr, len, pgoff, flags);
1166#else
1167	return addr;
1168#endif
1169}
1170
1171const struct file_operations bpf_map_fops = {
1172#ifdef CONFIG_PROC_FS
1173	.show_fdinfo	= bpf_map_show_fdinfo,
1174#endif
1175	.release	= bpf_map_release,
1176	.read		= bpf_dummy_read,
1177	.write		= bpf_dummy_write,
1178	.mmap		= bpf_map_mmap,
1179	.poll		= bpf_map_poll,
1180	.get_unmapped_area = bpf_get_unmapped_area,
1181};
1182
1183int bpf_map_new_fd(struct bpf_map *map, int flags)
1184{
1185	int ret;
1186
1187	ret = security_bpf_map(map, OPEN_FMODE(flags));
1188	if (ret < 0)
1189		return ret;
1190
1191	return anon_inode_getfd("bpf-map", &bpf_map_fops, map,
1192				flags | O_CLOEXEC);
1193}
1194
1195int bpf_get_file_flag(int flags)
1196{
1197	if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY))
1198		return -EINVAL;
1199	if (flags & BPF_F_RDONLY)
1200		return O_RDONLY;
1201	if (flags & BPF_F_WRONLY)
1202		return O_WRONLY;
1203	return O_RDWR;
1204}
1205
1206/* helper macro to check that unused fields 'union bpf_attr' are zero */
1207#define CHECK_ATTR(CMD) \
1208	memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
1209		   sizeof(attr->CMD##_LAST_FIELD), 0, \
1210		   sizeof(*attr) - \
1211		   offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
1212		   sizeof(attr->CMD##_LAST_FIELD)) != NULL
1213
1214/* dst and src must have at least "size" number of bytes.
1215 * Return strlen on success and < 0 on error.
1216 */
1217int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size)
1218{
1219	const char *end = src + size;
1220	const char *orig_src = src;
1221
1222	memset(dst, 0, size);
1223	/* Copy all isalnum(), '_' and '.' chars. */
1224	while (src < end && *src) {
1225		if (!isalnum(*src) &&
1226		    *src != '_' && *src != '.')
1227			return -EINVAL;
1228		*dst++ = *src++;
1229	}
1230
1231	/* No '\0' found in "size" number of bytes */
1232	if (src == end)
1233		return -EINVAL;
1234
1235	return src - orig_src;
1236}
1237EXPORT_SYMBOL_GPL(bpf_obj_name_cpy);
1238
1239int map_check_no_btf(const struct bpf_map *map,
1240		     const struct btf *btf,
1241		     const struct btf_type *key_type,
1242		     const struct btf_type *value_type)
1243{
1244	return -ENOTSUPP;
1245}
1246
1247static int map_check_btf(struct bpf_map *map, struct bpf_token *token,
1248			 const struct btf *btf, u32 btf_key_id, u32 btf_value_id)
1249{
1250	const struct btf_type *key_type, *value_type;
1251	u32 key_size, value_size;
1252	int ret = 0;
1253
1254	/* Some maps allow key to be unspecified. */
1255	if (btf_key_id) {
1256		key_type = btf_type_id_size(btf, &btf_key_id, &key_size);
1257		if (!key_type || key_size != map->key_size)
1258			return -EINVAL;
1259	} else {
1260		key_type = btf_type_by_id(btf, 0);
1261		if (!map->ops->map_check_btf)
1262			return -EINVAL;
1263	}
1264
1265	value_type = btf_type_id_size(btf, &btf_value_id, &value_size);
1266	if (!value_type || value_size != map->value_size)
1267		return -EINVAL;
1268
1269	map->record = btf_parse_fields(btf, value_type,
1270				       BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK | BPF_TIMER | BPF_KPTR | BPF_LIST_HEAD |
1271				       BPF_RB_ROOT | BPF_REFCOUNT | BPF_WORKQUEUE | BPF_UPTR |
1272				       BPF_TASK_WORK,
1273				       map->value_size);
1274	if (!IS_ERR_OR_NULL(map->record)) {
1275		int i;
1276
1277		if (!bpf_token_capable(token, CAP_BPF)) {
1278			ret = -EPERM;
1279			goto free_map_tab;
1280		}
1281		if (map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) {
1282			ret = -EACCES;
1283			goto free_map_tab;
1284		}
1285		for (i = 0; i < sizeof(map->record->field_mask) * 8; i++) {
1286			switch (map->record->field_mask & (1 << i)) {
1287			case 0:
1288				continue;
1289			case BPF_SPIN_LOCK:
1290			case BPF_RES_SPIN_LOCK:
1291				if (map->map_type != BPF_MAP_TYPE_HASH &&
1292				    map->map_type != BPF_MAP_TYPE_ARRAY &&
1293				    map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
1294				    map->map_type != BPF_MAP_TYPE_SK_STORAGE &&
1295				    map->map_type != BPF_MAP_TYPE_INODE_STORAGE &&
1296				    map->map_type != BPF_MAP_TYPE_TASK_STORAGE &&
1297				    map->map_type != BPF_MAP_TYPE_CGRP_STORAGE) {
1298					ret = -EOPNOTSUPP;
1299					goto free_map_tab;
1300				}
1301				break;
1302			case BPF_TIMER:
1303			case BPF_WORKQUEUE:
1304			case BPF_TASK_WORK:
1305				if (map->map_type != BPF_MAP_TYPE_HASH &&
1306				    map->map_type != BPF_MAP_TYPE_LRU_HASH &&
1307				    map->map_type != BPF_MAP_TYPE_ARRAY) {
1308					ret = -EOPNOTSUPP;
1309					goto free_map_tab;
1310				}
1311				break;
1312			case BPF_KPTR_UNREF:
1313			case BPF_KPTR_REF:
1314			case BPF_KPTR_PERCPU:
1315			case BPF_REFCOUNT:
1316				if (map->map_type != BPF_MAP_TYPE_HASH &&
1317				    map->map_type != BPF_MAP_TYPE_PERCPU_HASH &&
1318				    map->map_type != BPF_MAP_TYPE_LRU_HASH &&
1319				    map->map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH &&
1320				    map->map_type != BPF_MAP_TYPE_ARRAY &&
1321				    map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY &&
1322				    map->map_type != BPF_MAP_TYPE_SK_STORAGE &&
1323				    map->map_type != BPF_MAP_TYPE_INODE_STORAGE &&
1324				    map->map_type != BPF_MAP_TYPE_TASK_STORAGE &&
1325				    map->map_type != BPF_MAP_TYPE_CGRP_STORAGE) {
1326					ret = -EOPNOTSUPP;
1327					goto free_map_tab;
1328				}
1329				break;
1330			case BPF_UPTR:
1331				if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE) {
1332					ret = -EOPNOTSUPP;
1333					goto free_map_tab;
1334				}
1335				break;
1336			case BPF_LIST_HEAD:
1337			case BPF_RB_ROOT:
1338				if (map->map_type != BPF_MAP_TYPE_HASH &&
1339				    map->map_type != BPF_MAP_TYPE_LRU_HASH &&
1340				    map->map_type != BPF_MAP_TYPE_ARRAY) {
1341					ret = -EOPNOTSUPP;
1342					goto free_map_tab;
1343				}
1344				break;
1345			default:
1346				/* Fail if map_type checks are missing for a field type */
1347				ret = -EOPNOTSUPP;
1348				goto free_map_tab;
1349			}
1350		}
1351	}
1352
1353	ret = btf_check_and_fixup_fields(btf, map->record);
1354	if (ret < 0)
1355		goto free_map_tab;
1356
1357	if (map->ops->map_check_btf) {
1358		ret = map->ops->map_check_btf(map, btf, key_type, value_type);
1359		if (ret < 0)
1360			goto free_map_tab;
1361	}
1362
1363	return ret;
1364free_map_tab:
1365	bpf_map_free_record(map);
1366	return ret;
1367}
1368
1369#define BPF_MAP_CREATE_LAST_FIELD excl_prog_hash_size
1370/* called via syscall */
1371static int map_create(union bpf_attr *attr, bpfptr_t uattr)
1372{
1373	const struct bpf_map_ops *ops;
1374	struct bpf_token *token = NULL;
1375	int numa_node = bpf_map_attr_numa_node(attr);
1376	u32 map_type = attr->map_type;
1377	struct bpf_map *map;
1378	bool token_flag;
1379	int f_flags;
1380	int err;
1381
1382	err = CHECK_ATTR(BPF_MAP_CREATE);
1383	if (err)
1384		return -EINVAL;
1385
1386	/* check BPF_F_TOKEN_FD flag, remember if it's set, and then clear it
1387	 * to avoid per-map type checks tripping on unknown flag
1388	 */
1389	token_flag = attr->map_flags & BPF_F_TOKEN_FD;
1390	attr->map_flags &= ~BPF_F_TOKEN_FD;
1391
1392	if (attr->btf_vmlinux_value_type_id) {
1393		if (attr->map_type != BPF_MAP_TYPE_STRUCT_OPS ||
1394		    attr->btf_key_type_id || attr->btf_value_type_id)
1395			return -EINVAL;
1396	} else if (attr->btf_key_type_id && !attr->btf_value_type_id) {
1397		return -EINVAL;
1398	}
1399
1400	if (attr->map_type != BPF_MAP_TYPE_BLOOM_FILTER &&
1401	    attr->map_type != BPF_MAP_TYPE_ARENA &&
1402	    attr->map_extra != 0)
1403		return -EINVAL;
1404
1405	f_flags = bpf_get_file_flag(attr->map_flags);
1406	if (f_flags < 0)
1407		return f_flags;
1408
1409	if (numa_node != NUMA_NO_NODE &&
1410	    ((unsigned int)numa_node >= nr_node_ids ||
1411	     !node_online(numa_node)))
1412		return -EINVAL;
1413
1414	/* find map type and init map: hashtable vs rbtree vs bloom vs ... */
1415	map_type = attr->map_type;
1416	if (map_type >= ARRAY_SIZE(bpf_map_types))
1417		return -EINVAL;
1418	map_type = array_index_nospec(map_type, ARRAY_SIZE(bpf_map_types));
1419	ops = bpf_map_types[map_type];
1420	if (!ops)
1421		return -EINVAL;
1422
1423	if (ops->map_alloc_check) {
1424		err = ops->map_alloc_check(attr);
1425		if (err)
1426			return err;
1427	}
1428	if (attr->map_ifindex)
1429		ops = &bpf_map_offload_ops;
1430	if (!ops->map_mem_usage)
1431		return -EINVAL;
1432
1433	if (token_flag) {
1434		token = bpf_token_get_from_fd(attr->map_token_fd);
1435		if (IS_ERR(token))
1436			return PTR_ERR(token);
1437
1438		/* if current token doesn't grant map creation permissions,
1439		 * then we can't use this token, so ignore it and rely on
1440		 * system-wide capabilities checks
1441		 */
1442		if (!bpf_token_allow_cmd(token, BPF_MAP_CREATE) ||
1443		    !bpf_token_allow_map_type(token, attr->map_type)) {
1444			bpf_token_put(token);
1445			token = NULL;
1446		}
1447	}
1448
1449	err = -EPERM;
1450
1451	/* Intent here is for unprivileged_bpf_disabled to block BPF map
1452	 * creation for unprivileged users; other actions depend
1453	 * on fd availability and access to bpffs, so are dependent on
1454	 * object creation success. Even with unprivileged BPF disabled,
1455	 * capability checks are still carried out.
1456	 */
1457	if (sysctl_unprivileged_bpf_disabled && !bpf_token_capable(token, CAP_BPF))
1458		goto put_token;
1459
1460	/* check privileged map type permissions */
1461	switch (map_type) {
1462	case BPF_MAP_TYPE_ARRAY:
1463	case BPF_MAP_TYPE_PERCPU_ARRAY:
1464	case BPF_MAP_TYPE_PROG_ARRAY:
1465	case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
1466	case BPF_MAP_TYPE_CGROUP_ARRAY:
1467	case BPF_MAP_TYPE_ARRAY_OF_MAPS:
1468	case BPF_MAP_TYPE_HASH:
1469	case BPF_MAP_TYPE_PERCPU_HASH:
1470	case BPF_MAP_TYPE_HASH_OF_MAPS:
1471	case BPF_MAP_TYPE_RINGBUF:
1472	case BPF_MAP_TYPE_USER_RINGBUF:
1473	case BPF_MAP_TYPE_CGROUP_STORAGE:
1474	case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE:
1475		/* unprivileged */
1476		break;
1477	case BPF_MAP_TYPE_SK_STORAGE:
1478	case BPF_MAP_TYPE_INODE_STORAGE:
1479	case BPF_MAP_TYPE_TASK_STORAGE:
1480	case BPF_MAP_TYPE_CGRP_STORAGE:
1481	case BPF_MAP_TYPE_BLOOM_FILTER:
1482	case BPF_MAP_TYPE_LPM_TRIE:
1483	case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY:
1484	case BPF_MAP_TYPE_STACK_TRACE:
1485	case BPF_MAP_TYPE_QUEUE:
1486	case BPF_MAP_TYPE_STACK:
1487	case BPF_MAP_TYPE_LRU_HASH:
1488	case BPF_MAP_TYPE_LRU_PERCPU_HASH:
1489	case BPF_MAP_TYPE_STRUCT_OPS:
1490	case BPF_MAP_TYPE_CPUMAP:
1491	case BPF_MAP_TYPE_ARENA:
1492	case BPF_MAP_TYPE_INSN_ARRAY:
1493		if (!bpf_token_capable(token, CAP_BPF))
1494			goto put_token;
1495		break;
1496	case BPF_MAP_TYPE_SOCKMAP:
1497	case BPF_MAP_TYPE_SOCKHASH:
1498	case BPF_MAP_TYPE_DEVMAP:
1499	case BPF_MAP_TYPE_DEVMAP_HASH:
1500	case BPF_MAP_TYPE_XSKMAP:
1501		if (!bpf_token_capable(token, CAP_NET_ADMIN))
1502			goto put_token;
1503		break;
1504	default:
1505		WARN(1, "unsupported map type %d", map_type);
1506		goto put_token;
1507	}
1508
1509	map = ops->map_alloc(attr);
1510	if (IS_ERR(map)) {
1511		err = PTR_ERR(map);
1512		goto put_token;
1513	}
1514	map->ops = ops;
1515	map->map_type = map_type;
1516
1517	err = bpf_obj_name_cpy(map->name, attr->map_name,
1518			       sizeof(attr->map_name));
1519	if (err < 0)
1520		goto free_map;
1521
1522	preempt_disable();
1523	map->cookie = gen_cookie_next(&bpf_map_cookie);
1524	preempt_enable();
1525
1526	atomic64_set(&map->refcnt, 1);
1527	atomic64_set(&map->usercnt, 1);
1528	mutex_init(&map->freeze_mutex);
1529	spin_lock_init(&map->owner_lock);
1530
1531	if (attr->btf_key_type_id || attr->btf_value_type_id ||
1532	    /* Even the map's value is a kernel's struct,
1533	     * the bpf_prog.o must have BTF to begin with
1534	     * to figure out the corresponding kernel's
1535	     * counter part.  Thus, attr->btf_fd has
1536	     * to be valid also.
1537	     */
1538	    attr->btf_vmlinux_value_type_id) {
1539		struct btf *btf;
1540
1541		btf = btf_get_by_fd(attr->btf_fd);
1542		if (IS_ERR(btf)) {
1543			err = PTR_ERR(btf);
1544			goto free_map;
1545		}
1546		if (btf_is_kernel(btf)) {
1547			btf_put(btf);
1548			err = -EACCES;
1549			goto free_map;
1550		}
1551		map->btf = btf;
1552
1553		if (attr->btf_value_type_id) {
1554			err = map_check_btf(map, token, btf, attr->btf_key_type_id,
1555					    attr->btf_value_type_id);
1556			if (err)
1557				goto free_map;
1558		}
1559
1560		map->btf_key_type_id = attr->btf_key_type_id;
1561		map->btf_value_type_id = attr->btf_value_type_id;
1562		map->btf_vmlinux_value_type_id =
1563			attr->btf_vmlinux_value_type_id;
1564	}
1565
1566	if (attr->excl_prog_hash) {
1567		bpfptr_t uprog_hash = make_bpfptr(attr->excl_prog_hash, uattr.is_kernel);
1568
1569		if (attr->excl_prog_hash_size != SHA256_DIGEST_SIZE) {
1570			err = -EINVAL;
1571			goto free_map;
1572		}
1573
1574		map->excl_prog_sha = kzalloc(SHA256_DIGEST_SIZE, GFP_KERNEL);
1575		if (!map->excl_prog_sha) {
1576			err = -ENOMEM;
1577			goto free_map;
1578		}
1579
1580		if (copy_from_bpfptr(map->excl_prog_sha, uprog_hash, SHA256_DIGEST_SIZE)) {
1581			err = -EFAULT;
1582			goto free_map;
1583		}
1584	} else if (attr->excl_prog_hash_size) {
1585		err = -EINVAL;
1586		goto free_map;
1587	}
1588
1589	err = security_bpf_map_create(map, attr, token, uattr.is_kernel);
1590	if (err)
1591		goto free_map_sec;
1592
1593	err = bpf_map_alloc_id(map);
1594	if (err)
1595		goto free_map_sec;
1596
1597	bpf_map_save_memcg(map);
1598	bpf_token_put(token);
1599
1600	err = bpf_map_new_fd(map, f_flags);
1601	if (err < 0) {
1602		/* failed to allocate fd.
1603		 * bpf_map_put_with_uref() is needed because the above
1604		 * bpf_map_alloc_id() has published the map
1605		 * to the userspace and the userspace may
1606		 * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID.
1607		 */
1608		bpf_map_put_with_uref(map);
1609		return err;
1610	}
1611
1612	return err;
1613
1614free_map_sec:
1615	security_bpf_map_free(map);
1616free_map:
1617	bpf_map_free(map);
1618put_token:
1619	bpf_token_put(token);
1620	return err;
1621}
1622
1623void bpf_map_inc(struct bpf_map *map)
1624{
1625	atomic64_inc(&map->refcnt);
1626}
1627EXPORT_SYMBOL_GPL(bpf_map_inc);
1628
1629void bpf_map_inc_with_uref(struct bpf_map *map)
1630{
1631	atomic64_inc(&map->refcnt);
1632	atomic64_inc(&map->usercnt);
1633}
1634EXPORT_SYMBOL_GPL(bpf_map_inc_with_uref);
1635
1636struct bpf_map *bpf_map_get(u32 ufd)
1637{
1638	CLASS(fd, f)(ufd);
1639	struct bpf_map *map = __bpf_map_get(f);
1640
1641	if (!IS_ERR(map))
1642		bpf_map_inc(map);
1643
1644	return map;
1645}
1646EXPORT_SYMBOL_NS(bpf_map_get, "BPF_INTERNAL");
1647
1648struct bpf_map *bpf_map_get_with_uref(u32 ufd)
1649{
1650	CLASS(fd, f)(ufd);
1651	struct bpf_map *map = __bpf_map_get(f);
1652
1653	if (!IS_ERR(map))
1654		bpf_map_inc_with_uref(map);
1655
1656	return map;
1657}
1658
1659/* map_idr_lock should have been held or the map should have been
1660 * protected by rcu read lock.
1661 */
1662struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref)
1663{
1664	int refold;
1665
1666	refold = atomic64_fetch_add_unless(&map->refcnt, 1, 0);
1667	if (!refold)
1668		return ERR_PTR(-ENOENT);
1669	if (uref)
1670		atomic64_inc(&map->usercnt);
1671
1672	return map;
1673}
1674
1675struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map)
1676{
1677	lockdep_assert(rcu_read_lock_held());
1678	return __bpf_map_inc_not_zero(map, false);
1679}
1680EXPORT_SYMBOL_GPL(bpf_map_inc_not_zero);
1681
1682int __weak bpf_stackmap_extract(struct bpf_map *map, void *key, void *value,
1683				bool delete)
1684{
1685	return -ENOTSUPP;
1686}
1687
1688static void *__bpf_copy_key(void __user *ukey, u64 key_size)
1689{
1690	if (key_size)
1691		return vmemdup_user(ukey, key_size);
1692
1693	if (ukey)
1694		return ERR_PTR(-EINVAL);
1695
1696	return NULL;
1697}
1698
1699static void *___bpf_copy_key(bpfptr_t ukey, u64 key_size)
1700{
1701	if (key_size)
1702		return kvmemdup_bpfptr(ukey, key_size);
1703
1704	if (!bpfptr_is_null(ukey))
1705		return ERR_PTR(-EINVAL);
1706
1707	return NULL;
1708}
1709
1710/* last field in 'union bpf_attr' used by this command */
1711#define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags
1712
1713static int map_lookup_elem(union bpf_attr *attr)
1714{
1715	void __user *ukey = u64_to_user_ptr(attr->key);
1716	void __user *uvalue = u64_to_user_ptr(attr->value);
1717	struct bpf_map *map;
1718	void *key, *value;
1719	u32 value_size;
1720	int err;
1721
1722	if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
1723		return -EINVAL;
1724
1725	CLASS(fd, f)(attr->map_fd);
1726	map = __bpf_map_get(f);
1727	if (IS_ERR(map))
1728		return PTR_ERR(map);
1729	if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ))
1730		return -EPERM;
1731
1732	err = bpf_map_check_op_flags(map, attr->flags, BPF_F_LOCK);
1733	if (err)
1734		return err;
1735
1736	key = __bpf_copy_key(ukey, map->key_size);
1737	if (IS_ERR(key))
1738		return PTR_ERR(key);
1739
1740	value_size = bpf_map_value_size(map);
1741
1742	err = -ENOMEM;
1743	value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
1744	if (!value)
1745		goto free_key;
1746
1747	if (map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) {
1748		if (copy_from_user(value, uvalue, value_size))
1749			err = -EFAULT;
1750		else
1751			err = bpf_map_copy_value(map, key, value, attr->flags);
1752		goto free_value;
1753	}
1754
1755	err = bpf_map_copy_value(map, key, value, attr->flags);
1756	if (err)
1757		goto free_value;
1758
1759	err = -EFAULT;
1760	if (copy_to_user(uvalue, value, value_size) != 0)
1761		goto free_value;
1762
1763	err = 0;
1764
1765free_value:
1766	kvfree(value);
1767free_key:
1768	kvfree(key);
1769	return err;
1770}
1771
1772
1773#define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
1774
1775static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr)
1776{
1777	bpfptr_t ukey = make_bpfptr(attr->key, uattr.is_kernel);
1778	bpfptr_t uvalue = make_bpfptr(attr->value, uattr.is_kernel);
1779	struct bpf_map *map;
1780	void *key, *value;
1781	u32 value_size;
1782	int err;
1783
1784	if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM))
1785		return -EINVAL;
1786
1787	CLASS(fd, f)(attr->map_fd);
1788	map = __bpf_map_get(f);
1789	if (IS_ERR(map))
1790		return PTR_ERR(map);
1791	bpf_map_write_active_inc(map);
1792	if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
1793		err = -EPERM;
1794		goto err_put;
1795	}
1796
1797	err = bpf_map_check_op_flags(map, attr->flags, ~0);
1798	if (err)
1799		goto err_put;
1800
1801	key = ___bpf_copy_key(ukey, map->key_size);
1802	if (IS_ERR(key)) {
1803		err = PTR_ERR(key);
1804		goto err_put;
1805	}
1806
1807	value_size = bpf_map_value_size(map);
1808	value = kvmemdup_bpfptr(uvalue, value_size);
1809	if (IS_ERR(value)) {
1810		err = PTR_ERR(value);
1811		goto free_key;
1812	}
1813
1814	err = bpf_map_update_value(map, fd_file(f), key, value, attr->flags);
1815	if (!err)
1816		maybe_wait_bpf_programs(map);
1817
1818	kvfree(value);
1819free_key:
1820	kvfree(key);
1821err_put:
1822	bpf_map_write_active_dec(map);
1823	return err;
1824}
1825
1826#define BPF_MAP_DELETE_ELEM_LAST_FIELD key
1827
1828static int map_delete_elem(union bpf_attr *attr, bpfptr_t uattr)
1829{
1830	bpfptr_t ukey = make_bpfptr(attr->key, uattr.is_kernel);
1831	struct bpf_map *map;
1832	void *key;
1833	int err;
1834
1835	if (CHECK_ATTR(BPF_MAP_DELETE_ELEM))
1836		return -EINVAL;
1837
1838	CLASS(fd, f)(attr->map_fd);
1839	map = __bpf_map_get(f);
1840	if (IS_ERR(map))
1841		return PTR_ERR(map);
1842	bpf_map_write_active_inc(map);
1843	if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
1844		err = -EPERM;
1845		goto err_put;
1846	}
1847
1848	key = ___bpf_copy_key(ukey, map->key_size);
1849	if (IS_ERR(key)) {
1850		err = PTR_ERR(key);
1851		goto err_put;
1852	}
1853
1854	if (bpf_map_is_offloaded(map)) {
1855		err = bpf_map_offload_delete_elem(map, key);
1856		goto out;
1857	} else if (IS_FD_PROG_ARRAY(map) ||
1858		   map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
1859		/* These maps require sleepable context */
1860		err = map->ops->map_delete_elem(map, key);
1861		goto out;
1862	}
1863
1864	bpf_disable_instrumentation();
1865	rcu_read_lock();
1866	err = map->ops->map_delete_elem(map, key);
1867	rcu_read_unlock();
1868	bpf_enable_instrumentation();
1869	if (!err)
1870		maybe_wait_bpf_programs(map);
1871out:
1872	kvfree(key);
1873err_put:
1874	bpf_map_write_active_dec(map);
1875	return err;
1876}
1877
1878/* last field in 'union bpf_attr' used by this command */
1879#define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key
1880
1881static int map_get_next_key(union bpf_attr *attr)
1882{
1883	void __user *ukey = u64_to_user_ptr(attr->key);
1884	void __user *unext_key = u64_to_user_ptr(attr->next_key);
1885	struct bpf_map *map;
1886	void *key, *next_key;
1887	int err;
1888
1889	if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY))
1890		return -EINVAL;
1891
1892	CLASS(fd, f)(attr->map_fd);
1893	map = __bpf_map_get(f);
1894	if (IS_ERR(map))
1895		return PTR_ERR(map);
1896	if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ))
1897		return -EPERM;
1898
1899	if (ukey) {
1900		key = __bpf_copy_key(ukey, map->key_size);
1901		if (IS_ERR(key))
1902			return PTR_ERR(key);
1903	} else {
1904		key = NULL;
1905	}
1906
1907	err = -ENOMEM;
1908	next_key = kvmalloc(map->key_size, GFP_USER);
1909	if (!next_key)
1910		goto free_key;
1911
1912	if (bpf_map_is_offloaded(map)) {
1913		err = bpf_map_offload_get_next_key(map, key, next_key);
1914		goto out;
1915	}
1916
1917	rcu_read_lock();
1918	err = map->ops->map_get_next_key(map, key, next_key);
1919	rcu_read_unlock();
1920out:
1921	if (err)
1922		goto free_next_key;
1923
1924	err = -EFAULT;
1925	if (copy_to_user(unext_key, next_key, map->key_size) != 0)
1926		goto free_next_key;
1927
1928	err = 0;
1929
1930free_next_key:
1931	kvfree(next_key);
1932free_key:
1933	kvfree(key);
1934	return err;
1935}
1936
1937int generic_map_delete_batch(struct bpf_map *map,
1938			     const union bpf_attr *attr,
1939			     union bpf_attr __user *uattr)
1940{
1941	void __user *keys = u64_to_user_ptr(attr->batch.keys);
1942	u32 cp, max_count;
1943	int err = 0;
1944	void *key;
1945
1946	if (attr->batch.elem_flags & ~BPF_F_LOCK)
1947		return -EINVAL;
1948
1949	if ((attr->batch.elem_flags & BPF_F_LOCK) &&
1950	    !btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
1951		return -EINVAL;
1952	}
1953
1954	max_count = attr->batch.count;
1955	if (!max_count)
1956		return 0;
1957
1958	if (put_user(0, &uattr->batch.count))
1959		return -EFAULT;
1960
1961	key = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
1962	if (!key)
1963		return -ENOMEM;
1964
1965	for (cp = 0; cp < max_count; cp++) {
1966		err = -EFAULT;
1967		if (copy_from_user(key, keys + cp * map->key_size,
1968				   map->key_size))
1969			break;
1970
1971		if (bpf_map_is_offloaded(map)) {
1972			err = bpf_map_offload_delete_elem(map, key);
1973			break;
1974		}
1975
1976		bpf_disable_instrumentation();
1977		rcu_read_lock();
1978		err = map->ops->map_delete_elem(map, key);
1979		rcu_read_unlock();
1980		bpf_enable_instrumentation();
1981		if (err)
1982			break;
1983		cond_resched();
1984	}
1985	if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp)))
1986		err = -EFAULT;
1987
1988	kvfree(key);
1989
1990	return err;
1991}
1992
1993int generic_map_update_batch(struct bpf_map *map, struct file *map_file,
1994			     const union bpf_attr *attr,
1995			     union bpf_attr __user *uattr)
1996{
1997	void __user *values = u64_to_user_ptr(attr->batch.values);
1998	void __user *keys = u64_to_user_ptr(attr->batch.keys);
1999	u32 value_size, cp, max_count;
2000	void *key, *value;
2001	int err = 0;
2002
2003	err = bpf_map_check_op_flags(map, attr->batch.elem_flags, BPF_F_LOCK);
2004	if (err)
2005		return err;
2006
2007	value_size = bpf_map_value_size(map);
2008
2009	max_count = attr->batch.count;
2010	if (!max_count)
2011		return 0;
2012
2013	if (put_user(0, &uattr->batch.count))
2014		return -EFAULT;
2015
2016	key = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
2017	if (!key)
2018		return -ENOMEM;
2019
2020	value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
2021	if (!value) {
2022		kvfree(key);
2023		return -ENOMEM;
2024	}
2025
2026	for (cp = 0; cp < max_count; cp++) {
2027		err = -EFAULT;
2028		if (copy_from_user(key, keys + cp * map->key_size,
2029		    map->key_size) ||
2030		    copy_from_user(value, values + cp * value_size, value_size))
2031			break;
2032
2033		err = bpf_map_update_value(map, map_file, key, value,
2034					   attr->batch.elem_flags);
2035
2036		if (err)
2037			break;
2038		cond_resched();
2039	}
2040
2041	if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp)))
2042		err = -EFAULT;
2043
2044	kvfree(value);
2045	kvfree(key);
2046
2047	return err;
2048}
2049
2050int generic_map_lookup_batch(struct bpf_map *map,
2051				    const union bpf_attr *attr,
2052				    union bpf_attr __user *uattr)
2053{
2054	void __user *uobatch = u64_to_user_ptr(attr->batch.out_batch);
2055	void __user *ubatch = u64_to_user_ptr(attr->batch.in_batch);
2056	void __user *values = u64_to_user_ptr(attr->batch.values);
2057	void __user *keys = u64_to_user_ptr(attr->batch.keys);
2058	void *buf, *buf_prevkey, *prev_key, *key, *value;
2059	u32 value_size, cp, max_count;
2060	int err;
2061
2062	err = bpf_map_check_op_flags(map, attr->batch.elem_flags, BPF_F_LOCK);
2063	if (err)
2064		return err;
2065
2066	value_size = bpf_map_value_size(map);
2067
2068	max_count = attr->batch.count;
2069	if (!max_count)
2070		return 0;
2071
2072	if (put_user(0, &uattr->batch.count))
2073		return -EFAULT;
2074
2075	buf_prevkey = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
2076	if (!buf_prevkey)
2077		return -ENOMEM;
2078
2079	buf = kvmalloc(map->key_size + value_size, GFP_USER | __GFP_NOWARN);
2080	if (!buf) {
2081		kvfree(buf_prevkey);
2082		return -ENOMEM;
2083	}
2084
2085	err = -EFAULT;
2086	prev_key = NULL;
2087	if (ubatch && copy_from_user(buf_prevkey, ubatch, map->key_size))
2088		goto free_buf;
2089	key = buf;
2090	value = key + map->key_size;
2091	if (ubatch)
2092		prev_key = buf_prevkey;
2093
2094	for (cp = 0; cp < max_count;) {
2095		rcu_read_lock();
2096		err = map->ops->map_get_next_key(map, prev_key, key);
2097		rcu_read_unlock();
2098		if (err)
2099			break;
2100		err = bpf_map_copy_value(map, key, value,
2101					 attr->batch.elem_flags);
2102
2103		if (err == -ENOENT)
2104			goto next_key;
2105
2106		if (err)
2107			goto free_buf;
2108
2109		if (copy_to_user(keys + cp * map->key_size, key,
2110				 map->key_size)) {
2111			err = -EFAULT;
2112			goto free_buf;
2113		}
2114		if (copy_to_user(values + cp * value_size, value, value_size)) {
2115			err = -EFAULT;
2116			goto free_buf;
2117		}
2118
2119		cp++;
2120next_key:
2121		if (!prev_key)
2122			prev_key = buf_prevkey;
2123
2124		swap(prev_key, key);
2125		cond_resched();
2126	}
2127
2128	if (err == -EFAULT)
2129		goto free_buf;
2130
2131	if ((copy_to_user(&uattr->batch.count, &cp, sizeof(cp)) ||
2132		    (cp && copy_to_user(uobatch, prev_key, map->key_size))))
2133		err = -EFAULT;
2134
2135free_buf:
2136	kvfree(buf_prevkey);
2137	kvfree(buf);
2138	return err;
2139}
2140
2141#define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD flags
2142
2143static int map_lookup_and_delete_elem(union bpf_attr *attr)
2144{
2145	void __user *ukey = u64_to_user_ptr(attr->key);
2146	void __user *uvalue = u64_to_user_ptr(attr->value);
2147	struct bpf_map *map;
2148	void *key, *value;
2149	u32 value_size;
2150	int err;
2151
2152	if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM))
2153		return -EINVAL;
2154
2155	if (attr->flags & ~BPF_F_LOCK)
2156		return -EINVAL;
2157
2158	CLASS(fd, f)(attr->map_fd);
2159	map = __bpf_map_get(f);
2160	if (IS_ERR(map))
2161		return PTR_ERR(map);
2162	bpf_map_write_active_inc(map);
2163	if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ) ||
2164	    !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
2165		err = -EPERM;
2166		goto err_put;
2167	}
2168
2169	if (attr->flags &&
2170	    (map->map_type == BPF_MAP_TYPE_QUEUE ||
2171	     map->map_type == BPF_MAP_TYPE_STACK)) {
2172		err = -EINVAL;
2173		goto err_put;
2174	}
2175
2176	if ((attr->flags & BPF_F_LOCK) &&
2177	    !btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
2178		err = -EINVAL;
2179		goto err_put;
2180	}
2181
2182	key = __bpf_copy_key(ukey, map->key_size);
2183	if (IS_ERR(key)) {
2184		err = PTR_ERR(key);
2185		goto err_put;
2186	}
2187
2188	value_size = bpf_map_value_size(map);
2189
2190	err = -ENOMEM;
2191	value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
2192	if (!value)
2193		goto free_key;
2194
2195	err = -ENOTSUPP;
2196	if (map->map_type == BPF_MAP_TYPE_QUEUE ||
2197	    map->map_type == BPF_MAP_TYPE_STACK) {
2198		err = map->ops->map_pop_elem(map, value);
2199	} else if (map->map_type == BPF_MAP_TYPE_HASH ||
2200		   map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
2201		   map->map_type == BPF_MAP_TYPE_LRU_HASH ||
2202		   map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
2203		   map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
2204		if (!bpf_map_is_offloaded(map)) {
2205			bpf_disable_instrumentation();
2206			rcu_read_lock();
2207			err = map->ops->map_lookup_and_delete_elem(map, key, value, attr->flags);
2208			rcu_read_unlock();
2209			bpf_enable_instrumentation();
2210		}
2211	}
2212
2213	if (err)
2214		goto free_value;
2215
2216	if (copy_to_user(uvalue, value, value_size) != 0) {
2217		err = -EFAULT;
2218		goto free_value;
2219	}
2220
2221	err = 0;
2222
2223free_value:
2224	kvfree(value);
2225free_key:
2226	kvfree(key);
2227err_put:
2228	bpf_map_write_active_dec(map);
2229	return err;
2230}
2231
2232#define BPF_MAP_FREEZE_LAST_FIELD map_fd
2233
2234static int map_freeze(const union bpf_attr *attr)
2235{
2236	int err = 0;
2237	struct bpf_map *map;
2238
2239	if (CHECK_ATTR(BPF_MAP_FREEZE))
2240		return -EINVAL;
2241
2242	CLASS(fd, f)(attr->map_fd);
2243	map = __bpf_map_get(f);
2244	if (IS_ERR(map))
2245		return PTR_ERR(map);
2246
2247	if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS || !IS_ERR_OR_NULL(map->record))
2248		return -ENOTSUPP;
2249
2250	if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE))
2251		return -EPERM;
2252
2253	mutex_lock(&map->freeze_mutex);
2254	if (bpf_map_write_active(map)) {
2255		err = -EBUSY;
2256		goto err_put;
2257	}
2258	if (READ_ONCE(map->frozen)) {
2259		err = -EBUSY;
2260		goto err_put;
2261	}
2262
2263	WRITE_ONCE(map->frozen, true);
2264err_put:
2265	mutex_unlock(&map->freeze_mutex);
2266	return err;
2267}
2268
2269static const struct bpf_prog_ops * const bpf_prog_types[] = {
2270#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
2271	[_id] = & _name ## _prog_ops,
2272#define BPF_MAP_TYPE(_id, _ops)
2273#define BPF_LINK_TYPE(_id, _name)
2274#include <linux/bpf_types.h>
2275#undef BPF_PROG_TYPE
2276#undef BPF_MAP_TYPE
2277#undef BPF_LINK_TYPE
2278};
2279
2280static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
2281{
2282	const struct bpf_prog_ops *ops;
2283
2284	if (type >= ARRAY_SIZE(bpf_prog_types))
2285		return -EINVAL;
2286	type = array_index_nospec(type, ARRAY_SIZE(bpf_prog_types));
2287	ops = bpf_prog_types[type];
2288	if (!ops)
2289		return -EINVAL;
2290
2291	if (!bpf_prog_is_offloaded(prog->aux))
2292		prog->aux->ops = ops;
2293	else
2294		prog->aux->ops = &bpf_offload_prog_ops;
2295	prog->type = type;
2296	return 0;
2297}
2298
2299enum bpf_audit {
2300	BPF_AUDIT_LOAD,
2301	BPF_AUDIT_UNLOAD,
2302	BPF_AUDIT_MAX,
2303};
2304
2305static const char * const bpf_audit_str[BPF_AUDIT_MAX] = {
2306	[BPF_AUDIT_LOAD]   = "LOAD",
2307	[BPF_AUDIT_UNLOAD] = "UNLOAD",
2308};
2309
2310static void bpf_audit_prog(const struct bpf_prog *prog, unsigned int op)
2311{
2312	struct audit_context *ctx = NULL;
2313	struct audit_buffer *ab;
2314
2315	if (WARN_ON_ONCE(op >= BPF_AUDIT_MAX))
2316		return;
2317	if (audit_enabled == AUDIT_OFF)
2318		return;
2319	if (!in_hardirq() && !irqs_disabled())
2320		ctx = audit_context();
2321	ab = audit_log_start(ctx, GFP_ATOMIC, AUDIT_BPF);
2322	if (unlikely(!ab))
2323		return;
2324	audit_log_format(ab, "prog-id=%u op=%s",
2325			 prog->aux->id, bpf_audit_str[op]);
2326	audit_log_end(ab);
2327}
2328
2329static int bpf_prog_alloc_id(struct bpf_prog *prog)
2330{
2331	int id;
2332
2333	idr_preload(GFP_KERNEL);
2334	spin_lock_bh(&prog_idr_lock);
2335	id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC);
2336	if (id > 0)
2337		prog->aux->id = id;
2338	spin_unlock_bh(&prog_idr_lock);
2339	idr_preload_end();
2340
2341	/* id is in [1, INT_MAX) */
2342	if (WARN_ON_ONCE(!id))
2343		return -ENOSPC;
2344
2345	return id > 0 ? 0 : id;
2346}
2347
2348void bpf_prog_free_id(struct bpf_prog *prog)
2349{
2350	unsigned long flags;
2351
2352	/* cBPF to eBPF migrations are currently not in the idr store.
2353	 * Offloaded programs are removed from the store when their device
2354	 * disappears - even if someone grabs an fd to them they are unusable,
2355	 * simply waiting for refcnt to drop to be freed.
2356	 */
2357	if (!prog->aux->id)
2358		return;
2359
2360	spin_lock_irqsave(&prog_idr_lock, flags);
2361	idr_remove(&prog_idr, prog->aux->id);
2362	prog->aux->id = 0;
2363	spin_unlock_irqrestore(&prog_idr_lock, flags);
2364}
2365
2366static void __bpf_prog_put_rcu(struct rcu_head *rcu)
2367{
2368	struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
2369
2370	kvfree(aux->func_info);
2371	kfree(aux->func_info_aux);
2372	free_uid(aux->user);
2373	security_bpf_prog_free(aux->prog);
2374	bpf_prog_free(aux->prog);
2375}
2376
2377static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred)
2378{
2379	bpf_prog_kallsyms_del_all(prog);
2380	btf_put(prog->aux->btf);
2381	module_put(prog->aux->mod);
2382	kvfree(prog->aux->jited_linfo);
2383	kvfree(prog->aux->linfo);
2384	kfree(prog->aux->kfunc_tab);
2385	kfree(prog->aux->ctx_arg_info);
2386	if (prog->aux->attach_btf)
2387		btf_put(prog->aux->attach_btf);
2388
2389	if (deferred) {
2390		if (prog->sleepable)
2391			call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu);
2392		else
2393			call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
2394	} else {
2395		__bpf_prog_put_rcu(&prog->aux->rcu);
2396	}
2397}
2398
2399static void bpf_prog_put_deferred(struct work_struct *work)
2400{
2401	struct bpf_prog_aux *aux;
2402	struct bpf_prog *prog;
2403
2404	aux = container_of(work, struct bpf_prog_aux, work);
2405	prog = aux->prog;
2406	perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0);
2407	bpf_audit_prog(prog, BPF_AUDIT_UNLOAD);
2408	bpf_prog_free_id(prog);
2409	__bpf_prog_put_noref(prog, true);
2410}
2411
2412static void __bpf_prog_put(struct bpf_prog *prog)
2413{
2414	struct bpf_prog_aux *aux = prog->aux;
2415
2416	if (atomic64_dec_and_test(&aux->refcnt)) {
2417		if (in_hardirq() || irqs_disabled()) {
2418			INIT_WORK(&aux->work, bpf_prog_put_deferred);
2419			schedule_work(&aux->work);
2420		} else {
2421			bpf_prog_put_deferred(&aux->work);
2422		}
2423	}
2424}
2425
2426void bpf_prog_put(struct bpf_prog *prog)
2427{
2428	__bpf_prog_put(prog);
2429}
2430EXPORT_SYMBOL_GPL(bpf_prog_put);
2431
2432static int bpf_prog_release(struct inode *inode, struct file *filp)
2433{
2434	struct bpf_prog *prog = filp->private_data;
2435
2436	bpf_prog_put(prog);
2437	return 0;
2438}
2439
2440struct bpf_prog_kstats {
2441	u64 nsecs;
2442	u64 cnt;
2443	u64 misses;
2444};
2445
2446void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog)
2447{
2448	struct bpf_prog_stats *stats;
2449	unsigned int flags;
2450
2451	if (unlikely(!prog->stats))
2452		return;
2453
2454	stats = this_cpu_ptr(prog->stats);
2455	flags = u64_stats_update_begin_irqsave(&stats->syncp);
2456	u64_stats_inc(&stats->misses);
2457	u64_stats_update_end_irqrestore(&stats->syncp, flags);
2458}
2459
2460static void bpf_prog_get_stats(const struct bpf_prog *prog,
2461			       struct bpf_prog_kstats *stats)
2462{
2463	u64 nsecs = 0, cnt = 0, misses = 0;
2464	int cpu;
2465
2466	for_each_possible_cpu(cpu) {
2467		const struct bpf_prog_stats *st;
2468		unsigned int start;
2469		u64 tnsecs, tcnt, tmisses;
2470
2471		st = per_cpu_ptr(prog->stats, cpu);
2472		do {
2473			start = u64_stats_fetch_begin(&st->syncp);
2474			tnsecs = u64_stats_read(&st->nsecs);
2475			tcnt = u64_stats_read(&st->cnt);
2476			tmisses = u64_stats_read(&st->misses);
2477		} while (u64_stats_fetch_retry(&st->syncp, start));
2478		nsecs += tnsecs;
2479		cnt += tcnt;
2480		misses += tmisses;
2481	}
2482	stats->nsecs = nsecs;
2483	stats->cnt = cnt;
2484	stats->misses = misses;
2485}
2486
2487#ifdef CONFIG_PROC_FS
2488static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp)
2489{
2490	const struct bpf_prog *prog = filp->private_data;
2491	char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
2492	struct bpf_prog_kstats stats;
2493
2494	bpf_prog_get_stats(prog, &stats);
2495	bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
2496	seq_printf(m,
2497		   "prog_type:\t%u\n"
2498		   "prog_jited:\t%u\n"
2499		   "prog_tag:\t%s\n"
2500		   "memlock:\t%llu\n"
2501		   "prog_id:\t%u\n"
2502		   "run_time_ns:\t%llu\n"
2503		   "run_cnt:\t%llu\n"
2504		   "recursion_misses:\t%llu\n"
2505		   "verified_insns:\t%u\n",
2506		   prog->type,
2507		   prog->jited,
2508		   prog_tag,
2509		   prog->pages * 1ULL << PAGE_SHIFT,
2510		   prog->aux->id,
2511		   stats.nsecs,
2512		   stats.cnt,
2513		   stats.misses,
2514		   prog->aux->verified_insns);
2515}
2516#endif
2517
2518const struct file_operations bpf_prog_fops = {
2519#ifdef CONFIG_PROC_FS
2520	.show_fdinfo	= bpf_prog_show_fdinfo,
2521#endif
2522	.release	= bpf_prog_release,
2523	.read		= bpf_dummy_read,
2524	.write		= bpf_dummy_write,
2525};
2526
2527int bpf_prog_new_fd(struct bpf_prog *prog)
2528{
2529	int ret;
2530
2531	ret = security_bpf_prog(prog);
2532	if (ret < 0)
2533		return ret;
2534
2535	return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog,
2536				O_RDWR | O_CLOEXEC);
2537}
2538
2539void bpf_prog_add(struct bpf_prog *prog, int i)
2540{
2541	atomic64_add(i, &prog->aux->refcnt);
2542}
2543EXPORT_SYMBOL_GPL(bpf_prog_add);
2544
2545void bpf_prog_sub(struct bpf_prog *prog, int i)
2546{
2547	/* Only to be used for undoing previous bpf_prog_add() in some
2548	 * error path. We still know that another entity in our call
2549	 * path holds a reference to the program, thus atomic_sub() can
2550	 * be safely used in such cases!
2551	 */
2552	WARN_ON(atomic64_sub_return(i, &prog->aux->refcnt) == 0);
2553}
2554EXPORT_SYMBOL_GPL(bpf_prog_sub);
2555
2556void bpf_prog_inc(struct bpf_prog *prog)
2557{
2558	atomic64_inc(&prog->aux->refcnt);
2559}
2560EXPORT_SYMBOL_GPL(bpf_prog_inc);
2561
2562/* prog_idr_lock should have been held */
2563struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog)
2564{
2565	int refold;
2566
2567	refold = atomic64_fetch_add_unless(&prog->aux->refcnt, 1, 0);
2568
2569	if (!refold)
2570		return ERR_PTR(-ENOENT);
2571
2572	return prog;
2573}
2574EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero);
2575
2576bool bpf_prog_get_ok(struct bpf_prog *prog,
2577			    enum bpf_prog_type *attach_type, bool attach_drv)
2578{
2579	/* not an attachment, just a refcount inc, always allow */
2580	if (!attach_type)
2581		return true;
2582
2583	if (prog->type != *attach_type)
2584		return false;
2585	if (bpf_prog_is_offloaded(prog->aux) && !attach_drv)
2586		return false;
2587
2588	return true;
2589}
2590
2591static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type,
2592				       bool attach_drv)
2593{
2594	CLASS(fd, f)(ufd);
2595	struct bpf_prog *prog;
2596
2597	if (fd_empty(f))
2598		return ERR_PTR(-EBADF);
2599	if (fd_file(f)->f_op != &bpf_prog_fops)
2600		return ERR_PTR(-EINVAL);
2601
2602	prog = fd_file(f)->private_data;
2603	if (!bpf_prog_get_ok(prog, attach_type, attach_drv))
2604		return ERR_PTR(-EINVAL);
2605
2606	bpf_prog_inc(prog);
2607	return prog;
2608}
2609
2610struct bpf_prog *bpf_prog_get(u32 ufd)
2611{
2612	return __bpf_prog_get(ufd, NULL, false);
2613}
2614
2615struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
2616				       bool attach_drv)
2617{
2618	return __bpf_prog_get(ufd, &type, attach_drv);
2619}
2620EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev);
2621
2622/* Initially all BPF programs could be loaded w/o specifying
2623 * expected_attach_type. Later for some of them specifying expected_attach_type
2624 * at load time became required so that program could be validated properly.
2625 * Programs of types that are allowed to be loaded both w/ and w/o (for
2626 * backward compatibility) expected_attach_type, should have the default attach
2627 * type assigned to expected_attach_type for the latter case, so that it can be
2628 * validated later at attach time.
2629 *
2630 * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if
2631 * prog type requires it but has some attach types that have to be backward
2632 * compatible.
2633 */
2634static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr)
2635{
2636	switch (attr->prog_type) {
2637	case BPF_PROG_TYPE_CGROUP_SOCK:
2638		/* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't
2639		 * exist so checking for non-zero is the way to go here.
2640		 */
2641		if (!attr->expected_attach_type)
2642			attr->expected_attach_type =
2643				BPF_CGROUP_INET_SOCK_CREATE;
2644		break;
2645	case BPF_PROG_TYPE_SK_REUSEPORT:
2646		if (!attr->expected_attach_type)
2647			attr->expected_attach_type =
2648				BPF_SK_REUSEPORT_SELECT;
2649		break;
2650	}
2651}
2652
2653static int
2654bpf_prog_load_check_attach(enum bpf_prog_type prog_type,
2655			   enum bpf_attach_type expected_attach_type,
2656			   struct btf *attach_btf, u32 btf_id,
2657			   struct bpf_prog *dst_prog)
2658{
2659	if (btf_id) {
2660		if (btf_id > BTF_MAX_TYPE)
2661			return -EINVAL;
2662
2663		if (!attach_btf && !dst_prog)
2664			return -EINVAL;
2665
2666		switch (prog_type) {
2667		case BPF_PROG_TYPE_TRACING:
2668		case BPF_PROG_TYPE_LSM:
2669		case BPF_PROG_TYPE_STRUCT_OPS:
2670		case BPF_PROG_TYPE_EXT:
2671			break;
2672		default:
2673			return -EINVAL;
2674		}
2675	}
2676
2677	if (attach_btf && (!btf_id || dst_prog))
2678		return -EINVAL;
2679
2680	if (dst_prog && prog_type != BPF_PROG_TYPE_TRACING &&
2681	    prog_type != BPF_PROG_TYPE_EXT)
2682		return -EINVAL;
2683
2684	switch (prog_type) {
2685	case BPF_PROG_TYPE_CGROUP_SOCK:
2686		switch (expected_attach_type) {
2687		case BPF_CGROUP_INET_SOCK_CREATE:
2688		case BPF_CGROUP_INET_SOCK_RELEASE:
2689		case BPF_CGROUP_INET4_POST_BIND:
2690		case BPF_CGROUP_INET6_POST_BIND:
2691			return 0;
2692		default:
2693			return -EINVAL;
2694		}
2695	case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
2696		switch (expected_attach_type) {
2697		case BPF_CGROUP_INET4_BIND:
2698		case BPF_CGROUP_INET6_BIND:
2699		case BPF_CGROUP_INET4_CONNECT:
2700		case BPF_CGROUP_INET6_CONNECT:
2701		case BPF_CGROUP_UNIX_CONNECT:
2702		case BPF_CGROUP_INET4_GETPEERNAME:
2703		case BPF_CGROUP_INET6_GETPEERNAME:
2704		case BPF_CGROUP_UNIX_GETPEERNAME:
2705		case BPF_CGROUP_INET4_GETSOCKNAME:
2706		case BPF_CGROUP_INET6_GETSOCKNAME:
2707		case BPF_CGROUP_UNIX_GETSOCKNAME:
2708		case BPF_CGROUP_UDP4_SENDMSG:
2709		case BPF_CGROUP_UDP6_SENDMSG:
2710		case BPF_CGROUP_UNIX_SENDMSG:
2711		case BPF_CGROUP_UDP4_RECVMSG:
2712		case BPF_CGROUP_UDP6_RECVMSG:
2713		case BPF_CGROUP_UNIX_RECVMSG:
2714			return 0;
2715		default:
2716			return -EINVAL;
2717		}
2718	case BPF_PROG_TYPE_CGROUP_SKB:
2719		switch (expected_attach_type) {
2720		case BPF_CGROUP_INET_INGRESS:
2721		case BPF_CGROUP_INET_EGRESS:
2722			return 0;
2723		default:
2724			return -EINVAL;
2725		}
2726	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
2727		switch (expected_attach_type) {
2728		case BPF_CGROUP_SETSOCKOPT:
2729		case BPF_CGROUP_GETSOCKOPT:
2730			return 0;
2731		default:
2732			return -EINVAL;
2733		}
2734	case BPF_PROG_TYPE_SK_LOOKUP:
2735		if (expected_attach_type == BPF_SK_LOOKUP)
2736			return 0;
2737		return -EINVAL;
2738	case BPF_PROG_TYPE_SK_REUSEPORT:
2739		switch (expected_attach_type) {
2740		case BPF_SK_REUSEPORT_SELECT:
2741		case BPF_SK_REUSEPORT_SELECT_OR_MIGRATE:
2742			return 0;
2743		default:
2744			return -EINVAL;
2745		}
2746	case BPF_PROG_TYPE_NETFILTER:
2747		if (expected_attach_type == BPF_NETFILTER)
2748			return 0;
2749		return -EINVAL;
2750	case BPF_PROG_TYPE_SYSCALL:
2751	case BPF_PROG_TYPE_EXT:
2752		if (expected_attach_type)
2753			return -EINVAL;
2754		fallthrough;
2755	default:
2756		return 0;
2757	}
2758}
2759
2760static bool is_net_admin_prog_type(enum bpf_prog_type prog_type)
2761{
2762	switch (prog_type) {
2763	case BPF_PROG_TYPE_SCHED_CLS:
2764	case BPF_PROG_TYPE_SCHED_ACT:
2765	case BPF_PROG_TYPE_XDP:
2766	case BPF_PROG_TYPE_LWT_IN:
2767	case BPF_PROG_TYPE_LWT_OUT:
2768	case BPF_PROG_TYPE_LWT_XMIT:
2769	case BPF_PROG_TYPE_LWT_SEG6LOCAL:
2770	case BPF_PROG_TYPE_SK_SKB:
2771	case BPF_PROG_TYPE_SK_MSG:
2772	case BPF_PROG_TYPE_FLOW_DISSECTOR:
2773	case BPF_PROG_TYPE_CGROUP_DEVICE:
2774	case BPF_PROG_TYPE_CGROUP_SOCK:
2775	case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
2776	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
2777	case BPF_PROG_TYPE_CGROUP_SYSCTL:
2778	case BPF_PROG_TYPE_SOCK_OPS:
2779	case BPF_PROG_TYPE_EXT: /* extends any prog */
2780	case BPF_PROG_TYPE_NETFILTER:
2781		return true;
2782	case BPF_PROG_TYPE_CGROUP_SKB:
2783		/* always unpriv */
2784	case BPF_PROG_TYPE_SK_REUSEPORT:
2785		/* equivalent to SOCKET_FILTER. need CAP_BPF only */
2786	default:
2787		return false;
2788	}
2789}
2790
2791static bool is_perfmon_prog_type(enum bpf_prog_type prog_type)
2792{
2793	switch (prog_type) {
2794	case BPF_PROG_TYPE_KPROBE:
2795	case BPF_PROG_TYPE_TRACEPOINT:
2796	case BPF_PROG_TYPE_PERF_EVENT:
2797	case BPF_PROG_TYPE_RAW_TRACEPOINT:
2798	case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
2799	case BPF_PROG_TYPE_TRACING:
2800	case BPF_PROG_TYPE_LSM:
2801	case BPF_PROG_TYPE_STRUCT_OPS: /* has access to struct sock */
2802	case BPF_PROG_TYPE_EXT: /* extends any prog */
2803		return true;
2804	default:
2805		return false;
2806	}
2807}
2808
2809static int bpf_prog_verify_signature(struct bpf_prog *prog, union bpf_attr *attr,
2810				     bool is_kernel)
2811{
2812	bpfptr_t usig = make_bpfptr(attr->signature, is_kernel);
2813	struct bpf_dynptr_kern sig_ptr, insns_ptr;
2814	struct bpf_key *key = NULL;
2815	void *sig;
2816	int err = 0;
2817
2818	/*
2819	 * Don't attempt to use kmalloc_large or vmalloc for signatures.
2820	 * Practical signature for BPF program should be below this limit.
2821	 */
2822	if (attr->signature_size > KMALLOC_MAX_CACHE_SIZE)
2823		return -EINVAL;
2824
2825	if (system_keyring_id_check(attr->keyring_id) == 0)
2826		key = bpf_lookup_system_key(attr->keyring_id);
2827	else
2828		key = bpf_lookup_user_key(attr->keyring_id, 0);
2829
2830	if (!key)
2831		return -EINVAL;
2832
2833	sig = kvmemdup_bpfptr(usig, attr->signature_size);
2834	if (IS_ERR(sig)) {
2835		bpf_key_put(key);
2836		return -ENOMEM;
2837	}
2838
2839	bpf_dynptr_init(&sig_ptr, sig, BPF_DYNPTR_TYPE_LOCAL, 0,
2840			attr->signature_size);
2841	bpf_dynptr_init(&insns_ptr, prog->insnsi, BPF_DYNPTR_TYPE_LOCAL, 0,
2842			prog->len * sizeof(struct bpf_insn));
2843
2844	err = bpf_verify_pkcs7_signature((struct bpf_dynptr *)&insns_ptr,
2845					 (struct bpf_dynptr *)&sig_ptr, key);
2846
2847	bpf_key_put(key);
2848	kvfree(sig);
2849	return err;
2850}
2851
2852static int bpf_prog_mark_insn_arrays_ready(struct bpf_prog *prog)
2853{
2854	int err;
2855	int i;
2856
2857	for (i = 0; i < prog->aux->used_map_cnt; i++) {
2858		if (prog->aux->used_maps[i]->map_type != BPF_MAP_TYPE_INSN_ARRAY)
2859			continue;
2860
2861		err = bpf_insn_array_ready(prog->aux->used_maps[i]);
2862		if (err)
2863			return err;
2864	}
2865
2866	return 0;
2867}
2868
2869/* last field in 'union bpf_attr' used by this command */
2870#define BPF_PROG_LOAD_LAST_FIELD keyring_id
2871
2872static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size)
2873{
2874	enum bpf_prog_type type = attr->prog_type;
2875	struct bpf_prog *prog, *dst_prog = NULL;
2876	struct btf *attach_btf = NULL;
2877	struct bpf_token *token = NULL;
2878	bool bpf_cap;
2879	int err;
2880	char license[128];
2881
2882	if (CHECK_ATTR(BPF_PROG_LOAD))
2883		return -EINVAL;
2884
2885	if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT |
2886				 BPF_F_ANY_ALIGNMENT |
2887				 BPF_F_TEST_STATE_FREQ |
2888				 BPF_F_SLEEPABLE |
2889				 BPF_F_TEST_RND_HI32 |
2890				 BPF_F_XDP_HAS_FRAGS |
2891				 BPF_F_XDP_DEV_BOUND_ONLY |
2892				 BPF_F_TEST_REG_INVARIANTS |
2893				 BPF_F_TOKEN_FD))
2894		return -EINVAL;
2895
2896	bpf_prog_load_fixup_attach_type(attr);
2897
2898	if (attr->prog_flags & BPF_F_TOKEN_FD) {
2899		token = bpf_token_get_from_fd(attr->prog_token_fd);
2900		if (IS_ERR(token))
2901			return PTR_ERR(token);
2902		/* if current token doesn't grant prog loading permissions,
2903		 * then we can't use this token, so ignore it and rely on
2904		 * system-wide capabilities checks
2905		 */
2906		if (!bpf_token_allow_cmd(token, BPF_PROG_LOAD) ||
2907		    !bpf_token_allow_prog_type(token, attr->prog_type,
2908					       attr->expected_attach_type)) {
2909			bpf_token_put(token);
2910			token = NULL;
2911		}
2912	}
2913
2914	bpf_cap = bpf_token_capable(token, CAP_BPF);
2915	err = -EPERM;
2916
2917	if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
2918	    (attr->prog_flags & BPF_F_ANY_ALIGNMENT) &&
2919	    !bpf_cap)
2920		goto put_token;
2921
2922	/* Intent here is for unprivileged_bpf_disabled to block BPF program
2923	 * creation for unprivileged users; other actions depend
2924	 * on fd availability and access to bpffs, so are dependent on
2925	 * object creation success. Even with unprivileged BPF disabled,
2926	 * capability checks are still carried out for these
2927	 * and other operations.
2928	 */
2929	if (sysctl_unprivileged_bpf_disabled && !bpf_cap)
2930		goto put_token;
2931
2932	if (attr->insn_cnt == 0 ||
2933	    attr->insn_cnt > (bpf_cap ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) {
2934		err = -E2BIG;
2935		goto put_token;
2936	}
2937	if (type != BPF_PROG_TYPE_SOCKET_FILTER &&
2938	    type != BPF_PROG_TYPE_CGROUP_SKB &&
2939	    !bpf_cap)
2940		goto put_token;
2941
2942	if (is_net_admin_prog_type(type) && !bpf_token_capable(token, CAP_NET_ADMIN))
2943		goto put_token;
2944	if (is_perfmon_prog_type(type) && !bpf_token_capable(token, CAP_PERFMON))
2945		goto put_token;
2946
2947	/* attach_prog_fd/attach_btf_obj_fd can specify fd of either bpf_prog
2948	 * or btf, we need to check which one it is
2949	 */
2950	if (attr->attach_prog_fd) {
2951		dst_prog = bpf_prog_get(attr->attach_prog_fd);
2952		if (IS_ERR(dst_prog)) {
2953			dst_prog = NULL;
2954			attach_btf = btf_get_by_fd(attr->attach_btf_obj_fd);
2955			if (IS_ERR(attach_btf)) {
2956				err = -EINVAL;
2957				goto put_token;
2958			}
2959			if (!btf_is_kernel(attach_btf)) {
2960				/* attaching through specifying bpf_prog's BTF
2961				 * objects directly might be supported eventually
2962				 */
2963				btf_put(attach_btf);
2964				err = -ENOTSUPP;
2965				goto put_token;
2966			}
2967		}
2968	} else if (attr->attach_btf_id) {
2969		/* fall back to vmlinux BTF, if BTF type ID is specified */
2970		attach_btf = bpf_get_btf_vmlinux();
2971		if (IS_ERR(attach_btf)) {
2972			err = PTR_ERR(attach_btf);
2973			goto put_token;
2974		}
2975		if (!attach_btf) {
2976			err = -EINVAL;
2977			goto put_token;
2978		}
2979		btf_get(attach_btf);
2980	}
2981
2982	if (bpf_prog_load_check_attach(type, attr->expected_attach_type,
2983				       attach_btf, attr->attach_btf_id,
2984				       dst_prog)) {
2985		if (dst_prog)
2986			bpf_prog_put(dst_prog);
2987		if (attach_btf)
2988			btf_put(attach_btf);
2989		err = -EINVAL;
2990		goto put_token;
2991	}
2992
2993	/* plain bpf_prog allocation */
2994	prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
2995	if (!prog) {
2996		if (dst_prog)
2997			bpf_prog_put(dst_prog);
2998		if (attach_btf)
2999			btf_put(attach_btf);
3000		err = -EINVAL;
3001		goto put_token;
3002	}
3003
3004	prog->expected_attach_type = attr->expected_attach_type;
3005	prog->sleepable = !!(attr->prog_flags & BPF_F_SLEEPABLE);
3006	prog->aux->attach_btf = attach_btf;
3007	prog->aux->attach_btf_id = attr->attach_btf_id;
3008	prog->aux->dst_prog = dst_prog;
3009	prog->aux->dev_bound = !!attr->prog_ifindex;
3010	prog->aux->xdp_has_frags = attr->prog_flags & BPF_F_XDP_HAS_FRAGS;
3011
3012	/* move token into prog->aux, reuse taken refcnt */
3013	prog->aux->token = token;
3014	token = NULL;
3015
3016	prog->aux->user = get_current_user();
3017	prog->len = attr->insn_cnt;
3018
3019	err = -EFAULT;
3020	if (copy_from_bpfptr(prog->insns,
3021			     make_bpfptr(attr->insns, uattr.is_kernel),
3022			     bpf_prog_insn_size(prog)) != 0)
3023		goto free_prog;
3024	/* copy eBPF program license from user space */
3025	if (strncpy_from_bpfptr(license,
3026				make_bpfptr(attr->license, uattr.is_kernel),
3027				sizeof(license) - 1) < 0)
3028		goto free_prog;
3029	license[sizeof(license) - 1] = 0;
3030
3031	/* eBPF programs must be GPL compatible to use GPL-ed functions */
3032	prog->gpl_compatible = license_is_gpl_compatible(license) ? 1 : 0;
3033
3034	if (attr->signature) {
3035		err = bpf_prog_verify_signature(prog, attr, uattr.is_kernel);
3036		if (err)
3037			goto free_prog;
3038	}
3039
3040	prog->orig_prog = NULL;
3041	prog->jited = 0;
3042
3043	atomic64_set(&prog->aux->refcnt, 1);
3044
3045	if (bpf_prog_is_dev_bound(prog->aux)) {
3046		err = bpf_prog_dev_bound_init(prog, attr);
3047		if (err)
3048			goto free_prog;
3049	}
3050
3051	if (type == BPF_PROG_TYPE_EXT && dst_prog &&
3052	    bpf_prog_is_dev_bound(dst_prog->aux)) {
3053		err = bpf_prog_dev_bound_inherit(prog, dst_prog);
3054		if (err)
3055			goto free_prog;
3056	}
3057
3058	/*
3059	 * Bookkeeping for managing the program attachment chain.
3060	 *
3061	 * It might be tempting to set attach_tracing_prog flag at the attachment
3062	 * time, but this will not prevent from loading bunch of tracing prog
3063	 * first, then attach them one to another.
3064	 *
3065	 * The flag attach_tracing_prog is set for the whole program lifecycle, and
3066	 * doesn't have to be cleared in bpf_tracing_link_release, since tracing
3067	 * programs cannot change attachment target.
3068	 */
3069	if (type == BPF_PROG_TYPE_TRACING && dst_prog &&
3070	    dst_prog->type == BPF_PROG_TYPE_TRACING) {
3071		prog->aux->attach_tracing_prog = true;
3072	}
3073
3074	/* find program type: socket_filter vs tracing_filter */
3075	err = find_prog_type(type, prog);
3076	if (err < 0)
3077		goto free_prog;
3078
3079	prog->aux->load_time = ktime_get_boottime_ns();
3080	err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name,
3081			       sizeof(attr->prog_name));
3082	if (err < 0)
3083		goto free_prog;
3084
3085	err = security_bpf_prog_load(prog, attr, token, uattr.is_kernel);
3086	if (err)
3087		goto free_prog_sec;
3088
3089	/* run eBPF verifier */
3090	err = bpf_check(&prog, attr, uattr, uattr_size);
3091	if (err < 0)
3092		goto free_used_maps;
3093
3094	prog = bpf_prog_select_runtime(prog, &err);
3095	if (err < 0)
3096		goto free_used_maps;
3097
3098	err = bpf_prog_mark_insn_arrays_ready(prog);
3099	if (err < 0)
3100		goto free_used_maps;
3101
3102	err = bpf_prog_alloc_id(prog);
3103	if (err)
3104		goto free_used_maps;
3105
3106	/* Upon success of bpf_prog_alloc_id(), the BPF prog is
3107	 * effectively publicly exposed. However, retrieving via
3108	 * bpf_prog_get_fd_by_id() will take another reference,
3109	 * therefore it cannot be gone underneath us.
3110	 *
3111	 * Only for the time /after/ successful bpf_prog_new_fd()
3112	 * and before returning to userspace, we might just hold
3113	 * one reference and any parallel close on that fd could
3114	 * rip everything out. Hence, below notifications must
3115	 * happen before bpf_prog_new_fd().
3116	 *
3117	 * Also, any failure handling from this point onwards must
3118	 * be using bpf_prog_put() given the program is exposed.
3119	 */
3120	bpf_prog_kallsyms_add(prog);
3121	perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0);
3122	bpf_audit_prog(prog, BPF_AUDIT_LOAD);
3123
3124	err = bpf_prog_new_fd(prog);
3125	if (err < 0)
3126		bpf_prog_put(prog);
3127	return err;
3128
3129free_used_maps:
3130	/* In case we have subprogs, we need to wait for a grace
3131	 * period before we can tear down JIT memory since symbols
3132	 * are already exposed under kallsyms.
3133	 */
3134	__bpf_prog_put_noref(prog, prog->aux->real_func_cnt);
3135	return err;
3136
3137free_prog_sec:
3138	security_bpf_prog_free(prog);
3139free_prog:
3140	free_uid(prog->aux->user);
3141	if (prog->aux->attach_btf)
3142		btf_put(prog->aux->attach_btf);
3143	bpf_prog_free(prog);
3144put_token:
3145	bpf_token_put(token);
3146	return err;
3147}
3148
3149#define BPF_OBJ_LAST_FIELD path_fd
3150
3151static int bpf_obj_pin(const union bpf_attr *attr)
3152{
3153	int path_fd;
3154
3155	if (CHECK_ATTR(BPF_OBJ) || attr->file_flags & ~BPF_F_PATH_FD)
3156		return -EINVAL;
3157
3158	/* path_fd has to be accompanied by BPF_F_PATH_FD flag */
3159	if (!(attr->file_flags & BPF_F_PATH_FD) && attr->path_fd)
3160		return -EINVAL;
3161
3162	path_fd = attr->file_flags & BPF_F_PATH_FD ? attr->path_fd : AT_FDCWD;
3163	return bpf_obj_pin_user(attr->bpf_fd, path_fd,
3164				u64_to_user_ptr(attr->pathname));
3165}
3166
3167static int bpf_obj_get(const union bpf_attr *attr)
3168{
3169	int path_fd;
3170
3171	if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 ||
3172	    attr->file_flags & ~(BPF_OBJ_FLAG_MASK | BPF_F_PATH_FD))
3173		return -EINVAL;
3174
3175	/* path_fd has to be accompanied by BPF_F_PATH_FD flag */
3176	if (!(attr->file_flags & BPF_F_PATH_FD) && attr->path_fd)
3177		return -EINVAL;
3178
3179	path_fd = attr->file_flags & BPF_F_PATH_FD ? attr->path_fd : AT_FDCWD;
3180	return bpf_obj_get_user(path_fd, u64_to_user_ptr(attr->pathname),
3181				attr->file_flags);
3182}
3183
3184/* bpf_link_init_sleepable() allows to specify whether BPF link itself has
3185 * "sleepable" semantics, which normally would mean that BPF link's attach
3186 * hook can dereference link or link's underlying program for some time after
3187 * detachment due to RCU Tasks Trace-based lifetime protection scheme.
3188 * BPF program itself can be non-sleepable, yet, because it's transitively
3189 * reachable through BPF link, its freeing has to be delayed until after RCU
3190 * Tasks Trace GP.
3191 */
3192void bpf_link_init_sleepable(struct bpf_link *link, enum bpf_link_type type,
3193			     const struct bpf_link_ops *ops, struct bpf_prog *prog,
3194			     enum bpf_attach_type attach_type, bool sleepable)
3195{
3196	WARN_ON(ops->dealloc && ops->dealloc_deferred);
3197	atomic64_set(&link->refcnt, 1);
3198	link->type = type;
3199	link->sleepable = sleepable;
3200	link->id = 0;
3201	link->ops = ops;
3202	link->prog = prog;
3203	link->attach_type = attach_type;
3204}
3205
3206void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
3207		   const struct bpf_link_ops *ops, struct bpf_prog *prog,
3208		   enum bpf_attach_type attach_type)
3209{
3210	bpf_link_init_sleepable(link, type, ops, prog, attach_type, false);
3211}
3212
3213static void bpf_link_free_id(int id)
3214{
3215	if (!id)
3216		return;
3217
3218	spin_lock_bh(&link_idr_lock);
3219	idr_remove(&link_idr, id);
3220	spin_unlock_bh(&link_idr_lock);
3221}
3222
3223/* Clean up bpf_link and corresponding anon_inode file and FD. After
3224 * anon_inode is created, bpf_link can't be just kfree()'d due to deferred
3225 * anon_inode's release() call. This helper marks bpf_link as
3226 * defunct, releases anon_inode file and puts reserved FD. bpf_prog's refcnt
3227 * is not decremented, it's the responsibility of a calling code that failed
3228 * to complete bpf_link initialization.
3229 * This helper eventually calls link's dealloc callback, but does not call
3230 * link's release callback.
3231 */
3232void bpf_link_cleanup(struct bpf_link_primer *primer)
3233{
3234	primer->link->prog = NULL;
3235	bpf_link_free_id(primer->id);
3236	fput(primer->file);
3237	put_unused_fd(primer->fd);
3238}
3239
3240void bpf_link_inc(struct bpf_link *link)
3241{
3242	atomic64_inc(&link->refcnt);
3243}
3244
3245static void bpf_link_dealloc(struct bpf_link *link)
3246{
3247	/* now that we know that bpf_link itself can't be reached, put underlying BPF program */
3248	if (link->prog)
3249		bpf_prog_put(link->prog);
3250
3251	/* free bpf_link and its containing memory */
3252	if (link->ops->dealloc_deferred)
3253		link->ops->dealloc_deferred(link);
3254	else
3255		link->ops->dealloc(link);
3256}
3257
3258static void bpf_link_defer_dealloc_rcu_gp(struct rcu_head *rcu)
3259{
3260	struct bpf_link *link = container_of(rcu, struct bpf_link, rcu);
3261
3262	bpf_link_dealloc(link);
3263}
3264
3265static void bpf_link_defer_dealloc_mult_rcu_gp(struct rcu_head *rcu)
3266{
3267	if (rcu_trace_implies_rcu_gp())
3268		bpf_link_defer_dealloc_rcu_gp(rcu);
3269	else
3270		call_rcu(rcu, bpf_link_defer_dealloc_rcu_gp);
3271}
3272
3273/* bpf_link_free is guaranteed to be called from process context */
3274static void bpf_link_free(struct bpf_link *link)
3275{
3276	const struct bpf_link_ops *ops = link->ops;
3277
3278	bpf_link_free_id(link->id);
3279	/* detach BPF program, clean up used resources */
3280	if (link->prog)
3281		ops->release(link);
3282	if (ops->dealloc_deferred) {
3283		/* Schedule BPF link deallocation, which will only then
3284		 * trigger putting BPF program refcount.
3285		 * If underlying BPF program is sleepable or BPF link's target
3286		 * attach hookpoint is sleepable or otherwise requires RCU GPs
3287		 * to ensure link and its underlying BPF program is not
3288		 * reachable anymore, we need to first wait for RCU tasks
3289		 * trace sync, and then go through "classic" RCU grace period
3290		 */
3291		if (link->sleepable || (link->prog && link->prog->sleepable))
3292			call_rcu_tasks_trace(&link->rcu, bpf_link_defer_dealloc_mult_rcu_gp);
3293		else
3294			call_rcu(&link->rcu, bpf_link_defer_dealloc_rcu_gp);
3295	} else if (ops->dealloc) {
3296		bpf_link_dealloc(link);
3297	}
3298}
3299
3300static void bpf_link_put_deferred(struct work_struct *work)
3301{
3302	struct bpf_link *link = container_of(work, struct bpf_link, work);
3303
3304	bpf_link_free(link);
3305}
3306
3307/* bpf_link_put might be called from atomic context. It needs to be called
3308 * from sleepable context in order to acquire sleeping locks during the process.
3309 */
3310void bpf_link_put(struct bpf_link *link)
3311{
3312	if (!atomic64_dec_and_test(&link->refcnt))
3313		return;
3314
3315	INIT_WORK(&link->work, bpf_link_put_deferred);
3316	schedule_work(&link->work);
3317}
3318EXPORT_SYMBOL(bpf_link_put);
3319
3320static void bpf_link_put_direct(struct bpf_link *link)
3321{
3322	if (!atomic64_dec_and_test(&link->refcnt))
3323		return;
3324	bpf_link_free(link);
3325}
3326
3327static int bpf_link_release(struct inode *inode, struct file *filp)
3328{
3329	struct bpf_link *link = filp->private_data;
3330
3331	bpf_link_put_direct(link);
3332	return 0;
3333}
3334
3335#ifdef CONFIG_PROC_FS
3336#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type)
3337#define BPF_MAP_TYPE(_id, _ops)
3338#define BPF_LINK_TYPE(_id, _name) [_id] = #_name,
3339static const char *bpf_link_type_strs[] = {
3340	[BPF_LINK_TYPE_UNSPEC] = "<invalid>",
3341#include <linux/bpf_types.h>
3342};
3343#undef BPF_PROG_TYPE
3344#undef BPF_MAP_TYPE
3345#undef BPF_LINK_TYPE
3346
3347static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp)
3348{
3349	const struct bpf_link *link = filp->private_data;
3350	const struct bpf_prog *prog = link->prog;
3351	enum bpf_link_type type = link->type;
3352	char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
3353
3354	if (type < ARRAY_SIZE(bpf_link_type_strs) && bpf_link_type_strs[type]) {
3355		if (link->type == BPF_LINK_TYPE_KPROBE_MULTI)
3356			seq_printf(m, "link_type:\t%s\n", link->flags == BPF_F_KPROBE_MULTI_RETURN ?
3357				   "kretprobe_multi" : "kprobe_multi");
3358		else if (link->type == BPF_LINK_TYPE_UPROBE_MULTI)
3359			seq_printf(m, "link_type:\t%s\n", link->flags == BPF_F_UPROBE_MULTI_RETURN ?
3360				   "uretprobe_multi" : "uprobe_multi");
3361		else
3362			seq_printf(m, "link_type:\t%s\n", bpf_link_type_strs[type]);
3363	} else {
3364		WARN_ONCE(1, "missing BPF_LINK_TYPE(...) for link type %u\n", type);
3365		seq_printf(m, "link_type:\t<%u>\n", type);
3366	}
3367	seq_printf(m, "link_id:\t%u\n", link->id);
3368
3369	if (prog) {
3370		bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
3371		seq_printf(m,
3372			   "prog_tag:\t%s\n"
3373			   "prog_id:\t%u\n",
3374			   prog_tag,
3375			   prog->aux->id);
3376	}
3377	if (link->ops->show_fdinfo)
3378		link->ops->show_fdinfo(link, m);
3379}
3380#endif
3381
3382static __poll_t bpf_link_poll(struct file *file, struct poll_table_struct *pts)
3383{
3384	struct bpf_link *link = file->private_data;
3385
3386	return link->ops->poll(file, pts);
3387}
3388
3389static const struct file_operations bpf_link_fops = {
3390#ifdef CONFIG_PROC_FS
3391	.show_fdinfo	= bpf_link_show_fdinfo,
3392#endif
3393	.release	= bpf_link_release,
3394	.read		= bpf_dummy_read,
3395	.write		= bpf_dummy_write,
3396};
3397
3398static const struct file_operations bpf_link_fops_poll = {
3399#ifdef CONFIG_PROC_FS
3400	.show_fdinfo	= bpf_link_show_fdinfo,
3401#endif
3402	.release	= bpf_link_release,
3403	.read		= bpf_dummy_read,
3404	.write		= bpf_dummy_write,
3405	.poll		= bpf_link_poll,
3406};
3407
3408static int bpf_link_alloc_id(struct bpf_link *link)
3409{
3410	int id;
3411
3412	idr_preload(GFP_KERNEL);
3413	spin_lock_bh(&link_idr_lock);
3414	id = idr_alloc_cyclic(&link_idr, link, 1, INT_MAX, GFP_ATOMIC);
3415	spin_unlock_bh(&link_idr_lock);
3416	idr_preload_end();
3417
3418	return id;
3419}
3420
3421/* Prepare bpf_link to be exposed to user-space by allocating anon_inode file,
3422 * reserving unused FD and allocating ID from link_idr. This is to be paired
3423 * with bpf_link_settle() to install FD and ID and expose bpf_link to
3424 * user-space, if bpf_link is successfully attached. If not, bpf_link and
3425 * pre-allocated resources are to be freed with bpf_cleanup() call. All the
3426 * transient state is passed around in struct bpf_link_primer.
3427 * This is preferred way to create and initialize bpf_link, especially when
3428 * there are complicated and expensive operations in between creating bpf_link
3429 * itself and attaching it to BPF hook. By using bpf_link_prime() and
3430 * bpf_link_settle() kernel code using bpf_link doesn't have to perform
3431 * expensive (and potentially failing) roll back operations in a rare case
3432 * that file, FD, or ID can't be allocated.
3433 */
3434int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer)
3435{
3436	struct file *file;
3437	int fd, id;
3438
3439	fd = get_unused_fd_flags(O_CLOEXEC);
3440	if (fd < 0)
3441		return fd;
3442
3443
3444	id = bpf_link_alloc_id(link);
3445	if (id < 0) {
3446		put_unused_fd(fd);
3447		return id;
3448	}
3449
3450	file = anon_inode_getfile("bpf_link",
3451				  link->ops->poll ? &bpf_link_fops_poll : &bpf_link_fops,
3452				  link, O_CLOEXEC);
3453	if (IS_ERR(file)) {
3454		bpf_link_free_id(id);
3455		put_unused_fd(fd);
3456		return PTR_ERR(file);
3457	}
3458
3459	primer->link = link;
3460	primer->file = file;
3461	primer->fd = fd;
3462	primer->id = id;
3463	return 0;
3464}
3465
3466int bpf_link_settle(struct bpf_link_primer *primer)
3467{
3468	/* make bpf_link fetchable by ID */
3469	spin_lock_bh(&link_idr_lock);
3470	primer->link->id = primer->id;
3471	spin_unlock_bh(&link_idr_lock);
3472	/* make bpf_link fetchable by FD */
3473	fd_install(primer->fd, primer->file);
3474	/* pass through installed FD */
3475	return primer->fd;
3476}
3477
3478int bpf_link_new_fd(struct bpf_link *link)
3479{
3480	return anon_inode_getfd("bpf-link",
3481				link->ops->poll ? &bpf_link_fops_poll : &bpf_link_fops,
3482				link, O_CLOEXEC);
3483}
3484
3485struct bpf_link *bpf_link_get_from_fd(u32 ufd)
3486{
3487	CLASS(fd, f)(ufd);
3488	struct bpf_link *link;
3489
3490	if (fd_empty(f))
3491		return ERR_PTR(-EBADF);
3492	if (fd_file(f)->f_op != &bpf_link_fops && fd_file(f)->f_op != &bpf_link_fops_poll)
3493		return ERR_PTR(-EINVAL);
3494
3495	link = fd_file(f)->private_data;
3496	bpf_link_inc(link);
3497	return link;
3498}
3499EXPORT_SYMBOL_NS(bpf_link_get_from_fd, "BPF_INTERNAL");
3500
3501static void bpf_tracing_link_release(struct bpf_link *link)
3502{
3503	struct bpf_tracing_link *tr_link =
3504		container_of(link, struct bpf_tracing_link, link.link);
3505
3506	WARN_ON_ONCE(bpf_trampoline_unlink_prog(&tr_link->link,
3507						tr_link->trampoline,
3508						tr_link->tgt_prog));
3509
3510	bpf_trampoline_put(tr_link->trampoline);
3511
3512	/* tgt_prog is NULL if target is a kernel function */
3513	if (tr_link->tgt_prog)
3514		bpf_prog_put(tr_link->tgt_prog);
3515}
3516
3517static void bpf_tracing_link_dealloc(struct bpf_link *link)
3518{
3519	struct bpf_tracing_link *tr_link =
3520		container_of(link, struct bpf_tracing_link, link.link);
3521
3522	kfree(tr_link);
3523}
3524
3525static void bpf_tracing_link_show_fdinfo(const struct bpf_link *link,
3526					 struct seq_file *seq)
3527{
3528	struct bpf_tracing_link *tr_link =
3529		container_of(link, struct bpf_tracing_link, link.link);
3530	u32 target_btf_id, target_obj_id;
3531
3532	bpf_trampoline_unpack_key(tr_link->trampoline->key,
3533				  &target_obj_id, &target_btf_id);
3534	seq_printf(seq,
3535		   "attach_type:\t%d\n"
3536		   "target_obj_id:\t%u\n"
3537		   "target_btf_id:\t%u\n"
3538		   "cookie:\t%llu\n",
3539		   link->attach_type,
3540		   target_obj_id,
3541		   target_btf_id,
3542		   tr_link->link.cookie);
3543}
3544
3545static int bpf_tracing_link_fill_link_info(const struct bpf_link *link,
3546					   struct bpf_link_info *info)
3547{
3548	struct bpf_tracing_link *tr_link =
3549		container_of(link, struct bpf_tracing_link, link.link);
3550
3551	info->tracing.attach_type = link->attach_type;
3552	info->tracing.cookie = tr_link->link.cookie;
3553	bpf_trampoline_unpack_key(tr_link->trampoline->key,
3554				  &info->tracing.target_obj_id,
3555				  &info->tracing.target_btf_id);
3556
3557	return 0;
3558}
3559
3560static const struct bpf_link_ops bpf_tracing_link_lops = {
3561	.release = bpf_tracing_link_release,
3562	.dealloc = bpf_tracing_link_dealloc,
3563	.show_fdinfo = bpf_tracing_link_show_fdinfo,
3564	.fill_link_info = bpf_tracing_link_fill_link_info,
3565};
3566
3567static int bpf_tracing_prog_attach(struct bpf_prog *prog,
3568				   int tgt_prog_fd,
3569				   u32 btf_id,
3570				   u64 bpf_cookie,
3571				   enum bpf_attach_type attach_type)
3572{
3573	struct bpf_link_primer link_primer;
3574	struct bpf_prog *tgt_prog = NULL;
3575	struct bpf_trampoline *tr = NULL;
3576	struct bpf_tracing_link *link;
3577	u64 key = 0;
3578	int err;
3579
3580	switch (prog->type) {
3581	case BPF_PROG_TYPE_TRACING:
3582		if (prog->expected_attach_type != BPF_TRACE_FENTRY &&
3583		    prog->expected_attach_type != BPF_TRACE_FEXIT &&
3584		    prog->expected_attach_type != BPF_MODIFY_RETURN) {
3585			err = -EINVAL;
3586			goto out_put_prog;
3587		}
3588		break;
3589	case BPF_PROG_TYPE_EXT:
3590		if (prog->expected_attach_type != 0) {
3591			err = -EINVAL;
3592			goto out_put_prog;
3593		}
3594		break;
3595	case BPF_PROG_TYPE_LSM:
3596		if (prog->expected_attach_type != BPF_LSM_MAC) {
3597			err = -EINVAL;
3598			goto out_put_prog;
3599		}
3600		break;
3601	default:
3602		err = -EINVAL;
3603		goto out_put_prog;
3604	}
3605
3606	if (!!tgt_prog_fd != !!btf_id) {
3607		err = -EINVAL;
3608		goto out_put_prog;
3609	}
3610
3611	if (tgt_prog_fd) {
3612		/*
3613		 * For now we only allow new targets for BPF_PROG_TYPE_EXT. If this
3614		 * part would be changed to implement the same for
3615		 * BPF_PROG_TYPE_TRACING, do not forget to update the way how
3616		 * attach_tracing_prog flag is set.
3617		 */
3618		if (prog->type != BPF_PROG_TYPE_EXT) {
3619			err = -EINVAL;
3620			goto out_put_prog;
3621		}
3622
3623		tgt_prog = bpf_prog_get(tgt_prog_fd);
3624		if (IS_ERR(tgt_prog)) {
3625			err = PTR_ERR(tgt_prog);
3626			tgt_prog = NULL;
3627			goto out_put_prog;
3628		}
3629
3630		key = bpf_trampoline_compute_key(tgt_prog, NULL, btf_id);
3631	}
3632
3633	link = kzalloc(sizeof(*link), GFP_USER);
3634	if (!link) {
3635		err = -ENOMEM;
3636		goto out_put_prog;
3637	}
3638	bpf_link_init(&link->link.link, BPF_LINK_TYPE_TRACING,
3639		      &bpf_tracing_link_lops, prog, attach_type);
3640
3641	link->link.cookie = bpf_cookie;
3642
3643	mutex_lock(&prog->aux->dst_mutex);
3644
3645	/* There are a few possible cases here:
3646	 *
3647	 * - if prog->aux->dst_trampoline is set, the program was just loaded
3648	 *   and not yet attached to anything, so we can use the values stored
3649	 *   in prog->aux
3650	 *
3651	 * - if prog->aux->dst_trampoline is NULL, the program has already been
3652	 *   attached to a target and its initial target was cleared (below)
3653	 *
3654	 * - if tgt_prog != NULL, the caller specified tgt_prog_fd +
3655	 *   target_btf_id using the link_create API.
3656	 *
3657	 * - if tgt_prog == NULL when this function was called using the old
3658	 *   raw_tracepoint_open API, and we need a target from prog->aux
3659	 *
3660	 * - if prog->aux->dst_trampoline and tgt_prog is NULL, the program
3661	 *   was detached and is going for re-attachment.
3662	 *
3663	 * - if prog->aux->dst_trampoline is NULL and tgt_prog and prog->aux->attach_btf
3664	 *   are NULL, then program was already attached and user did not provide
3665	 *   tgt_prog_fd so we have no way to find out or create trampoline
3666	 */
3667	if (!prog->aux->dst_trampoline && !tgt_prog) {
3668		/*
3669		 * Allow re-attach for TRACING and LSM programs. If it's
3670		 * currently linked, bpf_trampoline_link_prog will fail.
3671		 * EXT programs need to specify tgt_prog_fd, so they
3672		 * re-attach in separate code path.
3673		 */
3674		if (prog->type != BPF_PROG_TYPE_TRACING &&
3675		    prog->type != BPF_PROG_TYPE_LSM) {
3676			err = -EINVAL;
3677			goto out_unlock;
3678		}
3679		/* We can allow re-attach only if we have valid attach_btf. */
3680		if (!prog->aux->attach_btf) {
3681			err = -EINVAL;
3682			goto out_unlock;
3683		}
3684		btf_id = prog->aux->attach_btf_id;
3685		key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf, btf_id);
3686	}
3687
3688	if (!prog->aux->dst_trampoline ||
3689	    (key && key != prog->aux->dst_trampoline->key)) {
3690		/* If there is no saved target, or the specified target is
3691		 * different from the destination specified at load time, we
3692		 * need a new trampoline and a check for compatibility
3693		 */
3694		struct bpf_attach_target_info tgt_info = {};
3695
3696		err = bpf_check_attach_target(NULL, prog, tgt_prog, btf_id,
3697					      &tgt_info);
3698		if (err)
3699			goto out_unlock;
3700
3701		if (tgt_info.tgt_mod) {
3702			module_put(prog->aux->mod);
3703			prog->aux->mod = tgt_info.tgt_mod;
3704		}
3705
3706		tr = bpf_trampoline_get(key, &tgt_info);
3707		if (!tr) {
3708			err = -ENOMEM;
3709			goto out_unlock;
3710		}
3711	} else {
3712		/* The caller didn't specify a target, or the target was the
3713		 * same as the destination supplied during program load. This
3714		 * means we can reuse the trampoline and reference from program
3715		 * load time, and there is no need to allocate a new one. This
3716		 * can only happen once for any program, as the saved values in
3717		 * prog->aux are cleared below.
3718		 */
3719		tr = prog->aux->dst_trampoline;
3720		tgt_prog = prog->aux->dst_prog;
3721	}
3722
3723	err = bpf_link_prime(&link->link.link, &link_primer);
3724	if (err)
3725		goto out_unlock;
3726
3727	err = bpf_trampoline_link_prog(&link->link, tr, tgt_prog);
3728	if (err) {
3729		bpf_link_cleanup(&link_primer);
3730		link = NULL;
3731		goto out_unlock;
3732	}
3733
3734	link->tgt_prog = tgt_prog;
3735	link->trampoline = tr;
3736
3737	/* Always clear the trampoline and target prog from prog->aux to make
3738	 * sure the original attach destination is not kept alive after a
3739	 * program is (re-)attached to another target.
3740	 */
3741	if (prog->aux->dst_prog &&
3742	    (tgt_prog_fd || tr != prog->aux->dst_trampoline))
3743		/* got extra prog ref from syscall, or attaching to different prog */
3744		bpf_prog_put(prog->aux->dst_prog);
3745	if (prog->aux->dst_trampoline && tr != prog->aux->dst_trampoline)
3746		/* we allocated a new trampoline, so free the old one */
3747		bpf_trampoline_put(prog->aux->dst_trampoline);
3748
3749	prog->aux->dst_prog = NULL;
3750	prog->aux->dst_trampoline = NULL;
3751	mutex_unlock(&prog->aux->dst_mutex);
3752
3753	return bpf_link_settle(&link_primer);
3754out_unlock:
3755	if (tr && tr != prog->aux->dst_trampoline)
3756		bpf_trampoline_put(tr);
3757	mutex_unlock(&prog->aux->dst_mutex);
3758	kfree(link);
3759out_put_prog:
3760	if (tgt_prog_fd && tgt_prog)
3761		bpf_prog_put(tgt_prog);
3762	return err;
3763}
3764
3765static void bpf_raw_tp_link_release(struct bpf_link *link)
3766{
3767	struct bpf_raw_tp_link *raw_tp =
3768		container_of(link, struct bpf_raw_tp_link, link);
3769
3770	bpf_probe_unregister(raw_tp->btp, raw_tp);
3771	bpf_put_raw_tracepoint(raw_tp->btp);
3772}
3773
3774static void bpf_raw_tp_link_dealloc(struct bpf_link *link)
3775{
3776	struct bpf_raw_tp_link *raw_tp =
3777		container_of(link, struct bpf_raw_tp_link, link);
3778
3779	kfree(raw_tp);
3780}
3781
3782static void bpf_raw_tp_link_show_fdinfo(const struct bpf_link *link,
3783					struct seq_file *seq)
3784{
3785	struct bpf_raw_tp_link *raw_tp_link =
3786		container_of(link, struct bpf_raw_tp_link, link);
3787
3788	seq_printf(seq,
3789		   "tp_name:\t%s\n"
3790		   "cookie:\t%llu\n",
3791		   raw_tp_link->btp->tp->name,
3792		   raw_tp_link->cookie);
3793}
3794
3795static int bpf_copy_to_user(char __user *ubuf, const char *buf, u32 ulen,
3796			    u32 len)
3797{
3798	if (ulen >= len + 1) {
3799		if (copy_to_user(ubuf, buf, len + 1))
3800			return -EFAULT;
3801	} else {
3802		char zero = '\0';
3803
3804		if (copy_to_user(ubuf, buf, ulen - 1))
3805			return -EFAULT;
3806		if (put_user(zero, ubuf + ulen - 1))
3807			return -EFAULT;
3808		return -ENOSPC;
3809	}
3810
3811	return 0;
3812}
3813
3814static int bpf_raw_tp_link_fill_link_info(const struct bpf_link *link,
3815					  struct bpf_link_info *info)
3816{
3817	struct bpf_raw_tp_link *raw_tp_link =
3818		container_of(link, struct bpf_raw_tp_link, link);
3819	char __user *ubuf = u64_to_user_ptr(info->raw_tracepoint.tp_name);
3820	const char *tp_name = raw_tp_link->btp->tp->name;
3821	u32 ulen = info->raw_tracepoint.tp_name_len;
3822	size_t tp_len = strlen(tp_name);
3823
3824	if (!ulen ^ !ubuf)
3825		return -EINVAL;
3826
3827	info->raw_tracepoint.tp_name_len = tp_len + 1;
3828	info->raw_tracepoint.cookie = raw_tp_link->cookie;
3829
3830	if (!ubuf)
3831		return 0;
3832
3833	return bpf_copy_to_user(ubuf, tp_name, ulen, tp_len);
3834}
3835
3836static const struct bpf_link_ops bpf_raw_tp_link_lops = {
3837	.release = bpf_raw_tp_link_release,
3838	.dealloc_deferred = bpf_raw_tp_link_dealloc,
3839	.show_fdinfo = bpf_raw_tp_link_show_fdinfo,
3840	.fill_link_info = bpf_raw_tp_link_fill_link_info,
3841};
3842
3843#ifdef CONFIG_PERF_EVENTS
3844struct bpf_perf_link {
3845	struct bpf_link link;
3846	struct file *perf_file;
3847};
3848
3849static void bpf_perf_link_release(struct bpf_link *link)
3850{
3851	struct bpf_perf_link *perf_link = container_of(link, struct bpf_perf_link, link);
3852	struct perf_event *event = perf_link->perf_file->private_data;
3853
3854	perf_event_free_bpf_prog(event);
3855	fput(perf_link->perf_file);
3856}
3857
3858static void bpf_perf_link_dealloc(struct bpf_link *link)
3859{
3860	struct bpf_perf_link *perf_link = container_of(link, struct bpf_perf_link, link);
3861
3862	kfree(perf_link);
3863}
3864
3865static int bpf_perf_link_fill_common(const struct perf_event *event,
3866				     char __user *uname, u32 *ulenp,
3867				     u64 *probe_offset, u64 *probe_addr,
3868				     u32 *fd_type, unsigned long *missed)
3869{
3870	const char *buf;
3871	u32 prog_id, ulen;
3872	size_t len;
3873	int err;
3874
3875	ulen = *ulenp;
3876	if (!ulen ^ !uname)
3877		return -EINVAL;
3878
3879	err = bpf_get_perf_event_info(event, &prog_id, fd_type, &buf,
3880				      probe_offset, probe_addr, missed);
3881	if (err)
3882		return err;
3883
3884	if (buf) {
3885		len = strlen(buf);
3886		*ulenp = len + 1;
3887	} else {
3888		*ulenp = 1;
3889	}
3890	if (!uname)
3891		return 0;
3892
3893	if (buf) {
3894		err = bpf_copy_to_user(uname, buf, ulen, len);
3895		if (err)
3896			return err;
3897	} else {
3898		char zero = '\0';
3899
3900		if (put_user(zero, uname))
3901			return -EFAULT;
3902	}
3903	return 0;
3904}
3905
3906#ifdef CONFIG_KPROBE_EVENTS
3907static int bpf_perf_link_fill_kprobe(const struct perf_event *event,
3908				     struct bpf_link_info *info)
3909{
3910	unsigned long missed;
3911	char __user *uname;
3912	u64 addr, offset;
3913	u32 ulen, type;
3914	int err;
3915
3916	uname = u64_to_user_ptr(info->perf_event.kprobe.func_name);
3917	ulen = info->perf_event.kprobe.name_len;
3918	err = bpf_perf_link_fill_common(event, uname, &ulen, &offset, &addr,
3919					&type, &missed);
3920	if (err)
3921		return err;
3922	if (type == BPF_FD_TYPE_KRETPROBE)
3923		info->perf_event.type = BPF_PERF_EVENT_KRETPROBE;
3924	else
3925		info->perf_event.type = BPF_PERF_EVENT_KPROBE;
3926	info->perf_event.kprobe.name_len = ulen;
3927	info->perf_event.kprobe.offset = offset;
3928	info->perf_event.kprobe.missed = missed;
3929	if (!kallsyms_show_value(current_cred()))
3930		addr = 0;
3931	info->perf_event.kprobe.addr = addr;
3932	info->perf_event.kprobe.cookie = event->bpf_cookie;
3933	return 0;
3934}
3935
3936static void bpf_perf_link_fdinfo_kprobe(const struct perf_event *event,
3937					struct seq_file *seq)
3938{
3939	const char *name;
3940	int err;
3941	u32 prog_id, type;
3942	u64 offset, addr;
3943	unsigned long missed;
3944
3945	err = bpf_get_perf_event_info(event, &prog_id, &type, &name,
3946				      &offset, &addr, &missed);
3947	if (err)
3948		return;
3949
3950	seq_printf(seq,
3951		   "name:\t%s\n"
3952		   "offset:\t%#llx\n"
3953		   "missed:\t%lu\n"
3954		   "addr:\t%#llx\n"
3955		   "event_type:\t%s\n"
3956		   "cookie:\t%llu\n",
3957		   name, offset, missed, addr,
3958		   type == BPF_FD_TYPE_KRETPROBE ?  "kretprobe" : "kprobe",
3959		   event->bpf_cookie);
3960}
3961#endif
3962
3963#ifdef CONFIG_UPROBE_EVENTS
3964static int bpf_perf_link_fill_uprobe(const struct perf_event *event,
3965				     struct bpf_link_info *info)
3966{
3967	u64 ref_ctr_offset, offset;
3968	char __user *uname;
3969	u32 ulen, type;
3970	int err;
3971
3972	uname = u64_to_user_ptr(info->perf_event.uprobe.file_name);
3973	ulen = info->perf_event.uprobe.name_len;
3974	err = bpf_perf_link_fill_common(event, uname, &ulen, &offset, &ref_ctr_offset,
3975					&type, NULL);
3976	if (err)
3977		return err;
3978
3979	if (type == BPF_FD_TYPE_URETPROBE)
3980		info->perf_event.type = BPF_PERF_EVENT_URETPROBE;
3981	else
3982		info->perf_event.type = BPF_PERF_EVENT_UPROBE;
3983	info->perf_event.uprobe.name_len = ulen;
3984	info->perf_event.uprobe.offset = offset;
3985	info->perf_event.uprobe.cookie = event->bpf_cookie;
3986	info->perf_event.uprobe.ref_ctr_offset = ref_ctr_offset;
3987	return 0;
3988}
3989
3990static void bpf_perf_link_fdinfo_uprobe(const struct perf_event *event,
3991					struct seq_file *seq)
3992{
3993	const char *name;
3994	int err;
3995	u32 prog_id, type;
3996	u64 offset, ref_ctr_offset;
3997	unsigned long missed;
3998
3999	err = bpf_get_perf_event_info(event, &prog_id, &type, &name,
4000				      &offset, &ref_ctr_offset, &missed);
4001	if (err)
4002		return;
4003
4004	seq_printf(seq,
4005		   "name:\t%s\n"
4006		   "offset:\t%#llx\n"
4007		   "ref_ctr_offset:\t%#llx\n"
4008		   "event_type:\t%s\n"
4009		   "cookie:\t%llu\n",
4010		   name, offset, ref_ctr_offset,
4011		   type == BPF_FD_TYPE_URETPROBE ?  "uretprobe" : "uprobe",
4012		   event->bpf_cookie);
4013}
4014#endif
4015
4016static int bpf_perf_link_fill_probe(const struct perf_event *event,
4017				    struct bpf_link_info *info)
4018{
4019#ifdef CONFIG_KPROBE_EVENTS
4020	if (event->tp_event->flags & TRACE_EVENT_FL_KPROBE)
4021		return bpf_perf_link_fill_kprobe(event, info);
4022#endif
4023#ifdef CONFIG_UPROBE_EVENTS
4024	if (event->tp_event->flags & TRACE_EVENT_FL_UPROBE)
4025		return bpf_perf_link_fill_uprobe(event, info);
4026#endif
4027	return -EOPNOTSUPP;
4028}
4029
4030static int bpf_perf_link_fill_tracepoint(const struct perf_event *event,
4031					 struct bpf_link_info *info)
4032{
4033	char __user *uname;
4034	u32 ulen;
4035	int err;
4036
4037	uname = u64_to_user_ptr(info->perf_event.tracepoint.tp_name);
4038	ulen = info->perf_event.tracepoint.name_len;
4039	err = bpf_perf_link_fill_common(event, uname, &ulen, NULL, NULL, NULL, NULL);
4040	if (err)
4041		return err;
4042
4043	info->perf_event.type = BPF_PERF_EVENT_TRACEPOINT;
4044	info->perf_event.tracepoint.name_len = ulen;
4045	info->perf_event.tracepoint.cookie = event->bpf_cookie;
4046	return 0;
4047}
4048
4049static int bpf_perf_link_fill_perf_event(const struct perf_event *event,
4050					 struct bpf_link_info *info)
4051{
4052	info->perf_event.event.type = event->attr.type;
4053	info->perf_event.event.config = event->attr.config;
4054	info->perf_event.event.cookie = event->bpf_cookie;
4055	info->perf_event.type = BPF_PERF_EVENT_EVENT;
4056	return 0;
4057}
4058
4059static int bpf_perf_link_fill_link_info(const struct bpf_link *link,
4060					struct bpf_link_info *info)
4061{
4062	struct bpf_perf_link *perf_link;
4063	const struct perf_event *event;
4064
4065	perf_link = container_of(link, struct bpf_perf_link, link);
4066	event = perf_get_event(perf_link->perf_file);
4067	if (IS_ERR(event))
4068		return PTR_ERR(event);
4069
4070	switch (event->prog->type) {
4071	case BPF_PROG_TYPE_PERF_EVENT:
4072		return bpf_perf_link_fill_perf_event(event, info);
4073	case BPF_PROG_TYPE_TRACEPOINT:
4074		return bpf_perf_link_fill_tracepoint(event, info);
4075	case BPF_PROG_TYPE_KPROBE:
4076		return bpf_perf_link_fill_probe(event, info);
4077	default:
4078		return -EOPNOTSUPP;
4079	}
4080}
4081
4082static void bpf_perf_event_link_show_fdinfo(const struct perf_event *event,
4083					    struct seq_file *seq)
4084{
4085	seq_printf(seq,
4086		   "type:\t%u\n"
4087		   "config:\t%llu\n"
4088		   "event_type:\t%s\n"
4089		   "cookie:\t%llu\n",
4090		   event->attr.type, event->attr.config,
4091		   "event", event->bpf_cookie);
4092}
4093
4094static void bpf_tracepoint_link_show_fdinfo(const struct perf_event *event,
4095					    struct seq_file *seq)
4096{
4097	int err;
4098	const char *name;
4099	u32 prog_id;
4100
4101	err = bpf_get_perf_event_info(event, &prog_id, NULL, &name, NULL,
4102				      NULL, NULL);
4103	if (err)
4104		return;
4105
4106	seq_printf(seq,
4107		   "tp_name:\t%s\n"
4108		   "event_type:\t%s\n"
4109		   "cookie:\t%llu\n",
4110		   name, "tracepoint", event->bpf_cookie);
4111}
4112
4113static void bpf_probe_link_show_fdinfo(const struct perf_event *event,
4114				       struct seq_file *seq)
4115{
4116#ifdef CONFIG_KPROBE_EVENTS
4117	if (event->tp_event->flags & TRACE_EVENT_FL_KPROBE)
4118		return bpf_perf_link_fdinfo_kprobe(event, seq);
4119#endif
4120
4121#ifdef CONFIG_UPROBE_EVENTS
4122	if (event->tp_event->flags & TRACE_EVENT_FL_UPROBE)
4123		return bpf_perf_link_fdinfo_uprobe(event, seq);
4124#endif
4125}
4126
4127static void bpf_perf_link_show_fdinfo(const struct bpf_link *link,
4128				      struct seq_file *seq)
4129{
4130	struct bpf_perf_link *perf_link;
4131	const struct perf_event *event;
4132
4133	perf_link = container_of(link, struct bpf_perf_link, link);
4134	event = perf_get_event(perf_link->perf_file);
4135	if (IS_ERR(event))
4136		return;
4137
4138	switch (event->prog->type) {
4139	case BPF_PROG_TYPE_PERF_EVENT:
4140		return bpf_perf_event_link_show_fdinfo(event, seq);
4141	case BPF_PROG_TYPE_TRACEPOINT:
4142		return bpf_tracepoint_link_show_fdinfo(event, seq);
4143	case BPF_PROG_TYPE_KPROBE:
4144		return bpf_probe_link_show_fdinfo(event, seq);
4145	default:
4146		return;
4147	}
4148}
4149
4150static const struct bpf_link_ops bpf_perf_link_lops = {
4151	.release = bpf_perf_link_release,
4152	.dealloc = bpf_perf_link_dealloc,
4153	.fill_link_info = bpf_perf_link_fill_link_info,
4154	.show_fdinfo = bpf_perf_link_show_fdinfo,
4155};
4156
4157static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
4158{
4159	struct bpf_link_primer link_primer;
4160	struct bpf_perf_link *link;
4161	struct perf_event *event;
4162	struct file *perf_file;
4163	int err;
4164
4165	if (attr->link_create.flags)
4166		return -EINVAL;
4167
4168	perf_file = perf_event_get(attr->link_create.target_fd);
4169	if (IS_ERR(perf_file))
4170		return PTR_ERR(perf_file);
4171
4172	link = kzalloc(sizeof(*link), GFP_USER);
4173	if (!link) {
4174		err = -ENOMEM;
4175		goto out_put_file;
4176	}
4177	bpf_link_init(&link->link, BPF_LINK_TYPE_PERF_EVENT, &bpf_perf_link_lops, prog,
4178		      attr->link_create.attach_type);
4179	link->perf_file = perf_file;
4180
4181	err = bpf_link_prime(&link->link, &link_primer);
4182	if (err) {
4183		kfree(link);
4184		goto out_put_file;
4185	}
4186
4187	event = perf_file->private_data;
4188	err = perf_event_set_bpf_prog(event, prog, attr->link_create.perf_event.bpf_cookie);
4189	if (err) {
4190		bpf_link_cleanup(&link_primer);
4191		goto out_put_file;
4192	}
4193	/* perf_event_set_bpf_prog() doesn't take its own refcnt on prog */
4194	bpf_prog_inc(prog);
4195
4196	return bpf_link_settle(&link_primer);
4197
4198out_put_file:
4199	fput(perf_file);
4200	return err;
4201}
4202#else
4203static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
4204{
4205	return -EOPNOTSUPP;
4206}
4207#endif /* CONFIG_PERF_EVENTS */
4208
4209static int bpf_raw_tp_link_attach(struct bpf_prog *prog,
4210				  const char __user *user_tp_name, u64 cookie,
4211				  enum bpf_attach_type attach_type)
4212{
4213	struct bpf_link_primer link_primer;
4214	struct bpf_raw_tp_link *link;
4215	struct bpf_raw_event_map *btp;
4216	const char *tp_name;
4217	char buf[128];
4218	int err;
4219
4220	switch (prog->type) {
4221	case BPF_PROG_TYPE_TRACING:
4222	case BPF_PROG_TYPE_EXT:
4223	case BPF_PROG_TYPE_LSM:
4224		if (user_tp_name)
4225			/* The attach point for this category of programs
4226			 * should be specified via btf_id during program load.
4227			 */
4228			return -EINVAL;
4229		if (prog->type == BPF_PROG_TYPE_TRACING &&
4230		    prog->expected_attach_type == BPF_TRACE_RAW_TP) {
4231			tp_name = prog->aux->attach_func_name;
4232			break;
4233		}
4234		return bpf_tracing_prog_attach(prog, 0, 0, 0, attach_type);
4235	case BPF_PROG_TYPE_RAW_TRACEPOINT:
4236	case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
4237		if (strncpy_from_user(buf, user_tp_name, sizeof(buf) - 1) < 0)
4238			return -EFAULT;
4239		buf[sizeof(buf) - 1] = 0;
4240		tp_name = buf;
4241		break;
4242	default:
4243		return -EINVAL;
4244	}
4245
4246	btp = bpf_get_raw_tracepoint(tp_name);
4247	if (!btp)
4248		return -ENOENT;
4249
4250	link = kzalloc(sizeof(*link), GFP_USER);
4251	if (!link) {
4252		err = -ENOMEM;
4253		goto out_put_btp;
4254	}
4255	bpf_link_init_sleepable(&link->link, BPF_LINK_TYPE_RAW_TRACEPOINT,
4256				&bpf_raw_tp_link_lops, prog, attach_type,
4257				tracepoint_is_faultable(btp->tp));
4258	link->btp = btp;
4259	link->cookie = cookie;
4260
4261	err = bpf_link_prime(&link->link, &link_primer);
4262	if (err) {
4263		kfree(link);
4264		goto out_put_btp;
4265	}
4266
4267	err = bpf_probe_register(link->btp, link);
4268	if (err) {
4269		bpf_link_cleanup(&link_primer);
4270		goto out_put_btp;
4271	}
4272
4273	return bpf_link_settle(&link_primer);
4274
4275out_put_btp:
4276	bpf_put_raw_tracepoint(btp);
4277	return err;
4278}
4279
4280#define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.cookie
4281
4282static int bpf_raw_tracepoint_open(const union bpf_attr *attr)
4283{
4284	struct bpf_prog *prog;
4285	void __user *tp_name;
4286	__u64 cookie;
4287	int fd;
4288
4289	if (CHECK_ATTR(BPF_RAW_TRACEPOINT_OPEN))
4290		return -EINVAL;
4291
4292	prog = bpf_prog_get(attr->raw_tracepoint.prog_fd);
4293	if (IS_ERR(prog))
4294		return PTR_ERR(prog);
4295
4296	tp_name = u64_to_user_ptr(attr->raw_tracepoint.name);
4297	cookie = attr->raw_tracepoint.cookie;
4298	fd = bpf_raw_tp_link_attach(prog, tp_name, cookie, prog->expected_attach_type);
4299	if (fd < 0)
4300		bpf_prog_put(prog);
4301	return fd;
4302}
4303
4304static enum bpf_prog_type
4305attach_type_to_prog_type(enum bpf_attach_type attach_type)
4306{
4307	switch (attach_type) {
4308	case BPF_CGROUP_INET_INGRESS:
4309	case BPF_CGROUP_INET_EGRESS:
4310		return BPF_PROG_TYPE_CGROUP_SKB;
4311	case BPF_CGROUP_INET_SOCK_CREATE:
4312	case BPF_CGROUP_INET_SOCK_RELEASE:
4313	case BPF_CGROUP_INET4_POST_BIND:
4314	case BPF_CGROUP_INET6_POST_BIND:
4315		return BPF_PROG_TYPE_CGROUP_SOCK;
4316	case BPF_CGROUP_INET4_BIND:
4317	case BPF_CGROUP_INET6_BIND:
4318	case BPF_CGROUP_INET4_CONNECT:
4319	case BPF_CGROUP_INET6_CONNECT:
4320	case BPF_CGROUP_UNIX_CONNECT:
4321	case BPF_CGROUP_INET4_GETPEERNAME:
4322	case BPF_CGROUP_INET6_GETPEERNAME:
4323	case BPF_CGROUP_UNIX_GETPEERNAME:
4324	case BPF_CGROUP_INET4_GETSOCKNAME:
4325	case BPF_CGROUP_INET6_GETSOCKNAME:
4326	case BPF_CGROUP_UNIX_GETSOCKNAME:
4327	case BPF_CGROUP_UDP4_SENDMSG:
4328	case BPF_CGROUP_UDP6_SENDMSG:
4329	case BPF_CGROUP_UNIX_SENDMSG:
4330	case BPF_CGROUP_UDP4_RECVMSG:
4331	case BPF_CGROUP_UDP6_RECVMSG:
4332	case BPF_CGROUP_UNIX_RECVMSG:
4333		return BPF_PROG_TYPE_CGROUP_SOCK_ADDR;
4334	case BPF_CGROUP_SOCK_OPS:
4335		return BPF_PROG_TYPE_SOCK_OPS;
4336	case BPF_CGROUP_DEVICE:
4337		return BPF_PROG_TYPE_CGROUP_DEVICE;
4338	case BPF_SK_MSG_VERDICT:
4339		return BPF_PROG_TYPE_SK_MSG;
4340	case BPF_SK_SKB_STREAM_PARSER:
4341	case BPF_SK_SKB_STREAM_VERDICT:
4342	case BPF_SK_SKB_VERDICT:
4343		return BPF_PROG_TYPE_SK_SKB;
4344	case BPF_LIRC_MODE2:
4345		return BPF_PROG_TYPE_LIRC_MODE2;
4346	case BPF_FLOW_DISSECTOR:
4347		return BPF_PROG_TYPE_FLOW_DISSECTOR;
4348	case BPF_CGROUP_SYSCTL:
4349		return BPF_PROG_TYPE_CGROUP_SYSCTL;
4350	case BPF_CGROUP_GETSOCKOPT:
4351	case BPF_CGROUP_SETSOCKOPT:
4352		return BPF_PROG_TYPE_CGROUP_SOCKOPT;
4353	case BPF_TRACE_ITER:
4354	case BPF_TRACE_RAW_TP:
4355	case BPF_TRACE_FENTRY:
4356	case BPF_TRACE_FEXIT:
4357	case BPF_MODIFY_RETURN:
4358		return BPF_PROG_TYPE_TRACING;
4359	case BPF_LSM_MAC:
4360		return BPF_PROG_TYPE_LSM;
4361	case BPF_SK_LOOKUP:
4362		return BPF_PROG_TYPE_SK_LOOKUP;
4363	case BPF_XDP:
4364		return BPF_PROG_TYPE_XDP;
4365	case BPF_LSM_CGROUP:
4366		return BPF_PROG_TYPE_LSM;
4367	case BPF_TCX_INGRESS:
4368	case BPF_TCX_EGRESS:
4369	case BPF_NETKIT_PRIMARY:
4370	case BPF_NETKIT_PEER:
4371		return BPF_PROG_TYPE_SCHED_CLS;
4372	default:
4373		return BPF_PROG_TYPE_UNSPEC;
4374	}
4375}
4376
4377static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog,
4378					     enum bpf_attach_type attach_type)
4379{
4380	enum bpf_prog_type ptype;
4381
4382	switch (prog->type) {
4383	case BPF_PROG_TYPE_CGROUP_SOCK:
4384	case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
4385	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
4386	case BPF_PROG_TYPE_SK_LOOKUP:
4387		return attach_type == prog->expected_attach_type ? 0 : -EINVAL;
4388	case BPF_PROG_TYPE_CGROUP_SKB:
4389		if (!bpf_token_capable(prog->aux->token, CAP_NET_ADMIN))
4390			/* cg-skb progs can be loaded by unpriv user.
4391			 * check permissions at attach time.
4392			 */
4393			return -EPERM;
4394
4395		ptype = attach_type_to_prog_type(attach_type);
4396		if (prog->type != ptype)
4397			return -EINVAL;
4398
4399		return prog->enforce_expected_attach_type &&
4400			prog->expected_attach_type != attach_type ?
4401			-EINVAL : 0;
4402	case BPF_PROG_TYPE_EXT:
4403		return 0;
4404	case BPF_PROG_TYPE_NETFILTER:
4405		if (attach_type != BPF_NETFILTER)
4406			return -EINVAL;
4407		return 0;
4408	case BPF_PROG_TYPE_PERF_EVENT:
4409	case BPF_PROG_TYPE_TRACEPOINT:
4410		if (attach_type != BPF_PERF_EVENT)
4411			return -EINVAL;
4412		return 0;
4413	case BPF_PROG_TYPE_KPROBE:
4414		if (prog->expected_attach_type == BPF_TRACE_KPROBE_MULTI &&
4415		    attach_type != BPF_TRACE_KPROBE_MULTI)
4416			return -EINVAL;
4417		if (prog->expected_attach_type == BPF_TRACE_KPROBE_SESSION &&
4418		    attach_type != BPF_TRACE_KPROBE_SESSION)
4419			return -EINVAL;
4420		if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI &&
4421		    attach_type != BPF_TRACE_UPROBE_MULTI)
4422			return -EINVAL;
4423		if (prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION &&
4424		    attach_type != BPF_TRACE_UPROBE_SESSION)
4425			return -EINVAL;
4426		if (attach_type != BPF_PERF_EVENT &&
4427		    attach_type != BPF_TRACE_KPROBE_MULTI &&
4428		    attach_type != BPF_TRACE_KPROBE_SESSION &&
4429		    attach_type != BPF_TRACE_UPROBE_MULTI &&
4430		    attach_type != BPF_TRACE_UPROBE_SESSION)
4431			return -EINVAL;
4432		return 0;
4433	case BPF_PROG_TYPE_SCHED_CLS:
4434		if (attach_type != BPF_TCX_INGRESS &&
4435		    attach_type != BPF_TCX_EGRESS &&
4436		    attach_type != BPF_NETKIT_PRIMARY &&
4437		    attach_type != BPF_NETKIT_PEER)
4438			return -EINVAL;
4439		return 0;
4440	default:
4441		ptype = attach_type_to_prog_type(attach_type);
4442		if (ptype == BPF_PROG_TYPE_UNSPEC || ptype != prog->type)
4443			return -EINVAL;
4444		return 0;
4445	}
4446}
4447
4448static bool is_cgroup_prog_type(enum bpf_prog_type ptype, enum bpf_attach_type atype,
4449				bool check_atype)
4450{
4451	switch (ptype) {
4452	case BPF_PROG_TYPE_CGROUP_DEVICE:
4453	case BPF_PROG_TYPE_CGROUP_SKB:
4454	case BPF_PROG_TYPE_CGROUP_SOCK:
4455	case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
4456	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
4457	case BPF_PROG_TYPE_CGROUP_SYSCTL:
4458	case BPF_PROG_TYPE_SOCK_OPS:
4459		return true;
4460	case BPF_PROG_TYPE_LSM:
4461		return check_atype ? atype == BPF_LSM_CGROUP : true;
4462	default:
4463		return false;
4464	}
4465}
4466
4467#define BPF_PROG_ATTACH_LAST_FIELD expected_revision
4468
4469#define BPF_F_ATTACH_MASK_BASE	\
4470	(BPF_F_ALLOW_OVERRIDE |	\
4471	 BPF_F_ALLOW_MULTI |	\
4472	 BPF_F_REPLACE |	\
4473	 BPF_F_PREORDER)
4474
4475#define BPF_F_ATTACH_MASK_MPROG	\
4476	(BPF_F_REPLACE |	\
4477	 BPF_F_BEFORE |		\
4478	 BPF_F_AFTER |		\
4479	 BPF_F_ID |		\
4480	 BPF_F_LINK)
4481
4482static int bpf_prog_attach(const union bpf_attr *attr)
4483{
4484	enum bpf_prog_type ptype;
4485	struct bpf_prog *prog;
4486	int ret;
4487
4488	if (CHECK_ATTR(BPF_PROG_ATTACH))
4489		return -EINVAL;
4490
4491	ptype = attach_type_to_prog_type(attr->attach_type);
4492	if (ptype == BPF_PROG_TYPE_UNSPEC)
4493		return -EINVAL;
4494	if (bpf_mprog_supported(ptype)) {
4495		if (attr->attach_flags & ~BPF_F_ATTACH_MASK_MPROG)
4496			return -EINVAL;
4497	} else if (is_cgroup_prog_type(ptype, 0, false)) {
4498		if (attr->attach_flags & ~(BPF_F_ATTACH_MASK_BASE | BPF_F_ATTACH_MASK_MPROG))
4499			return -EINVAL;
4500	} else {
4501		if (attr->attach_flags & ~BPF_F_ATTACH_MASK_BASE)
4502			return -EINVAL;
4503		if (attr->relative_fd ||
4504		    attr->expected_revision)
4505			return -EINVAL;
4506	}
4507
4508	prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
4509	if (IS_ERR(prog))
4510		return PTR_ERR(prog);
4511
4512	if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) {
4513		bpf_prog_put(prog);
4514		return -EINVAL;
4515	}
4516
4517	if (is_cgroup_prog_type(ptype, prog->expected_attach_type, true)) {
4518		ret = cgroup_bpf_prog_attach(attr, ptype, prog);
4519		goto out;
4520	}
4521
4522	switch (ptype) {
4523	case BPF_PROG_TYPE_SK_SKB:
4524	case BPF_PROG_TYPE_SK_MSG:
4525		ret = sock_map_get_from_fd(attr, prog);
4526		break;
4527	case BPF_PROG_TYPE_LIRC_MODE2:
4528		ret = lirc_prog_attach(attr, prog);
4529		break;
4530	case BPF_PROG_TYPE_FLOW_DISSECTOR:
4531		ret = netns_bpf_prog_attach(attr, prog);
4532		break;
4533	case BPF_PROG_TYPE_SCHED_CLS:
4534		if (attr->attach_type == BPF_TCX_INGRESS ||
4535		    attr->attach_type == BPF_TCX_EGRESS)
4536			ret = tcx_prog_attach(attr, prog);
4537		else
4538			ret = netkit_prog_attach(attr, prog);
4539		break;
4540	default:
4541		ret = -EINVAL;
4542	}
4543out:
4544	if (ret)
4545		bpf_prog_put(prog);
4546	return ret;
4547}
4548
4549#define BPF_PROG_DETACH_LAST_FIELD expected_revision
4550
4551static int bpf_prog_detach(const union bpf_attr *attr)
4552{
4553	struct bpf_prog *prog = NULL;
4554	enum bpf_prog_type ptype;
4555	int ret;
4556
4557	if (CHECK_ATTR(BPF_PROG_DETACH))
4558		return -EINVAL;
4559
4560	ptype = attach_type_to_prog_type(attr->attach_type);
4561	if (bpf_mprog_supported(ptype)) {
4562		if (ptype == BPF_PROG_TYPE_UNSPEC)
4563			return -EINVAL;
4564		if (attr->attach_flags & ~BPF_F_ATTACH_MASK_MPROG)
4565			return -EINVAL;
4566		if (attr->attach_bpf_fd) {
4567			prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
4568			if (IS_ERR(prog))
4569				return PTR_ERR(prog);
4570		} else if (!bpf_mprog_detach_empty(ptype)) {
4571			return -EPERM;
4572		}
4573	} else if (is_cgroup_prog_type(ptype, 0, false)) {
4574		if (attr->attach_flags || attr->relative_fd)
4575			return -EINVAL;
4576	} else if (attr->attach_flags ||
4577		   attr->relative_fd ||
4578		   attr->expected_revision) {
4579		return -EINVAL;
4580	}
4581
4582	switch (ptype) {
4583	case BPF_PROG_TYPE_SK_MSG:
4584	case BPF_PROG_TYPE_SK_SKB:
4585		ret = sock_map_prog_detach(attr, ptype);
4586		break;
4587	case BPF_PROG_TYPE_LIRC_MODE2:
4588		ret = lirc_prog_detach(attr);
4589		break;
4590	case BPF_PROG_TYPE_FLOW_DISSECTOR:
4591		ret = netns_bpf_prog_detach(attr, ptype);
4592		break;
4593	case BPF_PROG_TYPE_CGROUP_DEVICE:
4594	case BPF_PROG_TYPE_CGROUP_SKB:
4595	case BPF_PROG_TYPE_CGROUP_SOCK:
4596	case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
4597	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
4598	case BPF_PROG_TYPE_CGROUP_SYSCTL:
4599	case BPF_PROG_TYPE_SOCK_OPS:
4600	case BPF_PROG_TYPE_LSM:
4601		ret = cgroup_bpf_prog_detach(attr, ptype);
4602		break;
4603	case BPF_PROG_TYPE_SCHED_CLS:
4604		if (attr->attach_type == BPF_TCX_INGRESS ||
4605		    attr->attach_type == BPF_TCX_EGRESS)
4606			ret = tcx_prog_detach(attr, prog);
4607		else
4608			ret = netkit_prog_detach(attr, prog);
4609		break;
4610	default:
4611		ret = -EINVAL;
4612	}
4613
4614	if (prog)
4615		bpf_prog_put(prog);
4616	return ret;
4617}
4618
4619#define BPF_PROG_QUERY_LAST_FIELD query.revision
4620
4621static int bpf_prog_query(const union bpf_attr *attr,
4622			  union bpf_attr __user *uattr)
4623{
4624	if (!bpf_net_capable())
4625		return -EPERM;
4626	if (CHECK_ATTR(BPF_PROG_QUERY))
4627		return -EINVAL;
4628	if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE)
4629		return -EINVAL;
4630
4631	switch (attr->query.attach_type) {
4632	case BPF_CGROUP_INET_INGRESS:
4633	case BPF_CGROUP_INET_EGRESS:
4634	case BPF_CGROUP_INET_SOCK_CREATE:
4635	case BPF_CGROUP_INET_SOCK_RELEASE:
4636	case BPF_CGROUP_INET4_BIND:
4637	case BPF_CGROUP_INET6_BIND:
4638	case BPF_CGROUP_INET4_POST_BIND:
4639	case BPF_CGROUP_INET6_POST_BIND:
4640	case BPF_CGROUP_INET4_CONNECT:
4641	case BPF_CGROUP_INET6_CONNECT:
4642	case BPF_CGROUP_UNIX_CONNECT:
4643	case BPF_CGROUP_INET4_GETPEERNAME:
4644	case BPF_CGROUP_INET6_GETPEERNAME:
4645	case BPF_CGROUP_UNIX_GETPEERNAME:
4646	case BPF_CGROUP_INET4_GETSOCKNAME:
4647	case BPF_CGROUP_INET6_GETSOCKNAME:
4648	case BPF_CGROUP_UNIX_GETSOCKNAME:
4649	case BPF_CGROUP_UDP4_SENDMSG:
4650	case BPF_CGROUP_UDP6_SENDMSG:
4651	case BPF_CGROUP_UNIX_SENDMSG:
4652	case BPF_CGROUP_UDP4_RECVMSG:
4653	case BPF_CGROUP_UDP6_RECVMSG:
4654	case BPF_CGROUP_UNIX_RECVMSG:
4655	case BPF_CGROUP_SOCK_OPS:
4656	case BPF_CGROUP_DEVICE:
4657	case BPF_CGROUP_SYSCTL:
4658	case BPF_CGROUP_GETSOCKOPT:
4659	case BPF_CGROUP_SETSOCKOPT:
4660	case BPF_LSM_CGROUP:
4661		return cgroup_bpf_prog_query(attr, uattr);
4662	case BPF_LIRC_MODE2:
4663		return lirc_prog_query(attr, uattr);
4664	case BPF_FLOW_DISSECTOR:
4665	case BPF_SK_LOOKUP:
4666		return netns_bpf_prog_query(attr, uattr);
4667	case BPF_SK_SKB_STREAM_PARSER:
4668	case BPF_SK_SKB_STREAM_VERDICT:
4669	case BPF_SK_MSG_VERDICT:
4670	case BPF_SK_SKB_VERDICT:
4671		return sock_map_bpf_prog_query(attr, uattr);
4672	case BPF_TCX_INGRESS:
4673	case BPF_TCX_EGRESS:
4674		return tcx_prog_query(attr, uattr);
4675	case BPF_NETKIT_PRIMARY:
4676	case BPF_NETKIT_PEER:
4677		return netkit_prog_query(attr, uattr);
4678	default:
4679		return -EINVAL;
4680	}
4681}
4682
4683#define BPF_PROG_TEST_RUN_LAST_FIELD test.batch_size
4684
4685static int bpf_prog_test_run(const union bpf_attr *attr,
4686			     union bpf_attr __user *uattr)
4687{
4688	struct bpf_prog *prog;
4689	int ret = -ENOTSUPP;
4690
4691	if (CHECK_ATTR(BPF_PROG_TEST_RUN))
4692		return -EINVAL;
4693
4694	if ((attr->test.ctx_size_in && !attr->test.ctx_in) ||
4695	    (!attr->test.ctx_size_in && attr->test.ctx_in))
4696		return -EINVAL;
4697
4698	if ((attr->test.ctx_size_out && !attr->test.ctx_out) ||
4699	    (!attr->test.ctx_size_out && attr->test.ctx_out))
4700		return -EINVAL;
4701
4702	prog = bpf_prog_get(attr->test.prog_fd);
4703	if (IS_ERR(prog))
4704		return PTR_ERR(prog);
4705
4706	if (prog->aux->ops->test_run)
4707		ret = prog->aux->ops->test_run(prog, attr, uattr);
4708
4709	bpf_prog_put(prog);
4710	return ret;
4711}
4712
4713#define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id
4714
4715static int bpf_obj_get_next_id(const union bpf_attr *attr,
4716			       union bpf_attr __user *uattr,
4717			       struct idr *idr,
4718			       spinlock_t *lock)
4719{
4720	u32 next_id = attr->start_id;
4721	int err = 0;
4722
4723	if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX)
4724		return -EINVAL;
4725
4726	if (!capable(CAP_SYS_ADMIN))
4727		return -EPERM;
4728
4729	next_id++;
4730	spin_lock_bh(lock);
4731	if (!idr_get_next(idr, &next_id))
4732		err = -ENOENT;
4733	spin_unlock_bh(lock);
4734
4735	if (!err)
4736		err = put_user(next_id, &uattr->next_id);
4737
4738	return err;
4739}
4740
4741struct bpf_map *bpf_map_get_curr_or_next(u32 *id)
4742{
4743	struct bpf_map *map;
4744
4745	spin_lock_bh(&map_idr_lock);
4746again:
4747	map = idr_get_next(&map_idr, id);
4748	if (map) {
4749		map = __bpf_map_inc_not_zero(map, false);
4750		if (IS_ERR(map)) {
4751			(*id)++;
4752			goto again;
4753		}
4754	}
4755	spin_unlock_bh(&map_idr_lock);
4756
4757	return map;
4758}
4759
4760struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id)
4761{
4762	struct bpf_prog *prog;
4763
4764	spin_lock_bh(&prog_idr_lock);
4765again:
4766	prog = idr_get_next(&prog_idr, id);
4767	if (prog) {
4768		prog = bpf_prog_inc_not_zero(prog);
4769		if (IS_ERR(prog)) {
4770			(*id)++;
4771			goto again;
4772		}
4773	}
4774	spin_unlock_bh(&prog_idr_lock);
4775
4776	return prog;
4777}
4778
4779#define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id
4780
4781struct bpf_prog *bpf_prog_by_id(u32 id)
4782{
4783	struct bpf_prog *prog;
4784
4785	if (!id)
4786		return ERR_PTR(-ENOENT);
4787
4788	spin_lock_bh(&prog_idr_lock);
4789	prog = idr_find(&prog_idr, id);
4790	if (prog)
4791		prog = bpf_prog_inc_not_zero(prog);
4792	else
4793		prog = ERR_PTR(-ENOENT);
4794	spin_unlock_bh(&prog_idr_lock);
4795	return prog;
4796}
4797
4798static int bpf_prog_get_fd_by_id(const union bpf_attr *attr)
4799{
4800	struct bpf_prog *prog;
4801	u32 id = attr->prog_id;
4802	int fd;
4803
4804	if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID))
4805		return -EINVAL;
4806
4807	if (!capable(CAP_SYS_ADMIN))
4808		return -EPERM;
4809
4810	prog = bpf_prog_by_id(id);
4811	if (IS_ERR(prog))
4812		return PTR_ERR(prog);
4813
4814	fd = bpf_prog_new_fd(prog);
4815	if (fd < 0)
4816		bpf_prog_put(prog);
4817
4818	return fd;
4819}
4820
4821#define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags
4822
4823static int bpf_map_get_fd_by_id(const union bpf_attr *attr)
4824{
4825	struct bpf_map *map;
4826	u32 id = attr->map_id;
4827	int f_flags;
4828	int fd;
4829
4830	if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) ||
4831	    attr->open_flags & ~BPF_OBJ_FLAG_MASK)
4832		return -EINVAL;
4833
4834	if (!capable(CAP_SYS_ADMIN))
4835		return -EPERM;
4836
4837	f_flags = bpf_get_file_flag(attr->open_flags);
4838	if (f_flags < 0)
4839		return f_flags;
4840
4841	spin_lock_bh(&map_idr_lock);
4842	map = idr_find(&map_idr, id);
4843	if (map)
4844		map = __bpf_map_inc_not_zero(map, true);
4845	else
4846		map = ERR_PTR(-ENOENT);
4847	spin_unlock_bh(&map_idr_lock);
4848
4849	if (IS_ERR(map))
4850		return PTR_ERR(map);
4851
4852	fd = bpf_map_new_fd(map, f_flags);
4853	if (fd < 0)
4854		bpf_map_put_with_uref(map);
4855
4856	return fd;
4857}
4858
4859static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog,
4860					      unsigned long addr, u32 *off,
4861					      u32 *type)
4862{
4863	const struct bpf_map *map;
4864	int i;
4865
4866	mutex_lock(&prog->aux->used_maps_mutex);
4867	for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) {
4868		map = prog->aux->used_maps[i];
4869		if (map == (void *)addr) {
4870			*type = BPF_PSEUDO_MAP_FD;
4871			goto out;
4872		}
4873		if (!map->ops->map_direct_value_meta)
4874			continue;
4875		if (!map->ops->map_direct_value_meta(map, addr, off)) {
4876			*type = BPF_PSEUDO_MAP_VALUE;
4877			goto out;
4878		}
4879	}
4880	map = NULL;
4881
4882out:
4883	mutex_unlock(&prog->aux->used_maps_mutex);
4884	return map;
4885}
4886
4887static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog,
4888					      const struct cred *f_cred)
4889{
4890	const struct bpf_map *map;
4891	struct bpf_insn *insns;
4892	u32 off, type;
4893	u64 imm;
4894	u8 code;
4895	int i;
4896
4897	insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog),
4898			GFP_USER);
4899	if (!insns)
4900		return insns;
4901
4902	for (i = 0; i < prog->len; i++) {
4903		code = insns[i].code;
4904
4905		if (code == (BPF_JMP | BPF_TAIL_CALL)) {
4906			insns[i].code = BPF_JMP | BPF_CALL;
4907			insns[i].imm = BPF_FUNC_tail_call;
4908			/* fall-through */
4909		}
4910		if (code == (BPF_JMP | BPF_CALL) ||
4911		    code == (BPF_JMP | BPF_CALL_ARGS)) {
4912			if (code == (BPF_JMP | BPF_CALL_ARGS))
4913				insns[i].code = BPF_JMP | BPF_CALL;
4914			if (!bpf_dump_raw_ok(f_cred))
4915				insns[i].imm = 0;
4916			continue;
4917		}
4918		if (BPF_CLASS(code) == BPF_LDX && BPF_MODE(code) == BPF_PROBE_MEM) {
4919			insns[i].code = BPF_LDX | BPF_SIZE(code) | BPF_MEM;
4920			continue;
4921		}
4922
4923		if ((BPF_CLASS(code) == BPF_LDX || BPF_CLASS(code) == BPF_STX ||
4924		     BPF_CLASS(code) == BPF_ST) && BPF_MODE(code) == BPF_PROBE_MEM32) {
4925			insns[i].code = BPF_CLASS(code) | BPF_SIZE(code) | BPF_MEM;
4926			continue;
4927		}
4928
4929		if (code != (BPF_LD | BPF_IMM | BPF_DW))
4930			continue;
4931
4932		imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm;
4933		map = bpf_map_from_imm(prog, imm, &off, &type);
4934		if (map) {
4935			insns[i].src_reg = type;
4936			insns[i].imm = map->id;
4937			insns[i + 1].imm = off;
4938			continue;
4939		}
4940	}
4941
4942	return insns;
4943}
4944
4945static int set_info_rec_size(struct bpf_prog_info *info)
4946{
4947	/*
4948	 * Ensure info.*_rec_size is the same as kernel expected size
4949	 *
4950	 * or
4951	 *
4952	 * Only allow zero *_rec_size if both _rec_size and _cnt are
4953	 * zero.  In this case, the kernel will set the expected
4954	 * _rec_size back to the info.
4955	 */
4956
4957	if ((info->nr_func_info || info->func_info_rec_size) &&
4958	    info->func_info_rec_size != sizeof(struct bpf_func_info))
4959		return -EINVAL;
4960
4961	if ((info->nr_line_info || info->line_info_rec_size) &&
4962	    info->line_info_rec_size != sizeof(struct bpf_line_info))
4963		return -EINVAL;
4964
4965	if ((info->nr_jited_line_info || info->jited_line_info_rec_size) &&
4966	    info->jited_line_info_rec_size != sizeof(__u64))
4967		return -EINVAL;
4968
4969	info->func_info_rec_size = sizeof(struct bpf_func_info);
4970	info->line_info_rec_size = sizeof(struct bpf_line_info);
4971	info->jited_line_info_rec_size = sizeof(__u64);
4972
4973	return 0;
4974}
4975
4976static int bpf_prog_get_info_by_fd(struct file *file,
4977				   struct bpf_prog *prog,
4978				   const union bpf_attr *attr,
4979				   union bpf_attr __user *uattr)
4980{
4981	struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info);
4982	struct btf *attach_btf = bpf_prog_get_target_btf(prog);
4983	struct bpf_prog_info info;
4984	u32 info_len = attr->info.info_len;
4985	struct bpf_prog_kstats stats;
4986	char __user *uinsns;
4987	u32 ulen;
4988	int err;
4989
4990	err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len);
4991	if (err)
4992		return err;
4993	info_len = min_t(u32, sizeof(info), info_len);
4994
4995	memset(&info, 0, sizeof(info));
4996	if (copy_from_user(&info, uinfo, info_len))
4997		return -EFAULT;
4998
4999	info.type = prog->type;
5000	info.id = prog->aux->id;
5001	info.load_time = prog->aux->load_time;
5002	info.created_by_uid = from_kuid_munged(current_user_ns(),
5003					       prog->aux->user->uid);
5004	info.gpl_compatible = prog->gpl_compatible;
5005
5006	memcpy(info.tag, prog->tag, sizeof(prog->tag));
5007	memcpy(info.name, prog->aux->name, sizeof(prog->aux->name));
5008
5009	mutex_lock(&prog->aux->used_maps_mutex);
5010	ulen = info.nr_map_ids;
5011	info.nr_map_ids = prog->aux->used_map_cnt;
5012	ulen = min_t(u32, info.nr_map_ids, ulen);
5013	if (ulen) {
5014		u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids);
5015		u32 i;
5016
5017		for (i = 0; i < ulen; i++)
5018			if (put_user(prog->aux->used_maps[i]->id,
5019				     &user_map_ids[i])) {
5020				mutex_unlock(&prog->aux->used_maps_mutex);
5021				return -EFAULT;
5022			}
5023	}
5024	mutex_unlock(&prog->aux->used_maps_mutex);
5025
5026	err = set_info_rec_size(&info);
5027	if (err)
5028		return err;
5029
5030	bpf_prog_get_stats(prog, &stats);
5031	info.run_time_ns = stats.nsecs;
5032	info.run_cnt = stats.cnt;
5033	info.recursion_misses = stats.misses;
5034
5035	info.verified_insns = prog->aux->verified_insns;
5036	if (prog->aux->btf)
5037		info.btf_id = btf_obj_id(prog->aux->btf);
5038
5039	if (!bpf_capable()) {
5040		info.jited_prog_len = 0;
5041		info.xlated_prog_len = 0;
5042		info.nr_jited_ksyms = 0;
5043		info.nr_jited_func_lens = 0;
5044		info.nr_func_info = 0;
5045		info.nr_line_info = 0;
5046		info.nr_jited_line_info = 0;
5047		goto done;
5048	}
5049
5050	ulen = info.xlated_prog_len;
5051	info.xlated_prog_len = bpf_prog_insn_size(prog);
5052	if (info.xlated_prog_len && ulen) {
5053		struct bpf_insn *insns_sanitized;
5054		bool fault;
5055
5056		if (!prog->blinded || bpf_dump_raw_ok(file->f_cred)) {
5057			insns_sanitized = bpf_insn_prepare_dump(prog, file->f_cred);
5058			if (!insns_sanitized)
5059				return -ENOMEM;
5060			uinsns = u64_to_user_ptr(info.xlated_prog_insns);
5061			ulen = min_t(u32, info.xlated_prog_len, ulen);
5062			fault = copy_to_user(uinsns, insns_sanitized, ulen);
5063			kfree(insns_sanitized);
5064			if (fault)
5065				return -EFAULT;
5066		} else {
5067			info.xlated_prog_insns = 0;
5068		}
5069	}
5070
5071	if (bpf_prog_is_offloaded(prog->aux)) {
5072		err = bpf_prog_offload_info_fill(&info, prog);
5073		if (err)
5074			return err;
5075		goto done;
5076	}
5077
5078	/* NOTE: the following code is supposed to be skipped for offload.
5079	 * bpf_prog_offload_info_fill() is the place to fill similar fields
5080	 * for offload.
5081	 */
5082	ulen = info.jited_prog_len;
5083	if (prog->aux->func_cnt) {
5084		u32 i;
5085
5086		info.jited_prog_len = 0;
5087		for (i = 0; i < prog->aux->func_cnt; i++)
5088			info.jited_prog_len += prog->aux->func[i]->jited_len;
5089	} else {
5090		info.jited_prog_len = prog->jited_len;
5091	}
5092
5093	if (info.jited_prog_len && ulen) {
5094		if (bpf_dump_raw_ok(file->f_cred)) {
5095			uinsns = u64_to_user_ptr(info.jited_prog_insns);
5096			ulen = min_t(u32, info.jited_prog_len, ulen);
5097
5098			/* for multi-function programs, copy the JITed
5099			 * instructions for all the functions
5100			 */
5101			if (prog->aux->func_cnt) {
5102				u32 len, free, i;
5103				u8 *img;
5104
5105				free = ulen;
5106				for (i = 0; i < prog->aux->func_cnt; i++) {
5107					len = prog->aux->func[i]->jited_len;
5108					len = min_t(u32, len, free);
5109					img = (u8 *) prog->aux->func[i]->bpf_func;
5110					if (copy_to_user(uinsns, img, len))
5111						return -EFAULT;
5112					uinsns += len;
5113					free -= len;
5114					if (!free)
5115						break;
5116				}
5117			} else {
5118				if (copy_to_user(uinsns, prog->bpf_func, ulen))
5119					return -EFAULT;
5120			}
5121		} else {
5122			info.jited_prog_insns = 0;
5123		}
5124	}
5125
5126	ulen = info.nr_jited_ksyms;
5127	info.nr_jited_ksyms = prog->aux->func_cnt ? : 1;
5128	if (ulen) {
5129		if (bpf_dump_raw_ok(file->f_cred)) {
5130			unsigned long ksym_addr;
5131			u64 __user *user_ksyms;
5132			u32 i;
5133
5134			/* copy the address of the kernel symbol
5135			 * corresponding to each function
5136			 */
5137			ulen = min_t(u32, info.nr_jited_ksyms, ulen);
5138			user_ksyms = u64_to_user_ptr(info.jited_ksyms);
5139			if (prog->aux->func_cnt) {
5140				for (i = 0; i < ulen; i++) {
5141					ksym_addr = (unsigned long)
5142						prog->aux->func[i]->bpf_func;
5143					if (put_user((u64) ksym_addr,
5144						     &user_ksyms[i]))
5145						return -EFAULT;
5146				}
5147			} else {
5148				ksym_addr = (unsigned long) prog->bpf_func;
5149				if (put_user((u64) ksym_addr, &user_ksyms[0]))
5150					return -EFAULT;
5151			}
5152		} else {
5153			info.jited_ksyms = 0;
5154		}
5155	}
5156
5157	ulen = info.nr_jited_func_lens;
5158	info.nr_jited_func_lens = prog->aux->func_cnt ? : 1;
5159	if (ulen) {
5160		if (bpf_dump_raw_ok(file->f_cred)) {
5161			u32 __user *user_lens;
5162			u32 func_len, i;
5163
5164			/* copy the JITed image lengths for each function */
5165			ulen = min_t(u32, info.nr_jited_func_lens, ulen);
5166			user_lens = u64_to_user_ptr(info.jited_func_lens);
5167			if (prog->aux->func_cnt) {
5168				for (i = 0; i < ulen; i++) {
5169					func_len =
5170						prog->aux->func[i]->jited_len;
5171					if (put_user(func_len, &user_lens[i]))
5172						return -EFAULT;
5173				}
5174			} else {
5175				func_len = prog->jited_len;
5176				if (put_user(func_len, &user_lens[0]))
5177					return -EFAULT;
5178			}
5179		} else {
5180			info.jited_func_lens = 0;
5181		}
5182	}
5183
5184	info.attach_btf_id = prog->aux->attach_btf_id;
5185	if (attach_btf)
5186		info.attach_btf_obj_id = btf_obj_id(attach_btf);
5187
5188	ulen = info.nr_func_info;
5189	info.nr_func_info = prog->aux->func_info_cnt;
5190	if (info.nr_func_info && ulen) {
5191		char __user *user_finfo;
5192
5193		user_finfo = u64_to_user_ptr(info.func_info);
5194		ulen = min_t(u32, info.nr_func_info, ulen);
5195		if (copy_to_user(user_finfo, prog->aux->func_info,
5196				 info.func_info_rec_size * ulen))
5197			return -EFAULT;
5198	}
5199
5200	ulen = info.nr_line_info;
5201	info.nr_line_info = prog->aux->nr_linfo;
5202	if (info.nr_line_info && ulen) {
5203		__u8 __user *user_linfo;
5204
5205		user_linfo = u64_to_user_ptr(info.line_info);
5206		ulen = min_t(u32, info.nr_line_info, ulen);
5207		if (copy_to_user(user_linfo, prog->aux->linfo,
5208				 info.line_info_rec_size * ulen))
5209			return -EFAULT;
5210	}
5211
5212	ulen = info.nr_jited_line_info;
5213	if (prog->aux->jited_linfo)
5214		info.nr_jited_line_info = prog->aux->nr_linfo;
5215	else
5216		info.nr_jited_line_info = 0;
5217	if (info.nr_jited_line_info && ulen) {
5218		if (bpf_dump_raw_ok(file->f_cred)) {
5219			unsigned long line_addr;
5220			__u64 __user *user_linfo;
5221			u32 i;
5222
5223			user_linfo = u64_to_user_ptr(info.jited_line_info);
5224			ulen = min_t(u32, info.nr_jited_line_info, ulen);
5225			for (i = 0; i < ulen; i++) {
5226				line_addr = (unsigned long)prog->aux->jited_linfo[i];
5227				if (put_user((__u64)line_addr, &user_linfo[i]))
5228					return -EFAULT;
5229			}
5230		} else {
5231			info.jited_line_info = 0;
5232		}
5233	}
5234
5235	ulen = info.nr_prog_tags;
5236	info.nr_prog_tags = prog->aux->func_cnt ? : 1;
5237	if (ulen) {
5238		__u8 __user (*user_prog_tags)[BPF_TAG_SIZE];
5239		u32 i;
5240
5241		user_prog_tags = u64_to_user_ptr(info.prog_tags);
5242		ulen = min_t(u32, info.nr_prog_tags, ulen);
5243		if (prog->aux->func_cnt) {
5244			for (i = 0; i < ulen; i++) {
5245				if (copy_to_user(user_prog_tags[i],
5246						 prog->aux->func[i]->tag,
5247						 BPF_TAG_SIZE))
5248					return -EFAULT;
5249			}
5250		} else {
5251			if (copy_to_user(user_prog_tags[0],
5252					 prog->tag, BPF_TAG_SIZE))
5253				return -EFAULT;
5254		}
5255	}
5256
5257done:
5258	if (copy_to_user(uinfo, &info, info_len) ||
5259	    put_user(info_len, &uattr->info.info_len))
5260		return -EFAULT;
5261
5262	return 0;
5263}
5264
5265static int bpf_map_get_info_by_fd(struct file *file,
5266				  struct bpf_map *map,
5267				  const union bpf_attr *attr,
5268				  union bpf_attr __user *uattr)
5269{
5270	struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info);
5271	struct bpf_map_info info;
5272	u32 info_len = attr->info.info_len;
5273	int err;
5274
5275	err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len);
5276	if (err)
5277		return err;
5278	info_len = min_t(u32, sizeof(info), info_len);
5279
5280	memset(&info, 0, sizeof(info));
5281	if (copy_from_user(&info, uinfo, info_len))
5282		return -EFAULT;
5283
5284	info.type = map->map_type;
5285	info.id = map->id;
5286	info.key_size = map->key_size;
5287	info.value_size = map->value_size;
5288	info.max_entries = map->max_entries;
5289	info.map_flags = map->map_flags;
5290	info.map_extra = map->map_extra;
5291	memcpy(info.name, map->name, sizeof(map->name));
5292
5293	if (map->btf) {
5294		info.btf_id = btf_obj_id(map->btf);
5295		info.btf_key_type_id = map->btf_key_type_id;
5296		info.btf_value_type_id = map->btf_value_type_id;
5297	}
5298	info.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;
5299	if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS)
5300		bpf_map_struct_ops_info_fill(&info, map);
5301
5302	if (bpf_map_is_offloaded(map)) {
5303		err = bpf_map_offload_info_fill(&info, map);
5304		if (err)
5305			return err;
5306	}
5307
5308	if (info.hash) {
5309		char __user *uhash = u64_to_user_ptr(info.hash);
5310
5311		if (!map->ops->map_get_hash)
5312			return -EINVAL;
5313
5314		if (info.hash_size != SHA256_DIGEST_SIZE)
5315			return -EINVAL;
5316
5317		if (!READ_ONCE(map->frozen))
5318			return -EPERM;
5319
5320		err = map->ops->map_get_hash(map, SHA256_DIGEST_SIZE, map->sha);
5321		if (err != 0)
5322			return err;
5323
5324		if (copy_to_user(uhash, map->sha, SHA256_DIGEST_SIZE) != 0)
5325			return -EFAULT;
5326	} else if (info.hash_size) {
5327		return -EINVAL;
5328	}
5329
5330	if (copy_to_user(uinfo, &info, info_len) ||
5331	    put_user(info_len, &uattr->info.info_len))
5332		return -EFAULT;
5333
5334	return 0;
5335}
5336
5337static int bpf_btf_get_info_by_fd(struct file *file,
5338				  struct btf *btf,
5339				  const union bpf_attr *attr,
5340				  union bpf_attr __user *uattr)
5341{
5342	struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info);
5343	u32 info_len = attr->info.info_len;
5344	int err;
5345
5346	err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(*uinfo), info_len);
5347	if (err)
5348		return err;
5349
5350	return btf_get_info_by_fd(btf, attr, uattr);
5351}
5352
5353static int bpf_link_get_info_by_fd(struct file *file,
5354				  struct bpf_link *link,
5355				  const union bpf_attr *attr,
5356				  union bpf_attr __user *uattr)
5357{
5358	struct bpf_link_info __user *uinfo = u64_to_user_ptr(attr->info.info);
5359	struct bpf_link_info info;
5360	u32 info_len = attr->info.info_len;
5361	int err;
5362
5363	err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len);
5364	if (err)
5365		return err;
5366	info_len = min_t(u32, sizeof(info), info_len);
5367
5368	memset(&info, 0, sizeof(info));
5369	if (copy_from_user(&info, uinfo, info_len))
5370		return -EFAULT;
5371
5372	info.type = link->type;
5373	info.id = link->id;
5374	if (link->prog)
5375		info.prog_id = link->prog->aux->id;
5376
5377	if (link->ops->fill_link_info) {
5378		err = link->ops->fill_link_info(link, &info);
5379		if (err)
5380			return err;
5381	}
5382
5383	if (copy_to_user(uinfo, &info, info_len) ||
5384	    put_user(info_len, &uattr->info.info_len))
5385		return -EFAULT;
5386
5387	return 0;
5388}
5389
5390
5391static int token_get_info_by_fd(struct file *file,
5392				struct bpf_token *token,
5393				const union bpf_attr *attr,
5394				union bpf_attr __user *uattr)
5395{
5396	struct bpf_token_info __user *uinfo = u64_to_user_ptr(attr->info.info);
5397	u32 info_len = attr->info.info_len;
5398	int err;
5399
5400	err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(*uinfo), info_len);
5401	if (err)
5402		return err;
5403	return bpf_token_get_info_by_fd(token, attr, uattr);
5404}
5405
5406#define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info
5407
5408static int bpf_obj_get_info_by_fd(const union bpf_attr *attr,
5409				  union bpf_attr __user *uattr)
5410{
5411	if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD))
5412		return -EINVAL;
5413
5414	CLASS(fd, f)(attr->info.bpf_fd);
5415	if (fd_empty(f))
5416		return -EBADFD;
5417
5418	if (fd_file(f)->f_op == &bpf_prog_fops)
5419		return bpf_prog_get_info_by_fd(fd_file(f), fd_file(f)->private_data, attr,
5420					      uattr);
5421	else if (fd_file(f)->f_op == &bpf_map_fops)
5422		return bpf_map_get_info_by_fd(fd_file(f), fd_file(f)->private_data, attr,
5423					     uattr);
5424	else if (fd_file(f)->f_op == &btf_fops)
5425		return bpf_btf_get_info_by_fd(fd_file(f), fd_file(f)->private_data, attr, uattr);
5426	else if (fd_file(f)->f_op == &bpf_link_fops || fd_file(f)->f_op == &bpf_link_fops_poll)
5427		return bpf_link_get_info_by_fd(fd_file(f), fd_file(f)->private_data,
5428					      attr, uattr);
5429	else if (fd_file(f)->f_op == &bpf_token_fops)
5430		return token_get_info_by_fd(fd_file(f), fd_file(f)->private_data,
5431					    attr, uattr);
5432	return -EINVAL;
5433}
5434
5435#define BPF_BTF_LOAD_LAST_FIELD btf_token_fd
5436
5437static int bpf_btf_load(const union bpf_attr *attr, bpfptr_t uattr, __u32 uattr_size)
5438{
5439	struct bpf_token *token = NULL;
5440
5441	if (CHECK_ATTR(BPF_BTF_LOAD))
5442		return -EINVAL;
5443
5444	if (attr->btf_flags & ~BPF_F_TOKEN_FD)
5445		return -EINVAL;
5446
5447	if (attr->btf_flags & BPF_F_TOKEN_FD) {
5448		token = bpf_token_get_from_fd(attr->btf_token_fd);
5449		if (IS_ERR(token))
5450			return PTR_ERR(token);
5451		if (!bpf_token_allow_cmd(token, BPF_BTF_LOAD)) {
5452			bpf_token_put(token);
5453			token = NULL;
5454		}
5455	}
5456
5457	if (!bpf_token_capable(token, CAP_BPF)) {
5458		bpf_token_put(token);
5459		return -EPERM;
5460	}
5461
5462	bpf_token_put(token);
5463
5464	return btf_new_fd(attr, uattr, uattr_size);
5465}
5466
5467#define BPF_BTF_GET_FD_BY_ID_LAST_FIELD fd_by_id_token_fd
5468
5469static int bpf_btf_get_fd_by_id(const union bpf_attr *attr)
5470{
5471	struct bpf_token *token = NULL;
5472
5473	if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID))
5474		return -EINVAL;
5475
5476	if (attr->open_flags & ~BPF_F_TOKEN_FD)
5477		return -EINVAL;
5478
5479	if (attr->open_flags & BPF_F_TOKEN_FD) {
5480		token = bpf_token_get_from_fd(attr->fd_by_id_token_fd);
5481		if (IS_ERR(token))
5482			return PTR_ERR(token);
5483		if (!bpf_token_allow_cmd(token, BPF_BTF_GET_FD_BY_ID)) {
5484			bpf_token_put(token);
5485			token = NULL;
5486		}
5487	}
5488
5489	if (!bpf_token_capable(token, CAP_SYS_ADMIN)) {
5490		bpf_token_put(token);
5491		return -EPERM;
5492	}
5493
5494	bpf_token_put(token);
5495
5496	return btf_get_fd_by_id(attr->btf_id);
5497}
5498
5499static int bpf_task_fd_query_copy(const union bpf_attr *attr,
5500				    union bpf_attr __user *uattr,
5501				    u32 prog_id, u32 fd_type,
5502				    const char *buf, u64 probe_offset,
5503				    u64 probe_addr)
5504{
5505	char __user *ubuf = u64_to_user_ptr(attr->task_fd_query.buf);
5506	u32 len = buf ? strlen(buf) : 0, input_len;
5507	int err = 0;
5508
5509	if (put_user(len, &uattr->task_fd_query.buf_len))
5510		return -EFAULT;
5511	input_len = attr->task_fd_query.buf_len;
5512	if (input_len && ubuf) {
5513		if (!len) {
5514			/* nothing to copy, just make ubuf NULL terminated */
5515			char zero = '\0';
5516
5517			if (put_user(zero, ubuf))
5518				return -EFAULT;
5519		} else {
5520			err = bpf_copy_to_user(ubuf, buf, input_len, len);
5521			if (err == -EFAULT)
5522				return err;
5523		}
5524	}
5525
5526	if (put_user(prog_id, &uattr->task_fd_query.prog_id) ||
5527	    put_user(fd_type, &uattr->task_fd_query.fd_type) ||
5528	    put_user(probe_offset, &uattr->task_fd_query.probe_offset) ||
5529	    put_user(probe_addr, &uattr->task_fd_query.probe_addr))
5530		return -EFAULT;
5531
5532	return err;
5533}
5534
5535#define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr
5536
5537static int bpf_task_fd_query(const union bpf_attr *attr,
5538			     union bpf_attr __user *uattr)
5539{
5540	pid_t pid = attr->task_fd_query.pid;
5541	u32 fd = attr->task_fd_query.fd;
5542	const struct perf_event *event;
5543	struct task_struct *task;
5544	struct file *file;
5545	int err;
5546
5547	if (CHECK_ATTR(BPF_TASK_FD_QUERY))
5548		return -EINVAL;
5549
5550	if (!capable(CAP_SYS_ADMIN))
5551		return -EPERM;
5552
5553	if (attr->task_fd_query.flags != 0)
5554		return -EINVAL;
5555
5556	rcu_read_lock();
5557	task = get_pid_task(find_vpid(pid), PIDTYPE_PID);
5558	rcu_read_unlock();
5559	if (!task)
5560		return -ENOENT;
5561
5562	err = 0;
5563	file = fget_task(task, fd);
5564	put_task_struct(task);
5565	if (!file)
5566		return -EBADF;
5567
5568	if (file->f_op == &bpf_link_fops || file->f_op == &bpf_link_fops_poll) {
5569		struct bpf_link *link = file->private_data;
5570
5571		if (link->ops == &bpf_raw_tp_link_lops) {
5572			struct bpf_raw_tp_link *raw_tp =
5573				container_of(link, struct bpf_raw_tp_link, link);
5574			struct bpf_raw_event_map *btp = raw_tp->btp;
5575
5576			err = bpf_task_fd_query_copy(attr, uattr,
5577						     raw_tp->link.prog->aux->id,
5578						     BPF_FD_TYPE_RAW_TRACEPOINT,
5579						     btp->tp->name, 0, 0);
5580			goto put_file;
5581		}
5582		goto out_not_supp;
5583	}
5584
5585	event = perf_get_event(file);
5586	if (!IS_ERR(event)) {
5587		u64 probe_offset, probe_addr;
5588		u32 prog_id, fd_type;
5589		const char *buf;
5590
5591		err = bpf_get_perf_event_info(event, &prog_id, &fd_type,
5592					      &buf, &probe_offset,
5593					      &probe_addr, NULL);
5594		if (!err)
5595			err = bpf_task_fd_query_copy(attr, uattr, prog_id,
5596						     fd_type, buf,
5597						     probe_offset,
5598						     probe_addr);
5599		goto put_file;
5600	}
5601
5602out_not_supp:
5603	err = -ENOTSUPP;
5604put_file:
5605	fput(file);
5606	return err;
5607}
5608
5609#define BPF_MAP_BATCH_LAST_FIELD batch.flags
5610
5611#define BPF_DO_BATCH(fn, ...)			\
5612	do {					\
5613		if (!fn) {			\
5614			err = -ENOTSUPP;	\
5615			goto err_put;		\
5616		}				\
5617		err = fn(__VA_ARGS__);		\
5618	} while (0)
5619
5620static int bpf_map_do_batch(const union bpf_attr *attr,
5621			    union bpf_attr __user *uattr,
5622			    int cmd)
5623{
5624	bool has_read  = cmd == BPF_MAP_LOOKUP_BATCH ||
5625			 cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH;
5626	bool has_write = cmd != BPF_MAP_LOOKUP_BATCH;
5627	struct bpf_map *map;
5628	int err;
5629
5630	if (CHECK_ATTR(BPF_MAP_BATCH))
5631		return -EINVAL;
5632
5633	CLASS(fd, f)(attr->batch.map_fd);
5634
5635	map = __bpf_map_get(f);
5636	if (IS_ERR(map))
5637		return PTR_ERR(map);
5638	if (has_write)
5639		bpf_map_write_active_inc(map);
5640	if (has_read && !(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
5641		err = -EPERM;
5642		goto err_put;
5643	}
5644	if (has_write && !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
5645		err = -EPERM;
5646		goto err_put;
5647	}
5648
5649	if (cmd == BPF_MAP_LOOKUP_BATCH)
5650		BPF_DO_BATCH(map->ops->map_lookup_batch, map, attr, uattr);
5651	else if (cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH)
5652		BPF_DO_BATCH(map->ops->map_lookup_and_delete_batch, map, attr, uattr);
5653	else if (cmd == BPF_MAP_UPDATE_BATCH)
5654		BPF_DO_BATCH(map->ops->map_update_batch, map, fd_file(f), attr, uattr);
5655	else
5656		BPF_DO_BATCH(map->ops->map_delete_batch, map, attr, uattr);
5657err_put:
5658	if (has_write) {
5659		maybe_wait_bpf_programs(map);
5660		bpf_map_write_active_dec(map);
5661	}
5662	return err;
5663}
5664
5665#define BPF_LINK_CREATE_LAST_FIELD link_create.uprobe_multi.pid
5666static int link_create(union bpf_attr *attr, bpfptr_t uattr)
5667{
5668	struct bpf_prog *prog;
5669	int ret;
5670
5671	if (CHECK_ATTR(BPF_LINK_CREATE))
5672		return -EINVAL;
5673
5674	if (attr->link_create.attach_type == BPF_STRUCT_OPS)
5675		return bpf_struct_ops_link_create(attr);
5676
5677	prog = bpf_prog_get(attr->link_create.prog_fd);
5678	if (IS_ERR(prog))
5679		return PTR_ERR(prog);
5680
5681	ret = bpf_prog_attach_check_attach_type(prog,
5682						attr->link_create.attach_type);
5683	if (ret)
5684		goto out;
5685
5686	switch (prog->type) {
5687	case BPF_PROG_TYPE_CGROUP_SKB:
5688	case BPF_PROG_TYPE_CGROUP_SOCK:
5689	case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
5690	case BPF_PROG_TYPE_SOCK_OPS:
5691	case BPF_PROG_TYPE_CGROUP_DEVICE:
5692	case BPF_PROG_TYPE_CGROUP_SYSCTL:
5693	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
5694		ret = cgroup_bpf_link_attach(attr, prog);
5695		break;
5696	case BPF_PROG_TYPE_EXT:
5697		ret = bpf_tracing_prog_attach(prog,
5698					      attr->link_create.target_fd,
5699					      attr->link_create.target_btf_id,
5700					      attr->link_create.tracing.cookie,
5701					      attr->link_create.attach_type);
5702		break;
5703	case BPF_PROG_TYPE_LSM:
5704	case BPF_PROG_TYPE_TRACING:
5705		if (attr->link_create.attach_type != prog->expected_attach_type) {
5706			ret = -EINVAL;
5707			goto out;
5708		}
5709		if (prog->expected_attach_type == BPF_TRACE_RAW_TP)
5710			ret = bpf_raw_tp_link_attach(prog, NULL, attr->link_create.tracing.cookie,
5711						     attr->link_create.attach_type);
5712		else if (prog->expected_attach_type == BPF_TRACE_ITER)
5713			ret = bpf_iter_link_attach(attr, uattr, prog);
5714		else if (prog->expected_attach_type == BPF_LSM_CGROUP)
5715			ret = cgroup_bpf_link_attach(attr, prog);
5716		else
5717			ret = bpf_tracing_prog_attach(prog,
5718						      attr->link_create.target_fd,
5719						      attr->link_create.target_btf_id,
5720						      attr->link_create.tracing.cookie,
5721						      attr->link_create.attach_type);
5722		break;
5723	case BPF_PROG_TYPE_FLOW_DISSECTOR:
5724	case BPF_PROG_TYPE_SK_LOOKUP:
5725		ret = netns_bpf_link_create(attr, prog);
5726		break;
5727	case BPF_PROG_TYPE_SK_MSG:
5728	case BPF_PROG_TYPE_SK_SKB:
5729		ret = sock_map_link_create(attr, prog);
5730		break;
5731#ifdef CONFIG_NET
5732	case BPF_PROG_TYPE_XDP:
5733		ret = bpf_xdp_link_attach(attr, prog);
5734		break;
5735	case BPF_PROG_TYPE_SCHED_CLS:
5736		if (attr->link_create.attach_type == BPF_TCX_INGRESS ||
5737		    attr->link_create.attach_type == BPF_TCX_EGRESS)
5738			ret = tcx_link_attach(attr, prog);
5739		else
5740			ret = netkit_link_attach(attr, prog);
5741		break;
5742	case BPF_PROG_TYPE_NETFILTER:
5743		ret = bpf_nf_link_attach(attr, prog);
5744		break;
5745#endif
5746	case BPF_PROG_TYPE_PERF_EVENT:
5747	case BPF_PROG_TYPE_TRACEPOINT:
5748		ret = bpf_perf_link_attach(attr, prog);
5749		break;
5750	case BPF_PROG_TYPE_KPROBE:
5751		if (attr->link_create.attach_type == BPF_PERF_EVENT)
5752			ret = bpf_perf_link_attach(attr, prog);
5753		else if (attr->link_create.attach_type == BPF_TRACE_KPROBE_MULTI ||
5754			 attr->link_create.attach_type == BPF_TRACE_KPROBE_SESSION)
5755			ret = bpf_kprobe_multi_link_attach(attr, prog);
5756		else if (attr->link_create.attach_type == BPF_TRACE_UPROBE_MULTI ||
5757			 attr->link_create.attach_type == BPF_TRACE_UPROBE_SESSION)
5758			ret = bpf_uprobe_multi_link_attach(attr, prog);
5759		break;
5760	default:
5761		ret = -EINVAL;
5762	}
5763
5764out:
5765	if (ret < 0)
5766		bpf_prog_put(prog);
5767	return ret;
5768}
5769
5770static int link_update_map(struct bpf_link *link, union bpf_attr *attr)
5771{
5772	struct bpf_map *new_map, *old_map = NULL;
5773	int ret;
5774
5775	new_map = bpf_map_get(attr->link_update.new_map_fd);
5776	if (IS_ERR(new_map))
5777		return PTR_ERR(new_map);
5778
5779	if (attr->link_update.flags & BPF_F_REPLACE) {
5780		old_map = bpf_map_get(attr->link_update.old_map_fd);
5781		if (IS_ERR(old_map)) {
5782			ret = PTR_ERR(old_map);
5783			goto out_put;
5784		}
5785	} else if (attr->link_update.old_map_fd) {
5786		ret = -EINVAL;
5787		goto out_put;
5788	}
5789
5790	ret = link->ops->update_map(link, new_map, old_map);
5791
5792	if (old_map)
5793		bpf_map_put(old_map);
5794out_put:
5795	bpf_map_put(new_map);
5796	return ret;
5797}
5798
5799#define BPF_LINK_UPDATE_LAST_FIELD link_update.old_prog_fd
5800
5801static int link_update(union bpf_attr *attr)
5802{
5803	struct bpf_prog *old_prog = NULL, *new_prog;
5804	struct bpf_link *link;
5805	u32 flags;
5806	int ret;
5807
5808	if (CHECK_ATTR(BPF_LINK_UPDATE))
5809		return -EINVAL;
5810
5811	flags = attr->link_update.flags;
5812	if (flags & ~BPF_F_REPLACE)
5813		return -EINVAL;
5814
5815	link = bpf_link_get_from_fd(attr->link_update.link_fd);
5816	if (IS_ERR(link))
5817		return PTR_ERR(link);
5818
5819	if (link->ops->update_map) {
5820		ret = link_update_map(link, attr);
5821		goto out_put_link;
5822	}
5823
5824	new_prog = bpf_prog_get(attr->link_update.new_prog_fd);
5825	if (IS_ERR(new_prog)) {
5826		ret = PTR_ERR(new_prog);
5827		goto out_put_link;
5828	}
5829
5830	if (flags & BPF_F_REPLACE) {
5831		old_prog = bpf_prog_get(attr->link_update.old_prog_fd);
5832		if (IS_ERR(old_prog)) {
5833			ret = PTR_ERR(old_prog);
5834			old_prog = NULL;
5835			goto out_put_progs;
5836		}
5837	} else if (attr->link_update.old_prog_fd) {
5838		ret = -EINVAL;
5839		goto out_put_progs;
5840	}
5841
5842	if (link->ops->update_prog)
5843		ret = link->ops->update_prog(link, new_prog, old_prog);
5844	else
5845		ret = -EINVAL;
5846
5847out_put_progs:
5848	if (old_prog)
5849		bpf_prog_put(old_prog);
5850	if (ret)
5851		bpf_prog_put(new_prog);
5852out_put_link:
5853	bpf_link_put_direct(link);
5854	return ret;
5855}
5856
5857#define BPF_LINK_DETACH_LAST_FIELD link_detach.link_fd
5858
5859static int link_detach(union bpf_attr *attr)
5860{
5861	struct bpf_link *link;
5862	int ret;
5863
5864	if (CHECK_ATTR(BPF_LINK_DETACH))
5865		return -EINVAL;
5866
5867	link = bpf_link_get_from_fd(attr->link_detach.link_fd);
5868	if (IS_ERR(link))
5869		return PTR_ERR(link);
5870
5871	if (link->ops->detach)
5872		ret = link->ops->detach(link);
5873	else
5874		ret = -EOPNOTSUPP;
5875
5876	bpf_link_put_direct(link);
5877	return ret;
5878}
5879
5880struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link)
5881{
5882	return atomic64_fetch_add_unless(&link->refcnt, 1, 0) ? link : ERR_PTR(-ENOENT);
5883}
5884EXPORT_SYMBOL(bpf_link_inc_not_zero);
5885
5886struct bpf_link *bpf_link_by_id(u32 id)
5887{
5888	struct bpf_link *link;
5889
5890	if (!id)
5891		return ERR_PTR(-ENOENT);
5892
5893	spin_lock_bh(&link_idr_lock);
5894	/* before link is "settled", ID is 0, pretend it doesn't exist yet */
5895	link = idr_find(&link_idr, id);
5896	if (link) {
5897		if (link->id)
5898			link = bpf_link_inc_not_zero(link);
5899		else
5900			link = ERR_PTR(-EAGAIN);
5901	} else {
5902		link = ERR_PTR(-ENOENT);
5903	}
5904	spin_unlock_bh(&link_idr_lock);
5905	return link;
5906}
5907
5908struct bpf_link *bpf_link_get_curr_or_next(u32 *id)
5909{
5910	struct bpf_link *link;
5911
5912	spin_lock_bh(&link_idr_lock);
5913again:
5914	link = idr_get_next(&link_idr, id);
5915	if (link) {
5916		link = bpf_link_inc_not_zero(link);
5917		if (IS_ERR(link)) {
5918			(*id)++;
5919			goto again;
5920		}
5921	}
5922	spin_unlock_bh(&link_idr_lock);
5923
5924	return link;
5925}
5926
5927#define BPF_LINK_GET_FD_BY_ID_LAST_FIELD link_id
5928
5929static int bpf_link_get_fd_by_id(const union bpf_attr *attr)
5930{
5931	struct bpf_link *link;
5932	u32 id = attr->link_id;
5933	int fd;
5934
5935	if (CHECK_ATTR(BPF_LINK_GET_FD_BY_ID))
5936		return -EINVAL;
5937
5938	if (!capable(CAP_SYS_ADMIN))
5939		return -EPERM;
5940
5941	link = bpf_link_by_id(id);
5942	if (IS_ERR(link))
5943		return PTR_ERR(link);
5944
5945	fd = bpf_link_new_fd(link);
5946	if (fd < 0)
5947		bpf_link_put_direct(link);
5948
5949	return fd;
5950}
5951
5952DEFINE_MUTEX(bpf_stats_enabled_mutex);
5953
5954static int bpf_stats_release(struct inode *inode, struct file *file)
5955{
5956	mutex_lock(&bpf_stats_enabled_mutex);
5957	static_key_slow_dec(&bpf_stats_enabled_key.key);
5958	mutex_unlock(&bpf_stats_enabled_mutex);
5959	return 0;
5960}
5961
5962static const struct file_operations bpf_stats_fops = {
5963	.release = bpf_stats_release,
5964};
5965
5966static int bpf_enable_runtime_stats(void)
5967{
5968	int fd;
5969
5970	mutex_lock(&bpf_stats_enabled_mutex);
5971
5972	/* Set a very high limit to avoid overflow */
5973	if (static_key_count(&bpf_stats_enabled_key.key) > INT_MAX / 2) {
5974		mutex_unlock(&bpf_stats_enabled_mutex);
5975		return -EBUSY;
5976	}
5977
5978	fd = anon_inode_getfd("bpf-stats", &bpf_stats_fops, NULL, O_CLOEXEC);
5979	if (fd >= 0)
5980		static_key_slow_inc(&bpf_stats_enabled_key.key);
5981
5982	mutex_unlock(&bpf_stats_enabled_mutex);
5983	return fd;
5984}
5985
5986#define BPF_ENABLE_STATS_LAST_FIELD enable_stats.type
5987
5988static int bpf_enable_stats(union bpf_attr *attr)
5989{
5990
5991	if (CHECK_ATTR(BPF_ENABLE_STATS))
5992		return -EINVAL;
5993
5994	if (!capable(CAP_SYS_ADMIN))
5995		return -EPERM;
5996
5997	switch (attr->enable_stats.type) {
5998	case BPF_STATS_RUN_TIME:
5999		return bpf_enable_runtime_stats();
6000	default:
6001		break;
6002	}
6003	return -EINVAL;
6004}
6005
6006#define BPF_ITER_CREATE_LAST_FIELD iter_create.flags
6007
6008static int bpf_iter_create(union bpf_attr *attr)
6009{
6010	struct bpf_link *link;
6011	int err;
6012
6013	if (CHECK_ATTR(BPF_ITER_CREATE))
6014		return -EINVAL;
6015
6016	if (attr->iter_create.flags)
6017		return -EINVAL;
6018
6019	link = bpf_link_get_from_fd(attr->iter_create.link_fd);
6020	if (IS_ERR(link))
6021		return PTR_ERR(link);
6022
6023	err = bpf_iter_new_fd(link);
6024	bpf_link_put_direct(link);
6025
6026	return err;
6027}
6028
6029#define BPF_PROG_BIND_MAP_LAST_FIELD prog_bind_map.flags
6030
6031static int bpf_prog_bind_map(union bpf_attr *attr)
6032{
6033	struct bpf_prog *prog;
6034	struct bpf_map *map;
6035	struct bpf_map **used_maps_old, **used_maps_new;
6036	int i, ret = 0;
6037
6038	if (CHECK_ATTR(BPF_PROG_BIND_MAP))
6039		return -EINVAL;
6040
6041	if (attr->prog_bind_map.flags)
6042		return -EINVAL;
6043
6044	prog = bpf_prog_get(attr->prog_bind_map.prog_fd);
6045	if (IS_ERR(prog))
6046		return PTR_ERR(prog);
6047
6048	map = bpf_map_get(attr->prog_bind_map.map_fd);
6049	if (IS_ERR(map)) {
6050		ret = PTR_ERR(map);
6051		goto out_prog_put;
6052	}
6053
6054	mutex_lock(&prog->aux->used_maps_mutex);
6055
6056	used_maps_old = prog->aux->used_maps;
6057
6058	for (i = 0; i < prog->aux->used_map_cnt; i++)
6059		if (used_maps_old[i] == map) {
6060			bpf_map_put(map);
6061			goto out_unlock;
6062		}
6063
6064	used_maps_new = kmalloc_array(prog->aux->used_map_cnt + 1,
6065				      sizeof(used_maps_new[0]),
6066				      GFP_KERNEL);
6067	if (!used_maps_new) {
6068		ret = -ENOMEM;
6069		goto out_unlock;
6070	}
6071
6072	/* The bpf program will not access the bpf map, but for the sake of
6073	 * simplicity, increase sleepable_refcnt for sleepable program as well.
6074	 */
6075	if (prog->sleepable)
6076		atomic64_inc(&map->sleepable_refcnt);
6077	memcpy(used_maps_new, used_maps_old,
6078	       sizeof(used_maps_old[0]) * prog->aux->used_map_cnt);
6079	used_maps_new[prog->aux->used_map_cnt] = map;
6080
6081	prog->aux->used_map_cnt++;
6082	prog->aux->used_maps = used_maps_new;
6083
6084	kfree(used_maps_old);
6085
6086out_unlock:
6087	mutex_unlock(&prog->aux->used_maps_mutex);
6088
6089	if (ret)
6090		bpf_map_put(map);
6091out_prog_put:
6092	bpf_prog_put(prog);
6093	return ret;
6094}
6095
6096#define BPF_TOKEN_CREATE_LAST_FIELD token_create.bpffs_fd
6097
6098static int token_create(union bpf_attr *attr)
6099{
6100	if (CHECK_ATTR(BPF_TOKEN_CREATE))
6101		return -EINVAL;
6102
6103	/* no flags are supported yet */
6104	if (attr->token_create.flags)
6105		return -EINVAL;
6106
6107	return bpf_token_create(attr);
6108}
6109
6110#define BPF_PROG_STREAM_READ_BY_FD_LAST_FIELD prog_stream_read.prog_fd
6111
6112static int prog_stream_read(union bpf_attr *attr)
6113{
6114	char __user *buf = u64_to_user_ptr(attr->prog_stream_read.stream_buf);
6115	u32 len = attr->prog_stream_read.stream_buf_len;
6116	struct bpf_prog *prog;
6117	int ret;
6118
6119	if (CHECK_ATTR(BPF_PROG_STREAM_READ_BY_FD))
6120		return -EINVAL;
6121
6122	prog = bpf_prog_get(attr->prog_stream_read.prog_fd);
6123	if (IS_ERR(prog))
6124		return PTR_ERR(prog);
6125
6126	ret = bpf_prog_stream_read(prog, attr->prog_stream_read.stream_id, buf, len);
6127	bpf_prog_put(prog);
6128
6129	return ret;
6130}
6131
6132static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size)
6133{
6134	union bpf_attr attr;
6135	int err;
6136
6137	err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size);
6138	if (err)
6139		return err;
6140	size = min_t(u32, size, sizeof(attr));
6141
6142	/* copy attributes from user space, may be less than sizeof(bpf_attr) */
6143	memset(&attr, 0, sizeof(attr));
6144	if (copy_from_bpfptr(&attr, uattr, size) != 0)
6145		return -EFAULT;
6146
6147	err = security_bpf(cmd, &attr, size, uattr.is_kernel);
6148	if (err < 0)
6149		return err;
6150
6151	switch (cmd) {
6152	case BPF_MAP_CREATE:
6153		err = map_create(&attr, uattr);
6154		break;
6155	case BPF_MAP_LOOKUP_ELEM:
6156		err = map_lookup_elem(&attr);
6157		break;
6158	case BPF_MAP_UPDATE_ELEM:
6159		err = map_update_elem(&attr, uattr);
6160		break;
6161	case BPF_MAP_DELETE_ELEM:
6162		err = map_delete_elem(&attr, uattr);
6163		break;
6164	case BPF_MAP_GET_NEXT_KEY:
6165		err = map_get_next_key(&attr);
6166		break;
6167	case BPF_MAP_FREEZE:
6168		err = map_freeze(&attr);
6169		break;
6170	case BPF_PROG_LOAD:
6171		err = bpf_prog_load(&attr, uattr, size);
6172		break;
6173	case BPF_OBJ_PIN:
6174		err = bpf_obj_pin(&attr);
6175		break;
6176	case BPF_OBJ_GET:
6177		err = bpf_obj_get(&attr);
6178		break;
6179	case BPF_PROG_ATTACH:
6180		err = bpf_prog_attach(&attr);
6181		break;
6182	case BPF_PROG_DETACH:
6183		err = bpf_prog_detach(&attr);
6184		break;
6185	case BPF_PROG_QUERY:
6186		err = bpf_prog_query(&attr, uattr.user);
6187		break;
6188	case BPF_PROG_TEST_RUN:
6189		err = bpf_prog_test_run(&attr, uattr.user);
6190		break;
6191	case BPF_PROG_GET_NEXT_ID:
6192		err = bpf_obj_get_next_id(&attr, uattr.user,
6193					  &prog_idr, &prog_idr_lock);
6194		break;
6195	case BPF_MAP_GET_NEXT_ID:
6196		err = bpf_obj_get_next_id(&attr, uattr.user,
6197					  &map_idr, &map_idr_lock);
6198		break;
6199	case BPF_BTF_GET_NEXT_ID:
6200		err = bpf_obj_get_next_id(&attr, uattr.user,
6201					  &btf_idr, &btf_idr_lock);
6202		break;
6203	case BPF_PROG_GET_FD_BY_ID:
6204		err = bpf_prog_get_fd_by_id(&attr);
6205		break;
6206	case BPF_MAP_GET_FD_BY_ID:
6207		err = bpf_map_get_fd_by_id(&attr);
6208		break;
6209	case BPF_OBJ_GET_INFO_BY_FD:
6210		err = bpf_obj_get_info_by_fd(&attr, uattr.user);
6211		break;
6212	case BPF_RAW_TRACEPOINT_OPEN:
6213		err = bpf_raw_tracepoint_open(&attr);
6214		break;
6215	case BPF_BTF_LOAD:
6216		err = bpf_btf_load(&attr, uattr, size);
6217		break;
6218	case BPF_BTF_GET_FD_BY_ID:
6219		err = bpf_btf_get_fd_by_id(&attr);
6220		break;
6221	case BPF_TASK_FD_QUERY:
6222		err = bpf_task_fd_query(&attr, uattr.user);
6223		break;
6224	case BPF_MAP_LOOKUP_AND_DELETE_ELEM:
6225		err = map_lookup_and_delete_elem(&attr);
6226		break;
6227	case BPF_MAP_LOOKUP_BATCH:
6228		err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_LOOKUP_BATCH);
6229		break;
6230	case BPF_MAP_LOOKUP_AND_DELETE_BATCH:
6231		err = bpf_map_do_batch(&attr, uattr.user,
6232				       BPF_MAP_LOOKUP_AND_DELETE_BATCH);
6233		break;
6234	case BPF_MAP_UPDATE_BATCH:
6235		err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_UPDATE_BATCH);
6236		break;
6237	case BPF_MAP_DELETE_BATCH:
6238		err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_DELETE_BATCH);
6239		break;
6240	case BPF_LINK_CREATE:
6241		err = link_create(&attr, uattr);
6242		break;
6243	case BPF_LINK_UPDATE:
6244		err = link_update(&attr);
6245		break;
6246	case BPF_LINK_GET_FD_BY_ID:
6247		err = bpf_link_get_fd_by_id(&attr);
6248		break;
6249	case BPF_LINK_GET_NEXT_ID:
6250		err = bpf_obj_get_next_id(&attr, uattr.user,
6251					  &link_idr, &link_idr_lock);
6252		break;
6253	case BPF_ENABLE_STATS:
6254		err = bpf_enable_stats(&attr);
6255		break;
6256	case BPF_ITER_CREATE:
6257		err = bpf_iter_create(&attr);
6258		break;
6259	case BPF_LINK_DETACH:
6260		err = link_detach(&attr);
6261		break;
6262	case BPF_PROG_BIND_MAP:
6263		err = bpf_prog_bind_map(&attr);
6264		break;
6265	case BPF_TOKEN_CREATE:
6266		err = token_create(&attr);
6267		break;
6268	case BPF_PROG_STREAM_READ_BY_FD:
6269		err = prog_stream_read(&attr);
6270		break;
6271	default:
6272		err = -EINVAL;
6273		break;
6274	}
6275
6276	return err;
6277}
6278
6279SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
6280{
6281	return __sys_bpf(cmd, USER_BPFPTR(uattr), size);
6282}
6283
6284static bool syscall_prog_is_valid_access(int off, int size,
6285					 enum bpf_access_type type,
6286					 const struct bpf_prog *prog,
6287					 struct bpf_insn_access_aux *info)
6288{
6289	if (off < 0 || off >= U16_MAX)
6290		return false;
6291	if (off % size != 0)
6292		return false;
6293	return true;
6294}
6295
6296BPF_CALL_3(bpf_sys_bpf, int, cmd, union bpf_attr *, attr, u32, attr_size)
6297{
6298	switch (cmd) {
6299	case BPF_MAP_CREATE:
6300	case BPF_MAP_DELETE_ELEM:
6301	case BPF_MAP_UPDATE_ELEM:
6302	case BPF_MAP_FREEZE:
6303	case BPF_MAP_GET_FD_BY_ID:
6304	case BPF_PROG_LOAD:
6305	case BPF_BTF_LOAD:
6306	case BPF_LINK_CREATE:
6307	case BPF_RAW_TRACEPOINT_OPEN:
6308		break;
6309	default:
6310		return -EINVAL;
6311	}
6312	return __sys_bpf(cmd, KERNEL_BPFPTR(attr), attr_size);
6313}
6314
6315
6316/* To shut up -Wmissing-prototypes.
6317 * This function is used by the kernel light skeleton
6318 * to load bpf programs when modules are loaded or during kernel boot.
6319 * See tools/lib/bpf/skel_internal.h
6320 */
6321int kern_sys_bpf(int cmd, union bpf_attr *attr, unsigned int size);
6322
6323int kern_sys_bpf(int cmd, union bpf_attr *attr, unsigned int size)
6324{
6325	struct bpf_prog * __maybe_unused prog;
6326	struct bpf_tramp_run_ctx __maybe_unused run_ctx;
6327
6328	switch (cmd) {
6329#ifdef CONFIG_BPF_JIT /* __bpf_prog_enter_sleepable used by trampoline and JIT */
6330	case BPF_PROG_TEST_RUN:
6331		if (attr->test.data_in || attr->test.data_out ||
6332		    attr->test.ctx_out || attr->test.duration ||
6333		    attr->test.repeat || attr->test.flags)
6334			return -EINVAL;
6335
6336		prog = bpf_prog_get_type(attr->test.prog_fd, BPF_PROG_TYPE_SYSCALL);
6337		if (IS_ERR(prog))
6338			return PTR_ERR(prog);
6339
6340		if (attr->test.ctx_size_in < prog->aux->max_ctx_offset ||
6341		    attr->test.ctx_size_in > U16_MAX) {
6342			bpf_prog_put(prog);
6343			return -EINVAL;
6344		}
6345
6346		run_ctx.bpf_cookie = 0;
6347		if (!__bpf_prog_enter_sleepable_recur(prog, &run_ctx)) {
6348			/* recursion detected */
6349			__bpf_prog_exit_sleepable_recur(prog, 0, &run_ctx);
6350			bpf_prog_put(prog);
6351			return -EBUSY;
6352		}
6353		attr->test.retval = bpf_prog_run(prog, (void *) (long) attr->test.ctx_in);
6354		__bpf_prog_exit_sleepable_recur(prog, 0 /* bpf_prog_run does runtime stats */,
6355						&run_ctx);
6356		bpf_prog_put(prog);
6357		return 0;
6358#endif
6359	default:
6360		return ____bpf_sys_bpf(cmd, attr, size);
6361	}
6362}
6363EXPORT_SYMBOL_NS(kern_sys_bpf, "BPF_INTERNAL");
6364
6365static const struct bpf_func_proto bpf_sys_bpf_proto = {
6366	.func		= bpf_sys_bpf,
6367	.gpl_only	= false,
6368	.ret_type	= RET_INTEGER,
6369	.arg1_type	= ARG_ANYTHING,
6370	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
6371	.arg3_type	= ARG_CONST_SIZE,
6372};
6373
6374const struct bpf_func_proto * __weak
6375tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
6376{
6377	return bpf_base_func_proto(func_id, prog);
6378}
6379
6380BPF_CALL_1(bpf_sys_close, u32, fd)
6381{
6382	/* When bpf program calls this helper there should not be
6383	 * an fdget() without matching completed fdput().
6384	 * This helper is allowed in the following callchain only:
6385	 * sys_bpf->prog_test_run->bpf_prog->bpf_sys_close
6386	 */
6387	return close_fd(fd);
6388}
6389
6390static const struct bpf_func_proto bpf_sys_close_proto = {
6391	.func		= bpf_sys_close,
6392	.gpl_only	= false,
6393	.ret_type	= RET_INTEGER,
6394	.arg1_type	= ARG_ANYTHING,
6395};
6396
6397BPF_CALL_4(bpf_kallsyms_lookup_name, const char *, name, int, name_sz, int, flags, u64 *, res)
6398{
6399	*res = 0;
6400	if (flags)
6401		return -EINVAL;
6402
6403	if (name_sz <= 1 || name[name_sz - 1])
6404		return -EINVAL;
6405
6406	if (!bpf_dump_raw_ok(current_cred()))
6407		return -EPERM;
6408
6409	*res = kallsyms_lookup_name(name);
6410	return *res ? 0 : -ENOENT;
6411}
6412
6413static const struct bpf_func_proto bpf_kallsyms_lookup_name_proto = {
6414	.func		= bpf_kallsyms_lookup_name,
6415	.gpl_only	= false,
6416	.ret_type	= RET_INTEGER,
6417	.arg1_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
6418	.arg2_type	= ARG_CONST_SIZE_OR_ZERO,
6419	.arg3_type	= ARG_ANYTHING,
6420	.arg4_type	= ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED,
6421	.arg4_size	= sizeof(u64),
6422};
6423
6424static const struct bpf_func_proto *
6425syscall_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
6426{
6427	switch (func_id) {
6428	case BPF_FUNC_sys_bpf:
6429		return !bpf_token_capable(prog->aux->token, CAP_PERFMON)
6430		       ? NULL : &bpf_sys_bpf_proto;
6431	case BPF_FUNC_btf_find_by_name_kind:
6432		return &bpf_btf_find_by_name_kind_proto;
6433	case BPF_FUNC_sys_close:
6434		return &bpf_sys_close_proto;
6435	case BPF_FUNC_kallsyms_lookup_name:
6436		return &bpf_kallsyms_lookup_name_proto;
6437	default:
6438		return tracing_prog_func_proto(func_id, prog);
6439	}
6440}
6441
6442const struct bpf_verifier_ops bpf_syscall_verifier_ops = {
6443	.get_func_proto  = syscall_prog_func_proto,
6444	.is_valid_access = syscall_prog_is_valid_access,
6445};
6446
6447const struct bpf_prog_ops bpf_syscall_prog_ops = {
6448	.test_run = bpf_prog_test_run_syscall,
6449};
6450
6451#ifdef CONFIG_SYSCTL
6452static int bpf_stats_handler(const struct ctl_table *table, int write,
6453			     void *buffer, size_t *lenp, loff_t *ppos)
6454{
6455	struct static_key *key = (struct static_key *)table->data;
6456	static int saved_val;
6457	int val, ret;
6458	struct ctl_table tmp = {
6459		.data   = &val,
6460		.maxlen = sizeof(val),
6461		.mode   = table->mode,
6462		.extra1 = SYSCTL_ZERO,
6463		.extra2 = SYSCTL_ONE,
6464	};
6465
6466	if (write && !capable(CAP_SYS_ADMIN))
6467		return -EPERM;
6468
6469	mutex_lock(&bpf_stats_enabled_mutex);
6470	val = saved_val;
6471	ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);
6472	if (write && !ret && val != saved_val) {
6473		if (val)
6474			static_key_slow_inc(key);
6475		else
6476			static_key_slow_dec(key);
6477		saved_val = val;
6478	}
6479	mutex_unlock(&bpf_stats_enabled_mutex);
6480	return ret;
6481}
6482
6483void __weak unpriv_ebpf_notify(int new_state)
6484{
6485}
6486
6487static int bpf_unpriv_handler(const struct ctl_table *table, int write,
6488			      void *buffer, size_t *lenp, loff_t *ppos)
6489{
6490	int ret, unpriv_enable = *(int *)table->data;
6491	bool locked_state = unpriv_enable == 1;
6492	struct ctl_table tmp = *table;
6493
6494	if (write && !capable(CAP_SYS_ADMIN))
6495		return -EPERM;
6496
6497	tmp.data = &unpriv_enable;
6498	ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);
6499	if (write && !ret) {
6500		if (locked_state && unpriv_enable != 1)
6501			return -EPERM;
6502		*(int *)table->data = unpriv_enable;
6503	}
6504
6505	if (write)
6506		unpriv_ebpf_notify(unpriv_enable);
6507
6508	return ret;
6509}
6510
6511static const struct ctl_table bpf_syscall_table[] = {
6512	{
6513		.procname	= "unprivileged_bpf_disabled",
6514		.data		= &sysctl_unprivileged_bpf_disabled,
6515		.maxlen		= sizeof(sysctl_unprivileged_bpf_disabled),
6516		.mode		= 0644,
6517		.proc_handler	= bpf_unpriv_handler,
6518		.extra1		= SYSCTL_ZERO,
6519		.extra2		= SYSCTL_TWO,
6520	},
6521	{
6522		.procname	= "bpf_stats_enabled",
6523		.data		= &bpf_stats_enabled_key.key,
6524		.mode		= 0644,
6525		.proc_handler	= bpf_stats_handler,
6526	},
6527};
6528
6529static int __init bpf_syscall_sysctl_init(void)
6530{
6531	register_sysctl_init("kernel", bpf_syscall_table);
6532	return 0;
6533}
6534late_initcall(bpf_syscall_sysctl_init);
6535#endif /* CONFIG_SYSCTL */