test_run.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/* Copyright (c) 2017 Facebook
   3 */
   4#include <linux/bpf.h>
   5#include <linux/btf.h>
   6#include <linux/btf_ids.h>
   7#include <linux/slab.h>
   8#include <linux/init.h>
   9#include <linux/vmalloc.h>
  10#include <linux/etherdevice.h>
  11#include <linux/filter.h>
  12#include <linux/rcupdate_trace.h>
  13#include <linux/sched/signal.h>
  14#include <net/bpf_sk_storage.h>
  15#include <net/hotdata.h>
  16#include <net/sock.h>
  17#include <net/tcp.h>
  18#include <net/net_namespace.h>
  19#include <net/page_pool/helpers.h>
  20#include <linux/error-injection.h>
  21#include <linux/smp.h>
  22#include <linux/sock_diag.h>
  23#include <linux/netfilter.h>
  24#include <net/netdev_rx_queue.h>
  25#include <net/xdp.h>
  26#include <net/netfilter/nf_bpf_link.h>
  27
  28#define CREATE_TRACE_POINTS
  29#include <trace/events/bpf_test_run.h>
  30
  31struct bpf_test_timer {
  32	u32 i;
  33	u64 time_start, time_spent;
  34};
  35
  36static void bpf_test_timer_enter(struct bpf_test_timer *t)
  37	__acquires(rcu)
  38{
  39	rcu_read_lock_dont_migrate();
  40	t->time_start = ktime_get_ns();
  41}
  42
  43static void bpf_test_timer_leave(struct bpf_test_timer *t)
  44	__releases(rcu)
  45{
  46	t->time_start = 0;
  47	rcu_read_unlock_migrate();
  48}
  49
  50static bool bpf_test_timer_continue(struct bpf_test_timer *t, int iterations,
  51				    u32 repeat, int *err, u32 *duration)
  52	__must_hold(rcu)
  53{
  54	t->i += iterations;
  55	if (t->i >= repeat) {
  56		/* We're done. */
  57		t->time_spent += ktime_get_ns() - t->time_start;
  58		do_div(t->time_spent, t->i);
  59		*duration = t->time_spent > U32_MAX ? U32_MAX : (u32)t->time_spent;
  60		*err = 0;
  61		goto reset;
  62	}
  63
  64	if (signal_pending(current)) {
  65		/* During iteration: we've been cancelled, abort. */
  66		*err = -EINTR;
  67		goto reset;
  68	}
  69
  70	if (need_resched()) {
  71		/* During iteration: we need to reschedule between runs. */
  72		t->time_spent += ktime_get_ns() - t->time_start;
  73		bpf_test_timer_leave(t);
  74		cond_resched();
  75		bpf_test_timer_enter(t);
  76	}
  77
  78	/* Do another round. */
  79	return true;
  80
  81reset:
  82	t->i = 0;
  83	return false;
  84}
  85
  86/* We put this struct at the head of each page with a context and frame
  87 * initialised when the page is allocated, so we don't have to do this on each
  88 * repetition of the test run.
  89 */
  90struct xdp_page_head {
  91	struct xdp_buff orig_ctx;
  92	struct xdp_buff ctx;
  93	union {
  94		/* ::data_hard_start starts here */
  95		DECLARE_FLEX_ARRAY(struct xdp_frame, frame);
  96		DECLARE_FLEX_ARRAY(u8, data);
  97	};
  98};
  99
 100struct xdp_test_data {
 101	struct xdp_buff *orig_ctx;
 102	struct xdp_rxq_info rxq;
 103	struct net_device *dev;
 104	struct page_pool *pp;
 105	struct xdp_frame **frames;
 106	struct sk_buff **skbs;
 107	struct xdp_mem_info mem;
 108	u32 batch_size;
 109	u32 frame_cnt;
 110};
 111
 112/* tools/testing/selftests/bpf/prog_tests/xdp_do_redirect.c:%MAX_PKT_SIZE
 113 * must be updated accordingly this gets changed, otherwise BPF selftests
 114 * will fail.
 115 */
 116#define TEST_XDP_FRAME_SIZE (PAGE_SIZE - sizeof(struct xdp_page_head))
 117#define TEST_XDP_MAX_BATCH 256
 118
 119static void xdp_test_run_init_page(netmem_ref netmem, void *arg)
 120{
 121	struct xdp_page_head *head =
 122		phys_to_virt(page_to_phys(netmem_to_page(netmem)));
 123	struct xdp_buff *new_ctx, *orig_ctx;
 124	u32 headroom = XDP_PACKET_HEADROOM;
 125	struct xdp_test_data *xdp = arg;
 126	size_t frm_len, meta_len;
 127	struct xdp_frame *frm;
 128	void *data;
 129
 130	orig_ctx = xdp->orig_ctx;
 131	frm_len = orig_ctx->data_end - orig_ctx->data_meta;
 132	meta_len = orig_ctx->data - orig_ctx->data_meta;
 133	headroom -= meta_len;
 134
 135	new_ctx = &head->ctx;
 136	frm = head->frame;
 137	data = head->data;
 138	memcpy(data + headroom, orig_ctx->data_meta, frm_len);
 139
 140	xdp_init_buff(new_ctx, TEST_XDP_FRAME_SIZE, &xdp->rxq);
 141	xdp_prepare_buff(new_ctx, data, headroom, frm_len, true);
 142	new_ctx->data = new_ctx->data_meta + meta_len;
 143
 144	xdp_update_frame_from_buff(new_ctx, frm);
 145	frm->mem_type = new_ctx->rxq->mem.type;
 146
 147	memcpy(&head->orig_ctx, new_ctx, sizeof(head->orig_ctx));
 148}
 149
 150static int xdp_test_run_setup(struct xdp_test_data *xdp, struct xdp_buff *orig_ctx)
 151{
 152	struct page_pool *pp;
 153	int err = -ENOMEM;
 154	struct page_pool_params pp_params = {
 155		.order = 0,
 156		.flags = 0,
 157		.pool_size = xdp->batch_size,
 158		.nid = NUMA_NO_NODE,
 159		.init_callback = xdp_test_run_init_page,
 160		.init_arg = xdp,
 161	};
 162
 163	xdp->frames = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL);
 164	if (!xdp->frames)
 165		return -ENOMEM;
 166
 167	xdp->skbs = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL);
 168	if (!xdp->skbs)
 169		goto err_skbs;
 170
 171	pp = page_pool_create(&pp_params);
 172	if (IS_ERR(pp)) {
 173		err = PTR_ERR(pp);
 174		goto err_pp;
 175	}
 176
 177	/* will copy 'mem.id' into pp->xdp_mem_id */
 178	err = xdp_reg_mem_model(&xdp->mem, MEM_TYPE_PAGE_POOL, pp);
 179	if (err)
 180		goto err_mmodel;
 181
 182	xdp->pp = pp;
 183
 184	/* We create a 'fake' RXQ referencing the original dev, but with an
 185	 * xdp_mem_info pointing to our page_pool
 186	 */
 187	xdp_rxq_info_reg(&xdp->rxq, orig_ctx->rxq->dev, 0, 0);
 188	xdp->rxq.mem.type = MEM_TYPE_PAGE_POOL;
 189	xdp->rxq.mem.id = pp->xdp_mem_id;
 190	xdp->dev = orig_ctx->rxq->dev;
 191	xdp->orig_ctx = orig_ctx;
 192
 193	return 0;
 194
 195err_mmodel:
 196	page_pool_destroy(pp);
 197err_pp:
 198	kvfree(xdp->skbs);
 199err_skbs:
 200	kvfree(xdp->frames);
 201	return err;
 202}
 203
 204static void xdp_test_run_teardown(struct xdp_test_data *xdp)
 205{
 206	xdp_unreg_mem_model(&xdp->mem);
 207	page_pool_destroy(xdp->pp);
 208	kfree(xdp->frames);
 209	kfree(xdp->skbs);
 210}
 211
 212static bool frame_was_changed(const struct xdp_page_head *head)
 213{
 214	/* xdp_scrub_frame() zeroes the data pointer, flags is the last field,
 215	 * i.e. has the highest chances to be overwritten. If those two are
 216	 * untouched, it's most likely safe to skip the context reset.
 217	 */
 218	return head->frame->data != head->orig_ctx.data ||
 219	       head->frame->flags != head->orig_ctx.flags;
 220}
 221
 222static bool ctx_was_changed(struct xdp_page_head *head)
 223{
 224	return head->orig_ctx.data != head->ctx.data ||
 225		head->orig_ctx.data_meta != head->ctx.data_meta ||
 226		head->orig_ctx.data_end != head->ctx.data_end;
 227}
 228
 229static void reset_ctx(struct xdp_page_head *head)
 230{
 231	if (likely(!frame_was_changed(head) && !ctx_was_changed(head)))
 232		return;
 233
 234	head->ctx.data = head->orig_ctx.data;
 235	head->ctx.data_meta = head->orig_ctx.data_meta;
 236	head->ctx.data_end = head->orig_ctx.data_end;
 237	xdp_update_frame_from_buff(&head->ctx, head->frame);
 238	head->frame->mem_type = head->orig_ctx.rxq->mem.type;
 239}
 240
 241static int xdp_recv_frames(struct xdp_frame **frames, int nframes,
 242			   struct sk_buff **skbs,
 243			   struct net_device *dev)
 244{
 245	gfp_t gfp = __GFP_ZERO | GFP_ATOMIC;
 246	int i, n;
 247	LIST_HEAD(list);
 248
 249	n = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, gfp, nframes,
 250				  (void **)skbs);
 251	if (unlikely(n == 0)) {
 252		for (i = 0; i < nframes; i++)
 253			xdp_return_frame(frames[i]);
 254		return -ENOMEM;
 255	}
 256
 257	for (i = 0; i < nframes; i++) {
 258		struct xdp_frame *xdpf = frames[i];
 259		struct sk_buff *skb = skbs[i];
 260
 261		skb = __xdp_build_skb_from_frame(xdpf, skb, dev);
 262		if (!skb) {
 263			xdp_return_frame(xdpf);
 264			continue;
 265		}
 266
 267		list_add_tail(&skb->list, &list);
 268	}
 269	netif_receive_skb_list(&list);
 270
 271	return 0;
 272}
 273
 274static int xdp_test_run_batch(struct xdp_test_data *xdp, struct bpf_prog *prog,
 275			      u32 repeat)
 276{
 277	struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
 278	int err = 0, act, ret, i, nframes = 0, batch_sz;
 279	struct xdp_frame **frames = xdp->frames;
 280	struct bpf_redirect_info *ri;
 281	struct xdp_page_head *head;
 282	struct xdp_frame *frm;
 283	bool redirect = false;
 284	struct xdp_buff *ctx;
 285	struct page *page;
 286
 287	batch_sz = min_t(u32, repeat, xdp->batch_size);
 288
 289	local_bh_disable();
 290	bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
 291	ri = bpf_net_ctx_get_ri();
 292	xdp_set_return_frame_no_direct();
 293
 294	for (i = 0; i < batch_sz; i++) {
 295		page = page_pool_dev_alloc_pages(xdp->pp);
 296		if (!page) {
 297			err = -ENOMEM;
 298			goto out;
 299		}
 300
 301		head = phys_to_virt(page_to_phys(page));
 302		reset_ctx(head);
 303		ctx = &head->ctx;
 304		frm = head->frame;
 305		xdp->frame_cnt++;
 306
 307		act = bpf_prog_run_xdp(prog, ctx);
 308
 309		/* if program changed pkt bounds we need to update the xdp_frame */
 310		if (unlikely(ctx_was_changed(head))) {
 311			ret = xdp_update_frame_from_buff(ctx, frm);
 312			if (ret) {
 313				xdp_return_buff(ctx);
 314				continue;
 315			}
 316		}
 317
 318		switch (act) {
 319		case XDP_TX:
 320			/* we can't do a real XDP_TX since we're not in the
 321			 * driver, so turn it into a REDIRECT back to the same
 322			 * index
 323			 */
 324			ri->tgt_index = xdp->dev->ifindex;
 325			ri->map_id = INT_MAX;
 326			ri->map_type = BPF_MAP_TYPE_UNSPEC;
 327			fallthrough;
 328		case XDP_REDIRECT:
 329			redirect = true;
 330			ret = xdp_do_redirect_frame(xdp->dev, ctx, frm, prog);
 331			if (ret)
 332				xdp_return_buff(ctx);
 333			break;
 334		case XDP_PASS:
 335			frames[nframes++] = frm;
 336			break;
 337		default:
 338			bpf_warn_invalid_xdp_action(NULL, prog, act);
 339			fallthrough;
 340		case XDP_DROP:
 341			xdp_return_buff(ctx);
 342			break;
 343		}
 344	}
 345
 346out:
 347	if (redirect)
 348		xdp_do_flush();
 349	if (nframes) {
 350		ret = xdp_recv_frames(frames, nframes, xdp->skbs, xdp->dev);
 351		if (ret)
 352			err = ret;
 353	}
 354
 355	xdp_clear_return_frame_no_direct();
 356	bpf_net_ctx_clear(bpf_net_ctx);
 357	local_bh_enable();
 358	return err;
 359}
 360
 361static int bpf_test_run_xdp_live(struct bpf_prog *prog, struct xdp_buff *ctx,
 362				 u32 repeat, u32 batch_size, u32 *time)
 363
 364{
 365	struct xdp_test_data xdp = { .batch_size = batch_size };
 366	struct bpf_test_timer t = {};
 367	int ret;
 368
 369	if (!repeat)
 370		repeat = 1;
 371
 372	ret = xdp_test_run_setup(&xdp, ctx);
 373	if (ret)
 374		return ret;
 375
 376	bpf_test_timer_enter(&t);
 377	do {
 378		xdp.frame_cnt = 0;
 379		ret = xdp_test_run_batch(&xdp, prog, repeat - t.i);
 380		if (unlikely(ret < 0))
 381			break;
 382	} while (bpf_test_timer_continue(&t, xdp.frame_cnt, repeat, &ret, time));
 383	bpf_test_timer_leave(&t);
 384
 385	xdp_test_run_teardown(&xdp);
 386	return ret;
 387}
 388
 389static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat,
 390			u32 *retval, u32 *time, bool xdp)
 391{
 392	struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
 393	struct bpf_prog_array_item item = {.prog = prog};
 394	struct bpf_run_ctx *old_ctx;
 395	struct bpf_cg_run_ctx run_ctx;
 396	struct bpf_test_timer t = {};
 397	enum bpf_cgroup_storage_type stype;
 398	int ret;
 399
 400	for_each_cgroup_storage_type(stype) {
 401		item.cgroup_storage[stype] = bpf_cgroup_storage_alloc(prog, stype);
 402		if (IS_ERR(item.cgroup_storage[stype])) {
 403			item.cgroup_storage[stype] = NULL;
 404			for_each_cgroup_storage_type(stype)
 405				bpf_cgroup_storage_free(item.cgroup_storage[stype]);
 406			return -ENOMEM;
 407		}
 408	}
 409
 410	if (!repeat)
 411		repeat = 1;
 412
 413	bpf_test_timer_enter(&t);
 414	old_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
 415	do {
 416		run_ctx.prog_item = &item;
 417		local_bh_disable();
 418		bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
 419
 420		if (xdp)
 421			*retval = bpf_prog_run_xdp(prog, ctx);
 422		else
 423			*retval = bpf_prog_run(prog, ctx);
 424
 425		bpf_net_ctx_clear(bpf_net_ctx);
 426		local_bh_enable();
 427	} while (bpf_test_timer_continue(&t, 1, repeat, &ret, time));
 428	bpf_reset_run_ctx(old_ctx);
 429	bpf_test_timer_leave(&t);
 430
 431	for_each_cgroup_storage_type(stype)
 432		bpf_cgroup_storage_free(item.cgroup_storage[stype]);
 433
 434	return ret;
 435}
 436
 437static int bpf_test_finish(const union bpf_attr *kattr,
 438			   union bpf_attr __user *uattr, const void *data,
 439			   struct skb_shared_info *sinfo, u32 size, u32 frag_size,
 440			   u32 retval, u32 duration)
 441{
 442	void __user *data_out = u64_to_user_ptr(kattr->test.data_out);
 443	int err = -EFAULT;
 444	u32 copy_size = size;
 445
 446	/* Clamp copy if the user has provided a size hint, but copy the full
 447	 * buffer if not to retain old behaviour.
 448	 */
 449	if (kattr->test.data_size_out &&
 450	    copy_size > kattr->test.data_size_out) {
 451		copy_size = kattr->test.data_size_out;
 452		err = -ENOSPC;
 453	}
 454
 455	if (data_out) {
 456		int len = sinfo ? copy_size - frag_size : copy_size;
 457
 458		if (len < 0) {
 459			err = -ENOSPC;
 460			goto out;
 461		}
 462
 463		if (copy_to_user(data_out, data, len))
 464			goto out;
 465
 466		if (sinfo) {
 467			int i, offset = len;
 468			u32 data_len;
 469
 470			for (i = 0; i < sinfo->nr_frags; i++) {
 471				skb_frag_t *frag = &sinfo->frags[i];
 472
 473				if (offset >= copy_size) {
 474					err = -ENOSPC;
 475					break;
 476				}
 477
 478				data_len = min_t(u32, copy_size - offset,
 479						 skb_frag_size(frag));
 480
 481				if (copy_to_user(data_out + offset,
 482						 skb_frag_address(frag),
 483						 data_len))
 484					goto out;
 485
 486				offset += data_len;
 487			}
 488		}
 489	}
 490
 491	if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size)))
 492		goto out;
 493	if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
 494		goto out;
 495	if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration)))
 496		goto out;
 497	if (err != -ENOSPC)
 498		err = 0;
 499out:
 500	trace_bpf_test_finish(&err);
 501	return err;
 502}
 503
 504/* Integer types of various sizes and pointer combinations cover variety of
 505 * architecture dependent calling conventions. 7+ can be supported in the
 506 * future.
 507 */
 508__bpf_kfunc_start_defs();
 509
 510__bpf_kfunc int bpf_fentry_test1(int a)
 511{
 512	return a + 1;
 513}
 514EXPORT_SYMBOL_GPL(bpf_fentry_test1);
 515
 516noinline int bpf_fentry_test2(int a, u64 b)
 517{
 518	return a + b;
 519}
 520
 521noinline int bpf_fentry_test3(char a, int b, u64 c)
 522{
 523	return a + b + c;
 524}
 525
 526noinline int bpf_fentry_test4(void *a, char b, int c, u64 d)
 527{
 528	return (long)a + b + c + d;
 529}
 530
 531noinline int bpf_fentry_test5(u64 a, void *b, short c, int d, u64 e)
 532{
 533	return a + (long)b + c + d + e;
 534}
 535
 536noinline int bpf_fentry_test6(u64 a, void *b, short c, int d, void *e, u64 f)
 537{
 538	return a + (long)b + c + d + (long)e + f;
 539}
 540
 541struct bpf_fentry_test_t {
 542	struct bpf_fentry_test_t *a;
 543};
 544
 545noinline int bpf_fentry_test7(struct bpf_fentry_test_t *arg)
 546{
 547	asm volatile ("" : "+r"(arg));
 548	return (long)arg;
 549}
 550
 551noinline int bpf_fentry_test8(struct bpf_fentry_test_t *arg)
 552{
 553	return (long)arg->a;
 554}
 555
 556__bpf_kfunc u32 bpf_fentry_test9(u32 *a)
 557{
 558	return *a;
 559}
 560
 561noinline int bpf_fentry_test10(const void *a)
 562{
 563	return (long)a;
 564}
 565
 566noinline void bpf_fentry_test_sinfo(struct skb_shared_info *sinfo)
 567{
 568}
 569
 570__bpf_kfunc int bpf_modify_return_test(int a, int *b)
 571{
 572	*b += 1;
 573	return a + *b;
 574}
 575
 576__bpf_kfunc int bpf_modify_return_test2(int a, int *b, short c, int d,
 577					void *e, char f, int g)
 578{
 579	*b += 1;
 580	return a + *b + c + d + (long)e + f + g;
 581}
 582
 583__bpf_kfunc int bpf_modify_return_test_tp(int nonce)
 584{
 585	trace_bpf_trigger_tp(nonce);
 586
 587	return nonce;
 588}
 589
 590noinline int bpf_fentry_shadow_test(int a)
 591{
 592	return a + 1;
 593}
 594
 595struct prog_test_member1 {
 596	int a;
 597};
 598
 599struct prog_test_member {
 600	struct prog_test_member1 m;
 601	int c;
 602};
 603
 604struct prog_test_ref_kfunc {
 605	int a;
 606	int b;
 607	struct prog_test_member memb;
 608	struct prog_test_ref_kfunc *next;
 609	refcount_t cnt;
 610};
 611
 612__bpf_kfunc void bpf_kfunc_call_test_release(struct prog_test_ref_kfunc *p)
 613{
 614	refcount_dec(&p->cnt);
 615}
 616
 617__bpf_kfunc void bpf_kfunc_call_test_release_dtor(void *p)
 618{
 619	bpf_kfunc_call_test_release(p);
 620}
 621CFI_NOSEAL(bpf_kfunc_call_test_release_dtor);
 622
 623__bpf_kfunc void bpf_kfunc_call_memb_release(struct prog_test_member *p)
 624{
 625}
 626
 627__bpf_kfunc void bpf_kfunc_call_memb_release_dtor(void *p)
 628{
 629}
 630CFI_NOSEAL(bpf_kfunc_call_memb_release_dtor);
 631
 632__bpf_kfunc_end_defs();
 633
 634BTF_KFUNCS_START(bpf_test_modify_return_ids)
 635BTF_ID_FLAGS(func, bpf_modify_return_test)
 636BTF_ID_FLAGS(func, bpf_modify_return_test2)
 637BTF_ID_FLAGS(func, bpf_modify_return_test_tp)
 638BTF_ID_FLAGS(func, bpf_fentry_test1, KF_SLEEPABLE)
 639BTF_KFUNCS_END(bpf_test_modify_return_ids)
 640
 641static const struct btf_kfunc_id_set bpf_test_modify_return_set = {
 642	.owner = THIS_MODULE,
 643	.set   = &bpf_test_modify_return_ids,
 644};
 645
 646BTF_KFUNCS_START(test_sk_check_kfunc_ids)
 647BTF_ID_FLAGS(func, bpf_kfunc_call_test_release, KF_RELEASE)
 648BTF_ID_FLAGS(func, bpf_kfunc_call_memb_release, KF_RELEASE)
 649BTF_KFUNCS_END(test_sk_check_kfunc_ids)
 650
 651static void *bpf_test_init(const union bpf_attr *kattr, u32 user_size,
 652			   u32 size, u32 headroom, u32 tailroom)
 653{
 654	void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
 655	void *data;
 656
 657	if (user_size > PAGE_SIZE - headroom - tailroom)
 658		return ERR_PTR(-EINVAL);
 659
 660	size = SKB_DATA_ALIGN(size);
 661	data = kzalloc(size + headroom + tailroom, GFP_USER);
 662	if (!data)
 663		return ERR_PTR(-ENOMEM);
 664
 665	if (copy_from_user(data + headroom, data_in, user_size)) {
 666		kfree(data);
 667		return ERR_PTR(-EFAULT);
 668	}
 669
 670	return data;
 671}
 672
 673int bpf_prog_test_run_tracing(struct bpf_prog *prog,
 674			      const union bpf_attr *kattr,
 675			      union bpf_attr __user *uattr)
 676{
 677	struct bpf_fentry_test_t arg = {};
 678	u16 side_effect = 0, ret = 0;
 679	int b = 2, err = -EFAULT;
 680	u32 retval = 0;
 681
 682	if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)
 683		return -EINVAL;
 684
 685	switch (prog->expected_attach_type) {
 686	case BPF_TRACE_FENTRY:
 687	case BPF_TRACE_FEXIT:
 688		if (bpf_fentry_test1(1) != 2 ||
 689		    bpf_fentry_test2(2, 3) != 5 ||
 690		    bpf_fentry_test3(4, 5, 6) != 15 ||
 691		    bpf_fentry_test4((void *)7, 8, 9, 10) != 34 ||
 692		    bpf_fentry_test5(11, (void *)12, 13, 14, 15) != 65 ||
 693		    bpf_fentry_test6(16, (void *)17, 18, 19, (void *)20, 21) != 111 ||
 694		    bpf_fentry_test7((struct bpf_fentry_test_t *)0) != 0 ||
 695		    bpf_fentry_test8(&arg) != 0 ||
 696		    bpf_fentry_test9(&retval) != 0 ||
 697		    bpf_fentry_test10((void *)0) != 0)
 698			goto out;
 699		break;
 700	case BPF_MODIFY_RETURN:
 701		ret = bpf_modify_return_test(1, &b);
 702		if (b != 2)
 703			side_effect++;
 704		b = 2;
 705		ret += bpf_modify_return_test2(1, &b, 3, 4, (void *)5, 6, 7);
 706		if (b != 2)
 707			side_effect++;
 708		break;
 709	default:
 710		goto out;
 711	}
 712
 713	retval = ((u32)side_effect << 16) | ret;
 714	if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
 715		goto out;
 716
 717	err = 0;
 718out:
 719	trace_bpf_test_finish(&err);
 720	return err;
 721}
 722
 723struct bpf_raw_tp_test_run_info {
 724	struct bpf_prog *prog;
 725	void *ctx;
 726	u32 retval;
 727};
 728
 729static void
 730__bpf_prog_test_run_raw_tp(void *data)
 731{
 732	struct bpf_raw_tp_test_run_info *info = data;
 733	struct bpf_trace_run_ctx run_ctx = {};
 734	struct bpf_run_ctx *old_run_ctx;
 735
 736	old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
 737
 738	rcu_read_lock();
 739	info->retval = bpf_prog_run(info->prog, info->ctx);
 740	rcu_read_unlock();
 741
 742	bpf_reset_run_ctx(old_run_ctx);
 743}
 744
 745int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
 746			     const union bpf_attr *kattr,
 747			     union bpf_attr __user *uattr)
 748{
 749	void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in);
 750	__u32 ctx_size_in = kattr->test.ctx_size_in;
 751	struct bpf_raw_tp_test_run_info info;
 752	int cpu = kattr->test.cpu, err = 0;
 753	int current_cpu;
 754
 755	/* doesn't support data_in/out, ctx_out, duration, or repeat */
 756	if (kattr->test.data_in || kattr->test.data_out ||
 757	    kattr->test.ctx_out || kattr->test.duration ||
 758	    kattr->test.repeat || kattr->test.batch_size)
 759		return -EINVAL;
 760
 761	if (ctx_size_in < prog->aux->max_ctx_offset ||
 762	    ctx_size_in > MAX_BPF_FUNC_ARGS * sizeof(u64))
 763		return -EINVAL;
 764
 765	if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0)
 766		return -EINVAL;
 767
 768	if (ctx_size_in) {
 769		info.ctx = memdup_user(ctx_in, ctx_size_in);
 770		if (IS_ERR(info.ctx))
 771			return PTR_ERR(info.ctx);
 772	} else {
 773		info.ctx = NULL;
 774	}
 775
 776	info.prog = prog;
 777
 778	current_cpu = get_cpu();
 779	if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 ||
 780	    cpu == current_cpu) {
 781		__bpf_prog_test_run_raw_tp(&info);
 782	} else if (cpu >= nr_cpu_ids || !cpu_online(cpu)) {
 783		/* smp_call_function_single() also checks cpu_online()
 784		 * after csd_lock(). However, since cpu is from user
 785		 * space, let's do an extra quick check to filter out
 786		 * invalid value before smp_call_function_single().
 787		 */
 788		err = -ENXIO;
 789	} else {
 790		err = smp_call_function_single(cpu, __bpf_prog_test_run_raw_tp,
 791					       &info, 1);
 792	}
 793	put_cpu();
 794
 795	if (!err &&
 796	    copy_to_user(&uattr->test.retval, &info.retval, sizeof(u32)))
 797		err = -EFAULT;
 798
 799	kfree(info.ctx);
 800	return err;
 801}
 802
 803static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size)
 804{
 805	void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in);
 806	void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
 807	u32 size = kattr->test.ctx_size_in;
 808	void *data;
 809	int err;
 810
 811	if (!data_in && !data_out)
 812		return NULL;
 813
 814	data = kzalloc(max_size, GFP_USER);
 815	if (!data)
 816		return ERR_PTR(-ENOMEM);
 817
 818	if (data_in) {
 819		err = bpf_check_uarg_tail_zero(USER_BPFPTR(data_in), max_size, size);
 820		if (err) {
 821			kfree(data);
 822			return ERR_PTR(err);
 823		}
 824
 825		size = min_t(u32, max_size, size);
 826		if (copy_from_user(data, data_in, size)) {
 827			kfree(data);
 828			return ERR_PTR(-EFAULT);
 829		}
 830	}
 831	return data;
 832}
 833
 834static int bpf_ctx_finish(const union bpf_attr *kattr,
 835			  union bpf_attr __user *uattr, const void *data,
 836			  u32 size)
 837{
 838	void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
 839	int err = -EFAULT;
 840	u32 copy_size = size;
 841
 842	if (!data || !data_out)
 843		return 0;
 844
 845	if (copy_size > kattr->test.ctx_size_out) {
 846		copy_size = kattr->test.ctx_size_out;
 847		err = -ENOSPC;
 848	}
 849
 850	if (copy_to_user(data_out, data, copy_size))
 851		goto out;
 852	if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size)))
 853		goto out;
 854	if (err != -ENOSPC)
 855		err = 0;
 856out:
 857	return err;
 858}
 859
 860/**
 861 * range_is_zero - test whether buffer is initialized
 862 * @buf: buffer to check
 863 * @from: check from this position
 864 * @to: check up until (excluding) this position
 865 *
 866 * This function returns true if the there is a non-zero byte
 867 * in the buf in the range [from,to).
 868 */
 869static inline bool range_is_zero(void *buf, size_t from, size_t to)
 870{
 871	return !memchr_inv((u8 *)buf + from, 0, to - from);
 872}
 873
 874static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb)
 875{
 876	struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;
 877
 878	if (!__skb)
 879		return 0;
 880
 881	/* make sure the fields we don't use are zeroed */
 882	if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, mark)))
 883		return -EINVAL;
 884
 885	/* mark is allowed */
 886
 887	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, mark),
 888			   offsetof(struct __sk_buff, priority)))
 889		return -EINVAL;
 890
 891	/* priority is allowed */
 892	/* ingress_ifindex is allowed */
 893	/* ifindex is allowed */
 894
 895	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, ifindex),
 896			   offsetof(struct __sk_buff, cb)))
 897		return -EINVAL;
 898
 899	/* cb is allowed */
 900
 901	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, cb),
 902			   offsetof(struct __sk_buff, data_end)))
 903		return -EINVAL;
 904
 905	/* data_end is allowed, but not copied to skb */
 906
 907	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, data_end),
 908			   offsetof(struct __sk_buff, tstamp)))
 909		return -EINVAL;
 910
 911	/* tstamp is allowed */
 912	/* wire_len is allowed */
 913	/* gso_segs is allowed */
 914
 915	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_segs),
 916			   offsetof(struct __sk_buff, gso_size)))
 917		return -EINVAL;
 918
 919	/* gso_size is allowed */
 920
 921	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_size),
 922			   offsetof(struct __sk_buff, hwtstamp)))
 923		return -EINVAL;
 924
 925	/* hwtstamp is allowed */
 926
 927	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, hwtstamp),
 928			   sizeof(struct __sk_buff)))
 929		return -EINVAL;
 930
 931	skb->mark = __skb->mark;
 932	skb->priority = __skb->priority;
 933	skb->skb_iif = __skb->ingress_ifindex;
 934	skb->tstamp = __skb->tstamp;
 935	memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN);
 936
 937	if (__skb->wire_len == 0) {
 938		cb->pkt_len = skb->len;
 939	} else {
 940		if (__skb->wire_len < skb->len ||
 941		    __skb->wire_len > GSO_LEGACY_MAX_SIZE)
 942			return -EINVAL;
 943		cb->pkt_len = __skb->wire_len;
 944	}
 945
 946	if (__skb->gso_segs > GSO_MAX_SEGS)
 947		return -EINVAL;
 948
 949	/* Currently GSO type is zero/unset. If this gets extended with
 950	 * a small list of accepted GSO types in future, the filter for
 951	 * an unset GSO type in bpf_clone_redirect() can be lifted.
 952	 */
 953	skb_shinfo(skb)->gso_segs = __skb->gso_segs;
 954	skb_shinfo(skb)->gso_size = __skb->gso_size;
 955	skb_shinfo(skb)->hwtstamps.hwtstamp = __skb->hwtstamp;
 956
 957	return 0;
 958}
 959
 960static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb)
 961{
 962	struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;
 963
 964	if (!__skb)
 965		return;
 966
 967	__skb->mark = skb->mark;
 968	__skb->priority = skb->priority;
 969	__skb->ingress_ifindex = skb->skb_iif;
 970	__skb->ifindex = skb->dev->ifindex;
 971	__skb->tstamp = skb->tstamp;
 972	memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN);
 973	__skb->wire_len = cb->pkt_len;
 974	__skb->gso_segs = skb_shinfo(skb)->gso_segs;
 975	__skb->hwtstamp = skb_shinfo(skb)->hwtstamps.hwtstamp;
 976}
 977
 978static struct proto bpf_dummy_proto = {
 979	.name   = "bpf_dummy",
 980	.owner  = THIS_MODULE,
 981	.obj_size = sizeof(struct sock),
 982};
 983
 984int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
 985			  union bpf_attr __user *uattr)
 986{
 987	bool is_l2 = false, is_direct_pkt_access = false, is_lwt = false;
 988	u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 989	struct net *net = current->nsproxy->net_ns;
 990	struct net_device *dev = net->loopback_dev;
 991	u32 headroom = NET_SKB_PAD + NET_IP_ALIGN;
 992	u32 linear_sz = kattr->test.data_size_in;
 993	u32 repeat = kattr->test.repeat;
 994	struct __sk_buff *ctx = NULL;
 995	struct sk_buff *skb = NULL;
 996	struct sock *sk = NULL;
 997	u32 retval, duration;
 998	int hh_len = ETH_HLEN;
 999	void *data = NULL;
1000	int ret;
1001
1002	if ((kattr->test.flags & ~BPF_F_TEST_SKB_CHECKSUM_COMPLETE) ||
1003	    kattr->test.cpu || kattr->test.batch_size)
1004		return -EINVAL;
1005
1006	if (kattr->test.data_size_in < ETH_HLEN)
1007		return -EINVAL;
1008
1009	switch (prog->type) {
1010	case BPF_PROG_TYPE_SCHED_CLS:
1011	case BPF_PROG_TYPE_SCHED_ACT:
1012		is_direct_pkt_access = true;
1013		is_l2 = true;
1014		break;
1015	case BPF_PROG_TYPE_LWT_IN:
1016	case BPF_PROG_TYPE_LWT_OUT:
1017	case BPF_PROG_TYPE_LWT_XMIT:
1018		is_lwt = true;
1019		fallthrough;
1020	case BPF_PROG_TYPE_CGROUP_SKB:
1021		is_direct_pkt_access = true;
1022		break;
1023	default:
1024		break;
1025	}
1026
1027	ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff));
1028	if (IS_ERR(ctx))
1029		return PTR_ERR(ctx);
1030
1031	if (ctx) {
1032		if (ctx->data_end > kattr->test.data_size_in || ctx->data || ctx->data_meta) {
1033			ret = -EINVAL;
1034			goto out;
1035		}
1036		if (ctx->data_end) {
1037			/* Non-linear LWT test_run is unsupported for now. */
1038			if (is_lwt) {
1039				ret = -EINVAL;
1040				goto out;
1041			}
1042			linear_sz = max(ETH_HLEN, ctx->data_end);
1043		}
1044	}
1045
1046	linear_sz = min_t(u32, linear_sz, PAGE_SIZE - headroom - tailroom);
1047
1048	data = bpf_test_init(kattr, linear_sz, linear_sz, headroom, tailroom);
1049	if (IS_ERR(data)) {
1050		ret = PTR_ERR(data);
1051		data = NULL;
1052		goto out;
1053	}
1054
1055	sk = sk_alloc(net, AF_UNSPEC, GFP_USER, &bpf_dummy_proto, 1);
1056	if (!sk) {
1057		ret = -ENOMEM;
1058		goto out;
1059	}
1060	sock_init_data(NULL, sk);
1061
1062	skb = slab_build_skb(data);
1063	if (!skb) {
1064		ret = -ENOMEM;
1065		goto out;
1066	}
1067	skb->sk = sk;
1068
1069	data = NULL; /* data released via kfree_skb */
1070
1071	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
1072	__skb_put(skb, linear_sz);
1073
1074	if (unlikely(kattr->test.data_size_in > linear_sz)) {
1075		void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
1076		struct skb_shared_info *sinfo = skb_shinfo(skb);
1077		u32 copied = linear_sz;
1078
1079		while (copied < kattr->test.data_size_in) {
1080			struct page *page;
1081			u32 data_len;
1082
1083			if (sinfo->nr_frags == MAX_SKB_FRAGS) {
1084				ret = -ENOMEM;
1085				goto out;
1086			}
1087
1088			page = alloc_page(GFP_KERNEL);
1089			if (!page) {
1090				ret = -ENOMEM;
1091				goto out;
1092			}
1093
1094			data_len = min_t(u32, kattr->test.data_size_in - copied,
1095					 PAGE_SIZE);
1096			skb_fill_page_desc(skb, sinfo->nr_frags, page, 0, data_len);
1097
1098			if (copy_from_user(page_address(page), data_in + copied,
1099					   data_len)) {
1100				ret = -EFAULT;
1101				goto out;
1102			}
1103			skb->data_len += data_len;
1104			skb->truesize += PAGE_SIZE;
1105			skb->len += data_len;
1106			copied += data_len;
1107		}
1108	}
1109
1110	if (ctx && ctx->ifindex > 1) {
1111		dev = dev_get_by_index(net, ctx->ifindex);
1112		if (!dev) {
1113			ret = -ENODEV;
1114			goto out;
1115		}
1116	}
1117	skb->protocol = eth_type_trans(skb, dev);
1118	skb_reset_network_header(skb);
1119
1120	switch (skb->protocol) {
1121	case htons(ETH_P_IP):
1122		sk->sk_family = AF_INET;
1123		if (sizeof(struct iphdr) <= skb_headlen(skb)) {
1124			sk->sk_rcv_saddr = ip_hdr(skb)->saddr;
1125			sk->sk_daddr = ip_hdr(skb)->daddr;
1126		}
1127		break;
1128#if IS_ENABLED(CONFIG_IPV6)
1129	case htons(ETH_P_IPV6):
1130		sk->sk_family = AF_INET6;
1131		if (sizeof(struct ipv6hdr) <= skb_headlen(skb)) {
1132			sk->sk_v6_rcv_saddr = ipv6_hdr(skb)->saddr;
1133			sk->sk_v6_daddr = ipv6_hdr(skb)->daddr;
1134		}
1135		break;
1136#endif
1137	default:
1138		break;
1139	}
1140
1141	if (is_l2)
1142		__skb_push(skb, hh_len);
1143	if (is_direct_pkt_access)
1144		bpf_compute_data_pointers(skb);
1145
1146	ret = convert___skb_to_skb(skb, ctx);
1147	if (ret)
1148		goto out;
1149
1150	if (kattr->test.flags & BPF_F_TEST_SKB_CHECKSUM_COMPLETE) {
1151		const int off = skb_network_offset(skb);
1152		int len = skb->len - off;
1153
1154		skb->csum = skb_checksum(skb, off, len, 0);
1155		skb->ip_summed = CHECKSUM_COMPLETE;
1156	}
1157
1158	ret = bpf_test_run(prog, skb, repeat, &retval, &duration, false);
1159	if (ret)
1160		goto out;
1161	if (!is_l2) {
1162		if (skb_headroom(skb) < hh_len) {
1163			int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb));
1164
1165			if (pskb_expand_head(skb, nhead, 0, GFP_USER)) {
1166				ret = -ENOMEM;
1167				goto out;
1168			}
1169		}
1170		memset(__skb_push(skb, hh_len), 0, hh_len);
1171	}
1172
1173	if (kattr->test.flags & BPF_F_TEST_SKB_CHECKSUM_COMPLETE) {
1174		const int off = skb_network_offset(skb);
1175		int len = skb->len - off;
1176		__wsum csum;
1177
1178		csum = skb_checksum(skb, off, len, 0);
1179
1180		if (csum_fold(skb->csum) != csum_fold(csum)) {
1181			ret = -EBADMSG;
1182			goto out;
1183		}
1184	}
1185
1186	convert_skb_to___skb(skb, ctx);
1187
1188	if (skb_is_nonlinear(skb))
1189		/* bpf program can never convert linear skb to non-linear */
1190		WARN_ON_ONCE(linear_sz == kattr->test.data_size_in);
1191	ret = bpf_test_finish(kattr, uattr, skb->data, skb_shinfo(skb), skb->len,
1192			      skb->data_len, retval, duration);
1193	if (!ret)
1194		ret = bpf_ctx_finish(kattr, uattr, ctx,
1195				     sizeof(struct __sk_buff));
1196out:
1197	if (dev && dev != net->loopback_dev)
1198		dev_put(dev);
1199	kfree_skb(skb);
1200	kfree(data);
1201	if (sk)
1202		sk_free(sk);
1203	kfree(ctx);
1204	return ret;
1205}
1206
1207static int xdp_convert_md_to_buff(struct xdp_md *xdp_md, struct xdp_buff *xdp)
1208{
1209	unsigned int ingress_ifindex, rx_queue_index;
1210	struct netdev_rx_queue *rxqueue;
1211	struct net_device *device;
1212
1213	if (!xdp_md)
1214		return 0;
1215
1216	if (xdp_md->egress_ifindex != 0)
1217		return -EINVAL;
1218
1219	ingress_ifindex = xdp_md->ingress_ifindex;
1220	rx_queue_index = xdp_md->rx_queue_index;
1221
1222	if (!ingress_ifindex && rx_queue_index)
1223		return -EINVAL;
1224
1225	if (ingress_ifindex) {
1226		device = dev_get_by_index(current->nsproxy->net_ns,
1227					  ingress_ifindex);
1228		if (!device)
1229			return -ENODEV;
1230
1231		if (rx_queue_index >= device->real_num_rx_queues)
1232			goto free_dev;
1233
1234		rxqueue = __netif_get_rx_queue(device, rx_queue_index);
1235
1236		if (!xdp_rxq_info_is_reg(&rxqueue->xdp_rxq))
1237			goto free_dev;
1238
1239		xdp->rxq = &rxqueue->xdp_rxq;
1240		/* The device is now tracked in the xdp->rxq for later
1241		 * dev_put()
1242		 */
1243	}
1244
1245	xdp->data = xdp->data_meta + xdp_md->data;
1246	return 0;
1247
1248free_dev:
1249	dev_put(device);
1250	return -EINVAL;
1251}
1252
1253static void xdp_convert_buff_to_md(struct xdp_buff *xdp, struct xdp_md *xdp_md)
1254{
1255	if (!xdp_md)
1256		return;
1257
1258	xdp_md->data = xdp->data - xdp->data_meta;
1259	xdp_md->data_end = xdp->data_end - xdp->data_meta;
1260
1261	if (xdp_md->ingress_ifindex)
1262		dev_put(xdp->rxq->dev);
1263}
1264
1265int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
1266			  union bpf_attr __user *uattr)
1267{
1268	bool do_live = (kattr->test.flags & BPF_F_TEST_XDP_LIVE_FRAMES);
1269	u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1270	u32 retval = 0, meta_sz = 0, duration, max_linear_sz, size;
1271	u32 linear_sz = kattr->test.data_size_in;
1272	u32 batch_size = kattr->test.batch_size;
1273	u32 headroom = XDP_PACKET_HEADROOM;
1274	u32 repeat = kattr->test.repeat;
1275	struct netdev_rx_queue *rxqueue;
1276	struct skb_shared_info *sinfo;
1277	struct xdp_buff xdp = {};
1278	int i, ret = -EINVAL;
1279	struct xdp_md *ctx;
1280	void *data;
1281
1282	if (prog->expected_attach_type == BPF_XDP_DEVMAP ||
1283	    prog->expected_attach_type == BPF_XDP_CPUMAP)
1284		return -EINVAL;
1285
1286	if (kattr->test.flags & ~BPF_F_TEST_XDP_LIVE_FRAMES)
1287		return -EINVAL;
1288
1289	if (bpf_prog_is_dev_bound(prog->aux))
1290		return -EINVAL;
1291
1292	if (do_live) {
1293		if (!batch_size)
1294			batch_size = NAPI_POLL_WEIGHT;
1295		else if (batch_size > TEST_XDP_MAX_BATCH)
1296			return -E2BIG;
1297	} else if (batch_size) {
1298		return -EINVAL;
1299	}
1300
1301	ctx = bpf_ctx_init(kattr, sizeof(struct xdp_md));
1302	if (IS_ERR(ctx))
1303		return PTR_ERR(ctx);
1304
1305	if (ctx) {
1306		/* There can't be user provided data before the meta data */
1307		if (ctx->data_meta || ctx->data_end > kattr->test.data_size_in ||
1308		    ctx->data > ctx->data_end ||
1309		    (do_live && (kattr->test.data_out || kattr->test.ctx_out)))
1310			goto free_ctx;
1311
1312		meta_sz = ctx->data;
1313		if (xdp_metalen_invalid(meta_sz) || meta_sz > headroom - sizeof(struct xdp_frame))
1314			goto free_ctx;
1315
1316		/* Meta data is allocated from the headroom */
1317		headroom -= meta_sz;
1318		linear_sz = ctx->data_end;
1319	}
1320
1321	/* The xdp_page_head structure takes up space in each page, limiting the
1322         * size of the packet data; add the extra size to headroom here to make
1323         * sure it's accounted in the length checks below, but not in the
1324         * metadata size check above.
1325         */
1326        if (do_live)
1327		headroom += sizeof(struct xdp_page_head);
1328
1329	max_linear_sz = PAGE_SIZE - headroom - tailroom;
1330	linear_sz = min_t(u32, linear_sz, max_linear_sz);
1331
1332	/* disallow live data mode for jumbo frames */
1333	if (do_live && kattr->test.data_size_in > linear_sz)
1334		goto free_ctx;
1335
1336	if (kattr->test.data_size_in - meta_sz < ETH_HLEN)
1337		goto free_ctx;
1338
1339	data = bpf_test_init(kattr, linear_sz, max_linear_sz, headroom, tailroom);
1340	if (IS_ERR(data)) {
1341		ret = PTR_ERR(data);
1342		goto free_ctx;
1343	}
1344
1345	rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0);
1346	rxqueue->xdp_rxq.frag_size = PAGE_SIZE;
1347	xdp_init_buff(&xdp, rxqueue->xdp_rxq.frag_size, &rxqueue->xdp_rxq);
1348	xdp_prepare_buff(&xdp, data, headroom, linear_sz, true);
1349	sinfo = xdp_get_shared_info_from_buff(&xdp);
1350
1351	ret = xdp_convert_md_to_buff(ctx, &xdp);
1352	if (ret)
1353		goto free_data;
1354
1355	size = linear_sz;
1356	if (unlikely(kattr->test.data_size_in > size)) {
1357		void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
1358
1359		while (size < kattr->test.data_size_in) {
1360			struct page *page;
1361			skb_frag_t *frag;
1362			u32 data_len;
1363
1364			if (sinfo->nr_frags == MAX_SKB_FRAGS) {
1365				ret = -ENOMEM;
1366				goto out_put_dev;
1367			}
1368
1369			page = alloc_page(GFP_KERNEL);
1370			if (!page) {
1371				ret = -ENOMEM;
1372				goto out_put_dev;
1373			}
1374
1375			frag = &sinfo->frags[sinfo->nr_frags++];
1376
1377			data_len = min_t(u32, kattr->test.data_size_in - size,
1378					 PAGE_SIZE);
1379			skb_frag_fill_page_desc(frag, page, 0, data_len);
1380
1381			if (copy_from_user(page_address(page), data_in + size,
1382					   data_len)) {
1383				ret = -EFAULT;
1384				goto out_put_dev;
1385			}
1386			sinfo->xdp_frags_size += data_len;
1387			size += data_len;
1388		}
1389		xdp_buff_set_frags_flag(&xdp);
1390	}
1391
1392	if (repeat > 1)
1393		bpf_prog_change_xdp(NULL, prog);
1394
1395	if (do_live)
1396		ret = bpf_test_run_xdp_live(prog, &xdp, repeat, batch_size, &duration);
1397	else
1398		ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration, true);
1399out_put_dev:
1400	/* We convert the xdp_buff back to an xdp_md before checking the return
1401	 * code so the reference count of any held netdevice will be decremented
1402	 * even if the test run failed.
1403	 */
1404	xdp_convert_buff_to_md(&xdp, ctx);
1405	if (ret)
1406		goto out;
1407
1408	size = xdp.data_end - xdp.data_meta + sinfo->xdp_frags_size;
1409	ret = bpf_test_finish(kattr, uattr, xdp.data_meta, sinfo, size, sinfo->xdp_frags_size,
1410			      retval, duration);
1411	if (!ret)
1412		ret = bpf_ctx_finish(kattr, uattr, ctx,
1413				     sizeof(struct xdp_md));
1414
1415out:
1416	if (repeat > 1)
1417		bpf_prog_change_xdp(prog, NULL);
1418free_data:
1419	for (i = 0; i < sinfo->nr_frags; i++)
1420		__free_page(skb_frag_page(&sinfo->frags[i]));
1421	kfree(data);
1422free_ctx:
1423	kfree(ctx);
1424	return ret;
1425}
1426
1427static int verify_user_bpf_flow_keys(struct bpf_flow_keys *ctx)
1428{
1429	/* make sure the fields we don't use are zeroed */
1430	if (!range_is_zero(ctx, 0, offsetof(struct bpf_flow_keys, flags)))
1431		return -EINVAL;
1432
1433	/* flags is allowed */
1434
1435	if (!range_is_zero(ctx, offsetofend(struct bpf_flow_keys, flags),
1436			   sizeof(struct bpf_flow_keys)))
1437		return -EINVAL;
1438
1439	return 0;
1440}
1441
1442int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
1443				     const union bpf_attr *kattr,
1444				     union bpf_attr __user *uattr)
1445{
1446	struct bpf_test_timer t = {};
1447	u32 size = kattr->test.data_size_in;
1448	struct bpf_flow_dissector ctx = {};
1449	u32 repeat = kattr->test.repeat;
1450	struct bpf_flow_keys *user_ctx;
1451	struct bpf_flow_keys flow_keys;
1452	const struct ethhdr *eth;
1453	unsigned int flags = 0;
1454	u32 retval, duration;
1455	void *data;
1456	int ret;
1457
1458	if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)
1459		return -EINVAL;
1460
1461	if (size < ETH_HLEN)
1462		return -EINVAL;
1463
1464	data = bpf_test_init(kattr, kattr->test.data_size_in, size, 0, 0);
1465	if (IS_ERR(data))
1466		return PTR_ERR(data);
1467
1468	eth = (struct ethhdr *)data;
1469
1470	if (!repeat)
1471		repeat = 1;
1472
1473	user_ctx = bpf_ctx_init(kattr, sizeof(struct bpf_flow_keys));
1474	if (IS_ERR(user_ctx)) {
1475		kfree(data);
1476		return PTR_ERR(user_ctx);
1477	}
1478	if (user_ctx) {
1479		ret = verify_user_bpf_flow_keys(user_ctx);
1480		if (ret)
1481			goto out;
1482		flags = user_ctx->flags;
1483	}
1484
1485	ctx.flow_keys = &flow_keys;
1486	ctx.data = data;
1487	ctx.data_end = (__u8 *)data + size;
1488
1489	bpf_test_timer_enter(&t);
1490	do {
1491		retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN,
1492					  size, flags);
1493	} while (bpf_test_timer_continue(&t, 1, repeat, &ret, &duration));
1494	bpf_test_timer_leave(&t);
1495
1496	if (ret < 0)
1497		goto out;
1498
1499	ret = bpf_test_finish(kattr, uattr, &flow_keys, NULL,
1500			      sizeof(flow_keys), 0, retval, duration);
1501	if (!ret)
1502		ret = bpf_ctx_finish(kattr, uattr, user_ctx,
1503				     sizeof(struct bpf_flow_keys));
1504
1505out:
1506	kfree(user_ctx);
1507	kfree(data);
1508	return ret;
1509}
1510
1511int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kattr,
1512				union bpf_attr __user *uattr)
1513{
1514	struct bpf_test_timer t = {};
1515	struct bpf_prog_array *progs = NULL;
1516	struct bpf_sk_lookup_kern ctx = {};
1517	u32 repeat = kattr->test.repeat;
1518	struct bpf_sk_lookup *user_ctx;
1519	u32 retval, duration;
1520	int ret = -EINVAL;
1521
1522	if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)
1523		return -EINVAL;
1524
1525	if (kattr->test.data_in || kattr->test.data_size_in || kattr->test.data_out ||
1526	    kattr->test.data_size_out)
1527		return -EINVAL;
1528
1529	if (!repeat)
1530		repeat = 1;
1531
1532	user_ctx = bpf_ctx_init(kattr, sizeof(*user_ctx));
1533	if (IS_ERR(user_ctx))
1534		return PTR_ERR(user_ctx);
1535
1536	if (!user_ctx)
1537		return -EINVAL;
1538
1539	if (user_ctx->sk)
1540		goto out;
1541
1542	if (!range_is_zero(user_ctx, offsetofend(typeof(*user_ctx), local_port), sizeof(*user_ctx)))
1543		goto out;
1544
1545	if (user_ctx->local_port > U16_MAX) {
1546		ret = -ERANGE;
1547		goto out;
1548	}
1549
1550	ctx.family = (u16)user_ctx->family;
1551	ctx.protocol = (u16)user_ctx->protocol;
1552	ctx.dport = (u16)user_ctx->local_port;
1553	ctx.sport = user_ctx->remote_port;
1554
1555	switch (ctx.family) {
1556	case AF_INET:
1557		ctx.v4.daddr = (__force __be32)user_ctx->local_ip4;
1558		ctx.v4.saddr = (__force __be32)user_ctx->remote_ip4;
1559		break;
1560
1561#if IS_ENABLED(CONFIG_IPV6)
1562	case AF_INET6:
1563		ctx.v6.daddr = (struct in6_addr *)user_ctx->local_ip6;
1564		ctx.v6.saddr = (struct in6_addr *)user_ctx->remote_ip6;
1565		break;
1566#endif
1567
1568	default:
1569		ret = -EAFNOSUPPORT;
1570		goto out;
1571	}
1572
1573	progs = bpf_prog_array_alloc(1, GFP_KERNEL);
1574	if (!progs) {
1575		ret = -ENOMEM;
1576		goto out;
1577	}
1578
1579	progs->items[0].prog = prog;
1580
1581	bpf_test_timer_enter(&t);
1582	do {
1583		ctx.selected_sk = NULL;
1584		retval = BPF_PROG_SK_LOOKUP_RUN_ARRAY(progs, ctx, bpf_prog_run);
1585	} while (bpf_test_timer_continue(&t, 1, repeat, &ret, &duration));
1586	bpf_test_timer_leave(&t);
1587
1588	if (ret < 0)
1589		goto out;
1590
1591	user_ctx->cookie = 0;
1592	if (ctx.selected_sk) {
1593		if (ctx.selected_sk->sk_reuseport && !ctx.no_reuseport) {
1594			ret = -EOPNOTSUPP;
1595			goto out;
1596		}
1597
1598		user_ctx->cookie = sock_gen_cookie(ctx.selected_sk);
1599	}
1600
1601	ret = bpf_test_finish(kattr, uattr, NULL, NULL, 0, 0, retval, duration);
1602	if (!ret)
1603		ret = bpf_ctx_finish(kattr, uattr, user_ctx, sizeof(*user_ctx));
1604
1605out:
1606	bpf_prog_array_free(progs);
1607	kfree(user_ctx);
1608	return ret;
1609}
1610
1611int bpf_prog_test_run_syscall(struct bpf_prog *prog,
1612			      const union bpf_attr *kattr,
1613			      union bpf_attr __user *uattr)
1614{
1615	void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in);
1616	__u32 ctx_size_in = kattr->test.ctx_size_in;
1617	void *ctx = NULL;
1618	u32 retval;
1619	int err = 0;
1620
1621	/* doesn't support data_in/out, ctx_out, duration, or repeat or flags */
1622	if (kattr->test.data_in || kattr->test.data_out ||
1623	    kattr->test.ctx_out || kattr->test.duration ||
1624	    kattr->test.repeat || kattr->test.flags ||
1625	    kattr->test.batch_size)
1626		return -EINVAL;
1627
1628	if (ctx_size_in < prog->aux->max_ctx_offset ||
1629	    ctx_size_in > U16_MAX)
1630		return -EINVAL;
1631
1632	if (ctx_size_in) {
1633		ctx = memdup_user(ctx_in, ctx_size_in);
1634		if (IS_ERR(ctx))
1635			return PTR_ERR(ctx);
1636	}
1637
1638	rcu_read_lock_trace();
1639	retval = bpf_prog_run_pin_on_cpu(prog, ctx);
1640	rcu_read_unlock_trace();
1641
1642	if (copy_to_user(&uattr->test.retval, &retval, sizeof(u32))) {
1643		err = -EFAULT;
1644		goto out;
1645	}
1646	if (ctx_size_in)
1647		if (copy_to_user(ctx_in, ctx, ctx_size_in))
1648			err = -EFAULT;
1649out:
1650	kfree(ctx);
1651	return err;
1652}
1653
1654static int verify_and_copy_hook_state(struct nf_hook_state *state,
1655				      const struct nf_hook_state *user,
1656				      struct net_device *dev)
1657{
1658	if (user->in || user->out)
1659		return -EINVAL;
1660
1661	if (user->net || user->sk || user->okfn)
1662		return -EINVAL;
1663
1664	switch (user->pf) {
1665	case NFPROTO_IPV4:
1666	case NFPROTO_IPV6:
1667		switch (state->hook) {
1668		case NF_INET_PRE_ROUTING:
1669			state->in = dev;
1670			break;
1671		case NF_INET_LOCAL_IN:
1672			state->in = dev;
1673			break;
1674		case NF_INET_FORWARD:
1675			state->in = dev;
1676			state->out = dev;
1677			break;
1678		case NF_INET_LOCAL_OUT:
1679			state->out = dev;
1680			break;
1681		case NF_INET_POST_ROUTING:
1682			state->out = dev;
1683			break;
1684		}
1685
1686		break;
1687	default:
1688		return -EINVAL;
1689	}
1690
1691	state->pf = user->pf;
1692	state->hook = user->hook;
1693
1694	return 0;
1695}
1696
1697static __be16 nfproto_eth(int nfproto)
1698{
1699	switch (nfproto) {
1700	case NFPROTO_IPV4:
1701		return htons(ETH_P_IP);
1702	case NFPROTO_IPV6:
1703		break;
1704	}
1705
1706	return htons(ETH_P_IPV6);
1707}
1708
1709int bpf_prog_test_run_nf(struct bpf_prog *prog,
1710			 const union bpf_attr *kattr,
1711			 union bpf_attr __user *uattr)
1712{
1713	struct net *net = current->nsproxy->net_ns;
1714	struct net_device *dev = net->loopback_dev;
1715	struct nf_hook_state *user_ctx, hook_state = {
1716		.pf = NFPROTO_IPV4,
1717		.hook = NF_INET_LOCAL_OUT,
1718	};
1719	u32 size = kattr->test.data_size_in;
1720	u32 repeat = kattr->test.repeat;
1721	struct bpf_nf_ctx ctx = {
1722		.state = &hook_state,
1723	};
1724	struct sk_buff *skb = NULL;
1725	u32 retval, duration;
1726	void *data;
1727	int ret;
1728
1729	if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)
1730		return -EINVAL;
1731
1732	if (size < sizeof(struct iphdr))
1733		return -EINVAL;
1734
1735	data = bpf_test_init(kattr, kattr->test.data_size_in, size,
1736			     NET_SKB_PAD + NET_IP_ALIGN,
1737			     SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
1738	if (IS_ERR(data))
1739		return PTR_ERR(data);
1740
1741	if (!repeat)
1742		repeat = 1;
1743
1744	user_ctx = bpf_ctx_init(kattr, sizeof(struct nf_hook_state));
1745	if (IS_ERR(user_ctx)) {
1746		kfree(data);
1747		return PTR_ERR(user_ctx);
1748	}
1749
1750	if (user_ctx) {
1751		ret = verify_and_copy_hook_state(&hook_state, user_ctx, dev);
1752		if (ret)
1753			goto out;
1754	}
1755
1756	skb = slab_build_skb(data);
1757	if (!skb) {
1758		ret = -ENOMEM;
1759		goto out;
1760	}
1761
1762	data = NULL; /* data released via kfree_skb */
1763
1764	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
1765	__skb_put(skb, size);
1766
1767	ret = -EINVAL;
1768
1769	if (hook_state.hook != NF_INET_LOCAL_OUT) {
1770		if (size < ETH_HLEN + sizeof(struct iphdr))
1771			goto out;
1772
1773		skb->protocol = eth_type_trans(skb, dev);
1774		switch (skb->protocol) {
1775		case htons(ETH_P_IP):
1776			if (hook_state.pf == NFPROTO_IPV4)
1777				break;
1778			goto out;
1779		case htons(ETH_P_IPV6):
1780			if (size < ETH_HLEN + sizeof(struct ipv6hdr))
1781				goto out;
1782			if (hook_state.pf == NFPROTO_IPV6)
1783				break;
1784			goto out;
1785		default:
1786			ret = -EPROTO;
1787			goto out;
1788		}
1789
1790		skb_reset_network_header(skb);
1791	} else {
1792		skb->protocol = nfproto_eth(hook_state.pf);
1793	}
1794
1795	ctx.skb = skb;
1796
1797	ret = bpf_test_run(prog, &ctx, repeat, &retval, &duration, false);
1798	if (ret)
1799		goto out;
1800
1801	ret = bpf_test_finish(kattr, uattr, NULL, NULL, 0, 0, retval, duration);
1802
1803out:
1804	kfree(user_ctx);
1805	kfree_skb(skb);
1806	kfree(data);
1807	return ret;
1808}
1809
1810static const struct btf_kfunc_id_set bpf_prog_test_kfunc_set = {
1811	.owner = THIS_MODULE,
1812	.set   = &test_sk_check_kfunc_ids,
1813};
1814
1815BTF_ID_LIST(bpf_prog_test_dtor_kfunc_ids)
1816BTF_ID(struct, prog_test_ref_kfunc)
1817BTF_ID(func, bpf_kfunc_call_test_release_dtor)
1818BTF_ID(struct, prog_test_member)
1819BTF_ID(func, bpf_kfunc_call_memb_release_dtor)
1820
1821static int __init bpf_prog_test_run_init(void)
1822{
1823	const struct btf_id_dtor_kfunc bpf_prog_test_dtor_kfunc[] = {
1824		{
1825		  .btf_id       = bpf_prog_test_dtor_kfunc_ids[0],
1826		  .kfunc_btf_id = bpf_prog_test_dtor_kfunc_ids[1]
1827		},
1828		{
1829		  .btf_id	= bpf_prog_test_dtor_kfunc_ids[2],
1830		  .kfunc_btf_id = bpf_prog_test_dtor_kfunc_ids[3],
1831		},
1832	};
1833	int ret;
1834
1835	ret = register_btf_fmodret_id_set(&bpf_test_modify_return_set);
1836	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_prog_test_kfunc_set);
1837	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &bpf_prog_test_kfunc_set);
1838	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &bpf_prog_test_kfunc_set);
1839	return ret ?: register_btf_id_dtor_kfuncs(bpf_prog_test_dtor_kfunc,
1840						  ARRAY_SIZE(bpf_prog_test_dtor_kfunc),
1841						  THIS_MODULE);
1842}
1843late_initcall(bpf_prog_test_run_init);