dynptr_success.c

   1// SPDX-License-Identifier: GPL-2.0
   2/* Copyright (c) 2022 Facebook */
   3
   4#include <vmlinux.h>
   5#include <string.h>
   6#include <stdbool.h>
   7#include <bpf/bpf_helpers.h>
   8#include <bpf/bpf_tracing.h>
   9#include "bpf_misc.h"
  10#include "errno.h"
  11
  12#define PAGE_SIZE_64K 65536
  13
  14char _license[] SEC("license") = "GPL";
  15
  16int pid, err, val;
  17
  18struct ringbuf_sample {
  19	int pid;
  20	int seq;
  21	long value;
  22	char comm[16];
  23};
  24
  25struct {
  26	__uint(type, BPF_MAP_TYPE_RINGBUF);
  27	__uint(max_entries, 4096);
  28} ringbuf SEC(".maps");
  29
  30struct {
  31	__uint(type, BPF_MAP_TYPE_ARRAY);
  32	__uint(max_entries, 1);
  33	__type(key, __u32);
  34	__type(value, __u32);
  35} array_map SEC(".maps");
  36
  37SEC("?tp/syscalls/sys_enter_nanosleep")
  38int test_read_write(void *ctx)
  39{
  40	char write_data[64] = "hello there, world!!";
  41	char read_data[64] = {};
  42	struct bpf_dynptr ptr;
  43	int i;
  44
  45	if (bpf_get_current_pid_tgid() >> 32 != pid)
  46		return 0;
  47
  48	bpf_ringbuf_reserve_dynptr(&ringbuf, sizeof(write_data), 0, &ptr);
  49
  50	/* Write data into the dynptr */
  51	err = bpf_dynptr_write(&ptr, 0, write_data, sizeof(write_data), 0);
  52
  53	/* Read the data that was written into the dynptr */
  54	err = err ?: bpf_dynptr_read(read_data, sizeof(read_data), &ptr, 0, 0);
  55
  56	/* Ensure the data we read matches the data we wrote */
  57	for (i = 0; i < sizeof(read_data); i++) {
  58		if (read_data[i] != write_data[i]) {
  59			err = 1;
  60			break;
  61		}
  62	}
  63
  64	bpf_ringbuf_discard_dynptr(&ptr, 0);
  65	return 0;
  66}
  67
  68SEC("?tp/syscalls/sys_enter_nanosleep")
  69int test_dynptr_data(void *ctx)
  70{
  71	__u32 key = 0, val = 235, *map_val;
  72	struct bpf_dynptr ptr;
  73	__u32 map_val_size;
  74	void *data;
  75
  76	map_val_size = sizeof(*map_val);
  77
  78	if (bpf_get_current_pid_tgid() >> 32 != pid)
  79		return 0;
  80
  81	bpf_map_update_elem(&array_map, &key, &val, 0);
  82
  83	map_val = bpf_map_lookup_elem(&array_map, &key);
  84	if (!map_val) {
  85		err = 1;
  86		return 0;
  87	}
  88
  89	bpf_dynptr_from_mem(map_val, map_val_size, 0, &ptr);
  90
  91	/* Try getting a data slice that is out of range */
  92	data = bpf_dynptr_data(&ptr, map_val_size + 1, 1);
  93	if (data) {
  94		err = 2;
  95		return 0;
  96	}
  97
  98	/* Try getting more bytes than available */
  99	data = bpf_dynptr_data(&ptr, 0, map_val_size + 1);
 100	if (data) {
 101		err = 3;
 102		return 0;
 103	}
 104
 105	data = bpf_dynptr_data(&ptr, 0, sizeof(__u32));
 106	if (!data) {
 107		err = 4;
 108		return 0;
 109	}
 110
 111	*(__u32 *)data = 999;
 112
 113	err = bpf_probe_read_kernel(&val, sizeof(val), data);
 114	if (err)
 115		return 0;
 116
 117	if (val != *(int *)data)
 118		err = 5;
 119
 120	return 0;
 121}
 122
 123static int ringbuf_callback(__u32 index, void *data)
 124{
 125	struct ringbuf_sample *sample;
 126
 127	struct bpf_dynptr *ptr = (struct bpf_dynptr *)data;
 128
 129	sample = bpf_dynptr_data(ptr, 0, sizeof(*sample));
 130	if (!sample)
 131		err = 2;
 132	else
 133		sample->pid += index;
 134
 135	return 0;
 136}
 137
 138SEC("?tp/syscalls/sys_enter_nanosleep")
 139int test_ringbuf(void *ctx)
 140{
 141	struct bpf_dynptr ptr;
 142	struct ringbuf_sample *sample;
 143
 144	if (bpf_get_current_pid_tgid() >> 32 != pid)
 145		return 0;
 146
 147	val = 100;
 148
 149	/* check that you can reserve a dynamic size reservation */
 150	err = bpf_ringbuf_reserve_dynptr(&ringbuf, val, 0, &ptr);
 151
 152	sample = err ? NULL : bpf_dynptr_data(&ptr, 0, sizeof(*sample));
 153	if (!sample) {
 154		err = 1;
 155		goto done;
 156	}
 157
 158	sample->pid = 10;
 159
 160	/* Can pass dynptr to callback functions */
 161	bpf_loop(10, ringbuf_callback, &ptr, 0);
 162
 163	if (sample->pid != 55)
 164		err = 2;
 165
 166done:
 167	bpf_ringbuf_discard_dynptr(&ptr, 0);
 168	return 0;
 169}
 170
 171SEC("?cgroup_skb/egress")
 172int test_skb_readonly(struct __sk_buff *skb)
 173{
 174	__u8 write_data[2] = {1, 2};
 175	struct bpf_dynptr ptr;
 176	int ret;
 177
 178	if (bpf_dynptr_from_skb(skb, 0, &ptr)) {
 179		err = 1;
 180		return 1;
 181	}
 182
 183	/* since cgroup skbs are read only, writes should fail */
 184	ret = bpf_dynptr_write(&ptr, 0, write_data, sizeof(write_data), 0);
 185	if (ret != -EINVAL) {
 186		err = 2;
 187		return 1;
 188	}
 189
 190	return 1;
 191}
 192
 193SEC("?cgroup_skb/egress")
 194int test_dynptr_skb_data(struct __sk_buff *skb)
 195{
 196	struct bpf_dynptr ptr;
 197	__u64 *data;
 198
 199	if (bpf_dynptr_from_skb(skb, 0, &ptr)) {
 200		err = 1;
 201		return 1;
 202	}
 203
 204	/* This should return NULL. Must use bpf_dynptr_slice API */
 205	data = bpf_dynptr_data(&ptr, 0, 1);
 206	if (data) {
 207		err = 2;
 208		return 1;
 209	}
 210
 211	return 1;
 212}
 213
 214SEC("?tc")
 215int test_dynptr_skb_meta_data(struct __sk_buff *skb)
 216{
 217	struct bpf_dynptr meta;
 218	__u8 *md;
 219	int ret;
 220
 221	err = 1;
 222	ret = bpf_dynptr_from_skb_meta(skb, 0, &meta);
 223	if (ret)
 224		return 1;
 225
 226	/* This should return NULL. Must use bpf_dynptr_slice API */
 227	err = 2;
 228	md = bpf_dynptr_data(&meta, 0, sizeof(*md));
 229	if (md)
 230		return 1;
 231
 232	err = 0;
 233	return 1;
 234}
 235
 236/* Check that skb metadata dynptr ops don't accept any flags. */
 237SEC("?tc")
 238int test_dynptr_skb_meta_flags(struct __sk_buff *skb)
 239{
 240	const __u64 INVALID_FLAGS = ~0ULL;
 241	struct bpf_dynptr meta;
 242	__u8 buf;
 243	int ret;
 244
 245	err = 1;
 246	ret = bpf_dynptr_from_skb_meta(skb, INVALID_FLAGS, &meta);
 247	if (ret != -EINVAL)
 248		return 1;
 249
 250	err = 2;
 251	ret = bpf_dynptr_from_skb_meta(skb, 0, &meta);
 252	if (ret)
 253		return 1;
 254
 255	err = 3;
 256	ret = bpf_dynptr_read(&buf, 0, &meta, 0, INVALID_FLAGS);
 257	if (ret != -EINVAL)
 258		return 1;
 259
 260	err = 4;
 261	ret = bpf_dynptr_write(&meta, 0, &buf, 0, INVALID_FLAGS);
 262	if (ret != -EINVAL)
 263		return 1;
 264
 265	err = 0;
 266	return 1;
 267}
 268
 269SEC("tp/syscalls/sys_enter_nanosleep")
 270int test_adjust(void *ctx)
 271{
 272	struct bpf_dynptr ptr;
 273	__u32 bytes = 64;
 274	__u32 off = 10;
 275	__u32 trim = 15;
 276
 277	if (bpf_get_current_pid_tgid() >> 32 != pid)
 278		return 0;
 279
 280	err = bpf_ringbuf_reserve_dynptr(&ringbuf, bytes, 0, &ptr);
 281	if (err) {
 282		err = 1;
 283		goto done;
 284	}
 285
 286	if (bpf_dynptr_size(&ptr) != bytes) {
 287		err = 2;
 288		goto done;
 289	}
 290
 291	/* Advance the dynptr by off */
 292	err = bpf_dynptr_adjust(&ptr, off, bpf_dynptr_size(&ptr));
 293	if (err) {
 294		err = 3;
 295		goto done;
 296	}
 297
 298	if (bpf_dynptr_size(&ptr) != bytes - off) {
 299		err = 4;
 300		goto done;
 301	}
 302
 303	/* Trim the dynptr */
 304	err = bpf_dynptr_adjust(&ptr, off, 15);
 305	if (err) {
 306		err = 5;
 307		goto done;
 308	}
 309
 310	/* Check that the size was adjusted correctly */
 311	if (bpf_dynptr_size(&ptr) != trim - off) {
 312		err = 6;
 313		goto done;
 314	}
 315
 316done:
 317	bpf_ringbuf_discard_dynptr(&ptr, 0);
 318	return 0;
 319}
 320
 321SEC("tp/syscalls/sys_enter_nanosleep")
 322int test_adjust_err(void *ctx)
 323{
 324	char write_data[45] = "hello there, world!!";
 325	struct bpf_dynptr ptr;
 326	__u32 size = 64;
 327	__u32 off = 20;
 328
 329	if (bpf_get_current_pid_tgid() >> 32 != pid)
 330		return 0;
 331
 332	if (bpf_ringbuf_reserve_dynptr(&ringbuf, size, 0, &ptr)) {
 333		err = 1;
 334		goto done;
 335	}
 336
 337	/* Check that start can't be greater than end */
 338	if (bpf_dynptr_adjust(&ptr, 5, 1) != -EINVAL) {
 339		err = 2;
 340		goto done;
 341	}
 342
 343	/* Check that start can't be greater than size */
 344	if (bpf_dynptr_adjust(&ptr, size + 1, size + 1) != -ERANGE) {
 345		err = 3;
 346		goto done;
 347	}
 348
 349	/* Check that end can't be greater than size */
 350	if (bpf_dynptr_adjust(&ptr, 0, size + 1) != -ERANGE) {
 351		err = 4;
 352		goto done;
 353	}
 354
 355	if (bpf_dynptr_adjust(&ptr, off, size)) {
 356		err = 5;
 357		goto done;
 358	}
 359
 360	/* Check that you can't write more bytes than available into the dynptr
 361	 * after you've adjusted it
 362	 */
 363	if (bpf_dynptr_write(&ptr, 0, &write_data, sizeof(write_data), 0) != -E2BIG) {
 364		err = 6;
 365		goto done;
 366	}
 367
 368	/* Check that even after adjusting, submitting/discarding
 369	 * a ringbuf dynptr works
 370	 */
 371	bpf_ringbuf_submit_dynptr(&ptr, 0);
 372	return 0;
 373
 374done:
 375	bpf_ringbuf_discard_dynptr(&ptr, 0);
 376	return 0;
 377}
 378
 379SEC("tp/syscalls/sys_enter_nanosleep")
 380int test_zero_size_dynptr(void *ctx)
 381{
 382	char write_data = 'x', read_data;
 383	struct bpf_dynptr ptr;
 384	__u32 size = 64;
 385
 386	if (bpf_get_current_pid_tgid() >> 32 != pid)
 387		return 0;
 388
 389	if (bpf_ringbuf_reserve_dynptr(&ringbuf, size, 0, &ptr)) {
 390		err = 1;
 391		goto done;
 392	}
 393
 394	/* After this, the dynptr has a size of 0 */
 395	if (bpf_dynptr_adjust(&ptr, size, size)) {
 396		err = 2;
 397		goto done;
 398	}
 399
 400	/* Test that reading + writing non-zero bytes is not ok */
 401	if (bpf_dynptr_read(&read_data, sizeof(read_data), &ptr, 0, 0) != -E2BIG) {
 402		err = 3;
 403		goto done;
 404	}
 405
 406	if (bpf_dynptr_write(&ptr, 0, &write_data, sizeof(write_data), 0) != -E2BIG) {
 407		err = 4;
 408		goto done;
 409	}
 410
 411	/* Test that reading + writing 0 bytes from a 0-size dynptr is ok */
 412	if (bpf_dynptr_read(&read_data, 0, &ptr, 0, 0)) {
 413		err = 5;
 414		goto done;
 415	}
 416
 417	if (bpf_dynptr_write(&ptr, 0, &write_data, 0, 0)) {
 418		err = 6;
 419		goto done;
 420	}
 421
 422	err = 0;
 423
 424done:
 425	bpf_ringbuf_discard_dynptr(&ptr, 0);
 426	return 0;
 427}
 428
 429SEC("tp/syscalls/sys_enter_nanosleep")
 430int test_dynptr_is_null(void *ctx)
 431{
 432	struct bpf_dynptr ptr1;
 433	struct bpf_dynptr ptr2;
 434	__u64 size = 4;
 435
 436	if (bpf_get_current_pid_tgid() >> 32 != pid)
 437		return 0;
 438
 439	/* Pass in invalid flags, get back an invalid dynptr */
 440	if (bpf_ringbuf_reserve_dynptr(&ringbuf, size, 123, &ptr1) != -EINVAL) {
 441		err = 1;
 442		goto exit_early;
 443	}
 444
 445	/* Test that the invalid dynptr is null */
 446	if (!bpf_dynptr_is_null(&ptr1)) {
 447		err = 2;
 448		goto exit_early;
 449	}
 450
 451	/* Get a valid dynptr */
 452	if (bpf_ringbuf_reserve_dynptr(&ringbuf, size, 0, &ptr2)) {
 453		err = 3;
 454		goto exit;
 455	}
 456
 457	/* Test that the valid dynptr is not null */
 458	if (bpf_dynptr_is_null(&ptr2)) {
 459		err = 4;
 460		goto exit;
 461	}
 462
 463exit:
 464	bpf_ringbuf_discard_dynptr(&ptr2, 0);
 465exit_early:
 466	bpf_ringbuf_discard_dynptr(&ptr1, 0);
 467	return 0;
 468}
 469
 470SEC("cgroup_skb/egress")
 471int test_dynptr_is_rdonly(struct __sk_buff *skb)
 472{
 473	struct bpf_dynptr ptr1;
 474	struct bpf_dynptr ptr2;
 475	struct bpf_dynptr ptr3;
 476
 477	/* Pass in invalid flags, get back an invalid dynptr */
 478	if (bpf_dynptr_from_skb(skb, 123, &ptr1) != -EINVAL) {
 479		err = 1;
 480		return 0;
 481	}
 482
 483	/* Test that an invalid dynptr is_rdonly returns false */
 484	if (bpf_dynptr_is_rdonly(&ptr1)) {
 485		err = 2;
 486		return 0;
 487	}
 488
 489	/* Get a read-only dynptr */
 490	if (bpf_dynptr_from_skb(skb, 0, &ptr2)) {
 491		err = 3;
 492		return 0;
 493	}
 494
 495	/* Test that the dynptr is read-only */
 496	if (!bpf_dynptr_is_rdonly(&ptr2)) {
 497		err = 4;
 498		return 0;
 499	}
 500
 501	/* Get a read-writeable dynptr */
 502	if (bpf_ringbuf_reserve_dynptr(&ringbuf, 64, 0, &ptr3)) {
 503		err = 5;
 504		goto done;
 505	}
 506
 507	/* Test that the dynptr is read-only */
 508	if (bpf_dynptr_is_rdonly(&ptr3)) {
 509		err = 6;
 510		goto done;
 511	}
 512
 513done:
 514	bpf_ringbuf_discard_dynptr(&ptr3, 0);
 515	return 0;
 516}
 517
 518SEC("cgroup_skb/egress")
 519int test_dynptr_clone(struct __sk_buff *skb)
 520{
 521	struct bpf_dynptr ptr1;
 522	struct bpf_dynptr ptr2;
 523	__u32 off = 2, size;
 524
 525	/* Get a dynptr */
 526	if (bpf_dynptr_from_skb(skb, 0, &ptr1)) {
 527		err = 1;
 528		return 0;
 529	}
 530
 531	if (bpf_dynptr_adjust(&ptr1, off, bpf_dynptr_size(&ptr1))) {
 532		err = 2;
 533		return 0;
 534	}
 535
 536	/* Clone the dynptr */
 537	if (bpf_dynptr_clone(&ptr1, &ptr2)) {
 538		err = 3;
 539		return 0;
 540	}
 541
 542	size = bpf_dynptr_size(&ptr1);
 543
 544	/* Check that the clone has the same size and rd-only */
 545	if (bpf_dynptr_size(&ptr2) != size) {
 546		err = 4;
 547		return 0;
 548	}
 549
 550	if (bpf_dynptr_is_rdonly(&ptr2) != bpf_dynptr_is_rdonly(&ptr1)) {
 551		err = 5;
 552		return 0;
 553	}
 554
 555	/* Advance and trim the original dynptr */
 556	bpf_dynptr_adjust(&ptr1, 5, 5);
 557
 558	/* Check that only original dynptr was affected, and the clone wasn't */
 559	if (bpf_dynptr_size(&ptr2) != size) {
 560		err = 6;
 561		return 0;
 562	}
 563
 564	return 0;
 565}
 566
 567SEC("?cgroup_skb/egress")
 568int test_dynptr_skb_no_buff(struct __sk_buff *skb)
 569{
 570	struct bpf_dynptr ptr;
 571	__u64 *data;
 572
 573	if (bpf_dynptr_from_skb(skb, 0, &ptr)) {
 574		err = 1;
 575		return 1;
 576	}
 577
 578	/* This may return NULL. SKB may require a buffer */
 579	data = bpf_dynptr_slice(&ptr, 0, NULL, 1);
 580
 581	return !!data;
 582}
 583
 584SEC("?cgroup_skb/egress")
 585int test_dynptr_skb_strcmp(struct __sk_buff *skb)
 586{
 587	struct bpf_dynptr ptr;
 588	char *data;
 589
 590	if (bpf_dynptr_from_skb(skb, 0, &ptr)) {
 591		err = 1;
 592		return 1;
 593	}
 594
 595	/* This may return NULL. SKB may require a buffer */
 596	data = bpf_dynptr_slice(&ptr, 0, NULL, 10);
 597	if (data) {
 598		bpf_strncmp(data, 10, "foo");
 599		return 1;
 600	}
 601
 602	return 1;
 603}
 604
 605SEC("tp_btf/kfree_skb")
 606int BPF_PROG(test_dynptr_skb_tp_btf, void *skb, void *location)
 607{
 608	__u8 write_data[2] = {1, 2};
 609	struct bpf_dynptr ptr;
 610	int ret;
 611
 612	if (bpf_dynptr_from_skb(skb, 0, &ptr)) {
 613		err = 1;
 614		return 1;
 615	}
 616
 617	/* since tp_btf skbs are read only, writes should fail */
 618	ret = bpf_dynptr_write(&ptr, 0, write_data, sizeof(write_data), 0);
 619	if (ret != -EINVAL) {
 620		err = 2;
 621		return 1;
 622	}
 623
 624	return 1;
 625}
 626
 627static inline int bpf_memcmp(const char *a, const char *b, u32 size)
 628{
 629	int i;
 630
 631	bpf_for(i, 0, size) {
 632		if (a[i] != b[i])
 633			return a[i] < b[i] ? -1 : 1;
 634	}
 635	return 0;
 636}
 637
 638SEC("?tp/syscalls/sys_enter_nanosleep")
 639int test_dynptr_copy(void *ctx)
 640{
 641	char data[] = "hello there, world!!";
 642	char buf[32] = {'\0'};
 643	__u32 sz = sizeof(data);
 644	struct bpf_dynptr src, dst;
 645
 646	bpf_ringbuf_reserve_dynptr(&ringbuf, sz, 0, &src);
 647	bpf_ringbuf_reserve_dynptr(&ringbuf, sz, 0, &dst);
 648
 649	/* Test basic case of copying contiguous memory backed dynptrs */
 650	err = bpf_dynptr_write(&src, 0, data, sz, 0);
 651	err = err ?: bpf_dynptr_copy(&dst, 0, &src, 0, sz);
 652	err = err ?: bpf_dynptr_read(buf, sz, &dst, 0, 0);
 653	err = err ?: bpf_memcmp(data, buf, sz);
 654
 655	/* Test that offsets are handled correctly */
 656	err = err ?: bpf_dynptr_copy(&dst, 3, &src, 5, sz - 5);
 657	err = err ?: bpf_dynptr_read(buf, sz - 5, &dst, 3, 0);
 658	err = err ?: bpf_memcmp(data + 5, buf, sz - 5);
 659
 660	bpf_ringbuf_discard_dynptr(&src, 0);
 661	bpf_ringbuf_discard_dynptr(&dst, 0);
 662	return 0;
 663}
 664
 665SEC("xdp")
 666int test_dynptr_copy_xdp(struct xdp_md *xdp)
 667{
 668	struct bpf_dynptr ptr_buf, ptr_xdp;
 669	char data[] = "qwertyuiopasdfghjkl";
 670	char buf[32] = {'\0'};
 671	__u32 len = sizeof(data), xdp_data_size;
 672	int i, chunks = 200;
 673
 674	/* ptr_xdp is backed by non-contiguous memory */
 675	bpf_dynptr_from_xdp(xdp, 0, &ptr_xdp);
 676	xdp_data_size = bpf_dynptr_size(&ptr_xdp);
 677	bpf_ringbuf_reserve_dynptr(&ringbuf, len * chunks, 0, &ptr_buf);
 678
 679	/* Destination dynptr is backed by non-contiguous memory */
 680	bpf_for(i, 0, chunks) {
 681		err = bpf_dynptr_write(&ptr_buf, i * len, data, len, 0);
 682		if (err)
 683			goto out;
 684	}
 685
 686	err = bpf_dynptr_copy(&ptr_xdp, 0, &ptr_buf, 0, len * chunks);
 687	if (err)
 688		goto out;
 689
 690	bpf_for(i, 0, chunks) {
 691		__builtin_memset(buf, 0, sizeof(buf));
 692		err = bpf_dynptr_read(&buf, len, &ptr_xdp, i * len, 0);
 693		if (err)
 694			goto out;
 695		if (bpf_memcmp(data, buf, len) != 0)
 696			goto out;
 697	}
 698
 699	/* Source dynptr is backed by non-contiguous memory */
 700	__builtin_memset(buf, 0, sizeof(buf));
 701	bpf_for(i, 0, chunks) {
 702		err = bpf_dynptr_write(&ptr_buf, i * len, buf, len, 0);
 703		if (err)
 704			goto out;
 705	}
 706
 707	err = bpf_dynptr_copy(&ptr_buf, 0, &ptr_xdp, 0, len * chunks);
 708	if (err)
 709		goto out;
 710
 711	bpf_for(i, 0, chunks) {
 712		__builtin_memset(buf, 0, sizeof(buf));
 713		err = bpf_dynptr_read(&buf, len, &ptr_buf, i * len, 0);
 714		if (err)
 715			goto out;
 716		if (bpf_memcmp(data, buf, len) != 0)
 717			goto out;
 718	}
 719
 720	/* Both source and destination dynptrs are backed by non-contiguous memory */
 721	err = bpf_dynptr_copy(&ptr_xdp, 2, &ptr_xdp, len, len * (chunks - 1));
 722	if (err)
 723		goto out;
 724
 725	bpf_for(i, 0, chunks - 1) {
 726		__builtin_memset(buf, 0, sizeof(buf));
 727		err = bpf_dynptr_read(&buf, len, &ptr_xdp, 2 + i * len, 0);
 728		if (err)
 729			goto out;
 730		if (bpf_memcmp(data, buf, len) != 0)
 731			goto out;
 732	}
 733
 734	if (bpf_dynptr_copy(&ptr_xdp, xdp_data_size - 3000, &ptr_xdp, 0, len * chunks) != -E2BIG)
 735		err = 1;
 736
 737out:
 738	bpf_ringbuf_discard_dynptr(&ptr_buf, 0);
 739	return XDP_DROP;
 740}
 741
 742char memset_zero_data[] = "data to be zeroed";
 743
 744SEC("?tp/syscalls/sys_enter_nanosleep")
 745int test_dynptr_memset_zero(void *ctx)
 746{
 747	__u32 data_sz = sizeof(memset_zero_data);
 748	char zeroes[32] = {'\0'};
 749	struct bpf_dynptr ptr;
 750
 751	err = bpf_dynptr_from_mem(memset_zero_data, data_sz, 0, &ptr);
 752	err = err ?: bpf_dynptr_memset(&ptr, 0, data_sz, 0);
 753	err = err ?: bpf_memcmp(zeroes, memset_zero_data, data_sz);
 754
 755	return 0;
 756}
 757
 758#define DYNPTR_MEMSET_VAL 42
 759
 760char memset_notzero_data[] = "data to be overwritten";
 761
 762SEC("?tp/syscalls/sys_enter_nanosleep")
 763int test_dynptr_memset_notzero(void *ctx)
 764{
 765	u32 data_sz = sizeof(memset_notzero_data);
 766	struct bpf_dynptr ptr;
 767	char expected[32];
 768
 769	__builtin_memset(expected, DYNPTR_MEMSET_VAL, data_sz);
 770
 771	err = bpf_dynptr_from_mem(memset_notzero_data, data_sz, 0, &ptr);
 772	err = err ?: bpf_dynptr_memset(&ptr, 0, data_sz, DYNPTR_MEMSET_VAL);
 773	err = err ?: bpf_memcmp(expected, memset_notzero_data, data_sz);
 774
 775	return 0;
 776}
 777
 778char memset_zero_offset_data[] = "data to be zeroed partially";
 779
 780SEC("?tp/syscalls/sys_enter_nanosleep")
 781int test_dynptr_memset_zero_offset(void *ctx)
 782{
 783	char expected[] = "data to \0\0\0\0eroed partially";
 784	__u32 data_sz = sizeof(memset_zero_offset_data);
 785	struct bpf_dynptr ptr;
 786
 787	err = bpf_dynptr_from_mem(memset_zero_offset_data, data_sz, 0, &ptr);
 788	err = err ?: bpf_dynptr_memset(&ptr, 8, 4, 0);
 789	err = err ?: bpf_memcmp(expected, memset_zero_offset_data, data_sz);
 790
 791	return 0;
 792}
 793
 794char memset_zero_adjusted_data[] = "data to be zeroed partially";
 795
 796SEC("?tp/syscalls/sys_enter_nanosleep")
 797int test_dynptr_memset_zero_adjusted(void *ctx)
 798{
 799	char expected[] = "data\0\0\0\0be zeroed partially";
 800	__u32 data_sz = sizeof(memset_zero_adjusted_data);
 801	struct bpf_dynptr ptr;
 802
 803	err = bpf_dynptr_from_mem(memset_zero_adjusted_data, data_sz, 0, &ptr);
 804	err = err ?: bpf_dynptr_adjust(&ptr, 4, 8);
 805	err = err ?: bpf_dynptr_memset(&ptr, 0, bpf_dynptr_size(&ptr), 0);
 806	err = err ?: bpf_memcmp(expected, memset_zero_adjusted_data, data_sz);
 807
 808	return 0;
 809}
 810
 811char memset_overflow_data[] = "memset overflow data";
 812
 813SEC("?tp/syscalls/sys_enter_nanosleep")
 814int test_dynptr_memset_overflow(void *ctx)
 815{
 816	__u32 data_sz = sizeof(memset_overflow_data);
 817	struct bpf_dynptr ptr;
 818	int ret;
 819
 820	err = bpf_dynptr_from_mem(memset_overflow_data, data_sz, 0, &ptr);
 821	ret = bpf_dynptr_memset(&ptr, 0, data_sz + 1, 0);
 822	if (ret != -E2BIG)
 823		err = 1;
 824
 825	return 0;
 826}
 827
 828SEC("?tp/syscalls/sys_enter_nanosleep")
 829int test_dynptr_memset_overflow_offset(void *ctx)
 830{
 831	__u32 data_sz = sizeof(memset_overflow_data);
 832	struct bpf_dynptr ptr;
 833	int ret;
 834
 835	err = bpf_dynptr_from_mem(memset_overflow_data, data_sz, 0, &ptr);
 836	ret = bpf_dynptr_memset(&ptr, 1, data_sz, 0);
 837	if (ret != -E2BIG)
 838		err = 1;
 839
 840	return 0;
 841}
 842
 843SEC("?cgroup_skb/egress")
 844int test_dynptr_memset_readonly(struct __sk_buff *skb)
 845{
 846	struct bpf_dynptr ptr;
 847	int ret;
 848
 849	err = bpf_dynptr_from_skb(skb, 0, &ptr);
 850
 851	/* cgroup skbs are read only, memset should fail */
 852	ret = bpf_dynptr_memset(&ptr, 0, bpf_dynptr_size(&ptr), 0);
 853	if (ret != -EINVAL)
 854		err = 1;
 855
 856	return 0;
 857}
 858
 859#define min_t(type, x, y) ({		\
 860	type __x = (x);			\
 861	type __y = (y);			\
 862	__x < __y ? __x : __y; })
 863
 864SEC("xdp")
 865int test_dynptr_memset_xdp_chunks(struct xdp_md *xdp)
 866{
 867	u32 data_sz, chunk_sz, offset = 0;
 868	const int max_chunks = 200;
 869	struct bpf_dynptr ptr_xdp;
 870	char expected_buf[32];
 871	char buf[32];
 872	int i;
 873
 874	__builtin_memset(expected_buf, DYNPTR_MEMSET_VAL, sizeof(expected_buf));
 875
 876	/* ptr_xdp is backed by non-contiguous memory */
 877	bpf_dynptr_from_xdp(xdp, 0, &ptr_xdp);
 878	data_sz = bpf_dynptr_size(&ptr_xdp);
 879
 880	err = bpf_dynptr_memset(&ptr_xdp, 0, data_sz, DYNPTR_MEMSET_VAL);
 881	if (err) {
 882		/* bpf_dynptr_memset() eventually called bpf_xdp_pointer()
 883		 * where if data_sz is greater than 0xffff, -EFAULT will be
 884		 * returned. For 64K page size, data_sz is greater than
 885		 * 64K, so error is expected and let us zero out error and
 886		 * return success.
 887		 */
 888		if (data_sz >= PAGE_SIZE_64K)
 889			err = 0;
 890		goto out;
 891	}
 892
 893	bpf_for(i, 0, max_chunks) {
 894		offset = i * sizeof(buf);
 895		if (offset >= data_sz)
 896			goto out;
 897		chunk_sz = min_t(u32, sizeof(buf), data_sz - offset);
 898		err = bpf_dynptr_read(&buf, chunk_sz, &ptr_xdp, offset, 0);
 899		if (err)
 900			goto out;
 901		err = bpf_memcmp(buf, expected_buf, sizeof(buf));
 902		if (err)
 903			goto out;
 904	}
 905out:
 906	return XDP_DROP;
 907}
 908
 909void *user_ptr;
 910/* Contains the copy of the data pointed by user_ptr.
 911 * Size 384 to make it not fit into a single kernel chunk when copying
 912 * but less than the maximum bpf stack size (512).
 913 */
 914char expected_str[384];
 915__u32 test_len[7] = {0/* placeholder */, 0, 1, 2, 255, 256, 257};
 916
 917typedef int (*bpf_read_dynptr_fn_t)(struct bpf_dynptr *dptr, u64 off,
 918				    u64 size, const void *unsafe_ptr);
 919
 920/* Returns the offset just before the end of the maximum sized xdp fragment.
 921 * Any write larger than 32 bytes will be split between 2 fragments.
 922 */
 923__u32 xdp_near_frag_end_offset(void)
 924{
 925	const __u32 headroom = 256;
 926	const __u32 max_frag_size =  __PAGE_SIZE - headroom - sizeof(struct skb_shared_info);
 927
 928	/* 32 bytes before the approximate end of the fragment */
 929	return max_frag_size - 32;
 930}
 931
 932/* Use __always_inline on test_dynptr_probe[_str][_xdp]() and callbacks
 933 * of type bpf_read_dynptr_fn_t to prevent compiler from generating
 934 * indirect calls that make program fail to load with "unknown opcode" error.
 935 */
 936static __always_inline void test_dynptr_probe(void *ptr, bpf_read_dynptr_fn_t bpf_read_dynptr_fn)
 937{
 938	char buf[sizeof(expected_str)];
 939	struct bpf_dynptr ptr_buf;
 940	int i;
 941
 942	if (bpf_get_current_pid_tgid() >> 32 != pid)
 943		return;
 944
 945	err = bpf_ringbuf_reserve_dynptr(&ringbuf, sizeof(buf), 0, &ptr_buf);
 946
 947	bpf_for(i, 0, ARRAY_SIZE(test_len)) {
 948		__u32 len = test_len[i];
 949
 950		err = err ?: bpf_read_dynptr_fn(&ptr_buf, 0, test_len[i], ptr);
 951		if (len > sizeof(buf))
 952			break;
 953		err = err ?: bpf_dynptr_read(&buf, len, &ptr_buf, 0, 0);
 954
 955		if (err || bpf_memcmp(expected_str, buf, len))
 956			err = 1;
 957
 958		/* Reset buffer and dynptr */
 959		__builtin_memset(buf, 0, sizeof(buf));
 960		err = err ?: bpf_dynptr_write(&ptr_buf, 0, buf, len, 0);
 961	}
 962	bpf_ringbuf_discard_dynptr(&ptr_buf, 0);
 963}
 964
 965static __always_inline void test_dynptr_probe_str(void *ptr,
 966						  bpf_read_dynptr_fn_t bpf_read_dynptr_fn)
 967{
 968	char buf[sizeof(expected_str)];
 969	struct bpf_dynptr ptr_buf;
 970	__u32 cnt, i;
 971
 972	if (bpf_get_current_pid_tgid() >> 32 != pid)
 973		return;
 974
 975	bpf_ringbuf_reserve_dynptr(&ringbuf, sizeof(buf), 0, &ptr_buf);
 976
 977	bpf_for(i, 0, ARRAY_SIZE(test_len)) {
 978		__u32 len = test_len[i];
 979
 980		cnt = bpf_read_dynptr_fn(&ptr_buf, 0, len, ptr);
 981		if (cnt != len)
 982			err = 1;
 983
 984		if (len > sizeof(buf))
 985			continue;
 986		err = err ?: bpf_dynptr_read(&buf, len, &ptr_buf, 0, 0);
 987		if (!len)
 988			continue;
 989		if (err || bpf_memcmp(expected_str, buf, len - 1) || buf[len - 1] != '\0')
 990			err = 1;
 991	}
 992	bpf_ringbuf_discard_dynptr(&ptr_buf, 0);
 993}
 994
 995static __always_inline void test_dynptr_probe_xdp(struct xdp_md *xdp, void *ptr,
 996						  bpf_read_dynptr_fn_t bpf_read_dynptr_fn)
 997{
 998	struct bpf_dynptr ptr_xdp;
 999	char buf[sizeof(expected_str)];
1000	__u32 off, i;
1001
1002	if (bpf_get_current_pid_tgid() >> 32 != pid)
1003		return;
1004
1005	off = xdp_near_frag_end_offset();
1006	err = bpf_dynptr_from_xdp(xdp, 0, &ptr_xdp);
1007
1008	bpf_for(i, 0, ARRAY_SIZE(test_len)) {
1009		__u32 len = test_len[i];
1010
1011		err = err ?: bpf_read_dynptr_fn(&ptr_xdp, off, len, ptr);
1012		if (len > sizeof(buf))
1013			continue;
1014		err = err ?: bpf_dynptr_read(&buf, len, &ptr_xdp, off, 0);
1015		if (err || bpf_memcmp(expected_str, buf, len))
1016			err = 1;
1017		/* Reset buffer and dynptr */
1018		__builtin_memset(buf, 0, sizeof(buf));
1019		err = err ?: bpf_dynptr_write(&ptr_xdp, off, buf, len, 0);
1020	}
1021}
1022
1023static __always_inline void test_dynptr_probe_str_xdp(struct xdp_md *xdp, void *ptr,
1024						      bpf_read_dynptr_fn_t bpf_read_dynptr_fn)
1025{
1026	struct bpf_dynptr ptr_xdp;
1027	char buf[sizeof(expected_str)];
1028	__u32 cnt, off, i;
1029
1030	if (bpf_get_current_pid_tgid() >> 32 != pid)
1031		return;
1032
1033	off = xdp_near_frag_end_offset();
1034	err = bpf_dynptr_from_xdp(xdp, 0, &ptr_xdp);
1035	if (err)
1036		return;
1037
1038	bpf_for(i, 0, ARRAY_SIZE(test_len)) {
1039		__u32 len = test_len[i];
1040
1041		cnt = bpf_read_dynptr_fn(&ptr_xdp, off, len, ptr);
1042		if (cnt != len)
1043			err = 1;
1044
1045		if (len > sizeof(buf))
1046			continue;
1047		err = err ?: bpf_dynptr_read(&buf, len, &ptr_xdp, off, 0);
1048
1049		if (!len)
1050			continue;
1051		if (err || bpf_memcmp(expected_str, buf, len - 1) || buf[len - 1] != '\0')
1052			err = 1;
1053
1054		__builtin_memset(buf, 0, sizeof(buf));
1055		err = err ?: bpf_dynptr_write(&ptr_xdp, off, buf, len, 0);
1056	}
1057}
1058
1059SEC("xdp")
1060int test_probe_read_user_dynptr(struct xdp_md *xdp)
1061{
1062	test_dynptr_probe(user_ptr, bpf_probe_read_user_dynptr);
1063	if (!err)
1064		test_dynptr_probe_xdp(xdp, user_ptr, bpf_probe_read_user_dynptr);
1065	return XDP_PASS;
1066}
1067
1068SEC("xdp")
1069int test_probe_read_kernel_dynptr(struct xdp_md *xdp)
1070{
1071	test_dynptr_probe(expected_str, bpf_probe_read_kernel_dynptr);
1072	if (!err)
1073		test_dynptr_probe_xdp(xdp, expected_str, bpf_probe_read_kernel_dynptr);
1074	return XDP_PASS;
1075}
1076
1077SEC("xdp")
1078int test_probe_read_user_str_dynptr(struct xdp_md *xdp)
1079{
1080	test_dynptr_probe_str(user_ptr, bpf_probe_read_user_str_dynptr);
1081	if (!err)
1082		test_dynptr_probe_str_xdp(xdp, user_ptr, bpf_probe_read_user_str_dynptr);
1083	return XDP_PASS;
1084}
1085
1086SEC("xdp")
1087int test_probe_read_kernel_str_dynptr(struct xdp_md *xdp)
1088{
1089	test_dynptr_probe_str(expected_str, bpf_probe_read_kernel_str_dynptr);
1090	if (!err)
1091		test_dynptr_probe_str_xdp(xdp, expected_str, bpf_probe_read_kernel_str_dynptr);
1092	return XDP_PASS;
1093}
1094
1095SEC("fentry.s/" SYS_PREFIX "sys_nanosleep")
1096int test_copy_from_user_dynptr(void *ctx)
1097{
1098	test_dynptr_probe(user_ptr, bpf_copy_from_user_dynptr);
1099	return 0;
1100}
1101
1102SEC("fentry.s/" SYS_PREFIX "sys_nanosleep")
1103int test_copy_from_user_str_dynptr(void *ctx)
1104{
1105	test_dynptr_probe_str(user_ptr, bpf_copy_from_user_str_dynptr);
1106	return 0;
1107}
1108
1109static int bpf_copy_data_from_user_task(struct bpf_dynptr *dptr, u64 off,
1110					u64 size, const void *unsafe_ptr)
1111{
1112	struct task_struct *task = bpf_get_current_task_btf();
1113
1114	return bpf_copy_from_user_task_dynptr(dptr, off, size, unsafe_ptr, task);
1115}
1116
1117static int bpf_copy_data_from_user_task_str(struct bpf_dynptr *dptr, u64 off,
1118					    u64 size, const void *unsafe_ptr)
1119{
1120	struct task_struct *task = bpf_get_current_task_btf();
1121
1122	return bpf_copy_from_user_task_str_dynptr(dptr, off, size, unsafe_ptr, task);
1123}
1124
1125SEC("fentry.s/" SYS_PREFIX "sys_nanosleep")
1126int test_copy_from_user_task_dynptr(void *ctx)
1127{
1128	test_dynptr_probe(user_ptr, bpf_copy_data_from_user_task);
1129	return 0;
1130}
1131
1132SEC("fentry.s/" SYS_PREFIX "sys_nanosleep")
1133int test_copy_from_user_task_str_dynptr(void *ctx)
1134{
1135	test_dynptr_probe_str(user_ptr, bpf_copy_data_from_user_task_str);
1136	return 0;
1137}