stream.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
  3
  4#include <linux/bpf.h>
  5#include <linux/filter.h>
  6#include <linux/bpf_mem_alloc.h>
  7#include <linux/gfp.h>
  8#include <linux/memory.h>
  9#include <linux/mutex.h>
 10
 11static void bpf_stream_elem_init(struct bpf_stream_elem *elem, int len)
 12{
 13	init_llist_node(&elem->node);
 14	elem->total_len = len;
 15	elem->consumed_len = 0;
 16}
 17
 18static struct bpf_stream_elem *bpf_stream_elem_alloc(int len)
 19{
 20	const int max_len = ARRAY_SIZE((struct bpf_bprintf_buffers){}.buf);
 21	struct bpf_stream_elem *elem;
 22	size_t alloc_size;
 23
 24	/*
 25	 * Length denotes the amount of data to be written as part of stream element,
 26	 * thus includes '\0' byte. We're capped by how much bpf_bprintf_buffers can
 27	 * accomodate, therefore deny allocations that won't fit into them.
 28	 */
 29	if (len < 0 || len > max_len)
 30		return NULL;
 31
 32	alloc_size = offsetof(struct bpf_stream_elem, str[len]);
 33	elem = kmalloc_nolock(alloc_size, __GFP_ZERO, -1);
 34	if (!elem)
 35		return NULL;
 36
 37	bpf_stream_elem_init(elem, len);
 38
 39	return elem;
 40}
 41
 42static int __bpf_stream_push_str(struct llist_head *log, const char *str, int len)
 43{
 44	struct bpf_stream_elem *elem = NULL;
 45
 46	/*
 47	 * Allocate a bpf_prog_stream_elem and push it to the bpf_prog_stream
 48	 * log, elements will be popped at once and reversed to print the log.
 49	 */
 50	elem = bpf_stream_elem_alloc(len);
 51	if (!elem)
 52		return -ENOMEM;
 53
 54	memcpy(elem->str, str, len);
 55	llist_add(&elem->node, log);
 56
 57	return 0;
 58}
 59
 60static int bpf_stream_consume_capacity(struct bpf_stream *stream, int len)
 61{
 62	if (atomic_read(&stream->capacity) >= BPF_STREAM_MAX_CAPACITY)
 63		return -ENOSPC;
 64	if (atomic_add_return(len, &stream->capacity) >= BPF_STREAM_MAX_CAPACITY) {
 65		atomic_sub(len, &stream->capacity);
 66		return -ENOSPC;
 67	}
 68	return 0;
 69}
 70
 71static void bpf_stream_release_capacity(struct bpf_stream *stream, struct bpf_stream_elem *elem)
 72{
 73	int len = elem->total_len;
 74
 75	atomic_sub(len, &stream->capacity);
 76}
 77
 78static int bpf_stream_push_str(struct bpf_stream *stream, const char *str, int len)
 79{
 80	int ret = bpf_stream_consume_capacity(stream, len);
 81
 82	return ret ?: __bpf_stream_push_str(&stream->log, str, len);
 83}
 84
 85static struct bpf_stream *bpf_stream_get(enum bpf_stream_id stream_id, struct bpf_prog_aux *aux)
 86{
 87	if (stream_id != BPF_STDOUT && stream_id != BPF_STDERR)
 88		return NULL;
 89	return &aux->stream[stream_id - 1];
 90}
 91
 92static void bpf_stream_free_elem(struct bpf_stream_elem *elem)
 93{
 94	kfree_nolock(elem);
 95}
 96
 97static void bpf_stream_free_list(struct llist_node *list)
 98{
 99	struct bpf_stream_elem *elem, *tmp;
100
101	llist_for_each_entry_safe(elem, tmp, list, node)
102		bpf_stream_free_elem(elem);
103}
104
105static struct llist_node *bpf_stream_backlog_peek(struct bpf_stream *stream)
106{
107	return stream->backlog_head;
108}
109
110static struct llist_node *bpf_stream_backlog_pop(struct bpf_stream *stream)
111{
112	struct llist_node *node;
113
114	node = stream->backlog_head;
115	if (stream->backlog_head == stream->backlog_tail)
116		stream->backlog_head = stream->backlog_tail = NULL;
117	else
118		stream->backlog_head = node->next;
119	return node;
120}
121
122static void bpf_stream_backlog_fill(struct bpf_stream *stream)
123{
124	struct llist_node *head, *tail;
125
126	if (llist_empty(&stream->log))
127		return;
128	tail = llist_del_all(&stream->log);
129	if (!tail)
130		return;
131	head = llist_reverse_order(tail);
132
133	if (!stream->backlog_head) {
134		stream->backlog_head = head;
135		stream->backlog_tail = tail;
136	} else {
137		stream->backlog_tail->next = head;
138		stream->backlog_tail = tail;
139	}
140
141	return;
142}
143
144static bool bpf_stream_consume_elem(struct bpf_stream_elem *elem, int *len)
145{
146	int rem = elem->total_len - elem->consumed_len;
147	int used = min(rem, *len);
148
149	elem->consumed_len += used;
150	*len -= used;
151
152	return elem->consumed_len == elem->total_len;
153}
154
155static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len)
156{
157	int rem_len = len, cons_len, ret = 0;
158	struct bpf_stream_elem *elem = NULL;
159	struct llist_node *node;
160
161	mutex_lock(&stream->lock);
162
163	while (rem_len) {
164		int pos = len - rem_len;
165		bool cont;
166
167		node = bpf_stream_backlog_peek(stream);
168		if (!node) {
169			bpf_stream_backlog_fill(stream);
170			node = bpf_stream_backlog_peek(stream);
171		}
172		if (!node)
173			break;
174		elem = container_of(node, typeof(*elem), node);
175
176		cons_len = elem->consumed_len;
177		cont = bpf_stream_consume_elem(elem, &rem_len) == false;
178
179		ret = copy_to_user(buf + pos, elem->str + cons_len,
180				   elem->consumed_len - cons_len);
181		/* Restore in case of error. */
182		if (ret) {
183			ret = -EFAULT;
184			elem->consumed_len = cons_len;
185			break;
186		}
187
188		if (cont)
189			continue;
190		bpf_stream_backlog_pop(stream);
191		bpf_stream_release_capacity(stream, elem);
192		bpf_stream_free_elem(elem);
193	}
194
195	mutex_unlock(&stream->lock);
196	return ret ? ret : len - rem_len;
197}
198
199int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, int len)
200{
201	struct bpf_stream *stream;
202
203	stream = bpf_stream_get(stream_id, prog->aux);
204	if (!stream)
205		return -ENOENT;
206	return bpf_stream_read(stream, buf, len);
207}
208
209__bpf_kfunc_start_defs();
210
211/*
212 * Avoid using enum bpf_stream_id so that kfunc users don't have to pull in the
213 * enum in headers.
214 */
215__bpf_kfunc int bpf_stream_vprintk_impl(int stream_id, const char *fmt__str, const void *args,
216					u32 len__sz, void *aux__prog)
217{
218	struct bpf_bprintf_data data = {
219		.get_bin_args	= true,
220		.get_buf	= true,
221	};
222	struct bpf_prog_aux *aux = aux__prog;
223	u32 fmt_size = strlen(fmt__str) + 1;
224	struct bpf_stream *stream;
225	u32 data_len = len__sz;
226	int ret, num_args;
227
228	stream = bpf_stream_get(stream_id, aux);
229	if (!stream)
230		return -ENOENT;
231
232	if (data_len & 7 || data_len > MAX_BPRINTF_VARARGS * 8 ||
233	    (data_len && !args))
234		return -EINVAL;
235	num_args = data_len / 8;
236
237	ret = bpf_bprintf_prepare(fmt__str, fmt_size, args, num_args, &data);
238	if (ret < 0)
239		return ret;
240
241	ret = bstr_printf(data.buf, MAX_BPRINTF_BUF, fmt__str, data.bin_args);
242	/* Exclude NULL byte during push. */
243	ret = bpf_stream_push_str(stream, data.buf, ret);
244	bpf_bprintf_cleanup(&data);
245
246	return ret;
247}
248
249__bpf_kfunc_end_defs();
250
251/* Added kfunc to common_btf_ids */
252
253void bpf_prog_stream_init(struct bpf_prog *prog)
254{
255	int i;
256
257	for (i = 0; i < ARRAY_SIZE(prog->aux->stream); i++) {
258		atomic_set(&prog->aux->stream[i].capacity, 0);
259		init_llist_head(&prog->aux->stream[i].log);
260		mutex_init(&prog->aux->stream[i].lock);
261		prog->aux->stream[i].backlog_head = NULL;
262		prog->aux->stream[i].backlog_tail = NULL;
263	}
264}
265
266void bpf_prog_stream_free(struct bpf_prog *prog)
267{
268	struct llist_node *list;
269	int i;
270
271	for (i = 0; i < ARRAY_SIZE(prog->aux->stream); i++) {
272		list = llist_del_all(&prog->aux->stream[i].log);
273		bpf_stream_free_list(list);
274		bpf_stream_free_list(prog->aux->stream[i].backlog_head);
275	}
276}
277
278void bpf_stream_stage_init(struct bpf_stream_stage *ss)
279{
280	init_llist_head(&ss->log);
281	ss->len = 0;
282}
283
284void bpf_stream_stage_free(struct bpf_stream_stage *ss)
285{
286	struct llist_node *node;
287
288	node = llist_del_all(&ss->log);
289	bpf_stream_free_list(node);
290}
291
292int bpf_stream_stage_printk(struct bpf_stream_stage *ss, const char *fmt, ...)
293{
294	struct bpf_bprintf_buffers *buf;
295	va_list args;
296	int ret;
297
298	if (bpf_try_get_buffers(&buf))
299		return -EBUSY;
300
301	va_start(args, fmt);
302	ret = vsnprintf(buf->buf, ARRAY_SIZE(buf->buf), fmt, args);
303	va_end(args);
304	ss->len += ret;
305	/* Exclude NULL byte during push. */
306	ret = __bpf_stream_push_str(&ss->log, buf->buf, ret);
307	bpf_put_buffers();
308	return ret;
309}
310
311int bpf_stream_stage_commit(struct bpf_stream_stage *ss, struct bpf_prog *prog,
312			    enum bpf_stream_id stream_id)
313{
314	struct llist_node *list, *head, *tail;
315	struct bpf_stream *stream;
316	int ret;
317
318	stream = bpf_stream_get(stream_id, prog->aux);
319	if (!stream)
320		return -EINVAL;
321
322	ret = bpf_stream_consume_capacity(stream, ss->len);
323	if (ret)
324		return ret;
325
326	list = llist_del_all(&ss->log);
327	head = tail = list;
328
329	if (!list)
330		return 0;
331	while (llist_next(list)) {
332		tail = llist_next(list);
333		list = tail;
334	}
335	llist_add_batch(head, tail, &stream->log);
336	return 0;
337}
338
339struct dump_stack_ctx {
340	struct bpf_stream_stage *ss;
341	int err;
342};
343
344static bool dump_stack_cb(void *cookie, u64 ip, u64 sp, u64 bp)
345{
346	struct dump_stack_ctx *ctxp = cookie;
347	const char *file = "", *line = "";
348	struct bpf_prog *prog;
349	int num, ret;
350
351	rcu_read_lock();
352	prog = bpf_prog_ksym_find(ip);
353	rcu_read_unlock();
354	if (prog) {
355		ret = bpf_prog_get_file_line(prog, ip, &file, &line, &num);
356		if (ret < 0)
357			goto end;
358		ctxp->err = bpf_stream_stage_printk(ctxp->ss, "%pS\n  %s @ %s:%d\n",
359						    (void *)(long)ip, line, file, num);
360		return !ctxp->err;
361	}
362end:
363	ctxp->err = bpf_stream_stage_printk(ctxp->ss, "%pS\n", (void *)(long)ip);
364	return !ctxp->err;
365}
366
367int bpf_stream_stage_dump_stack(struct bpf_stream_stage *ss)
368{
369	struct dump_stack_ctx ctx = { .ss = ss };
370	int ret;
371
372	ret = bpf_stream_stage_printk(ss, "CPU: %d UID: %d PID: %d Comm: %s\n",
373				      raw_smp_processor_id(), __kuid_val(current_real_cred()->euid),
374				      current->pid, current->comm);
375	if (ret)
376		return ret;
377	ret = bpf_stream_stage_printk(ss, "Call trace:\n");
378	if (ret)
379		return ret;
380	arch_bpf_stack_walk(dump_stack_cb, &ctx);
381	if (ctx.err)
382		return ctx.err;
383	return bpf_stream_stage_printk(ss, "\n");
384}