bpf_sk_storage.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* Copyright (c) 2019 Facebook  */
  3#include <linux/rculist.h>
  4#include <linux/list.h>
  5#include <linux/hash.h>
  6#include <linux/types.h>
  7#include <linux/spinlock.h>
  8#include <linux/bpf.h>
  9#include <linux/btf.h>
 10#include <linux/btf_ids.h>
 11#include <linux/bpf_local_storage.h>
 12#include <net/bpf_sk_storage.h>
 13#include <net/sock.h>
 14#include <uapi/linux/sock_diag.h>
 15#include <uapi/linux/btf.h>
 16#include <linux/rcupdate_trace.h>
 17
 18DEFINE_BPF_STORAGE_CACHE(sk_cache);
 19
 20static struct bpf_local_storage_data *
 21bpf_sk_storage_lookup(struct sock *sk, struct bpf_map *map, bool cacheit_lockit)
 22{
 23	struct bpf_local_storage *sk_storage;
 24	struct bpf_local_storage_map *smap;
 25
 26	sk_storage =
 27		rcu_dereference_check(sk->sk_bpf_storage, bpf_rcu_lock_held());
 28	if (!sk_storage)
 29		return NULL;
 30
 31	smap = (struct bpf_local_storage_map *)map;
 32	return bpf_local_storage_lookup(sk_storage, smap, cacheit_lockit);
 33}
 34
 35static int bpf_sk_storage_del(struct sock *sk, struct bpf_map *map)
 36{
 37	struct bpf_local_storage_data *sdata;
 38
 39	sdata = bpf_sk_storage_lookup(sk, map, false);
 40	if (!sdata)
 41		return -ENOENT;
 42
 43	bpf_selem_unlink(SELEM(sdata), false);
 44
 45	return 0;
 46}
 47
 48/* Called by __sk_destruct() & bpf_sk_storage_clone() */
 49void bpf_sk_storage_free(struct sock *sk)
 50{
 51	struct bpf_local_storage *sk_storage;
 52
 53	rcu_read_lock_dont_migrate();
 54	sk_storage = rcu_dereference(sk->sk_bpf_storage);
 55	if (!sk_storage)
 56		goto out;
 57
 58	bpf_local_storage_destroy(sk_storage);
 59out:
 60	rcu_read_unlock_migrate();
 61}
 62
 63static void bpf_sk_storage_map_free(struct bpf_map *map)
 64{
 65	bpf_local_storage_map_free(map, &sk_cache, NULL);
 66}
 67
 68static struct bpf_map *bpf_sk_storage_map_alloc(union bpf_attr *attr)
 69{
 70	return bpf_local_storage_map_alloc(attr, &sk_cache, false);
 71}
 72
 73static int notsupp_get_next_key(struct bpf_map *map, void *key,
 74				void *next_key)
 75{
 76	return -ENOTSUPP;
 77}
 78
 79static void *bpf_fd_sk_storage_lookup_elem(struct bpf_map *map, void *key)
 80{
 81	struct bpf_local_storage_data *sdata;
 82	struct socket *sock;
 83	int fd, err;
 84
 85	fd = *(int *)key;
 86	sock = sockfd_lookup(fd, &err);
 87	if (sock) {
 88		sdata = bpf_sk_storage_lookup(sock->sk, map, true);
 89		sockfd_put(sock);
 90		return sdata ? sdata->data : NULL;
 91	}
 92
 93	return ERR_PTR(err);
 94}
 95
 96static long bpf_fd_sk_storage_update_elem(struct bpf_map *map, void *key,
 97					  void *value, u64 map_flags)
 98{
 99	struct bpf_local_storage_data *sdata;
100	struct socket *sock;
101	int fd, err;
102
103	fd = *(int *)key;
104	sock = sockfd_lookup(fd, &err);
105	if (sock) {
106		sdata = bpf_local_storage_update(
107			sock->sk, (struct bpf_local_storage_map *)map, value,
108			map_flags, false, GFP_ATOMIC);
109		sockfd_put(sock);
110		return PTR_ERR_OR_ZERO(sdata);
111	}
112
113	return err;
114}
115
116static long bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key)
117{
118	struct socket *sock;
119	int fd, err;
120
121	fd = *(int *)key;
122	sock = sockfd_lookup(fd, &err);
123	if (sock) {
124		err = bpf_sk_storage_del(sock->sk, map);
125		sockfd_put(sock);
126		return err;
127	}
128
129	return err;
130}
131
132static struct bpf_local_storage_elem *
133bpf_sk_storage_clone_elem(struct sock *newsk,
134			  struct bpf_local_storage_map *smap,
135			  struct bpf_local_storage_elem *selem)
136{
137	struct bpf_local_storage_elem *copy_selem;
138
139	copy_selem = bpf_selem_alloc(smap, newsk, NULL, false, GFP_ATOMIC);
140	if (!copy_selem)
141		return NULL;
142
143	if (btf_record_has_field(smap->map.record, BPF_SPIN_LOCK))
144		copy_map_value_locked(&smap->map, SDATA(copy_selem)->data,
145				      SDATA(selem)->data, true);
146	else
147		copy_map_value(&smap->map, SDATA(copy_selem)->data,
148			       SDATA(selem)->data);
149
150	return copy_selem;
151}
152
153int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk)
154{
155	struct bpf_local_storage *new_sk_storage = NULL;
156	struct bpf_local_storage *sk_storage;
157	struct bpf_local_storage_elem *selem;
158	int ret = 0;
159
160	RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL);
161
162	rcu_read_lock_dont_migrate();
163	sk_storage = rcu_dereference(sk->sk_bpf_storage);
164
165	if (!sk_storage || hlist_empty(&sk_storage->list))
166		goto out;
167
168	hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) {
169		struct bpf_local_storage_elem *copy_selem;
170		struct bpf_local_storage_map *smap;
171		struct bpf_map *map;
172
173		smap = rcu_dereference(SDATA(selem)->smap);
174		if (!(smap->map.map_flags & BPF_F_CLONE))
175			continue;
176
177		/* Note that for lockless listeners adding new element
178		 * here can race with cleanup in bpf_local_storage_map_free.
179		 * Try to grab map refcnt to make sure that it's still
180		 * alive and prevent concurrent removal.
181		 */
182		map = bpf_map_inc_not_zero(&smap->map);
183		if (IS_ERR(map))
184			continue;
185
186		copy_selem = bpf_sk_storage_clone_elem(newsk, smap, selem);
187		if (!copy_selem) {
188			ret = -ENOMEM;
189			bpf_map_put(map);
190			goto out;
191		}
192
193		if (new_sk_storage) {
194			bpf_selem_link_map(smap, copy_selem);
195			bpf_selem_link_storage_nolock(new_sk_storage, copy_selem);
196		} else {
197			ret = bpf_local_storage_alloc(newsk, smap, copy_selem, GFP_ATOMIC);
198			if (ret) {
199				bpf_selem_free(copy_selem, true);
200				atomic_sub(smap->elem_size,
201					   &newsk->sk_omem_alloc);
202				bpf_map_put(map);
203				goto out;
204			}
205
206			new_sk_storage =
207				rcu_dereference(copy_selem->local_storage);
208		}
209		bpf_map_put(map);
210	}
211
212out:
213	rcu_read_unlock_migrate();
214
215	/* In case of an error, don't free anything explicitly here, the
216	 * caller is responsible to call bpf_sk_storage_free.
217	 */
218
219	return ret;
220}
221
222/* *gfp_flags* is a hidden argument provided by the verifier */
223BPF_CALL_5(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk,
224	   void *, value, u64, flags, gfp_t, gfp_flags)
225{
226	struct bpf_local_storage_data *sdata;
227
228	WARN_ON_ONCE(!bpf_rcu_lock_held());
229	if (!sk || !sk_fullsock(sk) || flags > BPF_SK_STORAGE_GET_F_CREATE)
230		return (unsigned long)NULL;
231
232	sdata = bpf_sk_storage_lookup(sk, map, true);
233	if (sdata)
234		return (unsigned long)sdata->data;
235
236	if (flags == BPF_SK_STORAGE_GET_F_CREATE &&
237	    /* Cannot add new elem to a going away sk.
238	     * Otherwise, the new elem may become a leak
239	     * (and also other memory issues during map
240	     *  destruction).
241	     */
242	    refcount_inc_not_zero(&sk->sk_refcnt)) {
243		sdata = bpf_local_storage_update(
244			sk, (struct bpf_local_storage_map *)map, value,
245			BPF_NOEXIST, false, gfp_flags);
246		/* sk must be a fullsock (guaranteed by verifier),
247		 * so sock_gen_put() is unnecessary.
248		 */
249		sock_put(sk);
250		return IS_ERR(sdata) ?
251			(unsigned long)NULL : (unsigned long)sdata->data;
252	}
253
254	return (unsigned long)NULL;
255}
256
257BPF_CALL_2(bpf_sk_storage_delete, struct bpf_map *, map, struct sock *, sk)
258{
259	WARN_ON_ONCE(!bpf_rcu_lock_held());
260	if (!sk || !sk_fullsock(sk))
261		return -EINVAL;
262
263	if (refcount_inc_not_zero(&sk->sk_refcnt)) {
264		int err;
265
266		err = bpf_sk_storage_del(sk, map);
267		sock_put(sk);
268		return err;
269	}
270
271	return -ENOENT;
272}
273
274static int bpf_sk_storage_charge(struct bpf_local_storage_map *smap,
275				 void *owner, u32 size)
276{
277	struct sock *sk = (struct sock *)owner;
278	int optmem_max;
279
280	optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max);
281	/* same check as in sock_kmalloc() */
282	if (size <= optmem_max &&
283	    atomic_read(&sk->sk_omem_alloc) + size < optmem_max) {
284		atomic_add(size, &sk->sk_omem_alloc);
285		return 0;
286	}
287
288	return -ENOMEM;
289}
290
291static void bpf_sk_storage_uncharge(struct bpf_local_storage_map *smap,
292				    void *owner, u32 size)
293{
294	struct sock *sk = owner;
295
296	atomic_sub(size, &sk->sk_omem_alloc);
297}
298
299static struct bpf_local_storage __rcu **
300bpf_sk_storage_ptr(void *owner)
301{
302	struct sock *sk = owner;
303
304	return &sk->sk_bpf_storage;
305}
306
307const struct bpf_map_ops sk_storage_map_ops = {
308	.map_meta_equal = bpf_map_meta_equal,
309	.map_alloc_check = bpf_local_storage_map_alloc_check,
310	.map_alloc = bpf_sk_storage_map_alloc,
311	.map_free = bpf_sk_storage_map_free,
312	.map_get_next_key = notsupp_get_next_key,
313	.map_lookup_elem = bpf_fd_sk_storage_lookup_elem,
314	.map_update_elem = bpf_fd_sk_storage_update_elem,
315	.map_delete_elem = bpf_fd_sk_storage_delete_elem,
316	.map_check_btf = bpf_local_storage_map_check_btf,
317	.map_btf_id = &bpf_local_storage_map_btf_id[0],
318	.map_local_storage_charge = bpf_sk_storage_charge,
319	.map_local_storage_uncharge = bpf_sk_storage_uncharge,
320	.map_owner_storage_ptr = bpf_sk_storage_ptr,
321	.map_mem_usage = bpf_local_storage_map_mem_usage,
322};
323
324const struct bpf_func_proto bpf_sk_storage_get_proto = {
325	.func		= bpf_sk_storage_get,
326	.gpl_only	= false,
327	.ret_type	= RET_PTR_TO_MAP_VALUE_OR_NULL,
328	.arg1_type	= ARG_CONST_MAP_PTR,
329	.arg2_type	= ARG_PTR_TO_BTF_ID_SOCK_COMMON,
330	.arg3_type	= ARG_PTR_TO_MAP_VALUE_OR_NULL,
331	.arg4_type	= ARG_ANYTHING,
332};
333
334const struct bpf_func_proto bpf_sk_storage_get_cg_sock_proto = {
335	.func		= bpf_sk_storage_get,
336	.gpl_only	= false,
337	.ret_type	= RET_PTR_TO_MAP_VALUE_OR_NULL,
338	.arg1_type	= ARG_CONST_MAP_PTR,
339	.arg2_type	= ARG_PTR_TO_CTX, /* context is 'struct sock' */
340	.arg3_type	= ARG_PTR_TO_MAP_VALUE_OR_NULL,
341	.arg4_type	= ARG_ANYTHING,
342};
343
344const struct bpf_func_proto bpf_sk_storage_delete_proto = {
345	.func		= bpf_sk_storage_delete,
346	.gpl_only	= false,
347	.ret_type	= RET_INTEGER,
348	.arg1_type	= ARG_CONST_MAP_PTR,
349	.arg2_type	= ARG_PTR_TO_BTF_ID_SOCK_COMMON,
350};
351
352static bool bpf_sk_storage_tracing_allowed(const struct bpf_prog *prog)
353{
354	if (prog->aux->dst_prog)
355		return false;
356
357	/* Ensure the tracing program is not tracing
358	 * any bpf_sk_storage*() function and also
359	 * use the bpf_sk_storage_(get|delete) helper.
360	 */
361	switch (prog->expected_attach_type) {
362	case BPF_TRACE_ITER:
363	case BPF_TRACE_RAW_TP:
364		/* bpf_sk_storage has no trace point */
365		return true;
366	case BPF_TRACE_FENTRY:
367	case BPF_TRACE_FEXIT:
368		return !!strncmp(prog->aux->attach_func_name, "bpf_sk_storage",
369				 strlen("bpf_sk_storage"));
370	default:
371		return false;
372	}
373
374	return false;
375}
376
377/* *gfp_flags* is a hidden argument provided by the verifier */
378BPF_CALL_5(bpf_sk_storage_get_tracing, struct bpf_map *, map, struct sock *, sk,
379	   void *, value, u64, flags, gfp_t, gfp_flags)
380{
381	WARN_ON_ONCE(!bpf_rcu_lock_held());
382	if (in_hardirq() || in_nmi())
383		return (unsigned long)NULL;
384
385	return (unsigned long)____bpf_sk_storage_get(map, sk, value, flags,
386						     gfp_flags);
387}
388
389BPF_CALL_2(bpf_sk_storage_delete_tracing, struct bpf_map *, map,
390	   struct sock *, sk)
391{
392	WARN_ON_ONCE(!bpf_rcu_lock_held());
393	if (in_hardirq() || in_nmi())
394		return -EPERM;
395
396	return ____bpf_sk_storage_delete(map, sk);
397}
398
399const struct bpf_func_proto bpf_sk_storage_get_tracing_proto = {
400	.func		= bpf_sk_storage_get_tracing,
401	.gpl_only	= false,
402	.ret_type	= RET_PTR_TO_MAP_VALUE_OR_NULL,
403	.arg1_type	= ARG_CONST_MAP_PTR,
404	.arg2_type	= ARG_PTR_TO_BTF_ID_OR_NULL,
405	.arg2_btf_id	= &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
406	.arg3_type	= ARG_PTR_TO_MAP_VALUE_OR_NULL,
407	.arg4_type	= ARG_ANYTHING,
408	.allowed	= bpf_sk_storage_tracing_allowed,
409};
410
411const struct bpf_func_proto bpf_sk_storage_delete_tracing_proto = {
412	.func		= bpf_sk_storage_delete_tracing,
413	.gpl_only	= false,
414	.ret_type	= RET_INTEGER,
415	.arg1_type	= ARG_CONST_MAP_PTR,
416	.arg2_type	= ARG_PTR_TO_BTF_ID_OR_NULL,
417	.arg2_btf_id	= &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
418	.allowed	= bpf_sk_storage_tracing_allowed,
419};
420
421struct bpf_sk_storage_diag {
422	u32 nr_maps;
423	struct bpf_map *maps[];
424};
425
426/* The reply will be like:
427 * INET_DIAG_BPF_SK_STORAGES (nla_nest)
428 *	SK_DIAG_BPF_STORAGE (nla_nest)
429 *		SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32)
430 *		SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit)
431 *	SK_DIAG_BPF_STORAGE (nla_nest)
432 *		SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32)
433 *		SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit)
434 *	....
435 */
436static int nla_value_size(u32 value_size)
437{
438	/* SK_DIAG_BPF_STORAGE (nla_nest)
439	 *	SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32)
440	 *	SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit)
441	 */
442	return nla_total_size(0) + nla_total_size(sizeof(u32)) +
443		nla_total_size_64bit(value_size);
444}
445
446void bpf_sk_storage_diag_free(struct bpf_sk_storage_diag *diag)
447{
448	u32 i;
449
450	if (!diag)
451		return;
452
453	for (i = 0; i < diag->nr_maps; i++)
454		bpf_map_put(diag->maps[i]);
455
456	kfree(diag);
457}
458EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_free);
459
460static bool diag_check_dup(const struct bpf_sk_storage_diag *diag,
461			   const struct bpf_map *map)
462{
463	u32 i;
464
465	for (i = 0; i < diag->nr_maps; i++) {
466		if (diag->maps[i] == map)
467			return true;
468	}
469
470	return false;
471}
472
473struct bpf_sk_storage_diag *
474bpf_sk_storage_diag_alloc(const struct nlattr *nla_stgs)
475{
476	struct bpf_sk_storage_diag *diag;
477	struct nlattr *nla;
478	u32 nr_maps = 0;
479	int rem, err;
480
481	/* bpf_local_storage_map is currently limited to CAP_SYS_ADMIN as
482	 * the map_alloc_check() side also does.
483	 */
484	if (!bpf_capable())
485		return ERR_PTR(-EPERM);
486
487	nla_for_each_nested_type(nla, SK_DIAG_BPF_STORAGE_REQ_MAP_FD,
488				 nla_stgs, rem) {
489		if (nla_len(nla) != sizeof(u32))
490			return ERR_PTR(-EINVAL);
491		nr_maps++;
492	}
493
494	diag = kzalloc(struct_size(diag, maps, nr_maps), GFP_KERNEL);
495	if (!diag)
496		return ERR_PTR(-ENOMEM);
497
498	nla_for_each_nested_type(nla, SK_DIAG_BPF_STORAGE_REQ_MAP_FD,
499				 nla_stgs, rem) {
500		int map_fd = nla_get_u32(nla);
501		struct bpf_map *map = bpf_map_get(map_fd);
502
503		if (IS_ERR(map)) {
504			err = PTR_ERR(map);
505			goto err_free;
506		}
507		if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) {
508			bpf_map_put(map);
509			err = -EINVAL;
510			goto err_free;
511		}
512		if (diag_check_dup(diag, map)) {
513			bpf_map_put(map);
514			err = -EEXIST;
515			goto err_free;
516		}
517		diag->maps[diag->nr_maps++] = map;
518	}
519
520	return diag;
521
522err_free:
523	bpf_sk_storage_diag_free(diag);
524	return ERR_PTR(err);
525}
526EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_alloc);
527
528static int diag_get(struct bpf_local_storage_data *sdata, struct sk_buff *skb)
529{
530	struct nlattr *nla_stg, *nla_value;
531	struct bpf_local_storage_map *smap;
532
533	/* It cannot exceed max nlattr's payload */
534	BUILD_BUG_ON(U16_MAX - NLA_HDRLEN < BPF_LOCAL_STORAGE_MAX_VALUE_SIZE);
535
536	nla_stg = nla_nest_start(skb, SK_DIAG_BPF_STORAGE);
537	if (!nla_stg)
538		return -EMSGSIZE;
539
540	smap = rcu_dereference(sdata->smap);
541	if (nla_put_u32(skb, SK_DIAG_BPF_STORAGE_MAP_ID, smap->map.id))
542		goto errout;
543
544	nla_value = nla_reserve_64bit(skb, SK_DIAG_BPF_STORAGE_MAP_VALUE,
545				      smap->map.value_size,
546				      SK_DIAG_BPF_STORAGE_PAD);
547	if (!nla_value)
548		goto errout;
549
550	if (btf_record_has_field(smap->map.record, BPF_SPIN_LOCK))
551		copy_map_value_locked(&smap->map, nla_data(nla_value),
552				      sdata->data, true);
553	else
554		copy_map_value(&smap->map, nla_data(nla_value), sdata->data);
555
556	nla_nest_end(skb, nla_stg);
557	return 0;
558
559errout:
560	nla_nest_cancel(skb, nla_stg);
561	return -EMSGSIZE;
562}
563
564static int bpf_sk_storage_diag_put_all(struct sock *sk, struct sk_buff *skb,
565				       int stg_array_type,
566				       unsigned int *res_diag_size)
567{
568	/* stg_array_type (e.g. INET_DIAG_BPF_SK_STORAGES) */
569	unsigned int diag_size = nla_total_size(0);
570	struct bpf_local_storage *sk_storage;
571	struct bpf_local_storage_elem *selem;
572	struct bpf_local_storage_map *smap;
573	struct nlattr *nla_stgs;
574	unsigned int saved_len;
575	int err = 0;
576
577	rcu_read_lock();
578
579	sk_storage = rcu_dereference(sk->sk_bpf_storage);
580	if (!sk_storage || hlist_empty(&sk_storage->list)) {
581		rcu_read_unlock();
582		return 0;
583	}
584
585	nla_stgs = nla_nest_start(skb, stg_array_type);
586	if (!nla_stgs)
587		/* Continue to learn diag_size */
588		err = -EMSGSIZE;
589
590	saved_len = skb->len;
591	hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) {
592		smap = rcu_dereference(SDATA(selem)->smap);
593		diag_size += nla_value_size(smap->map.value_size);
594
595		if (nla_stgs && diag_get(SDATA(selem), skb))
596			/* Continue to learn diag_size */
597			err = -EMSGSIZE;
598	}
599
600	rcu_read_unlock();
601
602	if (nla_stgs) {
603		if (saved_len == skb->len)
604			nla_nest_cancel(skb, nla_stgs);
605		else
606			nla_nest_end(skb, nla_stgs);
607	}
608
609	if (diag_size == nla_total_size(0)) {
610		*res_diag_size = 0;
611		return 0;
612	}
613
614	*res_diag_size = diag_size;
615	return err;
616}
617
618int bpf_sk_storage_diag_put(struct bpf_sk_storage_diag *diag,
619			    struct sock *sk, struct sk_buff *skb,
620			    int stg_array_type,
621			    unsigned int *res_diag_size)
622{
623	/* stg_array_type (e.g. INET_DIAG_BPF_SK_STORAGES) */
624	unsigned int diag_size = nla_total_size(0);
625	struct bpf_local_storage *sk_storage;
626	struct bpf_local_storage_data *sdata;
627	struct nlattr *nla_stgs;
628	unsigned int saved_len;
629	int err = 0;
630	u32 i;
631
632	*res_diag_size = 0;
633
634	/* No map has been specified.  Dump all. */
635	if (!diag->nr_maps)
636		return bpf_sk_storage_diag_put_all(sk, skb, stg_array_type,
637						   res_diag_size);
638
639	rcu_read_lock();
640	sk_storage = rcu_dereference(sk->sk_bpf_storage);
641	if (!sk_storage || hlist_empty(&sk_storage->list)) {
642		rcu_read_unlock();
643		return 0;
644	}
645
646	nla_stgs = nla_nest_start(skb, stg_array_type);
647	if (!nla_stgs)
648		/* Continue to learn diag_size */
649		err = -EMSGSIZE;
650
651	saved_len = skb->len;
652	for (i = 0; i < diag->nr_maps; i++) {
653		sdata = bpf_local_storage_lookup(sk_storage,
654				(struct bpf_local_storage_map *)diag->maps[i],
655				false);
656
657		if (!sdata)
658			continue;
659
660		diag_size += nla_value_size(diag->maps[i]->value_size);
661
662		if (nla_stgs && diag_get(sdata, skb))
663			/* Continue to learn diag_size */
664			err = -EMSGSIZE;
665	}
666	rcu_read_unlock();
667
668	if (nla_stgs) {
669		if (saved_len == skb->len)
670			nla_nest_cancel(skb, nla_stgs);
671		else
672			nla_nest_end(skb, nla_stgs);
673	}
674
675	if (diag_size == nla_total_size(0)) {
676		*res_diag_size = 0;
677		return 0;
678	}
679
680	*res_diag_size = diag_size;
681	return err;
682}
683EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_put);
684
685struct bpf_iter_seq_sk_storage_map_info {
686	struct bpf_map *map;
687	unsigned int bucket_id;
688	unsigned skip_elems;
689};
690
691static struct bpf_local_storage_elem *
692bpf_sk_storage_map_seq_find_next(struct bpf_iter_seq_sk_storage_map_info *info,
693				 struct bpf_local_storage_elem *prev_selem)
694	__acquires(RCU) __releases(RCU)
695{
696	struct bpf_local_storage *sk_storage;
697	struct bpf_local_storage_elem *selem;
698	u32 skip_elems = info->skip_elems;
699	struct bpf_local_storage_map *smap;
700	u32 bucket_id = info->bucket_id;
701	u32 i, count, n_buckets;
702	struct bpf_local_storage_map_bucket *b;
703
704	smap = (struct bpf_local_storage_map *)info->map;
705	n_buckets = 1U << smap->bucket_log;
706	if (bucket_id >= n_buckets)
707		return NULL;
708
709	/* try to find next selem in the same bucket */
710	selem = prev_selem;
711	count = 0;
712	while (selem) {
713		selem = hlist_entry_safe(rcu_dereference(hlist_next_rcu(&selem->map_node)),
714					 struct bpf_local_storage_elem, map_node);
715		if (!selem) {
716			/* not found, unlock and go to the next bucket */
717			b = &smap->buckets[bucket_id++];
718			rcu_read_unlock();
719			skip_elems = 0;
720			break;
721		}
722		sk_storage = rcu_dereference(selem->local_storage);
723		if (sk_storage) {
724			info->skip_elems = skip_elems + count;
725			return selem;
726		}
727		count++;
728	}
729
730	for (i = bucket_id; i < (1U << smap->bucket_log); i++) {
731		b = &smap->buckets[i];
732		rcu_read_lock();
733		count = 0;
734		hlist_for_each_entry_rcu(selem, &b->list, map_node) {
735			sk_storage = rcu_dereference(selem->local_storage);
736			if (sk_storage && count >= skip_elems) {
737				info->bucket_id = i;
738				info->skip_elems = count;
739				return selem;
740			}
741			count++;
742		}
743		rcu_read_unlock();
744		skip_elems = 0;
745	}
746
747	info->bucket_id = i;
748	info->skip_elems = 0;
749	return NULL;
750}
751
752static void *bpf_sk_storage_map_seq_start(struct seq_file *seq, loff_t *pos)
753{
754	struct bpf_local_storage_elem *selem;
755
756	selem = bpf_sk_storage_map_seq_find_next(seq->private, NULL);
757	if (!selem)
758		return NULL;
759
760	if (*pos == 0)
761		++*pos;
762	return selem;
763}
764
765static void *bpf_sk_storage_map_seq_next(struct seq_file *seq, void *v,
766					 loff_t *pos)
767{
768	struct bpf_iter_seq_sk_storage_map_info *info = seq->private;
769
770	++*pos;
771	++info->skip_elems;
772	return bpf_sk_storage_map_seq_find_next(seq->private, v);
773}
774
775struct bpf_iter__bpf_sk_storage_map {
776	__bpf_md_ptr(struct bpf_iter_meta *, meta);
777	__bpf_md_ptr(struct bpf_map *, map);
778	__bpf_md_ptr(struct sock *, sk);
779	__bpf_md_ptr(void *, value);
780};
781
782DEFINE_BPF_ITER_FUNC(bpf_sk_storage_map, struct bpf_iter_meta *meta,
783		     struct bpf_map *map, struct sock *sk,
784		     void *value)
785
786static int __bpf_sk_storage_map_seq_show(struct seq_file *seq,
787					 struct bpf_local_storage_elem *selem)
788{
789	struct bpf_iter_seq_sk_storage_map_info *info = seq->private;
790	struct bpf_iter__bpf_sk_storage_map ctx = {};
791	struct bpf_local_storage *sk_storage;
792	struct bpf_iter_meta meta;
793	struct bpf_prog *prog;
794	int ret = 0;
795
796	meta.seq = seq;
797	prog = bpf_iter_get_info(&meta, selem == NULL);
798	if (prog) {
799		ctx.meta = &meta;
800		ctx.map = info->map;
801		if (selem) {
802			sk_storage = rcu_dereference(selem->local_storage);
803			ctx.sk = sk_storage->owner;
804			ctx.value = SDATA(selem)->data;
805		}
806		ret = bpf_iter_run_prog(prog, &ctx);
807	}
808
809	return ret;
810}
811
812static int bpf_sk_storage_map_seq_show(struct seq_file *seq, void *v)
813{
814	return __bpf_sk_storage_map_seq_show(seq, v);
815}
816
817static void bpf_sk_storage_map_seq_stop(struct seq_file *seq, void *v)
818	__releases(RCU)
819{
820	if (!v)
821		(void)__bpf_sk_storage_map_seq_show(seq, v);
822	else
823		rcu_read_unlock();
824}
825
826static int bpf_iter_init_sk_storage_map(void *priv_data,
827					struct bpf_iter_aux_info *aux)
828{
829	struct bpf_iter_seq_sk_storage_map_info *seq_info = priv_data;
830
831	bpf_map_inc_with_uref(aux->map);
832	seq_info->map = aux->map;
833	return 0;
834}
835
836static void bpf_iter_fini_sk_storage_map(void *priv_data)
837{
838	struct bpf_iter_seq_sk_storage_map_info *seq_info = priv_data;
839
840	bpf_map_put_with_uref(seq_info->map);
841}
842
843static int bpf_iter_attach_map(struct bpf_prog *prog,
844			       union bpf_iter_link_info *linfo,
845			       struct bpf_iter_aux_info *aux)
846{
847	struct bpf_map *map;
848	int err = -EINVAL;
849
850	if (!linfo->map.map_fd)
851		return -EBADF;
852
853	map = bpf_map_get_with_uref(linfo->map.map_fd);
854	if (IS_ERR(map))
855		return PTR_ERR(map);
856
857	if (map->map_type != BPF_MAP_TYPE_SK_STORAGE)
858		goto put_map;
859
860	if (prog->aux->max_rdwr_access > map->value_size) {
861		err = -EACCES;
862		goto put_map;
863	}
864
865	aux->map = map;
866	return 0;
867
868put_map:
869	bpf_map_put_with_uref(map);
870	return err;
871}
872
873static void bpf_iter_detach_map(struct bpf_iter_aux_info *aux)
874{
875	bpf_map_put_with_uref(aux->map);
876}
877
878static const struct seq_operations bpf_sk_storage_map_seq_ops = {
879	.start  = bpf_sk_storage_map_seq_start,
880	.next   = bpf_sk_storage_map_seq_next,
881	.stop   = bpf_sk_storage_map_seq_stop,
882	.show   = bpf_sk_storage_map_seq_show,
883};
884
885static const struct bpf_iter_seq_info iter_seq_info = {
886	.seq_ops		= &bpf_sk_storage_map_seq_ops,
887	.init_seq_private	= bpf_iter_init_sk_storage_map,
888	.fini_seq_private	= bpf_iter_fini_sk_storage_map,
889	.seq_priv_size		= sizeof(struct bpf_iter_seq_sk_storage_map_info),
890};
891
892static struct bpf_iter_reg bpf_sk_storage_map_reg_info = {
893	.target			= "bpf_sk_storage_map",
894	.attach_target		= bpf_iter_attach_map,
895	.detach_target		= bpf_iter_detach_map,
896	.show_fdinfo		= bpf_iter_map_show_fdinfo,
897	.fill_link_info		= bpf_iter_map_fill_link_info,
898	.ctx_arg_info_size	= 2,
899	.ctx_arg_info		= {
900		{ offsetof(struct bpf_iter__bpf_sk_storage_map, sk),
901		  PTR_TO_BTF_ID_OR_NULL },
902		{ offsetof(struct bpf_iter__bpf_sk_storage_map, value),
903		  PTR_TO_BUF | PTR_MAYBE_NULL },
904	},
905	.seq_info		= &iter_seq_info,
906};
907
908static int __init bpf_sk_storage_map_iter_init(void)
909{
910	bpf_sk_storage_map_reg_info.ctx_arg_info[0].btf_id =
911		btf_sock_ids[BTF_SOCK_TYPE_SOCK];
912	return bpf_iter_reg_target(&bpf_sk_storage_map_reg_info);
913}
914late_initcall(bpf_sk_storage_map_iter_init);