bpf_cgrp_storage.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
  4 */
  5
  6#include <linux/types.h>
  7#include <linux/bpf.h>
  8#include <linux/bpf_local_storage.h>
  9#include <uapi/linux/btf.h>
 10#include <linux/btf_ids.h>
 11
 12DEFINE_BPF_STORAGE_CACHE(cgroup_cache);
 13
 14static DEFINE_PER_CPU(int, bpf_cgrp_storage_busy);
 15
 16static void bpf_cgrp_storage_lock(void)
 17{
 18	cant_migrate();
 19	this_cpu_inc(bpf_cgrp_storage_busy);
 20}
 21
 22static void bpf_cgrp_storage_unlock(void)
 23{
 24	this_cpu_dec(bpf_cgrp_storage_busy);
 25}
 26
 27static bool bpf_cgrp_storage_trylock(void)
 28{
 29	cant_migrate();
 30	if (unlikely(this_cpu_inc_return(bpf_cgrp_storage_busy) != 1)) {
 31		this_cpu_dec(bpf_cgrp_storage_busy);
 32		return false;
 33	}
 34	return true;
 35}
 36
 37static struct bpf_local_storage __rcu **cgroup_storage_ptr(void *owner)
 38{
 39	struct cgroup *cg = owner;
 40
 41	return &cg->bpf_cgrp_storage;
 42}
 43
 44void bpf_cgrp_storage_free(struct cgroup *cgroup)
 45{
 46	struct bpf_local_storage *local_storage;
 47
 48	rcu_read_lock_dont_migrate();
 49	local_storage = rcu_dereference(cgroup->bpf_cgrp_storage);
 50	if (!local_storage)
 51		goto out;
 52
 53	bpf_cgrp_storage_lock();
 54	bpf_local_storage_destroy(local_storage);
 55	bpf_cgrp_storage_unlock();
 56out:
 57	rcu_read_unlock_migrate();
 58}
 59
 60static struct bpf_local_storage_data *
 61cgroup_storage_lookup(struct cgroup *cgroup, struct bpf_map *map, bool cacheit_lockit)
 62{
 63	struct bpf_local_storage *cgroup_storage;
 64	struct bpf_local_storage_map *smap;
 65
 66	cgroup_storage = rcu_dereference_check(cgroup->bpf_cgrp_storage,
 67					       bpf_rcu_lock_held());
 68	if (!cgroup_storage)
 69		return NULL;
 70
 71	smap = (struct bpf_local_storage_map *)map;
 72	return bpf_local_storage_lookup(cgroup_storage, smap, cacheit_lockit);
 73}
 74
 75static void *bpf_cgrp_storage_lookup_elem(struct bpf_map *map, void *key)
 76{
 77	struct bpf_local_storage_data *sdata;
 78	struct cgroup *cgroup;
 79	int fd;
 80
 81	fd = *(int *)key;
 82	cgroup = cgroup_v1v2_get_from_fd(fd);
 83	if (IS_ERR(cgroup))
 84		return ERR_CAST(cgroup);
 85
 86	bpf_cgrp_storage_lock();
 87	sdata = cgroup_storage_lookup(cgroup, map, true);
 88	bpf_cgrp_storage_unlock();
 89	cgroup_put(cgroup);
 90	return sdata ? sdata->data : NULL;
 91}
 92
 93static long bpf_cgrp_storage_update_elem(struct bpf_map *map, void *key,
 94					 void *value, u64 map_flags)
 95{
 96	struct bpf_local_storage_data *sdata;
 97	struct cgroup *cgroup;
 98	int fd;
 99
100	fd = *(int *)key;
101	cgroup = cgroup_v1v2_get_from_fd(fd);
102	if (IS_ERR(cgroup))
103		return PTR_ERR(cgroup);
104
105	bpf_cgrp_storage_lock();
106	sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,
107					 value, map_flags, false, GFP_ATOMIC);
108	bpf_cgrp_storage_unlock();
109	cgroup_put(cgroup);
110	return PTR_ERR_OR_ZERO(sdata);
111}
112
113static int cgroup_storage_delete(struct cgroup *cgroup, struct bpf_map *map)
114{
115	struct bpf_local_storage_data *sdata;
116
117	sdata = cgroup_storage_lookup(cgroup, map, false);
118	if (!sdata)
119		return -ENOENT;
120
121	bpf_selem_unlink(SELEM(sdata), false);
122	return 0;
123}
124
125static long bpf_cgrp_storage_delete_elem(struct bpf_map *map, void *key)
126{
127	struct cgroup *cgroup;
128	int err, fd;
129
130	fd = *(int *)key;
131	cgroup = cgroup_v1v2_get_from_fd(fd);
132	if (IS_ERR(cgroup))
133		return PTR_ERR(cgroup);
134
135	bpf_cgrp_storage_lock();
136	err = cgroup_storage_delete(cgroup, map);
137	bpf_cgrp_storage_unlock();
138	cgroup_put(cgroup);
139	return err;
140}
141
142static int notsupp_get_next_key(struct bpf_map *map, void *key, void *next_key)
143{
144	return -ENOTSUPP;
145}
146
147static struct bpf_map *cgroup_storage_map_alloc(union bpf_attr *attr)
148{
149	return bpf_local_storage_map_alloc(attr, &cgroup_cache, true);
150}
151
152static void cgroup_storage_map_free(struct bpf_map *map)
153{
154	bpf_local_storage_map_free(map, &cgroup_cache, &bpf_cgrp_storage_busy);
155}
156
157/* *gfp_flags* is a hidden argument provided by the verifier */
158BPF_CALL_5(bpf_cgrp_storage_get, struct bpf_map *, map, struct cgroup *, cgroup,
159	   void *, value, u64, flags, gfp_t, gfp_flags)
160{
161	struct bpf_local_storage_data *sdata;
162	bool nobusy;
163
164	WARN_ON_ONCE(!bpf_rcu_lock_held());
165	if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
166		return (unsigned long)NULL;
167
168	if (!cgroup)
169		return (unsigned long)NULL;
170
171	nobusy = bpf_cgrp_storage_trylock();
172
173	sdata = cgroup_storage_lookup(cgroup, map, nobusy);
174	if (sdata)
175		goto unlock;
176
177	/* only allocate new storage, when the cgroup is refcounted */
178	if (!percpu_ref_is_dying(&cgroup->self.refcnt) &&
179	    (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) && nobusy)
180		sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,
181						 value, BPF_NOEXIST, false, gfp_flags);
182
183unlock:
184	if (nobusy)
185		bpf_cgrp_storage_unlock();
186	return IS_ERR_OR_NULL(sdata) ? (unsigned long)NULL : (unsigned long)sdata->data;
187}
188
189BPF_CALL_2(bpf_cgrp_storage_delete, struct bpf_map *, map, struct cgroup *, cgroup)
190{
191	int ret;
192
193	WARN_ON_ONCE(!bpf_rcu_lock_held());
194	if (!cgroup)
195		return -EINVAL;
196
197	if (!bpf_cgrp_storage_trylock())
198		return -EBUSY;
199
200	ret = cgroup_storage_delete(cgroup, map);
201	bpf_cgrp_storage_unlock();
202	return ret;
203}
204
205const struct bpf_map_ops cgrp_storage_map_ops = {
206	.map_meta_equal = bpf_map_meta_equal,
207	.map_alloc_check = bpf_local_storage_map_alloc_check,
208	.map_alloc = cgroup_storage_map_alloc,
209	.map_free = cgroup_storage_map_free,
210	.map_get_next_key = notsupp_get_next_key,
211	.map_lookup_elem = bpf_cgrp_storage_lookup_elem,
212	.map_update_elem = bpf_cgrp_storage_update_elem,
213	.map_delete_elem = bpf_cgrp_storage_delete_elem,
214	.map_check_btf = bpf_local_storage_map_check_btf,
215	.map_mem_usage = bpf_local_storage_map_mem_usage,
216	.map_btf_id = &bpf_local_storage_map_btf_id[0],
217	.map_owner_storage_ptr = cgroup_storage_ptr,
218};
219
220const struct bpf_func_proto bpf_cgrp_storage_get_proto = {
221	.func		= bpf_cgrp_storage_get,
222	.gpl_only	= false,
223	.ret_type	= RET_PTR_TO_MAP_VALUE_OR_NULL,
224	.arg1_type	= ARG_CONST_MAP_PTR,
225	.arg2_type	= ARG_PTR_TO_BTF_ID_OR_NULL,
226	.arg2_btf_id	= &bpf_cgroup_btf_id[0],
227	.arg3_type	= ARG_PTR_TO_MAP_VALUE_OR_NULL,
228	.arg4_type	= ARG_ANYTHING,
229};
230
231const struct bpf_func_proto bpf_cgrp_storage_delete_proto = {
232	.func		= bpf_cgrp_storage_delete,
233	.gpl_only	= false,
234	.ret_type	= RET_INTEGER,
235	.arg1_type	= ARG_CONST_MAP_PTR,
236	.arg2_type	= ARG_PTR_TO_BTF_ID_OR_NULL,
237	.arg2_btf_id	= &bpf_cgroup_btf_id[0],
238};