offload.c

  1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
  2/* Copyright (C) 2016-2018 Netronome Systems, Inc. */
  3
  4/*
  5 * nfp_net_offload.c
  6 * Netronome network device driver: TC offload functions for PF and VF
  7 */
  8
  9#define pr_fmt(fmt)	"NFP net bpf: " fmt
 10
 11#include <linux/bpf.h>
 12#include <linux/kernel.h>
 13#include <linux/netdevice.h>
 14#include <linux/pci.h>
 15#include <linux/jiffies.h>
 16#include <linux/timer.h>
 17#include <linux/list.h>
 18#include <linux/mm.h>
 19
 20#include <net/pkt_cls.h>
 21#include <net/tc_act/tc_gact.h>
 22#include <net/tc_act/tc_mirred.h>
 23
 24#include "main.h"
 25#include "../ccm.h"
 26#include "../nfp_app.h"
 27#include "../nfp_net_ctrl.h"
 28#include "../nfp_net.h"
 29
 30static int
 31nfp_map_ptr_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog,
 32		   struct bpf_map *map)
 33{
 34	struct nfp_bpf_neutral_map *record;
 35	int err;
 36
 37	/* Reuse path - other offloaded program is already tracking this map. */
 38	record = rhashtable_lookup_fast(&bpf->maps_neutral, &map->id,
 39					nfp_bpf_maps_neutral_params);
 40	if (record) {
 41		nfp_prog->map_records[nfp_prog->map_records_cnt++] = record;
 42		record->count++;
 43		return 0;
 44	}
 45
 46	/* Grab a single ref to the map for our record.  The prog destroy ndo
 47	 * happens after free_used_maps().
 48	 */
 49	bpf_map_inc(map);
 50
 51	record = kmalloc(sizeof(*record), GFP_KERNEL);
 52	if (!record) {
 53		err = -ENOMEM;
 54		goto err_map_put;
 55	}
 56
 57	record->ptr = map;
 58	record->map_id = map->id;
 59	record->count = 1;
 60
 61	err = rhashtable_insert_fast(&bpf->maps_neutral, &record->l,
 62				     nfp_bpf_maps_neutral_params);
 63	if (err)
 64		goto err_free_rec;
 65
 66	nfp_prog->map_records[nfp_prog->map_records_cnt++] = record;
 67
 68	return 0;
 69
 70err_free_rec:
 71	kfree(record);
 72err_map_put:
 73	bpf_map_put(map);
 74	return err;
 75}
 76
 77static void
 78nfp_map_ptrs_forget(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog)
 79{
 80	bool freed = false;
 81	int i;
 82
 83	for (i = 0; i < nfp_prog->map_records_cnt; i++) {
 84		if (--nfp_prog->map_records[i]->count) {
 85			nfp_prog->map_records[i] = NULL;
 86			continue;
 87		}
 88
 89		WARN_ON(rhashtable_remove_fast(&bpf->maps_neutral,
 90					       &nfp_prog->map_records[i]->l,
 91					       nfp_bpf_maps_neutral_params));
 92		freed = true;
 93	}
 94
 95	if (freed) {
 96		synchronize_rcu();
 97
 98		for (i = 0; i < nfp_prog->map_records_cnt; i++)
 99			if (nfp_prog->map_records[i]) {
100				bpf_map_put(nfp_prog->map_records[i]->ptr);
101				kfree(nfp_prog->map_records[i]);
102			}
103	}
104
105	kfree(nfp_prog->map_records);
106	nfp_prog->map_records = NULL;
107	nfp_prog->map_records_cnt = 0;
108}
109
110static int
111nfp_map_ptrs_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog,
112		    struct bpf_prog *prog)
113{
114	int i, cnt, err = 0;
115
116	mutex_lock(&prog->aux->used_maps_mutex);
117
118	/* Quickly count the maps we will have to remember */
119	cnt = 0;
120	for (i = 0; i < prog->aux->used_map_cnt; i++)
121		if (bpf_map_offload_neutral(prog->aux->used_maps[i]))
122			cnt++;
123	if (!cnt)
124		goto out;
125
126	nfp_prog->map_records = kmalloc_array(cnt,
127					      sizeof(nfp_prog->map_records[0]),
128					      GFP_KERNEL);
129	if (!nfp_prog->map_records) {
130		err = -ENOMEM;
131		goto out;
132	}
133
134	for (i = 0; i < prog->aux->used_map_cnt; i++)
135		if (bpf_map_offload_neutral(prog->aux->used_maps[i])) {
136			err = nfp_map_ptr_record(bpf, nfp_prog,
137						 prog->aux->used_maps[i]);
138			if (err) {
139				nfp_map_ptrs_forget(bpf, nfp_prog);
140				goto out;
141			}
142		}
143	WARN_ON(cnt != nfp_prog->map_records_cnt);
144
145out:
146	mutex_unlock(&prog->aux->used_maps_mutex);
147	return err;
148}
149
150static int
151nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog,
152		 unsigned int cnt)
153{
154	struct nfp_insn_meta *meta;
155	unsigned int i;
156
157	for (i = 0; i < cnt; i++) {
158		meta = kzalloc(sizeof(*meta), GFP_KERNEL);
159		if (!meta)
160			return -ENOMEM;
161
162		meta->insn = prog[i];
163		meta->n = i;
164		if (is_mbpf_alu(meta)) {
165			meta->umin_src = U64_MAX;
166			meta->umin_dst = U64_MAX;
167		}
168
169		list_add_tail(&meta->l, &nfp_prog->insns);
170	}
171	nfp_prog->n_insns = cnt;
172
173	nfp_bpf_jit_prepare(nfp_prog);
174
175	return 0;
176}
177
178static void nfp_prog_free(struct nfp_prog *nfp_prog)
179{
180	struct nfp_insn_meta *meta, *tmp;
181
182	kfree(nfp_prog->subprog);
183
184	list_for_each_entry_safe(meta, tmp, &nfp_prog->insns, l) {
185		list_del(&meta->l);
186		kfree(meta);
187	}
188	kfree(nfp_prog);
189}
190
191static int nfp_bpf_verifier_prep(struct bpf_prog *prog)
192{
193	struct nfp_prog *nfp_prog;
194	int ret;
195
196	nfp_prog = kzalloc(sizeof(*nfp_prog), GFP_KERNEL);
197	if (!nfp_prog)
198		return -ENOMEM;
199	prog->aux->offload->dev_priv = nfp_prog;
200
201	INIT_LIST_HEAD(&nfp_prog->insns);
202	nfp_prog->type = prog->type;
203	nfp_prog->bpf = bpf_offload_dev_priv(prog->aux->offload->offdev);
204
205	ret = nfp_prog_prepare(nfp_prog, prog->insnsi, prog->len);
206	if (ret)
207		goto err_free;
208
209	nfp_prog->verifier_meta = nfp_prog_first_meta(nfp_prog);
210
211	return 0;
212
213err_free:
214	nfp_prog_free(nfp_prog);
215
216	return ret;
217}
218
219static int nfp_bpf_translate(struct bpf_prog *prog)
220{
221	struct nfp_net *nn = netdev_priv(prog->aux->offload->netdev);
222	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
223	unsigned int max_instr;
224	int err;
225
226	/* We depend on dead code elimination succeeding */
227	if (prog->aux->offload->opt_failed)
228		return -EINVAL;
229
230	max_instr = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN);
231	nfp_prog->__prog_alloc_len = max_instr * sizeof(u64);
232
233	nfp_prog->prog = kvmalloc(nfp_prog->__prog_alloc_len, GFP_KERNEL);
234	if (!nfp_prog->prog)
235		return -ENOMEM;
236
237	err = nfp_bpf_jit(nfp_prog);
238	if (err)
239		return err;
240
241	prog->aux->offload->jited_len = nfp_prog->prog_len * sizeof(u64);
242	prog->aux->offload->jited_image = nfp_prog->prog;
243
244	return nfp_map_ptrs_record(nfp_prog->bpf, nfp_prog, prog);
245}
246
247static void nfp_bpf_destroy(struct bpf_prog *prog)
248{
249	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
250
251	kvfree(nfp_prog->prog);
252	nfp_map_ptrs_forget(nfp_prog->bpf, nfp_prog);
253	nfp_prog_free(nfp_prog);
254}
255
256/* Atomic engine requires values to be in big endian, we need to byte swap
257 * the value words used with xadd.
258 */
259static void nfp_map_bpf_byte_swap(struct nfp_bpf_map *nfp_map, void *value)
260{
261	u32 *word = value;
262	unsigned int i;
263
264	for (i = 0; i < DIV_ROUND_UP(nfp_map->offmap->map.value_size, 4); i++)
265		if (nfp_map->use_map[i].type == NFP_MAP_USE_ATOMIC_CNT)
266			word[i] = (__force u32)cpu_to_be32(word[i]);
267}
268
269/* Mark value as unsafely initialized in case it becomes atomic later
270 * and we didn't byte swap something non-byte swap neutral.
271 */
272static void
273nfp_map_bpf_byte_swap_record(struct nfp_bpf_map *nfp_map, void *value)
274{
275	u32 *word = value;
276	unsigned int i;
277
278	for (i = 0; i < DIV_ROUND_UP(nfp_map->offmap->map.value_size, 4); i++)
279		if (nfp_map->use_map[i].type == NFP_MAP_UNUSED &&
280		    word[i] != (__force u32)cpu_to_be32(word[i]))
281			nfp_map->use_map[i].non_zero_update = 1;
282}
283
284static int
285nfp_bpf_map_lookup_entry(struct bpf_offloaded_map *offmap,
286			 void *key, void *value)
287{
288	int err;
289
290	err = nfp_bpf_ctrl_lookup_entry(offmap, key, value);
291	if (err)
292		return err;
293
294	nfp_map_bpf_byte_swap(offmap->dev_priv, value);
295	return 0;
296}
297
298static int
299nfp_bpf_map_update_entry(struct bpf_offloaded_map *offmap,
300			 void *key, void *value, u64 flags)
301{
302	nfp_map_bpf_byte_swap(offmap->dev_priv, value);
303	nfp_map_bpf_byte_swap_record(offmap->dev_priv, value);
304	return nfp_bpf_ctrl_update_entry(offmap, key, value, flags);
305}
306
307static int
308nfp_bpf_map_get_next_key(struct bpf_offloaded_map *offmap,
309			 void *key, void *next_key)
310{
311	if (!key)
312		return nfp_bpf_ctrl_getfirst_entry(offmap, next_key);
313	return nfp_bpf_ctrl_getnext_entry(offmap, key, next_key);
314}
315
316static int
317nfp_bpf_map_delete_elem(struct bpf_offloaded_map *offmap, void *key)
318{
319	if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY)
320		return -EINVAL;
321	return nfp_bpf_ctrl_del_entry(offmap, key);
322}
323
324static const struct bpf_map_dev_ops nfp_bpf_map_ops = {
325	.map_get_next_key	= nfp_bpf_map_get_next_key,
326	.map_lookup_elem	= nfp_bpf_map_lookup_entry,
327	.map_update_elem	= nfp_bpf_map_update_entry,
328	.map_delete_elem	= nfp_bpf_map_delete_elem,
329};
330
331static int
332nfp_bpf_map_alloc(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap)
333{
334	struct nfp_bpf_map *nfp_map;
335	unsigned int use_map_size;
336	long long int res;
337
338	if (!bpf->maps.types)
339		return -EOPNOTSUPP;
340
341	if (offmap->map.map_flags ||
342	    offmap->map.numa_node != NUMA_NO_NODE) {
343		pr_info("map flags are not supported\n");
344		return -EINVAL;
345	}
346
347	if (!(bpf->maps.types & 1 << offmap->map.map_type)) {
348		pr_info("map type not supported\n");
349		return -EOPNOTSUPP;
350	}
351	if (bpf->maps.max_maps == bpf->maps_in_use) {
352		pr_info("too many maps for a device\n");
353		return -ENOMEM;
354	}
355	if (bpf->maps.max_elems - bpf->map_elems_in_use <
356	    offmap->map.max_entries) {
357		pr_info("map with too many elements: %u, left: %u\n",
358			offmap->map.max_entries,
359			bpf->maps.max_elems - bpf->map_elems_in_use);
360		return -ENOMEM;
361	}
362
363	if (round_up(offmap->map.key_size, 8) +
364	    round_up(offmap->map.value_size, 8) > bpf->maps.max_elem_sz) {
365		pr_info("map elements too large: %u, FW max element size (key+value): %u\n",
366			round_up(offmap->map.key_size, 8) +
367			round_up(offmap->map.value_size, 8),
368			bpf->maps.max_elem_sz);
369		return -ENOMEM;
370	}
371	if (offmap->map.key_size > bpf->maps.max_key_sz) {
372		pr_info("map key size %u, FW max is %u\n",
373			offmap->map.key_size, bpf->maps.max_key_sz);
374		return -ENOMEM;
375	}
376	if (offmap->map.value_size > bpf->maps.max_val_sz) {
377		pr_info("map value size %u, FW max is %u\n",
378			offmap->map.value_size, bpf->maps.max_val_sz);
379		return -ENOMEM;
380	}
381
382	use_map_size = DIV_ROUND_UP(offmap->map.value_size, 4) *
383		       sizeof_field(struct nfp_bpf_map, use_map[0]);
384
385	nfp_map = kzalloc(sizeof(*nfp_map) + use_map_size, GFP_USER);
386	if (!nfp_map)
387		return -ENOMEM;
388
389	offmap->dev_priv = nfp_map;
390	nfp_map->offmap = offmap;
391	nfp_map->bpf = bpf;
392	spin_lock_init(&nfp_map->cache_lock);
393
394	res = nfp_bpf_ctrl_alloc_map(bpf, &offmap->map);
395	if (res < 0) {
396		kfree(nfp_map);
397		return res;
398	}
399
400	nfp_map->tid = res;
401	offmap->dev_ops = &nfp_bpf_map_ops;
402	bpf->maps_in_use++;
403	bpf->map_elems_in_use += offmap->map.max_entries;
404	list_add_tail(&nfp_map->l, &bpf->map_list);
405
406	return 0;
407}
408
409static int
410nfp_bpf_map_free(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap)
411{
412	struct nfp_bpf_map *nfp_map = offmap->dev_priv;
413
414	nfp_bpf_ctrl_free_map(bpf, nfp_map);
415	dev_consume_skb_any(nfp_map->cache);
416	WARN_ON_ONCE(nfp_map->cache_blockers);
417	list_del_init(&nfp_map->l);
418	bpf->map_elems_in_use -= offmap->map.max_entries;
419	bpf->maps_in_use--;
420	kfree(nfp_map);
421
422	return 0;
423}
424
425int nfp_ndo_bpf(struct nfp_app *app, struct nfp_net *nn, struct netdev_bpf *bpf)
426{
427	switch (bpf->command) {
428	case BPF_OFFLOAD_MAP_ALLOC:
429		return nfp_bpf_map_alloc(app->priv, bpf->offmap);
430	case BPF_OFFLOAD_MAP_FREE:
431		return nfp_bpf_map_free(app->priv, bpf->offmap);
432	default:
433		return -EINVAL;
434	}
435}
436
437static unsigned long
438nfp_bpf_perf_event_copy(void *dst, const void *src,
439			unsigned long off, unsigned long len)
440{
441	memcpy(dst, src + off, len);
442	return 0;
443}
444
445int nfp_bpf_event_output(struct nfp_app_bpf *bpf, const void *data,
446			 unsigned int len)
447{
448	struct cmsg_bpf_event *cbe = (void *)data;
449	struct nfp_bpf_neutral_map *record;
450	u32 pkt_size, data_size, map_id;
451	u64 map_id_full;
452
453	if (len < sizeof(struct cmsg_bpf_event))
454		return -EINVAL;
455
456	pkt_size = be32_to_cpu(cbe->pkt_size);
457	data_size = be32_to_cpu(cbe->data_size);
458	map_id_full = be64_to_cpu(cbe->map_ptr);
459	map_id = map_id_full;
460
461	if (size_add(pkt_size, data_size) > INT_MAX ||
462	    len < sizeof(struct cmsg_bpf_event) + pkt_size + data_size)
463		return -EINVAL;
464	if (cbe->hdr.ver != NFP_CCM_ABI_VERSION)
465		return -EINVAL;
466
467	rcu_read_lock();
468	record = rhashtable_lookup(&bpf->maps_neutral, &map_id,
469				   nfp_bpf_maps_neutral_params);
470	if (!record || map_id_full > U32_MAX) {
471		rcu_read_unlock();
472		cmsg_warn(bpf, "perf event: map id %lld (0x%llx) not recognized, dropping event\n",
473			  map_id_full, map_id_full);
474		return -EINVAL;
475	}
476
477	bpf_event_output(record->ptr, be32_to_cpu(cbe->cpu_id),
478			 &cbe->data[round_up(pkt_size, 4)], data_size,
479			 cbe->data, pkt_size, nfp_bpf_perf_event_copy);
480	rcu_read_unlock();
481
482	return 0;
483}
484
485bool nfp_bpf_offload_check_mtu(struct nfp_net *nn, struct bpf_prog *prog,
486			       unsigned int mtu)
487{
488	unsigned int fw_mtu, pkt_off;
489
490	fw_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32;
491	pkt_off = min(prog->aux->max_pkt_offset, mtu);
492
493	return fw_mtu < pkt_off;
494}
495
496static int
497nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog,
498		 struct netlink_ext_ack *extack)
499{
500	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
501	unsigned int max_stack, max_prog_len;
502	dma_addr_t dma_addr;
503	void *img;
504	int err;
505
506	if (nfp_bpf_offload_check_mtu(nn, prog, nn->dp.netdev->mtu)) {
507		NL_SET_ERR_MSG_MOD(extack, "BPF offload not supported with potential packet access beyond HW packet split boundary");
508		return -EOPNOTSUPP;
509	}
510
511	max_stack = nn_readb(nn, NFP_NET_CFG_BPF_STACK_SZ) * 64;
512	if (nfp_prog->stack_size > max_stack) {
513		NL_SET_ERR_MSG_MOD(extack, "stack too large");
514		return -EOPNOTSUPP;
515	}
516
517	max_prog_len = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN);
518	if (nfp_prog->prog_len > max_prog_len) {
519		NL_SET_ERR_MSG_MOD(extack, "program too long");
520		return -EOPNOTSUPP;
521	}
522
523	img = nfp_bpf_relo_for_vnic(nfp_prog, nn->app_priv);
524	if (IS_ERR(img))
525		return PTR_ERR(img);
526
527	dma_addr = dma_map_single(nn->dp.dev, img,
528				  nfp_prog->prog_len * sizeof(u64),
529				  DMA_TO_DEVICE);
530	if (dma_mapping_error(nn->dp.dev, dma_addr)) {
531		kfree(img);
532		return -ENOMEM;
533	}
534
535	nn_writew(nn, NFP_NET_CFG_BPF_SIZE, nfp_prog->prog_len);
536	nn_writeq(nn, NFP_NET_CFG_BPF_ADDR, dma_addr);
537
538	/* Load up the JITed code */
539	err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_BPF);
540	if (err)
541		NL_SET_ERR_MSG_MOD(extack,
542				   "FW command error while loading BPF");
543
544	dma_unmap_single(nn->dp.dev, dma_addr, nfp_prog->prog_len * sizeof(u64),
545			 DMA_TO_DEVICE);
546	kfree(img);
547
548	return err;
549}
550
551static void
552nfp_net_bpf_start(struct nfp_net *nn, struct netlink_ext_ack *extack)
553{
554	int err;
555
556	/* Enable passing packets through BPF function */
557	nn->dp.ctrl |= NFP_NET_CFG_CTRL_BPF;
558	nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
559	err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
560	if (err)
561		NL_SET_ERR_MSG_MOD(extack,
562				   "FW command error while enabling BPF");
563}
564
565static int nfp_net_bpf_stop(struct nfp_net *nn)
566{
567	if (!(nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF))
568		return 0;
569
570	nn->dp.ctrl &= ~NFP_NET_CFG_CTRL_BPF;
571	nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
572
573	return nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
574}
575
576int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog,
577			bool old_prog, struct netlink_ext_ack *extack)
578{
579	int err;
580
581	if (prog && !bpf_offload_dev_match(prog, nn->dp.netdev))
582		return -EINVAL;
583
584	if (prog && old_prog) {
585		u8 cap;
586
587		cap = nn_readb(nn, NFP_NET_CFG_BPF_CAP);
588		if (!(cap & NFP_NET_BPF_CAP_RELO)) {
589			NL_SET_ERR_MSG_MOD(extack,
590					   "FW does not support live reload");
591			return -EBUSY;
592		}
593	}
594
595	/* Something else is loaded, different program type? */
596	if (!old_prog && nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)
597		return -EBUSY;
598
599	if (old_prog && !prog)
600		return nfp_net_bpf_stop(nn);
601
602	err = nfp_net_bpf_load(nn, prog, extack);
603	if (err)
604		return err;
605
606	if (!old_prog)
607		nfp_net_bpf_start(nn, extack);
608
609	return 0;
610}
611
612const struct bpf_prog_offload_ops nfp_bpf_dev_ops = {
613	.insn_hook	= nfp_verify_insn,
614	.finalize	= nfp_bpf_finalize,
615	.replace_insn	= nfp_bpf_opt_replace_insn,
616	.remove_insns	= nfp_bpf_opt_remove_insns,
617	.prepare	= nfp_bpf_verifier_prep,
618	.translate	= nfp_bpf_translate,
619	.destroy	= nfp_bpf_destroy,
620};