ktls_rx.c

  1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
  2// Copyright (c) 2019 Mellanox Technologies.
  3
  4#include <net/inet6_hashtables.h>
  5#include "en_accel/en_accel.h"
  6#include "en_accel/ktls.h"
  7#include "en_accel/ktls_txrx.h"
  8#include "en_accel/ktls_utils.h"
  9#include "en_accel/fs_tcp.h"
 10
 11struct accel_rule {
 12	struct work_struct work;
 13	struct mlx5e_priv *priv;
 14	struct mlx5_flow_handle *rule;
 15};
 16
 17#define PROGRESS_PARAMS_WRITE_UNIT	64
 18#define PROGRESS_PARAMS_PADDED_SIZE	\
 19		(ALIGN(sizeof(struct mlx5_wqe_tls_progress_params_seg), \
 20		       PROGRESS_PARAMS_WRITE_UNIT))
 21
 22struct mlx5e_ktls_rx_resync_buf {
 23	union {
 24		struct mlx5_wqe_tls_progress_params_seg progress;
 25		u8 pad[PROGRESS_PARAMS_PADDED_SIZE];
 26	} ____cacheline_aligned_in_smp;
 27	dma_addr_t dma_addr;
 28	struct mlx5e_ktls_offload_context_rx *priv_rx;
 29};
 30
 31enum {
 32	MLX5E_PRIV_RX_FLAG_DELETING,
 33	MLX5E_NUM_PRIV_RX_FLAGS,
 34};
 35
 36struct mlx5e_ktls_rx_resync_ctx {
 37	struct tls_offload_resync_async core;
 38	struct work_struct work;
 39	struct mlx5e_priv *priv;
 40	refcount_t refcnt;
 41	__be64 sw_rcd_sn_be;
 42	u32 seq;
 43};
 44
 45struct mlx5e_ktls_offload_context_rx {
 46	union mlx5e_crypto_info crypto_info;
 47	struct accel_rule rule;
 48	struct sock *sk;
 49	struct mlx5e_rq_stats *rq_stats;
 50	struct mlx5e_tls_sw_stats *sw_stats;
 51	struct completion add_ctx;
 52	struct mlx5e_tir tir;
 53	struct mlx5_crypto_dek *dek;
 54	u32 rxq;
 55	DECLARE_BITMAP(flags, MLX5E_NUM_PRIV_RX_FLAGS);
 56
 57	/* resync */
 58	spinlock_t lock; /* protects resync fields */
 59	struct mlx5e_ktls_rx_resync_ctx resync;
 60	struct list_head list;
 61};
 62
 63static bool mlx5e_ktls_priv_rx_put(struct mlx5e_ktls_offload_context_rx *priv_rx)
 64{
 65	if (!refcount_dec_and_test(&priv_rx->resync.refcnt))
 66		return false;
 67
 68	kfree(priv_rx);
 69	return true;
 70}
 71
 72static void mlx5e_ktls_priv_rx_get(struct mlx5e_ktls_offload_context_rx *priv_rx)
 73{
 74	refcount_inc(&priv_rx->resync.refcnt);
 75}
 76
 77struct mlx5e_ktls_resync_resp {
 78	/* protects list changes */
 79	spinlock_t lock;
 80	struct list_head list;
 81};
 82
 83void mlx5e_ktls_rx_resync_destroy_resp_list(struct mlx5e_ktls_resync_resp *resp_list)
 84{
 85	kvfree(resp_list);
 86}
 87
 88struct mlx5e_ktls_resync_resp *
 89mlx5e_ktls_rx_resync_create_resp_list(void)
 90{
 91	struct mlx5e_ktls_resync_resp *resp_list;
 92
 93	resp_list = kvzalloc(sizeof(*resp_list), GFP_KERNEL);
 94	if (!resp_list)
 95		return ERR_PTR(-ENOMEM);
 96
 97	INIT_LIST_HEAD(&resp_list->list);
 98	spin_lock_init(&resp_list->lock);
 99
100	return resp_list;
101}
102
103static void accel_rule_handle_work(struct work_struct *work)
104{
105	struct mlx5e_ktls_offload_context_rx *priv_rx;
106	struct accel_rule *accel_rule;
107	struct mlx5_flow_handle *rule;
108
109	accel_rule = container_of(work, struct accel_rule, work);
110	priv_rx = container_of(accel_rule, struct mlx5e_ktls_offload_context_rx, rule);
111	if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags)))
112		goto out;
113
114	rule = mlx5e_accel_fs_add_sk(accel_rule->priv->fs, priv_rx->sk,
115				     mlx5e_tir_get_tirn(&priv_rx->tir),
116				     MLX5_FS_DEFAULT_FLOW_TAG);
117	if (!IS_ERR_OR_NULL(rule))
118		accel_rule->rule = rule;
119out:
120	complete(&priv_rx->add_ctx);
121}
122
123static void accel_rule_init(struct accel_rule *rule, struct mlx5e_priv *priv)
124{
125	INIT_WORK(&rule->work, accel_rule_handle_work);
126	rule->priv = priv;
127}
128
129static void icosq_fill_wi(struct mlx5e_icosq *sq, u16 pi,
130			  struct mlx5e_icosq_wqe_info *wi)
131{
132	sq->db.wqe_info[pi] = *wi;
133}
134
135static struct mlx5_wqe_ctrl_seg *
136post_static_params(struct mlx5e_icosq *sq,
137		   struct mlx5e_ktls_offload_context_rx *priv_rx)
138{
139	struct mlx5e_set_tls_static_params_wqe *wqe;
140	struct mlx5e_icosq_wqe_info wi;
141	u16 pi, num_wqebbs;
142
143	num_wqebbs = MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS;
144	if (unlikely(!mlx5e_icosq_can_post_wqe(sq, num_wqebbs)))
145		return ERR_PTR(-ENOSPC);
146
147	pi = mlx5e_icosq_get_next_pi(sq, num_wqebbs);
148	wqe = MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi);
149	mlx5e_ktls_build_static_params(wqe, sq->pc, sq->sqn, &priv_rx->crypto_info,
150				       mlx5e_tir_get_tirn(&priv_rx->tir),
151				       mlx5_crypto_dek_get_id(priv_rx->dek),
152				       priv_rx->resync.seq, false,
153				       TLS_OFFLOAD_CTX_DIR_RX);
154	wi = (struct mlx5e_icosq_wqe_info) {
155		.wqe_type = MLX5E_ICOSQ_WQE_UMR_TLS,
156		.num_wqebbs = num_wqebbs,
157		.tls_set_params.priv_rx = priv_rx,
158	};
159	icosq_fill_wi(sq, pi, &wi);
160	sq->pc += num_wqebbs;
161
162	return &wqe->ctrl;
163}
164
165static struct mlx5_wqe_ctrl_seg *
166post_progress_params(struct mlx5e_icosq *sq,
167		     struct mlx5e_ktls_offload_context_rx *priv_rx,
168		     u32 next_record_tcp_sn)
169{
170	struct mlx5e_set_tls_progress_params_wqe *wqe;
171	struct mlx5e_icosq_wqe_info wi;
172	u16 pi, num_wqebbs;
173
174	num_wqebbs = MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS;
175	if (unlikely(!mlx5e_icosq_can_post_wqe(sq, num_wqebbs)))
176		return ERR_PTR(-ENOSPC);
177
178	pi = mlx5e_icosq_get_next_pi(sq, num_wqebbs);
179	wqe = MLX5E_TLS_FETCH_SET_PROGRESS_PARAMS_WQE(sq, pi);
180	mlx5e_ktls_build_progress_params(wqe, sq->pc, sq->sqn,
181					 mlx5e_tir_get_tirn(&priv_rx->tir),
182					 false, next_record_tcp_sn,
183					 TLS_OFFLOAD_CTX_DIR_RX);
184	wi = (struct mlx5e_icosq_wqe_info) {
185		.wqe_type = MLX5E_ICOSQ_WQE_SET_PSV_TLS,
186		.num_wqebbs = num_wqebbs,
187		.tls_set_params.priv_rx = priv_rx,
188	};
189
190	icosq_fill_wi(sq, pi, &wi);
191	sq->pc += num_wqebbs;
192
193	return &wqe->ctrl;
194}
195
196static int post_rx_param_wqes(struct mlx5e_channel *c,
197			      struct mlx5e_ktls_offload_context_rx *priv_rx,
198			      u32 next_record_tcp_sn)
199{
200	struct mlx5_wqe_ctrl_seg *cseg;
201	struct mlx5e_icosq *sq;
202	int err;
203
204	err = 0;
205	sq = &c->async_icosq;
206	spin_lock_bh(&c->async_icosq_lock);
207
208	cseg = post_static_params(sq, priv_rx);
209	if (IS_ERR(cseg))
210		goto err_out;
211	cseg = post_progress_params(sq, priv_rx, next_record_tcp_sn);
212	if (IS_ERR(cseg))
213		goto err_out;
214
215	mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg);
216unlock:
217	spin_unlock_bh(&c->async_icosq_lock);
218
219	return err;
220
221err_out:
222	priv_rx->rq_stats->tls_resync_req_skip++;
223	err = PTR_ERR(cseg);
224	complete(&priv_rx->add_ctx);
225	goto unlock;
226}
227
228static void
229mlx5e_set_ktls_rx_priv_ctx(struct tls_context *tls_ctx,
230			   struct mlx5e_ktls_offload_context_rx *priv_rx)
231{
232	struct mlx5e_ktls_offload_context_rx **ctx =
233		__tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_RX);
234
235	BUILD_BUG_ON(sizeof(priv_rx) > TLS_DRIVER_STATE_SIZE_RX);
236
237	*ctx = priv_rx;
238}
239
240static struct mlx5e_ktls_offload_context_rx *
241mlx5e_get_ktls_rx_priv_ctx(struct tls_context *tls_ctx)
242{
243	struct mlx5e_ktls_offload_context_rx **ctx =
244		__tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_RX);
245
246	return *ctx;
247}
248
249/* Re-sync */
250/* Runs in work context */
251static int
252resync_post_get_progress_params(struct mlx5e_icosq *sq,
253				struct mlx5e_ktls_offload_context_rx *priv_rx)
254{
255	struct mlx5e_get_tls_progress_params_wqe *wqe;
256	struct mlx5e_ktls_rx_resync_buf *buf;
257	struct mlx5e_icosq_wqe_info wi;
258	struct mlx5_wqe_ctrl_seg *cseg;
259	struct mlx5_seg_get_psv *psv;
260	struct device *pdev;
261	int err;
262	u16 pi;
263
264	buf = kzalloc(sizeof(*buf), GFP_KERNEL);
265	if (unlikely(!buf)) {
266		err = -ENOMEM;
267		goto err_out;
268	}
269
270	pdev = mlx5_core_dma_dev(sq->channel->mdev);
271	buf->dma_addr = dma_map_single(pdev, &buf->progress,
272				       PROGRESS_PARAMS_PADDED_SIZE, DMA_FROM_DEVICE);
273	if (unlikely(dma_mapping_error(pdev, buf->dma_addr))) {
274		err = -ENOMEM;
275		goto err_free;
276	}
277
278	buf->priv_rx = priv_rx;
279
280	spin_lock_bh(&sq->channel->async_icosq_lock);
281
282	if (unlikely(!mlx5e_icosq_can_post_wqe(sq, MLX5E_KTLS_GET_PROGRESS_WQEBBS))) {
283		spin_unlock_bh(&sq->channel->async_icosq_lock);
284		err = -ENOSPC;
285		goto err_dma_unmap;
286	}
287
288	pi = mlx5e_icosq_get_next_pi(sq, MLX5E_KTLS_GET_PROGRESS_WQEBBS);
289	wqe = MLX5E_TLS_FETCH_GET_PROGRESS_PARAMS_WQE(sq, pi);
290
291#define GET_PSV_DS_CNT (DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS))
292
293	cseg = &wqe->ctrl;
294	cseg->opmod_idx_opcode =
295		cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_GET_PSV |
296			    (MLX5_OPC_MOD_TLS_TIR_PROGRESS_PARAMS << 24));
297	cseg->qpn_ds =
298		cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | GET_PSV_DS_CNT);
299
300	psv = &wqe->psv;
301	psv->num_psv      = 1 << 4;
302	psv->l_key        = sq->channel->mkey_be;
303	psv->psv_index[0] = cpu_to_be32(mlx5e_tir_get_tirn(&priv_rx->tir));
304	psv->va           = cpu_to_be64(buf->dma_addr);
305
306	wi = (struct mlx5e_icosq_wqe_info) {
307		.wqe_type = MLX5E_ICOSQ_WQE_GET_PSV_TLS,
308		.num_wqebbs = MLX5E_KTLS_GET_PROGRESS_WQEBBS,
309		.tls_get_params.buf = buf,
310	};
311	icosq_fill_wi(sq, pi, &wi);
312	sq->pc++;
313	mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg);
314	spin_unlock_bh(&sq->channel->async_icosq_lock);
315
316	return 0;
317
318err_dma_unmap:
319	dma_unmap_single(pdev, buf->dma_addr, PROGRESS_PARAMS_PADDED_SIZE, DMA_FROM_DEVICE);
320err_free:
321	kfree(buf);
322err_out:
323	return err;
324}
325
326/* Function is called with elevated refcount.
327 * It decreases it only if no WQE is posted.
328 */
329static void resync_handle_work(struct work_struct *work)
330{
331	struct mlx5e_ktls_offload_context_rx *priv_rx;
332	struct mlx5e_ktls_rx_resync_ctx *resync;
333	struct mlx5e_channel *c;
334	struct mlx5e_icosq *sq;
335
336	resync = container_of(work, struct mlx5e_ktls_rx_resync_ctx, work);
337	priv_rx = container_of(resync, struct mlx5e_ktls_offload_context_rx, resync);
338
339	if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) {
340		mlx5e_ktls_priv_rx_put(priv_rx);
341		priv_rx->rq_stats->tls_resync_req_skip++;
342		tls_offload_rx_resync_async_request_cancel(&resync->core);
343		return;
344	}
345
346	c = resync->priv->channels.c[priv_rx->rxq];
347	sq = &c->async_icosq;
348
349	if (resync_post_get_progress_params(sq, priv_rx)) {
350		priv_rx->rq_stats->tls_resync_req_skip++;
351		tls_offload_rx_resync_async_request_cancel(&resync->core);
352		mlx5e_ktls_priv_rx_put(priv_rx);
353	}
354}
355
356static void resync_init(struct mlx5e_ktls_rx_resync_ctx *resync,
357			struct mlx5e_priv *priv)
358{
359	INIT_WORK(&resync->work, resync_handle_work);
360	resync->priv = priv;
361	refcount_set(&resync->refcnt, 1);
362}
363
364/* Function can be called with the refcount being either elevated or not.
365 * It does not affect the refcount.
366 */
367static void resync_handle_seq_match(struct mlx5e_ktls_offload_context_rx *priv_rx,
368				    struct mlx5e_channel *c)
369{
370	struct mlx5e_ktls_resync_resp *ktls_resync;
371	struct mlx5e_icosq *sq;
372	bool trigger_poll;
373
374	sq = &c->async_icosq;
375	ktls_resync = sq->ktls_resync;
376	trigger_poll = false;
377
378	spin_lock_bh(&ktls_resync->lock);
379	spin_lock_bh(&priv_rx->lock);
380	switch (priv_rx->crypto_info.crypto_info.cipher_type) {
381	case TLS_CIPHER_AES_GCM_128: {
382		struct tls12_crypto_info_aes_gcm_128 *info =
383			&priv_rx->crypto_info.crypto_info_128;
384
385		memcpy(info->rec_seq, &priv_rx->resync.sw_rcd_sn_be,
386		       sizeof(info->rec_seq));
387		break;
388	}
389	case TLS_CIPHER_AES_GCM_256: {
390		struct tls12_crypto_info_aes_gcm_256 *info =
391			&priv_rx->crypto_info.crypto_info_256;
392
393		memcpy(info->rec_seq, &priv_rx->resync.sw_rcd_sn_be,
394		       sizeof(info->rec_seq));
395		break;
396	}
397	default:
398		WARN_ONCE(1, "Unsupported cipher type %u\n",
399			  priv_rx->crypto_info.crypto_info.cipher_type);
400		spin_unlock_bh(&priv_rx->lock);
401		spin_unlock_bh(&ktls_resync->lock);
402		return;
403	}
404
405	if (list_empty(&priv_rx->list)) {
406		list_add_tail(&priv_rx->list, &ktls_resync->list);
407		trigger_poll = !test_and_set_bit(MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC, &sq->state);
408	}
409	spin_unlock_bh(&priv_rx->lock);
410	spin_unlock_bh(&ktls_resync->lock);
411
412	if (!trigger_poll)
413		return;
414
415	if (!napi_if_scheduled_mark_missed(&c->napi)) {
416		spin_lock_bh(&c->async_icosq_lock);
417		mlx5e_trigger_irq(sq);
418		spin_unlock_bh(&c->async_icosq_lock);
419	}
420}
421
422/* Function can be called with the refcount being either elevated or not.
423 * It decreases the refcount and may free the kTLS priv context.
424 * Refcount is not elevated only if tls_dev_del has been called, but GET_PSV was
425 * already in flight.
426 */
427void mlx5e_ktls_handle_get_psv_completion(struct mlx5e_icosq_wqe_info *wi,
428					  struct mlx5e_icosq *sq)
429{
430	struct mlx5e_ktls_rx_resync_buf *buf = wi->tls_get_params.buf;
431	struct mlx5e_ktls_offload_context_rx *priv_rx;
432	struct tls_offload_resync_async *async_resync;
433	struct tls_offload_context_rx *rx_ctx;
434	u8 tracker_state, auth_state, *ctx;
435	struct device *dev;
436	u32 hw_seq;
437
438	priv_rx = buf->priv_rx;
439	dev = mlx5_core_dma_dev(sq->channel->mdev);
440	rx_ctx = tls_offload_ctx_rx(tls_get_ctx(priv_rx->sk));
441	async_resync = rx_ctx->resync_async;
442	if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) {
443		priv_rx->rq_stats->tls_resync_req_skip++;
444		tls_offload_rx_resync_async_request_cancel(async_resync);
445		goto out;
446	}
447
448	dma_sync_single_for_cpu(dev, buf->dma_addr, PROGRESS_PARAMS_PADDED_SIZE,
449				DMA_FROM_DEVICE);
450
451	ctx = buf->progress.ctx;
452	tracker_state = MLX5_GET(tls_progress_params, ctx, record_tracker_state);
453	auth_state = MLX5_GET(tls_progress_params, ctx, auth_state);
454	if (tracker_state != MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_TRACKING ||
455	    auth_state != MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD) {
456		priv_rx->rq_stats->tls_resync_req_skip++;
457		tls_offload_rx_resync_async_request_cancel(async_resync);
458		goto out;
459	}
460
461	hw_seq = MLX5_GET(tls_progress_params, ctx, hw_resync_tcp_sn);
462	tls_offload_rx_resync_async_request_end(async_resync,
463						cpu_to_be32(hw_seq));
464	priv_rx->rq_stats->tls_resync_req_end++;
465out:
466	mlx5e_ktls_priv_rx_put(priv_rx);
467	dma_unmap_single(dev, buf->dma_addr, PROGRESS_PARAMS_PADDED_SIZE, DMA_FROM_DEVICE);
468	kfree(buf);
469}
470
471/* Runs in NAPI.
472 * Function elevates the refcount, unless no work is queued.
473 */
474static bool resync_queue_get_psv(struct sock *sk)
475{
476	struct mlx5e_ktls_offload_context_rx *priv_rx;
477	struct mlx5e_ktls_rx_resync_ctx *resync;
478
479	priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_get_ctx(sk));
480	if (unlikely(!priv_rx))
481		return false;
482
483	if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags)))
484		return false;
485
486	resync = &priv_rx->resync;
487	mlx5e_ktls_priv_rx_get(priv_rx);
488	if (unlikely(!queue_work(resync->priv->tls->rx_wq, &resync->work))) {
489		mlx5e_ktls_priv_rx_put(priv_rx);
490		return false;
491	}
492
493	return true;
494}
495
496/* Runs in NAPI */
497static void resync_update_sn(struct mlx5e_rq *rq, struct sk_buff *skb)
498{
499	struct ethhdr *eth = (struct ethhdr *)(skb->data);
500	struct tls_offload_resync_async *resync_async;
501	struct net_device *netdev = rq->netdev;
502	struct net *net = dev_net(netdev);
503	struct sock *sk = NULL;
504	unsigned int datalen;
505	struct iphdr *iph;
506	struct tcphdr *th;
507	__be32 seq;
508	int depth = 0;
509
510	__vlan_get_protocol(skb, eth->h_proto, &depth);
511	iph = (struct iphdr *)(skb->data + depth);
512
513	if (iph->version == 4) {
514		depth += sizeof(struct iphdr);
515		th = (void *)iph + sizeof(struct iphdr);
516
517		sk = inet_lookup_established(net, iph->saddr, th->source,
518					     iph->daddr, th->dest,
519					     netdev->ifindex);
520#if IS_ENABLED(CONFIG_IPV6)
521	} else {
522		struct ipv6hdr *ipv6h = (struct ipv6hdr *)iph;
523
524		depth += sizeof(struct ipv6hdr);
525		th = (void *)ipv6h + sizeof(struct ipv6hdr);
526
527		sk = __inet6_lookup_established(net, &ipv6h->saddr, th->source,
528						&ipv6h->daddr, ntohs(th->dest),
529						netdev->ifindex, 0);
530#endif
531	}
532
533	depth += sizeof(struct tcphdr);
534
535	if (unlikely(!sk))
536		return;
537
538	if (unlikely(sk->sk_state == TCP_TIME_WAIT))
539		goto unref;
540
541	if (unlikely(!resync_queue_get_psv(sk)))
542		goto unref;
543
544	seq = th->seq;
545	datalen = skb->len - depth;
546	resync_async = tls_offload_ctx_rx(tls_get_ctx(sk))->resync_async;
547	tls_offload_rx_resync_async_request_start(resync_async, seq, datalen);
548	rq->stats->tls_resync_req_start++;
549
550unref:
551	sock_gen_put(sk);
552}
553
554void mlx5e_ktls_rx_resync(struct net_device *netdev, struct sock *sk,
555			  u32 seq, u8 *rcd_sn)
556{
557	struct mlx5e_ktls_offload_context_rx *priv_rx;
558	struct mlx5e_ktls_rx_resync_ctx *resync;
559	struct mlx5e_priv *priv;
560	struct mlx5e_channel *c;
561
562	priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_get_ctx(sk));
563	if (unlikely(!priv_rx))
564		return;
565
566	resync = &priv_rx->resync;
567	resync->sw_rcd_sn_be = *(__be64 *)rcd_sn;
568	resync->seq = seq;
569
570	priv = netdev_priv(netdev);
571	c = priv->channels.c[priv_rx->rxq];
572
573	resync_handle_seq_match(priv_rx, c);
574}
575
576void
577mlx5e_ktls_rx_resync_async_request_cancel(struct mlx5e_icosq_wqe_info *wi)
578{
579	struct mlx5e_ktls_offload_context_rx *priv_rx;
580	struct mlx5e_ktls_rx_resync_buf *buf;
581
582	buf = wi->tls_get_params.buf;
583	priv_rx = buf->priv_rx;
584	priv_rx->rq_stats->tls_resync_req_skip++;
585	tls_offload_rx_resync_async_request_cancel(&priv_rx->resync.core);
586}
587
588/* End of resync section */
589
590void mlx5e_ktls_handle_rx_skb(struct mlx5e_rq *rq, struct sk_buff *skb,
591			      struct mlx5_cqe64 *cqe, u32 *cqe_bcnt)
592{
593	struct mlx5e_rq_stats *stats = rq->stats;
594
595	switch (get_cqe_tls_offload(cqe)) {
596	case CQE_TLS_OFFLOAD_DECRYPTED:
597		skb->decrypted = 1;
598		stats->tls_decrypted_packets++;
599		stats->tls_decrypted_bytes += *cqe_bcnt;
600		break;
601	case CQE_TLS_OFFLOAD_RESYNC:
602		stats->tls_resync_req_pkt++;
603		resync_update_sn(rq, skb);
604		break;
605	default: /* CQE_TLS_OFFLOAD_ERROR: */
606		stats->tls_err++;
607		break;
608	}
609}
610
611void mlx5e_ktls_handle_ctx_completion(struct mlx5e_icosq_wqe_info *wi)
612{
613	struct mlx5e_ktls_offload_context_rx *priv_rx = wi->tls_set_params.priv_rx;
614	struct accel_rule *rule = &priv_rx->rule;
615
616	if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) {
617		complete(&priv_rx->add_ctx);
618		return;
619	}
620	queue_work(rule->priv->tls->rx_wq, &rule->work);
621}
622
623static int mlx5e_ktls_sk_get_rxq(struct sock *sk)
624{
625	int rxq = sk_rx_queue_get(sk);
626
627	if (unlikely(rxq == -1))
628		rxq = 0;
629
630	return rxq;
631}
632
633int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk,
634		      struct tls_crypto_info *crypto_info,
635		      u32 start_offload_tcp_sn)
636{
637	struct mlx5e_ktls_offload_context_rx *priv_rx;
638	struct mlx5e_ktls_rx_resync_ctx *resync;
639	struct tls_context *tls_ctx;
640	struct mlx5_crypto_dek *dek;
641	struct mlx5e_priv *priv;
642	int rxq, err;
643
644	tls_ctx = tls_get_ctx(sk);
645	priv = netdev_priv(netdev);
646	priv_rx = kzalloc(sizeof(*priv_rx), GFP_KERNEL);
647	if (unlikely(!priv_rx))
648		return -ENOMEM;
649
650	switch (crypto_info->cipher_type) {
651	case TLS_CIPHER_AES_GCM_128:
652		priv_rx->crypto_info.crypto_info_128 =
653			*(struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
654		break;
655	case TLS_CIPHER_AES_GCM_256:
656		priv_rx->crypto_info.crypto_info_256 =
657			*(struct tls12_crypto_info_aes_gcm_256 *)crypto_info;
658		break;
659	default:
660		WARN_ONCE(1, "Unsupported cipher type %u\n",
661			  crypto_info->cipher_type);
662		err = -EOPNOTSUPP;
663		goto err_cipher_type;
664	}
665
666	dek = mlx5_ktls_create_key(priv->tls->dek_pool, crypto_info);
667	if (IS_ERR(dek)) {
668		err = PTR_ERR(dek);
669		goto err_cipher_type;
670	}
671	priv_rx->dek = dek;
672
673	INIT_LIST_HEAD(&priv_rx->list);
674	spin_lock_init(&priv_rx->lock);
675
676	rxq = mlx5e_ktls_sk_get_rxq(sk);
677	priv_rx->rxq = rxq;
678	priv_rx->sk = sk;
679
680	priv_rx->rq_stats = &priv->channel_stats[rxq]->rq;
681	priv_rx->sw_stats = &priv->tls->sw_stats;
682	mlx5e_set_ktls_rx_priv_ctx(tls_ctx, priv_rx);
683
684	err = mlx5e_rx_res_tls_tir_create(priv->rx_res, rxq, &priv_rx->tir);
685	if (err)
686		goto err_create_tir;
687
688	init_completion(&priv_rx->add_ctx);
689
690	accel_rule_init(&priv_rx->rule, priv);
691	resync = &priv_rx->resync;
692	resync_init(resync, priv);
693	tls_offload_ctx_rx(tls_ctx)->resync_async = &resync->core;
694	tls_offload_rx_resync_set_type(sk, TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ_ASYNC);
695
696	err = post_rx_param_wqes(priv->channels.c[rxq], priv_rx, start_offload_tcp_sn);
697	if (err)
698		goto err_post_wqes;
699
700	atomic64_inc(&priv_rx->sw_stats->rx_tls_ctx);
701
702	return 0;
703
704err_post_wqes:
705	mlx5e_tir_destroy(&priv_rx->tir);
706err_create_tir:
707	mlx5_ktls_destroy_key(priv->tls->dek_pool, priv_rx->dek);
708err_cipher_type:
709	kfree(priv_rx);
710	return err;
711}
712
713void mlx5e_ktls_del_rx(struct net_device *netdev, struct tls_context *tls_ctx)
714{
715	struct mlx5e_ktls_offload_context_rx *priv_rx;
716	struct mlx5e_ktls_rx_resync_ctx *resync;
717	struct mlx5e_priv *priv;
718
719	priv = netdev_priv(netdev);
720
721	priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_ctx);
722	set_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags);
723	mlx5e_set_ktls_rx_priv_ctx(tls_ctx, NULL);
724	synchronize_net(); /* Sync with NAPI */
725	if (!cancel_work_sync(&priv_rx->rule.work))
726		/* completion is needed, as the priv_rx in the add flow
727		 * is maintained on the wqe info (wi), not on the socket.
728		 */
729		wait_for_completion(&priv_rx->add_ctx);
730	resync = &priv_rx->resync;
731	if (cancel_work_sync(&resync->work))
732		mlx5e_ktls_priv_rx_put(priv_rx);
733
734	atomic64_inc(&priv_rx->sw_stats->rx_tls_del);
735	if (priv_rx->rule.rule)
736		mlx5e_accel_fs_del_sk(priv_rx->rule.rule);
737
738	mlx5e_tir_destroy(&priv_rx->tir);
739	mlx5_ktls_destroy_key(priv->tls->dek_pool, priv_rx->dek);
740	/* priv_rx should normally be freed here, but if there is an outstanding
741	 * GET_PSV, deallocation will be delayed until the CQE for GET_PSV is
742	 * processed.
743	 */
744	mlx5e_ktls_priv_rx_put(priv_rx);
745}
746
747bool mlx5e_ktls_rx_handle_resync_list(struct mlx5e_channel *c, int budget)
748{
749	struct mlx5e_ktls_offload_context_rx *priv_rx, *tmp;
750	struct mlx5e_ktls_resync_resp *ktls_resync;
751	struct mlx5_wqe_ctrl_seg *db_cseg;
752	struct mlx5e_icosq *sq;
753	LIST_HEAD(local_list);
754	int i, j;
755
756	sq = &c->async_icosq;
757
758	if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
759		return false;
760
761	ktls_resync = sq->ktls_resync;
762	db_cseg = NULL;
763	i = 0;
764
765	spin_lock(&ktls_resync->lock);
766	list_for_each_entry_safe(priv_rx, tmp, &ktls_resync->list, list) {
767		list_move(&priv_rx->list, &local_list);
768		if (++i == budget)
769			break;
770	}
771	if (list_empty(&ktls_resync->list))
772		clear_bit(MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC, &sq->state);
773	spin_unlock(&ktls_resync->lock);
774
775	spin_lock(&c->async_icosq_lock);
776	for (j = 0; j < i; j++) {
777		struct mlx5_wqe_ctrl_seg *cseg;
778
779		priv_rx = list_first_entry(&local_list,
780					   struct mlx5e_ktls_offload_context_rx,
781					   list);
782		spin_lock(&priv_rx->lock);
783		cseg = post_static_params(sq, priv_rx);
784		if (IS_ERR(cseg)) {
785			spin_unlock(&priv_rx->lock);
786			break;
787		}
788		list_del_init(&priv_rx->list);
789		spin_unlock(&priv_rx->lock);
790		db_cseg = cseg;
791	}
792	if (db_cseg)
793		mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, db_cseg);
794	spin_unlock(&c->async_icosq_lock);
795
796	priv_rx->rq_stats->tls_resync_res_ok += j;
797
798	if (!list_empty(&local_list)) {
799		/* This happens only if ICOSQ is full.
800		 * There is no need to mark busy or explicitly ask for a NAPI cycle,
801		 * it will be triggered by the outstanding ICOSQ completions.
802		 */
803		spin_lock(&ktls_resync->lock);
804		list_splice(&local_list, &ktls_resync->list);
805		set_bit(MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC, &sq->state);
806		spin_unlock(&ktls_resync->lock);
807		priv_rx->rq_stats->tls_resync_res_retry++;
808	}
809
810	return i == budget;
811}