tls_device.c

   1/* Copyright (c) 2018, Mellanox Technologies All rights reserved.
   2 *
   3 * This software is available to you under a choice of one of two
   4 * licenses.  You may choose to be licensed under the terms of the GNU
   5 * General Public License (GPL) Version 2, available from the file
   6 * COPYING in the main directory of this source tree, or the
   7 * OpenIB.org BSD license below:
   8 *
   9 *     Redistribution and use in source and binary forms, with or
  10 *     without modification, are permitted provided that the following
  11 *     conditions are met:
  12 *
  13 *      - Redistributions of source code must retain the above
  14 *        copyright notice, this list of conditions and the following
  15 *        disclaimer.
  16 *
  17 *      - Redistributions in binary form must reproduce the above
  18 *        copyright notice, this list of conditions and the following
  19 *        disclaimer in the documentation and/or other materials
  20 *        provided with the distribution.
  21 *
  22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  23 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  24 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  25 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  26 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  27 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  28 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  29 * SOFTWARE.
  30 */
  31
  32#include <crypto/aead.h>
  33#include <linux/highmem.h>
  34#include <linux/module.h>
  35#include <linux/netdevice.h>
  36#include <net/dst.h>
  37#include <net/inet_connection_sock.h>
  38#include <net/tcp.h>
  39#include <net/tls.h>
  40#include <linux/skbuff_ref.h>
  41
  42#include "tls.h"
  43#include "trace.h"
  44
  45/* device_offload_lock is used to synchronize tls_dev_add
  46 * against NETDEV_DOWN notifications.
  47 */
  48static DECLARE_RWSEM(device_offload_lock);
  49
  50static struct workqueue_struct *destruct_wq __read_mostly;
  51
  52static LIST_HEAD(tls_device_list);
  53static LIST_HEAD(tls_device_down_list);
  54static DEFINE_SPINLOCK(tls_device_lock);
  55
  56static struct page *dummy_page;
  57
  58static void tls_device_free_ctx(struct tls_context *ctx)
  59{
  60	if (ctx->tx_conf == TLS_HW)
  61		kfree(tls_offload_ctx_tx(ctx));
  62
  63	if (ctx->rx_conf == TLS_HW)
  64		kfree(tls_offload_ctx_rx(ctx));
  65
  66	tls_ctx_free(NULL, ctx);
  67}
  68
  69static void tls_device_tx_del_task(struct work_struct *work)
  70{
  71	struct tls_offload_context_tx *offload_ctx =
  72		container_of(work, struct tls_offload_context_tx, destruct_work);
  73	struct tls_context *ctx = offload_ctx->ctx;
  74	struct net_device *netdev;
  75
  76	/* Safe, because this is the destroy flow, refcount is 0, so
  77	 * tls_device_down can't store this field in parallel.
  78	 */
  79	netdev = rcu_dereference_protected(ctx->netdev,
  80					   !refcount_read(&ctx->refcount));
  81
  82	netdev->tlsdev_ops->tls_dev_del(netdev, ctx, TLS_OFFLOAD_CTX_DIR_TX);
  83	dev_put(netdev);
  84	ctx->netdev = NULL;
  85	tls_device_free_ctx(ctx);
  86}
  87
  88static void tls_device_queue_ctx_destruction(struct tls_context *ctx)
  89{
  90	struct net_device *netdev;
  91	unsigned long flags;
  92	bool async_cleanup;
  93
  94	spin_lock_irqsave(&tls_device_lock, flags);
  95	if (unlikely(!refcount_dec_and_test(&ctx->refcount))) {
  96		spin_unlock_irqrestore(&tls_device_lock, flags);
  97		return;
  98	}
  99
 100	list_del(&ctx->list); /* Remove from tls_device_list / tls_device_down_list */
 101
 102	/* Safe, because this is the destroy flow, refcount is 0, so
 103	 * tls_device_down can't store this field in parallel.
 104	 */
 105	netdev = rcu_dereference_protected(ctx->netdev,
 106					   !refcount_read(&ctx->refcount));
 107
 108	async_cleanup = netdev && ctx->tx_conf == TLS_HW;
 109	if (async_cleanup) {
 110		struct tls_offload_context_tx *offload_ctx = tls_offload_ctx_tx(ctx);
 111
 112		/* queue_work inside the spinlock
 113		 * to make sure tls_device_down waits for that work.
 114		 */
 115		queue_work(destruct_wq, &offload_ctx->destruct_work);
 116	}
 117	spin_unlock_irqrestore(&tls_device_lock, flags);
 118
 119	if (!async_cleanup)
 120		tls_device_free_ctx(ctx);
 121}
 122
 123/* We assume that the socket is already connected */
 124static struct net_device *get_netdev_for_sock(struct sock *sk)
 125{
 126	struct net_device *dev, *lowest_dev = NULL;
 127	struct dst_entry *dst;
 128
 129	rcu_read_lock();
 130	dst = __sk_dst_get(sk);
 131	dev = dst ? dst_dev_rcu(dst) : NULL;
 132	if (likely(dev)) {
 133		lowest_dev = netdev_sk_get_lowest_dev(dev, sk);
 134		dev_hold(lowest_dev);
 135	}
 136	rcu_read_unlock();
 137
 138	return lowest_dev;
 139}
 140
 141static void destroy_record(struct tls_record_info *record)
 142{
 143	int i;
 144
 145	for (i = 0; i < record->num_frags; i++)
 146		__skb_frag_unref(&record->frags[i], false);
 147	kfree(record);
 148}
 149
 150static void delete_all_records(struct tls_offload_context_tx *offload_ctx)
 151{
 152	struct tls_record_info *info, *temp;
 153
 154	list_for_each_entry_safe(info, temp, &offload_ctx->records_list, list) {
 155		list_del(&info->list);
 156		destroy_record(info);
 157	}
 158
 159	offload_ctx->retransmit_hint = NULL;
 160}
 161
 162static void tls_tcp_clean_acked(struct sock *sk, u32 acked_seq)
 163{
 164	struct tls_context *tls_ctx = tls_get_ctx(sk);
 165	struct tls_record_info *info, *temp;
 166	struct tls_offload_context_tx *ctx;
 167	u64 deleted_records = 0;
 168	unsigned long flags;
 169
 170	if (!tls_ctx)
 171		return;
 172
 173	ctx = tls_offload_ctx_tx(tls_ctx);
 174
 175	spin_lock_irqsave(&ctx->lock, flags);
 176	info = ctx->retransmit_hint;
 177	if (info && !before(acked_seq, info->end_seq))
 178		ctx->retransmit_hint = NULL;
 179
 180	list_for_each_entry_safe(info, temp, &ctx->records_list, list) {
 181		if (before(acked_seq, info->end_seq))
 182			break;
 183		list_del(&info->list);
 184
 185		destroy_record(info);
 186		deleted_records++;
 187	}
 188
 189	ctx->unacked_record_sn += deleted_records;
 190	spin_unlock_irqrestore(&ctx->lock, flags);
 191}
 192
 193/* At this point, there should be no references on this
 194 * socket and no in-flight SKBs associated with this
 195 * socket, so it is safe to free all the resources.
 196 */
 197void tls_device_sk_destruct(struct sock *sk)
 198{
 199	struct tls_context *tls_ctx = tls_get_ctx(sk);
 200	struct tls_offload_context_tx *ctx = tls_offload_ctx_tx(tls_ctx);
 201
 202	tls_ctx->sk_destruct(sk);
 203
 204	if (tls_ctx->tx_conf == TLS_HW) {
 205		if (ctx->open_record)
 206			destroy_record(ctx->open_record);
 207		delete_all_records(ctx);
 208		crypto_free_aead(ctx->aead_send);
 209		clean_acked_data_disable(tcp_sk(sk));
 210	}
 211
 212	tls_device_queue_ctx_destruction(tls_ctx);
 213}
 214EXPORT_SYMBOL_GPL(tls_device_sk_destruct);
 215
 216void tls_device_free_resources_tx(struct sock *sk)
 217{
 218	struct tls_context *tls_ctx = tls_get_ctx(sk);
 219
 220	tls_free_partial_record(sk, tls_ctx);
 221}
 222
 223void tls_offload_tx_resync_request(struct sock *sk, u32 got_seq, u32 exp_seq)
 224{
 225	struct tls_context *tls_ctx = tls_get_ctx(sk);
 226
 227	trace_tls_device_tx_resync_req(sk, got_seq, exp_seq);
 228	WARN_ON(test_and_set_bit(TLS_TX_SYNC_SCHED, &tls_ctx->flags));
 229}
 230EXPORT_SYMBOL_GPL(tls_offload_tx_resync_request);
 231
 232static void tls_device_resync_tx(struct sock *sk, struct tls_context *tls_ctx,
 233				 u32 seq)
 234{
 235	struct net_device *netdev;
 236	int err = 0;
 237	u8 *rcd_sn;
 238
 239	tcp_write_collapse_fence(sk);
 240	rcd_sn = tls_ctx->tx.rec_seq;
 241
 242	trace_tls_device_tx_resync_send(sk, seq, rcd_sn);
 243	down_read(&device_offload_lock);
 244	netdev = rcu_dereference_protected(tls_ctx->netdev,
 245					   lockdep_is_held(&device_offload_lock));
 246	if (netdev)
 247		err = netdev->tlsdev_ops->tls_dev_resync(netdev, sk, seq,
 248							 rcd_sn,
 249							 TLS_OFFLOAD_CTX_DIR_TX);
 250	up_read(&device_offload_lock);
 251	if (err)
 252		return;
 253
 254	clear_bit_unlock(TLS_TX_SYNC_SCHED, &tls_ctx->flags);
 255}
 256
 257static void tls_append_frag(struct tls_record_info *record,
 258			    struct page_frag *pfrag,
 259			    int size)
 260{
 261	skb_frag_t *frag;
 262
 263	frag = &record->frags[record->num_frags - 1];
 264	if (skb_frag_page(frag) == pfrag->page &&
 265	    skb_frag_off(frag) + skb_frag_size(frag) == pfrag->offset) {
 266		skb_frag_size_add(frag, size);
 267	} else {
 268		++frag;
 269		skb_frag_fill_page_desc(frag, pfrag->page, pfrag->offset,
 270					size);
 271		++record->num_frags;
 272		get_page(pfrag->page);
 273	}
 274
 275	pfrag->offset += size;
 276	record->len += size;
 277}
 278
 279static int tls_push_record(struct sock *sk,
 280			   struct tls_context *ctx,
 281			   struct tls_offload_context_tx *offload_ctx,
 282			   struct tls_record_info *record,
 283			   int flags)
 284{
 285	struct tls_prot_info *prot = &ctx->prot_info;
 286	struct tcp_sock *tp = tcp_sk(sk);
 287	skb_frag_t *frag;
 288	int i;
 289
 290	record->end_seq = tp->write_seq + record->len;
 291	list_add_tail_rcu(&record->list, &offload_ctx->records_list);
 292	offload_ctx->open_record = NULL;
 293
 294	if (test_bit(TLS_TX_SYNC_SCHED, &ctx->flags))
 295		tls_device_resync_tx(sk, ctx, tp->write_seq);
 296
 297	tls_advance_record_sn(sk, prot, &ctx->tx);
 298
 299	for (i = 0; i < record->num_frags; i++) {
 300		frag = &record->frags[i];
 301		sg_unmark_end(&offload_ctx->sg_tx_data[i]);
 302		sg_set_page(&offload_ctx->sg_tx_data[i], skb_frag_page(frag),
 303			    skb_frag_size(frag), skb_frag_off(frag));
 304		sk_mem_charge(sk, skb_frag_size(frag));
 305		get_page(skb_frag_page(frag));
 306	}
 307	sg_mark_end(&offload_ctx->sg_tx_data[record->num_frags - 1]);
 308
 309	/* all ready, send */
 310	return tls_push_sg(sk, ctx, offload_ctx->sg_tx_data, 0, flags);
 311}
 312
 313static void tls_device_record_close(struct sock *sk,
 314				    struct tls_context *ctx,
 315				    struct tls_record_info *record,
 316				    struct page_frag *pfrag,
 317				    unsigned char record_type)
 318{
 319	struct tls_prot_info *prot = &ctx->prot_info;
 320	struct page_frag dummy_tag_frag;
 321
 322	/* append tag
 323	 * device will fill in the tag, we just need to append a placeholder
 324	 * use socket memory to improve coalescing (re-using a single buffer
 325	 * increases frag count)
 326	 * if we can't allocate memory now use the dummy page
 327	 */
 328	if (unlikely(pfrag->size - pfrag->offset < prot->tag_size) &&
 329	    !skb_page_frag_refill(prot->tag_size, pfrag, sk->sk_allocation)) {
 330		dummy_tag_frag.page = dummy_page;
 331		dummy_tag_frag.offset = 0;
 332		pfrag = &dummy_tag_frag;
 333	}
 334	tls_append_frag(record, pfrag, prot->tag_size);
 335
 336	/* fill prepend */
 337	tls_fill_prepend(ctx, skb_frag_address(&record->frags[0]),
 338			 record->len - prot->overhead_size,
 339			 record_type);
 340}
 341
 342static int tls_create_new_record(struct tls_offload_context_tx *offload_ctx,
 343				 struct page_frag *pfrag,
 344				 size_t prepend_size)
 345{
 346	struct tls_record_info *record;
 347	skb_frag_t *frag;
 348
 349	record = kmalloc(sizeof(*record), GFP_KERNEL);
 350	if (!record)
 351		return -ENOMEM;
 352
 353	frag = &record->frags[0];
 354	skb_frag_fill_page_desc(frag, pfrag->page, pfrag->offset,
 355				prepend_size);
 356
 357	get_page(pfrag->page);
 358	pfrag->offset += prepend_size;
 359
 360	record->num_frags = 1;
 361	record->len = prepend_size;
 362	offload_ctx->open_record = record;
 363	return 0;
 364}
 365
 366static int tls_do_allocation(struct sock *sk,
 367			     struct tls_offload_context_tx *offload_ctx,
 368			     struct page_frag *pfrag,
 369			     size_t prepend_size)
 370{
 371	int ret;
 372
 373	if (!offload_ctx->open_record) {
 374		if (unlikely(!skb_page_frag_refill(prepend_size, pfrag,
 375						   sk->sk_allocation))) {
 376			if (!sk->sk_bypass_prot_mem)
 377				READ_ONCE(sk->sk_prot)->enter_memory_pressure(sk);
 378			sk_stream_moderate_sndbuf(sk);
 379			return -ENOMEM;
 380		}
 381
 382		ret = tls_create_new_record(offload_ctx, pfrag, prepend_size);
 383		if (ret)
 384			return ret;
 385
 386		if (pfrag->size > pfrag->offset)
 387			return 0;
 388	}
 389
 390	if (!sk_page_frag_refill(sk, pfrag))
 391		return -ENOMEM;
 392
 393	return 0;
 394}
 395
 396static int tls_device_copy_data(void *addr, size_t bytes, struct iov_iter *i)
 397{
 398	size_t pre_copy, nocache;
 399
 400	pre_copy = ~((unsigned long)addr - 1) & (SMP_CACHE_BYTES - 1);
 401	if (pre_copy) {
 402		pre_copy = min(pre_copy, bytes);
 403		if (copy_from_iter(addr, pre_copy, i) != pre_copy)
 404			return -EFAULT;
 405		bytes -= pre_copy;
 406		addr += pre_copy;
 407	}
 408
 409	nocache = round_down(bytes, SMP_CACHE_BYTES);
 410	if (copy_from_iter_nocache(addr, nocache, i) != nocache)
 411		return -EFAULT;
 412	bytes -= nocache;
 413	addr += nocache;
 414
 415	if (bytes && copy_from_iter(addr, bytes, i) != bytes)
 416		return -EFAULT;
 417
 418	return 0;
 419}
 420
 421static int tls_push_data(struct sock *sk,
 422			 struct iov_iter *iter,
 423			 size_t size, int flags,
 424			 unsigned char record_type)
 425{
 426	struct tls_context *tls_ctx = tls_get_ctx(sk);
 427	struct tls_prot_info *prot = &tls_ctx->prot_info;
 428	struct tls_offload_context_tx *ctx = tls_offload_ctx_tx(tls_ctx);
 429	struct tls_record_info *record;
 430	int tls_push_record_flags;
 431	struct page_frag *pfrag;
 432	size_t orig_size = size;
 433	u32 max_open_record_len;
 434	bool more = false;
 435	bool done = false;
 436	int copy, rc = 0;
 437	long timeo;
 438
 439	if (flags &
 440	    ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL |
 441	      MSG_SPLICE_PAGES | MSG_EOR))
 442		return -EOPNOTSUPP;
 443
 444	if ((flags & (MSG_MORE | MSG_EOR)) == (MSG_MORE | MSG_EOR))
 445		return -EINVAL;
 446
 447	if (unlikely(sk->sk_err))
 448		return -sk->sk_err;
 449
 450	flags |= MSG_SENDPAGE_DECRYPTED;
 451	tls_push_record_flags = flags | MSG_MORE;
 452
 453	timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
 454	if (tls_is_partially_sent_record(tls_ctx)) {
 455		rc = tls_push_partial_record(sk, tls_ctx, flags);
 456		if (rc < 0)
 457			return rc;
 458	}
 459
 460	pfrag = sk_page_frag(sk);
 461
 462	/* TLS_HEADER_SIZE is not counted as part of the TLS record, and
 463	 * we need to leave room for an authentication tag.
 464	 */
 465	max_open_record_len = tls_ctx->tx_max_payload_len +
 466			      prot->prepend_size;
 467	do {
 468		rc = tls_do_allocation(sk, ctx, pfrag, prot->prepend_size);
 469		if (unlikely(rc)) {
 470			rc = sk_stream_wait_memory(sk, &timeo);
 471			if (!rc)
 472				continue;
 473
 474			record = ctx->open_record;
 475			if (!record)
 476				break;
 477handle_error:
 478			if (record_type != TLS_RECORD_TYPE_DATA) {
 479				/* avoid sending partial
 480				 * record with type !=
 481				 * application_data
 482				 */
 483				size = orig_size;
 484				destroy_record(record);
 485				ctx->open_record = NULL;
 486			} else if (record->len > prot->prepend_size) {
 487				goto last_record;
 488			}
 489
 490			break;
 491		}
 492
 493		record = ctx->open_record;
 494
 495		copy = min_t(size_t, size, max_open_record_len - record->len);
 496		if (copy && (flags & MSG_SPLICE_PAGES)) {
 497			struct page_frag zc_pfrag;
 498			struct page **pages = &zc_pfrag.page;
 499			size_t off;
 500
 501			rc = iov_iter_extract_pages(iter, &pages,
 502						    copy, 1, 0, &off);
 503			if (rc <= 0) {
 504				if (rc == 0)
 505					rc = -EIO;
 506				goto handle_error;
 507			}
 508			copy = rc;
 509
 510			if (WARN_ON_ONCE(!sendpage_ok(zc_pfrag.page))) {
 511				iov_iter_revert(iter, copy);
 512				rc = -EIO;
 513				goto handle_error;
 514			}
 515
 516			zc_pfrag.offset = off;
 517			zc_pfrag.size = copy;
 518			tls_append_frag(record, &zc_pfrag, copy);
 519		} else if (copy) {
 520			copy = min_t(size_t, copy, pfrag->size - pfrag->offset);
 521
 522			rc = tls_device_copy_data(page_address(pfrag->page) +
 523						  pfrag->offset, copy,
 524						  iter);
 525			if (rc)
 526				goto handle_error;
 527			tls_append_frag(record, pfrag, copy);
 528		}
 529
 530		size -= copy;
 531		if (!size) {
 532last_record:
 533			tls_push_record_flags = flags;
 534			if (flags & MSG_MORE) {
 535				more = true;
 536				break;
 537			}
 538
 539			done = true;
 540		}
 541
 542		if (done || record->len >= max_open_record_len ||
 543		    (record->num_frags >= MAX_SKB_FRAGS - 1)) {
 544			tls_device_record_close(sk, tls_ctx, record,
 545						pfrag, record_type);
 546
 547			rc = tls_push_record(sk,
 548					     tls_ctx,
 549					     ctx,
 550					     record,
 551					     tls_push_record_flags);
 552			if (rc < 0)
 553				break;
 554		}
 555	} while (!done);
 556
 557	tls_ctx->pending_open_record_frags = more;
 558
 559	if (orig_size - size > 0)
 560		rc = orig_size - size;
 561
 562	return rc;
 563}
 564
 565int tls_device_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
 566{
 567	unsigned char record_type = TLS_RECORD_TYPE_DATA;
 568	struct tls_context *tls_ctx = tls_get_ctx(sk);
 569	int rc;
 570
 571	if (!tls_ctx->zerocopy_sendfile)
 572		msg->msg_flags &= ~MSG_SPLICE_PAGES;
 573
 574	mutex_lock(&tls_ctx->tx_lock);
 575	lock_sock(sk);
 576
 577	if (unlikely(msg->msg_controllen)) {
 578		rc = tls_process_cmsg(sk, msg, &record_type);
 579		if (rc)
 580			goto out;
 581	}
 582
 583	rc = tls_push_data(sk, &msg->msg_iter, size, msg->msg_flags,
 584			   record_type);
 585
 586out:
 587	release_sock(sk);
 588	mutex_unlock(&tls_ctx->tx_lock);
 589	return rc;
 590}
 591
 592void tls_device_splice_eof(struct socket *sock)
 593{
 594	struct sock *sk = sock->sk;
 595	struct tls_context *tls_ctx = tls_get_ctx(sk);
 596	struct iov_iter iter = {};
 597
 598	if (!tls_is_partially_sent_record(tls_ctx))
 599		return;
 600
 601	mutex_lock(&tls_ctx->tx_lock);
 602	lock_sock(sk);
 603
 604	if (tls_is_partially_sent_record(tls_ctx)) {
 605		iov_iter_bvec(&iter, ITER_SOURCE, NULL, 0, 0);
 606		tls_push_data(sk, &iter, 0, 0, TLS_RECORD_TYPE_DATA);
 607	}
 608
 609	release_sock(sk);
 610	mutex_unlock(&tls_ctx->tx_lock);
 611}
 612
 613struct tls_record_info *tls_get_record(struct tls_offload_context_tx *context,
 614				       u32 seq, u64 *p_record_sn)
 615{
 616	u64 record_sn = context->hint_record_sn;
 617	struct tls_record_info *info, *last;
 618
 619	info = context->retransmit_hint;
 620	if (!info ||
 621	    before(seq, info->end_seq - info->len)) {
 622		/* if retransmit_hint is irrelevant start
 623		 * from the beginning of the list
 624		 */
 625		info = list_first_entry_or_null(&context->records_list,
 626						struct tls_record_info, list);
 627		if (!info)
 628			return NULL;
 629		/* send the start_marker record if seq number is before the
 630		 * tls offload start marker sequence number. This record is
 631		 * required to handle TCP packets which are before TLS offload
 632		 * started.
 633		 *  And if it's not start marker, look if this seq number
 634		 * belongs to the list.
 635		 */
 636		if (likely(!tls_record_is_start_marker(info))) {
 637			/* we have the first record, get the last record to see
 638			 * if this seq number belongs to the list.
 639			 */
 640			last = list_last_entry(&context->records_list,
 641					       struct tls_record_info, list);
 642
 643			if (!between(seq, tls_record_start_seq(info),
 644				     last->end_seq))
 645				return NULL;
 646		}
 647		record_sn = context->unacked_record_sn;
 648	}
 649
 650	/* We just need the _rcu for the READ_ONCE() */
 651	rcu_read_lock();
 652	list_for_each_entry_from_rcu(info, &context->records_list, list) {
 653		if (before(seq, info->end_seq)) {
 654			if (!context->retransmit_hint ||
 655			    after(info->end_seq,
 656				  context->retransmit_hint->end_seq)) {
 657				context->hint_record_sn = record_sn;
 658				context->retransmit_hint = info;
 659			}
 660			*p_record_sn = record_sn;
 661			goto exit_rcu_unlock;
 662		}
 663		record_sn++;
 664	}
 665	info = NULL;
 666
 667exit_rcu_unlock:
 668	rcu_read_unlock();
 669	return info;
 670}
 671EXPORT_SYMBOL(tls_get_record);
 672
 673static int tls_device_push_pending_record(struct sock *sk, int flags)
 674{
 675	struct iov_iter iter;
 676
 677	iov_iter_kvec(&iter, ITER_SOURCE, NULL, 0, 0);
 678	return tls_push_data(sk, &iter, 0, flags, TLS_RECORD_TYPE_DATA);
 679}
 680
 681void tls_device_write_space(struct sock *sk, struct tls_context *ctx)
 682{
 683	if (tls_is_partially_sent_record(ctx)) {
 684		gfp_t sk_allocation = sk->sk_allocation;
 685
 686		WARN_ON_ONCE(sk->sk_write_pending);
 687
 688		sk->sk_allocation = GFP_ATOMIC;
 689		tls_push_partial_record(sk, ctx,
 690					MSG_DONTWAIT | MSG_NOSIGNAL |
 691					MSG_SENDPAGE_DECRYPTED);
 692		sk->sk_allocation = sk_allocation;
 693	}
 694}
 695
 696static void tls_device_resync_rx(struct tls_context *tls_ctx,
 697				 struct sock *sk, u32 seq, u8 *rcd_sn)
 698{
 699	struct tls_offload_context_rx *rx_ctx = tls_offload_ctx_rx(tls_ctx);
 700	struct net_device *netdev;
 701
 702	trace_tls_device_rx_resync_send(sk, seq, rcd_sn, rx_ctx->resync_type);
 703	rcu_read_lock();
 704	netdev = rcu_dereference(tls_ctx->netdev);
 705	if (netdev)
 706		netdev->tlsdev_ops->tls_dev_resync(netdev, sk, seq, rcd_sn,
 707						   TLS_OFFLOAD_CTX_DIR_RX);
 708	rcu_read_unlock();
 709	TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXDEVICERESYNC);
 710}
 711
 712static bool
 713tls_device_rx_resync_async(struct tls_offload_resync_async *resync_async,
 714			   s64 resync_req, u32 *seq, u16 *rcd_delta)
 715{
 716	u32 is_async = resync_req & RESYNC_REQ_ASYNC;
 717	u32 req_seq = resync_req >> 32;
 718	u32 req_end = req_seq + ((resync_req >> 16) & 0xffff);
 719	u16 i;
 720
 721	*rcd_delta = 0;
 722
 723	if (is_async) {
 724		/* shouldn't get to wraparound:
 725		 * too long in async stage, something bad happened
 726		 */
 727		if (WARN_ON_ONCE(resync_async->rcd_delta == USHRT_MAX)) {
 728			tls_offload_rx_resync_async_request_cancel(resync_async);
 729			return false;
 730		}
 731
 732		/* asynchronous stage: log all headers seq such that
 733		 * req_seq <= seq <= end_seq, and wait for real resync request
 734		 */
 735		if (before(*seq, req_seq))
 736			return false;
 737		if (!after(*seq, req_end) &&
 738		    resync_async->loglen < TLS_DEVICE_RESYNC_ASYNC_LOGMAX)
 739			resync_async->log[resync_async->loglen++] = *seq;
 740
 741		resync_async->rcd_delta++;
 742
 743		return false;
 744	}
 745
 746	/* synchronous stage: check against the logged entries and
 747	 * proceed to check the next entries if no match was found
 748	 */
 749	for (i = 0; i < resync_async->loglen; i++)
 750		if (req_seq == resync_async->log[i] &&
 751		    atomic64_try_cmpxchg(&resync_async->req, &resync_req, 0)) {
 752			*rcd_delta = resync_async->rcd_delta - i;
 753			*seq = req_seq;
 754			resync_async->loglen = 0;
 755			resync_async->rcd_delta = 0;
 756			return true;
 757		}
 758
 759	resync_async->loglen = 0;
 760	resync_async->rcd_delta = 0;
 761
 762	if (req_seq == *seq &&
 763	    atomic64_try_cmpxchg(&resync_async->req,
 764				 &resync_req, 0))
 765		return true;
 766
 767	return false;
 768}
 769
 770void tls_device_rx_resync_new_rec(struct sock *sk, u32 rcd_len, u32 seq)
 771{
 772	struct tls_context *tls_ctx = tls_get_ctx(sk);
 773	struct tls_offload_context_rx *rx_ctx;
 774	u8 rcd_sn[TLS_MAX_REC_SEQ_SIZE];
 775	u32 sock_data, is_req_pending;
 776	struct tls_prot_info *prot;
 777	s64 resync_req;
 778	u16 rcd_delta;
 779	u32 req_seq;
 780
 781	if (tls_ctx->rx_conf != TLS_HW)
 782		return;
 783	if (unlikely(test_bit(TLS_RX_DEV_DEGRADED, &tls_ctx->flags)))
 784		return;
 785
 786	prot = &tls_ctx->prot_info;
 787	rx_ctx = tls_offload_ctx_rx(tls_ctx);
 788	memcpy(rcd_sn, tls_ctx->rx.rec_seq, prot->rec_seq_size);
 789
 790	switch (rx_ctx->resync_type) {
 791	case TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ:
 792		resync_req = atomic64_read(&rx_ctx->resync_req);
 793		req_seq = resync_req >> 32;
 794		seq += TLS_HEADER_SIZE - 1;
 795		is_req_pending = resync_req;
 796
 797		if (likely(!is_req_pending) || req_seq != seq ||
 798		    !atomic64_try_cmpxchg(&rx_ctx->resync_req, &resync_req, 0))
 799			return;
 800		break;
 801	case TLS_OFFLOAD_SYNC_TYPE_CORE_NEXT_HINT:
 802		if (likely(!rx_ctx->resync_nh_do_now))
 803			return;
 804
 805		/* head of next rec is already in, note that the sock_inq will
 806		 * include the currently parsed message when called from parser
 807		 */
 808		sock_data = tcp_inq(sk);
 809		if (sock_data > rcd_len) {
 810			trace_tls_device_rx_resync_nh_delay(sk, sock_data,
 811							    rcd_len);
 812			return;
 813		}
 814
 815		rx_ctx->resync_nh_do_now = 0;
 816		seq += rcd_len;
 817		tls_bigint_increment(rcd_sn, prot->rec_seq_size);
 818		break;
 819	case TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ_ASYNC:
 820		resync_req = atomic64_read(&rx_ctx->resync_async->req);
 821		is_req_pending = resync_req;
 822		if (likely(!is_req_pending))
 823			return;
 824
 825		if (!tls_device_rx_resync_async(rx_ctx->resync_async,
 826						resync_req, &seq, &rcd_delta))
 827			return;
 828		tls_bigint_subtract(rcd_sn, rcd_delta);
 829		break;
 830	}
 831
 832	tls_device_resync_rx(tls_ctx, sk, seq, rcd_sn);
 833}
 834
 835static void tls_device_core_ctrl_rx_resync(struct tls_context *tls_ctx,
 836					   struct tls_offload_context_rx *ctx,
 837					   struct sock *sk, struct sk_buff *skb)
 838{
 839	struct strp_msg *rxm;
 840
 841	/* device will request resyncs by itself based on stream scan */
 842	if (ctx->resync_type != TLS_OFFLOAD_SYNC_TYPE_CORE_NEXT_HINT)
 843		return;
 844	/* already scheduled */
 845	if (ctx->resync_nh_do_now)
 846		return;
 847	/* seen decrypted fragments since last fully-failed record */
 848	if (ctx->resync_nh_reset) {
 849		ctx->resync_nh_reset = 0;
 850		ctx->resync_nh.decrypted_failed = 1;
 851		ctx->resync_nh.decrypted_tgt = TLS_DEVICE_RESYNC_NH_START_IVAL;
 852		return;
 853	}
 854
 855	if (++ctx->resync_nh.decrypted_failed <= ctx->resync_nh.decrypted_tgt)
 856		return;
 857
 858	/* doing resync, bump the next target in case it fails */
 859	if (ctx->resync_nh.decrypted_tgt < TLS_DEVICE_RESYNC_NH_MAX_IVAL)
 860		ctx->resync_nh.decrypted_tgt *= 2;
 861	else
 862		ctx->resync_nh.decrypted_tgt += TLS_DEVICE_RESYNC_NH_MAX_IVAL;
 863
 864	rxm = strp_msg(skb);
 865
 866	/* head of next rec is already in, parser will sync for us */
 867	if (tcp_inq(sk) > rxm->full_len) {
 868		trace_tls_device_rx_resync_nh_schedule(sk);
 869		ctx->resync_nh_do_now = 1;
 870	} else {
 871		struct tls_prot_info *prot = &tls_ctx->prot_info;
 872		u8 rcd_sn[TLS_MAX_REC_SEQ_SIZE];
 873
 874		memcpy(rcd_sn, tls_ctx->rx.rec_seq, prot->rec_seq_size);
 875		tls_bigint_increment(rcd_sn, prot->rec_seq_size);
 876
 877		tls_device_resync_rx(tls_ctx, sk, tcp_sk(sk)->copied_seq,
 878				     rcd_sn);
 879	}
 880}
 881
 882static int
 883tls_device_reencrypt(struct sock *sk, struct tls_context *tls_ctx)
 884{
 885	struct tls_sw_context_rx *sw_ctx = tls_sw_ctx_rx(tls_ctx);
 886	const struct tls_cipher_desc *cipher_desc;
 887	int err, offset, copy, data_len, pos;
 888	struct sk_buff *skb, *skb_iter;
 889	struct scatterlist sg[1];
 890	struct strp_msg *rxm;
 891	char *orig_buf, *buf;
 892
 893	cipher_desc = get_cipher_desc(tls_ctx->crypto_recv.info.cipher_type);
 894	DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
 895
 896	rxm = strp_msg(tls_strp_msg(sw_ctx));
 897	orig_buf = kmalloc(rxm->full_len + TLS_HEADER_SIZE + cipher_desc->iv,
 898			   sk->sk_allocation);
 899	if (!orig_buf)
 900		return -ENOMEM;
 901	buf = orig_buf;
 902
 903	err = tls_strp_msg_cow(sw_ctx);
 904	if (unlikely(err))
 905		goto free_buf;
 906
 907	skb = tls_strp_msg(sw_ctx);
 908	rxm = strp_msg(skb);
 909	offset = rxm->offset;
 910
 911	sg_init_table(sg, 1);
 912	sg_set_buf(&sg[0], buf,
 913		   rxm->full_len + TLS_HEADER_SIZE + cipher_desc->iv);
 914	err = skb_copy_bits(skb, offset, buf, TLS_HEADER_SIZE + cipher_desc->iv);
 915	if (err)
 916		goto free_buf;
 917
 918	/* We are interested only in the decrypted data not the auth */
 919	err = decrypt_skb(sk, sg);
 920	if (err != -EBADMSG)
 921		goto free_buf;
 922	else
 923		err = 0;
 924
 925	data_len = rxm->full_len - cipher_desc->tag;
 926
 927	if (skb_pagelen(skb) > offset) {
 928		copy = min_t(int, skb_pagelen(skb) - offset, data_len);
 929
 930		if (skb->decrypted) {
 931			err = skb_store_bits(skb, offset, buf, copy);
 932			if (err)
 933				goto free_buf;
 934		}
 935
 936		offset += copy;
 937		buf += copy;
 938	}
 939
 940	pos = skb_pagelen(skb);
 941	skb_walk_frags(skb, skb_iter) {
 942		int frag_pos;
 943
 944		/* Practically all frags must belong to msg if reencrypt
 945		 * is needed with current strparser and coalescing logic,
 946		 * but strparser may "get optimized", so let's be safe.
 947		 */
 948		if (pos + skb_iter->len <= offset)
 949			goto done_with_frag;
 950		if (pos >= data_len + rxm->offset)
 951			break;
 952
 953		frag_pos = offset - pos;
 954		copy = min_t(int, skb_iter->len - frag_pos,
 955			     data_len + rxm->offset - offset);
 956
 957		if (skb_iter->decrypted) {
 958			err = skb_store_bits(skb_iter, frag_pos, buf, copy);
 959			if (err)
 960				goto free_buf;
 961		}
 962
 963		offset += copy;
 964		buf += copy;
 965done_with_frag:
 966		pos += skb_iter->len;
 967	}
 968
 969free_buf:
 970	kfree(orig_buf);
 971	return err;
 972}
 973
 974int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx)
 975{
 976	struct tls_offload_context_rx *ctx = tls_offload_ctx_rx(tls_ctx);
 977	struct tls_sw_context_rx *sw_ctx = tls_sw_ctx_rx(tls_ctx);
 978	struct sk_buff *skb = tls_strp_msg(sw_ctx);
 979	struct strp_msg *rxm = strp_msg(skb);
 980	int is_decrypted, is_encrypted;
 981
 982	if (!tls_strp_msg_mixed_decrypted(sw_ctx)) {
 983		is_decrypted = skb->decrypted;
 984		is_encrypted = !is_decrypted;
 985	} else {
 986		is_decrypted = 0;
 987		is_encrypted = 0;
 988	}
 989
 990	trace_tls_device_decrypted(sk, tcp_sk(sk)->copied_seq - rxm->full_len,
 991				   tls_ctx->rx.rec_seq, rxm->full_len,
 992				   is_encrypted, is_decrypted);
 993
 994	if (unlikely(test_bit(TLS_RX_DEV_DEGRADED, &tls_ctx->flags))) {
 995		if (likely(is_encrypted || is_decrypted))
 996			return is_decrypted;
 997
 998		/* After tls_device_down disables the offload, the next SKB will
 999		 * likely have initial fragments decrypted, and final ones not
1000		 * decrypted. We need to reencrypt that single SKB.
1001		 */
1002		return tls_device_reencrypt(sk, tls_ctx);
1003	}
1004
1005	/* Return immediately if the record is either entirely plaintext or
1006	 * entirely ciphertext. Otherwise handle reencrypt partially decrypted
1007	 * record.
1008	 */
1009	if (is_decrypted) {
1010		ctx->resync_nh_reset = 1;
1011		return is_decrypted;
1012	}
1013	if (is_encrypted) {
1014		tls_device_core_ctrl_rx_resync(tls_ctx, ctx, sk, skb);
1015		return 0;
1016	}
1017
1018	ctx->resync_nh_reset = 1;
1019	return tls_device_reencrypt(sk, tls_ctx);
1020}
1021
1022static void tls_device_attach(struct tls_context *ctx, struct sock *sk,
1023			      struct net_device *netdev)
1024{
1025	if (sk->sk_destruct != tls_device_sk_destruct) {
1026		refcount_set(&ctx->refcount, 1);
1027		dev_hold(netdev);
1028		RCU_INIT_POINTER(ctx->netdev, netdev);
1029		spin_lock_irq(&tls_device_lock);
1030		list_add_tail(&ctx->list, &tls_device_list);
1031		spin_unlock_irq(&tls_device_lock);
1032
1033		ctx->sk_destruct = sk->sk_destruct;
1034		smp_store_release(&sk->sk_destruct, tls_device_sk_destruct);
1035	}
1036}
1037
1038static struct tls_offload_context_tx *alloc_offload_ctx_tx(struct tls_context *ctx)
1039{
1040	struct tls_offload_context_tx *offload_ctx;
1041	__be64 rcd_sn;
1042
1043	offload_ctx = kzalloc(sizeof(*offload_ctx), GFP_KERNEL);
1044	if (!offload_ctx)
1045		return NULL;
1046
1047	INIT_WORK(&offload_ctx->destruct_work, tls_device_tx_del_task);
1048	INIT_LIST_HEAD(&offload_ctx->records_list);
1049	spin_lock_init(&offload_ctx->lock);
1050	sg_init_table(offload_ctx->sg_tx_data,
1051		      ARRAY_SIZE(offload_ctx->sg_tx_data));
1052
1053	/* start at rec_seq - 1 to account for the start marker record */
1054	memcpy(&rcd_sn, ctx->tx.rec_seq, sizeof(rcd_sn));
1055	offload_ctx->unacked_record_sn = be64_to_cpu(rcd_sn) - 1;
1056
1057	offload_ctx->ctx = ctx;
1058
1059	return offload_ctx;
1060}
1061
1062int tls_set_device_offload(struct sock *sk)
1063{
1064	struct tls_record_info *start_marker_record;
1065	struct tls_offload_context_tx *offload_ctx;
1066	const struct tls_cipher_desc *cipher_desc;
1067	struct tls_crypto_info *crypto_info;
1068	struct tls_prot_info *prot;
1069	struct net_device *netdev;
1070	struct tls_context *ctx;
1071	char *iv, *rec_seq;
1072	int rc;
1073
1074	ctx = tls_get_ctx(sk);
1075	prot = &ctx->prot_info;
1076
1077	if (ctx->priv_ctx_tx)
1078		return -EEXIST;
1079
1080	netdev = get_netdev_for_sock(sk);
1081	if (!netdev) {
1082		pr_err_ratelimited("%s: netdev not found\n", __func__);
1083		return -EINVAL;
1084	}
1085
1086	if (!(netdev->features & NETIF_F_HW_TLS_TX)) {
1087		rc = -EOPNOTSUPP;
1088		goto release_netdev;
1089	}
1090
1091	crypto_info = &ctx->crypto_send.info;
1092	if (crypto_info->version != TLS_1_2_VERSION) {
1093		rc = -EOPNOTSUPP;
1094		goto release_netdev;
1095	}
1096
1097	cipher_desc = get_cipher_desc(crypto_info->cipher_type);
1098	if (!cipher_desc || !cipher_desc->offloadable) {
1099		rc = -EINVAL;
1100		goto release_netdev;
1101	}
1102
1103	rc = init_prot_info(prot, crypto_info, cipher_desc);
1104	if (rc)
1105		goto release_netdev;
1106
1107	iv = crypto_info_iv(crypto_info, cipher_desc);
1108	rec_seq = crypto_info_rec_seq(crypto_info, cipher_desc);
1109
1110	memcpy(ctx->tx.iv + cipher_desc->salt, iv, cipher_desc->iv);
1111	memcpy(ctx->tx.rec_seq, rec_seq, cipher_desc->rec_seq);
1112
1113	start_marker_record = kmalloc(sizeof(*start_marker_record), GFP_KERNEL);
1114	if (!start_marker_record) {
1115		rc = -ENOMEM;
1116		goto release_netdev;
1117	}
1118
1119	offload_ctx = alloc_offload_ctx_tx(ctx);
1120	if (!offload_ctx) {
1121		rc = -ENOMEM;
1122		goto free_marker_record;
1123	}
1124
1125	rc = tls_sw_fallback_init(sk, offload_ctx, crypto_info);
1126	if (rc)
1127		goto free_offload_ctx;
1128
1129	start_marker_record->end_seq = tcp_sk(sk)->write_seq;
1130	start_marker_record->len = 0;
1131	start_marker_record->num_frags = 0;
1132	list_add_tail(&start_marker_record->list, &offload_ctx->records_list);
1133
1134	clean_acked_data_enable(tcp_sk(sk), &tls_tcp_clean_acked);
1135	ctx->push_pending_record = tls_device_push_pending_record;
1136
1137	/* TLS offload is greatly simplified if we don't send
1138	 * SKBs where only part of the payload needs to be encrypted.
1139	 * So mark the last skb in the write queue as end of record.
1140	 */
1141	tcp_write_collapse_fence(sk);
1142
1143	/* Avoid offloading if the device is down
1144	 * We don't want to offload new flows after
1145	 * the NETDEV_DOWN event
1146	 *
1147	 * device_offload_lock is taken in tls_devices's NETDEV_DOWN
1148	 * handler thus protecting from the device going down before
1149	 * ctx was added to tls_device_list.
1150	 */
1151	down_read(&device_offload_lock);
1152	if (!(netdev->flags & IFF_UP)) {
1153		rc = -EINVAL;
1154		goto release_lock;
1155	}
1156
1157	ctx->priv_ctx_tx = offload_ctx;
1158	rc = netdev->tlsdev_ops->tls_dev_add(netdev, sk, TLS_OFFLOAD_CTX_DIR_TX,
1159					     &ctx->crypto_send.info,
1160					     tcp_sk(sk)->write_seq);
1161	trace_tls_device_offload_set(sk, TLS_OFFLOAD_CTX_DIR_TX,
1162				     tcp_sk(sk)->write_seq, rec_seq, rc);
1163	if (rc)
1164		goto release_lock;
1165
1166	tls_device_attach(ctx, sk, netdev);
1167	up_read(&device_offload_lock);
1168
1169	/* following this assignment tls_is_skb_tx_device_offloaded
1170	 * will return true and the context might be accessed
1171	 * by the netdev's xmit function.
1172	 */
1173	smp_store_release(&sk->sk_validate_xmit_skb, tls_validate_xmit_skb);
1174	dev_put(netdev);
1175
1176	return 0;
1177
1178release_lock:
1179	up_read(&device_offload_lock);
1180	clean_acked_data_disable(tcp_sk(sk));
1181	crypto_free_aead(offload_ctx->aead_send);
1182free_offload_ctx:
1183	kfree(offload_ctx);
1184	ctx->priv_ctx_tx = NULL;
1185free_marker_record:
1186	kfree(start_marker_record);
1187release_netdev:
1188	dev_put(netdev);
1189	return rc;
1190}
1191
1192int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
1193{
1194	struct tls12_crypto_info_aes_gcm_128 *info;
1195	struct tls_offload_context_rx *context;
1196	struct net_device *netdev;
1197	int rc = 0;
1198
1199	if (ctx->crypto_recv.info.version != TLS_1_2_VERSION)
1200		return -EOPNOTSUPP;
1201
1202	netdev = get_netdev_for_sock(sk);
1203	if (!netdev) {
1204		pr_err_ratelimited("%s: netdev not found\n", __func__);
1205		return -EINVAL;
1206	}
1207
1208	if (!(netdev->features & NETIF_F_HW_TLS_RX)) {
1209		rc = -EOPNOTSUPP;
1210		goto release_netdev;
1211	}
1212
1213	/* Avoid offloading if the device is down
1214	 * We don't want to offload new flows after
1215	 * the NETDEV_DOWN event
1216	 *
1217	 * device_offload_lock is taken in tls_devices's NETDEV_DOWN
1218	 * handler thus protecting from the device going down before
1219	 * ctx was added to tls_device_list.
1220	 */
1221	down_read(&device_offload_lock);
1222	if (!(netdev->flags & IFF_UP)) {
1223		rc = -EINVAL;
1224		goto release_lock;
1225	}
1226
1227	context = kzalloc(sizeof(*context), GFP_KERNEL);
1228	if (!context) {
1229		rc = -ENOMEM;
1230		goto release_lock;
1231	}
1232	context->resync_nh_reset = 1;
1233
1234	ctx->priv_ctx_rx = context;
1235	rc = tls_set_sw_offload(sk, 0, NULL);
1236	if (rc)
1237		goto release_ctx;
1238
1239	rc = netdev->tlsdev_ops->tls_dev_add(netdev, sk, TLS_OFFLOAD_CTX_DIR_RX,
1240					     &ctx->crypto_recv.info,
1241					     tcp_sk(sk)->copied_seq);
1242	info = (void *)&ctx->crypto_recv.info;
1243	trace_tls_device_offload_set(sk, TLS_OFFLOAD_CTX_DIR_RX,
1244				     tcp_sk(sk)->copied_seq, info->rec_seq, rc);
1245	if (rc)
1246		goto free_sw_resources;
1247
1248	tls_device_attach(ctx, sk, netdev);
1249	up_read(&device_offload_lock);
1250
1251	dev_put(netdev);
1252
1253	return 0;
1254
1255free_sw_resources:
1256	up_read(&device_offload_lock);
1257	tls_sw_free_resources_rx(sk);
1258	down_read(&device_offload_lock);
1259release_ctx:
1260	ctx->priv_ctx_rx = NULL;
1261release_lock:
1262	up_read(&device_offload_lock);
1263release_netdev:
1264	dev_put(netdev);
1265	return rc;
1266}
1267
1268void tls_device_offload_cleanup_rx(struct sock *sk)
1269{
1270	struct tls_context *tls_ctx = tls_get_ctx(sk);
1271	struct net_device *netdev;
1272
1273	down_read(&device_offload_lock);
1274	netdev = rcu_dereference_protected(tls_ctx->netdev,
1275					   lockdep_is_held(&device_offload_lock));
1276	if (!netdev)
1277		goto out;
1278
1279	netdev->tlsdev_ops->tls_dev_del(netdev, tls_ctx,
1280					TLS_OFFLOAD_CTX_DIR_RX);
1281
1282	if (tls_ctx->tx_conf != TLS_HW) {
1283		dev_put(netdev);
1284		rcu_assign_pointer(tls_ctx->netdev, NULL);
1285	} else {
1286		set_bit(TLS_RX_DEV_CLOSED, &tls_ctx->flags);
1287	}
1288out:
1289	up_read(&device_offload_lock);
1290	tls_sw_release_resources_rx(sk);
1291}
1292
1293static int tls_device_down(struct net_device *netdev)
1294{
1295	struct tls_context *ctx, *tmp;
1296	unsigned long flags;
1297	LIST_HEAD(list);
1298
1299	/* Request a write lock to block new offload attempts */
1300	down_write(&device_offload_lock);
1301
1302	spin_lock_irqsave(&tls_device_lock, flags);
1303	list_for_each_entry_safe(ctx, tmp, &tls_device_list, list) {
1304		struct net_device *ctx_netdev =
1305			rcu_dereference_protected(ctx->netdev,
1306						  lockdep_is_held(&device_offload_lock));
1307
1308		if (ctx_netdev != netdev ||
1309		    !refcount_inc_not_zero(&ctx->refcount))
1310			continue;
1311
1312		list_move(&ctx->list, &list);
1313	}
1314	spin_unlock_irqrestore(&tls_device_lock, flags);
1315
1316	list_for_each_entry_safe(ctx, tmp, &list, list)	{
1317		/* Stop offloaded TX and switch to the fallback.
1318		 * tls_is_skb_tx_device_offloaded will return false.
1319		 */
1320		WRITE_ONCE(ctx->sk->sk_validate_xmit_skb, tls_validate_xmit_skb_sw);
1321
1322		/* Stop the RX and TX resync.
1323		 * tls_dev_resync must not be called after tls_dev_del.
1324		 */
1325		rcu_assign_pointer(ctx->netdev, NULL);
1326
1327		/* Start skipping the RX resync logic completely. */
1328		set_bit(TLS_RX_DEV_DEGRADED, &ctx->flags);
1329
1330		/* Sync with inflight packets. After this point:
1331		 * TX: no non-encrypted packets will be passed to the driver.
1332		 * RX: resync requests from the driver will be ignored.
1333		 */
1334		synchronize_net();
1335
1336		/* Release the offload context on the driver side. */
1337		if (ctx->tx_conf == TLS_HW)
1338			netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
1339							TLS_OFFLOAD_CTX_DIR_TX);
1340		if (ctx->rx_conf == TLS_HW &&
1341		    !test_bit(TLS_RX_DEV_CLOSED, &ctx->flags))
1342			netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
1343							TLS_OFFLOAD_CTX_DIR_RX);
1344
1345		dev_put(netdev);
1346
1347		/* Move the context to a separate list for two reasons:
1348		 * 1. When the context is deallocated, list_del is called.
1349		 * 2. It's no longer an offloaded context, so we don't want to
1350		 *    run offload-specific code on this context.
1351		 */
1352		spin_lock_irqsave(&tls_device_lock, flags);
1353		list_move_tail(&ctx->list, &tls_device_down_list);
1354		spin_unlock_irqrestore(&tls_device_lock, flags);
1355
1356		/* Device contexts for RX and TX will be freed in on sk_destruct
1357		 * by tls_device_free_ctx. rx_conf and tx_conf stay in TLS_HW.
1358		 * Now release the ref taken above.
1359		 */
1360		if (refcount_dec_and_test(&ctx->refcount)) {
1361			/* sk_destruct ran after tls_device_down took a ref, and
1362			 * it returned early. Complete the destruction here.
1363			 */
1364			list_del(&ctx->list);
1365			tls_device_free_ctx(ctx);
1366		}
1367	}
1368
1369	up_write(&device_offload_lock);
1370
1371	flush_workqueue(destruct_wq);
1372
1373	return NOTIFY_DONE;
1374}
1375
1376static int tls_dev_event(struct notifier_block *this, unsigned long event,
1377			 void *ptr)
1378{
1379	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
1380
1381	if (!dev->tlsdev_ops &&
1382	    !(dev->features & (NETIF_F_HW_TLS_RX | NETIF_F_HW_TLS_TX)))
1383		return NOTIFY_DONE;
1384
1385	switch (event) {
1386	case NETDEV_REGISTER:
1387	case NETDEV_FEAT_CHANGE:
1388		if (netif_is_bond_master(dev))
1389			return NOTIFY_DONE;
1390		if ((dev->features & NETIF_F_HW_TLS_RX) &&
1391		    !dev->tlsdev_ops->tls_dev_resync)
1392			return NOTIFY_BAD;
1393
1394		if  (dev->tlsdev_ops &&
1395		     dev->tlsdev_ops->tls_dev_add &&
1396		     dev->tlsdev_ops->tls_dev_del)
1397			return NOTIFY_DONE;
1398		else
1399			return NOTIFY_BAD;
1400	case NETDEV_DOWN:
1401		return tls_device_down(dev);
1402	}
1403	return NOTIFY_DONE;
1404}
1405
1406static struct notifier_block tls_dev_notifier = {
1407	.notifier_call	= tls_dev_event,
1408};
1409
1410int __init tls_device_init(void)
1411{
1412	int err;
1413
1414	dummy_page = alloc_page(GFP_KERNEL);
1415	if (!dummy_page)
1416		return -ENOMEM;
1417
1418	destruct_wq = alloc_workqueue("ktls_device_destruct", WQ_PERCPU, 0);
1419	if (!destruct_wq) {
1420		err = -ENOMEM;
1421		goto err_free_dummy;
1422	}
1423
1424	err = register_netdevice_notifier(&tls_dev_notifier);
1425	if (err)
1426		goto err_destroy_wq;
1427
1428	return 0;
1429
1430err_destroy_wq:
1431	destroy_workqueue(destruct_wq);
1432err_free_dummy:
1433	put_page(dummy_page);
1434	return err;
1435}
1436
1437void __exit tls_device_cleanup(void)
1438{
1439	unregister_netdevice_notifier(&tls_dev_notifier);
1440	destroy_workqueue(destruct_wq);
1441	clean_acked_data_flush();
1442	put_page(dummy_page);
1443}