tls_strp.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/* Copyright (c) 2016 Tom Herbert <tom@herbertland.com> */
  3
  4#include <linux/skbuff.h>
  5#include <linux/skbuff_ref.h>
  6#include <linux/workqueue.h>
  7#include <net/strparser.h>
  8#include <net/tcp.h>
  9#include <net/sock.h>
 10#include <net/tls.h>
 11
 12#include "tls.h"
 13
 14static struct workqueue_struct *tls_strp_wq;
 15
 16void tls_strp_abort_strp(struct tls_strparser *strp, int err)
 17{
 18	if (strp->stopped)
 19		return;
 20
 21	strp->stopped = 1;
 22
 23	/* Report an error on the lower socket */
 24	WRITE_ONCE(strp->sk->sk_err, -err);
 25	/* Paired with smp_rmb() in tcp_poll() */
 26	smp_wmb();
 27	sk_error_report(strp->sk);
 28}
 29
 30static void tls_strp_anchor_free(struct tls_strparser *strp)
 31{
 32	struct skb_shared_info *shinfo = skb_shinfo(strp->anchor);
 33
 34	DEBUG_NET_WARN_ON_ONCE(atomic_read(&shinfo->dataref) != 1);
 35	if (!strp->copy_mode)
 36		shinfo->frag_list = NULL;
 37	consume_skb(strp->anchor);
 38	strp->anchor = NULL;
 39}
 40
 41static struct sk_buff *
 42tls_strp_skb_copy(struct tls_strparser *strp, struct sk_buff *in_skb,
 43		  int offset, int len)
 44{
 45	struct sk_buff *skb;
 46	int i, err;
 47
 48	skb = alloc_skb_with_frags(0, len, TLS_PAGE_ORDER,
 49				   &err, strp->sk->sk_allocation);
 50	if (!skb)
 51		return NULL;
 52
 53	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 54		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 55
 56		WARN_ON_ONCE(skb_copy_bits(in_skb, offset,
 57					   skb_frag_address(frag),
 58					   skb_frag_size(frag)));
 59		offset += skb_frag_size(frag);
 60	}
 61
 62	skb->len = len;
 63	skb->data_len = len;
 64	skb_copy_header(skb, in_skb);
 65	return skb;
 66}
 67
 68/* Create a new skb with the contents of input copied to its page frags */
 69static struct sk_buff *tls_strp_msg_make_copy(struct tls_strparser *strp)
 70{
 71	struct strp_msg *rxm;
 72	struct sk_buff *skb;
 73
 74	skb = tls_strp_skb_copy(strp, strp->anchor, strp->stm.offset,
 75				strp->stm.full_len);
 76	if (!skb)
 77		return NULL;
 78
 79	rxm = strp_msg(skb);
 80	rxm->offset = 0;
 81	return skb;
 82}
 83
 84/* Steal the input skb, input msg is invalid after calling this function */
 85struct sk_buff *tls_strp_msg_detach(struct tls_sw_context_rx *ctx)
 86{
 87	struct tls_strparser *strp = &ctx->strp;
 88
 89#ifdef CONFIG_TLS_DEVICE
 90	DEBUG_NET_WARN_ON_ONCE(!strp->anchor->decrypted);
 91#else
 92	/* This function turns an input into an output,
 93	 * that can only happen if we have offload.
 94	 */
 95	WARN_ON(1);
 96#endif
 97
 98	if (strp->copy_mode) {
 99		struct sk_buff *skb;
100
101		/* Replace anchor with an empty skb, this is a little
102		 * dangerous but __tls_cur_msg() warns on empty skbs
103		 * so hopefully we'll catch abuses.
104		 */
105		skb = alloc_skb(0, strp->sk->sk_allocation);
106		if (!skb)
107			return NULL;
108
109		swap(strp->anchor, skb);
110		return skb;
111	}
112
113	return tls_strp_msg_make_copy(strp);
114}
115
116/* Force the input skb to be in copy mode. The data ownership remains
117 * with the input skb itself (meaning unpause will wipe it) but it can
118 * be modified.
119 */
120int tls_strp_msg_cow(struct tls_sw_context_rx *ctx)
121{
122	struct tls_strparser *strp = &ctx->strp;
123	struct sk_buff *skb;
124
125	if (strp->copy_mode)
126		return 0;
127
128	skb = tls_strp_msg_make_copy(strp);
129	if (!skb)
130		return -ENOMEM;
131
132	tls_strp_anchor_free(strp);
133	strp->anchor = skb;
134
135	tcp_read_done(strp->sk, strp->stm.full_len);
136	strp->copy_mode = 1;
137
138	return 0;
139}
140
141/* Make a clone (in the skb sense) of the input msg to keep a reference
142 * to the underlying data. The reference-holding skbs get placed on
143 * @dst.
144 */
145int tls_strp_msg_hold(struct tls_strparser *strp, struct sk_buff_head *dst)
146{
147	struct skb_shared_info *shinfo = skb_shinfo(strp->anchor);
148
149	if (strp->copy_mode) {
150		struct sk_buff *skb;
151
152		WARN_ON_ONCE(!shinfo->nr_frags);
153
154		/* We can't skb_clone() the anchor, it gets wiped by unpause */
155		skb = alloc_skb(0, strp->sk->sk_allocation);
156		if (!skb)
157			return -ENOMEM;
158
159		__skb_queue_tail(dst, strp->anchor);
160		strp->anchor = skb;
161	} else {
162		struct sk_buff *iter, *clone;
163		int chunk, len, offset;
164
165		offset = strp->stm.offset;
166		len = strp->stm.full_len;
167		iter = shinfo->frag_list;
168
169		while (len > 0) {
170			if (iter->len <= offset) {
171				offset -= iter->len;
172				goto next;
173			}
174
175			chunk = iter->len - offset;
176			offset = 0;
177
178			clone = skb_clone(iter, strp->sk->sk_allocation);
179			if (!clone)
180				return -ENOMEM;
181			__skb_queue_tail(dst, clone);
182
183			len -= chunk;
184next:
185			iter = iter->next;
186		}
187	}
188
189	return 0;
190}
191
192static void tls_strp_flush_anchor_copy(struct tls_strparser *strp)
193{
194	struct skb_shared_info *shinfo = skb_shinfo(strp->anchor);
195	int i;
196
197	DEBUG_NET_WARN_ON_ONCE(atomic_read(&shinfo->dataref) != 1);
198
199	for (i = 0; i < shinfo->nr_frags; i++)
200		__skb_frag_unref(&shinfo->frags[i], false);
201	shinfo->nr_frags = 0;
202	if (strp->copy_mode) {
203		kfree_skb_list(shinfo->frag_list);
204		shinfo->frag_list = NULL;
205	}
206	strp->copy_mode = 0;
207	strp->mixed_decrypted = 0;
208}
209
210static int tls_strp_copyin_frag(struct tls_strparser *strp, struct sk_buff *skb,
211				struct sk_buff *in_skb, unsigned int offset,
212				size_t in_len)
213{
214	unsigned int nfrag = skb->len / PAGE_SIZE;
215	size_t len, chunk;
216	skb_frag_t *frag;
217	int sz;
218
219	if (unlikely(nfrag >= skb_shinfo(skb)->nr_frags)) {
220		DEBUG_NET_WARN_ON_ONCE(1);
221		return -EMSGSIZE;
222	}
223
224	frag = &skb_shinfo(skb)->frags[nfrag];
225
226	len = in_len;
227	/* First make sure we got the header */
228	if (!strp->stm.full_len) {
229		/* Assume one page is more than enough for headers */
230		chunk =	min_t(size_t, len, PAGE_SIZE - skb_frag_size(frag));
231		WARN_ON_ONCE(skb_copy_bits(in_skb, offset,
232					   skb_frag_address(frag) +
233					   skb_frag_size(frag),
234					   chunk));
235
236		skb->len += chunk;
237		skb->data_len += chunk;
238		skb_frag_size_add(frag, chunk);
239
240		sz = tls_rx_msg_size(strp, skb);
241		if (sz < 0)
242			return sz;
243
244		/* We may have over-read, sz == 0 is guaranteed under-read */
245		if (unlikely(sz && sz < skb->len)) {
246			int over = skb->len - sz;
247
248			WARN_ON_ONCE(over > chunk);
249			skb->len -= over;
250			skb->data_len -= over;
251			skb_frag_size_add(frag, -over);
252
253			chunk -= over;
254		}
255
256		frag++;
257		len -= chunk;
258		offset += chunk;
259
260		strp->stm.full_len = sz;
261		if (!strp->stm.full_len)
262			goto read_done;
263	}
264
265	/* Load up more data */
266	while (len && strp->stm.full_len > skb->len) {
267		chunk =	min_t(size_t, len, strp->stm.full_len - skb->len);
268		chunk = min_t(size_t, chunk, PAGE_SIZE - skb_frag_size(frag));
269		WARN_ON_ONCE(skb_copy_bits(in_skb, offset,
270					   skb_frag_address(frag) +
271					   skb_frag_size(frag),
272					   chunk));
273
274		skb->len += chunk;
275		skb->data_len += chunk;
276		skb_frag_size_add(frag, chunk);
277		frag++;
278		len -= chunk;
279		offset += chunk;
280	}
281
282read_done:
283	return in_len - len;
284}
285
286static int tls_strp_copyin_skb(struct tls_strparser *strp, struct sk_buff *skb,
287			       struct sk_buff *in_skb, unsigned int offset,
288			       size_t in_len)
289{
290	struct sk_buff *nskb, *first, *last;
291	struct skb_shared_info *shinfo;
292	size_t chunk;
293	int sz;
294
295	if (strp->stm.full_len)
296		chunk = strp->stm.full_len - skb->len;
297	else
298		chunk = TLS_MAX_PAYLOAD_SIZE + PAGE_SIZE;
299	chunk = min(chunk, in_len);
300
301	nskb = tls_strp_skb_copy(strp, in_skb, offset, chunk);
302	if (!nskb)
303		return -ENOMEM;
304
305	shinfo = skb_shinfo(skb);
306	if (!shinfo->frag_list) {
307		shinfo->frag_list = nskb;
308		nskb->prev = nskb;
309	} else {
310		first = shinfo->frag_list;
311		last = first->prev;
312		last->next = nskb;
313		first->prev = nskb;
314	}
315
316	skb->len += chunk;
317	skb->data_len += chunk;
318
319	if (!strp->stm.full_len) {
320		sz = tls_rx_msg_size(strp, skb);
321		if (sz < 0)
322			return sz;
323
324		/* We may have over-read, sz == 0 is guaranteed under-read */
325		if (unlikely(sz && sz < skb->len)) {
326			int over = skb->len - sz;
327
328			WARN_ON_ONCE(over > chunk);
329			skb->len -= over;
330			skb->data_len -= over;
331			__pskb_trim(nskb, nskb->len - over);
332
333			chunk -= over;
334		}
335
336		strp->stm.full_len = sz;
337	}
338
339	return chunk;
340}
341
342static int tls_strp_copyin(read_descriptor_t *desc, struct sk_buff *in_skb,
343			   unsigned int offset, size_t in_len)
344{
345	struct tls_strparser *strp = (struct tls_strparser *)desc->arg.data;
346	struct sk_buff *skb;
347	int ret;
348
349	if (strp->msg_ready)
350		return 0;
351
352	skb = strp->anchor;
353	if (!skb->len)
354		skb_copy_decrypted(skb, in_skb);
355	else
356		strp->mixed_decrypted |= !!skb_cmp_decrypted(skb, in_skb);
357
358	if (IS_ENABLED(CONFIG_TLS_DEVICE) && strp->mixed_decrypted)
359		ret = tls_strp_copyin_skb(strp, skb, in_skb, offset, in_len);
360	else
361		ret = tls_strp_copyin_frag(strp, skb, in_skb, offset, in_len);
362	if (ret < 0) {
363		desc->error = ret;
364		ret = 0;
365	}
366
367	if (strp->stm.full_len && strp->stm.full_len == skb->len) {
368		desc->count = 0;
369
370		WRITE_ONCE(strp->msg_ready, 1);
371		tls_rx_msg_ready(strp);
372	}
373
374	return ret;
375}
376
377static int tls_strp_read_copyin(struct tls_strparser *strp)
378{
379	read_descriptor_t desc;
380
381	desc.arg.data = strp;
382	desc.error = 0;
383	desc.count = 1; /* give more than one skb per call */
384
385	/* sk should be locked here, so okay to do read_sock */
386	tcp_read_sock(strp->sk, &desc, tls_strp_copyin);
387
388	return desc.error;
389}
390
391static int tls_strp_read_copy(struct tls_strparser *strp, bool qshort)
392{
393	struct skb_shared_info *shinfo;
394	struct page *page;
395	int need_spc, len;
396
397	/* If the rbuf is small or rcv window has collapsed to 0 we need
398	 * to read the data out. Otherwise the connection will stall.
399	 * Without pressure threshold of INT_MAX will never be ready.
400	 */
401	if (likely(qshort && !tcp_epollin_ready(strp->sk, INT_MAX)))
402		return 0;
403
404	shinfo = skb_shinfo(strp->anchor);
405
406	/* If we don't know the length go max plus page for cipher overhead */
407	need_spc = strp->stm.full_len ?: TLS_MAX_PAYLOAD_SIZE + PAGE_SIZE;
408
409	for (len = need_spc; len > 0; len -= PAGE_SIZE) {
410		page = alloc_page(strp->sk->sk_allocation);
411		if (!page) {
412			tls_strp_flush_anchor_copy(strp);
413			return -ENOMEM;
414		}
415
416		skb_fill_page_desc(strp->anchor, shinfo->nr_frags++,
417				   page, 0, 0);
418	}
419
420	shinfo->frag_list = NULL;
421
422	strp->copy_mode = 1;
423	strp->stm.offset = 0;
424
425	strp->anchor->len = 0;
426	strp->anchor->data_len = 0;
427	strp->anchor->truesize = round_up(need_spc, PAGE_SIZE);
428
429	tls_strp_read_copyin(strp);
430
431	return 0;
432}
433
434static bool tls_strp_check_queue_ok(struct tls_strparser *strp)
435{
436	unsigned int len = strp->stm.offset + strp->stm.full_len;
437	struct sk_buff *first, *skb;
438	u32 seq;
439
440	first = skb_shinfo(strp->anchor)->frag_list;
441	skb = first;
442	seq = TCP_SKB_CB(first)->seq;
443
444	/* Make sure there's no duplicate data in the queue,
445	 * and the decrypted status matches.
446	 */
447	while (skb->len < len) {
448		seq += skb->len;
449		len -= skb->len;
450		skb = skb->next;
451
452		if (TCP_SKB_CB(skb)->seq != seq)
453			return false;
454		if (skb_cmp_decrypted(first, skb))
455			return false;
456	}
457
458	return true;
459}
460
461static void tls_strp_load_anchor_with_queue(struct tls_strparser *strp, int len)
462{
463	struct tcp_sock *tp = tcp_sk(strp->sk);
464	struct sk_buff *first;
465	u32 offset;
466
467	first = tcp_recv_skb(strp->sk, tp->copied_seq, &offset);
468	if (WARN_ON_ONCE(!first))
469		return;
470
471	/* Bestow the state onto the anchor */
472	strp->anchor->len = offset + len;
473	strp->anchor->data_len = offset + len;
474	strp->anchor->truesize = offset + len;
475
476	skb_shinfo(strp->anchor)->frag_list = first;
477
478	skb_copy_header(strp->anchor, first);
479	strp->anchor->destructor = NULL;
480
481	strp->stm.offset = offset;
482}
483
484bool tls_strp_msg_load(struct tls_strparser *strp, bool force_refresh)
485{
486	struct strp_msg *rxm;
487	struct tls_msg *tlm;
488
489	DEBUG_NET_WARN_ON_ONCE(!strp->msg_ready);
490	DEBUG_NET_WARN_ON_ONCE(!strp->stm.full_len);
491
492	if (!strp->copy_mode && force_refresh) {
493		if (unlikely(tcp_inq(strp->sk) < strp->stm.full_len)) {
494			WRITE_ONCE(strp->msg_ready, 0);
495			memset(&strp->stm, 0, sizeof(strp->stm));
496			return false;
497		}
498
499		tls_strp_load_anchor_with_queue(strp, strp->stm.full_len);
500	}
501
502	rxm = strp_msg(strp->anchor);
503	rxm->full_len	= strp->stm.full_len;
504	rxm->offset	= strp->stm.offset;
505	tlm = tls_msg(strp->anchor);
506	tlm->control	= strp->mark;
507
508	return true;
509}
510
511/* Called with lock held on lower socket */
512static int tls_strp_read_sock(struct tls_strparser *strp)
513{
514	int sz, inq;
515
516	inq = tcp_inq(strp->sk);
517	if (inq < 1)
518		return 0;
519
520	if (unlikely(strp->copy_mode))
521		return tls_strp_read_copyin(strp);
522
523	if (inq < strp->stm.full_len)
524		return tls_strp_read_copy(strp, true);
525
526	tls_strp_load_anchor_with_queue(strp, inq);
527	if (!strp->stm.full_len) {
528		sz = tls_rx_msg_size(strp, strp->anchor);
529		if (sz < 0)
530			return sz;
531
532		strp->stm.full_len = sz;
533
534		if (!strp->stm.full_len || inq < strp->stm.full_len)
535			return tls_strp_read_copy(strp, true);
536	}
537
538	if (!tls_strp_check_queue_ok(strp))
539		return tls_strp_read_copy(strp, false);
540
541	WRITE_ONCE(strp->msg_ready, 1);
542	tls_rx_msg_ready(strp);
543
544	return 0;
545}
546
547void tls_strp_check_rcv(struct tls_strparser *strp)
548{
549	if (unlikely(strp->stopped) || strp->msg_ready)
550		return;
551
552	if (tls_strp_read_sock(strp) == -ENOMEM)
553		queue_work(tls_strp_wq, &strp->work);
554}
555
556/* Lower sock lock held */
557void tls_strp_data_ready(struct tls_strparser *strp)
558{
559	/* This check is needed to synchronize with do_tls_strp_work.
560	 * do_tls_strp_work acquires a process lock (lock_sock) whereas
561	 * the lock held here is bh_lock_sock. The two locks can be
562	 * held by different threads at the same time, but bh_lock_sock
563	 * allows a thread in BH context to safely check if the process
564	 * lock is held. In this case, if the lock is held, queue work.
565	 */
566	if (sock_owned_by_user_nocheck(strp->sk)) {
567		queue_work(tls_strp_wq, &strp->work);
568		return;
569	}
570
571	tls_strp_check_rcv(strp);
572}
573
574static void tls_strp_work(struct work_struct *w)
575{
576	struct tls_strparser *strp =
577		container_of(w, struct tls_strparser, work);
578
579	lock_sock(strp->sk);
580	tls_strp_check_rcv(strp);
581	release_sock(strp->sk);
582}
583
584void tls_strp_msg_done(struct tls_strparser *strp)
585{
586	WARN_ON(!strp->stm.full_len);
587
588	if (likely(!strp->copy_mode))
589		tcp_read_done(strp->sk, strp->stm.full_len);
590	else
591		tls_strp_flush_anchor_copy(strp);
592
593	WRITE_ONCE(strp->msg_ready, 0);
594	memset(&strp->stm, 0, sizeof(strp->stm));
595
596	tls_strp_check_rcv(strp);
597}
598
599void tls_strp_stop(struct tls_strparser *strp)
600{
601	strp->stopped = 1;
602}
603
604int tls_strp_init(struct tls_strparser *strp, struct sock *sk)
605{
606	memset(strp, 0, sizeof(*strp));
607
608	strp->sk = sk;
609
610	strp->anchor = alloc_skb(0, GFP_KERNEL);
611	if (!strp->anchor)
612		return -ENOMEM;
613
614	INIT_WORK(&strp->work, tls_strp_work);
615
616	return 0;
617}
618
619/* strp must already be stopped so that tls_strp_recv will no longer be called.
620 * Note that tls_strp_done is not called with the lower socket held.
621 */
622void tls_strp_done(struct tls_strparser *strp)
623{
624	WARN_ON(!strp->stopped);
625
626	cancel_work_sync(&strp->work);
627	tls_strp_anchor_free(strp);
628}
629
630int __init tls_strp_dev_init(void)
631{
632	tls_strp_wq = create_workqueue("tls-strp");
633	if (unlikely(!tls_strp_wq))
634		return -ENOMEM;
635
636	return 0;
637}
638
639void tls_strp_dev_exit(void)
640{
641	destroy_workqueue(tls_strp_wq);
642}