call_accept.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2/* incoming call handling
  3 *
  4 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
  5 * Written by David Howells (dhowells@redhat.com)
  6 */
  7
  8#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  9
 10#include <linux/module.h>
 11#include <linux/net.h>
 12#include <linux/skbuff.h>
 13#include <linux/errqueue.h>
 14#include <linux/udp.h>
 15#include <linux/in.h>
 16#include <linux/in6.h>
 17#include <linux/icmp.h>
 18#include <linux/gfp.h>
 19#include <linux/circ_buf.h>
 20#include <net/sock.h>
 21#include <net/af_rxrpc.h>
 22#include <net/ip.h>
 23#include "ar-internal.h"
 24
 25static void rxrpc_dummy_notify(struct sock *sk, struct rxrpc_call *call,
 26			       unsigned long user_call_ID)
 27{
 28}
 29
 30/*
 31 * Preallocate a single service call, connection and peer and, if possible,
 32 * give them a user ID and attach the user's side of the ID to them.
 33 */
 34static int rxrpc_service_prealloc_one(struct rxrpc_sock *rx,
 35				      struct rxrpc_backlog *b,
 36				      rxrpc_notify_rx_t notify_rx,
 37				      unsigned long user_call_ID, gfp_t gfp,
 38				      unsigned int debug_id)
 39{
 40	struct rxrpc_call *call, *xcall;
 41	struct rxrpc_net *rxnet = rxrpc_net(sock_net(&rx->sk));
 42	struct rb_node *parent, **pp;
 43	int max, tmp;
 44	unsigned int size = RXRPC_BACKLOG_MAX;
 45	unsigned int head, tail, call_head, call_tail;
 46
 47	max = rx->sk.sk_max_ack_backlog;
 48	tmp = rx->sk.sk_ack_backlog;
 49	if (tmp >= max) {
 50		_leave(" = -ENOBUFS [full %u]", max);
 51		return -ENOBUFS;
 52	}
 53	max -= tmp;
 54
 55	/* We don't need more conns and peers than we have calls, but on the
 56	 * other hand, we shouldn't ever use more peers than conns or conns
 57	 * than calls.
 58	 */
 59	call_head = b->call_backlog_head;
 60	call_tail = READ_ONCE(b->call_backlog_tail);
 61	tmp = CIRC_CNT(call_head, call_tail, size);
 62	if (tmp >= max) {
 63		_leave(" = -ENOBUFS [enough %u]", tmp);
 64		return -ENOBUFS;
 65	}
 66	max = tmp + 1;
 67
 68	head = b->peer_backlog_head;
 69	tail = READ_ONCE(b->peer_backlog_tail);
 70	if (CIRC_CNT(head, tail, size) < max) {
 71		struct rxrpc_peer *peer;
 72
 73		peer = rxrpc_alloc_peer(rx->local, gfp, rxrpc_peer_new_prealloc);
 74		if (!peer)
 75			return -ENOMEM;
 76		b->peer_backlog[head] = peer;
 77		smp_store_release(&b->peer_backlog_head,
 78				  (head + 1) & (size - 1));
 79	}
 80
 81	head = b->conn_backlog_head;
 82	tail = READ_ONCE(b->conn_backlog_tail);
 83	if (CIRC_CNT(head, tail, size) < max) {
 84		struct rxrpc_connection *conn;
 85
 86		conn = rxrpc_prealloc_service_connection(rxnet, gfp);
 87		if (!conn)
 88			return -ENOMEM;
 89		b->conn_backlog[head] = conn;
 90		smp_store_release(&b->conn_backlog_head,
 91				  (head + 1) & (size - 1));
 92	}
 93
 94	/* Now it gets complicated, because calls get registered with the
 95	 * socket here, with a user ID preassigned by the user.
 96	 */
 97	call = rxrpc_alloc_call(rx, gfp, debug_id);
 98	if (!call)
 99		return -ENOMEM;
100	call->flags |= (1 << RXRPC_CALL_IS_SERVICE);
101	rxrpc_set_call_state(call, RXRPC_CALL_SERVER_PREALLOC);
102	__set_bit(RXRPC_CALL_EV_INITIAL_PING, &call->events);
103
104	trace_rxrpc_call(call->debug_id, refcount_read(&call->ref),
105			 user_call_ID, rxrpc_call_new_prealloc_service);
106
107	write_lock(&rx->call_lock);
108
109	/* Check the user ID isn't already in use */
110	pp = &rx->calls.rb_node;
111	parent = NULL;
112	while (*pp) {
113		parent = *pp;
114		xcall = rb_entry(parent, struct rxrpc_call, sock_node);
115		if (user_call_ID < xcall->user_call_ID)
116			pp = &(*pp)->rb_left;
117		else if (user_call_ID > xcall->user_call_ID)
118			pp = &(*pp)->rb_right;
119		else
120			goto id_in_use;
121	}
122
123	call->user_call_ID = user_call_ID;
124	call->notify_rx = notify_rx;
125	if (rx->app_ops &&
126	    rx->app_ops->user_attach_call) {
127		rxrpc_get_call(call, rxrpc_call_get_kernel_service);
128		rx->app_ops->user_attach_call(call, user_call_ID);
129	}
130
131	rxrpc_get_call(call, rxrpc_call_get_userid);
132	rb_link_node(&call->sock_node, parent, pp);
133	rb_insert_color(&call->sock_node, &rx->calls);
134	set_bit(RXRPC_CALL_HAS_USERID, &call->flags);
135
136	list_add(&call->sock_link, &rx->sock_calls);
137
138	write_unlock(&rx->call_lock);
139
140	rxnet = call->rxnet;
141	spin_lock(&rxnet->call_lock);
142	list_add_tail_rcu(&call->link, &rxnet->calls);
143	spin_unlock(&rxnet->call_lock);
144
145	b->call_backlog[call_head] = call;
146	smp_store_release(&b->call_backlog_head, (call_head + 1) & (size - 1));
147	_leave(" = 0 [%d -> %lx]", call->debug_id, user_call_ID);
148	return 0;
149
150id_in_use:
151	write_unlock(&rx->call_lock);
152	rxrpc_prefail_call(call, RXRPC_CALL_LOCAL_ERROR, -EBADSLT);
153	rxrpc_cleanup_call(call);
154	_leave(" = -EBADSLT");
155	return -EBADSLT;
156}
157
158/*
159 * Allocate the preallocation buffers for incoming service calls.  These must
160 * be charged manually.
161 */
162int rxrpc_service_prealloc(struct rxrpc_sock *rx, gfp_t gfp)
163{
164	struct rxrpc_backlog *b = rx->backlog;
165
166	if (!b) {
167		b = kzalloc(sizeof(struct rxrpc_backlog), gfp);
168		if (!b)
169			return -ENOMEM;
170		rx->backlog = b;
171	}
172
173	return 0;
174}
175
176/*
177 * Discard the preallocation on a service.
178 */
179void rxrpc_discard_prealloc(struct rxrpc_sock *rx)
180{
181	struct rxrpc_backlog *b = rx->backlog;
182	struct rxrpc_net *rxnet = rxrpc_net(sock_net(&rx->sk));
183	unsigned int size = RXRPC_BACKLOG_MAX, head, tail;
184
185	if (!b)
186		return;
187	rx->backlog = NULL;
188
189	/* Make sure that there aren't any incoming calls in progress before we
190	 * clear the preallocation buffers.
191	 */
192	spin_lock_irq(&rx->incoming_lock);
193	spin_unlock_irq(&rx->incoming_lock);
194
195	head = b->peer_backlog_head;
196	tail = b->peer_backlog_tail;
197	while (CIRC_CNT(head, tail, size) > 0) {
198		struct rxrpc_peer *peer = b->peer_backlog[tail];
199		rxrpc_put_local(peer->local, rxrpc_local_put_prealloc_peer);
200		kfree(peer);
201		tail = (tail + 1) & (size - 1);
202	}
203
204	head = b->conn_backlog_head;
205	tail = b->conn_backlog_tail;
206	while (CIRC_CNT(head, tail, size) > 0) {
207		struct rxrpc_connection *conn = b->conn_backlog[tail];
208		write_lock(&rxnet->conn_lock);
209		list_del(&conn->link);
210		list_del(&conn->proc_link);
211		write_unlock(&rxnet->conn_lock);
212		kfree(conn);
213		if (atomic_dec_and_test(&rxnet->nr_conns))
214			wake_up_var(&rxnet->nr_conns);
215		tail = (tail + 1) & (size - 1);
216	}
217
218	head = b->call_backlog_head;
219	tail = b->call_backlog_tail;
220	while (CIRC_CNT(head, tail, size) > 0) {
221		struct rxrpc_call *call = b->call_backlog[tail];
222		rxrpc_see_call(call, rxrpc_call_see_discard);
223		rcu_assign_pointer(call->socket, rx);
224		if (rx->app_ops &&
225		    rx->app_ops->discard_new_call) {
226			_debug("discard %lx", call->user_call_ID);
227			rx->app_ops->discard_new_call(call, call->user_call_ID);
228			if (call->notify_rx)
229				call->notify_rx = rxrpc_dummy_notify;
230			rxrpc_put_call(call, rxrpc_call_put_kernel);
231		}
232		rxrpc_call_completed(call);
233		rxrpc_release_call(rx, call);
234		rxrpc_put_call(call, rxrpc_call_put_discard_prealloc);
235		tail = (tail + 1) & (size - 1);
236	}
237
238	kfree(b);
239}
240
241/*
242 * Allocate a new incoming call from the prealloc pool, along with a connection
243 * and a peer as necessary.
244 */
245static struct rxrpc_call *rxrpc_alloc_incoming_call(struct rxrpc_sock *rx,
246						    struct rxrpc_local *local,
247						    struct rxrpc_peer *peer,
248						    struct rxrpc_connection *conn,
249						    const struct rxrpc_security *sec,
250						    struct sockaddr_rxrpc *peer_srx,
251						    struct sk_buff *skb)
252{
253	struct rxrpc_backlog *b = rx->backlog;
254	struct rxrpc_call *call;
255	unsigned short call_head, conn_head, peer_head;
256	unsigned short call_tail, conn_tail, peer_tail;
257	unsigned short call_count, conn_count;
258
259	if (!b)
260		return NULL;
261
262	/* #calls >= #conns >= #peers must hold true. */
263	call_head = smp_load_acquire(&b->call_backlog_head);
264	call_tail = b->call_backlog_tail;
265	call_count = CIRC_CNT(call_head, call_tail, RXRPC_BACKLOG_MAX);
266	conn_head = smp_load_acquire(&b->conn_backlog_head);
267	conn_tail = b->conn_backlog_tail;
268	conn_count = CIRC_CNT(conn_head, conn_tail, RXRPC_BACKLOG_MAX);
269	ASSERTCMP(conn_count, >=, call_count);
270	peer_head = smp_load_acquire(&b->peer_backlog_head);
271	peer_tail = b->peer_backlog_tail;
272	ASSERTCMP(CIRC_CNT(peer_head, peer_tail, RXRPC_BACKLOG_MAX), >=,
273		  conn_count);
274
275	if (call_count == 0)
276		return NULL;
277
278	if (!conn) {
279		if (peer && !rxrpc_get_peer_maybe(peer, rxrpc_peer_get_service_conn))
280			peer = NULL;
281		if (!peer) {
282			peer = b->peer_backlog[peer_tail];
283			peer->srx = *peer_srx;
284			b->peer_backlog[peer_tail] = NULL;
285			smp_store_release(&b->peer_backlog_tail,
286					  (peer_tail + 1) &
287					  (RXRPC_BACKLOG_MAX - 1));
288
289			rxrpc_new_incoming_peer(local, peer);
290		}
291
292		/* Now allocate and set up the connection */
293		conn = b->conn_backlog[conn_tail];
294		b->conn_backlog[conn_tail] = NULL;
295		smp_store_release(&b->conn_backlog_tail,
296				  (conn_tail + 1) & (RXRPC_BACKLOG_MAX - 1));
297		conn->local = rxrpc_get_local(local, rxrpc_local_get_prealloc_conn);
298		conn->peer = peer;
299		rxrpc_see_connection(conn, rxrpc_conn_see_new_service_conn);
300		rxrpc_new_incoming_connection(rx, conn, sec, skb);
301	} else {
302		rxrpc_get_connection(conn, rxrpc_conn_get_service_conn);
303		atomic_inc(&conn->active);
304	}
305
306	/* And now we can allocate and set up a new call */
307	call = b->call_backlog[call_tail];
308	b->call_backlog[call_tail] = NULL;
309	smp_store_release(&b->call_backlog_tail,
310			  (call_tail + 1) & (RXRPC_BACKLOG_MAX - 1));
311
312	rxrpc_see_call(call, rxrpc_call_see_accept);
313	call->local = rxrpc_get_local(conn->local, rxrpc_local_get_call);
314	call->conn = conn;
315	call->security = conn->security;
316	call->security_ix = conn->security_ix;
317	call->peer = rxrpc_get_peer(conn->peer, rxrpc_peer_get_accept);
318	call->dest_srx = peer->srx;
319	call->cong_ssthresh = call->peer->cong_ssthresh;
320	call->tx_last_sent = ktime_get_real();
321	return call;
322}
323
324/*
325 * Set up a new incoming call.  Called from the I/O thread.
326 *
327 * If this is for a kernel service, when we allocate the call, it will have
328 * three refs on it: (1) the kernel service, (2) the user_call_ID tree, (3) the
329 * retainer ref obtained from the backlog buffer.  Prealloc calls for userspace
330 * services only have the ref from the backlog buffer.
331 *
332 * If we want to report an error, we mark the skb with the packet type and
333 * abort code and return false.
334 */
335bool rxrpc_new_incoming_call(struct rxrpc_local *local,
336			     struct rxrpc_peer *peer,
337			     struct rxrpc_connection *conn,
338			     struct sockaddr_rxrpc *peer_srx,
339			     struct sk_buff *skb)
340{
341	const struct rxrpc_security *sec = NULL;
342	struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
343	struct rxrpc_call *call = NULL;
344	struct rxrpc_sock *rx;
345
346	_enter("");
347
348	/* Don't set up a call for anything other than a DATA packet. */
349	if (sp->hdr.type != RXRPC_PACKET_TYPE_DATA)
350		return rxrpc_protocol_error(skb, rxrpc_eproto_no_service_call);
351
352	read_lock_irq(&local->services_lock);
353
354	/* Weed out packets to services we're not offering.  Packets that would
355	 * begin a call are explicitly rejected and the rest are just
356	 * discarded.
357	 */
358	rx = local->service;
359	if (!rx || (sp->hdr.serviceId != rx->srx.srx_service &&
360		    sp->hdr.serviceId != rx->second_service)
361	    ) {
362		if (sp->hdr.type == RXRPC_PACKET_TYPE_DATA &&
363		    sp->hdr.seq == 1)
364			goto unsupported_service;
365		goto discard;
366	}
367
368	if (!conn) {
369		sec = rxrpc_get_incoming_security(rx, skb);
370		if (!sec)
371			goto unsupported_security;
372	}
373
374	spin_lock(&rx->incoming_lock);
375	if (rx->sk.sk_state == RXRPC_SERVER_LISTEN_DISABLED ||
376	    rx->sk.sk_state == RXRPC_CLOSE) {
377		rxrpc_direct_conn_abort(skb, rxrpc_abort_shut_down,
378					RX_INVALID_OPERATION, -ESHUTDOWN);
379		goto no_call;
380	}
381
382	call = rxrpc_alloc_incoming_call(rx, local, peer, conn, sec, peer_srx,
383					 skb);
384	if (!call) {
385		skb->mark = RXRPC_SKB_MARK_REJECT_BUSY;
386		goto no_call;
387	}
388
389	trace_rxrpc_receive(call, rxrpc_receive_incoming,
390			    sp->hdr.serial, sp->hdr.seq);
391
392	/* Make the call live. */
393	rxrpc_incoming_call(rx, call, skb);
394	conn = call->conn;
395
396	if (rx->app_ops &&
397	    rx->app_ops->notify_new_call)
398		rx->app_ops->notify_new_call(&rx->sk, call, call->user_call_ID);
399
400	spin_lock(&conn->state_lock);
401	if (conn->state == RXRPC_CONN_SERVICE_UNSECURED) {
402		conn->state = RXRPC_CONN_SERVICE_CHALLENGING;
403		set_bit(RXRPC_CONN_EV_CHALLENGE, &call->conn->events);
404		rxrpc_queue_conn(call->conn, rxrpc_conn_queue_challenge);
405	}
406	spin_unlock(&conn->state_lock);
407
408	spin_unlock(&rx->incoming_lock);
409	read_unlock_irq(&local->services_lock);
410	rxrpc_assess_MTU_size(local, call->peer);
411
412	if (hlist_unhashed(&call->error_link)) {
413		spin_lock_irq(&call->peer->lock);
414		hlist_add_head(&call->error_link, &call->peer->error_targets);
415		spin_unlock_irq(&call->peer->lock);
416	}
417
418	_leave(" = %p{%d}", call, call->debug_id);
419	rxrpc_queue_rx_call_packet(call, skb);
420	rxrpc_put_call(call, rxrpc_call_put_input);
421	return true;
422
423unsupported_service:
424	read_unlock_irq(&local->services_lock);
425	return rxrpc_direct_conn_abort(skb, rxrpc_abort_service_not_offered,
426				       RX_INVALID_OPERATION, -EOPNOTSUPP);
427unsupported_security:
428	read_unlock_irq(&local->services_lock);
429	return rxrpc_direct_conn_abort(skb, rxrpc_abort_service_not_offered,
430				       RX_INVALID_OPERATION, -EKEYREJECTED);
431no_call:
432	spin_unlock(&rx->incoming_lock);
433	read_unlock_irq(&local->services_lock);
434	_leave(" = f [%u]", skb->mark);
435	return false;
436discard:
437	read_unlock_irq(&local->services_lock);
438	return true;
439}
440
441/*
442 * Charge up socket with preallocated calls, attaching user call IDs.
443 */
444int rxrpc_user_charge_accept(struct rxrpc_sock *rx, unsigned long user_call_ID)
445{
446	struct rxrpc_backlog *b = rx->backlog;
447
448	if (rx->sk.sk_state == RXRPC_CLOSE)
449		return -ESHUTDOWN;
450
451	return rxrpc_service_prealloc_one(rx, b, NULL, user_call_ID, GFP_KERNEL,
452					  atomic_inc_return(&rxrpc_debug_id));
453}
454
455/*
456 * rxrpc_kernel_charge_accept - Charge up socket with preallocated calls
457 * @sock: The socket on which to preallocate
458 * @notify_rx: Event notification function for the call
459 * @user_call_ID: The tag to attach to the preallocated call
460 * @gfp: The allocation conditions.
461 * @debug_id: The tracing debug ID.
462 *
463 * Charge up the socket with preallocated calls, each with a user ID.  The
464 * ->user_attach_call() callback function should be provided to effect the
465 * attachment from the user's side.  The user is given a ref to hold on the
466 * call.
467 *
468 * Note that the call may be come connected before this function returns.
469 */
470int rxrpc_kernel_charge_accept(struct socket *sock, rxrpc_notify_rx_t notify_rx,
471			       unsigned long user_call_ID, gfp_t gfp,
472			       unsigned int debug_id)
473{
474	struct rxrpc_sock *rx = rxrpc_sk(sock->sk);
475	struct rxrpc_backlog *b = rx->backlog;
476
477	if (sock->sk->sk_state == RXRPC_CLOSE)
478		return -ESHUTDOWN;
479
480	return rxrpc_service_prealloc_one(rx, b, notify_rx, user_call_ID,
481					  gfp, debug_id);
482}
483EXPORT_SYMBOL(rxrpc_kernel_charge_accept);