transport.c

   1// SPDX-License-Identifier: LGPL-2.1
   2/*
   3 *
   4 *   Copyright (C) International Business Machines  Corp., 2002,2008
   5 *   Author(s): Steve French (sfrench@us.ibm.com)
   6 *   Jeremy Allison (jra@samba.org) 2006.
   7 *
   8 */
   9
  10#include <linux/fs.h>
  11#include <linux/list.h>
  12#include <linux/gfp.h>
  13#include <linux/wait.h>
  14#include <linux/net.h>
  15#include <linux/delay.h>
  16#include <linux/freezer.h>
  17#include <linux/tcp.h>
  18#include <linux/bvec.h>
  19#include <linux/highmem.h>
  20#include <linux/uaccess.h>
  21#include <linux/processor.h>
  22#include <linux/mempool.h>
  23#include <linux/sched/signal.h>
  24#include <linux/task_io_accounting_ops.h>
  25#include <linux/task_work.h>
  26#include "cifspdu.h"
  27#include "cifsglob.h"
  28#include "cifsproto.h"
  29#include "cifs_debug.h"
  30#include "smb2proto.h"
  31#include "smbdirect.h"
  32#include "compress.h"
  33
  34void
  35cifs_wake_up_task(struct TCP_Server_Info *server, struct mid_q_entry *mid)
  36{
  37	if (mid->mid_state == MID_RESPONSE_RECEIVED)
  38		mid->mid_state = MID_RESPONSE_READY;
  39	wake_up_process(mid->callback_data);
  40}
  41
  42void __release_mid(struct TCP_Server_Info *server, struct mid_q_entry *midEntry)
  43{
  44#ifdef CONFIG_CIFS_STATS2
  45	__le16 command = server->vals->lock_cmd;
  46	__u16 smb_cmd = le16_to_cpu(midEntry->command);
  47	unsigned long now;
  48	unsigned long roundtrip_time;
  49#endif
  50
  51	if (midEntry->resp_buf && (midEntry->wait_cancelled) &&
  52	    (midEntry->mid_state == MID_RESPONSE_RECEIVED ||
  53	     midEntry->mid_state == MID_RESPONSE_READY) &&
  54	    server->ops->handle_cancelled_mid)
  55		server->ops->handle_cancelled_mid(midEntry, server);
  56
  57	midEntry->mid_state = MID_FREE;
  58	atomic_dec(&mid_count);
  59	if (midEntry->large_buf)
  60		cifs_buf_release(midEntry->resp_buf);
  61	else
  62		cifs_small_buf_release(midEntry->resp_buf);
  63#ifdef CONFIG_CIFS_STATS2
  64	now = jiffies;
  65	if (now < midEntry->when_alloc)
  66		cifs_server_dbg(VFS, "Invalid mid allocation time\n");
  67	roundtrip_time = now - midEntry->when_alloc;
  68
  69	if (smb_cmd < NUMBER_OF_SMB2_COMMANDS) {
  70		if (atomic_read(&server->num_cmds[smb_cmd]) == 0) {
  71			server->slowest_cmd[smb_cmd] = roundtrip_time;
  72			server->fastest_cmd[smb_cmd] = roundtrip_time;
  73		} else {
  74			if (server->slowest_cmd[smb_cmd] < roundtrip_time)
  75				server->slowest_cmd[smb_cmd] = roundtrip_time;
  76			else if (server->fastest_cmd[smb_cmd] > roundtrip_time)
  77				server->fastest_cmd[smb_cmd] = roundtrip_time;
  78		}
  79		cifs_stats_inc(&server->num_cmds[smb_cmd]);
  80		server->time_per_cmd[smb_cmd] += roundtrip_time;
  81	}
  82	/*
  83	 * commands taking longer than one second (default) can be indications
  84	 * that something is wrong, unless it is quite a slow link or a very
  85	 * busy server. Note that this calc is unlikely or impossible to wrap
  86	 * as long as slow_rsp_threshold is not set way above recommended max
  87	 * value (32767 ie 9 hours) and is generally harmless even if wrong
  88	 * since only affects debug counters - so leaving the calc as simple
  89	 * comparison rather than doing multiple conversions and overflow
  90	 * checks
  91	 */
  92	if ((slow_rsp_threshold != 0) &&
  93	    time_after(now, midEntry->when_alloc + (slow_rsp_threshold * HZ)) &&
  94	    (midEntry->command != command)) {
  95		/*
  96		 * smb2slowcmd[NUMBER_OF_SMB2_COMMANDS] counts by command
  97		 * NB: le16_to_cpu returns unsigned so can not be negative below
  98		 */
  99		if (smb_cmd < NUMBER_OF_SMB2_COMMANDS)
 100			cifs_stats_inc(&server->smb2slowcmd[smb_cmd]);
 101
 102		trace_smb3_slow_rsp(smb_cmd, midEntry->mid, midEntry->pid,
 103			       midEntry->when_sent, midEntry->when_received);
 104		if (cifsFYI & CIFS_TIMER) {
 105			pr_debug("slow rsp: cmd %d mid %llu",
 106				 midEntry->command, midEntry->mid);
 107			cifs_info("A: 0x%lx S: 0x%lx R: 0x%lx\n",
 108				  now - midEntry->when_alloc,
 109				  now - midEntry->when_sent,
 110				  now - midEntry->when_received);
 111		}
 112	}
 113#endif
 114	put_task_struct(midEntry->creator);
 115
 116	mempool_free(midEntry, &cifs_mid_pool);
 117}
 118
 119void
 120delete_mid(struct TCP_Server_Info *server, struct mid_q_entry *mid)
 121{
 122	spin_lock(&server->mid_queue_lock);
 123
 124	if (!mid->deleted_from_q) {
 125		list_del_init(&mid->qhead);
 126		mid->deleted_from_q = true;
 127	}
 128	spin_unlock(&server->mid_queue_lock);
 129
 130	release_mid(server, mid);
 131}
 132
 133/*
 134 * smb_send_kvec - send an array of kvecs to the server
 135 * @server:	Server to send the data to
 136 * @smb_msg:	Message to send
 137 * @sent:	amount of data sent on socket is stored here
 138 *
 139 * Our basic "send data to server" function. Should be called with srv_mutex
 140 * held. The caller is responsible for handling the results.
 141 */
 142int
 143smb_send_kvec(struct TCP_Server_Info *server, struct msghdr *smb_msg,
 144	      size_t *sent)
 145{
 146	int rc = 0;
 147	int retries = 0;
 148	struct socket *ssocket = server->ssocket;
 149
 150	*sent = 0;
 151
 152	if (server->noblocksnd)
 153		smb_msg->msg_flags = MSG_DONTWAIT + MSG_NOSIGNAL;
 154	else
 155		smb_msg->msg_flags = MSG_NOSIGNAL;
 156
 157	while (msg_data_left(smb_msg)) {
 158		/*
 159		 * If blocking send, we try 3 times, since each can block
 160		 * for 5 seconds. For nonblocking  we have to try more
 161		 * but wait increasing amounts of time allowing time for
 162		 * socket to clear.  The overall time we wait in either
 163		 * case to send on the socket is about 15 seconds.
 164		 * Similarly we wait for 15 seconds for a response from
 165		 * the server in SendReceive[2] for the server to send
 166		 * a response back for most types of requests (except
 167		 * SMB Write past end of file which can be slow, and
 168		 * blocking lock operations). NFS waits slightly longer
 169		 * than CIFS, but this can make it take longer for
 170		 * nonresponsive servers to be detected and 15 seconds
 171		 * is more than enough time for modern networks to
 172		 * send a packet.  In most cases if we fail to send
 173		 * after the retries we will kill the socket and
 174		 * reconnect which may clear the network problem.
 175		 *
 176		 * Even if regular signals are masked, EINTR might be
 177		 * propagated from sk_stream_wait_memory() to here when
 178		 * TIF_NOTIFY_SIGNAL is used for task work. For example,
 179		 * certain io_uring completions will use that. Treat
 180		 * having EINTR with pending task work the same as EAGAIN
 181		 * to avoid unnecessary reconnects.
 182		 */
 183		rc = sock_sendmsg(ssocket, smb_msg);
 184		if (rc == -EAGAIN || unlikely(rc == -EINTR && task_work_pending(current))) {
 185			retries++;
 186			if (retries >= 14 ||
 187			    (!server->noblocksnd && (retries > 2))) {
 188				cifs_server_dbg(VFS, "sends on sock %p stuck for 15 seconds\n",
 189					 ssocket);
 190				return -EAGAIN;
 191			}
 192			msleep(1 << retries);
 193			continue;
 194		}
 195
 196		if (rc < 0)
 197			return rc;
 198
 199		if (rc == 0) {
 200			/* should never happen, letting socket clear before
 201			   retrying is our only obvious option here */
 202			cifs_server_dbg(VFS, "tcp sent no data\n");
 203			msleep(500);
 204			continue;
 205		}
 206
 207		/* send was at least partially successful */
 208		*sent += rc;
 209		retries = 0; /* in case we get ENOSPC on the next send */
 210	}
 211	return 0;
 212}
 213
 214unsigned long
 215smb_rqst_len(struct TCP_Server_Info *server, struct smb_rqst *rqst)
 216{
 217	unsigned int i;
 218	struct kvec *iov;
 219	int nvec;
 220	unsigned long buflen = 0;
 221
 222	if (!is_smb1(server) && rqst->rq_nvec >= 2 &&
 223	    rqst->rq_iov[0].iov_len == 4) {
 224		iov = &rqst->rq_iov[1];
 225		nvec = rqst->rq_nvec - 1;
 226	} else {
 227		iov = rqst->rq_iov;
 228		nvec = rqst->rq_nvec;
 229	}
 230
 231	/* total up iov array first */
 232	for (i = 0; i < nvec; i++)
 233		buflen += iov[i].iov_len;
 234
 235	buflen += iov_iter_count(&rqst->rq_iter);
 236	return buflen;
 237}
 238
 239int __smb_send_rqst(struct TCP_Server_Info *server, int num_rqst,
 240		    struct smb_rqst *rqst)
 241{
 242	int rc;
 243	struct kvec *iov;
 244	int n_vec;
 245	unsigned int send_length = 0;
 246	unsigned int i, j;
 247	sigset_t mask, oldmask;
 248	size_t total_len = 0, sent, size;
 249	struct socket *ssocket = server->ssocket;
 250	struct msghdr smb_msg = {};
 251	__be32 rfc1002_marker;
 252
 253	cifs_in_send_inc(server);
 254	if (cifs_rdma_enabled(server)) {
 255		/* return -EAGAIN when connecting or reconnecting */
 256		rc = -EAGAIN;
 257		if (server->smbd_conn)
 258			rc = smbd_send(server, num_rqst, rqst);
 259		goto smbd_done;
 260	}
 261
 262	rc = -EAGAIN;
 263	if (ssocket == NULL)
 264		goto out;
 265
 266	rc = -ERESTARTSYS;
 267	if (fatal_signal_pending(current)) {
 268		cifs_dbg(FYI, "signal pending before send request\n");
 269		goto out;
 270	}
 271
 272	rc = 0;
 273	/* cork the socket */
 274	tcp_sock_set_cork(ssocket->sk, true);
 275
 276	for (j = 0; j < num_rqst; j++)
 277		send_length += smb_rqst_len(server, &rqst[j]);
 278	rfc1002_marker = cpu_to_be32(send_length);
 279
 280	/*
 281	 * We should not allow signals to interrupt the network send because
 282	 * any partial send will cause session reconnects thus increasing
 283	 * latency of system calls and overload a server with unnecessary
 284	 * requests.
 285	 */
 286
 287	sigfillset(&mask);
 288	sigprocmask(SIG_BLOCK, &mask, &oldmask);
 289
 290	/* Generate a rfc1002 marker */
 291	{
 292		struct kvec hiov = {
 293			.iov_base = &rfc1002_marker,
 294			.iov_len  = 4
 295		};
 296		iov_iter_kvec(&smb_msg.msg_iter, ITER_SOURCE, &hiov, 1, 4);
 297		rc = smb_send_kvec(server, &smb_msg, &sent);
 298		if (rc < 0)
 299			goto unmask;
 300
 301		total_len += sent;
 302		send_length += 4;
 303	}
 304
 305	cifs_dbg(FYI, "Sending smb: smb_len=%u\n", send_length);
 306
 307	for (j = 0; j < num_rqst; j++) {
 308		iov = rqst[j].rq_iov;
 309		n_vec = rqst[j].rq_nvec;
 310
 311		size = 0;
 312		for (i = 0; i < n_vec; i++) {
 313			dump_smb(iov[i].iov_base, iov[i].iov_len);
 314			size += iov[i].iov_len;
 315		}
 316
 317		iov_iter_kvec(&smb_msg.msg_iter, ITER_SOURCE, iov, n_vec, size);
 318
 319		rc = smb_send_kvec(server, &smb_msg, &sent);
 320		if (rc < 0)
 321			goto unmask;
 322
 323		total_len += sent;
 324
 325		if (iov_iter_count(&rqst[j].rq_iter) > 0) {
 326			smb_msg.msg_iter = rqst[j].rq_iter;
 327			rc = smb_send_kvec(server, &smb_msg, &sent);
 328			if (rc < 0)
 329				break;
 330			total_len += sent;
 331		}
 332	}
 333
 334unmask:
 335	sigprocmask(SIG_SETMASK, &oldmask, NULL);
 336
 337	/*
 338	 * If signal is pending but we have already sent the whole packet to
 339	 * the server we need to return success status to allow a corresponding
 340	 * mid entry to be kept in the pending requests queue thus allowing
 341	 * to handle responses from the server by the client.
 342	 *
 343	 * If only part of the packet has been sent there is no need to hide
 344	 * interrupt because the session will be reconnected anyway, so there
 345	 * won't be any response from the server to handle.
 346	 */
 347
 348	if (signal_pending(current) && (total_len != send_length)) {
 349		cifs_dbg(FYI, "signal is pending after attempt to send\n");
 350		rc = -ERESTARTSYS;
 351	}
 352
 353	/* uncork it */
 354	tcp_sock_set_cork(ssocket->sk, false);
 355
 356	if ((total_len > 0) && (total_len != send_length)) {
 357		cifs_dbg(FYI, "partial send (wanted=%u sent=%zu): terminating session\n",
 358			 send_length, total_len);
 359		/*
 360		 * If we have only sent part of an SMB then the next SMB could
 361		 * be taken as the remainder of this one. We need to kill the
 362		 * socket so the server throws away the partial SMB
 363		 */
 364		cifs_signal_cifsd_for_reconnect(server, false);
 365		trace_smb3_partial_send_reconnect(server->current_mid,
 366						  server->conn_id, server->hostname);
 367	}
 368smbd_done:
 369	/*
 370	 * there's hardly any use for the layers above to know the
 371	 * actual error code here. All they should do at this point is
 372	 * to retry the connection and hope it goes away.
 373	 */
 374	if (rc < 0 && rc != -EINTR && rc != -EAGAIN) {
 375		cifs_server_dbg(VFS, "Error %d sending data on socket to server\n",
 376			 rc);
 377		rc = -ECONNABORTED;
 378		cifs_signal_cifsd_for_reconnect(server, false);
 379	} else if (rc > 0)
 380		rc = 0;
 381out:
 382	cifs_in_send_dec(server);
 383	return rc;
 384}
 385
 386static int
 387smb_send_rqst(struct TCP_Server_Info *server, int num_rqst,
 388	      struct smb_rqst *rqst, int flags)
 389{
 390	struct smb2_transform_hdr tr_hdr;
 391	struct smb_rqst new_rqst[MAX_COMPOUND] = {};
 392	struct kvec iov = {
 393		.iov_base = &tr_hdr,
 394		.iov_len = sizeof(tr_hdr),
 395	};
 396	int rc;
 397
 398	if (flags & CIFS_COMPRESS_REQ)
 399		return smb_compress(server, &rqst[0], __smb_send_rqst);
 400
 401	if (!(flags & CIFS_TRANSFORM_REQ))
 402		return __smb_send_rqst(server, num_rqst, rqst);
 403
 404	if (WARN_ON_ONCE(num_rqst > MAX_COMPOUND - 1))
 405		return smb_EIO1(smb_eio_trace_tx_max_compound, num_rqst);
 406
 407	if (!server->ops->init_transform_rq) {
 408		cifs_server_dbg(VFS, "Encryption requested but transform callback is missing\n");
 409		return smb_EIO(smb_eio_trace_tx_need_transform);
 410	}
 411
 412	new_rqst[0].rq_iov = &iov;
 413	new_rqst[0].rq_nvec = 1;
 414
 415	rc = server->ops->init_transform_rq(server, num_rqst + 1,
 416					    new_rqst, rqst);
 417	if (!rc) {
 418		rc = __smb_send_rqst(server, num_rqst + 1, new_rqst);
 419		smb3_free_compound_rqst(num_rqst, &new_rqst[1]);
 420	}
 421	return rc;
 422}
 423
 424static int
 425wait_for_free_credits(struct TCP_Server_Info *server, const int num_credits,
 426		      const int timeout, const int flags,
 427		      unsigned int *instance)
 428{
 429	long rc;
 430	int *credits;
 431	int optype;
 432	long int t;
 433	int scredits, in_flight;
 434
 435	if (timeout < 0)
 436		t = MAX_JIFFY_OFFSET;
 437	else
 438		t = msecs_to_jiffies(timeout);
 439
 440	optype = flags & CIFS_OP_MASK;
 441
 442	*instance = 0;
 443
 444	credits = server->ops->get_credits_field(server, optype);
 445	/* Since an echo is already inflight, no need to wait to send another */
 446	if (*credits <= 0 && optype == CIFS_ECHO_OP)
 447		return -EAGAIN;
 448
 449	spin_lock(&server->req_lock);
 450	if ((flags & CIFS_TIMEOUT_MASK) == CIFS_NON_BLOCKING) {
 451		/* oplock breaks must not be held up */
 452		server->in_flight++;
 453		if (server->in_flight > server->max_in_flight)
 454			server->max_in_flight = server->in_flight;
 455		*credits -= 1;
 456		*instance = server->reconnect_instance;
 457		scredits = *credits;
 458		in_flight = server->in_flight;
 459		spin_unlock(&server->req_lock);
 460
 461		trace_smb3_nblk_credits(server->current_mid,
 462				server->conn_id, server->hostname, scredits, -1, in_flight);
 463		cifs_dbg(FYI, "%s: remove %u credits total=%d\n",
 464				__func__, 1, scredits);
 465
 466		return 0;
 467	}
 468
 469	while (1) {
 470		spin_unlock(&server->req_lock);
 471
 472		spin_lock(&server->srv_lock);
 473		if (server->tcpStatus == CifsExiting) {
 474			spin_unlock(&server->srv_lock);
 475			return -ENOENT;
 476		}
 477		spin_unlock(&server->srv_lock);
 478
 479		spin_lock(&server->req_lock);
 480		if (*credits < num_credits) {
 481			scredits = *credits;
 482			spin_unlock(&server->req_lock);
 483
 484			cifs_num_waiters_inc(server);
 485			rc = wait_event_killable_timeout(server->request_q,
 486				has_credits(server, credits, num_credits), t);
 487			cifs_num_waiters_dec(server);
 488			if (!rc) {
 489				spin_lock(&server->req_lock);
 490				scredits = *credits;
 491				in_flight = server->in_flight;
 492				spin_unlock(&server->req_lock);
 493
 494				trace_smb3_credit_timeout(server->current_mid,
 495						server->conn_id, server->hostname, scredits,
 496						num_credits, in_flight);
 497				cifs_server_dbg(VFS, "wait timed out after %d ms\n",
 498						timeout);
 499				return -EBUSY;
 500			}
 501			if (rc == -ERESTARTSYS)
 502				return -ERESTARTSYS;
 503			spin_lock(&server->req_lock);
 504		} else {
 505			/*
 506			 * For normal commands, reserve the last MAX_COMPOUND
 507			 * credits to compound requests.
 508			 * Otherwise these compounds could be permanently
 509			 * starved for credits by single-credit requests.
 510			 *
 511			 * To prevent spinning CPU, block this thread until
 512			 * there are >MAX_COMPOUND credits available.
 513			 * But only do this is we already have a lot of
 514			 * credits in flight to avoid triggering this check
 515			 * for servers that are slow to hand out credits on
 516			 * new sessions.
 517			 */
 518			if (!optype && num_credits == 1 &&
 519			    server->in_flight > 2 * MAX_COMPOUND &&
 520			    *credits <= MAX_COMPOUND) {
 521				spin_unlock(&server->req_lock);
 522
 523				cifs_num_waiters_inc(server);
 524				rc = wait_event_killable_timeout(
 525					server->request_q,
 526					has_credits(server, credits,
 527						    MAX_COMPOUND + 1),
 528					t);
 529				cifs_num_waiters_dec(server);
 530				if (!rc) {
 531					spin_lock(&server->req_lock);
 532					scredits = *credits;
 533					in_flight = server->in_flight;
 534					spin_unlock(&server->req_lock);
 535
 536					trace_smb3_credit_timeout(
 537							server->current_mid,
 538							server->conn_id, server->hostname,
 539							scredits, num_credits, in_flight);
 540					cifs_server_dbg(VFS, "wait timed out after %d ms\n",
 541							timeout);
 542					return -EBUSY;
 543				}
 544				if (rc == -ERESTARTSYS)
 545					return -ERESTARTSYS;
 546				spin_lock(&server->req_lock);
 547				continue;
 548			}
 549
 550			/*
 551			 * Can not count locking commands against total
 552			 * as they are allowed to block on server.
 553			 */
 554
 555			/* update # of requests on the wire to server */
 556			if ((flags & CIFS_TIMEOUT_MASK) != CIFS_BLOCKING_OP) {
 557				*credits -= num_credits;
 558				server->in_flight += num_credits;
 559				if (server->in_flight > server->max_in_flight)
 560					server->max_in_flight = server->in_flight;
 561				*instance = server->reconnect_instance;
 562			}
 563			scredits = *credits;
 564			in_flight = server->in_flight;
 565			spin_unlock(&server->req_lock);
 566
 567			trace_smb3_waitff_credits(server->current_mid,
 568					server->conn_id, server->hostname, scredits,
 569					-(num_credits), in_flight);
 570			cifs_dbg(FYI, "%s: remove %u credits total=%d\n",
 571					__func__, num_credits, scredits);
 572			break;
 573		}
 574	}
 575	return 0;
 576}
 577
 578int wait_for_free_request(struct TCP_Server_Info *server, const int flags,
 579			  unsigned int *instance)
 580{
 581	return wait_for_free_credits(server, 1, -1, flags,
 582				     instance);
 583}
 584
 585static int
 586wait_for_compound_request(struct TCP_Server_Info *server, int num,
 587			  const int flags, unsigned int *instance)
 588{
 589	int *credits;
 590	int scredits, in_flight;
 591
 592	credits = server->ops->get_credits_field(server, flags & CIFS_OP_MASK);
 593
 594	spin_lock(&server->req_lock);
 595	scredits = *credits;
 596	in_flight = server->in_flight;
 597
 598	if (*credits < num) {
 599		/*
 600		 * If the server is tight on resources or just gives us less
 601		 * credits for other reasons (e.g. requests are coming out of
 602		 * order and the server delays granting more credits until it
 603		 * processes a missing mid) and we exhausted most available
 604		 * credits there may be situations when we try to send
 605		 * a compound request but we don't have enough credits. At this
 606		 * point the client needs to decide if it should wait for
 607		 * additional credits or fail the request. If at least one
 608		 * request is in flight there is a high probability that the
 609		 * server will return enough credits to satisfy this compound
 610		 * request.
 611		 *
 612		 * Return immediately if no requests in flight since we will be
 613		 * stuck on waiting for credits.
 614		 */
 615		if (server->in_flight == 0) {
 616			spin_unlock(&server->req_lock);
 617			trace_smb3_insufficient_credits(server->current_mid,
 618					server->conn_id, server->hostname, scredits,
 619					num, in_flight);
 620			cifs_dbg(FYI, "%s: %d requests in flight, needed %d total=%d\n",
 621					__func__, in_flight, num, scredits);
 622			return -EDEADLK;
 623		}
 624	}
 625	spin_unlock(&server->req_lock);
 626
 627	return wait_for_free_credits(server, num, 60000, flags,
 628				     instance);
 629}
 630
 631int
 632cifs_wait_mtu_credits(struct TCP_Server_Info *server, size_t size,
 633		      size_t *num, struct cifs_credits *credits)
 634{
 635	*num = size;
 636	credits->value = 0;
 637	credits->instance = server->reconnect_instance;
 638	return 0;
 639}
 640
 641int wait_for_response(struct TCP_Server_Info *server, struct mid_q_entry *mid)
 642{
 643	unsigned int sleep_state = TASK_KILLABLE;
 644	int error;
 645
 646	if (mid->sr_flags & CIFS_INTERRUPTIBLE_WAIT)
 647		sleep_state = TASK_INTERRUPTIBLE;
 648
 649	error = wait_event_state(server->response_q,
 650				 mid->mid_state != MID_REQUEST_SUBMITTED &&
 651				 mid->mid_state != MID_RESPONSE_RECEIVED,
 652				 (sleep_state | TASK_FREEZABLE_UNSAFE));
 653	if (error < 0)
 654		return -ERESTARTSYS;
 655
 656	return 0;
 657}
 658
 659/*
 660 * Send a SMB request and set the callback function in the mid to handle
 661 * the result. Caller is responsible for dealing with timeouts.
 662 */
 663int
 664cifs_call_async(struct TCP_Server_Info *server, struct smb_rqst *rqst,
 665		mid_receive_t receive, mid_callback_t callback,
 666		mid_handle_t handle, void *cbdata, const int flags,
 667		const struct cifs_credits *exist_credits)
 668{
 669	int rc;
 670	struct mid_q_entry *mid;
 671	struct cifs_credits credits = { .value = 0, .instance = 0 };
 672	unsigned int instance;
 673	int optype;
 674
 675	optype = flags & CIFS_OP_MASK;
 676
 677	if ((flags & CIFS_HAS_CREDITS) == 0) {
 678		rc = wait_for_free_request(server, flags, &instance);
 679		if (rc)
 680			return rc;
 681		credits.value = 1;
 682		credits.instance = instance;
 683	} else
 684		instance = exist_credits->instance;
 685
 686	cifs_server_lock(server);
 687
 688	/*
 689	 * We can't use credits obtained from the previous session to send this
 690	 * request. Check if there were reconnects after we obtained credits and
 691	 * return -EAGAIN in such cases to let callers handle it.
 692	 */
 693	if (instance != server->reconnect_instance) {
 694		cifs_server_unlock(server);
 695		add_credits_and_wake_if(server, &credits, optype);
 696		return -EAGAIN;
 697	}
 698
 699	mid = server->ops->setup_async_request(server, rqst);
 700	if (IS_ERR(mid)) {
 701		cifs_server_unlock(server);
 702		add_credits_and_wake_if(server, &credits, optype);
 703		return PTR_ERR(mid);
 704	}
 705
 706	mid->sr_flags = flags;
 707	mid->receive = receive;
 708	mid->callback = callback;
 709	mid->callback_data = cbdata;
 710	mid->handle = handle;
 711	mid->mid_state = MID_REQUEST_SUBMITTED;
 712
 713	/* put it on the pending_mid_q */
 714	spin_lock(&server->mid_queue_lock);
 715	list_add_tail(&mid->qhead, &server->pending_mid_q);
 716	spin_unlock(&server->mid_queue_lock);
 717
 718	/*
 719	 * Need to store the time in mid before calling I/O. For call_async,
 720	 * I/O response may come back and free the mid entry on another thread.
 721	 */
 722	cifs_save_when_sent(mid);
 723	rc = smb_send_rqst(server, 1, rqst, flags);
 724
 725	if (rc < 0) {
 726		revert_current_mid(server, mid->credits);
 727		server->sequence_number -= 2;
 728		delete_mid(server, mid);
 729	}
 730
 731	cifs_server_unlock(server);
 732
 733	if (rc == 0)
 734		return 0;
 735
 736	add_credits_and_wake_if(server, &credits, optype);
 737	return rc;
 738}
 739
 740int cifs_sync_mid_result(struct mid_q_entry *mid, struct TCP_Server_Info *server)
 741{
 742	int rc = 0;
 743
 744	cifs_dbg(FYI, "%s: cmd=%d mid=%llu state=%d\n",
 745		 __func__, le16_to_cpu(mid->command), mid->mid, mid->mid_state);
 746
 747	spin_lock(&server->mid_queue_lock);
 748	switch (mid->mid_state) {
 749	case MID_RESPONSE_READY:
 750		spin_unlock(&server->mid_queue_lock);
 751		return rc;
 752	case MID_RETRY_NEEDED:
 753		rc = -EAGAIN;
 754		break;
 755	case MID_RESPONSE_MALFORMED:
 756		rc = smb_EIO(smb_eio_trace_rx_sync_mid_malformed);
 757		break;
 758	case MID_SHUTDOWN:
 759		rc = -EHOSTDOWN;
 760		break;
 761	case MID_RC:
 762		rc = mid->mid_rc;
 763		break;
 764	default:
 765		if (mid->deleted_from_q == false) {
 766			list_del_init(&mid->qhead);
 767			mid->deleted_from_q = true;
 768		}
 769		spin_unlock(&server->mid_queue_lock);
 770		cifs_server_dbg(VFS, "%s: invalid mid state mid=%llu state=%d\n",
 771			 __func__, mid->mid, mid->mid_state);
 772		rc = smb_EIO1(smb_eio_trace_rx_sync_mid_invalid, mid->mid_state);
 773		goto sync_mid_done;
 774	}
 775	spin_unlock(&server->mid_queue_lock);
 776
 777sync_mid_done:
 778	release_mid(server, mid);
 779	return rc;
 780}
 781
 782static void
 783cifs_compound_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid)
 784{
 785	struct cifs_credits credits = {
 786		.value = server->ops->get_credits(mid),
 787		.instance = server->reconnect_instance,
 788	};
 789
 790	add_credits(server, &credits, mid->optype);
 791
 792	if (mid->mid_state == MID_RESPONSE_RECEIVED)
 793		mid->mid_state = MID_RESPONSE_READY;
 794}
 795
 796static void
 797cifs_compound_last_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid)
 798{
 799	cifs_compound_callback(server, mid);
 800	cifs_wake_up_task(server, mid);
 801}
 802
 803static void
 804cifs_cancelled_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid)
 805{
 806	cifs_compound_callback(server, mid);
 807	release_mid(server, mid);
 808}
 809
 810/*
 811 * cifs_pick_channel - pick an eligible channel for network operations
 812 *
 813 * @ses: session reference
 814 *
 815 * Select an eligible channel (not terminating and not marked as needing
 816 * reconnect), preferring the least loaded one. If no eligible channel is
 817 * found, fall back to the primary channel (index 0).
 818 *
 819 * Return: TCP_Server_Info pointer for the chosen channel, or NULL if @ses is
 820 * NULL.
 821 */
 822struct TCP_Server_Info *cifs_pick_channel(struct cifs_ses *ses)
 823{
 824	uint index = 0;
 825	unsigned int min_in_flight = UINT_MAX;
 826	struct TCP_Server_Info *server = NULL;
 827	int i, start, cur;
 828
 829	if (!ses)
 830		return NULL;
 831
 832	spin_lock(&ses->chan_lock);
 833	start = atomic_inc_return(&ses->chan_seq);
 834	for (i = 0; i < ses->chan_count; i++) {
 835		cur = (start + i) % ses->chan_count;
 836		server = ses->chans[cur].server;
 837		if (!server || server->terminate)
 838			continue;
 839
 840		if (CIFS_CHAN_NEEDS_RECONNECT(ses, cur))
 841			continue;
 842
 843		/*
 844		 * strictly speaking, we should pick up req_lock to read
 845		 * server->in_flight. But it shouldn't matter much here if we
 846		 * race while reading this data. The worst that can happen is
 847		 * that we could use a channel that's not least loaded. Avoiding
 848		 * taking the lock could help reduce wait time, which is
 849		 * important for this function
 850		 */
 851		if (server->in_flight < min_in_flight) {
 852			min_in_flight = server->in_flight;
 853			index = cur;
 854		}
 855	}
 856
 857	server = ses->chans[index].server;
 858	spin_unlock(&ses->chan_lock);
 859
 860	return server;
 861}
 862
 863int
 864compound_send_recv(const unsigned int xid, struct cifs_ses *ses,
 865		   struct TCP_Server_Info *server,
 866		   const int flags, const int num_rqst, struct smb_rqst *rqst,
 867		   int *resp_buf_type, struct kvec *resp_iov)
 868{
 869	int i, j, optype, rc = 0;
 870	struct mid_q_entry *mid[MAX_COMPOUND];
 871	bool cancelled_mid[MAX_COMPOUND] = {false};
 872	struct cifs_credits credits[MAX_COMPOUND] = {
 873		{ .value = 0, .instance = 0 }
 874	};
 875	unsigned int instance;
 876	char *buf;
 877
 878	optype = flags & CIFS_OP_MASK;
 879
 880	for (i = 0; i < num_rqst; i++)
 881		resp_buf_type[i] = CIFS_NO_BUFFER;  /* no response buf yet */
 882
 883	if (!ses || !ses->server || !server) {
 884		cifs_dbg(VFS, "Null session\n");
 885		return smb_EIO(smb_eio_trace_null_pointers);
 886	}
 887
 888	spin_lock(&server->srv_lock);
 889	if (server->tcpStatus == CifsExiting) {
 890		spin_unlock(&server->srv_lock);
 891		return -ENOENT;
 892	}
 893	spin_unlock(&server->srv_lock);
 894
 895	/*
 896	 * Wait for all the requests to become available.
 897	 * This approach still leaves the possibility to be stuck waiting for
 898	 * credits if the server doesn't grant credits to the outstanding
 899	 * requests and if the client is completely idle, not generating any
 900	 * other requests.
 901	 * This can be handled by the eventual session reconnect.
 902	 */
 903	rc = wait_for_compound_request(server, num_rqst, flags,
 904				       &instance);
 905	if (rc)
 906		return rc;
 907
 908	for (i = 0; i < num_rqst; i++) {
 909		credits[i].value = 1;
 910		credits[i].instance = instance;
 911	}
 912
 913	/*
 914	 * Make sure that we sign in the same order that we send on this socket
 915	 * and avoid races inside tcp sendmsg code that could cause corruption
 916	 * of smb data.
 917	 */
 918
 919	cifs_server_lock(server);
 920
 921	/*
 922	 * All the parts of the compound chain belong obtained credits from the
 923	 * same session. We can not use credits obtained from the previous
 924	 * session to send this request. Check if there were reconnects after
 925	 * we obtained credits and return -EAGAIN in such cases to let callers
 926	 * handle it.
 927	 */
 928	if (instance != server->reconnect_instance) {
 929		cifs_server_unlock(server);
 930		for (j = 0; j < num_rqst; j++)
 931			add_credits(server, &credits[j], optype);
 932		return -EAGAIN;
 933	}
 934
 935	for (i = 0; i < num_rqst; i++) {
 936		mid[i] = server->ops->setup_request(ses, server, &rqst[i]);
 937		if (IS_ERR(mid[i])) {
 938			revert_current_mid(server, i);
 939			for (j = 0; j < i; j++)
 940				delete_mid(server, mid[j]);
 941			cifs_server_unlock(server);
 942
 943			/* Update # of requests on wire to server */
 944			for (j = 0; j < num_rqst; j++)
 945				add_credits(server, &credits[j], optype);
 946			return PTR_ERR(mid[i]);
 947		}
 948
 949		mid[i]->sr_flags = flags;
 950		mid[i]->mid_state = MID_REQUEST_SUBMITTED;
 951		mid[i]->optype = optype;
 952		/*
 953		 * Invoke callback for every part of the compound chain
 954		 * to calculate credits properly. Wake up this thread only when
 955		 * the last element is received.
 956		 */
 957		if (i < num_rqst - 1)
 958			mid[i]->callback = cifs_compound_callback;
 959		else
 960			mid[i]->callback = cifs_compound_last_callback;
 961	}
 962	rc = smb_send_rqst(server, num_rqst, rqst, flags);
 963
 964	for (i = 0; i < num_rqst; i++)
 965		cifs_save_when_sent(mid[i]);
 966
 967	if (rc < 0) {
 968		revert_current_mid(server, num_rqst);
 969		server->sequence_number -= 2;
 970	}
 971
 972	cifs_server_unlock(server);
 973
 974	/*
 975	 * If sending failed for some reason or it is an oplock break that we
 976	 * will not receive a response to - return credits back
 977	 */
 978	if (rc < 0 || (flags & CIFS_NO_SRV_RSP)) {
 979		for (i = 0; i < num_rqst; i++)
 980			add_credits(server, &credits[i], optype);
 981		goto out;
 982	}
 983
 984	/*
 985	 * At this point the request is passed to the network stack - we assume
 986	 * that any credits taken from the server structure on the client have
 987	 * been spent and we can't return them back. Once we receive responses
 988	 * we will collect credits granted by the server in the mid callbacks
 989	 * and add those credits to the server structure.
 990	 */
 991
 992	/*
 993	 * Compounding is never used during session establish.
 994	 */
 995	spin_lock(&ses->ses_lock);
 996	if ((ses->ses_status == SES_NEW) || (optype & CIFS_NEG_OP) || (optype & CIFS_SESS_OP)) {
 997		spin_unlock(&ses->ses_lock);
 998
 999		if (WARN_ON_ONCE(num_rqst != 1 || !resp_iov))
1000			return -EINVAL;
1001
1002		cifs_server_lock(server);
1003		smb311_update_preauth_hash(ses, server, rqst[0].rq_iov, rqst[0].rq_nvec);
1004		cifs_server_unlock(server);
1005
1006		spin_lock(&ses->ses_lock);
1007	}
1008	spin_unlock(&ses->ses_lock);
1009
1010	for (i = 0; i < num_rqst; i++) {
1011		rc = wait_for_response(server, mid[i]);
1012		if (rc != 0)
1013			break;
1014	}
1015	if (rc != 0) {
1016		for (; i < num_rqst; i++) {
1017			cifs_server_dbg(FYI, "Cancelling wait for mid %llu cmd: %d\n",
1018				 mid[i]->mid, le16_to_cpu(mid[i]->command));
1019			send_cancel(ses, server, &rqst[i], mid[i], xid);
1020			spin_lock(&mid[i]->mid_lock);
1021			mid[i]->wait_cancelled = true;
1022			if (mid[i]->mid_state == MID_REQUEST_SUBMITTED ||
1023			    mid[i]->mid_state == MID_RESPONSE_RECEIVED) {
1024				mid[i]->callback = cifs_cancelled_callback;
1025				cancelled_mid[i] = true;
1026				credits[i].value = 0;
1027			}
1028			spin_unlock(&mid[i]->mid_lock);
1029		}
1030	}
1031
1032	for (i = 0; i < num_rqst; i++) {
1033		if (rc < 0)
1034			goto out;
1035
1036		rc = cifs_sync_mid_result(mid[i], server);
1037		if (rc != 0) {
1038			/* mark this mid as cancelled to not free it below */
1039			cancelled_mid[i] = true;
1040			goto out;
1041		}
1042
1043		if (!mid[i]->resp_buf ||
1044		    mid[i]->mid_state != MID_RESPONSE_READY) {
1045			rc = smb_EIO1(smb_eio_trace_rx_mid_unready, mid[i]->mid_state);
1046			cifs_dbg(FYI, "Bad MID state?\n");
1047			goto out;
1048		}
1049
1050		rc = server->ops->check_receive(mid[i], server,
1051						flags & CIFS_LOG_ERROR);
1052
1053		if (resp_iov) {
1054			buf = (char *)mid[i]->resp_buf;
1055			resp_iov[i].iov_base = buf;
1056			resp_iov[i].iov_len = mid[i]->resp_buf_size;
1057
1058			if (mid[i]->large_buf)
1059				resp_buf_type[i] = CIFS_LARGE_BUFFER;
1060			else
1061				resp_buf_type[i] = CIFS_SMALL_BUFFER;
1062
1063			/* mark it so buf will not be freed by delete_mid */
1064			if ((flags & CIFS_NO_RSP_BUF) == 0)
1065				mid[i]->resp_buf = NULL;
1066		}
1067	}
1068
1069	/*
1070	 * Compounding is never used during session establish.
1071	 */
1072	spin_lock(&ses->ses_lock);
1073	if ((ses->ses_status == SES_NEW) || (optype & CIFS_NEG_OP) || (optype & CIFS_SESS_OP)) {
1074		struct kvec iov = {
1075			.iov_base = resp_iov[0].iov_base,
1076			.iov_len = resp_iov[0].iov_len
1077		};
1078		spin_unlock(&ses->ses_lock);
1079		cifs_server_lock(server);
1080		smb311_update_preauth_hash(ses, server, &iov, 1);
1081		cifs_server_unlock(server);
1082		spin_lock(&ses->ses_lock);
1083	}
1084	spin_unlock(&ses->ses_lock);
1085
1086out:
1087	/*
1088	 * This will dequeue all mids. After this it is important that the
1089	 * demultiplex_thread will not process any of these mids any further.
1090	 * This is prevented above by using a noop callback that will not
1091	 * wake this thread except for the very last PDU.
1092	 */
1093	for (i = 0; i < num_rqst; i++) {
1094		if (!cancelled_mid[i])
1095			delete_mid(server, mid[i]);
1096	}
1097
1098	return rc;
1099}
1100
1101int
1102cifs_send_recv(const unsigned int xid, struct cifs_ses *ses,
1103	       struct TCP_Server_Info *server,
1104	       struct smb_rqst *rqst, int *resp_buf_type, const int flags,
1105	       struct kvec *resp_iov)
1106{
1107	return compound_send_recv(xid, ses, server, flags, 1,
1108				  rqst, resp_buf_type, resp_iov);
1109}
1110
1111
1112/*
1113 * Discard any remaining data in the current SMB. To do this, we borrow the
1114 * current bigbuf.
1115 */
1116int
1117cifs_discard_remaining_data(struct TCP_Server_Info *server)
1118{
1119	unsigned int rfclen = server->pdu_size;
1120	size_t remaining = rfclen - server->total_read;
1121
1122	while (remaining > 0) {
1123		ssize_t length;
1124
1125		length = cifs_discard_from_socket(server,
1126				min_t(size_t, remaining,
1127				      CIFSMaxBufSize + MAX_HEADER_SIZE(server)));
1128		if (length < 0)
1129			return length;
1130		server->total_read += length;
1131		remaining -= length;
1132	}
1133
1134	return 0;
1135}
1136
1137static int
1138__cifs_readv_discard(struct TCP_Server_Info *server, struct mid_q_entry *mid,
1139		     bool malformed)
1140{
1141	int length;
1142
1143	length = cifs_discard_remaining_data(server);
1144	dequeue_mid(server, mid, malformed);
1145	mid->resp_buf = server->smallbuf;
1146	server->smallbuf = NULL;
1147	return length;
1148}
1149
1150static int
1151cifs_readv_discard(struct TCP_Server_Info *server, struct mid_q_entry *mid)
1152{
1153	struct cifs_io_subrequest *rdata = mid->callback_data;
1154
1155	return  __cifs_readv_discard(server, mid, rdata->result);
1156}
1157
1158int
1159cifs_readv_receive(struct TCP_Server_Info *server, struct mid_q_entry *mid)
1160{
1161	int length, len;
1162	unsigned int data_offset, data_len;
1163	struct cifs_io_subrequest *rdata = mid->callback_data;
1164	char *buf = server->smallbuf;
1165	unsigned int buflen = server->pdu_size;
1166	bool use_rdma_mr = false;
1167
1168	cifs_dbg(FYI, "%s: mid=%llu offset=%llu bytes=%zu\n",
1169		 __func__, mid->mid, rdata->subreq.start, rdata->subreq.len);
1170
1171	/*
1172	 * read the rest of READ_RSP header (sans Data array), or whatever we
1173	 * can if there's not enough data. At this point, we've read down to
1174	 * the Mid.
1175	 */
1176	len = min_t(unsigned int, buflen, server->vals->read_rsp_size) -
1177							HEADER_SIZE(server) + 1;
1178
1179	length = cifs_read_from_socket(server,
1180				       buf + HEADER_SIZE(server) - 1, len);
1181	if (length < 0)
1182		return length;
1183	server->total_read += length;
1184
1185	if (server->ops->is_session_expired &&
1186	    server->ops->is_session_expired(buf)) {
1187		cifs_reconnect(server, true);
1188		return -1;
1189	}
1190
1191	if (server->ops->is_status_pending &&
1192	    server->ops->is_status_pending(buf, server)) {
1193		cifs_discard_remaining_data(server);
1194		return -1;
1195	}
1196
1197	/* set up first two iov for signature check and to get credits */
1198	rdata->iov[0].iov_base = buf;
1199	rdata->iov[0].iov_len = server->total_read;
1200	cifs_dbg(FYI, "0: iov_base=%p iov_len=%zu\n",
1201		 rdata->iov[0].iov_base, rdata->iov[0].iov_len);
1202
1203	/* Was the SMB read successful? */
1204	rdata->result = server->ops->map_error(buf, false);
1205	if (rdata->result != 0) {
1206		cifs_dbg(FYI, "%s: server returned error %d\n",
1207			 __func__, rdata->result);
1208		/* normal error on read response */
1209		return __cifs_readv_discard(server, mid, false);
1210	}
1211
1212	/* Is there enough to get to the rest of the READ_RSP header? */
1213	if (server->total_read < server->vals->read_rsp_size) {
1214		cifs_dbg(FYI, "%s: server returned short header. got=%u expected=%zu\n",
1215			 __func__, server->total_read,
1216			 server->vals->read_rsp_size);
1217		rdata->result = smb_EIO2(smb_eio_trace_read_rsp_short,
1218					 server->total_read, server->vals->read_rsp_size);
1219		return cifs_readv_discard(server, mid);
1220	}
1221
1222	data_offset = server->ops->read_data_offset(buf);
1223	if (data_offset < server->total_read) {
1224		/*
1225		 * win2k8 sometimes sends an offset of 0 when the read
1226		 * is beyond the EOF. Treat it as if the data starts just after
1227		 * the header.
1228		 */
1229		cifs_dbg(FYI, "%s: data offset (%u) inside read response header\n",
1230			 __func__, data_offset);
1231		data_offset = server->total_read;
1232	} else if (data_offset > MAX_CIFS_SMALL_BUFFER_SIZE) {
1233		/* data_offset is beyond the end of smallbuf */
1234		cifs_dbg(FYI, "%s: data offset (%u) beyond end of smallbuf\n",
1235			 __func__, data_offset);
1236		rdata->result = smb_EIO1(smb_eio_trace_read_overlarge,
1237					 data_offset);
1238		return cifs_readv_discard(server, mid);
1239	}
1240
1241	cifs_dbg(FYI, "%s: total_read=%u data_offset=%u\n",
1242		 __func__, server->total_read, data_offset);
1243
1244	len = data_offset - server->total_read;
1245	if (len > 0) {
1246		/* read any junk before data into the rest of smallbuf */
1247		length = cifs_read_from_socket(server,
1248					       buf + server->total_read, len);
1249		if (length < 0)
1250			return length;
1251		server->total_read += length;
1252		rdata->iov[0].iov_len = server->total_read;
1253	}
1254
1255	/* how much data is in the response? */
1256#ifdef CONFIG_CIFS_SMB_DIRECT
1257	use_rdma_mr = rdata->mr;
1258#endif
1259	data_len = server->ops->read_data_length(buf, use_rdma_mr);
1260	if (!use_rdma_mr && (data_offset + data_len > buflen)) {
1261		/* data_len is corrupt -- discard frame */
1262		rdata->result = smb_EIO2(smb_eio_trace_read_rsp_malformed,
1263					 data_offset + data_len, buflen);
1264		return cifs_readv_discard(server, mid);
1265	}
1266
1267#ifdef CONFIG_CIFS_SMB_DIRECT
1268	if (rdata->mr)
1269		length = data_len; /* An RDMA read is already done. */
1270	else
1271#endif
1272		length = cifs_read_iter_from_socket(server, &rdata->subreq.io_iter,
1273						    data_len);
1274	if (length > 0)
1275		rdata->got_bytes += length;
1276	server->total_read += length;
1277
1278	cifs_dbg(FYI, "total_read=%u buflen=%u remaining=%u\n",
1279		 server->total_read, buflen, data_len);
1280
1281	/* discard anything left over */
1282	if (server->total_read < buflen)
1283		return cifs_readv_discard(server, mid);
1284
1285	dequeue_mid(server, mid, false);
1286	mid->resp_buf = server->smallbuf;
1287	server->smallbuf = NULL;
1288	return length;
1289}