read_retry.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/* Network filesystem read subrequest retrying.
  3 *
  4 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
  5 * Written by David Howells (dhowells@redhat.com)
  6 */
  7
  8#include <linux/fs.h>
  9#include <linux/slab.h>
 10#include "internal.h"
 11
 12static void netfs_reissue_read(struct netfs_io_request *rreq,
 13			       struct netfs_io_subrequest *subreq)
 14{
 15	subreq->error = 0;
 16	__clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
 17	__set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
 18	netfs_stat(&netfs_n_rh_retry_read_subreq);
 19	subreq->rreq->netfs_ops->issue_read(subreq);
 20}
 21
 22/*
 23 * Go through the list of failed/short reads, retrying all retryable ones.  We
 24 * need to switch failed cache reads to network downloads.
 25 */
 26static void netfs_retry_read_subrequests(struct netfs_io_request *rreq)
 27{
 28	struct netfs_io_subrequest *subreq;
 29	struct netfs_io_stream *stream = &rreq->io_streams[0];
 30	struct list_head *next;
 31
 32	_enter("R=%x", rreq->debug_id);
 33
 34	if (list_empty(&stream->subrequests))
 35		return;
 36
 37	if (rreq->netfs_ops->retry_request)
 38		rreq->netfs_ops->retry_request(rreq, NULL);
 39
 40	/* If there's no renegotiation to do, just resend each retryable subreq
 41	 * up to the first permanently failed one.
 42	 */
 43	if (!rreq->netfs_ops->prepare_read &&
 44	    !rreq->cache_resources.ops) {
 45		list_for_each_entry(subreq, &stream->subrequests, rreq_link) {
 46			if (test_bit(NETFS_SREQ_FAILED, &subreq->flags))
 47				break;
 48			if (__test_and_clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) {
 49				__clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
 50				subreq->retry_count++;
 51				netfs_reset_iter(subreq);
 52				netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit);
 53				netfs_reissue_read(rreq, subreq);
 54			}
 55		}
 56		return;
 57	}
 58
 59	/* Okay, we need to renegotiate all the download requests and flip any
 60	 * failed cache reads over to being download requests and negotiate
 61	 * those also.  All fully successful subreqs have been removed from the
 62	 * list and any spare data from those has been donated.
 63	 *
 64	 * What we do is decant the list and rebuild it one subreq at a time so
 65	 * that we don't end up with donations jumping over a gap we're busy
 66	 * populating with smaller subrequests.  In the event that the subreq
 67	 * we just launched finishes before we insert the next subreq, it'll
 68	 * fill in rreq->prev_donated instead.
 69	 *
 70	 * Note: Alternatively, we could split the tail subrequest right before
 71	 * we reissue it and fix up the donations under lock.
 72	 */
 73	next = stream->subrequests.next;
 74
 75	do {
 76		struct netfs_io_subrequest *from, *to, *tmp;
 77		struct iov_iter source;
 78		unsigned long long start, len;
 79		size_t part;
 80		bool boundary = false, subreq_superfluous = false;
 81
 82		/* Go through the subreqs and find the next span of contiguous
 83		 * buffer that we then rejig (cifs, for example, needs the
 84		 * rsize renegotiating) and reissue.
 85		 */
 86		from = list_entry(next, struct netfs_io_subrequest, rreq_link);
 87		to = from;
 88		start = from->start + from->transferred;
 89		len   = from->len   - from->transferred;
 90
 91		_debug("from R=%08x[%x] s=%llx ctl=%zx/%zx",
 92		       rreq->debug_id, from->debug_index,
 93		       from->start, from->transferred, from->len);
 94
 95		if (test_bit(NETFS_SREQ_FAILED, &from->flags) ||
 96		    !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags)) {
 97			subreq = from;
 98			goto abandon;
 99		}
100
101		list_for_each_continue(next, &stream->subrequests) {
102			subreq = list_entry(next, struct netfs_io_subrequest, rreq_link);
103			if (subreq->start + subreq->transferred != start + len ||
104			    test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags) ||
105			    !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags))
106				break;
107			to = subreq;
108			len += to->len;
109		}
110
111		_debug(" - range: %llx-%llx %llx", start, start + len - 1, len);
112
113		/* Determine the set of buffers we're going to use.  Each
114		 * subreq gets a subset of a single overall contiguous buffer.
115		 */
116		netfs_reset_iter(from);
117		source = from->io_iter;
118		source.count = len;
119
120		/* Work through the sublist. */
121		subreq = from;
122		list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) {
123			if (!len) {
124				subreq_superfluous = true;
125				break;
126			}
127			subreq->source	= NETFS_DOWNLOAD_FROM_SERVER;
128			subreq->start	= start - subreq->transferred;
129			subreq->len	= len   + subreq->transferred;
130			__clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
131			__clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
132			subreq->retry_count++;
133
134			trace_netfs_sreq(subreq, netfs_sreq_trace_retry);
135
136			/* Renegotiate max_len (rsize) */
137			stream->sreq_max_len = subreq->len;
138			if (rreq->netfs_ops->prepare_read &&
139			    rreq->netfs_ops->prepare_read(subreq) < 0) {
140				trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed);
141				__set_bit(NETFS_SREQ_FAILED, &subreq->flags);
142				goto abandon;
143			}
144
145			part = umin(len, stream->sreq_max_len);
146			if (unlikely(stream->sreq_max_segs))
147				part = netfs_limit_iter(&source, 0, part, stream->sreq_max_segs);
148			subreq->len = subreq->transferred + part;
149			subreq->io_iter = source;
150			iov_iter_truncate(&subreq->io_iter, part);
151			iov_iter_advance(&source, part);
152			len -= part;
153			start += part;
154			if (!len) {
155				if (boundary)
156					__set_bit(NETFS_SREQ_BOUNDARY, &subreq->flags);
157			} else {
158				__clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags);
159			}
160
161			netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit);
162			netfs_reissue_read(rreq, subreq);
163			if (subreq == to) {
164				subreq_superfluous = false;
165				break;
166			}
167		}
168
169		/* If we managed to use fewer subreqs, we can discard the
170		 * excess; if we used the same number, then we're done.
171		 */
172		if (!len) {
173			if (!subreq_superfluous)
174				continue;
175			list_for_each_entry_safe_from(subreq, tmp,
176						      &stream->subrequests, rreq_link) {
177				trace_netfs_sreq(subreq, netfs_sreq_trace_superfluous);
178				list_del(&subreq->rreq_link);
179				netfs_put_subrequest(subreq, netfs_sreq_trace_put_done);
180				if (subreq == to)
181					break;
182			}
183			subreq = NULL;
184			continue;
185		}
186
187		/* We ran out of subrequests, so we need to allocate some more
188		 * and insert them after.
189		 */
190		do {
191			subreq = netfs_alloc_subrequest(rreq);
192			if (!subreq) {
193				subreq = to;
194				goto abandon_after;
195			}
196			subreq->source		= NETFS_DOWNLOAD_FROM_SERVER;
197			subreq->start		= start;
198			subreq->len		= len;
199			subreq->stream_nr	= stream->stream_nr;
200			subreq->retry_count	= 1;
201
202			trace_netfs_sreq_ref(rreq->debug_id, subreq->debug_index,
203					     refcount_read(&subreq->ref),
204					     netfs_sreq_trace_new);
205
206			list_add(&subreq->rreq_link, &to->rreq_link);
207			to = list_next_entry(to, rreq_link);
208			trace_netfs_sreq(subreq, netfs_sreq_trace_retry);
209
210			stream->sreq_max_len	= umin(len, rreq->rsize);
211			stream->sreq_max_segs	= 0;
212			if (unlikely(stream->sreq_max_segs))
213				part = netfs_limit_iter(&source, 0, part, stream->sreq_max_segs);
214
215			netfs_stat(&netfs_n_rh_download);
216			if (rreq->netfs_ops->prepare_read(subreq) < 0) {
217				trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed);
218				__set_bit(NETFS_SREQ_FAILED, &subreq->flags);
219				goto abandon;
220			}
221
222			part = umin(len, stream->sreq_max_len);
223			subreq->len = subreq->transferred + part;
224			subreq->io_iter = source;
225			iov_iter_truncate(&subreq->io_iter, part);
226			iov_iter_advance(&source, part);
227
228			len -= part;
229			start += part;
230			if (!len && boundary) {
231				__set_bit(NETFS_SREQ_BOUNDARY, &to->flags);
232				boundary = false;
233			}
234
235			netfs_reissue_read(rreq, subreq);
236		} while (len);
237
238	} while (!list_is_head(next, &stream->subrequests));
239
240	return;
241
242	/* If we hit an error, fail all remaining incomplete subrequests */
243abandon_after:
244	if (list_is_last(&subreq->rreq_link, &stream->subrequests))
245		return;
246	subreq = list_next_entry(subreq, rreq_link);
247abandon:
248	list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) {
249		if (!test_bit(NETFS_SREQ_FAILED, &subreq->flags) &&
250		    !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags))
251			continue;
252		subreq->error = -ENOMEM;
253		__set_bit(NETFS_SREQ_FAILED, &subreq->flags);
254		__clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
255	}
256}
257
258/*
259 * Retry reads.
260 */
261void netfs_retry_reads(struct netfs_io_request *rreq)
262{
263	struct netfs_io_stream *stream = &rreq->io_streams[0];
264
265	netfs_stat(&netfs_n_rh_retry_read_req);
266
267	/* Wait for all outstanding I/O to quiesce before performing retries as
268	 * we may need to renegotiate the I/O sizes.
269	 */
270	set_bit(NETFS_RREQ_RETRYING, &rreq->flags);
271	netfs_wait_for_in_progress_stream(rreq, stream);
272	clear_bit(NETFS_RREQ_RETRYING, &rreq->flags);
273
274	trace_netfs_rreq(rreq, netfs_rreq_trace_resubmit);
275	netfs_retry_read_subrequests(rreq);
276}
277
278/*
279 * Unlock any the pages that haven't been unlocked yet due to abandoned
280 * subrequests.
281 */
282void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq)
283{
284	struct folio_queue *p;
285
286	for (p = rreq->buffer.tail; p; p = p->next) {
287		for (int slot = 0; slot < folioq_count(p); slot++) {
288			struct folio *folio = folioq_folio(p, slot);
289
290			if (folio && !folioq_is_marked2(p, slot)) {
291				trace_netfs_folio(folio, netfs_folio_trace_abandon);
292				folio_unlock(folio);
293			}
294		}
295	}
296}