blk-mq-dma.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) 2025 Christoph Hellwig
  4 */
  5#include <linux/blk-integrity.h>
  6#include <linux/blk-mq-dma.h>
  7#include "blk.h"
  8
  9struct phys_vec {
 10	phys_addr_t	paddr;
 11	u32		len;
 12};
 13
 14static bool __blk_map_iter_next(struct blk_map_iter *iter)
 15{
 16	if (iter->iter.bi_size)
 17		return true;
 18	if (!iter->bio || !iter->bio->bi_next)
 19		return false;
 20
 21	iter->bio = iter->bio->bi_next;
 22	if (iter->is_integrity) {
 23		iter->iter = bio_integrity(iter->bio)->bip_iter;
 24		iter->bvecs = bio_integrity(iter->bio)->bip_vec;
 25	} else {
 26		iter->iter = iter->bio->bi_iter;
 27		iter->bvecs = iter->bio->bi_io_vec;
 28	}
 29	return true;
 30}
 31
 32static bool blk_map_iter_next(struct request *req, struct blk_map_iter *iter,
 33			      struct phys_vec *vec)
 34{
 35	unsigned int max_size;
 36	struct bio_vec bv;
 37
 38	if (!iter->iter.bi_size)
 39		return false;
 40
 41	bv = mp_bvec_iter_bvec(iter->bvecs, iter->iter);
 42	vec->paddr = bvec_phys(&bv);
 43	max_size = get_max_segment_size(&req->q->limits, vec->paddr, UINT_MAX);
 44	bv.bv_len = min(bv.bv_len, max_size);
 45	bvec_iter_advance_single(iter->bvecs, &iter->iter, bv.bv_len);
 46
 47	/*
 48	 * If we are entirely done with this bi_io_vec entry, check if the next
 49	 * one could be merged into it.  This typically happens when moving to
 50	 * the next bio, but some callers also don't pack bvecs tight.
 51	 */
 52	while (!iter->iter.bi_size || !iter->iter.bi_bvec_done) {
 53		struct bio_vec next;
 54
 55		if (!__blk_map_iter_next(iter))
 56			break;
 57
 58		next = mp_bvec_iter_bvec(iter->bvecs, iter->iter);
 59		if (bv.bv_len + next.bv_len > max_size ||
 60		    !biovec_phys_mergeable(req->q, &bv, &next))
 61			break;
 62
 63		bv.bv_len += next.bv_len;
 64		bvec_iter_advance_single(iter->bvecs, &iter->iter, next.bv_len);
 65	}
 66
 67	vec->len = bv.bv_len;
 68	return true;
 69}
 70
 71/*
 72 * The IOVA-based DMA API wants to be able to coalesce at the minimal IOMMU page
 73 * size granularity (which is guaranteed to be <= PAGE_SIZE and usually 4k), so
 74 * we need to ensure our segments are aligned to this as well.
 75 *
 76 * Note that there is no point in using the slightly more complicated IOVA based
 77 * path for single segment mappings.
 78 */
 79static inline bool blk_can_dma_map_iova(struct request *req,
 80		struct device *dma_dev)
 81{
 82	return !(req_phys_gap_mask(req) & dma_get_merge_boundary(dma_dev));
 83}
 84
 85static bool blk_dma_map_bus(struct blk_dma_iter *iter, struct phys_vec *vec)
 86{
 87	iter->addr = pci_p2pdma_bus_addr_map(iter->p2pdma.mem, vec->paddr);
 88	iter->len = vec->len;
 89	return true;
 90}
 91
 92static bool blk_dma_map_direct(struct request *req, struct device *dma_dev,
 93		struct blk_dma_iter *iter, struct phys_vec *vec)
 94{
 95	unsigned int attrs = 0;
 96
 97	if (iter->p2pdma.map == PCI_P2PDMA_MAP_THRU_HOST_BRIDGE)
 98		attrs |= DMA_ATTR_MMIO;
 99
100	iter->addr = dma_map_phys(dma_dev, vec->paddr, vec->len,
101			rq_dma_dir(req), attrs);
102	if (dma_mapping_error(dma_dev, iter->addr)) {
103		iter->status = BLK_STS_RESOURCE;
104		return false;
105	}
106	iter->len = vec->len;
107	return true;
108}
109
110static bool blk_rq_dma_map_iova(struct request *req, struct device *dma_dev,
111		struct dma_iova_state *state, struct blk_dma_iter *iter,
112		struct phys_vec *vec)
113{
114	enum dma_data_direction dir = rq_dma_dir(req);
115	unsigned int mapped = 0;
116	unsigned int attrs = 0;
117	int error;
118
119	iter->addr = state->addr;
120	iter->len = dma_iova_size(state);
121
122	if (iter->p2pdma.map == PCI_P2PDMA_MAP_THRU_HOST_BRIDGE)
123		attrs |= DMA_ATTR_MMIO;
124
125	do {
126		error = dma_iova_link(dma_dev, state, vec->paddr, mapped,
127				vec->len, dir, attrs);
128		if (error)
129			goto out_unlink;
130		mapped += vec->len;
131	} while (blk_map_iter_next(req, &iter->iter, vec));
132
133	error = dma_iova_sync(dma_dev, state, 0, mapped);
134	if (error)
135		goto out_unlink;
136
137	return true;
138
139out_unlink:
140	dma_iova_destroy(dma_dev, state, mapped, dir, attrs);
141	iter->status = errno_to_blk_status(error);
142	return false;
143}
144
145static inline void blk_rq_map_iter_init(struct request *rq,
146					struct blk_map_iter *iter)
147{
148	struct bio *bio = rq->bio;
149
150	if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) {
151		*iter = (struct blk_map_iter) {
152			.bvecs = &rq->special_vec,
153			.iter = {
154				.bi_size = rq->special_vec.bv_len,
155			}
156		};
157	} else if (bio) {
158		*iter = (struct blk_map_iter) {
159			.bio = bio,
160			.bvecs = bio->bi_io_vec,
161			.iter = bio->bi_iter,
162		};
163	} else {
164		/* the internal flush request may not have bio attached */
165		*iter = (struct blk_map_iter) {};
166	}
167}
168
169static bool blk_dma_map_iter_start(struct request *req, struct device *dma_dev,
170		struct dma_iova_state *state, struct blk_dma_iter *iter,
171		unsigned int total_len)
172{
173	struct phys_vec vec;
174
175	memset(&iter->p2pdma, 0, sizeof(iter->p2pdma));
176	iter->status = BLK_STS_OK;
177	iter->p2pdma.map = PCI_P2PDMA_MAP_NONE;
178
179	/*
180	 * Grab the first segment ASAP because we'll need it to check for P2P
181	 * transfers.
182	 */
183	if (!blk_map_iter_next(req, &iter->iter, &vec))
184		return false;
185
186	switch (pci_p2pdma_state(&iter->p2pdma, dma_dev,
187				 phys_to_page(vec.paddr))) {
188	case PCI_P2PDMA_MAP_BUS_ADDR:
189		return blk_dma_map_bus(iter, &vec);
190	case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE:
191		/*
192		 * P2P transfers through the host bridge are treated the
193		 * same as non-P2P transfers below and during unmap.
194		 */
195	case PCI_P2PDMA_MAP_NONE:
196		break;
197	default:
198		iter->status = BLK_STS_INVAL;
199		return false;
200	}
201
202	if (blk_can_dma_map_iova(req, dma_dev) &&
203	    dma_iova_try_alloc(dma_dev, state, vec.paddr, total_len))
204		return blk_rq_dma_map_iova(req, dma_dev, state, iter, &vec);
205	memset(state, 0, sizeof(*state));
206	return blk_dma_map_direct(req, dma_dev, iter, &vec);
207}
208
209/**
210 * blk_rq_dma_map_iter_start - map the first DMA segment for a request
211 * @req:	request to map
212 * @dma_dev:	device to map to
213 * @state:	DMA IOVA state
214 * @iter:	block layer DMA iterator
215 *
216 * Start DMA mapping @req to @dma_dev.  @state and @iter are provided by the
217 * caller and don't need to be initialized.  @state needs to be stored for use
218 * at unmap time, @iter is only needed at map time.
219 *
220 * Returns %false if there is no segment to map, including due to an error, or
221 * %true ft it did map a segment.
222 *
223 * If a segment was mapped, the DMA address for it is returned in @iter.addr and
224 * the length in @iter.len.  If no segment was mapped the status code is
225 * returned in @iter.status.
226 *
227 * The caller can call blk_rq_dma_map_coalesce() to check if further segments
228 * need to be mapped after this, or go straight to blk_rq_dma_map_iter_next()
229 * to try to map the following segments.
230 */
231bool blk_rq_dma_map_iter_start(struct request *req, struct device *dma_dev,
232		struct dma_iova_state *state, struct blk_dma_iter *iter)
233{
234	blk_rq_map_iter_init(req, &iter->iter);
235	return blk_dma_map_iter_start(req, dma_dev, state, iter,
236				      blk_rq_payload_bytes(req));
237}
238EXPORT_SYMBOL_GPL(blk_rq_dma_map_iter_start);
239
240/**
241 * blk_rq_dma_map_iter_next - map the next DMA segment for a request
242 * @req:	request to map
243 * @dma_dev:	device to map to
244 * @state:	DMA IOVA state
245 * @iter:	block layer DMA iterator
246 *
247 * Iterate to the next mapping after a previous call to
248 * blk_rq_dma_map_iter_start().  See there for a detailed description of the
249 * arguments.
250 *
251 * Returns %false if there is no segment to map, including due to an error, or
252 * %true ft it did map a segment.
253 *
254 * If a segment was mapped, the DMA address for it is returned in @iter.addr and
255 * the length in @iter.len.  If no segment was mapped the status code is
256 * returned in @iter.status.
257 */
258bool blk_rq_dma_map_iter_next(struct request *req, struct device *dma_dev,
259		struct dma_iova_state *state, struct blk_dma_iter *iter)
260{
261	struct phys_vec vec;
262
263	if (!blk_map_iter_next(req, &iter->iter, &vec))
264		return false;
265
266	if (iter->p2pdma.map == PCI_P2PDMA_MAP_BUS_ADDR)
267		return blk_dma_map_bus(iter, &vec);
268	return blk_dma_map_direct(req, dma_dev, iter, &vec);
269}
270EXPORT_SYMBOL_GPL(blk_rq_dma_map_iter_next);
271
272static inline struct scatterlist *
273blk_next_sg(struct scatterlist **sg, struct scatterlist *sglist)
274{
275	if (!*sg)
276		return sglist;
277
278	/*
279	 * If the driver previously mapped a shorter list, we could see a
280	 * termination bit prematurely unless it fully inits the sg table
281	 * on each mapping. We KNOW that there must be more entries here
282	 * or the driver would be buggy, so force clear the termination bit
283	 * to avoid doing a full sg_init_table() in drivers for each command.
284	 */
285	sg_unmark_end(*sg);
286	return sg_next(*sg);
287}
288
289/*
290 * Map a request to scatterlist, return number of sg entries setup. Caller
291 * must make sure sg can hold rq->nr_phys_segments entries.
292 */
293int __blk_rq_map_sg(struct request *rq, struct scatterlist *sglist,
294		    struct scatterlist **last_sg)
295{
296	struct blk_map_iter iter;
297	struct phys_vec vec;
298	int nsegs = 0;
299
300	blk_rq_map_iter_init(rq, &iter);
301	while (blk_map_iter_next(rq, &iter, &vec)) {
302		*last_sg = blk_next_sg(last_sg, sglist);
303		sg_set_page(*last_sg, phys_to_page(vec.paddr), vec.len,
304				offset_in_page(vec.paddr));
305		nsegs++;
306	}
307
308	if (*last_sg)
309		sg_mark_end(*last_sg);
310
311	/*
312	 * Something must have been wrong if the figured number of
313	 * segment is bigger than number of req's physical segments
314	 */
315	WARN_ON(nsegs > blk_rq_nr_phys_segments(rq));
316
317	return nsegs;
318}
319EXPORT_SYMBOL(__blk_rq_map_sg);
320
321#ifdef CONFIG_BLK_DEV_INTEGRITY
322/**
323 * blk_rq_integrity_dma_map_iter_start - map the first integrity DMA segment
324 * 					 for a request
325 * @req:	request to map
326 * @dma_dev:	device to map to
327 * @state:	DMA IOVA state
328 * @iter:	block layer DMA iterator
329 *
330 * Start DMA mapping @req integrity data to @dma_dev.  @state and @iter are
331 * provided by the caller and don't need to be initialized.  @state needs to be
332 * stored for use at unmap time, @iter is only needed at map time.
333 *
334 * Returns %false if there is no segment to map, including due to an error, or
335 * %true if it did map a segment.
336 *
337 * If a segment was mapped, the DMA address for it is returned in @iter.addr
338 * and the length in @iter.len.  If no segment was mapped the status code is
339 * returned in @iter.status.
340 *
341 * The caller can call blk_rq_dma_map_coalesce() to check if further segments
342 * need to be mapped after this, or go straight to blk_rq_dma_map_iter_next()
343 * to try to map the following segments.
344 */
345bool blk_rq_integrity_dma_map_iter_start(struct request *req,
346		struct device *dma_dev,  struct dma_iova_state *state,
347		struct blk_dma_iter *iter)
348{
349	unsigned len = bio_integrity_bytes(&req->q->limits.integrity,
350					   blk_rq_sectors(req));
351	struct bio *bio = req->bio;
352
353	iter->iter = (struct blk_map_iter) {
354		.bio = bio,
355		.iter = bio_integrity(bio)->bip_iter,
356		.bvecs = bio_integrity(bio)->bip_vec,
357		.is_integrity = true,
358	};
359	return blk_dma_map_iter_start(req, dma_dev, state, iter, len);
360}
361EXPORT_SYMBOL_GPL(blk_rq_integrity_dma_map_iter_start);
362
363/**
364 * blk_rq_integrity_dma_map_iter_next - map the next integrity DMA segment for
365 * 					 a request
366 * @req:	request to map
367 * @dma_dev:	device to map to
368 * @state:	DMA IOVA state
369 * @iter:	block layer DMA iterator
370 *
371 * Iterate to the next integrity mapping after a previous call to
372 * blk_rq_integrity_dma_map_iter_start().  See there for a detailed description
373 * of the arguments.
374 *
375 * Returns %false if there is no segment to map, including due to an error, or
376 * %true if it did map a segment.
377 *
378 * If a segment was mapped, the DMA address for it is returned in @iter.addr and
379 * the length in @iter.len.  If no segment was mapped the status code is
380 * returned in @iter.status.
381 */
382bool blk_rq_integrity_dma_map_iter_next(struct request *req,
383               struct device *dma_dev, struct blk_dma_iter *iter)
384{
385	struct phys_vec vec;
386
387	if (!blk_map_iter_next(req, &iter->iter, &vec))
388		return false;
389
390	if (iter->p2pdma.map == PCI_P2PDMA_MAP_BUS_ADDR)
391		return blk_dma_map_bus(iter, &vec);
392	return blk_dma_map_direct(req, dma_dev, iter, &vec);
393}
394EXPORT_SYMBOL_GPL(blk_rq_integrity_dma_map_iter_next);
395
396/**
397 * blk_rq_map_integrity_sg - Map integrity metadata into a scatterlist
398 * @rq:		request to map
399 * @sglist:	target scatterlist
400 *
401 * Description: Map the integrity vectors in request into a
402 * scatterlist.  The scatterlist must be big enough to hold all
403 * elements.  I.e. sized using blk_rq_count_integrity_sg() or
404 * rq->nr_integrity_segments.
405 */
406int blk_rq_map_integrity_sg(struct request *rq, struct scatterlist *sglist)
407{
408	struct request_queue *q = rq->q;
409	struct scatterlist *sg = NULL;
410	struct bio *bio = rq->bio;
411	unsigned int segments = 0;
412	struct phys_vec vec;
413
414	struct blk_map_iter iter = {
415		.bio = bio,
416		.iter = bio_integrity(bio)->bip_iter,
417		.bvecs = bio_integrity(bio)->bip_vec,
418		.is_integrity = true,
419	};
420
421	while (blk_map_iter_next(rq, &iter, &vec)) {
422		sg = blk_next_sg(&sg, sglist);
423		sg_set_page(sg, phys_to_page(vec.paddr), vec.len,
424				offset_in_page(vec.paddr));
425		segments++;
426	}
427
428	if (sg)
429	        sg_mark_end(sg);
430
431	/*
432	 * Something must have been wrong if the figured number of segment
433	 * is bigger than number of req's physical integrity segments
434	 */
435	BUG_ON(segments > rq->nr_integrity_segments);
436	BUG_ON(segments > queue_max_integrity_segments(q));
437	return segments;
438}
439EXPORT_SYMBOL(blk_rq_map_integrity_sg);
440#endif