fileio.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2/*
  3 * Copyright (C) 2024, Alibaba Cloud
  4 */
  5#include "internal.h"
  6#include <trace/events/erofs.h>
  7
  8struct erofs_fileio_rq {
  9	struct bio_vec bvecs[16];
 10	struct bio bio;
 11	struct kiocb iocb;
 12	struct super_block *sb;
 13	refcount_t ref;
 14};
 15
 16struct erofs_fileio {
 17	struct erofs_map_blocks map;
 18	struct erofs_map_dev dev;
 19	struct erofs_fileio_rq *rq;
 20};
 21
 22static void erofs_fileio_ki_complete(struct kiocb *iocb, long ret)
 23{
 24	struct erofs_fileio_rq *rq =
 25			container_of(iocb, struct erofs_fileio_rq, iocb);
 26	struct folio_iter fi;
 27
 28	if (ret >= 0 && ret != rq->bio.bi_iter.bi_size)
 29		ret = -EIO;
 30	if (!rq->bio.bi_end_io) {
 31		bio_for_each_folio_all(fi, &rq->bio) {
 32			DBG_BUGON(folio_test_uptodate(fi.folio));
 33			erofs_onlinefolio_end(fi.folio, ret < 0, false);
 34		}
 35	} else if (ret < 0 && !rq->bio.bi_status) {
 36		rq->bio.bi_status = errno_to_blk_status(ret);
 37	}
 38	bio_endio(&rq->bio);
 39	bio_uninit(&rq->bio);
 40	if (refcount_dec_and_test(&rq->ref))
 41		kfree(rq);
 42}
 43
 44static void erofs_fileio_rq_submit(struct erofs_fileio_rq *rq)
 45{
 46	struct iov_iter iter;
 47	ssize_t ret;
 48
 49	if (!rq)
 50		return;
 51	rq->iocb.ki_pos = rq->bio.bi_iter.bi_sector << SECTOR_SHIFT;
 52	rq->iocb.ki_ioprio = get_current_ioprio();
 53	rq->iocb.ki_complete = erofs_fileio_ki_complete;
 54	if (test_opt(&EROFS_SB(rq->sb)->opt, DIRECT_IO) &&
 55	    rq->iocb.ki_filp->f_mode & FMODE_CAN_ODIRECT)
 56		rq->iocb.ki_flags = IOCB_DIRECT;
 57	iov_iter_bvec(&iter, ITER_DEST, rq->bvecs, rq->bio.bi_vcnt,
 58		      rq->bio.bi_iter.bi_size);
 59	scoped_with_creds(rq->iocb.ki_filp->f_cred)
 60		ret = vfs_iocb_iter_read(rq->iocb.ki_filp, &rq->iocb, &iter);
 61	if (ret != -EIOCBQUEUED)
 62		erofs_fileio_ki_complete(&rq->iocb, ret);
 63	if (refcount_dec_and_test(&rq->ref))
 64		kfree(rq);
 65}
 66
 67static struct erofs_fileio_rq *erofs_fileio_rq_alloc(struct erofs_map_dev *mdev)
 68{
 69	struct erofs_fileio_rq *rq = kzalloc(sizeof(*rq),
 70					     GFP_KERNEL | __GFP_NOFAIL);
 71
 72	bio_init(&rq->bio, NULL, rq->bvecs, ARRAY_SIZE(rq->bvecs), REQ_OP_READ);
 73	rq->iocb.ki_filp = mdev->m_dif->file;
 74	rq->sb = mdev->m_sb;
 75	refcount_set(&rq->ref, 2);
 76	return rq;
 77}
 78
 79struct bio *erofs_fileio_bio_alloc(struct erofs_map_dev *mdev)
 80{
 81	return &erofs_fileio_rq_alloc(mdev)->bio;
 82}
 83
 84void erofs_fileio_submit_bio(struct bio *bio)
 85{
 86	return erofs_fileio_rq_submit(container_of(bio, struct erofs_fileio_rq,
 87						   bio));
 88}
 89
 90static int erofs_fileio_scan_folio(struct erofs_fileio *io, struct folio *folio)
 91{
 92	struct inode *inode = folio_inode(folio);
 93	struct erofs_map_blocks *map = &io->map;
 94	unsigned int cur = 0, end = folio_size(folio), len, attached = 0;
 95	loff_t pos = folio_pos(folio), ofs;
 96	int err = 0;
 97
 98	erofs_onlinefolio_init(folio);
 99	while (cur < end) {
100		if (!in_range(pos + cur, map->m_la, map->m_llen)) {
101			map->m_la = pos + cur;
102			map->m_llen = end - cur;
103			err = erofs_map_blocks(inode, map);
104			if (err)
105				break;
106		}
107
108		ofs = folio_pos(folio) + cur - map->m_la;
109		len = min_t(loff_t, map->m_llen - ofs, end - cur);
110		if (map->m_flags & EROFS_MAP_META) {
111			struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
112			void *src;
113
114			src = erofs_read_metabuf(&buf, inode->i_sb,
115				map->m_pa + ofs, erofs_inode_in_metabox(inode));
116			if (IS_ERR(src)) {
117				err = PTR_ERR(src);
118				break;
119			}
120			memcpy_to_folio(folio, cur, src, len);
121			erofs_put_metabuf(&buf);
122		} else if (!(map->m_flags & EROFS_MAP_MAPPED)) {
123			folio_zero_segment(folio, cur, cur + len);
124			attached = 0;
125		} else {
126			if (io->rq && (map->m_pa + ofs != io->dev.m_pa ||
127				       map->m_deviceid != io->dev.m_deviceid)) {
128io_retry:
129				erofs_fileio_rq_submit(io->rq);
130				io->rq = NULL;
131			}
132
133			if (!io->rq) {
134				io->dev = (struct erofs_map_dev) {
135					.m_pa = io->map.m_pa + ofs,
136					.m_deviceid = io->map.m_deviceid,
137				};
138				err = erofs_map_dev(inode->i_sb, &io->dev);
139				if (err)
140					break;
141				io->rq = erofs_fileio_rq_alloc(&io->dev);
142				io->rq->bio.bi_iter.bi_sector =
143					(io->dev.m_dif->fsoff + io->dev.m_pa) >> 9;
144				attached = 0;
145			}
146			if (!bio_add_folio(&io->rq->bio, folio, len, cur))
147				goto io_retry;
148			if (!attached++)
149				erofs_onlinefolio_split(folio);
150			io->dev.m_pa += len;
151		}
152		cur += len;
153	}
154	erofs_onlinefolio_end(folio, err, false);
155	return err;
156}
157
158static int erofs_fileio_read_folio(struct file *file, struct folio *folio)
159{
160	struct erofs_fileio io = {};
161	int err;
162
163	trace_erofs_read_folio(folio, true);
164	err = erofs_fileio_scan_folio(&io, folio);
165	erofs_fileio_rq_submit(io.rq);
166	return err;
167}
168
169static void erofs_fileio_readahead(struct readahead_control *rac)
170{
171	struct inode *inode = rac->mapping->host;
172	struct erofs_fileio io = {};
173	struct folio *folio;
174	int err;
175
176	trace_erofs_readahead(inode, readahead_index(rac),
177			      readahead_count(rac), true);
178	while ((folio = readahead_folio(rac))) {
179		err = erofs_fileio_scan_folio(&io, folio);
180		if (err && err != -EINTR)
181			erofs_err(inode->i_sb, "readahead error at folio %lu @ nid %llu",
182				  folio->index, EROFS_I(inode)->nid);
183	}
184	erofs_fileio_rq_submit(io.rq);
185}
186
187const struct address_space_operations erofs_fileio_aops = {
188	.read_folio = erofs_fileio_read_folio,
189	.readahead = erofs_fileio_readahead,
190};