io.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (c) 2016 Trond Myklebust
  4 *
  5 * I/O and data path helper functionality.
  6 */
  7
  8#include <linux/types.h>
  9#include <linux/kernel.h>
 10#include <linux/bitops.h>
 11#include <linux/rwsem.h>
 12#include <linux/fs.h>
 13#include <linux/nfs_fs.h>
 14
 15#include "internal.h"
 16
 17/**
 18 * nfs_start_io_read - declare the file is being used for buffered reads
 19 * @inode: file inode
 20 *
 21 * Declare that a buffered read operation is about to start, and ensure
 22 * that we block all direct I/O.
 23 * On exit, the function ensures that the NFS_INO_ODIRECT flag is unset,
 24 * and holds a shared lock on inode->i_rwsem to ensure that the flag
 25 * cannot be changed.
 26 * In practice, this means that buffered read operations are allowed to
 27 * execute in parallel, thanks to the shared lock, whereas direct I/O
 28 * operations need to wait to grab an exclusive lock in order to set
 29 * NFS_INO_ODIRECT.
 30 * Note that buffered writes and truncates both take a write lock on
 31 * inode->i_rwsem, meaning that those are serialised w.r.t. the reads.
 32 */
 33int
 34nfs_start_io_read(struct inode *inode)
 35{
 36	struct nfs_inode *nfsi = NFS_I(inode);
 37	int err;
 38
 39	/* Be an optimist! */
 40	err = down_read_killable(&inode->i_rwsem);
 41	if (err)
 42		return err;
 43	if (test_bit(NFS_INO_ODIRECT, &nfsi->flags) == 0)
 44		return 0;
 45	up_read(&inode->i_rwsem);
 46
 47	/* Slow path.... */
 48	err = down_write_killable(&inode->i_rwsem);
 49	if (err)
 50		return err;
 51	nfs_file_block_o_direct(nfsi);
 52	downgrade_write(&inode->i_rwsem);
 53
 54	return 0;
 55}
 56
 57/**
 58 * nfs_end_io_read - declare that the buffered read operation is done
 59 * @inode: file inode
 60 *
 61 * Declare that a buffered read operation is done, and release the shared
 62 * lock on inode->i_rwsem.
 63 */
 64void
 65nfs_end_io_read(struct inode *inode)
 66{
 67	up_read(&inode->i_rwsem);
 68}
 69
 70/**
 71 * nfs_start_io_write - declare the file is being used for buffered writes
 72 * @inode: file inode
 73 *
 74 * Declare that a buffered read operation is about to start, and ensure
 75 * that we block all direct I/O.
 76 */
 77int
 78nfs_start_io_write(struct inode *inode)
 79{
 80	int err;
 81
 82	err = down_write_killable(&inode->i_rwsem);
 83	if (!err)
 84		nfs_file_block_o_direct(NFS_I(inode));
 85	return err;
 86}
 87EXPORT_SYMBOL_GPL(nfs_start_io_write);
 88
 89/**
 90 * nfs_end_io_write - declare that the buffered write operation is done
 91 * @inode: file inode
 92 *
 93 * Declare that a buffered write operation is done, and release the
 94 * lock on inode->i_rwsem.
 95 */
 96void
 97nfs_end_io_write(struct inode *inode)
 98{
 99	up_write(&inode->i_rwsem);
100}
101EXPORT_SYMBOL_GPL(nfs_end_io_write);
102
103/* Call with exclusively locked inode->i_rwsem */
104static void nfs_block_buffered(struct nfs_inode *nfsi, struct inode *inode)
105{
106	if (!test_bit(NFS_INO_ODIRECT, &nfsi->flags)) {
107		set_bit(NFS_INO_ODIRECT, &nfsi->flags);
108		nfs_sync_mapping(inode->i_mapping);
109	}
110}
111
112/**
113 * nfs_start_io_direct - declare the file is being used for direct i/o
114 * @inode: file inode
115 *
116 * Declare that a direct I/O operation is about to start, and ensure
117 * that we block all buffered I/O.
118 * On exit, the function ensures that the NFS_INO_ODIRECT flag is set,
119 * and holds a shared lock on inode->i_rwsem to ensure that the flag
120 * cannot be changed.
121 * In practice, this means that direct I/O operations are allowed to
122 * execute in parallel, thanks to the shared lock, whereas buffered I/O
123 * operations need to wait to grab an exclusive lock in order to clear
124 * NFS_INO_ODIRECT.
125 * Note that buffered writes and truncates both take a write lock on
126 * inode->i_rwsem, meaning that those are serialised w.r.t. O_DIRECT.
127 */
128int
129nfs_start_io_direct(struct inode *inode)
130{
131	struct nfs_inode *nfsi = NFS_I(inode);
132	int err;
133
134	/* Be an optimist! */
135	err = down_read_killable(&inode->i_rwsem);
136	if (err)
137		return err;
138	if (test_bit(NFS_INO_ODIRECT, &nfsi->flags) != 0)
139		return 0;
140	up_read(&inode->i_rwsem);
141
142	/* Slow path.... */
143	err = down_write_killable(&inode->i_rwsem);
144	if (err)
145		return err;
146	nfs_block_buffered(nfsi, inode);
147	downgrade_write(&inode->i_rwsem);
148
149	return 0;
150}
151
152/**
153 * nfs_end_io_direct - declare that the direct i/o operation is done
154 * @inode: file inode
155 *
156 * Declare that a direct I/O operation is done, and release the shared
157 * lock on inode->i_rwsem.
158 */
159void
160nfs_end_io_direct(struct inode *inode)
161{
162	up_read(&inode->i_rwsem);
163}