reflink.c

  1// SPDX-License-Identifier: GPL-2.0
  2
  3#include <linux/blkdev.h>
  4#include <linux/fscrypt.h>
  5#include <linux/iversion.h>
  6#include "ctree.h"
  7#include "fs.h"
  8#include "messages.h"
  9#include "compression.h"
 10#include "delalloc-space.h"
 11#include "disk-io.h"
 12#include "reflink.h"
 13#include "transaction.h"
 14#include "subpage.h"
 15#include "accessors.h"
 16#include "file-item.h"
 17#include "file.h"
 18#include "super.h"
 19
 20#define BTRFS_MAX_DEDUPE_LEN	SZ_16M
 21
 22static int clone_finish_inode_update(struct btrfs_trans_handle *trans,
 23				     struct inode *inode,
 24				     u64 endoff,
 25				     const u64 destoff,
 26				     const u64 olen,
 27				     bool no_time_update)
 28{
 29	int ret;
 30
 31	inode_inc_iversion(inode);
 32	if (!no_time_update) {
 33		inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode));
 34	}
 35	/*
 36	 * We round up to the block size at eof when determining which
 37	 * extents to clone above, but shouldn't round up the file size.
 38	 */
 39	if (endoff > destoff + olen)
 40		endoff = destoff + olen;
 41	if (endoff > inode->i_size) {
 42		i_size_write(inode, endoff);
 43		btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), 0);
 44	}
 45
 46	ret = btrfs_update_inode(trans, BTRFS_I(inode));
 47	if (unlikely(ret)) {
 48		btrfs_abort_transaction(trans, ret);
 49		btrfs_end_transaction(trans);
 50		return ret;
 51	}
 52	return btrfs_end_transaction(trans);
 53}
 54
 55static int copy_inline_to_page(struct btrfs_inode *inode,
 56			       const u64 file_offset,
 57			       char *inline_data,
 58			       const u64 size,
 59			       const u64 datal,
 60			       const u8 comp_type)
 61{
 62	struct btrfs_fs_info *fs_info = inode->root->fs_info;
 63	const u32 block_size = fs_info->sectorsize;
 64	const u64 range_end = file_offset + block_size - 1;
 65	const size_t inline_size = size - btrfs_file_extent_calc_inline_size(0);
 66	char *data_start = inline_data + btrfs_file_extent_calc_inline_size(0);
 67	struct extent_changeset *data_reserved = NULL;
 68	struct folio *folio = NULL;
 69	struct address_space *mapping = inode->vfs_inode.i_mapping;
 70	int ret;
 71
 72	ASSERT(IS_ALIGNED(file_offset, block_size));
 73
 74	/*
 75	 * We have flushed and locked the ranges of the source and destination
 76	 * inodes, we also have locked the inodes, so we are safe to do a
 77	 * reservation here. Also we must not do the reservation while holding
 78	 * a transaction open, otherwise we would deadlock.
 79	 */
 80	ret = btrfs_delalloc_reserve_space(inode, &data_reserved, file_offset,
 81					   block_size);
 82	if (ret)
 83		goto out;
 84
 85	folio = __filemap_get_folio(mapping, file_offset >> PAGE_SHIFT,
 86					FGP_LOCK | FGP_ACCESSED | FGP_CREAT,
 87					btrfs_alloc_write_mask(mapping));
 88	if (IS_ERR(folio)) {
 89		ret = PTR_ERR(folio);
 90		goto out_unlock;
 91	}
 92
 93	ret = set_folio_extent_mapped(folio);
 94	if (ret < 0)
 95		goto out_unlock;
 96
 97	btrfs_clear_extent_bit(&inode->io_tree, file_offset, range_end,
 98			       EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, NULL);
 99	ret = btrfs_set_extent_delalloc(inode, file_offset, range_end, 0, NULL);
100	if (ret)
101		goto out_unlock;
102
103	/*
104	 * After dirtying the page our caller will need to start a transaction,
105	 * and if we are low on metadata free space, that can cause flushing of
106	 * delalloc for all inodes in order to get metadata space released.
107	 * However we are holding the range locked for the whole duration of
108	 * the clone/dedupe operation, so we may deadlock if that happens and no
109	 * other task releases enough space. So mark this inode as not being
110	 * possible to flush to avoid such deadlock. We will clear that flag
111	 * when we finish cloning all extents, since a transaction is started
112	 * after finding each extent to clone.
113	 */
114	set_bit(BTRFS_INODE_NO_DELALLOC_FLUSH, &inode->runtime_flags);
115
116	if (comp_type == BTRFS_COMPRESS_NONE) {
117		memcpy_to_folio(folio, offset_in_folio(folio, file_offset), data_start,
118					datal);
119	} else {
120		ret = btrfs_decompress(comp_type, data_start, folio,
121				       offset_in_folio(folio, file_offset),
122				       inline_size, datal);
123		if (ret)
124			goto out_unlock;
125		flush_dcache_folio(folio);
126	}
127
128	/*
129	 * If our inline data is smaller then the block/page size, then the
130	 * remaining of the block/page is equivalent to zeroes. We had something
131	 * like the following done:
132	 *
133	 * $ xfs_io -f -c "pwrite -S 0xab 0 500" file
134	 * $ sync  # (or fsync)
135	 * $ xfs_io -c "falloc 0 4K" file
136	 * $ xfs_io -c "pwrite -S 0xcd 4K 4K"
137	 *
138	 * So what's in the range [500, 4095] corresponds to zeroes.
139	 */
140	if (datal < block_size)
141		folio_zero_range(folio, datal, block_size - datal);
142
143	btrfs_folio_set_uptodate(fs_info, folio, file_offset, block_size);
144	btrfs_folio_clear_checked(fs_info, folio, file_offset, block_size);
145	btrfs_folio_set_dirty(fs_info, folio, file_offset, block_size);
146out_unlock:
147	if (!IS_ERR(folio)) {
148		folio_unlock(folio);
149		folio_put(folio);
150	}
151	if (ret)
152		btrfs_delalloc_release_space(inode, data_reserved, file_offset,
153					     block_size, true);
154	btrfs_delalloc_release_extents(inode, block_size);
155out:
156	extent_changeset_free(data_reserved);
157
158	return ret;
159}
160
161/*
162 * Deal with cloning of inline extents. We try to copy the inline extent from
163 * the source inode to destination inode when possible. When not possible we
164 * copy the inline extent's data into the respective page of the inode.
165 */
166static int clone_copy_inline_extent(struct btrfs_inode *inode,
167				    struct btrfs_path *path,
168				    struct btrfs_key *new_key,
169				    const u64 drop_start,
170				    const u64 datal,
171				    const u64 size,
172				    const u8 comp_type,
173				    char *inline_data,
174				    struct btrfs_trans_handle **trans_out)
175{
176	struct btrfs_root *root = inode->root;
177	struct btrfs_fs_info *fs_info = root->fs_info;
178	const u64 aligned_end = ALIGN(new_key->offset + datal,
179				      fs_info->sectorsize);
180	struct btrfs_trans_handle *trans = NULL;
181	struct btrfs_drop_extents_args drop_args = { 0 };
182	int ret;
183	struct btrfs_key key;
184
185	if (new_key->offset > 0) {
186		ret = copy_inline_to_page(inode, new_key->offset,
187					  inline_data, size, datal, comp_type);
188		goto out;
189	}
190
191	key.objectid = btrfs_ino(inode);
192	key.type = BTRFS_EXTENT_DATA_KEY;
193	key.offset = 0;
194	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
195	if (ret < 0) {
196		return ret;
197	} else if (ret > 0) {
198		if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) {
199			ret = btrfs_next_leaf(root, path);
200			if (ret < 0)
201				return ret;
202			else if (ret > 0)
203				goto copy_inline_extent;
204		}
205		btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
206		if (key.objectid == btrfs_ino(inode) &&
207		    key.type == BTRFS_EXTENT_DATA_KEY) {
208			/*
209			 * There's an implicit hole at file offset 0, copy the
210			 * inline extent's data to the page.
211			 */
212			ASSERT(key.offset > 0);
213			goto copy_to_page;
214		}
215	} else if (i_size_read(&inode->vfs_inode) <= datal) {
216		struct btrfs_file_extent_item *ei;
217
218		ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
219				    struct btrfs_file_extent_item);
220		/*
221		 * If it's an inline extent replace it with the source inline
222		 * extent, otherwise copy the source inline extent data into
223		 * the respective page at the destination inode.
224		 */
225		if (btrfs_file_extent_type(path->nodes[0], ei) ==
226		    BTRFS_FILE_EXTENT_INLINE)
227			goto copy_inline_extent;
228
229		goto copy_to_page;
230	}
231
232copy_inline_extent:
233	/*
234	 * We have no extent items, or we have an extent at offset 0 which may
235	 * or may not be inlined. All these cases are dealt the same way.
236	 */
237	if (i_size_read(&inode->vfs_inode) > datal) {
238		/*
239		 * At the destination offset 0 we have either a hole, a regular
240		 * extent or an inline extent larger then the one we want to
241		 * clone. Deal with all these cases by copying the inline extent
242		 * data into the respective page at the destination inode.
243		 */
244		goto copy_to_page;
245	}
246
247	/*
248	 * Release path before starting a new transaction so we don't hold locks
249	 * that would confuse lockdep.
250	 */
251	btrfs_release_path(path);
252	/*
253	 * If we end up here it means were copy the inline extent into a leaf
254	 * of the destination inode. We know we will drop or adjust at most one
255	 * extent item in the destination root.
256	 *
257	 * 1 unit - adjusting old extent (we may have to split it)
258	 * 1 unit - add new extent
259	 * 1 unit - inode update
260	 */
261	trans = btrfs_start_transaction(root, 3);
262	if (IS_ERR(trans)) {
263		ret = PTR_ERR(trans);
264		trans = NULL;
265		goto out;
266	}
267	drop_args.path = path;
268	drop_args.start = drop_start;
269	drop_args.end = aligned_end;
270	drop_args.drop_cache = true;
271	ret = btrfs_drop_extents(trans, root, inode, &drop_args);
272	if (unlikely(ret)) {
273		btrfs_abort_transaction(trans, ret);
274		goto out;
275	}
276	ret = btrfs_insert_empty_item(trans, root, path, new_key, size);
277	if (unlikely(ret)) {
278		btrfs_abort_transaction(trans, ret);
279		goto out;
280	}
281
282	write_extent_buffer(path->nodes[0], inline_data,
283			    btrfs_item_ptr_offset(path->nodes[0],
284						  path->slots[0]),
285			    size);
286	btrfs_update_inode_bytes(inode, datal, drop_args.bytes_found);
287	btrfs_set_inode_full_sync(inode);
288	ret = btrfs_inode_set_file_extent_range(inode, 0, aligned_end);
289	if (unlikely(ret))
290		btrfs_abort_transaction(trans, ret);
291out:
292	if (!ret && !trans) {
293		/*
294		 * No transaction here means we copied the inline extent into a
295		 * page of the destination inode.
296		 *
297		 * 1 unit to update inode item
298		 */
299		trans = btrfs_start_transaction(root, 1);
300		if (IS_ERR(trans)) {
301			ret = PTR_ERR(trans);
302			trans = NULL;
303		}
304	}
305	if (ret && trans)
306		btrfs_end_transaction(trans);
307	if (!ret)
308		*trans_out = trans;
309
310	return ret;
311
312copy_to_page:
313	/*
314	 * Release our path because we don't need it anymore and also because
315	 * copy_inline_to_page() needs to reserve data and metadata, which may
316	 * need to flush delalloc when we are low on available space and
317	 * therefore cause a deadlock if writeback of an inline extent needs to
318	 * write to the same leaf or an ordered extent completion needs to write
319	 * to the same leaf.
320	 */
321	btrfs_release_path(path);
322
323	ret = copy_inline_to_page(inode, new_key->offset,
324				  inline_data, size, datal, comp_type);
325	goto out;
326}
327
328/*
329 * Clone a range from inode file to another.
330 *
331 * @src:             Inode to clone from
332 * @inode:           Inode to clone to
333 * @off:             Offset within source to start clone from
334 * @olen:            Original length, passed by user, of range to clone
335 * @olen_aligned:    Block-aligned value of olen
336 * @destoff:         Offset within @inode to start clone
337 * @no_time_update:  Whether to update mtime/ctime on the target inode
338 */
339static int btrfs_clone(struct inode *src, struct inode *inode,
340		       const u64 off, const u64 olen, const u64 olen_aligned,
341		       const u64 destoff, bool no_time_update)
342{
343	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
344	BTRFS_PATH_AUTO_FREE(path);
345	struct extent_buffer *leaf;
346	struct btrfs_trans_handle *trans;
347	char AUTO_KVFREE(buf);
348	struct btrfs_key key;
349	u32 nritems;
350	int slot;
351	int ret;
352	const u64 len = olen_aligned;
353	u64 last_dest_end = destoff;
354	u64 prev_extent_end = off;
355
356	ret = -ENOMEM;
357	buf = kvmalloc(fs_info->nodesize, GFP_KERNEL);
358	if (!buf)
359		return ret;
360
361	path = btrfs_alloc_path();
362	if (!path)
363		return ret;
364
365	path->reada = READA_FORWARD;
366	/* Clone data */
367	key.objectid = btrfs_ino(BTRFS_I(src));
368	key.type = BTRFS_EXTENT_DATA_KEY;
369	key.offset = off;
370
371	while (1) {
372		struct btrfs_file_extent_item *extent;
373		u64 extent_gen;
374		int type;
375		u32 size;
376		struct btrfs_key new_key;
377		u64 disko = 0, diskl = 0;
378		u64 datao = 0, datal = 0;
379		u8 comp;
380		u64 drop_start;
381
382		/* Note the key will change type as we walk through the tree */
383		ret = btrfs_search_slot(NULL, BTRFS_I(src)->root, &key, path,
384				0, 0);
385		if (ret < 0)
386			goto out;
387		/*
388		 * First search, if no extent item that starts at offset off was
389		 * found but the previous item is an extent item, it's possible
390		 * it might overlap our target range, therefore process it.
391		 */
392		if (key.offset == off && ret > 0 && path->slots[0] > 0) {
393			btrfs_item_key_to_cpu(path->nodes[0], &key,
394					      path->slots[0] - 1);
395			if (key.type == BTRFS_EXTENT_DATA_KEY)
396				path->slots[0]--;
397		}
398
399		nritems = btrfs_header_nritems(path->nodes[0]);
400process_slot:
401		if (path->slots[0] >= nritems) {
402			ret = btrfs_next_leaf(BTRFS_I(src)->root, path);
403			if (ret < 0)
404				goto out;
405			if (ret > 0)
406				break;
407			nritems = btrfs_header_nritems(path->nodes[0]);
408		}
409		leaf = path->nodes[0];
410		slot = path->slots[0];
411
412		btrfs_item_key_to_cpu(leaf, &key, slot);
413		if (key.type > BTRFS_EXTENT_DATA_KEY ||
414		    key.objectid != btrfs_ino(BTRFS_I(src)))
415			break;
416
417		ASSERT(key.type == BTRFS_EXTENT_DATA_KEY);
418
419		extent = btrfs_item_ptr(leaf, slot,
420					struct btrfs_file_extent_item);
421		extent_gen = btrfs_file_extent_generation(leaf, extent);
422		comp = btrfs_file_extent_compression(leaf, extent);
423		type = btrfs_file_extent_type(leaf, extent);
424		if (type == BTRFS_FILE_EXTENT_REG ||
425		    type == BTRFS_FILE_EXTENT_PREALLOC) {
426			disko = btrfs_file_extent_disk_bytenr(leaf, extent);
427			diskl = btrfs_file_extent_disk_num_bytes(leaf, extent);
428			datao = btrfs_file_extent_offset(leaf, extent);
429			datal = btrfs_file_extent_num_bytes(leaf, extent);
430		} else if (type == BTRFS_FILE_EXTENT_INLINE) {
431			/* Take upper bound, may be compressed */
432			datal = btrfs_file_extent_ram_bytes(leaf, extent);
433		}
434
435		/*
436		 * The first search might have left us at an extent item that
437		 * ends before our target range's start, can happen if we have
438		 * holes and NO_HOLES feature enabled.
439		 *
440		 * Subsequent searches may leave us on a file range we have
441		 * processed before - this happens due to a race with ordered
442		 * extent completion for a file range that is outside our source
443		 * range, but that range was part of a file extent item that
444		 * also covered a leading part of our source range.
445		 */
446		if (key.offset + datal <= prev_extent_end) {
447			path->slots[0]++;
448			goto process_slot;
449		} else if (key.offset >= off + len) {
450			break;
451		}
452
453		prev_extent_end = key.offset + datal;
454		size = btrfs_item_size(leaf, slot);
455		read_extent_buffer(leaf, buf, btrfs_item_ptr_offset(leaf, slot),
456				   size);
457
458		btrfs_release_path(path);
459
460		memcpy(&new_key, &key, sizeof(new_key));
461		new_key.objectid = btrfs_ino(BTRFS_I(inode));
462		if (off <= key.offset)
463			new_key.offset = key.offset + destoff - off;
464		else
465			new_key.offset = destoff;
466
467		/*
468		 * Deal with a hole that doesn't have an extent item that
469		 * represents it (NO_HOLES feature enabled).
470		 * This hole is either in the middle of the cloning range or at
471		 * the beginning (fully overlaps it or partially overlaps it).
472		 */
473		if (new_key.offset != last_dest_end)
474			drop_start = last_dest_end;
475		else
476			drop_start = new_key.offset;
477
478		if (type == BTRFS_FILE_EXTENT_REG ||
479		    type == BTRFS_FILE_EXTENT_PREALLOC) {
480			struct btrfs_replace_extent_info clone_info;
481
482			/*
483			 *    a  | --- range to clone ---|  b
484			 * | ------------- extent ------------- |
485			 */
486
487			/* Subtract range b */
488			if (key.offset + datal > off + len)
489				datal = off + len - key.offset;
490
491			/* Subtract range a */
492			if (off > key.offset) {
493				datao += off - key.offset;
494				datal -= off - key.offset;
495			}
496
497			clone_info.disk_offset = disko;
498			clone_info.disk_len = diskl;
499			clone_info.data_offset = datao;
500			clone_info.data_len = datal;
501			clone_info.file_offset = new_key.offset;
502			clone_info.extent_buf = buf;
503			clone_info.is_new_extent = false;
504			clone_info.update_times = !no_time_update;
505			ret = btrfs_replace_file_extents(BTRFS_I(inode), path,
506					drop_start, new_key.offset + datal - 1,
507					&clone_info, &trans);
508			if (ret)
509				goto out;
510		} else {
511			ASSERT(type == BTRFS_FILE_EXTENT_INLINE);
512			/*
513			 * Inline extents always have to start at file offset 0
514			 * and can never be bigger then the sector size. We can
515			 * never clone only parts of an inline extent, since all
516			 * reflink operations must start at a sector size aligned
517			 * offset, and the length must be aligned too or end at
518			 * the i_size (which implies the whole inlined data).
519			 */
520			ASSERT(key.offset == 0);
521			ASSERT(datal <= fs_info->sectorsize);
522			if (WARN_ON(type != BTRFS_FILE_EXTENT_INLINE) ||
523			    WARN_ON(key.offset != 0) ||
524			    WARN_ON(datal > fs_info->sectorsize)) {
525				ret = -EUCLEAN;
526				goto out;
527			}
528
529			ret = clone_copy_inline_extent(BTRFS_I(inode), path, &new_key,
530						       drop_start, datal, size,
531						       comp, buf, &trans);
532			if (ret)
533				goto out;
534		}
535
536		btrfs_release_path(path);
537
538		/*
539		 * Whenever we share an extent we update the last_reflink_trans
540		 * of each inode to the current transaction. This is needed to
541		 * make sure fsync does not log multiple checksum items with
542		 * overlapping ranges (because some extent items might refer
543		 * only to sections of the original extent). For the destination
544		 * inode we do this regardless of the generation of the extents
545		 * or even if they are inline extents or explicit holes, to make
546		 * sure a full fsync does not skip them. For the source inode,
547		 * we only need to update last_reflink_trans in case it's a new
548		 * extent that is not a hole or an inline extent, to deal with
549		 * the checksums problem on fsync.
550		 */
551		if (extent_gen == trans->transid && disko > 0)
552			BTRFS_I(src)->last_reflink_trans = trans->transid;
553
554		BTRFS_I(inode)->last_reflink_trans = trans->transid;
555
556		last_dest_end = ALIGN(new_key.offset + datal,
557				      fs_info->sectorsize);
558		ret = clone_finish_inode_update(trans, inode, last_dest_end,
559						destoff, olen, no_time_update);
560		if (ret)
561			goto out;
562		if (new_key.offset + datal >= destoff + len)
563			break;
564
565		btrfs_release_path(path);
566		key.offset = prev_extent_end;
567
568		if (fatal_signal_pending(current)) {
569			ret = -EINTR;
570			goto out;
571		}
572
573		cond_resched();
574	}
575	ret = 0;
576
577	if (last_dest_end < destoff + len) {
578		/*
579		 * We have an implicit hole that fully or partially overlaps our
580		 * cloning range at its end. This means that we either have the
581		 * NO_HOLES feature enabled or the implicit hole happened due to
582		 * mixing buffered and direct IO writes against this file.
583		 */
584		btrfs_release_path(path);
585
586		/*
587		 * When using NO_HOLES and we are cloning a range that covers
588		 * only a hole (no extents) into a range beyond the current
589		 * i_size, punching a hole in the target range will not create
590		 * an extent map defining a hole, because the range starts at or
591		 * beyond current i_size. If the file previously had an i_size
592		 * greater than the new i_size set by this clone operation, we
593		 * need to make sure the next fsync is a full fsync, so that it
594		 * detects and logs a hole covering a range from the current
595		 * i_size to the new i_size. If the clone range covers extents,
596		 * besides a hole, then we know the full sync flag was already
597		 * set by previous calls to btrfs_replace_file_extents() that
598		 * replaced file extent items.
599		 */
600		if (last_dest_end >= i_size_read(inode))
601			btrfs_set_inode_full_sync(BTRFS_I(inode));
602
603		ret = btrfs_replace_file_extents(BTRFS_I(inode), path,
604				last_dest_end, destoff + len - 1, NULL, &trans);
605		if (ret)
606			goto out;
607
608		ret = clone_finish_inode_update(trans, inode, destoff + len,
609						destoff, olen, no_time_update);
610	}
611
612out:
613	clear_bit(BTRFS_INODE_NO_DELALLOC_FLUSH, &BTRFS_I(inode)->runtime_flags);
614
615	return ret;
616}
617
618static void btrfs_double_mmap_lock(struct btrfs_inode *inode1, struct btrfs_inode *inode2)
619{
620	if (inode1 < inode2)
621		swap(inode1, inode2);
622	down_write(&inode1->i_mmap_lock);
623	down_write_nested(&inode2->i_mmap_lock, SINGLE_DEPTH_NESTING);
624}
625
626static void btrfs_double_mmap_unlock(struct btrfs_inode *inode1, struct btrfs_inode *inode2)
627{
628	up_write(&inode1->i_mmap_lock);
629	up_write(&inode2->i_mmap_lock);
630}
631
632static int btrfs_extent_same_range(struct btrfs_inode *src, u64 loff, u64 len,
633				   struct btrfs_inode *dst, u64 dst_loff)
634{
635	const u64 end = dst_loff + len - 1;
636	struct extent_state *cached_state = NULL;
637	struct btrfs_fs_info *fs_info = src->root->fs_info;
638	const u64 bs = fs_info->sectorsize;
639	int ret;
640
641	/*
642	 * Lock destination range to serialize with concurrent readahead(), and
643	 * we are safe from concurrency with relocation of source extents
644	 * because we have already locked the inode's i_mmap_lock in exclusive
645	 * mode.
646	 */
647	btrfs_lock_extent(&dst->io_tree, dst_loff, end, &cached_state);
648	ret = btrfs_clone(&src->vfs_inode, &dst->vfs_inode, loff, len,
649			  ALIGN(len, bs), dst_loff, 1);
650	btrfs_unlock_extent(&dst->io_tree, dst_loff, end, &cached_state);
651
652	btrfs_btree_balance_dirty(fs_info);
653
654	return ret;
655}
656
657static int btrfs_extent_same(struct inode *src, u64 loff, u64 olen,
658			     struct inode *dst, u64 dst_loff)
659{
660	int ret = 0;
661	u64 i, tail_len, chunk_count;
662	struct btrfs_root *root_dst = BTRFS_I(dst)->root;
663
664	spin_lock(&root_dst->root_item_lock);
665	if (root_dst->send_in_progress) {
666		btrfs_warn_rl(root_dst->fs_info,
667"cannot deduplicate to root %llu while send operations are using it (%d in progress)",
668			      btrfs_root_id(root_dst),
669			      root_dst->send_in_progress);
670		spin_unlock(&root_dst->root_item_lock);
671		return -EAGAIN;
672	}
673	root_dst->dedupe_in_progress++;
674	spin_unlock(&root_dst->root_item_lock);
675
676	tail_len = olen % BTRFS_MAX_DEDUPE_LEN;
677	chunk_count = div_u64(olen, BTRFS_MAX_DEDUPE_LEN);
678
679	for (i = 0; i < chunk_count; i++) {
680		ret = btrfs_extent_same_range(BTRFS_I(src), loff, BTRFS_MAX_DEDUPE_LEN,
681					      BTRFS_I(dst), dst_loff);
682		if (ret)
683			goto out;
684
685		loff += BTRFS_MAX_DEDUPE_LEN;
686		dst_loff += BTRFS_MAX_DEDUPE_LEN;
687	}
688
689	if (tail_len > 0)
690		ret = btrfs_extent_same_range(BTRFS_I(src), loff, tail_len,
691					      BTRFS_I(dst), dst_loff);
692out:
693	spin_lock(&root_dst->root_item_lock);
694	root_dst->dedupe_in_progress--;
695	spin_unlock(&root_dst->root_item_lock);
696
697	return ret;
698}
699
700static noinline int btrfs_clone_files(struct file *file, struct file *file_src,
701					u64 off, u64 olen, u64 destoff)
702{
703	struct extent_state *cached_state = NULL;
704	struct inode *inode = file_inode(file);
705	struct inode *src = file_inode(file_src);
706	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
707	int ret;
708	u64 len = olen;
709	u64 bs = fs_info->sectorsize;
710	u64 end;
711
712	/*
713	 * VFS's generic_remap_file_range_prep() protects us from cloning the
714	 * eof block into the middle of a file, which would result in corruption
715	 * if the file size is not blocksize aligned. So we don't need to check
716	 * for that case here.
717	 */
718	if (off + len == src->i_size)
719		len = ALIGN(src->i_size, bs) - off;
720
721	if (destoff > inode->i_size) {
722		const u64 wb_start = ALIGN_DOWN(inode->i_size, bs);
723
724		ret = btrfs_cont_expand(BTRFS_I(inode), inode->i_size, destoff);
725		if (ret)
726			return ret;
727		/*
728		 * We may have truncated the last block if the inode's size is
729		 * not sector size aligned, so we need to wait for writeback to
730		 * complete before proceeding further, otherwise we can race
731		 * with cloning and attempt to increment a reference to an
732		 * extent that no longer exists (writeback completed right after
733		 * we found the previous extent covering eof and before we
734		 * attempted to increment its reference count).
735		 */
736		ret = btrfs_wait_ordered_range(BTRFS_I(inode), wb_start,
737					       destoff - wb_start);
738		if (ret)
739			return ret;
740	}
741
742	/*
743	 * Lock destination range to serialize with concurrent readahead(), and
744	 * we are safe from concurrency with relocation of source extents
745	 * because we have already locked the inode's i_mmap_lock in exclusive
746	 * mode.
747	 */
748	end = destoff + len - 1;
749	btrfs_lock_extent(&BTRFS_I(inode)->io_tree, destoff, end, &cached_state);
750	ret = btrfs_clone(src, inode, off, olen, len, destoff, 0);
751	btrfs_unlock_extent(&BTRFS_I(inode)->io_tree, destoff, end, &cached_state);
752	if (ret < 0)
753		return ret;
754
755	/*
756	 * We may have copied an inline extent into a page of the destination
757	 * range, so wait for writeback to complete before invalidating pages
758	 * from the page cache. This is a rare case.
759	 */
760	ret = btrfs_wait_ordered_range(BTRFS_I(inode), destoff, len);
761	if (ret < 0)
762		return ret;
763
764	/*
765	 * Invalidate page cache so that future reads will see the cloned data
766	 * immediately and not the previous data.
767	 */
768	ret = filemap_invalidate_inode(inode, false, destoff, end);
769	if (ret < 0)
770		return ret;
771
772	btrfs_btree_balance_dirty(fs_info);
773
774	return 0;
775}
776
777static int btrfs_remap_file_range_prep(struct file *file_in, loff_t pos_in,
778				       struct file *file_out, loff_t pos_out,
779				       loff_t *len, unsigned int remap_flags)
780{
781	struct btrfs_inode *inode_in = BTRFS_I(file_inode(file_in));
782	struct btrfs_inode *inode_out = BTRFS_I(file_inode(file_out));
783	u64 bs = inode_out->root->fs_info->sectorsize;
784	u64 wb_len;
785	int ret;
786
787	if (!(remap_flags & REMAP_FILE_DEDUP)) {
788		struct btrfs_root *root_out = inode_out->root;
789
790		if (btrfs_root_readonly(root_out))
791			return -EROFS;
792
793		ASSERT(inode_in->vfs_inode.i_sb == inode_out->vfs_inode.i_sb);
794	}
795
796	/* Can only reflink encrypted files if both files are encrypted. */
797	if (IS_ENCRYPTED(&inode_in->vfs_inode) != IS_ENCRYPTED(&inode_out->vfs_inode))
798		return -EINVAL;
799
800	/* Don't make the dst file partly checksummed */
801	if ((inode_in->flags & BTRFS_INODE_NODATASUM) !=
802	    (inode_out->flags & BTRFS_INODE_NODATASUM)) {
803		return -EINVAL;
804	}
805
806	/*
807	 * Now that the inodes are locked, we need to start writeback ourselves
808	 * and can not rely on the writeback from the VFS's generic helper
809	 * generic_remap_file_range_prep() because:
810	 *
811	 * 1) For compression we must call filemap_fdatawrite_range() range
812	 *    twice (btrfs_fdatawrite_range() does it for us), and the generic
813	 *    helper only calls it once;
814	 *
815	 * 2) filemap_fdatawrite_range(), called by the generic helper only
816	 *    waits for the writeback to complete, i.e. for IO to be done, and
817	 *    not for the ordered extents to complete. We need to wait for them
818	 *    to complete so that new file extent items are in the fs tree.
819	 */
820	if (*len == 0 && !(remap_flags & REMAP_FILE_DEDUP))
821		wb_len = ALIGN(inode_in->vfs_inode.i_size, bs) - ALIGN_DOWN(pos_in, bs);
822	else
823		wb_len = ALIGN(*len, bs);
824
825	/*
826	 * Workaround to make sure NOCOW buffered write reach disk as NOCOW.
827	 *
828	 * Btrfs' back references do not have a block level granularity, they
829	 * work at the whole extent level.
830	 * NOCOW buffered write without data space reserved may not be able
831	 * to fall back to CoW due to lack of data space, thus could cause
832	 * data loss.
833	 *
834	 * Here we take a shortcut by flushing the whole inode, so that all
835	 * nocow write should reach disk as nocow before we increase the
836	 * reference of the extent. We could do better by only flushing NOCOW
837	 * data, but that needs extra accounting.
838	 *
839	 * Also we don't need to check ASYNC_EXTENT, as async extent will be
840	 * CoWed anyway, not affecting nocow part.
841	 */
842	ret = filemap_flush(inode_in->vfs_inode.i_mapping);
843	if (ret < 0)
844		return ret;
845
846	ret = btrfs_wait_ordered_range(inode_in, ALIGN_DOWN(pos_in, bs), wb_len);
847	if (ret < 0)
848		return ret;
849	ret = btrfs_wait_ordered_range(inode_out, ALIGN_DOWN(pos_out, bs), wb_len);
850	if (ret < 0)
851		return ret;
852
853	return generic_remap_file_range_prep(file_in, pos_in, file_out, pos_out,
854					    len, remap_flags);
855}
856
857static bool file_sync_write(const struct file *file)
858{
859	if (file->f_flags & (__O_SYNC | O_DSYNC))
860		return true;
861	if (IS_SYNC(file_inode(file)))
862		return true;
863
864	return false;
865}
866
867loff_t btrfs_remap_file_range(struct file *src_file, loff_t off,
868		struct file *dst_file, loff_t destoff, loff_t len,
869		unsigned int remap_flags)
870{
871	struct btrfs_inode *src_inode = BTRFS_I(file_inode(src_file));
872	struct btrfs_inode *dst_inode = BTRFS_I(file_inode(dst_file));
873	bool same_inode = dst_inode == src_inode;
874	int ret;
875
876	if (unlikely(btrfs_is_shutdown(inode_to_fs_info(file_inode(src_file)))))
877		return -EIO;
878
879	if (remap_flags & ~(REMAP_FILE_DEDUP | REMAP_FILE_ADVISORY))
880		return -EINVAL;
881
882	if (same_inode) {
883		btrfs_inode_lock(src_inode, BTRFS_ILOCK_MMAP);
884	} else {
885		lock_two_nondirectories(&src_inode->vfs_inode, &dst_inode->vfs_inode);
886		btrfs_double_mmap_lock(src_inode, dst_inode);
887	}
888
889	ret = btrfs_remap_file_range_prep(src_file, off, dst_file, destoff,
890					  &len, remap_flags);
891	if (ret < 0 || len == 0)
892		goto out_unlock;
893
894	if (remap_flags & REMAP_FILE_DEDUP)
895		ret = btrfs_extent_same(&src_inode->vfs_inode, off, len,
896					&dst_inode->vfs_inode, destoff);
897	else
898		ret = btrfs_clone_files(dst_file, src_file, off, len, destoff);
899
900out_unlock:
901	if (same_inode) {
902		btrfs_inode_unlock(src_inode, BTRFS_ILOCK_MMAP);
903	} else {
904		btrfs_double_mmap_unlock(src_inode, dst_inode);
905		unlock_two_nondirectories(&src_inode->vfs_inode,
906					  &dst_inode->vfs_inode);
907	}
908
909	/*
910	 * If either the source or the destination file was opened with O_SYNC,
911	 * O_DSYNC or has the S_SYNC attribute, fsync both the destination and
912	 * source files/ranges, so that after a successful return (0) followed
913	 * by a power failure results in the reflinked data to be readable from
914	 * both files/ranges.
915	 */
916	if (ret == 0 && len > 0 &&
917	    (file_sync_write(src_file) || file_sync_write(dst_file))) {
918		ret = btrfs_sync_file(src_file, off, off + len - 1, 0);
919		if (ret == 0)
920			ret = btrfs_sync_file(dst_file, destoff,
921					      destoff + len - 1, 0);
922	}
923
924	return ret < 0 ? ret : len;
925}