copy_up.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 *
   4 * Copyright (C) 2011 Novell Inc.
   5 */
   6
   7#include <linux/module.h>
   8#include <linux/fs.h>
   9#include <linux/slab.h>
  10#include <linux/file.h>
  11#include <linux/fileattr.h>
  12#include <linux/splice.h>
  13#include <linux/xattr.h>
  14#include <linux/security.h>
  15#include <linux/uaccess.h>
  16#include <linux/sched/signal.h>
  17#include <linux/cred.h>
  18#include <linux/namei.h>
  19#include <linux/ratelimit.h>
  20#include <linux/exportfs.h>
  21#include "overlayfs.h"
  22
  23#define OVL_COPY_UP_CHUNK_SIZE (1 << 20)
  24
  25static int ovl_ccup_set(const char *buf, const struct kernel_param *param)
  26{
  27	pr_warn("\"check_copy_up\" module option is obsolete\n");
  28	return 0;
  29}
  30
  31static int ovl_ccup_get(char *buf, const struct kernel_param *param)
  32{
  33	return sprintf(buf, "N\n");
  34}
  35
  36module_param_call(check_copy_up, ovl_ccup_set, ovl_ccup_get, NULL, 0644);
  37MODULE_PARM_DESC(check_copy_up, "Obsolete; does nothing");
  38
  39static bool ovl_must_copy_xattr(const char *name)
  40{
  41	return !strcmp(name, XATTR_POSIX_ACL_ACCESS) ||
  42	       !strcmp(name, XATTR_POSIX_ACL_DEFAULT) ||
  43	       !strncmp(name, XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN);
  44}
  45
  46static int ovl_copy_acl(struct ovl_fs *ofs, const struct path *path,
  47			struct dentry *dentry, const char *acl_name)
  48{
  49	int err;
  50	struct posix_acl *clone, *real_acl = NULL;
  51
  52	real_acl = ovl_get_acl_path(path, acl_name, false);
  53	if (!real_acl)
  54		return 0;
  55
  56	if (IS_ERR(real_acl)) {
  57		err = PTR_ERR(real_acl);
  58		if (err == -ENODATA || err == -EOPNOTSUPP)
  59			return 0;
  60		return err;
  61	}
  62
  63	clone = posix_acl_clone(real_acl, GFP_KERNEL);
  64	posix_acl_release(real_acl); /* release original acl */
  65	if (!clone)
  66		return -ENOMEM;
  67
  68	err = ovl_do_set_acl(ofs, dentry, acl_name, clone);
  69
  70	/* release cloned acl */
  71	posix_acl_release(clone);
  72	return err;
  73}
  74
  75int ovl_copy_xattr(struct super_block *sb, const struct path *oldpath, struct dentry *new)
  76{
  77	struct dentry *old = oldpath->dentry;
  78	ssize_t list_size, size, value_size = 0;
  79	char *buf, *name, *value = NULL;
  80	int error = 0;
  81	size_t slen;
  82
  83	if (!old->d_inode->i_op->listxattr || !new->d_inode->i_op->listxattr)
  84		return 0;
  85
  86	list_size = vfs_listxattr(old, NULL, 0);
  87	if (list_size <= 0) {
  88		if (list_size == -EOPNOTSUPP)
  89			return 0;
  90		return list_size;
  91	}
  92
  93	buf = kvzalloc(list_size, GFP_KERNEL);
  94	if (!buf)
  95		return -ENOMEM;
  96
  97	list_size = vfs_listxattr(old, buf, list_size);
  98	if (list_size <= 0) {
  99		error = list_size;
 100		goto out;
 101	}
 102
 103	for (name = buf; list_size; name += slen) {
 104		slen = strnlen(name, list_size) + 1;
 105
 106		/* underlying fs providing us with an broken xattr list? */
 107		if (WARN_ON(slen > list_size)) {
 108			error = -EIO;
 109			break;
 110		}
 111		list_size -= slen;
 112
 113		if (ovl_is_private_xattr(sb, name))
 114			continue;
 115
 116		error = security_inode_copy_up_xattr(old, name);
 117		if (error == -ECANCELED) {
 118			error = 0;
 119			continue; /* Discard */
 120		}
 121		if (error < 0 && error != -EOPNOTSUPP)
 122			break;
 123
 124		if (is_posix_acl_xattr(name)) {
 125			error = ovl_copy_acl(OVL_FS(sb), oldpath, new, name);
 126			if (!error)
 127				continue;
 128			/* POSIX ACLs must be copied. */
 129			break;
 130		}
 131
 132retry:
 133		size = ovl_do_getxattr(oldpath, name, value, value_size);
 134		if (size == -ERANGE)
 135			size = ovl_do_getxattr(oldpath, name, NULL, 0);
 136
 137		if (size < 0) {
 138			error = size;
 139			break;
 140		}
 141
 142		if (size > value_size) {
 143			void *new;
 144
 145			new = kvmalloc(size, GFP_KERNEL);
 146			if (!new) {
 147				error = -ENOMEM;
 148				break;
 149			}
 150			kvfree(value);
 151			value = new;
 152			value_size = size;
 153			goto retry;
 154		}
 155
 156		error = ovl_do_setxattr(OVL_FS(sb), new, name, value, size, 0);
 157		if (error) {
 158			if (error != -EOPNOTSUPP || ovl_must_copy_xattr(name))
 159				break;
 160
 161			/* Ignore failure to copy unknown xattrs */
 162			error = 0;
 163		}
 164	}
 165	kvfree(value);
 166out:
 167	kvfree(buf);
 168	return error;
 169}
 170
 171static int ovl_copy_fileattr(struct inode *inode, const struct path *old,
 172			     const struct path *new)
 173{
 174	struct file_kattr oldfa = { .flags_valid = true };
 175	struct file_kattr newfa = { .flags_valid = true };
 176	int err;
 177
 178	err = ovl_real_fileattr_get(old, &oldfa);
 179	if (err) {
 180		/* Ntfs-3g returns -EINVAL for "no fileattr support" */
 181		if (err == -ENOTTY || err == -EINVAL)
 182			return 0;
 183		pr_warn("failed to retrieve lower fileattr (%pd2, err=%i)\n",
 184			old->dentry, err);
 185		return err;
 186	}
 187
 188	/*
 189	 * We cannot set immutable and append-only flags on upper inode,
 190	 * because we would not be able to link upper inode to upper dir
 191	 * not set overlay private xattr on upper inode.
 192	 * Store these flags in overlay.protattr xattr instead.
 193	 */
 194	if (oldfa.flags & OVL_PROT_FS_FLAGS_MASK) {
 195		err = ovl_set_protattr(inode, new->dentry, &oldfa);
 196		if (err == -EPERM)
 197			pr_warn_once("copying fileattr: no xattr on upper\n");
 198		else if (err)
 199			return err;
 200	}
 201
 202	/* Don't bother copying flags if none are set */
 203	if (!(oldfa.flags & OVL_COPY_FS_FLAGS_MASK))
 204		return 0;
 205
 206	err = ovl_real_fileattr_get(new, &newfa);
 207	if (err) {
 208		/*
 209		 * Returning an error if upper doesn't support fileattr will
 210		 * result in a regression, so revert to the old behavior.
 211		 */
 212		if (err == -ENOTTY || err == -EINVAL) {
 213			pr_warn_once("copying fileattr: no support on upper\n");
 214			return 0;
 215		}
 216		pr_warn("failed to retrieve upper fileattr (%pd2, err=%i)\n",
 217			new->dentry, err);
 218		return err;
 219	}
 220
 221	BUILD_BUG_ON(OVL_COPY_FS_FLAGS_MASK & ~FS_COMMON_FL);
 222	newfa.flags &= ~OVL_COPY_FS_FLAGS_MASK;
 223	newfa.flags |= (oldfa.flags & OVL_COPY_FS_FLAGS_MASK);
 224
 225	BUILD_BUG_ON(OVL_COPY_FSX_FLAGS_MASK & ~FS_XFLAG_COMMON);
 226	newfa.fsx_xflags &= ~OVL_COPY_FSX_FLAGS_MASK;
 227	newfa.fsx_xflags |= (oldfa.fsx_xflags & OVL_COPY_FSX_FLAGS_MASK);
 228
 229	return ovl_real_fileattr_set(new, &newfa);
 230}
 231
 232static int ovl_verify_area(loff_t pos, loff_t pos2, loff_t len, loff_t totlen)
 233{
 234	loff_t tmp;
 235
 236	if (pos != pos2)
 237		return -EIO;
 238	if (pos < 0 || len < 0 || totlen < 0)
 239		return -EIO;
 240	if (check_add_overflow(pos, len, &tmp))
 241		return -EIO;
 242	return 0;
 243}
 244
 245static int ovl_sync_file(const struct path *path)
 246{
 247	struct file *new_file;
 248	int err;
 249
 250	new_file = ovl_path_open(path, O_LARGEFILE | O_RDONLY);
 251	if (IS_ERR(new_file))
 252		return PTR_ERR(new_file);
 253
 254	err = vfs_fsync(new_file, 0);
 255	fput(new_file);
 256
 257	return err;
 258}
 259
 260static int ovl_copy_up_file(struct ovl_fs *ofs, struct dentry *dentry,
 261			    struct file *new_file, loff_t len,
 262			    bool datasync)
 263{
 264	struct path datapath;
 265	struct file *old_file;
 266	loff_t old_pos = 0;
 267	loff_t new_pos = 0;
 268	loff_t cloned;
 269	loff_t data_pos = -1;
 270	loff_t hole_len;
 271	bool skip_hole = false;
 272	int error = 0;
 273
 274	ovl_path_lowerdata(dentry, &datapath);
 275	if (WARN_ON_ONCE(datapath.dentry == NULL) ||
 276	    WARN_ON_ONCE(len < 0))
 277		return -EIO;
 278
 279	old_file = ovl_path_open(&datapath, O_LARGEFILE | O_RDONLY);
 280	if (IS_ERR(old_file))
 281		return PTR_ERR(old_file);
 282
 283	/* Try to use clone_file_range to clone up within the same fs */
 284	cloned = vfs_clone_file_range(old_file, 0, new_file, 0, len, 0);
 285	if (cloned == len)
 286		goto out_fput;
 287
 288	/* Couldn't clone, so now we try to copy the data */
 289	error = rw_verify_area(READ, old_file, &old_pos, len);
 290	if (!error)
 291		error = rw_verify_area(WRITE, new_file, &new_pos, len);
 292	if (error)
 293		goto out_fput;
 294
 295	/* Check if lower fs supports seek operation */
 296	if (old_file->f_mode & FMODE_LSEEK)
 297		skip_hole = true;
 298
 299	while (len) {
 300		size_t this_len = OVL_COPY_UP_CHUNK_SIZE;
 301		ssize_t bytes;
 302
 303		if (len < this_len)
 304			this_len = len;
 305
 306		if (signal_pending_state(TASK_KILLABLE, current)) {
 307			error = -EINTR;
 308			break;
 309		}
 310
 311		/*
 312		 * Fill zero for hole will cost unnecessary disk space
 313		 * and meanwhile slow down the copy-up speed, so we do
 314		 * an optimization for hole during copy-up, it relies
 315		 * on SEEK_DATA implementation in lower fs so if lower
 316		 * fs does not support it, copy-up will behave as before.
 317		 *
 318		 * Detail logic of hole detection as below:
 319		 * When we detect next data position is larger than current
 320		 * position we will skip that hole, otherwise we copy
 321		 * data in the size of OVL_COPY_UP_CHUNK_SIZE. Actually,
 322		 * it may not recognize all kind of holes and sometimes
 323		 * only skips partial of hole area. However, it will be
 324		 * enough for most of the use cases.
 325		 *
 326		 * We do not hold upper sb_writers throughout the loop to avert
 327		 * lockdep warning with llseek of lower file in nested overlay:
 328		 * - upper sb_writers
 329		 * -- lower ovl_inode_lock (ovl_llseek)
 330		 */
 331		if (skip_hole && data_pos < old_pos) {
 332			data_pos = vfs_llseek(old_file, old_pos, SEEK_DATA);
 333			if (data_pos > old_pos) {
 334				hole_len = data_pos - old_pos;
 335				len -= hole_len;
 336				old_pos = new_pos = data_pos;
 337				continue;
 338			} else if (data_pos == -ENXIO) {
 339				break;
 340			} else if (data_pos < 0) {
 341				skip_hole = false;
 342			}
 343		}
 344
 345		error = ovl_verify_area(old_pos, new_pos, this_len, len);
 346		if (error)
 347			break;
 348
 349		bytes = do_splice_direct(old_file, &old_pos,
 350					 new_file, &new_pos,
 351					 this_len, SPLICE_F_MOVE);
 352		if (bytes <= 0) {
 353			error = bytes;
 354			break;
 355		}
 356		WARN_ON(old_pos != new_pos);
 357
 358		len -= bytes;
 359	}
 360	/* call fsync once, either now or later along with metadata */
 361	if (!error && ovl_should_sync(ofs) && datasync)
 362		error = vfs_fsync(new_file, 0);
 363out_fput:
 364	fput(old_file);
 365	return error;
 366}
 367
 368static int ovl_set_size(struct ovl_fs *ofs,
 369			struct dentry *upperdentry, struct kstat *stat)
 370{
 371	struct iattr attr = {
 372		.ia_valid = ATTR_SIZE,
 373		.ia_size = stat->size,
 374	};
 375
 376	return ovl_do_notify_change(ofs, upperdentry, &attr);
 377}
 378
 379static int ovl_set_timestamps(struct ovl_fs *ofs, struct dentry *upperdentry,
 380			      struct kstat *stat)
 381{
 382	struct iattr attr = {
 383		.ia_valid =
 384		     ATTR_ATIME | ATTR_MTIME | ATTR_ATIME_SET | ATTR_MTIME_SET | ATTR_CTIME,
 385		.ia_atime = stat->atime,
 386		.ia_mtime = stat->mtime,
 387	};
 388
 389	return ovl_do_notify_change(ofs, upperdentry, &attr);
 390}
 391
 392int ovl_set_attr(struct ovl_fs *ofs, struct dentry *upperdentry,
 393		 struct kstat *stat)
 394{
 395	int err = 0;
 396
 397	if (!S_ISLNK(stat->mode)) {
 398		struct iattr attr = {
 399			.ia_valid = ATTR_MODE,
 400			.ia_mode = stat->mode,
 401		};
 402		err = ovl_do_notify_change(ofs, upperdentry, &attr);
 403	}
 404	if (!err) {
 405		struct iattr attr = {
 406			.ia_valid = ATTR_UID | ATTR_GID,
 407			.ia_vfsuid = VFSUIDT_INIT(stat->uid),
 408			.ia_vfsgid = VFSGIDT_INIT(stat->gid),
 409		};
 410		err = ovl_do_notify_change(ofs, upperdentry, &attr);
 411	}
 412	if (!err)
 413		ovl_set_timestamps(ofs, upperdentry, stat);
 414
 415	return err;
 416}
 417
 418struct ovl_fh *ovl_encode_real_fh(struct ovl_fs *ofs, struct inode *realinode,
 419				  bool is_upper)
 420{
 421	struct ovl_fh *fh;
 422	int fh_type, dwords;
 423	int buflen = MAX_HANDLE_SZ;
 424	uuid_t *uuid = &realinode->i_sb->s_uuid;
 425	int err;
 426
 427	/* Make sure the real fid stays 32bit aligned */
 428	BUILD_BUG_ON(OVL_FH_FID_OFFSET % 4);
 429	BUILD_BUG_ON(MAX_HANDLE_SZ + OVL_FH_FID_OFFSET > 255);
 430
 431	fh = kzalloc(buflen + OVL_FH_FID_OFFSET, GFP_KERNEL);
 432	if (!fh)
 433		return ERR_PTR(-ENOMEM);
 434
 435	/*
 436	 * We encode a non-connectable file handle for non-dir, because we
 437	 * only need to find the lower inode number and we don't want to pay
 438	 * the price or reconnecting the dentry.
 439	 */
 440	dwords = buflen >> 2;
 441	fh_type = exportfs_encode_inode_fh(realinode, (void *)fh->fb.fid,
 442					   &dwords, NULL, 0);
 443	buflen = (dwords << 2);
 444
 445	err = -EIO;
 446	if (fh_type < 0 || fh_type == FILEID_INVALID ||
 447	    WARN_ON(buflen > MAX_HANDLE_SZ))
 448		goto out_err;
 449
 450	fh->fb.version = OVL_FH_VERSION;
 451	fh->fb.magic = OVL_FH_MAGIC;
 452	fh->fb.type = fh_type;
 453	fh->fb.flags = OVL_FH_FLAG_CPU_ENDIAN;
 454	/*
 455	 * When we will want to decode an overlay dentry from this handle
 456	 * and all layers are on the same fs, if we get a disconncted real
 457	 * dentry when we decode fid, the only way to tell if we should assign
 458	 * it to upperdentry or to lowerstack is by checking this flag.
 459	 */
 460	if (is_upper)
 461		fh->fb.flags |= OVL_FH_FLAG_PATH_UPPER;
 462	fh->fb.len = sizeof(fh->fb) + buflen;
 463	if (ovl_origin_uuid(ofs))
 464		fh->fb.uuid = *uuid;
 465
 466	return fh;
 467
 468out_err:
 469	kfree(fh);
 470	return ERR_PTR(err);
 471}
 472
 473struct ovl_fh *ovl_get_origin_fh(struct ovl_fs *ofs, struct dentry *origin)
 474{
 475	/*
 476	 * When lower layer doesn't support export operations store a 'null' fh,
 477	 * so we can use the overlay.origin xattr to distignuish between a copy
 478	 * up and a pure upper inode.
 479	 */
 480	if (!ovl_can_decode_fh(origin->d_sb))
 481		return NULL;
 482
 483	return ovl_encode_real_fh(ofs, d_inode(origin), false);
 484}
 485
 486int ovl_set_origin_fh(struct ovl_fs *ofs, const struct ovl_fh *fh,
 487		      struct dentry *upper)
 488{
 489	int err;
 490
 491	/*
 492	 * Do not fail when upper doesn't support xattrs.
 493	 */
 494	err = ovl_check_setxattr(ofs, upper, OVL_XATTR_ORIGIN, fh->buf,
 495				 fh ? fh->fb.len : 0, 0);
 496
 497	/* Ignore -EPERM from setting "user.*" on symlink/special */
 498	return err == -EPERM ? 0 : err;
 499}
 500
 501/* Store file handle of @upper dir in @index dir entry */
 502static int ovl_set_upper_fh(struct ovl_fs *ofs, struct dentry *upper,
 503			    struct dentry *index)
 504{
 505	const struct ovl_fh *fh;
 506	int err;
 507
 508	fh = ovl_encode_real_fh(ofs, d_inode(upper), true);
 509	if (IS_ERR(fh))
 510		return PTR_ERR(fh);
 511
 512	err = ovl_setxattr(ofs, index, OVL_XATTR_UPPER, fh->buf, fh->fb.len);
 513
 514	kfree(fh);
 515	return err;
 516}
 517
 518/*
 519 * Create and install index entry.
 520 */
 521static int ovl_create_index(struct dentry *dentry, const struct ovl_fh *fh,
 522			    struct dentry *upper)
 523{
 524	struct ovl_fs *ofs = OVL_FS(dentry->d_sb);
 525	struct dentry *indexdir = ovl_indexdir(dentry->d_sb);
 526	struct dentry *temp = NULL;
 527	struct renamedata rd = {};
 528	struct qstr name = { };
 529	int err;
 530
 531	/*
 532	 * For now this is only used for creating index entry for directories,
 533	 * because non-dir are copied up directly to index and then hardlinked
 534	 * to upper dir.
 535	 *
 536	 * TODO: implement create index for non-dir, so we can call it when
 537	 * encoding file handle for non-dir in case index does not exist.
 538	 */
 539	if (WARN_ON(!d_is_dir(dentry)))
 540		return -EIO;
 541
 542	/* Directory not expected to be indexed before copy up */
 543	if (WARN_ON(ovl_test_flag(OVL_INDEX, d_inode(dentry))))
 544		return -EIO;
 545
 546	err = ovl_get_index_name_fh(fh, &name);
 547	if (err)
 548		return err;
 549
 550	temp = ovl_create_temp(ofs, indexdir, OVL_CATTR(S_IFDIR | 0));
 551	err = PTR_ERR(temp);
 552	if (IS_ERR(temp))
 553		goto free_name;
 554
 555	err = ovl_set_upper_fh(ofs, upper, temp);
 556	if (err)
 557		goto out;
 558
 559	rd.mnt_idmap = ovl_upper_mnt_idmap(ofs);
 560	rd.old_parent = indexdir;
 561	rd.new_parent = indexdir;
 562	err = start_renaming_dentry(&rd, 0, temp, &name);
 563	if (err)
 564		goto out;
 565
 566	err = ovl_do_rename_rd(&rd);
 567	end_renaming(&rd);
 568out:
 569	if (err)
 570		ovl_cleanup(ofs, indexdir, temp);
 571	dput(temp);
 572free_name:
 573	kfree(name.name);
 574	return err;
 575}
 576
 577struct ovl_copy_up_ctx {
 578	struct dentry *parent;
 579	struct dentry *dentry;
 580	struct path lowerpath;
 581	struct kstat stat;
 582	struct kstat pstat;
 583	const char *link;
 584	struct dentry *destdir;
 585	struct qstr destname;
 586	struct dentry *workdir;
 587	const struct ovl_fh *origin_fh;
 588	bool origin;
 589	bool indexed;
 590	bool metacopy;
 591	bool metacopy_digest;
 592	bool metadata_fsync;
 593};
 594
 595static int ovl_link_up(struct ovl_copy_up_ctx *c)
 596{
 597	int err;
 598	struct dentry *upper;
 599	struct dentry *upperdir = ovl_dentry_upper(c->parent);
 600	struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb);
 601	struct inode *udir = d_inode(upperdir);
 602
 603	ovl_start_write(c->dentry);
 604
 605	/* Mark parent "impure" because it may now contain non-pure upper */
 606	err = ovl_set_impure(c->parent, upperdir);
 607	if (err)
 608		goto out;
 609
 610	err = ovl_set_nlink_lower(c->dentry);
 611	if (err)
 612		goto out;
 613
 614	upper = ovl_start_creating_upper(ofs, upperdir,
 615					 &QSTR_LEN(c->dentry->d_name.name,
 616						   c->dentry->d_name.len));
 617	err = PTR_ERR(upper);
 618	if (!IS_ERR(upper)) {
 619		err = ovl_do_link(ofs, ovl_dentry_upper(c->dentry), udir, upper);
 620
 621		if (!err) {
 622			/* Restore timestamps on parent (best effort) */
 623			ovl_set_timestamps(ofs, upperdir, &c->pstat);
 624			ovl_dentry_set_upper_alias(c->dentry);
 625			ovl_dentry_update_reval(c->dentry, upper);
 626		}
 627		end_creating(upper);
 628	}
 629	if (err)
 630		goto out;
 631
 632	err = ovl_set_nlink_upper(c->dentry);
 633
 634out:
 635	ovl_end_write(c->dentry);
 636	return err;
 637}
 638
 639static int ovl_copy_up_data(struct ovl_copy_up_ctx *c, const struct path *temp)
 640{
 641	struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb);
 642	struct file *new_file;
 643	int err;
 644
 645	if (!S_ISREG(c->stat.mode) || c->metacopy || !c->stat.size)
 646		return 0;
 647
 648	new_file = ovl_path_open(temp, O_LARGEFILE | O_WRONLY);
 649	if (IS_ERR(new_file))
 650		return PTR_ERR(new_file);
 651
 652	err = ovl_copy_up_file(ofs, c->dentry, new_file, c->stat.size,
 653			       !c->metadata_fsync);
 654	fput(new_file);
 655
 656	return err;
 657}
 658
 659static int ovl_copy_up_metadata(struct ovl_copy_up_ctx *c, struct dentry *temp)
 660{
 661	struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb);
 662	struct inode *inode = d_inode(c->dentry);
 663	struct path upperpath = { .mnt = ovl_upper_mnt(ofs), .dentry = temp };
 664	int err;
 665
 666	err = ovl_copy_xattr(c->dentry->d_sb, &c->lowerpath, temp);
 667	if (err)
 668		return err;
 669
 670	if (inode->i_flags & OVL_FATTR_I_FLAGS_MASK &&
 671	    (S_ISREG(c->stat.mode) || S_ISDIR(c->stat.mode))) {
 672		/*
 673		 * Copy the fileattr inode flags that are the source of already
 674		 * copied i_flags
 675		 */
 676		err = ovl_copy_fileattr(inode, &c->lowerpath, &upperpath);
 677		if (err)
 678			return err;
 679	}
 680
 681	/*
 682	 * Store identifier of lower inode in upper inode xattr to
 683	 * allow lookup of the copy up origin inode.
 684	 *
 685	 * Don't set origin when we are breaking the association with a lower
 686	 * hard link.
 687	 */
 688	if (c->origin) {
 689		err = ovl_set_origin_fh(ofs, c->origin_fh, temp);
 690		if (err)
 691			return err;
 692	}
 693
 694	if (c->metacopy) {
 695		struct path lowerdatapath;
 696		struct ovl_metacopy metacopy_data = OVL_METACOPY_INIT;
 697
 698		ovl_path_lowerdata(c->dentry, &lowerdatapath);
 699		if (WARN_ON_ONCE(lowerdatapath.dentry == NULL))
 700			return -EIO;
 701		err = ovl_get_verity_digest(ofs, &lowerdatapath, &metacopy_data);
 702		if (err)
 703			return err;
 704
 705		if (metacopy_data.digest_algo)
 706			c->metacopy_digest = true;
 707
 708		err = ovl_set_metacopy_xattr(ofs, temp, &metacopy_data);
 709		if (err)
 710			return err;
 711	}
 712
 713	inode_lock(temp->d_inode);
 714	if (S_ISREG(c->stat.mode))
 715		err = ovl_set_size(ofs, temp, &c->stat);
 716	if (!err)
 717		err = ovl_set_attr(ofs, temp, &c->stat);
 718	inode_unlock(temp->d_inode);
 719
 720	/* fsync metadata before moving it into upper dir */
 721	if (!err && ovl_should_sync(ofs) && c->metadata_fsync)
 722		err = ovl_sync_file(&upperpath);
 723
 724	return err;
 725}
 726
 727static const struct cred *ovl_prepare_copy_up_creds(struct dentry *dentry)
 728{
 729	struct cred *copy_up_cred = NULL;
 730	int err;
 731
 732	err = security_inode_copy_up(dentry, &copy_up_cred);
 733	if (err < 0)
 734		return ERR_PTR(err);
 735
 736	if (!copy_up_cred)
 737		return NULL;
 738
 739	return override_creds(copy_up_cred);
 740}
 741
 742static void ovl_revert_copy_up_creds(const struct cred *orig_cred)
 743{
 744	const struct cred *copy_up_cred;
 745
 746	copy_up_cred = revert_creds(orig_cred);
 747	put_cred(copy_up_cred);
 748}
 749
 750DEFINE_CLASS(copy_up_creds, const struct cred *,
 751	     if (!IS_ERR_OR_NULL(_T)) ovl_revert_copy_up_creds(_T),
 752	     ovl_prepare_copy_up_creds(dentry), struct dentry *dentry)
 753
 754/*
 755 * Copyup using workdir to prepare temp file.  Used when copying up directories,
 756 * special files or when upper fs doesn't support O_TMPFILE.
 757 */
 758static int ovl_copy_up_workdir(struct ovl_copy_up_ctx *c)
 759{
 760	struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb);
 761	struct inode *inode;
 762	struct path path = { .mnt = ovl_upper_mnt(ofs) };
 763	struct renamedata rd = {};
 764	struct dentry *temp;
 765	int err;
 766	struct ovl_cattr cattr = {
 767		/* Can't properly set mode on creation because of the umask */
 768		.mode = c->stat.mode & S_IFMT,
 769		.rdev = c->stat.rdev,
 770		.link = c->link
 771	};
 772
 773	scoped_class(copy_up_creds, copy_up_creds, c->dentry) {
 774		if (IS_ERR(copy_up_creds))
 775			return PTR_ERR(copy_up_creds);
 776
 777		ovl_start_write(c->dentry);
 778		temp = ovl_create_temp(ofs, c->workdir, &cattr);
 779		ovl_end_write(c->dentry);
 780	}
 781
 782	if (IS_ERR(temp))
 783		return PTR_ERR(temp);
 784
 785	/*
 786	 * Copy up data first and then xattrs. Writing data after
 787	 * xattrs will remove security.capability xattr automatically.
 788	 */
 789	path.dentry = temp;
 790	err = ovl_copy_up_data(c, &path);
 791	ovl_start_write(c->dentry);
 792	if (err)
 793		goto cleanup_unlocked;
 794
 795	if (S_ISDIR(c->stat.mode) && c->indexed) {
 796		err = ovl_create_index(c->dentry, c->origin_fh, temp);
 797		if (err)
 798			goto cleanup_unlocked;
 799	}
 800
 801	/*
 802	 * We cannot hold lock_rename() throughout this helper, because of
 803	 * lock ordering with sb_writers, which shouldn't be held when calling
 804	 * ovl_copy_up_data(), so lock workdir and destdir and make sure that
 805	 * temp wasn't moved before copy up completion or cleanup.
 806	 */
 807	rd.mnt_idmap = ovl_upper_mnt_idmap(ofs);
 808	rd.old_parent = c->workdir;
 809	rd.new_parent = c->destdir;
 810	rd.flags = 0;
 811	err = start_renaming_dentry(&rd, 0, temp,
 812				    &QSTR_LEN(c->destname.name, c->destname.len));
 813	if (err) {
 814		/* temp or workdir moved underneath us? map to -EIO */
 815		err = -EIO;
 816	}
 817	if (err)
 818		goto cleanup_unlocked;
 819
 820	err = ovl_copy_up_metadata(c, temp);
 821	if (!err)
 822		err = ovl_do_rename_rd(&rd);
 823	end_renaming(&rd);
 824
 825	if (err)
 826		goto cleanup_unlocked;
 827
 828	inode = d_inode(c->dentry);
 829	if (c->metacopy_digest)
 830		ovl_set_flag(OVL_HAS_DIGEST, inode);
 831	else
 832		ovl_clear_flag(OVL_HAS_DIGEST, inode);
 833	ovl_clear_flag(OVL_VERIFIED_DIGEST, inode);
 834
 835	if (!c->metacopy)
 836		ovl_set_upperdata(inode);
 837	ovl_inode_update(inode, temp);
 838	if (S_ISDIR(inode->i_mode))
 839		ovl_set_flag(OVL_WHITEOUTS, inode);
 840out:
 841	ovl_end_write(c->dentry);
 842
 843	return err;
 844
 845cleanup_unlocked:
 846	ovl_cleanup(ofs, c->workdir, temp);
 847	dput(temp);
 848	goto out;
 849}
 850
 851/* Copyup using O_TMPFILE which does not require cross dir locking */
 852static int ovl_copy_up_tmpfile(struct ovl_copy_up_ctx *c)
 853{
 854	struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb);
 855	struct inode *udir = d_inode(c->destdir);
 856	struct dentry *temp, *upper;
 857	struct file *tmpfile;
 858	int err;
 859
 860	scoped_class(copy_up_creds, copy_up_creds, c->dentry) {
 861		if (IS_ERR(copy_up_creds))
 862			return PTR_ERR(copy_up_creds);
 863
 864		ovl_start_write(c->dentry);
 865		tmpfile = ovl_do_tmpfile(ofs, c->workdir, c->stat.mode);
 866		ovl_end_write(c->dentry);
 867	}
 868
 869	if (IS_ERR(tmpfile))
 870		return PTR_ERR(tmpfile);
 871
 872	temp = tmpfile->f_path.dentry;
 873	if (!c->metacopy && c->stat.size) {
 874		err = ovl_copy_up_file(ofs, c->dentry, tmpfile, c->stat.size,
 875				       !c->metadata_fsync);
 876		if (err)
 877			goto out_fput;
 878	}
 879
 880	ovl_start_write(c->dentry);
 881
 882	err = ovl_copy_up_metadata(c, temp);
 883	if (err)
 884		goto out;
 885
 886	upper = ovl_start_creating_upper(ofs, c->destdir,
 887					 &QSTR_LEN(c->destname.name,
 888						   c->destname.len));
 889	err = PTR_ERR(upper);
 890	if (!IS_ERR(upper)) {
 891		err = ovl_do_link(ofs, temp, udir, upper);
 892		end_creating(upper);
 893	}
 894
 895	if (err)
 896		goto out;
 897
 898	if (c->metacopy_digest)
 899		ovl_set_flag(OVL_HAS_DIGEST, d_inode(c->dentry));
 900	else
 901		ovl_clear_flag(OVL_HAS_DIGEST, d_inode(c->dentry));
 902	ovl_clear_flag(OVL_VERIFIED_DIGEST, d_inode(c->dentry));
 903
 904	if (!c->metacopy)
 905		ovl_set_upperdata(d_inode(c->dentry));
 906	ovl_inode_update(d_inode(c->dentry), dget(temp));
 907
 908out:
 909	ovl_end_write(c->dentry);
 910out_fput:
 911	fput(tmpfile);
 912	return err;
 913}
 914
 915/*
 916 * Copy up a single dentry
 917 *
 918 * All renames start with copy up of source if necessary.  The actual
 919 * rename will only proceed once the copy up was successful.  Copy up uses
 920 * upper parent i_mutex for exclusion.  Since rename can change d_parent it
 921 * is possible that the copy up will lock the old parent.  At that point
 922 * the file will have already been copied up anyway.
 923 */
 924static int ovl_do_copy_up(struct ovl_copy_up_ctx *c)
 925{
 926	int err;
 927	struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb);
 928	struct dentry *origin = c->lowerpath.dentry;
 929	struct ovl_fh *fh = NULL;
 930	bool to_index = false;
 931
 932	/*
 933	 * Indexed non-dir is copied up directly to the index entry and then
 934	 * hardlinked to upper dir. Indexed dir is copied up to indexdir,
 935	 * then index entry is created and then copied up dir installed.
 936	 * Copying dir up to indexdir instead of workdir simplifies locking.
 937	 */
 938	if (ovl_need_index(c->dentry)) {
 939		c->indexed = true;
 940		if (S_ISDIR(c->stat.mode))
 941			c->workdir = ovl_indexdir(c->dentry->d_sb);
 942		else
 943			to_index = true;
 944	}
 945
 946	if (S_ISDIR(c->stat.mode) || c->stat.nlink == 1 || to_index) {
 947		fh = ovl_get_origin_fh(ofs, origin);
 948		if (IS_ERR(fh))
 949			return PTR_ERR(fh);
 950
 951		/* origin_fh may be NULL */
 952		c->origin_fh = fh;
 953		c->origin = true;
 954	}
 955
 956	if (to_index) {
 957		c->destdir = ovl_indexdir(c->dentry->d_sb);
 958		err = ovl_get_index_name(ofs, origin, &c->destname);
 959		if (err)
 960			goto out_free_fh;
 961	} else if (WARN_ON(!c->parent)) {
 962		/* Disconnected dentry must be copied up to index dir */
 963		err = -EIO;
 964		goto out_free_fh;
 965	} else {
 966		/*
 967		 * c->dentry->d_name is stabilzed by ovl_copy_up_start(),
 968		 * because if we got here, it means that c->dentry has no upper
 969		 * alias and changing ->d_name means going through ovl_rename()
 970		 * that will call ovl_copy_up() on source and target dentry.
 971		 */
 972		c->destname = c->dentry->d_name;
 973		/*
 974		 * Mark parent "impure" because it may now contain non-pure
 975		 * upper
 976		 */
 977		ovl_start_write(c->dentry);
 978		err = ovl_set_impure(c->parent, c->destdir);
 979		ovl_end_write(c->dentry);
 980		if (err)
 981			goto out_free_fh;
 982	}
 983
 984	/* Should we copyup with O_TMPFILE or with workdir? */
 985	if (S_ISREG(c->stat.mode) && ofs->tmpfile)
 986		err = ovl_copy_up_tmpfile(c);
 987	else
 988		err = ovl_copy_up_workdir(c);
 989	if (err)
 990		goto out;
 991
 992	if (c->indexed)
 993		ovl_set_flag(OVL_INDEX, d_inode(c->dentry));
 994
 995	ovl_start_write(c->dentry);
 996	if (to_index) {
 997		/* Initialize nlink for copy up of disconnected dentry */
 998		err = ovl_set_nlink_upper(c->dentry);
 999	} else {
1000		struct inode *udir = d_inode(c->destdir);
1001
1002		/* Restore timestamps on parent (best effort) */
1003		inode_lock(udir);
1004		ovl_set_timestamps(ofs, c->destdir, &c->pstat);
1005		inode_unlock(udir);
1006
1007		ovl_dentry_set_upper_alias(c->dentry);
1008		ovl_dentry_update_reval(c->dentry, ovl_dentry_upper(c->dentry));
1009	}
1010	ovl_end_write(c->dentry);
1011
1012out:
1013	if (to_index)
1014		kfree(c->destname.name);
1015out_free_fh:
1016	kfree(fh);
1017	return err;
1018}
1019
1020static bool ovl_need_meta_copy_up(struct dentry *dentry, umode_t mode,
1021				  int flags)
1022{
1023	struct ovl_fs *ofs = OVL_FS(dentry->d_sb);
1024
1025	if (!ofs->config.metacopy)
1026		return false;
1027
1028	if (!S_ISREG(mode))
1029		return false;
1030
1031	if (flags && ((OPEN_FMODE(flags) & FMODE_WRITE) || (flags & O_TRUNC)))
1032		return false;
1033
1034	/* Fall back to full copy if no fsverity on source data and we require verity */
1035	if (ofs->config.verity_mode == OVL_VERITY_REQUIRE) {
1036		struct path lowerdata;
1037
1038		ovl_path_lowerdata(dentry, &lowerdata);
1039
1040		if (WARN_ON_ONCE(lowerdata.dentry == NULL) ||
1041		    ovl_ensure_verity_loaded(&lowerdata) ||
1042		    !fsverity_active(d_inode(lowerdata.dentry))) {
1043			return false;
1044		}
1045	}
1046
1047	return true;
1048}
1049
1050static ssize_t ovl_getxattr_value(const struct path *path, char *name, char **value)
1051{
1052	ssize_t res;
1053	char *buf;
1054
1055	res = ovl_do_getxattr(path, name, NULL, 0);
1056	if (res == -ENODATA || res == -EOPNOTSUPP)
1057		res = 0;
1058
1059	if (res > 0) {
1060		buf = kzalloc(res, GFP_KERNEL);
1061		if (!buf)
1062			return -ENOMEM;
1063
1064		res = ovl_do_getxattr(path, name, buf, res);
1065		if (res < 0)
1066			kfree(buf);
1067		else
1068			*value = buf;
1069	}
1070	return res;
1071}
1072
1073/* Copy up data of an inode which was copied up metadata only in the past. */
1074static int ovl_copy_up_meta_inode_data(struct ovl_copy_up_ctx *c)
1075{
1076	struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb);
1077	struct path upperpath;
1078	int err;
1079	char *capability = NULL;
1080	ssize_t cap_size;
1081
1082	ovl_path_upper(c->dentry, &upperpath);
1083	if (WARN_ON(upperpath.dentry == NULL))
1084		return -EIO;
1085
1086	if (c->stat.size) {
1087		err = cap_size = ovl_getxattr_value(&upperpath, XATTR_NAME_CAPS,
1088						    &capability);
1089		if (cap_size < 0)
1090			goto out;
1091	}
1092
1093	err = ovl_copy_up_data(c, &upperpath);
1094	if (err)
1095		goto out_free;
1096
1097	/*
1098	 * Writing to upper file will clear security.capability xattr. We
1099	 * don't want that to happen for normal copy-up operation.
1100	 */
1101	ovl_start_write(c->dentry);
1102	if (capability) {
1103		err = ovl_do_setxattr(ofs, upperpath.dentry, XATTR_NAME_CAPS,
1104				      capability, cap_size, 0);
1105	}
1106	if (!err) {
1107		err = ovl_removexattr(ofs, upperpath.dentry,
1108				      OVL_XATTR_METACOPY);
1109	}
1110	ovl_end_write(c->dentry);
1111	if (err)
1112		goto out_free;
1113
1114	ovl_clear_flag(OVL_HAS_DIGEST, d_inode(c->dentry));
1115	ovl_clear_flag(OVL_VERIFIED_DIGEST, d_inode(c->dentry));
1116	ovl_set_upperdata(d_inode(c->dentry));
1117out_free:
1118	kfree(capability);
1119out:
1120	return err;
1121}
1122
1123static int ovl_copy_up_one(struct dentry *parent, struct dentry *dentry,
1124			   int flags)
1125{
1126	int err;
1127	DEFINE_DELAYED_CALL(done);
1128	struct path parentpath;
1129	struct ovl_copy_up_ctx ctx = {
1130		.parent = parent,
1131		.dentry = dentry,
1132		.workdir = ovl_workdir(dentry),
1133	};
1134
1135	if (WARN_ON(!ctx.workdir))
1136		return -EROFS;
1137
1138	ovl_path_lower(dentry, &ctx.lowerpath);
1139	err = vfs_getattr(&ctx.lowerpath, &ctx.stat,
1140			  STATX_BASIC_STATS, AT_STATX_SYNC_AS_STAT);
1141	if (err)
1142		return err;
1143
1144	if (!kuid_has_mapping(current_user_ns(), ctx.stat.uid) ||
1145	    !kgid_has_mapping(current_user_ns(), ctx.stat.gid))
1146		return -EOVERFLOW;
1147
1148	/*
1149	 * With "fsync=strict", we fsync after final metadata copyup, for
1150	 * both regular files and directories to get atomic copyup semantics
1151	 * on filesystems that do not use strict metadata ordering (e.g. ubifs).
1152	 *
1153	 * By default, we want to avoid fsync on all meta copyup, because
1154	 * that will hurt performance of workloads such as chown -R, so we
1155	 * only fsync on data copyup as legacy behavior.
1156	 */
1157	ctx.metadata_fsync = ovl_should_sync_metadata(OVL_FS(dentry->d_sb)) &&
1158			     (S_ISREG(ctx.stat.mode) || S_ISDIR(ctx.stat.mode));
1159	ctx.metacopy = ovl_need_meta_copy_up(dentry, ctx.stat.mode, flags);
1160
1161	if (parent) {
1162		ovl_path_upper(parent, &parentpath);
1163		ctx.destdir = parentpath.dentry;
1164
1165		err = vfs_getattr(&parentpath, &ctx.pstat,
1166				  STATX_ATIME | STATX_MTIME,
1167				  AT_STATX_SYNC_AS_STAT);
1168		if (err)
1169			return err;
1170	}
1171
1172	/* maybe truncate regular file. this has no effect on dirs */
1173	if (flags & O_TRUNC)
1174		ctx.stat.size = 0;
1175
1176	if (S_ISLNK(ctx.stat.mode)) {
1177		ctx.link = vfs_get_link(ctx.lowerpath.dentry, &done);
1178		if (IS_ERR(ctx.link))
1179			return PTR_ERR(ctx.link);
1180	}
1181
1182	err = ovl_copy_up_start(dentry, flags);
1183	/* err < 0: interrupted, err > 0: raced with another copy-up */
1184	if (unlikely(err)) {
1185		if (err > 0)
1186			err = 0;
1187	} else {
1188		if (!ovl_dentry_upper(dentry))
1189			err = ovl_do_copy_up(&ctx);
1190		if (!err && parent && !ovl_dentry_has_upper_alias(dentry))
1191			err = ovl_link_up(&ctx);
1192		if (!err && ovl_dentry_needs_data_copy_up_locked(dentry, flags))
1193			err = ovl_copy_up_meta_inode_data(&ctx);
1194		ovl_copy_up_end(dentry);
1195	}
1196	do_delayed_call(&done);
1197
1198	return err;
1199}
1200
1201static int ovl_copy_up_flags(struct dentry *dentry, int flags)
1202{
1203	int err = 0;
1204	bool disconnected = (dentry->d_flags & DCACHE_DISCONNECTED);
1205
1206	/*
1207	 * With NFS export, copy up can get called for a disconnected non-dir.
1208	 * In this case, we will copy up lower inode to index dir without
1209	 * linking it to upper dir.
1210	 */
1211	if (WARN_ON(disconnected && d_is_dir(dentry)))
1212		return -EIO;
1213
1214	/*
1215	 * We may not need lowerdata if we are only doing metacopy up, but it is
1216	 * not very important to optimize this case, so do lazy lowerdata lookup
1217	 * before any copy up, so we can do it before taking ovl_inode_lock().
1218	 */
1219	err = ovl_verify_lowerdata(dentry);
1220	if (err)
1221		return err;
1222
1223	while (!err) {
1224		struct dentry *next;
1225		struct dentry *parent = NULL;
1226
1227		if (ovl_already_copied_up(dentry, flags))
1228			break;
1229
1230		next = dget(dentry);
1231		/* find the topmost dentry not yet copied up */
1232		for (; !disconnected;) {
1233			parent = dget_parent(next);
1234
1235			if (ovl_dentry_upper(parent))
1236				break;
1237
1238			dput(next);
1239			next = parent;
1240		}
1241
1242		with_ovl_creds(dentry->d_sb)
1243			err = ovl_copy_up_one(parent, next, flags);
1244
1245		dput(parent);
1246		dput(next);
1247	}
1248
1249	return err;
1250}
1251
1252static bool ovl_open_need_copy_up(struct dentry *dentry, int flags)
1253{
1254	/* Copy up of disconnected dentry does not set upper alias */
1255	if (ovl_already_copied_up(dentry, flags))
1256		return false;
1257
1258	if (special_file(d_inode(dentry)->i_mode))
1259		return false;
1260
1261	if (!ovl_open_flags_need_copy_up(flags))
1262		return false;
1263
1264	return true;
1265}
1266
1267int ovl_maybe_copy_up(struct dentry *dentry, int flags)
1268{
1269	if (!ovl_open_need_copy_up(dentry, flags))
1270		return 0;
1271
1272	return ovl_copy_up_flags(dentry, flags);
1273}
1274
1275int ovl_copy_up_with_data(struct dentry *dentry)
1276{
1277	return ovl_copy_up_flags(dentry, O_WRONLY);
1278}
1279
1280int ovl_copy_up(struct dentry *dentry)
1281{
1282	return ovl_copy_up_flags(dentry, 0);
1283}