ublk_drv.c

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Userspace block device - block device which IO is handled from userspace
   4 *
   5 * Take full use of io_uring passthrough command for communicating with
   6 * ublk userspace daemon(ublksrvd) for handling basic IO request.
   7 *
   8 * Copyright 2022 Ming Lei <ming.lei@redhat.com>
   9 *
  10 * (part of code stolen from loop.c)
  11 */
  12#include <linux/module.h>
  13#include <linux/moduleparam.h>
  14#include <linux/sched.h>
  15#include <linux/fs.h>
  16#include <linux/pagemap.h>
  17#include <linux/file.h>
  18#include <linux/stat.h>
  19#include <linux/errno.h>
  20#include <linux/major.h>
  21#include <linux/wait.h>
  22#include <linux/blkdev.h>
  23#include <linux/init.h>
  24#include <linux/swap.h>
  25#include <linux/slab.h>
  26#include <linux/compat.h>
  27#include <linux/mutex.h>
  28#include <linux/writeback.h>
  29#include <linux/completion.h>
  30#include <linux/highmem.h>
  31#include <linux/sysfs.h>
  32#include <linux/miscdevice.h>
  33#include <linux/falloc.h>
  34#include <linux/uio.h>
  35#include <linux/ioprio.h>
  36#include <linux/sched/mm.h>
  37#include <linux/uaccess.h>
  38#include <linux/cdev.h>
  39#include <linux/io_uring/cmd.h>
  40#include <linux/blk-mq.h>
  41#include <linux/delay.h>
  42#include <linux/mm.h>
  43#include <asm/page.h>
  44#include <linux/task_work.h>
  45#include <linux/namei.h>
  46#include <linux/kref.h>
  47#include <uapi/linux/ublk_cmd.h>
  48
  49#define UBLK_MINORS		(1U << MINORBITS)
  50
  51#define UBLK_INVALID_BUF_IDX 	((u16)-1)
  52
  53/* private ioctl command mirror */
  54#define UBLK_CMD_DEL_DEV_ASYNC	_IOC_NR(UBLK_U_CMD_DEL_DEV_ASYNC)
  55#define UBLK_CMD_UPDATE_SIZE	_IOC_NR(UBLK_U_CMD_UPDATE_SIZE)
  56#define UBLK_CMD_QUIESCE_DEV	_IOC_NR(UBLK_U_CMD_QUIESCE_DEV)
  57
  58#define UBLK_IO_REGISTER_IO_BUF		_IOC_NR(UBLK_U_IO_REGISTER_IO_BUF)
  59#define UBLK_IO_UNREGISTER_IO_BUF	_IOC_NR(UBLK_U_IO_UNREGISTER_IO_BUF)
  60
  61/* All UBLK_F_* have to be included into UBLK_F_ALL */
  62#define UBLK_F_ALL (UBLK_F_SUPPORT_ZERO_COPY \
  63		| UBLK_F_URING_CMD_COMP_IN_TASK \
  64		| UBLK_F_NEED_GET_DATA \
  65		| UBLK_F_USER_RECOVERY \
  66		| UBLK_F_USER_RECOVERY_REISSUE \
  67		| UBLK_F_UNPRIVILEGED_DEV \
  68		| UBLK_F_CMD_IOCTL_ENCODE \
  69		| UBLK_F_USER_COPY \
  70		| UBLK_F_ZONED \
  71		| UBLK_F_USER_RECOVERY_FAIL_IO \
  72		| UBLK_F_UPDATE_SIZE \
  73		| UBLK_F_AUTO_BUF_REG \
  74		| UBLK_F_QUIESCE \
  75		| UBLK_F_PER_IO_DAEMON \
  76		| UBLK_F_BUF_REG_OFF_DAEMON)
  77
  78#define UBLK_F_ALL_RECOVERY_FLAGS (UBLK_F_USER_RECOVERY \
  79		| UBLK_F_USER_RECOVERY_REISSUE \
  80		| UBLK_F_USER_RECOVERY_FAIL_IO)
  81
  82/* All UBLK_PARAM_TYPE_* should be included here */
  83#define UBLK_PARAM_TYPE_ALL                                \
  84	(UBLK_PARAM_TYPE_BASIC | UBLK_PARAM_TYPE_DISCARD | \
  85	 UBLK_PARAM_TYPE_DEVT | UBLK_PARAM_TYPE_ZONED |    \
  86	 UBLK_PARAM_TYPE_DMA_ALIGN | UBLK_PARAM_TYPE_SEGMENT)
  87
  88struct ublk_uring_cmd_pdu {
  89	/*
  90	 * Store requests in same batch temporarily for queuing them to
  91	 * daemon context.
  92	 *
  93	 * It should have been stored to request payload, but we do want
  94	 * to avoid extra pre-allocation, and uring_cmd payload is always
  95	 * free for us
  96	 */
  97	union {
  98		struct request *req;
  99		struct request *req_list;
 100	};
 101
 102	/*
 103	 * The following two are valid in this cmd whole lifetime, and
 104	 * setup in ublk uring_cmd handler
 105	 */
 106	struct ublk_queue *ubq;
 107
 108	u16 tag;
 109};
 110
 111/*
 112 * io command is active: sqe cmd is received, and its cqe isn't done
 113 *
 114 * If the flag is set, the io command is owned by ublk driver, and waited
 115 * for incoming blk-mq request from the ublk block device.
 116 *
 117 * If the flag is cleared, the io command will be completed, and owned by
 118 * ublk server.
 119 */
 120#define UBLK_IO_FLAG_ACTIVE	0x01
 121
 122/*
 123 * IO command is completed via cqe, and it is being handled by ublksrv, and
 124 * not committed yet
 125 *
 126 * Basically exclusively with UBLK_IO_FLAG_ACTIVE, so can be served for
 127 * cross verification
 128 */
 129#define UBLK_IO_FLAG_OWNED_BY_SRV 0x02
 130
 131/*
 132 * UBLK_IO_FLAG_NEED_GET_DATA is set because IO command requires
 133 * get data buffer address from ublksrv.
 134 *
 135 * Then, bio data could be copied into this data buffer for a WRITE request
 136 * after the IO command is issued again and UBLK_IO_FLAG_NEED_GET_DATA is unset.
 137 */
 138#define UBLK_IO_FLAG_NEED_GET_DATA 0x08
 139
 140/*
 141 * request buffer is registered automatically, so we have to unregister it
 142 * before completing this request.
 143 *
 144 * io_uring will unregister buffer automatically for us during exiting.
 145 */
 146#define UBLK_IO_FLAG_AUTO_BUF_REG 	0x10
 147
 148/* atomic RW with ubq->cancel_lock */
 149#define UBLK_IO_FLAG_CANCELED	0x80000000
 150
 151/*
 152 * Initialize refcount to a large number to include any registered buffers.
 153 * UBLK_IO_COMMIT_AND_FETCH_REQ will release these references minus those for
 154 * any buffers registered on the io daemon task.
 155 */
 156#define UBLK_REFCOUNT_INIT (REFCOUNT_MAX / 2)
 157
 158union ublk_io_buf {
 159	__u64	addr;
 160	struct ublk_auto_buf_reg auto_reg;
 161};
 162
 163struct ublk_io {
 164	union ublk_io_buf buf;
 165	unsigned int flags;
 166	int res;
 167
 168	union {
 169		/* valid if UBLK_IO_FLAG_ACTIVE is set */
 170		struct io_uring_cmd *cmd;
 171		/* valid if UBLK_IO_FLAG_OWNED_BY_SRV is set */
 172		struct request *req;
 173	};
 174
 175	struct task_struct *task;
 176
 177	/*
 178	 * The number of uses of this I/O by the ublk server
 179	 * if user copy or zero copy are enabled:
 180	 * - UBLK_REFCOUNT_INIT from dispatch to the server
 181	 *   until UBLK_IO_COMMIT_AND_FETCH_REQ
 182	 * - 1 for each inflight ublk_ch_{read,write}_iter() call
 183	 * - 1 for each io_uring registered buffer not registered on task
 184	 * The I/O can only be completed once all references are dropped.
 185	 * User copy and buffer registration operations are only permitted
 186	 * if the reference count is nonzero.
 187	 */
 188	refcount_t ref;
 189	/* Count of buffers registered on task and not yet unregistered */
 190	unsigned task_registered_buffers;
 191
 192	void *buf_ctx_handle;
 193} ____cacheline_aligned_in_smp;
 194
 195struct ublk_queue {
 196	int q_id;
 197	int q_depth;
 198
 199	unsigned long flags;
 200	struct ublksrv_io_desc *io_cmd_buf;
 201
 202	bool force_abort;
 203	bool canceling;
 204	bool fail_io; /* copy of dev->state == UBLK_S_DEV_FAIL_IO */
 205	spinlock_t		cancel_lock;
 206	struct ublk_device *dev;
 207	struct ublk_io ios[] __counted_by(q_depth);
 208};
 209
 210struct ublk_device {
 211	struct gendisk		*ub_disk;
 212
 213	struct ublksrv_ctrl_dev_info	dev_info;
 214
 215	struct blk_mq_tag_set	tag_set;
 216
 217	struct cdev		cdev;
 218	struct device		cdev_dev;
 219
 220#define UB_STATE_OPEN		0
 221#define UB_STATE_USED		1
 222#define UB_STATE_DELETED	2
 223	unsigned long		state;
 224	int			ub_number;
 225
 226	struct mutex		mutex;
 227
 228	spinlock_t		lock;
 229	struct mm_struct	*mm;
 230
 231	struct ublk_params	params;
 232
 233	struct completion	completion;
 234	u32			nr_io_ready;
 235	bool 			unprivileged_daemons;
 236	struct mutex cancel_mutex;
 237	bool canceling;
 238	pid_t 	ublksrv_tgid;
 239	struct delayed_work	exit_work;
 240	struct work_struct	partition_scan_work;
 241
 242	struct ublk_queue       *queues[];
 243};
 244
 245/* header of ublk_params */
 246struct ublk_params_header {
 247	__u32	len;
 248	__u32	types;
 249};
 250
 251static void ublk_io_release(void *priv);
 252static void ublk_stop_dev_unlocked(struct ublk_device *ub);
 253static void ublk_abort_queue(struct ublk_device *ub, struct ublk_queue *ubq);
 254static inline struct request *__ublk_check_and_get_req(struct ublk_device *ub,
 255		u16 q_id, u16 tag, struct ublk_io *io, size_t offset);
 256static inline unsigned int ublk_req_build_flags(struct request *req);
 257
 258static inline struct ublksrv_io_desc *
 259ublk_get_iod(const struct ublk_queue *ubq, unsigned tag)
 260{
 261	return &ubq->io_cmd_buf[tag];
 262}
 263
 264static inline bool ublk_dev_is_zoned(const struct ublk_device *ub)
 265{
 266	return ub->dev_info.flags & UBLK_F_ZONED;
 267}
 268
 269static inline bool ublk_queue_is_zoned(const struct ublk_queue *ubq)
 270{
 271	return ubq->flags & UBLK_F_ZONED;
 272}
 273
 274#ifdef CONFIG_BLK_DEV_ZONED
 275
 276struct ublk_zoned_report_desc {
 277	__u64 sector;
 278	__u32 operation;
 279	__u32 nr_zones;
 280};
 281
 282static DEFINE_XARRAY(ublk_zoned_report_descs);
 283
 284static int ublk_zoned_insert_report_desc(const struct request *req,
 285		struct ublk_zoned_report_desc *desc)
 286{
 287	return xa_insert(&ublk_zoned_report_descs, (unsigned long)req,
 288			    desc, GFP_KERNEL);
 289}
 290
 291static struct ublk_zoned_report_desc *ublk_zoned_erase_report_desc(
 292		const struct request *req)
 293{
 294	return xa_erase(&ublk_zoned_report_descs, (unsigned long)req);
 295}
 296
 297static struct ublk_zoned_report_desc *ublk_zoned_get_report_desc(
 298		const struct request *req)
 299{
 300	return xa_load(&ublk_zoned_report_descs, (unsigned long)req);
 301}
 302
 303static int ublk_get_nr_zones(const struct ublk_device *ub)
 304{
 305	const struct ublk_param_basic *p = &ub->params.basic;
 306
 307	/* Zone size is a power of 2 */
 308	return p->dev_sectors >> ilog2(p->chunk_sectors);
 309}
 310
 311static int ublk_revalidate_disk_zones(struct ublk_device *ub)
 312{
 313	return blk_revalidate_disk_zones(ub->ub_disk);
 314}
 315
 316static int ublk_dev_param_zoned_validate(const struct ublk_device *ub)
 317{
 318	const struct ublk_param_zoned *p = &ub->params.zoned;
 319	int nr_zones;
 320
 321	if (!ublk_dev_is_zoned(ub))
 322		return -EINVAL;
 323
 324	if (!p->max_zone_append_sectors)
 325		return -EINVAL;
 326
 327	nr_zones = ublk_get_nr_zones(ub);
 328
 329	if (p->max_active_zones > nr_zones)
 330		return -EINVAL;
 331
 332	if (p->max_open_zones > nr_zones)
 333		return -EINVAL;
 334
 335	return 0;
 336}
 337
 338static void ublk_dev_param_zoned_apply(struct ublk_device *ub)
 339{
 340	ub->ub_disk->nr_zones = ublk_get_nr_zones(ub);
 341}
 342
 343/* Based on virtblk_alloc_report_buffer */
 344static void *ublk_alloc_report_buffer(struct ublk_device *ublk,
 345				      unsigned int nr_zones, size_t *buflen)
 346{
 347	struct request_queue *q = ublk->ub_disk->queue;
 348	size_t bufsize;
 349	void *buf;
 350
 351	nr_zones = min_t(unsigned int, nr_zones,
 352			 ublk->ub_disk->nr_zones);
 353
 354	bufsize = nr_zones * sizeof(struct blk_zone);
 355	bufsize =
 356		min_t(size_t, bufsize, queue_max_hw_sectors(q) << SECTOR_SHIFT);
 357
 358	while (bufsize >= sizeof(struct blk_zone)) {
 359		buf = kvmalloc(bufsize, GFP_KERNEL | __GFP_NORETRY);
 360		if (buf) {
 361			*buflen = bufsize;
 362			return buf;
 363		}
 364		bufsize >>= 1;
 365	}
 366
 367	*buflen = 0;
 368	return NULL;
 369}
 370
 371static int ublk_report_zones(struct gendisk *disk, sector_t sector,
 372		      unsigned int nr_zones, struct blk_report_zones_args *args)
 373{
 374	struct ublk_device *ub = disk->private_data;
 375	unsigned int zone_size_sectors = disk->queue->limits.chunk_sectors;
 376	unsigned int first_zone = sector >> ilog2(zone_size_sectors);
 377	unsigned int done_zones = 0;
 378	unsigned int max_zones_per_request;
 379	int ret;
 380	struct blk_zone *buffer;
 381	size_t buffer_length;
 382
 383	nr_zones = min_t(unsigned int, ub->ub_disk->nr_zones - first_zone,
 384			 nr_zones);
 385
 386	buffer = ublk_alloc_report_buffer(ub, nr_zones, &buffer_length);
 387	if (!buffer)
 388		return -ENOMEM;
 389
 390	max_zones_per_request = buffer_length / sizeof(struct blk_zone);
 391
 392	while (done_zones < nr_zones) {
 393		unsigned int remaining_zones = nr_zones - done_zones;
 394		unsigned int zones_in_request =
 395			min_t(unsigned int, remaining_zones, max_zones_per_request);
 396		struct request *req;
 397		struct ublk_zoned_report_desc desc;
 398		blk_status_t status;
 399
 400		memset(buffer, 0, buffer_length);
 401
 402		req = blk_mq_alloc_request(disk->queue, REQ_OP_DRV_IN, 0);
 403		if (IS_ERR(req)) {
 404			ret = PTR_ERR(req);
 405			goto out;
 406		}
 407
 408		desc.operation = UBLK_IO_OP_REPORT_ZONES;
 409		desc.sector = sector;
 410		desc.nr_zones = zones_in_request;
 411		ret = ublk_zoned_insert_report_desc(req, &desc);
 412		if (ret)
 413			goto free_req;
 414
 415		ret = blk_rq_map_kern(req, buffer, buffer_length, GFP_KERNEL);
 416		if (ret)
 417			goto erase_desc;
 418
 419		status = blk_execute_rq(req, 0);
 420		ret = blk_status_to_errno(status);
 421erase_desc:
 422		ublk_zoned_erase_report_desc(req);
 423free_req:
 424		blk_mq_free_request(req);
 425		if (ret)
 426			goto out;
 427
 428		for (unsigned int i = 0; i < zones_in_request; i++) {
 429			struct blk_zone *zone = buffer + i;
 430
 431			/* A zero length zone means no more zones in this response */
 432			if (!zone->len)
 433				break;
 434
 435			ret = disk_report_zone(disk, zone, i, args);
 436			if (ret)
 437				goto out;
 438
 439			done_zones++;
 440			sector += zone_size_sectors;
 441
 442		}
 443	}
 444
 445	ret = done_zones;
 446
 447out:
 448	kvfree(buffer);
 449	return ret;
 450}
 451
 452static blk_status_t ublk_setup_iod_zoned(struct ublk_queue *ubq,
 453					 struct request *req)
 454{
 455	struct ublksrv_io_desc *iod = ublk_get_iod(ubq, req->tag);
 456	struct ublk_io *io = &ubq->ios[req->tag];
 457	struct ublk_zoned_report_desc *desc;
 458	u32 ublk_op;
 459
 460	switch (req_op(req)) {
 461	case REQ_OP_ZONE_OPEN:
 462		ublk_op = UBLK_IO_OP_ZONE_OPEN;
 463		break;
 464	case REQ_OP_ZONE_CLOSE:
 465		ublk_op = UBLK_IO_OP_ZONE_CLOSE;
 466		break;
 467	case REQ_OP_ZONE_FINISH:
 468		ublk_op = UBLK_IO_OP_ZONE_FINISH;
 469		break;
 470	case REQ_OP_ZONE_RESET:
 471		ublk_op = UBLK_IO_OP_ZONE_RESET;
 472		break;
 473	case REQ_OP_ZONE_APPEND:
 474		ublk_op = UBLK_IO_OP_ZONE_APPEND;
 475		break;
 476	case REQ_OP_ZONE_RESET_ALL:
 477		ublk_op = UBLK_IO_OP_ZONE_RESET_ALL;
 478		break;
 479	case REQ_OP_DRV_IN:
 480		desc = ublk_zoned_get_report_desc(req);
 481		if (!desc)
 482			return BLK_STS_IOERR;
 483		ublk_op = desc->operation;
 484		switch (ublk_op) {
 485		case UBLK_IO_OP_REPORT_ZONES:
 486			iod->op_flags = ublk_op | ublk_req_build_flags(req);
 487			iod->nr_zones = desc->nr_zones;
 488			iod->start_sector = desc->sector;
 489			return BLK_STS_OK;
 490		default:
 491			return BLK_STS_IOERR;
 492		}
 493	case REQ_OP_DRV_OUT:
 494		/* We do not support drv_out */
 495		return BLK_STS_NOTSUPP;
 496	default:
 497		return BLK_STS_IOERR;
 498	}
 499
 500	iod->op_flags = ublk_op | ublk_req_build_flags(req);
 501	iod->nr_sectors = blk_rq_sectors(req);
 502	iod->start_sector = blk_rq_pos(req);
 503	iod->addr = io->buf.addr;
 504
 505	return BLK_STS_OK;
 506}
 507
 508#else
 509
 510#define ublk_report_zones (NULL)
 511
 512static int ublk_dev_param_zoned_validate(const struct ublk_device *ub)
 513{
 514	return -EOPNOTSUPP;
 515}
 516
 517static void ublk_dev_param_zoned_apply(struct ublk_device *ub)
 518{
 519}
 520
 521static int ublk_revalidate_disk_zones(struct ublk_device *ub)
 522{
 523	return 0;
 524}
 525
 526static blk_status_t ublk_setup_iod_zoned(struct ublk_queue *ubq,
 527					 struct request *req)
 528{
 529	return BLK_STS_NOTSUPP;
 530}
 531
 532#endif
 533
 534static inline void __ublk_complete_rq(struct request *req, struct ublk_io *io,
 535				      bool need_map);
 536
 537static dev_t ublk_chr_devt;
 538static const struct class ublk_chr_class = {
 539	.name = "ublk-char",
 540};
 541
 542static DEFINE_IDR(ublk_index_idr);
 543static DEFINE_SPINLOCK(ublk_idr_lock);
 544static wait_queue_head_t ublk_idr_wq;	/* wait until one idr is freed */
 545
 546static DEFINE_MUTEX(ublk_ctl_mutex);
 547
 548
 549#define UBLK_MAX_UBLKS UBLK_MINORS
 550
 551/*
 552 * Max unprivileged ublk devices allowed to add
 553 *
 554 * It can be extended to one per-user limit in future or even controlled
 555 * by cgroup.
 556 */
 557static unsigned int unprivileged_ublks_max = 64;
 558static unsigned int unprivileged_ublks_added; /* protected by ublk_ctl_mutex */
 559
 560static struct miscdevice ublk_misc;
 561
 562static inline unsigned ublk_pos_to_hwq(loff_t pos)
 563{
 564	return ((pos - UBLKSRV_IO_BUF_OFFSET) >> UBLK_QID_OFF) &
 565		UBLK_QID_BITS_MASK;
 566}
 567
 568static inline unsigned ublk_pos_to_buf_off(loff_t pos)
 569{
 570	return (pos - UBLKSRV_IO_BUF_OFFSET) & UBLK_IO_BUF_BITS_MASK;
 571}
 572
 573static inline unsigned ublk_pos_to_tag(loff_t pos)
 574{
 575	return ((pos - UBLKSRV_IO_BUF_OFFSET) >> UBLK_TAG_OFF) &
 576		UBLK_TAG_BITS_MASK;
 577}
 578
 579static void ublk_dev_param_basic_apply(struct ublk_device *ub)
 580{
 581	const struct ublk_param_basic *p = &ub->params.basic;
 582
 583	if (p->attrs & UBLK_ATTR_READ_ONLY)
 584		set_disk_ro(ub->ub_disk, true);
 585
 586	set_capacity(ub->ub_disk, p->dev_sectors);
 587}
 588
 589static int ublk_validate_params(const struct ublk_device *ub)
 590{
 591	/* basic param is the only one which must be set */
 592	if (ub->params.types & UBLK_PARAM_TYPE_BASIC) {
 593		const struct ublk_param_basic *p = &ub->params.basic;
 594
 595		if (p->logical_bs_shift > PAGE_SHIFT || p->logical_bs_shift < 9)
 596			return -EINVAL;
 597
 598		if (p->logical_bs_shift > p->physical_bs_shift)
 599			return -EINVAL;
 600
 601		if (p->max_sectors > (ub->dev_info.max_io_buf_bytes >> 9))
 602			return -EINVAL;
 603
 604		if (ublk_dev_is_zoned(ub) && !p->chunk_sectors)
 605			return -EINVAL;
 606	} else
 607		return -EINVAL;
 608
 609	if (ub->params.types & UBLK_PARAM_TYPE_DISCARD) {
 610		const struct ublk_param_discard *p = &ub->params.discard;
 611
 612		/* So far, only support single segment discard */
 613		if (p->max_discard_sectors && p->max_discard_segments != 1)
 614			return -EINVAL;
 615
 616		if (!p->discard_granularity)
 617			return -EINVAL;
 618	}
 619
 620	/* dev_t is read-only */
 621	if (ub->params.types & UBLK_PARAM_TYPE_DEVT)
 622		return -EINVAL;
 623
 624	if (ub->params.types & UBLK_PARAM_TYPE_ZONED)
 625		return ublk_dev_param_zoned_validate(ub);
 626	else if (ublk_dev_is_zoned(ub))
 627		return -EINVAL;
 628
 629	if (ub->params.types & UBLK_PARAM_TYPE_DMA_ALIGN) {
 630		const struct ublk_param_dma_align *p = &ub->params.dma;
 631
 632		if (p->alignment >= PAGE_SIZE)
 633			return -EINVAL;
 634
 635		if (!is_power_of_2(p->alignment + 1))
 636			return -EINVAL;
 637	}
 638
 639	if (ub->params.types & UBLK_PARAM_TYPE_SEGMENT) {
 640		const struct ublk_param_segment *p = &ub->params.seg;
 641
 642		if (!is_power_of_2(p->seg_boundary_mask + 1))
 643			return -EINVAL;
 644
 645		if (p->seg_boundary_mask + 1 < UBLK_MIN_SEGMENT_SIZE)
 646			return -EINVAL;
 647		if (p->max_segment_size < UBLK_MIN_SEGMENT_SIZE)
 648			return -EINVAL;
 649	}
 650
 651	return 0;
 652}
 653
 654static void ublk_apply_params(struct ublk_device *ub)
 655{
 656	ublk_dev_param_basic_apply(ub);
 657
 658	if (ub->params.types & UBLK_PARAM_TYPE_ZONED)
 659		ublk_dev_param_zoned_apply(ub);
 660}
 661
 662static inline bool ublk_support_zero_copy(const struct ublk_queue *ubq)
 663{
 664	return ubq->flags & UBLK_F_SUPPORT_ZERO_COPY;
 665}
 666
 667static inline bool ublk_dev_support_zero_copy(const struct ublk_device *ub)
 668{
 669	return ub->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY;
 670}
 671
 672static inline bool ublk_support_auto_buf_reg(const struct ublk_queue *ubq)
 673{
 674	return ubq->flags & UBLK_F_AUTO_BUF_REG;
 675}
 676
 677static inline bool ublk_dev_support_auto_buf_reg(const struct ublk_device *ub)
 678{
 679	return ub->dev_info.flags & UBLK_F_AUTO_BUF_REG;
 680}
 681
 682static inline bool ublk_support_user_copy(const struct ublk_queue *ubq)
 683{
 684	return ubq->flags & UBLK_F_USER_COPY;
 685}
 686
 687static inline bool ublk_dev_support_user_copy(const struct ublk_device *ub)
 688{
 689	return ub->dev_info.flags & UBLK_F_USER_COPY;
 690}
 691
 692static inline bool ublk_need_map_io(const struct ublk_queue *ubq)
 693{
 694	return !ublk_support_user_copy(ubq) && !ublk_support_zero_copy(ubq) &&
 695		!ublk_support_auto_buf_reg(ubq);
 696}
 697
 698static inline bool ublk_dev_need_map_io(const struct ublk_device *ub)
 699{
 700	return !ublk_dev_support_user_copy(ub) &&
 701	       !ublk_dev_support_zero_copy(ub) &&
 702	       !ublk_dev_support_auto_buf_reg(ub);
 703}
 704
 705static inline bool ublk_need_req_ref(const struct ublk_queue *ubq)
 706{
 707	/*
 708	 * read()/write() is involved in user copy, so request reference
 709	 * has to be grabbed
 710	 *
 711	 * for zero copy, request buffer need to be registered to io_uring
 712	 * buffer table, so reference is needed
 713	 *
 714	 * For auto buffer register, ublk server still may issue
 715	 * UBLK_IO_COMMIT_AND_FETCH_REQ before one registered buffer is used up,
 716	 * so reference is required too.
 717	 */
 718	return ublk_support_user_copy(ubq) || ublk_support_zero_copy(ubq) ||
 719		ublk_support_auto_buf_reg(ubq);
 720}
 721
 722static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub)
 723{
 724	return ublk_dev_support_user_copy(ub) ||
 725	       ublk_dev_support_zero_copy(ub) ||
 726	       ublk_dev_support_auto_buf_reg(ub);
 727}
 728
 729static inline void ublk_init_req_ref(const struct ublk_queue *ubq,
 730		struct ublk_io *io)
 731{
 732	if (ublk_need_req_ref(ubq))
 733		refcount_set(&io->ref, UBLK_REFCOUNT_INIT);
 734}
 735
 736static inline bool ublk_get_req_ref(struct ublk_io *io)
 737{
 738	return refcount_inc_not_zero(&io->ref);
 739}
 740
 741static inline void ublk_put_req_ref(struct ublk_io *io, struct request *req)
 742{
 743	if (!refcount_dec_and_test(&io->ref))
 744		return;
 745
 746	/* ublk_need_map_io() and ublk_need_req_ref() are mutually exclusive */
 747	__ublk_complete_rq(req, io, false);
 748}
 749
 750static inline bool ublk_sub_req_ref(struct ublk_io *io)
 751{
 752	unsigned sub_refs = UBLK_REFCOUNT_INIT - io->task_registered_buffers;
 753
 754	io->task_registered_buffers = 0;
 755	return refcount_sub_and_test(sub_refs, &io->ref);
 756}
 757
 758static inline bool ublk_need_get_data(const struct ublk_queue *ubq)
 759{
 760	return ubq->flags & UBLK_F_NEED_GET_DATA;
 761}
 762
 763static inline bool ublk_dev_need_get_data(const struct ublk_device *ub)
 764{
 765	return ub->dev_info.flags & UBLK_F_NEED_GET_DATA;
 766}
 767
 768/* Called in slow path only, keep it noinline for trace purpose */
 769static noinline struct ublk_device *ublk_get_device(struct ublk_device *ub)
 770{
 771	if (kobject_get_unless_zero(&ub->cdev_dev.kobj))
 772		return ub;
 773	return NULL;
 774}
 775
 776/* Called in slow path only, keep it noinline for trace purpose */
 777static noinline void ublk_put_device(struct ublk_device *ub)
 778{
 779	put_device(&ub->cdev_dev);
 780}
 781
 782static inline struct ublk_queue *ublk_get_queue(struct ublk_device *dev,
 783		int qid)
 784{
 785	return dev->queues[qid];
 786}
 787
 788static inline bool ublk_rq_has_data(const struct request *rq)
 789{
 790	return bio_has_data(rq->bio);
 791}
 792
 793static inline struct ublksrv_io_desc *
 794ublk_queue_cmd_buf(struct ublk_device *ub, int q_id)
 795{
 796	return ublk_get_queue(ub, q_id)->io_cmd_buf;
 797}
 798
 799static inline int __ublk_queue_cmd_buf_size(int depth)
 800{
 801	return round_up(depth * sizeof(struct ublksrv_io_desc), PAGE_SIZE);
 802}
 803
 804static inline int ublk_queue_cmd_buf_size(struct ublk_device *ub)
 805{
 806	return __ublk_queue_cmd_buf_size(ub->dev_info.queue_depth);
 807}
 808
 809static int ublk_max_cmd_buf_size(void)
 810{
 811	return __ublk_queue_cmd_buf_size(UBLK_MAX_QUEUE_DEPTH);
 812}
 813
 814/*
 815 * Should I/O outstanding to the ublk server when it exits be reissued?
 816 * If not, outstanding I/O will get errors.
 817 */
 818static inline bool ublk_nosrv_should_reissue_outstanding(struct ublk_device *ub)
 819{
 820	return (ub->dev_info.flags & UBLK_F_USER_RECOVERY) &&
 821	       (ub->dev_info.flags & UBLK_F_USER_RECOVERY_REISSUE);
 822}
 823
 824/*
 825 * Should I/O issued while there is no ublk server queue? If not, I/O
 826 * issued while there is no ublk server will get errors.
 827 */
 828static inline bool ublk_nosrv_dev_should_queue_io(struct ublk_device *ub)
 829{
 830	return (ub->dev_info.flags & UBLK_F_USER_RECOVERY) &&
 831	       !(ub->dev_info.flags & UBLK_F_USER_RECOVERY_FAIL_IO);
 832}
 833
 834/*
 835 * Same as ublk_nosrv_dev_should_queue_io, but uses a queue-local copy
 836 * of the device flags for smaller cache footprint - better for fast
 837 * paths.
 838 */
 839static inline bool ublk_nosrv_should_queue_io(struct ublk_queue *ubq)
 840{
 841	return (ubq->flags & UBLK_F_USER_RECOVERY) &&
 842	       !(ubq->flags & UBLK_F_USER_RECOVERY_FAIL_IO);
 843}
 844
 845/*
 846 * Should ublk devices be stopped (i.e. no recovery possible) when the
 847 * ublk server exits? If not, devices can be used again by a future
 848 * incarnation of a ublk server via the start_recovery/end_recovery
 849 * commands.
 850 */
 851static inline bool ublk_nosrv_should_stop_dev(struct ublk_device *ub)
 852{
 853	return !(ub->dev_info.flags & UBLK_F_USER_RECOVERY);
 854}
 855
 856static inline bool ublk_dev_in_recoverable_state(struct ublk_device *ub)
 857{
 858	return ub->dev_info.state == UBLK_S_DEV_QUIESCED ||
 859	       ub->dev_info.state == UBLK_S_DEV_FAIL_IO;
 860}
 861
 862static void ublk_free_disk(struct gendisk *disk)
 863{
 864	struct ublk_device *ub = disk->private_data;
 865
 866	clear_bit(UB_STATE_USED, &ub->state);
 867	ublk_put_device(ub);
 868}
 869
 870static void ublk_store_owner_uid_gid(unsigned int *owner_uid,
 871		unsigned int *owner_gid)
 872{
 873	kuid_t uid;
 874	kgid_t gid;
 875
 876	current_uid_gid(&uid, &gid);
 877
 878	*owner_uid = from_kuid(&init_user_ns, uid);
 879	*owner_gid = from_kgid(&init_user_ns, gid);
 880}
 881
 882static int ublk_open(struct gendisk *disk, blk_mode_t mode)
 883{
 884	struct ublk_device *ub = disk->private_data;
 885
 886	if (capable(CAP_SYS_ADMIN))
 887		return 0;
 888
 889	/*
 890	 * If it is one unprivileged device, only owner can open
 891	 * the disk. Otherwise it could be one trap made by one
 892	 * evil user who grants this disk's privileges to other
 893	 * users deliberately.
 894	 *
 895	 * This way is reasonable too given anyone can create
 896	 * unprivileged device, and no need other's grant.
 897	 */
 898	if (ub->dev_info.flags & UBLK_F_UNPRIVILEGED_DEV) {
 899		unsigned int curr_uid, curr_gid;
 900
 901		ublk_store_owner_uid_gid(&curr_uid, &curr_gid);
 902
 903		if (curr_uid != ub->dev_info.owner_uid || curr_gid !=
 904				ub->dev_info.owner_gid)
 905			return -EPERM;
 906	}
 907
 908	return 0;
 909}
 910
 911static const struct block_device_operations ub_fops = {
 912	.owner =	THIS_MODULE,
 913	.open =		ublk_open,
 914	.free_disk =	ublk_free_disk,
 915	.report_zones =	ublk_report_zones,
 916};
 917
 918/*
 919 * Copy data between request pages and io_iter, and 'offset'
 920 * is the start point of linear offset of request.
 921 */
 922static size_t ublk_copy_user_pages(const struct request *req,
 923		unsigned offset, struct iov_iter *uiter, int dir)
 924{
 925	struct req_iterator iter;
 926	struct bio_vec bv;
 927	size_t done = 0;
 928
 929	rq_for_each_segment(bv, req, iter) {
 930		unsigned len;
 931		void *bv_buf;
 932		size_t copied;
 933
 934		if (offset >= bv.bv_len) {
 935			offset -= bv.bv_len;
 936			continue;
 937		}
 938
 939		len = bv.bv_len - offset;
 940		bv_buf = kmap_local_page(bv.bv_page) + bv.bv_offset + offset;
 941		if (dir == ITER_DEST)
 942			copied = copy_to_iter(bv_buf, len, uiter);
 943		else
 944			copied = copy_from_iter(bv_buf, len, uiter);
 945
 946		kunmap_local(bv_buf);
 947
 948		done += copied;
 949		if (copied < len)
 950			break;
 951
 952		offset = 0;
 953	}
 954	return done;
 955}
 956
 957static inline bool ublk_need_map_req(const struct request *req)
 958{
 959	return ublk_rq_has_data(req) && req_op(req) == REQ_OP_WRITE;
 960}
 961
 962static inline bool ublk_need_unmap_req(const struct request *req)
 963{
 964	return ublk_rq_has_data(req) &&
 965	       (req_op(req) == REQ_OP_READ || req_op(req) == REQ_OP_DRV_IN);
 966}
 967
 968static unsigned int ublk_map_io(const struct ublk_queue *ubq,
 969				const struct request *req,
 970				const struct ublk_io *io)
 971{
 972	const unsigned int rq_bytes = blk_rq_bytes(req);
 973
 974	if (!ublk_need_map_io(ubq))
 975		return rq_bytes;
 976
 977	/*
 978	 * no zero copy, we delay copy WRITE request data into ublksrv
 979	 * context and the big benefit is that pinning pages in current
 980	 * context is pretty fast, see ublk_pin_user_pages
 981	 */
 982	if (ublk_need_map_req(req)) {
 983		struct iov_iter iter;
 984		const int dir = ITER_DEST;
 985
 986		import_ubuf(dir, u64_to_user_ptr(io->buf.addr), rq_bytes, &iter);
 987		return ublk_copy_user_pages(req, 0, &iter, dir);
 988	}
 989	return rq_bytes;
 990}
 991
 992static unsigned int ublk_unmap_io(bool need_map,
 993		const struct request *req,
 994		const struct ublk_io *io)
 995{
 996	const unsigned int rq_bytes = blk_rq_bytes(req);
 997
 998	if (!need_map)
 999		return rq_bytes;
1000
1001	if (ublk_need_unmap_req(req)) {
1002		struct iov_iter iter;
1003		const int dir = ITER_SOURCE;
1004
1005		WARN_ON_ONCE(io->res > rq_bytes);
1006
1007		import_ubuf(dir, u64_to_user_ptr(io->buf.addr), io->res, &iter);
1008		return ublk_copy_user_pages(req, 0, &iter, dir);
1009	}
1010	return rq_bytes;
1011}
1012
1013static inline unsigned int ublk_req_build_flags(struct request *req)
1014{
1015	unsigned flags = 0;
1016
1017	if (req->cmd_flags & REQ_FAILFAST_DEV)
1018		flags |= UBLK_IO_F_FAILFAST_DEV;
1019
1020	if (req->cmd_flags & REQ_FAILFAST_TRANSPORT)
1021		flags |= UBLK_IO_F_FAILFAST_TRANSPORT;
1022
1023	if (req->cmd_flags & REQ_FAILFAST_DRIVER)
1024		flags |= UBLK_IO_F_FAILFAST_DRIVER;
1025
1026	if (req->cmd_flags & REQ_META)
1027		flags |= UBLK_IO_F_META;
1028
1029	if (req->cmd_flags & REQ_FUA)
1030		flags |= UBLK_IO_F_FUA;
1031
1032	if (req->cmd_flags & REQ_NOUNMAP)
1033		flags |= UBLK_IO_F_NOUNMAP;
1034
1035	if (req->cmd_flags & REQ_SWAP)
1036		flags |= UBLK_IO_F_SWAP;
1037
1038	return flags;
1039}
1040
1041static blk_status_t ublk_setup_iod(struct ublk_queue *ubq, struct request *req)
1042{
1043	struct ublksrv_io_desc *iod = ublk_get_iod(ubq, req->tag);
1044	struct ublk_io *io = &ubq->ios[req->tag];
1045	u32 ublk_op;
1046
1047	switch (req_op(req)) {
1048	case REQ_OP_READ:
1049		ublk_op = UBLK_IO_OP_READ;
1050		break;
1051	case REQ_OP_WRITE:
1052		ublk_op = UBLK_IO_OP_WRITE;
1053		break;
1054	case REQ_OP_FLUSH:
1055		ublk_op = UBLK_IO_OP_FLUSH;
1056		break;
1057	case REQ_OP_DISCARD:
1058		ublk_op = UBLK_IO_OP_DISCARD;
1059		break;
1060	case REQ_OP_WRITE_ZEROES:
1061		ublk_op = UBLK_IO_OP_WRITE_ZEROES;
1062		break;
1063	default:
1064		if (ublk_queue_is_zoned(ubq))
1065			return ublk_setup_iod_zoned(ubq, req);
1066		return BLK_STS_IOERR;
1067	}
1068
1069	/* need to translate since kernel may change */
1070	iod->op_flags = ublk_op | ublk_req_build_flags(req);
1071	iod->nr_sectors = blk_rq_sectors(req);
1072	iod->start_sector = blk_rq_pos(req);
1073	iod->addr = io->buf.addr;
1074
1075	return BLK_STS_OK;
1076}
1077
1078static inline struct ublk_uring_cmd_pdu *ublk_get_uring_cmd_pdu(
1079		struct io_uring_cmd *ioucmd)
1080{
1081	return io_uring_cmd_to_pdu(ioucmd, struct ublk_uring_cmd_pdu);
1082}
1083
1084static void ublk_end_request(struct request *req, blk_status_t error)
1085{
1086	local_bh_disable();
1087	blk_mq_end_request(req, error);
1088	local_bh_enable();
1089}
1090
1091/* todo: handle partial completion */
1092static inline void __ublk_complete_rq(struct request *req, struct ublk_io *io,
1093				      bool need_map)
1094{
1095	unsigned int unmapped_bytes;
1096	blk_status_t res = BLK_STS_OK;
1097	bool requeue;
1098
1099	/* failed read IO if nothing is read */
1100	if (!io->res && req_op(req) == REQ_OP_READ)
1101		io->res = -EIO;
1102
1103	if (io->res < 0) {
1104		res = errno_to_blk_status(io->res);
1105		goto exit;
1106	}
1107
1108	/*
1109	 * FLUSH, DISCARD or WRITE_ZEROES usually won't return bytes returned, so end them
1110	 * directly.
1111	 *
1112	 * Both the two needn't unmap.
1113	 */
1114	if (req_op(req) != REQ_OP_READ && req_op(req) != REQ_OP_WRITE &&
1115	    req_op(req) != REQ_OP_DRV_IN)
1116		goto exit;
1117
1118	/* for READ request, writing data in iod->addr to rq buffers */
1119	unmapped_bytes = ublk_unmap_io(need_map, req, io);
1120
1121	/*
1122	 * Extremely impossible since we got data filled in just before
1123	 *
1124	 * Re-read simply for this unlikely case.
1125	 */
1126	if (unlikely(unmapped_bytes < io->res))
1127		io->res = unmapped_bytes;
1128
1129	/*
1130	 * Run bio->bi_end_io() with softirqs disabled. If the final fput
1131	 * happens off this path, then that will prevent ublk's blkdev_release()
1132	 * from being called on current's task work, see fput() implementation.
1133	 *
1134	 * Otherwise, ublk server may not provide forward progress in case of
1135	 * reading the partition table from bdev_open() with disk->open_mutex
1136	 * held, and causes dead lock as we could already be holding
1137	 * disk->open_mutex here.
1138	 *
1139	 * Preferably we would not be doing IO with a mutex held that is also
1140	 * used for release, but this work-around will suffice for now.
1141	 */
1142	local_bh_disable();
1143	requeue = blk_update_request(req, BLK_STS_OK, io->res);
1144	local_bh_enable();
1145	if (requeue)
1146		blk_mq_requeue_request(req, true);
1147	else if (likely(!blk_should_fake_timeout(req->q)))
1148		__blk_mq_end_request(req, BLK_STS_OK);
1149
1150	return;
1151exit:
1152	ublk_end_request(req, res);
1153}
1154
1155static struct io_uring_cmd *__ublk_prep_compl_io_cmd(struct ublk_io *io,
1156						     struct request *req)
1157{
1158	/* read cmd first because req will overwrite it */
1159	struct io_uring_cmd *cmd = io->cmd;
1160
1161	/* mark this cmd owned by ublksrv */
1162	io->flags |= UBLK_IO_FLAG_OWNED_BY_SRV;
1163
1164	/*
1165	 * clear ACTIVE since we are done with this sqe/cmd slot
1166	 * We can only accept io cmd in case of being not active.
1167	 */
1168	io->flags &= ~UBLK_IO_FLAG_ACTIVE;
1169
1170	io->req = req;
1171	return cmd;
1172}
1173
1174static void ublk_complete_io_cmd(struct ublk_io *io, struct request *req,
1175				 int res, unsigned issue_flags)
1176{
1177	struct io_uring_cmd *cmd = __ublk_prep_compl_io_cmd(io, req);
1178
1179	/* tell ublksrv one io request is coming */
1180	io_uring_cmd_done(cmd, res, issue_flags);
1181}
1182
1183#define UBLK_REQUEUE_DELAY_MS	3
1184
1185static inline void __ublk_abort_rq(struct ublk_queue *ubq,
1186		struct request *rq)
1187{
1188	/* We cannot process this rq so just requeue it. */
1189	if (ublk_nosrv_dev_should_queue_io(ubq->dev))
1190		blk_mq_requeue_request(rq, false);
1191	else
1192		ublk_end_request(rq, BLK_STS_IOERR);
1193}
1194
1195static void
1196ublk_auto_buf_reg_fallback(const struct ublk_queue *ubq, unsigned tag)
1197{
1198	struct ublksrv_io_desc *iod = ublk_get_iod(ubq, tag);
1199
1200	iod->op_flags |= UBLK_IO_F_NEED_REG_BUF;
1201}
1202
1203enum auto_buf_reg_res {
1204	AUTO_BUF_REG_FAIL,
1205	AUTO_BUF_REG_FALLBACK,
1206	AUTO_BUF_REG_OK,
1207};
1208
1209static void ublk_prep_auto_buf_reg_io(const struct ublk_queue *ubq,
1210				      struct request *req, struct ublk_io *io,
1211				      struct io_uring_cmd *cmd,
1212				      enum auto_buf_reg_res res)
1213{
1214	if (res == AUTO_BUF_REG_OK) {
1215		io->task_registered_buffers = 1;
1216		io->buf_ctx_handle = io_uring_cmd_ctx_handle(cmd);
1217		io->flags |= UBLK_IO_FLAG_AUTO_BUF_REG;
1218	}
1219	ublk_init_req_ref(ubq, io);
1220	__ublk_prep_compl_io_cmd(io, req);
1221}
1222
1223static enum auto_buf_reg_res
1224__ublk_do_auto_buf_reg(const struct ublk_queue *ubq, struct request *req,
1225		       struct ublk_io *io, struct io_uring_cmd *cmd,
1226		       unsigned int issue_flags)
1227{
1228	int ret;
1229
1230	ret = io_buffer_register_bvec(cmd, req, ublk_io_release,
1231				      io->buf.auto_reg.index, issue_flags);
1232	if (ret) {
1233		if (io->buf.auto_reg.flags & UBLK_AUTO_BUF_REG_FALLBACK) {
1234			ublk_auto_buf_reg_fallback(ubq, req->tag);
1235			return AUTO_BUF_REG_FALLBACK;
1236		}
1237		ublk_end_request(req, BLK_STS_IOERR);
1238		return AUTO_BUF_REG_FAIL;
1239	}
1240
1241	return AUTO_BUF_REG_OK;
1242}
1243
1244static void ublk_do_auto_buf_reg(const struct ublk_queue *ubq, struct request *req,
1245				 struct ublk_io *io, struct io_uring_cmd *cmd,
1246				 unsigned int issue_flags)
1247{
1248	enum auto_buf_reg_res res = __ublk_do_auto_buf_reg(ubq, req, io, cmd,
1249			issue_flags);
1250
1251	if (res != AUTO_BUF_REG_FAIL) {
1252		ublk_prep_auto_buf_reg_io(ubq, req, io, cmd, res);
1253		io_uring_cmd_done(cmd, UBLK_IO_RES_OK, issue_flags);
1254	}
1255}
1256
1257static bool ublk_start_io(const struct ublk_queue *ubq, struct request *req,
1258			  struct ublk_io *io)
1259{
1260	unsigned mapped_bytes = ublk_map_io(ubq, req, io);
1261
1262	/* partially mapped, update io descriptor */
1263	if (unlikely(mapped_bytes != blk_rq_bytes(req))) {
1264		/*
1265		 * Nothing mapped, retry until we succeed.
1266		 *
1267		 * We may never succeed in mapping any bytes here because
1268		 * of OOM. TODO: reserve one buffer with single page pinned
1269		 * for providing forward progress guarantee.
1270		 */
1271		if (unlikely(!mapped_bytes)) {
1272			blk_mq_requeue_request(req, false);
1273			blk_mq_delay_kick_requeue_list(req->q,
1274					UBLK_REQUEUE_DELAY_MS);
1275			return false;
1276		}
1277
1278		ublk_get_iod(ubq, req->tag)->nr_sectors =
1279			mapped_bytes >> 9;
1280	}
1281
1282	return true;
1283}
1284
1285static void ublk_dispatch_req(struct ublk_queue *ubq, struct request *req)
1286{
1287	unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS;
1288	int tag = req->tag;
1289	struct ublk_io *io = &ubq->ios[tag];
1290
1291	pr_devel("%s: complete: qid %d tag %d io_flags %x addr %llx\n",
1292			__func__, ubq->q_id, req->tag, io->flags,
1293			ublk_get_iod(ubq, req->tag)->addr);
1294
1295	/*
1296	 * Task is exiting if either:
1297	 *
1298	 * (1) current != io->task.
1299	 * io_uring_cmd_complete_in_task() tries to run task_work
1300	 * in a workqueue if cmd's task is PF_EXITING.
1301	 *
1302	 * (2) current->flags & PF_EXITING.
1303	 */
1304	if (unlikely(current != io->task || current->flags & PF_EXITING)) {
1305		__ublk_abort_rq(ubq, req);
1306		return;
1307	}
1308
1309	if (ublk_need_get_data(ubq) && ublk_need_map_req(req)) {
1310		/*
1311		 * We have not handled UBLK_IO_NEED_GET_DATA command yet,
1312		 * so immediately pass UBLK_IO_RES_NEED_GET_DATA to ublksrv
1313		 * and notify it.
1314		 */
1315		io->flags |= UBLK_IO_FLAG_NEED_GET_DATA;
1316		pr_devel("%s: need get data. qid %d tag %d io_flags %x\n",
1317				__func__, ubq->q_id, req->tag, io->flags);
1318		ublk_complete_io_cmd(io, req, UBLK_IO_RES_NEED_GET_DATA,
1319				     issue_flags);
1320		return;
1321	}
1322
1323	if (!ublk_start_io(ubq, req, io))
1324		return;
1325
1326	if (ublk_support_auto_buf_reg(ubq) && ublk_rq_has_data(req)) {
1327		ublk_do_auto_buf_reg(ubq, req, io, io->cmd, issue_flags);
1328	} else {
1329		ublk_init_req_ref(ubq, io);
1330		ublk_complete_io_cmd(io, req, UBLK_IO_RES_OK, issue_flags);
1331	}
1332}
1333
1334static void ublk_cmd_tw_cb(struct io_tw_req tw_req, io_tw_token_t tw)
1335{
1336	struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);
1337	struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);
1338	struct ublk_queue *ubq = pdu->ubq;
1339
1340	ublk_dispatch_req(ubq, pdu->req);
1341}
1342
1343static void ublk_queue_cmd(struct ublk_queue *ubq, struct request *rq)
1344{
1345	struct io_uring_cmd *cmd = ubq->ios[rq->tag].cmd;
1346	struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);
1347
1348	pdu->req = rq;
1349	io_uring_cmd_complete_in_task(cmd, ublk_cmd_tw_cb);
1350}
1351
1352static void ublk_cmd_list_tw_cb(struct io_tw_req tw_req, io_tw_token_t tw)
1353{
1354	struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);
1355	struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);
1356	struct request *rq = pdu->req_list;
1357	struct request *next;
1358
1359	do {
1360		next = rq->rq_next;
1361		rq->rq_next = NULL;
1362		ublk_dispatch_req(rq->mq_hctx->driver_data, rq);
1363		rq = next;
1364	} while (rq);
1365}
1366
1367static void ublk_queue_cmd_list(struct ublk_io *io, struct rq_list *l)
1368{
1369	struct io_uring_cmd *cmd = io->cmd;
1370	struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);
1371
1372	pdu->req_list = rq_list_peek(l);
1373	rq_list_init(l);
1374	io_uring_cmd_complete_in_task(cmd, ublk_cmd_list_tw_cb);
1375}
1376
1377static enum blk_eh_timer_return ublk_timeout(struct request *rq)
1378{
1379	struct ublk_queue *ubq = rq->mq_hctx->driver_data;
1380	pid_t tgid = ubq->dev->ublksrv_tgid;
1381	struct task_struct *p;
1382	struct pid *pid;
1383
1384	if (!(ubq->flags & UBLK_F_UNPRIVILEGED_DEV))
1385		return BLK_EH_RESET_TIMER;
1386
1387	if (unlikely(!tgid))
1388		return BLK_EH_RESET_TIMER;
1389
1390	rcu_read_lock();
1391	pid = find_vpid(tgid);
1392	p = pid_task(pid, PIDTYPE_PID);
1393	if (p)
1394		send_sig(SIGKILL, p, 0);
1395	rcu_read_unlock();
1396	return BLK_EH_DONE;
1397}
1398
1399static blk_status_t ublk_prep_req(struct ublk_queue *ubq, struct request *rq,
1400				  bool check_cancel)
1401{
1402	blk_status_t res;
1403
1404	if (unlikely(READ_ONCE(ubq->fail_io)))
1405		return BLK_STS_TARGET;
1406
1407	/* With recovery feature enabled, force_abort is set in
1408	 * ublk_stop_dev() before calling del_gendisk(). We have to
1409	 * abort all requeued and new rqs here to let del_gendisk()
1410	 * move on. Besides, we cannot not call io_uring_cmd_complete_in_task()
1411	 * to avoid UAF on io_uring ctx.
1412	 *
1413	 * Note: force_abort is guaranteed to be seen because it is set
1414	 * before request queue is unqiuesced.
1415	 */
1416	if (ublk_nosrv_should_queue_io(ubq) &&
1417	    unlikely(READ_ONCE(ubq->force_abort)))
1418		return BLK_STS_IOERR;
1419
1420	if (check_cancel && unlikely(ubq->canceling))
1421		return BLK_STS_IOERR;
1422
1423	/* fill iod to slot in io cmd buffer */
1424	res = ublk_setup_iod(ubq, rq);
1425	if (unlikely(res != BLK_STS_OK))
1426		return BLK_STS_IOERR;
1427
1428	blk_mq_start_request(rq);
1429	return BLK_STS_OK;
1430}
1431
1432static blk_status_t ublk_queue_rq(struct blk_mq_hw_ctx *hctx,
1433		const struct blk_mq_queue_data *bd)
1434{
1435	struct ublk_queue *ubq = hctx->driver_data;
1436	struct request *rq = bd->rq;
1437	blk_status_t res;
1438
1439	res = ublk_prep_req(ubq, rq, false);
1440	if (res != BLK_STS_OK)
1441		return res;
1442
1443	/*
1444	 * ->canceling has to be handled after ->force_abort and ->fail_io
1445	 * is dealt with, otherwise this request may not be failed in case
1446	 * of recovery, and cause hang when deleting disk
1447	 */
1448	if (unlikely(ubq->canceling)) {
1449		__ublk_abort_rq(ubq, rq);
1450		return BLK_STS_OK;
1451	}
1452
1453	ublk_queue_cmd(ubq, rq);
1454	return BLK_STS_OK;
1455}
1456
1457static inline bool ublk_belong_to_same_batch(const struct ublk_io *io,
1458					     const struct ublk_io *io2)
1459{
1460	return (io_uring_cmd_ctx_handle(io->cmd) ==
1461		io_uring_cmd_ctx_handle(io2->cmd)) &&
1462		(io->task == io2->task);
1463}
1464
1465static void ublk_queue_rqs(struct rq_list *rqlist)
1466{
1467	struct rq_list requeue_list = { };
1468	struct rq_list submit_list = { };
1469	struct ublk_io *io = NULL;
1470	struct request *req;
1471
1472	while ((req = rq_list_pop(rqlist))) {
1473		struct ublk_queue *this_q = req->mq_hctx->driver_data;
1474		struct ublk_io *this_io = &this_q->ios[req->tag];
1475
1476		if (ublk_prep_req(this_q, req, true) != BLK_STS_OK) {
1477			rq_list_add_tail(&requeue_list, req);
1478			continue;
1479		}
1480
1481		if (io && !ublk_belong_to_same_batch(io, this_io) &&
1482				!rq_list_empty(&submit_list))
1483			ublk_queue_cmd_list(io, &submit_list);
1484		io = this_io;
1485		rq_list_add_tail(&submit_list, req);
1486	}
1487
1488	if (!rq_list_empty(&submit_list))
1489		ublk_queue_cmd_list(io, &submit_list);
1490	*rqlist = requeue_list;
1491}
1492
1493static int ublk_init_hctx(struct blk_mq_hw_ctx *hctx, void *driver_data,
1494		unsigned int hctx_idx)
1495{
1496	struct ublk_device *ub = driver_data;
1497	struct ublk_queue *ubq = ublk_get_queue(ub, hctx->queue_num);
1498
1499	hctx->driver_data = ubq;
1500	return 0;
1501}
1502
1503static const struct blk_mq_ops ublk_mq_ops = {
1504	.queue_rq       = ublk_queue_rq,
1505	.queue_rqs      = ublk_queue_rqs,
1506	.init_hctx	= ublk_init_hctx,
1507	.timeout	= ublk_timeout,
1508};
1509
1510static void ublk_queue_reinit(struct ublk_device *ub, struct ublk_queue *ubq)
1511{
1512	int i;
1513
1514	for (i = 0; i < ubq->q_depth; i++) {
1515		struct ublk_io *io = &ubq->ios[i];
1516
1517		/*
1518		 * UBLK_IO_FLAG_CANCELED is kept for avoiding to touch
1519		 * io->cmd
1520		 */
1521		io->flags &= UBLK_IO_FLAG_CANCELED;
1522		io->cmd = NULL;
1523		io->buf.addr = 0;
1524
1525		/*
1526		 * old task is PF_EXITING, put it now
1527		 *
1528		 * It could be NULL in case of closing one quiesced
1529		 * device.
1530		 */
1531		if (io->task) {
1532			put_task_struct(io->task);
1533			io->task = NULL;
1534		}
1535
1536		WARN_ON_ONCE(refcount_read(&io->ref));
1537		WARN_ON_ONCE(io->task_registered_buffers);
1538	}
1539}
1540
1541static int ublk_ch_open(struct inode *inode, struct file *filp)
1542{
1543	struct ublk_device *ub = container_of(inode->i_cdev,
1544			struct ublk_device, cdev);
1545
1546	if (test_and_set_bit(UB_STATE_OPEN, &ub->state))
1547		return -EBUSY;
1548	filp->private_data = ub;
1549	ub->ublksrv_tgid = current->tgid;
1550	return 0;
1551}
1552
1553static void ublk_reset_ch_dev(struct ublk_device *ub)
1554{
1555	int i;
1556
1557	for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
1558		ublk_queue_reinit(ub, ublk_get_queue(ub, i));
1559
1560	/* set to NULL, otherwise new tasks cannot mmap io_cmd_buf */
1561	ub->mm = NULL;
1562	ub->nr_io_ready = 0;
1563	ub->unprivileged_daemons = false;
1564	ub->ublksrv_tgid = -1;
1565}
1566
1567static struct gendisk *ublk_get_disk(struct ublk_device *ub)
1568{
1569	struct gendisk *disk;
1570
1571	spin_lock(&ub->lock);
1572	disk = ub->ub_disk;
1573	if (disk)
1574		get_device(disk_to_dev(disk));
1575	spin_unlock(&ub->lock);
1576
1577	return disk;
1578}
1579
1580static void ublk_put_disk(struct gendisk *disk)
1581{
1582	if (disk)
1583		put_device(disk_to_dev(disk));
1584}
1585
1586static void ublk_partition_scan_work(struct work_struct *work)
1587{
1588	struct ublk_device *ub =
1589		container_of(work, struct ublk_device, partition_scan_work);
1590	/* Hold disk reference to prevent UAF during concurrent teardown */
1591	struct gendisk *disk = ublk_get_disk(ub);
1592
1593	if (!disk)
1594		return;
1595
1596	if (WARN_ON_ONCE(!test_and_clear_bit(GD_SUPPRESS_PART_SCAN,
1597					     &disk->state)))
1598		goto out;
1599
1600	mutex_lock(&disk->open_mutex);
1601	bdev_disk_changed(disk, false);
1602	mutex_unlock(&disk->open_mutex);
1603out:
1604	ublk_put_disk(disk);
1605}
1606
1607/*
1608 * Use this function to ensure that ->canceling is consistently set for
1609 * the device and all queues. Do not set these flags directly.
1610 *
1611 * Caller must ensure that:
1612 * - cancel_mutex is held. This ensures that there is no concurrent
1613 *   access to ub->canceling and no concurrent writes to ubq->canceling.
1614 * - there are no concurrent reads of ubq->canceling from the queue_rq
1615 *   path. This can be done by quiescing the queue, or through other
1616 *   means.
1617 */
1618static void ublk_set_canceling(struct ublk_device *ub, bool canceling)
1619	__must_hold(&ub->cancel_mutex)
1620{
1621	int i;
1622
1623	ub->canceling = canceling;
1624	for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
1625		ublk_get_queue(ub, i)->canceling = canceling;
1626}
1627
1628static bool ublk_check_and_reset_active_ref(struct ublk_device *ub)
1629{
1630	int i, j;
1631
1632	if (!ublk_dev_need_req_ref(ub))
1633		return false;
1634
1635	for (i = 0; i < ub->dev_info.nr_hw_queues; i++) {
1636		struct ublk_queue *ubq = ublk_get_queue(ub, i);
1637
1638		for (j = 0; j < ubq->q_depth; j++) {
1639			struct ublk_io *io = &ubq->ios[j];
1640			unsigned int refs = refcount_read(&io->ref) +
1641				io->task_registered_buffers;
1642
1643			/*
1644			 * UBLK_REFCOUNT_INIT or zero means no active
1645			 * reference
1646			 */
1647			if (refs != UBLK_REFCOUNT_INIT && refs != 0)
1648				return true;
1649
1650			/* reset to zero if the io hasn't active references */
1651			refcount_set(&io->ref, 0);
1652			io->task_registered_buffers = 0;
1653		}
1654	}
1655	return false;
1656}
1657
1658static void ublk_ch_release_work_fn(struct work_struct *work)
1659{
1660	struct ublk_device *ub =
1661		container_of(work, struct ublk_device, exit_work.work);
1662	struct gendisk *disk;
1663	int i;
1664
1665	/*
1666	 * For zero-copy and auto buffer register modes, I/O references
1667	 * might not be dropped naturally when the daemon is killed, but
1668	 * io_uring guarantees that registered bvec kernel buffers are
1669	 * unregistered finally when freeing io_uring context, then the
1670	 * active references are dropped.
1671	 *
1672	 * Wait until active references are dropped for avoiding use-after-free
1673	 *
1674	 * registered buffer may be unregistered in io_ring's release hander,
1675	 * so have to wait by scheduling work function for avoiding the two
1676	 * file release dependency.
1677	 */
1678	if (ublk_check_and_reset_active_ref(ub)) {
1679		schedule_delayed_work(&ub->exit_work, 1);
1680		return;
1681	}
1682
1683	/*
1684	 * disk isn't attached yet, either device isn't live, or it has
1685	 * been removed already, so we needn't to do anything
1686	 */
1687	disk = ublk_get_disk(ub);
1688	if (!disk)
1689		goto out;
1690
1691	/*
1692	 * All uring_cmd are done now, so abort any request outstanding to
1693	 * the ublk server
1694	 *
1695	 * This can be done in lockless way because ublk server has been
1696	 * gone
1697	 *
1698	 * More importantly, we have to provide forward progress guarantee
1699	 * without holding ub->mutex, otherwise control task grabbing
1700	 * ub->mutex triggers deadlock
1701	 *
1702	 * All requests may be inflight, so ->canceling may not be set, set
1703	 * it now.
1704	 */
1705	mutex_lock(&ub->cancel_mutex);
1706	ublk_set_canceling(ub, true);
1707	for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
1708		ublk_abort_queue(ub, ublk_get_queue(ub, i));
1709	mutex_unlock(&ub->cancel_mutex);
1710	blk_mq_kick_requeue_list(disk->queue);
1711
1712	/*
1713	 * All infligh requests have been completed or requeued and any new
1714	 * request will be failed or requeued via `->canceling` now, so it is
1715	 * fine to grab ub->mutex now.
1716	 */
1717	mutex_lock(&ub->mutex);
1718
1719	/* double check after grabbing lock */
1720	if (!ub->ub_disk)
1721		goto unlock;
1722
1723	/*
1724	 * Transition the device to the nosrv state. What exactly this
1725	 * means depends on the recovery flags
1726	 */
1727	if (ublk_nosrv_should_stop_dev(ub)) {
1728		/*
1729		 * Allow any pending/future I/O to pass through quickly
1730		 * with an error. This is needed because del_gendisk
1731		 * waits for all pending I/O to complete
1732		 */
1733		for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
1734			WRITE_ONCE(ublk_get_queue(ub, i)->force_abort, true);
1735
1736		ublk_stop_dev_unlocked(ub);
1737	} else {
1738		if (ublk_nosrv_dev_should_queue_io(ub)) {
1739			/* ->canceling is set and all requests are aborted */
1740			ub->dev_info.state = UBLK_S_DEV_QUIESCED;
1741		} else {
1742			ub->dev_info.state = UBLK_S_DEV_FAIL_IO;
1743			for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
1744				WRITE_ONCE(ublk_get_queue(ub, i)->fail_io, true);
1745		}
1746	}
1747unlock:
1748	mutex_unlock(&ub->mutex);
1749	ublk_put_disk(disk);
1750
1751	/* all uring_cmd has been done now, reset device & ubq */
1752	ublk_reset_ch_dev(ub);
1753out:
1754	clear_bit(UB_STATE_OPEN, &ub->state);
1755
1756	/* put the reference grabbed in ublk_ch_release() */
1757	ublk_put_device(ub);
1758}
1759
1760static int ublk_ch_release(struct inode *inode, struct file *filp)
1761{
1762	struct ublk_device *ub = filp->private_data;
1763
1764	/*
1765	 * Grab ublk device reference, so it won't be gone until we are
1766	 * really released from work function.
1767	 */
1768	ublk_get_device(ub);
1769
1770	INIT_DELAYED_WORK(&ub->exit_work, ublk_ch_release_work_fn);
1771	schedule_delayed_work(&ub->exit_work, 0);
1772	return 0;
1773}
1774
1775/* map pre-allocated per-queue cmd buffer to ublksrv daemon */
1776static int ublk_ch_mmap(struct file *filp, struct vm_area_struct *vma)
1777{
1778	struct ublk_device *ub = filp->private_data;
1779	size_t sz = vma->vm_end - vma->vm_start;
1780	unsigned max_sz = ublk_max_cmd_buf_size();
1781	unsigned long pfn, end, phys_off = vma->vm_pgoff << PAGE_SHIFT;
1782	int q_id, ret = 0;
1783
1784	spin_lock(&ub->lock);
1785	if (!ub->mm)
1786		ub->mm = current->mm;
1787	if (current->mm != ub->mm)
1788		ret = -EINVAL;
1789	spin_unlock(&ub->lock);
1790
1791	if (ret)
1792		return ret;
1793
1794	if (vma->vm_flags & VM_WRITE)
1795		return -EPERM;
1796
1797	end = UBLKSRV_CMD_BUF_OFFSET + ub->dev_info.nr_hw_queues * max_sz;
1798	if (phys_off < UBLKSRV_CMD_BUF_OFFSET || phys_off >= end)
1799		return -EINVAL;
1800
1801	q_id = (phys_off - UBLKSRV_CMD_BUF_OFFSET) / max_sz;
1802	pr_devel("%s: qid %d, pid %d, addr %lx pg_off %lx sz %lu\n",
1803			__func__, q_id, current->pid, vma->vm_start,
1804			phys_off, (unsigned long)sz);
1805
1806	if (sz != ublk_queue_cmd_buf_size(ub))
1807		return -EINVAL;
1808
1809	pfn = virt_to_phys(ublk_queue_cmd_buf(ub, q_id)) >> PAGE_SHIFT;
1810	return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
1811}
1812
1813static void __ublk_fail_req(struct ublk_device *ub, struct ublk_io *io,
1814		struct request *req)
1815{
1816	WARN_ON_ONCE(io->flags & UBLK_IO_FLAG_ACTIVE);
1817
1818	if (ublk_nosrv_should_reissue_outstanding(ub))
1819		blk_mq_requeue_request(req, false);
1820	else {
1821		io->res = -EIO;
1822		__ublk_complete_rq(req, io, ublk_dev_need_map_io(ub));
1823	}
1824}
1825
1826/*
1827 * Called from ublk char device release handler, when any uring_cmd is
1828 * done, meantime request queue is "quiesced" since all inflight requests
1829 * can't be completed because ublk server is dead.
1830 *
1831 * So no one can hold our request IO reference any more, simply ignore the
1832 * reference, and complete the request immediately
1833 */
1834static void ublk_abort_queue(struct ublk_device *ub, struct ublk_queue *ubq)
1835{
1836	int i;
1837
1838	for (i = 0; i < ubq->q_depth; i++) {
1839		struct ublk_io *io = &ubq->ios[i];
1840
1841		if (io->flags & UBLK_IO_FLAG_OWNED_BY_SRV)
1842			__ublk_fail_req(ub, io, io->req);
1843	}
1844}
1845
1846static void ublk_start_cancel(struct ublk_device *ub)
1847{
1848	struct gendisk *disk = ublk_get_disk(ub);
1849
1850	/* Our disk has been dead */
1851	if (!disk)
1852		return;
1853
1854	mutex_lock(&ub->cancel_mutex);
1855	if (ub->canceling)
1856		goto out;
1857	/*
1858	 * Now we are serialized with ublk_queue_rq()
1859	 *
1860	 * Make sure that ubq->canceling is set when queue is frozen,
1861	 * because ublk_queue_rq() has to rely on this flag for avoiding to
1862	 * touch completed uring_cmd
1863	 */
1864	blk_mq_quiesce_queue(disk->queue);
1865	ublk_set_canceling(ub, true);
1866	blk_mq_unquiesce_queue(disk->queue);
1867out:
1868	mutex_unlock(&ub->cancel_mutex);
1869	ublk_put_disk(disk);
1870}
1871
1872static void ublk_cancel_cmd(struct ublk_queue *ubq, unsigned tag,
1873		unsigned int issue_flags)
1874{
1875	struct ublk_io *io = &ubq->ios[tag];
1876	struct ublk_device *ub = ubq->dev;
1877	struct request *req;
1878	bool done;
1879
1880	if (!(io->flags & UBLK_IO_FLAG_ACTIVE))
1881		return;
1882
1883	/*
1884	 * Don't try to cancel this command if the request is started for
1885	 * avoiding race between io_uring_cmd_done() and
1886	 * io_uring_cmd_complete_in_task().
1887	 *
1888	 * Either the started request will be aborted via __ublk_abort_rq(),
1889	 * then this uring_cmd is canceled next time, or it will be done in
1890	 * task work function ublk_dispatch_req() because io_uring guarantees
1891	 * that ublk_dispatch_req() is always called
1892	 */
1893	req = blk_mq_tag_to_rq(ub->tag_set.tags[ubq->q_id], tag);
1894	if (req && blk_mq_request_started(req) && req->tag == tag)
1895		return;
1896
1897	spin_lock(&ubq->cancel_lock);
1898	done = !!(io->flags & UBLK_IO_FLAG_CANCELED);
1899	if (!done)
1900		io->flags |= UBLK_IO_FLAG_CANCELED;
1901	spin_unlock(&ubq->cancel_lock);
1902
1903	if (!done)
1904		io_uring_cmd_done(io->cmd, UBLK_IO_RES_ABORT, issue_flags);
1905}
1906
1907/*
1908 * The ublk char device won't be closed when calling cancel fn, so both
1909 * ublk device and queue are guaranteed to be live
1910 *
1911 * Two-stage cancel:
1912 *
1913 * - make every active uring_cmd done in ->cancel_fn()
1914 *
1915 * - aborting inflight ublk IO requests in ublk char device release handler,
1916 *   which depends on 1st stage because device can only be closed iff all
1917 *   uring_cmd are done
1918 *
1919 * Do _not_ try to acquire ub->mutex before all inflight requests are
1920 * aborted, otherwise deadlock may be caused.
1921 */
1922static void ublk_uring_cmd_cancel_fn(struct io_uring_cmd *cmd,
1923		unsigned int issue_flags)
1924{
1925	struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);
1926	struct ublk_queue *ubq = pdu->ubq;
1927	struct task_struct *task;
1928	struct ublk_io *io;
1929
1930	if (WARN_ON_ONCE(!ubq))
1931		return;
1932
1933	if (WARN_ON_ONCE(pdu->tag >= ubq->q_depth))
1934		return;
1935
1936	task = io_uring_cmd_get_task(cmd);
1937	io = &ubq->ios[pdu->tag];
1938	if (WARN_ON_ONCE(task && task != io->task))
1939		return;
1940
1941	ublk_start_cancel(ubq->dev);
1942
1943	WARN_ON_ONCE(io->cmd != cmd);
1944	ublk_cancel_cmd(ubq, pdu->tag, issue_flags);
1945}
1946
1947static inline bool ublk_dev_ready(const struct ublk_device *ub)
1948{
1949	u32 total = (u32)ub->dev_info.nr_hw_queues * ub->dev_info.queue_depth;
1950
1951	return ub->nr_io_ready == total;
1952}
1953
1954static void ublk_cancel_queue(struct ublk_queue *ubq)
1955{
1956	int i;
1957
1958	for (i = 0; i < ubq->q_depth; i++)
1959		ublk_cancel_cmd(ubq, i, IO_URING_F_UNLOCKED);
1960}
1961
1962/* Cancel all pending commands, must be called after del_gendisk() returns */
1963static void ublk_cancel_dev(struct ublk_device *ub)
1964{
1965	int i;
1966
1967	for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
1968		ublk_cancel_queue(ublk_get_queue(ub, i));
1969}
1970
1971static bool ublk_check_inflight_rq(struct request *rq, void *data)
1972{
1973	bool *idle = data;
1974
1975	if (blk_mq_request_started(rq)) {
1976		*idle = false;
1977		return false;
1978	}
1979	return true;
1980}
1981
1982static void ublk_wait_tagset_rqs_idle(struct ublk_device *ub)
1983{
1984	bool idle;
1985
1986	WARN_ON_ONCE(!blk_queue_quiesced(ub->ub_disk->queue));
1987	while (true) {
1988		idle = true;
1989		blk_mq_tagset_busy_iter(&ub->tag_set,
1990				ublk_check_inflight_rq, &idle);
1991		if (idle)
1992			break;
1993		msleep(UBLK_REQUEUE_DELAY_MS);
1994	}
1995}
1996
1997static void ublk_force_abort_dev(struct ublk_device *ub)
1998{
1999	int i;
2000
2001	pr_devel("%s: force abort ub: dev_id %d state %s\n",
2002			__func__, ub->dev_info.dev_id,
2003			ub->dev_info.state == UBLK_S_DEV_LIVE ?
2004			"LIVE" : "QUIESCED");
2005	blk_mq_quiesce_queue(ub->ub_disk->queue);
2006	if (ub->dev_info.state == UBLK_S_DEV_LIVE)
2007		ublk_wait_tagset_rqs_idle(ub);
2008
2009	for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
2010		ublk_get_queue(ub, i)->force_abort = true;
2011	blk_mq_unquiesce_queue(ub->ub_disk->queue);
2012	/* We may have requeued some rqs in ublk_quiesce_queue() */
2013	blk_mq_kick_requeue_list(ub->ub_disk->queue);
2014}
2015
2016static struct gendisk *ublk_detach_disk(struct ublk_device *ub)
2017{
2018	struct gendisk *disk;
2019
2020	/* Sync with ublk_abort_queue() by holding the lock */
2021	spin_lock(&ub->lock);
2022	disk = ub->ub_disk;
2023	ub->dev_info.state = UBLK_S_DEV_DEAD;
2024	ub->dev_info.ublksrv_pid = -1;
2025	ub->ub_disk = NULL;
2026	spin_unlock(&ub->lock);
2027
2028	return disk;
2029}
2030
2031static void ublk_stop_dev_unlocked(struct ublk_device *ub)
2032	__must_hold(&ub->mutex)
2033{
2034	struct gendisk *disk;
2035
2036	if (ub->dev_info.state == UBLK_S_DEV_DEAD)
2037		return;
2038
2039	if (ublk_nosrv_dev_should_queue_io(ub))
2040		ublk_force_abort_dev(ub);
2041	del_gendisk(ub->ub_disk);
2042	disk = ublk_detach_disk(ub);
2043	put_disk(disk);
2044}
2045
2046static void ublk_stop_dev(struct ublk_device *ub)
2047{
2048	mutex_lock(&ub->mutex);
2049	ublk_stop_dev_unlocked(ub);
2050	mutex_unlock(&ub->mutex);
2051	cancel_work_sync(&ub->partition_scan_work);
2052	ublk_cancel_dev(ub);
2053}
2054
2055/* reset ublk io_uring queue & io flags */
2056static void ublk_reset_io_flags(struct ublk_device *ub)
2057{
2058	int i, j;
2059
2060	for (i = 0; i < ub->dev_info.nr_hw_queues; i++) {
2061		struct ublk_queue *ubq = ublk_get_queue(ub, i);
2062
2063		/* UBLK_IO_FLAG_CANCELED can be cleared now */
2064		spin_lock(&ubq->cancel_lock);
2065		for (j = 0; j < ubq->q_depth; j++)
2066			ubq->ios[j].flags &= ~UBLK_IO_FLAG_CANCELED;
2067		spin_unlock(&ubq->cancel_lock);
2068		ubq->fail_io = false;
2069	}
2070	mutex_lock(&ub->cancel_mutex);
2071	ublk_set_canceling(ub, false);
2072	mutex_unlock(&ub->cancel_mutex);
2073}
2074
2075/* device can only be started after all IOs are ready */
2076static void ublk_mark_io_ready(struct ublk_device *ub)
2077	__must_hold(&ub->mutex)
2078{
2079	if (!ub->unprivileged_daemons && !capable(CAP_SYS_ADMIN))
2080		ub->unprivileged_daemons = true;
2081
2082	ub->nr_io_ready++;
2083	if (ublk_dev_ready(ub)) {
2084		/* now we are ready for handling ublk io request */
2085		ublk_reset_io_flags(ub);
2086		complete_all(&ub->completion);
2087	}
2088}
2089
2090static inline int ublk_check_cmd_op(u32 cmd_op)
2091{
2092	u32 ioc_type = _IOC_TYPE(cmd_op);
2093
2094	if (!IS_ENABLED(CONFIG_BLKDEV_UBLK_LEGACY_OPCODES) && ioc_type != 'u')
2095		return -EOPNOTSUPP;
2096
2097	if (ioc_type != 'u' && ioc_type != 0)
2098		return -EOPNOTSUPP;
2099
2100	return 0;
2101}
2102
2103static inline int ublk_set_auto_buf_reg(struct ublk_io *io, struct io_uring_cmd *cmd)
2104{
2105	struct ublk_auto_buf_reg buf;
2106
2107	buf = ublk_sqe_addr_to_auto_buf_reg(READ_ONCE(cmd->sqe->addr));
2108
2109	if (buf.reserved0 || buf.reserved1)
2110		return -EINVAL;
2111
2112	if (buf.flags & ~UBLK_AUTO_BUF_REG_F_MASK)
2113		return -EINVAL;
2114	io->buf.auto_reg = buf;
2115	return 0;
2116}
2117
2118static int ublk_handle_auto_buf_reg(struct ublk_io *io,
2119				    struct io_uring_cmd *cmd,
2120				    u16 *buf_idx)
2121{
2122	if (io->flags & UBLK_IO_FLAG_AUTO_BUF_REG) {
2123		io->flags &= ~UBLK_IO_FLAG_AUTO_BUF_REG;
2124
2125		/*
2126		 * `UBLK_F_AUTO_BUF_REG` only works iff `UBLK_IO_FETCH_REQ`
2127		 * and `UBLK_IO_COMMIT_AND_FETCH_REQ` are issued from same
2128		 * `io_ring_ctx`.
2129		 *
2130		 * If this uring_cmd's io_ring_ctx isn't same with the
2131		 * one for registering the buffer, it is ublk server's
2132		 * responsibility for unregistering the buffer, otherwise
2133		 * this ublk request gets stuck.
2134		 */
2135		if (io->buf_ctx_handle == io_uring_cmd_ctx_handle(cmd))
2136			*buf_idx = io->buf.auto_reg.index;
2137	}
2138
2139	return ublk_set_auto_buf_reg(io, cmd);
2140}
2141
2142/* Once we return, `io->req` can't be used any more */
2143static inline struct request *
2144ublk_fill_io_cmd(struct ublk_io *io, struct io_uring_cmd *cmd)
2145{
2146	struct request *req = io->req;
2147
2148	io->cmd = cmd;
2149	io->flags |= UBLK_IO_FLAG_ACTIVE;
2150	/* now this cmd slot is owned by ublk driver */
2151	io->flags &= ~UBLK_IO_FLAG_OWNED_BY_SRV;
2152
2153	return req;
2154}
2155
2156static inline int
2157ublk_config_io_buf(const struct ublk_device *ub, struct ublk_io *io,
2158		   struct io_uring_cmd *cmd, unsigned long buf_addr,
2159		   u16 *buf_idx)
2160{
2161	if (ublk_dev_support_auto_buf_reg(ub))
2162		return ublk_handle_auto_buf_reg(io, cmd, buf_idx);
2163
2164	io->buf.addr = buf_addr;
2165	return 0;
2166}
2167
2168static inline void ublk_prep_cancel(struct io_uring_cmd *cmd,
2169				    unsigned int issue_flags,
2170				    struct ublk_queue *ubq, unsigned int tag)
2171{
2172	struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);
2173
2174	/*
2175	 * Safe to refer to @ubq since ublk_queue won't be died until its
2176	 * commands are completed
2177	 */
2178	pdu->ubq = ubq;
2179	pdu->tag = tag;
2180	io_uring_cmd_mark_cancelable(cmd, issue_flags);
2181}
2182
2183static void ublk_io_release(void *priv)
2184{
2185	struct request *rq = priv;
2186	struct ublk_queue *ubq = rq->mq_hctx->driver_data;
2187	struct ublk_io *io = &ubq->ios[rq->tag];
2188
2189	/*
2190	 * task_registered_buffers may be 0 if buffers were registered off task
2191	 * but unregistered on task. Or after UBLK_IO_COMMIT_AND_FETCH_REQ.
2192	 */
2193	if (current == io->task && io->task_registered_buffers)
2194		io->task_registered_buffers--;
2195	else
2196		ublk_put_req_ref(io, rq);
2197}
2198
2199static int ublk_register_io_buf(struct io_uring_cmd *cmd,
2200				struct ublk_device *ub,
2201				u16 q_id, u16 tag,
2202				struct ublk_io *io,
2203				unsigned int index, unsigned int issue_flags)
2204{
2205	struct request *req;
2206	int ret;
2207
2208	if (!ublk_dev_support_zero_copy(ub))
2209		return -EINVAL;
2210
2211	req = __ublk_check_and_get_req(ub, q_id, tag, io, 0);
2212	if (!req)
2213		return -EINVAL;
2214
2215	ret = io_buffer_register_bvec(cmd, req, ublk_io_release, index,
2216				      issue_flags);
2217	if (ret) {
2218		ublk_put_req_ref(io, req);
2219		return ret;
2220	}
2221
2222	return 0;
2223}
2224
2225static int
2226ublk_daemon_register_io_buf(struct io_uring_cmd *cmd,
2227			    struct ublk_device *ub,
2228			    u16 q_id, u16 tag, struct ublk_io *io,
2229			    unsigned index, unsigned issue_flags)
2230{
2231	unsigned new_registered_buffers;
2232	struct request *req = io->req;
2233	int ret;
2234
2235	/*
2236	 * Ensure there are still references for ublk_sub_req_ref() to release.
2237	 * If not, fall back on the thread-safe buffer registration.
2238	 */
2239	new_registered_buffers = io->task_registered_buffers + 1;
2240	if (unlikely(new_registered_buffers >= UBLK_REFCOUNT_INIT))
2241		return ublk_register_io_buf(cmd, ub, q_id, tag, io, index,
2242					    issue_flags);
2243
2244	if (!ublk_dev_support_zero_copy(ub) || !ublk_rq_has_data(req))
2245		return -EINVAL;
2246
2247	ret = io_buffer_register_bvec(cmd, req, ublk_io_release, index,
2248				      issue_flags);
2249	if (ret)
2250		return ret;
2251
2252	io->task_registered_buffers = new_registered_buffers;
2253	return 0;
2254}
2255
2256static int ublk_unregister_io_buf(struct io_uring_cmd *cmd,
2257				  const struct ublk_device *ub,
2258				  unsigned int index, unsigned int issue_flags)
2259{
2260	if (!(ub->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY))
2261		return -EINVAL;
2262
2263	return io_buffer_unregister_bvec(cmd, index, issue_flags);
2264}
2265
2266static int ublk_check_fetch_buf(const struct ublk_device *ub, __u64 buf_addr)
2267{
2268	if (ublk_dev_need_map_io(ub)) {
2269		/*
2270		 * FETCH_RQ has to provide IO buffer if NEED GET
2271		 * DATA is not enabled
2272		 */
2273		if (!buf_addr && !ublk_dev_need_get_data(ub))
2274			return -EINVAL;
2275	} else if (buf_addr) {
2276		/* User copy requires addr to be unset */
2277		return -EINVAL;
2278	}
2279	return 0;
2280}
2281
2282static int __ublk_fetch(struct io_uring_cmd *cmd, struct ublk_device *ub,
2283			struct ublk_io *io)
2284{
2285	/* UBLK_IO_FETCH_REQ is only allowed before dev is setup */
2286	if (ublk_dev_ready(ub))
2287		return -EBUSY;
2288
2289	/* allow each command to be FETCHed at most once */
2290	if (io->flags & UBLK_IO_FLAG_ACTIVE)
2291		return -EINVAL;
2292
2293	WARN_ON_ONCE(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV);
2294
2295	ublk_fill_io_cmd(io, cmd);
2296
2297	WRITE_ONCE(io->task, get_task_struct(current));
2298	ublk_mark_io_ready(ub);
2299
2300	return 0;
2301}
2302
2303static int ublk_fetch(struct io_uring_cmd *cmd, struct ublk_device *ub,
2304		      struct ublk_io *io, __u64 buf_addr)
2305{
2306	int ret;
2307
2308	/*
2309	 * When handling FETCH command for setting up ublk uring queue,
2310	 * ub->mutex is the innermost lock, and we won't block for handling
2311	 * FETCH, so it is fine even for IO_URING_F_NONBLOCK.
2312	 */
2313	mutex_lock(&ub->mutex);
2314	ret = __ublk_fetch(cmd, ub, io);
2315	if (!ret)
2316		ret = ublk_config_io_buf(ub, io, cmd, buf_addr, NULL);
2317	mutex_unlock(&ub->mutex);
2318	return ret;
2319}
2320
2321static int ublk_check_commit_and_fetch(const struct ublk_device *ub,
2322				       struct ublk_io *io, __u64 buf_addr)
2323{
2324	struct request *req = io->req;
2325
2326	if (ublk_dev_need_map_io(ub)) {
2327		/*
2328		 * COMMIT_AND_FETCH_REQ has to provide IO buffer if
2329		 * NEED GET DATA is not enabled or it is Read IO.
2330		 */
2331		if (!buf_addr && (!ublk_dev_need_get_data(ub) ||
2332					req_op(req) == REQ_OP_READ))
2333			return -EINVAL;
2334	} else if (req_op(req) != REQ_OP_ZONE_APPEND && buf_addr) {
2335		/*
2336		 * User copy requires addr to be unset when command is
2337		 * not zone append
2338		 */
2339		return -EINVAL;
2340	}
2341
2342	return 0;
2343}
2344
2345static bool ublk_need_complete_req(const struct ublk_device *ub,
2346				   struct ublk_io *io)
2347{
2348	if (ublk_dev_need_req_ref(ub))
2349		return ublk_sub_req_ref(io);
2350	return true;
2351}
2352
2353static bool ublk_get_data(const struct ublk_queue *ubq, struct ublk_io *io,
2354			  struct request *req)
2355{
2356	/*
2357	 * We have handled UBLK_IO_NEED_GET_DATA command,
2358	 * so clear UBLK_IO_FLAG_NEED_GET_DATA now and just
2359	 * do the copy work.
2360	 */
2361	io->flags &= ~UBLK_IO_FLAG_NEED_GET_DATA;
2362	/* update iod->addr because ublksrv may have passed a new io buffer */
2363	ublk_get_iod(ubq, req->tag)->addr = io->buf.addr;
2364	pr_devel("%s: update iod->addr: qid %d tag %d io_flags %x addr %llx\n",
2365			__func__, ubq->q_id, req->tag, io->flags,
2366			ublk_get_iod(ubq, req->tag)->addr);
2367
2368	return ublk_start_io(ubq, req, io);
2369}
2370
2371static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd,
2372		unsigned int issue_flags)
2373{
2374	/* May point to userspace-mapped memory */
2375	const struct ublksrv_io_cmd *ub_src = io_uring_sqe_cmd(cmd->sqe);
2376	u16 buf_idx = UBLK_INVALID_BUF_IDX;
2377	struct ublk_device *ub = cmd->file->private_data;
2378	struct ublk_queue *ubq;
2379	struct ublk_io *io = NULL;
2380	u32 cmd_op = cmd->cmd_op;
2381	u16 q_id = READ_ONCE(ub_src->q_id);
2382	u16 tag = READ_ONCE(ub_src->tag);
2383	s32 result = READ_ONCE(ub_src->result);
2384	u64 addr = READ_ONCE(ub_src->addr); /* unioned with zone_append_lba */
2385	struct request *req;
2386	int ret;
2387	bool compl;
2388
2389	WARN_ON_ONCE(issue_flags & IO_URING_F_UNLOCKED);
2390
2391	pr_devel("%s: received: cmd op %d queue %d tag %d result %d\n",
2392			__func__, cmd->cmd_op, q_id, tag, result);
2393
2394	ret = ublk_check_cmd_op(cmd_op);
2395	if (ret)
2396		goto out;
2397
2398	/*
2399	 * io_buffer_unregister_bvec() doesn't access the ubq or io,
2400	 * so no need to validate the q_id, tag, or task
2401	 */
2402	if (_IOC_NR(cmd_op) == UBLK_IO_UNREGISTER_IO_BUF)
2403		return ublk_unregister_io_buf(cmd, ub, addr, issue_flags);
2404
2405	ret = -EINVAL;
2406	if (q_id >= ub->dev_info.nr_hw_queues)
2407		goto out;
2408
2409	ubq = ublk_get_queue(ub, q_id);
2410
2411	if (tag >= ub->dev_info.queue_depth)
2412		goto out;
2413
2414	io = &ubq->ios[tag];
2415	/* UBLK_IO_FETCH_REQ can be handled on any task, which sets io->task */
2416	if (unlikely(_IOC_NR(cmd_op) == UBLK_IO_FETCH_REQ)) {
2417		ret = ublk_check_fetch_buf(ub, addr);
2418		if (ret)
2419			goto out;
2420		ret = ublk_fetch(cmd, ub, io, addr);
2421		if (ret)
2422			goto out;
2423
2424		ublk_prep_cancel(cmd, issue_flags, ubq, tag);
2425		return -EIOCBQUEUED;
2426	}
2427
2428	if (READ_ONCE(io->task) != current) {
2429		/*
2430		 * ublk_register_io_buf() accesses only the io's refcount,
2431		 * so can be handled on any task
2432		 */
2433		if (_IOC_NR(cmd_op) == UBLK_IO_REGISTER_IO_BUF)
2434			return ublk_register_io_buf(cmd, ub, q_id, tag, io,
2435						    addr, issue_flags);
2436
2437		goto out;
2438	}
2439
2440	/* there is pending io cmd, something must be wrong */
2441	if (!(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV)) {
2442		ret = -EBUSY;
2443		goto out;
2444	}
2445
2446	/*
2447	 * ensure that the user issues UBLK_IO_NEED_GET_DATA
2448	 * iff the driver have set the UBLK_IO_FLAG_NEED_GET_DATA.
2449	 */
2450	if ((!!(io->flags & UBLK_IO_FLAG_NEED_GET_DATA))
2451			^ (_IOC_NR(cmd_op) == UBLK_IO_NEED_GET_DATA))
2452		goto out;
2453
2454	switch (_IOC_NR(cmd_op)) {
2455	case UBLK_IO_REGISTER_IO_BUF:
2456		return ublk_daemon_register_io_buf(cmd, ub, q_id, tag, io, addr,
2457						   issue_flags);
2458	case UBLK_IO_COMMIT_AND_FETCH_REQ:
2459		ret = ublk_check_commit_and_fetch(ub, io, addr);
2460		if (ret)
2461			goto out;
2462		io->res = result;
2463		req = ublk_fill_io_cmd(io, cmd);
2464		ret = ublk_config_io_buf(ub, io, cmd, addr, &buf_idx);
2465		if (buf_idx != UBLK_INVALID_BUF_IDX)
2466			io_buffer_unregister_bvec(cmd, buf_idx, issue_flags);
2467		compl = ublk_need_complete_req(ub, io);
2468
2469		/* can't touch 'ublk_io' any more */
2470		if (req_op(req) == REQ_OP_ZONE_APPEND)
2471			req->__sector = addr;
2472		if (compl)
2473			__ublk_complete_rq(req, io, ublk_dev_need_map_io(ub));
2474
2475		if (ret)
2476			goto out;
2477		break;
2478	case UBLK_IO_NEED_GET_DATA:
2479		/*
2480		 * ublk_get_data() may fail and fallback to requeue, so keep
2481		 * uring_cmd active first and prepare for handling new requeued
2482		 * request
2483		 */
2484		req = ublk_fill_io_cmd(io, cmd);
2485		ret = ublk_config_io_buf(ub, io, cmd, addr, NULL);
2486		WARN_ON_ONCE(ret);
2487		if (likely(ublk_get_data(ubq, io, req))) {
2488			__ublk_prep_compl_io_cmd(io, req);
2489			return UBLK_IO_RES_OK;
2490		}
2491		break;
2492	default:
2493		goto out;
2494	}
2495	ublk_prep_cancel(cmd, issue_flags, ubq, tag);
2496	return -EIOCBQUEUED;
2497
2498 out:
2499	pr_devel("%s: complete: cmd op %d, tag %d ret %x io_flags %x\n",
2500			__func__, cmd_op, tag, ret, io ? io->flags : 0);
2501	return ret;
2502}
2503
2504static inline struct request *__ublk_check_and_get_req(struct ublk_device *ub,
2505		u16 q_id, u16 tag, struct ublk_io *io, size_t offset)
2506{
2507	struct request *req;
2508
2509	/*
2510	 * can't use io->req in case of concurrent UBLK_IO_COMMIT_AND_FETCH_REQ,
2511	 * which would overwrite it with io->cmd
2512	 */
2513	req = blk_mq_tag_to_rq(ub->tag_set.tags[q_id], tag);
2514	if (!req)
2515		return NULL;
2516
2517	if (!ublk_get_req_ref(io))
2518		return NULL;
2519
2520	if (unlikely(!blk_mq_request_started(req) || req->tag != tag))
2521		goto fail_put;
2522
2523	if (!ublk_rq_has_data(req))
2524		goto fail_put;
2525
2526	if (offset > blk_rq_bytes(req))
2527		goto fail_put;
2528
2529	return req;
2530fail_put:
2531	ublk_put_req_ref(io, req);
2532	return NULL;
2533}
2534
2535static void ublk_ch_uring_cmd_cb(struct io_tw_req tw_req, io_tw_token_t tw)
2536{
2537	unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS;
2538	struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);
2539	int ret = ublk_ch_uring_cmd_local(cmd, issue_flags);
2540
2541	if (ret != -EIOCBQUEUED)
2542		io_uring_cmd_done(cmd, ret, issue_flags);
2543}
2544
2545static int ublk_ch_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags)
2546{
2547	if (unlikely(issue_flags & IO_URING_F_CANCEL)) {
2548		ublk_uring_cmd_cancel_fn(cmd, issue_flags);
2549		return 0;
2550	}
2551
2552	/* well-implemented server won't run into unlocked */
2553	if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) {
2554		io_uring_cmd_complete_in_task(cmd, ublk_ch_uring_cmd_cb);
2555		return -EIOCBQUEUED;
2556	}
2557
2558	return ublk_ch_uring_cmd_local(cmd, issue_flags);
2559}
2560
2561static inline bool ublk_check_ubuf_dir(const struct request *req,
2562		int ubuf_dir)
2563{
2564	/* copy ubuf to request pages */
2565	if ((req_op(req) == REQ_OP_READ || req_op(req) == REQ_OP_DRV_IN) &&
2566	    ubuf_dir == ITER_SOURCE)
2567		return true;
2568
2569	/* copy request pages to ubuf */
2570	if ((req_op(req) == REQ_OP_WRITE ||
2571	     req_op(req) == REQ_OP_ZONE_APPEND) &&
2572	    ubuf_dir == ITER_DEST)
2573		return true;
2574
2575	return false;
2576}
2577
2578static struct request *ublk_check_and_get_req(struct kiocb *iocb,
2579		struct iov_iter *iter, size_t *off, int dir,
2580		struct ublk_io **io)
2581{
2582	struct ublk_device *ub = iocb->ki_filp->private_data;
2583	struct ublk_queue *ubq;
2584	struct request *req;
2585	size_t buf_off;
2586	u16 tag, q_id;
2587
2588	if (!user_backed_iter(iter))
2589		return ERR_PTR(-EACCES);
2590
2591	if (ub->dev_info.state == UBLK_S_DEV_DEAD)
2592		return ERR_PTR(-EACCES);
2593
2594	tag = ublk_pos_to_tag(iocb->ki_pos);
2595	q_id = ublk_pos_to_hwq(iocb->ki_pos);
2596	buf_off = ublk_pos_to_buf_off(iocb->ki_pos);
2597
2598	if (q_id >= ub->dev_info.nr_hw_queues)
2599		return ERR_PTR(-EINVAL);
2600
2601	ubq = ublk_get_queue(ub, q_id);
2602	if (!ublk_dev_support_user_copy(ub))
2603		return ERR_PTR(-EACCES);
2604
2605	if (tag >= ub->dev_info.queue_depth)
2606		return ERR_PTR(-EINVAL);
2607
2608	*io = &ubq->ios[tag];
2609	req = __ublk_check_and_get_req(ub, q_id, tag, *io, buf_off);
2610	if (!req)
2611		return ERR_PTR(-EINVAL);
2612
2613	if (!ublk_check_ubuf_dir(req, dir))
2614		goto fail;
2615
2616	*off = buf_off;
2617	return req;
2618fail:
2619	ublk_put_req_ref(*io, req);
2620	return ERR_PTR(-EACCES);
2621}
2622
2623static ssize_t ublk_ch_read_iter(struct kiocb *iocb, struct iov_iter *to)
2624{
2625	struct request *req;
2626	struct ublk_io *io;
2627	size_t buf_off;
2628	size_t ret;
2629
2630	req = ublk_check_and_get_req(iocb, to, &buf_off, ITER_DEST, &io);
2631	if (IS_ERR(req))
2632		return PTR_ERR(req);
2633
2634	ret = ublk_copy_user_pages(req, buf_off, to, ITER_DEST);
2635	ublk_put_req_ref(io, req);
2636
2637	return ret;
2638}
2639
2640static ssize_t ublk_ch_write_iter(struct kiocb *iocb, struct iov_iter *from)
2641{
2642	struct request *req;
2643	struct ublk_io *io;
2644	size_t buf_off;
2645	size_t ret;
2646
2647	req = ublk_check_and_get_req(iocb, from, &buf_off, ITER_SOURCE, &io);
2648	if (IS_ERR(req))
2649		return PTR_ERR(req);
2650
2651	ret = ublk_copy_user_pages(req, buf_off, from, ITER_SOURCE);
2652	ublk_put_req_ref(io, req);
2653
2654	return ret;
2655}
2656
2657static const struct file_operations ublk_ch_fops = {
2658	.owner = THIS_MODULE,
2659	.open = ublk_ch_open,
2660	.release = ublk_ch_release,
2661	.read_iter = ublk_ch_read_iter,
2662	.write_iter = ublk_ch_write_iter,
2663	.uring_cmd = ublk_ch_uring_cmd,
2664	.mmap = ublk_ch_mmap,
2665};
2666
2667static void ublk_deinit_queue(struct ublk_device *ub, int q_id)
2668{
2669	struct ublk_queue *ubq = ub->queues[q_id];
2670	int size, i;
2671
2672	if (!ubq)
2673		return;
2674
2675	size = ublk_queue_cmd_buf_size(ub);
2676
2677	for (i = 0; i < ubq->q_depth; i++) {
2678		struct ublk_io *io = &ubq->ios[i];
2679		if (io->task)
2680			put_task_struct(io->task);
2681		WARN_ON_ONCE(refcount_read(&io->ref));
2682		WARN_ON_ONCE(io->task_registered_buffers);
2683	}
2684
2685	if (ubq->io_cmd_buf)
2686		free_pages((unsigned long)ubq->io_cmd_buf, get_order(size));
2687
2688	kvfree(ubq);
2689	ub->queues[q_id] = NULL;
2690}
2691
2692static int ublk_get_queue_numa_node(struct ublk_device *ub, int q_id)
2693{
2694	unsigned int cpu;
2695
2696	/* Find first CPU mapped to this queue */
2697	for_each_possible_cpu(cpu) {
2698		if (ub->tag_set.map[HCTX_TYPE_DEFAULT].mq_map[cpu] == q_id)
2699			return cpu_to_node(cpu);
2700	}
2701
2702	return NUMA_NO_NODE;
2703}
2704
2705static int ublk_init_queue(struct ublk_device *ub, int q_id)
2706{
2707	int depth = ub->dev_info.queue_depth;
2708	gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO;
2709	struct ublk_queue *ubq;
2710	struct page *page;
2711	int numa_node;
2712	int size;
2713
2714	/* Determine NUMA node based on queue's CPU affinity */
2715	numa_node = ublk_get_queue_numa_node(ub, q_id);
2716
2717	/* Allocate queue structure on local NUMA node */
2718	ubq = kvzalloc_node(struct_size(ubq, ios, depth), GFP_KERNEL,
2719			    numa_node);
2720	if (!ubq)
2721		return -ENOMEM;
2722
2723	spin_lock_init(&ubq->cancel_lock);
2724	ubq->flags = ub->dev_info.flags;
2725	ubq->q_id = q_id;
2726	ubq->q_depth = depth;
2727	size = ublk_queue_cmd_buf_size(ub);
2728
2729	/* Allocate I/O command buffer on local NUMA node */
2730	page = alloc_pages_node(numa_node, gfp_flags, get_order(size));
2731	if (!page) {
2732		kvfree(ubq);
2733		return -ENOMEM;
2734	}
2735	ubq->io_cmd_buf = page_address(page);
2736
2737	ub->queues[q_id] = ubq;
2738	ubq->dev = ub;
2739	return 0;
2740}
2741
2742static void ublk_deinit_queues(struct ublk_device *ub)
2743{
2744	int i;
2745
2746	for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
2747		ublk_deinit_queue(ub, i);
2748}
2749
2750static int ublk_init_queues(struct ublk_device *ub)
2751{
2752	int i, ret;
2753
2754	for (i = 0; i < ub->dev_info.nr_hw_queues; i++) {
2755		ret = ublk_init_queue(ub, i);
2756		if (ret)
2757			goto fail;
2758	}
2759
2760	init_completion(&ub->completion);
2761	return 0;
2762
2763 fail:
2764	ublk_deinit_queues(ub);
2765	return ret;
2766}
2767
2768static int ublk_alloc_dev_number(struct ublk_device *ub, int idx)
2769{
2770	int i = idx;
2771	int err;
2772
2773	spin_lock(&ublk_idr_lock);
2774	/* allocate id, if @id >= 0, we're requesting that specific id */
2775	if (i >= 0) {
2776		err = idr_alloc(&ublk_index_idr, ub, i, i + 1, GFP_NOWAIT);
2777		if (err == -ENOSPC)
2778			err = -EEXIST;
2779	} else {
2780		err = idr_alloc(&ublk_index_idr, ub, 0, UBLK_MAX_UBLKS,
2781				GFP_NOWAIT);
2782	}
2783	spin_unlock(&ublk_idr_lock);
2784
2785	if (err >= 0)
2786		ub->ub_number = err;
2787
2788	return err;
2789}
2790
2791static void ublk_free_dev_number(struct ublk_device *ub)
2792{
2793	spin_lock(&ublk_idr_lock);
2794	idr_remove(&ublk_index_idr, ub->ub_number);
2795	wake_up_all(&ublk_idr_wq);
2796	spin_unlock(&ublk_idr_lock);
2797}
2798
2799static void ublk_cdev_rel(struct device *dev)
2800{
2801	struct ublk_device *ub = container_of(dev, struct ublk_device, cdev_dev);
2802
2803	blk_mq_free_tag_set(&ub->tag_set);
2804	ublk_deinit_queues(ub);
2805	ublk_free_dev_number(ub);
2806	mutex_destroy(&ub->mutex);
2807	mutex_destroy(&ub->cancel_mutex);
2808	kfree(ub);
2809}
2810
2811static int ublk_add_chdev(struct ublk_device *ub)
2812{
2813	struct device *dev = &ub->cdev_dev;
2814	int minor = ub->ub_number;
2815	int ret;
2816
2817	dev->parent = ublk_misc.this_device;
2818	dev->devt = MKDEV(MAJOR(ublk_chr_devt), minor);
2819	dev->class = &ublk_chr_class;
2820	dev->release = ublk_cdev_rel;
2821	device_initialize(dev);
2822
2823	ret = dev_set_name(dev, "ublkc%d", minor);
2824	if (ret)
2825		goto fail;
2826
2827	cdev_init(&ub->cdev, &ublk_ch_fops);
2828	ret = cdev_device_add(&ub->cdev, dev);
2829	if (ret)
2830		goto fail;
2831
2832	if (ub->dev_info.flags & UBLK_F_UNPRIVILEGED_DEV)
2833		unprivileged_ublks_added++;
2834	return 0;
2835 fail:
2836	put_device(dev);
2837	return ret;
2838}
2839
2840/* align max io buffer size with PAGE_SIZE */
2841static void ublk_align_max_io_size(struct ublk_device *ub)
2842{
2843	unsigned int max_io_bytes = ub->dev_info.max_io_buf_bytes;
2844
2845	ub->dev_info.max_io_buf_bytes =
2846		round_down(max_io_bytes, PAGE_SIZE);
2847}
2848
2849static int ublk_add_tag_set(struct ublk_device *ub)
2850{
2851	ub->tag_set.ops = &ublk_mq_ops;
2852	ub->tag_set.nr_hw_queues = ub->dev_info.nr_hw_queues;
2853	ub->tag_set.queue_depth = ub->dev_info.queue_depth;
2854	ub->tag_set.numa_node = NUMA_NO_NODE;
2855	ub->tag_set.driver_data = ub;
2856	return blk_mq_alloc_tag_set(&ub->tag_set);
2857}
2858
2859static void ublk_remove(struct ublk_device *ub)
2860{
2861	bool unprivileged;
2862
2863	ublk_stop_dev(ub);
2864	cdev_device_del(&ub->cdev, &ub->cdev_dev);
2865	unprivileged = ub->dev_info.flags & UBLK_F_UNPRIVILEGED_DEV;
2866	ublk_put_device(ub);
2867
2868	if (unprivileged)
2869		unprivileged_ublks_added--;
2870}
2871
2872static struct ublk_device *ublk_get_device_from_id(int idx)
2873{
2874	struct ublk_device *ub = NULL;
2875
2876	if (idx < 0)
2877		return NULL;
2878
2879	spin_lock(&ublk_idr_lock);
2880	ub = idr_find(&ublk_index_idr, idx);
2881	if (ub)
2882		ub = ublk_get_device(ub);
2883	spin_unlock(&ublk_idr_lock);
2884
2885	return ub;
2886}
2887
2888static bool ublk_validate_user_pid(struct ublk_device *ub, pid_t ublksrv_pid)
2889{
2890	rcu_read_lock();
2891	ublksrv_pid = pid_nr(find_vpid(ublksrv_pid));
2892	rcu_read_unlock();
2893
2894	return ub->ublksrv_tgid == ublksrv_pid;
2895}
2896
2897static int ublk_ctrl_start_dev(struct ublk_device *ub,
2898		const struct ublksrv_ctrl_cmd *header)
2899{
2900	const struct ublk_param_basic *p = &ub->params.basic;
2901	int ublksrv_pid = (int)header->data[0];
2902	struct queue_limits lim = {
2903		.logical_block_size	= 1 << p->logical_bs_shift,
2904		.physical_block_size	= 1 << p->physical_bs_shift,
2905		.io_min			= 1 << p->io_min_shift,
2906		.io_opt			= 1 << p->io_opt_shift,
2907		.max_hw_sectors		= p->max_sectors,
2908		.chunk_sectors		= p->chunk_sectors,
2909		.virt_boundary_mask	= p->virt_boundary_mask,
2910		.max_segments		= USHRT_MAX,
2911		.max_segment_size	= UINT_MAX,
2912		.dma_alignment		= 3,
2913	};
2914	struct gendisk *disk;
2915	int ret = -EINVAL;
2916
2917	if (ublksrv_pid <= 0)
2918		return -EINVAL;
2919	if (!(ub->params.types & UBLK_PARAM_TYPE_BASIC))
2920		return -EINVAL;
2921
2922	if (ub->params.types & UBLK_PARAM_TYPE_DISCARD) {
2923		const struct ublk_param_discard *pd = &ub->params.discard;
2924
2925		lim.discard_alignment = pd->discard_alignment;
2926		lim.discard_granularity = pd->discard_granularity;
2927		lim.max_hw_discard_sectors = pd->max_discard_sectors;
2928		lim.max_write_zeroes_sectors = pd->max_write_zeroes_sectors;
2929		lim.max_discard_segments = pd->max_discard_segments;
2930	}
2931
2932	if (ub->params.types & UBLK_PARAM_TYPE_ZONED) {
2933		const struct ublk_param_zoned *p = &ub->params.zoned;
2934
2935		if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED))
2936			return -EOPNOTSUPP;
2937
2938		lim.features |= BLK_FEAT_ZONED;
2939		lim.max_active_zones = p->max_active_zones;
2940		lim.max_open_zones =  p->max_open_zones;
2941		lim.max_hw_zone_append_sectors = p->max_zone_append_sectors;
2942	}
2943
2944	if (ub->params.basic.attrs & UBLK_ATTR_VOLATILE_CACHE) {
2945		lim.features |= BLK_FEAT_WRITE_CACHE;
2946		if (ub->params.basic.attrs & UBLK_ATTR_FUA)
2947			lim.features |= BLK_FEAT_FUA;
2948	}
2949
2950	if (ub->params.basic.attrs & UBLK_ATTR_ROTATIONAL)
2951		lim.features |= BLK_FEAT_ROTATIONAL;
2952
2953	if (ub->params.types & UBLK_PARAM_TYPE_DMA_ALIGN)
2954		lim.dma_alignment = ub->params.dma.alignment;
2955
2956	if (ub->params.types & UBLK_PARAM_TYPE_SEGMENT) {
2957		lim.seg_boundary_mask = ub->params.seg.seg_boundary_mask;
2958		lim.max_segment_size = ub->params.seg.max_segment_size;
2959		lim.max_segments = ub->params.seg.max_segments;
2960	}
2961
2962	if (wait_for_completion_interruptible(&ub->completion) != 0)
2963		return -EINTR;
2964
2965	if (!ublk_validate_user_pid(ub, ublksrv_pid))
2966		return -EINVAL;
2967
2968	mutex_lock(&ub->mutex);
2969	if (ub->dev_info.state == UBLK_S_DEV_LIVE ||
2970	    test_bit(UB_STATE_USED, &ub->state)) {
2971		ret = -EEXIST;
2972		goto out_unlock;
2973	}
2974
2975	disk = blk_mq_alloc_disk(&ub->tag_set, &lim, NULL);
2976	if (IS_ERR(disk)) {
2977		ret = PTR_ERR(disk);
2978		goto out_unlock;
2979	}
2980	sprintf(disk->disk_name, "ublkb%d", ub->ub_number);
2981	disk->fops = &ub_fops;
2982	disk->private_data = ub;
2983
2984	ub->dev_info.ublksrv_pid = ub->ublksrv_tgid;
2985	ub->ub_disk = disk;
2986
2987	ublk_apply_params(ub);
2988
2989	/*
2990	 * Suppress partition scan to avoid potential IO hang.
2991	 *
2992	 * If ublk server error occurs during partition scan, the IO may
2993	 * wait while holding ub->mutex, which can deadlock with other
2994	 * operations that need the mutex. Defer partition scan to async
2995	 * work.
2996	 * For unprivileged daemons, keep GD_SUPPRESS_PART_SCAN set
2997	 * permanently.
2998	 */
2999	set_bit(GD_SUPPRESS_PART_SCAN, &disk->state);
3000
3001	ublk_get_device(ub);
3002	ub->dev_info.state = UBLK_S_DEV_LIVE;
3003
3004	if (ublk_dev_is_zoned(ub)) {
3005		ret = ublk_revalidate_disk_zones(ub);
3006		if (ret)
3007			goto out_put_cdev;
3008	}
3009
3010	ret = add_disk(disk);
3011	if (ret)
3012		goto out_put_cdev;
3013
3014	set_bit(UB_STATE_USED, &ub->state);
3015
3016	/* Schedule async partition scan for trusted daemons */
3017	if (!ub->unprivileged_daemons)
3018		schedule_work(&ub->partition_scan_work);
3019
3020out_put_cdev:
3021	if (ret) {
3022		ublk_detach_disk(ub);
3023		ublk_put_device(ub);
3024	}
3025	if (ret)
3026		put_disk(disk);
3027out_unlock:
3028	mutex_unlock(&ub->mutex);
3029	return ret;
3030}
3031
3032static int ublk_ctrl_get_queue_affinity(struct ublk_device *ub,
3033		const struct ublksrv_ctrl_cmd *header)
3034{
3035	void __user *argp = (void __user *)(unsigned long)header->addr;
3036	cpumask_var_t cpumask;
3037	unsigned long queue;
3038	unsigned int retlen;
3039	unsigned int i;
3040	int ret;
3041
3042	if (header->len * BITS_PER_BYTE < nr_cpu_ids)
3043		return -EINVAL;
3044	if (header->len & (sizeof(unsigned long)-1))
3045		return -EINVAL;
3046	if (!header->addr)
3047		return -EINVAL;
3048
3049	queue = header->data[0];
3050	if (queue >= ub->dev_info.nr_hw_queues)
3051		return -EINVAL;
3052
3053	if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL))
3054		return -ENOMEM;
3055
3056	for_each_possible_cpu(i) {
3057		if (ub->tag_set.map[HCTX_TYPE_DEFAULT].mq_map[i] == queue)
3058			cpumask_set_cpu(i, cpumask);
3059	}
3060
3061	ret = -EFAULT;
3062	retlen = min_t(unsigned short, header->len, cpumask_size());
3063	if (copy_to_user(argp, cpumask, retlen))
3064		goto out_free_cpumask;
3065	if (retlen != header->len &&
3066	    clear_user(argp + retlen, header->len - retlen))
3067		goto out_free_cpumask;
3068
3069	ret = 0;
3070out_free_cpumask:
3071	free_cpumask_var(cpumask);
3072	return ret;
3073}
3074
3075static inline void ublk_dump_dev_info(struct ublksrv_ctrl_dev_info *info)
3076{
3077	pr_devel("%s: dev id %d flags %llx\n", __func__,
3078			info->dev_id, info->flags);
3079	pr_devel("\t nr_hw_queues %d queue_depth %d\n",
3080			info->nr_hw_queues, info->queue_depth);
3081}
3082
3083static int ublk_ctrl_add_dev(const struct ublksrv_ctrl_cmd *header)
3084{
3085	void __user *argp = (void __user *)(unsigned long)header->addr;
3086	struct ublksrv_ctrl_dev_info info;
3087	struct ublk_device *ub;
3088	int ret = -EINVAL;
3089
3090	if (header->len < sizeof(info) || !header->addr)
3091		return -EINVAL;
3092	if (header->queue_id != (u16)-1) {
3093		pr_warn("%s: queue_id is wrong %x\n",
3094			__func__, header->queue_id);
3095		return -EINVAL;
3096	}
3097
3098	if (copy_from_user(&info, argp, sizeof(info)))
3099		return -EFAULT;
3100
3101	if (info.queue_depth > UBLK_MAX_QUEUE_DEPTH || !info.queue_depth ||
3102	    info.nr_hw_queues > UBLK_MAX_NR_QUEUES || !info.nr_hw_queues)
3103		return -EINVAL;
3104
3105	if (capable(CAP_SYS_ADMIN))
3106		info.flags &= ~UBLK_F_UNPRIVILEGED_DEV;
3107	else if (!(info.flags & UBLK_F_UNPRIVILEGED_DEV))
3108		return -EPERM;
3109
3110	/* forbid nonsense combinations of recovery flags */
3111	switch (info.flags & UBLK_F_ALL_RECOVERY_FLAGS) {
3112	case 0:
3113	case UBLK_F_USER_RECOVERY:
3114	case (UBLK_F_USER_RECOVERY | UBLK_F_USER_RECOVERY_REISSUE):
3115	case (UBLK_F_USER_RECOVERY | UBLK_F_USER_RECOVERY_FAIL_IO):
3116		break;
3117	default:
3118		pr_warn("%s: invalid recovery flags %llx\n", __func__,
3119			info.flags & UBLK_F_ALL_RECOVERY_FLAGS);
3120		return -EINVAL;
3121	}
3122
3123	if ((info.flags & UBLK_F_QUIESCE) && !(info.flags & UBLK_F_USER_RECOVERY)) {
3124		pr_warn("UBLK_F_QUIESCE requires UBLK_F_USER_RECOVERY\n");
3125		return -EINVAL;
3126	}
3127
3128	/*
3129	 * unprivileged device can't be trusted, but RECOVERY and
3130	 * RECOVERY_REISSUE still may hang error handling, so can't
3131	 * support recovery features for unprivileged ublk now
3132	 *
3133	 * TODO: provide forward progress for RECOVERY handler, so that
3134	 * unprivileged device can benefit from it
3135	 */
3136	if (info.flags & UBLK_F_UNPRIVILEGED_DEV) {
3137		info.flags &= ~(UBLK_F_USER_RECOVERY_REISSUE |
3138				UBLK_F_USER_RECOVERY);
3139
3140		/*
3141		 * For USER_COPY, we depends on userspace to fill request
3142		 * buffer by pwrite() to ublk char device, which can't be
3143		 * used for unprivileged device
3144		 *
3145		 * Same with zero copy or auto buffer register.
3146		 */
3147		if (info.flags & (UBLK_F_USER_COPY | UBLK_F_SUPPORT_ZERO_COPY |
3148					UBLK_F_AUTO_BUF_REG))
3149			return -EINVAL;
3150	}
3151
3152	/* the created device is always owned by current user */
3153	ublk_store_owner_uid_gid(&info.owner_uid, &info.owner_gid);
3154
3155	if (header->dev_id != info.dev_id) {
3156		pr_warn("%s: dev id not match %u %u\n",
3157			__func__, header->dev_id, info.dev_id);
3158		return -EINVAL;
3159	}
3160
3161	if (header->dev_id != U32_MAX && header->dev_id >= UBLK_MAX_UBLKS) {
3162		pr_warn("%s: dev id is too large. Max supported is %d\n",
3163			__func__, UBLK_MAX_UBLKS - 1);
3164		return -EINVAL;
3165	}
3166
3167	ublk_dump_dev_info(&info);
3168
3169	ret = mutex_lock_killable(&ublk_ctl_mutex);
3170	if (ret)
3171		return ret;
3172
3173	ret = -EACCES;
3174	if ((info.flags & UBLK_F_UNPRIVILEGED_DEV) &&
3175	    unprivileged_ublks_added >= unprivileged_ublks_max)
3176		goto out_unlock;
3177
3178	ret = -ENOMEM;
3179	ub = kzalloc(struct_size(ub, queues, info.nr_hw_queues), GFP_KERNEL);
3180	if (!ub)
3181		goto out_unlock;
3182	mutex_init(&ub->mutex);
3183	spin_lock_init(&ub->lock);
3184	mutex_init(&ub->cancel_mutex);
3185	INIT_WORK(&ub->partition_scan_work, ublk_partition_scan_work);
3186
3187	ret = ublk_alloc_dev_number(ub, header->dev_id);
3188	if (ret < 0)
3189		goto out_free_ub;
3190
3191	memcpy(&ub->dev_info, &info, sizeof(info));
3192
3193	/* update device id */
3194	ub->dev_info.dev_id = ub->ub_number;
3195
3196	/*
3197	 * 64bit flags will be copied back to userspace as feature
3198	 * negotiation result, so have to clear flags which driver
3199	 * doesn't support yet, then userspace can get correct flags
3200	 * (features) to handle.
3201	 */
3202	ub->dev_info.flags &= UBLK_F_ALL;
3203
3204	ub->dev_info.flags |= UBLK_F_CMD_IOCTL_ENCODE |
3205		UBLK_F_URING_CMD_COMP_IN_TASK |
3206		UBLK_F_PER_IO_DAEMON |
3207		UBLK_F_BUF_REG_OFF_DAEMON;
3208
3209	/* GET_DATA isn't needed any more with USER_COPY or ZERO COPY */
3210	if (ub->dev_info.flags & (UBLK_F_USER_COPY | UBLK_F_SUPPORT_ZERO_COPY |
3211				UBLK_F_AUTO_BUF_REG))
3212		ub->dev_info.flags &= ~UBLK_F_NEED_GET_DATA;
3213
3214	/*
3215	 * Zoned storage support requires reuse `ublksrv_io_cmd->addr` for
3216	 * returning write_append_lba, which is only allowed in case of
3217	 * user copy or zero copy
3218	 */
3219	if (ublk_dev_is_zoned(ub) &&
3220	    (!IS_ENABLED(CONFIG_BLK_DEV_ZONED) || !(ub->dev_info.flags &
3221	     (UBLK_F_USER_COPY | UBLK_F_SUPPORT_ZERO_COPY)))) {
3222		ret = -EINVAL;
3223		goto out_free_dev_number;
3224	}
3225
3226	ub->dev_info.nr_hw_queues = min_t(unsigned int,
3227			ub->dev_info.nr_hw_queues, nr_cpu_ids);
3228	ublk_align_max_io_size(ub);
3229
3230	ret = ublk_add_tag_set(ub);
3231	if (ret)
3232		goto out_free_dev_number;
3233
3234	ret = ublk_init_queues(ub);
3235	if (ret)
3236		goto out_free_tag_set;
3237
3238	ret = -EFAULT;
3239	if (copy_to_user(argp, &ub->dev_info, sizeof(info)))
3240		goto out_deinit_queues;
3241
3242	/*
3243	 * Add the char dev so that ublksrv daemon can be setup.
3244	 * ublk_add_chdev() will cleanup everything if it fails.
3245	 */
3246	ret = ublk_add_chdev(ub);
3247	goto out_unlock;
3248
3249out_deinit_queues:
3250	ublk_deinit_queues(ub);
3251out_free_tag_set:
3252	blk_mq_free_tag_set(&ub->tag_set);
3253out_free_dev_number:
3254	ublk_free_dev_number(ub);
3255out_free_ub:
3256	mutex_destroy(&ub->mutex);
3257	mutex_destroy(&ub->cancel_mutex);
3258	kfree(ub);
3259out_unlock:
3260	mutex_unlock(&ublk_ctl_mutex);
3261	return ret;
3262}
3263
3264static inline bool ublk_idr_freed(int id)
3265{
3266	void *ptr;
3267
3268	spin_lock(&ublk_idr_lock);
3269	ptr = idr_find(&ublk_index_idr, id);
3270	spin_unlock(&ublk_idr_lock);
3271
3272	return ptr == NULL;
3273}
3274
3275static int ublk_ctrl_del_dev(struct ublk_device **p_ub, bool wait)
3276{
3277	struct ublk_device *ub = *p_ub;
3278	int idx = ub->ub_number;
3279	int ret;
3280
3281	ret = mutex_lock_killable(&ublk_ctl_mutex);
3282	if (ret)
3283		return ret;
3284
3285	if (!test_bit(UB_STATE_DELETED, &ub->state)) {
3286		ublk_remove(ub);
3287		set_bit(UB_STATE_DELETED, &ub->state);
3288	}
3289
3290	/* Mark the reference as consumed */
3291	*p_ub = NULL;
3292	ublk_put_device(ub);
3293	mutex_unlock(&ublk_ctl_mutex);
3294
3295	/*
3296	 * Wait until the idr is removed, then it can be reused after
3297	 * DEL_DEV command is returned.
3298	 *
3299	 * If we returns because of user interrupt, future delete command
3300	 * may come:
3301	 *
3302	 * - the device number isn't freed, this device won't or needn't
3303	 *   be deleted again, since UB_STATE_DELETED is set, and device
3304	 *   will be released after the last reference is dropped
3305	 *
3306	 * - the device number is freed already, we will not find this
3307	 *   device via ublk_get_device_from_id()
3308	 */
3309	if (wait && wait_event_interruptible(ublk_idr_wq, ublk_idr_freed(idx)))
3310		return -EINTR;
3311	return 0;
3312}
3313
3314static inline void ublk_ctrl_cmd_dump(u32 cmd_op,
3315				      const struct ublksrv_ctrl_cmd *header)
3316{
3317	pr_devel("%s: cmd_op %x, dev id %d qid %d data %llx buf %llx len %u\n",
3318			__func__, cmd_op, header->dev_id, header->queue_id,
3319			header->data[0], header->addr, header->len);
3320}
3321
3322static int ublk_ctrl_stop_dev(struct ublk_device *ub)
3323{
3324	ublk_stop_dev(ub);
3325	return 0;
3326}
3327
3328static int ublk_ctrl_get_dev_info(struct ublk_device *ub,
3329		const struct ublksrv_ctrl_cmd *header)
3330{
3331	struct task_struct *p;
3332	struct pid *pid;
3333	struct ublksrv_ctrl_dev_info dev_info;
3334	pid_t init_ublksrv_tgid = ub->dev_info.ublksrv_pid;
3335	void __user *argp = (void __user *)(unsigned long)header->addr;
3336
3337	if (header->len < sizeof(struct ublksrv_ctrl_dev_info) || !header->addr)
3338		return -EINVAL;
3339
3340	memcpy(&dev_info, &ub->dev_info, sizeof(dev_info));
3341	dev_info.ublksrv_pid = -1;
3342
3343	if (init_ublksrv_tgid > 0) {
3344		rcu_read_lock();
3345		pid = find_pid_ns(init_ublksrv_tgid, &init_pid_ns);
3346		p = pid_task(pid, PIDTYPE_TGID);
3347		if (p) {
3348			int vnr = task_tgid_vnr(p);
3349
3350			if (vnr)
3351				dev_info.ublksrv_pid = vnr;
3352		}
3353		rcu_read_unlock();
3354	}
3355
3356	if (copy_to_user(argp, &dev_info, sizeof(dev_info)))
3357		return -EFAULT;
3358
3359	return 0;
3360}
3361
3362/* TYPE_DEVT is readonly, so fill it up before returning to userspace */
3363static void ublk_ctrl_fill_params_devt(struct ublk_device *ub)
3364{
3365	ub->params.devt.char_major = MAJOR(ub->cdev_dev.devt);
3366	ub->params.devt.char_minor = MINOR(ub->cdev_dev.devt);
3367
3368	if (ub->ub_disk) {
3369		ub->params.devt.disk_major = MAJOR(disk_devt(ub->ub_disk));
3370		ub->params.devt.disk_minor = MINOR(disk_devt(ub->ub_disk));
3371	} else {
3372		ub->params.devt.disk_major = 0;
3373		ub->params.devt.disk_minor = 0;
3374	}
3375	ub->params.types |= UBLK_PARAM_TYPE_DEVT;
3376}
3377
3378static int ublk_ctrl_get_params(struct ublk_device *ub,
3379		const struct ublksrv_ctrl_cmd *header)
3380{
3381	void __user *argp = (void __user *)(unsigned long)header->addr;
3382	struct ublk_params_header ph;
3383	int ret;
3384
3385	if (header->len <= sizeof(ph) || !header->addr)
3386		return -EINVAL;
3387
3388	if (copy_from_user(&ph, argp, sizeof(ph)))
3389		return -EFAULT;
3390
3391	if (ph.len > header->len || !ph.len)
3392		return -EINVAL;
3393
3394	if (ph.len > sizeof(struct ublk_params))
3395		ph.len = sizeof(struct ublk_params);
3396
3397	mutex_lock(&ub->mutex);
3398	ublk_ctrl_fill_params_devt(ub);
3399	if (copy_to_user(argp, &ub->params, ph.len))
3400		ret = -EFAULT;
3401	else
3402		ret = 0;
3403	mutex_unlock(&ub->mutex);
3404
3405	return ret;
3406}
3407
3408static int ublk_ctrl_set_params(struct ublk_device *ub,
3409		const struct ublksrv_ctrl_cmd *header)
3410{
3411	void __user *argp = (void __user *)(unsigned long)header->addr;
3412	struct ublk_params_header ph;
3413	int ret = -EFAULT;
3414
3415	if (header->len <= sizeof(ph) || !header->addr)
3416		return -EINVAL;
3417
3418	if (copy_from_user(&ph, argp, sizeof(ph)))
3419		return -EFAULT;
3420
3421	if (ph.len > header->len || !ph.len || !ph.types)
3422		return -EINVAL;
3423
3424	if (ph.len > sizeof(struct ublk_params))
3425		ph.len = sizeof(struct ublk_params);
3426
3427	mutex_lock(&ub->mutex);
3428	if (test_bit(UB_STATE_USED, &ub->state)) {
3429		/*
3430		 * Parameters can only be changed when device hasn't
3431		 * been started yet
3432		 */
3433		ret = -EACCES;
3434	} else if (copy_from_user(&ub->params, argp, ph.len)) {
3435		ret = -EFAULT;
3436	} else {
3437		/* clear all we don't support yet */
3438		ub->params.types &= UBLK_PARAM_TYPE_ALL;
3439		ret = ublk_validate_params(ub);
3440		if (ret)
3441			ub->params.types = 0;
3442	}
3443	mutex_unlock(&ub->mutex);
3444
3445	return ret;
3446}
3447
3448static int ublk_ctrl_start_recovery(struct ublk_device *ub,
3449		const struct ublksrv_ctrl_cmd *header)
3450{
3451	int ret = -EINVAL;
3452
3453	mutex_lock(&ub->mutex);
3454	if (ublk_nosrv_should_stop_dev(ub))
3455		goto out_unlock;
3456	/*
3457	 * START_RECOVERY is only allowd after:
3458	 *
3459	 * (1) UB_STATE_OPEN is not set, which means the dying process is exited
3460	 *     and related io_uring ctx is freed so file struct of /dev/ublkcX is
3461	 *     released.
3462	 *
3463	 * and one of the following holds
3464	 *
3465	 * (2) UBLK_S_DEV_QUIESCED is set, which means the quiesce_work:
3466	 *     (a)has quiesced request queue
3467	 *     (b)has requeued every inflight rqs whose io_flags is ACTIVE
3468	 *     (c)has requeued/aborted every inflight rqs whose io_flags is NOT ACTIVE
3469	 *     (d)has completed/camceled all ioucmds owned by ther dying process
3470	 *
3471	 * (3) UBLK_S_DEV_FAIL_IO is set, which means the queue is not
3472	 *     quiesced, but all I/O is being immediately errored
3473	 */
3474	if (test_bit(UB_STATE_OPEN, &ub->state) || !ublk_dev_in_recoverable_state(ub)) {
3475		ret = -EBUSY;
3476		goto out_unlock;
3477	}
3478	pr_devel("%s: start recovery for dev id %d.\n", __func__, header->dev_id);
3479	init_completion(&ub->completion);
3480	ret = 0;
3481 out_unlock:
3482	mutex_unlock(&ub->mutex);
3483	return ret;
3484}
3485
3486static int ublk_ctrl_end_recovery(struct ublk_device *ub,
3487		const struct ublksrv_ctrl_cmd *header)
3488{
3489	int ublksrv_pid = (int)header->data[0];
3490	int ret = -EINVAL;
3491
3492	pr_devel("%s: Waiting for all FETCH_REQs, dev id %d...\n", __func__,
3493		 header->dev_id);
3494
3495	if (wait_for_completion_interruptible(&ub->completion))
3496		return -EINTR;
3497
3498	pr_devel("%s: All FETCH_REQs received, dev id %d\n", __func__,
3499		 header->dev_id);
3500
3501	if (!ublk_validate_user_pid(ub, ublksrv_pid))
3502		return -EINVAL;
3503
3504	mutex_lock(&ub->mutex);
3505	if (ublk_nosrv_should_stop_dev(ub))
3506		goto out_unlock;
3507
3508	if (!ublk_dev_in_recoverable_state(ub)) {
3509		ret = -EBUSY;
3510		goto out_unlock;
3511	}
3512	ub->dev_info.ublksrv_pid = ub->ublksrv_tgid;
3513	ub->dev_info.state = UBLK_S_DEV_LIVE;
3514	pr_devel("%s: new ublksrv_pid %d, dev id %d\n",
3515			__func__, ublksrv_pid, header->dev_id);
3516	blk_mq_kick_requeue_list(ub->ub_disk->queue);
3517	ret = 0;
3518 out_unlock:
3519	mutex_unlock(&ub->mutex);
3520	return ret;
3521}
3522
3523static int ublk_ctrl_get_features(const struct ublksrv_ctrl_cmd *header)
3524{
3525	void __user *argp = (void __user *)(unsigned long)header->addr;
3526	u64 features = UBLK_F_ALL;
3527
3528	if (header->len != UBLK_FEATURES_LEN || !header->addr)
3529		return -EINVAL;
3530
3531	if (copy_to_user(argp, &features, UBLK_FEATURES_LEN))
3532		return -EFAULT;
3533
3534	return 0;
3535}
3536
3537static int ublk_ctrl_set_size(struct ublk_device *ub, const struct ublksrv_ctrl_cmd *header)
3538{
3539	struct ublk_param_basic *p = &ub->params.basic;
3540	u64 new_size = header->data[0];
3541	int ret = 0;
3542
3543	mutex_lock(&ub->mutex);
3544	if (!ub->ub_disk) {
3545		ret = -ENODEV;
3546		goto out;
3547	}
3548	p->dev_sectors = new_size;
3549	set_capacity_and_notify(ub->ub_disk, p->dev_sectors);
3550out:
3551	mutex_unlock(&ub->mutex);
3552	return ret;
3553}
3554
3555struct count_busy {
3556	const struct ublk_queue *ubq;
3557	unsigned int nr_busy;
3558};
3559
3560static bool ublk_count_busy_req(struct request *rq, void *data)
3561{
3562	struct count_busy *idle = data;
3563
3564	if (!blk_mq_request_started(rq) && rq->mq_hctx->driver_data == idle->ubq)
3565		idle->nr_busy += 1;
3566	return true;
3567}
3568
3569/* uring_cmd is guaranteed to be active if the associated request is idle */
3570static bool ubq_has_idle_io(const struct ublk_queue *ubq)
3571{
3572	struct count_busy data = {
3573		.ubq = ubq,
3574	};
3575
3576	blk_mq_tagset_busy_iter(&ubq->dev->tag_set, ublk_count_busy_req, &data);
3577	return data.nr_busy < ubq->q_depth;
3578}
3579
3580/* Wait until each hw queue has at least one idle IO */
3581static int ublk_wait_for_idle_io(struct ublk_device *ub,
3582				 unsigned int timeout_ms)
3583{
3584	unsigned int elapsed = 0;
3585	int ret;
3586
3587	while (elapsed < timeout_ms && !signal_pending(current)) {
3588		unsigned int queues_cancelable = 0;
3589		int i;
3590
3591		for (i = 0; i < ub->dev_info.nr_hw_queues; i++) {
3592			struct ublk_queue *ubq = ublk_get_queue(ub, i);
3593
3594			queues_cancelable += !!ubq_has_idle_io(ubq);
3595		}
3596
3597		/*
3598		 * Each queue needs at least one active command for
3599		 * notifying ublk server
3600		 */
3601		if (queues_cancelable == ub->dev_info.nr_hw_queues)
3602			break;
3603
3604		msleep(UBLK_REQUEUE_DELAY_MS);
3605		elapsed += UBLK_REQUEUE_DELAY_MS;
3606	}
3607
3608	if (signal_pending(current))
3609		ret = -EINTR;
3610	else if (elapsed >= timeout_ms)
3611		ret = -EBUSY;
3612	else
3613		ret = 0;
3614
3615	return ret;
3616}
3617
3618static int ublk_ctrl_quiesce_dev(struct ublk_device *ub,
3619				 const struct ublksrv_ctrl_cmd *header)
3620{
3621	/* zero means wait forever */
3622	u64 timeout_ms = header->data[0];
3623	struct gendisk *disk;
3624	int ret = -ENODEV;
3625
3626	if (!(ub->dev_info.flags & UBLK_F_QUIESCE))
3627		return -EOPNOTSUPP;
3628
3629	mutex_lock(&ub->mutex);
3630	disk = ublk_get_disk(ub);
3631	if (!disk)
3632		goto unlock;
3633	if (ub->dev_info.state == UBLK_S_DEV_DEAD)
3634		goto put_disk;
3635
3636	ret = 0;
3637	/* already in expected state */
3638	if (ub->dev_info.state != UBLK_S_DEV_LIVE)
3639		goto put_disk;
3640
3641	/* Mark the device as canceling */
3642	mutex_lock(&ub->cancel_mutex);
3643	blk_mq_quiesce_queue(disk->queue);
3644	ublk_set_canceling(ub, true);
3645	blk_mq_unquiesce_queue(disk->queue);
3646	mutex_unlock(&ub->cancel_mutex);
3647
3648	if (!timeout_ms)
3649		timeout_ms = UINT_MAX;
3650	ret = ublk_wait_for_idle_io(ub, timeout_ms);
3651
3652put_disk:
3653	ublk_put_disk(disk);
3654unlock:
3655	mutex_unlock(&ub->mutex);
3656
3657	/* Cancel pending uring_cmd */
3658	if (!ret)
3659		ublk_cancel_dev(ub);
3660	return ret;
3661}
3662
3663/*
3664 * All control commands are sent via /dev/ublk-control, so we have to check
3665 * the destination device's permission
3666 */
3667static int ublk_char_dev_permission(struct ublk_device *ub,
3668		const char *dev_path, int mask)
3669{
3670	int err;
3671	struct path path;
3672	struct kstat stat;
3673
3674	err = kern_path(dev_path, LOOKUP_FOLLOW, &path);
3675	if (err)
3676		return err;
3677
3678	err = vfs_getattr(&path, &stat, STATX_TYPE, AT_STATX_SYNC_AS_STAT);
3679	if (err)
3680		goto exit;
3681
3682	err = -EPERM;
3683	if (stat.rdev != ub->cdev_dev.devt || !S_ISCHR(stat.mode))
3684		goto exit;
3685
3686	err = inode_permission(&nop_mnt_idmap,
3687			d_backing_inode(path.dentry), mask);
3688exit:
3689	path_put(&path);
3690	return err;
3691}
3692
3693static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub,
3694		u32 cmd_op, struct ublksrv_ctrl_cmd *header)
3695{
3696	bool unprivileged = ub->dev_info.flags & UBLK_F_UNPRIVILEGED_DEV;
3697	void __user *argp = (void __user *)(unsigned long)header->addr;
3698	char *dev_path = NULL;
3699	int ret = 0;
3700	int mask;
3701
3702	if (!unprivileged) {
3703		if (!capable(CAP_SYS_ADMIN))
3704			return -EPERM;
3705		/*
3706		 * The new added command of UBLK_CMD_GET_DEV_INFO2 includes
3707		 * char_dev_path in payload too, since userspace may not
3708		 * know if the specified device is created as unprivileged
3709		 * mode.
3710		 */
3711		if (_IOC_NR(cmd_op) != UBLK_CMD_GET_DEV_INFO2)
3712			return 0;
3713	}
3714
3715	/*
3716	 * User has to provide the char device path for unprivileged ublk
3717	 *
3718	 * header->addr always points to the dev path buffer, and
3719	 * header->dev_path_len records length of dev path buffer.
3720	 */
3721	if (!header->dev_path_len || header->dev_path_len > PATH_MAX)
3722		return -EINVAL;
3723
3724	if (header->len < header->dev_path_len)
3725		return -EINVAL;
3726
3727	dev_path = memdup_user_nul(argp, header->dev_path_len);
3728	if (IS_ERR(dev_path))
3729		return PTR_ERR(dev_path);
3730
3731	ret = -EINVAL;
3732	switch (_IOC_NR(cmd_op)) {
3733	case UBLK_CMD_GET_DEV_INFO:
3734	case UBLK_CMD_GET_DEV_INFO2:
3735	case UBLK_CMD_GET_QUEUE_AFFINITY:
3736	case UBLK_CMD_GET_PARAMS:
3737	case (_IOC_NR(UBLK_U_CMD_GET_FEATURES)):
3738		mask = MAY_READ;
3739		break;
3740	case UBLK_CMD_START_DEV:
3741	case UBLK_CMD_STOP_DEV:
3742	case UBLK_CMD_ADD_DEV:
3743	case UBLK_CMD_DEL_DEV:
3744	case UBLK_CMD_SET_PARAMS:
3745	case UBLK_CMD_START_USER_RECOVERY:
3746	case UBLK_CMD_END_USER_RECOVERY:
3747	case UBLK_CMD_UPDATE_SIZE:
3748	case UBLK_CMD_QUIESCE_DEV:
3749		mask = MAY_READ | MAY_WRITE;
3750		break;
3751	default:
3752		goto exit;
3753	}
3754
3755	ret = ublk_char_dev_permission(ub, dev_path, mask);
3756	if (!ret) {
3757		header->len -= header->dev_path_len;
3758		header->addr += header->dev_path_len;
3759	}
3760	pr_devel("%s: dev id %d cmd_op %x uid %d gid %d path %s ret %d\n",
3761			__func__, ub->ub_number, cmd_op,
3762			ub->dev_info.owner_uid, ub->dev_info.owner_gid,
3763			dev_path, ret);
3764exit:
3765	kfree(dev_path);
3766	return ret;
3767}
3768
3769static bool ublk_ctrl_uring_cmd_may_sleep(u32 cmd_op)
3770{
3771	switch (_IOC_NR(cmd_op)) {
3772	case UBLK_CMD_GET_QUEUE_AFFINITY:
3773	case UBLK_CMD_GET_DEV_INFO:
3774	case UBLK_CMD_GET_DEV_INFO2:
3775	case _IOC_NR(UBLK_U_CMD_GET_FEATURES):
3776		return false;
3777	default:
3778		return true;
3779	}
3780}
3781
3782static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd,
3783		unsigned int issue_flags)
3784{
3785	/* May point to userspace-mapped memory */
3786	const struct ublksrv_ctrl_cmd *ub_src = io_uring_sqe_cmd(cmd->sqe);
3787	struct ublksrv_ctrl_cmd header;
3788	struct ublk_device *ub = NULL;
3789	u32 cmd_op = cmd->cmd_op;
3790	int ret = -EINVAL;
3791
3792	if (ublk_ctrl_uring_cmd_may_sleep(cmd_op) &&
3793	    issue_flags & IO_URING_F_NONBLOCK)
3794		return -EAGAIN;
3795
3796	if (!(issue_flags & IO_URING_F_SQE128))
3797		return -EINVAL;
3798
3799	header.dev_id = READ_ONCE(ub_src->dev_id);
3800	header.queue_id = READ_ONCE(ub_src->queue_id);
3801	header.len = READ_ONCE(ub_src->len);
3802	header.addr = READ_ONCE(ub_src->addr);
3803	header.data[0] = READ_ONCE(ub_src->data[0]);
3804	header.dev_path_len = READ_ONCE(ub_src->dev_path_len);
3805	ublk_ctrl_cmd_dump(cmd_op, &header);
3806
3807	ret = ublk_check_cmd_op(cmd_op);
3808	if (ret)
3809		goto out;
3810
3811	if (cmd_op == UBLK_U_CMD_GET_FEATURES) {
3812		ret = ublk_ctrl_get_features(&header);
3813		goto out;
3814	}
3815
3816	if (_IOC_NR(cmd_op) != UBLK_CMD_ADD_DEV) {
3817		ret = -ENODEV;
3818		ub = ublk_get_device_from_id(header.dev_id);
3819		if (!ub)
3820			goto out;
3821
3822		ret = ublk_ctrl_uring_cmd_permission(ub, cmd_op, &header);
3823		if (ret)
3824			goto put_dev;
3825	}
3826
3827	switch (_IOC_NR(cmd_op)) {
3828	case UBLK_CMD_START_DEV:
3829		ret = ublk_ctrl_start_dev(ub, &header);
3830		break;
3831	case UBLK_CMD_STOP_DEV:
3832		ret = ublk_ctrl_stop_dev(ub);
3833		break;
3834	case UBLK_CMD_GET_DEV_INFO:
3835	case UBLK_CMD_GET_DEV_INFO2:
3836		ret = ublk_ctrl_get_dev_info(ub, &header);
3837		break;
3838	case UBLK_CMD_ADD_DEV:
3839		ret = ublk_ctrl_add_dev(&header);
3840		break;
3841	case UBLK_CMD_DEL_DEV:
3842		ret = ublk_ctrl_del_dev(&ub, true);
3843		break;
3844	case UBLK_CMD_DEL_DEV_ASYNC:
3845		ret = ublk_ctrl_del_dev(&ub, false);
3846		break;
3847	case UBLK_CMD_GET_QUEUE_AFFINITY:
3848		ret = ublk_ctrl_get_queue_affinity(ub, &header);
3849		break;
3850	case UBLK_CMD_GET_PARAMS:
3851		ret = ublk_ctrl_get_params(ub, &header);
3852		break;
3853	case UBLK_CMD_SET_PARAMS:
3854		ret = ublk_ctrl_set_params(ub, &header);
3855		break;
3856	case UBLK_CMD_START_USER_RECOVERY:
3857		ret = ublk_ctrl_start_recovery(ub, &header);
3858		break;
3859	case UBLK_CMD_END_USER_RECOVERY:
3860		ret = ublk_ctrl_end_recovery(ub, &header);
3861		break;
3862	case UBLK_CMD_UPDATE_SIZE:
3863		ret = ublk_ctrl_set_size(ub, &header);
3864		break;
3865	case UBLK_CMD_QUIESCE_DEV:
3866		ret = ublk_ctrl_quiesce_dev(ub, &header);
3867		break;
3868	default:
3869		ret = -EOPNOTSUPP;
3870		break;
3871	}
3872
3873 put_dev:
3874	if (ub)
3875		ublk_put_device(ub);
3876 out:
3877	pr_devel("%s: cmd done ret %d cmd_op %x, dev id %d qid %d\n",
3878			__func__, ret, cmd_op, header.dev_id, header.queue_id);
3879	return ret;
3880}
3881
3882static const struct file_operations ublk_ctl_fops = {
3883	.open		= nonseekable_open,
3884	.uring_cmd      = ublk_ctrl_uring_cmd,
3885	.owner		= THIS_MODULE,
3886	.llseek		= noop_llseek,
3887};
3888
3889static struct miscdevice ublk_misc = {
3890	.minor		= MISC_DYNAMIC_MINOR,
3891	.name		= "ublk-control",
3892	.fops		= &ublk_ctl_fops,
3893};
3894
3895static int __init ublk_init(void)
3896{
3897	int ret;
3898
3899	BUILD_BUG_ON((u64)UBLKSRV_IO_BUF_OFFSET +
3900			UBLKSRV_IO_BUF_TOTAL_SIZE < UBLKSRV_IO_BUF_OFFSET);
3901	BUILD_BUG_ON(sizeof(struct ublk_auto_buf_reg) != 8);
3902
3903	init_waitqueue_head(&ublk_idr_wq);
3904
3905	ret = misc_register(&ublk_misc);
3906	if (ret)
3907		return ret;
3908
3909	ret = alloc_chrdev_region(&ublk_chr_devt, 0, UBLK_MINORS, "ublk-char");
3910	if (ret)
3911		goto unregister_mis;
3912
3913	ret = class_register(&ublk_chr_class);
3914	if (ret)
3915		goto free_chrdev_region;
3916
3917	return 0;
3918
3919free_chrdev_region:
3920	unregister_chrdev_region(ublk_chr_devt, UBLK_MINORS);
3921unregister_mis:
3922	misc_deregister(&ublk_misc);
3923	return ret;
3924}
3925
3926static void __exit ublk_exit(void)
3927{
3928	struct ublk_device *ub;
3929	int id;
3930
3931	idr_for_each_entry(&ublk_index_idr, ub, id)
3932		ublk_remove(ub);
3933
3934	class_unregister(&ublk_chr_class);
3935	misc_deregister(&ublk_misc);
3936
3937	idr_destroy(&ublk_index_idr);
3938	unregister_chrdev_region(ublk_chr_devt, UBLK_MINORS);
3939}
3940
3941module_init(ublk_init);
3942module_exit(ublk_exit);
3943
3944static int ublk_set_max_unprivileged_ublks(const char *buf,
3945					   const struct kernel_param *kp)
3946{
3947	return param_set_uint_minmax(buf, kp, 0, UBLK_MAX_UBLKS);
3948}
3949
3950static int ublk_get_max_unprivileged_ublks(char *buf,
3951					   const struct kernel_param *kp)
3952{
3953	return sysfs_emit(buf, "%u\n", unprivileged_ublks_max);
3954}
3955
3956static const struct kernel_param_ops ublk_max_unprivileged_ublks_ops = {
3957	.set = ublk_set_max_unprivileged_ublks,
3958	.get = ublk_get_max_unprivileged_ublks,
3959};
3960
3961module_param_cb(ublks_max, &ublk_max_unprivileged_ublks_ops,
3962		&unprivileged_ublks_max, 0644);
3963MODULE_PARM_DESC(ublks_max, "max number of unprivileged ublk devices allowed to add(default: 64)");
3964
3965MODULE_AUTHOR("Ming Lei <ming.lei@redhat.com>");
3966MODULE_DESCRIPTION("Userspace block device");
3967MODULE_LICENSE("GPL");