pci.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * NVM Express device driver
   4 * Copyright (c) 2011-2014, Intel Corporation.
   5 */
   6
   7#include <linux/acpi.h>
   8#include <linux/async.h>
   9#include <linux/blkdev.h>
  10#include <linux/blk-mq-dma.h>
  11#include <linux/blk-integrity.h>
  12#include <linux/dmi.h>
  13#include <linux/init.h>
  14#include <linux/interrupt.h>
  15#include <linux/io.h>
  16#include <linux/kstrtox.h>
  17#include <linux/memremap.h>
  18#include <linux/mm.h>
  19#include <linux/module.h>
  20#include <linux/mutex.h>
  21#include <linux/nodemask.h>
  22#include <linux/once.h>
  23#include <linux/pci.h>
  24#include <linux/suspend.h>
  25#include <linux/t10-pi.h>
  26#include <linux/types.h>
  27#include <linux/io-64-nonatomic-lo-hi.h>
  28#include <linux/io-64-nonatomic-hi-lo.h>
  29#include <linux/sed-opal.h>
  30
  31#include "trace.h"
  32#include "nvme.h"
  33
  34#define SQ_SIZE(q)	((q)->q_depth << (q)->sqes)
  35#define CQ_SIZE(q)	((q)->q_depth * sizeof(struct nvme_completion))
  36
  37/* Optimisation for I/Os between 4k and 128k */
  38#define NVME_SMALL_POOL_SIZE	256
  39
  40/*
  41 * Arbitrary upper bound.
  42 */
  43#define NVME_MAX_BYTES		SZ_8M
  44#define NVME_MAX_NR_DESCRIPTORS	5
  45
  46/*
  47 * For data SGLs we support a single descriptors worth of SGL entries.
  48 * For PRPs, segments don't matter at all.
  49 */
  50#define NVME_MAX_SEGS \
  51	(NVME_CTRL_PAGE_SIZE / sizeof(struct nvme_sgl_desc))
  52
  53/*
  54 * For metadata SGLs, only the small descriptor is supported, and the first
  55 * entry is the segment descriptor, which for the data pointer sits in the SQE.
  56 */
  57#define NVME_MAX_META_SEGS \
  58	((NVME_SMALL_POOL_SIZE / sizeof(struct nvme_sgl_desc)) - 1)
  59
  60/*
  61 * The last entry is used to link to the next descriptor.
  62 */
  63#define PRPS_PER_PAGE \
  64	(((NVME_CTRL_PAGE_SIZE / sizeof(__le64))) - 1)
  65
  66/*
  67 * I/O could be non-aligned both at the beginning and end.
  68 */
  69#define MAX_PRP_RANGE \
  70	(NVME_MAX_BYTES + 2 * (NVME_CTRL_PAGE_SIZE - 1))
  71
  72static_assert(MAX_PRP_RANGE / NVME_CTRL_PAGE_SIZE <=
  73	(1 /* prp1 */ + NVME_MAX_NR_DESCRIPTORS * PRPS_PER_PAGE));
  74
  75static int use_threaded_interrupts;
  76module_param(use_threaded_interrupts, int, 0444);
  77
  78static bool use_cmb_sqes = true;
  79module_param(use_cmb_sqes, bool, 0444);
  80MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes");
  81
  82static unsigned int max_host_mem_size_mb = 128;
  83module_param(max_host_mem_size_mb, uint, 0444);
  84MODULE_PARM_DESC(max_host_mem_size_mb,
  85	"Maximum Host Memory Buffer (HMB) size per controller (in MiB)");
  86
  87static unsigned int sgl_threshold = SZ_32K;
  88module_param(sgl_threshold, uint, 0644);
  89MODULE_PARM_DESC(sgl_threshold,
  90		"Use SGLs when average request segment size is larger or equal to "
  91		"this size. Use 0 to disable SGLs.");
  92
  93#define NVME_PCI_MIN_QUEUE_SIZE 2
  94#define NVME_PCI_MAX_QUEUE_SIZE 4095
  95static int io_queue_depth_set(const char *val, const struct kernel_param *kp);
  96static const struct kernel_param_ops io_queue_depth_ops = {
  97	.set = io_queue_depth_set,
  98	.get = param_get_uint,
  99};
 100
 101static unsigned int io_queue_depth = 1024;
 102module_param_cb(io_queue_depth, &io_queue_depth_ops, &io_queue_depth, 0644);
 103MODULE_PARM_DESC(io_queue_depth, "set io queue depth, should >= 2 and < 4096");
 104
 105static int io_queue_count_set(const char *val, const struct kernel_param *kp)
 106{
 107	unsigned int n;
 108	int ret;
 109
 110	ret = kstrtouint(val, 10, &n);
 111	if (ret != 0 || n > blk_mq_num_possible_queues(0))
 112		return -EINVAL;
 113	return param_set_uint(val, kp);
 114}
 115
 116static const struct kernel_param_ops io_queue_count_ops = {
 117	.set = io_queue_count_set,
 118	.get = param_get_uint,
 119};
 120
 121static unsigned int write_queues;
 122module_param_cb(write_queues, &io_queue_count_ops, &write_queues, 0644);
 123MODULE_PARM_DESC(write_queues,
 124	"Number of queues to use for writes. If not set, reads and writes "
 125	"will share a queue set.");
 126
 127static unsigned int poll_queues;
 128module_param_cb(poll_queues, &io_queue_count_ops, &poll_queues, 0644);
 129MODULE_PARM_DESC(poll_queues, "Number of queues to use for polled IO.");
 130
 131static bool noacpi;
 132module_param(noacpi, bool, 0444);
 133MODULE_PARM_DESC(noacpi, "disable acpi bios quirks");
 134
 135struct nvme_dev;
 136struct nvme_queue;
 137
 138static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown);
 139static void nvme_delete_io_queues(struct nvme_dev *dev);
 140static void nvme_update_attrs(struct nvme_dev *dev);
 141
 142struct nvme_descriptor_pools {
 143	struct dma_pool *large;
 144	struct dma_pool *small;
 145};
 146
 147/*
 148 * Represents an NVM Express device.  Each nvme_dev is a PCI function.
 149 */
 150struct nvme_dev {
 151	struct nvme_queue *queues;
 152	struct blk_mq_tag_set tagset;
 153	struct blk_mq_tag_set admin_tagset;
 154	u32 __iomem *dbs;
 155	struct device *dev;
 156	unsigned online_queues;
 157	unsigned max_qid;
 158	unsigned io_queues[HCTX_MAX_TYPES];
 159	unsigned int num_vecs;
 160	u32 q_depth;
 161	int io_sqes;
 162	u32 db_stride;
 163	void __iomem *bar;
 164	unsigned long bar_mapped_size;
 165	struct mutex shutdown_lock;
 166	bool subsystem;
 167	u64 cmb_size;
 168	bool cmb_use_sqes;
 169	u32 cmbsz;
 170	u32 cmbloc;
 171	struct nvme_ctrl ctrl;
 172	u32 last_ps;
 173	bool hmb;
 174	struct sg_table *hmb_sgt;
 175	mempool_t *dmavec_mempool;
 176
 177	/* shadow doorbell buffer support: */
 178	__le32 *dbbuf_dbs;
 179	dma_addr_t dbbuf_dbs_dma_addr;
 180	__le32 *dbbuf_eis;
 181	dma_addr_t dbbuf_eis_dma_addr;
 182
 183	/* host memory buffer support: */
 184	u64 host_mem_size;
 185	u32 nr_host_mem_descs;
 186	u32 host_mem_descs_size;
 187	dma_addr_t host_mem_descs_dma;
 188	struct nvme_host_mem_buf_desc *host_mem_descs;
 189	void **host_mem_desc_bufs;
 190	unsigned int nr_allocated_queues;
 191	unsigned int nr_write_queues;
 192	unsigned int nr_poll_queues;
 193	struct nvme_descriptor_pools descriptor_pools[];
 194};
 195
 196static int io_queue_depth_set(const char *val, const struct kernel_param *kp)
 197{
 198	return param_set_uint_minmax(val, kp, NVME_PCI_MIN_QUEUE_SIZE,
 199			NVME_PCI_MAX_QUEUE_SIZE);
 200}
 201
 202static inline unsigned int sq_idx(unsigned int qid, u32 stride)
 203{
 204	return qid * 2 * stride;
 205}
 206
 207static inline unsigned int cq_idx(unsigned int qid, u32 stride)
 208{
 209	return (qid * 2 + 1) * stride;
 210}
 211
 212static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl)
 213{
 214	return container_of(ctrl, struct nvme_dev, ctrl);
 215}
 216
 217/*
 218 * An NVM Express queue.  Each device has at least two (one for admin
 219 * commands and one for I/O commands).
 220 */
 221struct nvme_queue {
 222	struct nvme_dev *dev;
 223	struct nvme_descriptor_pools descriptor_pools;
 224	spinlock_t sq_lock;
 225	void *sq_cmds;
 226	 /* only used for poll queues: */
 227	spinlock_t cq_poll_lock ____cacheline_aligned_in_smp;
 228	struct nvme_completion *cqes;
 229	dma_addr_t sq_dma_addr;
 230	dma_addr_t cq_dma_addr;
 231	u32 __iomem *q_db;
 232	u32 q_depth;
 233	u16 cq_vector;
 234	u16 sq_tail;
 235	u16 last_sq_tail;
 236	u16 cq_head;
 237	u16 qid;
 238	u8 cq_phase;
 239	u8 sqes;
 240	unsigned long flags;
 241#define NVMEQ_ENABLED		0
 242#define NVMEQ_SQ_CMB		1
 243#define NVMEQ_DELETE_ERROR	2
 244#define NVMEQ_POLLED		3
 245	__le32 *dbbuf_sq_db;
 246	__le32 *dbbuf_cq_db;
 247	__le32 *dbbuf_sq_ei;
 248	__le32 *dbbuf_cq_ei;
 249	struct completion delete_done;
 250};
 251
 252/* bits for iod->flags */
 253enum nvme_iod_flags {
 254	/* this command has been aborted by the timeout handler */
 255	IOD_ABORTED		= 1U << 0,
 256
 257	/* uses the small descriptor pool */
 258	IOD_SMALL_DESCRIPTOR	= 1U << 1,
 259
 260	/* single segment dma mapping */
 261	IOD_SINGLE_SEGMENT	= 1U << 2,
 262
 263	/* Data payload contains p2p memory */
 264	IOD_DATA_P2P		= 1U << 3,
 265
 266	/* Metadata contains p2p memory */
 267	IOD_META_P2P		= 1U << 4,
 268
 269	/* Data payload contains MMIO memory */
 270	IOD_DATA_MMIO		= 1U << 5,
 271
 272	/* Metadata contains MMIO memory */
 273	IOD_META_MMIO		= 1U << 6,
 274
 275	/* Metadata using non-coalesced MPTR */
 276	IOD_SINGLE_META_SEGMENT	= 1U << 7,
 277};
 278
 279struct nvme_dma_vec {
 280	dma_addr_t addr;
 281	unsigned int len;
 282};
 283
 284/*
 285 * The nvme_iod describes the data in an I/O.
 286 */
 287struct nvme_iod {
 288	struct nvme_request req;
 289	struct nvme_command cmd;
 290	u8 flags;
 291	u8 nr_descriptors;
 292
 293	unsigned int total_len;
 294	struct dma_iova_state dma_state;
 295	void *descriptors[NVME_MAX_NR_DESCRIPTORS];
 296	struct nvme_dma_vec *dma_vecs;
 297	unsigned int nr_dma_vecs;
 298
 299	dma_addr_t meta_dma;
 300	unsigned int meta_total_len;
 301	struct dma_iova_state meta_dma_state;
 302	struct nvme_sgl_desc *meta_descriptor;
 303};
 304
 305static inline unsigned int nvme_dbbuf_size(struct nvme_dev *dev)
 306{
 307	return dev->nr_allocated_queues * 8 * dev->db_stride;
 308}
 309
 310static void nvme_dbbuf_dma_alloc(struct nvme_dev *dev)
 311{
 312	unsigned int mem_size = nvme_dbbuf_size(dev);
 313
 314	if (!(dev->ctrl.oacs & NVME_CTRL_OACS_DBBUF_SUPP))
 315		return;
 316
 317	if (dev->dbbuf_dbs) {
 318		/*
 319		 * Clear the dbbuf memory so the driver doesn't observe stale
 320		 * values from the previous instantiation.
 321		 */
 322		memset(dev->dbbuf_dbs, 0, mem_size);
 323		memset(dev->dbbuf_eis, 0, mem_size);
 324		return;
 325	}
 326
 327	dev->dbbuf_dbs = dma_alloc_coherent(dev->dev, mem_size,
 328					    &dev->dbbuf_dbs_dma_addr,
 329					    GFP_KERNEL);
 330	if (!dev->dbbuf_dbs)
 331		goto fail;
 332	dev->dbbuf_eis = dma_alloc_coherent(dev->dev, mem_size,
 333					    &dev->dbbuf_eis_dma_addr,
 334					    GFP_KERNEL);
 335	if (!dev->dbbuf_eis)
 336		goto fail_free_dbbuf_dbs;
 337	return;
 338
 339fail_free_dbbuf_dbs:
 340	dma_free_coherent(dev->dev, mem_size, dev->dbbuf_dbs,
 341			  dev->dbbuf_dbs_dma_addr);
 342	dev->dbbuf_dbs = NULL;
 343fail:
 344	dev_warn(dev->dev, "unable to allocate dma for dbbuf\n");
 345}
 346
 347static void nvme_dbbuf_dma_free(struct nvme_dev *dev)
 348{
 349	unsigned int mem_size = nvme_dbbuf_size(dev);
 350
 351	if (dev->dbbuf_dbs) {
 352		dma_free_coherent(dev->dev, mem_size,
 353				  dev->dbbuf_dbs, dev->dbbuf_dbs_dma_addr);
 354		dev->dbbuf_dbs = NULL;
 355	}
 356	if (dev->dbbuf_eis) {
 357		dma_free_coherent(dev->dev, mem_size,
 358				  dev->dbbuf_eis, dev->dbbuf_eis_dma_addr);
 359		dev->dbbuf_eis = NULL;
 360	}
 361}
 362
 363static void nvme_dbbuf_init(struct nvme_dev *dev,
 364			    struct nvme_queue *nvmeq, int qid)
 365{
 366	if (!dev->dbbuf_dbs || !qid)
 367		return;
 368
 369	nvmeq->dbbuf_sq_db = &dev->dbbuf_dbs[sq_idx(qid, dev->db_stride)];
 370	nvmeq->dbbuf_cq_db = &dev->dbbuf_dbs[cq_idx(qid, dev->db_stride)];
 371	nvmeq->dbbuf_sq_ei = &dev->dbbuf_eis[sq_idx(qid, dev->db_stride)];
 372	nvmeq->dbbuf_cq_ei = &dev->dbbuf_eis[cq_idx(qid, dev->db_stride)];
 373}
 374
 375static void nvme_dbbuf_free(struct nvme_queue *nvmeq)
 376{
 377	if (!nvmeq->qid)
 378		return;
 379
 380	nvmeq->dbbuf_sq_db = NULL;
 381	nvmeq->dbbuf_cq_db = NULL;
 382	nvmeq->dbbuf_sq_ei = NULL;
 383	nvmeq->dbbuf_cq_ei = NULL;
 384}
 385
 386static void nvme_dbbuf_set(struct nvme_dev *dev)
 387{
 388	struct nvme_command c = { };
 389	unsigned int i;
 390
 391	if (!dev->dbbuf_dbs)
 392		return;
 393
 394	c.dbbuf.opcode = nvme_admin_dbbuf;
 395	c.dbbuf.prp1 = cpu_to_le64(dev->dbbuf_dbs_dma_addr);
 396	c.dbbuf.prp2 = cpu_to_le64(dev->dbbuf_eis_dma_addr);
 397
 398	if (nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0)) {
 399		dev_warn(dev->ctrl.device, "unable to set dbbuf\n");
 400		/* Free memory and continue on */
 401		nvme_dbbuf_dma_free(dev);
 402
 403		for (i = 1; i < dev->online_queues; i++)
 404			nvme_dbbuf_free(&dev->queues[i]);
 405	}
 406}
 407
 408static inline int nvme_dbbuf_need_event(u16 event_idx, u16 new_idx, u16 old)
 409{
 410	return (u16)(new_idx - event_idx - 1) < (u16)(new_idx - old);
 411}
 412
 413/* Update dbbuf and return true if an MMIO is required */
 414static bool nvme_dbbuf_update_and_check_event(u16 value, __le32 *dbbuf_db,
 415					      volatile __le32 *dbbuf_ei)
 416{
 417	if (dbbuf_db) {
 418		u16 old_value, event_idx;
 419
 420		/*
 421		 * Ensure that the queue is written before updating
 422		 * the doorbell in memory
 423		 */
 424		wmb();
 425
 426		old_value = le32_to_cpu(*dbbuf_db);
 427		*dbbuf_db = cpu_to_le32(value);
 428
 429		/*
 430		 * Ensure that the doorbell is updated before reading the event
 431		 * index from memory.  The controller needs to provide similar
 432		 * ordering to ensure the event index is updated before reading
 433		 * the doorbell.
 434		 */
 435		mb();
 436
 437		event_idx = le32_to_cpu(*dbbuf_ei);
 438		if (!nvme_dbbuf_need_event(event_idx, value, old_value))
 439			return false;
 440	}
 441
 442	return true;
 443}
 444
 445static struct nvme_descriptor_pools *
 446nvme_setup_descriptor_pools(struct nvme_dev *dev, unsigned numa_node)
 447{
 448	struct nvme_descriptor_pools *pools = &dev->descriptor_pools[numa_node];
 449	size_t small_align = NVME_SMALL_POOL_SIZE;
 450
 451	if (pools->small)
 452		return pools; /* already initialized */
 453
 454	pools->large = dma_pool_create_node("nvme descriptor page", dev->dev,
 455			NVME_CTRL_PAGE_SIZE, NVME_CTRL_PAGE_SIZE, 0, numa_node);
 456	if (!pools->large)
 457		return ERR_PTR(-ENOMEM);
 458
 459	if (dev->ctrl.quirks & NVME_QUIRK_DMAPOOL_ALIGN_512)
 460		small_align = 512;
 461
 462	pools->small = dma_pool_create_node("nvme descriptor small", dev->dev,
 463			NVME_SMALL_POOL_SIZE, small_align, 0, numa_node);
 464	if (!pools->small) {
 465		dma_pool_destroy(pools->large);
 466		pools->large = NULL;
 467		return ERR_PTR(-ENOMEM);
 468	}
 469
 470	return pools;
 471}
 472
 473static void nvme_release_descriptor_pools(struct nvme_dev *dev)
 474{
 475	unsigned i;
 476
 477	for (i = 0; i < nr_node_ids; i++) {
 478		struct nvme_descriptor_pools *pools = &dev->descriptor_pools[i];
 479
 480		dma_pool_destroy(pools->large);
 481		dma_pool_destroy(pools->small);
 482	}
 483}
 484
 485static int nvme_init_hctx_common(struct blk_mq_hw_ctx *hctx, void *data,
 486		unsigned qid)
 487{
 488	struct nvme_dev *dev = to_nvme_dev(data);
 489	struct nvme_queue *nvmeq = &dev->queues[qid];
 490	struct nvme_descriptor_pools *pools;
 491	struct blk_mq_tags *tags;
 492
 493	tags = qid ? dev->tagset.tags[qid - 1] : dev->admin_tagset.tags[0];
 494	WARN_ON(tags != hctx->tags);
 495	pools = nvme_setup_descriptor_pools(dev, hctx->numa_node);
 496	if (IS_ERR(pools))
 497		return PTR_ERR(pools);
 498
 499	nvmeq->descriptor_pools = *pools;
 500	hctx->driver_data = nvmeq;
 501	return 0;
 502}
 503
 504static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 505				unsigned int hctx_idx)
 506{
 507	WARN_ON(hctx_idx != 0);
 508	return nvme_init_hctx_common(hctx, data, 0);
 509}
 510
 511static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 512			     unsigned int hctx_idx)
 513{
 514	return nvme_init_hctx_common(hctx, data, hctx_idx + 1);
 515}
 516
 517static int nvme_pci_init_request(struct blk_mq_tag_set *set,
 518		struct request *req, unsigned int hctx_idx,
 519		unsigned int numa_node)
 520{
 521	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 522
 523	nvme_req(req)->ctrl = set->driver_data;
 524	nvme_req(req)->cmd = &iod->cmd;
 525	return 0;
 526}
 527
 528static int queue_irq_offset(struct nvme_dev *dev)
 529{
 530	/* if we have more than 1 vec, admin queue offsets us by 1 */
 531	if (dev->num_vecs > 1)
 532		return 1;
 533
 534	return 0;
 535}
 536
 537static void nvme_pci_map_queues(struct blk_mq_tag_set *set)
 538{
 539	struct nvme_dev *dev = to_nvme_dev(set->driver_data);
 540	int i, qoff, offset;
 541
 542	offset = queue_irq_offset(dev);
 543	for (i = 0, qoff = 0; i < set->nr_maps; i++) {
 544		struct blk_mq_queue_map *map = &set->map[i];
 545
 546		map->nr_queues = dev->io_queues[i];
 547		if (!map->nr_queues) {
 548			BUG_ON(i == HCTX_TYPE_DEFAULT);
 549			continue;
 550		}
 551
 552		/*
 553		 * The poll queue(s) doesn't have an IRQ (and hence IRQ
 554		 * affinity), so use the regular blk-mq cpu mapping
 555		 */
 556		map->queue_offset = qoff;
 557		if (i != HCTX_TYPE_POLL && offset)
 558			blk_mq_map_hw_queues(map, dev->dev, offset);
 559		else
 560			blk_mq_map_queues(map);
 561		qoff += map->nr_queues;
 562		offset += map->nr_queues;
 563	}
 564}
 565
 566/*
 567 * Write sq tail if we are asked to, or if the next command would wrap.
 568 */
 569static inline void nvme_write_sq_db(struct nvme_queue *nvmeq, bool write_sq)
 570{
 571	if (!write_sq) {
 572		u16 next_tail = nvmeq->sq_tail + 1;
 573
 574		if (next_tail == nvmeq->q_depth)
 575			next_tail = 0;
 576		if (next_tail != nvmeq->last_sq_tail)
 577			return;
 578	}
 579
 580	if (nvme_dbbuf_update_and_check_event(nvmeq->sq_tail,
 581			nvmeq->dbbuf_sq_db, nvmeq->dbbuf_sq_ei))
 582		writel(nvmeq->sq_tail, nvmeq->q_db);
 583	nvmeq->last_sq_tail = nvmeq->sq_tail;
 584}
 585
 586static inline void nvme_sq_copy_cmd(struct nvme_queue *nvmeq,
 587				    struct nvme_command *cmd)
 588{
 589	memcpy(nvmeq->sq_cmds + (nvmeq->sq_tail << nvmeq->sqes),
 590		absolute_pointer(cmd), sizeof(*cmd));
 591	if (++nvmeq->sq_tail == nvmeq->q_depth)
 592		nvmeq->sq_tail = 0;
 593}
 594
 595static void nvme_commit_rqs(struct blk_mq_hw_ctx *hctx)
 596{
 597	struct nvme_queue *nvmeq = hctx->driver_data;
 598
 599	spin_lock(&nvmeq->sq_lock);
 600	if (nvmeq->sq_tail != nvmeq->last_sq_tail)
 601		nvme_write_sq_db(nvmeq, true);
 602	spin_unlock(&nvmeq->sq_lock);
 603}
 604
 605enum nvme_use_sgl {
 606	SGL_UNSUPPORTED,
 607	SGL_SUPPORTED,
 608	SGL_FORCED,
 609};
 610
 611static inline bool nvme_pci_metadata_use_sgls(struct request *req)
 612{
 613	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
 614	struct nvme_dev *dev = nvmeq->dev;
 615
 616	if (!nvme_ctrl_meta_sgl_supported(&dev->ctrl))
 617		return false;
 618	return req->nr_integrity_segments > 1 ||
 619		nvme_req(req)->flags & NVME_REQ_USERCMD;
 620}
 621
 622static inline enum nvme_use_sgl nvme_pci_use_sgls(struct nvme_dev *dev,
 623		struct request *req)
 624{
 625	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
 626
 627	if (nvmeq->qid && nvme_ctrl_sgl_supported(&dev->ctrl)) {
 628		/*
 629		 * When the controller is capable of using SGL, there are
 630		 * several conditions that we force to use it:
 631		 *
 632		 * 1. A request containing page gaps within the controller's
 633		 *    mask can not use the PRP format.
 634		 *
 635		 * 2. User commands use SGL because that lets the device
 636		 *    validate the requested transfer lengths.
 637		 *
 638		 * 3. Multiple integrity segments must use SGL as that's the
 639		 *    only way to describe such a command in NVMe.
 640		 */
 641		if (req_phys_gap_mask(req) & (NVME_CTRL_PAGE_SIZE - 1) ||
 642		    nvme_req(req)->flags & NVME_REQ_USERCMD ||
 643		    req->nr_integrity_segments > 1)
 644			return SGL_FORCED;
 645		return SGL_SUPPORTED;
 646	}
 647
 648	return SGL_UNSUPPORTED;
 649}
 650
 651static unsigned int nvme_pci_avg_seg_size(struct request *req)
 652{
 653	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 654	unsigned int nseg;
 655
 656	if (blk_rq_dma_map_coalesce(&iod->dma_state))
 657		nseg = 1;
 658	else
 659		nseg = blk_rq_nr_phys_segments(req);
 660	return DIV_ROUND_UP(blk_rq_payload_bytes(req), nseg);
 661}
 662
 663static inline struct dma_pool *nvme_dma_pool(struct nvme_queue *nvmeq,
 664		struct nvme_iod *iod)
 665{
 666	if (iod->flags & IOD_SMALL_DESCRIPTOR)
 667		return nvmeq->descriptor_pools.small;
 668	return nvmeq->descriptor_pools.large;
 669}
 670
 671static inline bool nvme_pci_cmd_use_meta_sgl(struct nvme_command *cmd)
 672{
 673	return (cmd->common.flags & NVME_CMD_SGL_ALL) == NVME_CMD_SGL_METASEG;
 674}
 675
 676static inline bool nvme_pci_cmd_use_sgl(struct nvme_command *cmd)
 677{
 678	return cmd->common.flags &
 679		(NVME_CMD_SGL_METABUF | NVME_CMD_SGL_METASEG);
 680}
 681
 682static inline dma_addr_t nvme_pci_first_desc_dma_addr(struct nvme_command *cmd)
 683{
 684	if (nvme_pci_cmd_use_sgl(cmd))
 685		return le64_to_cpu(cmd->common.dptr.sgl.addr);
 686	return le64_to_cpu(cmd->common.dptr.prp2);
 687}
 688
 689static void nvme_free_descriptors(struct request *req)
 690{
 691	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
 692	const int last_prp = NVME_CTRL_PAGE_SIZE / sizeof(__le64) - 1;
 693	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 694	dma_addr_t dma_addr = nvme_pci_first_desc_dma_addr(&iod->cmd);
 695	int i;
 696
 697	if (iod->nr_descriptors == 1) {
 698		dma_pool_free(nvme_dma_pool(nvmeq, iod), iod->descriptors[0],
 699				dma_addr);
 700		return;
 701	}
 702
 703	for (i = 0; i < iod->nr_descriptors; i++) {
 704		__le64 *prp_list = iod->descriptors[i];
 705		dma_addr_t next_dma_addr = le64_to_cpu(prp_list[last_prp]);
 706
 707		dma_pool_free(nvmeq->descriptor_pools.large, prp_list,
 708				dma_addr);
 709		dma_addr = next_dma_addr;
 710	}
 711}
 712
 713static void nvme_free_prps(struct request *req, unsigned int attrs)
 714{
 715	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 716	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
 717	unsigned int i;
 718
 719	for (i = 0; i < iod->nr_dma_vecs; i++)
 720		dma_unmap_phys(nvmeq->dev->dev, iod->dma_vecs[i].addr,
 721			       iod->dma_vecs[i].len, rq_dma_dir(req), attrs);
 722	mempool_free(iod->dma_vecs, nvmeq->dev->dmavec_mempool);
 723}
 724
 725static void nvme_free_sgls(struct request *req, struct nvme_sgl_desc *sge,
 726		struct nvme_sgl_desc *sg_list, unsigned int attrs)
 727{
 728	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
 729	enum dma_data_direction dir = rq_dma_dir(req);
 730	unsigned int len = le32_to_cpu(sge->length);
 731	struct device *dma_dev = nvmeq->dev->dev;
 732	unsigned int i;
 733
 734	if (sge->type == (NVME_SGL_FMT_DATA_DESC << 4)) {
 735		dma_unmap_phys(dma_dev, le64_to_cpu(sge->addr), len, dir,
 736			       attrs);
 737		return;
 738	}
 739
 740	for (i = 0; i < len / sizeof(*sg_list); i++)
 741		dma_unmap_phys(dma_dev, le64_to_cpu(sg_list[i].addr),
 742			le32_to_cpu(sg_list[i].length), dir, attrs);
 743}
 744
 745static void nvme_unmap_metadata(struct request *req)
 746{
 747	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
 748	enum pci_p2pdma_map_type map = PCI_P2PDMA_MAP_NONE;
 749	enum dma_data_direction dir = rq_dma_dir(req);
 750	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 751	struct device *dma_dev = nvmeq->dev->dev;
 752	struct nvme_sgl_desc *sge = iod->meta_descriptor;
 753	unsigned int attrs = 0;
 754
 755	if (iod->flags & IOD_SINGLE_META_SEGMENT) {
 756		dma_unmap_page(dma_dev, iod->meta_dma,
 757			       rq_integrity_vec(req).bv_len,
 758			       rq_dma_dir(req));
 759		return;
 760	}
 761
 762	if (iod->flags & IOD_META_P2P)
 763		map = PCI_P2PDMA_MAP_BUS_ADDR;
 764	else if (iod->flags & IOD_META_MMIO) {
 765		map = PCI_P2PDMA_MAP_THRU_HOST_BRIDGE;
 766		attrs |= DMA_ATTR_MMIO;
 767	}
 768
 769	if (!blk_rq_dma_unmap(req, dma_dev, &iod->meta_dma_state,
 770			      iod->meta_total_len, map)) {
 771		if (nvme_pci_cmd_use_meta_sgl(&iod->cmd))
 772			nvme_free_sgls(req, sge, &sge[1], attrs);
 773		else
 774			dma_unmap_phys(dma_dev, iod->meta_dma,
 775				       iod->meta_total_len, dir, attrs);
 776	}
 777
 778	if (iod->meta_descriptor)
 779		dma_pool_free(nvmeq->descriptor_pools.small,
 780			      iod->meta_descriptor, iod->meta_dma);
 781}
 782
 783static void nvme_unmap_data(struct request *req)
 784{
 785	enum pci_p2pdma_map_type map = PCI_P2PDMA_MAP_NONE;
 786	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 787	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
 788	struct device *dma_dev = nvmeq->dev->dev;
 789	unsigned int attrs = 0;
 790
 791	if (iod->flags & IOD_SINGLE_SEGMENT) {
 792		static_assert(offsetof(union nvme_data_ptr, prp1) ==
 793				offsetof(union nvme_data_ptr, sgl.addr));
 794		dma_unmap_page(dma_dev, le64_to_cpu(iod->cmd.common.dptr.prp1),
 795				iod->total_len, rq_dma_dir(req));
 796		return;
 797	}
 798
 799	if (iod->flags & IOD_DATA_P2P)
 800		map = PCI_P2PDMA_MAP_BUS_ADDR;
 801	else if (iod->flags & IOD_DATA_MMIO) {
 802		map = PCI_P2PDMA_MAP_THRU_HOST_BRIDGE;
 803		attrs |= DMA_ATTR_MMIO;
 804	}
 805
 806	if (!blk_rq_dma_unmap(req, dma_dev, &iod->dma_state, iod->total_len,
 807			      map)) {
 808		if (nvme_pci_cmd_use_sgl(&iod->cmd))
 809			nvme_free_sgls(req, &iod->cmd.common.dptr.sgl,
 810			               iod->descriptors[0], attrs);
 811		else
 812			nvme_free_prps(req, attrs);
 813	}
 814
 815	if (iod->nr_descriptors)
 816		nvme_free_descriptors(req);
 817}
 818
 819static bool nvme_pci_prp_save_mapping(struct request *req,
 820				      struct device *dma_dev,
 821				      struct blk_dma_iter *iter)
 822{
 823	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 824
 825	if (dma_use_iova(&iod->dma_state) || !dma_need_unmap(dma_dev))
 826		return true;
 827
 828	if (!iod->nr_dma_vecs) {
 829		struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
 830
 831		iod->dma_vecs = mempool_alloc(nvmeq->dev->dmavec_mempool,
 832				GFP_ATOMIC);
 833		if (!iod->dma_vecs) {
 834			iter->status = BLK_STS_RESOURCE;
 835			return false;
 836		}
 837	}
 838
 839	iod->dma_vecs[iod->nr_dma_vecs].addr = iter->addr;
 840	iod->dma_vecs[iod->nr_dma_vecs].len = iter->len;
 841	iod->nr_dma_vecs++;
 842	return true;
 843}
 844
 845static bool nvme_pci_prp_iter_next(struct request *req, struct device *dma_dev,
 846		struct blk_dma_iter *iter)
 847{
 848	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 849
 850	if (iter->len)
 851		return true;
 852	if (!blk_rq_dma_map_iter_next(req, dma_dev, &iod->dma_state, iter))
 853		return false;
 854	return nvme_pci_prp_save_mapping(req, dma_dev, iter);
 855}
 856
 857static blk_status_t nvme_pci_setup_data_prp(struct request *req,
 858		struct blk_dma_iter *iter)
 859{
 860	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 861	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
 862	unsigned int length = blk_rq_payload_bytes(req);
 863	dma_addr_t prp1_dma, prp2_dma = 0;
 864	unsigned int prp_len, i;
 865	__le64 *prp_list;
 866
 867	if (!nvme_pci_prp_save_mapping(req, nvmeq->dev->dev, iter))
 868		return iter->status;
 869
 870	/*
 871	 * PRP1 always points to the start of the DMA transfers.
 872	 *
 873	 * This is the only PRP (except for the list entries) that could be
 874	 * non-aligned.
 875	 */
 876	prp1_dma = iter->addr;
 877	prp_len = min(length, NVME_CTRL_PAGE_SIZE -
 878			(iter->addr & (NVME_CTRL_PAGE_SIZE - 1)));
 879	iod->total_len += prp_len;
 880	iter->addr += prp_len;
 881	iter->len -= prp_len;
 882	length -= prp_len;
 883	if (!length)
 884		goto done;
 885
 886	if (!nvme_pci_prp_iter_next(req, nvmeq->dev->dev, iter)) {
 887		if (WARN_ON_ONCE(!iter->status))
 888			goto bad_sgl;
 889		goto done;
 890	}
 891
 892	/*
 893	 * PRP2 is usually a list, but can point to data if all data to be
 894	 * transferred fits into PRP1 + PRP2:
 895	 */
 896	if (length <= NVME_CTRL_PAGE_SIZE) {
 897		prp2_dma = iter->addr;
 898		iod->total_len += length;
 899		goto done;
 900	}
 901
 902	if (DIV_ROUND_UP(length, NVME_CTRL_PAGE_SIZE) <=
 903	    NVME_SMALL_POOL_SIZE / sizeof(__le64))
 904		iod->flags |= IOD_SMALL_DESCRIPTOR;
 905
 906	prp_list = dma_pool_alloc(nvme_dma_pool(nvmeq, iod), GFP_ATOMIC,
 907			&prp2_dma);
 908	if (!prp_list) {
 909		iter->status = BLK_STS_RESOURCE;
 910		goto done;
 911	}
 912	iod->descriptors[iod->nr_descriptors++] = prp_list;
 913
 914	i = 0;
 915	for (;;) {
 916		prp_list[i++] = cpu_to_le64(iter->addr);
 917		prp_len = min(length, NVME_CTRL_PAGE_SIZE);
 918		if (WARN_ON_ONCE(iter->len < prp_len))
 919			goto bad_sgl;
 920
 921		iod->total_len += prp_len;
 922		iter->addr += prp_len;
 923		iter->len -= prp_len;
 924		length -= prp_len;
 925		if (!length)
 926			break;
 927
 928		if (!nvme_pci_prp_iter_next(req, nvmeq->dev->dev, iter)) {
 929			if (WARN_ON_ONCE(!iter->status))
 930				goto bad_sgl;
 931			goto done;
 932		}
 933
 934		/*
 935		 * If we've filled the entire descriptor, allocate a new that is
 936		 * pointed to be the last entry in the previous PRP list.  To
 937		 * accommodate for that move the last actual entry to the new
 938		 * descriptor.
 939		 */
 940		if (i == NVME_CTRL_PAGE_SIZE >> 3) {
 941			__le64 *old_prp_list = prp_list;
 942			dma_addr_t prp_list_dma;
 943
 944			prp_list = dma_pool_alloc(nvmeq->descriptor_pools.large,
 945					GFP_ATOMIC, &prp_list_dma);
 946			if (!prp_list) {
 947				iter->status = BLK_STS_RESOURCE;
 948				goto done;
 949			}
 950			iod->descriptors[iod->nr_descriptors++] = prp_list;
 951
 952			prp_list[0] = old_prp_list[i - 1];
 953			old_prp_list[i - 1] = cpu_to_le64(prp_list_dma);
 954			i = 1;
 955		}
 956	}
 957
 958done:
 959	/*
 960	 * nvme_unmap_data uses the DPT field in the SQE to tear down the
 961	 * mapping, so initialize it even for failures.
 962	 */
 963	iod->cmd.common.dptr.prp1 = cpu_to_le64(prp1_dma);
 964	iod->cmd.common.dptr.prp2 = cpu_to_le64(prp2_dma);
 965	if (unlikely(iter->status))
 966		nvme_unmap_data(req);
 967	return iter->status;
 968
 969bad_sgl:
 970	dev_err_once(nvmeq->dev->dev,
 971		"Incorrectly formed request for payload:%d nents:%d\n",
 972		blk_rq_payload_bytes(req), blk_rq_nr_phys_segments(req));
 973	return BLK_STS_IOERR;
 974}
 975
 976static void nvme_pci_sgl_set_data(struct nvme_sgl_desc *sge,
 977		struct blk_dma_iter *iter)
 978{
 979	sge->addr = cpu_to_le64(iter->addr);
 980	sge->length = cpu_to_le32(iter->len);
 981	sge->type = NVME_SGL_FMT_DATA_DESC << 4;
 982}
 983
 984static void nvme_pci_sgl_set_seg(struct nvme_sgl_desc *sge,
 985		dma_addr_t dma_addr, int entries)
 986{
 987	sge->addr = cpu_to_le64(dma_addr);
 988	sge->length = cpu_to_le32(entries * sizeof(*sge));
 989	sge->type = NVME_SGL_FMT_LAST_SEG_DESC << 4;
 990}
 991
 992static blk_status_t nvme_pci_setup_data_sgl(struct request *req,
 993		struct blk_dma_iter *iter)
 994{
 995	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 996	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
 997	unsigned int entries = blk_rq_nr_phys_segments(req);
 998	struct nvme_sgl_desc *sg_list;
 999	dma_addr_t sgl_dma;
1000	unsigned int mapped = 0;
1001
1002	/* set the transfer type as SGL */
1003	iod->cmd.common.flags = NVME_CMD_SGL_METABUF;
1004
1005	if (entries == 1 || blk_rq_dma_map_coalesce(&iod->dma_state)) {
1006		nvme_pci_sgl_set_data(&iod->cmd.common.dptr.sgl, iter);
1007		iod->total_len += iter->len;
1008		return BLK_STS_OK;
1009	}
1010
1011	if (entries <= NVME_SMALL_POOL_SIZE / sizeof(*sg_list))
1012		iod->flags |= IOD_SMALL_DESCRIPTOR;
1013
1014	sg_list = dma_pool_alloc(nvme_dma_pool(nvmeq, iod), GFP_ATOMIC,
1015			&sgl_dma);
1016	if (!sg_list)
1017		return BLK_STS_RESOURCE;
1018	iod->descriptors[iod->nr_descriptors++] = sg_list;
1019
1020	do {
1021		if (WARN_ON_ONCE(mapped == entries)) {
1022			iter->status = BLK_STS_IOERR;
1023			break;
1024		}
1025		nvme_pci_sgl_set_data(&sg_list[mapped++], iter);
1026		iod->total_len += iter->len;
1027	} while (blk_rq_dma_map_iter_next(req, nvmeq->dev->dev, &iod->dma_state,
1028				iter));
1029
1030	nvme_pci_sgl_set_seg(&iod->cmd.common.dptr.sgl, sgl_dma, mapped);
1031	if (unlikely(iter->status))
1032		nvme_unmap_data(req);
1033	return iter->status;
1034}
1035
1036static blk_status_t nvme_pci_setup_data_simple(struct request *req,
1037		enum nvme_use_sgl use_sgl)
1038{
1039	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
1040	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
1041	struct bio_vec bv = req_bvec(req);
1042	unsigned int prp1_offset = bv.bv_offset & (NVME_CTRL_PAGE_SIZE - 1);
1043	bool prp_possible = prp1_offset + bv.bv_len <= NVME_CTRL_PAGE_SIZE * 2;
1044	dma_addr_t dma_addr;
1045
1046	if (!use_sgl && !prp_possible)
1047		return BLK_STS_AGAIN;
1048	if (is_pci_p2pdma_page(bv.bv_page))
1049		return BLK_STS_AGAIN;
1050
1051	dma_addr = dma_map_bvec(nvmeq->dev->dev, &bv, rq_dma_dir(req), 0);
1052	if (dma_mapping_error(nvmeq->dev->dev, dma_addr))
1053		return BLK_STS_RESOURCE;
1054	iod->total_len = bv.bv_len;
1055	iod->flags |= IOD_SINGLE_SEGMENT;
1056
1057	if (use_sgl == SGL_FORCED || !prp_possible) {
1058		iod->cmd.common.flags = NVME_CMD_SGL_METABUF;
1059		iod->cmd.common.dptr.sgl.addr = cpu_to_le64(dma_addr);
1060		iod->cmd.common.dptr.sgl.length = cpu_to_le32(bv.bv_len);
1061		iod->cmd.common.dptr.sgl.type = NVME_SGL_FMT_DATA_DESC << 4;
1062	} else {
1063		unsigned int first_prp_len = NVME_CTRL_PAGE_SIZE - prp1_offset;
1064
1065		iod->cmd.common.dptr.prp1 = cpu_to_le64(dma_addr);
1066		iod->cmd.common.dptr.prp2 = 0;
1067		if (bv.bv_len > first_prp_len)
1068			iod->cmd.common.dptr.prp2 =
1069				cpu_to_le64(dma_addr + first_prp_len);
1070	}
1071
1072	return BLK_STS_OK;
1073}
1074
1075static blk_status_t nvme_map_data(struct request *req)
1076{
1077	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
1078	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
1079	struct nvme_dev *dev = nvmeq->dev;
1080	enum nvme_use_sgl use_sgl = nvme_pci_use_sgls(dev, req);
1081	struct blk_dma_iter iter;
1082	blk_status_t ret;
1083
1084	/*
1085	 * Try to skip the DMA iterator for single segment requests, as that
1086	 * significantly improves performances for small I/O sizes.
1087	 */
1088	if (blk_rq_nr_phys_segments(req) == 1) {
1089		ret = nvme_pci_setup_data_simple(req, use_sgl);
1090		if (ret != BLK_STS_AGAIN)
1091			return ret;
1092	}
1093
1094	if (!blk_rq_dma_map_iter_start(req, dev->dev, &iod->dma_state, &iter))
1095		return iter.status;
1096
1097	switch (iter.p2pdma.map) {
1098	case PCI_P2PDMA_MAP_BUS_ADDR:
1099		iod->flags |= IOD_DATA_P2P;
1100		break;
1101	case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE:
1102		iod->flags |= IOD_DATA_MMIO;
1103		break;
1104	case PCI_P2PDMA_MAP_NONE:
1105		break;
1106	default:
1107		return BLK_STS_RESOURCE;
1108	}
1109
1110	if (use_sgl == SGL_FORCED ||
1111	    (use_sgl == SGL_SUPPORTED &&
1112	     (sgl_threshold && nvme_pci_avg_seg_size(req) >= sgl_threshold)))
1113		return nvme_pci_setup_data_sgl(req, &iter);
1114	return nvme_pci_setup_data_prp(req, &iter);
1115}
1116
1117static blk_status_t nvme_pci_setup_meta_iter(struct request *req)
1118{
1119	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
1120	unsigned int entries = req->nr_integrity_segments;
1121	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
1122	struct nvme_dev *dev = nvmeq->dev;
1123	struct nvme_sgl_desc *sg_list;
1124	struct blk_dma_iter iter;
1125	dma_addr_t sgl_dma;
1126	int i = 0;
1127
1128	if (!blk_rq_integrity_dma_map_iter_start(req, dev->dev,
1129						&iod->meta_dma_state, &iter))
1130		return iter.status;
1131
1132	switch (iter.p2pdma.map) {
1133	case PCI_P2PDMA_MAP_BUS_ADDR:
1134		iod->flags |= IOD_META_P2P;
1135		break;
1136	case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE:
1137		iod->flags |= IOD_META_MMIO;
1138		break;
1139	case PCI_P2PDMA_MAP_NONE:
1140		break;
1141	default:
1142		return BLK_STS_RESOURCE;
1143	}
1144
1145	if (blk_rq_dma_map_coalesce(&iod->meta_dma_state))
1146		entries = 1;
1147
1148	/*
1149	 * The NVMe MPTR descriptor has an implicit length that the host and
1150	 * device must agree on to avoid data/memory corruption. We trust the
1151	 * kernel allocated correctly based on the format's parameters, so use
1152	 * the more efficient MPTR to avoid extra dma pool allocations for the
1153	 * SGL indirection.
1154	 *
1155	 * But for user commands, we don't necessarily know what they do, so
1156	 * the driver can't validate the metadata buffer size. The SGL
1157	 * descriptor provides an explicit length, so we're relying on that
1158	 * mechanism to catch any misunderstandings between the application and
1159	 * device.
1160	 *
1161	 * P2P DMA also needs to use the blk_dma_iter method, so mptr setup
1162	 * leverages this routine when that happens.
1163	 */
1164	if (!nvme_ctrl_meta_sgl_supported(&dev->ctrl) ||
1165	    (entries == 1 && !(nvme_req(req)->flags & NVME_REQ_USERCMD))) {
1166		iod->cmd.common.metadata = cpu_to_le64(iter.addr);
1167		iod->meta_total_len = iter.len;
1168		iod->meta_dma = iter.addr;
1169		iod->meta_descriptor = NULL;
1170		return BLK_STS_OK;
1171	}
1172
1173	sg_list = dma_pool_alloc(nvmeq->descriptor_pools.small, GFP_ATOMIC,
1174			&sgl_dma);
1175	if (!sg_list)
1176		return BLK_STS_RESOURCE;
1177
1178	iod->meta_descriptor = sg_list;
1179	iod->meta_dma = sgl_dma;
1180	iod->cmd.common.flags = NVME_CMD_SGL_METASEG;
1181	iod->cmd.common.metadata = cpu_to_le64(sgl_dma);
1182	if (entries == 1) {
1183		iod->meta_total_len = iter.len;
1184		nvme_pci_sgl_set_data(sg_list, &iter);
1185		return BLK_STS_OK;
1186	}
1187
1188	sgl_dma += sizeof(*sg_list);
1189	do {
1190		nvme_pci_sgl_set_data(&sg_list[++i], &iter);
1191		iod->meta_total_len += iter.len;
1192	} while (blk_rq_integrity_dma_map_iter_next(req, dev->dev, &iter));
1193
1194	nvme_pci_sgl_set_seg(sg_list, sgl_dma, i);
1195	if (unlikely(iter.status))
1196		nvme_unmap_metadata(req);
1197	return iter.status;
1198}
1199
1200static blk_status_t nvme_pci_setup_meta_mptr(struct request *req)
1201{
1202	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
1203	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
1204	struct bio_vec bv = rq_integrity_vec(req);
1205
1206	if (is_pci_p2pdma_page(bv.bv_page))
1207		return nvme_pci_setup_meta_iter(req);
1208
1209	iod->meta_dma = dma_map_bvec(nvmeq->dev->dev, &bv, rq_dma_dir(req), 0);
1210	if (dma_mapping_error(nvmeq->dev->dev, iod->meta_dma))
1211		return BLK_STS_IOERR;
1212	iod->cmd.common.metadata = cpu_to_le64(iod->meta_dma);
1213	iod->flags |= IOD_SINGLE_META_SEGMENT;
1214	return BLK_STS_OK;
1215}
1216
1217static blk_status_t nvme_map_metadata(struct request *req)
1218{
1219	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
1220
1221	if ((iod->cmd.common.flags & NVME_CMD_SGL_METABUF) &&
1222	    nvme_pci_metadata_use_sgls(req))
1223		return nvme_pci_setup_meta_iter(req);
1224	return nvme_pci_setup_meta_mptr(req);
1225}
1226
1227static blk_status_t nvme_prep_rq(struct request *req)
1228{
1229	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
1230	blk_status_t ret;
1231
1232	iod->flags = 0;
1233	iod->nr_descriptors = 0;
1234	iod->total_len = 0;
1235	iod->meta_total_len = 0;
1236	iod->nr_dma_vecs = 0;
1237
1238	ret = nvme_setup_cmd(req->q->queuedata, req);
1239	if (ret)
1240		return ret;
1241
1242	if (blk_rq_nr_phys_segments(req)) {
1243		ret = nvme_map_data(req);
1244		if (ret)
1245			goto out_free_cmd;
1246	}
1247
1248	if (blk_integrity_rq(req)) {
1249		ret = nvme_map_metadata(req);
1250		if (ret)
1251			goto out_unmap_data;
1252	}
1253
1254	nvme_start_request(req);
1255	return BLK_STS_OK;
1256out_unmap_data:
1257	if (blk_rq_nr_phys_segments(req))
1258		nvme_unmap_data(req);
1259out_free_cmd:
1260	nvme_cleanup_cmd(req);
1261	return ret;
1262}
1263
1264static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
1265			 const struct blk_mq_queue_data *bd)
1266{
1267	struct nvme_queue *nvmeq = hctx->driver_data;
1268	struct nvme_dev *dev = nvmeq->dev;
1269	struct request *req = bd->rq;
1270	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
1271	blk_status_t ret;
1272
1273	/*
1274	 * We should not need to do this, but we're still using this to
1275	 * ensure we can drain requests on a dying queue.
1276	 */
1277	if (unlikely(!test_bit(NVMEQ_ENABLED, &nvmeq->flags)))
1278		return BLK_STS_IOERR;
1279
1280	if (unlikely(!nvme_check_ready(&dev->ctrl, req, true)))
1281		return nvme_fail_nonready_command(&dev->ctrl, req);
1282
1283	ret = nvme_prep_rq(req);
1284	if (unlikely(ret))
1285		return ret;
1286	spin_lock(&nvmeq->sq_lock);
1287	nvme_sq_copy_cmd(nvmeq, &iod->cmd);
1288	nvme_write_sq_db(nvmeq, bd->last);
1289	spin_unlock(&nvmeq->sq_lock);
1290	return BLK_STS_OK;
1291}
1292
1293static void nvme_submit_cmds(struct nvme_queue *nvmeq, struct rq_list *rqlist)
1294{
1295	struct request *req;
1296
1297	if (rq_list_empty(rqlist))
1298		return;
1299
1300	spin_lock(&nvmeq->sq_lock);
1301	while ((req = rq_list_pop(rqlist))) {
1302		struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
1303
1304		nvme_sq_copy_cmd(nvmeq, &iod->cmd);
1305	}
1306	nvme_write_sq_db(nvmeq, true);
1307	spin_unlock(&nvmeq->sq_lock);
1308}
1309
1310static bool nvme_prep_rq_batch(struct nvme_queue *nvmeq, struct request *req)
1311{
1312	/*
1313	 * We should not need to do this, but we're still using this to
1314	 * ensure we can drain requests on a dying queue.
1315	 */
1316	if (unlikely(!test_bit(NVMEQ_ENABLED, &nvmeq->flags)))
1317		return false;
1318	if (unlikely(!nvme_check_ready(&nvmeq->dev->ctrl, req, true)))
1319		return false;
1320
1321	return nvme_prep_rq(req) == BLK_STS_OK;
1322}
1323
1324static void nvme_queue_rqs(struct rq_list *rqlist)
1325{
1326	struct rq_list submit_list = { };
1327	struct rq_list requeue_list = { };
1328	struct nvme_queue *nvmeq = NULL;
1329	struct request *req;
1330
1331	while ((req = rq_list_pop(rqlist))) {
1332		if (nvmeq && nvmeq != req->mq_hctx->driver_data)
1333			nvme_submit_cmds(nvmeq, &submit_list);
1334		nvmeq = req->mq_hctx->driver_data;
1335
1336		if (nvme_prep_rq_batch(nvmeq, req))
1337			rq_list_add_tail(&submit_list, req);
1338		else
1339			rq_list_add_tail(&requeue_list, req);
1340	}
1341
1342	if (nvmeq)
1343		nvme_submit_cmds(nvmeq, &submit_list);
1344	*rqlist = requeue_list;
1345}
1346
1347static __always_inline void nvme_pci_unmap_rq(struct request *req)
1348{
1349	if (blk_integrity_rq(req))
1350		nvme_unmap_metadata(req);
1351	if (blk_rq_nr_phys_segments(req))
1352		nvme_unmap_data(req);
1353}
1354
1355static void nvme_pci_complete_rq(struct request *req)
1356{
1357	nvme_pci_unmap_rq(req);
1358	nvme_complete_rq(req);
1359}
1360
1361static void nvme_pci_complete_batch(struct io_comp_batch *iob)
1362{
1363	nvme_complete_batch(iob, nvme_pci_unmap_rq);
1364}
1365
1366/* We read the CQE phase first to check if the rest of the entry is valid */
1367static inline bool nvme_cqe_pending(struct nvme_queue *nvmeq)
1368{
1369	struct nvme_completion *hcqe = &nvmeq->cqes[nvmeq->cq_head];
1370
1371	return (le16_to_cpu(READ_ONCE(hcqe->status)) & 1) == nvmeq->cq_phase;
1372}
1373
1374static inline void nvme_ring_cq_doorbell(struct nvme_queue *nvmeq)
1375{
1376	u16 head = nvmeq->cq_head;
1377
1378	if (nvme_dbbuf_update_and_check_event(head, nvmeq->dbbuf_cq_db,
1379					      nvmeq->dbbuf_cq_ei))
1380		writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
1381}
1382
1383static inline struct blk_mq_tags *nvme_queue_tagset(struct nvme_queue *nvmeq)
1384{
1385	if (!nvmeq->qid)
1386		return nvmeq->dev->admin_tagset.tags[0];
1387	return nvmeq->dev->tagset.tags[nvmeq->qid - 1];
1388}
1389
1390static inline void nvme_handle_cqe(struct nvme_queue *nvmeq,
1391				   struct io_comp_batch *iob, u16 idx)
1392{
1393	struct nvme_completion *cqe = &nvmeq->cqes[idx];
1394	__u16 command_id = READ_ONCE(cqe->command_id);
1395	struct request *req;
1396
1397	/*
1398	 * AEN requests are special as they don't time out and can
1399	 * survive any kind of queue freeze and often don't respond to
1400	 * aborts.  We don't even bother to allocate a struct request
1401	 * for them but rather special case them here.
1402	 */
1403	if (unlikely(nvme_is_aen_req(nvmeq->qid, command_id))) {
1404		nvme_complete_async_event(&nvmeq->dev->ctrl,
1405				cqe->status, &cqe->result);
1406		return;
1407	}
1408
1409	req = nvme_find_rq(nvme_queue_tagset(nvmeq), command_id);
1410	if (unlikely(!req)) {
1411		dev_warn(nvmeq->dev->ctrl.device,
1412			"invalid id %d completed on queue %d\n",
1413			command_id, le16_to_cpu(cqe->sq_id));
1414		return;
1415	}
1416
1417	trace_nvme_sq(req, cqe->sq_head, nvmeq->sq_tail);
1418	if (!nvme_try_complete_req(req, cqe->status, cqe->result) &&
1419	    !blk_mq_add_to_batch(req, iob,
1420				 nvme_req(req)->status != NVME_SC_SUCCESS,
1421				 nvme_pci_complete_batch))
1422		nvme_pci_complete_rq(req);
1423}
1424
1425static inline void nvme_update_cq_head(struct nvme_queue *nvmeq)
1426{
1427	u32 tmp = nvmeq->cq_head + 1;
1428
1429	if (tmp == nvmeq->q_depth) {
1430		nvmeq->cq_head = 0;
1431		nvmeq->cq_phase ^= 1;
1432	} else {
1433		nvmeq->cq_head = tmp;
1434	}
1435}
1436
1437static inline bool nvme_poll_cq(struct nvme_queue *nvmeq,
1438			        struct io_comp_batch *iob)
1439{
1440	bool found = false;
1441
1442	while (nvme_cqe_pending(nvmeq)) {
1443		found = true;
1444		/*
1445		 * load-load control dependency between phase and the rest of
1446		 * the cqe requires a full read memory barrier
1447		 */
1448		dma_rmb();
1449		nvme_handle_cqe(nvmeq, iob, nvmeq->cq_head);
1450		nvme_update_cq_head(nvmeq);
1451	}
1452
1453	if (found)
1454		nvme_ring_cq_doorbell(nvmeq);
1455	return found;
1456}
1457
1458static irqreturn_t nvme_irq(int irq, void *data)
1459{
1460	struct nvme_queue *nvmeq = data;
1461	DEFINE_IO_COMP_BATCH(iob);
1462
1463	if (nvme_poll_cq(nvmeq, &iob)) {
1464		if (!rq_list_empty(&iob.req_list))
1465			nvme_pci_complete_batch(&iob);
1466		return IRQ_HANDLED;
1467	}
1468	return IRQ_NONE;
1469}
1470
1471static irqreturn_t nvme_irq_check(int irq, void *data)
1472{
1473	struct nvme_queue *nvmeq = data;
1474
1475	if (nvme_cqe_pending(nvmeq))
1476		return IRQ_WAKE_THREAD;
1477	return IRQ_NONE;
1478}
1479
1480/*
1481 * Poll for completions for any interrupt driven queue
1482 * Can be called from any context.
1483 */
1484static void nvme_poll_irqdisable(struct nvme_queue *nvmeq)
1485{
1486	struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev);
1487	int irq;
1488
1489	WARN_ON_ONCE(test_bit(NVMEQ_POLLED, &nvmeq->flags));
1490
1491	irq = pci_irq_vector(pdev, nvmeq->cq_vector);
1492	disable_irq(irq);
1493	spin_lock(&nvmeq->cq_poll_lock);
1494	nvme_poll_cq(nvmeq, NULL);
1495	spin_unlock(&nvmeq->cq_poll_lock);
1496	enable_irq(irq);
1497}
1498
1499static int nvme_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
1500{
1501	struct nvme_queue *nvmeq = hctx->driver_data;
1502	bool found;
1503
1504	if (!test_bit(NVMEQ_POLLED, &nvmeq->flags) ||
1505	    !nvme_cqe_pending(nvmeq))
1506		return 0;
1507
1508	spin_lock(&nvmeq->cq_poll_lock);
1509	found = nvme_poll_cq(nvmeq, iob);
1510	spin_unlock(&nvmeq->cq_poll_lock);
1511
1512	return found;
1513}
1514
1515static void nvme_pci_submit_async_event(struct nvme_ctrl *ctrl)
1516{
1517	struct nvme_dev *dev = to_nvme_dev(ctrl);
1518	struct nvme_queue *nvmeq = &dev->queues[0];
1519	struct nvme_command c = { };
1520
1521	c.common.opcode = nvme_admin_async_event;
1522	c.common.command_id = NVME_AQ_BLK_MQ_DEPTH;
1523
1524	spin_lock(&nvmeq->sq_lock);
1525	nvme_sq_copy_cmd(nvmeq, &c);
1526	nvme_write_sq_db(nvmeq, true);
1527	spin_unlock(&nvmeq->sq_lock);
1528}
1529
1530static int nvme_pci_subsystem_reset(struct nvme_ctrl *ctrl)
1531{
1532	struct nvme_dev *dev = to_nvme_dev(ctrl);
1533	int ret = 0;
1534
1535	/*
1536	 * Taking the shutdown_lock ensures the BAR mapping is not being
1537	 * altered by reset_work. Holding this lock before the RESETTING state
1538	 * change, if successful, also ensures nvme_remove won't be able to
1539	 * proceed to iounmap until we're done.
1540	 */
1541	mutex_lock(&dev->shutdown_lock);
1542	if (!dev->bar_mapped_size) {
1543		ret = -ENODEV;
1544		goto unlock;
1545	}
1546
1547	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) {
1548		ret = -EBUSY;
1549		goto unlock;
1550	}
1551
1552	writel(NVME_SUBSYS_RESET, dev->bar + NVME_REG_NSSR);
1553
1554	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING) ||
1555	    !nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE))
1556		goto unlock;
1557
1558	/*
1559	 * Read controller status to flush the previous write and trigger a
1560	 * pcie read error.
1561	 */
1562	readl(dev->bar + NVME_REG_CSTS);
1563unlock:
1564	mutex_unlock(&dev->shutdown_lock);
1565	return ret;
1566}
1567
1568static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
1569{
1570	struct nvme_command c = { };
1571
1572	c.delete_queue.opcode = opcode;
1573	c.delete_queue.qid = cpu_to_le16(id);
1574
1575	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
1576}
1577
1578static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
1579		struct nvme_queue *nvmeq, s16 vector)
1580{
1581	struct nvme_command c = { };
1582	int flags = NVME_QUEUE_PHYS_CONTIG;
1583
1584	if (!test_bit(NVMEQ_POLLED, &nvmeq->flags))
1585		flags |= NVME_CQ_IRQ_ENABLED;
1586
1587	/*
1588	 * Note: we (ab)use the fact that the prp fields survive if no data
1589	 * is attached to the request.
1590	 */
1591	c.create_cq.opcode = nvme_admin_create_cq;
1592	c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
1593	c.create_cq.cqid = cpu_to_le16(qid);
1594	c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
1595	c.create_cq.cq_flags = cpu_to_le16(flags);
1596	c.create_cq.irq_vector = cpu_to_le16(vector);
1597
1598	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
1599}
1600
1601static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
1602						struct nvme_queue *nvmeq)
1603{
1604	struct nvme_ctrl *ctrl = &dev->ctrl;
1605	struct nvme_command c = { };
1606	int flags = NVME_QUEUE_PHYS_CONTIG;
1607
1608	/*
1609	 * Some drives have a bug that auto-enables WRRU if MEDIUM isn't
1610	 * set. Since URGENT priority is zeroes, it makes all queues
1611	 * URGENT.
1612	 */
1613	if (ctrl->quirks & NVME_QUIRK_MEDIUM_PRIO_SQ)
1614		flags |= NVME_SQ_PRIO_MEDIUM;
1615
1616	/*
1617	 * Note: we (ab)use the fact that the prp fields survive if no data
1618	 * is attached to the request.
1619	 */
1620	c.create_sq.opcode = nvme_admin_create_sq;
1621	c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
1622	c.create_sq.sqid = cpu_to_le16(qid);
1623	c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
1624	c.create_sq.sq_flags = cpu_to_le16(flags);
1625	c.create_sq.cqid = cpu_to_le16(qid);
1626
1627	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
1628}
1629
1630static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
1631{
1632	return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
1633}
1634
1635static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
1636{
1637	return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
1638}
1639
1640static enum rq_end_io_ret abort_endio(struct request *req, blk_status_t error)
1641{
1642	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
1643
1644	dev_warn(nvmeq->dev->ctrl.device,
1645		 "Abort status: 0x%x", nvme_req(req)->status);
1646	atomic_inc(&nvmeq->dev->ctrl.abort_limit);
1647	blk_mq_free_request(req);
1648	return RQ_END_IO_NONE;
1649}
1650
1651static bool nvme_should_reset(struct nvme_dev *dev, u32 csts)
1652{
1653	/* If true, indicates loss of adapter communication, possibly by a
1654	 * NVMe Subsystem reset.
1655	 */
1656	bool nssro = dev->subsystem && (csts & NVME_CSTS_NSSRO);
1657
1658	/* If there is a reset/reinit ongoing, we shouldn't reset again. */
1659	switch (nvme_ctrl_state(&dev->ctrl)) {
1660	case NVME_CTRL_RESETTING:
1661	case NVME_CTRL_CONNECTING:
1662		return false;
1663	default:
1664		break;
1665	}
1666
1667	/* We shouldn't reset unless the controller is on fatal error state
1668	 * _or_ if we lost the communication with it.
1669	 */
1670	if (!(csts & NVME_CSTS_CFS) && !nssro)
1671		return false;
1672
1673	return true;
1674}
1675
1676static void nvme_warn_reset(struct nvme_dev *dev, u32 csts)
1677{
1678	/* Read a config register to help see what died. */
1679	u16 pci_status;
1680	int result;
1681
1682	result = pci_read_config_word(to_pci_dev(dev->dev), PCI_STATUS,
1683				      &pci_status);
1684	if (result == PCIBIOS_SUCCESSFUL)
1685		dev_warn(dev->ctrl.device,
1686			 "controller is down; will reset: CSTS=0x%x, PCI_STATUS=0x%hx\n",
1687			 csts, pci_status);
1688	else
1689		dev_warn(dev->ctrl.device,
1690			 "controller is down; will reset: CSTS=0x%x, PCI_STATUS read failed (%d)\n",
1691			 csts, result);
1692
1693	if (csts != ~0)
1694		return;
1695
1696	dev_warn(dev->ctrl.device,
1697		 "Does your device have a faulty power saving mode enabled?\n");
1698	dev_warn(dev->ctrl.device,
1699		 "Try \"nvme_core.default_ps_max_latency_us=0 pcie_aspm=off pcie_port_pm=off\" and report a bug\n");
1700}
1701
1702static enum blk_eh_timer_return nvme_timeout(struct request *req)
1703{
1704	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
1705	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
1706	struct nvme_dev *dev = nvmeq->dev;
1707	struct request *abort_req;
1708	struct nvme_command cmd = { };
1709	struct pci_dev *pdev = to_pci_dev(dev->dev);
1710	u32 csts = readl(dev->bar + NVME_REG_CSTS);
1711	u8 opcode;
1712
1713	/*
1714	 * Shutdown the device immediately if we see it is disconnected. This
1715	 * unblocks PCIe error handling if the nvme driver is waiting in
1716	 * error_resume for a device that has been removed. We can't unbind the
1717	 * driver while the driver's error callback is waiting to complete, so
1718	 * we're relying on a timeout to break that deadlock if a removal
1719	 * occurs while reset work is running.
1720	 */
1721	if (pci_dev_is_disconnected(pdev))
1722		nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING);
1723	if (nvme_state_terminal(&dev->ctrl))
1724		goto disable;
1725
1726	/* If PCI error recovery process is happening, we cannot reset or
1727	 * the recovery mechanism will surely fail.
1728	 */
1729	mb();
1730	if (pci_channel_offline(pdev))
1731		return BLK_EH_RESET_TIMER;
1732
1733	/*
1734	 * Reset immediately if the controller is failed
1735	 */
1736	if (nvme_should_reset(dev, csts)) {
1737		nvme_warn_reset(dev, csts);
1738		goto disable;
1739	}
1740
1741	/*
1742	 * Did we miss an interrupt?
1743	 */
1744	if (test_bit(NVMEQ_POLLED, &nvmeq->flags))
1745		nvme_poll(req->mq_hctx, NULL);
1746	else
1747		nvme_poll_irqdisable(nvmeq);
1748
1749	if (blk_mq_rq_state(req) != MQ_RQ_IN_FLIGHT) {
1750		dev_warn(dev->ctrl.device,
1751			 "I/O tag %d (%04x) QID %d timeout, completion polled\n",
1752			 req->tag, nvme_cid(req), nvmeq->qid);
1753		return BLK_EH_DONE;
1754	}
1755
1756	/*
1757	 * Shutdown immediately if controller times out while starting. The
1758	 * reset work will see the pci device disabled when it gets the forced
1759	 * cancellation error. All outstanding requests are completed on
1760	 * shutdown, so we return BLK_EH_DONE.
1761	 */
1762	switch (nvme_ctrl_state(&dev->ctrl)) {
1763	case NVME_CTRL_CONNECTING:
1764		nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING);
1765		fallthrough;
1766	case NVME_CTRL_DELETING:
1767		dev_warn_ratelimited(dev->ctrl.device,
1768			 "I/O tag %d (%04x) QID %d timeout, disable controller\n",
1769			 req->tag, nvme_cid(req), nvmeq->qid);
1770		nvme_req(req)->flags |= NVME_REQ_CANCELLED;
1771		nvme_dev_disable(dev, true);
1772		return BLK_EH_DONE;
1773	case NVME_CTRL_RESETTING:
1774		return BLK_EH_RESET_TIMER;
1775	default:
1776		break;
1777	}
1778
1779	/*
1780	 * Shutdown the controller immediately and schedule a reset if the
1781	 * command was already aborted once before and still hasn't been
1782	 * returned to the driver, or if this is the admin queue.
1783	 */
1784	opcode = nvme_req(req)->cmd->common.opcode;
1785	if (!nvmeq->qid || (iod->flags & IOD_ABORTED)) {
1786		dev_warn(dev->ctrl.device,
1787			 "I/O tag %d (%04x) opcode %#x (%s) QID %d timeout, reset controller\n",
1788			 req->tag, nvme_cid(req), opcode,
1789			 nvme_opcode_str(nvmeq->qid, opcode), nvmeq->qid);
1790		nvme_req(req)->flags |= NVME_REQ_CANCELLED;
1791		goto disable;
1792	}
1793
1794	if (atomic_dec_return(&dev->ctrl.abort_limit) < 0) {
1795		atomic_inc(&dev->ctrl.abort_limit);
1796		return BLK_EH_RESET_TIMER;
1797	}
1798	iod->flags |= IOD_ABORTED;
1799
1800	cmd.abort.opcode = nvme_admin_abort_cmd;
1801	cmd.abort.cid = nvme_cid(req);
1802	cmd.abort.sqid = cpu_to_le16(nvmeq->qid);
1803
1804	dev_warn(nvmeq->dev->ctrl.device,
1805		 "I/O tag %d (%04x) opcode %#x (%s) QID %d timeout, aborting req_op:%s(%u) size:%u\n",
1806		 req->tag, nvme_cid(req), opcode, nvme_get_opcode_str(opcode),
1807		 nvmeq->qid, blk_op_str(req_op(req)), req_op(req),
1808		 blk_rq_bytes(req));
1809
1810	abort_req = blk_mq_alloc_request(dev->ctrl.admin_q, nvme_req_op(&cmd),
1811					 BLK_MQ_REQ_NOWAIT);
1812	if (IS_ERR(abort_req)) {
1813		atomic_inc(&dev->ctrl.abort_limit);
1814		return BLK_EH_RESET_TIMER;
1815	}
1816	nvme_init_request(abort_req, &cmd);
1817
1818	abort_req->end_io = abort_endio;
1819	abort_req->end_io_data = NULL;
1820	blk_execute_rq_nowait(abort_req, false);
1821
1822	/*
1823	 * The aborted req will be completed on receiving the abort req.
1824	 * We enable the timer again. If hit twice, it'll cause a device reset,
1825	 * as the device then is in a faulty state.
1826	 */
1827	return BLK_EH_RESET_TIMER;
1828
1829disable:
1830	if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_RESETTING)) {
1831		if (nvme_state_terminal(&dev->ctrl))
1832			nvme_dev_disable(dev, true);
1833		return BLK_EH_DONE;
1834	}
1835
1836	nvme_dev_disable(dev, false);
1837	if (nvme_try_sched_reset(&dev->ctrl))
1838		nvme_unquiesce_io_queues(&dev->ctrl);
1839	return BLK_EH_DONE;
1840}
1841
1842static void nvme_free_queue(struct nvme_queue *nvmeq)
1843{
1844	dma_free_coherent(nvmeq->dev->dev, CQ_SIZE(nvmeq),
1845				(void *)nvmeq->cqes, nvmeq->cq_dma_addr);
1846	if (!nvmeq->sq_cmds)
1847		return;
1848
1849	if (test_and_clear_bit(NVMEQ_SQ_CMB, &nvmeq->flags)) {
1850		pci_free_p2pmem(to_pci_dev(nvmeq->dev->dev),
1851				nvmeq->sq_cmds, SQ_SIZE(nvmeq));
1852	} else {
1853		dma_free_coherent(nvmeq->dev->dev, SQ_SIZE(nvmeq),
1854				nvmeq->sq_cmds, nvmeq->sq_dma_addr);
1855	}
1856}
1857
1858static void nvme_free_queues(struct nvme_dev *dev, int lowest)
1859{
1860	int i;
1861
1862	for (i = dev->ctrl.queue_count - 1; i >= lowest; i--) {
1863		dev->ctrl.queue_count--;
1864		nvme_free_queue(&dev->queues[i]);
1865	}
1866}
1867
1868static void nvme_suspend_queue(struct nvme_dev *dev, unsigned int qid)
1869{
1870	struct nvme_queue *nvmeq = &dev->queues[qid];
1871
1872	if (!test_and_clear_bit(NVMEQ_ENABLED, &nvmeq->flags))
1873		return;
1874
1875	/* ensure that nvme_queue_rq() sees NVMEQ_ENABLED cleared */
1876	mb();
1877
1878	nvmeq->dev->online_queues--;
1879	if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q)
1880		nvme_quiesce_admin_queue(&nvmeq->dev->ctrl);
1881	if (!test_and_clear_bit(NVMEQ_POLLED, &nvmeq->flags))
1882		pci_free_irq(to_pci_dev(dev->dev), nvmeq->cq_vector, nvmeq);
1883}
1884
1885static void nvme_suspend_io_queues(struct nvme_dev *dev)
1886{
1887	int i;
1888
1889	for (i = dev->ctrl.queue_count - 1; i > 0; i--)
1890		nvme_suspend_queue(dev, i);
1891}
1892
1893/*
1894 * Called only on a device that has been disabled and after all other threads
1895 * that can check this device's completion queues have synced, except
1896 * nvme_poll(). This is the last chance for the driver to see a natural
1897 * completion before nvme_cancel_request() terminates all incomplete requests.
1898 */
1899static void nvme_reap_pending_cqes(struct nvme_dev *dev)
1900{
1901	int i;
1902
1903	for (i = dev->ctrl.queue_count - 1; i > 0; i--) {
1904		spin_lock(&dev->queues[i].cq_poll_lock);
1905		nvme_poll_cq(&dev->queues[i], NULL);
1906		spin_unlock(&dev->queues[i].cq_poll_lock);
1907	}
1908}
1909
1910static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues,
1911				int entry_size)
1912{
1913	int q_depth = dev->q_depth;
1914	unsigned q_size_aligned = roundup(q_depth * entry_size,
1915					  NVME_CTRL_PAGE_SIZE);
1916
1917	if (q_size_aligned * nr_io_queues > dev->cmb_size) {
1918		u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues);
1919
1920		mem_per_q = round_down(mem_per_q, NVME_CTRL_PAGE_SIZE);
1921		q_depth = div_u64(mem_per_q, entry_size);
1922
1923		/*
1924		 * Ensure the reduced q_depth is above some threshold where it
1925		 * would be better to map queues in system memory with the
1926		 * original depth
1927		 */
1928		if (q_depth < 64)
1929			return -ENOMEM;
1930	}
1931
1932	return q_depth;
1933}
1934
1935static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq,
1936				int qid)
1937{
1938	struct pci_dev *pdev = to_pci_dev(dev->dev);
1939
1940	if (qid && dev->cmb_use_sqes && (dev->cmbsz & NVME_CMBSZ_SQS)) {
1941		nvmeq->sq_cmds = pci_alloc_p2pmem(pdev, SQ_SIZE(nvmeq));
1942		if (nvmeq->sq_cmds) {
1943			nvmeq->sq_dma_addr = pci_p2pmem_virt_to_bus(pdev,
1944							nvmeq->sq_cmds);
1945			if (nvmeq->sq_dma_addr) {
1946				set_bit(NVMEQ_SQ_CMB, &nvmeq->flags);
1947				return 0;
1948			}
1949
1950			pci_free_p2pmem(pdev, nvmeq->sq_cmds, SQ_SIZE(nvmeq));
1951		}
1952	}
1953
1954	nvmeq->sq_cmds = dma_alloc_coherent(dev->dev, SQ_SIZE(nvmeq),
1955				&nvmeq->sq_dma_addr, GFP_KERNEL);
1956	if (!nvmeq->sq_cmds)
1957		return -ENOMEM;
1958	return 0;
1959}
1960
1961static int nvme_alloc_queue(struct nvme_dev *dev, int qid, int depth)
1962{
1963	struct nvme_queue *nvmeq = &dev->queues[qid];
1964
1965	if (dev->ctrl.queue_count > qid)
1966		return 0;
1967
1968	nvmeq->sqes = qid ? dev->io_sqes : NVME_ADM_SQES;
1969	nvmeq->q_depth = depth;
1970	nvmeq->cqes = dma_alloc_coherent(dev->dev, CQ_SIZE(nvmeq),
1971					 &nvmeq->cq_dma_addr, GFP_KERNEL);
1972	if (!nvmeq->cqes)
1973		goto free_nvmeq;
1974
1975	if (nvme_alloc_sq_cmds(dev, nvmeq, qid))
1976		goto free_cqdma;
1977
1978	nvmeq->dev = dev;
1979	spin_lock_init(&nvmeq->sq_lock);
1980	spin_lock_init(&nvmeq->cq_poll_lock);
1981	nvmeq->cq_head = 0;
1982	nvmeq->cq_phase = 1;
1983	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
1984	nvmeq->qid = qid;
1985	dev->ctrl.queue_count++;
1986
1987	return 0;
1988
1989 free_cqdma:
1990	dma_free_coherent(dev->dev, CQ_SIZE(nvmeq), (void *)nvmeq->cqes,
1991			  nvmeq->cq_dma_addr);
1992 free_nvmeq:
1993	return -ENOMEM;
1994}
1995
1996static int queue_request_irq(struct nvme_queue *nvmeq)
1997{
1998	struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev);
1999	int nr = nvmeq->dev->ctrl.instance;
2000
2001	if (use_threaded_interrupts) {
2002		return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq_check,
2003				nvme_irq, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
2004	} else {
2005		return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq,
2006				NULL, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
2007	}
2008}
2009
2010static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
2011{
2012	struct nvme_dev *dev = nvmeq->dev;
2013
2014	nvmeq->sq_tail = 0;
2015	nvmeq->last_sq_tail = 0;
2016	nvmeq->cq_head = 0;
2017	nvmeq->cq_phase = 1;
2018	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
2019	memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq));
2020	nvme_dbbuf_init(dev, nvmeq, qid);
2021	dev->online_queues++;
2022	wmb(); /* ensure the first interrupt sees the initialization */
2023}
2024
2025/*
2026 * Try getting shutdown_lock while setting up IO queues.
2027 */
2028static int nvme_setup_io_queues_trylock(struct nvme_dev *dev)
2029{
2030	/*
2031	 * Give up if the lock is being held by nvme_dev_disable.
2032	 */
2033	if (!mutex_trylock(&dev->shutdown_lock))
2034		return -ENODEV;
2035
2036	/*
2037	 * Controller is in wrong state, fail early.
2038	 */
2039	if (nvme_ctrl_state(&dev->ctrl) != NVME_CTRL_CONNECTING) {
2040		mutex_unlock(&dev->shutdown_lock);
2041		return -ENODEV;
2042	}
2043
2044	return 0;
2045}
2046
2047static int nvme_create_queue(struct nvme_queue *nvmeq, int qid, bool polled)
2048{
2049	struct nvme_dev *dev = nvmeq->dev;
2050	int result;
2051	u16 vector = 0;
2052
2053	clear_bit(NVMEQ_DELETE_ERROR, &nvmeq->flags);
2054
2055	/*
2056	 * A queue's vector matches the queue identifier unless the controller
2057	 * has only one vector available.
2058	 */
2059	if (!polled)
2060		vector = dev->num_vecs == 1 ? 0 : qid;
2061	else
2062		set_bit(NVMEQ_POLLED, &nvmeq->flags);
2063
2064	result = adapter_alloc_cq(dev, qid, nvmeq, vector);
2065	if (result)
2066		return result;
2067
2068	result = adapter_alloc_sq(dev, qid, nvmeq);
2069	if (result < 0)
2070		return result;
2071	if (result)
2072		goto release_cq;
2073
2074	nvmeq->cq_vector = vector;
2075
2076	result = nvme_setup_io_queues_trylock(dev);
2077	if (result)
2078		return result;
2079	nvme_init_queue(nvmeq, qid);
2080	if (!polled) {
2081		result = queue_request_irq(nvmeq);
2082		if (result < 0)
2083			goto release_sq;
2084	}
2085
2086	set_bit(NVMEQ_ENABLED, &nvmeq->flags);
2087	mutex_unlock(&dev->shutdown_lock);
2088	return result;
2089
2090release_sq:
2091	dev->online_queues--;
2092	mutex_unlock(&dev->shutdown_lock);
2093	adapter_delete_sq(dev, qid);
2094release_cq:
2095	adapter_delete_cq(dev, qid);
2096	return result;
2097}
2098
2099static const struct blk_mq_ops nvme_mq_admin_ops = {
2100	.queue_rq	= nvme_queue_rq,
2101	.complete	= nvme_pci_complete_rq,
2102	.init_hctx	= nvme_admin_init_hctx,
2103	.init_request	= nvme_pci_init_request,
2104	.timeout	= nvme_timeout,
2105};
2106
2107static const struct blk_mq_ops nvme_mq_ops = {
2108	.queue_rq	= nvme_queue_rq,
2109	.queue_rqs	= nvme_queue_rqs,
2110	.complete	= nvme_pci_complete_rq,
2111	.commit_rqs	= nvme_commit_rqs,
2112	.init_hctx	= nvme_init_hctx,
2113	.init_request	= nvme_pci_init_request,
2114	.map_queues	= nvme_pci_map_queues,
2115	.timeout	= nvme_timeout,
2116	.poll		= nvme_poll,
2117};
2118
2119static void nvme_dev_remove_admin(struct nvme_dev *dev)
2120{
2121	if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) {
2122		/*
2123		 * If the controller was reset during removal, it's possible
2124		 * user requests may be waiting on a stopped queue. Start the
2125		 * queue to flush these to completion.
2126		 */
2127		nvme_unquiesce_admin_queue(&dev->ctrl);
2128		nvme_remove_admin_tag_set(&dev->ctrl);
2129	}
2130}
2131
2132static unsigned long db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues)
2133{
2134	return NVME_REG_DBS + ((nr_io_queues + 1) * 8 * dev->db_stride);
2135}
2136
2137static int nvme_remap_bar(struct nvme_dev *dev, unsigned long size)
2138{
2139	struct pci_dev *pdev = to_pci_dev(dev->dev);
2140
2141	if (size <= dev->bar_mapped_size)
2142		return 0;
2143	if (size > pci_resource_len(pdev, 0))
2144		return -ENOMEM;
2145	if (dev->bar)
2146		iounmap(dev->bar);
2147	dev->bar = ioremap(pci_resource_start(pdev, 0), size);
2148	if (!dev->bar) {
2149		dev->bar_mapped_size = 0;
2150		return -ENOMEM;
2151	}
2152	dev->bar_mapped_size = size;
2153	dev->dbs = dev->bar + NVME_REG_DBS;
2154
2155	return 0;
2156}
2157
2158static int nvme_pci_configure_admin_queue(struct nvme_dev *dev)
2159{
2160	int result;
2161	u32 aqa;
2162	struct nvme_queue *nvmeq;
2163
2164	result = nvme_remap_bar(dev, db_bar_size(dev, 0));
2165	if (result < 0)
2166		return result;
2167
2168	dev->subsystem = readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 1, 0) ?
2169				NVME_CAP_NSSRC(dev->ctrl.cap) : 0;
2170
2171	if (dev->subsystem &&
2172	    (readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_NSSRO))
2173		writel(NVME_CSTS_NSSRO, dev->bar + NVME_REG_CSTS);
2174
2175	/*
2176	 * If the device has been passed off to us in an enabled state, just
2177	 * clear the enabled bit.  The spec says we should set the 'shutdown
2178	 * notification bits', but doing so may cause the device to complete
2179	 * commands to the admin queue ... and we don't know what memory that
2180	 * might be pointing at!
2181	 */
2182	result = nvme_disable_ctrl(&dev->ctrl, false);
2183	if (result < 0) {
2184		struct pci_dev *pdev = to_pci_dev(dev->dev);
2185
2186		/*
2187		 * The NVMe Controller Reset method did not get an expected
2188		 * CSTS.RDY transition, so something with the device appears to
2189		 * be stuck. Use the lower level and bigger hammer PCIe
2190		 * Function Level Reset to attempt restoring the device to its
2191		 * initial state, and try again.
2192		 */
2193		result = pcie_reset_flr(pdev, false);
2194		if (result < 0)
2195			return result;
2196
2197		pci_restore_state(pdev);
2198		result = nvme_disable_ctrl(&dev->ctrl, false);
2199		if (result < 0)
2200			return result;
2201
2202		dev_info(dev->ctrl.device,
2203			"controller reset completed after pcie flr\n");
2204	}
2205
2206	result = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH);
2207	if (result)
2208		return result;
2209
2210	dev->ctrl.numa_node = dev_to_node(dev->dev);
2211
2212	nvmeq = &dev->queues[0];
2213	aqa = nvmeq->q_depth - 1;
2214	aqa |= aqa << 16;
2215
2216	writel(aqa, dev->bar + NVME_REG_AQA);
2217	lo_hi_writeq(nvmeq->sq_dma_addr, dev->bar + NVME_REG_ASQ);
2218	lo_hi_writeq(nvmeq->cq_dma_addr, dev->bar + NVME_REG_ACQ);
2219
2220	result = nvme_enable_ctrl(&dev->ctrl);
2221	if (result)
2222		return result;
2223
2224	nvmeq->cq_vector = 0;
2225	nvme_init_queue(nvmeq, 0);
2226	result = queue_request_irq(nvmeq);
2227	if (result) {
2228		dev->online_queues--;
2229		return result;
2230	}
2231
2232	set_bit(NVMEQ_ENABLED, &nvmeq->flags);
2233	return result;
2234}
2235
2236static int nvme_create_io_queues(struct nvme_dev *dev)
2237{
2238	unsigned i, max, rw_queues;
2239	int ret = 0;
2240
2241	for (i = dev->ctrl.queue_count; i <= dev->max_qid; i++) {
2242		if (nvme_alloc_queue(dev, i, dev->q_depth)) {
2243			ret = -ENOMEM;
2244			break;
2245		}
2246	}
2247
2248	max = min(dev->max_qid, dev->ctrl.queue_count - 1);
2249	if (max != 1 && dev->io_queues[HCTX_TYPE_POLL]) {
2250		rw_queues = dev->io_queues[HCTX_TYPE_DEFAULT] +
2251				dev->io_queues[HCTX_TYPE_READ];
2252	} else {
2253		rw_queues = max;
2254	}
2255
2256	for (i = dev->online_queues; i <= max; i++) {
2257		bool polled = i > rw_queues;
2258
2259		ret = nvme_create_queue(&dev->queues[i], i, polled);
2260		if (ret)
2261			break;
2262	}
2263
2264	/*
2265	 * Ignore failing Create SQ/CQ commands, we can continue with less
2266	 * than the desired amount of queues, and even a controller without
2267	 * I/O queues can still be used to issue admin commands.  This might
2268	 * be useful to upgrade a buggy firmware for example.
2269	 */
2270	return ret >= 0 ? 0 : ret;
2271}
2272
2273static u64 nvme_cmb_size_unit(struct nvme_dev *dev)
2274{
2275	u8 szu = (dev->cmbsz >> NVME_CMBSZ_SZU_SHIFT) & NVME_CMBSZ_SZU_MASK;
2276
2277	return 1ULL << (12 + 4 * szu);
2278}
2279
2280static u32 nvme_cmb_size(struct nvme_dev *dev)
2281{
2282	return (dev->cmbsz >> NVME_CMBSZ_SZ_SHIFT) & NVME_CMBSZ_SZ_MASK;
2283}
2284
2285static void nvme_map_cmb(struct nvme_dev *dev)
2286{
2287	u64 size, offset;
2288	resource_size_t bar_size;
2289	struct pci_dev *pdev = to_pci_dev(dev->dev);
2290	int bar;
2291
2292	if (dev->cmb_size)
2293		return;
2294
2295	if (NVME_CAP_CMBS(dev->ctrl.cap))
2296		writel(NVME_CMBMSC_CRE, dev->bar + NVME_REG_CMBMSC);
2297
2298	dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ);
2299	if (!dev->cmbsz)
2300		return;
2301	dev->cmbloc = readl(dev->bar + NVME_REG_CMBLOC);
2302
2303	size = nvme_cmb_size_unit(dev) * nvme_cmb_size(dev);
2304	offset = nvme_cmb_size_unit(dev) * NVME_CMB_OFST(dev->cmbloc);
2305	bar = NVME_CMB_BIR(dev->cmbloc);
2306	bar_size = pci_resource_len(pdev, bar);
2307
2308	if (offset > bar_size)
2309		return;
2310
2311	/*
2312	 * Controllers may support a CMB size larger than their BAR, for
2313	 * example, due to being behind a bridge. Reduce the CMB to the
2314	 * reported size of the BAR
2315	 */
2316	size = min(size, bar_size - offset);
2317
2318	if (!IS_ALIGNED(size, memremap_compat_align()) ||
2319	    !IS_ALIGNED(pci_resource_start(pdev, bar),
2320			memremap_compat_align()))
2321		return;
2322
2323	/*
2324	 * Tell the controller about the host side address mapping the CMB,
2325	 * and enable CMB decoding for the NVMe 1.4+ scheme:
2326	 */
2327	if (NVME_CAP_CMBS(dev->ctrl.cap)) {
2328		hi_lo_writeq(NVME_CMBMSC_CRE | NVME_CMBMSC_CMSE |
2329			     (pci_bus_address(pdev, bar) + offset),
2330			     dev->bar + NVME_REG_CMBMSC);
2331	}
2332
2333	if (pci_p2pdma_add_resource(pdev, bar, size, offset)) {
2334		dev_warn(dev->ctrl.device,
2335			 "failed to register the CMB\n");
2336		hi_lo_writeq(0, dev->bar + NVME_REG_CMBMSC);
2337		return;
2338	}
2339
2340	dev->cmb_size = size;
2341	dev->cmb_use_sqes = use_cmb_sqes && (dev->cmbsz & NVME_CMBSZ_SQS);
2342
2343	if ((dev->cmbsz & (NVME_CMBSZ_WDS | NVME_CMBSZ_RDS)) ==
2344			(NVME_CMBSZ_WDS | NVME_CMBSZ_RDS))
2345		pci_p2pmem_publish(pdev, true);
2346}
2347
2348static int nvme_set_host_mem(struct nvme_dev *dev, u32 bits)
2349{
2350	u32 host_mem_size = dev->host_mem_size >> NVME_CTRL_PAGE_SHIFT;
2351	u64 dma_addr = dev->host_mem_descs_dma;
2352	struct nvme_command c = { };
2353	int ret;
2354
2355	c.features.opcode	= nvme_admin_set_features;
2356	c.features.fid		= cpu_to_le32(NVME_FEAT_HOST_MEM_BUF);
2357	c.features.dword11	= cpu_to_le32(bits);
2358	c.features.dword12	= cpu_to_le32(host_mem_size);
2359	c.features.dword13	= cpu_to_le32(lower_32_bits(dma_addr));
2360	c.features.dword14	= cpu_to_le32(upper_32_bits(dma_addr));
2361	c.features.dword15	= cpu_to_le32(dev->nr_host_mem_descs);
2362
2363	ret = nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
2364	if (ret) {
2365		dev_warn(dev->ctrl.device,
2366			 "failed to set host mem (err %d, flags %#x).\n",
2367			 ret, bits);
2368	} else
2369		dev->hmb = bits & NVME_HOST_MEM_ENABLE;
2370
2371	return ret;
2372}
2373
2374static void nvme_free_host_mem_multi(struct nvme_dev *dev)
2375{
2376	int i;
2377
2378	for (i = 0; i < dev->nr_host_mem_descs; i++) {
2379		struct nvme_host_mem_buf_desc *desc = &dev->host_mem_descs[i];
2380		size_t size = le32_to_cpu(desc->size) * NVME_CTRL_PAGE_SIZE;
2381
2382		dma_free_attrs(dev->dev, size, dev->host_mem_desc_bufs[i],
2383			       le64_to_cpu(desc->addr),
2384			       DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_NO_WARN);
2385	}
2386
2387	kfree(dev->host_mem_desc_bufs);
2388	dev->host_mem_desc_bufs = NULL;
2389}
2390
2391static void nvme_free_host_mem(struct nvme_dev *dev)
2392{
2393	if (dev->hmb_sgt)
2394		dma_free_noncontiguous(dev->dev, dev->host_mem_size,
2395				dev->hmb_sgt, DMA_BIDIRECTIONAL);
2396	else
2397		nvme_free_host_mem_multi(dev);
2398
2399	dma_free_coherent(dev->dev, dev->host_mem_descs_size,
2400			dev->host_mem_descs, dev->host_mem_descs_dma);
2401	dev->host_mem_descs = NULL;
2402	dev->host_mem_descs_size = 0;
2403	dev->nr_host_mem_descs = 0;
2404}
2405
2406static int nvme_alloc_host_mem_single(struct nvme_dev *dev, u64 size)
2407{
2408	dev->hmb_sgt = dma_alloc_noncontiguous(dev->dev, size,
2409				DMA_BIDIRECTIONAL, GFP_KERNEL, 0);
2410	if (!dev->hmb_sgt)
2411		return -ENOMEM;
2412
2413	dev->host_mem_descs = dma_alloc_coherent(dev->dev,
2414			sizeof(*dev->host_mem_descs), &dev->host_mem_descs_dma,
2415			GFP_KERNEL);
2416	if (!dev->host_mem_descs) {
2417		dma_free_noncontiguous(dev->dev, size, dev->hmb_sgt,
2418				DMA_BIDIRECTIONAL);
2419		dev->hmb_sgt = NULL;
2420		return -ENOMEM;
2421	}
2422	dev->host_mem_size = size;
2423	dev->host_mem_descs_size = sizeof(*dev->host_mem_descs);
2424	dev->nr_host_mem_descs = 1;
2425
2426	dev->host_mem_descs[0].addr =
2427		cpu_to_le64(dev->hmb_sgt->sgl->dma_address);
2428	dev->host_mem_descs[0].size = cpu_to_le32(size / NVME_CTRL_PAGE_SIZE);
2429	return 0;
2430}
2431
2432static int nvme_alloc_host_mem_multi(struct nvme_dev *dev, u64 preferred,
2433		u32 chunk_size)
2434{
2435	struct nvme_host_mem_buf_desc *descs;
2436	u32 max_entries, len, descs_size;
2437	dma_addr_t descs_dma;
2438	int i = 0;
2439	void **bufs;
2440	u64 size, tmp;
2441
2442	tmp = (preferred + chunk_size - 1);
2443	do_div(tmp, chunk_size);
2444	max_entries = tmp;
2445
2446	if (dev->ctrl.hmmaxd && dev->ctrl.hmmaxd < max_entries)
2447		max_entries = dev->ctrl.hmmaxd;
2448
2449	descs_size = max_entries * sizeof(*descs);
2450	descs = dma_alloc_coherent(dev->dev, descs_size, &descs_dma,
2451			GFP_KERNEL);
2452	if (!descs)
2453		goto out;
2454
2455	bufs = kcalloc(max_entries, sizeof(*bufs), GFP_KERNEL);
2456	if (!bufs)
2457		goto out_free_descs;
2458
2459	for (size = 0; size < preferred && i < max_entries; size += len) {
2460		dma_addr_t dma_addr;
2461
2462		len = min_t(u64, chunk_size, preferred - size);
2463		bufs[i] = dma_alloc_attrs(dev->dev, len, &dma_addr, GFP_KERNEL,
2464				DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_NO_WARN);
2465		if (!bufs[i])
2466			break;
2467
2468		descs[i].addr = cpu_to_le64(dma_addr);
2469		descs[i].size = cpu_to_le32(len / NVME_CTRL_PAGE_SIZE);
2470		i++;
2471	}
2472
2473	if (!size)
2474		goto out_free_bufs;
2475
2476	dev->nr_host_mem_descs = i;
2477	dev->host_mem_size = size;
2478	dev->host_mem_descs = descs;
2479	dev->host_mem_descs_dma = descs_dma;
2480	dev->host_mem_descs_size = descs_size;
2481	dev->host_mem_desc_bufs = bufs;
2482	return 0;
2483
2484out_free_bufs:
2485	kfree(bufs);
2486out_free_descs:
2487	dma_free_coherent(dev->dev, descs_size, descs, descs_dma);
2488out:
2489	dev->host_mem_descs = NULL;
2490	return -ENOMEM;
2491}
2492
2493static int nvme_alloc_host_mem(struct nvme_dev *dev, u64 min, u64 preferred)
2494{
2495	unsigned long dma_merge_boundary = dma_get_merge_boundary(dev->dev);
2496	u64 min_chunk = min_t(u64, preferred, PAGE_SIZE * MAX_ORDER_NR_PAGES);
2497	u64 hmminds = max_t(u32, dev->ctrl.hmminds * 4096, PAGE_SIZE * 2);
2498	u64 chunk_size;
2499
2500	/*
2501	 * If there is an IOMMU that can merge pages, try a virtually
2502	 * non-contiguous allocation for a single segment first.
2503	 */
2504	if (dma_merge_boundary && (PAGE_SIZE & dma_merge_boundary) == 0) {
2505		if (!nvme_alloc_host_mem_single(dev, preferred))
2506			return 0;
2507	}
2508
2509	/* start big and work our way down */
2510	for (chunk_size = min_chunk; chunk_size >= hmminds; chunk_size /= 2) {
2511		if (!nvme_alloc_host_mem_multi(dev, preferred, chunk_size)) {
2512			if (!min || dev->host_mem_size >= min)
2513				return 0;
2514			nvme_free_host_mem(dev);
2515		}
2516	}
2517
2518	return -ENOMEM;
2519}
2520
2521static int nvme_setup_host_mem(struct nvme_dev *dev)
2522{
2523	u64 max = (u64)max_host_mem_size_mb * SZ_1M;
2524	u64 preferred = (u64)dev->ctrl.hmpre * 4096;
2525	u64 min = (u64)dev->ctrl.hmmin * 4096;
2526	u32 enable_bits = NVME_HOST_MEM_ENABLE;
2527	int ret;
2528
2529	if (!dev->ctrl.hmpre)
2530		return 0;
2531
2532	preferred = min(preferred, max);
2533	if (min > max) {
2534		dev_warn(dev->ctrl.device,
2535			"min host memory (%lld MiB) above limit (%d MiB).\n",
2536			min >> ilog2(SZ_1M), max_host_mem_size_mb);
2537		nvme_free_host_mem(dev);
2538		return 0;
2539	}
2540
2541	/*
2542	 * If we already have a buffer allocated check if we can reuse it.
2543	 */
2544	if (dev->host_mem_descs) {
2545		if (dev->host_mem_size >= min)
2546			enable_bits |= NVME_HOST_MEM_RETURN;
2547		else
2548			nvme_free_host_mem(dev);
2549	}
2550
2551	if (!dev->host_mem_descs) {
2552		if (nvme_alloc_host_mem(dev, min, preferred)) {
2553			dev_warn(dev->ctrl.device,
2554				"failed to allocate host memory buffer.\n");
2555			return 0; /* controller must work without HMB */
2556		}
2557
2558		dev_info(dev->ctrl.device,
2559			"allocated %lld MiB host memory buffer (%u segment%s).\n",
2560			dev->host_mem_size >> ilog2(SZ_1M),
2561			dev->nr_host_mem_descs,
2562			str_plural(dev->nr_host_mem_descs));
2563	}
2564
2565	ret = nvme_set_host_mem(dev, enable_bits);
2566	if (ret)
2567		nvme_free_host_mem(dev);
2568	return ret;
2569}
2570
2571static ssize_t cmb_show(struct device *dev, struct device_attribute *attr,
2572		char *buf)
2573{
2574	struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
2575
2576	return sysfs_emit(buf, "cmbloc : 0x%08x\ncmbsz  : 0x%08x\n",
2577		       ndev->cmbloc, ndev->cmbsz);
2578}
2579static DEVICE_ATTR_RO(cmb);
2580
2581static ssize_t cmbloc_show(struct device *dev, struct device_attribute *attr,
2582		char *buf)
2583{
2584	struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
2585
2586	return sysfs_emit(buf, "%u\n", ndev->cmbloc);
2587}
2588static DEVICE_ATTR_RO(cmbloc);
2589
2590static ssize_t cmbsz_show(struct device *dev, struct device_attribute *attr,
2591		char *buf)
2592{
2593	struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
2594
2595	return sysfs_emit(buf, "%u\n", ndev->cmbsz);
2596}
2597static DEVICE_ATTR_RO(cmbsz);
2598
2599static ssize_t hmb_show(struct device *dev, struct device_attribute *attr,
2600			char *buf)
2601{
2602	struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
2603
2604	return sysfs_emit(buf, "%d\n", ndev->hmb);
2605}
2606
2607static ssize_t hmb_store(struct device *dev, struct device_attribute *attr,
2608			 const char *buf, size_t count)
2609{
2610	struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
2611	bool new;
2612	int ret;
2613
2614	if (kstrtobool(buf, &new) < 0)
2615		return -EINVAL;
2616
2617	if (new == ndev->hmb)
2618		return count;
2619
2620	if (new) {
2621		ret = nvme_setup_host_mem(ndev);
2622	} else {
2623		ret = nvme_set_host_mem(ndev, 0);
2624		if (!ret)
2625			nvme_free_host_mem(ndev);
2626	}
2627
2628	if (ret < 0)
2629		return ret;
2630
2631	return count;
2632}
2633static DEVICE_ATTR_RW(hmb);
2634
2635static umode_t nvme_pci_attrs_are_visible(struct kobject *kobj,
2636		struct attribute *a, int n)
2637{
2638	struct nvme_ctrl *ctrl =
2639		dev_get_drvdata(container_of(kobj, struct device, kobj));
2640	struct nvme_dev *dev = to_nvme_dev(ctrl);
2641
2642	if (a == &dev_attr_cmb.attr ||
2643	    a == &dev_attr_cmbloc.attr ||
2644	    a == &dev_attr_cmbsz.attr) {
2645	    	if (!dev->cmbsz)
2646			return 0;
2647	}
2648	if (a == &dev_attr_hmb.attr && !ctrl->hmpre)
2649		return 0;
2650
2651	return a->mode;
2652}
2653
2654static struct attribute *nvme_pci_attrs[] = {
2655	&dev_attr_cmb.attr,
2656	&dev_attr_cmbloc.attr,
2657	&dev_attr_cmbsz.attr,
2658	&dev_attr_hmb.attr,
2659	NULL,
2660};
2661
2662static const struct attribute_group nvme_pci_dev_attrs_group = {
2663	.attrs		= nvme_pci_attrs,
2664	.is_visible	= nvme_pci_attrs_are_visible,
2665};
2666
2667static const struct attribute_group *nvme_pci_dev_attr_groups[] = {
2668	&nvme_dev_attrs_group,
2669	&nvme_pci_dev_attrs_group,
2670	NULL,
2671};
2672
2673static void nvme_update_attrs(struct nvme_dev *dev)
2674{
2675	sysfs_update_group(&dev->ctrl.device->kobj, &nvme_pci_dev_attrs_group);
2676}
2677
2678/*
2679 * nirqs is the number of interrupts available for write and read
2680 * queues. The core already reserved an interrupt for the admin queue.
2681 */
2682static void nvme_calc_irq_sets(struct irq_affinity *affd, unsigned int nrirqs)
2683{
2684	struct nvme_dev *dev = affd->priv;
2685	unsigned int nr_read_queues, nr_write_queues = dev->nr_write_queues;
2686
2687	/*
2688	 * If there is no interrupt available for queues, ensure that
2689	 * the default queue is set to 1. The affinity set size is
2690	 * also set to one, but the irq core ignores it for this case.
2691	 *
2692	 * If only one interrupt is available or 'write_queue' == 0, combine
2693	 * write and read queues.
2694	 *
2695	 * If 'write_queues' > 0, ensure it leaves room for at least one read
2696	 * queue.
2697	 */
2698	if (!nrirqs) {
2699		nrirqs = 1;
2700		nr_read_queues = 0;
2701	} else if (nrirqs == 1 || !nr_write_queues) {
2702		nr_read_queues = 0;
2703	} else if (nr_write_queues >= nrirqs) {
2704		nr_read_queues = 1;
2705	} else {
2706		nr_read_queues = nrirqs - nr_write_queues;
2707	}
2708
2709	dev->io_queues[HCTX_TYPE_DEFAULT] = nrirqs - nr_read_queues;
2710	affd->set_size[HCTX_TYPE_DEFAULT] = nrirqs - nr_read_queues;
2711	dev->io_queues[HCTX_TYPE_READ] = nr_read_queues;
2712	affd->set_size[HCTX_TYPE_READ] = nr_read_queues;
2713	affd->nr_sets = nr_read_queues ? 2 : 1;
2714}
2715
2716static int nvme_setup_irqs(struct nvme_dev *dev, unsigned int nr_io_queues)
2717{
2718	struct pci_dev *pdev = to_pci_dev(dev->dev);
2719	struct irq_affinity affd = {
2720		.pre_vectors	= 1,
2721		.calc_sets	= nvme_calc_irq_sets,
2722		.priv		= dev,
2723	};
2724	unsigned int irq_queues, poll_queues;
2725	unsigned int flags = PCI_IRQ_ALL_TYPES | PCI_IRQ_AFFINITY;
2726
2727	/*
2728	 * Poll queues don't need interrupts, but we need at least one I/O queue
2729	 * left over for non-polled I/O.
2730	 */
2731	poll_queues = min(dev->nr_poll_queues, nr_io_queues - 1);
2732	dev->io_queues[HCTX_TYPE_POLL] = poll_queues;
2733
2734	/*
2735	 * Initialize for the single interrupt case, will be updated in
2736	 * nvme_calc_irq_sets().
2737	 */
2738	dev->io_queues[HCTX_TYPE_DEFAULT] = 1;
2739	dev->io_queues[HCTX_TYPE_READ] = 0;
2740
2741	/*
2742	 * We need interrupts for the admin queue and each non-polled I/O queue,
2743	 * but some Apple controllers require all queues to use the first
2744	 * vector.
2745	 */
2746	irq_queues = 1;
2747	if (!(dev->ctrl.quirks & NVME_QUIRK_SINGLE_VECTOR))
2748		irq_queues += (nr_io_queues - poll_queues);
2749	if (dev->ctrl.quirks & NVME_QUIRK_BROKEN_MSI)
2750		flags &= ~PCI_IRQ_MSI;
2751	return pci_alloc_irq_vectors_affinity(pdev, 1, irq_queues, flags,
2752					      &affd);
2753}
2754
2755static unsigned int nvme_max_io_queues(struct nvme_dev *dev)
2756{
2757	/*
2758	 * If tags are shared with admin queue (Apple bug), then
2759	 * make sure we only use one IO queue.
2760	 */
2761	if (dev->ctrl.quirks & NVME_QUIRK_SHARED_TAGS)
2762		return 1;
2763	return blk_mq_num_possible_queues(0) + dev->nr_write_queues +
2764		dev->nr_poll_queues;
2765}
2766
2767static int nvme_setup_io_queues(struct nvme_dev *dev)
2768{
2769	struct nvme_queue *adminq = &dev->queues[0];
2770	struct pci_dev *pdev = to_pci_dev(dev->dev);
2771	unsigned int nr_io_queues;
2772	unsigned long size;
2773	int result;
2774
2775	/*
2776	 * Sample the module parameters once at reset time so that we have
2777	 * stable values to work with.
2778	 */
2779	dev->nr_write_queues = write_queues;
2780	dev->nr_poll_queues = poll_queues;
2781
2782	/*
2783	 * The initial number of allocated queue slots may be too large if the
2784	 * user reduced the special queue parameters. Cap the value to the
2785	 * number we need for this round.
2786	 */
2787	nr_io_queues = min(nvme_max_io_queues(dev),
2788			   dev->nr_allocated_queues - 1);
2789	result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues);
2790	if (result < 0)
2791		return result;
2792
2793	if (nr_io_queues == 0)
2794		return 0;
2795
2796	/*
2797	 * Free IRQ resources as soon as NVMEQ_ENABLED bit transitions
2798	 * from set to unset. If there is a window to it is truely freed,
2799	 * pci_free_irq_vectors() jumping into this window will crash.
2800	 * And take lock to avoid racing with pci_free_irq_vectors() in
2801	 * nvme_dev_disable() path.
2802	 */
2803	result = nvme_setup_io_queues_trylock(dev);
2804	if (result)
2805		return result;
2806	if (test_and_clear_bit(NVMEQ_ENABLED, &adminq->flags))
2807		pci_free_irq(pdev, 0, adminq);
2808
2809	if (dev->cmb_use_sqes) {
2810		result = nvme_cmb_qdepth(dev, nr_io_queues,
2811				sizeof(struct nvme_command));
2812		if (result > 0) {
2813			dev->q_depth = result;
2814			dev->ctrl.sqsize = result - 1;
2815		} else {
2816			dev->cmb_use_sqes = false;
2817		}
2818	}
2819
2820	do {
2821		size = db_bar_size(dev, nr_io_queues);
2822		result = nvme_remap_bar(dev, size);
2823		if (!result)
2824			break;
2825		if (!--nr_io_queues) {
2826			result = -ENOMEM;
2827			goto out_unlock;
2828		}
2829	} while (1);
2830	adminq->q_db = dev->dbs;
2831
2832 retry:
2833	/* Deregister the admin queue's interrupt */
2834	if (test_and_clear_bit(NVMEQ_ENABLED, &adminq->flags))
2835		pci_free_irq(pdev, 0, adminq);
2836
2837	/*
2838	 * If we enable msix early due to not intx, disable it again before
2839	 * setting up the full range we need.
2840	 */
2841	pci_free_irq_vectors(pdev);
2842
2843	result = nvme_setup_irqs(dev, nr_io_queues);
2844	if (result <= 0) {
2845		result = -EIO;
2846		goto out_unlock;
2847	}
2848
2849	dev->num_vecs = result;
2850	result = max(result - 1, 1);
2851	dev->max_qid = result + dev->io_queues[HCTX_TYPE_POLL];
2852
2853	/*
2854	 * Should investigate if there's a performance win from allocating
2855	 * more queues than interrupt vectors; it might allow the submission
2856	 * path to scale better, even if the receive path is limited by the
2857	 * number of interrupts.
2858	 */
2859	result = queue_request_irq(adminq);
2860	if (result)
2861		goto out_unlock;
2862	set_bit(NVMEQ_ENABLED, &adminq->flags);
2863	mutex_unlock(&dev->shutdown_lock);
2864
2865	result = nvme_create_io_queues(dev);
2866	if (result || dev->online_queues < 2)
2867		return result;
2868
2869	if (dev->online_queues - 1 < dev->max_qid) {
2870		nr_io_queues = dev->online_queues - 1;
2871		nvme_delete_io_queues(dev);
2872		result = nvme_setup_io_queues_trylock(dev);
2873		if (result)
2874			return result;
2875		nvme_suspend_io_queues(dev);
2876		goto retry;
2877	}
2878	dev_info(dev->ctrl.device, "%d/%d/%d default/read/poll queues\n",
2879					dev->io_queues[HCTX_TYPE_DEFAULT],
2880					dev->io_queues[HCTX_TYPE_READ],
2881					dev->io_queues[HCTX_TYPE_POLL]);
2882	return 0;
2883out_unlock:
2884	mutex_unlock(&dev->shutdown_lock);
2885	return result;
2886}
2887
2888static enum rq_end_io_ret nvme_del_queue_end(struct request *req,
2889					     blk_status_t error)
2890{
2891	struct nvme_queue *nvmeq = req->end_io_data;
2892
2893	blk_mq_free_request(req);
2894	complete(&nvmeq->delete_done);
2895	return RQ_END_IO_NONE;
2896}
2897
2898static enum rq_end_io_ret nvme_del_cq_end(struct request *req,
2899					  blk_status_t error)
2900{
2901	struct nvme_queue *nvmeq = req->end_io_data;
2902
2903	if (error)
2904		set_bit(NVMEQ_DELETE_ERROR, &nvmeq->flags);
2905
2906	return nvme_del_queue_end(req, error);
2907}
2908
2909static int nvme_delete_queue(struct nvme_queue *nvmeq, u8 opcode)
2910{
2911	struct request_queue *q = nvmeq->dev->ctrl.admin_q;
2912	struct request *req;
2913	struct nvme_command cmd = { };
2914
2915	cmd.delete_queue.opcode = opcode;
2916	cmd.delete_queue.qid = cpu_to_le16(nvmeq->qid);
2917
2918	req = blk_mq_alloc_request(q, nvme_req_op(&cmd), BLK_MQ_REQ_NOWAIT);
2919	if (IS_ERR(req))
2920		return PTR_ERR(req);
2921	nvme_init_request(req, &cmd);
2922
2923	if (opcode == nvme_admin_delete_cq)
2924		req->end_io = nvme_del_cq_end;
2925	else
2926		req->end_io = nvme_del_queue_end;
2927	req->end_io_data = nvmeq;
2928
2929	init_completion(&nvmeq->delete_done);
2930	blk_execute_rq_nowait(req, false);
2931	return 0;
2932}
2933
2934static bool __nvme_delete_io_queues(struct nvme_dev *dev, u8 opcode)
2935{
2936	int nr_queues = dev->online_queues - 1, sent = 0;
2937	unsigned long timeout;
2938
2939 retry:
2940	timeout = NVME_ADMIN_TIMEOUT;
2941	while (nr_queues > 0) {
2942		if (nvme_delete_queue(&dev->queues[nr_queues], opcode))
2943			break;
2944		nr_queues--;
2945		sent++;
2946	}
2947	while (sent) {
2948		struct nvme_queue *nvmeq = &dev->queues[nr_queues + sent];
2949
2950		timeout = wait_for_completion_io_timeout(&nvmeq->delete_done,
2951				timeout);
2952		if (timeout == 0)
2953			return false;
2954
2955		sent--;
2956		if (nr_queues)
2957			goto retry;
2958	}
2959	return true;
2960}
2961
2962static void nvme_delete_io_queues(struct nvme_dev *dev)
2963{
2964	if (__nvme_delete_io_queues(dev, nvme_admin_delete_sq))
2965		__nvme_delete_io_queues(dev, nvme_admin_delete_cq);
2966}
2967
2968static unsigned int nvme_pci_nr_maps(struct nvme_dev *dev)
2969{
2970	if (dev->io_queues[HCTX_TYPE_POLL])
2971		return 3;
2972	if (dev->io_queues[HCTX_TYPE_READ])
2973		return 2;
2974	return 1;
2975}
2976
2977static bool nvme_pci_update_nr_queues(struct nvme_dev *dev)
2978{
2979	if (!dev->ctrl.tagset) {
2980		nvme_alloc_io_tag_set(&dev->ctrl, &dev->tagset, &nvme_mq_ops,
2981				nvme_pci_nr_maps(dev), sizeof(struct nvme_iod));
2982		return true;
2983	}
2984
2985	/* Give up if we are racing with nvme_dev_disable() */
2986	if (!mutex_trylock(&dev->shutdown_lock))
2987		return false;
2988
2989	/* Check if nvme_dev_disable() has been executed already */
2990	if (!dev->online_queues) {
2991		mutex_unlock(&dev->shutdown_lock);
2992		return false;
2993	}
2994
2995	blk_mq_update_nr_hw_queues(&dev->tagset, dev->online_queues - 1);
2996	/* free previously allocated queues that are no longer usable */
2997	nvme_free_queues(dev, dev->online_queues);
2998	mutex_unlock(&dev->shutdown_lock);
2999	return true;
3000}
3001
3002static int nvme_pci_enable(struct nvme_dev *dev)
3003{
3004	int result = -ENOMEM;
3005	struct pci_dev *pdev = to_pci_dev(dev->dev);
3006	unsigned int flags = PCI_IRQ_ALL_TYPES;
3007
3008	if (pci_enable_device_mem(pdev))
3009		return result;
3010
3011	pci_set_master(pdev);
3012
3013	if (readl(dev->bar + NVME_REG_CSTS) == -1) {
3014		dev_dbg(dev->ctrl.device, "reading CSTS register failed\n");
3015		result = -ENODEV;
3016		goto disable;
3017	}
3018
3019	/*
3020	 * Some devices and/or platforms don't advertise or work with INTx
3021	 * interrupts. Pre-enable a single MSIX or MSI vec for setup. We'll
3022	 * adjust this later.
3023	 */
3024	if (dev->ctrl.quirks & NVME_QUIRK_BROKEN_MSI)
3025		flags &= ~PCI_IRQ_MSI;
3026	result = pci_alloc_irq_vectors(pdev, 1, 1, flags);
3027	if (result < 0)
3028		goto disable;
3029
3030	dev->ctrl.cap = lo_hi_readq(dev->bar + NVME_REG_CAP);
3031
3032	dev->q_depth = min_t(u32, NVME_CAP_MQES(dev->ctrl.cap) + 1,
3033				io_queue_depth);
3034	dev->db_stride = 1 << NVME_CAP_STRIDE(dev->ctrl.cap);
3035	dev->dbs = dev->bar + 4096;
3036
3037	/*
3038	 * Some Apple controllers require a non-standard SQE size.
3039	 * Interestingly they also seem to ignore the CC:IOSQES register
3040	 * so we don't bother updating it here.
3041	 */
3042	if (dev->ctrl.quirks & NVME_QUIRK_128_BYTES_SQES)
3043		dev->io_sqes = 7;
3044	else
3045		dev->io_sqes = NVME_NVM_IOSQES;
3046
3047	if (dev->ctrl.quirks & NVME_QUIRK_QDEPTH_ONE) {
3048		dev->q_depth = 2;
3049	} else if (pdev->vendor == PCI_VENDOR_ID_SAMSUNG &&
3050		   (pdev->device == 0xa821 || pdev->device == 0xa822) &&
3051		   NVME_CAP_MQES(dev->ctrl.cap) == 0) {
3052		dev->q_depth = 64;
3053		dev_err(dev->ctrl.device, "detected PM1725 NVMe controller, "
3054                        "set queue depth=%u\n", dev->q_depth);
3055	}
3056
3057	/*
3058	 * Controllers with the shared tags quirk need the IO queue to be
3059	 * big enough so that we get 32 tags for the admin queue
3060	 */
3061	if ((dev->ctrl.quirks & NVME_QUIRK_SHARED_TAGS) &&
3062	    (dev->q_depth < (NVME_AQ_DEPTH + 2))) {
3063		dev->q_depth = NVME_AQ_DEPTH + 2;
3064		dev_warn(dev->ctrl.device, "IO queue depth clamped to %d\n",
3065			 dev->q_depth);
3066	}
3067	dev->ctrl.sqsize = dev->q_depth - 1; /* 0's based queue depth */
3068
3069	nvme_map_cmb(dev);
3070
3071	pci_save_state(pdev);
3072
3073	result = nvme_pci_configure_admin_queue(dev);
3074	if (result)
3075		goto free_irq;
3076	return result;
3077
3078 free_irq:
3079	pci_free_irq_vectors(pdev);
3080 disable:
3081	pci_disable_device(pdev);
3082	return result;
3083}
3084
3085static void nvme_dev_unmap(struct nvme_dev *dev)
3086{
3087	if (dev->bar)
3088		iounmap(dev->bar);
3089	pci_release_mem_regions(to_pci_dev(dev->dev));
3090}
3091
3092static bool nvme_pci_ctrl_is_dead(struct nvme_dev *dev)
3093{
3094	struct pci_dev *pdev = to_pci_dev(dev->dev);
3095	u32 csts;
3096
3097	if (!pci_is_enabled(pdev) || !pci_device_is_present(pdev))
3098		return true;
3099	if (pdev->error_state != pci_channel_io_normal)
3100		return true;
3101
3102	csts = readl(dev->bar + NVME_REG_CSTS);
3103	return (csts & NVME_CSTS_CFS) || !(csts & NVME_CSTS_RDY);
3104}
3105
3106static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown)
3107{
3108	enum nvme_ctrl_state state = nvme_ctrl_state(&dev->ctrl);
3109	struct pci_dev *pdev = to_pci_dev(dev->dev);
3110	bool dead;
3111
3112	mutex_lock(&dev->shutdown_lock);
3113	dead = nvme_pci_ctrl_is_dead(dev);
3114	if (state == NVME_CTRL_LIVE || state == NVME_CTRL_RESETTING) {
3115		if (pci_is_enabled(pdev))
3116			nvme_start_freeze(&dev->ctrl);
3117		/*
3118		 * Give the controller a chance to complete all entered requests
3119		 * if doing a safe shutdown.
3120		 */
3121		if (!dead && shutdown)
3122			nvme_wait_freeze_timeout(&dev->ctrl, NVME_IO_TIMEOUT);
3123	}
3124
3125	nvme_quiesce_io_queues(&dev->ctrl);
3126
3127	if (!dead && dev->ctrl.queue_count > 0) {
3128		nvme_delete_io_queues(dev);
3129		nvme_disable_ctrl(&dev->ctrl, shutdown);
3130		nvme_poll_irqdisable(&dev->queues[0]);
3131	}
3132	nvme_suspend_io_queues(dev);
3133	nvme_suspend_queue(dev, 0);
3134	pci_free_irq_vectors(pdev);
3135	if (pci_is_enabled(pdev))
3136		pci_disable_device(pdev);
3137	nvme_reap_pending_cqes(dev);
3138
3139	nvme_cancel_tagset(&dev->ctrl);
3140	nvme_cancel_admin_tagset(&dev->ctrl);
3141
3142	/*
3143	 * The driver will not be starting up queues again if shutting down so
3144	 * must flush all entered requests to their failed completion to avoid
3145	 * deadlocking blk-mq hot-cpu notifier.
3146	 */
3147	if (shutdown) {
3148		nvme_unquiesce_io_queues(&dev->ctrl);
3149		if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q))
3150			nvme_unquiesce_admin_queue(&dev->ctrl);
3151	}
3152	mutex_unlock(&dev->shutdown_lock);
3153}
3154
3155static int nvme_disable_prepare_reset(struct nvme_dev *dev, bool shutdown)
3156{
3157	if (!nvme_wait_reset(&dev->ctrl))
3158		return -EBUSY;
3159	nvme_dev_disable(dev, shutdown);
3160	return 0;
3161}
3162
3163static int nvme_pci_alloc_iod_mempool(struct nvme_dev *dev)
3164{
3165	size_t alloc_size = sizeof(struct nvme_dma_vec) * NVME_MAX_SEGS;
3166
3167	dev->dmavec_mempool = mempool_create_node(1,
3168			mempool_kmalloc, mempool_kfree,
3169			(void *)alloc_size, GFP_KERNEL,
3170			dev_to_node(dev->dev));
3171	if (!dev->dmavec_mempool)
3172		return -ENOMEM;
3173	return 0;
3174}
3175
3176static void nvme_free_tagset(struct nvme_dev *dev)
3177{
3178	if (dev->tagset.tags)
3179		nvme_remove_io_tag_set(&dev->ctrl);
3180	dev->ctrl.tagset = NULL;
3181}
3182
3183/* pairs with nvme_pci_alloc_dev */
3184static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl)
3185{
3186	struct nvme_dev *dev = to_nvme_dev(ctrl);
3187
3188	nvme_free_tagset(dev);
3189	put_device(dev->dev);
3190	kfree(dev->queues);
3191	kfree(dev);
3192}
3193
3194static void nvme_reset_work(struct work_struct *work)
3195{
3196	struct nvme_dev *dev =
3197		container_of(work, struct nvme_dev, ctrl.reset_work);
3198	bool was_suspend = !!(dev->ctrl.ctrl_config & NVME_CC_SHN_NORMAL);
3199	int result;
3200
3201	if (nvme_ctrl_state(&dev->ctrl) != NVME_CTRL_RESETTING) {
3202		dev_warn(dev->ctrl.device, "ctrl state %d is not RESETTING\n",
3203			 dev->ctrl.state);
3204		result = -ENODEV;
3205		goto out;
3206	}
3207
3208	/*
3209	 * If we're called to reset a live controller first shut it down before
3210	 * moving on.
3211	 */
3212	if (dev->ctrl.ctrl_config & NVME_CC_ENABLE)
3213		nvme_dev_disable(dev, false);
3214	nvme_sync_queues(&dev->ctrl);
3215
3216	mutex_lock(&dev->shutdown_lock);
3217	result = nvme_pci_enable(dev);
3218	if (result)
3219		goto out_unlock;
3220	nvme_unquiesce_admin_queue(&dev->ctrl);
3221	mutex_unlock(&dev->shutdown_lock);
3222
3223	/*
3224	 * Introduce CONNECTING state from nvme-fc/rdma transports to mark the
3225	 * initializing procedure here.
3226	 */
3227	if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_CONNECTING)) {
3228		dev_warn(dev->ctrl.device,
3229			"failed to mark controller CONNECTING\n");
3230		result = -EBUSY;
3231		goto out;
3232	}
3233
3234	result = nvme_init_ctrl_finish(&dev->ctrl, was_suspend);
3235	if (result)
3236		goto out;
3237
3238	if (nvme_ctrl_meta_sgl_supported(&dev->ctrl))
3239		dev->ctrl.max_integrity_segments = NVME_MAX_META_SEGS;
3240	else
3241		dev->ctrl.max_integrity_segments = 1;
3242
3243	nvme_dbbuf_dma_alloc(dev);
3244
3245	result = nvme_setup_host_mem(dev);
3246	if (result < 0)
3247		goto out;
3248
3249	nvme_update_attrs(dev);
3250
3251	result = nvme_setup_io_queues(dev);
3252	if (result)
3253		goto out;
3254
3255	/*
3256	 * Freeze and update the number of I/O queues as those might have
3257	 * changed.  If there are no I/O queues left after this reset, keep the
3258	 * controller around but remove all namespaces.
3259	 */
3260	if (dev->online_queues > 1) {
3261		nvme_dbbuf_set(dev);
3262		nvme_unquiesce_io_queues(&dev->ctrl);
3263		nvme_wait_freeze(&dev->ctrl);
3264		if (!nvme_pci_update_nr_queues(dev))
3265			goto out;
3266		nvme_unfreeze(&dev->ctrl);
3267	} else {
3268		dev_warn(dev->ctrl.device, "IO queues lost\n");
3269		nvme_mark_namespaces_dead(&dev->ctrl);
3270		nvme_unquiesce_io_queues(&dev->ctrl);
3271		nvme_remove_namespaces(&dev->ctrl);
3272		nvme_free_tagset(dev);
3273	}
3274
3275	/*
3276	 * If only admin queue live, keep it to do further investigation or
3277	 * recovery.
3278	 */
3279	if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_LIVE)) {
3280		dev_warn(dev->ctrl.device,
3281			"failed to mark controller live state\n");
3282		result = -ENODEV;
3283		goto out;
3284	}
3285
3286	nvme_start_ctrl(&dev->ctrl);
3287	return;
3288
3289 out_unlock:
3290	mutex_unlock(&dev->shutdown_lock);
3291 out:
3292	/*
3293	 * Set state to deleting now to avoid blocking nvme_wait_reset(), which
3294	 * may be holding this pci_dev's device lock.
3295	 */
3296	dev_warn(dev->ctrl.device, "Disabling device after reset failure: %d\n",
3297		 result);
3298	nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING);
3299	nvme_dev_disable(dev, true);
3300	nvme_sync_queues(&dev->ctrl);
3301	nvme_mark_namespaces_dead(&dev->ctrl);
3302	nvme_unquiesce_io_queues(&dev->ctrl);
3303	nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DEAD);
3304}
3305
3306static int nvme_pci_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
3307{
3308	*val = readl(to_nvme_dev(ctrl)->bar + off);
3309	return 0;
3310}
3311
3312static int nvme_pci_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val)
3313{
3314	writel(val, to_nvme_dev(ctrl)->bar + off);
3315	return 0;
3316}
3317
3318static int nvme_pci_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val)
3319{
3320	*val = lo_hi_readq(to_nvme_dev(ctrl)->bar + off);
3321	return 0;
3322}
3323
3324static int nvme_pci_get_address(struct nvme_ctrl *ctrl, char *buf, int size)
3325{
3326	struct pci_dev *pdev = to_pci_dev(to_nvme_dev(ctrl)->dev);
3327
3328	return snprintf(buf, size, "%s\n", dev_name(&pdev->dev));
3329}
3330
3331static void nvme_pci_print_device_info(struct nvme_ctrl *ctrl)
3332{
3333	struct pci_dev *pdev = to_pci_dev(to_nvme_dev(ctrl)->dev);
3334	struct nvme_subsystem *subsys = ctrl->subsys;
3335
3336	dev_err(ctrl->device,
3337		"VID:DID %04x:%04x model:%.*s firmware:%.*s\n",
3338		pdev->vendor, pdev->device,
3339		nvme_strlen(subsys->model, sizeof(subsys->model)),
3340		subsys->model, nvme_strlen(subsys->firmware_rev,
3341					   sizeof(subsys->firmware_rev)),
3342		subsys->firmware_rev);
3343}
3344
3345static bool nvme_pci_supports_pci_p2pdma(struct nvme_ctrl *ctrl)
3346{
3347	struct nvme_dev *dev = to_nvme_dev(ctrl);
3348
3349	return dma_pci_p2pdma_supported(dev->dev);
3350}
3351
3352static unsigned long nvme_pci_get_virt_boundary(struct nvme_ctrl *ctrl,
3353						bool is_admin)
3354{
3355	if (!nvme_ctrl_sgl_supported(ctrl) || is_admin)
3356		return NVME_CTRL_PAGE_SIZE - 1;
3357	return 0;
3358}
3359
3360static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
3361	.name			= "pcie",
3362	.module			= THIS_MODULE,
3363	.flags			= NVME_F_METADATA_SUPPORTED,
3364	.dev_attr_groups	= nvme_pci_dev_attr_groups,
3365	.reg_read32		= nvme_pci_reg_read32,
3366	.reg_write32		= nvme_pci_reg_write32,
3367	.reg_read64		= nvme_pci_reg_read64,
3368	.free_ctrl		= nvme_pci_free_ctrl,
3369	.submit_async_event	= nvme_pci_submit_async_event,
3370	.subsystem_reset	= nvme_pci_subsystem_reset,
3371	.get_address		= nvme_pci_get_address,
3372	.print_device_info	= nvme_pci_print_device_info,
3373	.supports_pci_p2pdma	= nvme_pci_supports_pci_p2pdma,
3374	.get_virt_boundary	= nvme_pci_get_virt_boundary,
3375};
3376
3377static int nvme_dev_map(struct nvme_dev *dev)
3378{
3379	struct pci_dev *pdev = to_pci_dev(dev->dev);
3380
3381	if (pci_request_mem_regions(pdev, "nvme"))
3382		return -ENODEV;
3383
3384	if (nvme_remap_bar(dev, NVME_REG_DBS + 4096))
3385		goto release;
3386
3387	return 0;
3388  release:
3389	pci_release_mem_regions(pdev);
3390	return -ENODEV;
3391}
3392
3393static unsigned long check_vendor_combination_bug(struct pci_dev *pdev)
3394{
3395	if (pdev->vendor == 0x144d && pdev->device == 0xa802) {
3396		/*
3397		 * Several Samsung devices seem to drop off the PCIe bus
3398		 * randomly when APST is on and uses the deepest sleep state.
3399		 * This has been observed on a Samsung "SM951 NVMe SAMSUNG
3400		 * 256GB", a "PM951 NVMe SAMSUNG 512GB", and a "Samsung SSD
3401		 * 950 PRO 256GB", but it seems to be restricted to two Dell
3402		 * laptops.
3403		 */
3404		if (dmi_match(DMI_SYS_VENDOR, "Dell Inc.") &&
3405		    (dmi_match(DMI_PRODUCT_NAME, "XPS 15 9550") ||
3406		     dmi_match(DMI_PRODUCT_NAME, "Precision 5510")))
3407			return NVME_QUIRK_NO_DEEPEST_PS;
3408	} else if (pdev->vendor == 0x144d && pdev->device == 0xa804) {
3409		/*
3410		 * Samsung SSD 960 EVO drops off the PCIe bus after system
3411		 * suspend on a Ryzen board, ASUS PRIME B350M-A, as well as
3412		 * within few minutes after bootup on a Coffee Lake board -
3413		 * ASUS PRIME Z370-A
3414		 */
3415		if (dmi_match(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC.") &&
3416		    (dmi_match(DMI_BOARD_NAME, "PRIME B350M-A") ||
3417		     dmi_match(DMI_BOARD_NAME, "PRIME Z370-A")))
3418			return NVME_QUIRK_NO_APST;
3419	} else if ((pdev->vendor == 0x144d && (pdev->device == 0xa801 ||
3420		    pdev->device == 0xa808 || pdev->device == 0xa809)) ||
3421		   (pdev->vendor == 0x1e0f && pdev->device == 0x0001)) {
3422		/*
3423		 * Forcing to use host managed nvme power settings for
3424		 * lowest idle power with quick resume latency on
3425		 * Samsung and Toshiba SSDs based on suspend behavior
3426		 * on Coffee Lake board for LENOVO C640
3427		 */
3428		if ((dmi_match(DMI_BOARD_VENDOR, "LENOVO")) &&
3429		     dmi_match(DMI_BOARD_NAME, "LNVNB161216"))
3430			return NVME_QUIRK_SIMPLE_SUSPEND;
3431	} else if (pdev->vendor == 0x2646 && (pdev->device == 0x2263 ||
3432		   pdev->device == 0x500f)) {
3433		/*
3434		 * Exclude some Kingston NV1 and A2000 devices from
3435		 * NVME_QUIRK_SIMPLE_SUSPEND. Do a full suspend to save a
3436		 * lot of energy with s2idle sleep on some TUXEDO platforms.
3437		 */
3438		if (dmi_match(DMI_BOARD_NAME, "NS5X_NS7XAU") ||
3439		    dmi_match(DMI_BOARD_NAME, "NS5x_7xAU") ||
3440		    dmi_match(DMI_BOARD_NAME, "NS5x_7xPU") ||
3441		    dmi_match(DMI_BOARD_NAME, "PH4PRX1_PH6PRX1"))
3442			return NVME_QUIRK_FORCE_NO_SIMPLE_SUSPEND;
3443	} else if (pdev->vendor == 0x144d && pdev->device == 0xa80d) {
3444		/*
3445		 * Exclude Samsung 990 Evo from NVME_QUIRK_SIMPLE_SUSPEND
3446		 * because of high power consumption (> 2 Watt) in s2idle
3447		 * sleep. Only some boards with Intel CPU are affected.
3448		 * (Note for testing: Samsung 990 Evo Plus has same PCI ID)
3449		 */
3450		if (dmi_match(DMI_BOARD_NAME, "DN50Z-140HC-YD") ||
3451		    dmi_match(DMI_BOARD_NAME, "GMxPXxx") ||
3452		    dmi_match(DMI_BOARD_NAME, "GXxMRXx") ||
3453		    dmi_match(DMI_BOARD_NAME, "NS5X_NS7XAU") ||
3454		    dmi_match(DMI_BOARD_NAME, "PH4PG31") ||
3455		    dmi_match(DMI_BOARD_NAME, "PH4PRX1_PH6PRX1") ||
3456		    dmi_match(DMI_BOARD_NAME, "PH6PG01_PH6PG71"))
3457			return NVME_QUIRK_FORCE_NO_SIMPLE_SUSPEND;
3458	}
3459
3460	/*
3461	 * NVMe SSD drops off the PCIe bus after system idle
3462	 * for 10 hours on a Lenovo N60z board.
3463	 */
3464	if (dmi_match(DMI_BOARD_NAME, "LXKT-ZXEG-N6"))
3465		return NVME_QUIRK_NO_APST;
3466
3467	return 0;
3468}
3469
3470static struct nvme_dev *nvme_pci_alloc_dev(struct pci_dev *pdev,
3471		const struct pci_device_id *id)
3472{
3473	unsigned long quirks = id->driver_data;
3474	int node = dev_to_node(&pdev->dev);
3475	struct nvme_dev *dev;
3476	int ret = -ENOMEM;
3477
3478	dev = kzalloc_node(struct_size(dev, descriptor_pools, nr_node_ids),
3479			GFP_KERNEL, node);
3480	if (!dev)
3481		return ERR_PTR(-ENOMEM);
3482	INIT_WORK(&dev->ctrl.reset_work, nvme_reset_work);
3483	mutex_init(&dev->shutdown_lock);
3484
3485	dev->nr_write_queues = write_queues;
3486	dev->nr_poll_queues = poll_queues;
3487	dev->nr_allocated_queues = nvme_max_io_queues(dev) + 1;
3488	dev->queues = kcalloc_node(dev->nr_allocated_queues,
3489			sizeof(struct nvme_queue), GFP_KERNEL, node);
3490	if (!dev->queues)
3491		goto out_free_dev;
3492
3493	dev->dev = get_device(&pdev->dev);
3494
3495	quirks |= check_vendor_combination_bug(pdev);
3496	if (!noacpi &&
3497	    !(quirks & NVME_QUIRK_FORCE_NO_SIMPLE_SUSPEND) &&
3498	    acpi_storage_d3(&pdev->dev)) {
3499		/*
3500		 * Some systems use a bios work around to ask for D3 on
3501		 * platforms that support kernel managed suspend.
3502		 */
3503		dev_info(&pdev->dev,
3504			 "platform quirk: setting simple suspend\n");
3505		quirks |= NVME_QUIRK_SIMPLE_SUSPEND;
3506	}
3507	ret = nvme_init_ctrl(&dev->ctrl, &pdev->dev, &nvme_pci_ctrl_ops,
3508			     quirks);
3509	if (ret)
3510		goto out_put_device;
3511
3512	if (dev->ctrl.quirks & NVME_QUIRK_DMA_ADDRESS_BITS_48)
3513		dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48));
3514	else
3515		dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3516	dma_set_min_align_mask(&pdev->dev, NVME_CTRL_PAGE_SIZE - 1);
3517	dma_set_max_seg_size(&pdev->dev, 0xffffffff);
3518
3519	/*
3520	 * Limit the max command size to prevent iod->sg allocations going
3521	 * over a single page.
3522	 */
3523	dev->ctrl.max_hw_sectors = min_t(u32,
3524			NVME_MAX_BYTES >> SECTOR_SHIFT,
3525			dma_opt_mapping_size(&pdev->dev) >> 9);
3526	dev->ctrl.max_segments = NVME_MAX_SEGS;
3527	dev->ctrl.max_integrity_segments = 1;
3528	return dev;
3529
3530out_put_device:
3531	put_device(dev->dev);
3532	kfree(dev->queues);
3533out_free_dev:
3534	kfree(dev);
3535	return ERR_PTR(ret);
3536}
3537
3538static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
3539{
3540	struct nvme_dev *dev;
3541	int result = -ENOMEM;
3542
3543	dev = nvme_pci_alloc_dev(pdev, id);
3544	if (IS_ERR(dev))
3545		return PTR_ERR(dev);
3546
3547	result = nvme_add_ctrl(&dev->ctrl);
3548	if (result)
3549		goto out_put_ctrl;
3550
3551	result = nvme_dev_map(dev);
3552	if (result)
3553		goto out_uninit_ctrl;
3554
3555	result = nvme_pci_alloc_iod_mempool(dev);
3556	if (result)
3557		goto out_dev_unmap;
3558
3559	dev_info(dev->ctrl.device, "pci function %s\n", dev_name(&pdev->dev));
3560
3561	result = nvme_pci_enable(dev);
3562	if (result)
3563		goto out_release_iod_mempool;
3564
3565	result = nvme_alloc_admin_tag_set(&dev->ctrl, &dev->admin_tagset,
3566				&nvme_mq_admin_ops, sizeof(struct nvme_iod));
3567	if (result)
3568		goto out_disable;
3569
3570	/*
3571	 * Mark the controller as connecting before sending admin commands to
3572	 * allow the timeout handler to do the right thing.
3573	 */
3574	if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_CONNECTING)) {
3575		dev_warn(dev->ctrl.device,
3576			"failed to mark controller CONNECTING\n");
3577		result = -EBUSY;
3578		goto out_disable;
3579	}
3580
3581	result = nvme_init_ctrl_finish(&dev->ctrl, false);
3582	if (result)
3583		goto out_disable;
3584
3585	if (nvme_ctrl_meta_sgl_supported(&dev->ctrl))
3586		dev->ctrl.max_integrity_segments = NVME_MAX_META_SEGS;
3587	else
3588		dev->ctrl.max_integrity_segments = 1;
3589
3590	nvme_dbbuf_dma_alloc(dev);
3591
3592	result = nvme_setup_host_mem(dev);
3593	if (result < 0)
3594		goto out_disable;
3595
3596	nvme_update_attrs(dev);
3597
3598	result = nvme_setup_io_queues(dev);
3599	if (result)
3600		goto out_disable;
3601
3602	if (dev->online_queues > 1) {
3603		nvme_alloc_io_tag_set(&dev->ctrl, &dev->tagset, &nvme_mq_ops,
3604				nvme_pci_nr_maps(dev), sizeof(struct nvme_iod));
3605		nvme_dbbuf_set(dev);
3606	}
3607
3608	if (!dev->ctrl.tagset)
3609		dev_warn(dev->ctrl.device, "IO queues not created\n");
3610
3611	if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_LIVE)) {
3612		dev_warn(dev->ctrl.device,
3613			"failed to mark controller live state\n");
3614		result = -ENODEV;
3615		goto out_disable;
3616	}
3617
3618	pci_set_drvdata(pdev, dev);
3619
3620	nvme_start_ctrl(&dev->ctrl);
3621	nvme_put_ctrl(&dev->ctrl);
3622	flush_work(&dev->ctrl.scan_work);
3623	return 0;
3624
3625out_disable:
3626	nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING);
3627	nvme_dev_disable(dev, true);
3628	nvme_free_host_mem(dev);
3629	nvme_dev_remove_admin(dev);
3630	nvme_dbbuf_dma_free(dev);
3631	nvme_free_queues(dev, 0);
3632out_release_iod_mempool:
3633	mempool_destroy(dev->dmavec_mempool);
3634out_dev_unmap:
3635	nvme_dev_unmap(dev);
3636out_uninit_ctrl:
3637	nvme_uninit_ctrl(&dev->ctrl);
3638out_put_ctrl:
3639	nvme_put_ctrl(&dev->ctrl);
3640	dev_err_probe(&pdev->dev, result, "probe failed\n");
3641	return result;
3642}
3643
3644static void nvme_reset_prepare(struct pci_dev *pdev)
3645{
3646	struct nvme_dev *dev = pci_get_drvdata(pdev);
3647
3648	/*
3649	 * We don't need to check the return value from waiting for the reset
3650	 * state as pci_dev device lock is held, making it impossible to race
3651	 * with ->remove().
3652	 */
3653	nvme_disable_prepare_reset(dev, false);
3654	nvme_sync_queues(&dev->ctrl);
3655}
3656
3657static void nvme_reset_done(struct pci_dev *pdev)
3658{
3659	struct nvme_dev *dev = pci_get_drvdata(pdev);
3660
3661	if (!nvme_try_sched_reset(&dev->ctrl))
3662		flush_work(&dev->ctrl.reset_work);
3663}
3664
3665static void nvme_shutdown(struct pci_dev *pdev)
3666{
3667	struct nvme_dev *dev = pci_get_drvdata(pdev);
3668
3669	nvme_disable_prepare_reset(dev, true);
3670}
3671
3672/*
3673 * The driver's remove may be called on a device in a partially initialized
3674 * state. This function must not have any dependencies on the device state in
3675 * order to proceed.
3676 */
3677static void nvme_remove(struct pci_dev *pdev)
3678{
3679	struct nvme_dev *dev = pci_get_drvdata(pdev);
3680
3681	nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING);
3682	pci_set_drvdata(pdev, NULL);
3683
3684	if (!pci_device_is_present(pdev)) {
3685		nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DEAD);
3686		nvme_dev_disable(dev, true);
3687	}
3688
3689	flush_work(&dev->ctrl.reset_work);
3690	nvme_stop_ctrl(&dev->ctrl);
3691	nvme_remove_namespaces(&dev->ctrl);
3692	nvme_dev_disable(dev, true);
3693	nvme_free_host_mem(dev);
3694	nvme_dev_remove_admin(dev);
3695	nvme_dbbuf_dma_free(dev);
3696	nvme_free_queues(dev, 0);
3697	mempool_destroy(dev->dmavec_mempool);
3698	nvme_release_descriptor_pools(dev);
3699	nvme_dev_unmap(dev);
3700	nvme_uninit_ctrl(&dev->ctrl);
3701}
3702
3703#ifdef CONFIG_PM_SLEEP
3704static int nvme_get_power_state(struct nvme_ctrl *ctrl, u32 *ps)
3705{
3706	return nvme_get_features(ctrl, NVME_FEAT_POWER_MGMT, 0, NULL, 0, ps);
3707}
3708
3709static int nvme_set_power_state(struct nvme_ctrl *ctrl, u32 ps)
3710{
3711	return nvme_set_features(ctrl, NVME_FEAT_POWER_MGMT, ps, NULL, 0, NULL);
3712}
3713
3714static int nvme_resume(struct device *dev)
3715{
3716	struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev));
3717	struct nvme_ctrl *ctrl = &ndev->ctrl;
3718
3719	if (ndev->last_ps == U32_MAX ||
3720	    nvme_set_power_state(ctrl, ndev->last_ps) != 0)
3721		goto reset;
3722	if (ctrl->hmpre && nvme_setup_host_mem(ndev))
3723		goto reset;
3724
3725	return 0;
3726reset:
3727	return nvme_try_sched_reset(ctrl);
3728}
3729
3730static int nvme_suspend(struct device *dev)
3731{
3732	struct pci_dev *pdev = to_pci_dev(dev);
3733	struct nvme_dev *ndev = pci_get_drvdata(pdev);
3734	struct nvme_ctrl *ctrl = &ndev->ctrl;
3735	int ret = -EBUSY;
3736
3737	ndev->last_ps = U32_MAX;
3738
3739	/*
3740	 * The platform does not remove power for a kernel managed suspend so
3741	 * use host managed nvme power settings for lowest idle power if
3742	 * possible. This should have quicker resume latency than a full device
3743	 * shutdown.  But if the firmware is involved after the suspend or the
3744	 * device does not support any non-default power states, shut down the
3745	 * device fully.
3746	 *
3747	 * If ASPM is not enabled for the device, shut down the device and allow
3748	 * the PCI bus layer to put it into D3 in order to take the PCIe link
3749	 * down, so as to allow the platform to achieve its minimum low-power
3750	 * state (which may not be possible if the link is up).
3751	 */
3752	if (pm_suspend_via_firmware() || !ctrl->npss ||
3753	    !pcie_aspm_enabled(pdev) ||
3754	    (ndev->ctrl.quirks & NVME_QUIRK_SIMPLE_SUSPEND))
3755		return nvme_disable_prepare_reset(ndev, true);
3756
3757	nvme_start_freeze(ctrl);
3758	nvme_wait_freeze(ctrl);
3759	nvme_sync_queues(ctrl);
3760
3761	if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE)
3762		goto unfreeze;
3763
3764	/*
3765	 * Host memory access may not be successful in a system suspend state,
3766	 * but the specification allows the controller to access memory in a
3767	 * non-operational power state.
3768	 */
3769	if (ndev->hmb) {
3770		ret = nvme_set_host_mem(ndev, 0);
3771		if (ret < 0)
3772			goto unfreeze;
3773	}
3774
3775	ret = nvme_get_power_state(ctrl, &ndev->last_ps);
3776	if (ret < 0)
3777		goto unfreeze;
3778
3779	/*
3780	 * A saved state prevents pci pm from generically controlling the
3781	 * device's power. If we're using protocol specific settings, we don't
3782	 * want pci interfering.
3783	 */
3784	pci_save_state(pdev);
3785
3786	ret = nvme_set_power_state(ctrl, ctrl->npss);
3787	if (ret < 0)
3788		goto unfreeze;
3789
3790	if (ret) {
3791		/* discard the saved state */
3792		pci_load_saved_state(pdev, NULL);
3793
3794		/*
3795		 * Clearing npss forces a controller reset on resume. The
3796		 * correct value will be rediscovered then.
3797		 */
3798		ret = nvme_disable_prepare_reset(ndev, true);
3799		ctrl->npss = 0;
3800	}
3801unfreeze:
3802	nvme_unfreeze(ctrl);
3803	return ret;
3804}
3805
3806static int nvme_simple_suspend(struct device *dev)
3807{
3808	struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev));
3809
3810	return nvme_disable_prepare_reset(ndev, true);
3811}
3812
3813static int nvme_simple_resume(struct device *dev)
3814{
3815	struct pci_dev *pdev = to_pci_dev(dev);
3816	struct nvme_dev *ndev = pci_get_drvdata(pdev);
3817
3818	return nvme_try_sched_reset(&ndev->ctrl);
3819}
3820
3821static const struct dev_pm_ops nvme_dev_pm_ops = {
3822	.suspend	= nvme_suspend,
3823	.resume		= nvme_resume,
3824	.freeze		= nvme_simple_suspend,
3825	.thaw		= nvme_simple_resume,
3826	.poweroff	= nvme_simple_suspend,
3827	.restore	= nvme_simple_resume,
3828};
3829#endif /* CONFIG_PM_SLEEP */
3830
3831static pci_ers_result_t nvme_error_detected(struct pci_dev *pdev,
3832						pci_channel_state_t state)
3833{
3834	struct nvme_dev *dev = pci_get_drvdata(pdev);
3835
3836	/*
3837	 * A frozen channel requires a reset. When detected, this method will
3838	 * shutdown the controller to quiesce. The controller will be restarted
3839	 * after the slot reset through driver's slot_reset callback.
3840	 */
3841	switch (state) {
3842	case pci_channel_io_normal:
3843		return PCI_ERS_RESULT_CAN_RECOVER;
3844	case pci_channel_io_frozen:
3845		dev_warn(dev->ctrl.device,
3846			"frozen state error detected, reset controller\n");
3847		if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_RESETTING)) {
3848			nvme_dev_disable(dev, true);
3849			return PCI_ERS_RESULT_DISCONNECT;
3850		}
3851		nvme_dev_disable(dev, false);
3852		return PCI_ERS_RESULT_NEED_RESET;
3853	case pci_channel_io_perm_failure:
3854		dev_warn(dev->ctrl.device,
3855			"failure state error detected, request disconnect\n");
3856		return PCI_ERS_RESULT_DISCONNECT;
3857	}
3858	return PCI_ERS_RESULT_NEED_RESET;
3859}
3860
3861static pci_ers_result_t nvme_slot_reset(struct pci_dev *pdev)
3862{
3863	struct nvme_dev *dev = pci_get_drvdata(pdev);
3864
3865	dev_info(dev->ctrl.device, "restart after slot reset\n");
3866	pci_restore_state(pdev);
3867	if (nvme_try_sched_reset(&dev->ctrl))
3868		nvme_unquiesce_io_queues(&dev->ctrl);
3869	return PCI_ERS_RESULT_RECOVERED;
3870}
3871
3872static void nvme_error_resume(struct pci_dev *pdev)
3873{
3874	struct nvme_dev *dev = pci_get_drvdata(pdev);
3875
3876	flush_work(&dev->ctrl.reset_work);
3877}
3878
3879static const struct pci_error_handlers nvme_err_handler = {
3880	.error_detected	= nvme_error_detected,
3881	.slot_reset	= nvme_slot_reset,
3882	.resume		= nvme_error_resume,
3883	.reset_prepare	= nvme_reset_prepare,
3884	.reset_done	= nvme_reset_done,
3885};
3886
3887static const struct pci_device_id nvme_id_table[] = {
3888	{ PCI_VDEVICE(INTEL, 0x0953),	/* Intel 750/P3500/P3600/P3700 */
3889		.driver_data = NVME_QUIRK_STRIPE_SIZE |
3890				NVME_QUIRK_DEALLOCATE_ZEROES, },
3891	{ PCI_VDEVICE(INTEL, 0x0a53),	/* Intel P3520 */
3892		.driver_data = NVME_QUIRK_STRIPE_SIZE |
3893				NVME_QUIRK_DEALLOCATE_ZEROES, },
3894	{ PCI_VDEVICE(INTEL, 0x0a54),	/* Intel P4500/P4600 */
3895		.driver_data = NVME_QUIRK_STRIPE_SIZE |
3896				NVME_QUIRK_IGNORE_DEV_SUBNQN |
3897				NVME_QUIRK_BOGUS_NID, },
3898	{ PCI_VDEVICE(INTEL, 0x0a55),	/* Dell Express Flash P4600 */
3899		.driver_data = NVME_QUIRK_STRIPE_SIZE, },
3900	{ PCI_VDEVICE(INTEL, 0xf1a5),	/* Intel 600P/P3100 */
3901		.driver_data = NVME_QUIRK_NO_DEEPEST_PS |
3902				NVME_QUIRK_MEDIUM_PRIO_SQ |
3903				NVME_QUIRK_NO_TEMP_THRESH_CHANGE |
3904				NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3905	{ PCI_VDEVICE(INTEL, 0xf1a6),	/* Intel 760p/Pro 7600p */
3906		.driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN, },
3907	{ PCI_VDEVICE(INTEL, 0x5845),	/* Qemu emulated controller */
3908		.driver_data = NVME_QUIRK_IDENTIFY_CNS |
3909				NVME_QUIRK_DISABLE_WRITE_ZEROES |
3910				NVME_QUIRK_BOGUS_NID, },
3911	{ PCI_VDEVICE(REDHAT, 0x0010),	/* Qemu emulated controller */
3912		.driver_data = NVME_QUIRK_BOGUS_NID, },
3913	{ PCI_DEVICE(0x1217, 0x8760), /* O2 Micro 64GB Steam Deck */
3914		.driver_data = NVME_QUIRK_DMAPOOL_ALIGN_512, },
3915	{ PCI_DEVICE(0x126f, 0x1001),	/* Silicon Motion generic */
3916		.driver_data = NVME_QUIRK_NO_DEEPEST_PS |
3917				NVME_QUIRK_IGNORE_DEV_SUBNQN, },
3918	{ PCI_DEVICE(0x126f, 0x2262),	/* Silicon Motion generic */
3919		.driver_data = NVME_QUIRK_NO_DEEPEST_PS |
3920				NVME_QUIRK_BOGUS_NID, },
3921	{ PCI_DEVICE(0x126f, 0x2263),	/* Silicon Motion unidentified */
3922		.driver_data = NVME_QUIRK_NO_NS_DESC_LIST |
3923				NVME_QUIRK_BOGUS_NID, },
3924	{ PCI_DEVICE(0x1bb1, 0x0100),   /* Seagate Nytro Flash Storage */
3925		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY |
3926				NVME_QUIRK_NO_NS_DESC_LIST, },
3927	{ PCI_DEVICE(0x1c58, 0x0003),	/* HGST adapter */
3928		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
3929	{ PCI_DEVICE(0x1c58, 0x0023),	/* WDC SN200 adapter */
3930		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
3931	{ PCI_DEVICE(0x1c5f, 0x0540),	/* Memblaze Pblaze4 adapter */
3932		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
3933	{ PCI_DEVICE(0x144d, 0xa821),   /* Samsung PM1725 */
3934		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
3935	{ PCI_DEVICE(0x144d, 0xa822),   /* Samsung PM1725a */
3936		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY |
3937				NVME_QUIRK_DISABLE_WRITE_ZEROES|
3938				NVME_QUIRK_IGNORE_DEV_SUBNQN, },
3939	{ PCI_DEVICE(0x15b7, 0x5008),   /* Sandisk SN530 */
3940		.driver_data = NVME_QUIRK_BROKEN_MSI },
3941	{ PCI_DEVICE(0x15b7, 0x5009),   /* Sandisk SN550 */
3942		.driver_data = NVME_QUIRK_BROKEN_MSI |
3943				NVME_QUIRK_NO_DEEPEST_PS },
3944	{ PCI_DEVICE(0x1987, 0x5012),	/* Phison E12 */
3945		.driver_data = NVME_QUIRK_BOGUS_NID, },
3946	{ PCI_DEVICE(0x1987, 0x5016),	/* Phison E16 */
3947		.driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN |
3948				NVME_QUIRK_BOGUS_NID, },
3949	{ PCI_DEVICE(0x1987, 0x5019),  /* phison E19 */
3950		.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3951	{ PCI_DEVICE(0x1987, 0x5021),   /* Phison E21 */
3952		.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3953	{ PCI_DEVICE(0x1b4b, 0x1092),	/* Lexar 256 GB SSD */
3954		.driver_data = NVME_QUIRK_NO_NS_DESC_LIST |
3955				NVME_QUIRK_IGNORE_DEV_SUBNQN, },
3956	{ PCI_DEVICE(0x1cc1, 0x33f8),   /* ADATA IM2P33F8ABR1 1 TB */
3957		.driver_data = NVME_QUIRK_BOGUS_NID, },
3958	{ PCI_DEVICE(0x10ec, 0x5762),   /* ADATA SX6000LNP */
3959		.driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN |
3960				NVME_QUIRK_BOGUS_NID, },
3961	{ PCI_DEVICE(0x10ec, 0x5763),  /* ADATA SX6000PNP */
3962		.driver_data = NVME_QUIRK_BOGUS_NID, },
3963	{ PCI_DEVICE(0x1cc1, 0x8201),   /* ADATA SX8200PNP 512GB */
3964		.driver_data = NVME_QUIRK_NO_DEEPEST_PS |
3965				NVME_QUIRK_IGNORE_DEV_SUBNQN, },
3966	 { PCI_DEVICE(0x1344, 0x5407), /* Micron Technology Inc NVMe SSD */
3967		.driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN },
3968	 { PCI_DEVICE(0x1344, 0x6001),   /* Micron Nitro NVMe */
3969		 .driver_data = NVME_QUIRK_BOGUS_NID, },
3970	{ PCI_DEVICE(0x1c5c, 0x1504),   /* SK Hynix PC400 */
3971		.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3972	{ PCI_DEVICE(0x1c5c, 0x174a),   /* SK Hynix P31 SSD */
3973		.driver_data = NVME_QUIRK_BOGUS_NID, },
3974	{ PCI_DEVICE(0x1c5c, 0x1D59),   /* SK Hynix BC901 */
3975		.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3976	{ PCI_DEVICE(0x15b7, 0x2001),   /*  Sandisk Skyhawk */
3977		.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3978	{ PCI_DEVICE(0x1d97, 0x2263),   /* SPCC */
3979		.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3980	{ PCI_DEVICE(0x144d, 0xa80b),   /* Samsung PM9B1 256G and 512G */
3981		.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES |
3982				NVME_QUIRK_BOGUS_NID, },
3983	{ PCI_DEVICE(0x144d, 0xa809),   /* Samsung MZALQ256HBJD 256G */
3984		.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3985	{ PCI_DEVICE(0x144d, 0xa802),   /* Samsung SM953 */
3986		.driver_data = NVME_QUIRK_BOGUS_NID, },
3987	{ PCI_DEVICE(0x1cc4, 0x6303),   /* UMIS RPJTJ512MGE1QDY 512G */
3988		.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3989	{ PCI_DEVICE(0x1cc4, 0x6302),   /* UMIS RPJTJ256MGE1QDY 256G */
3990		.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3991	{ PCI_DEVICE(0x2646, 0x2262),   /* KINGSTON SKC2000 NVMe SSD */
3992		.driver_data = NVME_QUIRK_NO_DEEPEST_PS, },
3993	{ PCI_DEVICE(0x2646, 0x2263),   /* KINGSTON A2000 NVMe SSD  */
3994		.driver_data = NVME_QUIRK_NO_DEEPEST_PS, },
3995	{ PCI_DEVICE(0x2646, 0x5013),   /* Kingston KC3000, Kingston FURY Renegade */
3996		.driver_data = NVME_QUIRK_NO_SECONDARY_TEMP_THRESH, },
3997	{ PCI_DEVICE(0x2646, 0x5018),   /* KINGSTON OM8SFP4xxxxP OS21012 NVMe SSD */
3998		.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3999	{ PCI_DEVICE(0x2646, 0x5016),   /* KINGSTON OM3PGP4xxxxP OS21011 NVMe SSD */
4000		.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
4001	{ PCI_DEVICE(0x2646, 0x501A),   /* KINGSTON OM8PGP4xxxxP OS21005 NVMe SSD */
4002		.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
4003	{ PCI_DEVICE(0x2646, 0x501B),   /* KINGSTON OM8PGP4xxxxQ OS21005 NVMe SSD */
4004		.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
4005	{ PCI_DEVICE(0x2646, 0x501E),   /* KINGSTON OM3PGP4xxxxQ OS21011 NVMe SSD */
4006		.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
4007	{ PCI_DEVICE(0x1f40, 0x1202),   /* Netac Technologies Co. NV3000 NVMe SSD */
4008		.driver_data = NVME_QUIRK_BOGUS_NID, },
4009	{ PCI_DEVICE(0x1f40, 0x5236),   /* Netac Technologies Co. NV7000 NVMe SSD */
4010		.driver_data = NVME_QUIRK_BOGUS_NID, },
4011	{ PCI_DEVICE(0x1e4B, 0x1001),   /* MAXIO MAP1001 */
4012		.driver_data = NVME_QUIRK_BOGUS_NID, },
4013	{ PCI_DEVICE(0x1e4B, 0x1002),   /* MAXIO MAP1002 */
4014		.driver_data = NVME_QUIRK_BOGUS_NID, },
4015	{ PCI_DEVICE(0x1e4B, 0x1202),   /* MAXIO MAP1202 */
4016		.driver_data = NVME_QUIRK_BOGUS_NID, },
4017	{ PCI_DEVICE(0x1e4B, 0x1602),   /* MAXIO MAP1602 */
4018		.driver_data = NVME_QUIRK_BOGUS_NID, },
4019	{ PCI_DEVICE(0x1cc1, 0x5350),   /* ADATA XPG GAMMIX S50 */
4020		.driver_data = NVME_QUIRK_BOGUS_NID, },
4021	{ PCI_DEVICE(0x1dbe, 0x5216),   /* Acer/INNOGRIT FA100/5216 NVMe SSD */
4022		.driver_data = NVME_QUIRK_BOGUS_NID, },
4023	{ PCI_DEVICE(0x1dbe, 0x5236),   /* ADATA XPG GAMMIX S70 */
4024		.driver_data = NVME_QUIRK_BOGUS_NID, },
4025	{ PCI_DEVICE(0x1e49, 0x0021),   /* ZHITAI TiPro5000 NVMe SSD */
4026		.driver_data = NVME_QUIRK_NO_DEEPEST_PS, },
4027	{ PCI_DEVICE(0x1e49, 0x0041),   /* ZHITAI TiPro7000 NVMe SSD */
4028		.driver_data = NVME_QUIRK_NO_DEEPEST_PS, },
4029	{ PCI_DEVICE(0x1fa0, 0x2283),   /* Wodposit WPBSNM8-256GTP */
4030		.driver_data = NVME_QUIRK_NO_SECONDARY_TEMP_THRESH, },
4031	{ PCI_DEVICE(0x025e, 0xf1ac),   /* SOLIDIGM  P44 pro SSDPFKKW020X7  */
4032		.driver_data = NVME_QUIRK_NO_DEEPEST_PS, },
4033	{ PCI_DEVICE(0xc0a9, 0x540a),   /* Crucial P2 */
4034		.driver_data = NVME_QUIRK_BOGUS_NID, },
4035	{ PCI_DEVICE(0x1d97, 0x2263), /* Lexar NM610 */
4036		.driver_data = NVME_QUIRK_BOGUS_NID, },
4037	{ PCI_DEVICE(0x1d97, 0x1d97), /* Lexar NM620 */
4038		.driver_data = NVME_QUIRK_BOGUS_NID, },
4039	{ PCI_DEVICE(0x1d97, 0x2269), /* Lexar NM760 */
4040		.driver_data = NVME_QUIRK_BOGUS_NID |
4041				NVME_QUIRK_IGNORE_DEV_SUBNQN, },
4042	{ PCI_DEVICE(0x10ec, 0x5763), /* TEAMGROUP T-FORCE CARDEA ZERO Z330 SSD */
4043		.driver_data = NVME_QUIRK_BOGUS_NID, },
4044	{ PCI_DEVICE(0x1e4b, 0x1602), /* HS-SSD-FUTURE 2048G  */
4045		.driver_data = NVME_QUIRK_BOGUS_NID, },
4046	{ PCI_DEVICE(0x10ec, 0x5765), /* TEAMGROUP MP33 2TB SSD */
4047		.driver_data = NVME_QUIRK_BOGUS_NID, },
4048	{ PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0x0061),
4049		.driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, },
4050	{ PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0x0065),
4051		.driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, },
4052	{ PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0x8061),
4053		.driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, },
4054	{ PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0xcd00),
4055		.driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, },
4056	{ PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0xcd01),
4057		.driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, },
4058	{ PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0xcd02),
4059		.driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, },
4060	{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001),
4061		/*
4062		 * Fix for the Apple controller found in the MacBook8,1 and
4063		 * some MacBook7,1 to avoid controller resets and data loss.
4064		 */
4065		.driver_data = NVME_QUIRK_SINGLE_VECTOR |
4066				NVME_QUIRK_QDEPTH_ONE },
4067	{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2003) },
4068	{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2005),
4069		.driver_data = NVME_QUIRK_SINGLE_VECTOR |
4070				NVME_QUIRK_128_BYTES_SQES |
4071				NVME_QUIRK_SHARED_TAGS |
4072				NVME_QUIRK_SKIP_CID_GEN |
4073				NVME_QUIRK_IDENTIFY_CNS },
4074	{ PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
4075	{ 0, }
4076};
4077MODULE_DEVICE_TABLE(pci, nvme_id_table);
4078
4079static struct pci_driver nvme_driver = {
4080	.name		= "nvme",
4081	.id_table	= nvme_id_table,
4082	.probe		= nvme_probe,
4083	.remove		= nvme_remove,
4084	.shutdown	= nvme_shutdown,
4085	.driver		= {
4086		.probe_type	= PROBE_PREFER_ASYNCHRONOUS,
4087#ifdef CONFIG_PM_SLEEP
4088		.pm		= &nvme_dev_pm_ops,
4089#endif
4090	},
4091	.sriov_configure = pci_sriov_configure_simple,
4092	.err_handler	= &nvme_err_handler,
4093};
4094
4095static int __init nvme_init(void)
4096{
4097	BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
4098	BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
4099	BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
4100	BUILD_BUG_ON(IRQ_AFFINITY_MAX_SETS < 2);
4101
4102	return pci_register_driver(&nvme_driver);
4103}
4104
4105static void __exit nvme_exit(void)
4106{
4107	pci_unregister_driver(&nvme_driver);
4108	flush_workqueue(nvme_wq);
4109}
4110
4111MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
4112MODULE_LICENSE("GPL");
4113MODULE_VERSION("1.0");
4114MODULE_DESCRIPTION("NVMe host PCIe transport driver");
4115module_init(nvme_init);
4116module_exit(nvme_exit);