hisi_acc_vfio_pci.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (c) 2021, HiSilicon Ltd.
   4 */
   5
   6#include <linux/device.h>
   7#include <linux/eventfd.h>
   8#include <linux/file.h>
   9#include <linux/hisi_acc_qm.h>
  10#include <linux/interrupt.h>
  11#include <linux/module.h>
  12#include <linux/pci.h>
  13#include <linux/vfio.h>
  14#include <linux/vfio_pci_core.h>
  15#include <linux/anon_inodes.h>
  16
  17#include "hisi_acc_vfio_pci.h"
  18
  19/* Return 0 on VM acc device ready, -ETIMEDOUT hardware timeout */
  20static int qm_wait_dev_not_ready(struct hisi_qm *qm)
  21{
  22	u32 val;
  23
  24	return readl_relaxed_poll_timeout(qm->io_base + QM_VF_STATE,
  25				val, !(val & 0x1), MB_POLL_PERIOD_US,
  26				MB_POLL_TIMEOUT_US);
  27}
  28
  29/*
  30 * Each state Reg is checked 100 times,
  31 * with a delay of 100 microseconds after each check
  32 */
  33static u32 qm_check_reg_state(struct hisi_qm *qm, u32 regs)
  34{
  35	int check_times = 0;
  36	u32 state;
  37
  38	state = readl(qm->io_base + regs);
  39	while (state && check_times < ERROR_CHECK_TIMEOUT) {
  40		udelay(CHECK_DELAY_TIME);
  41		state = readl(qm->io_base + regs);
  42		check_times++;
  43	}
  44
  45	return state;
  46}
  47
  48static int qm_read_regs(struct hisi_qm *qm, u32 reg_addr,
  49			u32 *data, u8 nums)
  50{
  51	int i;
  52
  53	if (nums < 1 || nums > QM_REGS_MAX_LEN)
  54		return -EINVAL;
  55
  56	for (i = 0; i < nums; i++) {
  57		data[i] = readl(qm->io_base + reg_addr);
  58		reg_addr += QM_REG_ADDR_OFFSET;
  59	}
  60
  61	return 0;
  62}
  63
  64static int qm_write_regs(struct hisi_qm *qm, u32 reg,
  65			 u32 *data, u8 nums)
  66{
  67	int i;
  68
  69	if (nums < 1 || nums > QM_REGS_MAX_LEN)
  70		return -EINVAL;
  71
  72	for (i = 0; i < nums; i++)
  73		writel(data[i], qm->io_base + reg + i * QM_REG_ADDR_OFFSET);
  74
  75	return 0;
  76}
  77
  78static int qm_get_vft(struct hisi_qm *qm, u32 *base)
  79{
  80	u64 sqc_vft;
  81	u32 qp_num;
  82	int ret;
  83
  84	ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_VFT_V2, 0, 0, 1);
  85	if (ret)
  86		return ret;
  87
  88	sqc_vft = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
  89		  ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) <<
  90		  QM_XQC_ADDR_OFFSET);
  91	*base = QM_SQC_VFT_BASE_MASK_V2 & (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2);
  92	qp_num = (QM_SQC_VFT_NUM_MASK_V2 &
  93		  (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1;
  94
  95	return qp_num;
  96}
  97
  98static int qm_get_sqc(struct hisi_qm *qm, u64 *addr)
  99{
 100	int ret;
 101
 102	ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, 0, 0, 1);
 103	if (ret)
 104		return ret;
 105
 106	*addr = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
 107		  ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) <<
 108		  QM_XQC_ADDR_OFFSET);
 109
 110	return 0;
 111}
 112
 113static int qm_get_cqc(struct hisi_qm *qm, u64 *addr)
 114{
 115	int ret;
 116
 117	ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, 0, 0, 1);
 118	if (ret)
 119		return ret;
 120
 121	*addr = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
 122		  ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) <<
 123		  QM_XQC_ADDR_OFFSET);
 124
 125	return 0;
 126}
 127
 128static void qm_xqc_reg_offsets(struct hisi_qm *qm,
 129			       u32 *eqc_addr, u32 *aeqc_addr)
 130{
 131	struct hisi_acc_vf_core_device *hisi_acc_vdev =
 132		container_of(qm, struct hisi_acc_vf_core_device, vf_qm);
 133
 134	if (hisi_acc_vdev->drv_mode == HW_ACC_MIG_VF_CTRL) {
 135		*eqc_addr = QM_EQC_VF_DW0;
 136		*aeqc_addr = QM_AEQC_VF_DW0;
 137	} else {
 138		*eqc_addr = QM_EQC_PF_DW0;
 139		*aeqc_addr = QM_AEQC_PF_DW0;
 140	}
 141}
 142
 143static int qm_get_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data)
 144{
 145	struct device *dev = &qm->pdev->dev;
 146	u32 eqc_addr, aeqc_addr;
 147	int ret;
 148
 149	ret = qm_read_regs(qm, QM_VF_AEQ_INT_MASK, &vf_data->aeq_int_mask, 1);
 150	if (ret) {
 151		dev_err(dev, "failed to read QM_VF_AEQ_INT_MASK\n");
 152		return ret;
 153	}
 154
 155	ret = qm_read_regs(qm, QM_VF_EQ_INT_MASK, &vf_data->eq_int_mask, 1);
 156	if (ret) {
 157		dev_err(dev, "failed to read QM_VF_EQ_INT_MASK\n");
 158		return ret;
 159	}
 160
 161	ret = qm_read_regs(qm, QM_IFC_INT_SOURCE_V,
 162			   &vf_data->ifc_int_source, 1);
 163	if (ret) {
 164		dev_err(dev, "failed to read QM_IFC_INT_SOURCE_V\n");
 165		return ret;
 166	}
 167
 168	ret = qm_read_regs(qm, QM_IFC_INT_MASK, &vf_data->ifc_int_mask, 1);
 169	if (ret) {
 170		dev_err(dev, "failed to read QM_IFC_INT_MASK\n");
 171		return ret;
 172	}
 173
 174	ret = qm_read_regs(qm, QM_IFC_INT_SET_V, &vf_data->ifc_int_set, 1);
 175	if (ret) {
 176		dev_err(dev, "failed to read QM_IFC_INT_SET_V\n");
 177		return ret;
 178	}
 179
 180	ret = qm_read_regs(qm, QM_PAGE_SIZE, &vf_data->page_size, 1);
 181	if (ret) {
 182		dev_err(dev, "failed to read QM_PAGE_SIZE\n");
 183		return ret;
 184	}
 185
 186	qm_xqc_reg_offsets(qm, &eqc_addr, &aeqc_addr);
 187	/* QM_EQC_DW has 7 regs */
 188	ret = qm_read_regs(qm, eqc_addr, vf_data->qm_eqc_dw, 7);
 189	if (ret) {
 190		dev_err(dev, "failed to read QM_EQC_DW\n");
 191		return ret;
 192	}
 193
 194	/* QM_AEQC_DW has 7 regs */
 195	ret = qm_read_regs(qm, aeqc_addr, vf_data->qm_aeqc_dw, 7);
 196	if (ret) {
 197		dev_err(dev, "failed to read QM_AEQC_DW\n");
 198		return ret;
 199	}
 200
 201	return 0;
 202}
 203
 204static int qm_set_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data)
 205{
 206	struct device *dev = &qm->pdev->dev;
 207	u32 eqc_addr, aeqc_addr;
 208	int ret;
 209
 210	/* Check VF state */
 211	ret = hisi_qm_wait_mb_ready(qm);
 212	if (unlikely(ret)) {
 213		dev_err(&qm->pdev->dev, "QM device is not ready to write\n");
 214		return ret;
 215	}
 216
 217	ret = qm_write_regs(qm, QM_VF_AEQ_INT_MASK, &vf_data->aeq_int_mask, 1);
 218	if (ret) {
 219		dev_err(dev, "failed to write QM_VF_AEQ_INT_MASK\n");
 220		return ret;
 221	}
 222
 223	ret = qm_write_regs(qm, QM_VF_EQ_INT_MASK, &vf_data->eq_int_mask, 1);
 224	if (ret) {
 225		dev_err(dev, "failed to write QM_VF_EQ_INT_MASK\n");
 226		return ret;
 227	}
 228
 229	ret = qm_write_regs(qm, QM_IFC_INT_SOURCE_V,
 230			    &vf_data->ifc_int_source, 1);
 231	if (ret) {
 232		dev_err(dev, "failed to write QM_IFC_INT_SOURCE_V\n");
 233		return ret;
 234	}
 235
 236	ret = qm_write_regs(qm, QM_IFC_INT_MASK, &vf_data->ifc_int_mask, 1);
 237	if (ret) {
 238		dev_err(dev, "failed to write QM_IFC_INT_MASK\n");
 239		return ret;
 240	}
 241
 242	ret = qm_write_regs(qm, QM_IFC_INT_SET_V, &vf_data->ifc_int_set, 1);
 243	if (ret) {
 244		dev_err(dev, "failed to write QM_IFC_INT_SET_V\n");
 245		return ret;
 246	}
 247
 248	ret = qm_write_regs(qm, QM_QUE_ISO_CFG_V, &vf_data->que_iso_cfg, 1);
 249	if (ret) {
 250		dev_err(dev, "failed to write QM_QUE_ISO_CFG_V\n");
 251		return ret;
 252	}
 253
 254	ret = qm_write_regs(qm, QM_PAGE_SIZE, &vf_data->page_size, 1);
 255	if (ret) {
 256		dev_err(dev, "failed to write QM_PAGE_SIZE\n");
 257		return ret;
 258	}
 259
 260	qm_xqc_reg_offsets(qm, &eqc_addr, &aeqc_addr);
 261	/* QM_EQC_DW has 7 regs */
 262	ret = qm_write_regs(qm, eqc_addr, vf_data->qm_eqc_dw, 7);
 263	if (ret) {
 264		dev_err(dev, "failed to write QM_EQC_DW\n");
 265		return ret;
 266	}
 267
 268	/* QM_AEQC_DW has 7 regs */
 269	ret = qm_write_regs(qm, aeqc_addr, vf_data->qm_aeqc_dw, 7);
 270	if (ret) {
 271		dev_err(dev, "failed to write QM_AEQC_DW\n");
 272		return ret;
 273	}
 274
 275	return 0;
 276}
 277
 278static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd,
 279		  u16 index, u8 priority)
 280{
 281	u64 doorbell;
 282	u64 dbase;
 283	u16 randata = 0;
 284
 285	if (cmd == QM_DOORBELL_CMD_SQ || cmd == QM_DOORBELL_CMD_CQ)
 286		dbase = QM_DOORBELL_SQ_CQ_BASE_V2;
 287	else
 288		dbase = QM_DOORBELL_EQ_AEQ_BASE_V2;
 289
 290	doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V2) |
 291		   ((u64)randata << QM_DB_RAND_SHIFT_V2) |
 292		   ((u64)index << QM_DB_INDEX_SHIFT_V2)	 |
 293		   ((u64)priority << QM_DB_PRIORITY_SHIFT_V2);
 294
 295	writeq(doorbell, qm->io_base + dbase);
 296}
 297
 298static int pf_qm_get_qp_num(struct hisi_qm *qm, int vf_id, u32 *rbase)
 299{
 300	unsigned int val;
 301	u64 sqc_vft;
 302	u32 qp_num;
 303	int ret;
 304
 305	ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
 306					 val & BIT(0), MB_POLL_PERIOD_US,
 307					 MB_POLL_TIMEOUT_US);
 308	if (ret)
 309		return ret;
 310
 311	writel(0x1, qm->io_base + QM_VFT_CFG_OP_WR);
 312	/* 0 mean SQC VFT */
 313	writel(0x0, qm->io_base + QM_VFT_CFG_TYPE);
 314	writel(vf_id, qm->io_base + QM_VFT_CFG);
 315
 316	writel(0x0, qm->io_base + QM_VFT_CFG_RDY);
 317	writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE);
 318
 319	ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
 320					 val & BIT(0), MB_POLL_PERIOD_US,
 321					 MB_POLL_TIMEOUT_US);
 322	if (ret)
 323		return ret;
 324
 325	sqc_vft = readl(qm->io_base + QM_VFT_CFG_DATA_L) |
 326		  ((u64)readl(qm->io_base + QM_VFT_CFG_DATA_H) <<
 327		  QM_XQC_ADDR_OFFSET);
 328	*rbase = QM_SQC_VFT_BASE_MASK_V2 &
 329		  (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2);
 330	qp_num = (QM_SQC_VFT_NUM_MASK_V2 &
 331		  (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1;
 332
 333	return qp_num;
 334}
 335
 336static void qm_dev_cmd_init(struct hisi_qm *qm)
 337{
 338	/* Clear VF communication status registers. */
 339	writel(0x1, qm->io_base + QM_IFC_INT_SOURCE_V);
 340
 341	/* Enable pf and vf communication. */
 342	writel(0x0, qm->io_base + QM_IFC_INT_MASK);
 343}
 344
 345static int vf_qm_cache_wb(struct hisi_qm *qm)
 346{
 347	unsigned int val;
 348	int ret;
 349
 350	writel(0x1, qm->io_base + QM_CACHE_WB_START);
 351	ret = readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE,
 352				       val, val & BIT(0), MB_POLL_PERIOD_US,
 353				       MB_POLL_TIMEOUT_US);
 354	if (ret) {
 355		dev_err(&qm->pdev->dev, "vf QM writeback sqc cache fail\n");
 356		return ret;
 357	}
 358
 359	return 0;
 360}
 361
 362static void vf_qm_fun_reset(struct hisi_qm *qm)
 363{
 364	int i;
 365
 366	for (i = 0; i < qm->qp_num; i++)
 367		qm_db(qm, i, QM_DOORBELL_CMD_SQ, 0, 1);
 368}
 369
 370static int vf_qm_func_stop(struct hisi_qm *qm)
 371{
 372	return hisi_qm_mb(qm, QM_MB_CMD_PAUSE_QM, 0, 0, 0);
 373}
 374
 375static int vf_qm_version_check(struct acc_vf_data *vf_data, struct device *dev)
 376{
 377	switch (vf_data->acc_magic) {
 378	case ACC_DEV_MAGIC_V2:
 379		if (vf_data->major_ver != ACC_DRV_MAJOR_VER) {
 380			dev_info(dev, "migration driver version<%u.%u> not match!\n",
 381				 vf_data->major_ver, vf_data->minor_ver);
 382			return -EINVAL;
 383		}
 384		break;
 385	case ACC_DEV_MAGIC_V1:
 386		/* Correct dma address */
 387		vf_data->eqe_dma = vf_data->qm_eqc_dw[QM_XQC_ADDR_HIGH];
 388		vf_data->eqe_dma <<= QM_XQC_ADDR_OFFSET;
 389		vf_data->eqe_dma |= vf_data->qm_eqc_dw[QM_XQC_ADDR_LOW];
 390		vf_data->aeqe_dma = vf_data->qm_aeqc_dw[QM_XQC_ADDR_HIGH];
 391		vf_data->aeqe_dma <<= QM_XQC_ADDR_OFFSET;
 392		vf_data->aeqe_dma |= vf_data->qm_aeqc_dw[QM_XQC_ADDR_LOW];
 393		break;
 394	default:
 395		return -EINVAL;
 396	}
 397
 398	return 0;
 399}
 400
 401static int vf_qm_check_match(struct hisi_acc_vf_core_device *hisi_acc_vdev,
 402			     struct hisi_acc_vf_migration_file *migf)
 403{
 404	struct acc_vf_data *vf_data = &migf->vf_data;
 405	struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
 406	struct hisi_qm *pf_qm = hisi_acc_vdev->pf_qm;
 407	struct device *dev = &vf_qm->pdev->dev;
 408	u32 que_iso_state;
 409	int ret;
 410
 411	if (migf->total_length < QM_MATCH_SIZE || hisi_acc_vdev->match_done)
 412		return 0;
 413
 414	ret = vf_qm_version_check(vf_data, dev);
 415	if (ret) {
 416		dev_err(dev, "failed to match ACC_DEV_MAGIC\n");
 417		return ret;
 418	}
 419
 420	if (vf_data->dev_id != hisi_acc_vdev->vf_dev->device) {
 421		dev_err(dev, "failed to match VF devices\n");
 422		return -EINVAL;
 423	}
 424
 425	/* VF qp num check */
 426	ret = qm_get_vft(vf_qm, &vf_qm->qp_base);
 427	if (ret <= 0) {
 428		dev_err(dev, "failed to get vft qp nums\n");
 429		return ret < 0 ? ret : -EINVAL;
 430	}
 431
 432	if (ret != vf_data->qp_num) {
 433		dev_err(dev, "failed to match VF qp num\n");
 434		return -EINVAL;
 435	}
 436
 437	vf_qm->qp_num = ret;
 438
 439	/* VF isolation state check */
 440	ret = qm_read_regs(pf_qm, QM_QUE_ISO_CFG_V, &que_iso_state, 1);
 441	if (ret) {
 442		dev_err(dev, "failed to read QM_QUE_ISO_CFG_V\n");
 443		return ret;
 444	}
 445
 446	if (vf_data->que_iso_cfg != que_iso_state) {
 447		dev_err(dev, "failed to match isolation state\n");
 448		return -EINVAL;
 449	}
 450
 451	hisi_acc_vdev->match_done = true;
 452	return 0;
 453}
 454
 455static int vf_qm_get_match_data(struct hisi_acc_vf_core_device *hisi_acc_vdev,
 456				struct acc_vf_data *vf_data)
 457{
 458	struct hisi_qm *pf_qm = hisi_acc_vdev->pf_qm;
 459	struct device *dev = &pf_qm->pdev->dev;
 460	int vf_id = hisi_acc_vdev->vf_id;
 461	int ret;
 462
 463	vf_data->acc_magic = ACC_DEV_MAGIC_V2;
 464	vf_data->major_ver = ACC_DRV_MAJOR_VER;
 465	vf_data->minor_ver = ACC_DRV_MINOR_VER;
 466	/* Save device id */
 467	vf_data->dev_id = hisi_acc_vdev->vf_dev->device;
 468
 469	/* VF qp num save from PF */
 470	ret = pf_qm_get_qp_num(pf_qm, vf_id, &vf_data->qp_base);
 471	if (ret <= 0) {
 472		dev_err(dev, "failed to get vft qp nums!\n");
 473		return -EINVAL;
 474	}
 475
 476	vf_data->qp_num = ret;
 477
 478	/* VF isolation state save from PF */
 479	ret = qm_read_regs(pf_qm, QM_QUE_ISO_CFG_V, &vf_data->que_iso_cfg, 1);
 480	if (ret) {
 481		dev_err(dev, "failed to read QM_QUE_ISO_CFG_V!\n");
 482		return ret;
 483	}
 484
 485	return 0;
 486}
 487
 488static void vf_qm_xeqc_save(struct hisi_qm *qm,
 489			    struct hisi_acc_vf_migration_file *migf)
 490{
 491	struct acc_vf_data *vf_data = &migf->vf_data;
 492	u16 eq_head, aeq_head;
 493
 494	eq_head = vf_data->qm_eqc_dw[0] & 0xFFFF;
 495	qm_db(qm, 0, QM_DOORBELL_CMD_EQ, eq_head, 0);
 496
 497	aeq_head = vf_data->qm_aeqc_dw[0] & 0xFFFF;
 498	qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, aeq_head, 0);
 499}
 500
 501static int vf_qm_load_data(struct hisi_acc_vf_core_device *hisi_acc_vdev,
 502			   struct hisi_acc_vf_migration_file *migf)
 503{
 504	struct hisi_qm *qm = &hisi_acc_vdev->vf_qm;
 505	struct device *dev = &qm->pdev->dev;
 506	struct acc_vf_data *vf_data = &migf->vf_data;
 507	int ret;
 508
 509	/* Return if only match data was transferred */
 510	if (migf->total_length == QM_MATCH_SIZE)
 511		return 0;
 512
 513	if (migf->total_length < sizeof(struct acc_vf_data))
 514		return -EINVAL;
 515
 516	if (!vf_data->eqe_dma || !vf_data->aeqe_dma ||
 517	    !vf_data->sqc_dma || !vf_data->cqc_dma) {
 518		dev_info(dev, "resume dma addr is NULL!\n");
 519		hisi_acc_vdev->vf_qm_state = QM_NOT_READY;
 520		return 0;
 521	}
 522
 523	ret = qm_write_regs(qm, QM_VF_STATE, &vf_data->vf_qm_state, 1);
 524	if (ret) {
 525		dev_err(dev, "failed to write QM_VF_STATE\n");
 526		return ret;
 527	}
 528	hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state;
 529
 530	qm->eqe_dma = vf_data->eqe_dma;
 531	qm->aeqe_dma = vf_data->aeqe_dma;
 532	qm->sqc_dma = vf_data->sqc_dma;
 533	qm->cqc_dma = vf_data->cqc_dma;
 534
 535	qm->qp_base = vf_data->qp_base;
 536	qm->qp_num = vf_data->qp_num;
 537
 538	ret = qm_set_regs(qm, vf_data);
 539	if (ret) {
 540		dev_err(dev, "set VF regs failed\n");
 541		return ret;
 542	}
 543
 544	ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, 0);
 545	if (ret) {
 546		dev_err(dev, "set sqc failed\n");
 547		return ret;
 548	}
 549
 550	ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, qm->cqc_dma, 0, 0);
 551	if (ret) {
 552		dev_err(dev, "set cqc failed\n");
 553		return ret;
 554	}
 555
 556	qm_dev_cmd_init(qm);
 557	return 0;
 558}
 559
 560static int vf_qm_read_data(struct hisi_qm *vf_qm, struct acc_vf_data *vf_data)
 561{
 562	struct device *dev = &vf_qm->pdev->dev;
 563	int ret;
 564
 565	ret = qm_get_regs(vf_qm, vf_data);
 566	if (ret)
 567		return ret;
 568
 569	/* Every reg is 32 bit, the dma address is 64 bit. */
 570	vf_data->eqe_dma = vf_data->qm_eqc_dw[QM_XQC_ADDR_HIGH];
 571	vf_data->eqe_dma <<= QM_XQC_ADDR_OFFSET;
 572	vf_data->eqe_dma |= vf_data->qm_eqc_dw[QM_XQC_ADDR_LOW];
 573	vf_data->aeqe_dma = vf_data->qm_aeqc_dw[QM_XQC_ADDR_HIGH];
 574	vf_data->aeqe_dma <<= QM_XQC_ADDR_OFFSET;
 575	vf_data->aeqe_dma |= vf_data->qm_aeqc_dw[QM_XQC_ADDR_LOW];
 576
 577	/* Through SQC_BT/CQC_BT to get sqc and cqc address */
 578	ret = qm_get_sqc(vf_qm, &vf_data->sqc_dma);
 579	if (ret) {
 580		dev_err(dev, "failed to read SQC addr!\n");
 581		return ret;
 582	}
 583
 584	ret = qm_get_cqc(vf_qm, &vf_data->cqc_dma);
 585	if (ret) {
 586		dev_err(dev, "failed to read CQC addr!\n");
 587		return ret;
 588	}
 589
 590	return 0;
 591}
 592
 593static int vf_qm_state_save(struct hisi_acc_vf_core_device *hisi_acc_vdev,
 594			    struct hisi_acc_vf_migration_file *migf)
 595{
 596	struct acc_vf_data *vf_data = &migf->vf_data;
 597	struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
 598	int ret;
 599
 600	if (unlikely(qm_wait_dev_not_ready(vf_qm))) {
 601		/* Update state and return with match data */
 602		vf_data->vf_qm_state = QM_NOT_READY;
 603		hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state;
 604		migf->total_length = QM_MATCH_SIZE;
 605		return 0;
 606	}
 607
 608	vf_data->vf_qm_state = QM_READY;
 609	hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state;
 610
 611	ret = vf_qm_read_data(vf_qm, vf_data);
 612	if (ret)
 613		return ret;
 614
 615	migf->total_length = sizeof(struct acc_vf_data);
 616	/* Save eqc and aeqc interrupt information */
 617	vf_qm_xeqc_save(vf_qm, migf);
 618
 619	return 0;
 620}
 621
 622static struct hisi_acc_vf_core_device *hisi_acc_drvdata(struct pci_dev *pdev)
 623{
 624	struct vfio_pci_core_device *core_device = dev_get_drvdata(&pdev->dev);
 625
 626	return container_of(core_device, struct hisi_acc_vf_core_device,
 627			    core_device);
 628}
 629
 630/* Check the PF's RAS state and Function INT state */
 631static int
 632hisi_acc_check_int_state(struct hisi_acc_vf_core_device *hisi_acc_vdev)
 633{
 634	struct hisi_qm *vfqm = &hisi_acc_vdev->vf_qm;
 635	struct hisi_qm *qm = hisi_acc_vdev->pf_qm;
 636	struct pci_dev *vf_pdev = hisi_acc_vdev->vf_dev;
 637	struct device *dev = &qm->pdev->dev;
 638	u32 state;
 639
 640	/* Check RAS state */
 641	state = qm_check_reg_state(qm, QM_ABNORMAL_INT_STATUS);
 642	if (state) {
 643		dev_err(dev, "failed to check QM RAS state!\n");
 644		return -EBUSY;
 645	}
 646
 647	/* Check Function Communication state between PF and VF */
 648	state = qm_check_reg_state(vfqm, QM_IFC_INT_STATUS);
 649	if (state) {
 650		dev_err(dev, "failed to check QM IFC INT state!\n");
 651		return -EBUSY;
 652	}
 653	state = qm_check_reg_state(vfqm, QM_IFC_INT_SET_V);
 654	if (state) {
 655		dev_err(dev, "failed to check QM IFC INT SET state!\n");
 656		return -EBUSY;
 657	}
 658
 659	/* Check submodule task state */
 660	switch (vf_pdev->device) {
 661	case PCI_DEVICE_ID_HUAWEI_SEC_VF:
 662		state = qm_check_reg_state(qm, SEC_CORE_INT_STATUS);
 663		if (state) {
 664			dev_err(dev, "failed to check QM SEC Core INT state!\n");
 665			return -EBUSY;
 666		}
 667		return 0;
 668	case PCI_DEVICE_ID_HUAWEI_HPRE_VF:
 669		state = qm_check_reg_state(qm, HPRE_HAC_INT_STATUS);
 670		if (state) {
 671			dev_err(dev, "failed to check QM HPRE HAC INT state!\n");
 672			return -EBUSY;
 673		}
 674		return 0;
 675	case PCI_DEVICE_ID_HUAWEI_ZIP_VF:
 676		state = qm_check_reg_state(qm, HZIP_CORE_INT_STATUS);
 677		if (state) {
 678			dev_err(dev, "failed to check QM ZIP Core INT state!\n");
 679			return -EBUSY;
 680		}
 681		return 0;
 682	default:
 683		dev_err(dev, "failed to detect acc module type!\n");
 684		return -EINVAL;
 685	}
 686}
 687
 688static void hisi_acc_vf_disable_fd(struct hisi_acc_vf_migration_file *migf)
 689{
 690	mutex_lock(&migf->lock);
 691	migf->disabled = true;
 692	migf->total_length = 0;
 693	migf->filp->f_pos = 0;
 694	mutex_unlock(&migf->lock);
 695}
 696
 697static void
 698hisi_acc_debug_migf_copy(struct hisi_acc_vf_core_device *hisi_acc_vdev,
 699			 struct hisi_acc_vf_migration_file *src_migf)
 700{
 701	struct hisi_acc_vf_migration_file *dst_migf = hisi_acc_vdev->debug_migf;
 702
 703	if (!dst_migf)
 704		return;
 705
 706	dst_migf->total_length = src_migf->total_length;
 707	memcpy(&dst_migf->vf_data, &src_migf->vf_data,
 708	       sizeof(struct acc_vf_data));
 709}
 710
 711static void hisi_acc_vf_disable_fds(struct hisi_acc_vf_core_device *hisi_acc_vdev)
 712{
 713	if (hisi_acc_vdev->resuming_migf) {
 714		hisi_acc_debug_migf_copy(hisi_acc_vdev, hisi_acc_vdev->resuming_migf);
 715		hisi_acc_vf_disable_fd(hisi_acc_vdev->resuming_migf);
 716		fput(hisi_acc_vdev->resuming_migf->filp);
 717		hisi_acc_vdev->resuming_migf = NULL;
 718	}
 719
 720	if (hisi_acc_vdev->saving_migf) {
 721		hisi_acc_debug_migf_copy(hisi_acc_vdev, hisi_acc_vdev->saving_migf);
 722		hisi_acc_vf_disable_fd(hisi_acc_vdev->saving_migf);
 723		fput(hisi_acc_vdev->saving_migf->filp);
 724		hisi_acc_vdev->saving_migf = NULL;
 725	}
 726}
 727
 728static struct hisi_acc_vf_core_device *hisi_acc_get_vf_dev(struct vfio_device *vdev)
 729{
 730	return container_of(vdev, struct hisi_acc_vf_core_device,
 731			    core_device.vdev);
 732}
 733
 734static void hisi_acc_vf_reset(struct hisi_acc_vf_core_device *hisi_acc_vdev)
 735{
 736	hisi_acc_vdev->vf_qm_state = QM_NOT_READY;
 737	hisi_acc_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING;
 738	hisi_acc_vf_disable_fds(hisi_acc_vdev);
 739}
 740
 741static void hisi_acc_vf_start_device(struct hisi_acc_vf_core_device *hisi_acc_vdev)
 742{
 743	struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
 744
 745	if (hisi_acc_vdev->vf_qm_state != QM_READY)
 746		return;
 747
 748	/* Make sure the device is enabled */
 749	qm_dev_cmd_init(vf_qm);
 750
 751	vf_qm_fun_reset(vf_qm);
 752}
 753
 754static int hisi_acc_vf_load_state(struct hisi_acc_vf_core_device *hisi_acc_vdev)
 755{
 756	struct device *dev = &hisi_acc_vdev->vf_dev->dev;
 757	struct hisi_acc_vf_migration_file *migf = hisi_acc_vdev->resuming_migf;
 758	int ret;
 759
 760	/* Recover data to VF */
 761	ret = vf_qm_load_data(hisi_acc_vdev, migf);
 762	if (ret) {
 763		dev_err(dev, "failed to recover the VF!\n");
 764		return ret;
 765	}
 766
 767	return 0;
 768}
 769
 770static int hisi_acc_vf_release_file(struct inode *inode, struct file *filp)
 771{
 772	struct hisi_acc_vf_migration_file *migf = filp->private_data;
 773
 774	hisi_acc_vf_disable_fd(migf);
 775	mutex_destroy(&migf->lock);
 776	kfree(migf);
 777	return 0;
 778}
 779
 780static ssize_t hisi_acc_vf_resume_write(struct file *filp, const char __user *buf,
 781					size_t len, loff_t *pos)
 782{
 783	struct hisi_acc_vf_migration_file *migf = filp->private_data;
 784	u8 *vf_data = (u8 *)&migf->vf_data;
 785	loff_t requested_length;
 786	ssize_t done = 0;
 787	int ret;
 788
 789	if (pos)
 790		return -ESPIPE;
 791	pos = &filp->f_pos;
 792
 793	if (*pos < 0 ||
 794	    check_add_overflow((loff_t)len, *pos, &requested_length))
 795		return -EINVAL;
 796
 797	if (requested_length > sizeof(struct acc_vf_data))
 798		return -ENOMEM;
 799
 800	mutex_lock(&migf->lock);
 801	if (migf->disabled) {
 802		done = -ENODEV;
 803		goto out_unlock;
 804	}
 805
 806	ret = copy_from_user(vf_data + *pos, buf, len);
 807	if (ret) {
 808		done = -EFAULT;
 809		goto out_unlock;
 810	}
 811	*pos += len;
 812	done = len;
 813	migf->total_length += len;
 814
 815	ret = vf_qm_check_match(migf->hisi_acc_vdev, migf);
 816	if (ret)
 817		done = -EFAULT;
 818out_unlock:
 819	mutex_unlock(&migf->lock);
 820	return done;
 821}
 822
 823static const struct file_operations hisi_acc_vf_resume_fops = {
 824	.owner = THIS_MODULE,
 825	.write = hisi_acc_vf_resume_write,
 826	.release = hisi_acc_vf_release_file,
 827};
 828
 829static struct hisi_acc_vf_migration_file *
 830hisi_acc_vf_pci_resume(struct hisi_acc_vf_core_device *hisi_acc_vdev)
 831{
 832	struct hisi_acc_vf_migration_file *migf;
 833
 834	migf = kzalloc(sizeof(*migf), GFP_KERNEL_ACCOUNT);
 835	if (!migf)
 836		return ERR_PTR(-ENOMEM);
 837
 838	migf->filp = anon_inode_getfile("hisi_acc_vf_mig", &hisi_acc_vf_resume_fops, migf,
 839					O_WRONLY);
 840	if (IS_ERR(migf->filp)) {
 841		int err = PTR_ERR(migf->filp);
 842
 843		kfree(migf);
 844		return ERR_PTR(err);
 845	}
 846
 847	stream_open(migf->filp->f_inode, migf->filp);
 848	mutex_init(&migf->lock);
 849	migf->hisi_acc_vdev = hisi_acc_vdev;
 850	return migf;
 851}
 852
 853static long hisi_acc_vf_precopy_ioctl(struct file *filp,
 854				      unsigned int cmd, unsigned long arg)
 855{
 856	struct hisi_acc_vf_migration_file *migf = filp->private_data;
 857	struct hisi_acc_vf_core_device *hisi_acc_vdev = migf->hisi_acc_vdev;
 858	loff_t *pos = &filp->f_pos;
 859	struct vfio_precopy_info info;
 860	unsigned long minsz;
 861	int ret;
 862
 863	if (cmd != VFIO_MIG_GET_PRECOPY_INFO)
 864		return -ENOTTY;
 865
 866	minsz = offsetofend(struct vfio_precopy_info, dirty_bytes);
 867
 868	if (copy_from_user(&info, (void __user *)arg, minsz))
 869		return -EFAULT;
 870	if (info.argsz < minsz)
 871		return -EINVAL;
 872
 873	mutex_lock(&hisi_acc_vdev->state_mutex);
 874	if (hisi_acc_vdev->mig_state != VFIO_DEVICE_STATE_PRE_COPY) {
 875		mutex_unlock(&hisi_acc_vdev->state_mutex);
 876		return -EINVAL;
 877	}
 878
 879	mutex_lock(&migf->lock);
 880
 881	if (migf->disabled) {
 882		ret = -ENODEV;
 883		goto out;
 884	}
 885
 886	if (*pos > migf->total_length) {
 887		ret = -EINVAL;
 888		goto out;
 889	}
 890
 891	info.dirty_bytes = 0;
 892	info.initial_bytes = migf->total_length - *pos;
 893	mutex_unlock(&migf->lock);
 894	mutex_unlock(&hisi_acc_vdev->state_mutex);
 895
 896	return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
 897out:
 898	mutex_unlock(&migf->lock);
 899	mutex_unlock(&hisi_acc_vdev->state_mutex);
 900	return ret;
 901}
 902
 903static ssize_t hisi_acc_vf_save_read(struct file *filp, char __user *buf, size_t len,
 904				     loff_t *pos)
 905{
 906	struct hisi_acc_vf_migration_file *migf = filp->private_data;
 907	ssize_t done = 0;
 908	int ret;
 909
 910	if (pos)
 911		return -ESPIPE;
 912	pos = &filp->f_pos;
 913
 914	mutex_lock(&migf->lock);
 915	if (*pos > migf->total_length) {
 916		done = -EINVAL;
 917		goto out_unlock;
 918	}
 919
 920	if (migf->disabled) {
 921		done = -ENODEV;
 922		goto out_unlock;
 923	}
 924
 925	len = min_t(size_t, migf->total_length - *pos, len);
 926	if (len) {
 927		u8 *vf_data = (u8 *)&migf->vf_data;
 928
 929		ret = copy_to_user(buf, vf_data + *pos, len);
 930		if (ret) {
 931			done = -EFAULT;
 932			goto out_unlock;
 933		}
 934		*pos += len;
 935		done = len;
 936	}
 937out_unlock:
 938	mutex_unlock(&migf->lock);
 939	return done;
 940}
 941
 942static const struct file_operations hisi_acc_vf_save_fops = {
 943	.owner = THIS_MODULE,
 944	.read = hisi_acc_vf_save_read,
 945	.unlocked_ioctl = hisi_acc_vf_precopy_ioctl,
 946	.compat_ioctl = compat_ptr_ioctl,
 947	.release = hisi_acc_vf_release_file,
 948};
 949
 950static struct hisi_acc_vf_migration_file *
 951hisi_acc_open_saving_migf(struct hisi_acc_vf_core_device *hisi_acc_vdev)
 952{
 953	struct hisi_acc_vf_migration_file *migf;
 954	int ret;
 955
 956	migf = kzalloc(sizeof(*migf), GFP_KERNEL_ACCOUNT);
 957	if (!migf)
 958		return ERR_PTR(-ENOMEM);
 959
 960	migf->filp = anon_inode_getfile("hisi_acc_vf_mig", &hisi_acc_vf_save_fops, migf,
 961					O_RDONLY);
 962	if (IS_ERR(migf->filp)) {
 963		int err = PTR_ERR(migf->filp);
 964
 965		kfree(migf);
 966		return ERR_PTR(err);
 967	}
 968
 969	stream_open(migf->filp->f_inode, migf->filp);
 970	mutex_init(&migf->lock);
 971	migf->hisi_acc_vdev = hisi_acc_vdev;
 972
 973	ret = vf_qm_get_match_data(hisi_acc_vdev, &migf->vf_data);
 974	if (ret) {
 975		fput(migf->filp);
 976		return ERR_PTR(ret);
 977	}
 978
 979	return migf;
 980}
 981
 982static struct hisi_acc_vf_migration_file *
 983hisi_acc_vf_pre_copy(struct hisi_acc_vf_core_device *hisi_acc_vdev)
 984{
 985	struct hisi_acc_vf_migration_file *migf;
 986
 987	migf = hisi_acc_open_saving_migf(hisi_acc_vdev);
 988	if (IS_ERR(migf))
 989		return migf;
 990
 991	migf->total_length = QM_MATCH_SIZE;
 992	return migf;
 993}
 994
 995static struct hisi_acc_vf_migration_file *
 996hisi_acc_vf_stop_copy(struct hisi_acc_vf_core_device *hisi_acc_vdev, bool open)
 997{
 998	int ret;
 999	struct hisi_acc_vf_migration_file *migf = NULL;
1000
1001	if (open) {
1002		/*
1003		 * Userspace didn't use PRECOPY support. Hence saving_migf
1004		 * is not opened yet.
1005		 */
1006		migf = hisi_acc_open_saving_migf(hisi_acc_vdev);
1007		if (IS_ERR(migf))
1008			return migf;
1009	} else {
1010		migf = hisi_acc_vdev->saving_migf;
1011	}
1012
1013	ret = vf_qm_state_save(hisi_acc_vdev, migf);
1014	if (ret)
1015		return ERR_PTR(ret);
1016
1017	return open ? migf : NULL;
1018}
1019
1020static int hisi_acc_vf_stop_device(struct hisi_acc_vf_core_device *hisi_acc_vdev)
1021{
1022	struct device *dev = &hisi_acc_vdev->vf_dev->dev;
1023	struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
1024	int ret;
1025
1026	ret = vf_qm_func_stop(vf_qm);
1027	if (ret) {
1028		dev_err(dev, "failed to stop QM VF function!\n");
1029		return ret;
1030	}
1031
1032	ret = hisi_acc_check_int_state(hisi_acc_vdev);
1033	if (ret) {
1034		dev_err(dev, "failed to check QM INT state!\n");
1035		return ret;
1036	}
1037
1038	ret = vf_qm_cache_wb(vf_qm);
1039	if (ret) {
1040		dev_err(dev, "failed to writeback QM cache!\n");
1041		return ret;
1042	}
1043
1044	return 0;
1045}
1046
1047static struct file *
1048hisi_acc_vf_set_device_state(struct hisi_acc_vf_core_device *hisi_acc_vdev,
1049			     u32 new)
1050{
1051	u32 cur = hisi_acc_vdev->mig_state;
1052	int ret;
1053
1054	if (cur == VFIO_DEVICE_STATE_RUNNING && new == VFIO_DEVICE_STATE_PRE_COPY) {
1055		struct hisi_acc_vf_migration_file *migf;
1056
1057		migf = hisi_acc_vf_pre_copy(hisi_acc_vdev);
1058		if (IS_ERR(migf))
1059			return ERR_CAST(migf);
1060		get_file(migf->filp);
1061		hisi_acc_vdev->saving_migf = migf;
1062		return migf->filp;
1063	}
1064
1065	if (cur == VFIO_DEVICE_STATE_PRE_COPY && new == VFIO_DEVICE_STATE_STOP_COPY) {
1066		struct hisi_acc_vf_migration_file *migf;
1067
1068		ret = hisi_acc_vf_stop_device(hisi_acc_vdev);
1069		if (ret)
1070			return ERR_PTR(ret);
1071
1072		migf = hisi_acc_vf_stop_copy(hisi_acc_vdev, false);
1073		if (IS_ERR(migf))
1074			return ERR_CAST(migf);
1075
1076		return NULL;
1077	}
1078
1079	if (cur == VFIO_DEVICE_STATE_RUNNING && new == VFIO_DEVICE_STATE_STOP) {
1080		ret = hisi_acc_vf_stop_device(hisi_acc_vdev);
1081		if (ret)
1082			return ERR_PTR(ret);
1083		return NULL;
1084	}
1085
1086	if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_STOP_COPY) {
1087		struct hisi_acc_vf_migration_file *migf;
1088
1089		migf = hisi_acc_vf_stop_copy(hisi_acc_vdev, true);
1090		if (IS_ERR(migf))
1091			return ERR_CAST(migf);
1092		get_file(migf->filp);
1093		hisi_acc_vdev->saving_migf = migf;
1094		return migf->filp;
1095	}
1096
1097	if ((cur == VFIO_DEVICE_STATE_STOP_COPY && new == VFIO_DEVICE_STATE_STOP)) {
1098		hisi_acc_vf_disable_fds(hisi_acc_vdev);
1099		return NULL;
1100	}
1101
1102	if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RESUMING) {
1103		struct hisi_acc_vf_migration_file *migf;
1104
1105		migf = hisi_acc_vf_pci_resume(hisi_acc_vdev);
1106		if (IS_ERR(migf))
1107			return ERR_CAST(migf);
1108		get_file(migf->filp);
1109		hisi_acc_vdev->resuming_migf = migf;
1110		return migf->filp;
1111	}
1112
1113	if (cur == VFIO_DEVICE_STATE_RESUMING && new == VFIO_DEVICE_STATE_STOP) {
1114		ret = hisi_acc_vf_load_state(hisi_acc_vdev);
1115		if (ret)
1116			return ERR_PTR(ret);
1117		hisi_acc_vf_disable_fds(hisi_acc_vdev);
1118		return NULL;
1119	}
1120
1121	if (cur == VFIO_DEVICE_STATE_PRE_COPY && new == VFIO_DEVICE_STATE_RUNNING) {
1122		hisi_acc_vf_disable_fds(hisi_acc_vdev);
1123		return NULL;
1124	}
1125
1126	if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RUNNING) {
1127		hisi_acc_vf_start_device(hisi_acc_vdev);
1128		return NULL;
1129	}
1130
1131	/*
1132	 * vfio_mig_get_next_state() does not use arcs other than the above
1133	 */
1134	WARN_ON(true);
1135	return ERR_PTR(-EINVAL);
1136}
1137
1138static struct file *
1139hisi_acc_vfio_pci_set_device_state(struct vfio_device *vdev,
1140				   enum vfio_device_mig_state new_state)
1141{
1142	struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_get_vf_dev(vdev);
1143	enum vfio_device_mig_state next_state;
1144	struct file *res = NULL;
1145	int ret;
1146
1147	mutex_lock(&hisi_acc_vdev->state_mutex);
1148	while (new_state != hisi_acc_vdev->mig_state) {
1149		ret = vfio_mig_get_next_state(vdev,
1150					      hisi_acc_vdev->mig_state,
1151					      new_state, &next_state);
1152		if (ret) {
1153			res = ERR_PTR(-EINVAL);
1154			break;
1155		}
1156
1157		res = hisi_acc_vf_set_device_state(hisi_acc_vdev, next_state);
1158		if (IS_ERR(res))
1159			break;
1160		hisi_acc_vdev->mig_state = next_state;
1161		if (WARN_ON(res && new_state != hisi_acc_vdev->mig_state)) {
1162			fput(res);
1163			res = ERR_PTR(-EINVAL);
1164			break;
1165		}
1166	}
1167	mutex_unlock(&hisi_acc_vdev->state_mutex);
1168	return res;
1169}
1170
1171static int
1172hisi_acc_vfio_pci_get_data_size(struct vfio_device *vdev,
1173				unsigned long *stop_copy_length)
1174{
1175	*stop_copy_length = sizeof(struct acc_vf_data);
1176	return 0;
1177}
1178
1179static int
1180hisi_acc_vfio_pci_get_device_state(struct vfio_device *vdev,
1181				   enum vfio_device_mig_state *curr_state)
1182{
1183	struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_get_vf_dev(vdev);
1184
1185	mutex_lock(&hisi_acc_vdev->state_mutex);
1186	*curr_state = hisi_acc_vdev->mig_state;
1187	mutex_unlock(&hisi_acc_vdev->state_mutex);
1188	return 0;
1189}
1190
1191static void hisi_acc_vf_pci_reset_prepare(struct pci_dev *pdev)
1192{
1193	struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev);
1194	struct hisi_qm *qm = hisi_acc_vdev->pf_qm;
1195	struct device *dev = &qm->pdev->dev;
1196	u32 delay = 0;
1197
1198	/* All reset requests need to be queued for processing */
1199	while (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) {
1200		msleep(1);
1201		if (++delay > QM_RESET_WAIT_TIMEOUT) {
1202			dev_err(dev, "reset prepare failed\n");
1203			return;
1204		}
1205	}
1206
1207	hisi_acc_vdev->set_reset_flag = true;
1208}
1209
1210static void hisi_acc_vf_pci_aer_reset_done(struct pci_dev *pdev)
1211{
1212	struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev);
1213	struct hisi_qm *qm = hisi_acc_vdev->pf_qm;
1214
1215	if (hisi_acc_vdev->set_reset_flag)
1216		clear_bit(QM_RESETTING, &qm->misc_ctl);
1217
1218	if (!hisi_acc_vdev->core_device.vdev.mig_ops)
1219		return;
1220
1221	mutex_lock(&hisi_acc_vdev->state_mutex);
1222	hisi_acc_vf_reset(hisi_acc_vdev);
1223	mutex_unlock(&hisi_acc_vdev->state_mutex);
1224}
1225
1226static int hisi_acc_vf_qm_init(struct hisi_acc_vf_core_device *hisi_acc_vdev)
1227{
1228	struct vfio_pci_core_device *vdev = &hisi_acc_vdev->core_device;
1229	struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
1230	struct hisi_qm *pf_qm = hisi_acc_vdev->pf_qm;
1231	struct pci_dev *vf_dev = vdev->pdev;
1232	u32 val;
1233
1234	val = readl(pf_qm->io_base + QM_MIG_REGION_SEL);
1235	if (pf_qm->ver > QM_HW_V3 && (val & QM_MIG_REGION_EN))
1236		hisi_acc_vdev->drv_mode = HW_ACC_MIG_PF_CTRL;
1237	else
1238		hisi_acc_vdev->drv_mode = HW_ACC_MIG_VF_CTRL;
1239
1240	if (hisi_acc_vdev->drv_mode == HW_ACC_MIG_PF_CTRL) {
1241		/*
1242		 * On hardware platforms greater than QM_HW_V3, the migration function
1243		 * register is placed in the BAR2 configuration region of the PF,
1244		 * and each VF device occupies 8KB of configuration space.
1245		 */
1246		vf_qm->io_base = pf_qm->io_base + QM_MIG_REGION_OFFSET +
1247				 hisi_acc_vdev->vf_id * QM_MIG_REGION_SIZE;
1248	} else {
1249		/*
1250		 * ACC VF dev BAR2 region consists of both functional register space
1251		 * and migration control register space. For migration to work, we
1252		 * need access to both. Hence, we map the entire BAR2 region here.
1253		 * But unnecessarily exposing the migration BAR region to the Guest
1254		 * has the potential to prevent/corrupt the Guest migration. Hence,
1255		 * we restrict access to the migration control space from
1256		 * Guest(Please see mmap/ioctl/read/write override functions).
1257		 *
1258		 * Please note that it is OK to expose the entire VF BAR if migration
1259		 * is not supported or required as this cannot affect the ACC PF
1260		 * configurations.
1261		 *
1262		 * Also the HiSilicon ACC VF devices supported by this driver on
1263		 * HiSilicon hardware platforms are integrated end point devices
1264		 * and the platform lacks the capability to perform any PCIe P2P
1265		 * between these devices.
1266		 */
1267
1268		vf_qm->io_base =
1269			ioremap(pci_resource_start(vf_dev, VFIO_PCI_BAR2_REGION_INDEX),
1270				pci_resource_len(vf_dev, VFIO_PCI_BAR2_REGION_INDEX));
1271		if (!vf_qm->io_base)
1272			return -EIO;
1273	}
1274	vf_qm->fun_type = QM_HW_VF;
1275	vf_qm->ver = pf_qm->ver;
1276	vf_qm->pdev = vf_dev;
1277	mutex_init(&vf_qm->mailbox_lock);
1278
1279	return 0;
1280}
1281
1282static struct hisi_qm *hisi_acc_get_pf_qm(struct pci_dev *pdev)
1283{
1284	struct hisi_qm	*pf_qm;
1285	struct pci_driver *pf_driver;
1286
1287	if (!pdev->is_virtfn)
1288		return NULL;
1289
1290	switch (pdev->device) {
1291	case PCI_DEVICE_ID_HUAWEI_SEC_VF:
1292		pf_driver = hisi_sec_get_pf_driver();
1293		break;
1294	case PCI_DEVICE_ID_HUAWEI_HPRE_VF:
1295		pf_driver = hisi_hpre_get_pf_driver();
1296		break;
1297	case PCI_DEVICE_ID_HUAWEI_ZIP_VF:
1298		pf_driver = hisi_zip_get_pf_driver();
1299		break;
1300	default:
1301		return NULL;
1302	}
1303
1304	if (!pf_driver)
1305		return NULL;
1306
1307	pf_qm = pci_iov_get_pf_drvdata(pdev, pf_driver);
1308
1309	return !IS_ERR(pf_qm) ? pf_qm : NULL;
1310}
1311
1312static size_t hisi_acc_get_resource_len(struct vfio_pci_core_device *vdev,
1313					unsigned int index)
1314{
1315	struct hisi_acc_vf_core_device *hisi_acc_vdev =
1316			hisi_acc_drvdata(vdev->pdev);
1317
1318	/*
1319	 * On the old HW_ACC_MIG_VF_CTRL mode device, the ACC VF device
1320	 * BAR2 region encompasses both functional register space
1321	 * and migration control register space.
1322	 * only the functional region should be report to Guest.
1323	 */
1324	if (hisi_acc_vdev->drv_mode == HW_ACC_MIG_VF_CTRL)
1325		return (pci_resource_len(vdev->pdev, index) >> 1);
1326	/*
1327	 * On the new HW device, the migration control register
1328	 * has been moved to the PF device BAR2 region.
1329	 * The VF device BAR2 is entirely functional register space.
1330	 */
1331	return pci_resource_len(vdev->pdev, index);
1332}
1333
1334static int hisi_acc_pci_rw_access_check(struct vfio_device *core_vdev,
1335					size_t count, loff_t *ppos,
1336					size_t *new_count)
1337{
1338	unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
1339	struct vfio_pci_core_device *vdev =
1340		container_of(core_vdev, struct vfio_pci_core_device, vdev);
1341
1342	if (index == VFIO_PCI_BAR2_REGION_INDEX) {
1343		loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
1344		resource_size_t end;
1345
1346		end = hisi_acc_get_resource_len(vdev, index);
1347		/* Check if access is for migration control region */
1348		if (pos >= end)
1349			return -EINVAL;
1350
1351		*new_count = min(count, (size_t)(end - pos));
1352	}
1353
1354	return 0;
1355}
1356
1357static int hisi_acc_vfio_pci_mmap(struct vfio_device *core_vdev,
1358				  struct vm_area_struct *vma)
1359{
1360	struct vfio_pci_core_device *vdev =
1361		container_of(core_vdev, struct vfio_pci_core_device, vdev);
1362	unsigned int index;
1363
1364	index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
1365	if (index == VFIO_PCI_BAR2_REGION_INDEX) {
1366		u64 req_len, pgoff, req_start;
1367		resource_size_t end;
1368
1369		end = hisi_acc_get_resource_len(vdev, index);
1370		req_len = vma->vm_end - vma->vm_start;
1371		pgoff = vma->vm_pgoff &
1372			((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
1373		req_start = pgoff << PAGE_SHIFT;
1374
1375		if (req_start + req_len > end)
1376			return -EINVAL;
1377	}
1378
1379	return vfio_pci_core_mmap(core_vdev, vma);
1380}
1381
1382static ssize_t hisi_acc_vfio_pci_write(struct vfio_device *core_vdev,
1383				       const char __user *buf, size_t count,
1384				       loff_t *ppos)
1385{
1386	size_t new_count = count;
1387	int ret;
1388
1389	ret = hisi_acc_pci_rw_access_check(core_vdev, count, ppos, &new_count);
1390	if (ret)
1391		return ret;
1392
1393	return vfio_pci_core_write(core_vdev, buf, new_count, ppos);
1394}
1395
1396static ssize_t hisi_acc_vfio_pci_read(struct vfio_device *core_vdev,
1397				      char __user *buf, size_t count,
1398				      loff_t *ppos)
1399{
1400	size_t new_count = count;
1401	int ret;
1402
1403	ret = hisi_acc_pci_rw_access_check(core_vdev, count, ppos, &new_count);
1404	if (ret)
1405		return ret;
1406
1407	return vfio_pci_core_read(core_vdev, buf, new_count, ppos);
1408}
1409
1410static int hisi_acc_vfio_ioctl_get_region(struct vfio_device *core_vdev,
1411					  struct vfio_region_info *info,
1412					  struct vfio_info_cap *caps)
1413{
1414	struct vfio_pci_core_device *vdev =
1415		container_of(core_vdev, struct vfio_pci_core_device, vdev);
1416
1417	if (info->index != VFIO_PCI_BAR2_REGION_INDEX)
1418		return vfio_pci_ioctl_get_region_info(core_vdev, info, caps);
1419
1420	info->offset = VFIO_PCI_INDEX_TO_OFFSET(info->index);
1421
1422	info->size = hisi_acc_get_resource_len(vdev, info->index);
1423
1424	info->flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE |
1425		     VFIO_REGION_INFO_FLAG_MMAP;
1426	return 0;
1427}
1428
1429static int hisi_acc_vf_debug_check(struct seq_file *seq, struct vfio_device *vdev)
1430{
1431	struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_get_vf_dev(vdev);
1432	struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
1433	int ret;
1434
1435	lockdep_assert_held(&hisi_acc_vdev->open_mutex);
1436	/*
1437	 * When the device is not opened, the io_base is not mapped.
1438	 * The driver cannot perform device read and write operations.
1439	 */
1440	if (!hisi_acc_vdev->dev_opened) {
1441		seq_puts(seq, "device not opened!\n");
1442		return -EINVAL;
1443	}
1444
1445	ret = qm_wait_dev_not_ready(vf_qm);
1446	if (ret) {
1447		seq_puts(seq, "VF device not ready!\n");
1448		return ret;
1449	}
1450
1451	return 0;
1452}
1453
1454static int hisi_acc_vf_debug_cmd(struct seq_file *seq, void *data)
1455{
1456	struct device *vf_dev = seq->private;
1457	struct vfio_pci_core_device *core_device = dev_get_drvdata(vf_dev);
1458	struct vfio_device *vdev = &core_device->vdev;
1459	struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_get_vf_dev(vdev);
1460	struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
1461	u64 value;
1462	int ret;
1463
1464	mutex_lock(&hisi_acc_vdev->open_mutex);
1465	ret = hisi_acc_vf_debug_check(seq, vdev);
1466	if (ret) {
1467		mutex_unlock(&hisi_acc_vdev->open_mutex);
1468		return ret;
1469	}
1470
1471	value = readl(vf_qm->io_base + QM_MB_CMD_SEND_BASE);
1472	if (value == QM_MB_CMD_NOT_READY) {
1473		mutex_unlock(&hisi_acc_vdev->open_mutex);
1474		seq_puts(seq, "mailbox cmd channel not ready!\n");
1475		return -EINVAL;
1476	}
1477	mutex_unlock(&hisi_acc_vdev->open_mutex);
1478	seq_puts(seq, "mailbox cmd channel ready!\n");
1479
1480	return 0;
1481}
1482
1483static int hisi_acc_vf_dev_read(struct seq_file *seq, void *data)
1484{
1485	struct device *vf_dev = seq->private;
1486	struct vfio_pci_core_device *core_device = dev_get_drvdata(vf_dev);
1487	struct vfio_device *vdev = &core_device->vdev;
1488	struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_get_vf_dev(vdev);
1489	size_t vf_data_sz = offsetofend(struct acc_vf_data, padding);
1490	struct acc_vf_data *vf_data;
1491	int ret;
1492
1493	mutex_lock(&hisi_acc_vdev->open_mutex);
1494	ret = hisi_acc_vf_debug_check(seq, vdev);
1495	if (ret) {
1496		mutex_unlock(&hisi_acc_vdev->open_mutex);
1497		return ret;
1498	}
1499
1500	mutex_lock(&hisi_acc_vdev->state_mutex);
1501	vf_data = kzalloc(sizeof(*vf_data), GFP_KERNEL);
1502	if (!vf_data) {
1503		ret = -ENOMEM;
1504		goto mutex_release;
1505	}
1506
1507	vf_data->vf_qm_state = hisi_acc_vdev->vf_qm_state;
1508	ret = vf_qm_read_data(&hisi_acc_vdev->vf_qm, vf_data);
1509	if (ret)
1510		goto migf_err;
1511
1512	seq_hex_dump(seq, "Dev Data:", DUMP_PREFIX_OFFSET, 16, 1,
1513		     (const void *)vf_data, vf_data_sz, false);
1514
1515	seq_printf(seq,
1516		   "guest driver load: %u\n"
1517		   "data size: %lu\n",
1518		   hisi_acc_vdev->vf_qm_state,
1519		   sizeof(struct acc_vf_data));
1520
1521migf_err:
1522	kfree(vf_data);
1523mutex_release:
1524	mutex_unlock(&hisi_acc_vdev->state_mutex);
1525	mutex_unlock(&hisi_acc_vdev->open_mutex);
1526
1527	return ret;
1528}
1529
1530static int hisi_acc_vf_migf_read(struct seq_file *seq, void *data)
1531{
1532	struct device *vf_dev = seq->private;
1533	struct vfio_pci_core_device *core_device = dev_get_drvdata(vf_dev);
1534	struct vfio_device *vdev = &core_device->vdev;
1535	struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_get_vf_dev(vdev);
1536	size_t vf_data_sz = offsetofend(struct acc_vf_data, padding);
1537	struct hisi_acc_vf_migration_file *debug_migf = hisi_acc_vdev->debug_migf;
1538
1539	/* Check whether the live migration operation has been performed */
1540	if (debug_migf->total_length < QM_MATCH_SIZE) {
1541		seq_puts(seq, "device not migrated!\n");
1542		return -EAGAIN;
1543	}
1544
1545	seq_hex_dump(seq, "Mig Data:", DUMP_PREFIX_OFFSET, 16, 1,
1546		     (const void *)&debug_migf->vf_data, vf_data_sz, false);
1547	seq_printf(seq, "migrate data length: %lu\n", debug_migf->total_length);
1548
1549	return 0;
1550}
1551
1552static int hisi_acc_vfio_pci_open_device(struct vfio_device *core_vdev)
1553{
1554	struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_get_vf_dev(core_vdev);
1555	struct vfio_pci_core_device *vdev = &hisi_acc_vdev->core_device;
1556	int ret;
1557
1558	ret = vfio_pci_core_enable(vdev);
1559	if (ret)
1560		return ret;
1561
1562	if (core_vdev->mig_ops) {
1563		mutex_lock(&hisi_acc_vdev->open_mutex);
1564		ret = hisi_acc_vf_qm_init(hisi_acc_vdev);
1565		if (ret) {
1566			mutex_unlock(&hisi_acc_vdev->open_mutex);
1567			vfio_pci_core_disable(vdev);
1568			return ret;
1569		}
1570		hisi_acc_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING;
1571		hisi_acc_vdev->dev_opened = true;
1572		hisi_acc_vdev->match_done = 0;
1573		mutex_unlock(&hisi_acc_vdev->open_mutex);
1574	}
1575
1576	vfio_pci_core_finish_enable(vdev);
1577	return 0;
1578}
1579
1580static void hisi_acc_vfio_pci_close_device(struct vfio_device *core_vdev)
1581{
1582	struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_get_vf_dev(core_vdev);
1583	struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
1584
1585	hisi_acc_vf_disable_fds(hisi_acc_vdev);
1586	mutex_lock(&hisi_acc_vdev->open_mutex);
1587	hisi_acc_vdev->dev_opened = false;
1588	if (hisi_acc_vdev->drv_mode == HW_ACC_MIG_VF_CTRL)
1589		iounmap(vf_qm->io_base);
1590	mutex_unlock(&hisi_acc_vdev->open_mutex);
1591	vfio_pci_core_close_device(core_vdev);
1592}
1593
1594static const struct vfio_migration_ops hisi_acc_vfio_pci_migrn_state_ops = {
1595	.migration_set_state = hisi_acc_vfio_pci_set_device_state,
1596	.migration_get_state = hisi_acc_vfio_pci_get_device_state,
1597	.migration_get_data_size = hisi_acc_vfio_pci_get_data_size,
1598};
1599
1600static int hisi_acc_vfio_pci_migrn_init_dev(struct vfio_device *core_vdev)
1601{
1602	struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_get_vf_dev(core_vdev);
1603	struct pci_dev *pdev = to_pci_dev(core_vdev->dev);
1604	struct hisi_qm *pf_qm = hisi_acc_get_pf_qm(pdev);
1605
1606	hisi_acc_vdev->vf_id = pci_iov_vf_id(pdev) + 1;
1607	hisi_acc_vdev->pf_qm = pf_qm;
1608	hisi_acc_vdev->vf_dev = pdev;
1609	hisi_acc_vdev->vf_qm_state = QM_NOT_READY;
1610	mutex_init(&hisi_acc_vdev->state_mutex);
1611	mutex_init(&hisi_acc_vdev->open_mutex);
1612
1613	core_vdev->migration_flags = VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_PRE_COPY;
1614	core_vdev->mig_ops = &hisi_acc_vfio_pci_migrn_state_ops;
1615
1616	return vfio_pci_core_init_dev(core_vdev);
1617}
1618
1619static const struct vfio_device_ops hisi_acc_vfio_pci_migrn_ops = {
1620	.name = "hisi-acc-vfio-pci-migration",
1621	.init = hisi_acc_vfio_pci_migrn_init_dev,
1622	.release = vfio_pci_core_release_dev,
1623	.open_device = hisi_acc_vfio_pci_open_device,
1624	.close_device = hisi_acc_vfio_pci_close_device,
1625	.ioctl = vfio_pci_core_ioctl,
1626	.get_region_info_caps = hisi_acc_vfio_ioctl_get_region,
1627	.device_feature = vfio_pci_core_ioctl_feature,
1628	.read = hisi_acc_vfio_pci_read,
1629	.write = hisi_acc_vfio_pci_write,
1630	.mmap = hisi_acc_vfio_pci_mmap,
1631	.request = vfio_pci_core_request,
1632	.match = vfio_pci_core_match,
1633	.match_token_uuid = vfio_pci_core_match_token_uuid,
1634	.bind_iommufd = vfio_iommufd_physical_bind,
1635	.unbind_iommufd = vfio_iommufd_physical_unbind,
1636	.attach_ioas = vfio_iommufd_physical_attach_ioas,
1637	.detach_ioas = vfio_iommufd_physical_detach_ioas,
1638};
1639
1640static const struct vfio_device_ops hisi_acc_vfio_pci_ops = {
1641	.name = "hisi-acc-vfio-pci",
1642	.init = vfio_pci_core_init_dev,
1643	.release = vfio_pci_core_release_dev,
1644	.open_device = hisi_acc_vfio_pci_open_device,
1645	.close_device = vfio_pci_core_close_device,
1646	.ioctl = vfio_pci_core_ioctl,
1647	.get_region_info_caps = vfio_pci_ioctl_get_region_info,
1648	.device_feature = vfio_pci_core_ioctl_feature,
1649	.read = vfio_pci_core_read,
1650	.write = vfio_pci_core_write,
1651	.mmap = vfio_pci_core_mmap,
1652	.request = vfio_pci_core_request,
1653	.match = vfio_pci_core_match,
1654	.match_token_uuid = vfio_pci_core_match_token_uuid,
1655	.bind_iommufd = vfio_iommufd_physical_bind,
1656	.unbind_iommufd = vfio_iommufd_physical_unbind,
1657	.attach_ioas = vfio_iommufd_physical_attach_ioas,
1658	.detach_ioas = vfio_iommufd_physical_detach_ioas,
1659};
1660
1661static void hisi_acc_vfio_debug_init(struct hisi_acc_vf_core_device *hisi_acc_vdev)
1662{
1663	struct vfio_device *vdev = &hisi_acc_vdev->core_device.vdev;
1664	struct hisi_acc_vf_migration_file *migf;
1665	struct dentry *vfio_dev_migration;
1666	struct dentry *vfio_hisi_acc;
1667	struct device *dev = vdev->dev;
1668
1669	if (!debugfs_initialized() ||
1670	    !IS_ENABLED(CONFIG_VFIO_DEBUGFS))
1671		return;
1672
1673	if (vdev->ops != &hisi_acc_vfio_pci_migrn_ops)
1674		return;
1675
1676	vfio_dev_migration = debugfs_lookup("migration", vdev->debug_root);
1677	if (!vfio_dev_migration) {
1678		dev_err(dev, "failed to lookup migration debugfs file!\n");
1679		return;
1680	}
1681
1682	migf = kzalloc(sizeof(*migf), GFP_KERNEL);
1683	if (!migf) {
1684		dput(vfio_dev_migration);
1685		return;
1686	}
1687	hisi_acc_vdev->debug_migf = migf;
1688
1689	vfio_hisi_acc = debugfs_create_dir("hisi_acc", vfio_dev_migration);
1690	debugfs_create_devm_seqfile(dev, "dev_data", vfio_hisi_acc,
1691				    hisi_acc_vf_dev_read);
1692	debugfs_create_devm_seqfile(dev, "migf_data", vfio_hisi_acc,
1693				    hisi_acc_vf_migf_read);
1694	debugfs_create_devm_seqfile(dev, "cmd_state", vfio_hisi_acc,
1695				    hisi_acc_vf_debug_cmd);
1696
1697	dput(vfio_dev_migration);
1698}
1699
1700static void hisi_acc_vf_debugfs_exit(struct hisi_acc_vf_core_device *hisi_acc_vdev)
1701{
1702	kfree(hisi_acc_vdev->debug_migf);
1703	hisi_acc_vdev->debug_migf = NULL;
1704}
1705
1706static int hisi_acc_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1707{
1708	struct hisi_acc_vf_core_device *hisi_acc_vdev;
1709	const struct vfio_device_ops *ops = &hisi_acc_vfio_pci_ops;
1710	struct hisi_qm *pf_qm;
1711	int vf_id;
1712	int ret;
1713
1714	pf_qm = hisi_acc_get_pf_qm(pdev);
1715	if (pf_qm && pf_qm->ver >= QM_HW_V3) {
1716		vf_id = pci_iov_vf_id(pdev);
1717		if (vf_id >= 0)
1718			ops = &hisi_acc_vfio_pci_migrn_ops;
1719		else
1720			pci_warn(pdev, "migration support failed, continue with generic interface\n");
1721	}
1722
1723	hisi_acc_vdev = vfio_alloc_device(hisi_acc_vf_core_device,
1724					  core_device.vdev, &pdev->dev, ops);
1725	if (IS_ERR(hisi_acc_vdev))
1726		return PTR_ERR(hisi_acc_vdev);
1727
1728	dev_set_drvdata(&pdev->dev, &hisi_acc_vdev->core_device);
1729	ret = vfio_pci_core_register_device(&hisi_acc_vdev->core_device);
1730	if (ret)
1731		goto out_put_vdev;
1732
1733	hisi_acc_vfio_debug_init(hisi_acc_vdev);
1734	return 0;
1735
1736out_put_vdev:
1737	vfio_put_device(&hisi_acc_vdev->core_device.vdev);
1738	return ret;
1739}
1740
1741static void hisi_acc_vfio_pci_remove(struct pci_dev *pdev)
1742{
1743	struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev);
1744
1745	vfio_pci_core_unregister_device(&hisi_acc_vdev->core_device);
1746	hisi_acc_vf_debugfs_exit(hisi_acc_vdev);
1747	vfio_put_device(&hisi_acc_vdev->core_device.vdev);
1748}
1749
1750static const struct pci_device_id hisi_acc_vfio_pci_table[] = {
1751	{ PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_SEC_VF) },
1752	{ PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_HPRE_VF) },
1753	{ PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_ZIP_VF) },
1754	{ }
1755};
1756
1757MODULE_DEVICE_TABLE(pci, hisi_acc_vfio_pci_table);
1758
1759static const struct pci_error_handlers hisi_acc_vf_err_handlers = {
1760	.reset_prepare = hisi_acc_vf_pci_reset_prepare,
1761	.reset_done = hisi_acc_vf_pci_aer_reset_done,
1762	.error_detected = vfio_pci_core_aer_err_detected,
1763};
1764
1765static struct pci_driver hisi_acc_vfio_pci_driver = {
1766	.name = KBUILD_MODNAME,
1767	.id_table = hisi_acc_vfio_pci_table,
1768	.probe = hisi_acc_vfio_pci_probe,
1769	.remove = hisi_acc_vfio_pci_remove,
1770	.err_handler = &hisi_acc_vf_err_handlers,
1771	.driver_managed_dma = true,
1772};
1773
1774module_pci_driver(hisi_acc_vfio_pci_driver);
1775
1776MODULE_LICENSE("GPL v2");
1777MODULE_AUTHOR("Liu Longfang <liulongfang@huawei.com>");
1778MODULE_AUTHOR("Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>");
1779MODULE_DESCRIPTION("HiSilicon VFIO PCI - VFIO PCI driver with live migration support for HiSilicon ACC device family");