FILE
qm.c
File preview with syntax highlighting.
linux-6.19.14/drivers/crypto/hisilicon/qm.c
1// SPDX-License-Identifier: GPL-2.0 2/* Copyright (c) 2019 HiSilicon Limited. */ 3#include <asm/page.h> 4#include <linux/acpi.h> 5#include <linux/bitmap.h> 6#include <linux/dma-mapping.h> 7#include <linux/idr.h> 8#include <linux/io.h> 9#include <linux/irqreturn.h> 10#include <linux/log2.h> 11#include <linux/pm_runtime.h> 12#include <linux/seq_file.h> 13#include <linux/slab.h> 14#include <linux/uacce.h> 15#include <linux/uaccess.h> 16#include <uapi/misc/uacce/hisi_qm.h> 17#include <linux/hisi_acc_qm.h> 18#include "qm_common.h" 19 20/* eq/aeq irq enable */ 21#define QM_VF_AEQ_INT_SOURCE 0x0 22#define QM_VF_AEQ_INT_MASK 0x4 23#define QM_VF_EQ_INT_SOURCE 0x8 24#define QM_VF_EQ_INT_MASK 0xc 25 26#define QM_IRQ_VECTOR_MASK GENMASK(15, 0) 27#define QM_IRQ_TYPE_MASK GENMASK(15, 0) 28#define QM_IRQ_TYPE_SHIFT 16 29#define QM_ABN_IRQ_TYPE_MASK GENMASK(7, 0) 30 31/* mailbox */ 32#define QM_MB_PING_ALL_VFS 0xffff 33#define QM_MB_STATUS_MASK GENMASK(12, 9) 34 35/* sqc shift */ 36#define QM_SQ_HOP_NUM_SHIFT 0 37#define QM_SQ_PAGE_SIZE_SHIFT 4 38#define QM_SQ_BUF_SIZE_SHIFT 8 39#define QM_SQ_SQE_SIZE_SHIFT 12 40#define QM_SQ_PRIORITY_SHIFT 0 41#define QM_SQ_ORDERS_SHIFT 4 42#define QM_SQ_TYPE_SHIFT 8 43#define QM_QC_PASID_ENABLE 0x1 44#define QM_QC_PASID_ENABLE_SHIFT 7 45 46#define QM_SQ_TYPE_MASK GENMASK(3, 0) 47#define QM_SQ_TAIL_IDX(sqc) ((le16_to_cpu((sqc).w11) >> 6) & 0x1) 48#define QM_SQC_DISABLE_QP (1U << 6) 49#define QM_XQC_RANDOM_DATA 0xaaaa 50 51/* cqc shift */ 52#define QM_CQ_HOP_NUM_SHIFT 0 53#define QM_CQ_PAGE_SIZE_SHIFT 4 54#define QM_CQ_BUF_SIZE_SHIFT 8 55#define QM_CQ_CQE_SIZE_SHIFT 12 56#define QM_CQ_PHASE_SHIFT 0 57#define QM_CQ_FLAG_SHIFT 1 58 59#define QM_CQE_PHASE(cqe) (le16_to_cpu((cqe)->w7) & 0x1) 60#define QM_QC_CQE_SIZE 4 61#define QM_CQ_TAIL_IDX(cqc) ((le16_to_cpu((cqc).w11) >> 6) & 0x1) 62 63/* eqc shift */ 64#define QM_EQE_AEQE_SIZE (2UL << 12) 65#define QM_EQC_PHASE_SHIFT 16 66 67#define QM_EQE_PHASE(dw0) (((dw0) >> 16) & 0x1) 68#define QM_EQE_CQN_MASK GENMASK(15, 0) 69 70#define QM_AEQE_PHASE(dw0) (((dw0) >> 16) & 0x1) 71#define QM_AEQE_TYPE_SHIFT 17 72#define QM_AEQE_TYPE_MASK 0xf 73#define QM_AEQE_CQN_MASK GENMASK(15, 0) 74#define QM_CQ_OVERFLOW 0 75#define QM_EQ_OVERFLOW 1 76#define QM_CQE_ERROR 2 77 78#define QM_XQ_DEPTH_SHIFT 16 79#define QM_XQ_DEPTH_MASK GENMASK(15, 0) 80 81#define QM_DOORBELL_CMD_SQ 0 82#define QM_DOORBELL_CMD_CQ 1 83#define QM_DOORBELL_CMD_EQ 2 84#define QM_DOORBELL_CMD_AEQ 3 85 86#define QM_DOORBELL_BASE_V1 0x340 87#define QM_DB_CMD_SHIFT_V1 16 88#define QM_DB_INDEX_SHIFT_V1 32 89#define QM_DB_PRIORITY_SHIFT_V1 48 90#define QM_PAGE_SIZE 0x0034 91#define QM_QP_DB_INTERVAL 0x10000 92#define QM_DB_TIMEOUT_CFG 0x100074 93#define QM_DB_TIMEOUT_SET 0x1fffff 94 95#define QM_MEM_START_INIT 0x100040 96#define QM_MEM_INIT_DONE 0x100044 97#define QM_VFT_CFG_RDY 0x10006c 98#define QM_VFT_CFG_OP_WR 0x100058 99#define QM_VFT_CFG_TYPE 0x10005c 100#define QM_VFT_CFG 0x100060 101#define QM_VFT_CFG_OP_ENABLE 0x100054 102#define QM_PM_CTRL 0x100148 103#define QM_IDLE_DISABLE BIT(9) 104 105#define QM_SUB_VERSION_ID 0x210 106 107#define QM_VFT_CFG_DATA_L 0x100064 108#define QM_VFT_CFG_DATA_H 0x100068 109#define QM_SQC_VFT_BUF_SIZE (7ULL << 8) 110#define QM_SQC_VFT_SQC_SIZE (5ULL << 12) 111#define QM_SQC_VFT_INDEX_NUMBER (1ULL << 16) 112#define QM_SQC_VFT_START_SQN_SHIFT 28 113#define QM_SQC_VFT_VALID (1ULL << 44) 114#define QM_SQC_VFT_SQN_SHIFT 45 115#define QM_CQC_VFT_BUF_SIZE (7ULL << 8) 116#define QM_CQC_VFT_SQC_SIZE (5ULL << 12) 117#define QM_CQC_VFT_INDEX_NUMBER (1ULL << 16) 118#define QM_CQC_VFT_VALID (1ULL << 28) 119 120#define QM_SQC_VFT_BASE_SHIFT_V2 28 121#define QM_SQC_VFT_BASE_MASK_V2 GENMASK(15, 0) 122#define QM_SQC_VFT_NUM_SHIFT_V2 45 123#define QM_SQC_VFT_NUM_MASK_V2 GENMASK(9, 0) 124#define QM_MAX_QC_TYPE 2 125 126#define QM_ABNORMAL_INT_SOURCE 0x100000 127#define QM_ABNORMAL_INT_MASK 0x100004 128#define QM_ABNORMAL_INT_MASK_VALUE 0x7fff 129#define QM_ABNORMAL_INT_STATUS 0x100008 130#define QM_ABNORMAL_INT_SET 0x10000c 131#define QM_ABNORMAL_INF00 0x100010 132#define QM_FIFO_OVERFLOW_TYPE 0xc0 133#define QM_FIFO_OVERFLOW_TYPE_SHIFT 6 134#define QM_FIFO_OVERFLOW_VF 0x3f 135#define QM_FIFO_OVERFLOW_QP_SHIFT 16 136#define QM_ABNORMAL_INF01 0x100014 137#define QM_DB_TIMEOUT_TYPE 0xc0 138#define QM_DB_TIMEOUT_TYPE_SHIFT 6 139#define QM_DB_TIMEOUT_VF 0x3f 140#define QM_DB_TIMEOUT_QP_SHIFT 16 141#define QM_ABNORMAL_INF02 0x100018 142#define QM_AXI_POISON_ERR BIT(22) 143#define QM_RAS_CE_ENABLE 0x1000ec 144#define QM_RAS_FE_ENABLE 0x1000f0 145#define QM_RAS_NFE_ENABLE 0x1000f4 146#define QM_RAS_CE_THRESHOLD 0x1000f8 147#define QM_RAS_CE_TIMES_PER_IRQ 1 148#define QM_OOO_SHUTDOWN_SEL 0x1040f8 149#define QM_AXI_RRESP_ERR BIT(0) 150#define QM_DB_TIMEOUT BIT(10) 151#define QM_OF_FIFO_OF BIT(11) 152#define QM_RAS_AXI_ERROR (BIT(0) | BIT(1) | BIT(12)) 153 154#define QM_RESET_WAIT_TIMEOUT 400 155#define QM_PEH_VENDOR_ID 0x1000d8 156#define ACC_VENDOR_ID_VALUE 0x5a5a 157#define QM_PEH_DFX_INFO0 0x1000fc 158#define QM_PEH_DFX_INFO1 0x100100 159#define QM_PEH_DFX_MASK (BIT(0) | BIT(2)) 160#define QM_PEH_MSI_FINISH_MASK GENMASK(19, 16) 161#define ACC_PEH_SRIOV_CTRL_VF_MSE_SHIFT 3 162#define ACC_PEH_MSI_DISABLE GENMASK(31, 0) 163#define ACC_MASTER_GLOBAL_CTRL_SHUTDOWN 0x1 164#define ACC_MASTER_TRANS_RETURN_RW 3 165#define ACC_MASTER_TRANS_RETURN 0x300150 166#define ACC_MASTER_GLOBAL_CTRL 0x300000 167#define ACC_AM_CFG_PORT_WR_EN 0x30001c 168#define ACC_AM_ROB_ECC_INT_STS 0x300104 169#define ACC_ROB_ECC_ERR_MULTPL BIT(1) 170#define QM_MSI_CAP_ENABLE BIT(16) 171 172/* interfunction communication */ 173#define QM_IFC_READY_STATUS 0x100128 174#define QM_IFC_INT_SET_P 0x100130 175#define QM_IFC_INT_CFG 0x100134 176#define QM_IFC_INT_SOURCE_P 0x100138 177#define QM_IFC_INT_SOURCE_V 0x0020 178#define QM_IFC_INT_MASK 0x0024 179#define QM_IFC_INT_STATUS 0x0028 180#define QM_IFC_INT_SET_V 0x002C 181#define QM_PF2VF_PF_W 0x104700 182#define QM_VF2PF_PF_R 0x104800 183#define QM_VF2PF_VF_W 0x320 184#define QM_PF2VF_VF_R 0x380 185#define QM_IFC_SEND_ALL_VFS GENMASK(6, 0) 186#define QM_IFC_INT_SOURCE_CLR GENMASK(63, 0) 187#define QM_IFC_INT_SOURCE_MASK BIT(0) 188#define QM_IFC_INT_DISABLE BIT(0) 189#define QM_IFC_INT_STATUS_MASK BIT(0) 190#define QM_IFC_INT_SET_MASK BIT(0) 191#define QM_WAIT_DST_ACK 10 192#define QM_MAX_PF_WAIT_COUNT 10 193#define QM_MAX_VF_WAIT_COUNT 40 194#define QM_VF_RESET_WAIT_US 20000 195#define QM_VF_RESET_WAIT_CNT 3000 196#define QM_VF2PF_REG_SIZE 4 197#define QM_IFC_CMD_MASK GENMASK(31, 0) 198#define QM_IFC_DATA_SHIFT 32 199#define QM_VF_RESET_WAIT_TIMEOUT_US \ 200 (QM_VF_RESET_WAIT_US * QM_VF_RESET_WAIT_CNT) 201 202#define POLL_PERIOD 10 203#define POLL_TIMEOUT 1000 204#define WAIT_PERIOD_US_MAX 200 205#define WAIT_PERIOD_US_MIN 100 206#define MAX_WAIT_COUNTS 1000 207#define QM_CACHE_WB_START 0x204 208#define QM_CACHE_WB_DONE 0x208 209#define QM_FUNC_CAPS_REG 0x3100 210#define QM_CAPBILITY_VERSION GENMASK(7, 0) 211 212#define PCI_BAR_2 2 213#define PCI_BAR_4 4 214#define QMC_ALIGN(sz) ALIGN(sz, 32) 215 216#define QM_DBG_READ_LEN 256 217#define QM_PCI_COMMAND_INVALID ~0 218#define QM_RESET_STOP_TX_OFFSET 1 219#define QM_RESET_STOP_RX_OFFSET 2 220 221#define WAIT_PERIOD 20 222#define REMOVE_WAIT_DELAY 10 223 224#define QM_QOS_PARAM_NUM 2 225#define QM_QOS_MAX_VAL 1000 226#define QM_QOS_RATE 100 227#define QM_QOS_EXPAND_RATE 1000 228#define QM_SHAPER_CIR_B_MASK GENMASK(7, 0) 229#define QM_SHAPER_CIR_U_MASK GENMASK(10, 8) 230#define QM_SHAPER_CIR_S_MASK GENMASK(14, 11) 231#define QM_SHAPER_FACTOR_CIR_U_SHIFT 8 232#define QM_SHAPER_FACTOR_CIR_S_SHIFT 11 233#define QM_SHAPER_FACTOR_CBS_B_SHIFT 15 234#define QM_SHAPER_FACTOR_CBS_S_SHIFT 19 235#define QM_SHAPER_CBS_B 1 236#define QM_SHAPER_VFT_OFFSET 6 237#define QM_QOS_MIN_ERROR_RATE 5 238#define QM_SHAPER_MIN_CBS_S 8 239#define QM_QOS_TICK 0x300U 240#define QM_QOS_DIVISOR_CLK 0x1f40U 241#define QM_QOS_MAX_CIR_B 200 242#define QM_QOS_MIN_CIR_B 100 243#define QM_QOS_MAX_CIR_U 6 244#define QM_AUTOSUSPEND_DELAY 3000 245 246 /* abnormal status value for stopping queue */ 247#define QM_STOP_QUEUE_FAIL 1 248#define QM_DUMP_SQC_FAIL 3 249#define QM_DUMP_CQC_FAIL 4 250#define QM_FINISH_WAIT 5 251 252#define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \ 253 (((hop_num) << QM_CQ_HOP_NUM_SHIFT) | \ 254 ((pg_sz) << QM_CQ_PAGE_SIZE_SHIFT) | \ 255 ((buf_sz) << QM_CQ_BUF_SIZE_SHIFT) | \ 256 ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT)) 257 258#define QM_MK_CQC_DW3_V2(cqe_sz, cq_depth) \ 259 ((((u32)cq_depth) - 1) | ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT)) 260 261#define QM_MK_SQC_W13(priority, orders, alg_type) \ 262 (((priority) << QM_SQ_PRIORITY_SHIFT) | \ 263 ((orders) << QM_SQ_ORDERS_SHIFT) | \ 264 (((alg_type) & QM_SQ_TYPE_MASK) << QM_SQ_TYPE_SHIFT)) 265 266#define QM_MK_SQC_DW3_V1(hop_num, pg_sz, buf_sz, sqe_sz) \ 267 (((hop_num) << QM_SQ_HOP_NUM_SHIFT) | \ 268 ((pg_sz) << QM_SQ_PAGE_SIZE_SHIFT) | \ 269 ((buf_sz) << QM_SQ_BUF_SIZE_SHIFT) | \ 270 ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT)) 271 272#define QM_MK_SQC_DW3_V2(sqe_sz, sq_depth) \ 273 ((((u32)sq_depth) - 1) | ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT)) 274 275enum vft_type { 276 SQC_VFT = 0, 277 CQC_VFT, 278 SHAPER_VFT, 279}; 280 281enum qm_alg_type { 282 ALG_TYPE_0, 283 ALG_TYPE_1, 284}; 285 286enum qm_ifc_cmd { 287 QM_PF_FLR_PREPARE = 0x01, 288 QM_PF_SRST_PREPARE, 289 QM_PF_RESET_DONE, 290 QM_VF_PREPARE_DONE, 291 QM_VF_PREPARE_FAIL, 292 QM_VF_START_DONE, 293 QM_VF_START_FAIL, 294 QM_PF_SET_QOS, 295 QM_VF_GET_QOS, 296}; 297 298enum qm_basic_type { 299 QM_TOTAL_QP_NUM_CAP = 0x0, 300 QM_FUNC_MAX_QP_CAP, 301 QM_XEQ_DEPTH_CAP, 302 QM_QP_DEPTH_CAP, 303 QM_EQ_IRQ_TYPE_CAP, 304 QM_AEQ_IRQ_TYPE_CAP, 305 QM_ABN_IRQ_TYPE_CAP, 306 QM_PF2VF_IRQ_TYPE_CAP, 307 QM_PF_IRQ_NUM_CAP, 308 QM_VF_IRQ_NUM_CAP, 309}; 310 311enum qm_cap_table_type { 312 QM_CAP_VF = 0x0, 313 QM_AEQE_NUM, 314 QM_SCQE_NUM, 315 QM_EQ_IRQ, 316 QM_AEQ_IRQ, 317 QM_ABNORMAL_IRQ, 318 QM_MB_IRQ, 319 MAX_IRQ_NUM, 320 EXT_BAR_INDEX, 321}; 322 323static const struct hisi_qm_cap_query_info qm_cap_query_info[] = { 324 {QM_CAP_VF, "QM_CAP_VF ", 0x3100, 0x0, 0x0, 0x6F01}, 325 {QM_AEQE_NUM, "QM_AEQE_NUM ", 0x3104, 0x800, 0x4000800, 0x4000800}, 326 {QM_SCQE_NUM, "QM_SCQE_NUM ", 327 0x3108, 0x4000400, 0x4000400, 0x4000400}, 328 {QM_EQ_IRQ, "QM_EQ_IRQ ", 0x310c, 0x10000, 0x10000, 0x10000}, 329 {QM_AEQ_IRQ, "QM_AEQ_IRQ ", 0x3110, 0x0, 0x10001, 0x10001}, 330 {QM_ABNORMAL_IRQ, "QM_ABNORMAL_IRQ ", 0x3114, 0x0, 0x10003, 0x10003}, 331 {QM_MB_IRQ, "QM_MB_IRQ ", 0x3118, 0x0, 0x0, 0x10002}, 332 {MAX_IRQ_NUM, "MAX_IRQ_NUM ", 0x311c, 0x10001, 0x40002, 0x40003}, 333 {EXT_BAR_INDEX, "EXT_BAR_INDEX ", 0x3120, 0x0, 0x0, 0x14}, 334}; 335 336static const struct hisi_qm_cap_info qm_cap_info_comm[] = { 337 {QM_SUPPORT_DB_ISOLATION, 0x30, 0, BIT(0), 0x0, 0x0, 0x0}, 338 {QM_SUPPORT_FUNC_QOS, 0x3100, 0, BIT(8), 0x0, 0x0, 0x1}, 339 {QM_SUPPORT_STOP_QP, 0x3100, 0, BIT(9), 0x0, 0x0, 0x1}, 340 {QM_SUPPORT_STOP_FUNC, 0x3100, 0, BIT(10), 0x0, 0x0, 0x1}, 341 {QM_SUPPORT_MB_COMMAND, 0x3100, 0, BIT(11), 0x0, 0x0, 0x1}, 342 {QM_SUPPORT_SVA_PREFETCH, 0x3100, 0, BIT(14), 0x0, 0x0, 0x1}, 343 {QM_SUPPORT_DAE, 0x3100, 0, BIT(15), 0x0, 0x0, 0x0}, 344}; 345 346static const struct hisi_qm_cap_info qm_cap_info_pf[] = { 347 {QM_SUPPORT_RPM, 0x3100, 0, BIT(13), 0x0, 0x0, 0x1}, 348}; 349 350static const struct hisi_qm_cap_info qm_cap_info_vf[] = { 351 {QM_SUPPORT_RPM, 0x3100, 0, BIT(12), 0x0, 0x0, 0x0}, 352}; 353 354static const struct hisi_qm_cap_info qm_basic_info[] = { 355 {QM_TOTAL_QP_NUM_CAP, 0x100158, 0, GENMASK(10, 0), 0x1000, 0x400, 0x400}, 356 {QM_FUNC_MAX_QP_CAP, 0x100158, 11, GENMASK(10, 0), 0x1000, 0x400, 0x400}, 357 {QM_XEQ_DEPTH_CAP, 0x3104, 0, GENMASK(31, 0), 0x800, 0x4000800, 0x4000800}, 358 {QM_QP_DEPTH_CAP, 0x3108, 0, GENMASK(31, 0), 0x4000400, 0x4000400, 0x4000400}, 359 {QM_EQ_IRQ_TYPE_CAP, 0x310c, 0, GENMASK(31, 0), 0x10000, 0x10000, 0x10000}, 360 {QM_AEQ_IRQ_TYPE_CAP, 0x3110, 0, GENMASK(31, 0), 0x0, 0x10001, 0x10001}, 361 {QM_ABN_IRQ_TYPE_CAP, 0x3114, 0, GENMASK(31, 0), 0x0, 0x10003, 0x10003}, 362 {QM_PF2VF_IRQ_TYPE_CAP, 0x3118, 0, GENMASK(31, 0), 0x0, 0x0, 0x10002}, 363 {QM_PF_IRQ_NUM_CAP, 0x311c, 16, GENMASK(15, 0), 0x1, 0x4, 0x4}, 364 {QM_VF_IRQ_NUM_CAP, 0x311c, 0, GENMASK(15, 0), 0x1, 0x2, 0x3}, 365}; 366 367struct qm_mailbox { 368 __le16 w0; 369 __le16 queue_num; 370 __le32 base_l; 371 __le32 base_h; 372 __le32 rsvd; 373}; 374 375struct qm_doorbell { 376 __le16 queue_num; 377 __le16 cmd; 378 __le16 index; 379 __le16 priority; 380}; 381 382struct hisi_qm_resource { 383 struct hisi_qm *qm; 384 int distance; 385 struct list_head list; 386}; 387 388/** 389 * struct qm_hw_err - Structure describing the device errors 390 * @list: hardware error list 391 * @timestamp: timestamp when the error occurred 392 */ 393struct qm_hw_err { 394 struct list_head list; 395 unsigned long long timestamp; 396}; 397 398struct hisi_qm_hw_ops { 399 int (*get_vft)(struct hisi_qm *qm, u32 *base, u32 *number); 400 void (*qm_db)(struct hisi_qm *qm, u16 qn, 401 u8 cmd, u16 index, u8 priority); 402 int (*debug_init)(struct hisi_qm *qm); 403 void (*hw_error_init)(struct hisi_qm *qm); 404 void (*hw_error_uninit)(struct hisi_qm *qm); 405 enum acc_err_result (*hw_error_handle)(struct hisi_qm *qm); 406 int (*set_msi)(struct hisi_qm *qm, bool set); 407 408 /* (u64)msg = (u32)data << 32 | (enum qm_ifc_cmd)cmd */ 409 int (*set_ifc_begin)(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num); 410 void (*set_ifc_end)(struct hisi_qm *qm); 411 int (*get_ifc)(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num); 412}; 413 414struct hisi_qm_hw_error { 415 u32 int_msk; 416 const char *msg; 417}; 418 419static const struct hisi_qm_hw_error qm_hw_error[] = { 420 { .int_msk = BIT(0), .msg = "qm_axi_rresp" }, 421 { .int_msk = BIT(1), .msg = "qm_axi_bresp" }, 422 { .int_msk = BIT(2), .msg = "qm_ecc_mbit" }, 423 { .int_msk = BIT(3), .msg = "qm_ecc_1bit" }, 424 { .int_msk = BIT(4), .msg = "qm_acc_get_task_timeout" }, 425 { .int_msk = BIT(5), .msg = "qm_acc_do_task_timeout" }, 426 { .int_msk = BIT(6), .msg = "qm_acc_wb_not_ready_timeout" }, 427 { .int_msk = BIT(7), .msg = "qm_sq_cq_vf_invalid" }, 428 { .int_msk = BIT(8), .msg = "qm_cq_vf_invalid" }, 429 { .int_msk = BIT(9), .msg = "qm_sq_vf_invalid" }, 430 { .int_msk = BIT(10), .msg = "qm_db_timeout" }, 431 { .int_msk = BIT(11), .msg = "qm_of_fifo_of" }, 432 { .int_msk = BIT(12), .msg = "qm_db_random_invalid" }, 433 { .int_msk = BIT(13), .msg = "qm_mailbox_timeout" }, 434 { .int_msk = BIT(14), .msg = "qm_flr_timeout" }, 435}; 436 437static const char * const qm_db_timeout[] = { 438 "sq", "cq", "eq", "aeq", 439}; 440 441static const char * const qm_fifo_overflow[] = { 442 "cq", "eq", "aeq", 443}; 444 445struct qm_typical_qos_table { 446 u32 start; 447 u32 end; 448 u32 val; 449}; 450 451/* the qos step is 100 */ 452static struct qm_typical_qos_table shaper_cir_s[] = { 453 {100, 100, 4}, 454 {200, 200, 3}, 455 {300, 500, 2}, 456 {600, 1000, 1}, 457 {1100, 100000, 0}, 458}; 459 460static struct qm_typical_qos_table shaper_cbs_s[] = { 461 {100, 200, 9}, 462 {300, 500, 11}, 463 {600, 1000, 12}, 464 {1100, 10000, 16}, 465 {10100, 25000, 17}, 466 {25100, 50000, 18}, 467 {50100, 100000, 19} 468}; 469 470static void qm_irqs_unregister(struct hisi_qm *qm); 471static int qm_reset_device(struct hisi_qm *qm); 472int hisi_qm_q_num_set(const char *val, const struct kernel_param *kp, 473 unsigned int device) 474{ 475 struct pci_dev *pdev; 476 u32 n, q_num; 477 int ret; 478 479 if (!val) 480 return -EINVAL; 481 482 pdev = pci_get_device(PCI_VENDOR_ID_HUAWEI, device, NULL); 483 if (!pdev) { 484 q_num = min_t(u32, QM_QNUM_V1, QM_QNUM_V2); 485 pr_info("No device found currently, suppose queue number is %u\n", 486 q_num); 487 } else { 488 if (pdev->revision == QM_HW_V1) 489 q_num = QM_QNUM_V1; 490 else 491 q_num = QM_QNUM_V2; 492 493 pci_dev_put(pdev); 494 } 495 496 ret = kstrtou32(val, 10, &n); 497 if (ret || n < QM_MIN_QNUM || n > q_num) 498 return -EINVAL; 499 500 return param_set_int(val, kp); 501} 502EXPORT_SYMBOL_GPL(hisi_qm_q_num_set); 503 504static u32 qm_get_hw_error_status(struct hisi_qm *qm) 505{ 506 return readl(qm->io_base + QM_ABNORMAL_INT_STATUS); 507} 508 509static u32 qm_get_dev_err_status(struct hisi_qm *qm) 510{ 511 return qm->err_ini->get_dev_hw_err_status(qm); 512} 513 514/* Check if the error causes the master ooo block */ 515static bool qm_check_dev_error(struct hisi_qm *qm) 516{ 517 struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); 518 u32 err_status; 519 520 if (pf_qm->fun_type == QM_HW_VF) 521 return false; 522 523 err_status = qm_get_hw_error_status(pf_qm); 524 if (err_status & pf_qm->err_info.qm_err.shutdown_mask) 525 return true; 526 527 if (pf_qm->err_ini->dev_is_abnormal) 528 return pf_qm->err_ini->dev_is_abnormal(pf_qm); 529 530 return false; 531} 532 533static int qm_wait_reset_finish(struct hisi_qm *qm) 534{ 535 int delay = 0; 536 537 /* All reset requests need to be queued for processing */ 538 while (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) { 539 msleep(++delay); 540 if (delay > QM_RESET_WAIT_TIMEOUT) 541 return -EBUSY; 542 } 543 544 return 0; 545} 546 547static int qm_reset_prepare_ready(struct hisi_qm *qm) 548{ 549 struct pci_dev *pdev = qm->pdev; 550 struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev)); 551 552 /* 553 * PF and VF on host doesnot support resetting at the 554 * same time on Kunpeng920. 555 */ 556 if (qm->ver < QM_HW_V3) 557 return qm_wait_reset_finish(pf_qm); 558 559 return qm_wait_reset_finish(qm); 560} 561 562static void qm_reset_bit_clear(struct hisi_qm *qm) 563{ 564 struct pci_dev *pdev = qm->pdev; 565 struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev)); 566 567 if (qm->ver < QM_HW_V3) 568 clear_bit(QM_RESETTING, &pf_qm->misc_ctl); 569 570 clear_bit(QM_RESETTING, &qm->misc_ctl); 571} 572 573static void qm_mb_pre_init(struct qm_mailbox *mailbox, u8 cmd, 574 u64 base, u16 queue, bool op) 575{ 576 mailbox->w0 = cpu_to_le16((cmd) | 577 ((op) ? 0x1 << QM_MB_OP_SHIFT : 0) | 578 (0x1 << QM_MB_BUSY_SHIFT)); 579 mailbox->queue_num = cpu_to_le16(queue); 580 mailbox->base_l = cpu_to_le32(lower_32_bits(base)); 581 mailbox->base_h = cpu_to_le32(upper_32_bits(base)); 582 mailbox->rsvd = 0; 583} 584 585/* return 0 mailbox ready, -ETIMEDOUT hardware timeout */ 586int hisi_qm_wait_mb_ready(struct hisi_qm *qm) 587{ 588 u32 val; 589 590 return readl_relaxed_poll_timeout(qm->io_base + QM_MB_CMD_SEND_BASE, 591 val, !((val >> QM_MB_BUSY_SHIFT) & 592 0x1), POLL_PERIOD, POLL_TIMEOUT); 593} 594EXPORT_SYMBOL_GPL(hisi_qm_wait_mb_ready); 595 596/* 128 bit should be written to hardware at one time to trigger a mailbox */ 597static void qm_mb_write(struct hisi_qm *qm, const void *src) 598{ 599 void __iomem *fun_base = qm->io_base + QM_MB_CMD_SEND_BASE; 600 601#if IS_ENABLED(CONFIG_ARM64) 602 unsigned long tmp0 = 0, tmp1 = 0; 603#endif 604 605 if (!IS_ENABLED(CONFIG_ARM64)) { 606 memcpy_toio(fun_base, src, 16); 607 dma_wmb(); 608 return; 609 } 610 611#if IS_ENABLED(CONFIG_ARM64) 612 /* 613 * The dmb oshst instruction ensures that the data in the 614 * mailbox is written before it is sent to the hardware. 615 */ 616 asm volatile("ldp %0, %1, %3\n" 617 "dmb oshst\n" 618 "stp %0, %1, %2\n" 619 : "=&r" (tmp0), 620 "=&r" (tmp1), 621 "+Q" (*((char __iomem *)fun_base)) 622 : "Q" (*((char *)src)) 623 : "memory"); 624#endif 625} 626 627static int qm_mb_nolock(struct hisi_qm *qm, struct qm_mailbox *mailbox) 628{ 629 int ret; 630 u32 val; 631 632 if (unlikely(hisi_qm_wait_mb_ready(qm))) { 633 dev_err(&qm->pdev->dev, "QM mailbox is busy to start!\n"); 634 ret = -EBUSY; 635 goto mb_busy; 636 } 637 638 qm_mb_write(qm, mailbox); 639 640 if (unlikely(hisi_qm_wait_mb_ready(qm))) { 641 dev_err(&qm->pdev->dev, "QM mailbox operation timeout!\n"); 642 ret = -ETIMEDOUT; 643 goto mb_busy; 644 } 645 646 val = readl(qm->io_base + QM_MB_CMD_SEND_BASE); 647 if (val & QM_MB_STATUS_MASK) { 648 dev_err(&qm->pdev->dev, "QM mailbox operation failed!\n"); 649 ret = -EIO; 650 goto mb_busy; 651 } 652 653 return 0; 654 655mb_busy: 656 atomic64_inc(&qm->debug.dfx.mb_err_cnt); 657 return ret; 658} 659 660int hisi_qm_mb(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, u16 queue, 661 bool op) 662{ 663 struct qm_mailbox mailbox; 664 int ret; 665 666 qm_mb_pre_init(&mailbox, cmd, dma_addr, queue, op); 667 668 mutex_lock(&qm->mailbox_lock); 669 ret = qm_mb_nolock(qm, &mailbox); 670 mutex_unlock(&qm->mailbox_lock); 671 672 return ret; 673} 674EXPORT_SYMBOL_GPL(hisi_qm_mb); 675 676/* op 0: set xqc information to hardware, 1: get xqc information from hardware. */ 677int qm_set_and_get_xqc(struct hisi_qm *qm, u8 cmd, void *xqc, u32 qp_id, bool op) 678{ 679 struct qm_mailbox mailbox; 680 dma_addr_t xqc_dma; 681 void *tmp_xqc; 682 size_t size; 683 int ret; 684 685 switch (cmd) { 686 case QM_MB_CMD_SQC: 687 size = sizeof(struct qm_sqc); 688 tmp_xqc = qm->xqc_buf.sqc; 689 xqc_dma = qm->xqc_buf.sqc_dma; 690 break; 691 case QM_MB_CMD_CQC: 692 size = sizeof(struct qm_cqc); 693 tmp_xqc = qm->xqc_buf.cqc; 694 xqc_dma = qm->xqc_buf.cqc_dma; 695 break; 696 case QM_MB_CMD_EQC: 697 size = sizeof(struct qm_eqc); 698 tmp_xqc = qm->xqc_buf.eqc; 699 xqc_dma = qm->xqc_buf.eqc_dma; 700 break; 701 case QM_MB_CMD_AEQC: 702 size = sizeof(struct qm_aeqc); 703 tmp_xqc = qm->xqc_buf.aeqc; 704 xqc_dma = qm->xqc_buf.aeqc_dma; 705 break; 706 default: 707 dev_err(&qm->pdev->dev, "unknown mailbox cmd %u\n", cmd); 708 return -EINVAL; 709 } 710 711 /* Setting xqc will fail if master OOO is blocked. */ 712 if (qm_check_dev_error(qm)) { 713 dev_err(&qm->pdev->dev, "failed to send mailbox since qm is stop!\n"); 714 return -EIO; 715 } 716 717 mutex_lock(&qm->mailbox_lock); 718 if (!op) 719 memcpy(tmp_xqc, xqc, size); 720 721 qm_mb_pre_init(&mailbox, cmd, xqc_dma, qp_id, op); 722 ret = qm_mb_nolock(qm, &mailbox); 723 if (!ret && op) 724 memcpy(xqc, tmp_xqc, size); 725 726 mutex_unlock(&qm->mailbox_lock); 727 728 return ret; 729} 730 731static void qm_db_v1(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority) 732{ 733 u64 doorbell; 734 735 doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V1) | 736 ((u64)index << QM_DB_INDEX_SHIFT_V1) | 737 ((u64)priority << QM_DB_PRIORITY_SHIFT_V1); 738 739 writeq(doorbell, qm->io_base + QM_DOORBELL_BASE_V1); 740} 741 742static void qm_db_v2(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority) 743{ 744 void __iomem *io_base = qm->io_base; 745 u16 randata = 0; 746 u64 doorbell; 747 748 if (cmd == QM_DOORBELL_CMD_SQ || cmd == QM_DOORBELL_CMD_CQ) 749 io_base = qm->db_io_base + (u64)qn * qm->db_interval + 750 QM_DOORBELL_SQ_CQ_BASE_V2; 751 else 752 io_base += QM_DOORBELL_EQ_AEQ_BASE_V2; 753 754 doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V2) | 755 ((u64)randata << QM_DB_RAND_SHIFT_V2) | 756 ((u64)index << QM_DB_INDEX_SHIFT_V2) | 757 ((u64)priority << QM_DB_PRIORITY_SHIFT_V2); 758 759 writeq(doorbell, io_base); 760} 761 762static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority) 763{ 764 dev_dbg(&qm->pdev->dev, "QM doorbell request: qn=%u, cmd=%u, index=%u\n", 765 qn, cmd, index); 766 767 qm->ops->qm_db(qm, qn, cmd, index, priority); 768} 769 770static void qm_disable_clock_gate(struct hisi_qm *qm) 771{ 772 u32 val; 773 774 /* if qm enables clock gating in Kunpeng930, qos will be inaccurate. */ 775 if (qm->ver < QM_HW_V3) 776 return; 777 778 val = readl(qm->io_base + QM_PM_CTRL); 779 val |= QM_IDLE_DISABLE; 780 writel(val, qm->io_base + QM_PM_CTRL); 781} 782 783static int qm_dev_mem_reset(struct hisi_qm *qm) 784{ 785 u32 val; 786 787 writel(0x1, qm->io_base + QM_MEM_START_INIT); 788 return readl_relaxed_poll_timeout(qm->io_base + QM_MEM_INIT_DONE, val, 789 val & BIT(0), POLL_PERIOD, 790 POLL_TIMEOUT); 791} 792 793/** 794 * hisi_qm_get_hw_info() - Get device information. 795 * @qm: The qm which want to get information. 796 * @info_table: Array for storing device information. 797 * @index: Index in info_table. 798 * @is_read: Whether read from reg, 0: not support read from reg. 799 * 800 * This function returns device information the caller needs. 801 */ 802u32 hisi_qm_get_hw_info(struct hisi_qm *qm, 803 const struct hisi_qm_cap_info *info_table, 804 u32 index, bool is_read) 805{ 806 u32 val; 807 808 switch (qm->ver) { 809 case QM_HW_V1: 810 return info_table[index].v1_val; 811 case QM_HW_V2: 812 return info_table[index].v2_val; 813 default: 814 if (!is_read) 815 return info_table[index].v3_val; 816 817 val = readl(qm->io_base + info_table[index].offset); 818 return (val >> info_table[index].shift) & info_table[index].mask; 819 } 820} 821EXPORT_SYMBOL_GPL(hisi_qm_get_hw_info); 822 823u32 hisi_qm_get_cap_value(struct hisi_qm *qm, 824 const struct hisi_qm_cap_query_info *info_table, 825 u32 index, bool is_read) 826{ 827 u32 val; 828 829 switch (qm->ver) { 830 case QM_HW_V1: 831 return info_table[index].v1_val; 832 case QM_HW_V2: 833 return info_table[index].v2_val; 834 default: 835 if (!is_read) 836 return info_table[index].v3_val; 837 838 val = readl(qm->io_base + info_table[index].offset); 839 return val; 840 } 841} 842EXPORT_SYMBOL_GPL(hisi_qm_get_cap_value); 843 844static void qm_get_xqc_depth(struct hisi_qm *qm, u16 *low_bits, 845 u16 *high_bits, enum qm_basic_type type) 846{ 847 u32 depth; 848 849 depth = hisi_qm_get_hw_info(qm, qm_basic_info, type, qm->cap_ver); 850 *low_bits = depth & QM_XQ_DEPTH_MASK; 851 *high_bits = (depth >> QM_XQ_DEPTH_SHIFT) & QM_XQ_DEPTH_MASK; 852} 853 854int hisi_qm_set_algs(struct hisi_qm *qm, u64 alg_msk, const struct qm_dev_alg *dev_algs, 855 u32 dev_algs_size) 856{ 857 struct device *dev = &qm->pdev->dev; 858 char *algs, *ptr; 859 int i; 860 861 if (!qm->uacce) 862 return 0; 863 864 if (dev_algs_size >= QM_DEV_ALG_MAX_LEN) { 865 dev_err(dev, "algs size %u is equal or larger than %d.\n", 866 dev_algs_size, QM_DEV_ALG_MAX_LEN); 867 return -EINVAL; 868 } 869 870 algs = devm_kzalloc(dev, QM_DEV_ALG_MAX_LEN, GFP_KERNEL); 871 if (!algs) 872 return -ENOMEM; 873 874 for (i = 0; i < dev_algs_size; i++) 875 if (alg_msk & dev_algs[i].alg_msk) 876 strcat(algs, dev_algs[i].alg); 877 878 ptr = strrchr(algs, '\n'); 879 if (ptr) 880 *ptr = '\0'; 881 882 qm->uacce->algs = algs; 883 884 return 0; 885} 886EXPORT_SYMBOL_GPL(hisi_qm_set_algs); 887 888static u32 qm_get_irq_num(struct hisi_qm *qm) 889{ 890 if (qm->fun_type == QM_HW_PF) 891 return hisi_qm_get_hw_info(qm, qm_basic_info, QM_PF_IRQ_NUM_CAP, qm->cap_ver); 892 893 return hisi_qm_get_hw_info(qm, qm_basic_info, QM_VF_IRQ_NUM_CAP, qm->cap_ver); 894} 895 896static int qm_pm_get_sync(struct hisi_qm *qm) 897{ 898 struct device *dev = &qm->pdev->dev; 899 int ret; 900 901 if (!test_bit(QM_SUPPORT_RPM, &qm->caps)) 902 return 0; 903 904 ret = pm_runtime_resume_and_get(dev); 905 if (ret < 0) { 906 dev_err(dev, "failed to get_sync(%d).\n", ret); 907 return ret; 908 } 909 910 return 0; 911} 912 913static void qm_pm_put_sync(struct hisi_qm *qm) 914{ 915 struct device *dev = &qm->pdev->dev; 916 917 if (!test_bit(QM_SUPPORT_RPM, &qm->caps)) 918 return; 919 920 pm_runtime_put_autosuspend(dev); 921} 922 923static void qm_cq_head_update(struct hisi_qp *qp) 924{ 925 if (qp->qp_status.cq_head == qp->cq_depth - 1) { 926 qp->qp_status.cqc_phase = !qp->qp_status.cqc_phase; 927 qp->qp_status.cq_head = 0; 928 } else { 929 qp->qp_status.cq_head++; 930 } 931} 932 933static void qm_poll_req_cb(struct hisi_qp *qp) 934{ 935 struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head; 936 struct hisi_qm *qm = qp->qm; 937 938 while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) { 939 dma_rmb(); 940 qp->req_cb(qp, qp->sqe + qm->sqe_size * 941 le16_to_cpu(cqe->sq_head)); 942 qm_cq_head_update(qp); 943 cqe = qp->cqe + qp->qp_status.cq_head; 944 qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ, 945 qp->qp_status.cq_head, 0); 946 atomic_dec(&qp->qp_status.used); 947 948 cond_resched(); 949 } 950 951 /* set c_flag */ 952 qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ, qp->qp_status.cq_head, 1); 953} 954 955static void qm_work_process(struct work_struct *work) 956{ 957 struct hisi_qm_poll_data *poll_data = 958 container_of(work, struct hisi_qm_poll_data, work); 959 struct hisi_qm *qm = poll_data->qm; 960 u16 eqe_num = poll_data->eqe_num; 961 struct hisi_qp *qp; 962 int i; 963 964 for (i = eqe_num - 1; i >= 0; i--) { 965 qp = &qm->qp_array[poll_data->qp_finish_id[i]]; 966 if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP)) 967 continue; 968 969 if (qp->event_cb) { 970 qp->event_cb(qp); 971 continue; 972 } 973 974 if (likely(qp->req_cb)) 975 qm_poll_req_cb(qp); 976 } 977} 978 979static void qm_get_complete_eqe_num(struct hisi_qm *qm) 980{ 981 struct qm_eqe *eqe = qm->eqe + qm->status.eq_head; 982 struct hisi_qm_poll_data *poll_data = NULL; 983 u32 dw0 = le32_to_cpu(eqe->dw0); 984 u16 eq_depth = qm->eq_depth; 985 u16 cqn, eqe_num = 0; 986 987 if (QM_EQE_PHASE(dw0) != qm->status.eqc_phase) { 988 atomic64_inc(&qm->debug.dfx.err_irq_cnt); 989 qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); 990 return; 991 } 992 993 cqn = dw0 & QM_EQE_CQN_MASK; 994 if (unlikely(cqn >= qm->qp_num)) 995 return; 996 poll_data = &qm->poll_data[cqn]; 997 998 do { 999 poll_data->qp_finish_id[eqe_num] = dw0 & QM_EQE_CQN_MASK; 1000 eqe_num++; 1001 1002 if (qm->status.eq_head == eq_depth - 1) { 1003 qm->status.eqc_phase = !qm->status.eqc_phase; 1004 eqe = qm->eqe; 1005 qm->status.eq_head = 0; 1006 } else { 1007 eqe++; 1008 qm->status.eq_head++; 1009 } 1010 1011 dw0 = le32_to_cpu(eqe->dw0); 1012 if (QM_EQE_PHASE(dw0) != qm->status.eqc_phase) 1013 break; 1014 } while (eqe_num < (eq_depth >> 1) - 1); 1015 1016 poll_data->eqe_num = eqe_num; 1017 queue_work(qm->wq, &poll_data->work); 1018 qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); 1019} 1020 1021static irqreturn_t qm_eq_irq(int irq, void *data) 1022{ 1023 struct hisi_qm *qm = data; 1024 1025 /* Get qp id of completed tasks and re-enable the interrupt */ 1026 qm_get_complete_eqe_num(qm); 1027 1028 return IRQ_HANDLED; 1029} 1030 1031static irqreturn_t qm_mb_cmd_irq(int irq, void *data) 1032{ 1033 struct hisi_qm *qm = data; 1034 u32 val; 1035 1036 val = readl(qm->io_base + QM_IFC_INT_STATUS); 1037 val &= QM_IFC_INT_STATUS_MASK; 1038 if (!val) 1039 return IRQ_NONE; 1040 1041 if (test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl)) { 1042 dev_warn(&qm->pdev->dev, "Driver is down, message cannot be processed!\n"); 1043 return IRQ_HANDLED; 1044 } 1045 1046 schedule_work(&qm->cmd_process); 1047 1048 return IRQ_HANDLED; 1049} 1050 1051static void qm_set_qp_disable(struct hisi_qp *qp, int offset) 1052{ 1053 u32 *addr; 1054 1055 if (qp->is_in_kernel) 1056 return; 1057 1058 addr = (u32 *)(qp->qdma.va + qp->qdma.size) - offset; 1059 *addr = 1; 1060 1061 /* make sure setup is completed */ 1062 smp_wmb(); 1063} 1064 1065static void qm_disable_qp(struct hisi_qm *qm, u32 qp_id) 1066{ 1067 struct hisi_qp *qp = &qm->qp_array[qp_id]; 1068 1069 qm_set_qp_disable(qp, QM_RESET_STOP_TX_OFFSET); 1070 hisi_qm_stop_qp(qp); 1071 qm_set_qp_disable(qp, QM_RESET_STOP_RX_OFFSET); 1072} 1073 1074static void qm_reset_function(struct hisi_qm *qm) 1075{ 1076 struct device *dev = &qm->pdev->dev; 1077 int ret; 1078 1079 if (qm_check_dev_error(qm)) 1080 return; 1081 1082 ret = qm_reset_prepare_ready(qm); 1083 if (ret) { 1084 dev_err(dev, "reset function not ready\n"); 1085 return; 1086 } 1087 1088 ret = hisi_qm_stop(qm, QM_DOWN); 1089 if (ret) { 1090 dev_err(dev, "failed to stop qm when reset function\n"); 1091 goto clear_bit; 1092 } 1093 1094 ret = hisi_qm_start(qm); 1095 if (ret) 1096 dev_err(dev, "failed to start qm when reset function\n"); 1097 1098clear_bit: 1099 qm_reset_bit_clear(qm); 1100} 1101 1102static irqreturn_t qm_aeq_thread(int irq, void *data) 1103{ 1104 struct hisi_qm *qm = data; 1105 struct qm_aeqe *aeqe = qm->aeqe + qm->status.aeq_head; 1106 u32 dw0 = le32_to_cpu(aeqe->dw0); 1107 u16 aeq_depth = qm->aeq_depth; 1108 u32 type, qp_id; 1109 1110 atomic64_inc(&qm->debug.dfx.aeq_irq_cnt); 1111 1112 while (QM_AEQE_PHASE(dw0) == qm->status.aeqc_phase) { 1113 type = (dw0 >> QM_AEQE_TYPE_SHIFT) & QM_AEQE_TYPE_MASK; 1114 qp_id = dw0 & QM_AEQE_CQN_MASK; 1115 1116 switch (type) { 1117 case QM_EQ_OVERFLOW: 1118 dev_err(&qm->pdev->dev, "eq overflow, reset function\n"); 1119 qm_reset_function(qm); 1120 return IRQ_HANDLED; 1121 case QM_CQ_OVERFLOW: 1122 dev_err(&qm->pdev->dev, "cq overflow, stop qp(%u)\n", 1123 qp_id); 1124 fallthrough; 1125 case QM_CQE_ERROR: 1126 qm_disable_qp(qm, qp_id); 1127 break; 1128 default: 1129 dev_err(&qm->pdev->dev, "unknown error type %u\n", 1130 type); 1131 break; 1132 } 1133 1134 if (qm->status.aeq_head == aeq_depth - 1) { 1135 qm->status.aeqc_phase = !qm->status.aeqc_phase; 1136 aeqe = qm->aeqe; 1137 qm->status.aeq_head = 0; 1138 } else { 1139 aeqe++; 1140 qm->status.aeq_head++; 1141 } 1142 dw0 = le32_to_cpu(aeqe->dw0); 1143 } 1144 1145 qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0); 1146 1147 return IRQ_HANDLED; 1148} 1149 1150static void qm_init_qp_status(struct hisi_qp *qp) 1151{ 1152 struct hisi_qp_status *qp_status = &qp->qp_status; 1153 1154 qp_status->sq_tail = 0; 1155 qp_status->cq_head = 0; 1156 qp_status->cqc_phase = true; 1157 atomic_set(&qp_status->used, 0); 1158} 1159 1160static void qm_init_prefetch(struct hisi_qm *qm) 1161{ 1162 struct device *dev = &qm->pdev->dev; 1163 u32 page_type = 0x0; 1164 1165 if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) 1166 return; 1167 1168 switch (PAGE_SIZE) { 1169 case SZ_4K: 1170 page_type = 0x0; 1171 break; 1172 case SZ_16K: 1173 page_type = 0x1; 1174 break; 1175 case SZ_64K: 1176 page_type = 0x2; 1177 break; 1178 default: 1179 dev_err(dev, "system page size is not support: %lu, default set to 4KB", 1180 PAGE_SIZE); 1181 } 1182 1183 writel(page_type, qm->io_base + QM_PAGE_SIZE); 1184} 1185 1186/* 1187 * acc_shaper_para_calc() Get the IR value by the qos formula, the return value 1188 * is the expected qos calculated. 1189 * the formula: 1190 * IR = X Mbps if ir = 1 means IR = 100 Mbps, if ir = 10000 means = 10Gbps 1191 * 1192 * IR_b * (2 ^ IR_u) * 8000 1193 * IR(Mbps) = ------------------------- 1194 * Tick * (2 ^ IR_s) 1195 */ 1196static u32 acc_shaper_para_calc(u64 cir_b, u64 cir_u, u64 cir_s) 1197{ 1198 return ((cir_b * QM_QOS_DIVISOR_CLK) * (1 << cir_u)) / 1199 (QM_QOS_TICK * (1 << cir_s)); 1200} 1201 1202static u32 acc_shaper_calc_cbs_s(u32 ir) 1203{ 1204 int table_size = ARRAY_SIZE(shaper_cbs_s); 1205 int i; 1206 1207 for (i = 0; i < table_size; i++) { 1208 if (ir >= shaper_cbs_s[i].start && ir <= shaper_cbs_s[i].end) 1209 return shaper_cbs_s[i].val; 1210 } 1211 1212 return QM_SHAPER_MIN_CBS_S; 1213} 1214 1215static u32 acc_shaper_calc_cir_s(u32 ir) 1216{ 1217 int table_size = ARRAY_SIZE(shaper_cir_s); 1218 int i; 1219 1220 for (i = 0; i < table_size; i++) { 1221 if (ir >= shaper_cir_s[i].start && ir <= shaper_cir_s[i].end) 1222 return shaper_cir_s[i].val; 1223 } 1224 1225 return 0; 1226} 1227 1228static int qm_get_shaper_para(u32 ir, struct qm_shaper_factor *factor) 1229{ 1230 u32 cir_b, cir_u, cir_s, ir_calc; 1231 u32 error_rate; 1232 1233 factor->cbs_s = acc_shaper_calc_cbs_s(ir); 1234 cir_s = acc_shaper_calc_cir_s(ir); 1235 1236 for (cir_b = QM_QOS_MIN_CIR_B; cir_b <= QM_QOS_MAX_CIR_B; cir_b++) { 1237 for (cir_u = 0; cir_u <= QM_QOS_MAX_CIR_U; cir_u++) { 1238 ir_calc = acc_shaper_para_calc(cir_b, cir_u, cir_s); 1239 1240 error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir; 1241 if (error_rate <= QM_QOS_MIN_ERROR_RATE) { 1242 factor->cir_b = cir_b; 1243 factor->cir_u = cir_u; 1244 factor->cir_s = cir_s; 1245 return 0; 1246 } 1247 } 1248 } 1249 1250 return -EINVAL; 1251} 1252 1253static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base, 1254 u32 number, struct qm_shaper_factor *factor) 1255{ 1256 u64 tmp = 0; 1257 1258 if (number > 0) { 1259 switch (type) { 1260 case SQC_VFT: 1261 if (qm->ver == QM_HW_V1) { 1262 tmp = QM_SQC_VFT_BUF_SIZE | 1263 QM_SQC_VFT_SQC_SIZE | 1264 QM_SQC_VFT_INDEX_NUMBER | 1265 QM_SQC_VFT_VALID | 1266 (u64)base << QM_SQC_VFT_START_SQN_SHIFT; 1267 } else { 1268 tmp = (u64)base << QM_SQC_VFT_START_SQN_SHIFT | 1269 QM_SQC_VFT_VALID | 1270 (u64)(number - 1) << QM_SQC_VFT_SQN_SHIFT; 1271 } 1272 break; 1273 case CQC_VFT: 1274 if (qm->ver == QM_HW_V1) { 1275 tmp = QM_CQC_VFT_BUF_SIZE | 1276 QM_CQC_VFT_SQC_SIZE | 1277 QM_CQC_VFT_INDEX_NUMBER | 1278 QM_CQC_VFT_VALID; 1279 } else { 1280 tmp = QM_CQC_VFT_VALID; 1281 } 1282 break; 1283 case SHAPER_VFT: 1284 if (factor) { 1285 tmp = factor->cir_b | 1286 (factor->cir_u << QM_SHAPER_FACTOR_CIR_U_SHIFT) | 1287 (factor->cir_s << QM_SHAPER_FACTOR_CIR_S_SHIFT) | 1288 (QM_SHAPER_CBS_B << QM_SHAPER_FACTOR_CBS_B_SHIFT) | 1289 (factor->cbs_s << QM_SHAPER_FACTOR_CBS_S_SHIFT); 1290 } 1291 break; 1292 /* 1293 * Note: The current logic only needs to handle the above three types 1294 * If new types are added, they need to be supplemented here, 1295 * otherwise undefined behavior may occur. 1296 */ 1297 default: 1298 break; 1299 } 1300 } 1301 1302 writel(lower_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_L); 1303 writel(upper_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_H); 1304} 1305 1306static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type, 1307 u32 fun_num, u32 base, u32 number) 1308{ 1309 struct qm_shaper_factor *factor = NULL; 1310 unsigned int val; 1311 int ret; 1312 1313 if (type == SHAPER_VFT && test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) 1314 factor = &qm->factor[fun_num]; 1315 1316 ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, 1317 val & BIT(0), POLL_PERIOD, 1318 POLL_TIMEOUT); 1319 if (ret) 1320 return ret; 1321 1322 writel(0x0, qm->io_base + QM_VFT_CFG_OP_WR); 1323 writel(type, qm->io_base + QM_VFT_CFG_TYPE); 1324 if (type == SHAPER_VFT) 1325 fun_num |= base << QM_SHAPER_VFT_OFFSET; 1326 1327 writel(fun_num, qm->io_base + QM_VFT_CFG); 1328 1329 qm_vft_data_cfg(qm, type, base, number, factor); 1330 1331 writel(0x0, qm->io_base + QM_VFT_CFG_RDY); 1332 writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE); 1333 1334 return readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, 1335 val & BIT(0), POLL_PERIOD, 1336 POLL_TIMEOUT); 1337} 1338 1339static int qm_shaper_init_vft(struct hisi_qm *qm, u32 fun_num) 1340{ 1341 u32 qos = qm->factor[fun_num].func_qos; 1342 int ret, i; 1343 1344 ret = qm_get_shaper_para(qos * QM_QOS_RATE, &qm->factor[fun_num]); 1345 if (ret) { 1346 dev_err(&qm->pdev->dev, "failed to calculate shaper parameter!\n"); 1347 return ret; 1348 } 1349 writel(qm->type_rate, qm->io_base + QM_SHAPER_CFG); 1350 for (i = ALG_TYPE_0; i <= ALG_TYPE_1; i++) { 1351 /* The base number of queue reuse for different alg type */ 1352 ret = qm_set_vft_common(qm, SHAPER_VFT, fun_num, i, 1); 1353 if (ret) 1354 return ret; 1355 } 1356 1357 return 0; 1358} 1359 1360/* The config should be conducted after qm_dev_mem_reset() */ 1361static int qm_set_sqc_cqc_vft(struct hisi_qm *qm, u32 fun_num, u32 base, 1362 u32 number) 1363{ 1364 int ret, i; 1365 1366 for (i = SQC_VFT; i <= CQC_VFT; i++) { 1367 ret = qm_set_vft_common(qm, i, fun_num, base, number); 1368 if (ret) 1369 return ret; 1370 } 1371 1372 /* init default shaper qos val */ 1373 if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) { 1374 ret = qm_shaper_init_vft(qm, fun_num); 1375 if (ret) 1376 goto back_sqc_cqc; 1377 } 1378 1379 return 0; 1380back_sqc_cqc: 1381 for (i = SQC_VFT; i <= CQC_VFT; i++) 1382 qm_set_vft_common(qm, i, fun_num, 0, 0); 1383 1384 return ret; 1385} 1386 1387static int qm_get_vft_v2(struct hisi_qm *qm, u32 *base, u32 *number) 1388{ 1389 u64 sqc_vft; 1390 int ret; 1391 1392 ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_VFT_V2, 0, 0, 1); 1393 if (ret) 1394 return ret; 1395 1396 sqc_vft = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) | 1397 ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 32); 1398 *base = QM_SQC_VFT_BASE_MASK_V2 & (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2); 1399 *number = (QM_SQC_VFT_NUM_MASK_V2 & 1400 (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1; 1401 1402 return 0; 1403} 1404 1405static void qm_hw_error_init_v1(struct hisi_qm *qm) 1406{ 1407 writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK); 1408} 1409 1410static void qm_hw_error_cfg(struct hisi_qm *qm) 1411{ 1412 struct hisi_qm_err_mask *qm_err = &qm->err_info.qm_err; 1413 1414 qm->error_mask = qm_err->nfe | qm_err->ce | qm_err->fe; 1415 /* clear QM hw residual error source */ 1416 writel(qm->error_mask, qm->io_base + QM_ABNORMAL_INT_SOURCE); 1417 1418 /* configure error type */ 1419 writel(qm_err->ce, qm->io_base + QM_RAS_CE_ENABLE); 1420 writel(QM_RAS_CE_TIMES_PER_IRQ, qm->io_base + QM_RAS_CE_THRESHOLD); 1421 writel(qm_err->nfe, qm->io_base + QM_RAS_NFE_ENABLE); 1422 writel(qm_err->fe, qm->io_base + QM_RAS_FE_ENABLE); 1423} 1424 1425static void qm_hw_error_init_v2(struct hisi_qm *qm) 1426{ 1427 u32 irq_unmask; 1428 1429 qm_hw_error_cfg(qm); 1430 1431 irq_unmask = ~qm->error_mask; 1432 irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK); 1433 writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK); 1434} 1435 1436static void qm_hw_error_uninit_v2(struct hisi_qm *qm) 1437{ 1438 u32 irq_mask = qm->error_mask; 1439 1440 irq_mask |= readl(qm->io_base + QM_ABNORMAL_INT_MASK); 1441 writel(irq_mask, qm->io_base + QM_ABNORMAL_INT_MASK); 1442} 1443 1444static void qm_hw_error_init_v3(struct hisi_qm *qm) 1445{ 1446 u32 irq_unmask; 1447 1448 qm_hw_error_cfg(qm); 1449 1450 /* enable close master ooo when hardware error happened */ 1451 writel(qm->err_info.qm_err.shutdown_mask, qm->io_base + QM_OOO_SHUTDOWN_SEL); 1452 1453 irq_unmask = ~qm->error_mask; 1454 irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK); 1455 writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK); 1456} 1457 1458static void qm_hw_error_uninit_v3(struct hisi_qm *qm) 1459{ 1460 u32 irq_mask = qm->error_mask; 1461 1462 irq_mask |= readl(qm->io_base + QM_ABNORMAL_INT_MASK); 1463 writel(irq_mask, qm->io_base + QM_ABNORMAL_INT_MASK); 1464 1465 /* disable close master ooo when hardware error happened */ 1466 writel(0x0, qm->io_base + QM_OOO_SHUTDOWN_SEL); 1467} 1468 1469static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status) 1470{ 1471 const struct hisi_qm_hw_error *err; 1472 struct device *dev = &qm->pdev->dev; 1473 u32 reg_val, type, vf_num, qp_id; 1474 int i; 1475 1476 for (i = 0; i < ARRAY_SIZE(qm_hw_error); i++) { 1477 err = &qm_hw_error[i]; 1478 if (!(err->int_msk & error_status)) 1479 continue; 1480 1481 dev_err(dev, "%s [error status=0x%x] found\n", 1482 err->msg, err->int_msk); 1483 1484 if (err->int_msk & QM_DB_TIMEOUT) { 1485 reg_val = readl(qm->io_base + QM_ABNORMAL_INF01); 1486 type = (reg_val & QM_DB_TIMEOUT_TYPE) >> 1487 QM_DB_TIMEOUT_TYPE_SHIFT; 1488 vf_num = reg_val & QM_DB_TIMEOUT_VF; 1489 qp_id = reg_val >> QM_DB_TIMEOUT_QP_SHIFT; 1490 dev_err(dev, "qm %s doorbell timeout in function %u qp %u\n", 1491 qm_db_timeout[type], vf_num, qp_id); 1492 } else if (err->int_msk & QM_OF_FIFO_OF) { 1493 reg_val = readl(qm->io_base + QM_ABNORMAL_INF00); 1494 type = (reg_val & QM_FIFO_OVERFLOW_TYPE) >> 1495 QM_FIFO_OVERFLOW_TYPE_SHIFT; 1496 vf_num = reg_val & QM_FIFO_OVERFLOW_VF; 1497 qp_id = reg_val >> QM_FIFO_OVERFLOW_QP_SHIFT; 1498 if (type < ARRAY_SIZE(qm_fifo_overflow)) 1499 dev_err(dev, "qm %s fifo overflow in function %u qp %u\n", 1500 qm_fifo_overflow[type], vf_num, qp_id); 1501 else 1502 dev_err(dev, "unknown error type\n"); 1503 } else if (err->int_msk & QM_AXI_RRESP_ERR) { 1504 reg_val = readl(qm->io_base + QM_ABNORMAL_INF02); 1505 if (reg_val & QM_AXI_POISON_ERR) 1506 dev_err(dev, "qm axi poison error happened\n"); 1507 } 1508 } 1509} 1510 1511static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm) 1512{ 1513 struct hisi_qm_err_mask *qm_err = &qm->err_info.qm_err; 1514 u32 error_status; 1515 1516 error_status = qm_get_hw_error_status(qm); 1517 if (error_status & qm->error_mask) { 1518 if (error_status & QM_ECC_MBIT) 1519 qm->err_status.is_qm_ecc_mbit = true; 1520 1521 qm_log_hw_error(qm, error_status); 1522 if (error_status & qm_err->reset_mask) { 1523 /* Disable the same error reporting until device is recovered. */ 1524 writel(qm_err->nfe & (~error_status), qm->io_base + QM_RAS_NFE_ENABLE); 1525 return ACC_ERR_NEED_RESET; 1526 } 1527 1528 /* Clear error source if not need reset. */ 1529 writel(error_status, qm->io_base + QM_ABNORMAL_INT_SOURCE); 1530 writel(qm_err->nfe, qm->io_base + QM_RAS_NFE_ENABLE); 1531 writel(qm_err->ce, qm->io_base + QM_RAS_CE_ENABLE); 1532 } 1533 1534 return ACC_ERR_RECOVERED; 1535} 1536 1537static int qm_get_mb_cmd(struct hisi_qm *qm, u64 *msg, u16 fun_num) 1538{ 1539 struct qm_mailbox mailbox; 1540 int ret; 1541 1542 qm_mb_pre_init(&mailbox, QM_MB_CMD_DST, 0, fun_num, 0); 1543 mutex_lock(&qm->mailbox_lock); 1544 ret = qm_mb_nolock(qm, &mailbox); 1545 if (ret) 1546 goto err_unlock; 1547 1548 *msg = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) | 1549 ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 32); 1550 1551err_unlock: 1552 mutex_unlock(&qm->mailbox_lock); 1553 return ret; 1554} 1555 1556static void qm_clear_cmd_interrupt(struct hisi_qm *qm, u64 vf_mask) 1557{ 1558 u32 val; 1559 1560 if (qm->fun_type == QM_HW_PF) 1561 writeq(vf_mask, qm->io_base + QM_IFC_INT_SOURCE_P); 1562 1563 val = readl(qm->io_base + QM_IFC_INT_SOURCE_V); 1564 val |= QM_IFC_INT_SOURCE_MASK; 1565 writel(val, qm->io_base + QM_IFC_INT_SOURCE_V); 1566} 1567 1568static void qm_handle_vf_msg(struct hisi_qm *qm, u32 vf_id) 1569{ 1570 struct device *dev = &qm->pdev->dev; 1571 enum qm_ifc_cmd cmd; 1572 int ret; 1573 1574 ret = qm->ops->get_ifc(qm, &cmd, NULL, vf_id); 1575 if (ret) { 1576 dev_err(dev, "failed to get command from VF(%u)!\n", vf_id); 1577 return; 1578 } 1579 1580 switch (cmd) { 1581 case QM_VF_PREPARE_FAIL: 1582 dev_err(dev, "failed to stop VF(%u)!\n", vf_id); 1583 break; 1584 case QM_VF_START_FAIL: 1585 dev_err(dev, "failed to start VF(%u)!\n", vf_id); 1586 break; 1587 case QM_VF_PREPARE_DONE: 1588 case QM_VF_START_DONE: 1589 break; 1590 default: 1591 dev_err(dev, "unsupported command(0x%x) sent by VF(%u)!\n", cmd, vf_id); 1592 break; 1593 } 1594} 1595 1596static int qm_wait_vf_prepare_finish(struct hisi_qm *qm) 1597{ 1598 struct device *dev = &qm->pdev->dev; 1599 u32 vfs_num = qm->vfs_num; 1600 int cnt = 0; 1601 int ret = 0; 1602 u64 val; 1603 u32 i; 1604 1605 if (!qm->vfs_num || !test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) 1606 return 0; 1607 1608 while (true) { 1609 val = readq(qm->io_base + QM_IFC_INT_SOURCE_P); 1610 /* All VFs send command to PF, break */ 1611 if ((val & GENMASK(vfs_num, 1)) == GENMASK(vfs_num, 1)) 1612 break; 1613 1614 if (++cnt > QM_MAX_PF_WAIT_COUNT) { 1615 ret = -EBUSY; 1616 break; 1617 } 1618 1619 msleep(QM_WAIT_DST_ACK); 1620 } 1621 1622 /* PF check VFs msg */ 1623 for (i = 1; i <= vfs_num; i++) { 1624 if (val & BIT(i)) 1625 qm_handle_vf_msg(qm, i); 1626 else 1627 dev_err(dev, "VF(%u) not ping PF!\n", i); 1628 } 1629 1630 /* PF clear interrupt to ack VFs */ 1631 qm_clear_cmd_interrupt(qm, val); 1632 1633 return ret; 1634} 1635 1636static void qm_trigger_vf_interrupt(struct hisi_qm *qm, u32 fun_num) 1637{ 1638 u32 val; 1639 1640 val = readl(qm->io_base + QM_IFC_INT_CFG); 1641 val &= ~QM_IFC_SEND_ALL_VFS; 1642 val |= fun_num; 1643 writel(val, qm->io_base + QM_IFC_INT_CFG); 1644 1645 val = readl(qm->io_base + QM_IFC_INT_SET_P); 1646 val |= QM_IFC_INT_SET_MASK; 1647 writel(val, qm->io_base + QM_IFC_INT_SET_P); 1648} 1649 1650static void qm_trigger_pf_interrupt(struct hisi_qm *qm) 1651{ 1652 u32 val; 1653 1654 val = readl(qm->io_base + QM_IFC_INT_SET_V); 1655 val |= QM_IFC_INT_SET_MASK; 1656 writel(val, qm->io_base + QM_IFC_INT_SET_V); 1657} 1658 1659static int qm_ping_single_vf(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num) 1660{ 1661 struct device *dev = &qm->pdev->dev; 1662 int cnt = 0; 1663 u64 val; 1664 int ret; 1665 1666 ret = qm->ops->set_ifc_begin(qm, cmd, data, fun_num); 1667 if (ret) { 1668 dev_err(dev, "failed to send command to vf(%u)!\n", fun_num); 1669 goto err_unlock; 1670 } 1671 1672 qm_trigger_vf_interrupt(qm, fun_num); 1673 while (true) { 1674 msleep(QM_WAIT_DST_ACK); 1675 val = readq(qm->io_base + QM_IFC_READY_STATUS); 1676 /* if VF respond, PF notifies VF successfully. */ 1677 if (!(val & BIT(fun_num))) 1678 goto err_unlock; 1679 1680 if (++cnt > QM_MAX_PF_WAIT_COUNT) { 1681 dev_err(dev, "failed to get response from VF(%u)!\n", fun_num); 1682 ret = -ETIMEDOUT; 1683 break; 1684 } 1685 } 1686 1687err_unlock: 1688 qm->ops->set_ifc_end(qm); 1689 return ret; 1690} 1691 1692static int qm_ping_all_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd) 1693{ 1694 struct device *dev = &qm->pdev->dev; 1695 u32 vfs_num = qm->vfs_num; 1696 u64 val = 0; 1697 int cnt = 0; 1698 int ret; 1699 u32 i; 1700 1701 ret = qm->ops->set_ifc_begin(qm, cmd, 0, QM_MB_PING_ALL_VFS); 1702 if (ret) { 1703 dev_err(dev, "failed to send command(0x%x) to all vfs!\n", cmd); 1704 qm->ops->set_ifc_end(qm); 1705 return ret; 1706 } 1707 1708 qm_trigger_vf_interrupt(qm, QM_IFC_SEND_ALL_VFS); 1709 while (true) { 1710 msleep(QM_WAIT_DST_ACK); 1711 val = readq(qm->io_base + QM_IFC_READY_STATUS); 1712 /* If all VFs acked, PF notifies VFs successfully. */ 1713 if (!(val & GENMASK(vfs_num, 1))) { 1714 qm->ops->set_ifc_end(qm); 1715 return 0; 1716 } 1717 1718 if (++cnt > QM_MAX_PF_WAIT_COUNT) 1719 break; 1720 } 1721 1722 qm->ops->set_ifc_end(qm); 1723 1724 /* Check which vf respond timeout. */ 1725 for (i = 1; i <= vfs_num; i++) { 1726 if (val & BIT(i)) 1727 dev_err(dev, "failed to get response from VF(%u)!\n", i); 1728 } 1729 1730 return -ETIMEDOUT; 1731} 1732 1733static int qm_ping_pf(struct hisi_qm *qm, enum qm_ifc_cmd cmd) 1734{ 1735 int cnt = 0; 1736 u32 val; 1737 int ret; 1738 1739 ret = qm->ops->set_ifc_begin(qm, cmd, 0, 0); 1740 if (ret) { 1741 dev_err(&qm->pdev->dev, "failed to send command(0x%x) to PF!\n", cmd); 1742 goto unlock; 1743 } 1744 1745 qm_trigger_pf_interrupt(qm); 1746 /* Waiting for PF response */ 1747 while (true) { 1748 msleep(QM_WAIT_DST_ACK); 1749 val = readl(qm->io_base + QM_IFC_INT_SET_V); 1750 if (!(val & QM_IFC_INT_STATUS_MASK)) 1751 break; 1752 1753 if (++cnt > QM_MAX_VF_WAIT_COUNT) { 1754 ret = -ETIMEDOUT; 1755 break; 1756 } 1757 } 1758 1759unlock: 1760 qm->ops->set_ifc_end(qm); 1761 1762 return ret; 1763} 1764 1765static int qm_drain_qm(struct hisi_qm *qm) 1766{ 1767 return hisi_qm_mb(qm, QM_MB_CMD_FLUSH_QM, 0, 0, 0); 1768} 1769 1770static int qm_stop_qp(struct hisi_qp *qp) 1771{ 1772 return hisi_qm_mb(qp->qm, QM_MB_CMD_STOP_QP, 0, qp->qp_id, 0); 1773} 1774 1775static int qm_set_msi(struct hisi_qm *qm, bool set) 1776{ 1777 struct pci_dev *pdev = qm->pdev; 1778 1779 if (set) { 1780 pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64, 1781 0); 1782 } else { 1783 pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64, 1784 ACC_PEH_MSI_DISABLE); 1785 if (qm->err_status.is_qm_ecc_mbit || 1786 qm->err_status.is_dev_ecc_mbit) 1787 return 0; 1788 1789 mdelay(1); 1790 if (readl(qm->io_base + QM_PEH_DFX_INFO0)) 1791 return -EFAULT; 1792 } 1793 1794 return 0; 1795} 1796 1797static void qm_wait_msi_finish(struct hisi_qm *qm) 1798{ 1799 struct pci_dev *pdev = qm->pdev; 1800 u32 cmd = ~0; 1801 int cnt = 0; 1802 u32 val; 1803 int ret; 1804 1805 while (true) { 1806 pci_read_config_dword(pdev, pdev->msi_cap + 1807 PCI_MSI_PENDING_64, &cmd); 1808 if (!cmd) 1809 break; 1810 1811 if (++cnt > MAX_WAIT_COUNTS) { 1812 pci_warn(pdev, "failed to empty MSI PENDING!\n"); 1813 break; 1814 } 1815 1816 udelay(1); 1817 } 1818 1819 ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_DFX_INFO0, 1820 val, !(val & QM_PEH_DFX_MASK), 1821 POLL_PERIOD, POLL_TIMEOUT); 1822 if (ret) 1823 pci_warn(pdev, "failed to empty PEH MSI!\n"); 1824 1825 ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_DFX_INFO1, 1826 val, !(val & QM_PEH_MSI_FINISH_MASK), 1827 POLL_PERIOD, POLL_TIMEOUT); 1828 if (ret) 1829 pci_warn(pdev, "failed to finish MSI operation!\n"); 1830} 1831 1832static int qm_set_msi_v3(struct hisi_qm *qm, bool set) 1833{ 1834 struct pci_dev *pdev = qm->pdev; 1835 int ret = -ETIMEDOUT; 1836 u32 cmd, i; 1837 1838 pci_read_config_dword(pdev, pdev->msi_cap, &cmd); 1839 if (set) 1840 cmd |= QM_MSI_CAP_ENABLE; 1841 else 1842 cmd &= ~QM_MSI_CAP_ENABLE; 1843 1844 pci_write_config_dword(pdev, pdev->msi_cap, cmd); 1845 if (set) { 1846 for (i = 0; i < MAX_WAIT_COUNTS; i++) { 1847 pci_read_config_dword(pdev, pdev->msi_cap, &cmd); 1848 if (cmd & QM_MSI_CAP_ENABLE) 1849 return 0; 1850 1851 udelay(1); 1852 } 1853 } else { 1854 udelay(WAIT_PERIOD_US_MIN); 1855 qm_wait_msi_finish(qm); 1856 ret = 0; 1857 } 1858 1859 return ret; 1860} 1861 1862static int qm_set_ifc_begin_v3(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num) 1863{ 1864 struct qm_mailbox mailbox; 1865 u64 msg; 1866 1867 msg = cmd | (u64)data << QM_IFC_DATA_SHIFT; 1868 1869 qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, msg, fun_num, 0); 1870 mutex_lock(&qm->mailbox_lock); 1871 return qm_mb_nolock(qm, &mailbox); 1872} 1873 1874static void qm_set_ifc_end_v3(struct hisi_qm *qm) 1875{ 1876 mutex_unlock(&qm->mailbox_lock); 1877} 1878 1879static int qm_get_ifc_v3(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num) 1880{ 1881 u64 msg; 1882 int ret; 1883 1884 ret = qm_get_mb_cmd(qm, &msg, fun_num); 1885 if (ret) 1886 return ret; 1887 1888 *cmd = msg & QM_IFC_CMD_MASK; 1889 1890 if (data) 1891 *data = msg >> QM_IFC_DATA_SHIFT; 1892 1893 return 0; 1894} 1895 1896static int qm_set_ifc_begin_v4(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num) 1897{ 1898 uintptr_t offset; 1899 u64 msg; 1900 1901 if (qm->fun_type == QM_HW_PF) 1902 offset = QM_PF2VF_PF_W; 1903 else 1904 offset = QM_VF2PF_VF_W; 1905 1906 msg = cmd | (u64)data << QM_IFC_DATA_SHIFT; 1907 1908 mutex_lock(&qm->ifc_lock); 1909 writeq(msg, qm->io_base + offset); 1910 1911 return 0; 1912} 1913 1914static void qm_set_ifc_end_v4(struct hisi_qm *qm) 1915{ 1916 mutex_unlock(&qm->ifc_lock); 1917} 1918 1919static u64 qm_get_ifc_pf(struct hisi_qm *qm, u32 fun_num) 1920{ 1921 uintptr_t offset; 1922 1923 offset = QM_VF2PF_PF_R + QM_VF2PF_REG_SIZE * fun_num; 1924 1925 return (u64)readl(qm->io_base + offset); 1926} 1927 1928static u64 qm_get_ifc_vf(struct hisi_qm *qm) 1929{ 1930 return readq(qm->io_base + QM_PF2VF_VF_R); 1931} 1932 1933static int qm_get_ifc_v4(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num) 1934{ 1935 u64 msg; 1936 1937 if (qm->fun_type == QM_HW_PF) 1938 msg = qm_get_ifc_pf(qm, fun_num); 1939 else 1940 msg = qm_get_ifc_vf(qm); 1941 1942 *cmd = msg & QM_IFC_CMD_MASK; 1943 1944 if (data) 1945 *data = msg >> QM_IFC_DATA_SHIFT; 1946 1947 return 0; 1948} 1949 1950static const struct hisi_qm_hw_ops qm_hw_ops_v1 = { 1951 .qm_db = qm_db_v1, 1952 .hw_error_init = qm_hw_error_init_v1, 1953 .set_msi = qm_set_msi, 1954}; 1955 1956static const struct hisi_qm_hw_ops qm_hw_ops_v2 = { 1957 .get_vft = qm_get_vft_v2, 1958 .qm_db = qm_db_v2, 1959 .hw_error_init = qm_hw_error_init_v2, 1960 .hw_error_uninit = qm_hw_error_uninit_v2, 1961 .hw_error_handle = qm_hw_error_handle_v2, 1962 .set_msi = qm_set_msi, 1963}; 1964 1965static const struct hisi_qm_hw_ops qm_hw_ops_v3 = { 1966 .get_vft = qm_get_vft_v2, 1967 .qm_db = qm_db_v2, 1968 .hw_error_init = qm_hw_error_init_v3, 1969 .hw_error_uninit = qm_hw_error_uninit_v3, 1970 .hw_error_handle = qm_hw_error_handle_v2, 1971 .set_msi = qm_set_msi_v3, 1972 .set_ifc_begin = qm_set_ifc_begin_v3, 1973 .set_ifc_end = qm_set_ifc_end_v3, 1974 .get_ifc = qm_get_ifc_v3, 1975}; 1976 1977static const struct hisi_qm_hw_ops qm_hw_ops_v4 = { 1978 .get_vft = qm_get_vft_v2, 1979 .qm_db = qm_db_v2, 1980 .hw_error_init = qm_hw_error_init_v3, 1981 .hw_error_uninit = qm_hw_error_uninit_v3, 1982 .hw_error_handle = qm_hw_error_handle_v2, 1983 .set_msi = qm_set_msi_v3, 1984 .set_ifc_begin = qm_set_ifc_begin_v4, 1985 .set_ifc_end = qm_set_ifc_end_v4, 1986 .get_ifc = qm_get_ifc_v4, 1987}; 1988 1989static void *qm_get_avail_sqe(struct hisi_qp *qp) 1990{ 1991 struct hisi_qp_status *qp_status = &qp->qp_status; 1992 u16 sq_tail = qp_status->sq_tail; 1993 1994 if (unlikely(atomic_read(&qp->qp_status.used) == qp->sq_depth - 1)) 1995 return NULL; 1996 1997 return qp->sqe + sq_tail * qp->qm->sqe_size; 1998} 1999 2000static void hisi_qm_unset_hw_reset(struct hisi_qp *qp) 2001{ 2002 u64 *addr; 2003 2004 /* Use last 64 bits of DUS to reset status. */ 2005 addr = (u64 *)(qp->qdma.va + qp->qdma.size) - QM_RESET_STOP_TX_OFFSET; 2006 *addr = 0; 2007} 2008 2009static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type) 2010{ 2011 struct device *dev = &qm->pdev->dev; 2012 struct hisi_qp *qp; 2013 int qp_id; 2014 2015 if (atomic_read(&qm->status.flags) == QM_STOP) { 2016 dev_info_ratelimited(dev, "failed to create qp as qm is stop!\n"); 2017 return ERR_PTR(-EPERM); 2018 } 2019 2020 if (qm->qp_in_used == qm->qp_num) { 2021 dev_info_ratelimited(dev, "All %u queues of QM are busy!\n", 2022 qm->qp_num); 2023 atomic64_inc(&qm->debug.dfx.create_qp_err_cnt); 2024 return ERR_PTR(-EBUSY); 2025 } 2026 2027 qp_id = idr_alloc_cyclic(&qm->qp_idr, NULL, 0, qm->qp_num, GFP_ATOMIC); 2028 if (qp_id < 0) { 2029 dev_info_ratelimited(dev, "All %u queues of QM are busy!\n", 2030 qm->qp_num); 2031 atomic64_inc(&qm->debug.dfx.create_qp_err_cnt); 2032 return ERR_PTR(-EBUSY); 2033 } 2034 2035 qp = &qm->qp_array[qp_id]; 2036 hisi_qm_unset_hw_reset(qp); 2037 memset(qp->cqe, 0, sizeof(struct qm_cqe) * qp->cq_depth); 2038 2039 qp->event_cb = NULL; 2040 qp->req_cb = NULL; 2041 qp->qp_id = qp_id; 2042 qp->alg_type = alg_type; 2043 qp->is_in_kernel = true; 2044 qm->qp_in_used++; 2045 2046 return qp; 2047} 2048 2049/** 2050 * hisi_qm_create_qp() - Create a queue pair from qm. 2051 * @qm: The qm we create a qp from. 2052 * @alg_type: Accelerator specific algorithm type in sqc. 2053 * 2054 * Return created qp, negative error code if failed. 2055 */ 2056static struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, u8 alg_type) 2057{ 2058 struct hisi_qp *qp; 2059 int ret; 2060 2061 ret = qm_pm_get_sync(qm); 2062 if (ret) 2063 return ERR_PTR(ret); 2064 2065 down_write(&qm->qps_lock); 2066 qp = qm_create_qp_nolock(qm, alg_type); 2067 up_write(&qm->qps_lock); 2068 2069 if (IS_ERR(qp)) 2070 qm_pm_put_sync(qm); 2071 2072 return qp; 2073} 2074 2075/** 2076 * hisi_qm_release_qp() - Release a qp back to its qm. 2077 * @qp: The qp we want to release. 2078 * 2079 * This function releases the resource of a qp. 2080 */ 2081static void hisi_qm_release_qp(struct hisi_qp *qp) 2082{ 2083 struct hisi_qm *qm = qp->qm; 2084 2085 down_write(&qm->qps_lock); 2086 2087 qm->qp_in_used--; 2088 idr_remove(&qm->qp_idr, qp->qp_id); 2089 2090 up_write(&qm->qps_lock); 2091 2092 qm_pm_put_sync(qm); 2093} 2094 2095static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) 2096{ 2097 struct hisi_qm *qm = qp->qm; 2098 enum qm_hw_ver ver = qm->ver; 2099 struct qm_sqc sqc = {0}; 2100 2101 if (ver == QM_HW_V1) { 2102 sqc.dw3 = cpu_to_le32(QM_MK_SQC_DW3_V1(0, 0, 0, qm->sqe_size)); 2103 sqc.w8 = cpu_to_le16(qp->sq_depth - 1); 2104 } else { 2105 sqc.dw3 = cpu_to_le32(QM_MK_SQC_DW3_V2(qm->sqe_size, qp->sq_depth)); 2106 sqc.w8 = 0; /* rand_qc */ 2107 } 2108 sqc.w13 = cpu_to_le16(QM_MK_SQC_W13(0, 1, qp->alg_type)); 2109 sqc.base_l = cpu_to_le32(lower_32_bits(qp->sqe_dma)); 2110 sqc.base_h = cpu_to_le32(upper_32_bits(qp->sqe_dma)); 2111 sqc.cq_num = cpu_to_le16(qp_id); 2112 sqc.pasid = cpu_to_le16(pasid); 2113 2114 if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel) 2115 sqc.w11 = cpu_to_le16(QM_QC_PASID_ENABLE << 2116 QM_QC_PASID_ENABLE_SHIFT); 2117 2118 return qm_set_and_get_xqc(qm, QM_MB_CMD_SQC, &sqc, qp_id, 0); 2119} 2120 2121static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) 2122{ 2123 struct hisi_qm *qm = qp->qm; 2124 enum qm_hw_ver ver = qm->ver; 2125 struct qm_cqc cqc = {0}; 2126 2127 if (ver == QM_HW_V1) { 2128 cqc.dw3 = cpu_to_le32(QM_MK_CQC_DW3_V1(0, 0, 0, QM_QC_CQE_SIZE)); 2129 cqc.w8 = cpu_to_le16(qp->cq_depth - 1); 2130 } else { 2131 cqc.dw3 = cpu_to_le32(QM_MK_CQC_DW3_V2(QM_QC_CQE_SIZE, qp->cq_depth)); 2132 cqc.w8 = 0; /* rand_qc */ 2133 } 2134 /* 2135 * Enable request finishing interrupts defaultly. 2136 * So, there will be some interrupts until disabling 2137 * this. 2138 */ 2139 cqc.dw6 = cpu_to_le32(1 << QM_CQ_PHASE_SHIFT | 1 << QM_CQ_FLAG_SHIFT); 2140 cqc.base_l = cpu_to_le32(lower_32_bits(qp->cqe_dma)); 2141 cqc.base_h = cpu_to_le32(upper_32_bits(qp->cqe_dma)); 2142 cqc.pasid = cpu_to_le16(pasid); 2143 2144 if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel) 2145 cqc.w11 = cpu_to_le16(QM_QC_PASID_ENABLE); 2146 2147 return qm_set_and_get_xqc(qm, QM_MB_CMD_CQC, &cqc, qp_id, 0); 2148} 2149 2150static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) 2151{ 2152 int ret; 2153 2154 qm_init_qp_status(qp); 2155 2156 ret = qm_sq_ctx_cfg(qp, qp_id, pasid); 2157 if (ret) 2158 return ret; 2159 2160 return qm_cq_ctx_cfg(qp, qp_id, pasid); 2161} 2162 2163static int qm_start_qp_nolock(struct hisi_qp *qp, unsigned long arg) 2164{ 2165 struct hisi_qm *qm = qp->qm; 2166 struct device *dev = &qm->pdev->dev; 2167 int qp_id = qp->qp_id; 2168 u32 pasid = arg; 2169 int ret; 2170 2171 if (atomic_read(&qm->status.flags) == QM_STOP) { 2172 dev_info_ratelimited(dev, "failed to start qp as qm is stop!\n"); 2173 return -EPERM; 2174 } 2175 2176 ret = qm_qp_ctx_cfg(qp, qp_id, pasid); 2177 if (ret) 2178 return ret; 2179 2180 atomic_set(&qp->qp_status.flags, QP_START); 2181 dev_dbg(dev, "queue %d started\n", qp_id); 2182 2183 return 0; 2184} 2185 2186/** 2187 * hisi_qm_start_qp() - Start a qp into running. 2188 * @qp: The qp we want to start to run. 2189 * @arg: Accelerator specific argument. 2190 * 2191 * After this function, qp can receive request from user. Return 0 if 2192 * successful, negative error code if failed. 2193 */ 2194int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg) 2195{ 2196 struct hisi_qm *qm = qp->qm; 2197 int ret; 2198 2199 down_write(&qm->qps_lock); 2200 ret = qm_start_qp_nolock(qp, arg); 2201 up_write(&qm->qps_lock); 2202 2203 return ret; 2204} 2205EXPORT_SYMBOL_GPL(hisi_qm_start_qp); 2206 2207/** 2208 * qp_stop_fail_cb() - call request cb. 2209 * @qp: stopped failed qp. 2210 * 2211 * Callback function should be called whether task completed or not. 2212 */ 2213static void qp_stop_fail_cb(struct hisi_qp *qp) 2214{ 2215 int qp_used = atomic_read(&qp->qp_status.used); 2216 u16 cur_tail = qp->qp_status.sq_tail; 2217 u16 sq_depth = qp->sq_depth; 2218 u16 cur_head = (cur_tail + sq_depth - qp_used) % sq_depth; 2219 struct hisi_qm *qm = qp->qm; 2220 u16 pos; 2221 int i; 2222 2223 for (i = 0; i < qp_used; i++) { 2224 pos = (i + cur_head) % sq_depth; 2225 qp->req_cb(qp, qp->sqe + (u32)(qm->sqe_size * pos)); 2226 qm_cq_head_update(qp); 2227 atomic_dec(&qp->qp_status.used); 2228 } 2229} 2230 2231static int qm_wait_qp_empty(struct hisi_qm *qm, u32 *state, u32 qp_id) 2232{ 2233 struct device *dev = &qm->pdev->dev; 2234 struct qm_sqc sqc; 2235 struct qm_cqc cqc; 2236 int ret, i = 0; 2237 2238 while (++i) { 2239 ret = qm_set_and_get_xqc(qm, QM_MB_CMD_SQC, &sqc, qp_id, 1); 2240 if (ret) { 2241 dev_err_ratelimited(dev, "Failed to dump sqc!\n"); 2242 *state = QM_DUMP_SQC_FAIL; 2243 return ret; 2244 } 2245 2246 ret = qm_set_and_get_xqc(qm, QM_MB_CMD_CQC, &cqc, qp_id, 1); 2247 if (ret) { 2248 dev_err_ratelimited(dev, "Failed to dump cqc!\n"); 2249 *state = QM_DUMP_CQC_FAIL; 2250 return ret; 2251 } 2252 2253 if ((sqc.tail == cqc.tail) && 2254 (QM_SQ_TAIL_IDX(sqc) == QM_CQ_TAIL_IDX(cqc))) 2255 break; 2256 2257 if (i == MAX_WAIT_COUNTS) { 2258 dev_err(dev, "Fail to empty queue %u!\n", qp_id); 2259 *state = QM_STOP_QUEUE_FAIL; 2260 return -ETIMEDOUT; 2261 } 2262 2263 usleep_range(WAIT_PERIOD_US_MIN, WAIT_PERIOD_US_MAX); 2264 } 2265 2266 return 0; 2267} 2268 2269/** 2270 * qm_drain_qp() - Drain a qp. 2271 * @qp: The qp we want to drain. 2272 * 2273 * If the device does not support stopping queue by sending mailbox, 2274 * determine whether the queue is cleared by judging the tail pointers of 2275 * sq and cq. 2276 */ 2277static int qm_drain_qp(struct hisi_qp *qp) 2278{ 2279 struct hisi_qm *qm = qp->qm; 2280 u32 state = 0; 2281 int ret; 2282 2283 /* No need to judge if master OOO is blocked. */ 2284 if (qm_check_dev_error(qm)) 2285 return 0; 2286 2287 /* HW V3 supports drain qp by device */ 2288 if (test_bit(QM_SUPPORT_STOP_QP, &qm->caps)) { 2289 ret = qm_stop_qp(qp); 2290 if (ret) { 2291 dev_err(&qm->pdev->dev, "Failed to stop qp!\n"); 2292 state = QM_STOP_QUEUE_FAIL; 2293 goto set_dev_state; 2294 } 2295 return ret; 2296 } 2297 2298 ret = qm_wait_qp_empty(qm, &state, qp->qp_id); 2299 if (ret) 2300 goto set_dev_state; 2301 2302 return 0; 2303 2304set_dev_state: 2305 if (qm->debug.dev_dfx.dev_timeout) 2306 qm->debug.dev_dfx.dev_state = state; 2307 2308 return ret; 2309} 2310 2311static void qm_stop_qp_nolock(struct hisi_qp *qp) 2312{ 2313 struct hisi_qm *qm = qp->qm; 2314 struct device *dev = &qm->pdev->dev; 2315 int ret; 2316 2317 /* 2318 * It is allowed to stop and release qp when reset, If the qp is 2319 * stopped when reset but still want to be released then, the 2320 * is_resetting flag should be set negative so that this qp will not 2321 * be restarted after reset. 2322 */ 2323 if (atomic_read(&qp->qp_status.flags) != QP_START) { 2324 qp->is_resetting = false; 2325 return; 2326 } 2327 2328 atomic_set(&qp->qp_status.flags, QP_STOP); 2329 2330 /* V3 supports direct stop function when FLR prepare */ 2331 if (qm->ver < QM_HW_V3 || qm->status.stop_reason == QM_NORMAL) { 2332 ret = qm_drain_qp(qp); 2333 if (ret) 2334 dev_err(dev, "Failed to drain out data for stopping qp(%u)!\n", qp->qp_id); 2335 } 2336 2337 flush_workqueue(qm->wq); 2338 if (unlikely(qp->is_resetting && atomic_read(&qp->qp_status.used))) 2339 qp_stop_fail_cb(qp); 2340 2341 dev_dbg(dev, "stop queue %u!", qp->qp_id); 2342} 2343 2344/** 2345 * hisi_qm_stop_qp() - Stop a qp in qm. 2346 * @qp: The qp we want to stop. 2347 * 2348 * This function is reverse of hisi_qm_start_qp. 2349 */ 2350void hisi_qm_stop_qp(struct hisi_qp *qp) 2351{ 2352 down_write(&qp->qm->qps_lock); 2353 qm_stop_qp_nolock(qp); 2354 up_write(&qp->qm->qps_lock); 2355} 2356EXPORT_SYMBOL_GPL(hisi_qm_stop_qp); 2357 2358/** 2359 * hisi_qp_send() - Queue up a task in the hardware queue. 2360 * @qp: The qp in which to put the message. 2361 * @msg: The message. 2362 * 2363 * This function will return -EBUSY if qp is currently full, and -EAGAIN 2364 * if qp related qm is resetting. 2365 * 2366 * Note: This function may run with qm_irq_thread and ACC reset at same time. 2367 * It has no race with qm_irq_thread. However, during hisi_qp_send, ACC 2368 * reset may happen, we have no lock here considering performance. This 2369 * causes current qm_db sending fail or can not receive sended sqe. QM 2370 * sync/async receive function should handle the error sqe. ACC reset 2371 * done function should clear used sqe to 0. 2372 */ 2373int hisi_qp_send(struct hisi_qp *qp, const void *msg) 2374{ 2375 struct hisi_qp_status *qp_status = &qp->qp_status; 2376 u16 sq_tail, sq_tail_next; 2377 void *sqe; 2378 2379 spin_lock_bh(&qp->qp_lock); 2380 if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP || 2381 atomic_read(&qp->qm->status.flags) == QM_STOP || 2382 qp->is_resetting)) { 2383 spin_unlock_bh(&qp->qp_lock); 2384 dev_info_ratelimited(&qp->qm->pdev->dev, "QP is stopped or resetting\n"); 2385 return -EAGAIN; 2386 } 2387 2388 sqe = qm_get_avail_sqe(qp); 2389 if (!sqe) { 2390 spin_unlock_bh(&qp->qp_lock); 2391 return -EBUSY; 2392 } 2393 2394 sq_tail = qp_status->sq_tail; 2395 sq_tail_next = (sq_tail + 1) % qp->sq_depth; 2396 memcpy(sqe, msg, qp->qm->sqe_size); 2397 qp->msg[sq_tail] = msg; 2398 2399 qm_db(qp->qm, qp->qp_id, QM_DOORBELL_CMD_SQ, sq_tail_next, 0); 2400 atomic_inc(&qp->qp_status.used); 2401 qp_status->sq_tail = sq_tail_next; 2402 spin_unlock_bh(&qp->qp_lock); 2403 2404 return 0; 2405} 2406EXPORT_SYMBOL_GPL(hisi_qp_send); 2407 2408static void hisi_qm_cache_wb(struct hisi_qm *qm) 2409{ 2410 unsigned int val; 2411 2412 if (qm->ver == QM_HW_V1) 2413 return; 2414 2415 writel(0x1, qm->io_base + QM_CACHE_WB_START); 2416 if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE, 2417 val, val & BIT(0), POLL_PERIOD, 2418 POLL_TIMEOUT)) 2419 dev_err(&qm->pdev->dev, "QM writeback sqc cache fail!\n"); 2420} 2421 2422static void qm_qp_event_notifier(struct hisi_qp *qp) 2423{ 2424 wake_up_interruptible(&qp->uacce_q->wait); 2425} 2426 2427 /* This function returns free number of qp in qm. */ 2428static int hisi_qm_get_available_instances(struct uacce_device *uacce) 2429{ 2430 struct hisi_qm *qm = uacce->priv; 2431 int ret; 2432 2433 down_read(&qm->qps_lock); 2434 ret = qm->qp_num - qm->qp_in_used; 2435 up_read(&qm->qps_lock); 2436 2437 return ret; 2438} 2439 2440static void hisi_qm_set_hw_reset(struct hisi_qm *qm, int offset) 2441{ 2442 int i; 2443 2444 for (i = 0; i < qm->qp_num; i++) 2445 qm_set_qp_disable(&qm->qp_array[i], offset); 2446} 2447 2448static int hisi_qm_uacce_get_queue(struct uacce_device *uacce, 2449 unsigned long arg, 2450 struct uacce_queue *q) 2451{ 2452 struct hisi_qm *qm = uacce->priv; 2453 struct hisi_qp *qp; 2454 u8 alg_type = 0; 2455 2456 qp = hisi_qm_create_qp(qm, alg_type); 2457 if (IS_ERR(qp)) 2458 return PTR_ERR(qp); 2459 2460 q->priv = qp; 2461 q->uacce = uacce; 2462 qp->uacce_q = q; 2463 qp->event_cb = qm_qp_event_notifier; 2464 qp->pasid = arg; 2465 qp->is_in_kernel = false; 2466 2467 return 0; 2468} 2469 2470static void hisi_qm_uacce_put_queue(struct uacce_queue *q) 2471{ 2472 struct hisi_qp *qp = q->priv; 2473 2474 hisi_qm_release_qp(qp); 2475} 2476 2477/* map sq/cq/doorbell to user space */ 2478static int hisi_qm_uacce_mmap(struct uacce_queue *q, 2479 struct vm_area_struct *vma, 2480 struct uacce_qfile_region *qfr) 2481{ 2482 struct hisi_qp *qp = q->priv; 2483 struct hisi_qm *qm = qp->qm; 2484 resource_size_t phys_base = qm->db_phys_base + 2485 qp->qp_id * qm->db_interval; 2486 size_t sz = vma->vm_end - vma->vm_start; 2487 struct pci_dev *pdev = qm->pdev; 2488 struct device *dev = &pdev->dev; 2489 unsigned long vm_pgoff; 2490 int ret; 2491 2492 switch (qfr->type) { 2493 case UACCE_QFRT_MMIO: 2494 if (qm->ver == QM_HW_V1) { 2495 if (sz > PAGE_SIZE * QM_DOORBELL_PAGE_NR) 2496 return -EINVAL; 2497 } else if (!test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) { 2498 if (sz > PAGE_SIZE * (QM_DOORBELL_PAGE_NR + 2499 QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE)) 2500 return -EINVAL; 2501 } else { 2502 if (sz > qm->db_interval) 2503 return -EINVAL; 2504 } 2505 2506 vm_flags_set(vma, VM_IO); 2507 2508 return remap_pfn_range(vma, vma->vm_start, 2509 phys_base >> PAGE_SHIFT, 2510 sz, pgprot_noncached(vma->vm_page_prot)); 2511 case UACCE_QFRT_DUS: 2512 if (sz != qp->qdma.size) 2513 return -EINVAL; 2514 2515 /* 2516 * dma_mmap_coherent() requires vm_pgoff as 0 2517 * restore vm_pfoff to initial value for mmap() 2518 */ 2519 vm_pgoff = vma->vm_pgoff; 2520 vma->vm_pgoff = 0; 2521 ret = dma_mmap_coherent(dev, vma, qp->qdma.va, 2522 qp->qdma.dma, sz); 2523 vma->vm_pgoff = vm_pgoff; 2524 return ret; 2525 2526 default: 2527 return -EINVAL; 2528 } 2529} 2530 2531static int hisi_qm_uacce_start_queue(struct uacce_queue *q) 2532{ 2533 struct hisi_qp *qp = q->priv; 2534 2535 return hisi_qm_start_qp(qp, qp->pasid); 2536} 2537 2538static void hisi_qm_uacce_stop_queue(struct uacce_queue *q) 2539{ 2540 struct hisi_qp *qp = q->priv; 2541 struct hisi_qm *qm = qp->qm; 2542 struct qm_dev_dfx *dev_dfx = &qm->debug.dev_dfx; 2543 u32 i = 0; 2544 2545 hisi_qm_stop_qp(qp); 2546 2547 if (!dev_dfx->dev_timeout || !dev_dfx->dev_state) 2548 return; 2549 2550 /* 2551 * After the queue fails to be stopped, 2552 * wait for a period of time before releasing the queue. 2553 */ 2554 while (++i) { 2555 msleep(WAIT_PERIOD); 2556 2557 /* Since dev_timeout maybe modified, check i >= dev_timeout */ 2558 if (i >= dev_dfx->dev_timeout) { 2559 dev_err(&qm->pdev->dev, "Stop q %u timeout, state %u\n", 2560 qp->qp_id, dev_dfx->dev_state); 2561 dev_dfx->dev_state = QM_FINISH_WAIT; 2562 break; 2563 } 2564 } 2565} 2566 2567static int hisi_qm_is_q_updated(struct uacce_queue *q) 2568{ 2569 struct hisi_qp *qp = q->priv; 2570 struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head; 2571 int updated = 0; 2572 2573 while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) { 2574 /* make sure to read data from memory */ 2575 dma_rmb(); 2576 qm_cq_head_update(qp); 2577 cqe = qp->cqe + qp->qp_status.cq_head; 2578 updated = 1; 2579 } 2580 2581 return updated; 2582} 2583 2584static void qm_set_sqctype(struct uacce_queue *q, u16 type) 2585{ 2586 struct hisi_qm *qm = q->uacce->priv; 2587 struct hisi_qp *qp = q->priv; 2588 2589 down_write(&qm->qps_lock); 2590 qp->alg_type = type; 2591 up_write(&qm->qps_lock); 2592} 2593 2594static long hisi_qm_uacce_ioctl(struct uacce_queue *q, unsigned int cmd, 2595 unsigned long arg) 2596{ 2597 struct hisi_qp *qp = q->priv; 2598 struct hisi_qp_info qp_info; 2599 struct hisi_qp_ctx qp_ctx; 2600 2601 if (cmd == UACCE_CMD_QM_SET_QP_CTX) { 2602 if (copy_from_user(&qp_ctx, (void __user *)arg, 2603 sizeof(struct hisi_qp_ctx))) 2604 return -EFAULT; 2605 2606 if (qp_ctx.qc_type > QM_MAX_QC_TYPE) 2607 return -EINVAL; 2608 2609 qm_set_sqctype(q, qp_ctx.qc_type); 2610 qp_ctx.id = qp->qp_id; 2611 2612 if (copy_to_user((void __user *)arg, &qp_ctx, 2613 sizeof(struct hisi_qp_ctx))) 2614 return -EFAULT; 2615 2616 return 0; 2617 } else if (cmd == UACCE_CMD_QM_SET_QP_INFO) { 2618 if (copy_from_user(&qp_info, (void __user *)arg, 2619 sizeof(struct hisi_qp_info))) 2620 return -EFAULT; 2621 2622 qp_info.sqe_size = qp->qm->sqe_size; 2623 qp_info.sq_depth = qp->sq_depth; 2624 qp_info.cq_depth = qp->cq_depth; 2625 2626 if (copy_to_user((void __user *)arg, &qp_info, 2627 sizeof(struct hisi_qp_info))) 2628 return -EFAULT; 2629 2630 return 0; 2631 } 2632 2633 return -EINVAL; 2634} 2635 2636/** 2637 * qm_hw_err_isolate() - Try to set the isolation status of the uacce device 2638 * according to user's configuration of error threshold. 2639 * @qm: the uacce device 2640 */ 2641static int qm_hw_err_isolate(struct hisi_qm *qm) 2642{ 2643 struct qm_hw_err *err, *tmp, *hw_err; 2644 struct qm_err_isolate *isolate; 2645 u32 count = 0; 2646 2647 isolate = &qm->isolate_data; 2648 2649#define SECONDS_PER_HOUR 3600 2650 2651 /* All the hw errs are processed by PF driver */ 2652 if (qm->uacce->is_vf || isolate->is_isolate || !isolate->err_threshold) 2653 return 0; 2654 2655 hw_err = kzalloc(sizeof(*hw_err), GFP_KERNEL); 2656 if (!hw_err) 2657 return -ENOMEM; 2658 2659 /* 2660 * Time-stamp every slot AER error. Then check the AER error log when the 2661 * next device AER error occurred. if the device slot AER error count exceeds 2662 * the setting error threshold in one hour, the isolated state will be set 2663 * to true. And the AER error logs that exceed one hour will be cleared. 2664 */ 2665 mutex_lock(&isolate->isolate_lock); 2666 hw_err->timestamp = jiffies; 2667 list_for_each_entry_safe(err, tmp, &isolate->qm_hw_errs, list) { 2668 if ((hw_err->timestamp - err->timestamp) / HZ > 2669 SECONDS_PER_HOUR) { 2670 list_del(&err->list); 2671 kfree(err); 2672 } else { 2673 count++; 2674 } 2675 } 2676 list_add(&hw_err->list, &isolate->qm_hw_errs); 2677 2678 if (count >= isolate->err_threshold) 2679 isolate->is_isolate = true; 2680 mutex_unlock(&isolate->isolate_lock); 2681 2682 return 0; 2683} 2684 2685static void qm_hw_err_destroy(struct hisi_qm *qm) 2686{ 2687 struct qm_hw_err *err, *tmp; 2688 2689 list_for_each_entry_safe(err, tmp, &qm->isolate_data.qm_hw_errs, list) { 2690 list_del(&err->list); 2691 kfree(err); 2692 } 2693} 2694 2695static enum uacce_dev_state hisi_qm_get_isolate_state(struct uacce_device *uacce) 2696{ 2697 struct hisi_qm *qm = uacce->priv; 2698 struct hisi_qm *pf_qm; 2699 2700 if (uacce->is_vf) 2701 pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); 2702 else 2703 pf_qm = qm; 2704 2705 return pf_qm->isolate_data.is_isolate ? 2706 UACCE_DEV_ISOLATE : UACCE_DEV_NORMAL; 2707} 2708 2709static int hisi_qm_isolate_threshold_write(struct uacce_device *uacce, u32 num) 2710{ 2711 struct hisi_qm *qm = uacce->priv; 2712 2713 /* Must be set by PF */ 2714 if (uacce->is_vf) 2715 return -EPERM; 2716 2717 if (qm->isolate_data.is_isolate) 2718 return -EPERM; 2719 2720 mutex_lock(&qm->isolate_data.isolate_lock); 2721 qm->isolate_data.err_threshold = num; 2722 2723 /* After the policy is updated, need to reset the hardware err list */ 2724 qm_hw_err_destroy(qm); 2725 mutex_unlock(&qm->isolate_data.isolate_lock); 2726 2727 return 0; 2728} 2729 2730static u32 hisi_qm_isolate_threshold_read(struct uacce_device *uacce) 2731{ 2732 struct hisi_qm *qm = uacce->priv; 2733 struct hisi_qm *pf_qm; 2734 2735 if (uacce->is_vf) { 2736 pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); 2737 return pf_qm->isolate_data.err_threshold; 2738 } 2739 2740 return qm->isolate_data.err_threshold; 2741} 2742 2743static const struct uacce_ops uacce_qm_ops = { 2744 .get_available_instances = hisi_qm_get_available_instances, 2745 .get_queue = hisi_qm_uacce_get_queue, 2746 .put_queue = hisi_qm_uacce_put_queue, 2747 .start_queue = hisi_qm_uacce_start_queue, 2748 .stop_queue = hisi_qm_uacce_stop_queue, 2749 .mmap = hisi_qm_uacce_mmap, 2750 .ioctl = hisi_qm_uacce_ioctl, 2751 .is_q_updated = hisi_qm_is_q_updated, 2752 .get_isolate_state = hisi_qm_get_isolate_state, 2753 .isolate_err_threshold_write = hisi_qm_isolate_threshold_write, 2754 .isolate_err_threshold_read = hisi_qm_isolate_threshold_read, 2755}; 2756 2757static void qm_remove_uacce(struct hisi_qm *qm) 2758{ 2759 struct uacce_device *uacce = qm->uacce; 2760 2761 if (qm->use_sva) { 2762 mutex_lock(&qm->isolate_data.isolate_lock); 2763 qm_hw_err_destroy(qm); 2764 mutex_unlock(&qm->isolate_data.isolate_lock); 2765 2766 uacce_remove(uacce); 2767 qm->uacce = NULL; 2768 } 2769} 2770 2771static void qm_uacce_api_ver_init(struct hisi_qm *qm) 2772{ 2773 struct uacce_device *uacce = qm->uacce; 2774 2775 switch (qm->ver) { 2776 case QM_HW_V1: 2777 uacce->api_ver = HISI_QM_API_VER_BASE; 2778 break; 2779 case QM_HW_V2: 2780 uacce->api_ver = HISI_QM_API_VER2_BASE; 2781 break; 2782 case QM_HW_V3: 2783 case QM_HW_V4: 2784 uacce->api_ver = HISI_QM_API_VER3_BASE; 2785 break; 2786 default: 2787 uacce->api_ver = HISI_QM_API_VER5_BASE; 2788 break; 2789 } 2790} 2791 2792static int qm_alloc_uacce(struct hisi_qm *qm) 2793{ 2794 struct pci_dev *pdev = qm->pdev; 2795 struct uacce_device *uacce; 2796 unsigned long mmio_page_nr; 2797 unsigned long dus_page_nr; 2798 u16 sq_depth, cq_depth; 2799 struct uacce_interface interface = { 2800 .flags = UACCE_DEV_SVA, 2801 .ops = &uacce_qm_ops, 2802 }; 2803 int ret; 2804 2805 ret = strscpy(interface.name, dev_driver_string(&pdev->dev), 2806 sizeof(interface.name)); 2807 if (ret < 0) 2808 return -ENAMETOOLONG; 2809 2810 uacce = uacce_alloc(&pdev->dev, &interface); 2811 if (IS_ERR(uacce)) 2812 return PTR_ERR(uacce); 2813 2814 if (uacce->flags & UACCE_DEV_SVA) { 2815 qm->use_sva = true; 2816 } else { 2817 /* only consider sva case */ 2818 qm_remove_uacce(qm); 2819 return -EINVAL; 2820 } 2821 2822 uacce->is_vf = pdev->is_virtfn; 2823 uacce->priv = qm; 2824 2825 if (qm->ver == QM_HW_V1) 2826 mmio_page_nr = QM_DOORBELL_PAGE_NR; 2827 else if (!test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) 2828 mmio_page_nr = QM_DOORBELL_PAGE_NR + 2829 QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE; 2830 else 2831 mmio_page_nr = qm->db_interval / PAGE_SIZE; 2832 2833 qm_get_xqc_depth(qm, &sq_depth, &cq_depth, QM_QP_DEPTH_CAP); 2834 2835 /* Add one more page for device or qp status */ 2836 dus_page_nr = (PAGE_SIZE - 1 + qm->sqe_size * sq_depth + 2837 sizeof(struct qm_cqe) * cq_depth + PAGE_SIZE) >> 2838 PAGE_SHIFT; 2839 2840 uacce->qf_pg_num[UACCE_QFRT_MMIO] = mmio_page_nr; 2841 uacce->qf_pg_num[UACCE_QFRT_DUS] = dus_page_nr; 2842 2843 qm->uacce = uacce; 2844 qm_uacce_api_ver_init(qm); 2845 INIT_LIST_HEAD(&qm->isolate_data.qm_hw_errs); 2846 mutex_init(&qm->isolate_data.isolate_lock); 2847 2848 return 0; 2849} 2850 2851/** 2852 * qm_frozen() - Try to froze QM to cut continuous queue request. If 2853 * there is user on the QM, return failure without doing anything. 2854 * @qm: The qm needed to be fronzen. 2855 * 2856 * This function frozes QM, then we can do SRIOV disabling. 2857 */ 2858static int qm_frozen(struct hisi_qm *qm) 2859{ 2860 if (test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl)) 2861 return 0; 2862 2863 down_write(&qm->qps_lock); 2864 2865 if (!qm->qp_in_used) { 2866 qm->qp_in_used = qm->qp_num; 2867 up_write(&qm->qps_lock); 2868 set_bit(QM_DRIVER_REMOVING, &qm->misc_ctl); 2869 return 0; 2870 } 2871 2872 up_write(&qm->qps_lock); 2873 2874 return -EBUSY; 2875} 2876 2877static int qm_try_frozen_vfs(struct pci_dev *pdev, 2878 struct hisi_qm_list *qm_list) 2879{ 2880 struct hisi_qm *qm, *vf_qm; 2881 struct pci_dev *dev; 2882 int ret = 0; 2883 2884 if (!qm_list || !pdev) 2885 return -EINVAL; 2886 2887 /* Try to frozen all the VFs as disable SRIOV */ 2888 mutex_lock(&qm_list->lock); 2889 list_for_each_entry(qm, &qm_list->list, list) { 2890 dev = qm->pdev; 2891 if (dev == pdev) 2892 continue; 2893 if (pci_physfn(dev) == pdev) { 2894 vf_qm = pci_get_drvdata(dev); 2895 ret = qm_frozen(vf_qm); 2896 if (ret) 2897 goto frozen_fail; 2898 } 2899 } 2900 2901frozen_fail: 2902 mutex_unlock(&qm_list->lock); 2903 2904 return ret; 2905} 2906 2907/** 2908 * hisi_qm_wait_task_finish() - Wait until the task is finished 2909 * when removing the driver. 2910 * @qm: The qm needed to wait for the task to finish. 2911 * @qm_list: The list of all available devices. 2912 */ 2913void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list) 2914{ 2915 while (qm_frozen(qm) || 2916 ((qm->fun_type == QM_HW_PF) && 2917 qm_try_frozen_vfs(qm->pdev, qm_list))) { 2918 msleep(WAIT_PERIOD); 2919 } 2920 2921 while (test_bit(QM_RST_SCHED, &qm->misc_ctl) || 2922 test_bit(QM_RESETTING, &qm->misc_ctl)) 2923 msleep(WAIT_PERIOD); 2924 2925 if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) 2926 flush_work(&qm->cmd_process); 2927 2928 udelay(REMOVE_WAIT_DELAY); 2929} 2930EXPORT_SYMBOL_GPL(hisi_qm_wait_task_finish); 2931 2932static void hisi_qp_memory_uninit(struct hisi_qm *qm, int num) 2933{ 2934 struct device *dev = &qm->pdev->dev; 2935 struct hisi_qp *qp; 2936 int i; 2937 2938 for (i = num - 1; i >= 0; i--) { 2939 qp = &qm->qp_array[i]; 2940 dma_free_coherent(dev, qp->qdma.size, qp->qdma.va, qp->qdma.dma); 2941 kfree(qp->msg); 2942 kfree(qm->poll_data[i].qp_finish_id); 2943 } 2944 2945 kfree(qm->poll_data); 2946 kfree(qm->qp_array); 2947} 2948 2949static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id, 2950 u16 sq_depth, u16 cq_depth) 2951{ 2952 struct device *dev = &qm->pdev->dev; 2953 size_t off = qm->sqe_size * sq_depth; 2954 struct hisi_qp *qp; 2955 int ret = -ENOMEM; 2956 2957 qm->poll_data[id].qp_finish_id = kcalloc(qm->qp_num, sizeof(u16), 2958 GFP_KERNEL); 2959 if (!qm->poll_data[id].qp_finish_id) 2960 return -ENOMEM; 2961 2962 qp = &qm->qp_array[id]; 2963 qp->msg = kmalloc_array(sq_depth, sizeof(void *), GFP_KERNEL); 2964 if (!qp->msg) 2965 goto err_free_qp_finish_id; 2966 2967 qp->qdma.va = dma_alloc_coherent(dev, dma_size, &qp->qdma.dma, 2968 GFP_KERNEL); 2969 if (!qp->qdma.va) 2970 goto err_free_qp_msg; 2971 2972 qp->sqe = qp->qdma.va; 2973 qp->sqe_dma = qp->qdma.dma; 2974 qp->cqe = qp->qdma.va + off; 2975 qp->cqe_dma = qp->qdma.dma + off; 2976 qp->qdma.size = dma_size; 2977 qp->sq_depth = sq_depth; 2978 qp->cq_depth = cq_depth; 2979 qp->qm = qm; 2980 qp->qp_id = id; 2981 2982 spin_lock_init(&qp->qp_lock); 2983 spin_lock_init(&qp->backlog.lock); 2984 INIT_LIST_HEAD(&qp->backlog.list); 2985 2986 return 0; 2987 2988err_free_qp_msg: 2989 kfree(qp->msg); 2990err_free_qp_finish_id: 2991 kfree(qm->poll_data[id].qp_finish_id); 2992 return ret; 2993} 2994 2995static void hisi_qm_pre_init(struct hisi_qm *qm) 2996{ 2997 struct pci_dev *pdev = qm->pdev; 2998 2999 if (qm->ver == QM_HW_V1) 3000 qm->ops = &qm_hw_ops_v1; 3001 else if (qm->ver == QM_HW_V2) 3002 qm->ops = &qm_hw_ops_v2; 3003 else if (qm->ver == QM_HW_V3) 3004 qm->ops = &qm_hw_ops_v3; 3005 else 3006 qm->ops = &qm_hw_ops_v4; 3007 3008 pci_set_drvdata(pdev, qm); 3009 mutex_init(&qm->mailbox_lock); 3010 mutex_init(&qm->ifc_lock); 3011 init_rwsem(&qm->qps_lock); 3012 qm->qp_in_used = 0; 3013 if (test_bit(QM_SUPPORT_RPM, &qm->caps)) { 3014 if (!acpi_device_power_manageable(ACPI_COMPANION(&pdev->dev))) 3015 dev_info(&pdev->dev, "_PS0 and _PR0 are not defined"); 3016 } 3017} 3018 3019static void qm_cmd_uninit(struct hisi_qm *qm) 3020{ 3021 u32 val; 3022 3023 if (!test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) 3024 return; 3025 3026 val = readl(qm->io_base + QM_IFC_INT_MASK); 3027 val |= QM_IFC_INT_DISABLE; 3028 writel(val, qm->io_base + QM_IFC_INT_MASK); 3029} 3030 3031static void qm_cmd_init(struct hisi_qm *qm) 3032{ 3033 u32 val; 3034 3035 if (!test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) 3036 return; 3037 3038 /* Clear communication interrupt source */ 3039 qm_clear_cmd_interrupt(qm, QM_IFC_INT_SOURCE_CLR); 3040 3041 /* Enable pf to vf communication reg. */ 3042 val = readl(qm->io_base + QM_IFC_INT_MASK); 3043 val &= ~QM_IFC_INT_DISABLE; 3044 writel(val, qm->io_base + QM_IFC_INT_MASK); 3045} 3046 3047static void qm_put_pci_res(struct hisi_qm *qm) 3048{ 3049 struct pci_dev *pdev = qm->pdev; 3050 3051 if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) 3052 iounmap(qm->db_io_base); 3053 3054 iounmap(qm->io_base); 3055 pci_release_mem_regions(pdev); 3056} 3057 3058static void hisi_mig_region_clear(struct hisi_qm *qm) 3059{ 3060 u32 val; 3061 3062 /* Clear migration region set of PF */ 3063 if (qm->fun_type == QM_HW_PF && qm->ver > QM_HW_V3) { 3064 val = readl(qm->io_base + QM_MIG_REGION_SEL); 3065 val &= ~QM_MIG_REGION_EN; 3066 writel(val, qm->io_base + QM_MIG_REGION_SEL); 3067 } 3068} 3069 3070static void hisi_mig_region_enable(struct hisi_qm *qm) 3071{ 3072 u32 val; 3073 3074 /* Select migration region of PF */ 3075 if (qm->fun_type == QM_HW_PF && qm->ver > QM_HW_V3) { 3076 val = readl(qm->io_base + QM_MIG_REGION_SEL); 3077 val |= QM_MIG_REGION_EN; 3078 writel(val, qm->io_base + QM_MIG_REGION_SEL); 3079 } 3080} 3081 3082static void hisi_qm_pci_uninit(struct hisi_qm *qm) 3083{ 3084 struct pci_dev *pdev = qm->pdev; 3085 3086 pci_free_irq_vectors(pdev); 3087 hisi_mig_region_clear(qm); 3088 qm_put_pci_res(qm); 3089 pci_disable_device(pdev); 3090} 3091 3092static void hisi_qm_set_state(struct hisi_qm *qm, u8 state) 3093{ 3094 if (qm->ver > QM_HW_V2 && qm->fun_type == QM_HW_VF) 3095 writel(state, qm->io_base + QM_VF_STATE); 3096} 3097 3098static void hisi_qm_unint_work(struct hisi_qm *qm) 3099{ 3100 destroy_workqueue(qm->wq); 3101} 3102 3103static void hisi_qm_free_rsv_buf(struct hisi_qm *qm) 3104{ 3105 struct qm_dma *xqc_dma = &qm->xqc_buf.qcdma; 3106 struct device *dev = &qm->pdev->dev; 3107 3108 dma_free_coherent(dev, xqc_dma->size, xqc_dma->va, xqc_dma->dma); 3109} 3110 3111static void hisi_qm_memory_uninit(struct hisi_qm *qm) 3112{ 3113 struct device *dev = &qm->pdev->dev; 3114 3115 hisi_qp_memory_uninit(qm, qm->qp_num); 3116 hisi_qm_free_rsv_buf(qm); 3117 if (qm->qdma.va) { 3118 hisi_qm_cache_wb(qm); 3119 dma_free_coherent(dev, qm->qdma.size, 3120 qm->qdma.va, qm->qdma.dma); 3121 } 3122 3123 idr_destroy(&qm->qp_idr); 3124 3125 if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) 3126 kfree(qm->factor); 3127} 3128 3129/** 3130 * hisi_qm_uninit() - Uninitialize qm. 3131 * @qm: The qm needed uninit. 3132 * 3133 * This function uninits qm related device resources. 3134 */ 3135void hisi_qm_uninit(struct hisi_qm *qm) 3136{ 3137 qm_cmd_uninit(qm); 3138 hisi_qm_unint_work(qm); 3139 3140 down_write(&qm->qps_lock); 3141 hisi_qm_memory_uninit(qm); 3142 hisi_qm_set_state(qm, QM_NOT_READY); 3143 up_write(&qm->qps_lock); 3144 3145 qm_remove_uacce(qm); 3146 qm_irqs_unregister(qm); 3147 hisi_qm_pci_uninit(qm); 3148} 3149EXPORT_SYMBOL_GPL(hisi_qm_uninit); 3150 3151/** 3152 * hisi_qm_get_vft() - Get vft from a qm. 3153 * @qm: The qm we want to get its vft. 3154 * @base: The base number of queue in vft. 3155 * @number: The number of queues in vft. 3156 * 3157 * We can allocate multiple queues to a qm by configuring virtual function 3158 * table. We get related configures by this function. Normally, we call this 3159 * function in VF driver to get the queue information. 3160 * 3161 * qm hw v1 does not support this interface. 3162 */ 3163static int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number) 3164{ 3165 if (!base || !number) 3166 return -EINVAL; 3167 3168 if (!qm->ops->get_vft) { 3169 dev_err(&qm->pdev->dev, "Don't support vft read!\n"); 3170 return -EINVAL; 3171 } 3172 3173 return qm->ops->get_vft(qm, base, number); 3174} 3175 3176/** 3177 * hisi_qm_set_vft() - Set vft to a qm. 3178 * @qm: The qm we want to set its vft. 3179 * @fun_num: The function number. 3180 * @base: The base number of queue in vft. 3181 * @number: The number of queues in vft. 3182 * 3183 * This function is alway called in PF driver, it is used to assign queues 3184 * among PF and VFs. 3185 * 3186 * Assign queues A~B to PF: hisi_qm_set_vft(qm, 0, A, B - A + 1) 3187 * Assign queues A~B to VF: hisi_qm_set_vft(qm, 2, A, B - A + 1) 3188 * (VF function number 0x2) 3189 */ 3190static int hisi_qm_set_vft(struct hisi_qm *qm, u32 fun_num, u32 base, 3191 u32 number) 3192{ 3193 u32 max_q_num = qm->ctrl_qp_num; 3194 3195 if (base >= max_q_num || number > max_q_num || 3196 (base + number) > max_q_num) 3197 return -EINVAL; 3198 3199 return qm_set_sqc_cqc_vft(qm, fun_num, base, number); 3200} 3201 3202static void qm_init_eq_aeq_status(struct hisi_qm *qm) 3203{ 3204 struct hisi_qm_status *status = &qm->status; 3205 3206 status->eq_head = 0; 3207 status->aeq_head = 0; 3208 status->eqc_phase = true; 3209 status->aeqc_phase = true; 3210} 3211 3212static void qm_enable_eq_aeq_interrupts(struct hisi_qm *qm) 3213{ 3214 /* Clear eq/aeq interrupt source */ 3215 qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0); 3216 qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); 3217 3218 writel(0x0, qm->io_base + QM_VF_EQ_INT_MASK); 3219 writel(0x0, qm->io_base + QM_VF_AEQ_INT_MASK); 3220} 3221 3222static void qm_disable_eq_aeq_interrupts(struct hisi_qm *qm) 3223{ 3224 writel(0x1, qm->io_base + QM_VF_EQ_INT_MASK); 3225 writel(0x1, qm->io_base + QM_VF_AEQ_INT_MASK); 3226} 3227 3228static int qm_eq_ctx_cfg(struct hisi_qm *qm) 3229{ 3230 struct qm_eqc eqc = {0}; 3231 3232 eqc.base_l = cpu_to_le32(lower_32_bits(qm->eqe_dma)); 3233 eqc.base_h = cpu_to_le32(upper_32_bits(qm->eqe_dma)); 3234 if (qm->ver == QM_HW_V1) 3235 eqc.dw3 = cpu_to_le32(QM_EQE_AEQE_SIZE); 3236 eqc.dw6 = cpu_to_le32(((u32)qm->eq_depth - 1) | (1 << QM_EQC_PHASE_SHIFT)); 3237 3238 return qm_set_and_get_xqc(qm, QM_MB_CMD_EQC, &eqc, 0, 0); 3239} 3240 3241static int qm_aeq_ctx_cfg(struct hisi_qm *qm) 3242{ 3243 struct qm_aeqc aeqc = {0}; 3244 3245 aeqc.base_l = cpu_to_le32(lower_32_bits(qm->aeqe_dma)); 3246 aeqc.base_h = cpu_to_le32(upper_32_bits(qm->aeqe_dma)); 3247 aeqc.dw6 = cpu_to_le32(((u32)qm->aeq_depth - 1) | (1 << QM_EQC_PHASE_SHIFT)); 3248 3249 return qm_set_and_get_xqc(qm, QM_MB_CMD_AEQC, &aeqc, 0, 0); 3250} 3251 3252static int qm_eq_aeq_ctx_cfg(struct hisi_qm *qm) 3253{ 3254 struct device *dev = &qm->pdev->dev; 3255 int ret; 3256 3257 qm_init_eq_aeq_status(qm); 3258 3259 /* Before starting the dev, clear the memory and then configure to device using. */ 3260 memset(qm->qdma.va, 0, qm->qdma.size); 3261 3262 ret = qm_eq_ctx_cfg(qm); 3263 if (ret) { 3264 dev_err(dev, "Set eqc failed!\n"); 3265 return ret; 3266 } 3267 3268 return qm_aeq_ctx_cfg(qm); 3269} 3270 3271static int __hisi_qm_start(struct hisi_qm *qm) 3272{ 3273 struct device *dev = &qm->pdev->dev; 3274 int ret; 3275 3276 if (!qm->qdma.va) { 3277 dev_err(dev, "qm qdma is NULL!\n"); 3278 return -EINVAL; 3279 } 3280 3281 if (qm->fun_type == QM_HW_PF) { 3282 ret = hisi_qm_set_vft(qm, 0, qm->qp_base, qm->qp_num); 3283 if (ret) 3284 return ret; 3285 } 3286 3287 ret = qm_eq_aeq_ctx_cfg(qm); 3288 if (ret) 3289 return ret; 3290 3291 ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, 0); 3292 if (ret) 3293 return ret; 3294 3295 ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, qm->cqc_dma, 0, 0); 3296 if (ret) 3297 return ret; 3298 3299 qm_init_prefetch(qm); 3300 qm_enable_eq_aeq_interrupts(qm); 3301 3302 return 0; 3303} 3304 3305/** 3306 * hisi_qm_start() - start qm 3307 * @qm: The qm to be started. 3308 * 3309 * This function starts a qm, then we can allocate qp from this qm. 3310 */ 3311int hisi_qm_start(struct hisi_qm *qm) 3312{ 3313 struct device *dev = &qm->pdev->dev; 3314 int ret = 0; 3315 3316 down_write(&qm->qps_lock); 3317 3318 dev_dbg(dev, "qm start with %u queue pairs\n", qm->qp_num); 3319 3320 if (!qm->qp_num) { 3321 dev_err(dev, "qp_num should not be 0\n"); 3322 ret = -EINVAL; 3323 goto err_unlock; 3324 } 3325 3326 ret = __hisi_qm_start(qm); 3327 if (ret) 3328 goto err_unlock; 3329 3330 atomic_set(&qm->status.flags, QM_WORK); 3331 hisi_qm_set_state(qm, QM_READY); 3332 3333err_unlock: 3334 up_write(&qm->qps_lock); 3335 return ret; 3336} 3337EXPORT_SYMBOL_GPL(hisi_qm_start); 3338 3339static int qm_restart(struct hisi_qm *qm) 3340{ 3341 struct device *dev = &qm->pdev->dev; 3342 struct hisi_qp *qp; 3343 int ret, i; 3344 3345 ret = hisi_qm_start(qm); 3346 if (ret < 0) 3347 return ret; 3348 3349 down_write(&qm->qps_lock); 3350 for (i = 0; i < qm->qp_num; i++) { 3351 qp = &qm->qp_array[i]; 3352 if (atomic_read(&qp->qp_status.flags) == QP_STOP && 3353 qp->is_resetting == true && qp->is_in_kernel == true) { 3354 ret = qm_start_qp_nolock(qp, 0); 3355 if (ret < 0) { 3356 dev_err(dev, "Failed to start qp%d!\n", i); 3357 3358 up_write(&qm->qps_lock); 3359 return ret; 3360 } 3361 qp->is_resetting = false; 3362 } 3363 } 3364 up_write(&qm->qps_lock); 3365 3366 return 0; 3367} 3368 3369/* Stop started qps in reset flow */ 3370static void qm_stop_started_qp(struct hisi_qm *qm) 3371{ 3372 struct hisi_qp *qp; 3373 int i; 3374 3375 for (i = 0; i < qm->qp_num; i++) { 3376 qp = &qm->qp_array[i]; 3377 if (atomic_read(&qp->qp_status.flags) == QP_START) { 3378 qp->is_resetting = true; 3379 qm_stop_qp_nolock(qp); 3380 } 3381 } 3382} 3383 3384/** 3385 * qm_invalid_queues() - invalid all queues in use. 3386 * @qm: The qm in which the queues will be invalidated. 3387 * 3388 * This function invalid all queues in use. If the doorbell command is sent 3389 * to device in user space after the device is reset, the device discards 3390 * the doorbell command. 3391 */ 3392static void qm_invalid_queues(struct hisi_qm *qm) 3393{ 3394 struct hisi_qp *qp; 3395 struct qm_sqc *sqc; 3396 struct qm_cqc *cqc; 3397 int i; 3398 3399 /* 3400 * Normal stop queues is no longer used and does not need to be 3401 * invalid queues. 3402 */ 3403 if (qm->status.stop_reason == QM_NORMAL) 3404 return; 3405 3406 if (qm->status.stop_reason == QM_DOWN) 3407 hisi_qm_cache_wb(qm); 3408 3409 for (i = 0; i < qm->qp_num; i++) { 3410 qp = &qm->qp_array[i]; 3411 if (!qp->is_resetting) 3412 continue; 3413 3414 /* Modify random data and set sqc close bit to invalid queue. */ 3415 sqc = qm->sqc + i; 3416 cqc = qm->cqc + i; 3417 sqc->w8 = cpu_to_le16(QM_XQC_RANDOM_DATA); 3418 sqc->w13 = cpu_to_le16(QM_SQC_DISABLE_QP); 3419 cqc->w8 = cpu_to_le16(QM_XQC_RANDOM_DATA); 3420 if (qp->is_in_kernel) 3421 memset(qp->qdma.va, 0, qp->qdma.size); 3422 } 3423} 3424 3425/** 3426 * hisi_qm_stop() - Stop a qm. 3427 * @qm: The qm which will be stopped. 3428 * @r: The reason to stop qm. 3429 * 3430 * This function stops qm and its qps, then qm can not accept request. 3431 * Related resources are not released at this state, we can use hisi_qm_start 3432 * to let qm start again. 3433 */ 3434int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r) 3435{ 3436 struct device *dev = &qm->pdev->dev; 3437 int ret = 0; 3438 3439 down_write(&qm->qps_lock); 3440 3441 if (atomic_read(&qm->status.flags) == QM_STOP) 3442 goto err_unlock; 3443 3444 /* Stop all the request sending at first. */ 3445 atomic_set(&qm->status.flags, QM_STOP); 3446 qm->status.stop_reason = r; 3447 3448 if (qm->status.stop_reason != QM_NORMAL) { 3449 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET); 3450 /* 3451 * When performing soft reset, the hardware will no longer 3452 * do tasks, and the tasks in the device will be flushed 3453 * out directly since the master ooo is closed. 3454 */ 3455 if (test_bit(QM_SUPPORT_STOP_FUNC, &qm->caps) && 3456 r != QM_SOFT_RESET) { 3457 ret = qm_drain_qm(qm); 3458 if (ret) { 3459 dev_err(dev, "failed to drain qm!\n"); 3460 goto err_unlock; 3461 } 3462 } 3463 3464 qm_stop_started_qp(qm); 3465 3466 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); 3467 } 3468 3469 qm_disable_eq_aeq_interrupts(qm); 3470 if (qm->fun_type == QM_HW_PF) { 3471 ret = hisi_qm_set_vft(qm, 0, 0, 0); 3472 if (ret < 0) { 3473 dev_err(dev, "Failed to set vft!\n"); 3474 ret = -EBUSY; 3475 goto err_unlock; 3476 } 3477 } 3478 3479 qm_invalid_queues(qm); 3480 qm->status.stop_reason = QM_NORMAL; 3481 3482err_unlock: 3483 up_write(&qm->qps_lock); 3484 return ret; 3485} 3486EXPORT_SYMBOL_GPL(hisi_qm_stop); 3487 3488static void qm_hw_error_init(struct hisi_qm *qm) 3489{ 3490 if (!qm->ops->hw_error_init) { 3491 dev_err(&qm->pdev->dev, "QM doesn't support hw error handling!\n"); 3492 return; 3493 } 3494 3495 qm->ops->hw_error_init(qm); 3496} 3497 3498static void qm_hw_error_uninit(struct hisi_qm *qm) 3499{ 3500 if (!qm->ops->hw_error_uninit) { 3501 dev_err(&qm->pdev->dev, "Unexpected QM hw error uninit!\n"); 3502 return; 3503 } 3504 3505 qm->ops->hw_error_uninit(qm); 3506} 3507 3508static enum acc_err_result qm_hw_error_handle(struct hisi_qm *qm) 3509{ 3510 if (!qm->ops->hw_error_handle) { 3511 dev_err(&qm->pdev->dev, "QM doesn't support hw error report!\n"); 3512 return ACC_ERR_NONE; 3513 } 3514 3515 return qm->ops->hw_error_handle(qm); 3516} 3517 3518/** 3519 * hisi_qm_dev_err_init() - Initialize device error configuration. 3520 * @qm: The qm for which we want to do error initialization. 3521 * 3522 * Initialize QM and device error related configuration. 3523 */ 3524void hisi_qm_dev_err_init(struct hisi_qm *qm) 3525{ 3526 if (qm->fun_type == QM_HW_VF) 3527 return; 3528 3529 qm_hw_error_init(qm); 3530 3531 if (!qm->err_ini->hw_err_enable) { 3532 dev_err(&qm->pdev->dev, "Device doesn't support hw error init!\n"); 3533 return; 3534 } 3535 qm->err_ini->hw_err_enable(qm); 3536} 3537EXPORT_SYMBOL_GPL(hisi_qm_dev_err_init); 3538 3539/** 3540 * hisi_qm_dev_err_uninit() - Uninitialize device error configuration. 3541 * @qm: The qm for which we want to do error uninitialization. 3542 * 3543 * Uninitialize QM and device error related configuration. 3544 */ 3545void hisi_qm_dev_err_uninit(struct hisi_qm *qm) 3546{ 3547 if (qm->fun_type == QM_HW_VF) 3548 return; 3549 3550 qm_hw_error_uninit(qm); 3551 3552 if (!qm->err_ini->hw_err_disable) { 3553 dev_err(&qm->pdev->dev, "Unexpected device hw error uninit!\n"); 3554 return; 3555 } 3556 qm->err_ini->hw_err_disable(qm); 3557} 3558EXPORT_SYMBOL_GPL(hisi_qm_dev_err_uninit); 3559 3560static void qm_release_qp_nolock(struct hisi_qp *qp) 3561{ 3562 struct hisi_qm *qm = qp->qm; 3563 3564 qm->qp_in_used--; 3565 idr_remove(&qm->qp_idr, qp->qp_id); 3566} 3567 3568/** 3569 * hisi_qm_free_qps() - free multiple queue pairs. 3570 * @qps: The queue pairs need to be freed. 3571 * @qp_num: The num of queue pairs. 3572 */ 3573void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num) 3574{ 3575 int i; 3576 3577 if (!qps || qp_num <= 0) 3578 return; 3579 3580 down_write(&qps[0]->qm->qps_lock); 3581 3582 for (i = qp_num - 1; i >= 0; i--) { 3583 qm_stop_qp_nolock(qps[i]); 3584 qm_release_qp_nolock(qps[i]); 3585 } 3586 3587 up_write(&qps[0]->qm->qps_lock); 3588 qm_pm_put_sync(qps[0]->qm); 3589} 3590EXPORT_SYMBOL_GPL(hisi_qm_free_qps); 3591 3592static void free_list(struct list_head *head) 3593{ 3594 struct hisi_qm_resource *res, *tmp; 3595 3596 list_for_each_entry_safe(res, tmp, head, list) { 3597 list_del(&res->list); 3598 kfree(res); 3599 } 3600} 3601 3602static int qm_get_and_start_qp(struct hisi_qm *qm, int qp_num, struct hisi_qp **qps, u8 *alg_type) 3603{ 3604 int i, ret; 3605 3606 ret = qm_pm_get_sync(qm); 3607 if (ret) 3608 return ret; 3609 3610 down_write(&qm->qps_lock); 3611 for (i = 0; i < qp_num; i++) { 3612 qps[i] = qm_create_qp_nolock(qm, alg_type[i]); 3613 if (IS_ERR(qps[i])) { 3614 ret = -ENODEV; 3615 goto stop_and_free; 3616 } 3617 3618 ret = qm_start_qp_nolock(qps[i], 0); 3619 if (ret) { 3620 qm_release_qp_nolock(qps[i]); 3621 goto stop_and_free; 3622 } 3623 } 3624 up_write(&qm->qps_lock); 3625 3626 return 0; 3627 3628stop_and_free: 3629 for (i--; i >= 0; i--) { 3630 qm_stop_qp_nolock(qps[i]); 3631 qm_release_qp_nolock(qps[i]); 3632 } 3633 up_write(&qm->qps_lock); 3634 qm_pm_put_sync(qm); 3635 3636 return ret; 3637} 3638 3639static int hisi_qm_sort_devices(int node, struct list_head *head, 3640 struct hisi_qm_list *qm_list) 3641{ 3642 struct hisi_qm_resource *res, *tmp; 3643 struct hisi_qm *qm; 3644 struct list_head *n; 3645 struct device *dev; 3646 int dev_node; 3647 3648 list_for_each_entry(qm, &qm_list->list, list) { 3649 dev = &qm->pdev->dev; 3650 3651 dev_node = dev_to_node(dev); 3652 if (dev_node < 0) 3653 dev_node = 0; 3654 3655 res = kzalloc(sizeof(*res), GFP_KERNEL); 3656 if (!res) 3657 return -ENOMEM; 3658 3659 res->qm = qm; 3660 res->distance = node_distance(dev_node, node); 3661 n = head; 3662 list_for_each_entry(tmp, head, list) { 3663 if (res->distance < tmp->distance) { 3664 n = &tmp->list; 3665 break; 3666 } 3667 } 3668 list_add_tail(&res->list, n); 3669 } 3670 3671 return 0; 3672} 3673 3674/** 3675 * hisi_qm_alloc_qps_node() - Create multiple queue pairs. 3676 * @qm_list: The list of all available devices. 3677 * @qp_num: The number of queue pairs need created. 3678 * @alg_type: The algorithm type. 3679 * @node: The numa node. 3680 * @qps: The queue pairs need created. 3681 * 3682 * This function will sort all available device according to numa distance. 3683 * Then try to create all queue pairs from one device, if all devices do 3684 * not meet the requirements will return error. 3685 */ 3686int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num, 3687 u8 *alg_type, int node, struct hisi_qp **qps) 3688{ 3689 struct hisi_qm_resource *tmp; 3690 int ret = -ENODEV; 3691 LIST_HEAD(head); 3692 3693 if (!qps || !qm_list || qp_num <= 0) 3694 return -EINVAL; 3695 3696 mutex_lock(&qm_list->lock); 3697 if (hisi_qm_sort_devices(node, &head, qm_list)) { 3698 mutex_unlock(&qm_list->lock); 3699 goto err; 3700 } 3701 3702 list_for_each_entry(tmp, &head, list) { 3703 ret = qm_get_and_start_qp(tmp->qm, qp_num, qps, alg_type); 3704 if (!ret) 3705 break; 3706 } 3707 3708 mutex_unlock(&qm_list->lock); 3709 if (ret) 3710 pr_info("Failed to create qps, node[%d], qp[%d]!\n", 3711 node, qp_num); 3712 3713err: 3714 free_list(&head); 3715 return ret; 3716} 3717EXPORT_SYMBOL_GPL(hisi_qm_alloc_qps_node); 3718 3719static int qm_vf_q_assign(struct hisi_qm *qm, u32 num_vfs) 3720{ 3721 u32 remain_q_num, vfs_q_num, act_q_num, q_num, i, j; 3722 u32 max_qp_num = qm->max_qp_num; 3723 u32 q_base = qm->qp_num; 3724 int ret; 3725 3726 if (!num_vfs) 3727 return -EINVAL; 3728 3729 vfs_q_num = qm->ctrl_qp_num - qm->qp_num; 3730 3731 /* If vfs_q_num is less than num_vfs, return error. */ 3732 if (vfs_q_num < num_vfs) 3733 return -EINVAL; 3734 3735 q_num = vfs_q_num / num_vfs; 3736 remain_q_num = vfs_q_num % num_vfs; 3737 3738 for (i = num_vfs; i > 0; i--) { 3739 /* 3740 * if q_num + remain_q_num > max_qp_num in last vf, divide the 3741 * remaining queues equally. 3742 */ 3743 if (i == num_vfs && q_num + remain_q_num <= max_qp_num) { 3744 act_q_num = q_num + remain_q_num; 3745 remain_q_num = 0; 3746 } else if (remain_q_num > 0) { 3747 act_q_num = q_num + 1; 3748 remain_q_num--; 3749 } else { 3750 act_q_num = q_num; 3751 } 3752 3753 act_q_num = min(act_q_num, max_qp_num); 3754 ret = hisi_qm_set_vft(qm, i, q_base, act_q_num); 3755 if (ret) { 3756 for (j = num_vfs; j > i; j--) 3757 hisi_qm_set_vft(qm, j, 0, 0); 3758 return ret; 3759 } 3760 q_base += act_q_num; 3761 } 3762 3763 return 0; 3764} 3765 3766static void qm_clear_vft_config(struct hisi_qm *qm) 3767{ 3768 u32 i; 3769 3770 /* 3771 * When disabling SR-IOV, clear the configuration of each VF in the hardware 3772 * sequentially. Failure to clear a single VF should not affect the clearing 3773 * operation of other VFs. 3774 */ 3775 for (i = 1; i <= qm->vfs_num; i++) 3776 (void)hisi_qm_set_vft(qm, i, 0, 0); 3777 3778 qm->vfs_num = 0; 3779} 3780 3781static int qm_func_shaper_enable(struct hisi_qm *qm, u32 fun_index, u32 qos) 3782{ 3783 struct device *dev = &qm->pdev->dev; 3784 struct qm_shaper_factor t_factor; 3785 u32 ir = qos * QM_QOS_RATE; 3786 int ret, total_vfs, i; 3787 3788 total_vfs = pci_sriov_get_totalvfs(qm->pdev); 3789 if (fun_index > total_vfs) 3790 return -EINVAL; 3791 3792 memcpy(&t_factor, &qm->factor[fun_index], sizeof(t_factor)); 3793 qm->factor[fun_index].func_qos = qos; 3794 3795 ret = qm_get_shaper_para(ir, &qm->factor[fun_index]); 3796 if (ret) { 3797 dev_err(dev, "failed to calculate shaper parameter!\n"); 3798 return -EINVAL; 3799 } 3800 3801 for (i = ALG_TYPE_0; i <= ALG_TYPE_1; i++) { 3802 /* The base number of queue reuse for different alg type */ 3803 ret = qm_set_vft_common(qm, SHAPER_VFT, fun_index, i, 1); 3804 if (ret) { 3805 dev_err(dev, "type: %d, failed to set shaper vft!\n", i); 3806 goto back_func_qos; 3807 } 3808 } 3809 3810 return 0; 3811 3812back_func_qos: 3813 memcpy(&qm->factor[fun_index], &t_factor, sizeof(t_factor)); 3814 for (i--; i >= ALG_TYPE_0; i--) { 3815 ret = qm_set_vft_common(qm, SHAPER_VFT, fun_index, i, 1); 3816 if (ret) 3817 dev_err(dev, "failed to restore shaper vft during rollback!\n"); 3818 } 3819 3820 return -EINVAL; 3821} 3822 3823static u32 qm_get_shaper_vft_qos(struct hisi_qm *qm, u32 fun_index) 3824{ 3825 u64 cir_u = 0, cir_b = 0, cir_s = 0; 3826 u64 shaper_vft, ir_calc, ir; 3827 unsigned int val; 3828 u32 error_rate; 3829 int ret; 3830 3831 ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, 3832 val & BIT(0), POLL_PERIOD, 3833 POLL_TIMEOUT); 3834 if (ret) 3835 return 0; 3836 3837 writel(0x1, qm->io_base + QM_VFT_CFG_OP_WR); 3838 writel(SHAPER_VFT, qm->io_base + QM_VFT_CFG_TYPE); 3839 writel(fun_index, qm->io_base + QM_VFT_CFG); 3840 3841 writel(0x0, qm->io_base + QM_VFT_CFG_RDY); 3842 writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE); 3843 3844 ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, 3845 val & BIT(0), POLL_PERIOD, 3846 POLL_TIMEOUT); 3847 if (ret) 3848 return 0; 3849 3850 shaper_vft = readl(qm->io_base + QM_VFT_CFG_DATA_L) | 3851 ((u64)readl(qm->io_base + QM_VFT_CFG_DATA_H) << 32); 3852 3853 cir_b = shaper_vft & QM_SHAPER_CIR_B_MASK; 3854 cir_u = shaper_vft & QM_SHAPER_CIR_U_MASK; 3855 cir_u = cir_u >> QM_SHAPER_FACTOR_CIR_U_SHIFT; 3856 3857 cir_s = shaper_vft & QM_SHAPER_CIR_S_MASK; 3858 cir_s = cir_s >> QM_SHAPER_FACTOR_CIR_S_SHIFT; 3859 3860 ir_calc = acc_shaper_para_calc(cir_b, cir_u, cir_s); 3861 3862 ir = qm->factor[fun_index].func_qos * QM_QOS_RATE; 3863 3864 error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir; 3865 if (error_rate > QM_QOS_MIN_ERROR_RATE) { 3866 pci_err(qm->pdev, "error_rate: %u, get function qos is error!\n", error_rate); 3867 return 0; 3868 } 3869 3870 return ir; 3871} 3872 3873static void qm_vf_get_qos(struct hisi_qm *qm, u32 fun_num) 3874{ 3875 struct device *dev = &qm->pdev->dev; 3876 u32 qos; 3877 int ret; 3878 3879 qos = qm_get_shaper_vft_qos(qm, fun_num); 3880 if (!qos) { 3881 dev_err(dev, "function(%u) failed to get qos by PF!\n", fun_num); 3882 return; 3883 } 3884 3885 ret = qm_ping_single_vf(qm, QM_PF_SET_QOS, qos, fun_num); 3886 if (ret) 3887 dev_err(dev, "failed to send command(0x%x) to VF(%u)!\n", QM_PF_SET_QOS, fun_num); 3888} 3889 3890static int qm_vf_read_qos(struct hisi_qm *qm) 3891{ 3892 int cnt = 0; 3893 int ret = -EINVAL; 3894 3895 /* reset mailbox qos val */ 3896 qm->mb_qos = 0; 3897 3898 /* vf ping pf to get function qos */ 3899 ret = qm_ping_pf(qm, QM_VF_GET_QOS); 3900 if (ret) { 3901 pci_err(qm->pdev, "failed to send cmd to PF to get qos!\n"); 3902 return ret; 3903 } 3904 3905 while (true) { 3906 msleep(QM_WAIT_DST_ACK); 3907 if (qm->mb_qos) 3908 break; 3909 3910 if (++cnt > QM_MAX_VF_WAIT_COUNT) { 3911 pci_err(qm->pdev, "PF ping VF timeout!\n"); 3912 return -ETIMEDOUT; 3913 } 3914 } 3915 3916 return ret; 3917} 3918 3919static ssize_t qm_algqos_read(struct file *filp, char __user *buf, 3920 size_t count, loff_t *pos) 3921{ 3922 struct hisi_qm *qm = filp->private_data; 3923 char tbuf[QM_DBG_READ_LEN]; 3924 u32 qos_val, ir; 3925 int ret; 3926 3927 ret = hisi_qm_get_dfx_access(qm); 3928 if (ret) 3929 return ret; 3930 3931 /* Mailbox and reset cannot be operated at the same time */ 3932 if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) { 3933 pci_err(qm->pdev, "dev resetting, read alg qos failed!\n"); 3934 ret = -EAGAIN; 3935 goto err_put_dfx_access; 3936 } 3937 3938 if (qm->fun_type == QM_HW_PF) { 3939 ir = qm_get_shaper_vft_qos(qm, 0); 3940 } else { 3941 ret = qm_vf_read_qos(qm); 3942 if (ret) 3943 goto err_get_status; 3944 ir = qm->mb_qos; 3945 } 3946 3947 qos_val = ir / QM_QOS_RATE; 3948 ret = scnprintf(tbuf, QM_DBG_READ_LEN, "%u\n", qos_val); 3949 3950 ret = simple_read_from_buffer(buf, count, pos, tbuf, ret); 3951 3952err_get_status: 3953 clear_bit(QM_RESETTING, &qm->misc_ctl); 3954err_put_dfx_access: 3955 hisi_qm_put_dfx_access(qm); 3956 return ret; 3957} 3958 3959static ssize_t qm_get_qos_value(struct hisi_qm *qm, const char *buf, 3960 unsigned long *val, 3961 unsigned int *fun_index) 3962{ 3963 const struct bus_type *bus_type = qm->pdev->dev.bus; 3964 char tbuf_bdf[QM_DBG_READ_LEN] = {0}; 3965 char val_buf[QM_DBG_READ_LEN] = {0}; 3966 struct pci_dev *pdev; 3967 struct device *dev; 3968 int ret; 3969 3970 ret = sscanf(buf, "%s %s", tbuf_bdf, val_buf); 3971 if (ret != QM_QOS_PARAM_NUM) 3972 return -EINVAL; 3973 3974 ret = kstrtoul(val_buf, 10, val); 3975 if (ret || *val == 0 || *val > QM_QOS_MAX_VAL) { 3976 pci_err(qm->pdev, "input qos value is error, please set 1~1000!\n"); 3977 return -EINVAL; 3978 } 3979 3980 dev = bus_find_device_by_name(bus_type, NULL, tbuf_bdf); 3981 if (!dev) { 3982 pci_err(qm->pdev, "input pci bdf number is error!\n"); 3983 return -ENODEV; 3984 } 3985 3986 pdev = container_of(dev, struct pci_dev, dev); 3987 if (pci_physfn(pdev) != qm->pdev) { 3988 pci_err(qm->pdev, "the pdev input does not match the pf!\n"); 3989 put_device(dev); 3990 return -EINVAL; 3991 } 3992 3993 *fun_index = pdev->devfn; 3994 put_device(dev); 3995 3996 return 0; 3997} 3998 3999static ssize_t qm_algqos_write(struct file *filp, const char __user *buf, 4000 size_t count, loff_t *pos) 4001{ 4002 struct hisi_qm *qm = filp->private_data; 4003 char tbuf[QM_DBG_READ_LEN]; 4004 unsigned int fun_index; 4005 unsigned long val; 4006 int len, ret; 4007 4008 if (*pos != 0) 4009 return 0; 4010 4011 if (count >= QM_DBG_READ_LEN) 4012 return -ENOSPC; 4013 4014 len = simple_write_to_buffer(tbuf, QM_DBG_READ_LEN - 1, pos, buf, count); 4015 if (len < 0) 4016 return len; 4017 4018 tbuf[len] = '\0'; 4019 ret = qm_get_qos_value(qm, tbuf, &val, &fun_index); 4020 if (ret) 4021 return ret; 4022 4023 /* Mailbox and reset cannot be operated at the same time */ 4024 if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) { 4025 pci_err(qm->pdev, "dev resetting, write alg qos failed!\n"); 4026 return -EAGAIN; 4027 } 4028 4029 ret = qm_pm_get_sync(qm); 4030 if (ret) { 4031 ret = -EINVAL; 4032 goto err_get_status; 4033 } 4034 4035 ret = qm_func_shaper_enable(qm, fun_index, val); 4036 if (ret) { 4037 pci_err(qm->pdev, "failed to enable function shaper!\n"); 4038 ret = -EINVAL; 4039 goto err_put_sync; 4040 } 4041 4042 pci_info(qm->pdev, "the qos value of function%u is set to %lu.\n", 4043 fun_index, val); 4044 ret = count; 4045 4046err_put_sync: 4047 qm_pm_put_sync(qm); 4048err_get_status: 4049 clear_bit(QM_RESETTING, &qm->misc_ctl); 4050 return ret; 4051} 4052 4053static const struct file_operations qm_algqos_fops = { 4054 .owner = THIS_MODULE, 4055 .open = simple_open, 4056 .read = qm_algqos_read, 4057 .write = qm_algqos_write, 4058}; 4059 4060/** 4061 * hisi_qm_set_algqos_init() - Initialize function qos debugfs files. 4062 * @qm: The qm for which we want to add debugfs files. 4063 * 4064 * Create function qos debugfs files, VF ping PF to get function qos. 4065 */ 4066void hisi_qm_set_algqos_init(struct hisi_qm *qm) 4067{ 4068 if (qm->fun_type == QM_HW_PF) 4069 debugfs_create_file("alg_qos", 0644, qm->debug.debug_root, 4070 qm, &qm_algqos_fops); 4071 else if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) 4072 debugfs_create_file("alg_qos", 0444, qm->debug.debug_root, 4073 qm, &qm_algqos_fops); 4074} 4075 4076static void hisi_qm_init_vf_qos(struct hisi_qm *qm, int total_func) 4077{ 4078 int i; 4079 4080 for (i = 1; i <= total_func; i++) 4081 qm->factor[i].func_qos = QM_QOS_MAX_VAL; 4082} 4083 4084/** 4085 * hisi_qm_sriov_enable() - enable virtual functions 4086 * @pdev: the PCIe device 4087 * @max_vfs: the number of virtual functions to enable 4088 * 4089 * Returns the number of enabled VFs. If there are VFs enabled already or 4090 * max_vfs is more than the total number of device can be enabled, returns 4091 * failure. 4092 */ 4093int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs) 4094{ 4095 struct hisi_qm *qm = pci_get_drvdata(pdev); 4096 int pre_existing_vfs, num_vfs, total_vfs, ret; 4097 4098 ret = qm_pm_get_sync(qm); 4099 if (ret) 4100 return ret; 4101 4102 total_vfs = pci_sriov_get_totalvfs(pdev); 4103 pre_existing_vfs = pci_num_vf(pdev); 4104 if (pre_existing_vfs) { 4105 pci_err(pdev, "%d VFs already enabled. Please disable pre-enabled VFs!\n", 4106 pre_existing_vfs); 4107 goto err_put_sync; 4108 } 4109 4110 if (max_vfs > total_vfs) { 4111 pci_err(pdev, "%d VFs is more than total VFs %d!\n", max_vfs, total_vfs); 4112 ret = -ERANGE; 4113 goto err_put_sync; 4114 } 4115 4116 num_vfs = max_vfs; 4117 4118 if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) 4119 hisi_qm_init_vf_qos(qm, num_vfs); 4120 4121 ret = qm_vf_q_assign(qm, num_vfs); 4122 if (ret) { 4123 pci_err(pdev, "Can't assign queues for VF!\n"); 4124 goto err_put_sync; 4125 } 4126 4127 qm->vfs_num = num_vfs; 4128 ret = pci_enable_sriov(pdev, num_vfs); 4129 if (ret) { 4130 pci_err(pdev, "Can't enable VF!\n"); 4131 qm_clear_vft_config(qm); 4132 goto err_put_sync; 4133 } 4134 4135 pci_info(pdev, "VF enabled, vfs_num(=%d)!\n", num_vfs); 4136 4137 return num_vfs; 4138 4139err_put_sync: 4140 qm_pm_put_sync(qm); 4141 return ret; 4142} 4143EXPORT_SYMBOL_GPL(hisi_qm_sriov_enable); 4144 4145/** 4146 * hisi_qm_sriov_disable - disable virtual functions 4147 * @pdev: the PCI device. 4148 * @is_frozen: true when all the VFs are frozen. 4149 * 4150 * Return failure if there are VFs assigned already or VF is in used. 4151 */ 4152int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen) 4153{ 4154 struct hisi_qm *qm = pci_get_drvdata(pdev); 4155 4156 if (pci_vfs_assigned(pdev)) { 4157 pci_err(pdev, "Failed to disable VFs as VFs are assigned!\n"); 4158 return -EPERM; 4159 } 4160 4161 /* While VF is in used, SRIOV cannot be disabled. */ 4162 if (!is_frozen && qm_try_frozen_vfs(pdev, qm->qm_list)) { 4163 pci_err(pdev, "Task is using its VF!\n"); 4164 return -EBUSY; 4165 } 4166 4167 pci_disable_sriov(pdev); 4168 qm_clear_vft_config(qm); 4169 qm_pm_put_sync(qm); 4170 4171 return 0; 4172} 4173EXPORT_SYMBOL_GPL(hisi_qm_sriov_disable); 4174 4175/** 4176 * hisi_qm_sriov_configure - configure the number of VFs 4177 * @pdev: The PCI device 4178 * @num_vfs: The number of VFs need enabled 4179 * 4180 * Enable SR-IOV according to num_vfs, 0 means disable. 4181 */ 4182int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs) 4183{ 4184 if (num_vfs == 0) 4185 return hisi_qm_sriov_disable(pdev, false); 4186 else 4187 return hisi_qm_sriov_enable(pdev, num_vfs); 4188} 4189EXPORT_SYMBOL_GPL(hisi_qm_sriov_configure); 4190 4191static enum acc_err_result qm_dev_err_handle(struct hisi_qm *qm) 4192{ 4193 if (!qm->err_ini->get_err_result) { 4194 dev_err(&qm->pdev->dev, "Device doesn't support reset!\n"); 4195 return ACC_ERR_NONE; 4196 } 4197 4198 return qm->err_ini->get_err_result(qm); 4199} 4200 4201static enum acc_err_result qm_process_dev_error(struct hisi_qm *qm) 4202{ 4203 enum acc_err_result qm_ret, dev_ret; 4204 4205 /* log qm error */ 4206 qm_ret = qm_hw_error_handle(qm); 4207 4208 /* log device error */ 4209 dev_ret = qm_dev_err_handle(qm); 4210 4211 return (qm_ret == ACC_ERR_NEED_RESET || 4212 dev_ret == ACC_ERR_NEED_RESET) ? 4213 ACC_ERR_NEED_RESET : ACC_ERR_RECOVERED; 4214} 4215 4216/** 4217 * hisi_qm_dev_err_detected() - Get device and qm error status then log it. 4218 * @pdev: The PCI device which need report error. 4219 * @state: The connectivity between CPU and device. 4220 * 4221 * We register this function into PCIe AER handlers, It will report device or 4222 * qm hardware error status when error occur. 4223 */ 4224pci_ers_result_t hisi_qm_dev_err_detected(struct pci_dev *pdev, 4225 pci_channel_state_t state) 4226{ 4227 struct hisi_qm *qm = pci_get_drvdata(pdev); 4228 enum acc_err_result ret; 4229 4230 if (pdev->is_virtfn) 4231 return PCI_ERS_RESULT_NONE; 4232 4233 pci_info(pdev, "PCI error detected, state(=%u)!!\n", state); 4234 if (state == pci_channel_io_perm_failure) 4235 return PCI_ERS_RESULT_DISCONNECT; 4236 4237 ret = qm_process_dev_error(qm); 4238 if (ret == ACC_ERR_NEED_RESET) 4239 return PCI_ERS_RESULT_NEED_RESET; 4240 4241 return PCI_ERS_RESULT_RECOVERED; 4242} 4243EXPORT_SYMBOL_GPL(hisi_qm_dev_err_detected); 4244 4245static int qm_check_req_recv(struct hisi_qm *qm) 4246{ 4247 struct pci_dev *pdev = qm->pdev; 4248 int ret; 4249 u32 val; 4250 4251 if (qm->ver >= QM_HW_V3) 4252 return 0; 4253 4254 writel(ACC_VENDOR_ID_VALUE, qm->io_base + QM_PEH_VENDOR_ID); 4255 ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_VENDOR_ID, val, 4256 (val == ACC_VENDOR_ID_VALUE), 4257 POLL_PERIOD, POLL_TIMEOUT); 4258 if (ret) { 4259 dev_err(&pdev->dev, "Fails to read QM reg!\n"); 4260 return ret; 4261 } 4262 4263 writel(PCI_VENDOR_ID_HUAWEI, qm->io_base + QM_PEH_VENDOR_ID); 4264 ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_VENDOR_ID, val, 4265 (val == PCI_VENDOR_ID_HUAWEI), 4266 POLL_PERIOD, POLL_TIMEOUT); 4267 if (ret) 4268 dev_err(&pdev->dev, "Fails to read QM reg in the second time!\n"); 4269 4270 return ret; 4271} 4272 4273static int qm_set_pf_mse(struct hisi_qm *qm, bool set) 4274{ 4275 struct pci_dev *pdev = qm->pdev; 4276 u16 cmd; 4277 int i; 4278 4279 pci_read_config_word(pdev, PCI_COMMAND, &cmd); 4280 if (set) 4281 cmd |= PCI_COMMAND_MEMORY; 4282 else 4283 cmd &= ~PCI_COMMAND_MEMORY; 4284 4285 pci_write_config_word(pdev, PCI_COMMAND, cmd); 4286 for (i = 0; i < MAX_WAIT_COUNTS; i++) { 4287 pci_read_config_word(pdev, PCI_COMMAND, &cmd); 4288 if (set == ((cmd & PCI_COMMAND_MEMORY) >> 1)) 4289 return 0; 4290 4291 udelay(1); 4292 } 4293 4294 return -ETIMEDOUT; 4295} 4296 4297static int qm_set_vf_mse(struct hisi_qm *qm, bool set) 4298{ 4299 struct pci_dev *pdev = qm->pdev; 4300 u16 sriov_ctrl; 4301 int pos; 4302 int i; 4303 4304 /* 4305 * Since function qm_set_vf_mse is called only after SRIOV is enabled, 4306 * pci_find_ext_capability cannot return 0, pos does not need to be 4307 * checked. 4308 */ 4309 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV); 4310 pci_read_config_word(pdev, pos + PCI_SRIOV_CTRL, &sriov_ctrl); 4311 if (set) 4312 sriov_ctrl |= PCI_SRIOV_CTRL_MSE; 4313 else 4314 sriov_ctrl &= ~PCI_SRIOV_CTRL_MSE; 4315 pci_write_config_word(pdev, pos + PCI_SRIOV_CTRL, sriov_ctrl); 4316 4317 for (i = 0; i < MAX_WAIT_COUNTS; i++) { 4318 pci_read_config_word(pdev, pos + PCI_SRIOV_CTRL, &sriov_ctrl); 4319 if (set == (sriov_ctrl & PCI_SRIOV_CTRL_MSE) >> 4320 ACC_PEH_SRIOV_CTRL_VF_MSE_SHIFT) 4321 return 0; 4322 4323 udelay(1); 4324 } 4325 4326 return -ETIMEDOUT; 4327} 4328 4329static void qm_dev_ecc_mbit_handle(struct hisi_qm *qm) 4330{ 4331 u32 nfe_enb = 0; 4332 4333 /* Kunpeng930 hardware automatically close master ooo when NFE occurs */ 4334 if (qm->ver >= QM_HW_V3) 4335 return; 4336 4337 if (!qm->err_status.is_dev_ecc_mbit && 4338 qm->err_status.is_qm_ecc_mbit && 4339 qm->err_ini->close_axi_master_ooo) { 4340 qm->err_ini->close_axi_master_ooo(qm); 4341 } else if (qm->err_status.is_dev_ecc_mbit && 4342 !qm->err_status.is_qm_ecc_mbit && 4343 !qm->err_ini->close_axi_master_ooo) { 4344 nfe_enb = readl(qm->io_base + QM_RAS_NFE_ENABLE); 4345 writel(nfe_enb & ~qm->err_info.qm_err.ecc_2bits_mask, 4346 qm->io_base + QM_RAS_NFE_ENABLE); 4347 writel(qm->err_info.qm_err.ecc_2bits_mask, qm->io_base + QM_ABNORMAL_INT_SET); 4348 } 4349} 4350 4351static int qm_vf_reset_prepare(struct hisi_qm *qm, 4352 enum qm_stop_reason stop_reason) 4353{ 4354 struct hisi_qm_list *qm_list = qm->qm_list; 4355 struct pci_dev *pdev = qm->pdev; 4356 struct pci_dev *virtfn; 4357 struct hisi_qm *vf_qm; 4358 int ret = 0; 4359 4360 mutex_lock(&qm_list->lock); 4361 list_for_each_entry(vf_qm, &qm_list->list, list) { 4362 virtfn = vf_qm->pdev; 4363 if (virtfn == pdev) 4364 continue; 4365 4366 if (pci_physfn(virtfn) == pdev) { 4367 /* save VFs PCIE BAR configuration */ 4368 pci_save_state(virtfn); 4369 4370 ret = hisi_qm_stop(vf_qm, stop_reason); 4371 if (ret) 4372 goto stop_fail; 4373 } 4374 } 4375 4376stop_fail: 4377 mutex_unlock(&qm_list->lock); 4378 return ret; 4379} 4380 4381static int qm_try_stop_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd, 4382 enum qm_stop_reason stop_reason) 4383{ 4384 struct pci_dev *pdev = qm->pdev; 4385 int ret; 4386 4387 if (!qm->vfs_num) 4388 return 0; 4389 4390 /* Kunpeng930 supports to notify VFs to stop before PF reset */ 4391 if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) { 4392 ret = qm_ping_all_vfs(qm, cmd); 4393 if (ret) 4394 pci_err(pdev, "failed to send command to all VFs before PF reset!\n"); 4395 } else { 4396 ret = qm_vf_reset_prepare(qm, stop_reason); 4397 if (ret) 4398 pci_err(pdev, "failed to prepare reset, ret = %d.\n", ret); 4399 } 4400 4401 return ret; 4402} 4403 4404static int qm_controller_reset_prepare(struct hisi_qm *qm) 4405{ 4406 struct pci_dev *pdev = qm->pdev; 4407 int ret; 4408 4409 if (qm->err_ini->set_priv_status) { 4410 ret = qm->err_ini->set_priv_status(qm); 4411 if (ret) 4412 return ret; 4413 } 4414 4415 ret = qm_reset_prepare_ready(qm); 4416 if (ret) { 4417 pci_err(pdev, "Controller reset not ready!\n"); 4418 return ret; 4419 } 4420 4421 qm_dev_ecc_mbit_handle(qm); 4422 4423 /* PF obtains the information of VF by querying the register. */ 4424 qm_cmd_uninit(qm); 4425 4426 /* Whether VFs stop successfully, soft reset will continue. */ 4427 ret = qm_try_stop_vfs(qm, QM_PF_SRST_PREPARE, QM_SOFT_RESET); 4428 if (ret) 4429 pci_err(pdev, "failed to stop vfs by pf in soft reset.\n"); 4430 4431 ret = hisi_qm_stop(qm, QM_SOFT_RESET); 4432 if (ret) { 4433 pci_err(pdev, "Fails to stop QM!\n"); 4434 qm_reset_bit_clear(qm); 4435 return ret; 4436 } 4437 4438 if (qm->use_sva) { 4439 ret = qm_hw_err_isolate(qm); 4440 if (ret) 4441 pci_err(pdev, "failed to isolate hw err!\n"); 4442 } 4443 4444 ret = qm_wait_vf_prepare_finish(qm); 4445 if (ret) 4446 pci_err(pdev, "failed to stop by vfs in soft reset!\n"); 4447 4448 clear_bit(QM_RST_SCHED, &qm->misc_ctl); 4449 4450 return 0; 4451} 4452 4453static int qm_master_ooo_check(struct hisi_qm *qm) 4454{ 4455 u32 val; 4456 int ret; 4457 4458 /* Check the ooo register of the device before resetting the device. */ 4459 writel(ACC_MASTER_GLOBAL_CTRL_SHUTDOWN, qm->io_base + ACC_MASTER_GLOBAL_CTRL); 4460 ret = readl_relaxed_poll_timeout(qm->io_base + ACC_MASTER_TRANS_RETURN, 4461 val, (val == ACC_MASTER_TRANS_RETURN_RW), 4462 POLL_PERIOD, POLL_TIMEOUT); 4463 if (ret) 4464 pci_warn(qm->pdev, "Bus lock! Please reset system.\n"); 4465 4466 return ret; 4467} 4468 4469static int qm_soft_reset_prepare(struct hisi_qm *qm) 4470{ 4471 struct pci_dev *pdev = qm->pdev; 4472 int ret; 4473 4474 /* Ensure all doorbells and mailboxes received by QM */ 4475 ret = qm_check_req_recv(qm); 4476 if (ret) 4477 return ret; 4478 4479 if (qm->vfs_num) { 4480 ret = qm_set_vf_mse(qm, false); 4481 if (ret) { 4482 pci_err(pdev, "Fails to disable vf MSE bit.\n"); 4483 return ret; 4484 } 4485 } 4486 4487 ret = qm->ops->set_msi(qm, false); 4488 if (ret) { 4489 pci_err(pdev, "Fails to disable PEH MSI bit.\n"); 4490 return ret; 4491 } 4492 4493 ret = qm_master_ooo_check(qm); 4494 if (ret) 4495 return ret; 4496 4497 if (qm->err_ini->close_sva_prefetch) 4498 qm->err_ini->close_sva_prefetch(qm); 4499 4500 ret = qm_set_pf_mse(qm, false); 4501 if (ret) 4502 pci_err(pdev, "Fails to disable pf MSE bit.\n"); 4503 4504 return ret; 4505} 4506 4507static int qm_reset_device(struct hisi_qm *qm) 4508{ 4509 struct pci_dev *pdev = qm->pdev; 4510 4511 /* The reset related sub-control registers are not in PCI BAR */ 4512 if (ACPI_HANDLE(&pdev->dev)) { 4513 unsigned long long value = 0; 4514 acpi_status s; 4515 4516 s = acpi_evaluate_integer(ACPI_HANDLE(&pdev->dev), 4517 qm->err_info.acpi_rst, 4518 NULL, &value); 4519 if (ACPI_FAILURE(s)) { 4520 pci_err(pdev, "NO controller reset method!\n"); 4521 return -EIO; 4522 } 4523 4524 if (value) { 4525 pci_err(pdev, "Reset step %llu failed!\n", value); 4526 return -EIO; 4527 } 4528 4529 return 0; 4530 } 4531 4532 pci_err(pdev, "No reset method!\n"); 4533 return -EINVAL; 4534} 4535 4536static int qm_soft_reset(struct hisi_qm *qm) 4537{ 4538 int ret; 4539 4540 ret = qm_soft_reset_prepare(qm); 4541 if (ret) 4542 return ret; 4543 4544 return qm_reset_device(qm); 4545} 4546 4547static int qm_vf_reset_done(struct hisi_qm *qm) 4548{ 4549 struct hisi_qm_list *qm_list = qm->qm_list; 4550 struct pci_dev *pdev = qm->pdev; 4551 struct pci_dev *virtfn; 4552 struct hisi_qm *vf_qm; 4553 int ret = 0; 4554 4555 mutex_lock(&qm_list->lock); 4556 list_for_each_entry(vf_qm, &qm_list->list, list) { 4557 virtfn = vf_qm->pdev; 4558 if (virtfn == pdev) 4559 continue; 4560 4561 if (pci_physfn(virtfn) == pdev) { 4562 /* enable VFs PCIE BAR configuration */ 4563 pci_restore_state(virtfn); 4564 4565 ret = qm_restart(vf_qm); 4566 if (ret) 4567 goto restart_fail; 4568 } 4569 } 4570 4571restart_fail: 4572 mutex_unlock(&qm_list->lock); 4573 return ret; 4574} 4575 4576static int qm_try_start_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd) 4577{ 4578 struct pci_dev *pdev = qm->pdev; 4579 int ret; 4580 4581 if (!qm->vfs_num) 4582 return 0; 4583 4584 ret = qm_vf_q_assign(qm, qm->vfs_num); 4585 if (ret) { 4586 pci_err(pdev, "failed to assign VFs, ret = %d.\n", ret); 4587 return ret; 4588 } 4589 4590 /* Kunpeng930 supports to notify VFs to start after PF reset. */ 4591 if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) { 4592 ret = qm_ping_all_vfs(qm, cmd); 4593 if (ret) 4594 pci_warn(pdev, "failed to send cmd to all VFs after PF reset!\n"); 4595 } else { 4596 ret = qm_vf_reset_done(qm); 4597 if (ret) 4598 pci_warn(pdev, "failed to start vfs, ret = %d.\n", ret); 4599 } 4600 4601 return ret; 4602} 4603 4604static int qm_dev_hw_init(struct hisi_qm *qm) 4605{ 4606 return qm->err_ini->hw_init(qm); 4607} 4608 4609static void qm_restart_prepare(struct hisi_qm *qm) 4610{ 4611 u32 value; 4612 4613 if (qm->ver >= QM_HW_V3) 4614 return; 4615 4616 if (!qm->err_status.is_qm_ecc_mbit && 4617 !qm->err_status.is_dev_ecc_mbit) 4618 return; 4619 4620 /* temporarily close the OOO port used for PEH to write out MSI */ 4621 value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN); 4622 writel(value & ~qm->err_info.msi_wr_port, 4623 qm->io_base + ACC_AM_CFG_PORT_WR_EN); 4624 4625 /* clear dev ecc 2bit error source if having */ 4626 value = qm_get_dev_err_status(qm) & qm->err_info.dev_err.ecc_2bits_mask; 4627 if (value && qm->err_ini->clear_dev_hw_err_status) 4628 qm->err_ini->clear_dev_hw_err_status(qm, value); 4629 4630 /* clear QM ecc mbit error source */ 4631 writel(qm->err_info.qm_err.ecc_2bits_mask, qm->io_base + QM_ABNORMAL_INT_SOURCE); 4632 4633 /* clear AM Reorder Buffer ecc mbit source */ 4634 writel(ACC_ROB_ECC_ERR_MULTPL, qm->io_base + ACC_AM_ROB_ECC_INT_STS); 4635} 4636 4637static void qm_restart_done(struct hisi_qm *qm) 4638{ 4639 u32 value; 4640 4641 if (qm->ver >= QM_HW_V3) 4642 goto clear_flags; 4643 4644 if (!qm->err_status.is_qm_ecc_mbit && 4645 !qm->err_status.is_dev_ecc_mbit) 4646 return; 4647 4648 /* open the OOO port for PEH to write out MSI */ 4649 value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN); 4650 value |= qm->err_info.msi_wr_port; 4651 writel(value, qm->io_base + ACC_AM_CFG_PORT_WR_EN); 4652 4653clear_flags: 4654 qm->err_status.is_qm_ecc_mbit = false; 4655 qm->err_status.is_dev_ecc_mbit = false; 4656} 4657 4658static void qm_disable_axi_error(struct hisi_qm *qm) 4659{ 4660 struct hisi_qm_err_mask *qm_err = &qm->err_info.qm_err; 4661 u32 val; 4662 4663 val = ~(qm->error_mask & (~QM_RAS_AXI_ERROR)); 4664 writel(val, qm->io_base + QM_ABNORMAL_INT_MASK); 4665 if (qm->ver > QM_HW_V2) 4666 writel(qm_err->shutdown_mask & (~QM_RAS_AXI_ERROR), 4667 qm->io_base + QM_OOO_SHUTDOWN_SEL); 4668 4669 if (qm->err_ini->disable_axi_error) 4670 qm->err_ini->disable_axi_error(qm); 4671} 4672 4673static void qm_enable_axi_error(struct hisi_qm *qm) 4674{ 4675 /* clear axi error source */ 4676 writel(QM_RAS_AXI_ERROR, qm->io_base + QM_ABNORMAL_INT_SOURCE); 4677 4678 writel(~qm->error_mask, qm->io_base + QM_ABNORMAL_INT_MASK); 4679 if (qm->ver > QM_HW_V2) 4680 writel(qm->err_info.qm_err.shutdown_mask, qm->io_base + QM_OOO_SHUTDOWN_SEL); 4681 4682 if (qm->err_ini->enable_axi_error) 4683 qm->err_ini->enable_axi_error(qm); 4684} 4685 4686static int qm_controller_reset_done(struct hisi_qm *qm) 4687{ 4688 struct pci_dev *pdev = qm->pdev; 4689 int ret; 4690 4691 ret = qm->ops->set_msi(qm, true); 4692 if (ret) { 4693 pci_err(pdev, "Fails to enable PEH MSI bit!\n"); 4694 return ret; 4695 } 4696 4697 ret = qm_set_pf_mse(qm, true); 4698 if (ret) { 4699 pci_err(pdev, "Fails to enable pf MSE bit!\n"); 4700 return ret; 4701 } 4702 4703 if (qm->vfs_num) { 4704 ret = qm_set_vf_mse(qm, true); 4705 if (ret) { 4706 pci_err(pdev, "Fails to enable vf MSE bit!\n"); 4707 return ret; 4708 } 4709 } 4710 4711 ret = qm_dev_hw_init(qm); 4712 if (ret) { 4713 pci_err(pdev, "Failed to init device\n"); 4714 return ret; 4715 } 4716 4717 qm_restart_prepare(qm); 4718 hisi_qm_dev_err_init(qm); 4719 qm_disable_axi_error(qm); 4720 if (qm->err_ini->open_axi_master_ooo) 4721 qm->err_ini->open_axi_master_ooo(qm); 4722 4723 ret = qm_dev_mem_reset(qm); 4724 if (ret) { 4725 pci_err(pdev, "failed to reset device memory\n"); 4726 return ret; 4727 } 4728 4729 ret = qm_restart(qm); 4730 if (ret) { 4731 pci_err(pdev, "Failed to start QM!\n"); 4732 return ret; 4733 } 4734 4735 ret = qm_try_start_vfs(qm, QM_PF_RESET_DONE); 4736 if (ret) 4737 pci_err(pdev, "failed to start vfs by pf in soft reset.\n"); 4738 4739 ret = qm_wait_vf_prepare_finish(qm); 4740 if (ret) 4741 pci_err(pdev, "failed to start by vfs in soft reset!\n"); 4742 qm_enable_axi_error(qm); 4743 qm_cmd_init(qm); 4744 qm_restart_done(qm); 4745 4746 qm_reset_bit_clear(qm); 4747 4748 return 0; 4749} 4750 4751static int qm_controller_reset(struct hisi_qm *qm) 4752{ 4753 struct pci_dev *pdev = qm->pdev; 4754 int ret; 4755 4756 pci_info(pdev, "Controller resetting...\n"); 4757 4758 ret = qm_controller_reset_prepare(qm); 4759 if (ret) { 4760 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET); 4761 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); 4762 clear_bit(QM_RST_SCHED, &qm->misc_ctl); 4763 return ret; 4764 } 4765 4766 hisi_qm_show_last_dfx_regs(qm); 4767 if (qm->err_ini->show_last_dfx_regs) 4768 qm->err_ini->show_last_dfx_regs(qm); 4769 4770 ret = qm_soft_reset(qm); 4771 if (ret) 4772 goto err_reset; 4773 4774 ret = qm_controller_reset_done(qm); 4775 if (ret) 4776 goto err_reset; 4777 4778 pci_info(pdev, "Controller reset complete\n"); 4779 4780 return 0; 4781 4782err_reset: 4783 pci_err(pdev, "Controller reset failed (%d)\n", ret); 4784 qm_reset_bit_clear(qm); 4785 4786 /* if resetting fails, isolate the device */ 4787 if (qm->use_sva) 4788 qm->isolate_data.is_isolate = true; 4789 return ret; 4790} 4791 4792/** 4793 * hisi_qm_dev_slot_reset() - slot reset 4794 * @pdev: the PCIe device 4795 * 4796 * This function offers QM relate PCIe device reset interface. Drivers which 4797 * use QM can use this function as slot_reset in its struct pci_error_handlers. 4798 */ 4799pci_ers_result_t hisi_qm_dev_slot_reset(struct pci_dev *pdev) 4800{ 4801 struct hisi_qm *qm = pci_get_drvdata(pdev); 4802 int ret; 4803 4804 if (pdev->is_virtfn) 4805 return PCI_ERS_RESULT_RECOVERED; 4806 4807 /* reset pcie device controller */ 4808 ret = qm_controller_reset(qm); 4809 if (ret) { 4810 pci_err(pdev, "Controller reset failed (%d)\n", ret); 4811 return PCI_ERS_RESULT_DISCONNECT; 4812 } 4813 4814 return PCI_ERS_RESULT_RECOVERED; 4815} 4816EXPORT_SYMBOL_GPL(hisi_qm_dev_slot_reset); 4817 4818void hisi_qm_reset_prepare(struct pci_dev *pdev) 4819{ 4820 struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev)); 4821 struct hisi_qm *qm = pci_get_drvdata(pdev); 4822 u32 delay = 0; 4823 int ret; 4824 4825 hisi_qm_dev_err_uninit(pf_qm); 4826 4827 /* 4828 * Check whether there is an ECC mbit error, If it occurs, need to 4829 * wait for soft reset to fix it. 4830 */ 4831 while (qm_check_dev_error(qm)) { 4832 msleep(++delay); 4833 if (delay > QM_RESET_WAIT_TIMEOUT) 4834 return; 4835 } 4836 4837 ret = qm_reset_prepare_ready(qm); 4838 if (ret) { 4839 pci_err(pdev, "FLR not ready!\n"); 4840 return; 4841 } 4842 4843 /* PF obtains the information of VF by querying the register. */ 4844 if (qm->fun_type == QM_HW_PF) 4845 qm_cmd_uninit(qm); 4846 4847 ret = qm_try_stop_vfs(qm, QM_PF_FLR_PREPARE, QM_DOWN); 4848 if (ret) 4849 pci_err(pdev, "failed to stop vfs by pf in FLR.\n"); 4850 4851 ret = hisi_qm_stop(qm, QM_DOWN); 4852 if (ret) { 4853 pci_err(pdev, "Failed to stop QM, ret = %d.\n", ret); 4854 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET); 4855 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); 4856 return; 4857 } 4858 4859 ret = qm_wait_vf_prepare_finish(qm); 4860 if (ret) 4861 pci_err(pdev, "failed to stop by vfs in FLR!\n"); 4862 4863 pci_info(pdev, "FLR resetting...\n"); 4864} 4865EXPORT_SYMBOL_GPL(hisi_qm_reset_prepare); 4866 4867static bool qm_flr_reset_complete(struct pci_dev *pdev) 4868{ 4869 struct pci_dev *pf_pdev = pci_physfn(pdev); 4870 struct hisi_qm *qm = pci_get_drvdata(pf_pdev); 4871 u32 id; 4872 4873 pci_read_config_dword(qm->pdev, PCI_COMMAND, &id); 4874 if (id == QM_PCI_COMMAND_INVALID) { 4875 pci_err(pdev, "Device can not be used!\n"); 4876 return false; 4877 } 4878 4879 return true; 4880} 4881 4882void hisi_qm_reset_done(struct pci_dev *pdev) 4883{ 4884 struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev)); 4885 struct hisi_qm *qm = pci_get_drvdata(pdev); 4886 int ret; 4887 4888 if (qm->fun_type == QM_HW_PF) { 4889 ret = qm_dev_hw_init(qm); 4890 if (ret) { 4891 pci_err(pdev, "Failed to init PF, ret = %d.\n", ret); 4892 goto flr_done; 4893 } 4894 } 4895 4896 hisi_qm_dev_err_init(pf_qm); 4897 4898 ret = qm_restart(qm); 4899 if (ret) { 4900 pci_err(pdev, "Failed to start QM, ret = %d.\n", ret); 4901 goto flr_done; 4902 } 4903 4904 ret = qm_try_start_vfs(qm, QM_PF_RESET_DONE); 4905 if (ret) 4906 pci_err(pdev, "failed to start vfs by pf in FLR.\n"); 4907 4908 ret = qm_wait_vf_prepare_finish(qm); 4909 if (ret) 4910 pci_err(pdev, "failed to start by vfs in FLR!\n"); 4911 4912flr_done: 4913 if (qm->fun_type == QM_HW_PF) 4914 qm_cmd_init(qm); 4915 4916 if (qm_flr_reset_complete(pdev)) 4917 pci_info(pdev, "FLR reset complete\n"); 4918 4919 qm_reset_bit_clear(qm); 4920} 4921EXPORT_SYMBOL_GPL(hisi_qm_reset_done); 4922 4923static irqreturn_t qm_rsvd_irq(int irq, void *data) 4924{ 4925 struct hisi_qm *qm = data; 4926 4927 dev_info(&qm->pdev->dev, "Reserved interrupt, ignore!\n"); 4928 4929 return IRQ_HANDLED; 4930} 4931 4932static irqreturn_t qm_abnormal_irq(int irq, void *data) 4933{ 4934 struct hisi_qm *qm = data; 4935 enum acc_err_result ret; 4936 4937 atomic64_inc(&qm->debug.dfx.abnormal_irq_cnt); 4938 ret = qm_process_dev_error(qm); 4939 if (ret == ACC_ERR_NEED_RESET && 4940 !test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl) && 4941 !test_and_set_bit(QM_RST_SCHED, &qm->misc_ctl)) 4942 schedule_work(&qm->rst_work); 4943 4944 return IRQ_HANDLED; 4945} 4946 4947/** 4948 * hisi_qm_dev_shutdown() - Shutdown device. 4949 * @pdev: The device will be shutdown. 4950 * 4951 * This function will stop qm when OS shutdown or rebooting. 4952 */ 4953void hisi_qm_dev_shutdown(struct pci_dev *pdev) 4954{ 4955 struct hisi_qm *qm = pci_get_drvdata(pdev); 4956 int ret; 4957 4958 ret = hisi_qm_stop(qm, QM_DOWN); 4959 if (ret) 4960 dev_err(&pdev->dev, "Fail to stop qm in shutdown!\n"); 4961} 4962EXPORT_SYMBOL_GPL(hisi_qm_dev_shutdown); 4963 4964static void hisi_qm_controller_reset(struct work_struct *rst_work) 4965{ 4966 struct hisi_qm *qm = container_of(rst_work, struct hisi_qm, rst_work); 4967 int ret; 4968 4969 ret = qm_pm_get_sync(qm); 4970 if (ret) { 4971 clear_bit(QM_RST_SCHED, &qm->misc_ctl); 4972 return; 4973 } 4974 4975 /* reset pcie device controller */ 4976 ret = qm_controller_reset(qm); 4977 if (ret) 4978 dev_err(&qm->pdev->dev, "controller reset failed (%d)\n", ret); 4979 4980 qm_pm_put_sync(qm); 4981} 4982 4983static void qm_pf_reset_vf_prepare(struct hisi_qm *qm, 4984 enum qm_stop_reason stop_reason) 4985{ 4986 enum qm_ifc_cmd cmd = QM_VF_PREPARE_DONE; 4987 struct pci_dev *pdev = qm->pdev; 4988 int ret; 4989 4990 ret = qm_reset_prepare_ready(qm); 4991 if (ret) { 4992 dev_err(&pdev->dev, "reset prepare not ready!\n"); 4993 atomic_set(&qm->status.flags, QM_STOP); 4994 cmd = QM_VF_PREPARE_FAIL; 4995 goto err_prepare; 4996 } 4997 4998 ret = hisi_qm_stop(qm, stop_reason); 4999 if (ret) { 5000 dev_err(&pdev->dev, "failed to stop QM, ret = %d.\n", ret); 5001 atomic_set(&qm->status.flags, QM_STOP); 5002 cmd = QM_VF_PREPARE_FAIL; 5003 goto err_prepare; 5004 } else { 5005 goto out; 5006 } 5007 5008err_prepare: 5009 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET); 5010 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); 5011out: 5012 pci_save_state(pdev); 5013 ret = qm_ping_pf(qm, cmd); 5014 if (ret) 5015 dev_warn(&pdev->dev, "PF responds timeout in reset prepare!\n"); 5016} 5017 5018static void qm_pf_reset_vf_done(struct hisi_qm *qm) 5019{ 5020 enum qm_ifc_cmd cmd = QM_VF_START_DONE; 5021 struct pci_dev *pdev = qm->pdev; 5022 int ret; 5023 5024 pci_restore_state(pdev); 5025 ret = hisi_qm_start(qm); 5026 if (ret) { 5027 dev_err(&pdev->dev, "failed to start QM, ret = %d.\n", ret); 5028 cmd = QM_VF_START_FAIL; 5029 } 5030 5031 qm_cmd_init(qm); 5032 ret = qm_ping_pf(qm, cmd); 5033 if (ret) 5034 dev_warn(&pdev->dev, "PF responds timeout in reset done!\n"); 5035 5036 qm_reset_bit_clear(qm); 5037} 5038 5039static int qm_wait_pf_reset_finish(struct hisi_qm *qm) 5040{ 5041 struct device *dev = &qm->pdev->dev; 5042 u32 val, cmd; 5043 int ret; 5044 5045 /* Wait for reset to finish */ 5046 ret = readl_relaxed_poll_timeout(qm->io_base + QM_IFC_INT_SOURCE_V, val, 5047 val == BIT(0), QM_VF_RESET_WAIT_US, 5048 QM_VF_RESET_WAIT_TIMEOUT_US); 5049 /* hardware completion status should be available by this time */ 5050 if (ret) { 5051 dev_err(dev, "couldn't get reset done status from PF, timeout!\n"); 5052 return -ETIMEDOUT; 5053 } 5054 5055 /* 5056 * Whether message is got successfully, 5057 * VF needs to ack PF by clearing the interrupt. 5058 */ 5059 ret = qm->ops->get_ifc(qm, &cmd, NULL, 0); 5060 qm_clear_cmd_interrupt(qm, 0); 5061 if (ret) { 5062 dev_err(dev, "failed to get command from PF in reset done!\n"); 5063 return ret; 5064 } 5065 5066 if (cmd != QM_PF_RESET_DONE) { 5067 dev_err(dev, "the command(0x%x) is not reset done!\n", cmd); 5068 ret = -EINVAL; 5069 } 5070 5071 return ret; 5072} 5073 5074static void qm_pf_reset_vf_process(struct hisi_qm *qm, 5075 enum qm_stop_reason stop_reason) 5076{ 5077 struct device *dev = &qm->pdev->dev; 5078 int ret; 5079 5080 dev_info(dev, "device reset start...\n"); 5081 5082 /* The message is obtained by querying the register during resetting */ 5083 qm_cmd_uninit(qm); 5084 qm_pf_reset_vf_prepare(qm, stop_reason); 5085 5086 ret = qm_wait_pf_reset_finish(qm); 5087 if (ret) 5088 goto err_get_status; 5089 5090 qm_pf_reset_vf_done(qm); 5091 5092 dev_info(dev, "device reset done.\n"); 5093 5094 return; 5095 5096err_get_status: 5097 qm_cmd_init(qm); 5098 qm_reset_bit_clear(qm); 5099} 5100 5101static void qm_handle_cmd_msg(struct hisi_qm *qm, u32 fun_num) 5102{ 5103 struct device *dev = &qm->pdev->dev; 5104 enum qm_ifc_cmd cmd; 5105 u32 data; 5106 int ret; 5107 5108 /* 5109 * Get the msg from source by sending mailbox. Whether message is got 5110 * successfully, destination needs to ack source by clearing the interrupt. 5111 */ 5112 ret = qm->ops->get_ifc(qm, &cmd, &data, fun_num); 5113 qm_clear_cmd_interrupt(qm, BIT(fun_num)); 5114 if (ret) { 5115 dev_err(dev, "failed to get command from source!\n"); 5116 return; 5117 } 5118 5119 switch (cmd) { 5120 case QM_PF_FLR_PREPARE: 5121 qm_pf_reset_vf_process(qm, QM_DOWN); 5122 break; 5123 case QM_PF_SRST_PREPARE: 5124 qm_pf_reset_vf_process(qm, QM_SOFT_RESET); 5125 break; 5126 case QM_VF_GET_QOS: 5127 qm_vf_get_qos(qm, fun_num); 5128 break; 5129 case QM_PF_SET_QOS: 5130 qm->mb_qos = data; 5131 break; 5132 default: 5133 dev_err(dev, "unsupported command(0x%x) sent by function(%u)!\n", cmd, fun_num); 5134 break; 5135 } 5136} 5137 5138static void qm_cmd_process(struct work_struct *cmd_process) 5139{ 5140 struct hisi_qm *qm = container_of(cmd_process, 5141 struct hisi_qm, cmd_process); 5142 u32 vfs_num = qm->vfs_num; 5143 u64 val; 5144 u32 i; 5145 5146 if (qm->fun_type == QM_HW_PF) { 5147 val = readq(qm->io_base + QM_IFC_INT_SOURCE_P); 5148 if (!val) 5149 return; 5150 5151 for (i = 1; i <= vfs_num; i++) { 5152 if (val & BIT(i)) 5153 qm_handle_cmd_msg(qm, i); 5154 } 5155 5156 return; 5157 } 5158 5159 qm_handle_cmd_msg(qm, 0); 5160} 5161 5162/** 5163 * hisi_qm_alg_register() - Register alg to crypto. 5164 * @qm: The qm needs add. 5165 * @qm_list: The qm list. 5166 * @guard: Guard of qp_num. 5167 * 5168 * Register algorithm to crypto when the function is satisfy guard. 5169 */ 5170int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list, int guard) 5171{ 5172 struct device *dev = &qm->pdev->dev; 5173 5174 if (qm->ver <= QM_HW_V2 && qm->use_sva) { 5175 dev_info(dev, "HW V2 not both use uacce sva mode and hardware crypto algs.\n"); 5176 return 0; 5177 } 5178 5179 if (qm->qp_num < guard) { 5180 dev_info(dev, "qp_num is less than task need.\n"); 5181 return 0; 5182 } 5183 5184 return qm_list->register_to_crypto(qm); 5185} 5186EXPORT_SYMBOL_GPL(hisi_qm_alg_register); 5187 5188/** 5189 * hisi_qm_alg_unregister() - Unregister alg from crypto. 5190 * @qm: The qm needs delete. 5191 * @qm_list: The qm list. 5192 * @guard: Guard of qp_num. 5193 * 5194 * Unregister algorithm from crypto when the last function is satisfy guard. 5195 */ 5196void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list, int guard) 5197{ 5198 if (qm->ver <= QM_HW_V2 && qm->use_sva) 5199 return; 5200 5201 if (qm->qp_num < guard) 5202 return; 5203 5204 qm_list->unregister_from_crypto(qm); 5205} 5206EXPORT_SYMBOL_GPL(hisi_qm_alg_unregister); 5207 5208static void qm_unregister_abnormal_irq(struct hisi_qm *qm) 5209{ 5210 struct pci_dev *pdev = qm->pdev; 5211 u32 irq_vector, val; 5212 5213 if (qm->fun_type == QM_HW_VF && qm->ver < QM_HW_V3) 5214 return; 5215 5216 val = qm->cap_tables.qm_cap_table[QM_ABNORMAL_IRQ].cap_val; 5217 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK)) 5218 return; 5219 5220 irq_vector = val & QM_IRQ_VECTOR_MASK; 5221 free_irq(pci_irq_vector(pdev, irq_vector), qm); 5222} 5223 5224static int qm_register_abnormal_irq(struct hisi_qm *qm) 5225{ 5226 struct pci_dev *pdev = qm->pdev; 5227 u32 irq_vector, val; 5228 int ret; 5229 5230 val = qm->cap_tables.qm_cap_table[QM_ABNORMAL_IRQ].cap_val; 5231 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK)) 5232 return 0; 5233 irq_vector = val & QM_IRQ_VECTOR_MASK; 5234 5235 /* For VF, this is a reserved interrupt in V3 version. */ 5236 if (qm->fun_type == QM_HW_VF) { 5237 if (qm->ver < QM_HW_V3) 5238 return 0; 5239 5240 ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_rsvd_irq, 5241 IRQF_NO_AUTOEN, qm->dev_name, qm); 5242 if (ret) { 5243 dev_err(&pdev->dev, "failed to request reserved irq, ret = %d!\n", ret); 5244 return ret; 5245 } 5246 return 0; 5247 } 5248 5249 ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_abnormal_irq, 0, qm->dev_name, qm); 5250 if (ret) 5251 dev_err(&qm->pdev->dev, "failed to request abnormal irq, ret = %d!\n", ret); 5252 5253 return ret; 5254} 5255 5256static void qm_unregister_mb_cmd_irq(struct hisi_qm *qm) 5257{ 5258 struct pci_dev *pdev = qm->pdev; 5259 u32 irq_vector, val; 5260 5261 val = qm->cap_tables.qm_cap_table[QM_MB_IRQ].cap_val; 5262 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) 5263 return; 5264 5265 irq_vector = val & QM_IRQ_VECTOR_MASK; 5266 free_irq(pci_irq_vector(pdev, irq_vector), qm); 5267} 5268 5269static int qm_register_mb_cmd_irq(struct hisi_qm *qm) 5270{ 5271 struct pci_dev *pdev = qm->pdev; 5272 u32 irq_vector, val; 5273 int ret; 5274 5275 val = qm->cap_tables.qm_cap_table[QM_MB_IRQ].cap_val; 5276 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) 5277 return 0; 5278 5279 irq_vector = val & QM_IRQ_VECTOR_MASK; 5280 ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_mb_cmd_irq, 0, qm->dev_name, qm); 5281 if (ret) 5282 dev_err(&pdev->dev, "failed to request function communication irq, ret = %d", ret); 5283 5284 return ret; 5285} 5286 5287static void qm_unregister_aeq_irq(struct hisi_qm *qm) 5288{ 5289 struct pci_dev *pdev = qm->pdev; 5290 u32 irq_vector, val; 5291 5292 val = qm->cap_tables.qm_cap_table[QM_AEQ_IRQ].cap_val; 5293 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) 5294 return; 5295 5296 irq_vector = val & QM_IRQ_VECTOR_MASK; 5297 free_irq(pci_irq_vector(pdev, irq_vector), qm); 5298} 5299 5300static int qm_register_aeq_irq(struct hisi_qm *qm) 5301{ 5302 struct pci_dev *pdev = qm->pdev; 5303 u32 irq_vector, val; 5304 int ret; 5305 5306 val = qm->cap_tables.qm_cap_table[QM_AEQ_IRQ].cap_val; 5307 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) 5308 return 0; 5309 5310 irq_vector = val & QM_IRQ_VECTOR_MASK; 5311 ret = request_threaded_irq(pci_irq_vector(pdev, irq_vector), NULL, 5312 qm_aeq_thread, IRQF_ONESHOT, qm->dev_name, qm); 5313 if (ret) 5314 dev_err(&pdev->dev, "failed to request eq irq, ret = %d", ret); 5315 5316 return ret; 5317} 5318 5319static void qm_unregister_eq_irq(struct hisi_qm *qm) 5320{ 5321 struct pci_dev *pdev = qm->pdev; 5322 u32 irq_vector, val; 5323 5324 val = qm->cap_tables.qm_cap_table[QM_EQ_IRQ].cap_val; 5325 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) 5326 return; 5327 5328 irq_vector = val & QM_IRQ_VECTOR_MASK; 5329 free_irq(pci_irq_vector(pdev, irq_vector), qm); 5330} 5331 5332static int qm_register_eq_irq(struct hisi_qm *qm) 5333{ 5334 struct pci_dev *pdev = qm->pdev; 5335 u32 irq_vector, val; 5336 int ret; 5337 5338 val = qm->cap_tables.qm_cap_table[QM_EQ_IRQ].cap_val; 5339 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) 5340 return 0; 5341 5342 irq_vector = val & QM_IRQ_VECTOR_MASK; 5343 ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_eq_irq, 0, qm->dev_name, qm); 5344 if (ret) 5345 dev_err(&pdev->dev, "failed to request eq irq, ret = %d", ret); 5346 5347 return ret; 5348} 5349 5350static void qm_irqs_unregister(struct hisi_qm *qm) 5351{ 5352 qm_unregister_mb_cmd_irq(qm); 5353 qm_unregister_abnormal_irq(qm); 5354 qm_unregister_aeq_irq(qm); 5355 qm_unregister_eq_irq(qm); 5356} 5357 5358static int qm_irqs_register(struct hisi_qm *qm) 5359{ 5360 int ret; 5361 5362 ret = qm_register_eq_irq(qm); 5363 if (ret) 5364 return ret; 5365 5366 ret = qm_register_aeq_irq(qm); 5367 if (ret) 5368 goto free_eq_irq; 5369 5370 ret = qm_register_abnormal_irq(qm); 5371 if (ret) 5372 goto free_aeq_irq; 5373 5374 ret = qm_register_mb_cmd_irq(qm); 5375 if (ret) 5376 goto free_abnormal_irq; 5377 5378 return 0; 5379 5380free_abnormal_irq: 5381 qm_unregister_abnormal_irq(qm); 5382free_aeq_irq: 5383 qm_unregister_aeq_irq(qm); 5384free_eq_irq: 5385 qm_unregister_eq_irq(qm); 5386 return ret; 5387} 5388 5389static int qm_get_qp_num(struct hisi_qm *qm) 5390{ 5391 struct device *dev = &qm->pdev->dev; 5392 bool is_db_isolation; 5393 5394 /* VF's qp_num assigned by PF in v2, and VF can get qp_num by vft. */ 5395 if (qm->fun_type == QM_HW_VF) { 5396 if (qm->ver != QM_HW_V1) 5397 /* v2 starts to support get vft by mailbox */ 5398 return hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num); 5399 5400 return 0; 5401 } 5402 5403 is_db_isolation = test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps); 5404 qm->ctrl_qp_num = hisi_qm_get_hw_info(qm, qm_basic_info, QM_TOTAL_QP_NUM_CAP, true); 5405 qm->max_qp_num = hisi_qm_get_hw_info(qm, qm_basic_info, 5406 QM_FUNC_MAX_QP_CAP, is_db_isolation); 5407 5408 if (qm->qp_num <= qm->max_qp_num) 5409 return 0; 5410 5411 if (test_bit(QM_MODULE_PARAM, &qm->misc_ctl)) { 5412 /* Check whether the set qp number is valid */ 5413 dev_err(dev, "qp num(%u) is more than max qp num(%u)!\n", 5414 qm->qp_num, qm->max_qp_num); 5415 return -EINVAL; 5416 } 5417 5418 dev_info(dev, "Default qp num(%u) is too big, reset it to Function's max qp num(%u)!\n", 5419 qm->qp_num, qm->max_qp_num); 5420 qm->qp_num = qm->max_qp_num; 5421 qm->debug.curr_qm_qp_num = qm->qp_num; 5422 5423 return 0; 5424} 5425 5426static int qm_pre_store_caps(struct hisi_qm *qm) 5427{ 5428 struct hisi_qm_cap_record *qm_cap; 5429 struct pci_dev *pdev = qm->pdev; 5430 size_t i, size; 5431 5432 size = ARRAY_SIZE(qm_cap_query_info); 5433 qm_cap = devm_kcalloc(&pdev->dev, sizeof(*qm_cap), size, GFP_KERNEL); 5434 if (!qm_cap) 5435 return -ENOMEM; 5436 5437 for (i = 0; i < size; i++) { 5438 qm_cap[i].type = qm_cap_query_info[i].type; 5439 qm_cap[i].name = qm_cap_query_info[i].name; 5440 qm_cap[i].cap_val = hisi_qm_get_cap_value(qm, qm_cap_query_info, 5441 i, qm->cap_ver); 5442 } 5443 5444 qm->cap_tables.qm_cap_table = qm_cap; 5445 qm->cap_tables.qm_cap_size = size; 5446 5447 return 0; 5448} 5449 5450static int qm_get_hw_caps(struct hisi_qm *qm) 5451{ 5452 const struct hisi_qm_cap_info *cap_info = qm->fun_type == QM_HW_PF ? 5453 qm_cap_info_pf : qm_cap_info_vf; 5454 u32 size = qm->fun_type == QM_HW_PF ? ARRAY_SIZE(qm_cap_info_pf) : 5455 ARRAY_SIZE(qm_cap_info_vf); 5456 u32 val, i; 5457 5458 /* Doorbell isolate register is a independent register. */ 5459 val = hisi_qm_get_hw_info(qm, qm_cap_info_comm, QM_SUPPORT_DB_ISOLATION, true); 5460 if (val) 5461 set_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps); 5462 5463 if (qm->ver >= QM_HW_V3) { 5464 val = readl(qm->io_base + QM_FUNC_CAPS_REG); 5465 qm->cap_ver = val & QM_CAPBILITY_VERSION; 5466 } 5467 5468 /* Get PF/VF common capbility */ 5469 for (i = 1; i < ARRAY_SIZE(qm_cap_info_comm); i++) { 5470 val = hisi_qm_get_hw_info(qm, qm_cap_info_comm, i, qm->cap_ver); 5471 if (val) 5472 set_bit(qm_cap_info_comm[i].type, &qm->caps); 5473 } 5474 5475 /* Get PF/VF different capbility */ 5476 for (i = 0; i < size; i++) { 5477 val = hisi_qm_get_hw_info(qm, cap_info, i, qm->cap_ver); 5478 if (val) 5479 set_bit(cap_info[i].type, &qm->caps); 5480 } 5481 5482 /* Fetch and save the value of qm capability registers */ 5483 return qm_pre_store_caps(qm); 5484} 5485 5486static void qm_get_version(struct hisi_qm *qm) 5487{ 5488 struct pci_dev *pdev = qm->pdev; 5489 u32 sub_version_id; 5490 5491 qm->ver = pdev->revision; 5492 5493 if (pdev->revision == QM_HW_V3) { 5494 sub_version_id = readl(qm->io_base + QM_SUB_VERSION_ID); 5495 if (sub_version_id) 5496 qm->ver = sub_version_id; 5497 } 5498} 5499 5500static int qm_get_pci_res(struct hisi_qm *qm) 5501{ 5502 struct pci_dev *pdev = qm->pdev; 5503 struct device *dev = &pdev->dev; 5504 int ret; 5505 5506 ret = pci_request_mem_regions(pdev, qm->dev_name); 5507 if (ret < 0) { 5508 dev_err(dev, "Failed to request mem regions!\n"); 5509 return ret; 5510 } 5511 5512 qm->phys_base = pci_resource_start(pdev, PCI_BAR_2); 5513 qm->io_base = ioremap(qm->phys_base, pci_resource_len(pdev, PCI_BAR_2)); 5514 if (!qm->io_base) { 5515 ret = -EIO; 5516 goto err_request_mem_regions; 5517 } 5518 5519 qm_get_version(qm); 5520 5521 ret = qm_get_hw_caps(qm); 5522 if (ret) 5523 goto err_ioremap; 5524 5525 if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) { 5526 qm->db_interval = QM_QP_DB_INTERVAL; 5527 qm->db_phys_base = pci_resource_start(pdev, PCI_BAR_4); 5528 qm->db_io_base = ioremap(qm->db_phys_base, 5529 pci_resource_len(pdev, PCI_BAR_4)); 5530 if (!qm->db_io_base) { 5531 ret = -EIO; 5532 goto err_ioremap; 5533 } 5534 } else { 5535 qm->db_phys_base = qm->phys_base; 5536 qm->db_io_base = qm->io_base; 5537 qm->db_interval = 0; 5538 } 5539 5540 hisi_qm_pre_init(qm); 5541 ret = qm_get_qp_num(qm); 5542 if (ret) 5543 goto err_db_ioremap; 5544 5545 return 0; 5546 5547err_db_ioremap: 5548 if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) 5549 iounmap(qm->db_io_base); 5550err_ioremap: 5551 iounmap(qm->io_base); 5552err_request_mem_regions: 5553 pci_release_mem_regions(pdev); 5554 return ret; 5555} 5556 5557static int qm_clear_device(struct hisi_qm *qm) 5558{ 5559 acpi_handle handle = ACPI_HANDLE(&qm->pdev->dev); 5560 int ret; 5561 5562 if (qm->fun_type == QM_HW_VF) 5563 return 0; 5564 5565 /* Device does not support reset, return */ 5566 if (!qm->err_ini->err_info_init) 5567 return 0; 5568 qm->err_ini->err_info_init(qm); 5569 5570 if (!handle) 5571 return 0; 5572 5573 /* No reset method, return */ 5574 if (!acpi_has_method(handle, qm->err_info.acpi_rst)) 5575 return 0; 5576 5577 ret = qm_master_ooo_check(qm); 5578 if (ret) { 5579 writel(0x0, qm->io_base + ACC_MASTER_GLOBAL_CTRL); 5580 return ret; 5581 } 5582 5583 if (qm->err_ini->set_priv_status) { 5584 ret = qm->err_ini->set_priv_status(qm); 5585 if (ret) { 5586 writel(0x0, qm->io_base + ACC_MASTER_GLOBAL_CTRL); 5587 return ret; 5588 } 5589 } 5590 5591 return qm_reset_device(qm); 5592} 5593 5594static int hisi_qm_pci_init(struct hisi_qm *qm) 5595{ 5596 struct pci_dev *pdev = qm->pdev; 5597 struct device *dev = &pdev->dev; 5598 unsigned int num_vec; 5599 int ret; 5600 5601 ret = pci_enable_device_mem(pdev); 5602 if (ret < 0) { 5603 dev_err(dev, "Failed to enable device mem!\n"); 5604 return ret; 5605 } 5606 5607 ret = qm_get_pci_res(qm); 5608 if (ret) 5609 goto err_disable_pcidev; 5610 5611 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); 5612 if (ret < 0) 5613 goto err_get_pci_res; 5614 pci_set_master(pdev); 5615 5616 num_vec = qm_get_irq_num(qm); 5617 if (!num_vec) { 5618 dev_err(dev, "Device irq num is zero!\n"); 5619 ret = -EINVAL; 5620 goto err_get_pci_res; 5621 } 5622 num_vec = roundup_pow_of_two(num_vec); 5623 ret = pci_alloc_irq_vectors(pdev, num_vec, num_vec, PCI_IRQ_MSI); 5624 if (ret < 0) { 5625 dev_err(dev, "Failed to enable MSI vectors!\n"); 5626 goto err_get_pci_res; 5627 } 5628 5629 ret = qm_clear_device(qm); 5630 if (ret) 5631 goto err_free_vectors; 5632 5633 return 0; 5634 5635err_free_vectors: 5636 pci_free_irq_vectors(pdev); 5637err_get_pci_res: 5638 qm_put_pci_res(qm); 5639err_disable_pcidev: 5640 pci_disable_device(pdev); 5641 return ret; 5642} 5643 5644static int hisi_qm_init_work(struct hisi_qm *qm) 5645{ 5646 int i; 5647 5648 for (i = 0; i < qm->qp_num; i++) 5649 INIT_WORK(&qm->poll_data[i].work, qm_work_process); 5650 5651 if (qm->fun_type == QM_HW_PF) 5652 INIT_WORK(&qm->rst_work, hisi_qm_controller_reset); 5653 5654 if (qm->ver > QM_HW_V2) 5655 INIT_WORK(&qm->cmd_process, qm_cmd_process); 5656 5657 qm->wq = alloc_workqueue("%s", WQ_HIGHPRI | WQ_MEM_RECLAIM | 5658 WQ_UNBOUND, num_online_cpus(), 5659 pci_name(qm->pdev)); 5660 if (!qm->wq) { 5661 pci_err(qm->pdev, "failed to alloc workqueue!\n"); 5662 return -ENOMEM; 5663 } 5664 5665 return 0; 5666} 5667 5668static int hisi_qp_alloc_memory(struct hisi_qm *qm) 5669{ 5670 struct device *dev = &qm->pdev->dev; 5671 u16 sq_depth, cq_depth; 5672 size_t qp_dma_size; 5673 int i, ret; 5674 5675 qm->qp_array = kcalloc(qm->qp_num, sizeof(struct hisi_qp), GFP_KERNEL); 5676 if (!qm->qp_array) 5677 return -ENOMEM; 5678 5679 qm->poll_data = kcalloc(qm->qp_num, sizeof(struct hisi_qm_poll_data), GFP_KERNEL); 5680 if (!qm->poll_data) { 5681 kfree(qm->qp_array); 5682 return -ENOMEM; 5683 } 5684 5685 qm_get_xqc_depth(qm, &sq_depth, &cq_depth, QM_QP_DEPTH_CAP); 5686 5687 /* one more page for device or qp statuses */ 5688 qp_dma_size = qm->sqe_size * sq_depth + sizeof(struct qm_cqe) * cq_depth; 5689 qp_dma_size = PAGE_ALIGN(qp_dma_size) + PAGE_SIZE; 5690 for (i = 0; i < qm->qp_num; i++) { 5691 qm->poll_data[i].qm = qm; 5692 ret = hisi_qp_memory_init(qm, qp_dma_size, i, sq_depth, cq_depth); 5693 if (ret) 5694 goto err_init_qp_mem; 5695 5696 dev_dbg(dev, "allocate qp dma buf size=%zx)\n", qp_dma_size); 5697 } 5698 5699 return 0; 5700err_init_qp_mem: 5701 hisi_qp_memory_uninit(qm, i); 5702 5703 return ret; 5704} 5705 5706static int hisi_qm_alloc_rsv_buf(struct hisi_qm *qm) 5707{ 5708 struct qm_rsv_buf *xqc_buf = &qm->xqc_buf; 5709 struct qm_dma *xqc_dma = &xqc_buf->qcdma; 5710 struct device *dev = &qm->pdev->dev; 5711 size_t off = 0; 5712 5713#define QM_XQC_BUF_INIT(xqc_buf, type) do { \ 5714 (xqc_buf)->type = ((xqc_buf)->qcdma.va + (off)); \ 5715 (xqc_buf)->type##_dma = (xqc_buf)->qcdma.dma + (off); \ 5716 off += QMC_ALIGN(sizeof(struct qm_##type)); \ 5717} while (0) 5718 5719 xqc_dma->size = QMC_ALIGN(sizeof(struct qm_eqc)) + 5720 QMC_ALIGN(sizeof(struct qm_aeqc)) + 5721 QMC_ALIGN(sizeof(struct qm_sqc)) + 5722 QMC_ALIGN(sizeof(struct qm_cqc)); 5723 xqc_dma->va = dma_alloc_coherent(dev, xqc_dma->size, 5724 &xqc_dma->dma, GFP_KERNEL); 5725 if (!xqc_dma->va) 5726 return -ENOMEM; 5727 5728 QM_XQC_BUF_INIT(xqc_buf, eqc); 5729 QM_XQC_BUF_INIT(xqc_buf, aeqc); 5730 QM_XQC_BUF_INIT(xqc_buf, sqc); 5731 QM_XQC_BUF_INIT(xqc_buf, cqc); 5732 5733 return 0; 5734} 5735 5736static int hisi_qm_memory_init(struct hisi_qm *qm) 5737{ 5738 struct device *dev = &qm->pdev->dev; 5739 int ret, total_func; 5740 size_t off = 0; 5741 5742 if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) { 5743 total_func = pci_sriov_get_totalvfs(qm->pdev) + 1; 5744 qm->factor = kcalloc(total_func, sizeof(struct qm_shaper_factor), GFP_KERNEL); 5745 if (!qm->factor) 5746 return -ENOMEM; 5747 5748 /* Only the PF value needs to be initialized */ 5749 qm->factor[0].func_qos = QM_QOS_MAX_VAL; 5750 } 5751 5752#define QM_INIT_BUF(qm, type, num) do { \ 5753 (qm)->type = ((qm)->qdma.va + (off)); \ 5754 (qm)->type##_dma = (qm)->qdma.dma + (off); \ 5755 off += QMC_ALIGN(sizeof(struct qm_##type) * (num)); \ 5756} while (0) 5757 5758 idr_init(&qm->qp_idr); 5759 qm_get_xqc_depth(qm, &qm->eq_depth, &qm->aeq_depth, QM_XEQ_DEPTH_CAP); 5760 qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * qm->eq_depth) + 5761 QMC_ALIGN(sizeof(struct qm_aeqe) * qm->aeq_depth) + 5762 QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) + 5763 QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num); 5764 qm->qdma.va = dma_alloc_coherent(dev, qm->qdma.size, &qm->qdma.dma, 5765 GFP_ATOMIC); 5766 dev_dbg(dev, "allocate qm dma buf size=%zx)\n", qm->qdma.size); 5767 if (!qm->qdma.va) { 5768 ret = -ENOMEM; 5769 goto err_destroy_idr; 5770 } 5771 5772 QM_INIT_BUF(qm, eqe, qm->eq_depth); 5773 QM_INIT_BUF(qm, aeqe, qm->aeq_depth); 5774 QM_INIT_BUF(qm, sqc, qm->qp_num); 5775 QM_INIT_BUF(qm, cqc, qm->qp_num); 5776 5777 ret = hisi_qm_alloc_rsv_buf(qm); 5778 if (ret) 5779 goto err_free_qdma; 5780 5781 ret = hisi_qp_alloc_memory(qm); 5782 if (ret) 5783 goto err_free_reserve_buf; 5784 5785 return 0; 5786 5787err_free_reserve_buf: 5788 hisi_qm_free_rsv_buf(qm); 5789err_free_qdma: 5790 dma_free_coherent(dev, qm->qdma.size, qm->qdma.va, qm->qdma.dma); 5791err_destroy_idr: 5792 idr_destroy(&qm->qp_idr); 5793 if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) 5794 kfree(qm->factor); 5795 5796 return ret; 5797} 5798 5799/** 5800 * hisi_qm_init() - Initialize configures about qm. 5801 * @qm: The qm needing init. 5802 * 5803 * This function init qm, then we can call hisi_qm_start to put qm into work. 5804 */ 5805int hisi_qm_init(struct hisi_qm *qm) 5806{ 5807 struct pci_dev *pdev = qm->pdev; 5808 struct device *dev = &pdev->dev; 5809 int ret; 5810 5811 ret = hisi_qm_pci_init(qm); 5812 if (ret) 5813 return ret; 5814 5815 ret = qm_irqs_register(qm); 5816 if (ret) 5817 goto err_pci_init; 5818 5819 if (qm->fun_type == QM_HW_PF) { 5820 /* Set the doorbell timeout to QM_DB_TIMEOUT_CFG ns. */ 5821 writel(QM_DB_TIMEOUT_SET, qm->io_base + QM_DB_TIMEOUT_CFG); 5822 qm_disable_clock_gate(qm); 5823 ret = qm_dev_mem_reset(qm); 5824 if (ret) { 5825 dev_err(dev, "failed to reset device memory\n"); 5826 goto err_irq_register; 5827 } 5828 } 5829 5830 if (qm->mode == UACCE_MODE_SVA) { 5831 ret = qm_alloc_uacce(qm); 5832 if (ret < 0) 5833 dev_warn(dev, "fail to alloc uacce (%d)\n", ret); 5834 } 5835 5836 ret = hisi_qm_memory_init(qm); 5837 if (ret) 5838 goto err_alloc_uacce; 5839 5840 ret = hisi_qm_init_work(qm); 5841 if (ret) 5842 goto err_free_qm_memory; 5843 5844 qm_cmd_init(qm); 5845 hisi_mig_region_enable(qm); 5846 5847 return 0; 5848 5849err_free_qm_memory: 5850 hisi_qm_memory_uninit(qm); 5851err_alloc_uacce: 5852 qm_remove_uacce(qm); 5853err_irq_register: 5854 qm_irqs_unregister(qm); 5855err_pci_init: 5856 hisi_qm_pci_uninit(qm); 5857 return ret; 5858} 5859EXPORT_SYMBOL_GPL(hisi_qm_init); 5860 5861/** 5862 * hisi_qm_get_dfx_access() - Try to get dfx access. 5863 * @qm: pointer to accelerator device. 5864 * 5865 * Try to get dfx access, then user can get message. 5866 * 5867 * If device is in suspended, return failure, otherwise 5868 * bump up the runtime PM usage counter. 5869 */ 5870int hisi_qm_get_dfx_access(struct hisi_qm *qm) 5871{ 5872 struct device *dev = &qm->pdev->dev; 5873 5874 if (pm_runtime_suspended(dev)) { 5875 dev_info(dev, "can not read/write - device in suspended.\n"); 5876 return -EAGAIN; 5877 } 5878 5879 return qm_pm_get_sync(qm); 5880} 5881EXPORT_SYMBOL_GPL(hisi_qm_get_dfx_access); 5882 5883/** 5884 * hisi_qm_put_dfx_access() - Put dfx access. 5885 * @qm: pointer to accelerator device. 5886 * 5887 * Put dfx access, drop runtime PM usage counter. 5888 */ 5889void hisi_qm_put_dfx_access(struct hisi_qm *qm) 5890{ 5891 qm_pm_put_sync(qm); 5892} 5893EXPORT_SYMBOL_GPL(hisi_qm_put_dfx_access); 5894 5895/** 5896 * hisi_qm_pm_init() - Initialize qm runtime PM. 5897 * @qm: pointer to accelerator device. 5898 * 5899 * Function that initialize qm runtime PM. 5900 */ 5901void hisi_qm_pm_init(struct hisi_qm *qm) 5902{ 5903 struct device *dev = &qm->pdev->dev; 5904 5905 if (!test_bit(QM_SUPPORT_RPM, &qm->caps)) 5906 return; 5907 5908 pm_runtime_set_autosuspend_delay(dev, QM_AUTOSUSPEND_DELAY); 5909 pm_runtime_use_autosuspend(dev); 5910 pm_runtime_put_noidle(dev); 5911} 5912EXPORT_SYMBOL_GPL(hisi_qm_pm_init); 5913 5914/** 5915 * hisi_qm_pm_uninit() - Uninitialize qm runtime PM. 5916 * @qm: pointer to accelerator device. 5917 * 5918 * Function that uninitialize qm runtime PM. 5919 */ 5920void hisi_qm_pm_uninit(struct hisi_qm *qm) 5921{ 5922 struct device *dev = &qm->pdev->dev; 5923 5924 if (!test_bit(QM_SUPPORT_RPM, &qm->caps)) 5925 return; 5926 5927 pm_runtime_get_noresume(dev); 5928 pm_runtime_dont_use_autosuspend(dev); 5929} 5930EXPORT_SYMBOL_GPL(hisi_qm_pm_uninit); 5931 5932static int qm_prepare_for_suspend(struct hisi_qm *qm) 5933{ 5934 struct pci_dev *pdev = qm->pdev; 5935 int ret; 5936 5937 ret = qm->ops->set_msi(qm, false); 5938 if (ret) { 5939 pci_err(pdev, "failed to disable MSI before suspending!\n"); 5940 return ret; 5941 } 5942 5943 ret = qm_master_ooo_check(qm); 5944 if (ret) 5945 return ret; 5946 5947 if (qm->err_ini->set_priv_status) { 5948 ret = qm->err_ini->set_priv_status(qm); 5949 if (ret) 5950 return ret; 5951 } 5952 5953 ret = qm_set_pf_mse(qm, false); 5954 if (ret) 5955 pci_err(pdev, "failed to disable MSE before suspending!\n"); 5956 5957 return ret; 5958} 5959 5960static int qm_rebuild_for_resume(struct hisi_qm *qm) 5961{ 5962 struct pci_dev *pdev = qm->pdev; 5963 int ret; 5964 5965 ret = qm_set_pf_mse(qm, true); 5966 if (ret) { 5967 pci_err(pdev, "failed to enable MSE after resuming!\n"); 5968 return ret; 5969 } 5970 5971 ret = qm->ops->set_msi(qm, true); 5972 if (ret) { 5973 pci_err(pdev, "failed to enable MSI after resuming!\n"); 5974 return ret; 5975 } 5976 5977 ret = qm_dev_hw_init(qm); 5978 if (ret) { 5979 pci_err(pdev, "failed to init device after resuming\n"); 5980 return ret; 5981 } 5982 5983 qm_cmd_init(qm); 5984 hisi_mig_region_enable(qm); 5985 hisi_qm_dev_err_init(qm); 5986 /* Set the doorbell timeout to QM_DB_TIMEOUT_CFG ns. */ 5987 writel(QM_DB_TIMEOUT_SET, qm->io_base + QM_DB_TIMEOUT_CFG); 5988 qm_disable_clock_gate(qm); 5989 ret = qm_dev_mem_reset(qm); 5990 if (ret) 5991 pci_err(pdev, "failed to reset device memory\n"); 5992 5993 return ret; 5994} 5995 5996/** 5997 * hisi_qm_suspend() - Runtime suspend of given device. 5998 * @dev: device to suspend. 5999 * 6000 * Function that suspend the device. 6001 */ 6002int hisi_qm_suspend(struct device *dev) 6003{ 6004 struct pci_dev *pdev = to_pci_dev(dev); 6005 struct hisi_qm *qm = pci_get_drvdata(pdev); 6006 int ret; 6007 6008 pci_info(pdev, "entering suspended state\n"); 6009 6010 ret = hisi_qm_stop(qm, QM_NORMAL); 6011 if (ret) { 6012 pci_err(pdev, "failed to stop qm(%d)\n", ret); 6013 return ret; 6014 } 6015 6016 ret = qm_prepare_for_suspend(qm); 6017 if (ret) 6018 pci_err(pdev, "failed to prepare suspended(%d)\n", ret); 6019 6020 return ret; 6021} 6022EXPORT_SYMBOL_GPL(hisi_qm_suspend); 6023 6024/** 6025 * hisi_qm_resume() - Runtime resume of given device. 6026 * @dev: device to resume. 6027 * 6028 * Function that resume the device. 6029 */ 6030int hisi_qm_resume(struct device *dev) 6031{ 6032 struct pci_dev *pdev = to_pci_dev(dev); 6033 struct hisi_qm *qm = pci_get_drvdata(pdev); 6034 int ret; 6035 6036 pci_info(pdev, "resuming from suspend state\n"); 6037 6038 ret = qm_rebuild_for_resume(qm); 6039 if (ret) { 6040 pci_err(pdev, "failed to rebuild resume(%d)\n", ret); 6041 return ret; 6042 } 6043 6044 ret = hisi_qm_start(qm); 6045 if (ret) { 6046 if (qm_check_dev_error(qm)) { 6047 pci_info(pdev, "failed to start qm due to device error, device will be reset!\n"); 6048 return 0; 6049 } 6050 6051 pci_err(pdev, "failed to start qm(%d)!\n", ret); 6052 } 6053 6054 return ret; 6055} 6056EXPORT_SYMBOL_GPL(hisi_qm_resume); 6057 6058MODULE_LICENSE("GPL v2"); 6059MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>"); 6060MODULE_DESCRIPTION("HiSilicon Accelerator queue manager driver");
~/
Read-only file preview.