FILE
sec_main.c
File preview with syntax highlighting.
linux-6.19.14/drivers/crypto/hisilicon/sec2/sec_main.c
1// SPDX-License-Identifier: GPL-2.0 2/* Copyright (c) 2019 HiSilicon Limited. */ 3 4#include <linux/acpi.h> 5#include <linux/bitops.h> 6#include <linux/debugfs.h> 7#include <linux/init.h> 8#include <linux/io.h> 9#include <linux/iommu.h> 10#include <linux/kernel.h> 11#include <linux/module.h> 12#include <linux/pci.h> 13#include <linux/pm_runtime.h> 14#include <linux/seq_file.h> 15#include <linux/topology.h> 16#include <linux/uacce.h> 17#include "sec.h" 18 19#define CAP_FILE_PERMISSION 0444 20#define SEC_VF_NUM 63 21#define SEC_QUEUE_NUM_V1 4096 22#define PCI_DEVICE_ID_HUAWEI_SEC_PF 0xa255 23 24#define SEC_BD_ERR_CHK_EN0 0xEFFFFFFF 25#define SEC_BD_ERR_CHK_EN1 0x7ffff7fd 26#define SEC_BD_ERR_CHK_EN3 0xffffbfff 27 28#define SEC_SQE_SIZE 128 29#define SEC_PF_DEF_Q_NUM 256 30#define SEC_PF_DEF_Q_BASE 0 31#define SEC_CTX_Q_NUM_DEF 2 32#define SEC_CTX_Q_NUM_MAX 32 33 34#define SEC_CTRL_CNT_CLR_CE 0x301120 35#define SEC_CTRL_CNT_CLR_CE_BIT BIT(0) 36#define SEC_CORE_INT_SOURCE 0x301010 37#define SEC_CORE_INT_MASK 0x301000 38#define SEC_CORE_INT_STATUS 0x301008 39#define SEC_CORE_SRAM_ECC_ERR_INFO 0x301C14 40#define SEC_ECC_NUM 16 41#define SEC_ECC_MASH 0xFF 42#define SEC_CORE_INT_DISABLE 0x0 43 44#define SEC_RAS_CE_REG 0x301050 45#define SEC_RAS_FE_REG 0x301054 46#define SEC_RAS_NFE_REG 0x301058 47#define SEC_RAS_FE_ENB_MSK 0x0 48#define SEC_OOO_SHUTDOWN_SEL 0x301014 49#define SEC_RAS_DISABLE 0x0 50#define SEC_AXI_ERROR_MASK (BIT(0) | BIT(1)) 51 52#define SEC_MEM_START_INIT_REG 0x301100 53#define SEC_MEM_INIT_DONE_REG 0x301104 54 55/* clock gating */ 56#define SEC_CONTROL_REG 0x301200 57#define SEC_DYNAMIC_GATE_REG 0x30121c 58#define SEC_CORE_AUTO_GATE 0x30212c 59#define SEC_DYNAMIC_GATE_EN 0x7fff 60#define SEC_CORE_AUTO_GATE_EN GENMASK(3, 0) 61#define SEC_CLK_GATE_ENABLE BIT(3) 62#define SEC_CLK_GATE_DISABLE (~BIT(3)) 63 64#define SEC_TRNG_EN_SHIFT 8 65#define SEC_AXI_SHUTDOWN_ENABLE BIT(12) 66#define SEC_AXI_SHUTDOWN_DISABLE 0xFFFFEFFF 67 68#define SEC_INTERFACE_USER_CTRL0_REG 0x301220 69#define SEC_INTERFACE_USER_CTRL1_REG 0x301224 70#define SEC_SAA_EN_REG 0x301270 71#define SEC_BD_ERR_CHK_EN_REG0 0x301380 72#define SEC_BD_ERR_CHK_EN_REG1 0x301384 73#define SEC_BD_ERR_CHK_EN_REG3 0x30138c 74 75#define SEC_USER0_SMMU_NORMAL (BIT(23) | BIT(15)) 76#define SEC_USER1_SMMU_NORMAL (BIT(31) | BIT(23) | BIT(15) | BIT(7)) 77#define SEC_USER1_ENABLE_CONTEXT_SSV BIT(24) 78#define SEC_USER1_ENABLE_DATA_SSV BIT(16) 79#define SEC_USER1_WB_CONTEXT_SSV BIT(8) 80#define SEC_USER1_WB_DATA_SSV BIT(0) 81#define SEC_USER1_SVA_SET (SEC_USER1_ENABLE_CONTEXT_SSV | \ 82 SEC_USER1_ENABLE_DATA_SSV | \ 83 SEC_USER1_WB_CONTEXT_SSV | \ 84 SEC_USER1_WB_DATA_SSV) 85#define SEC_USER1_SMMU_SVA (SEC_USER1_SMMU_NORMAL | SEC_USER1_SVA_SET) 86#define SEC_USER1_SMMU_MASK (~SEC_USER1_SVA_SET) 87#define SEC_INTERFACE_USER_CTRL0_REG_V3 0x302220 88#define SEC_INTERFACE_USER_CTRL1_REG_V3 0x302224 89#define SEC_USER1_SMMU_NORMAL_V3 (BIT(23) | BIT(17) | BIT(11) | BIT(5)) 90#define SEC_USER1_SMMU_MASK_V3 0xFF79E79E 91#define SEC_CORE_INT_STATUS_M_ECC BIT(2) 92 93#define SEC_PREFETCH_CFG 0x301130 94#define SEC_SVA_TRANS 0x301EC4 95#define SEC_PREFETCH_ENABLE (~(BIT(0) | BIT(1) | BIT(11))) 96#define SEC_PREFETCH_DISABLE BIT(1) 97#define SEC_SVA_DISABLE_READY (BIT(7) | BIT(11)) 98#define SEC_SVA_PREFETCH_INFO 0x301ED4 99#define SEC_SVA_STALL_NUM GENMASK(23, 8) 100#define SEC_SVA_PREFETCH_NUM GENMASK(2, 0) 101#define SEC_WAIT_SVA_READY 500000 102#define SEC_READ_SVA_STATUS_TIMES 3 103#define SEC_WAIT_US_MIN 10 104#define SEC_WAIT_US_MAX 20 105#define SEC_WAIT_QP_US_MIN 1000 106#define SEC_WAIT_QP_US_MAX 2000 107#define SEC_MAX_WAIT_TIMES 2000 108 109#define SEC_DELAY_10_US 10 110#define SEC_POLL_TIMEOUT_US 1000 111#define SEC_DBGFS_VAL_MAX_LEN 20 112#define SEC_SINGLE_PORT_MAX_TRANS 0x2060 113 114#define SEC_SQE_MASK_OFFSET 16 115#define SEC_SQE_MASK_LEN 108 116#define SEC_SHAPER_TYPE_RATE 400 117 118#define SEC_DFX_BASE 0x301000 119#define SEC_DFX_CORE 0x302100 120#define SEC_DFX_COMMON1 0x301600 121#define SEC_DFX_COMMON2 0x301C00 122#define SEC_DFX_BASE_LEN 0x9D 123#define SEC_DFX_CORE_LEN 0x32B 124#define SEC_DFX_COMMON1_LEN 0x45 125#define SEC_DFX_COMMON2_LEN 0xBA 126 127#define SEC_ALG_BITMAP_SHIFT 32 128 129#define SEC_CIPHER_BITMAP (GENMASK_ULL(5, 0) | GENMASK_ULL(16, 12) | \ 130 GENMASK(24, 21)) 131#define SEC_DIGEST_BITMAP (GENMASK_ULL(11, 8) | GENMASK_ULL(20, 19) | \ 132 GENMASK_ULL(42, 25)) 133#define SEC_AEAD_BITMAP (GENMASK_ULL(7, 6) | GENMASK_ULL(18, 17) | \ 134 GENMASK_ULL(45, 43)) 135 136struct sec_hw_error { 137 u32 int_msk; 138 const char *msg; 139}; 140 141struct sec_dfx_item { 142 const char *name; 143 u32 offset; 144}; 145 146static const char sec_name[] = "hisi_sec2"; 147static struct dentry *sec_debugfs_root; 148 149static struct hisi_qm_list sec_devices = { 150 .register_to_crypto = sec_register_to_crypto, 151 .unregister_from_crypto = sec_unregister_from_crypto, 152}; 153 154static const struct hisi_qm_cap_info sec_basic_info[] = { 155 {SEC_QM_NFE_MASK_CAP, 0x3124, 0, GENMASK(31, 0), 0x0, 0x1C77, 0x7C77}, 156 {SEC_QM_RESET_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0xC77, 0x6C77}, 157 {SEC_QM_OOO_SHUTDOWN_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0x4, 0x6C77}, 158 {SEC_QM_CE_MASK_CAP, 0x312C, 0, GENMASK(31, 0), 0x0, 0x8, 0x8}, 159 {SEC_NFE_MASK_CAP, 0x3130, 0, GENMASK(31, 0), 0x0, 0x177, 0x60177}, 160 {SEC_RESET_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x177, 0x177}, 161 {SEC_OOO_SHUTDOWN_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x4, 0x177}, 162 {SEC_CE_MASK_CAP, 0x3138, 0, GENMASK(31, 0), 0x0, 0x88, 0xC088}, 163 {SEC_CLUSTER_NUM_CAP, 0x313c, 20, GENMASK(3, 0), 0x1, 0x1, 0x1}, 164 {SEC_CORE_TYPE_NUM_CAP, 0x313c, 16, GENMASK(3, 0), 0x1, 0x1, 0x1}, 165 {SEC_CORE_NUM_CAP, 0x313c, 8, GENMASK(7, 0), 0x4, 0x4, 0x4}, 166 {SEC_CORES_PER_CLUSTER_NUM_CAP, 0x313c, 0, GENMASK(7, 0), 0x4, 0x4, 0x4}, 167 {SEC_CORE_ENABLE_BITMAP, 0x3140, 0, GENMASK(31, 0), 0x17F, 0x17F, 0xF}, 168 {SEC_DRV_ALG_BITMAP_LOW, 0x3144, 0, GENMASK(31, 0), 0x18050CB, 0x18050CB, 0x18670CF}, 169 {SEC_DRV_ALG_BITMAP_HIGH, 0x3148, 0, GENMASK(31, 0), 0x395C, 0x395C, 0x395C}, 170 {SEC_DEV_ALG_BITMAP_LOW, 0x314c, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 171 {SEC_DEV_ALG_BITMAP_HIGH, 0x3150, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, 172 {SEC_CORE1_ALG_BITMAP_LOW, 0x3154, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 173 {SEC_CORE1_ALG_BITMAP_HIGH, 0x3158, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, 174 {SEC_CORE2_ALG_BITMAP_LOW, 0x315c, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 175 {SEC_CORE2_ALG_BITMAP_HIGH, 0x3160, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, 176 {SEC_CORE3_ALG_BITMAP_LOW, 0x3164, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 177 {SEC_CORE3_ALG_BITMAP_HIGH, 0x3168, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, 178 {SEC_CORE4_ALG_BITMAP_LOW, 0x316c, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 179 {SEC_CORE4_ALG_BITMAP_HIGH, 0x3170, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, 180}; 181 182static const struct hisi_qm_cap_query_info sec_cap_query_info[] = { 183 {QM_RAS_NFE_TYPE, "QM_RAS_NFE_TYPE ", 0x3124, 0x0, 0x1C77, 0x7C77}, 184 {QM_RAS_NFE_RESET, "QM_RAS_NFE_RESET ", 0x3128, 0x0, 0xC77, 0x6C77}, 185 {QM_RAS_CE_TYPE, "QM_RAS_CE_TYPE ", 0x312C, 0x0, 0x8, 0x8}, 186 {SEC_RAS_NFE_TYPE, "SEC_RAS_NFE_TYPE ", 0x3130, 0x0, 0x177, 0x60177}, 187 {SEC_RAS_NFE_RESET, "SEC_RAS_NFE_RESET ", 0x3134, 0x0, 0x177, 0x177}, 188 {SEC_RAS_CE_TYPE, "SEC_RAS_CE_TYPE ", 0x3138, 0x0, 0x88, 0xC088}, 189 {SEC_CORE_INFO, "SEC_CORE_INFO ", 0x313c, 0x110404, 0x110404, 0x110404}, 190 {SEC_CORE_EN, "SEC_CORE_EN ", 0x3140, 0x17F, 0x17F, 0xF}, 191 {SEC_DRV_ALG_BITMAP_LOW_TB, "SEC_DRV_ALG_BITMAP_LOW ", 192 0x3144, 0x18050CB, 0x18050CB, 0x18670CF}, 193 {SEC_DRV_ALG_BITMAP_HIGH_TB, "SEC_DRV_ALG_BITMAP_HIGH ", 194 0x3148, 0x395C, 0x395C, 0x395C}, 195 {SEC_ALG_BITMAP_LOW, "SEC_ALG_BITMAP_LOW ", 196 0x314c, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 197 {SEC_ALG_BITMAP_HIGH, "SEC_ALG_BITMAP_HIGH ", 0x3150, 0x3FFF, 0x3FFF, 0x3FFF}, 198 {SEC_CORE1_BITMAP_LOW, "SEC_CORE1_BITMAP_LOW ", 199 0x3154, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 200 {SEC_CORE1_BITMAP_HIGH, "SEC_CORE1_BITMAP_HIGH ", 0x3158, 0x3FFF, 0x3FFF, 0x3FFF}, 201 {SEC_CORE2_BITMAP_LOW, "SEC_CORE2_BITMAP_LOW ", 202 0x315c, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 203 {SEC_CORE2_BITMAP_HIGH, "SEC_CORE2_BITMAP_HIGH ", 0x3160, 0x3FFF, 0x3FFF, 0x3FFF}, 204 {SEC_CORE3_BITMAP_LOW, "SEC_CORE3_BITMAP_LOW ", 205 0x3164, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 206 {SEC_CORE3_BITMAP_HIGH, "SEC_CORE3_BITMAP_HIGH ", 0x3168, 0x3FFF, 0x3FFF, 0x3FFF}, 207 {SEC_CORE4_BITMAP_LOW, "SEC_CORE4_BITMAP_LOW ", 208 0x316c, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 209 {SEC_CORE4_BITMAP_HIGH, "SEC_CORE4_BITMAP_HIGH ", 0x3170, 0x3FFF, 0x3FFF, 0x3FFF}, 210}; 211 212static const struct qm_dev_alg sec_dev_algs[] = { { 213 .alg_msk = SEC_CIPHER_BITMAP, 214 .alg = "cipher\n", 215 }, { 216 .alg_msk = SEC_DIGEST_BITMAP, 217 .alg = "digest\n", 218 }, { 219 .alg_msk = SEC_AEAD_BITMAP, 220 .alg = "aead\n", 221 }, 222}; 223 224static const struct sec_hw_error sec_hw_errors[] = { 225 { 226 .int_msk = BIT(0), 227 .msg = "sec_axi_rresp_err_rint" 228 }, 229 { 230 .int_msk = BIT(1), 231 .msg = "sec_axi_bresp_err_rint" 232 }, 233 { 234 .int_msk = BIT(2), 235 .msg = "sec_ecc_2bit_err_rint" 236 }, 237 { 238 .int_msk = BIT(3), 239 .msg = "sec_ecc_1bit_err_rint" 240 }, 241 { 242 .int_msk = BIT(4), 243 .msg = "sec_req_trng_timeout_rint" 244 }, 245 { 246 .int_msk = BIT(5), 247 .msg = "sec_fsm_hbeat_rint" 248 }, 249 { 250 .int_msk = BIT(6), 251 .msg = "sec_channel_req_rng_timeout_rint" 252 }, 253 { 254 .int_msk = BIT(7), 255 .msg = "sec_bd_err_rint" 256 }, 257 { 258 .int_msk = BIT(8), 259 .msg = "sec_chain_buff_err_rint" 260 }, 261 { 262 .int_msk = BIT(14), 263 .msg = "sec_no_secure_access" 264 }, 265 { 266 .int_msk = BIT(15), 267 .msg = "sec_wrapping_key_auth_err" 268 }, 269 { 270 .int_msk = BIT(16), 271 .msg = "sec_km_key_crc_fail" 272 }, 273 { 274 .int_msk = BIT(17), 275 .msg = "sec_axi_poison_err" 276 }, 277 { 278 .int_msk = BIT(18), 279 .msg = "sec_sva_err" 280 }, 281 {} 282}; 283 284static const char * const sec_dbg_file_name[] = { 285 [SEC_CLEAR_ENABLE] = "clear_enable", 286}; 287 288static struct sec_dfx_item sec_dfx_labels[] = { 289 {"send_cnt", offsetof(struct sec_dfx, send_cnt)}, 290 {"recv_cnt", offsetof(struct sec_dfx, recv_cnt)}, 291 {"send_busy_cnt", offsetof(struct sec_dfx, send_busy_cnt)}, 292 {"recv_busy_cnt", offsetof(struct sec_dfx, recv_busy_cnt)}, 293 {"err_bd_cnt", offsetof(struct sec_dfx, err_bd_cnt)}, 294 {"invalid_req_cnt", offsetof(struct sec_dfx, invalid_req_cnt)}, 295 {"done_flag_cnt", offsetof(struct sec_dfx, done_flag_cnt)}, 296}; 297 298static const struct debugfs_reg32 sec_dfx_regs[] = { 299 {"SEC_PF_ABNORMAL_INT_SOURCE ", 0x301010}, 300 {"SEC_SAA_EN ", 0x301270}, 301 {"SEC_BD_LATENCY_MIN ", 0x301600}, 302 {"SEC_BD_LATENCY_MAX ", 0x301608}, 303 {"SEC_BD_LATENCY_AVG ", 0x30160C}, 304 {"SEC_BD_NUM_IN_SAA0 ", 0x301670}, 305 {"SEC_BD_NUM_IN_SAA1 ", 0x301674}, 306 {"SEC_BD_NUM_IN_SEC ", 0x301680}, 307 {"SEC_ECC_1BIT_CNT ", 0x301C00}, 308 {"SEC_ECC_1BIT_INFO ", 0x301C04}, 309 {"SEC_ECC_2BIT_CNT ", 0x301C10}, 310 {"SEC_ECC_2BIT_INFO ", 0x301C14}, 311 {"SEC_BD_SAA0 ", 0x301C20}, 312 {"SEC_BD_SAA1 ", 0x301C24}, 313 {"SEC_BD_SAA2 ", 0x301C28}, 314 {"SEC_BD_SAA3 ", 0x301C2C}, 315 {"SEC_BD_SAA4 ", 0x301C30}, 316 {"SEC_BD_SAA5 ", 0x301C34}, 317 {"SEC_BD_SAA6 ", 0x301C38}, 318 {"SEC_BD_SAA7 ", 0x301C3C}, 319 {"SEC_BD_SAA8 ", 0x301C40}, 320 {"SEC_RAS_CE_ENABLE ", 0x301050}, 321 {"SEC_RAS_FE_ENABLE ", 0x301054}, 322 {"SEC_RAS_NFE_ENABLE ", 0x301058}, 323 {"SEC_REQ_TRNG_TIME_TH ", 0x30112C}, 324 {"SEC_CHANNEL_RNG_REQ_THLD ", 0x302110}, 325}; 326 327/* define the SEC's dfx regs region and region length */ 328static struct dfx_diff_registers sec_diff_regs[] = { 329 { 330 .reg_offset = SEC_DFX_BASE, 331 .reg_len = SEC_DFX_BASE_LEN, 332 }, { 333 .reg_offset = SEC_DFX_COMMON1, 334 .reg_len = SEC_DFX_COMMON1_LEN, 335 }, { 336 .reg_offset = SEC_DFX_COMMON2, 337 .reg_len = SEC_DFX_COMMON2_LEN, 338 }, { 339 .reg_offset = SEC_DFX_CORE, 340 .reg_len = SEC_DFX_CORE_LEN, 341 }, 342}; 343 344static int sec_diff_regs_show(struct seq_file *s, void *unused) 345{ 346 struct hisi_qm *qm = s->private; 347 348 hisi_qm_acc_diff_regs_dump(qm, s, qm->debug.acc_diff_regs, 349 ARRAY_SIZE(sec_diff_regs)); 350 351 return 0; 352} 353DEFINE_SHOW_ATTRIBUTE(sec_diff_regs); 354 355static bool pf_q_num_flag; 356static int sec_pf_q_num_set(const char *val, const struct kernel_param *kp) 357{ 358 pf_q_num_flag = true; 359 360 return hisi_qm_q_num_set(val, kp, PCI_DEVICE_ID_HUAWEI_SEC_PF); 361} 362 363static const struct kernel_param_ops sec_pf_q_num_ops = { 364 .set = sec_pf_q_num_set, 365 .get = param_get_int, 366}; 367 368static u32 pf_q_num = SEC_PF_DEF_Q_NUM; 369module_param_cb(pf_q_num, &sec_pf_q_num_ops, &pf_q_num, 0444); 370MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 2-4096, v2 2-1024)"); 371 372static int sec_ctx_q_num_set(const char *val, const struct kernel_param *kp) 373{ 374 u32 ctx_q_num; 375 int ret; 376 377 if (!val) 378 return -EINVAL; 379 380 ret = kstrtou32(val, 10, &ctx_q_num); 381 if (ret) 382 return -EINVAL; 383 384 if (!ctx_q_num || ctx_q_num > SEC_CTX_Q_NUM_MAX || ctx_q_num & 0x1) { 385 pr_err("ctx queue num[%u] is invalid!\n", ctx_q_num); 386 return -EINVAL; 387 } 388 389 return param_set_int(val, kp); 390} 391 392static const struct kernel_param_ops sec_ctx_q_num_ops = { 393 .set = sec_ctx_q_num_set, 394 .get = param_get_int, 395}; 396static u32 ctx_q_num = SEC_CTX_Q_NUM_DEF; 397module_param_cb(ctx_q_num, &sec_ctx_q_num_ops, &ctx_q_num, 0444); 398MODULE_PARM_DESC(ctx_q_num, "Queue num in ctx (2 default, 2, 4, ..., 32)"); 399 400static const struct kernel_param_ops vfs_num_ops = { 401 .set = vfs_num_set, 402 .get = param_get_int, 403}; 404 405static u32 vfs_num; 406module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444); 407MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)"); 408 409void sec_destroy_qps(struct hisi_qp **qps, int qp_num) 410{ 411 hisi_qm_free_qps(qps, qp_num); 412 kfree(qps); 413} 414 415struct hisi_qp **sec_create_qps(void) 416{ 417 int node = cpu_to_node(raw_smp_processor_id()); 418 u32 ctx_num = ctx_q_num; 419 struct hisi_qp **qps; 420 u8 *type; 421 int ret; 422 423 qps = kcalloc(ctx_num, sizeof(struct hisi_qp *), GFP_KERNEL); 424 if (!qps) 425 return NULL; 426 427 /* The type of SEC is all 0, so just allocated by kcalloc */ 428 type = kcalloc(ctx_num, sizeof(u8), GFP_KERNEL); 429 if (!type) { 430 kfree(qps); 431 return NULL; 432 } 433 434 ret = hisi_qm_alloc_qps_node(&sec_devices, ctx_num, type, node, qps); 435 if (ret) { 436 kfree(type); 437 kfree(qps); 438 return NULL; 439 } 440 441 kfree(type); 442 return qps; 443} 444 445u64 sec_get_alg_bitmap(struct hisi_qm *qm, u32 high, u32 low) 446{ 447 u32 cap_val_h, cap_val_l; 448 449 cap_val_h = qm->cap_tables.dev_cap_table[high].cap_val; 450 cap_val_l = qm->cap_tables.dev_cap_table[low].cap_val; 451 452 return ((u64)cap_val_h << SEC_ALG_BITMAP_SHIFT) | (u64)cap_val_l; 453} 454 455static const struct kernel_param_ops sec_uacce_mode_ops = { 456 .set = uacce_mode_set, 457 .get = param_get_int, 458}; 459 460/* 461 * uacce_mode = 0 means sec only register to crypto, 462 * uacce_mode = 1 means sec both register to crypto and uacce. 463 */ 464static u32 uacce_mode = UACCE_MODE_NOUACCE; 465module_param_cb(uacce_mode, &sec_uacce_mode_ops, &uacce_mode, 0444); 466MODULE_PARM_DESC(uacce_mode, UACCE_MODE_DESC); 467 468static const struct pci_device_id sec_dev_ids[] = { 469 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_SEC_PF) }, 470 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_SEC_VF) }, 471 { 0, } 472}; 473MODULE_DEVICE_TABLE(pci, sec_dev_ids); 474 475static void sec_set_endian(struct hisi_qm *qm) 476{ 477 u32 reg; 478 479 reg = readl_relaxed(qm->io_base + SEC_CONTROL_REG); 480 reg &= ~(BIT(1) | BIT(0)); 481 if (!IS_ENABLED(CONFIG_64BIT)) 482 reg |= BIT(1); 483 484 if (!IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN)) 485 reg |= BIT(0); 486 487 writel_relaxed(reg, qm->io_base + SEC_CONTROL_REG); 488} 489 490static int sec_wait_sva_ready(struct hisi_qm *qm, __u32 offset, __u32 mask) 491{ 492 u32 val, try_times = 0; 493 u8 count = 0; 494 495 /* 496 * Read the register value every 10-20us. If the value is 0 for three 497 * consecutive times, the SVA module is ready. 498 */ 499 do { 500 val = readl(qm->io_base + offset); 501 if (val & mask) 502 count = 0; 503 else if (++count == SEC_READ_SVA_STATUS_TIMES) 504 break; 505 506 usleep_range(SEC_WAIT_US_MIN, SEC_WAIT_US_MAX); 507 } while (++try_times < SEC_WAIT_SVA_READY); 508 509 if (try_times == SEC_WAIT_SVA_READY) { 510 pci_err(qm->pdev, "failed to wait sva prefetch ready\n"); 511 return -ETIMEDOUT; 512 } 513 514 return 0; 515} 516 517static void sec_close_sva_prefetch(struct hisi_qm *qm) 518{ 519 u32 val; 520 int ret; 521 522 if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) 523 return; 524 525 val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG); 526 val |= SEC_PREFETCH_DISABLE; 527 writel(val, qm->io_base + SEC_PREFETCH_CFG); 528 529 ret = readl_relaxed_poll_timeout(qm->io_base + SEC_SVA_TRANS, 530 val, !(val & SEC_SVA_DISABLE_READY), 531 SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US); 532 if (ret) 533 pci_err(qm->pdev, "failed to close sva prefetch\n"); 534 535 (void)sec_wait_sva_ready(qm, SEC_SVA_PREFETCH_INFO, SEC_SVA_STALL_NUM); 536} 537 538static void sec_open_sva_prefetch(struct hisi_qm *qm) 539{ 540 u32 val; 541 int ret; 542 543 if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) 544 return; 545 546 /* Enable prefetch */ 547 val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG); 548 val &= SEC_PREFETCH_ENABLE; 549 writel(val, qm->io_base + SEC_PREFETCH_CFG); 550 551 ret = readl_relaxed_poll_timeout(qm->io_base + SEC_PREFETCH_CFG, 552 val, !(val & SEC_PREFETCH_DISABLE), 553 SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US); 554 if (ret) { 555 pci_err(qm->pdev, "failed to open sva prefetch\n"); 556 sec_close_sva_prefetch(qm); 557 return; 558 } 559 560 ret = sec_wait_sva_ready(qm, SEC_SVA_TRANS, SEC_SVA_PREFETCH_NUM); 561 if (ret) 562 sec_close_sva_prefetch(qm); 563} 564 565static void sec_engine_sva_config(struct hisi_qm *qm) 566{ 567 u32 reg; 568 569 if (qm->ver > QM_HW_V2) { 570 reg = readl_relaxed(qm->io_base + 571 SEC_INTERFACE_USER_CTRL0_REG_V3); 572 reg |= SEC_USER0_SMMU_NORMAL; 573 writel_relaxed(reg, qm->io_base + 574 SEC_INTERFACE_USER_CTRL0_REG_V3); 575 576 reg = readl_relaxed(qm->io_base + 577 SEC_INTERFACE_USER_CTRL1_REG_V3); 578 reg &= SEC_USER1_SMMU_MASK_V3; 579 reg |= SEC_USER1_SMMU_NORMAL_V3; 580 writel_relaxed(reg, qm->io_base + 581 SEC_INTERFACE_USER_CTRL1_REG_V3); 582 } else { 583 reg = readl_relaxed(qm->io_base + 584 SEC_INTERFACE_USER_CTRL0_REG); 585 reg |= SEC_USER0_SMMU_NORMAL; 586 writel_relaxed(reg, qm->io_base + 587 SEC_INTERFACE_USER_CTRL0_REG); 588 reg = readl_relaxed(qm->io_base + 589 SEC_INTERFACE_USER_CTRL1_REG); 590 reg &= SEC_USER1_SMMU_MASK; 591 if (qm->use_sva) 592 reg |= SEC_USER1_SMMU_SVA; 593 else 594 reg |= SEC_USER1_SMMU_NORMAL; 595 writel_relaxed(reg, qm->io_base + 596 SEC_INTERFACE_USER_CTRL1_REG); 597 } 598 sec_open_sva_prefetch(qm); 599} 600 601static void sec_enable_clock_gate(struct hisi_qm *qm) 602{ 603 u32 val; 604 605 if (qm->ver < QM_HW_V3) 606 return; 607 608 val = readl_relaxed(qm->io_base + SEC_CONTROL_REG); 609 val |= SEC_CLK_GATE_ENABLE; 610 writel_relaxed(val, qm->io_base + SEC_CONTROL_REG); 611 612 val = readl(qm->io_base + SEC_DYNAMIC_GATE_REG); 613 val |= SEC_DYNAMIC_GATE_EN; 614 writel(val, qm->io_base + SEC_DYNAMIC_GATE_REG); 615 616 val = readl(qm->io_base + SEC_CORE_AUTO_GATE); 617 val |= SEC_CORE_AUTO_GATE_EN; 618 writel(val, qm->io_base + SEC_CORE_AUTO_GATE); 619} 620 621static void sec_disable_clock_gate(struct hisi_qm *qm) 622{ 623 u32 val; 624 625 /* Kunpeng920 needs to close clock gating */ 626 val = readl_relaxed(qm->io_base + SEC_CONTROL_REG); 627 val &= SEC_CLK_GATE_DISABLE; 628 writel_relaxed(val, qm->io_base + SEC_CONTROL_REG); 629} 630 631static int sec_engine_init(struct hisi_qm *qm) 632{ 633 int ret; 634 u32 reg; 635 636 /* disable clock gate control before mem init */ 637 sec_disable_clock_gate(qm); 638 639 writel_relaxed(0x1, qm->io_base + SEC_MEM_START_INIT_REG); 640 641 ret = readl_relaxed_poll_timeout(qm->io_base + SEC_MEM_INIT_DONE_REG, 642 reg, reg & 0x1, SEC_DELAY_10_US, 643 SEC_POLL_TIMEOUT_US); 644 if (ret) { 645 pci_err(qm->pdev, "fail to init sec mem\n"); 646 return ret; 647 } 648 649 reg = readl_relaxed(qm->io_base + SEC_CONTROL_REG); 650 reg |= (0x1 << SEC_TRNG_EN_SHIFT); 651 writel_relaxed(reg, qm->io_base + SEC_CONTROL_REG); 652 653 sec_engine_sva_config(qm); 654 655 writel(SEC_SINGLE_PORT_MAX_TRANS, 656 qm->io_base + AM_CFG_SINGLE_PORT_MAX_TRANS); 657 658 reg = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_CORE_ENABLE_BITMAP, qm->cap_ver); 659 writel(reg, qm->io_base + SEC_SAA_EN_REG); 660 661 if (qm->ver < QM_HW_V3) { 662 /* HW V2 enable sm4 extra mode, as ctr/ecb */ 663 writel_relaxed(SEC_BD_ERR_CHK_EN0, 664 qm->io_base + SEC_BD_ERR_CHK_EN_REG0); 665 666 /* HW V2 enable sm4 xts mode multiple iv */ 667 writel_relaxed(SEC_BD_ERR_CHK_EN1, 668 qm->io_base + SEC_BD_ERR_CHK_EN_REG1); 669 writel_relaxed(SEC_BD_ERR_CHK_EN3, 670 qm->io_base + SEC_BD_ERR_CHK_EN_REG3); 671 } 672 673 /* config endian */ 674 sec_set_endian(qm); 675 676 sec_enable_clock_gate(qm); 677 678 return 0; 679} 680 681static int sec_set_user_domain_and_cache(struct hisi_qm *qm) 682{ 683 /* qm user domain */ 684 writel(AXUSER_BASE, qm->io_base + QM_ARUSER_M_CFG_1); 685 writel(ARUSER_M_CFG_ENABLE, qm->io_base + QM_ARUSER_M_CFG_ENABLE); 686 writel(AXUSER_BASE, qm->io_base + QM_AWUSER_M_CFG_1); 687 writel(AWUSER_M_CFG_ENABLE, qm->io_base + QM_AWUSER_M_CFG_ENABLE); 688 writel(WUSER_M_CFG_ENABLE, qm->io_base + QM_WUSER_M_CFG_ENABLE); 689 690 /* qm cache */ 691 writel(AXI_M_CFG, qm->io_base + QM_AXI_M_CFG); 692 writel(AXI_M_CFG_ENABLE, qm->io_base + QM_AXI_M_CFG_ENABLE); 693 694 /* disable FLR triggered by BME(bus master enable) */ 695 writel(PEH_AXUSER_CFG, qm->io_base + QM_PEH_AXUSER_CFG); 696 writel(PEH_AXUSER_CFG_ENABLE, qm->io_base + QM_PEH_AXUSER_CFG_ENABLE); 697 698 /* enable sqc,cqc writeback */ 699 writel(SQC_CACHE_ENABLE | CQC_CACHE_ENABLE | SQC_CACHE_WB_ENABLE | 700 CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) | 701 FIELD_PREP(CQC_CACHE_WB_THRD, 1), qm->io_base + QM_CACHE_CTL); 702 703 return sec_engine_init(qm); 704} 705 706/* sec_debug_regs_clear() - clear the sec debug regs */ 707static void sec_debug_regs_clear(struct hisi_qm *qm) 708{ 709 int i; 710 711 /* clear sec dfx regs */ 712 writel(0x1, qm->io_base + SEC_CTRL_CNT_CLR_CE); 713 for (i = 0; i < ARRAY_SIZE(sec_dfx_regs); i++) 714 readl(qm->io_base + sec_dfx_regs[i].offset); 715 716 /* clear rdclr_en */ 717 writel(0x0, qm->io_base + SEC_CTRL_CNT_CLR_CE); 718 719 hisi_qm_debug_regs_clear(qm); 720} 721 722static void sec_master_ooo_ctrl(struct hisi_qm *qm, bool enable) 723{ 724 u32 val1, val2; 725 726 val1 = readl(qm->io_base + SEC_CONTROL_REG); 727 if (enable) { 728 val1 |= SEC_AXI_SHUTDOWN_ENABLE; 729 val2 = qm->err_info.dev_err.shutdown_mask; 730 } else { 731 val1 &= SEC_AXI_SHUTDOWN_DISABLE; 732 val2 = 0x0; 733 } 734 735 if (qm->ver > QM_HW_V2) 736 writel(val2, qm->io_base + SEC_OOO_SHUTDOWN_SEL); 737 738 writel(val1, qm->io_base + SEC_CONTROL_REG); 739} 740 741static void sec_hw_error_enable(struct hisi_qm *qm) 742{ 743 struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; 744 u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; 745 746 if (qm->ver == QM_HW_V1) { 747 writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK); 748 pci_info(qm->pdev, "V1 not support hw error handle\n"); 749 return; 750 } 751 752 /* clear SEC hw error source if having */ 753 writel(err_mask, qm->io_base + SEC_CORE_INT_SOURCE); 754 755 /* enable RAS int */ 756 writel(dev_err->ce, qm->io_base + SEC_RAS_CE_REG); 757 writel(dev_err->fe, qm->io_base + SEC_RAS_FE_REG); 758 writel(dev_err->nfe, qm->io_base + SEC_RAS_NFE_REG); 759 760 /* enable SEC block master OOO when nfe occurs on Kunpeng930 */ 761 sec_master_ooo_ctrl(qm, true); 762 763 /* enable SEC hw error interrupts */ 764 writel(err_mask, qm->io_base + SEC_CORE_INT_MASK); 765} 766 767static void sec_hw_error_disable(struct hisi_qm *qm) 768{ 769 /* disable SEC hw error interrupts */ 770 writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK); 771 772 /* disable SEC block master OOO when nfe occurs on Kunpeng930 */ 773 sec_master_ooo_ctrl(qm, false); 774 775 /* disable RAS int */ 776 writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_CE_REG); 777 writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_FE_REG); 778 writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_NFE_REG); 779} 780 781static u32 sec_clear_enable_read(struct hisi_qm *qm) 782{ 783 return readl(qm->io_base + SEC_CTRL_CNT_CLR_CE) & 784 SEC_CTRL_CNT_CLR_CE_BIT; 785} 786 787static int sec_clear_enable_write(struct hisi_qm *qm, u32 val) 788{ 789 u32 tmp; 790 791 if (val != 1 && val) 792 return -EINVAL; 793 794 tmp = (readl(qm->io_base + SEC_CTRL_CNT_CLR_CE) & 795 ~SEC_CTRL_CNT_CLR_CE_BIT) | val; 796 writel(tmp, qm->io_base + SEC_CTRL_CNT_CLR_CE); 797 798 return 0; 799} 800 801static ssize_t sec_debug_read(struct file *filp, char __user *buf, 802 size_t count, loff_t *pos) 803{ 804 struct sec_debug_file *file = filp->private_data; 805 char tbuf[SEC_DBGFS_VAL_MAX_LEN]; 806 struct hisi_qm *qm = file->qm; 807 u32 val; 808 int ret; 809 810 ret = hisi_qm_get_dfx_access(qm); 811 if (ret) 812 return ret; 813 814 spin_lock_irq(&file->lock); 815 816 switch (file->index) { 817 case SEC_CLEAR_ENABLE: 818 val = sec_clear_enable_read(qm); 819 break; 820 default: 821 goto err_input; 822 } 823 824 spin_unlock_irq(&file->lock); 825 826 hisi_qm_put_dfx_access(qm); 827 ret = snprintf(tbuf, SEC_DBGFS_VAL_MAX_LEN, "%u\n", val); 828 return simple_read_from_buffer(buf, count, pos, tbuf, ret); 829 830err_input: 831 spin_unlock_irq(&file->lock); 832 hisi_qm_put_dfx_access(qm); 833 return -EINVAL; 834} 835 836static ssize_t sec_debug_write(struct file *filp, const char __user *buf, 837 size_t count, loff_t *pos) 838{ 839 struct sec_debug_file *file = filp->private_data; 840 char tbuf[SEC_DBGFS_VAL_MAX_LEN]; 841 struct hisi_qm *qm = file->qm; 842 unsigned long val; 843 int len, ret; 844 845 if (*pos != 0) 846 return 0; 847 848 if (count >= SEC_DBGFS_VAL_MAX_LEN) 849 return -ENOSPC; 850 851 len = simple_write_to_buffer(tbuf, SEC_DBGFS_VAL_MAX_LEN - 1, 852 pos, buf, count); 853 if (len < 0) 854 return len; 855 856 tbuf[len] = '\0'; 857 if (kstrtoul(tbuf, 0, &val)) 858 return -EFAULT; 859 860 ret = hisi_qm_get_dfx_access(qm); 861 if (ret) 862 return ret; 863 864 spin_lock_irq(&file->lock); 865 866 switch (file->index) { 867 case SEC_CLEAR_ENABLE: 868 ret = sec_clear_enable_write(qm, val); 869 if (ret) 870 goto err_input; 871 break; 872 default: 873 ret = -EINVAL; 874 goto err_input; 875 } 876 877 ret = count; 878 879 err_input: 880 spin_unlock_irq(&file->lock); 881 hisi_qm_put_dfx_access(qm); 882 return ret; 883} 884 885static const struct file_operations sec_dbg_fops = { 886 .owner = THIS_MODULE, 887 .open = simple_open, 888 .read = sec_debug_read, 889 .write = sec_debug_write, 890}; 891 892static int sec_debugfs_atomic64_get(void *data, u64 *val) 893{ 894 *val = atomic64_read((atomic64_t *)data); 895 896 return 0; 897} 898 899static int sec_debugfs_atomic64_set(void *data, u64 val) 900{ 901 if (val) 902 return -EINVAL; 903 904 atomic64_set((atomic64_t *)data, 0); 905 906 return 0; 907} 908 909DEFINE_DEBUGFS_ATTRIBUTE(sec_atomic64_ops, sec_debugfs_atomic64_get, 910 sec_debugfs_atomic64_set, "%lld\n"); 911 912static int sec_regs_show(struct seq_file *s, void *unused) 913{ 914 hisi_qm_regs_dump(s, s->private); 915 916 return 0; 917} 918 919DEFINE_SHOW_ATTRIBUTE(sec_regs); 920 921static int sec_cap_regs_show(struct seq_file *s, void *unused) 922{ 923 struct hisi_qm *qm = s->private; 924 u32 i, size; 925 926 size = qm->cap_tables.qm_cap_size; 927 for (i = 0; i < size; i++) 928 seq_printf(s, "%s= 0x%08x\n", qm->cap_tables.qm_cap_table[i].name, 929 qm->cap_tables.qm_cap_table[i].cap_val); 930 931 size = qm->cap_tables.dev_cap_size; 932 for (i = 0; i < size; i++) 933 seq_printf(s, "%s= 0x%08x\n", qm->cap_tables.dev_cap_table[i].name, 934 qm->cap_tables.dev_cap_table[i].cap_val); 935 936 return 0; 937} 938 939DEFINE_SHOW_ATTRIBUTE(sec_cap_regs); 940 941static int sec_core_debug_init(struct hisi_qm *qm) 942{ 943 struct dfx_diff_registers *sec_regs = qm->debug.acc_diff_regs; 944 struct sec_dev *sec = container_of(qm, struct sec_dev, qm); 945 struct device *dev = &qm->pdev->dev; 946 struct sec_dfx *dfx = &sec->debug.dfx; 947 struct debugfs_regset32 *regset; 948 struct dentry *tmp_d; 949 int i; 950 951 tmp_d = debugfs_create_dir("sec_dfx", qm->debug.debug_root); 952 953 regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL); 954 if (!regset) 955 return -ENOMEM; 956 957 regset->regs = sec_dfx_regs; 958 regset->nregs = ARRAY_SIZE(sec_dfx_regs); 959 regset->base = qm->io_base; 960 regset->dev = dev; 961 962 if (qm->pdev->device == PCI_DEVICE_ID_HUAWEI_SEC_PF) 963 debugfs_create_file("regs", 0444, tmp_d, regset, &sec_regs_fops); 964 if (qm->fun_type == QM_HW_PF && sec_regs) 965 debugfs_create_file("diff_regs", 0444, tmp_d, 966 qm, &sec_diff_regs_fops); 967 968 for (i = 0; i < ARRAY_SIZE(sec_dfx_labels); i++) { 969 atomic64_t *data = (atomic64_t *)((uintptr_t)dfx + 970 sec_dfx_labels[i].offset); 971 debugfs_create_file(sec_dfx_labels[i].name, 0644, 972 tmp_d, data, &sec_atomic64_ops); 973 } 974 975 debugfs_create_file("cap_regs", CAP_FILE_PERMISSION, 976 qm->debug.debug_root, qm, &sec_cap_regs_fops); 977 978 return 0; 979} 980 981static int sec_debug_init(struct hisi_qm *qm) 982{ 983 struct sec_dev *sec = container_of(qm, struct sec_dev, qm); 984 int i; 985 986 if (qm->pdev->device == PCI_DEVICE_ID_HUAWEI_SEC_PF) { 987 for (i = SEC_CLEAR_ENABLE; i < SEC_DEBUG_FILE_NUM; i++) { 988 spin_lock_init(&sec->debug.files[i].lock); 989 sec->debug.files[i].index = i; 990 sec->debug.files[i].qm = qm; 991 992 debugfs_create_file(sec_dbg_file_name[i], 0600, 993 qm->debug.debug_root, 994 sec->debug.files + i, 995 &sec_dbg_fops); 996 } 997 } 998 999 return sec_core_debug_init(qm); 1000} 1001 1002static int sec_debugfs_init(struct hisi_qm *qm) 1003{ 1004 struct device *dev = &qm->pdev->dev; 1005 int ret; 1006 1007 ret = hisi_qm_regs_debugfs_init(qm, sec_diff_regs, ARRAY_SIZE(sec_diff_regs)); 1008 if (ret) { 1009 dev_warn(dev, "Failed to init SEC diff regs!\n"); 1010 return ret; 1011 } 1012 1013 qm->debug.debug_root = debugfs_create_dir(dev_name(dev), 1014 sec_debugfs_root); 1015 qm->debug.sqe_mask_offset = SEC_SQE_MASK_OFFSET; 1016 qm->debug.sqe_mask_len = SEC_SQE_MASK_LEN; 1017 1018 hisi_qm_debug_init(qm); 1019 1020 ret = sec_debug_init(qm); 1021 if (ret) 1022 goto debugfs_remove; 1023 1024 return 0; 1025 1026debugfs_remove: 1027 debugfs_remove_recursive(qm->debug.debug_root); 1028 hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(sec_diff_regs)); 1029 return ret; 1030} 1031 1032static void sec_debugfs_exit(struct hisi_qm *qm) 1033{ 1034 debugfs_remove_recursive(qm->debug.debug_root); 1035 1036 hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(sec_diff_regs)); 1037} 1038 1039static int sec_show_last_regs_init(struct hisi_qm *qm) 1040{ 1041 struct qm_debug *debug = &qm->debug; 1042 int i; 1043 1044 debug->last_words = kcalloc(ARRAY_SIZE(sec_dfx_regs), 1045 sizeof(unsigned int), GFP_KERNEL); 1046 if (!debug->last_words) 1047 return -ENOMEM; 1048 1049 for (i = 0; i < ARRAY_SIZE(sec_dfx_regs); i++) 1050 debug->last_words[i] = readl_relaxed(qm->io_base + 1051 sec_dfx_regs[i].offset); 1052 1053 return 0; 1054} 1055 1056static void sec_show_last_regs_uninit(struct hisi_qm *qm) 1057{ 1058 struct qm_debug *debug = &qm->debug; 1059 1060 if (qm->fun_type == QM_HW_VF || !debug->last_words) 1061 return; 1062 1063 kfree(debug->last_words); 1064 debug->last_words = NULL; 1065} 1066 1067static void sec_show_last_dfx_regs(struct hisi_qm *qm) 1068{ 1069 struct qm_debug *debug = &qm->debug; 1070 struct pci_dev *pdev = qm->pdev; 1071 u32 val; 1072 int i; 1073 1074 if (qm->fun_type == QM_HW_VF || !debug->last_words) 1075 return; 1076 1077 /* dumps last word of the debugging registers during controller reset */ 1078 for (i = 0; i < ARRAY_SIZE(sec_dfx_regs); i++) { 1079 val = readl_relaxed(qm->io_base + sec_dfx_regs[i].offset); 1080 if (val != debug->last_words[i]) 1081 pci_info(pdev, "%s \t= 0x%08x => 0x%08x\n", 1082 sec_dfx_regs[i].name, debug->last_words[i], val); 1083 } 1084} 1085 1086static void sec_log_hw_error(struct hisi_qm *qm, u32 err_sts) 1087{ 1088 const struct sec_hw_error *errs = sec_hw_errors; 1089 struct device *dev = &qm->pdev->dev; 1090 u32 err_val; 1091 1092 while (errs->msg) { 1093 if (errs->int_msk & err_sts) { 1094 dev_err(dev, "%s [error status=0x%x] found\n", 1095 errs->msg, errs->int_msk); 1096 1097 if (SEC_CORE_INT_STATUS_M_ECC & errs->int_msk) { 1098 err_val = readl(qm->io_base + 1099 SEC_CORE_SRAM_ECC_ERR_INFO); 1100 dev_err(dev, "multi ecc sram num=0x%x\n", 1101 ((err_val) >> SEC_ECC_NUM) & 1102 SEC_ECC_MASH); 1103 } 1104 } 1105 errs++; 1106 } 1107} 1108 1109static u32 sec_get_hw_err_status(struct hisi_qm *qm) 1110{ 1111 return readl(qm->io_base + SEC_CORE_INT_STATUS); 1112} 1113 1114static void sec_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) 1115{ 1116 writel(err_sts, qm->io_base + SEC_CORE_INT_SOURCE); 1117} 1118 1119static void sec_disable_error_report(struct hisi_qm *qm, u32 err_type) 1120{ 1121 u32 nfe_mask = qm->err_info.dev_err.nfe; 1122 1123 writel(nfe_mask & (~err_type), qm->io_base + SEC_RAS_NFE_REG); 1124} 1125 1126static void sec_enable_error_report(struct hisi_qm *qm) 1127{ 1128 u32 nfe_mask = qm->err_info.dev_err.nfe; 1129 u32 ce_mask = qm->err_info.dev_err.ce; 1130 1131 writel(nfe_mask, qm->io_base + SEC_RAS_NFE_REG); 1132 writel(ce_mask, qm->io_base + SEC_RAS_CE_REG); 1133} 1134 1135static void sec_open_axi_master_ooo(struct hisi_qm *qm) 1136{ 1137 u32 val; 1138 1139 val = readl(qm->io_base + SEC_CONTROL_REG); 1140 writel(val & SEC_AXI_SHUTDOWN_DISABLE, qm->io_base + SEC_CONTROL_REG); 1141 writel(val | SEC_AXI_SHUTDOWN_ENABLE, qm->io_base + SEC_CONTROL_REG); 1142} 1143 1144static enum acc_err_result sec_get_err_result(struct hisi_qm *qm) 1145{ 1146 u32 err_status; 1147 1148 err_status = sec_get_hw_err_status(qm); 1149 if (err_status) { 1150 if (err_status & qm->err_info.dev_err.ecc_2bits_mask) 1151 qm->err_status.is_dev_ecc_mbit = true; 1152 sec_log_hw_error(qm, err_status); 1153 1154 if (err_status & qm->err_info.dev_err.reset_mask) { 1155 /* Disable the same error reporting until device is recovered. */ 1156 sec_disable_error_report(qm, err_status); 1157 return ACC_ERR_NEED_RESET; 1158 } 1159 sec_clear_hw_err_status(qm, err_status); 1160 /* Avoid firmware disable error report, re-enable. */ 1161 sec_enable_error_report(qm); 1162 } 1163 1164 return ACC_ERR_RECOVERED; 1165} 1166 1167static bool sec_dev_is_abnormal(struct hisi_qm *qm) 1168{ 1169 u32 err_status; 1170 1171 err_status = sec_get_hw_err_status(qm); 1172 if (err_status & qm->err_info.dev_err.shutdown_mask) 1173 return true; 1174 1175 return false; 1176} 1177 1178static void sec_disable_axi_error(struct hisi_qm *qm) 1179{ 1180 struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; 1181 u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; 1182 1183 writel(err_mask & ~SEC_AXI_ERROR_MASK, qm->io_base + SEC_CORE_INT_MASK); 1184 1185 if (qm->ver > QM_HW_V2) 1186 writel(dev_err->shutdown_mask & (~SEC_AXI_ERROR_MASK), 1187 qm->io_base + SEC_OOO_SHUTDOWN_SEL); 1188} 1189 1190static void sec_enable_axi_error(struct hisi_qm *qm) 1191{ 1192 struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err; 1193 u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe; 1194 1195 /* clear axi error source */ 1196 writel(SEC_AXI_ERROR_MASK, qm->io_base + SEC_CORE_INT_SOURCE); 1197 1198 writel(err_mask, qm->io_base + SEC_CORE_INT_MASK); 1199 1200 if (qm->ver > QM_HW_V2) 1201 writel(dev_err->shutdown_mask, qm->io_base + SEC_OOO_SHUTDOWN_SEL); 1202} 1203 1204static void sec_err_info_init(struct hisi_qm *qm) 1205{ 1206 struct hisi_qm_err_info *err_info = &qm->err_info; 1207 struct hisi_qm_err_mask *qm_err = &err_info->qm_err; 1208 struct hisi_qm_err_mask *dev_err = &err_info->dev_err; 1209 1210 qm_err->fe = SEC_RAS_FE_ENB_MSK; 1211 qm_err->ce = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_QM_CE_MASK_CAP, qm->cap_ver); 1212 qm_err->nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_QM_NFE_MASK_CAP, qm->cap_ver); 1213 qm_err->shutdown_mask = hisi_qm_get_hw_info(qm, sec_basic_info, 1214 SEC_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); 1215 qm_err->reset_mask = hisi_qm_get_hw_info(qm, sec_basic_info, 1216 SEC_QM_RESET_MASK_CAP, qm->cap_ver); 1217 qm_err->ecc_2bits_mask = QM_ECC_MBIT; 1218 1219 dev_err->fe = SEC_RAS_FE_ENB_MSK; 1220 dev_err->ce = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_CE_MASK_CAP, qm->cap_ver); 1221 dev_err->nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_NFE_MASK_CAP, qm->cap_ver); 1222 dev_err->shutdown_mask = hisi_qm_get_hw_info(qm, sec_basic_info, 1223 SEC_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); 1224 dev_err->reset_mask = hisi_qm_get_hw_info(qm, sec_basic_info, 1225 SEC_RESET_MASK_CAP, qm->cap_ver); 1226 dev_err->ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC; 1227 1228 err_info->msi_wr_port = BIT(0); 1229 err_info->acpi_rst = "SRST"; 1230} 1231 1232static const struct hisi_qm_err_ini sec_err_ini = { 1233 .hw_init = sec_set_user_domain_and_cache, 1234 .hw_err_enable = sec_hw_error_enable, 1235 .hw_err_disable = sec_hw_error_disable, 1236 .get_dev_hw_err_status = sec_get_hw_err_status, 1237 .clear_dev_hw_err_status = sec_clear_hw_err_status, 1238 .open_axi_master_ooo = sec_open_axi_master_ooo, 1239 .open_sva_prefetch = sec_open_sva_prefetch, 1240 .close_sva_prefetch = sec_close_sva_prefetch, 1241 .show_last_dfx_regs = sec_show_last_dfx_regs, 1242 .err_info_init = sec_err_info_init, 1243 .get_err_result = sec_get_err_result, 1244 .dev_is_abnormal = sec_dev_is_abnormal, 1245 .disable_axi_error = sec_disable_axi_error, 1246 .enable_axi_error = sec_enable_axi_error, 1247}; 1248 1249static int sec_pf_probe_init(struct sec_dev *sec) 1250{ 1251 struct hisi_qm *qm = &sec->qm; 1252 int ret; 1253 1254 ret = sec_set_user_domain_and_cache(qm); 1255 if (ret) 1256 return ret; 1257 1258 hisi_qm_dev_err_init(qm); 1259 sec_debug_regs_clear(qm); 1260 ret = sec_show_last_regs_init(qm); 1261 if (ret) 1262 pci_err(qm->pdev, "Failed to init last word regs!\n"); 1263 1264 return ret; 1265} 1266 1267static int sec_pre_store_cap_reg(struct hisi_qm *qm) 1268{ 1269 struct hisi_qm_cap_record *sec_cap; 1270 struct pci_dev *pdev = qm->pdev; 1271 size_t i, size; 1272 1273 size = ARRAY_SIZE(sec_cap_query_info); 1274 sec_cap = devm_kcalloc(&pdev->dev, size, sizeof(*sec_cap), GFP_KERNEL); 1275 if (!sec_cap) 1276 return -ENOMEM; 1277 1278 for (i = 0; i < size; i++) { 1279 sec_cap[i].type = sec_cap_query_info[i].type; 1280 sec_cap[i].name = sec_cap_query_info[i].name; 1281 sec_cap[i].cap_val = hisi_qm_get_cap_value(qm, sec_cap_query_info, 1282 i, qm->cap_ver); 1283 } 1284 1285 qm->cap_tables.dev_cap_table = sec_cap; 1286 qm->cap_tables.dev_cap_size = size; 1287 1288 return 0; 1289} 1290 1291static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) 1292{ 1293 u64 alg_msk; 1294 int ret; 1295 1296 qm->pdev = pdev; 1297 qm->mode = uacce_mode; 1298 qm->sqe_size = SEC_SQE_SIZE; 1299 qm->dev_name = sec_name; 1300 1301 qm->fun_type = (pdev->device == PCI_DEVICE_ID_HUAWEI_SEC_PF) ? 1302 QM_HW_PF : QM_HW_VF; 1303 if (qm->fun_type == QM_HW_PF) { 1304 qm->qp_base = SEC_PF_DEF_Q_BASE; 1305 qm->qp_num = pf_q_num; 1306 qm->debug.curr_qm_qp_num = pf_q_num; 1307 qm->qm_list = &sec_devices; 1308 qm->err_ini = &sec_err_ini; 1309 if (pf_q_num_flag) 1310 set_bit(QM_MODULE_PARAM, &qm->misc_ctl); 1311 } else if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V1) { 1312 /* 1313 * have no way to get qm configure in VM in v1 hardware, 1314 * so currently force PF to uses SEC_PF_DEF_Q_NUM, and force 1315 * to trigger only one VF in v1 hardware. 1316 * v2 hardware has no such problem. 1317 */ 1318 qm->qp_base = SEC_PF_DEF_Q_NUM; 1319 qm->qp_num = SEC_QUEUE_NUM_V1 - SEC_PF_DEF_Q_NUM; 1320 } 1321 1322 ret = hisi_qm_init(qm); 1323 if (ret) { 1324 pci_err(qm->pdev, "Failed to init sec qm configures!\n"); 1325 return ret; 1326 } 1327 1328 /* Fetch and save the value of capability registers */ 1329 ret = sec_pre_store_cap_reg(qm); 1330 if (ret) { 1331 pci_err(qm->pdev, "Failed to pre-store capability registers!\n"); 1332 hisi_qm_uninit(qm); 1333 return ret; 1334 } 1335 alg_msk = sec_get_alg_bitmap(qm, SEC_ALG_BITMAP_HIGH, SEC_ALG_BITMAP_LOW); 1336 ret = hisi_qm_set_algs(qm, alg_msk, sec_dev_algs, ARRAY_SIZE(sec_dev_algs)); 1337 if (ret) { 1338 pci_err(qm->pdev, "Failed to set sec algs!\n"); 1339 hisi_qm_uninit(qm); 1340 } 1341 1342 return ret; 1343} 1344 1345static void sec_qm_uninit(struct hisi_qm *qm) 1346{ 1347 hisi_qm_uninit(qm); 1348} 1349 1350static int sec_probe_init(struct sec_dev *sec) 1351{ 1352 u32 type_rate = SEC_SHAPER_TYPE_RATE; 1353 struct hisi_qm *qm = &sec->qm; 1354 int ret; 1355 1356 if (qm->fun_type == QM_HW_PF) { 1357 ret = sec_pf_probe_init(sec); 1358 if (ret) 1359 return ret; 1360 /* enable shaper type 0 */ 1361 if (qm->ver >= QM_HW_V3) { 1362 type_rate |= QM_SHAPER_ENABLE; 1363 qm->type_rate = type_rate; 1364 } 1365 } 1366 1367 return 0; 1368} 1369 1370static void sec_probe_uninit(struct hisi_qm *qm) 1371{ 1372 if (qm->fun_type == QM_HW_VF) 1373 return; 1374 1375 sec_debug_regs_clear(qm); 1376 sec_show_last_regs_uninit(qm); 1377 sec_close_sva_prefetch(qm); 1378 hisi_qm_dev_err_uninit(qm); 1379} 1380 1381static void sec_iommu_used_check(struct sec_dev *sec) 1382{ 1383 struct iommu_domain *domain; 1384 struct device *dev = &sec->qm.pdev->dev; 1385 1386 domain = iommu_get_domain_for_dev(dev); 1387 1388 /* Check if iommu is used */ 1389 sec->iommu_used = false; 1390 if (domain) { 1391 if (domain->type & __IOMMU_DOMAIN_PAGING) 1392 sec->iommu_used = true; 1393 dev_info(dev, "SMMU Opened, the iommu type = %u\n", 1394 domain->type); 1395 } 1396} 1397 1398static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) 1399{ 1400 struct sec_dev *sec; 1401 struct hisi_qm *qm; 1402 int ret; 1403 1404 sec = devm_kzalloc(&pdev->dev, sizeof(*sec), GFP_KERNEL); 1405 if (!sec) 1406 return -ENOMEM; 1407 1408 qm = &sec->qm; 1409 ret = sec_qm_init(qm, pdev); 1410 if (ret) { 1411 pci_err(pdev, "Failed to init SEC QM (%d)!\n", ret); 1412 return ret; 1413 } 1414 1415 sec->ctx_q_num = ctx_q_num; 1416 sec_iommu_used_check(sec); 1417 1418 ret = sec_probe_init(sec); 1419 if (ret) { 1420 pci_err(pdev, "Failed to probe!\n"); 1421 goto err_qm_uninit; 1422 } 1423 1424 ret = hisi_qm_start(qm); 1425 if (ret) { 1426 pci_err(pdev, "Failed to start sec qm!\n"); 1427 goto err_probe_uninit; 1428 } 1429 1430 ret = sec_debugfs_init(qm); 1431 if (ret) 1432 pci_warn(pdev, "Failed to init debugfs!\n"); 1433 1434 hisi_qm_add_list(qm, &sec_devices); 1435 ret = hisi_qm_alg_register(qm, &sec_devices, ctx_q_num); 1436 if (ret < 0) { 1437 pr_err("Failed to register driver to crypto.\n"); 1438 goto err_qm_del_list; 1439 } 1440 1441 if (qm->uacce) { 1442 ret = uacce_register(qm->uacce); 1443 if (ret) { 1444 pci_err(pdev, "failed to register uacce (%d)!\n", ret); 1445 goto err_alg_unregister; 1446 } 1447 } 1448 1449 if (qm->fun_type == QM_HW_PF && vfs_num) { 1450 ret = hisi_qm_sriov_enable(pdev, vfs_num); 1451 if (ret < 0) 1452 goto err_alg_unregister; 1453 } 1454 1455 hisi_qm_pm_init(qm); 1456 1457 return 0; 1458 1459err_alg_unregister: 1460 hisi_qm_alg_unregister(qm, &sec_devices, ctx_q_num); 1461err_qm_del_list: 1462 hisi_qm_del_list(qm, &sec_devices); 1463 sec_debugfs_exit(qm); 1464 hisi_qm_stop(qm, QM_NORMAL); 1465err_probe_uninit: 1466 sec_probe_uninit(qm); 1467err_qm_uninit: 1468 sec_qm_uninit(qm); 1469 return ret; 1470} 1471 1472static void sec_remove(struct pci_dev *pdev) 1473{ 1474 struct hisi_qm *qm = pci_get_drvdata(pdev); 1475 1476 hisi_qm_pm_uninit(qm); 1477 hisi_qm_wait_task_finish(qm, &sec_devices); 1478 hisi_qm_alg_unregister(qm, &sec_devices, ctx_q_num); 1479 hisi_qm_del_list(qm, &sec_devices); 1480 1481 if (qm->fun_type == QM_HW_PF && qm->vfs_num) 1482 hisi_qm_sriov_disable(pdev, true); 1483 1484 sec_debugfs_exit(qm); 1485 1486 (void)hisi_qm_stop(qm, QM_NORMAL); 1487 sec_probe_uninit(qm); 1488 1489 sec_qm_uninit(qm); 1490} 1491 1492static const struct dev_pm_ops sec_pm_ops = { 1493 SET_RUNTIME_PM_OPS(hisi_qm_suspend, hisi_qm_resume, NULL) 1494}; 1495 1496static const struct pci_error_handlers sec_err_handler = { 1497 .error_detected = hisi_qm_dev_err_detected, 1498 .slot_reset = hisi_qm_dev_slot_reset, 1499 .reset_prepare = hisi_qm_reset_prepare, 1500 .reset_done = hisi_qm_reset_done, 1501}; 1502 1503static struct pci_driver sec_pci_driver = { 1504 .name = "hisi_sec2", 1505 .id_table = sec_dev_ids, 1506 .probe = sec_probe, 1507 .remove = sec_remove, 1508 .err_handler = &sec_err_handler, 1509 .sriov_configure = IS_ENABLED(CONFIG_PCI_IOV) ? 1510 hisi_qm_sriov_configure : NULL, 1511 .shutdown = hisi_qm_dev_shutdown, 1512 .driver.pm = &sec_pm_ops, 1513}; 1514 1515struct pci_driver *hisi_sec_get_pf_driver(void) 1516{ 1517 return &sec_pci_driver; 1518} 1519EXPORT_SYMBOL_GPL(hisi_sec_get_pf_driver); 1520 1521static void sec_register_debugfs(void) 1522{ 1523 if (!debugfs_initialized()) 1524 return; 1525 1526 sec_debugfs_root = debugfs_create_dir("hisi_sec2", NULL); 1527} 1528 1529static void sec_unregister_debugfs(void) 1530{ 1531 debugfs_remove_recursive(sec_debugfs_root); 1532} 1533 1534static int __init sec_init(void) 1535{ 1536 int ret; 1537 1538 hisi_qm_init_list(&sec_devices); 1539 sec_register_debugfs(); 1540 1541 ret = pci_register_driver(&sec_pci_driver); 1542 if (ret < 0) { 1543 sec_unregister_debugfs(); 1544 pr_err("Failed to register pci driver.\n"); 1545 return ret; 1546 } 1547 1548 return 0; 1549} 1550 1551static void __exit sec_exit(void) 1552{ 1553 pci_unregister_driver(&sec_pci_driver); 1554 sec_unregister_debugfs(); 1555} 1556 1557module_init(sec_init); 1558module_exit(sec_exit); 1559 1560MODULE_LICENSE("GPL v2"); 1561MODULE_AUTHOR("Zaibo Xu <xuzaibo@huawei.com>"); 1562MODULE_AUTHOR("Longfang Liu <liulongfang@huawei.com>"); 1563MODULE_AUTHOR("Kai Ye <yekai13@huawei.com>"); 1564MODULE_AUTHOR("Wei Zhang <zhangwei375@huawei.com>"); 1565MODULE_DESCRIPTION("Driver for HiSilicon SEC accelerator");
~/
Read-only file preview.