FILE
gaudi2.c
File preview with syntax highlighting.
linux-6.19.14/drivers/accel/habanalabs/gaudi2/gaudi2.c
1// SPDX-License-Identifier: GPL-2.0 2 3/* 4 * Copyright 2020-2022 HabanaLabs, Ltd. 5 * All Rights Reserved. 6 */ 7 8#include "gaudi2P.h" 9#include "gaudi2_masks.h" 10#include "../include/gaudi2/gaudi2_special_blocks.h" 11#include "../include/hw_ip/mmu/mmu_general.h" 12#include "../include/hw_ip/mmu/mmu_v2_0.h" 13#include "../include/gaudi2/gaudi2_packets.h" 14#include "../include/gaudi2/gaudi2_reg_map.h" 15#include "../include/gaudi2/gaudi2_async_ids_map_extended.h" 16#include "../include/gaudi2/arc/gaudi2_arc_common_packets.h" 17 18#include <linux/module.h> 19#include <linux/pci.h> 20#include <linux/hwmon.h> 21#include <linux/iommu.h> 22 23#define GAUDI2_DMA_POOL_BLK_SIZE SZ_256 /* 256 bytes */ 24 25#define GAUDI2_RESET_TIMEOUT_MSEC 2000 /* 2000ms */ 26 27#define GAUDI2_RESET_POLL_TIMEOUT_USEC 500000 /* 500ms */ 28#define GAUDI2_PLDM_HRESET_TIMEOUT_MSEC 25000 /* 25s */ 29#define GAUDI2_PLDM_SRESET_TIMEOUT_MSEC 25000 /* 25s */ 30#define GAUDI2_PLDM_RESET_POLL_TIMEOUT_USEC 3000000 /* 3s */ 31#define GAUDI2_RESET_POLL_CNT 3 32#define GAUDI2_RESET_WAIT_MSEC 1 /* 1ms */ 33#define GAUDI2_CPU_RESET_WAIT_MSEC 100 /* 100ms */ 34#define GAUDI2_PLDM_RESET_WAIT_MSEC 1000 /* 1s */ 35#define GAUDI2_CB_POOL_CB_CNT 512 36#define GAUDI2_CB_POOL_CB_SIZE SZ_128K /* 128KB */ 37#define GAUDI2_MSG_TO_CPU_TIMEOUT_USEC 4000000 /* 4s */ 38#define GAUDI2_WAIT_FOR_BL_TIMEOUT_USEC 25000000 /* 25s */ 39#define GAUDI2_TEST_QUEUE_WAIT_USEC 100000 /* 100ms */ 40#define GAUDI2_PLDM_TEST_QUEUE_WAIT_USEC 1000000 /* 1s */ 41 42#define GAUDI2_ALLOC_CPU_MEM_RETRY_CNT 3 43 44/* 45 * since the code already has built-in support for binning of up to MAX_FAULTY_TPCS TPCs 46 * and the code relies on that value (for array size etc..) we define another value 47 * for MAX faulty TPCs which reflects the cluster binning requirements 48 */ 49#define MAX_CLUSTER_BINNING_FAULTY_TPCS 1 50#define MAX_FAULTY_XBARS 1 51#define MAX_FAULTY_EDMAS 1 52#define MAX_FAULTY_DECODERS 1 53 54#define GAUDI2_TPC_FULL_MASK 0x1FFFFFF 55#define GAUDI2_HIF_HMMU_FULL_MASK 0xFFFF 56#define GAUDI2_DECODER_FULL_MASK 0x3FF 57 58#define GAUDI2_NA_EVENT_CAUSE 0xFF 59#define GAUDI2_NUM_OF_QM_ERR_CAUSE 18 60#define GAUDI2_NUM_OF_LOWER_QM_ERR_CAUSE 25 61#define GAUDI2_NUM_OF_QM_ARB_ERR_CAUSE 3 62#define GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE 14 63#define GAUDI2_NUM_OF_CPU_SEI_ERR_CAUSE 3 64#define GAUDI2_NUM_OF_QM_SEI_ERR_CAUSE 2 65#define GAUDI2_NUM_OF_ROT_ERR_CAUSE 22 66#define GAUDI2_NUM_OF_TPC_INTR_CAUSE 31 67#define GAUDI2_NUM_OF_DEC_ERR_CAUSE 25 68#define GAUDI2_NUM_OF_MME_ERR_CAUSE 16 69#define GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE 7 70#define GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE 8 71#define GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE 19 72#define GAUDI2_NUM_OF_HBM_SEI_CAUSE 9 73#define GAUDI2_NUM_OF_SM_SEI_ERR_CAUSE 3 74#define GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE 3 75#define GAUDI2_NUM_OF_PMMU_FATAL_ERR_CAUSE 2 76#define GAUDI2_NUM_OF_HIF_FATAL_ERR_CAUSE 2 77#define GAUDI2_NUM_OF_AXI_DRAIN_ERR_CAUSE 2 78#define GAUDI2_NUM_OF_HBM_MC_SPI_CAUSE 5 79 80#define GAUDI2_MMU_CACHE_INV_TIMEOUT_USEC (MMU_CONFIG_TIMEOUT_USEC * 10) 81#define GAUDI2_PLDM_MMU_TIMEOUT_USEC (MMU_CONFIG_TIMEOUT_USEC * 200) 82#define GAUDI2_ARB_WDT_TIMEOUT (0x1000000) 83 84#define GAUDI2_VDEC_TIMEOUT_USEC 10000 /* 10ms */ 85#define GAUDI2_PLDM_VDEC_TIMEOUT_USEC (GAUDI2_VDEC_TIMEOUT_USEC * 100) 86 87#define KDMA_TIMEOUT_USEC USEC_PER_SEC 88 89#define IS_DMA_IDLE(dma_core_sts0) \ 90 (!((dma_core_sts0) & (DCORE0_EDMA0_CORE_STS0_BUSY_MASK))) 91 92#define IS_DMA_HALTED(dma_core_sts1) \ 93 ((dma_core_sts1) & (DCORE0_EDMA0_CORE_STS1_IS_HALT_MASK)) 94 95#define IS_MME_IDLE(mme_arch_sts) (((mme_arch_sts) & MME_ARCH_IDLE_MASK) == MME_ARCH_IDLE_MASK) 96 97#define IS_TPC_IDLE(tpc_cfg_sts) (((tpc_cfg_sts) & (TPC_IDLE_MASK)) == (TPC_IDLE_MASK)) 98 99#define IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) \ 100 ((((qm_glbl_sts0) & (QM_IDLE_MASK)) == (QM_IDLE_MASK)) && \ 101 (((qm_glbl_sts1) & (QM_ARC_IDLE_MASK)) == (QM_ARC_IDLE_MASK)) && \ 102 (((qm_cgm_sts) & (CGM_IDLE_MASK)) == (CGM_IDLE_MASK))) 103 104#define PCIE_DEC_EN_MASK 0x300 105#define DEC_WORK_STATE_IDLE 0 106#define DEC_WORK_STATE_PEND 3 107#define IS_DEC_IDLE(dec_swreg15) \ 108 (((dec_swreg15) & DCORE0_DEC0_CMD_SWREG15_SW_WORK_STATE_MASK) == DEC_WORK_STATE_IDLE || \ 109 ((dec_swreg15) & DCORE0_DEC0_CMD_SWREG15_SW_WORK_STATE_MASK) == DEC_WORK_STATE_PEND) 110 111/* HBM MMU address scrambling parameters */ 112#define GAUDI2_HBM_MMU_SCRM_MEM_SIZE SZ_8M 113#define GAUDI2_HBM_MMU_SCRM_DIV_SHIFT 26 114#define GAUDI2_HBM_MMU_SCRM_MOD_SHIFT 0 115#define GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK DRAM_VA_HINT_MASK 116#define GAUDI2_COMPENSATE_TLB_PAGE_SIZE_FACTOR 16 117#define MMU_RANGE_INV_VA_LSB_SHIFT 12 118#define MMU_RANGE_INV_VA_MSB_SHIFT 44 119#define MMU_RANGE_INV_EN_SHIFT 0 120#define MMU_RANGE_INV_ASID_EN_SHIFT 1 121#define MMU_RANGE_INV_ASID_SHIFT 2 122 123/* The last SPI_SEI cause bit, "burst_fifo_full", is expected to be triggered in PMMU because it has 124 * a 2 entries FIFO, and hence it is not enabled for it. 125 */ 126#define GAUDI2_PMMU_SPI_SEI_ENABLE_MASK GENMASK(GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE - 2, 0) 127#define GAUDI2_HMMU_SPI_SEI_ENABLE_MASK GENMASK(GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE - 1, 0) 128 129#define GAUDI2_MAX_STRING_LEN 64 130 131#define GAUDI2_VDEC_MSIX_ENTRIES (GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM - \ 132 GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + 1) 133 134#define ENGINE_ID_DCORE_OFFSET (GAUDI2_DCORE1_ENGINE_ID_EDMA_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0) 135 136/* RAZWI initiator coordinates */ 137#define RAZWI_GET_AXUSER_XY(x) \ 138 ((x & 0xF8001FF0) >> 4) 139 140#define RAZWI_GET_AXUSER_LOW_XY(x) \ 141 ((x & 0x00001FF0) >> 4) 142 143#define RAZWI_INITIATOR_AXUER_L_X_SHIFT 0 144#define RAZWI_INITIATOR_AXUER_L_X_MASK 0x1F 145#define RAZWI_INITIATOR_AXUER_L_Y_SHIFT 5 146#define RAZWI_INITIATOR_AXUER_L_Y_MASK 0xF 147 148#define RAZWI_INITIATOR_AXUER_H_X_SHIFT 23 149#define RAZWI_INITIATOR_AXUER_H_X_MASK 0x1F 150 151#define RAZWI_INITIATOR_ID_X_Y_LOW(x, y) \ 152 ((((y) & RAZWI_INITIATOR_AXUER_L_Y_MASK) << RAZWI_INITIATOR_AXUER_L_Y_SHIFT) | \ 153 (((x) & RAZWI_INITIATOR_AXUER_L_X_MASK) << RAZWI_INITIATOR_AXUER_L_X_SHIFT)) 154 155#define RAZWI_INITIATOR_ID_X_HIGH(x) \ 156 (((x) & RAZWI_INITIATOR_AXUER_H_X_MASK) << RAZWI_INITIATOR_AXUER_H_X_SHIFT) 157 158#define RAZWI_INITIATOR_ID_X_Y(xl, yl, xh) \ 159 (RAZWI_INITIATOR_ID_X_Y_LOW(xl, yl) | RAZWI_INITIATOR_ID_X_HIGH(xh)) 160 161#define PSOC_RAZWI_ENG_STR_SIZE 128 162#define PSOC_RAZWI_MAX_ENG_PER_RTR 5 163 164/* HW scrambles only bits 0-25 */ 165#define HW_UNSCRAMBLED_BITS_MASK GENMASK_ULL(63, 26) 166 167#define GAUDI2_GLBL_ERR_MAX_CAUSE_NUM 17 168 169struct gaudi2_razwi_info { 170 u32 axuser_xy; 171 u32 rtr_ctrl; 172 u16 eng_id; 173 char *eng_name; 174}; 175 176static struct gaudi2_razwi_info common_razwi_info[] = { 177 {RAZWI_INITIATOR_ID_X_Y(2, 4, 0), mmDCORE0_RTR0_CTRL_BASE, 178 GAUDI2_DCORE0_ENGINE_ID_DEC_0, "DEC0"}, 179 {RAZWI_INITIATOR_ID_X_Y(2, 4, 4), mmDCORE0_RTR0_CTRL_BASE, 180 GAUDI2_DCORE0_ENGINE_ID_DEC_1, "DEC1"}, 181 {RAZWI_INITIATOR_ID_X_Y(17, 4, 18), mmDCORE1_RTR7_CTRL_BASE, 182 GAUDI2_DCORE1_ENGINE_ID_DEC_0, "DEC2"}, 183 {RAZWI_INITIATOR_ID_X_Y(17, 4, 14), mmDCORE1_RTR7_CTRL_BASE, 184 GAUDI2_DCORE1_ENGINE_ID_DEC_1, "DEC3"}, 185 {RAZWI_INITIATOR_ID_X_Y(2, 11, 0), mmDCORE2_RTR0_CTRL_BASE, 186 GAUDI2_DCORE2_ENGINE_ID_DEC_0, "DEC4"}, 187 {RAZWI_INITIATOR_ID_X_Y(2, 11, 4), mmDCORE2_RTR0_CTRL_BASE, 188 GAUDI2_DCORE2_ENGINE_ID_DEC_1, "DEC5"}, 189 {RAZWI_INITIATOR_ID_X_Y(17, 11, 18), mmDCORE3_RTR7_CTRL_BASE, 190 GAUDI2_DCORE3_ENGINE_ID_DEC_0, "DEC6"}, 191 {RAZWI_INITIATOR_ID_X_Y(17, 11, 14), mmDCORE3_RTR7_CTRL_BASE, 192 GAUDI2_DCORE3_ENGINE_ID_DEC_1, "DEC7"}, 193 {RAZWI_INITIATOR_ID_X_Y(2, 4, 6), mmDCORE0_RTR0_CTRL_BASE, 194 GAUDI2_PCIE_ENGINE_ID_DEC_0, "DEC8"}, 195 {RAZWI_INITIATOR_ID_X_Y(2, 4, 7), mmDCORE0_RTR0_CTRL_BASE, 196 GAUDI2_PCIE_ENGINE_ID_DEC_0, "DEC9"}, 197 {RAZWI_INITIATOR_ID_X_Y(3, 4, 2), mmDCORE0_RTR1_CTRL_BASE, 198 GAUDI2_DCORE0_ENGINE_ID_TPC_0, "TPC0"}, 199 {RAZWI_INITIATOR_ID_X_Y(3, 4, 4), mmDCORE0_RTR1_CTRL_BASE, 200 GAUDI2_DCORE0_ENGINE_ID_TPC_1, "TPC1"}, 201 {RAZWI_INITIATOR_ID_X_Y(4, 4, 2), mmDCORE0_RTR2_CTRL_BASE, 202 GAUDI2_DCORE0_ENGINE_ID_TPC_2, "TPC2"}, 203 {RAZWI_INITIATOR_ID_X_Y(4, 4, 4), mmDCORE0_RTR2_CTRL_BASE, 204 GAUDI2_DCORE0_ENGINE_ID_TPC_3, "TPC3"}, 205 {RAZWI_INITIATOR_ID_X_Y(5, 4, 2), mmDCORE0_RTR3_CTRL_BASE, 206 GAUDI2_DCORE0_ENGINE_ID_TPC_4, "TPC4"}, 207 {RAZWI_INITIATOR_ID_X_Y(5, 4, 4), mmDCORE0_RTR3_CTRL_BASE, 208 GAUDI2_DCORE0_ENGINE_ID_TPC_5, "TPC5"}, 209 {RAZWI_INITIATOR_ID_X_Y(16, 4, 14), mmDCORE1_RTR6_CTRL_BASE, 210 GAUDI2_DCORE1_ENGINE_ID_TPC_0, "TPC6"}, 211 {RAZWI_INITIATOR_ID_X_Y(16, 4, 16), mmDCORE1_RTR6_CTRL_BASE, 212 GAUDI2_DCORE1_ENGINE_ID_TPC_1, "TPC7"}, 213 {RAZWI_INITIATOR_ID_X_Y(15, 4, 14), mmDCORE1_RTR5_CTRL_BASE, 214 GAUDI2_DCORE1_ENGINE_ID_TPC_2, "TPC8"}, 215 {RAZWI_INITIATOR_ID_X_Y(15, 4, 16), mmDCORE1_RTR5_CTRL_BASE, 216 GAUDI2_DCORE1_ENGINE_ID_TPC_3, "TPC9"}, 217 {RAZWI_INITIATOR_ID_X_Y(14, 4, 14), mmDCORE1_RTR4_CTRL_BASE, 218 GAUDI2_DCORE1_ENGINE_ID_TPC_4, "TPC10"}, 219 {RAZWI_INITIATOR_ID_X_Y(14, 4, 16), mmDCORE1_RTR4_CTRL_BASE, 220 GAUDI2_DCORE1_ENGINE_ID_TPC_5, "TPC11"}, 221 {RAZWI_INITIATOR_ID_X_Y(5, 11, 2), mmDCORE2_RTR3_CTRL_BASE, 222 GAUDI2_DCORE2_ENGINE_ID_TPC_0, "TPC12"}, 223 {RAZWI_INITIATOR_ID_X_Y(5, 11, 4), mmDCORE2_RTR3_CTRL_BASE, 224 GAUDI2_DCORE2_ENGINE_ID_TPC_1, "TPC13"}, 225 {RAZWI_INITIATOR_ID_X_Y(4, 11, 2), mmDCORE2_RTR2_CTRL_BASE, 226 GAUDI2_DCORE2_ENGINE_ID_TPC_2, "TPC14"}, 227 {RAZWI_INITIATOR_ID_X_Y(4, 11, 4), mmDCORE2_RTR2_CTRL_BASE, 228 GAUDI2_DCORE2_ENGINE_ID_TPC_3, "TPC15"}, 229 {RAZWI_INITIATOR_ID_X_Y(3, 11, 2), mmDCORE2_RTR1_CTRL_BASE, 230 GAUDI2_DCORE2_ENGINE_ID_TPC_4, "TPC16"}, 231 {RAZWI_INITIATOR_ID_X_Y(3, 11, 4), mmDCORE2_RTR1_CTRL_BASE, 232 GAUDI2_DCORE2_ENGINE_ID_TPC_5, "TPC17"}, 233 {RAZWI_INITIATOR_ID_X_Y(14, 11, 14), mmDCORE3_RTR4_CTRL_BASE, 234 GAUDI2_DCORE3_ENGINE_ID_TPC_0, "TPC18"}, 235 {RAZWI_INITIATOR_ID_X_Y(14, 11, 16), mmDCORE3_RTR4_CTRL_BASE, 236 GAUDI2_DCORE3_ENGINE_ID_TPC_1, "TPC19"}, 237 {RAZWI_INITIATOR_ID_X_Y(15, 11, 14), mmDCORE3_RTR5_CTRL_BASE, 238 GAUDI2_DCORE3_ENGINE_ID_TPC_2, "TPC20"}, 239 {RAZWI_INITIATOR_ID_X_Y(15, 11, 16), mmDCORE3_RTR5_CTRL_BASE, 240 GAUDI2_DCORE3_ENGINE_ID_TPC_3, "TPC21"}, 241 {RAZWI_INITIATOR_ID_X_Y(16, 11, 14), mmDCORE3_RTR6_CTRL_BASE, 242 GAUDI2_DCORE3_ENGINE_ID_TPC_4, "TPC22"}, 243 {RAZWI_INITIATOR_ID_X_Y(16, 11, 16), mmDCORE3_RTR6_CTRL_BASE, 244 GAUDI2_DCORE3_ENGINE_ID_TPC_5, "TPC23"}, 245 {RAZWI_INITIATOR_ID_X_Y(2, 4, 2), mmDCORE0_RTR0_CTRL_BASE, 246 GAUDI2_DCORE3_ENGINE_ID_TPC_5, "TPC24"}, 247 {RAZWI_INITIATOR_ID_X_Y(17, 4, 8), mmDCORE1_RTR7_CTRL_BASE, 248 GAUDI2_ENGINE_ID_NIC0_0, "NIC0"}, 249 {RAZWI_INITIATOR_ID_X_Y(17, 4, 10), mmDCORE1_RTR7_CTRL_BASE, 250 GAUDI2_ENGINE_ID_NIC0_1, "NIC1"}, 251 {RAZWI_INITIATOR_ID_X_Y(17, 4, 12), mmDCORE1_RTR7_CTRL_BASE, 252 GAUDI2_ENGINE_ID_NIC1_0, "NIC2"}, 253 {RAZWI_INITIATOR_ID_X_Y(17, 4, 14), mmDCORE1_RTR7_CTRL_BASE, 254 GAUDI2_ENGINE_ID_NIC1_1, "NIC3"}, 255 {RAZWI_INITIATOR_ID_X_Y(17, 4, 15), mmDCORE1_RTR7_CTRL_BASE, 256 GAUDI2_ENGINE_ID_NIC2_0, "NIC4"}, 257 {RAZWI_INITIATOR_ID_X_Y(2, 11, 2), mmDCORE2_RTR0_CTRL_BASE, 258 GAUDI2_ENGINE_ID_NIC2_1, "NIC5"}, 259 {RAZWI_INITIATOR_ID_X_Y(2, 11, 4), mmDCORE2_RTR0_CTRL_BASE, 260 GAUDI2_ENGINE_ID_NIC3_0, "NIC6"}, 261 {RAZWI_INITIATOR_ID_X_Y(2, 11, 6), mmDCORE2_RTR0_CTRL_BASE, 262 GAUDI2_ENGINE_ID_NIC3_1, "NIC7"}, 263 {RAZWI_INITIATOR_ID_X_Y(2, 11, 8), mmDCORE2_RTR0_CTRL_BASE, 264 GAUDI2_ENGINE_ID_NIC4_0, "NIC8"}, 265 {RAZWI_INITIATOR_ID_X_Y(17, 11, 12), mmDCORE3_RTR7_CTRL_BASE, 266 GAUDI2_ENGINE_ID_NIC4_1, "NIC9"}, 267 {RAZWI_INITIATOR_ID_X_Y(17, 11, 14), mmDCORE3_RTR7_CTRL_BASE, 268 GAUDI2_ENGINE_ID_NIC5_0, "NIC10"}, 269 {RAZWI_INITIATOR_ID_X_Y(17, 11, 16), mmDCORE3_RTR7_CTRL_BASE, 270 GAUDI2_ENGINE_ID_NIC5_1, "NIC11"}, 271 {RAZWI_INITIATOR_ID_X_Y(2, 4, 2), mmDCORE0_RTR0_CTRL_BASE, 272 GAUDI2_ENGINE_ID_PDMA_0, "PDMA0"}, 273 {RAZWI_INITIATOR_ID_X_Y(2, 4, 3), mmDCORE0_RTR0_CTRL_BASE, 274 GAUDI2_ENGINE_ID_PDMA_1, "PDMA1"}, 275 {RAZWI_INITIATOR_ID_X_Y(2, 4, 4), mmDCORE0_RTR0_CTRL_BASE, 276 GAUDI2_ENGINE_ID_SIZE, "PMMU"}, 277 {RAZWI_INITIATOR_ID_X_Y(2, 4, 5), mmDCORE0_RTR0_CTRL_BASE, 278 GAUDI2_ENGINE_ID_SIZE, "PCIE"}, 279 {RAZWI_INITIATOR_ID_X_Y(17, 4, 16), mmDCORE1_RTR7_CTRL_BASE, 280 GAUDI2_ENGINE_ID_ARC_FARM, "ARC_FARM"}, 281 {RAZWI_INITIATOR_ID_X_Y(17, 4, 17), mmDCORE1_RTR7_CTRL_BASE, 282 GAUDI2_ENGINE_ID_KDMA, "KDMA"}, 283 {RAZWI_INITIATOR_ID_X_Y(1, 5, 1), mmSFT0_HBW_RTR_IF1_RTR_CTRL_BASE, 284 GAUDI2_DCORE0_ENGINE_ID_EDMA_0, "EDMA0"}, 285 {RAZWI_INITIATOR_ID_X_Y(1, 5, 1), mmSFT0_HBW_RTR_IF0_RTR_CTRL_BASE, 286 GAUDI2_DCORE0_ENGINE_ID_EDMA_1, "EDMA1"}, 287 {RAZWI_INITIATOR_ID_X_Y(18, 5, 18), mmSFT1_HBW_RTR_IF1_RTR_CTRL_BASE, 288 GAUDI2_DCORE1_ENGINE_ID_EDMA_0, "EDMA2"}, 289 {RAZWI_INITIATOR_ID_X_Y(18, 5, 18), mmSFT1_HBW_RTR_IF0_RTR_CTRL_BASE, 290 GAUDI2_DCORE1_ENGINE_ID_EDMA_1, "EDMA3"}, 291 {RAZWI_INITIATOR_ID_X_Y(1, 10, 1), mmSFT2_HBW_RTR_IF0_RTR_CTRL_BASE, 292 GAUDI2_DCORE2_ENGINE_ID_EDMA_0, "EDMA4"}, 293 {RAZWI_INITIATOR_ID_X_Y(1, 10, 1), mmSFT2_HBW_RTR_IF1_RTR_CTRL_BASE, 294 GAUDI2_DCORE2_ENGINE_ID_EDMA_1, "EDMA5"}, 295 {RAZWI_INITIATOR_ID_X_Y(18, 10, 18), mmSFT2_HBW_RTR_IF0_RTR_CTRL_BASE, 296 GAUDI2_DCORE3_ENGINE_ID_EDMA_0, "EDMA6"}, 297 {RAZWI_INITIATOR_ID_X_Y(18, 10, 18), mmSFT2_HBW_RTR_IF1_RTR_CTRL_BASE, 298 GAUDI2_DCORE3_ENGINE_ID_EDMA_1, "EDMA7"}, 299 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE, 300 GAUDI2_ENGINE_ID_SIZE, "HMMU0"}, 301 {RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE, 302 GAUDI2_ENGINE_ID_SIZE, "HMMU1"}, 303 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE, 304 GAUDI2_ENGINE_ID_SIZE, "HMMU2"}, 305 {RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE, 306 GAUDI2_ENGINE_ID_SIZE, "HMMU3"}, 307 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE, 308 GAUDI2_ENGINE_ID_SIZE, "HMMU4"}, 309 {RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE, 310 GAUDI2_ENGINE_ID_SIZE, "HMMU5"}, 311 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE, 312 GAUDI2_ENGINE_ID_SIZE, "HMMU6"}, 313 {RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE, 314 GAUDI2_ENGINE_ID_SIZE, "HMMU7"}, 315 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE, 316 GAUDI2_ENGINE_ID_SIZE, "HMMU8"}, 317 {RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE, 318 GAUDI2_ENGINE_ID_SIZE, "HMMU9"}, 319 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE, 320 GAUDI2_ENGINE_ID_SIZE, "HMMU10"}, 321 {RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE, 322 GAUDI2_ENGINE_ID_SIZE, "HMMU11"}, 323 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE, 324 GAUDI2_ENGINE_ID_SIZE, "HMMU12"}, 325 {RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE, 326 GAUDI2_ENGINE_ID_SIZE, "HMMU13"}, 327 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE, 328 GAUDI2_ENGINE_ID_SIZE, "HMMU14"}, 329 {RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE, 330 GAUDI2_ENGINE_ID_SIZE, "HMMU15"}, 331 {RAZWI_INITIATOR_ID_X_Y(2, 11, 2), mmDCORE2_RTR0_CTRL_BASE, 332 GAUDI2_ENGINE_ID_ROT_0, "ROT0"}, 333 {RAZWI_INITIATOR_ID_X_Y(17, 11, 16), mmDCORE3_RTR7_CTRL_BASE, 334 GAUDI2_ENGINE_ID_ROT_1, "ROT1"}, 335 {RAZWI_INITIATOR_ID_X_Y(2, 11, 2), mmDCORE2_RTR0_CTRL_BASE, 336 GAUDI2_ENGINE_ID_PSOC, "CPU"}, 337 {RAZWI_INITIATOR_ID_X_Y(17, 11, 11), mmDCORE3_RTR7_CTRL_BASE, 338 GAUDI2_ENGINE_ID_PSOC, "PSOC"} 339}; 340 341static struct gaudi2_razwi_info mme_razwi_info[] = { 342 /* MME X high coordinate is N/A, hence using only low coordinates */ 343 {RAZWI_INITIATOR_ID_X_Y_LOW(7, 4), mmDCORE0_RTR5_CTRL_BASE, 344 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_WAP0"}, 345 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 4), mmDCORE0_RTR7_CTRL_BASE, 346 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_WAP1"}, 347 {RAZWI_INITIATOR_ID_X_Y_LOW(8, 4), mmDCORE0_RTR6_CTRL_BASE, 348 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_CTRL_WR"}, 349 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 4), mmDCORE0_RTR7_CTRL_BASE, 350 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_CTRL_RD"}, 351 {RAZWI_INITIATOR_ID_X_Y_LOW(6, 4), mmDCORE0_RTR4_CTRL_BASE, 352 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE0"}, 353 {RAZWI_INITIATOR_ID_X_Y_LOW(6, 4), mmDCORE0_RTR4_CTRL_BASE, 354 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE1"}, 355 {RAZWI_INITIATOR_ID_X_Y_LOW(7, 4), mmDCORE0_RTR5_CTRL_BASE, 356 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE2"}, 357 {RAZWI_INITIATOR_ID_X_Y_LOW(8, 4), mmDCORE0_RTR6_CTRL_BASE, 358 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE3"}, 359 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 4), mmDCORE0_RTR7_CTRL_BASE, 360 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE4"}, 361 {RAZWI_INITIATOR_ID_X_Y_LOW(12, 4), mmDCORE1_RTR2_CTRL_BASE, 362 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_WAP0"}, 363 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 4), mmDCORE1_RTR0_CTRL_BASE, 364 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_WAP1"}, 365 {RAZWI_INITIATOR_ID_X_Y_LOW(11, 4), mmDCORE1_RTR1_CTRL_BASE, 366 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_CTRL_WR"}, 367 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 4), mmDCORE1_RTR0_CTRL_BASE, 368 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_CTRL_RD"}, 369 {RAZWI_INITIATOR_ID_X_Y_LOW(13, 4), mmDCORE1_RTR3_CTRL_BASE, 370 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE0"}, 371 {RAZWI_INITIATOR_ID_X_Y_LOW(13, 4), mmDCORE1_RTR3_CTRL_BASE, 372 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE1"}, 373 {RAZWI_INITIATOR_ID_X_Y_LOW(12, 4), mmDCORE1_RTR2_CTRL_BASE, 374 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE2"}, 375 {RAZWI_INITIATOR_ID_X_Y_LOW(11, 4), mmDCORE1_RTR1_CTRL_BASE, 376 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE3"}, 377 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 4), mmDCORE1_RTR0_CTRL_BASE, 378 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE4"}, 379 {RAZWI_INITIATOR_ID_X_Y_LOW(7, 11), mmDCORE2_RTR5_CTRL_BASE, 380 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_WAP0"}, 381 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 11), mmDCORE2_RTR7_CTRL_BASE, 382 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_WAP1"}, 383 {RAZWI_INITIATOR_ID_X_Y_LOW(8, 11), mmDCORE2_RTR6_CTRL_BASE, 384 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_CTRL_WR"}, 385 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 11), mmDCORE2_RTR7_CTRL_BASE, 386 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_CTRL_RD"}, 387 {RAZWI_INITIATOR_ID_X_Y_LOW(6, 11), mmDCORE2_RTR4_CTRL_BASE, 388 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE0"}, 389 {RAZWI_INITIATOR_ID_X_Y_LOW(6, 11), mmDCORE2_RTR4_CTRL_BASE, 390 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE1"}, 391 {RAZWI_INITIATOR_ID_X_Y_LOW(7, 11), mmDCORE2_RTR5_CTRL_BASE, 392 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE2"}, 393 {RAZWI_INITIATOR_ID_X_Y_LOW(8, 11), mmDCORE2_RTR6_CTRL_BASE, 394 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE3"}, 395 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 11), mmDCORE2_RTR7_CTRL_BASE, 396 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE4"}, 397 {RAZWI_INITIATOR_ID_X_Y_LOW(12, 11), mmDCORE3_RTR2_CTRL_BASE, 398 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_WAP0"}, 399 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 11), mmDCORE3_RTR0_CTRL_BASE, 400 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_WAP1"}, 401 {RAZWI_INITIATOR_ID_X_Y_LOW(11, 11), mmDCORE3_RTR1_CTRL_BASE, 402 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_CTRL_WR"}, 403 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 11), mmDCORE3_RTR0_CTRL_BASE, 404 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_CTRL_RD"}, 405 {RAZWI_INITIATOR_ID_X_Y_LOW(13, 11), mmDCORE3_RTR3_CTRL_BASE, 406 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE0"}, 407 {RAZWI_INITIATOR_ID_X_Y_LOW(13, 11), mmDCORE3_RTR3_CTRL_BASE, 408 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE1"}, 409 {RAZWI_INITIATOR_ID_X_Y_LOW(12, 11), mmDCORE3_RTR2_CTRL_BASE, 410 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE2"}, 411 {RAZWI_INITIATOR_ID_X_Y_LOW(11, 11), mmDCORE3_RTR1_CTRL_BASE, 412 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE3"}, 413 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 11), mmDCORE3_RTR0_CTRL_BASE, 414 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE4"} 415}; 416 417enum hl_pmmu_fatal_cause { 418 LATENCY_RD_OUT_FIFO_OVERRUN, 419 LATENCY_WR_OUT_FIFO_OVERRUN, 420}; 421 422enum hl_pcie_drain_ind_cause { 423 LBW_AXI_DRAIN_IND, 424 HBW_AXI_DRAIN_IND 425}; 426 427static const u32 cluster_hmmu_hif_enabled_mask[GAUDI2_HBM_NUM] = { 428 [HBM_ID0] = 0xFFFC, 429 [HBM_ID1] = 0xFFCF, 430 [HBM_ID2] = 0xF7F7, 431 [HBM_ID3] = 0x7F7F, 432 [HBM_ID4] = 0xFCFF, 433 [HBM_ID5] = 0xCFFF, 434}; 435 436static const u8 xbar_edge_to_hbm_cluster[EDMA_ID_SIZE] = { 437 [0] = HBM_ID0, 438 [1] = HBM_ID1, 439 [2] = HBM_ID4, 440 [3] = HBM_ID5, 441}; 442 443static const u8 edma_to_hbm_cluster[EDMA_ID_SIZE] = { 444 [EDMA_ID_DCORE0_INSTANCE0] = HBM_ID0, 445 [EDMA_ID_DCORE0_INSTANCE1] = HBM_ID2, 446 [EDMA_ID_DCORE1_INSTANCE0] = HBM_ID1, 447 [EDMA_ID_DCORE1_INSTANCE1] = HBM_ID3, 448 [EDMA_ID_DCORE2_INSTANCE0] = HBM_ID2, 449 [EDMA_ID_DCORE2_INSTANCE1] = HBM_ID4, 450 [EDMA_ID_DCORE3_INSTANCE0] = HBM_ID3, 451 [EDMA_ID_DCORE3_INSTANCE1] = HBM_ID5, 452}; 453 454static const int gaudi2_qman_async_event_id[] = { 455 [GAUDI2_QUEUE_ID_PDMA_0_0] = GAUDI2_EVENT_PDMA0_QM, 456 [GAUDI2_QUEUE_ID_PDMA_0_1] = GAUDI2_EVENT_PDMA0_QM, 457 [GAUDI2_QUEUE_ID_PDMA_0_2] = GAUDI2_EVENT_PDMA0_QM, 458 [GAUDI2_QUEUE_ID_PDMA_0_3] = GAUDI2_EVENT_PDMA0_QM, 459 [GAUDI2_QUEUE_ID_PDMA_1_0] = GAUDI2_EVENT_PDMA1_QM, 460 [GAUDI2_QUEUE_ID_PDMA_1_1] = GAUDI2_EVENT_PDMA1_QM, 461 [GAUDI2_QUEUE_ID_PDMA_1_2] = GAUDI2_EVENT_PDMA1_QM, 462 [GAUDI2_QUEUE_ID_PDMA_1_3] = GAUDI2_EVENT_PDMA1_QM, 463 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0] = GAUDI2_EVENT_HDMA0_QM, 464 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_1] = GAUDI2_EVENT_HDMA0_QM, 465 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_2] = GAUDI2_EVENT_HDMA0_QM, 466 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = GAUDI2_EVENT_HDMA0_QM, 467 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0] = GAUDI2_EVENT_HDMA1_QM, 468 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_1] = GAUDI2_EVENT_HDMA1_QM, 469 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_2] = GAUDI2_EVENT_HDMA1_QM, 470 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = GAUDI2_EVENT_HDMA1_QM, 471 [GAUDI2_QUEUE_ID_DCORE0_MME_0_0] = GAUDI2_EVENT_MME0_QM, 472 [GAUDI2_QUEUE_ID_DCORE0_MME_0_1] = GAUDI2_EVENT_MME0_QM, 473 [GAUDI2_QUEUE_ID_DCORE0_MME_0_2] = GAUDI2_EVENT_MME0_QM, 474 [GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = GAUDI2_EVENT_MME0_QM, 475 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_0] = GAUDI2_EVENT_TPC0_QM, 476 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_1] = GAUDI2_EVENT_TPC0_QM, 477 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_2] = GAUDI2_EVENT_TPC0_QM, 478 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = GAUDI2_EVENT_TPC0_QM, 479 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_0] = GAUDI2_EVENT_TPC1_QM, 480 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_1] = GAUDI2_EVENT_TPC1_QM, 481 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_2] = GAUDI2_EVENT_TPC1_QM, 482 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = GAUDI2_EVENT_TPC1_QM, 483 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_0] = GAUDI2_EVENT_TPC2_QM, 484 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_1] = GAUDI2_EVENT_TPC2_QM, 485 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_2] = GAUDI2_EVENT_TPC2_QM, 486 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = GAUDI2_EVENT_TPC2_QM, 487 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_0] = GAUDI2_EVENT_TPC3_QM, 488 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_1] = GAUDI2_EVENT_TPC3_QM, 489 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_2] = GAUDI2_EVENT_TPC3_QM, 490 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = GAUDI2_EVENT_TPC3_QM, 491 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_0] = GAUDI2_EVENT_TPC4_QM, 492 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_1] = GAUDI2_EVENT_TPC4_QM, 493 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_2] = GAUDI2_EVENT_TPC4_QM, 494 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = GAUDI2_EVENT_TPC4_QM, 495 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_0] = GAUDI2_EVENT_TPC5_QM, 496 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_1] = GAUDI2_EVENT_TPC5_QM, 497 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_2] = GAUDI2_EVENT_TPC5_QM, 498 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = GAUDI2_EVENT_TPC5_QM, 499 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_0] = GAUDI2_EVENT_TPC24_QM, 500 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_1] = GAUDI2_EVENT_TPC24_QM, 501 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_2] = GAUDI2_EVENT_TPC24_QM, 502 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = GAUDI2_EVENT_TPC24_QM, 503 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0] = GAUDI2_EVENT_HDMA2_QM, 504 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_1] = GAUDI2_EVENT_HDMA2_QM, 505 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_2] = GAUDI2_EVENT_HDMA2_QM, 506 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = GAUDI2_EVENT_HDMA2_QM, 507 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0] = GAUDI2_EVENT_HDMA3_QM, 508 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_1] = GAUDI2_EVENT_HDMA3_QM, 509 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_2] = GAUDI2_EVENT_HDMA3_QM, 510 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = GAUDI2_EVENT_HDMA3_QM, 511 [GAUDI2_QUEUE_ID_DCORE1_MME_0_0] = GAUDI2_EVENT_MME1_QM, 512 [GAUDI2_QUEUE_ID_DCORE1_MME_0_1] = GAUDI2_EVENT_MME1_QM, 513 [GAUDI2_QUEUE_ID_DCORE1_MME_0_2] = GAUDI2_EVENT_MME1_QM, 514 [GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = GAUDI2_EVENT_MME1_QM, 515 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_0] = GAUDI2_EVENT_TPC6_QM, 516 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_1] = GAUDI2_EVENT_TPC6_QM, 517 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_2] = GAUDI2_EVENT_TPC6_QM, 518 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = GAUDI2_EVENT_TPC6_QM, 519 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_0] = GAUDI2_EVENT_TPC7_QM, 520 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_1] = GAUDI2_EVENT_TPC7_QM, 521 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_2] = GAUDI2_EVENT_TPC7_QM, 522 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = GAUDI2_EVENT_TPC7_QM, 523 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_0] = GAUDI2_EVENT_TPC8_QM, 524 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_1] = GAUDI2_EVENT_TPC8_QM, 525 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_2] = GAUDI2_EVENT_TPC8_QM, 526 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = GAUDI2_EVENT_TPC8_QM, 527 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_0] = GAUDI2_EVENT_TPC9_QM, 528 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_1] = GAUDI2_EVENT_TPC9_QM, 529 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_2] = GAUDI2_EVENT_TPC9_QM, 530 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = GAUDI2_EVENT_TPC9_QM, 531 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_0] = GAUDI2_EVENT_TPC10_QM, 532 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_1] = GAUDI2_EVENT_TPC10_QM, 533 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_2] = GAUDI2_EVENT_TPC10_QM, 534 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = GAUDI2_EVENT_TPC10_QM, 535 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_0] = GAUDI2_EVENT_TPC11_QM, 536 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_1] = GAUDI2_EVENT_TPC11_QM, 537 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_2] = GAUDI2_EVENT_TPC11_QM, 538 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = GAUDI2_EVENT_TPC11_QM, 539 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0] = GAUDI2_EVENT_HDMA4_QM, 540 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_1] = GAUDI2_EVENT_HDMA4_QM, 541 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_2] = GAUDI2_EVENT_HDMA4_QM, 542 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = GAUDI2_EVENT_HDMA4_QM, 543 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0] = GAUDI2_EVENT_HDMA5_QM, 544 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_1] = GAUDI2_EVENT_HDMA5_QM, 545 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_2] = GAUDI2_EVENT_HDMA5_QM, 546 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = GAUDI2_EVENT_HDMA5_QM, 547 [GAUDI2_QUEUE_ID_DCORE2_MME_0_0] = GAUDI2_EVENT_MME2_QM, 548 [GAUDI2_QUEUE_ID_DCORE2_MME_0_1] = GAUDI2_EVENT_MME2_QM, 549 [GAUDI2_QUEUE_ID_DCORE2_MME_0_2] = GAUDI2_EVENT_MME2_QM, 550 [GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = GAUDI2_EVENT_MME2_QM, 551 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_0] = GAUDI2_EVENT_TPC12_QM, 552 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_1] = GAUDI2_EVENT_TPC12_QM, 553 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_2] = GAUDI2_EVENT_TPC12_QM, 554 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = GAUDI2_EVENT_TPC12_QM, 555 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_0] = GAUDI2_EVENT_TPC13_QM, 556 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_1] = GAUDI2_EVENT_TPC13_QM, 557 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_2] = GAUDI2_EVENT_TPC13_QM, 558 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = GAUDI2_EVENT_TPC13_QM, 559 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_0] = GAUDI2_EVENT_TPC14_QM, 560 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_1] = GAUDI2_EVENT_TPC14_QM, 561 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_2] = GAUDI2_EVENT_TPC14_QM, 562 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = GAUDI2_EVENT_TPC14_QM, 563 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_0] = GAUDI2_EVENT_TPC15_QM, 564 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_1] = GAUDI2_EVENT_TPC15_QM, 565 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_2] = GAUDI2_EVENT_TPC15_QM, 566 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = GAUDI2_EVENT_TPC15_QM, 567 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_0] = GAUDI2_EVENT_TPC16_QM, 568 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_1] = GAUDI2_EVENT_TPC16_QM, 569 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_2] = GAUDI2_EVENT_TPC16_QM, 570 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = GAUDI2_EVENT_TPC16_QM, 571 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_0] = GAUDI2_EVENT_TPC17_QM, 572 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_1] = GAUDI2_EVENT_TPC17_QM, 573 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_2] = GAUDI2_EVENT_TPC17_QM, 574 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = GAUDI2_EVENT_TPC17_QM, 575 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0] = GAUDI2_EVENT_HDMA6_QM, 576 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_1] = GAUDI2_EVENT_HDMA6_QM, 577 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_2] = GAUDI2_EVENT_HDMA6_QM, 578 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = GAUDI2_EVENT_HDMA6_QM, 579 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0] = GAUDI2_EVENT_HDMA7_QM, 580 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1] = GAUDI2_EVENT_HDMA7_QM, 581 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2] = GAUDI2_EVENT_HDMA7_QM, 582 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = GAUDI2_EVENT_HDMA7_QM, 583 [GAUDI2_QUEUE_ID_DCORE3_MME_0_0] = GAUDI2_EVENT_MME3_QM, 584 [GAUDI2_QUEUE_ID_DCORE3_MME_0_1] = GAUDI2_EVENT_MME3_QM, 585 [GAUDI2_QUEUE_ID_DCORE3_MME_0_2] = GAUDI2_EVENT_MME3_QM, 586 [GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = GAUDI2_EVENT_MME3_QM, 587 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_0] = GAUDI2_EVENT_TPC18_QM, 588 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_1] = GAUDI2_EVENT_TPC18_QM, 589 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_2] = GAUDI2_EVENT_TPC18_QM, 590 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = GAUDI2_EVENT_TPC18_QM, 591 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_0] = GAUDI2_EVENT_TPC19_QM, 592 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_1] = GAUDI2_EVENT_TPC19_QM, 593 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_2] = GAUDI2_EVENT_TPC19_QM, 594 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = GAUDI2_EVENT_TPC19_QM, 595 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_0] = GAUDI2_EVENT_TPC20_QM, 596 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_1] = GAUDI2_EVENT_TPC20_QM, 597 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_2] = GAUDI2_EVENT_TPC20_QM, 598 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = GAUDI2_EVENT_TPC20_QM, 599 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_0] = GAUDI2_EVENT_TPC21_QM, 600 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_1] = GAUDI2_EVENT_TPC21_QM, 601 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_2] = GAUDI2_EVENT_TPC21_QM, 602 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = GAUDI2_EVENT_TPC21_QM, 603 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_0] = GAUDI2_EVENT_TPC22_QM, 604 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_1] = GAUDI2_EVENT_TPC22_QM, 605 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_2] = GAUDI2_EVENT_TPC22_QM, 606 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = GAUDI2_EVENT_TPC22_QM, 607 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_0] = GAUDI2_EVENT_TPC23_QM, 608 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_1] = GAUDI2_EVENT_TPC23_QM, 609 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_2] = GAUDI2_EVENT_TPC23_QM, 610 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = GAUDI2_EVENT_TPC23_QM, 611 [GAUDI2_QUEUE_ID_NIC_0_0] = GAUDI2_EVENT_NIC0_QM0, 612 [GAUDI2_QUEUE_ID_NIC_0_1] = GAUDI2_EVENT_NIC0_QM0, 613 [GAUDI2_QUEUE_ID_NIC_0_2] = GAUDI2_EVENT_NIC0_QM0, 614 [GAUDI2_QUEUE_ID_NIC_0_3] = GAUDI2_EVENT_NIC0_QM0, 615 [GAUDI2_QUEUE_ID_NIC_1_0] = GAUDI2_EVENT_NIC0_QM1, 616 [GAUDI2_QUEUE_ID_NIC_1_1] = GAUDI2_EVENT_NIC0_QM1, 617 [GAUDI2_QUEUE_ID_NIC_1_2] = GAUDI2_EVENT_NIC0_QM1, 618 [GAUDI2_QUEUE_ID_NIC_1_3] = GAUDI2_EVENT_NIC0_QM1, 619 [GAUDI2_QUEUE_ID_NIC_2_0] = GAUDI2_EVENT_NIC1_QM0, 620 [GAUDI2_QUEUE_ID_NIC_2_1] = GAUDI2_EVENT_NIC1_QM0, 621 [GAUDI2_QUEUE_ID_NIC_2_2] = GAUDI2_EVENT_NIC1_QM0, 622 [GAUDI2_QUEUE_ID_NIC_2_3] = GAUDI2_EVENT_NIC1_QM0, 623 [GAUDI2_QUEUE_ID_NIC_3_0] = GAUDI2_EVENT_NIC1_QM1, 624 [GAUDI2_QUEUE_ID_NIC_3_1] = GAUDI2_EVENT_NIC1_QM1, 625 [GAUDI2_QUEUE_ID_NIC_3_2] = GAUDI2_EVENT_NIC1_QM1, 626 [GAUDI2_QUEUE_ID_NIC_3_3] = GAUDI2_EVENT_NIC1_QM1, 627 [GAUDI2_QUEUE_ID_NIC_4_0] = GAUDI2_EVENT_NIC2_QM0, 628 [GAUDI2_QUEUE_ID_NIC_4_1] = GAUDI2_EVENT_NIC2_QM0, 629 [GAUDI2_QUEUE_ID_NIC_4_2] = GAUDI2_EVENT_NIC2_QM0, 630 [GAUDI2_QUEUE_ID_NIC_4_3] = GAUDI2_EVENT_NIC2_QM0, 631 [GAUDI2_QUEUE_ID_NIC_5_0] = GAUDI2_EVENT_NIC2_QM1, 632 [GAUDI2_QUEUE_ID_NIC_5_1] = GAUDI2_EVENT_NIC2_QM1, 633 [GAUDI2_QUEUE_ID_NIC_5_2] = GAUDI2_EVENT_NIC2_QM1, 634 [GAUDI2_QUEUE_ID_NIC_5_3] = GAUDI2_EVENT_NIC2_QM1, 635 [GAUDI2_QUEUE_ID_NIC_6_0] = GAUDI2_EVENT_NIC3_QM0, 636 [GAUDI2_QUEUE_ID_NIC_6_1] = GAUDI2_EVENT_NIC3_QM0, 637 [GAUDI2_QUEUE_ID_NIC_6_2] = GAUDI2_EVENT_NIC3_QM0, 638 [GAUDI2_QUEUE_ID_NIC_6_3] = GAUDI2_EVENT_NIC3_QM0, 639 [GAUDI2_QUEUE_ID_NIC_7_0] = GAUDI2_EVENT_NIC3_QM1, 640 [GAUDI2_QUEUE_ID_NIC_7_1] = GAUDI2_EVENT_NIC3_QM1, 641 [GAUDI2_QUEUE_ID_NIC_7_2] = GAUDI2_EVENT_NIC3_QM1, 642 [GAUDI2_QUEUE_ID_NIC_7_3] = GAUDI2_EVENT_NIC3_QM1, 643 [GAUDI2_QUEUE_ID_NIC_8_0] = GAUDI2_EVENT_NIC4_QM0, 644 [GAUDI2_QUEUE_ID_NIC_8_1] = GAUDI2_EVENT_NIC4_QM0, 645 [GAUDI2_QUEUE_ID_NIC_8_2] = GAUDI2_EVENT_NIC4_QM0, 646 [GAUDI2_QUEUE_ID_NIC_8_3] = GAUDI2_EVENT_NIC4_QM0, 647 [GAUDI2_QUEUE_ID_NIC_9_0] = GAUDI2_EVENT_NIC4_QM1, 648 [GAUDI2_QUEUE_ID_NIC_9_1] = GAUDI2_EVENT_NIC4_QM1, 649 [GAUDI2_QUEUE_ID_NIC_9_2] = GAUDI2_EVENT_NIC4_QM1, 650 [GAUDI2_QUEUE_ID_NIC_9_3] = GAUDI2_EVENT_NIC4_QM1, 651 [GAUDI2_QUEUE_ID_NIC_10_0] = GAUDI2_EVENT_NIC5_QM0, 652 [GAUDI2_QUEUE_ID_NIC_10_1] = GAUDI2_EVENT_NIC5_QM0, 653 [GAUDI2_QUEUE_ID_NIC_10_2] = GAUDI2_EVENT_NIC5_QM0, 654 [GAUDI2_QUEUE_ID_NIC_10_3] = GAUDI2_EVENT_NIC5_QM0, 655 [GAUDI2_QUEUE_ID_NIC_11_0] = GAUDI2_EVENT_NIC5_QM1, 656 [GAUDI2_QUEUE_ID_NIC_11_1] = GAUDI2_EVENT_NIC5_QM1, 657 [GAUDI2_QUEUE_ID_NIC_11_2] = GAUDI2_EVENT_NIC5_QM1, 658 [GAUDI2_QUEUE_ID_NIC_11_3] = GAUDI2_EVENT_NIC5_QM1, 659 [GAUDI2_QUEUE_ID_NIC_12_0] = GAUDI2_EVENT_NIC6_QM0, 660 [GAUDI2_QUEUE_ID_NIC_12_1] = GAUDI2_EVENT_NIC6_QM0, 661 [GAUDI2_QUEUE_ID_NIC_12_2] = GAUDI2_EVENT_NIC6_QM0, 662 [GAUDI2_QUEUE_ID_NIC_12_3] = GAUDI2_EVENT_NIC6_QM0, 663 [GAUDI2_QUEUE_ID_NIC_13_0] = GAUDI2_EVENT_NIC6_QM1, 664 [GAUDI2_QUEUE_ID_NIC_13_1] = GAUDI2_EVENT_NIC6_QM1, 665 [GAUDI2_QUEUE_ID_NIC_13_2] = GAUDI2_EVENT_NIC6_QM1, 666 [GAUDI2_QUEUE_ID_NIC_13_3] = GAUDI2_EVENT_NIC6_QM1, 667 [GAUDI2_QUEUE_ID_NIC_14_0] = GAUDI2_EVENT_NIC7_QM0, 668 [GAUDI2_QUEUE_ID_NIC_14_1] = GAUDI2_EVENT_NIC7_QM0, 669 [GAUDI2_QUEUE_ID_NIC_14_2] = GAUDI2_EVENT_NIC7_QM0, 670 [GAUDI2_QUEUE_ID_NIC_14_3] = GAUDI2_EVENT_NIC7_QM0, 671 [GAUDI2_QUEUE_ID_NIC_15_0] = GAUDI2_EVENT_NIC7_QM1, 672 [GAUDI2_QUEUE_ID_NIC_15_1] = GAUDI2_EVENT_NIC7_QM1, 673 [GAUDI2_QUEUE_ID_NIC_15_2] = GAUDI2_EVENT_NIC7_QM1, 674 [GAUDI2_QUEUE_ID_NIC_15_3] = GAUDI2_EVENT_NIC7_QM1, 675 [GAUDI2_QUEUE_ID_NIC_16_0] = GAUDI2_EVENT_NIC8_QM0, 676 [GAUDI2_QUEUE_ID_NIC_16_1] = GAUDI2_EVENT_NIC8_QM0, 677 [GAUDI2_QUEUE_ID_NIC_16_2] = GAUDI2_EVENT_NIC8_QM0, 678 [GAUDI2_QUEUE_ID_NIC_16_3] = GAUDI2_EVENT_NIC8_QM0, 679 [GAUDI2_QUEUE_ID_NIC_17_0] = GAUDI2_EVENT_NIC8_QM1, 680 [GAUDI2_QUEUE_ID_NIC_17_1] = GAUDI2_EVENT_NIC8_QM1, 681 [GAUDI2_QUEUE_ID_NIC_17_2] = GAUDI2_EVENT_NIC8_QM1, 682 [GAUDI2_QUEUE_ID_NIC_17_3] = GAUDI2_EVENT_NIC8_QM1, 683 [GAUDI2_QUEUE_ID_NIC_18_0] = GAUDI2_EVENT_NIC9_QM0, 684 [GAUDI2_QUEUE_ID_NIC_18_1] = GAUDI2_EVENT_NIC9_QM0, 685 [GAUDI2_QUEUE_ID_NIC_18_2] = GAUDI2_EVENT_NIC9_QM0, 686 [GAUDI2_QUEUE_ID_NIC_18_3] = GAUDI2_EVENT_NIC9_QM0, 687 [GAUDI2_QUEUE_ID_NIC_19_0] = GAUDI2_EVENT_NIC9_QM1, 688 [GAUDI2_QUEUE_ID_NIC_19_1] = GAUDI2_EVENT_NIC9_QM1, 689 [GAUDI2_QUEUE_ID_NIC_19_2] = GAUDI2_EVENT_NIC9_QM1, 690 [GAUDI2_QUEUE_ID_NIC_19_3] = GAUDI2_EVENT_NIC9_QM1, 691 [GAUDI2_QUEUE_ID_NIC_20_0] = GAUDI2_EVENT_NIC10_QM0, 692 [GAUDI2_QUEUE_ID_NIC_20_1] = GAUDI2_EVENT_NIC10_QM0, 693 [GAUDI2_QUEUE_ID_NIC_20_2] = GAUDI2_EVENT_NIC10_QM0, 694 [GAUDI2_QUEUE_ID_NIC_20_3] = GAUDI2_EVENT_NIC10_QM0, 695 [GAUDI2_QUEUE_ID_NIC_21_0] = GAUDI2_EVENT_NIC10_QM1, 696 [GAUDI2_QUEUE_ID_NIC_21_1] = GAUDI2_EVENT_NIC10_QM1, 697 [GAUDI2_QUEUE_ID_NIC_21_2] = GAUDI2_EVENT_NIC10_QM1, 698 [GAUDI2_QUEUE_ID_NIC_21_3] = GAUDI2_EVENT_NIC10_QM1, 699 [GAUDI2_QUEUE_ID_NIC_22_0] = GAUDI2_EVENT_NIC11_QM0, 700 [GAUDI2_QUEUE_ID_NIC_22_1] = GAUDI2_EVENT_NIC11_QM0, 701 [GAUDI2_QUEUE_ID_NIC_22_2] = GAUDI2_EVENT_NIC11_QM0, 702 [GAUDI2_QUEUE_ID_NIC_22_3] = GAUDI2_EVENT_NIC11_QM0, 703 [GAUDI2_QUEUE_ID_NIC_23_0] = GAUDI2_EVENT_NIC11_QM1, 704 [GAUDI2_QUEUE_ID_NIC_23_1] = GAUDI2_EVENT_NIC11_QM1, 705 [GAUDI2_QUEUE_ID_NIC_23_2] = GAUDI2_EVENT_NIC11_QM1, 706 [GAUDI2_QUEUE_ID_NIC_23_3] = GAUDI2_EVENT_NIC11_QM1, 707 [GAUDI2_QUEUE_ID_ROT_0_0] = GAUDI2_EVENT_ROTATOR0_ROT0_QM, 708 [GAUDI2_QUEUE_ID_ROT_0_1] = GAUDI2_EVENT_ROTATOR0_ROT0_QM, 709 [GAUDI2_QUEUE_ID_ROT_0_2] = GAUDI2_EVENT_ROTATOR0_ROT0_QM, 710 [GAUDI2_QUEUE_ID_ROT_0_3] = GAUDI2_EVENT_ROTATOR0_ROT0_QM, 711 [GAUDI2_QUEUE_ID_ROT_1_0] = GAUDI2_EVENT_ROTATOR1_ROT1_QM, 712 [GAUDI2_QUEUE_ID_ROT_1_1] = GAUDI2_EVENT_ROTATOR1_ROT1_QM, 713 [GAUDI2_QUEUE_ID_ROT_1_2] = GAUDI2_EVENT_ROTATOR1_ROT1_QM, 714 [GAUDI2_QUEUE_ID_ROT_1_3] = GAUDI2_EVENT_ROTATOR1_ROT1_QM 715}; 716 717static const int gaudi2_dma_core_async_event_id[] = { 718 [DMA_CORE_ID_EDMA0] = GAUDI2_EVENT_HDMA0_CORE, 719 [DMA_CORE_ID_EDMA1] = GAUDI2_EVENT_HDMA1_CORE, 720 [DMA_CORE_ID_EDMA2] = GAUDI2_EVENT_HDMA2_CORE, 721 [DMA_CORE_ID_EDMA3] = GAUDI2_EVENT_HDMA3_CORE, 722 [DMA_CORE_ID_EDMA4] = GAUDI2_EVENT_HDMA4_CORE, 723 [DMA_CORE_ID_EDMA5] = GAUDI2_EVENT_HDMA5_CORE, 724 [DMA_CORE_ID_EDMA6] = GAUDI2_EVENT_HDMA6_CORE, 725 [DMA_CORE_ID_EDMA7] = GAUDI2_EVENT_HDMA7_CORE, 726 [DMA_CORE_ID_PDMA0] = GAUDI2_EVENT_PDMA0_CORE, 727 [DMA_CORE_ID_PDMA1] = GAUDI2_EVENT_PDMA1_CORE, 728 [DMA_CORE_ID_KDMA] = GAUDI2_EVENT_KDMA0_CORE, 729}; 730 731const char *gaudi2_engine_id_str[] = { 732 __stringify(GAUDI2_DCORE0_ENGINE_ID_EDMA_0), 733 __stringify(GAUDI2_DCORE0_ENGINE_ID_EDMA_1), 734 __stringify(GAUDI2_DCORE0_ENGINE_ID_MME), 735 __stringify(GAUDI2_DCORE0_ENGINE_ID_TPC_0), 736 __stringify(GAUDI2_DCORE0_ENGINE_ID_TPC_1), 737 __stringify(GAUDI2_DCORE0_ENGINE_ID_TPC_2), 738 __stringify(GAUDI2_DCORE0_ENGINE_ID_TPC_3), 739 __stringify(GAUDI2_DCORE0_ENGINE_ID_TPC_4), 740 __stringify(GAUDI2_DCORE0_ENGINE_ID_TPC_5), 741 __stringify(GAUDI2_DCORE0_ENGINE_ID_DEC_0), 742 __stringify(GAUDI2_DCORE0_ENGINE_ID_DEC_1), 743 __stringify(GAUDI2_DCORE1_ENGINE_ID_EDMA_0), 744 __stringify(GAUDI2_DCORE1_ENGINE_ID_EDMA_1), 745 __stringify(GAUDI2_DCORE1_ENGINE_ID_MME), 746 __stringify(GAUDI2_DCORE1_ENGINE_ID_TPC_0), 747 __stringify(GAUDI2_DCORE1_ENGINE_ID_TPC_1), 748 __stringify(GAUDI2_DCORE1_ENGINE_ID_TPC_2), 749 __stringify(GAUDI2_DCORE1_ENGINE_ID_TPC_3), 750 __stringify(GAUDI2_DCORE1_ENGINE_ID_TPC_4), 751 __stringify(GAUDI2_DCORE1_ENGINE_ID_TPC_5), 752 __stringify(GAUDI2_DCORE1_ENGINE_ID_DEC_0), 753 __stringify(GAUDI2_DCORE1_ENGINE_ID_DEC_1), 754 __stringify(GAUDI2_DCORE2_ENGINE_ID_EDMA_0), 755 __stringify(GAUDI2_DCORE2_ENGINE_ID_EDMA_1), 756 __stringify(GAUDI2_DCORE2_ENGINE_ID_MME), 757 __stringify(GAUDI2_DCORE2_ENGINE_ID_TPC_0), 758 __stringify(GAUDI2_DCORE2_ENGINE_ID_TPC_1), 759 __stringify(GAUDI2_DCORE2_ENGINE_ID_TPC_2), 760 __stringify(GAUDI2_DCORE2_ENGINE_ID_TPC_3), 761 __stringify(GAUDI2_DCORE2_ENGINE_ID_TPC_4), 762 __stringify(GAUDI2_DCORE2_ENGINE_ID_TPC_5), 763 __stringify(GAUDI2_DCORE2_ENGINE_ID_DEC_0), 764 __stringify(GAUDI2_DCORE2_ENGINE_ID_DEC_1), 765 __stringify(GAUDI2_DCORE3_ENGINE_ID_EDMA_0), 766 __stringify(GAUDI2_DCORE3_ENGINE_ID_EDMA_1), 767 __stringify(GAUDI2_DCORE3_ENGINE_ID_MME), 768 __stringify(GAUDI2_DCORE3_ENGINE_ID_TPC_0), 769 __stringify(GAUDI2_DCORE3_ENGINE_ID_TPC_1), 770 __stringify(GAUDI2_DCORE3_ENGINE_ID_TPC_2), 771 __stringify(GAUDI2_DCORE3_ENGINE_ID_TPC_3), 772 __stringify(GAUDI2_DCORE3_ENGINE_ID_TPC_4), 773 __stringify(GAUDI2_DCORE3_ENGINE_ID_TPC_5), 774 __stringify(GAUDI2_DCORE3_ENGINE_ID_DEC_0), 775 __stringify(GAUDI2_DCORE3_ENGINE_ID_DEC_1), 776 __stringify(GAUDI2_DCORE0_ENGINE_ID_TPC_6), 777 __stringify(GAUDI2_ENGINE_ID_PDMA_0), 778 __stringify(GAUDI2_ENGINE_ID_PDMA_1), 779 __stringify(GAUDI2_ENGINE_ID_ROT_0), 780 __stringify(GAUDI2_ENGINE_ID_ROT_1), 781 __stringify(GAUDI2_PCIE_ENGINE_ID_DEC_0), 782 __stringify(GAUDI2_PCIE_ENGINE_ID_DEC_1), 783 __stringify(GAUDI2_ENGINE_ID_NIC0_0), 784 __stringify(GAUDI2_ENGINE_ID_NIC0_1), 785 __stringify(GAUDI2_ENGINE_ID_NIC1_0), 786 __stringify(GAUDI2_ENGINE_ID_NIC1_1), 787 __stringify(GAUDI2_ENGINE_ID_NIC2_0), 788 __stringify(GAUDI2_ENGINE_ID_NIC2_1), 789 __stringify(GAUDI2_ENGINE_ID_NIC3_0), 790 __stringify(GAUDI2_ENGINE_ID_NIC3_1), 791 __stringify(GAUDI2_ENGINE_ID_NIC4_0), 792 __stringify(GAUDI2_ENGINE_ID_NIC4_1), 793 __stringify(GAUDI2_ENGINE_ID_NIC5_0), 794 __stringify(GAUDI2_ENGINE_ID_NIC5_1), 795 __stringify(GAUDI2_ENGINE_ID_NIC6_0), 796 __stringify(GAUDI2_ENGINE_ID_NIC6_1), 797 __stringify(GAUDI2_ENGINE_ID_NIC7_0), 798 __stringify(GAUDI2_ENGINE_ID_NIC7_1), 799 __stringify(GAUDI2_ENGINE_ID_NIC8_0), 800 __stringify(GAUDI2_ENGINE_ID_NIC8_1), 801 __stringify(GAUDI2_ENGINE_ID_NIC9_0), 802 __stringify(GAUDI2_ENGINE_ID_NIC9_1), 803 __stringify(GAUDI2_ENGINE_ID_NIC10_0), 804 __stringify(GAUDI2_ENGINE_ID_NIC10_1), 805 __stringify(GAUDI2_ENGINE_ID_NIC11_0), 806 __stringify(GAUDI2_ENGINE_ID_NIC11_1), 807 __stringify(GAUDI2_ENGINE_ID_PCIE), 808 __stringify(GAUDI2_ENGINE_ID_PSOC), 809 __stringify(GAUDI2_ENGINE_ID_ARC_FARM), 810 __stringify(GAUDI2_ENGINE_ID_KDMA), 811 __stringify(GAUDI2_ENGINE_ID_SIZE), 812}; 813 814const char *gaudi2_queue_id_str[] = { 815 __stringify(GAUDI2_QUEUE_ID_PDMA_0_0), 816 __stringify(GAUDI2_QUEUE_ID_PDMA_0_1), 817 __stringify(GAUDI2_QUEUE_ID_PDMA_0_2), 818 __stringify(GAUDI2_QUEUE_ID_PDMA_0_3), 819 __stringify(GAUDI2_QUEUE_ID_PDMA_1_0), 820 __stringify(GAUDI2_QUEUE_ID_PDMA_1_1), 821 __stringify(GAUDI2_QUEUE_ID_PDMA_1_2), 822 __stringify(GAUDI2_QUEUE_ID_PDMA_1_3), 823 __stringify(GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0), 824 __stringify(GAUDI2_QUEUE_ID_DCORE0_EDMA_0_1), 825 __stringify(GAUDI2_QUEUE_ID_DCORE0_EDMA_0_2), 826 __stringify(GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3), 827 __stringify(GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0), 828 __stringify(GAUDI2_QUEUE_ID_DCORE0_EDMA_1_1), 829 __stringify(GAUDI2_QUEUE_ID_DCORE0_EDMA_1_2), 830 __stringify(GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3), 831 __stringify(GAUDI2_QUEUE_ID_DCORE0_MME_0_0), 832 __stringify(GAUDI2_QUEUE_ID_DCORE0_MME_0_1), 833 __stringify(GAUDI2_QUEUE_ID_DCORE0_MME_0_2), 834 __stringify(GAUDI2_QUEUE_ID_DCORE0_MME_0_3), 835 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_0_0), 836 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_0_1), 837 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_0_2), 838 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_0_3), 839 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_1_0), 840 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_1_1), 841 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_1_2), 842 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_1_3), 843 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_2_0), 844 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_2_1), 845 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_2_2), 846 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_2_3), 847 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_3_0), 848 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_3_1), 849 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_3_2), 850 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_3_3), 851 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_4_0), 852 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_4_1), 853 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_4_2), 854 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_4_3), 855 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_5_0), 856 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_5_1), 857 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_5_2), 858 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_5_3), 859 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_6_0), 860 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_6_1), 861 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_6_2), 862 __stringify(GAUDI2_QUEUE_ID_DCORE0_TPC_6_3), 863 __stringify(GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0), 864 __stringify(GAUDI2_QUEUE_ID_DCORE1_EDMA_0_1), 865 __stringify(GAUDI2_QUEUE_ID_DCORE1_EDMA_0_2), 866 __stringify(GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3), 867 __stringify(GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0), 868 __stringify(GAUDI2_QUEUE_ID_DCORE1_EDMA_1_1), 869 __stringify(GAUDI2_QUEUE_ID_DCORE1_EDMA_1_2), 870 __stringify(GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3), 871 __stringify(GAUDI2_QUEUE_ID_DCORE1_MME_0_0), 872 __stringify(GAUDI2_QUEUE_ID_DCORE1_MME_0_1), 873 __stringify(GAUDI2_QUEUE_ID_DCORE1_MME_0_2), 874 __stringify(GAUDI2_QUEUE_ID_DCORE1_MME_0_3), 875 __stringify(GAUDI2_QUEUE_ID_DCORE1_TPC_0_0), 876 __stringify(GAUDI2_QUEUE_ID_DCORE1_TPC_0_1), 877 __stringify(GAUDI2_QUEUE_ID_DCORE1_TPC_0_2), 878 __stringify(GAUDI2_QUEUE_ID_DCORE1_TPC_0_3), 879 __stringify(GAUDI2_QUEUE_ID_DCORE1_TPC_1_0), 880 __stringify(GAUDI2_QUEUE_ID_DCORE1_TPC_1_1), 881 __stringify(GAUDI2_QUEUE_ID_DCORE1_TPC_1_2), 882 __stringify(GAUDI2_QUEUE_ID_DCORE1_TPC_1_3), 883 __stringify(GAUDI2_QUEUE_ID_DCORE1_TPC_2_0), 884 __stringify(GAUDI2_QUEUE_ID_DCORE1_TPC_2_1), 885 __stringify(GAUDI2_QUEUE_ID_DCORE1_TPC_2_2), 886 __stringify(GAUDI2_QUEUE_ID_DCORE1_TPC_2_3), 887 __stringify(GAUDI2_QUEUE_ID_DCORE1_TPC_3_0), 888 __stringify(GAUDI2_QUEUE_ID_DCORE1_TPC_3_1), 889 __stringify(GAUDI2_QUEUE_ID_DCORE1_TPC_3_2), 890 __stringify(GAUDI2_QUEUE_ID_DCORE1_TPC_3_3), 891 __stringify(GAUDI2_QUEUE_ID_DCORE1_TPC_4_0), 892 __stringify(GAUDI2_QUEUE_ID_DCORE1_TPC_4_1), 893 __stringify(GAUDI2_QUEUE_ID_DCORE1_TPC_4_2), 894 __stringify(GAUDI2_QUEUE_ID_DCORE1_TPC_4_3), 895 __stringify(GAUDI2_QUEUE_ID_DCORE1_TPC_5_0), 896 __stringify(GAUDI2_QUEUE_ID_DCORE1_TPC_5_1), 897 __stringify(GAUDI2_QUEUE_ID_DCORE1_TPC_5_2), 898 __stringify(GAUDI2_QUEUE_ID_DCORE1_TPC_5_3), 899 __stringify(GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0), 900 __stringify(GAUDI2_QUEUE_ID_DCORE2_EDMA_0_1), 901 __stringify(GAUDI2_QUEUE_ID_DCORE2_EDMA_0_2), 902 __stringify(GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3), 903 __stringify(GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0), 904 __stringify(GAUDI2_QUEUE_ID_DCORE2_EDMA_1_1), 905 __stringify(GAUDI2_QUEUE_ID_DCORE2_EDMA_1_2), 906 __stringify(GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3), 907 __stringify(GAUDI2_QUEUE_ID_DCORE2_MME_0_0), 908 __stringify(GAUDI2_QUEUE_ID_DCORE2_MME_0_1), 909 __stringify(GAUDI2_QUEUE_ID_DCORE2_MME_0_2), 910 __stringify(GAUDI2_QUEUE_ID_DCORE2_MME_0_3), 911 __stringify(GAUDI2_QUEUE_ID_DCORE2_TPC_0_0), 912 __stringify(GAUDI2_QUEUE_ID_DCORE2_TPC_0_1), 913 __stringify(GAUDI2_QUEUE_ID_DCORE2_TPC_0_2), 914 __stringify(GAUDI2_QUEUE_ID_DCORE2_TPC_0_3), 915 __stringify(GAUDI2_QUEUE_ID_DCORE2_TPC_1_0), 916 __stringify(GAUDI2_QUEUE_ID_DCORE2_TPC_1_1), 917 __stringify(GAUDI2_QUEUE_ID_DCORE2_TPC_1_2), 918 __stringify(GAUDI2_QUEUE_ID_DCORE2_TPC_1_3), 919 __stringify(GAUDI2_QUEUE_ID_DCORE2_TPC_2_0), 920 __stringify(GAUDI2_QUEUE_ID_DCORE2_TPC_2_1), 921 __stringify(GAUDI2_QUEUE_ID_DCORE2_TPC_2_2), 922 __stringify(GAUDI2_QUEUE_ID_DCORE2_TPC_2_3), 923 __stringify(GAUDI2_QUEUE_ID_DCORE2_TPC_3_0), 924 __stringify(GAUDI2_QUEUE_ID_DCORE2_TPC_3_1), 925 __stringify(GAUDI2_QUEUE_ID_DCORE2_TPC_3_2), 926 __stringify(GAUDI2_QUEUE_ID_DCORE2_TPC_3_3), 927 __stringify(GAUDI2_QUEUE_ID_DCORE2_TPC_4_0), 928 __stringify(GAUDI2_QUEUE_ID_DCORE2_TPC_4_1), 929 __stringify(GAUDI2_QUEUE_ID_DCORE2_TPC_4_2), 930 __stringify(GAUDI2_QUEUE_ID_DCORE2_TPC_4_3), 931 __stringify(GAUDI2_QUEUE_ID_DCORE2_TPC_5_0), 932 __stringify(GAUDI2_QUEUE_ID_DCORE2_TPC_5_1), 933 __stringify(GAUDI2_QUEUE_ID_DCORE2_TPC_5_2), 934 __stringify(GAUDI2_QUEUE_ID_DCORE2_TPC_5_3), 935 __stringify(GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0), 936 __stringify(GAUDI2_QUEUE_ID_DCORE3_EDMA_0_1), 937 __stringify(GAUDI2_QUEUE_ID_DCORE3_EDMA_0_2), 938 __stringify(GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3), 939 __stringify(GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0), 940 __stringify(GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1), 941 __stringify(GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2), 942 __stringify(GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3), 943 __stringify(GAUDI2_QUEUE_ID_DCORE3_MME_0_0), 944 __stringify(GAUDI2_QUEUE_ID_DCORE3_MME_0_1), 945 __stringify(GAUDI2_QUEUE_ID_DCORE3_MME_0_2), 946 __stringify(GAUDI2_QUEUE_ID_DCORE3_MME_0_3), 947 __stringify(GAUDI2_QUEUE_ID_DCORE3_TPC_0_0), 948 __stringify(GAUDI2_QUEUE_ID_DCORE3_TPC_0_1), 949 __stringify(GAUDI2_QUEUE_ID_DCORE3_TPC_0_2), 950 __stringify(GAUDI2_QUEUE_ID_DCORE3_TPC_0_3), 951 __stringify(GAUDI2_QUEUE_ID_DCORE3_TPC_1_0), 952 __stringify(GAUDI2_QUEUE_ID_DCORE3_TPC_1_1), 953 __stringify(GAUDI2_QUEUE_ID_DCORE3_TPC_1_2), 954 __stringify(GAUDI2_QUEUE_ID_DCORE3_TPC_1_3), 955 __stringify(GAUDI2_QUEUE_ID_DCORE3_TPC_2_0), 956 __stringify(GAUDI2_QUEUE_ID_DCORE3_TPC_2_1), 957 __stringify(GAUDI2_QUEUE_ID_DCORE3_TPC_2_2), 958 __stringify(GAUDI2_QUEUE_ID_DCORE3_TPC_2_3), 959 __stringify(GAUDI2_QUEUE_ID_DCORE3_TPC_3_0), 960 __stringify(GAUDI2_QUEUE_ID_DCORE3_TPC_3_1), 961 __stringify(GAUDI2_QUEUE_ID_DCORE3_TPC_3_2), 962 __stringify(GAUDI2_QUEUE_ID_DCORE3_TPC_3_3), 963 __stringify(GAUDI2_QUEUE_ID_DCORE3_TPC_4_0), 964 __stringify(GAUDI2_QUEUE_ID_DCORE3_TPC_4_1), 965 __stringify(GAUDI2_QUEUE_ID_DCORE3_TPC_4_2), 966 __stringify(GAUDI2_QUEUE_ID_DCORE3_TPC_4_3), 967 __stringify(GAUDI2_QUEUE_ID_DCORE3_TPC_5_0), 968 __stringify(GAUDI2_QUEUE_ID_DCORE3_TPC_5_1), 969 __stringify(GAUDI2_QUEUE_ID_DCORE3_TPC_5_2), 970 __stringify(GAUDI2_QUEUE_ID_DCORE3_TPC_5_3), 971 __stringify(GAUDI2_QUEUE_ID_NIC_0_0), 972 __stringify(GAUDI2_QUEUE_ID_NIC_0_1), 973 __stringify(GAUDI2_QUEUE_ID_NIC_0_2), 974 __stringify(GAUDI2_QUEUE_ID_NIC_0_3), 975 __stringify(GAUDI2_QUEUE_ID_NIC_1_0), 976 __stringify(GAUDI2_QUEUE_ID_NIC_1_1), 977 __stringify(GAUDI2_QUEUE_ID_NIC_1_2), 978 __stringify(GAUDI2_QUEUE_ID_NIC_1_3), 979 __stringify(GAUDI2_QUEUE_ID_NIC_2_0), 980 __stringify(GAUDI2_QUEUE_ID_NIC_2_1), 981 __stringify(GAUDI2_QUEUE_ID_NIC_2_2), 982 __stringify(GAUDI2_QUEUE_ID_NIC_2_3), 983 __stringify(GAUDI2_QUEUE_ID_NIC_3_0), 984 __stringify(GAUDI2_QUEUE_ID_NIC_3_1), 985 __stringify(GAUDI2_QUEUE_ID_NIC_3_2), 986 __stringify(GAUDI2_QUEUE_ID_NIC_3_3), 987 __stringify(GAUDI2_QUEUE_ID_NIC_4_0), 988 __stringify(GAUDI2_QUEUE_ID_NIC_4_1), 989 __stringify(GAUDI2_QUEUE_ID_NIC_4_2), 990 __stringify(GAUDI2_QUEUE_ID_NIC_4_3), 991 __stringify(GAUDI2_QUEUE_ID_NIC_5_0), 992 __stringify(GAUDI2_QUEUE_ID_NIC_5_1), 993 __stringify(GAUDI2_QUEUE_ID_NIC_5_2), 994 __stringify(GAUDI2_QUEUE_ID_NIC_5_3), 995 __stringify(GAUDI2_QUEUE_ID_NIC_6_0), 996 __stringify(GAUDI2_QUEUE_ID_NIC_6_1), 997 __stringify(GAUDI2_QUEUE_ID_NIC_6_2), 998 __stringify(GAUDI2_QUEUE_ID_NIC_6_3), 999 __stringify(GAUDI2_QUEUE_ID_NIC_7_0), 1000 __stringify(GAUDI2_QUEUE_ID_NIC_7_1), 1001 __stringify(GAUDI2_QUEUE_ID_NIC_7_2), 1002 __stringify(GAUDI2_QUEUE_ID_NIC_7_3), 1003 __stringify(GAUDI2_QUEUE_ID_NIC_8_0), 1004 __stringify(GAUDI2_QUEUE_ID_NIC_8_1), 1005 __stringify(GAUDI2_QUEUE_ID_NIC_8_2), 1006 __stringify(GAUDI2_QUEUE_ID_NIC_8_3), 1007 __stringify(GAUDI2_QUEUE_ID_NIC_9_0), 1008 __stringify(GAUDI2_QUEUE_ID_NIC_9_1), 1009 __stringify(GAUDI2_QUEUE_ID_NIC_9_2), 1010 __stringify(GAUDI2_QUEUE_ID_NIC_9_3), 1011 __stringify(GAUDI2_QUEUE_ID_NIC_10_0), 1012 __stringify(GAUDI2_QUEUE_ID_NIC_10_1), 1013 __stringify(GAUDI2_QUEUE_ID_NIC_10_2), 1014 __stringify(GAUDI2_QUEUE_ID_NIC_10_3), 1015 __stringify(GAUDI2_QUEUE_ID_NIC_11_0), 1016 __stringify(GAUDI2_QUEUE_ID_NIC_11_1), 1017 __stringify(GAUDI2_QUEUE_ID_NIC_11_2), 1018 __stringify(GAUDI2_QUEUE_ID_NIC_11_3), 1019 __stringify(GAUDI2_QUEUE_ID_NIC_12_0), 1020 __stringify(GAUDI2_QUEUE_ID_NIC_12_1), 1021 __stringify(GAUDI2_QUEUE_ID_NIC_12_2), 1022 __stringify(GAUDI2_QUEUE_ID_NIC_12_3), 1023 __stringify(GAUDI2_QUEUE_ID_NIC_13_0), 1024 __stringify(GAUDI2_QUEUE_ID_NIC_13_1), 1025 __stringify(GAUDI2_QUEUE_ID_NIC_13_2), 1026 __stringify(GAUDI2_QUEUE_ID_NIC_13_3), 1027 __stringify(GAUDI2_QUEUE_ID_NIC_14_0), 1028 __stringify(GAUDI2_QUEUE_ID_NIC_14_1), 1029 __stringify(GAUDI2_QUEUE_ID_NIC_14_2), 1030 __stringify(GAUDI2_QUEUE_ID_NIC_14_3), 1031 __stringify(GAUDI2_QUEUE_ID_NIC_15_0), 1032 __stringify(GAUDI2_QUEUE_ID_NIC_15_1), 1033 __stringify(GAUDI2_QUEUE_ID_NIC_15_2), 1034 __stringify(GAUDI2_QUEUE_ID_NIC_15_3), 1035 __stringify(GAUDI2_QUEUE_ID_NIC_16_0), 1036 __stringify(GAUDI2_QUEUE_ID_NIC_16_1), 1037 __stringify(GAUDI2_QUEUE_ID_NIC_16_2), 1038 __stringify(GAUDI2_QUEUE_ID_NIC_16_3), 1039 __stringify(GAUDI2_QUEUE_ID_NIC_17_0), 1040 __stringify(GAUDI2_QUEUE_ID_NIC_17_1), 1041 __stringify(GAUDI2_QUEUE_ID_NIC_17_2), 1042 __stringify(GAUDI2_QUEUE_ID_NIC_17_3), 1043 __stringify(GAUDI2_QUEUE_ID_NIC_18_0), 1044 __stringify(GAUDI2_QUEUE_ID_NIC_18_1), 1045 __stringify(GAUDI2_QUEUE_ID_NIC_18_2), 1046 __stringify(GAUDI2_QUEUE_ID_NIC_18_3), 1047 __stringify(GAUDI2_QUEUE_ID_NIC_19_0), 1048 __stringify(GAUDI2_QUEUE_ID_NIC_19_1), 1049 __stringify(GAUDI2_QUEUE_ID_NIC_19_2), 1050 __stringify(GAUDI2_QUEUE_ID_NIC_19_3), 1051 __stringify(GAUDI2_QUEUE_ID_NIC_20_0), 1052 __stringify(GAUDI2_QUEUE_ID_NIC_20_1), 1053 __stringify(GAUDI2_QUEUE_ID_NIC_20_2), 1054 __stringify(GAUDI2_QUEUE_ID_NIC_20_3), 1055 __stringify(GAUDI2_QUEUE_ID_NIC_21_0), 1056 __stringify(GAUDI2_QUEUE_ID_NIC_21_1), 1057 __stringify(GAUDI2_QUEUE_ID_NIC_21_2), 1058 __stringify(GAUDI2_QUEUE_ID_NIC_21_3), 1059 __stringify(GAUDI2_QUEUE_ID_NIC_22_0), 1060 __stringify(GAUDI2_QUEUE_ID_NIC_22_1), 1061 __stringify(GAUDI2_QUEUE_ID_NIC_22_2), 1062 __stringify(GAUDI2_QUEUE_ID_NIC_22_3), 1063 __stringify(GAUDI2_QUEUE_ID_NIC_23_0), 1064 __stringify(GAUDI2_QUEUE_ID_NIC_23_1), 1065 __stringify(GAUDI2_QUEUE_ID_NIC_23_2), 1066 __stringify(GAUDI2_QUEUE_ID_NIC_23_3), 1067 __stringify(GAUDI2_QUEUE_ID_ROT_0_0), 1068 __stringify(GAUDI2_QUEUE_ID_ROT_0_1), 1069 __stringify(GAUDI2_QUEUE_ID_ROT_0_2), 1070 __stringify(GAUDI2_QUEUE_ID_ROT_0_3), 1071 __stringify(GAUDI2_QUEUE_ID_ROT_1_0), 1072 __stringify(GAUDI2_QUEUE_ID_ROT_1_1), 1073 __stringify(GAUDI2_QUEUE_ID_ROT_1_2), 1074 __stringify(GAUDI2_QUEUE_ID_ROT_1_3), 1075 __stringify(GAUDI2_QUEUE_ID_CPU_PQ), 1076 __stringify(GAUDI2_QUEUE_ID_SIZE), 1077}; 1078 1079static const char * const gaudi2_qm_sei_error_cause[GAUDI2_NUM_OF_QM_SEI_ERR_CAUSE] = { 1080 "qman sei intr", 1081 "arc sei intr" 1082}; 1083 1084static const char * const gaudi2_cpu_sei_error_cause[GAUDI2_NUM_OF_CPU_SEI_ERR_CAUSE] = { 1085 "AXI_TERMINATOR WR", 1086 "AXI_TERMINATOR RD", 1087 "AXI SPLIT SEI Status" 1088}; 1089 1090static const char * const gaudi2_arc_sei_error_cause[GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE] = { 1091 "cbu_bresp_sei_intr_cause", 1092 "cbu_rresp_sei_intr_cause", 1093 "lbu_bresp_sei_intr_cause", 1094 "lbu_rresp_sei_intr_cause", 1095 "cbu_axi_split_intr_cause", 1096 "lbu_axi_split_intr_cause", 1097 "arc_ip_excptn_sei_intr_cause", 1098 "dmi_bresp_sei_intr_cause", 1099 "aux2apb_err_sei_intr_cause", 1100 "cfg_lbw_wr_terminated_intr_cause", 1101 "cfg_lbw_rd_terminated_intr_cause", 1102 "cfg_dccm_wr_terminated_intr_cause", 1103 "cfg_dccm_rd_terminated_intr_cause", 1104 "cfg_hbw_rd_terminated_intr_cause" 1105}; 1106 1107static const char * const gaudi2_dec_error_cause[GAUDI2_NUM_OF_DEC_ERR_CAUSE] = { 1108 "msix_vcd_hbw_sei", 1109 "msix_l2c_hbw_sei", 1110 "msix_nrm_hbw_sei", 1111 "msix_abnrm_hbw_sei", 1112 "msix_vcd_lbw_sei", 1113 "msix_l2c_lbw_sei", 1114 "msix_nrm_lbw_sei", 1115 "msix_abnrm_lbw_sei", 1116 "apb_vcd_lbw_sei", 1117 "apb_l2c_lbw_sei", 1118 "apb_nrm_lbw_sei", 1119 "apb_abnrm_lbw_sei", 1120 "dec_sei", 1121 "dec_apb_sei", 1122 "trc_apb_sei", 1123 "lbw_mstr_if_sei", 1124 "axi_split_bresp_err_sei", 1125 "hbw_axi_wr_viol_sei", 1126 "hbw_axi_rd_viol_sei", 1127 "lbw_axi_wr_viol_sei", 1128 "lbw_axi_rd_viol_sei", 1129 "vcd_spi", 1130 "l2c_spi", 1131 "nrm_spi", 1132 "abnrm_spi", 1133}; 1134 1135static const char * const gaudi2_qman_error_cause[GAUDI2_NUM_OF_QM_ERR_CAUSE] = { 1136 "PQ AXI HBW error", 1137 "CQ AXI HBW error", 1138 "CP AXI HBW error", 1139 "CP error due to undefined OPCODE", 1140 "CP encountered STOP OPCODE", 1141 "CP AXI LBW error", 1142 "CP WRREG32 or WRBULK returned error", 1143 "N/A", 1144 "FENCE 0 inc over max value and clipped", 1145 "FENCE 1 inc over max value and clipped", 1146 "FENCE 2 inc over max value and clipped", 1147 "FENCE 3 inc over max value and clipped", 1148 "FENCE 0 dec under min value and clipped", 1149 "FENCE 1 dec under min value and clipped", 1150 "FENCE 2 dec under min value and clipped", 1151 "FENCE 3 dec under min value and clipped", 1152 "CPDMA Up overflow", 1153 "PQC L2H error" 1154}; 1155 1156static const char * const gaudi2_lower_qman_error_cause[GAUDI2_NUM_OF_LOWER_QM_ERR_CAUSE] = { 1157 "RSVD0", 1158 "CQ AXI HBW error", 1159 "CP AXI HBW error", 1160 "CP error due to undefined OPCODE", 1161 "CP encountered STOP OPCODE", 1162 "CP AXI LBW error", 1163 "CP WRREG32 or WRBULK returned error", 1164 "N/A", 1165 "FENCE 0 inc over max value and clipped", 1166 "FENCE 1 inc over max value and clipped", 1167 "FENCE 2 inc over max value and clipped", 1168 "FENCE 3 inc over max value and clipped", 1169 "FENCE 0 dec under min value and clipped", 1170 "FENCE 1 dec under min value and clipped", 1171 "FENCE 2 dec under min value and clipped", 1172 "FENCE 3 dec under min value and clipped", 1173 "CPDMA Up overflow", 1174 "RSVD17", 1175 "CQ_WR_IFIFO_CI_ERR", 1176 "CQ_WR_CTL_CI_ERR", 1177 "ARC_CQF_RD_ERR", 1178 "ARC_CQ_WR_IFIFO_CI_ERR", 1179 "ARC_CQ_WR_CTL_CI_ERR", 1180 "ARC_AXI_ERR", 1181 "CP_SWITCH_WDT_ERR" 1182}; 1183 1184static const char * const gaudi2_qman_arb_error_cause[GAUDI2_NUM_OF_QM_ARB_ERR_CAUSE] = { 1185 "Choice push while full error", 1186 "Choice Q watchdog error", 1187 "MSG AXI LBW returned with error" 1188}; 1189 1190static const char * const guadi2_rot_error_cause[GAUDI2_NUM_OF_ROT_ERR_CAUSE] = { 1191 "qm_axi_err", 1192 "qm_trace_fence_events", 1193 "qm_sw_err", 1194 "qm_cp_sw_stop", 1195 "lbw_mstr_rresp_err", 1196 "lbw_mstr_bresp_err", 1197 "lbw_msg_slverr", 1198 "hbw_msg_slverr", 1199 "wbc_slverr", 1200 "hbw_mstr_rresp_err", 1201 "hbw_mstr_bresp_err", 1202 "sb_resp_intr", 1203 "mrsb_resp_intr", 1204 "core_dw_status_0", 1205 "core_dw_status_1", 1206 "core_dw_status_2", 1207 "core_dw_status_3", 1208 "core_dw_status_4", 1209 "core_dw_status_5", 1210 "core_dw_status_6", 1211 "core_dw_status_7", 1212 "async_arc2cpu_sei_intr", 1213}; 1214 1215static const char * const gaudi2_tpc_interrupts_cause[GAUDI2_NUM_OF_TPC_INTR_CAUSE] = { 1216 "tpc_address_exceed_slm", 1217 "tpc_div_by_0", 1218 "tpc_spu_mac_overflow", 1219 "tpc_spu_addsub_overflow", 1220 "tpc_spu_abs_overflow", 1221 "tpc_spu_fma_fp_dst_nan", 1222 "tpc_spu_fma_fp_dst_inf", 1223 "tpc_spu_convert_fp_dst_nan", 1224 "tpc_spu_convert_fp_dst_inf", 1225 "tpc_spu_fp_dst_denorm", 1226 "tpc_vpu_mac_overflow", 1227 "tpc_vpu_addsub_overflow", 1228 "tpc_vpu_abs_overflow", 1229 "tpc_vpu_convert_fp_dst_nan", 1230 "tpc_vpu_convert_fp_dst_inf", 1231 "tpc_vpu_fma_fp_dst_nan", 1232 "tpc_vpu_fma_fp_dst_inf", 1233 "tpc_vpu_fp_dst_denorm", 1234 "tpc_assertions", 1235 "tpc_illegal_instruction", 1236 "tpc_pc_wrap_around", 1237 "tpc_qm_sw_err", 1238 "tpc_hbw_rresp_err", 1239 "tpc_hbw_bresp_err", 1240 "tpc_lbw_rresp_err", 1241 "tpc_lbw_bresp_err", 1242 "st_unlock_already_locked", 1243 "invalid_lock_access", 1244 "LD_L protection violation", 1245 "ST_L protection violation", 1246 "D$ L0CS mismatch", 1247}; 1248 1249static const char * const guadi2_mme_error_cause[GAUDI2_NUM_OF_MME_ERR_CAUSE] = { 1250 "agu_resp_intr", 1251 "qman_axi_err", 1252 "wap sei (wbc axi err)", 1253 "arc sei", 1254 "cfg access error", 1255 "qm_sw_err", 1256 "sbte_dbg_intr_0", 1257 "sbte_dbg_intr_1", 1258 "sbte_dbg_intr_2", 1259 "sbte_dbg_intr_3", 1260 "sbte_dbg_intr_4", 1261 "sbte_prtn_intr_0", 1262 "sbte_prtn_intr_1", 1263 "sbte_prtn_intr_2", 1264 "sbte_prtn_intr_3", 1265 "sbte_prtn_intr_4", 1266}; 1267 1268static const char * const guadi2_mme_wap_error_cause[GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE] = { 1269 "WBC ERR RESP_0", 1270 "WBC ERR RESP_1", 1271 "AP SOURCE POS INF", 1272 "AP SOURCE NEG INF", 1273 "AP SOURCE NAN", 1274 "AP RESULT POS INF", 1275 "AP RESULT NEG INF", 1276}; 1277 1278static const char * const gaudi2_dma_core_interrupts_cause[GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE] = { 1279 "HBW Read returned with error RRESP", 1280 "HBW write returned with error BRESP", 1281 "LBW write returned with error BRESP", 1282 "descriptor_fifo_overflow", 1283 "KDMA SB LBW Read returned with error", 1284 "KDMA WBC LBW Write returned with error", 1285 "TRANSPOSE ENGINE DESC FIFO OVERFLOW", 1286 "WRONG CFG FOR COMMIT IN LIN DMA" 1287}; 1288 1289static const char * const gaudi2_kdma_core_interrupts_cause[GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE] = { 1290 "HBW/LBW Read returned with error RRESP", 1291 "HBW/LBW write returned with error BRESP", 1292 "LBW write returned with error BRESP", 1293 "descriptor_fifo_overflow", 1294 "KDMA SB LBW Read returned with error", 1295 "KDMA WBC LBW Write returned with error", 1296 "TRANSPOSE ENGINE DESC FIFO OVERFLOW", 1297 "WRONG CFG FOR COMMIT IN LIN DMA" 1298}; 1299 1300struct gaudi2_sm_sei_cause_data { 1301 const char *cause_name; 1302 const char *log_name; 1303}; 1304 1305static const struct gaudi2_sm_sei_cause_data 1306gaudi2_sm_sei_cause[GAUDI2_NUM_OF_SM_SEI_ERR_CAUSE] = { 1307 {"calculated SO value overflow/underflow", "SOB ID"}, 1308 {"payload address of monitor is not aligned to 4B", "monitor addr"}, 1309 {"armed monitor write got BRESP (SLVERR or DECERR)", "AXI id"}, 1310}; 1311 1312static const char * const 1313gaudi2_pmmu_fatal_interrupts_cause[GAUDI2_NUM_OF_PMMU_FATAL_ERR_CAUSE] = { 1314 "LATENCY_RD_OUT_FIFO_OVERRUN", 1315 "LATENCY_WR_OUT_FIFO_OVERRUN", 1316}; 1317 1318static const char * const 1319gaudi2_hif_fatal_interrupts_cause[GAUDI2_NUM_OF_HIF_FATAL_ERR_CAUSE] = { 1320 "LATENCY_RD_OUT_FIFO_OVERRUN", 1321 "LATENCY_WR_OUT_FIFO_OVERRUN", 1322}; 1323 1324static const char * const 1325gaudi2_psoc_axi_drain_interrupts_cause[GAUDI2_NUM_OF_AXI_DRAIN_ERR_CAUSE] = { 1326 "AXI drain HBW", 1327 "AXI drain LBW", 1328}; 1329 1330static const char * const 1331gaudi2_pcie_addr_dec_error_cause[GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE] = { 1332 "HBW error response", 1333 "LBW error response", 1334 "TLP is blocked by RR" 1335}; 1336 1337static const int gaudi2_queue_id_to_engine_id[] = { 1338 [GAUDI2_QUEUE_ID_PDMA_0_0...GAUDI2_QUEUE_ID_PDMA_0_3] = GAUDI2_ENGINE_ID_PDMA_0, 1339 [GAUDI2_QUEUE_ID_PDMA_1_0...GAUDI2_QUEUE_ID_PDMA_1_3] = GAUDI2_ENGINE_ID_PDMA_1, 1340 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = 1341 GAUDI2_DCORE0_ENGINE_ID_EDMA_0, 1342 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = 1343 GAUDI2_DCORE0_ENGINE_ID_EDMA_1, 1344 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = 1345 GAUDI2_DCORE1_ENGINE_ID_EDMA_0, 1346 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = 1347 GAUDI2_DCORE1_ENGINE_ID_EDMA_1, 1348 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = 1349 GAUDI2_DCORE2_ENGINE_ID_EDMA_0, 1350 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = 1351 GAUDI2_DCORE2_ENGINE_ID_EDMA_1, 1352 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = 1353 GAUDI2_DCORE3_ENGINE_ID_EDMA_0, 1354 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = 1355 GAUDI2_DCORE3_ENGINE_ID_EDMA_1, 1356 [GAUDI2_QUEUE_ID_DCORE0_MME_0_0...GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = 1357 GAUDI2_DCORE0_ENGINE_ID_MME, 1358 [GAUDI2_QUEUE_ID_DCORE1_MME_0_0...GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = 1359 GAUDI2_DCORE1_ENGINE_ID_MME, 1360 [GAUDI2_QUEUE_ID_DCORE2_MME_0_0...GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = 1361 GAUDI2_DCORE2_ENGINE_ID_MME, 1362 [GAUDI2_QUEUE_ID_DCORE3_MME_0_0...GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = 1363 GAUDI2_DCORE3_ENGINE_ID_MME, 1364 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_0...GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = 1365 GAUDI2_DCORE0_ENGINE_ID_TPC_0, 1366 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_0...GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = 1367 GAUDI2_DCORE0_ENGINE_ID_TPC_1, 1368 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_0...GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = 1369 GAUDI2_DCORE0_ENGINE_ID_TPC_2, 1370 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_0...GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = 1371 GAUDI2_DCORE0_ENGINE_ID_TPC_3, 1372 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_0...GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = 1373 GAUDI2_DCORE0_ENGINE_ID_TPC_4, 1374 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_0...GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = 1375 GAUDI2_DCORE0_ENGINE_ID_TPC_5, 1376 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_0...GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = 1377 GAUDI2_DCORE0_ENGINE_ID_TPC_6, 1378 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_0...GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = 1379 GAUDI2_DCORE1_ENGINE_ID_TPC_0, 1380 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_0...GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = 1381 GAUDI2_DCORE1_ENGINE_ID_TPC_1, 1382 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_0...GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = 1383 GAUDI2_DCORE1_ENGINE_ID_TPC_2, 1384 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_0...GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = 1385 GAUDI2_DCORE1_ENGINE_ID_TPC_3, 1386 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_0...GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = 1387 GAUDI2_DCORE1_ENGINE_ID_TPC_4, 1388 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_0...GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = 1389 GAUDI2_DCORE1_ENGINE_ID_TPC_5, 1390 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_0...GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = 1391 GAUDI2_DCORE2_ENGINE_ID_TPC_0, 1392 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_0...GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = 1393 GAUDI2_DCORE2_ENGINE_ID_TPC_1, 1394 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_0...GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = 1395 GAUDI2_DCORE2_ENGINE_ID_TPC_2, 1396 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_0...GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = 1397 GAUDI2_DCORE2_ENGINE_ID_TPC_3, 1398 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_0...GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = 1399 GAUDI2_DCORE2_ENGINE_ID_TPC_4, 1400 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_0...GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = 1401 GAUDI2_DCORE2_ENGINE_ID_TPC_5, 1402 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_0...GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = 1403 GAUDI2_DCORE3_ENGINE_ID_TPC_0, 1404 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_0...GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = 1405 GAUDI2_DCORE3_ENGINE_ID_TPC_1, 1406 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_0...GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = 1407 GAUDI2_DCORE3_ENGINE_ID_TPC_2, 1408 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_0...GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = 1409 GAUDI2_DCORE3_ENGINE_ID_TPC_3, 1410 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_0...GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = 1411 GAUDI2_DCORE3_ENGINE_ID_TPC_4, 1412 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_0...GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = 1413 GAUDI2_DCORE3_ENGINE_ID_TPC_5, 1414 [GAUDI2_QUEUE_ID_NIC_0_0...GAUDI2_QUEUE_ID_NIC_0_3] = GAUDI2_ENGINE_ID_NIC0_0, 1415 [GAUDI2_QUEUE_ID_NIC_1_0...GAUDI2_QUEUE_ID_NIC_1_3] = GAUDI2_ENGINE_ID_NIC0_1, 1416 [GAUDI2_QUEUE_ID_NIC_2_0...GAUDI2_QUEUE_ID_NIC_2_3] = GAUDI2_ENGINE_ID_NIC1_0, 1417 [GAUDI2_QUEUE_ID_NIC_3_0...GAUDI2_QUEUE_ID_NIC_3_3] = GAUDI2_ENGINE_ID_NIC1_1, 1418 [GAUDI2_QUEUE_ID_NIC_4_0...GAUDI2_QUEUE_ID_NIC_4_3] = GAUDI2_ENGINE_ID_NIC2_0, 1419 [GAUDI2_QUEUE_ID_NIC_5_0...GAUDI2_QUEUE_ID_NIC_5_3] = GAUDI2_ENGINE_ID_NIC2_1, 1420 [GAUDI2_QUEUE_ID_NIC_6_0...GAUDI2_QUEUE_ID_NIC_6_3] = GAUDI2_ENGINE_ID_NIC3_0, 1421 [GAUDI2_QUEUE_ID_NIC_7_0...GAUDI2_QUEUE_ID_NIC_7_3] = GAUDI2_ENGINE_ID_NIC3_1, 1422 [GAUDI2_QUEUE_ID_NIC_8_0...GAUDI2_QUEUE_ID_NIC_8_3] = GAUDI2_ENGINE_ID_NIC4_0, 1423 [GAUDI2_QUEUE_ID_NIC_9_0...GAUDI2_QUEUE_ID_NIC_9_3] = GAUDI2_ENGINE_ID_NIC4_1, 1424 [GAUDI2_QUEUE_ID_NIC_10_0...GAUDI2_QUEUE_ID_NIC_10_3] = GAUDI2_ENGINE_ID_NIC5_0, 1425 [GAUDI2_QUEUE_ID_NIC_11_0...GAUDI2_QUEUE_ID_NIC_11_3] = GAUDI2_ENGINE_ID_NIC5_1, 1426 [GAUDI2_QUEUE_ID_NIC_12_0...GAUDI2_QUEUE_ID_NIC_12_3] = GAUDI2_ENGINE_ID_NIC6_0, 1427 [GAUDI2_QUEUE_ID_NIC_13_0...GAUDI2_QUEUE_ID_NIC_13_3] = GAUDI2_ENGINE_ID_NIC6_1, 1428 [GAUDI2_QUEUE_ID_NIC_14_0...GAUDI2_QUEUE_ID_NIC_14_3] = GAUDI2_ENGINE_ID_NIC7_0, 1429 [GAUDI2_QUEUE_ID_NIC_15_0...GAUDI2_QUEUE_ID_NIC_15_3] = GAUDI2_ENGINE_ID_NIC7_1, 1430 [GAUDI2_QUEUE_ID_NIC_16_0...GAUDI2_QUEUE_ID_NIC_16_3] = GAUDI2_ENGINE_ID_NIC8_0, 1431 [GAUDI2_QUEUE_ID_NIC_17_0...GAUDI2_QUEUE_ID_NIC_17_3] = GAUDI2_ENGINE_ID_NIC8_1, 1432 [GAUDI2_QUEUE_ID_NIC_18_0...GAUDI2_QUEUE_ID_NIC_18_3] = GAUDI2_ENGINE_ID_NIC9_0, 1433 [GAUDI2_QUEUE_ID_NIC_19_0...GAUDI2_QUEUE_ID_NIC_19_3] = GAUDI2_ENGINE_ID_NIC9_1, 1434 [GAUDI2_QUEUE_ID_NIC_20_0...GAUDI2_QUEUE_ID_NIC_20_3] = GAUDI2_ENGINE_ID_NIC10_0, 1435 [GAUDI2_QUEUE_ID_NIC_21_0...GAUDI2_QUEUE_ID_NIC_21_3] = GAUDI2_ENGINE_ID_NIC10_1, 1436 [GAUDI2_QUEUE_ID_NIC_22_0...GAUDI2_QUEUE_ID_NIC_22_3] = GAUDI2_ENGINE_ID_NIC11_0, 1437 [GAUDI2_QUEUE_ID_NIC_23_0...GAUDI2_QUEUE_ID_NIC_23_3] = GAUDI2_ENGINE_ID_NIC11_1, 1438 [GAUDI2_QUEUE_ID_ROT_0_0...GAUDI2_QUEUE_ID_ROT_0_3] = GAUDI2_ENGINE_ID_ROT_0, 1439 [GAUDI2_QUEUE_ID_ROT_1_0...GAUDI2_QUEUE_ID_ROT_1_3] = GAUDI2_ENGINE_ID_ROT_1, 1440}; 1441 1442const u32 gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_SIZE] = { 1443 [GAUDI2_QUEUE_ID_PDMA_0_0] = mmPDMA0_QM_BASE, 1444 [GAUDI2_QUEUE_ID_PDMA_0_1] = mmPDMA0_QM_BASE, 1445 [GAUDI2_QUEUE_ID_PDMA_0_2] = mmPDMA0_QM_BASE, 1446 [GAUDI2_QUEUE_ID_PDMA_0_3] = mmPDMA0_QM_BASE, 1447 [GAUDI2_QUEUE_ID_PDMA_1_0] = mmPDMA1_QM_BASE, 1448 [GAUDI2_QUEUE_ID_PDMA_1_1] = mmPDMA1_QM_BASE, 1449 [GAUDI2_QUEUE_ID_PDMA_1_2] = mmPDMA1_QM_BASE, 1450 [GAUDI2_QUEUE_ID_PDMA_1_3] = mmPDMA1_QM_BASE, 1451 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0] = mmDCORE0_EDMA0_QM_BASE, 1452 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_1] = mmDCORE0_EDMA0_QM_BASE, 1453 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_2] = mmDCORE0_EDMA0_QM_BASE, 1454 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = mmDCORE0_EDMA0_QM_BASE, 1455 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0] = mmDCORE0_EDMA1_QM_BASE, 1456 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_1] = mmDCORE0_EDMA1_QM_BASE, 1457 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_2] = mmDCORE0_EDMA1_QM_BASE, 1458 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = mmDCORE0_EDMA1_QM_BASE, 1459 [GAUDI2_QUEUE_ID_DCORE0_MME_0_0] = mmDCORE0_MME_QM_BASE, 1460 [GAUDI2_QUEUE_ID_DCORE0_MME_0_1] = mmDCORE0_MME_QM_BASE, 1461 [GAUDI2_QUEUE_ID_DCORE0_MME_0_2] = mmDCORE0_MME_QM_BASE, 1462 [GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = mmDCORE0_MME_QM_BASE, 1463 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_0] = mmDCORE0_TPC0_QM_BASE, 1464 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_1] = mmDCORE0_TPC0_QM_BASE, 1465 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_2] = mmDCORE0_TPC0_QM_BASE, 1466 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = mmDCORE0_TPC0_QM_BASE, 1467 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_0] = mmDCORE0_TPC1_QM_BASE, 1468 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_1] = mmDCORE0_TPC1_QM_BASE, 1469 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_2] = mmDCORE0_TPC1_QM_BASE, 1470 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = mmDCORE0_TPC1_QM_BASE, 1471 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_0] = mmDCORE0_TPC2_QM_BASE, 1472 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_1] = mmDCORE0_TPC2_QM_BASE, 1473 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_2] = mmDCORE0_TPC2_QM_BASE, 1474 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = mmDCORE0_TPC2_QM_BASE, 1475 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_0] = mmDCORE0_TPC3_QM_BASE, 1476 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_1] = mmDCORE0_TPC3_QM_BASE, 1477 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_2] = mmDCORE0_TPC3_QM_BASE, 1478 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = mmDCORE0_TPC3_QM_BASE, 1479 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_0] = mmDCORE0_TPC4_QM_BASE, 1480 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_1] = mmDCORE0_TPC4_QM_BASE, 1481 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_2] = mmDCORE0_TPC4_QM_BASE, 1482 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = mmDCORE0_TPC4_QM_BASE, 1483 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_0] = mmDCORE0_TPC5_QM_BASE, 1484 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_1] = mmDCORE0_TPC5_QM_BASE, 1485 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_2] = mmDCORE0_TPC5_QM_BASE, 1486 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = mmDCORE0_TPC5_QM_BASE, 1487 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_0] = mmDCORE0_TPC6_QM_BASE, 1488 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_1] = mmDCORE0_TPC6_QM_BASE, 1489 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_2] = mmDCORE0_TPC6_QM_BASE, 1490 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = mmDCORE0_TPC6_QM_BASE, 1491 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0] = mmDCORE1_EDMA0_QM_BASE, 1492 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_1] = mmDCORE1_EDMA0_QM_BASE, 1493 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_2] = mmDCORE1_EDMA0_QM_BASE, 1494 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = mmDCORE1_EDMA0_QM_BASE, 1495 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0] = mmDCORE1_EDMA1_QM_BASE, 1496 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_1] = mmDCORE1_EDMA1_QM_BASE, 1497 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_2] = mmDCORE1_EDMA1_QM_BASE, 1498 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = mmDCORE1_EDMA1_QM_BASE, 1499 [GAUDI2_QUEUE_ID_DCORE1_MME_0_0] = mmDCORE1_MME_QM_BASE, 1500 [GAUDI2_QUEUE_ID_DCORE1_MME_0_1] = mmDCORE1_MME_QM_BASE, 1501 [GAUDI2_QUEUE_ID_DCORE1_MME_0_2] = mmDCORE1_MME_QM_BASE, 1502 [GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = mmDCORE1_MME_QM_BASE, 1503 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_0] = mmDCORE1_TPC0_QM_BASE, 1504 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_1] = mmDCORE1_TPC0_QM_BASE, 1505 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_2] = mmDCORE1_TPC0_QM_BASE, 1506 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = mmDCORE1_TPC0_QM_BASE, 1507 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_0] = mmDCORE1_TPC1_QM_BASE, 1508 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_1] = mmDCORE1_TPC1_QM_BASE, 1509 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_2] = mmDCORE1_TPC1_QM_BASE, 1510 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = mmDCORE1_TPC1_QM_BASE, 1511 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_0] = mmDCORE1_TPC2_QM_BASE, 1512 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_1] = mmDCORE1_TPC2_QM_BASE, 1513 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_2] = mmDCORE1_TPC2_QM_BASE, 1514 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = mmDCORE1_TPC2_QM_BASE, 1515 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_0] = mmDCORE1_TPC3_QM_BASE, 1516 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_1] = mmDCORE1_TPC3_QM_BASE, 1517 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_2] = mmDCORE1_TPC3_QM_BASE, 1518 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = mmDCORE1_TPC3_QM_BASE, 1519 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_0] = mmDCORE1_TPC4_QM_BASE, 1520 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_1] = mmDCORE1_TPC4_QM_BASE, 1521 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_2] = mmDCORE1_TPC4_QM_BASE, 1522 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = mmDCORE1_TPC4_QM_BASE, 1523 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_0] = mmDCORE1_TPC5_QM_BASE, 1524 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_1] = mmDCORE1_TPC5_QM_BASE, 1525 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_2] = mmDCORE1_TPC5_QM_BASE, 1526 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = mmDCORE1_TPC5_QM_BASE, 1527 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0] = mmDCORE2_EDMA0_QM_BASE, 1528 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_1] = mmDCORE2_EDMA0_QM_BASE, 1529 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_2] = mmDCORE2_EDMA0_QM_BASE, 1530 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = mmDCORE2_EDMA0_QM_BASE, 1531 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0] = mmDCORE2_EDMA1_QM_BASE, 1532 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_1] = mmDCORE2_EDMA1_QM_BASE, 1533 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_2] = mmDCORE2_EDMA1_QM_BASE, 1534 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = mmDCORE2_EDMA1_QM_BASE, 1535 [GAUDI2_QUEUE_ID_DCORE2_MME_0_0] = mmDCORE2_MME_QM_BASE, 1536 [GAUDI2_QUEUE_ID_DCORE2_MME_0_1] = mmDCORE2_MME_QM_BASE, 1537 [GAUDI2_QUEUE_ID_DCORE2_MME_0_2] = mmDCORE2_MME_QM_BASE, 1538 [GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = mmDCORE2_MME_QM_BASE, 1539 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_0] = mmDCORE2_TPC0_QM_BASE, 1540 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_1] = mmDCORE2_TPC0_QM_BASE, 1541 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_2] = mmDCORE2_TPC0_QM_BASE, 1542 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = mmDCORE2_TPC0_QM_BASE, 1543 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_0] = mmDCORE2_TPC1_QM_BASE, 1544 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_1] = mmDCORE2_TPC1_QM_BASE, 1545 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_2] = mmDCORE2_TPC1_QM_BASE, 1546 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = mmDCORE2_TPC1_QM_BASE, 1547 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_0] = mmDCORE2_TPC2_QM_BASE, 1548 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_1] = mmDCORE2_TPC2_QM_BASE, 1549 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_2] = mmDCORE2_TPC2_QM_BASE, 1550 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = mmDCORE2_TPC2_QM_BASE, 1551 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_0] = mmDCORE2_TPC3_QM_BASE, 1552 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_1] = mmDCORE2_TPC3_QM_BASE, 1553 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_2] = mmDCORE2_TPC3_QM_BASE, 1554 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = mmDCORE2_TPC3_QM_BASE, 1555 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_0] = mmDCORE2_TPC4_QM_BASE, 1556 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_1] = mmDCORE2_TPC4_QM_BASE, 1557 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_2] = mmDCORE2_TPC4_QM_BASE, 1558 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = mmDCORE2_TPC4_QM_BASE, 1559 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_0] = mmDCORE2_TPC5_QM_BASE, 1560 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_1] = mmDCORE2_TPC5_QM_BASE, 1561 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_2] = mmDCORE2_TPC5_QM_BASE, 1562 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = mmDCORE2_TPC5_QM_BASE, 1563 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0] = mmDCORE3_EDMA0_QM_BASE, 1564 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_1] = mmDCORE3_EDMA0_QM_BASE, 1565 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_2] = mmDCORE3_EDMA0_QM_BASE, 1566 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = mmDCORE3_EDMA0_QM_BASE, 1567 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0] = mmDCORE3_EDMA1_QM_BASE, 1568 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1] = mmDCORE3_EDMA1_QM_BASE, 1569 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2] = mmDCORE3_EDMA1_QM_BASE, 1570 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = mmDCORE3_EDMA1_QM_BASE, 1571 [GAUDI2_QUEUE_ID_DCORE3_MME_0_0] = mmDCORE3_MME_QM_BASE, 1572 [GAUDI2_QUEUE_ID_DCORE3_MME_0_1] = mmDCORE3_MME_QM_BASE, 1573 [GAUDI2_QUEUE_ID_DCORE3_MME_0_2] = mmDCORE3_MME_QM_BASE, 1574 [GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = mmDCORE3_MME_QM_BASE, 1575 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_0] = mmDCORE3_TPC0_QM_BASE, 1576 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_1] = mmDCORE3_TPC0_QM_BASE, 1577 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_2] = mmDCORE3_TPC0_QM_BASE, 1578 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = mmDCORE3_TPC0_QM_BASE, 1579 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_0] = mmDCORE3_TPC1_QM_BASE, 1580 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_1] = mmDCORE3_TPC1_QM_BASE, 1581 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_2] = mmDCORE3_TPC1_QM_BASE, 1582 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = mmDCORE3_TPC1_QM_BASE, 1583 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_0] = mmDCORE3_TPC2_QM_BASE, 1584 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_1] = mmDCORE3_TPC2_QM_BASE, 1585 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_2] = mmDCORE3_TPC2_QM_BASE, 1586 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = mmDCORE3_TPC2_QM_BASE, 1587 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_0] = mmDCORE3_TPC3_QM_BASE, 1588 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_1] = mmDCORE3_TPC3_QM_BASE, 1589 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_2] = mmDCORE3_TPC3_QM_BASE, 1590 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = mmDCORE3_TPC3_QM_BASE, 1591 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_0] = mmDCORE3_TPC4_QM_BASE, 1592 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_1] = mmDCORE3_TPC4_QM_BASE, 1593 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_2] = mmDCORE3_TPC4_QM_BASE, 1594 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = mmDCORE3_TPC4_QM_BASE, 1595 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_0] = mmDCORE3_TPC5_QM_BASE, 1596 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_1] = mmDCORE3_TPC5_QM_BASE, 1597 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_2] = mmDCORE3_TPC5_QM_BASE, 1598 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = mmDCORE3_TPC5_QM_BASE, 1599 [GAUDI2_QUEUE_ID_NIC_0_0] = mmNIC0_QM0_BASE, 1600 [GAUDI2_QUEUE_ID_NIC_0_1] = mmNIC0_QM0_BASE, 1601 [GAUDI2_QUEUE_ID_NIC_0_2] = mmNIC0_QM0_BASE, 1602 [GAUDI2_QUEUE_ID_NIC_0_3] = mmNIC0_QM0_BASE, 1603 [GAUDI2_QUEUE_ID_NIC_1_0] = mmNIC0_QM1_BASE, 1604 [GAUDI2_QUEUE_ID_NIC_1_1] = mmNIC0_QM1_BASE, 1605 [GAUDI2_QUEUE_ID_NIC_1_2] = mmNIC0_QM1_BASE, 1606 [GAUDI2_QUEUE_ID_NIC_1_3] = mmNIC0_QM1_BASE, 1607 [GAUDI2_QUEUE_ID_NIC_2_0] = mmNIC1_QM0_BASE, 1608 [GAUDI2_QUEUE_ID_NIC_2_1] = mmNIC1_QM0_BASE, 1609 [GAUDI2_QUEUE_ID_NIC_2_2] = mmNIC1_QM0_BASE, 1610 [GAUDI2_QUEUE_ID_NIC_2_3] = mmNIC1_QM0_BASE, 1611 [GAUDI2_QUEUE_ID_NIC_3_0] = mmNIC1_QM1_BASE, 1612 [GAUDI2_QUEUE_ID_NIC_3_1] = mmNIC1_QM1_BASE, 1613 [GAUDI2_QUEUE_ID_NIC_3_2] = mmNIC1_QM1_BASE, 1614 [GAUDI2_QUEUE_ID_NIC_3_3] = mmNIC1_QM1_BASE, 1615 [GAUDI2_QUEUE_ID_NIC_4_0] = mmNIC2_QM0_BASE, 1616 [GAUDI2_QUEUE_ID_NIC_4_1] = mmNIC2_QM0_BASE, 1617 [GAUDI2_QUEUE_ID_NIC_4_2] = mmNIC2_QM0_BASE, 1618 [GAUDI2_QUEUE_ID_NIC_4_3] = mmNIC2_QM0_BASE, 1619 [GAUDI2_QUEUE_ID_NIC_5_0] = mmNIC2_QM1_BASE, 1620 [GAUDI2_QUEUE_ID_NIC_5_1] = mmNIC2_QM1_BASE, 1621 [GAUDI2_QUEUE_ID_NIC_5_2] = mmNIC2_QM1_BASE, 1622 [GAUDI2_QUEUE_ID_NIC_5_3] = mmNIC2_QM1_BASE, 1623 [GAUDI2_QUEUE_ID_NIC_6_0] = mmNIC3_QM0_BASE, 1624 [GAUDI2_QUEUE_ID_NIC_6_1] = mmNIC3_QM0_BASE, 1625 [GAUDI2_QUEUE_ID_NIC_6_2] = mmNIC3_QM0_BASE, 1626 [GAUDI2_QUEUE_ID_NIC_6_3] = mmNIC3_QM0_BASE, 1627 [GAUDI2_QUEUE_ID_NIC_7_0] = mmNIC3_QM1_BASE, 1628 [GAUDI2_QUEUE_ID_NIC_7_1] = mmNIC3_QM1_BASE, 1629 [GAUDI2_QUEUE_ID_NIC_7_2] = mmNIC3_QM1_BASE, 1630 [GAUDI2_QUEUE_ID_NIC_7_3] = mmNIC3_QM1_BASE, 1631 [GAUDI2_QUEUE_ID_NIC_8_0] = mmNIC4_QM0_BASE, 1632 [GAUDI2_QUEUE_ID_NIC_8_1] = mmNIC4_QM0_BASE, 1633 [GAUDI2_QUEUE_ID_NIC_8_2] = mmNIC4_QM0_BASE, 1634 [GAUDI2_QUEUE_ID_NIC_8_3] = mmNIC4_QM0_BASE, 1635 [GAUDI2_QUEUE_ID_NIC_9_0] = mmNIC4_QM1_BASE, 1636 [GAUDI2_QUEUE_ID_NIC_9_1] = mmNIC4_QM1_BASE, 1637 [GAUDI2_QUEUE_ID_NIC_9_2] = mmNIC4_QM1_BASE, 1638 [GAUDI2_QUEUE_ID_NIC_9_3] = mmNIC4_QM1_BASE, 1639 [GAUDI2_QUEUE_ID_NIC_10_0] = mmNIC5_QM0_BASE, 1640 [GAUDI2_QUEUE_ID_NIC_10_1] = mmNIC5_QM0_BASE, 1641 [GAUDI2_QUEUE_ID_NIC_10_2] = mmNIC5_QM0_BASE, 1642 [GAUDI2_QUEUE_ID_NIC_10_3] = mmNIC5_QM0_BASE, 1643 [GAUDI2_QUEUE_ID_NIC_11_0] = mmNIC5_QM1_BASE, 1644 [GAUDI2_QUEUE_ID_NIC_11_1] = mmNIC5_QM1_BASE, 1645 [GAUDI2_QUEUE_ID_NIC_11_2] = mmNIC5_QM1_BASE, 1646 [GAUDI2_QUEUE_ID_NIC_11_3] = mmNIC5_QM1_BASE, 1647 [GAUDI2_QUEUE_ID_NIC_12_0] = mmNIC6_QM0_BASE, 1648 [GAUDI2_QUEUE_ID_NIC_12_1] = mmNIC6_QM0_BASE, 1649 [GAUDI2_QUEUE_ID_NIC_12_2] = mmNIC6_QM0_BASE, 1650 [GAUDI2_QUEUE_ID_NIC_12_3] = mmNIC6_QM0_BASE, 1651 [GAUDI2_QUEUE_ID_NIC_13_0] = mmNIC6_QM1_BASE, 1652 [GAUDI2_QUEUE_ID_NIC_13_1] = mmNIC6_QM1_BASE, 1653 [GAUDI2_QUEUE_ID_NIC_13_2] = mmNIC6_QM1_BASE, 1654 [GAUDI2_QUEUE_ID_NIC_13_3] = mmNIC6_QM1_BASE, 1655 [GAUDI2_QUEUE_ID_NIC_14_0] = mmNIC7_QM0_BASE, 1656 [GAUDI2_QUEUE_ID_NIC_14_1] = mmNIC7_QM0_BASE, 1657 [GAUDI2_QUEUE_ID_NIC_14_2] = mmNIC7_QM0_BASE, 1658 [GAUDI2_QUEUE_ID_NIC_14_3] = mmNIC7_QM0_BASE, 1659 [GAUDI2_QUEUE_ID_NIC_15_0] = mmNIC7_QM1_BASE, 1660 [GAUDI2_QUEUE_ID_NIC_15_1] = mmNIC7_QM1_BASE, 1661 [GAUDI2_QUEUE_ID_NIC_15_2] = mmNIC7_QM1_BASE, 1662 [GAUDI2_QUEUE_ID_NIC_15_3] = mmNIC7_QM1_BASE, 1663 [GAUDI2_QUEUE_ID_NIC_16_0] = mmNIC8_QM0_BASE, 1664 [GAUDI2_QUEUE_ID_NIC_16_1] = mmNIC8_QM0_BASE, 1665 [GAUDI2_QUEUE_ID_NIC_16_2] = mmNIC8_QM0_BASE, 1666 [GAUDI2_QUEUE_ID_NIC_16_3] = mmNIC8_QM0_BASE, 1667 [GAUDI2_QUEUE_ID_NIC_17_0] = mmNIC8_QM1_BASE, 1668 [GAUDI2_QUEUE_ID_NIC_17_1] = mmNIC8_QM1_BASE, 1669 [GAUDI2_QUEUE_ID_NIC_17_2] = mmNIC8_QM1_BASE, 1670 [GAUDI2_QUEUE_ID_NIC_17_3] = mmNIC8_QM1_BASE, 1671 [GAUDI2_QUEUE_ID_NIC_18_0] = mmNIC9_QM0_BASE, 1672 [GAUDI2_QUEUE_ID_NIC_18_1] = mmNIC9_QM0_BASE, 1673 [GAUDI2_QUEUE_ID_NIC_18_2] = mmNIC9_QM0_BASE, 1674 [GAUDI2_QUEUE_ID_NIC_18_3] = mmNIC9_QM0_BASE, 1675 [GAUDI2_QUEUE_ID_NIC_19_0] = mmNIC9_QM1_BASE, 1676 [GAUDI2_QUEUE_ID_NIC_19_1] = mmNIC9_QM1_BASE, 1677 [GAUDI2_QUEUE_ID_NIC_19_2] = mmNIC9_QM1_BASE, 1678 [GAUDI2_QUEUE_ID_NIC_19_3] = mmNIC9_QM1_BASE, 1679 [GAUDI2_QUEUE_ID_NIC_20_0] = mmNIC10_QM0_BASE, 1680 [GAUDI2_QUEUE_ID_NIC_20_1] = mmNIC10_QM0_BASE, 1681 [GAUDI2_QUEUE_ID_NIC_20_2] = mmNIC10_QM0_BASE, 1682 [GAUDI2_QUEUE_ID_NIC_20_3] = mmNIC10_QM0_BASE, 1683 [GAUDI2_QUEUE_ID_NIC_21_0] = mmNIC10_QM1_BASE, 1684 [GAUDI2_QUEUE_ID_NIC_21_1] = mmNIC10_QM1_BASE, 1685 [GAUDI2_QUEUE_ID_NIC_21_2] = mmNIC10_QM1_BASE, 1686 [GAUDI2_QUEUE_ID_NIC_21_3] = mmNIC10_QM1_BASE, 1687 [GAUDI2_QUEUE_ID_NIC_22_0] = mmNIC11_QM0_BASE, 1688 [GAUDI2_QUEUE_ID_NIC_22_1] = mmNIC11_QM0_BASE, 1689 [GAUDI2_QUEUE_ID_NIC_22_2] = mmNIC11_QM0_BASE, 1690 [GAUDI2_QUEUE_ID_NIC_22_3] = mmNIC11_QM0_BASE, 1691 [GAUDI2_QUEUE_ID_NIC_23_0] = mmNIC11_QM1_BASE, 1692 [GAUDI2_QUEUE_ID_NIC_23_1] = mmNIC11_QM1_BASE, 1693 [GAUDI2_QUEUE_ID_NIC_23_2] = mmNIC11_QM1_BASE, 1694 [GAUDI2_QUEUE_ID_NIC_23_3] = mmNIC11_QM1_BASE, 1695 [GAUDI2_QUEUE_ID_ROT_0_0] = mmROT0_QM_BASE, 1696 [GAUDI2_QUEUE_ID_ROT_0_1] = mmROT0_QM_BASE, 1697 [GAUDI2_QUEUE_ID_ROT_0_2] = mmROT0_QM_BASE, 1698 [GAUDI2_QUEUE_ID_ROT_0_3] = mmROT0_QM_BASE, 1699 [GAUDI2_QUEUE_ID_ROT_1_0] = mmROT1_QM_BASE, 1700 [GAUDI2_QUEUE_ID_ROT_1_1] = mmROT1_QM_BASE, 1701 [GAUDI2_QUEUE_ID_ROT_1_2] = mmROT1_QM_BASE, 1702 [GAUDI2_QUEUE_ID_ROT_1_3] = mmROT1_QM_BASE 1703}; 1704 1705static const u32 gaudi2_arc_blocks_bases[NUM_ARC_CPUS] = { 1706 [CPU_ID_SCHED_ARC0] = mmARC_FARM_ARC0_AUX_BASE, 1707 [CPU_ID_SCHED_ARC1] = mmARC_FARM_ARC1_AUX_BASE, 1708 [CPU_ID_SCHED_ARC2] = mmARC_FARM_ARC2_AUX_BASE, 1709 [CPU_ID_SCHED_ARC3] = mmARC_FARM_ARC3_AUX_BASE, 1710 [CPU_ID_SCHED_ARC4] = mmDCORE1_MME_QM_ARC_AUX_BASE, 1711 [CPU_ID_SCHED_ARC5] = mmDCORE3_MME_QM_ARC_AUX_BASE, 1712 [CPU_ID_TPC_QMAN_ARC0] = mmDCORE0_TPC0_QM_ARC_AUX_BASE, 1713 [CPU_ID_TPC_QMAN_ARC1] = mmDCORE0_TPC1_QM_ARC_AUX_BASE, 1714 [CPU_ID_TPC_QMAN_ARC2] = mmDCORE0_TPC2_QM_ARC_AUX_BASE, 1715 [CPU_ID_TPC_QMAN_ARC3] = mmDCORE0_TPC3_QM_ARC_AUX_BASE, 1716 [CPU_ID_TPC_QMAN_ARC4] = mmDCORE0_TPC4_QM_ARC_AUX_BASE, 1717 [CPU_ID_TPC_QMAN_ARC5] = mmDCORE0_TPC5_QM_ARC_AUX_BASE, 1718 [CPU_ID_TPC_QMAN_ARC6] = mmDCORE1_TPC0_QM_ARC_AUX_BASE, 1719 [CPU_ID_TPC_QMAN_ARC7] = mmDCORE1_TPC1_QM_ARC_AUX_BASE, 1720 [CPU_ID_TPC_QMAN_ARC8] = mmDCORE1_TPC2_QM_ARC_AUX_BASE, 1721 [CPU_ID_TPC_QMAN_ARC9] = mmDCORE1_TPC3_QM_ARC_AUX_BASE, 1722 [CPU_ID_TPC_QMAN_ARC10] = mmDCORE1_TPC4_QM_ARC_AUX_BASE, 1723 [CPU_ID_TPC_QMAN_ARC11] = mmDCORE1_TPC5_QM_ARC_AUX_BASE, 1724 [CPU_ID_TPC_QMAN_ARC12] = mmDCORE2_TPC0_QM_ARC_AUX_BASE, 1725 [CPU_ID_TPC_QMAN_ARC13] = mmDCORE2_TPC1_QM_ARC_AUX_BASE, 1726 [CPU_ID_TPC_QMAN_ARC14] = mmDCORE2_TPC2_QM_ARC_AUX_BASE, 1727 [CPU_ID_TPC_QMAN_ARC15] = mmDCORE2_TPC3_QM_ARC_AUX_BASE, 1728 [CPU_ID_TPC_QMAN_ARC16] = mmDCORE2_TPC4_QM_ARC_AUX_BASE, 1729 [CPU_ID_TPC_QMAN_ARC17] = mmDCORE2_TPC5_QM_ARC_AUX_BASE, 1730 [CPU_ID_TPC_QMAN_ARC18] = mmDCORE3_TPC0_QM_ARC_AUX_BASE, 1731 [CPU_ID_TPC_QMAN_ARC19] = mmDCORE3_TPC1_QM_ARC_AUX_BASE, 1732 [CPU_ID_TPC_QMAN_ARC20] = mmDCORE3_TPC2_QM_ARC_AUX_BASE, 1733 [CPU_ID_TPC_QMAN_ARC21] = mmDCORE3_TPC3_QM_ARC_AUX_BASE, 1734 [CPU_ID_TPC_QMAN_ARC22] = mmDCORE3_TPC4_QM_ARC_AUX_BASE, 1735 [CPU_ID_TPC_QMAN_ARC23] = mmDCORE3_TPC5_QM_ARC_AUX_BASE, 1736 [CPU_ID_TPC_QMAN_ARC24] = mmDCORE0_TPC6_QM_ARC_AUX_BASE, 1737 [CPU_ID_MME_QMAN_ARC0] = mmDCORE0_MME_QM_ARC_AUX_BASE, 1738 [CPU_ID_MME_QMAN_ARC1] = mmDCORE2_MME_QM_ARC_AUX_BASE, 1739 [CPU_ID_EDMA_QMAN_ARC0] = mmDCORE0_EDMA0_QM_ARC_AUX_BASE, 1740 [CPU_ID_EDMA_QMAN_ARC1] = mmDCORE0_EDMA1_QM_ARC_AUX_BASE, 1741 [CPU_ID_EDMA_QMAN_ARC2] = mmDCORE1_EDMA0_QM_ARC_AUX_BASE, 1742 [CPU_ID_EDMA_QMAN_ARC3] = mmDCORE1_EDMA1_QM_ARC_AUX_BASE, 1743 [CPU_ID_EDMA_QMAN_ARC4] = mmDCORE2_EDMA0_QM_ARC_AUX_BASE, 1744 [CPU_ID_EDMA_QMAN_ARC5] = mmDCORE2_EDMA1_QM_ARC_AUX_BASE, 1745 [CPU_ID_EDMA_QMAN_ARC6] = mmDCORE3_EDMA0_QM_ARC_AUX_BASE, 1746 [CPU_ID_EDMA_QMAN_ARC7] = mmDCORE3_EDMA1_QM_ARC_AUX_BASE, 1747 [CPU_ID_PDMA_QMAN_ARC0] = mmPDMA0_QM_ARC_AUX_BASE, 1748 [CPU_ID_PDMA_QMAN_ARC1] = mmPDMA1_QM_ARC_AUX_BASE, 1749 [CPU_ID_ROT_QMAN_ARC0] = mmROT0_QM_ARC_AUX_BASE, 1750 [CPU_ID_ROT_QMAN_ARC1] = mmROT1_QM_ARC_AUX_BASE, 1751 [CPU_ID_NIC_QMAN_ARC0] = mmNIC0_QM_ARC_AUX0_BASE, 1752 [CPU_ID_NIC_QMAN_ARC1] = mmNIC0_QM_ARC_AUX1_BASE, 1753 [CPU_ID_NIC_QMAN_ARC2] = mmNIC1_QM_ARC_AUX0_BASE, 1754 [CPU_ID_NIC_QMAN_ARC3] = mmNIC1_QM_ARC_AUX1_BASE, 1755 [CPU_ID_NIC_QMAN_ARC4] = mmNIC2_QM_ARC_AUX0_BASE, 1756 [CPU_ID_NIC_QMAN_ARC5] = mmNIC2_QM_ARC_AUX1_BASE, 1757 [CPU_ID_NIC_QMAN_ARC6] = mmNIC3_QM_ARC_AUX0_BASE, 1758 [CPU_ID_NIC_QMAN_ARC7] = mmNIC3_QM_ARC_AUX1_BASE, 1759 [CPU_ID_NIC_QMAN_ARC8] = mmNIC4_QM_ARC_AUX0_BASE, 1760 [CPU_ID_NIC_QMAN_ARC9] = mmNIC4_QM_ARC_AUX1_BASE, 1761 [CPU_ID_NIC_QMAN_ARC10] = mmNIC5_QM_ARC_AUX0_BASE, 1762 [CPU_ID_NIC_QMAN_ARC11] = mmNIC5_QM_ARC_AUX1_BASE, 1763 [CPU_ID_NIC_QMAN_ARC12] = mmNIC6_QM_ARC_AUX0_BASE, 1764 [CPU_ID_NIC_QMAN_ARC13] = mmNIC6_QM_ARC_AUX1_BASE, 1765 [CPU_ID_NIC_QMAN_ARC14] = mmNIC7_QM_ARC_AUX0_BASE, 1766 [CPU_ID_NIC_QMAN_ARC15] = mmNIC7_QM_ARC_AUX1_BASE, 1767 [CPU_ID_NIC_QMAN_ARC16] = mmNIC8_QM_ARC_AUX0_BASE, 1768 [CPU_ID_NIC_QMAN_ARC17] = mmNIC8_QM_ARC_AUX1_BASE, 1769 [CPU_ID_NIC_QMAN_ARC18] = mmNIC9_QM_ARC_AUX0_BASE, 1770 [CPU_ID_NIC_QMAN_ARC19] = mmNIC9_QM_ARC_AUX1_BASE, 1771 [CPU_ID_NIC_QMAN_ARC20] = mmNIC10_QM_ARC_AUX0_BASE, 1772 [CPU_ID_NIC_QMAN_ARC21] = mmNIC10_QM_ARC_AUX1_BASE, 1773 [CPU_ID_NIC_QMAN_ARC22] = mmNIC11_QM_ARC_AUX0_BASE, 1774 [CPU_ID_NIC_QMAN_ARC23] = mmNIC11_QM_ARC_AUX1_BASE, 1775}; 1776 1777static const u32 gaudi2_arc_dccm_bases[NUM_ARC_CPUS] = { 1778 [CPU_ID_SCHED_ARC0] = mmARC_FARM_ARC0_DCCM0_BASE, 1779 [CPU_ID_SCHED_ARC1] = mmARC_FARM_ARC1_DCCM0_BASE, 1780 [CPU_ID_SCHED_ARC2] = mmARC_FARM_ARC2_DCCM0_BASE, 1781 [CPU_ID_SCHED_ARC3] = mmARC_FARM_ARC3_DCCM0_BASE, 1782 [CPU_ID_SCHED_ARC4] = mmDCORE1_MME_QM_ARC_DCCM_BASE, 1783 [CPU_ID_SCHED_ARC5] = mmDCORE3_MME_QM_ARC_DCCM_BASE, 1784 [CPU_ID_TPC_QMAN_ARC0] = mmDCORE0_TPC0_QM_DCCM_BASE, 1785 [CPU_ID_TPC_QMAN_ARC1] = mmDCORE0_TPC1_QM_DCCM_BASE, 1786 [CPU_ID_TPC_QMAN_ARC2] = mmDCORE0_TPC2_QM_DCCM_BASE, 1787 [CPU_ID_TPC_QMAN_ARC3] = mmDCORE0_TPC3_QM_DCCM_BASE, 1788 [CPU_ID_TPC_QMAN_ARC4] = mmDCORE0_TPC4_QM_DCCM_BASE, 1789 [CPU_ID_TPC_QMAN_ARC5] = mmDCORE0_TPC5_QM_DCCM_BASE, 1790 [CPU_ID_TPC_QMAN_ARC6] = mmDCORE1_TPC0_QM_DCCM_BASE, 1791 [CPU_ID_TPC_QMAN_ARC7] = mmDCORE1_TPC1_QM_DCCM_BASE, 1792 [CPU_ID_TPC_QMAN_ARC8] = mmDCORE1_TPC2_QM_DCCM_BASE, 1793 [CPU_ID_TPC_QMAN_ARC9] = mmDCORE1_TPC3_QM_DCCM_BASE, 1794 [CPU_ID_TPC_QMAN_ARC10] = mmDCORE1_TPC4_QM_DCCM_BASE, 1795 [CPU_ID_TPC_QMAN_ARC11] = mmDCORE1_TPC5_QM_DCCM_BASE, 1796 [CPU_ID_TPC_QMAN_ARC12] = mmDCORE2_TPC0_QM_DCCM_BASE, 1797 [CPU_ID_TPC_QMAN_ARC13] = mmDCORE2_TPC1_QM_DCCM_BASE, 1798 [CPU_ID_TPC_QMAN_ARC14] = mmDCORE2_TPC2_QM_DCCM_BASE, 1799 [CPU_ID_TPC_QMAN_ARC15] = mmDCORE2_TPC3_QM_DCCM_BASE, 1800 [CPU_ID_TPC_QMAN_ARC16] = mmDCORE2_TPC4_QM_DCCM_BASE, 1801 [CPU_ID_TPC_QMAN_ARC17] = mmDCORE2_TPC5_QM_DCCM_BASE, 1802 [CPU_ID_TPC_QMAN_ARC18] = mmDCORE3_TPC0_QM_DCCM_BASE, 1803 [CPU_ID_TPC_QMAN_ARC19] = mmDCORE3_TPC1_QM_DCCM_BASE, 1804 [CPU_ID_TPC_QMAN_ARC20] = mmDCORE3_TPC2_QM_DCCM_BASE, 1805 [CPU_ID_TPC_QMAN_ARC21] = mmDCORE3_TPC3_QM_DCCM_BASE, 1806 [CPU_ID_TPC_QMAN_ARC22] = mmDCORE3_TPC4_QM_DCCM_BASE, 1807 [CPU_ID_TPC_QMAN_ARC23] = mmDCORE3_TPC5_QM_DCCM_BASE, 1808 [CPU_ID_TPC_QMAN_ARC24] = mmDCORE0_TPC6_QM_DCCM_BASE, 1809 [CPU_ID_MME_QMAN_ARC0] = mmDCORE0_MME_QM_ARC_DCCM_BASE, 1810 [CPU_ID_MME_QMAN_ARC1] = mmDCORE2_MME_QM_ARC_DCCM_BASE, 1811 [CPU_ID_EDMA_QMAN_ARC0] = mmDCORE0_EDMA0_QM_DCCM_BASE, 1812 [CPU_ID_EDMA_QMAN_ARC1] = mmDCORE0_EDMA1_QM_DCCM_BASE, 1813 [CPU_ID_EDMA_QMAN_ARC2] = mmDCORE1_EDMA0_QM_DCCM_BASE, 1814 [CPU_ID_EDMA_QMAN_ARC3] = mmDCORE1_EDMA1_QM_DCCM_BASE, 1815 [CPU_ID_EDMA_QMAN_ARC4] = mmDCORE2_EDMA0_QM_DCCM_BASE, 1816 [CPU_ID_EDMA_QMAN_ARC5] = mmDCORE2_EDMA1_QM_DCCM_BASE, 1817 [CPU_ID_EDMA_QMAN_ARC6] = mmDCORE3_EDMA0_QM_DCCM_BASE, 1818 [CPU_ID_EDMA_QMAN_ARC7] = mmDCORE3_EDMA1_QM_DCCM_BASE, 1819 [CPU_ID_PDMA_QMAN_ARC0] = mmPDMA0_QM_ARC_DCCM_BASE, 1820 [CPU_ID_PDMA_QMAN_ARC1] = mmPDMA1_QM_ARC_DCCM_BASE, 1821 [CPU_ID_ROT_QMAN_ARC0] = mmROT0_QM_ARC_DCCM_BASE, 1822 [CPU_ID_ROT_QMAN_ARC1] = mmROT1_QM_ARC_DCCM_BASE, 1823 [CPU_ID_NIC_QMAN_ARC0] = mmNIC0_QM_DCCM0_BASE, 1824 [CPU_ID_NIC_QMAN_ARC1] = mmNIC0_QM_DCCM1_BASE, 1825 [CPU_ID_NIC_QMAN_ARC2] = mmNIC1_QM_DCCM0_BASE, 1826 [CPU_ID_NIC_QMAN_ARC3] = mmNIC1_QM_DCCM1_BASE, 1827 [CPU_ID_NIC_QMAN_ARC4] = mmNIC2_QM_DCCM0_BASE, 1828 [CPU_ID_NIC_QMAN_ARC5] = mmNIC2_QM_DCCM1_BASE, 1829 [CPU_ID_NIC_QMAN_ARC6] = mmNIC3_QM_DCCM0_BASE, 1830 [CPU_ID_NIC_QMAN_ARC7] = mmNIC3_QM_DCCM1_BASE, 1831 [CPU_ID_NIC_QMAN_ARC8] = mmNIC4_QM_DCCM0_BASE, 1832 [CPU_ID_NIC_QMAN_ARC9] = mmNIC4_QM_DCCM1_BASE, 1833 [CPU_ID_NIC_QMAN_ARC10] = mmNIC5_QM_DCCM0_BASE, 1834 [CPU_ID_NIC_QMAN_ARC11] = mmNIC5_QM_DCCM1_BASE, 1835 [CPU_ID_NIC_QMAN_ARC12] = mmNIC6_QM_DCCM0_BASE, 1836 [CPU_ID_NIC_QMAN_ARC13] = mmNIC6_QM_DCCM1_BASE, 1837 [CPU_ID_NIC_QMAN_ARC14] = mmNIC7_QM_DCCM0_BASE, 1838 [CPU_ID_NIC_QMAN_ARC15] = mmNIC7_QM_DCCM1_BASE, 1839 [CPU_ID_NIC_QMAN_ARC16] = mmNIC8_QM_DCCM0_BASE, 1840 [CPU_ID_NIC_QMAN_ARC17] = mmNIC8_QM_DCCM1_BASE, 1841 [CPU_ID_NIC_QMAN_ARC18] = mmNIC9_QM_DCCM0_BASE, 1842 [CPU_ID_NIC_QMAN_ARC19] = mmNIC9_QM_DCCM1_BASE, 1843 [CPU_ID_NIC_QMAN_ARC20] = mmNIC10_QM_DCCM0_BASE, 1844 [CPU_ID_NIC_QMAN_ARC21] = mmNIC10_QM_DCCM1_BASE, 1845 [CPU_ID_NIC_QMAN_ARC22] = mmNIC11_QM_DCCM0_BASE, 1846 [CPU_ID_NIC_QMAN_ARC23] = mmNIC11_QM_DCCM1_BASE, 1847}; 1848 1849const u32 gaudi2_mme_ctrl_lo_blocks_bases[MME_ID_SIZE] = { 1850 [MME_ID_DCORE0] = mmDCORE0_MME_CTRL_LO_BASE, 1851 [MME_ID_DCORE1] = mmDCORE1_MME_CTRL_LO_BASE, 1852 [MME_ID_DCORE2] = mmDCORE2_MME_CTRL_LO_BASE, 1853 [MME_ID_DCORE3] = mmDCORE3_MME_CTRL_LO_BASE, 1854}; 1855 1856static const u32 gaudi2_queue_id_to_arc_id[GAUDI2_QUEUE_ID_SIZE] = { 1857 [GAUDI2_QUEUE_ID_PDMA_0_0] = CPU_ID_PDMA_QMAN_ARC0, 1858 [GAUDI2_QUEUE_ID_PDMA_0_1] = CPU_ID_PDMA_QMAN_ARC0, 1859 [GAUDI2_QUEUE_ID_PDMA_0_2] = CPU_ID_PDMA_QMAN_ARC0, 1860 [GAUDI2_QUEUE_ID_PDMA_0_3] = CPU_ID_PDMA_QMAN_ARC0, 1861 [GAUDI2_QUEUE_ID_PDMA_1_0] = CPU_ID_PDMA_QMAN_ARC1, 1862 [GAUDI2_QUEUE_ID_PDMA_1_1] = CPU_ID_PDMA_QMAN_ARC1, 1863 [GAUDI2_QUEUE_ID_PDMA_1_2] = CPU_ID_PDMA_QMAN_ARC1, 1864 [GAUDI2_QUEUE_ID_PDMA_1_3] = CPU_ID_PDMA_QMAN_ARC1, 1865 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC0, 1866 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC0, 1867 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC0, 1868 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC0, 1869 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC1, 1870 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC1, 1871 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC1, 1872 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC1, 1873 [GAUDI2_QUEUE_ID_DCORE0_MME_0_0] = CPU_ID_MME_QMAN_ARC0, 1874 [GAUDI2_QUEUE_ID_DCORE0_MME_0_1] = CPU_ID_MME_QMAN_ARC0, 1875 [GAUDI2_QUEUE_ID_DCORE0_MME_0_2] = CPU_ID_MME_QMAN_ARC0, 1876 [GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = CPU_ID_MME_QMAN_ARC0, 1877 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_0] = CPU_ID_TPC_QMAN_ARC0, 1878 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_1] = CPU_ID_TPC_QMAN_ARC0, 1879 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_2] = CPU_ID_TPC_QMAN_ARC0, 1880 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = CPU_ID_TPC_QMAN_ARC0, 1881 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_0] = CPU_ID_TPC_QMAN_ARC1, 1882 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_1] = CPU_ID_TPC_QMAN_ARC1, 1883 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_2] = CPU_ID_TPC_QMAN_ARC1, 1884 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = CPU_ID_TPC_QMAN_ARC1, 1885 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_0] = CPU_ID_TPC_QMAN_ARC2, 1886 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_1] = CPU_ID_TPC_QMAN_ARC2, 1887 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_2] = CPU_ID_TPC_QMAN_ARC2, 1888 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = CPU_ID_TPC_QMAN_ARC2, 1889 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_0] = CPU_ID_TPC_QMAN_ARC3, 1890 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_1] = CPU_ID_TPC_QMAN_ARC3, 1891 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_2] = CPU_ID_TPC_QMAN_ARC3, 1892 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = CPU_ID_TPC_QMAN_ARC3, 1893 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_0] = CPU_ID_TPC_QMAN_ARC4, 1894 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_1] = CPU_ID_TPC_QMAN_ARC4, 1895 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_2] = CPU_ID_TPC_QMAN_ARC4, 1896 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = CPU_ID_TPC_QMAN_ARC4, 1897 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_0] = CPU_ID_TPC_QMAN_ARC5, 1898 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_1] = CPU_ID_TPC_QMAN_ARC5, 1899 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_2] = CPU_ID_TPC_QMAN_ARC5, 1900 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = CPU_ID_TPC_QMAN_ARC5, 1901 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_0] = CPU_ID_TPC_QMAN_ARC24, 1902 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_1] = CPU_ID_TPC_QMAN_ARC24, 1903 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_2] = CPU_ID_TPC_QMAN_ARC24, 1904 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = CPU_ID_TPC_QMAN_ARC24, 1905 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC2, 1906 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC2, 1907 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC2, 1908 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC2, 1909 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC3, 1910 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC3, 1911 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC3, 1912 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC3, 1913 [GAUDI2_QUEUE_ID_DCORE1_MME_0_0] = CPU_ID_SCHED_ARC4, 1914 [GAUDI2_QUEUE_ID_DCORE1_MME_0_1] = CPU_ID_SCHED_ARC4, 1915 [GAUDI2_QUEUE_ID_DCORE1_MME_0_2] = CPU_ID_SCHED_ARC4, 1916 [GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = CPU_ID_SCHED_ARC4, 1917 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_0] = CPU_ID_TPC_QMAN_ARC6, 1918 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_1] = CPU_ID_TPC_QMAN_ARC6, 1919 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_2] = CPU_ID_TPC_QMAN_ARC6, 1920 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = CPU_ID_TPC_QMAN_ARC6, 1921 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_0] = CPU_ID_TPC_QMAN_ARC7, 1922 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_1] = CPU_ID_TPC_QMAN_ARC7, 1923 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_2] = CPU_ID_TPC_QMAN_ARC7, 1924 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = CPU_ID_TPC_QMAN_ARC7, 1925 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_0] = CPU_ID_TPC_QMAN_ARC8, 1926 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_1] = CPU_ID_TPC_QMAN_ARC8, 1927 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_2] = CPU_ID_TPC_QMAN_ARC8, 1928 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = CPU_ID_TPC_QMAN_ARC8, 1929 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_0] = CPU_ID_TPC_QMAN_ARC9, 1930 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_1] = CPU_ID_TPC_QMAN_ARC9, 1931 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_2] = CPU_ID_TPC_QMAN_ARC9, 1932 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = CPU_ID_TPC_QMAN_ARC9, 1933 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_0] = CPU_ID_TPC_QMAN_ARC10, 1934 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_1] = CPU_ID_TPC_QMAN_ARC10, 1935 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_2] = CPU_ID_TPC_QMAN_ARC10, 1936 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = CPU_ID_TPC_QMAN_ARC10, 1937 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_0] = CPU_ID_TPC_QMAN_ARC11, 1938 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_1] = CPU_ID_TPC_QMAN_ARC11, 1939 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_2] = CPU_ID_TPC_QMAN_ARC11, 1940 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = CPU_ID_TPC_QMAN_ARC11, 1941 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC4, 1942 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC4, 1943 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC4, 1944 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC4, 1945 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC5, 1946 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC5, 1947 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC5, 1948 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC5, 1949 [GAUDI2_QUEUE_ID_DCORE2_MME_0_0] = CPU_ID_MME_QMAN_ARC1, 1950 [GAUDI2_QUEUE_ID_DCORE2_MME_0_1] = CPU_ID_MME_QMAN_ARC1, 1951 [GAUDI2_QUEUE_ID_DCORE2_MME_0_2] = CPU_ID_MME_QMAN_ARC1, 1952 [GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = CPU_ID_MME_QMAN_ARC1, 1953 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_0] = CPU_ID_TPC_QMAN_ARC12, 1954 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_1] = CPU_ID_TPC_QMAN_ARC12, 1955 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_2] = CPU_ID_TPC_QMAN_ARC12, 1956 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = CPU_ID_TPC_QMAN_ARC12, 1957 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_0] = CPU_ID_TPC_QMAN_ARC13, 1958 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_1] = CPU_ID_TPC_QMAN_ARC13, 1959 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_2] = CPU_ID_TPC_QMAN_ARC13, 1960 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = CPU_ID_TPC_QMAN_ARC13, 1961 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_0] = CPU_ID_TPC_QMAN_ARC14, 1962 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_1] = CPU_ID_TPC_QMAN_ARC14, 1963 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_2] = CPU_ID_TPC_QMAN_ARC14, 1964 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = CPU_ID_TPC_QMAN_ARC14, 1965 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_0] = CPU_ID_TPC_QMAN_ARC15, 1966 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_1] = CPU_ID_TPC_QMAN_ARC15, 1967 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_2] = CPU_ID_TPC_QMAN_ARC15, 1968 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = CPU_ID_TPC_QMAN_ARC15, 1969 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_0] = CPU_ID_TPC_QMAN_ARC16, 1970 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_1] = CPU_ID_TPC_QMAN_ARC16, 1971 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_2] = CPU_ID_TPC_QMAN_ARC16, 1972 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = CPU_ID_TPC_QMAN_ARC16, 1973 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_0] = CPU_ID_TPC_QMAN_ARC17, 1974 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_1] = CPU_ID_TPC_QMAN_ARC17, 1975 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_2] = CPU_ID_TPC_QMAN_ARC17, 1976 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = CPU_ID_TPC_QMAN_ARC17, 1977 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC6, 1978 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC6, 1979 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC6, 1980 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC6, 1981 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC7, 1982 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC7, 1983 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC7, 1984 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC7, 1985 [GAUDI2_QUEUE_ID_DCORE3_MME_0_0] = CPU_ID_SCHED_ARC5, 1986 [GAUDI2_QUEUE_ID_DCORE3_MME_0_1] = CPU_ID_SCHED_ARC5, 1987 [GAUDI2_QUEUE_ID_DCORE3_MME_0_2] = CPU_ID_SCHED_ARC5, 1988 [GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = CPU_ID_SCHED_ARC5, 1989 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_0] = CPU_ID_TPC_QMAN_ARC18, 1990 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_1] = CPU_ID_TPC_QMAN_ARC18, 1991 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_2] = CPU_ID_TPC_QMAN_ARC18, 1992 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = CPU_ID_TPC_QMAN_ARC18, 1993 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_0] = CPU_ID_TPC_QMAN_ARC19, 1994 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_1] = CPU_ID_TPC_QMAN_ARC19, 1995 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_2] = CPU_ID_TPC_QMAN_ARC19, 1996 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = CPU_ID_TPC_QMAN_ARC19, 1997 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_0] = CPU_ID_TPC_QMAN_ARC20, 1998 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_1] = CPU_ID_TPC_QMAN_ARC20, 1999 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_2] = CPU_ID_TPC_QMAN_ARC20, 2000 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = CPU_ID_TPC_QMAN_ARC20, 2001 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_0] = CPU_ID_TPC_QMAN_ARC21, 2002 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_1] = CPU_ID_TPC_QMAN_ARC21, 2003 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_2] = CPU_ID_TPC_QMAN_ARC21, 2004 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = CPU_ID_TPC_QMAN_ARC21, 2005 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_0] = CPU_ID_TPC_QMAN_ARC22, 2006 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_1] = CPU_ID_TPC_QMAN_ARC22, 2007 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_2] = CPU_ID_TPC_QMAN_ARC22, 2008 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = CPU_ID_TPC_QMAN_ARC22, 2009 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_0] = CPU_ID_TPC_QMAN_ARC23, 2010 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_1] = CPU_ID_TPC_QMAN_ARC23, 2011 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_2] = CPU_ID_TPC_QMAN_ARC23, 2012 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = CPU_ID_TPC_QMAN_ARC23, 2013 [GAUDI2_QUEUE_ID_NIC_0_0] = CPU_ID_NIC_QMAN_ARC0, 2014 [GAUDI2_QUEUE_ID_NIC_0_1] = CPU_ID_NIC_QMAN_ARC0, 2015 [GAUDI2_QUEUE_ID_NIC_0_2] = CPU_ID_NIC_QMAN_ARC0, 2016 [GAUDI2_QUEUE_ID_NIC_0_3] = CPU_ID_NIC_QMAN_ARC0, 2017 [GAUDI2_QUEUE_ID_NIC_1_0] = CPU_ID_NIC_QMAN_ARC1, 2018 [GAUDI2_QUEUE_ID_NIC_1_1] = CPU_ID_NIC_QMAN_ARC1, 2019 [GAUDI2_QUEUE_ID_NIC_1_2] = CPU_ID_NIC_QMAN_ARC1, 2020 [GAUDI2_QUEUE_ID_NIC_1_3] = CPU_ID_NIC_QMAN_ARC1, 2021 [GAUDI2_QUEUE_ID_NIC_2_0] = CPU_ID_NIC_QMAN_ARC2, 2022 [GAUDI2_QUEUE_ID_NIC_2_1] = CPU_ID_NIC_QMAN_ARC2, 2023 [GAUDI2_QUEUE_ID_NIC_2_2] = CPU_ID_NIC_QMAN_ARC2, 2024 [GAUDI2_QUEUE_ID_NIC_2_3] = CPU_ID_NIC_QMAN_ARC2, 2025 [GAUDI2_QUEUE_ID_NIC_3_0] = CPU_ID_NIC_QMAN_ARC3, 2026 [GAUDI2_QUEUE_ID_NIC_3_1] = CPU_ID_NIC_QMAN_ARC3, 2027 [GAUDI2_QUEUE_ID_NIC_3_2] = CPU_ID_NIC_QMAN_ARC3, 2028 [GAUDI2_QUEUE_ID_NIC_3_3] = CPU_ID_NIC_QMAN_ARC3, 2029 [GAUDI2_QUEUE_ID_NIC_4_0] = CPU_ID_NIC_QMAN_ARC4, 2030 [GAUDI2_QUEUE_ID_NIC_4_1] = CPU_ID_NIC_QMAN_ARC4, 2031 [GAUDI2_QUEUE_ID_NIC_4_2] = CPU_ID_NIC_QMAN_ARC4, 2032 [GAUDI2_QUEUE_ID_NIC_4_3] = CPU_ID_NIC_QMAN_ARC4, 2033 [GAUDI2_QUEUE_ID_NIC_5_0] = CPU_ID_NIC_QMAN_ARC5, 2034 [GAUDI2_QUEUE_ID_NIC_5_1] = CPU_ID_NIC_QMAN_ARC5, 2035 [GAUDI2_QUEUE_ID_NIC_5_2] = CPU_ID_NIC_QMAN_ARC5, 2036 [GAUDI2_QUEUE_ID_NIC_5_3] = CPU_ID_NIC_QMAN_ARC5, 2037 [GAUDI2_QUEUE_ID_NIC_6_0] = CPU_ID_NIC_QMAN_ARC6, 2038 [GAUDI2_QUEUE_ID_NIC_6_1] = CPU_ID_NIC_QMAN_ARC6, 2039 [GAUDI2_QUEUE_ID_NIC_6_2] = CPU_ID_NIC_QMAN_ARC6, 2040 [GAUDI2_QUEUE_ID_NIC_6_3] = CPU_ID_NIC_QMAN_ARC6, 2041 [GAUDI2_QUEUE_ID_NIC_7_0] = CPU_ID_NIC_QMAN_ARC7, 2042 [GAUDI2_QUEUE_ID_NIC_7_1] = CPU_ID_NIC_QMAN_ARC7, 2043 [GAUDI2_QUEUE_ID_NIC_7_2] = CPU_ID_NIC_QMAN_ARC7, 2044 [GAUDI2_QUEUE_ID_NIC_7_3] = CPU_ID_NIC_QMAN_ARC7, 2045 [GAUDI2_QUEUE_ID_NIC_8_0] = CPU_ID_NIC_QMAN_ARC8, 2046 [GAUDI2_QUEUE_ID_NIC_8_1] = CPU_ID_NIC_QMAN_ARC8, 2047 [GAUDI2_QUEUE_ID_NIC_8_2] = CPU_ID_NIC_QMAN_ARC8, 2048 [GAUDI2_QUEUE_ID_NIC_8_3] = CPU_ID_NIC_QMAN_ARC8, 2049 [GAUDI2_QUEUE_ID_NIC_9_0] = CPU_ID_NIC_QMAN_ARC9, 2050 [GAUDI2_QUEUE_ID_NIC_9_1] = CPU_ID_NIC_QMAN_ARC9, 2051 [GAUDI2_QUEUE_ID_NIC_9_2] = CPU_ID_NIC_QMAN_ARC9, 2052 [GAUDI2_QUEUE_ID_NIC_9_3] = CPU_ID_NIC_QMAN_ARC9, 2053 [GAUDI2_QUEUE_ID_NIC_10_0] = CPU_ID_NIC_QMAN_ARC10, 2054 [GAUDI2_QUEUE_ID_NIC_10_1] = CPU_ID_NIC_QMAN_ARC10, 2055 [GAUDI2_QUEUE_ID_NIC_10_2] = CPU_ID_NIC_QMAN_ARC10, 2056 [GAUDI2_QUEUE_ID_NIC_10_3] = CPU_ID_NIC_QMAN_ARC10, 2057 [GAUDI2_QUEUE_ID_NIC_11_0] = CPU_ID_NIC_QMAN_ARC11, 2058 [GAUDI2_QUEUE_ID_NIC_11_1] = CPU_ID_NIC_QMAN_ARC11, 2059 [GAUDI2_QUEUE_ID_NIC_11_2] = CPU_ID_NIC_QMAN_ARC11, 2060 [GAUDI2_QUEUE_ID_NIC_11_3] = CPU_ID_NIC_QMAN_ARC11, 2061 [GAUDI2_QUEUE_ID_NIC_12_0] = CPU_ID_NIC_QMAN_ARC12, 2062 [GAUDI2_QUEUE_ID_NIC_12_1] = CPU_ID_NIC_QMAN_ARC12, 2063 [GAUDI2_QUEUE_ID_NIC_12_2] = CPU_ID_NIC_QMAN_ARC12, 2064 [GAUDI2_QUEUE_ID_NIC_12_3] = CPU_ID_NIC_QMAN_ARC12, 2065 [GAUDI2_QUEUE_ID_NIC_13_0] = CPU_ID_NIC_QMAN_ARC13, 2066 [GAUDI2_QUEUE_ID_NIC_13_1] = CPU_ID_NIC_QMAN_ARC13, 2067 [GAUDI2_QUEUE_ID_NIC_13_2] = CPU_ID_NIC_QMAN_ARC13, 2068 [GAUDI2_QUEUE_ID_NIC_13_3] = CPU_ID_NIC_QMAN_ARC13, 2069 [GAUDI2_QUEUE_ID_NIC_14_0] = CPU_ID_NIC_QMAN_ARC14, 2070 [GAUDI2_QUEUE_ID_NIC_14_1] = CPU_ID_NIC_QMAN_ARC14, 2071 [GAUDI2_QUEUE_ID_NIC_14_2] = CPU_ID_NIC_QMAN_ARC14, 2072 [GAUDI2_QUEUE_ID_NIC_14_3] = CPU_ID_NIC_QMAN_ARC14, 2073 [GAUDI2_QUEUE_ID_NIC_15_0] = CPU_ID_NIC_QMAN_ARC15, 2074 [GAUDI2_QUEUE_ID_NIC_15_1] = CPU_ID_NIC_QMAN_ARC15, 2075 [GAUDI2_QUEUE_ID_NIC_15_2] = CPU_ID_NIC_QMAN_ARC15, 2076 [GAUDI2_QUEUE_ID_NIC_15_3] = CPU_ID_NIC_QMAN_ARC15, 2077 [GAUDI2_QUEUE_ID_NIC_16_0] = CPU_ID_NIC_QMAN_ARC16, 2078 [GAUDI2_QUEUE_ID_NIC_16_1] = CPU_ID_NIC_QMAN_ARC16, 2079 [GAUDI2_QUEUE_ID_NIC_16_2] = CPU_ID_NIC_QMAN_ARC16, 2080 [GAUDI2_QUEUE_ID_NIC_16_3] = CPU_ID_NIC_QMAN_ARC16, 2081 [GAUDI2_QUEUE_ID_NIC_17_0] = CPU_ID_NIC_QMAN_ARC17, 2082 [GAUDI2_QUEUE_ID_NIC_17_1] = CPU_ID_NIC_QMAN_ARC17, 2083 [GAUDI2_QUEUE_ID_NIC_17_2] = CPU_ID_NIC_QMAN_ARC17, 2084 [GAUDI2_QUEUE_ID_NIC_17_3] = CPU_ID_NIC_QMAN_ARC17, 2085 [GAUDI2_QUEUE_ID_NIC_18_0] = CPU_ID_NIC_QMAN_ARC18, 2086 [GAUDI2_QUEUE_ID_NIC_18_1] = CPU_ID_NIC_QMAN_ARC18, 2087 [GAUDI2_QUEUE_ID_NIC_18_2] = CPU_ID_NIC_QMAN_ARC18, 2088 [GAUDI2_QUEUE_ID_NIC_18_3] = CPU_ID_NIC_QMAN_ARC18, 2089 [GAUDI2_QUEUE_ID_NIC_19_0] = CPU_ID_NIC_QMAN_ARC19, 2090 [GAUDI2_QUEUE_ID_NIC_19_1] = CPU_ID_NIC_QMAN_ARC19, 2091 [GAUDI2_QUEUE_ID_NIC_19_2] = CPU_ID_NIC_QMAN_ARC19, 2092 [GAUDI2_QUEUE_ID_NIC_19_3] = CPU_ID_NIC_QMAN_ARC19, 2093 [GAUDI2_QUEUE_ID_NIC_20_0] = CPU_ID_NIC_QMAN_ARC20, 2094 [GAUDI2_QUEUE_ID_NIC_20_1] = CPU_ID_NIC_QMAN_ARC20, 2095 [GAUDI2_QUEUE_ID_NIC_20_2] = CPU_ID_NIC_QMAN_ARC20, 2096 [GAUDI2_QUEUE_ID_NIC_20_3] = CPU_ID_NIC_QMAN_ARC20, 2097 [GAUDI2_QUEUE_ID_NIC_21_0] = CPU_ID_NIC_QMAN_ARC21, 2098 [GAUDI2_QUEUE_ID_NIC_21_1] = CPU_ID_NIC_QMAN_ARC21, 2099 [GAUDI2_QUEUE_ID_NIC_21_2] = CPU_ID_NIC_QMAN_ARC21, 2100 [GAUDI2_QUEUE_ID_NIC_21_3] = CPU_ID_NIC_QMAN_ARC21, 2101 [GAUDI2_QUEUE_ID_NIC_22_0] = CPU_ID_NIC_QMAN_ARC22, 2102 [GAUDI2_QUEUE_ID_NIC_22_1] = CPU_ID_NIC_QMAN_ARC22, 2103 [GAUDI2_QUEUE_ID_NIC_22_2] = CPU_ID_NIC_QMAN_ARC22, 2104 [GAUDI2_QUEUE_ID_NIC_22_3] = CPU_ID_NIC_QMAN_ARC22, 2105 [GAUDI2_QUEUE_ID_NIC_23_0] = CPU_ID_NIC_QMAN_ARC23, 2106 [GAUDI2_QUEUE_ID_NIC_23_1] = CPU_ID_NIC_QMAN_ARC23, 2107 [GAUDI2_QUEUE_ID_NIC_23_2] = CPU_ID_NIC_QMAN_ARC23, 2108 [GAUDI2_QUEUE_ID_NIC_23_3] = CPU_ID_NIC_QMAN_ARC23, 2109 [GAUDI2_QUEUE_ID_ROT_0_0] = CPU_ID_ROT_QMAN_ARC0, 2110 [GAUDI2_QUEUE_ID_ROT_0_1] = CPU_ID_ROT_QMAN_ARC0, 2111 [GAUDI2_QUEUE_ID_ROT_0_2] = CPU_ID_ROT_QMAN_ARC0, 2112 [GAUDI2_QUEUE_ID_ROT_0_3] = CPU_ID_ROT_QMAN_ARC0, 2113 [GAUDI2_QUEUE_ID_ROT_1_0] = CPU_ID_ROT_QMAN_ARC1, 2114 [GAUDI2_QUEUE_ID_ROT_1_1] = CPU_ID_ROT_QMAN_ARC1, 2115 [GAUDI2_QUEUE_ID_ROT_1_2] = CPU_ID_ROT_QMAN_ARC1, 2116 [GAUDI2_QUEUE_ID_ROT_1_3] = CPU_ID_ROT_QMAN_ARC1 2117}; 2118 2119const u32 gaudi2_dma_core_blocks_bases[DMA_CORE_ID_SIZE] = { 2120 [DMA_CORE_ID_PDMA0] = mmPDMA0_CORE_BASE, 2121 [DMA_CORE_ID_PDMA1] = mmPDMA1_CORE_BASE, 2122 [DMA_CORE_ID_EDMA0] = mmDCORE0_EDMA0_CORE_BASE, 2123 [DMA_CORE_ID_EDMA1] = mmDCORE0_EDMA1_CORE_BASE, 2124 [DMA_CORE_ID_EDMA2] = mmDCORE1_EDMA0_CORE_BASE, 2125 [DMA_CORE_ID_EDMA3] = mmDCORE1_EDMA1_CORE_BASE, 2126 [DMA_CORE_ID_EDMA4] = mmDCORE2_EDMA0_CORE_BASE, 2127 [DMA_CORE_ID_EDMA5] = mmDCORE2_EDMA1_CORE_BASE, 2128 [DMA_CORE_ID_EDMA6] = mmDCORE3_EDMA0_CORE_BASE, 2129 [DMA_CORE_ID_EDMA7] = mmDCORE3_EDMA1_CORE_BASE, 2130 [DMA_CORE_ID_KDMA] = mmARC_FARM_KDMA_BASE 2131}; 2132 2133const u32 gaudi2_mme_acc_blocks_bases[MME_ID_SIZE] = { 2134 [MME_ID_DCORE0] = mmDCORE0_MME_ACC_BASE, 2135 [MME_ID_DCORE1] = mmDCORE1_MME_ACC_BASE, 2136 [MME_ID_DCORE2] = mmDCORE2_MME_ACC_BASE, 2137 [MME_ID_DCORE3] = mmDCORE3_MME_ACC_BASE 2138}; 2139 2140static const u32 gaudi2_tpc_cfg_blocks_bases[TPC_ID_SIZE] = { 2141 [TPC_ID_DCORE0_TPC0] = mmDCORE0_TPC0_CFG_BASE, 2142 [TPC_ID_DCORE0_TPC1] = mmDCORE0_TPC1_CFG_BASE, 2143 [TPC_ID_DCORE0_TPC2] = mmDCORE0_TPC2_CFG_BASE, 2144 [TPC_ID_DCORE0_TPC3] = mmDCORE0_TPC3_CFG_BASE, 2145 [TPC_ID_DCORE0_TPC4] = mmDCORE0_TPC4_CFG_BASE, 2146 [TPC_ID_DCORE0_TPC5] = mmDCORE0_TPC5_CFG_BASE, 2147 [TPC_ID_DCORE1_TPC0] = mmDCORE1_TPC0_CFG_BASE, 2148 [TPC_ID_DCORE1_TPC1] = mmDCORE1_TPC1_CFG_BASE, 2149 [TPC_ID_DCORE1_TPC2] = mmDCORE1_TPC2_CFG_BASE, 2150 [TPC_ID_DCORE1_TPC3] = mmDCORE1_TPC3_CFG_BASE, 2151 [TPC_ID_DCORE1_TPC4] = mmDCORE1_TPC4_CFG_BASE, 2152 [TPC_ID_DCORE1_TPC5] = mmDCORE1_TPC5_CFG_BASE, 2153 [TPC_ID_DCORE2_TPC0] = mmDCORE2_TPC0_CFG_BASE, 2154 [TPC_ID_DCORE2_TPC1] = mmDCORE2_TPC1_CFG_BASE, 2155 [TPC_ID_DCORE2_TPC2] = mmDCORE2_TPC2_CFG_BASE, 2156 [TPC_ID_DCORE2_TPC3] = mmDCORE2_TPC3_CFG_BASE, 2157 [TPC_ID_DCORE2_TPC4] = mmDCORE2_TPC4_CFG_BASE, 2158 [TPC_ID_DCORE2_TPC5] = mmDCORE2_TPC5_CFG_BASE, 2159 [TPC_ID_DCORE3_TPC0] = mmDCORE3_TPC0_CFG_BASE, 2160 [TPC_ID_DCORE3_TPC1] = mmDCORE3_TPC1_CFG_BASE, 2161 [TPC_ID_DCORE3_TPC2] = mmDCORE3_TPC2_CFG_BASE, 2162 [TPC_ID_DCORE3_TPC3] = mmDCORE3_TPC3_CFG_BASE, 2163 [TPC_ID_DCORE3_TPC4] = mmDCORE3_TPC4_CFG_BASE, 2164 [TPC_ID_DCORE3_TPC5] = mmDCORE3_TPC5_CFG_BASE, 2165 [TPC_ID_DCORE0_TPC6] = mmDCORE0_TPC6_CFG_BASE, 2166}; 2167 2168static const u32 gaudi2_tpc_eml_cfg_blocks_bases[TPC_ID_SIZE] = { 2169 [TPC_ID_DCORE0_TPC0] = mmDCORE0_TPC0_EML_CFG_BASE, 2170 [TPC_ID_DCORE0_TPC1] = mmDCORE0_TPC1_EML_CFG_BASE, 2171 [TPC_ID_DCORE0_TPC2] = mmDCORE0_TPC2_EML_CFG_BASE, 2172 [TPC_ID_DCORE0_TPC3] = mmDCORE0_TPC3_EML_CFG_BASE, 2173 [TPC_ID_DCORE0_TPC4] = mmDCORE0_TPC4_EML_CFG_BASE, 2174 [TPC_ID_DCORE0_TPC5] = mmDCORE0_TPC5_EML_CFG_BASE, 2175 [TPC_ID_DCORE1_TPC0] = mmDCORE1_TPC0_EML_CFG_BASE, 2176 [TPC_ID_DCORE1_TPC1] = mmDCORE1_TPC1_EML_CFG_BASE, 2177 [TPC_ID_DCORE1_TPC2] = mmDCORE1_TPC2_EML_CFG_BASE, 2178 [TPC_ID_DCORE1_TPC3] = mmDCORE1_TPC3_EML_CFG_BASE, 2179 [TPC_ID_DCORE1_TPC4] = mmDCORE1_TPC4_EML_CFG_BASE, 2180 [TPC_ID_DCORE1_TPC5] = mmDCORE1_TPC5_EML_CFG_BASE, 2181 [TPC_ID_DCORE2_TPC0] = mmDCORE2_TPC0_EML_CFG_BASE, 2182 [TPC_ID_DCORE2_TPC1] = mmDCORE2_TPC1_EML_CFG_BASE, 2183 [TPC_ID_DCORE2_TPC2] = mmDCORE2_TPC2_EML_CFG_BASE, 2184 [TPC_ID_DCORE2_TPC3] = mmDCORE2_TPC3_EML_CFG_BASE, 2185 [TPC_ID_DCORE2_TPC4] = mmDCORE2_TPC4_EML_CFG_BASE, 2186 [TPC_ID_DCORE2_TPC5] = mmDCORE2_TPC5_EML_CFG_BASE, 2187 [TPC_ID_DCORE3_TPC0] = mmDCORE3_TPC0_EML_CFG_BASE, 2188 [TPC_ID_DCORE3_TPC1] = mmDCORE3_TPC1_EML_CFG_BASE, 2189 [TPC_ID_DCORE3_TPC2] = mmDCORE3_TPC2_EML_CFG_BASE, 2190 [TPC_ID_DCORE3_TPC3] = mmDCORE3_TPC3_EML_CFG_BASE, 2191 [TPC_ID_DCORE3_TPC4] = mmDCORE3_TPC4_EML_CFG_BASE, 2192 [TPC_ID_DCORE3_TPC5] = mmDCORE3_TPC5_EML_CFG_BASE, 2193 [TPC_ID_DCORE0_TPC6] = mmDCORE0_TPC6_EML_CFG_BASE, 2194}; 2195 2196const u32 gaudi2_rot_blocks_bases[ROTATOR_ID_SIZE] = { 2197 [ROTATOR_ID_0] = mmROT0_BASE, 2198 [ROTATOR_ID_1] = mmROT1_BASE 2199}; 2200 2201static const u32 gaudi2_tpc_id_to_queue_id[TPC_ID_SIZE] = { 2202 [TPC_ID_DCORE0_TPC0] = GAUDI2_QUEUE_ID_DCORE0_TPC_0_0, 2203 [TPC_ID_DCORE0_TPC1] = GAUDI2_QUEUE_ID_DCORE0_TPC_1_0, 2204 [TPC_ID_DCORE0_TPC2] = GAUDI2_QUEUE_ID_DCORE0_TPC_2_0, 2205 [TPC_ID_DCORE0_TPC3] = GAUDI2_QUEUE_ID_DCORE0_TPC_3_0, 2206 [TPC_ID_DCORE0_TPC4] = GAUDI2_QUEUE_ID_DCORE0_TPC_4_0, 2207 [TPC_ID_DCORE0_TPC5] = GAUDI2_QUEUE_ID_DCORE0_TPC_5_0, 2208 [TPC_ID_DCORE1_TPC0] = GAUDI2_QUEUE_ID_DCORE1_TPC_0_0, 2209 [TPC_ID_DCORE1_TPC1] = GAUDI2_QUEUE_ID_DCORE1_TPC_1_0, 2210 [TPC_ID_DCORE1_TPC2] = GAUDI2_QUEUE_ID_DCORE1_TPC_2_0, 2211 [TPC_ID_DCORE1_TPC3] = GAUDI2_QUEUE_ID_DCORE1_TPC_3_0, 2212 [TPC_ID_DCORE1_TPC4] = GAUDI2_QUEUE_ID_DCORE1_TPC_4_0, 2213 [TPC_ID_DCORE1_TPC5] = GAUDI2_QUEUE_ID_DCORE1_TPC_5_0, 2214 [TPC_ID_DCORE2_TPC0] = GAUDI2_QUEUE_ID_DCORE2_TPC_0_0, 2215 [TPC_ID_DCORE2_TPC1] = GAUDI2_QUEUE_ID_DCORE2_TPC_1_0, 2216 [TPC_ID_DCORE2_TPC2] = GAUDI2_QUEUE_ID_DCORE2_TPC_2_0, 2217 [TPC_ID_DCORE2_TPC3] = GAUDI2_QUEUE_ID_DCORE2_TPC_3_0, 2218 [TPC_ID_DCORE2_TPC4] = GAUDI2_QUEUE_ID_DCORE2_TPC_4_0, 2219 [TPC_ID_DCORE2_TPC5] = GAUDI2_QUEUE_ID_DCORE2_TPC_5_0, 2220 [TPC_ID_DCORE3_TPC0] = GAUDI2_QUEUE_ID_DCORE3_TPC_0_0, 2221 [TPC_ID_DCORE3_TPC1] = GAUDI2_QUEUE_ID_DCORE3_TPC_1_0, 2222 [TPC_ID_DCORE3_TPC2] = GAUDI2_QUEUE_ID_DCORE3_TPC_2_0, 2223 [TPC_ID_DCORE3_TPC3] = GAUDI2_QUEUE_ID_DCORE3_TPC_3_0, 2224 [TPC_ID_DCORE3_TPC4] = GAUDI2_QUEUE_ID_DCORE3_TPC_4_0, 2225 [TPC_ID_DCORE3_TPC5] = GAUDI2_QUEUE_ID_DCORE3_TPC_5_0, 2226 [TPC_ID_DCORE0_TPC6] = GAUDI2_QUEUE_ID_DCORE0_TPC_6_0, 2227}; 2228 2229static const u32 gaudi2_rot_id_to_queue_id[ROTATOR_ID_SIZE] = { 2230 [ROTATOR_ID_0] = GAUDI2_QUEUE_ID_ROT_0_0, 2231 [ROTATOR_ID_1] = GAUDI2_QUEUE_ID_ROT_1_0, 2232}; 2233 2234static const u32 gaudi2_tpc_engine_id_to_tpc_id[] = { 2235 [GAUDI2_DCORE0_ENGINE_ID_TPC_0] = TPC_ID_DCORE0_TPC0, 2236 [GAUDI2_DCORE0_ENGINE_ID_TPC_1] = TPC_ID_DCORE0_TPC1, 2237 [GAUDI2_DCORE0_ENGINE_ID_TPC_2] = TPC_ID_DCORE0_TPC2, 2238 [GAUDI2_DCORE0_ENGINE_ID_TPC_3] = TPC_ID_DCORE0_TPC3, 2239 [GAUDI2_DCORE0_ENGINE_ID_TPC_4] = TPC_ID_DCORE0_TPC4, 2240 [GAUDI2_DCORE0_ENGINE_ID_TPC_5] = TPC_ID_DCORE0_TPC5, 2241 [GAUDI2_DCORE1_ENGINE_ID_TPC_0] = TPC_ID_DCORE1_TPC0, 2242 [GAUDI2_DCORE1_ENGINE_ID_TPC_1] = TPC_ID_DCORE1_TPC1, 2243 [GAUDI2_DCORE1_ENGINE_ID_TPC_2] = TPC_ID_DCORE1_TPC2, 2244 [GAUDI2_DCORE1_ENGINE_ID_TPC_3] = TPC_ID_DCORE1_TPC3, 2245 [GAUDI2_DCORE1_ENGINE_ID_TPC_4] = TPC_ID_DCORE1_TPC4, 2246 [GAUDI2_DCORE1_ENGINE_ID_TPC_5] = TPC_ID_DCORE1_TPC5, 2247 [GAUDI2_DCORE2_ENGINE_ID_TPC_0] = TPC_ID_DCORE2_TPC0, 2248 [GAUDI2_DCORE2_ENGINE_ID_TPC_1] = TPC_ID_DCORE2_TPC1, 2249 [GAUDI2_DCORE2_ENGINE_ID_TPC_2] = TPC_ID_DCORE2_TPC2, 2250 [GAUDI2_DCORE2_ENGINE_ID_TPC_3] = TPC_ID_DCORE2_TPC3, 2251 [GAUDI2_DCORE2_ENGINE_ID_TPC_4] = TPC_ID_DCORE2_TPC4, 2252 [GAUDI2_DCORE2_ENGINE_ID_TPC_5] = TPC_ID_DCORE2_TPC5, 2253 [GAUDI2_DCORE3_ENGINE_ID_TPC_0] = TPC_ID_DCORE3_TPC0, 2254 [GAUDI2_DCORE3_ENGINE_ID_TPC_1] = TPC_ID_DCORE3_TPC1, 2255 [GAUDI2_DCORE3_ENGINE_ID_TPC_2] = TPC_ID_DCORE3_TPC2, 2256 [GAUDI2_DCORE3_ENGINE_ID_TPC_3] = TPC_ID_DCORE3_TPC3, 2257 [GAUDI2_DCORE3_ENGINE_ID_TPC_4] = TPC_ID_DCORE3_TPC4, 2258 [GAUDI2_DCORE3_ENGINE_ID_TPC_5] = TPC_ID_DCORE3_TPC5, 2259 /* the PCI TPC is placed last (mapped liked HW) */ 2260 [GAUDI2_DCORE0_ENGINE_ID_TPC_6] = TPC_ID_DCORE0_TPC6, 2261}; 2262 2263static const u32 gaudi2_mme_engine_id_to_mme_id[] = { 2264 [GAUDI2_DCORE0_ENGINE_ID_MME] = MME_ID_DCORE0, 2265 [GAUDI2_DCORE1_ENGINE_ID_MME] = MME_ID_DCORE1, 2266 [GAUDI2_DCORE2_ENGINE_ID_MME] = MME_ID_DCORE2, 2267 [GAUDI2_DCORE3_ENGINE_ID_MME] = MME_ID_DCORE3, 2268}; 2269 2270static const u32 gaudi2_edma_engine_id_to_edma_id[] = { 2271 [GAUDI2_ENGINE_ID_PDMA_0] = DMA_CORE_ID_PDMA0, 2272 [GAUDI2_ENGINE_ID_PDMA_1] = DMA_CORE_ID_PDMA1, 2273 [GAUDI2_DCORE0_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA0, 2274 [GAUDI2_DCORE0_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA1, 2275 [GAUDI2_DCORE1_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA2, 2276 [GAUDI2_DCORE1_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA3, 2277 [GAUDI2_DCORE2_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA4, 2278 [GAUDI2_DCORE2_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA5, 2279 [GAUDI2_DCORE3_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA6, 2280 [GAUDI2_DCORE3_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA7, 2281 [GAUDI2_ENGINE_ID_KDMA] = DMA_CORE_ID_KDMA, 2282}; 2283 2284const u32 edma_stream_base[NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES] = { 2285 GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0, 2286 GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0, 2287 GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0, 2288 GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0, 2289 GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0, 2290 GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0, 2291 GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0, 2292 GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0, 2293}; 2294 2295static const char gaudi2_vdec_irq_name[GAUDI2_VDEC_MSIX_ENTRIES][GAUDI2_MAX_STRING_LEN] = { 2296 "gaudi2 vdec 0_0", "gaudi2 vdec 0_0 abnormal", 2297 "gaudi2 vdec 0_1", "gaudi2 vdec 0_1 abnormal", 2298 "gaudi2 vdec 1_0", "gaudi2 vdec 1_0 abnormal", 2299 "gaudi2 vdec 1_1", "gaudi2 vdec 1_1 abnormal", 2300 "gaudi2 vdec 2_0", "gaudi2 vdec 2_0 abnormal", 2301 "gaudi2 vdec 2_1", "gaudi2 vdec 2_1 abnormal", 2302 "gaudi2 vdec 3_0", "gaudi2 vdec 3_0 abnormal", 2303 "gaudi2 vdec 3_1", "gaudi2 vdec 3_1 abnormal", 2304 "gaudi2 vdec s_0", "gaudi2 vdec s_0 abnormal", 2305 "gaudi2 vdec s_1", "gaudi2 vdec s_1 abnormal" 2306}; 2307 2308enum rtr_id { 2309 DCORE0_RTR0, 2310 DCORE0_RTR1, 2311 DCORE0_RTR2, 2312 DCORE0_RTR3, 2313 DCORE0_RTR4, 2314 DCORE0_RTR5, 2315 DCORE0_RTR6, 2316 DCORE0_RTR7, 2317 DCORE1_RTR0, 2318 DCORE1_RTR1, 2319 DCORE1_RTR2, 2320 DCORE1_RTR3, 2321 DCORE1_RTR4, 2322 DCORE1_RTR5, 2323 DCORE1_RTR6, 2324 DCORE1_RTR7, 2325 DCORE2_RTR0, 2326 DCORE2_RTR1, 2327 DCORE2_RTR2, 2328 DCORE2_RTR3, 2329 DCORE2_RTR4, 2330 DCORE2_RTR5, 2331 DCORE2_RTR6, 2332 DCORE2_RTR7, 2333 DCORE3_RTR0, 2334 DCORE3_RTR1, 2335 DCORE3_RTR2, 2336 DCORE3_RTR3, 2337 DCORE3_RTR4, 2338 DCORE3_RTR5, 2339 DCORE3_RTR6, 2340 DCORE3_RTR7, 2341}; 2342 2343static const u32 gaudi2_tpc_initiator_hbw_rtr_id[NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1] = { 2344 DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR2, DCORE0_RTR2, DCORE0_RTR3, DCORE0_RTR3, 2345 DCORE1_RTR6, DCORE1_RTR6, DCORE1_RTR5, DCORE1_RTR5, DCORE1_RTR4, DCORE1_RTR4, 2346 DCORE2_RTR3, DCORE2_RTR3, DCORE2_RTR2, DCORE2_RTR2, DCORE2_RTR1, DCORE2_RTR1, 2347 DCORE3_RTR4, DCORE3_RTR4, DCORE3_RTR5, DCORE3_RTR5, DCORE3_RTR6, DCORE3_RTR6, 2348 DCORE0_RTR0 2349}; 2350 2351static const u32 gaudi2_tpc_initiator_lbw_rtr_id[NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1] = { 2352 DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR2, DCORE0_RTR2, 2353 DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR6, DCORE1_RTR6, DCORE1_RTR5, DCORE1_RTR5, 2354 DCORE2_RTR2, DCORE2_RTR2, DCORE2_RTR1, DCORE2_RTR1, DCORE2_RTR0, DCORE2_RTR0, 2355 DCORE3_RTR5, DCORE3_RTR5, DCORE3_RTR6, DCORE3_RTR6, DCORE3_RTR7, DCORE3_RTR7, 2356 DCORE0_RTR0 2357}; 2358 2359static const u32 gaudi2_dec_initiator_hbw_rtr_id[NUMBER_OF_DEC] = { 2360 DCORE0_RTR0, DCORE0_RTR0, DCORE1_RTR7, DCORE1_RTR7, DCORE2_RTR0, DCORE2_RTR0, 2361 DCORE3_RTR7, DCORE3_RTR7, DCORE0_RTR0, DCORE0_RTR0 2362}; 2363 2364static const u32 gaudi2_dec_initiator_lbw_rtr_id[NUMBER_OF_DEC] = { 2365 DCORE0_RTR1, DCORE0_RTR1, DCORE1_RTR6, DCORE1_RTR6, DCORE2_RTR1, DCORE2_RTR1, 2366 DCORE3_RTR6, DCORE3_RTR6, DCORE0_RTR0, DCORE0_RTR0 2367}; 2368 2369static const u32 gaudi2_nic_initiator_hbw_rtr_id[NIC_NUMBER_OF_MACROS] = { 2370 DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE2_RTR0, 2371 DCORE2_RTR0, DCORE2_RTR0, DCORE2_RTR0, DCORE3_RTR7, DCORE3_RTR7, DCORE3_RTR7 2372}; 2373 2374static const u32 gaudi2_nic_initiator_lbw_rtr_id[NIC_NUMBER_OF_MACROS] = { 2375 DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE2_RTR0, 2376 DCORE2_RTR0, DCORE2_RTR0, DCORE2_RTR0, DCORE3_RTR7, DCORE3_RTR7, DCORE3_RTR7 2377}; 2378 2379static const u32 gaudi2_edma_initiator_hbw_sft[NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES] = { 2380 mmSFT0_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE, 2381 mmSFT0_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE, 2382 mmSFT1_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE, 2383 mmSFT1_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE, 2384 mmSFT2_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE, 2385 mmSFT2_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE, 2386 mmSFT3_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE, 2387 mmSFT3_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE 2388}; 2389 2390static const u32 gaudi2_pdma_initiator_hbw_rtr_id[NUM_OF_PDMA] = { 2391 DCORE0_RTR0, DCORE0_RTR0 2392}; 2393 2394static const u32 gaudi2_pdma_initiator_lbw_rtr_id[NUM_OF_PDMA] = { 2395 DCORE0_RTR2, DCORE0_RTR2 2396}; 2397 2398static const u32 gaudi2_rot_initiator_hbw_rtr_id[NUM_OF_ROT] = { 2399 DCORE2_RTR0, DCORE3_RTR7 2400}; 2401 2402static const u32 gaudi2_rot_initiator_lbw_rtr_id[NUM_OF_ROT] = { 2403 DCORE2_RTR2, DCORE3_RTR5 2404}; 2405 2406struct mme_initiators_rtr_id { 2407 u32 wap0; 2408 u32 wap1; 2409 u32 write; 2410 u32 read; 2411 u32 sbte0; 2412 u32 sbte1; 2413 u32 sbte2; 2414 u32 sbte3; 2415 u32 sbte4; 2416}; 2417 2418enum mme_initiators { 2419 MME_WAP0 = 0, 2420 MME_WAP1, 2421 MME_WRITE, 2422 MME_READ, 2423 MME_SBTE0, 2424 MME_SBTE1, 2425 MME_SBTE2, 2426 MME_SBTE3, 2427 MME_SBTE4, 2428 MME_INITIATORS_MAX 2429}; 2430 2431static const struct mme_initiators_rtr_id 2432gaudi2_mme_initiator_rtr_id[NUM_OF_MME_PER_DCORE * NUM_OF_DCORES] = { 2433 { .wap0 = 5, .wap1 = 7, .write = 6, .read = 7, 2434 .sbte0 = 7, .sbte1 = 4, .sbte2 = 4, .sbte3 = 5, .sbte4 = 6}, 2435 { .wap0 = 10, .wap1 = 8, .write = 9, .read = 8, 2436 .sbte0 = 11, .sbte1 = 11, .sbte2 = 10, .sbte3 = 9, .sbte4 = 8}, 2437 { .wap0 = 21, .wap1 = 23, .write = 22, .read = 23, 2438 .sbte0 = 20, .sbte1 = 20, .sbte2 = 21, .sbte3 = 22, .sbte4 = 23}, 2439 { .wap0 = 30, .wap1 = 28, .write = 29, .read = 30, 2440 .sbte0 = 31, .sbte1 = 31, .sbte2 = 30, .sbte3 = 29, .sbte4 = 28}, 2441}; 2442 2443enum razwi_event_sources { 2444 RAZWI_TPC, 2445 RAZWI_MME, 2446 RAZWI_EDMA, 2447 RAZWI_PDMA, 2448 RAZWI_NIC, 2449 RAZWI_DEC, 2450 RAZWI_ROT, 2451 RAZWI_ARC_FARM 2452}; 2453 2454struct hbm_mc_error_causes { 2455 u32 mask; 2456 char cause[50]; 2457}; 2458 2459static struct hl_special_block_info gaudi2_special_blocks[] = GAUDI2_SPECIAL_BLOCKS; 2460 2461/* Special blocks iterator is currently used to configure security protection bits, 2462 * and read global errors. Most HW blocks are addressable and those who aren't (N/A)- 2463 * must be skipped. Following configurations are commonly used for both PB config 2464 * and global error reading, since currently they both share the same settings. 2465 * Once it changes, we must remember to use separate configurations for either one. 2466 */ 2467static int gaudi2_iterator_skip_block_types[] = { 2468 GAUDI2_BLOCK_TYPE_PLL, 2469 GAUDI2_BLOCK_TYPE_EU_BIST, 2470 GAUDI2_BLOCK_TYPE_HBM, 2471 GAUDI2_BLOCK_TYPE_XFT 2472}; 2473 2474static struct range gaudi2_iterator_skip_block_ranges[] = { 2475 /* Skip all PSOC blocks except for PSOC_GLOBAL_CONF */ 2476 {mmPSOC_I2C_M0_BASE, mmPSOC_EFUSE_BASE}, 2477 {mmPSOC_BTL_BASE, mmPSOC_MSTR_IF_RR_SHRD_HBW_BASE}, 2478 /* Skip all CPU blocks except for CPU_IF */ 2479 {mmCPU_CA53_CFG_BASE, mmCPU_CA53_CFG_BASE}, 2480 {mmCPU_TIMESTAMP_BASE, mmCPU_MSTR_IF_RR_SHRD_HBW_BASE} 2481}; 2482 2483static struct hbm_mc_error_causes hbm_mc_spi[GAUDI2_NUM_OF_HBM_MC_SPI_CAUSE] = { 2484 {HBM_MC_SPI_TEMP_PIN_CHG_MASK, "temperature pins changed"}, 2485 {HBM_MC_SPI_THR_ENG_MASK, "temperature-based throttling engaged"}, 2486 {HBM_MC_SPI_THR_DIS_ENG_MASK, "temperature-based throttling disengaged"}, 2487 {HBM_MC_SPI_IEEE1500_COMP_MASK, "IEEE1500 op comp"}, 2488 {HBM_MC_SPI_IEEE1500_PAUSED_MASK, "IEEE1500 op paused"}, 2489}; 2490 2491static const char * const hbm_mc_sei_cause[GAUDI2_NUM_OF_HBM_SEI_CAUSE] = { 2492 [HBM_SEI_CMD_PARITY_EVEN] = "SEI C/A parity even", 2493 [HBM_SEI_CMD_PARITY_ODD] = "SEI C/A parity odd", 2494 [HBM_SEI_READ_ERR] = "SEI read data error", 2495 [HBM_SEI_WRITE_DATA_PARITY_ERR] = "SEI write data parity error", 2496 [HBM_SEI_CATTRIP] = "SEI CATTRIP asserted", 2497 [HBM_SEI_MEM_BIST_FAIL] = "SEI memory BIST fail", 2498 [HBM_SEI_DFI] = "SEI DFI error", 2499 [HBM_SEI_INV_TEMP_READ_OUT] = "SEI invalid temp read", 2500 [HBM_SEI_BIST_FAIL] = "SEI BIST fail" 2501}; 2502 2503struct mmu_spi_sei_cause { 2504 char cause[50]; 2505 int clear_bit; 2506}; 2507 2508static const struct mmu_spi_sei_cause gaudi2_mmu_spi_sei[GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE] = { 2509 {"page fault", 1}, /* INTERRUPT_CLR[1] */ 2510 {"page access", 1}, /* INTERRUPT_CLR[1] */ 2511 {"bypass ddr", 2}, /* INTERRUPT_CLR[2] */ 2512 {"multi hit", 2}, /* INTERRUPT_CLR[2] */ 2513 {"mmu rei0", -1}, /* no clear register bit */ 2514 {"mmu rei1", -1}, /* no clear register bit */ 2515 {"stlb rei0", -1}, /* no clear register bit */ 2516 {"stlb rei1", -1}, /* no clear register bit */ 2517 {"rr privileged write hit", 2}, /* INTERRUPT_CLR[2] */ 2518 {"rr privileged read hit", 2}, /* INTERRUPT_CLR[2] */ 2519 {"rr secure write hit", 2}, /* INTERRUPT_CLR[2] */ 2520 {"rr secure read hit", 2}, /* INTERRUPT_CLR[2] */ 2521 {"bist_fail no use", 2}, /* INTERRUPT_CLR[2] */ 2522 {"bist_fail no use", 2}, /* INTERRUPT_CLR[2] */ 2523 {"bist_fail no use", 2}, /* INTERRUPT_CLR[2] */ 2524 {"bist_fail no use", 2}, /* INTERRUPT_CLR[2] */ 2525 {"slave error", 16}, /* INTERRUPT_CLR[16] */ 2526 {"dec error", 17}, /* INTERRUPT_CLR[17] */ 2527 {"burst fifo full", 2} /* INTERRUPT_CLR[2] */ 2528}; 2529 2530struct gaudi2_cache_invld_params { 2531 u64 start_va; 2532 u64 end_va; 2533 u32 inv_start_val; 2534 u32 flags; 2535 bool range_invalidation; 2536}; 2537 2538struct gaudi2_tpc_idle_data { 2539 struct engines_data *e; 2540 unsigned long *mask; 2541 bool *is_idle; 2542 const char *tpc_fmt; 2543}; 2544 2545struct gaudi2_tpc_mmu_data { 2546 u32 rw_asid; 2547}; 2548 2549static s64 gaudi2_state_dump_specs_props[SP_MAX] = {0}; 2550 2551static int gaudi2_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, u64 val); 2552static bool gaudi2_is_queue_enabled(struct hl_device *hdev, u32 hw_queue_id); 2553static bool gaudi2_is_arc_enabled(struct hl_device *hdev, u64 arc_id); 2554static void gaudi2_clr_arc_id_cap(struct hl_device *hdev, u64 arc_id); 2555static void gaudi2_set_arc_id_cap(struct hl_device *hdev, u64 arc_id); 2556static void gaudi2_memset_device_lbw(struct hl_device *hdev, u32 addr, u32 size, u32 val); 2557static int gaudi2_send_job_to_kdma(struct hl_device *hdev, u64 src_addr, u64 dst_addr, u32 size, 2558 bool is_memset); 2559static bool gaudi2_get_tpc_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 2560 struct engines_data *e); 2561static bool gaudi2_get_mme_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 2562 struct engines_data *e); 2563static bool gaudi2_get_edma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 2564 struct engines_data *e); 2565static u64 gaudi2_mmu_scramble_addr(struct hl_device *hdev, u64 raw_addr); 2566static u64 gaudi2_mmu_descramble_addr(struct hl_device *hdev, u64 scrambled_addr); 2567 2568static void gaudi2_init_scrambler_hbm(struct hl_device *hdev) 2569{ 2570 2571} 2572 2573static u32 gaudi2_get_signal_cb_size(struct hl_device *hdev) 2574{ 2575 return sizeof(struct packet_msg_short); 2576} 2577 2578static u32 gaudi2_get_wait_cb_size(struct hl_device *hdev) 2579{ 2580 return sizeof(struct packet_msg_short) * 4 + sizeof(struct packet_fence); 2581} 2582 2583void gaudi2_iterate_tpcs(struct hl_device *hdev, struct iterate_module_ctx *ctx) 2584{ 2585 struct asic_fixed_properties *prop = &hdev->asic_prop; 2586 int dcore, inst, tpc_seq; 2587 u32 offset; 2588 2589 /* init the return code */ 2590 ctx->rc = 0; 2591 2592 for (dcore = 0; dcore < NUM_OF_DCORES; dcore++) { 2593 for (inst = 0; inst < NUM_OF_TPC_PER_DCORE; inst++) { 2594 tpc_seq = dcore * NUM_OF_TPC_PER_DCORE + inst; 2595 2596 if (!(prop->tpc_enabled_mask & BIT(tpc_seq))) 2597 continue; 2598 2599 offset = (DCORE_OFFSET * dcore) + (DCORE_TPC_OFFSET * inst); 2600 2601 ctx->fn(hdev, dcore, inst, offset, ctx); 2602 if (ctx->rc) { 2603 dev_err(hdev->dev, "TPC iterator failed for DCORE%d TPC%d\n", 2604 dcore, inst); 2605 return; 2606 } 2607 } 2608 } 2609 2610 if (!(prop->tpc_enabled_mask & BIT(TPC_ID_DCORE0_TPC6))) 2611 return; 2612 2613 /* special check for PCI TPC (DCORE0_TPC6) */ 2614 offset = DCORE_TPC_OFFSET * (NUM_DCORE0_TPC - 1); 2615 ctx->fn(hdev, 0, NUM_DCORE0_TPC - 1, offset, ctx); 2616 if (ctx->rc) 2617 dev_err(hdev->dev, "TPC iterator failed for DCORE0 TPC6\n"); 2618} 2619 2620static bool gaudi2_host_phys_addr_valid(u64 addr) 2621{ 2622 if ((addr < HOST_PHYS_BASE_0 + HOST_PHYS_SIZE_0) || (addr >= HOST_PHYS_BASE_1)) 2623 return true; 2624 2625 return false; 2626} 2627 2628static int set_number_of_functional_hbms(struct hl_device *hdev) 2629{ 2630 struct asic_fixed_properties *prop = &hdev->asic_prop; 2631 u8 faulty_hbms = hweight64(hdev->dram_binning); 2632 2633 /* check if all HBMs should be used */ 2634 if (!faulty_hbms) { 2635 dev_dbg(hdev->dev, "All HBM are in use (no binning)\n"); 2636 prop->num_functional_hbms = GAUDI2_HBM_NUM; 2637 return 0; 2638 } 2639 2640 /* 2641 * check for error condition in which number of binning 2642 * candidates is higher than the maximum supported by the 2643 * driver (in which case binning mask shall be ignored and driver will 2644 * set the default) 2645 */ 2646 if (faulty_hbms > MAX_FAULTY_HBMS) { 2647 dev_err(hdev->dev, 2648 "HBM binning supports max of %d faulty HBMs, supplied mask 0x%llx.\n", 2649 MAX_FAULTY_HBMS, hdev->dram_binning); 2650 return -EINVAL; 2651 } 2652 2653 /* 2654 * by default, number of functional HBMs in Gaudi2 is always 2655 * GAUDI2_HBM_NUM - 1. 2656 */ 2657 prop->num_functional_hbms = GAUDI2_HBM_NUM - faulty_hbms; 2658 return 0; 2659} 2660 2661static bool gaudi2_is_edma_queue_id(u32 queue_id) 2662{ 2663 2664 switch (queue_id) { 2665 case GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3: 2666 case GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3: 2667 case GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3: 2668 case GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3: 2669 return true; 2670 default: 2671 return false; 2672 } 2673} 2674 2675static int gaudi2_set_dram_properties(struct hl_device *hdev) 2676{ 2677 struct asic_fixed_properties *prop = &hdev->asic_prop; 2678 u64 hbm_drv_base_offset = 0, edma_pq_base_addr; 2679 u32 basic_hbm_page_size, edma_idx = 0; 2680 int rc, i; 2681 2682 rc = set_number_of_functional_hbms(hdev); 2683 if (rc) 2684 return -EINVAL; 2685 2686 /* 2687 * Due to HW bug in which TLB size is x16 smaller than expected we use a workaround 2688 * in which we are using x16 bigger page size to be able to populate the entire 2689 * HBM mappings in the TLB 2690 */ 2691 basic_hbm_page_size = prop->num_functional_hbms * SZ_8M; 2692 prop->dram_page_size = GAUDI2_COMPENSATE_TLB_PAGE_SIZE_FACTOR * basic_hbm_page_size; 2693 prop->device_mem_alloc_default_page_size = prop->dram_page_size; 2694 prop->dram_size = prop->num_functional_hbms * SZ_16G; 2695 prop->dram_base_address = DRAM_PHYS_BASE; 2696 prop->dram_end_address = prop->dram_base_address + prop->dram_size; 2697 prop->dram_supports_virtual_memory = true; 2698 2699 prop->dram_user_base_address = DRAM_PHYS_BASE + prop->dram_page_size; 2700 prop->dram_hints_align_mask = ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK; 2701 prop->hints_dram_reserved_va_range.start_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HBM_START; 2702 prop->hints_dram_reserved_va_range.end_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HBM_END; 2703 2704 /* since DRAM page size differs from DMMU page size we need to allocate 2705 * DRAM memory in units of dram_page size and mapping this memory in 2706 * units of DMMU page size. we overcome this size mismatch using a 2707 * scrambling routine which takes a DRAM page and converts it to a DMMU 2708 * page. 2709 * We therefore: 2710 * 1. partition the virtual address space to DRAM-page (whole) pages. 2711 * (suppose we get n such pages) 2712 * 2. limit the amount of virtual address space we got from 1 above to 2713 * a multiple of 64M as we don't want the scrambled address to cross 2714 * the DRAM virtual address space. 2715 * ( m = (n * DRAM_page_size) / DMMU_page_size). 2716 * 3. determine the and address accordingly 2717 * end_addr = start_addr + m * 48M 2718 * 2719 * the DRAM address MSBs (63:48) are not part of the roundup calculation 2720 */ 2721 prop->dmmu.start_addr = prop->dram_base_address + 2722 (prop->dram_page_size * 2723 DIV_ROUND_UP_SECTOR_T(prop->dram_size, prop->dram_page_size)); 2724 prop->dmmu.end_addr = prop->dmmu.start_addr + prop->dram_page_size * 2725 div_u64((VA_HBM_SPACE_END - prop->dmmu.start_addr), prop->dmmu.page_size); 2726 /* 2727 * Driver can't share an (48MB) HBM page with the F/W in order to prevent FW to block 2728 * the driver part by range register, so it must start at the next (48MB) page 2729 */ 2730 hbm_drv_base_offset = roundup(CPU_FW_IMAGE_SIZE, prop->num_functional_hbms * SZ_8M); 2731 2732 /* 2733 * The NIC driver section size and the HMMU page tables section in the HBM needs 2734 * to be the remaining size in the first dram page after taking into 2735 * account the F/W image size 2736 */ 2737 2738 /* Reserve region in HBM for HMMU page tables */ 2739 prop->mmu_pgt_addr = DRAM_PHYS_BASE + hbm_drv_base_offset + 2740 ((prop->dram_page_size - hbm_drv_base_offset) - 2741 (HMMU_PAGE_TABLES_SIZE + EDMA_PQS_SIZE + EDMA_SCRATCHPAD_SIZE)); 2742 2743 /* Set EDMA PQs HBM addresses */ 2744 edma_pq_base_addr = prop->mmu_pgt_addr + HMMU_PAGE_TABLES_SIZE; 2745 2746 for (i = 0 ; i < GAUDI2_QUEUE_ID_CPU_PQ ; i++) { 2747 if (gaudi2_is_edma_queue_id(i)) { 2748 prop->hw_queues_props[i].q_dram_bd_address = edma_pq_base_addr + 2749 (edma_idx * HL_QUEUE_SIZE_IN_BYTES); 2750 edma_idx++; 2751 } 2752 } 2753 2754 return 0; 2755} 2756 2757static int gaudi2_set_fixed_properties(struct hl_device *hdev) 2758{ 2759 struct asic_fixed_properties *prop = &hdev->asic_prop; 2760 struct hw_queue_properties *q_props; 2761 u32 num_sync_stream_queues = 0; 2762 int i, rc; 2763 2764 prop->max_queues = GAUDI2_QUEUE_ID_SIZE; 2765 prop->hw_queues_props = kcalloc(prop->max_queues, sizeof(struct hw_queue_properties), 2766 GFP_KERNEL); 2767 2768 if (!prop->hw_queues_props) 2769 return -ENOMEM; 2770 2771 q_props = prop->hw_queues_props; 2772 2773 for (i = 0 ; i < GAUDI2_QUEUE_ID_CPU_PQ ; i++) { 2774 q_props[i].type = QUEUE_TYPE_HW; 2775 q_props[i].driver_only = 0; 2776 2777 if (i >= GAUDI2_QUEUE_ID_NIC_0_0 && i <= GAUDI2_QUEUE_ID_NIC_23_3) { 2778 q_props[i].supports_sync_stream = 0; 2779 } else { 2780 q_props[i].supports_sync_stream = 1; 2781 num_sync_stream_queues++; 2782 } 2783 2784 q_props[i].cb_alloc_flags = CB_ALLOC_USER; 2785 2786 if (gaudi2_is_edma_queue_id(i)) 2787 q_props[i].dram_bd = 1; 2788 } 2789 2790 q_props[GAUDI2_QUEUE_ID_CPU_PQ].type = QUEUE_TYPE_CPU; 2791 q_props[GAUDI2_QUEUE_ID_CPU_PQ].driver_only = 1; 2792 q_props[GAUDI2_QUEUE_ID_CPU_PQ].cb_alloc_flags = CB_ALLOC_KERNEL; 2793 2794 prop->cache_line_size = DEVICE_CACHE_LINE_SIZE; 2795 prop->cfg_base_address = CFG_BASE; 2796 prop->device_dma_offset_for_host_access = HOST_PHYS_BASE_0; 2797 prop->host_base_address = HOST_PHYS_BASE_0; 2798 prop->host_end_address = prop->host_base_address + HOST_PHYS_SIZE_0; 2799 prop->max_pending_cs = GAUDI2_MAX_PENDING_CS; 2800 prop->completion_queues_count = GAUDI2_RESERVED_CQ_NUMBER; 2801 prop->user_dec_intr_count = NUMBER_OF_DEC; 2802 prop->user_interrupt_count = GAUDI2_IRQ_NUM_USER_LAST - GAUDI2_IRQ_NUM_USER_FIRST + 1; 2803 prop->completion_mode = HL_COMPLETION_MODE_CS; 2804 prop->sync_stream_first_sob = GAUDI2_RESERVED_SOB_NUMBER; 2805 prop->sync_stream_first_mon = GAUDI2_RESERVED_MON_NUMBER; 2806 2807 prop->sram_base_address = SRAM_BASE_ADDR; 2808 prop->sram_size = SRAM_SIZE; 2809 prop->sram_end_address = prop->sram_base_address + prop->sram_size; 2810 prop->sram_user_base_address = prop->sram_base_address + SRAM_USER_BASE_OFFSET; 2811 2812 prop->hints_range_reservation = true; 2813 2814 prop->rotator_enabled_mask = BIT(NUM_OF_ROT) - 1; 2815 2816 prop->max_asid = 2; 2817 2818 prop->dmmu.pgt_size = HMMU_PAGE_TABLES_SIZE; 2819 prop->mmu_pte_size = HL_PTE_SIZE; 2820 2821 prop->dmmu.hop_shifts[MMU_HOP0] = DHOP0_SHIFT; 2822 prop->dmmu.hop_shifts[MMU_HOP1] = DHOP1_SHIFT; 2823 prop->dmmu.hop_shifts[MMU_HOP2] = DHOP2_SHIFT; 2824 prop->dmmu.hop_shifts[MMU_HOP3] = DHOP3_SHIFT; 2825 prop->dmmu.hop_masks[MMU_HOP0] = DHOP0_MASK; 2826 prop->dmmu.hop_masks[MMU_HOP1] = DHOP1_MASK; 2827 prop->dmmu.hop_masks[MMU_HOP2] = DHOP2_MASK; 2828 prop->dmmu.hop_masks[MMU_HOP3] = DHOP3_MASK; 2829 prop->dmmu.page_size = PAGE_SIZE_1GB; 2830 prop->dmmu.num_hops = MMU_ARCH_4_HOPS; 2831 prop->dmmu.last_mask = LAST_MASK; 2832 prop->dmmu.host_resident = 0; 2833 prop->dmmu.hop_table_size = HOP_TABLE_SIZE_512_PTE; 2834 prop->dmmu.hop0_tables_total_size = HOP_TABLE_SIZE_512_PTE * prop->max_asid; 2835 2836 /* As we need to set the pgt address in dram for HMMU init so we cannot 2837 * wait to the fw cpucp info to set the dram props as mmu init comes before 2838 * hw init 2839 */ 2840 rc = hdev->asic_funcs->set_dram_properties(hdev); 2841 if (rc) 2842 goto free_qprops; 2843 2844 prop->mmu_pgt_size = PMMU_PAGE_TABLES_SIZE; 2845 2846 prop->pmmu.pgt_size = prop->mmu_pgt_size; 2847 hdev->pmmu_huge_range = true; 2848 prop->pmmu.host_resident = 1; 2849 prop->pmmu.num_hops = MMU_ARCH_6_HOPS; 2850 prop->pmmu.last_mask = LAST_MASK; 2851 prop->pmmu.hop_table_size = HOP_TABLE_SIZE_512_PTE; 2852 prop->pmmu.hop0_tables_total_size = HOP_TABLE_SIZE_512_PTE * prop->max_asid; 2853 2854 prop->hints_host_reserved_va_range.start_addr = RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START; 2855 prop->hints_host_reserved_va_range.end_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HOST_END; 2856 prop->hints_host_hpage_reserved_va_range.start_addr = 2857 RESERVED_VA_RANGE_FOR_ARC_ON_HOST_HPAGE_START; 2858 prop->hints_host_hpage_reserved_va_range.end_addr = 2859 RESERVED_VA_RANGE_FOR_ARC_ON_HOST_HPAGE_END; 2860 2861 if (PAGE_SIZE == SZ_64K) { 2862 prop->pmmu.hop_shifts[MMU_HOP0] = HOP0_SHIFT_64K; 2863 prop->pmmu.hop_shifts[MMU_HOP1] = HOP1_SHIFT_64K; 2864 prop->pmmu.hop_shifts[MMU_HOP2] = HOP2_SHIFT_64K; 2865 prop->pmmu.hop_shifts[MMU_HOP3] = HOP3_SHIFT_64K; 2866 prop->pmmu.hop_shifts[MMU_HOP4] = HOP4_SHIFT_64K; 2867 prop->pmmu.hop_shifts[MMU_HOP5] = HOP5_SHIFT_64K; 2868 prop->pmmu.hop_masks[MMU_HOP0] = HOP0_MASK_64K; 2869 prop->pmmu.hop_masks[MMU_HOP1] = HOP1_MASK_64K; 2870 prop->pmmu.hop_masks[MMU_HOP2] = HOP2_MASK_64K; 2871 prop->pmmu.hop_masks[MMU_HOP3] = HOP3_MASK_64K; 2872 prop->pmmu.hop_masks[MMU_HOP4] = HOP4_MASK_64K; 2873 prop->pmmu.hop_masks[MMU_HOP5] = HOP5_MASK_64K; 2874 prop->pmmu.start_addr = VA_HOST_SPACE_PAGE_START; 2875 prop->pmmu.end_addr = VA_HOST_SPACE_PAGE_END; 2876 prop->pmmu.page_size = PAGE_SIZE_64KB; 2877 2878 /* shifts and masks are the same in PMMU and HPMMU */ 2879 memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu)); 2880 prop->pmmu_huge.page_size = PAGE_SIZE_16MB; 2881 prop->pmmu_huge.start_addr = VA_HOST_SPACE_HPAGE_START; 2882 prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END; 2883 } else { 2884 prop->pmmu.hop_shifts[MMU_HOP0] = HOP0_SHIFT_4K; 2885 prop->pmmu.hop_shifts[MMU_HOP1] = HOP1_SHIFT_4K; 2886 prop->pmmu.hop_shifts[MMU_HOP2] = HOP2_SHIFT_4K; 2887 prop->pmmu.hop_shifts[MMU_HOP3] = HOP3_SHIFT_4K; 2888 prop->pmmu.hop_shifts[MMU_HOP4] = HOP4_SHIFT_4K; 2889 prop->pmmu.hop_shifts[MMU_HOP5] = HOP5_SHIFT_4K; 2890 prop->pmmu.hop_masks[MMU_HOP0] = HOP0_MASK_4K; 2891 prop->pmmu.hop_masks[MMU_HOP1] = HOP1_MASK_4K; 2892 prop->pmmu.hop_masks[MMU_HOP2] = HOP2_MASK_4K; 2893 prop->pmmu.hop_masks[MMU_HOP3] = HOP3_MASK_4K; 2894 prop->pmmu.hop_masks[MMU_HOP4] = HOP4_MASK_4K; 2895 prop->pmmu.hop_masks[MMU_HOP5] = HOP5_MASK_4K; 2896 prop->pmmu.start_addr = VA_HOST_SPACE_PAGE_START; 2897 prop->pmmu.end_addr = VA_HOST_SPACE_PAGE_END; 2898 prop->pmmu.page_size = PAGE_SIZE_4KB; 2899 2900 /* shifts and masks are the same in PMMU and HPMMU */ 2901 memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu)); 2902 prop->pmmu_huge.page_size = PAGE_SIZE_2MB; 2903 prop->pmmu_huge.start_addr = VA_HOST_SPACE_HPAGE_START; 2904 prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END; 2905 } 2906 2907 prop->max_num_of_engines = GAUDI2_ENGINE_ID_SIZE; 2908 prop->num_engine_cores = CPU_ID_MAX; 2909 prop->cfg_size = CFG_SIZE; 2910 prop->num_of_events = GAUDI2_EVENT_SIZE; 2911 2912 prop->supports_engine_modes = true; 2913 2914 prop->dc_power_default = DC_POWER_DEFAULT; 2915 2916 prop->cb_pool_cb_cnt = GAUDI2_CB_POOL_CB_CNT; 2917 prop->cb_pool_cb_size = GAUDI2_CB_POOL_CB_SIZE; 2918 prop->pcie_dbi_base_address = CFG_BASE + mmPCIE_DBI_BASE; 2919 prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI; 2920 2921 strscpy_pad(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, CARD_NAME_MAX_LEN); 2922 2923 prop->mme_master_slave_mode = 1; 2924 2925 prop->first_available_user_sob[0] = GAUDI2_RESERVED_SOB_NUMBER + 2926 (num_sync_stream_queues * HL_RSVD_SOBS); 2927 2928 prop->first_available_user_mon[0] = GAUDI2_RESERVED_MON_NUMBER + 2929 (num_sync_stream_queues * HL_RSVD_MONS); 2930 2931 prop->first_available_user_interrupt = GAUDI2_IRQ_NUM_USER_FIRST; 2932 prop->tpc_interrupt_id = GAUDI2_IRQ_NUM_TPC_ASSERT; 2933 prop->eq_interrupt_id = GAUDI2_IRQ_NUM_EVENT_QUEUE; 2934 2935 prop->first_available_cq[0] = GAUDI2_RESERVED_CQ_NUMBER; 2936 2937 prop->fw_cpu_boot_dev_sts0_valid = false; 2938 prop->fw_cpu_boot_dev_sts1_valid = false; 2939 prop->hard_reset_done_by_fw = false; 2940 prop->gic_interrupts_enable = true; 2941 2942 prop->server_type = HL_SERVER_TYPE_UNKNOWN; 2943 2944 prop->max_dec = NUMBER_OF_DEC; 2945 2946 prop->clk_pll_index = HL_GAUDI2_MME_PLL; 2947 2948 prop->dma_mask = 64; 2949 2950 prop->hbw_flush_reg = mmPCIE_WRAP_SPECIAL_GLBL_SPARE_0; 2951 2952 prop->supports_advanced_cpucp_rc = true; 2953 2954 return 0; 2955 2956free_qprops: 2957 kfree(prop->hw_queues_props); 2958 return rc; 2959} 2960 2961static int gaudi2_pci_bars_map(struct hl_device *hdev) 2962{ 2963 static const char * const name[] = {"CFG_SRAM", "MSIX", "DRAM"}; 2964 bool is_wc[3] = {false, false, true}; 2965 int rc; 2966 2967 rc = hl_pci_bars_map(hdev, name, is_wc); 2968 if (rc) 2969 return rc; 2970 2971 hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] + (CFG_BASE - STM_FLASH_BASE_ADDR); 2972 2973 return 0; 2974} 2975 2976static u64 gaudi2_set_hbm_bar_base(struct hl_device *hdev, u64 addr) 2977{ 2978 struct gaudi2_device *gaudi2 = hdev->asic_specific; 2979 struct hl_inbound_pci_region pci_region; 2980 u64 old_addr = addr; 2981 int rc; 2982 2983 if ((gaudi2) && (gaudi2->dram_bar_cur_addr == addr)) 2984 return old_addr; 2985 2986 if (hdev->asic_prop.iatu_done_by_fw) 2987 return U64_MAX; 2988 2989 /* Inbound Region 2 - Bar 4 - Point to DRAM */ 2990 pci_region.mode = PCI_BAR_MATCH_MODE; 2991 pci_region.bar = DRAM_BAR_ID; 2992 pci_region.addr = addr; 2993 rc = hl_pci_set_inbound_region(hdev, 2, &pci_region); 2994 if (rc) 2995 return U64_MAX; 2996 2997 if (gaudi2) { 2998 old_addr = gaudi2->dram_bar_cur_addr; 2999 gaudi2->dram_bar_cur_addr = addr; 3000 } 3001 3002 return old_addr; 3003} 3004 3005static int gaudi2_init_iatu(struct hl_device *hdev) 3006{ 3007 struct hl_inbound_pci_region inbound_region; 3008 struct hl_outbound_pci_region outbound_region; 3009 u32 bar_addr_low, bar_addr_high; 3010 int rc; 3011 3012 if (hdev->asic_prop.iatu_done_by_fw) 3013 return 0; 3014 3015 /* Temporary inbound Region 0 - Bar 0 - Point to CFG 3016 * We must map this region in BAR match mode in order to 3017 * fetch BAR physical base address 3018 */ 3019 inbound_region.mode = PCI_BAR_MATCH_MODE; 3020 inbound_region.bar = SRAM_CFG_BAR_ID; 3021 /* Base address must be aligned to Bar size which is 256 MB */ 3022 inbound_region.addr = STM_FLASH_BASE_ADDR - STM_FLASH_ALIGNED_OFF; 3023 rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region); 3024 if (rc) 3025 return rc; 3026 3027 /* Fetch physical BAR address */ 3028 bar_addr_high = RREG32(mmPCIE_DBI_BAR1_REG + STM_FLASH_ALIGNED_OFF); 3029 bar_addr_low = RREG32(mmPCIE_DBI_BAR0_REG + STM_FLASH_ALIGNED_OFF) & ~0xF; 3030 3031 hdev->pcie_bar_phys[SRAM_CFG_BAR_ID] = (u64)bar_addr_high << 32 | bar_addr_low; 3032 3033 /* Inbound Region 0 - Bar 0 - Point to CFG */ 3034 inbound_region.mode = PCI_ADDRESS_MATCH_MODE; 3035 inbound_region.bar = SRAM_CFG_BAR_ID; 3036 inbound_region.offset_in_bar = 0; 3037 inbound_region.addr = STM_FLASH_BASE_ADDR; 3038 inbound_region.size = CFG_REGION_SIZE; 3039 rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region); 3040 if (rc) 3041 return rc; 3042 3043 /* Inbound Region 1 - Bar 0 - Point to BAR0_RESERVED + SRAM */ 3044 inbound_region.mode = PCI_ADDRESS_MATCH_MODE; 3045 inbound_region.bar = SRAM_CFG_BAR_ID; 3046 inbound_region.offset_in_bar = CFG_REGION_SIZE; 3047 inbound_region.addr = BAR0_RSRVD_BASE_ADDR; 3048 inbound_region.size = BAR0_RSRVD_SIZE + SRAM_SIZE; 3049 rc = hl_pci_set_inbound_region(hdev, 1, &inbound_region); 3050 if (rc) 3051 return rc; 3052 3053 /* Inbound Region 2 - Bar 4 - Point to DRAM */ 3054 inbound_region.mode = PCI_BAR_MATCH_MODE; 3055 inbound_region.bar = DRAM_BAR_ID; 3056 inbound_region.addr = DRAM_PHYS_BASE; 3057 rc = hl_pci_set_inbound_region(hdev, 2, &inbound_region); 3058 if (rc) 3059 return rc; 3060 3061 /* Outbound Region 0 - Point to Host */ 3062 outbound_region.addr = HOST_PHYS_BASE_0; 3063 outbound_region.size = HOST_PHYS_SIZE_0; 3064 rc = hl_pci_set_outbound_region(hdev, &outbound_region); 3065 3066 return rc; 3067} 3068 3069static enum hl_device_hw_state gaudi2_get_hw_state(struct hl_device *hdev) 3070{ 3071 return RREG32(mmHW_STATE); 3072} 3073 3074static int gaudi2_tpc_binning_init_prop(struct hl_device *hdev) 3075{ 3076 struct asic_fixed_properties *prop = &hdev->asic_prop; 3077 3078 /* 3079 * check for error condition in which number of binning candidates 3080 * is higher than the maximum supported by the driver 3081 */ 3082 if (hweight64(hdev->tpc_binning) > MAX_CLUSTER_BINNING_FAULTY_TPCS) { 3083 dev_err(hdev->dev, "TPC binning is supported for max of %d faulty TPCs, provided mask 0x%llx\n", 3084 MAX_CLUSTER_BINNING_FAULTY_TPCS, 3085 hdev->tpc_binning); 3086 return -EINVAL; 3087 } 3088 3089 prop->tpc_binning_mask = hdev->tpc_binning; 3090 prop->tpc_enabled_mask = GAUDI2_TPC_FULL_MASK; 3091 3092 return 0; 3093} 3094 3095static int gaudi2_set_tpc_binning_masks(struct hl_device *hdev) 3096{ 3097 struct asic_fixed_properties *prop = &hdev->asic_prop; 3098 struct hw_queue_properties *q_props = prop->hw_queues_props; 3099 u64 tpc_binning_mask; 3100 u8 subst_idx = 0; 3101 int i, rc; 3102 3103 rc = gaudi2_tpc_binning_init_prop(hdev); 3104 if (rc) 3105 return rc; 3106 3107 tpc_binning_mask = prop->tpc_binning_mask; 3108 3109 for (i = 0 ; i < MAX_FAULTY_TPCS ; i++) { 3110 u8 subst_seq, binned, qid_base; 3111 3112 if (tpc_binning_mask == 0) 3113 break; 3114 3115 if (subst_idx == 0) { 3116 subst_seq = TPC_ID_DCORE0_TPC6; 3117 qid_base = GAUDI2_QUEUE_ID_DCORE0_TPC_6_0; 3118 } else { 3119 subst_seq = TPC_ID_DCORE3_TPC5; 3120 qid_base = GAUDI2_QUEUE_ID_DCORE3_TPC_5_0; 3121 } 3122 3123 3124 /* clear bit from mask */ 3125 binned = __ffs(tpc_binning_mask); 3126 /* 3127 * Coverity complains about possible out-of-bound access in 3128 * clear_bit 3129 */ 3130 if (binned >= TPC_ID_SIZE) { 3131 dev_err(hdev->dev, 3132 "Invalid binned TPC (binning mask: %llx)\n", 3133 tpc_binning_mask); 3134 return -EINVAL; 3135 } 3136 clear_bit(binned, (unsigned long *)&tpc_binning_mask); 3137 3138 /* also clear replacing TPC bit from enabled mask */ 3139 clear_bit(subst_seq, (unsigned long *)&prop->tpc_enabled_mask); 3140 3141 /* bin substite TPC's Qs */ 3142 q_props[qid_base].binned = 1; 3143 q_props[qid_base + 1].binned = 1; 3144 q_props[qid_base + 2].binned = 1; 3145 q_props[qid_base + 3].binned = 1; 3146 3147 subst_idx++; 3148 } 3149 3150 return 0; 3151} 3152 3153static int gaudi2_set_dec_binning_masks(struct hl_device *hdev) 3154{ 3155 struct asic_fixed_properties *prop = &hdev->asic_prop; 3156 u8 num_faulty; 3157 3158 num_faulty = hweight32(hdev->decoder_binning); 3159 3160 /* 3161 * check for error condition in which number of binning candidates 3162 * is higher than the maximum supported by the driver 3163 */ 3164 if (num_faulty > MAX_FAULTY_DECODERS) { 3165 dev_err(hdev->dev, "decoder binning is supported for max of single faulty decoder, provided mask 0x%x\n", 3166 hdev->decoder_binning); 3167 return -EINVAL; 3168 } 3169 3170 prop->decoder_binning_mask = (hdev->decoder_binning & GAUDI2_DECODER_FULL_MASK); 3171 3172 if (prop->decoder_binning_mask) 3173 prop->decoder_enabled_mask = (GAUDI2_DECODER_FULL_MASK & ~BIT(DEC_ID_PCIE_VDEC1)); 3174 else 3175 prop->decoder_enabled_mask = GAUDI2_DECODER_FULL_MASK; 3176 3177 return 0; 3178} 3179 3180static void gaudi2_set_dram_binning_masks(struct hl_device *hdev) 3181{ 3182 struct asic_fixed_properties *prop = &hdev->asic_prop; 3183 3184 /* check if we should override default binning */ 3185 if (!hdev->dram_binning) { 3186 prop->dram_binning_mask = 0; 3187 prop->dram_enabled_mask = GAUDI2_DRAM_FULL_MASK; 3188 return; 3189 } 3190 3191 /* set DRAM binning constraints */ 3192 prop->faulty_dram_cluster_map |= hdev->dram_binning; 3193 prop->dram_binning_mask = hdev->dram_binning; 3194 prop->dram_enabled_mask = GAUDI2_DRAM_FULL_MASK & ~BIT(HBM_ID5); 3195} 3196 3197static int gaudi2_set_edma_binning_masks(struct hl_device *hdev) 3198{ 3199 struct asic_fixed_properties *prop = &hdev->asic_prop; 3200 struct hw_queue_properties *q_props; 3201 u8 seq, num_faulty; 3202 3203 num_faulty = hweight32(hdev->edma_binning); 3204 3205 /* 3206 * check for error condition in which number of binning candidates 3207 * is higher than the maximum supported by the driver 3208 */ 3209 if (num_faulty > MAX_FAULTY_EDMAS) { 3210 dev_err(hdev->dev, 3211 "EDMA binning is supported for max of single faulty EDMA, provided mask 0x%x\n", 3212 hdev->edma_binning); 3213 return -EINVAL; 3214 } 3215 3216 if (!hdev->edma_binning) { 3217 prop->edma_binning_mask = 0; 3218 prop->edma_enabled_mask = GAUDI2_EDMA_FULL_MASK; 3219 return 0; 3220 } 3221 3222 seq = __ffs((unsigned long)hdev->edma_binning); 3223 3224 /* set binning constraints */ 3225 prop->faulty_dram_cluster_map |= BIT(edma_to_hbm_cluster[seq]); 3226 prop->edma_binning_mask = hdev->edma_binning; 3227 prop->edma_enabled_mask = GAUDI2_EDMA_FULL_MASK & ~BIT(EDMA_ID_DCORE3_INSTANCE1); 3228 3229 /* bin substitute EDMA's queue */ 3230 q_props = prop->hw_queues_props; 3231 q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0].binned = 1; 3232 q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1].binned = 1; 3233 q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2].binned = 1; 3234 q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3].binned = 1; 3235 3236 return 0; 3237} 3238 3239static int gaudi2_set_xbar_edge_enable_mask(struct hl_device *hdev, u32 xbar_edge_iso_mask) 3240{ 3241 struct asic_fixed_properties *prop = &hdev->asic_prop; 3242 u8 num_faulty, seq; 3243 3244 /* check if we should override default binning */ 3245 if (!xbar_edge_iso_mask) { 3246 prop->xbar_edge_enabled_mask = GAUDI2_XBAR_EDGE_FULL_MASK; 3247 return 0; 3248 } 3249 3250 /* 3251 * note that it can be set to value other than 0 only after cpucp packet (i.e. 3252 * only the FW can set a redundancy value). for user it'll always be 0. 3253 */ 3254 num_faulty = hweight32(xbar_edge_iso_mask); 3255 3256 /* 3257 * check for error condition in which number of binning candidates 3258 * is higher than the maximum supported by the driver 3259 */ 3260 if (num_faulty > MAX_FAULTY_XBARS) { 3261 dev_err(hdev->dev, "we cannot have more than %d faulty XBAR EDGE\n", 3262 MAX_FAULTY_XBARS); 3263 return -EINVAL; 3264 } 3265 3266 seq = __ffs((unsigned long)xbar_edge_iso_mask); 3267 3268 /* set binning constraints */ 3269 prop->faulty_dram_cluster_map |= BIT(xbar_edge_to_hbm_cluster[seq]); 3270 prop->xbar_edge_enabled_mask = (~xbar_edge_iso_mask) & GAUDI2_XBAR_EDGE_FULL_MASK; 3271 3272 return 0; 3273} 3274 3275static int gaudi2_set_cluster_binning_masks_common(struct hl_device *hdev, u8 xbar_edge_iso_mask) 3276{ 3277 int rc; 3278 3279 /* 3280 * mark all clusters as good, each component will "fail" cluster 3281 * based on eFuse/user values. 3282 * If more than single cluster is faulty- the chip is unusable 3283 */ 3284 hdev->asic_prop.faulty_dram_cluster_map = 0; 3285 3286 gaudi2_set_dram_binning_masks(hdev); 3287 3288 rc = gaudi2_set_edma_binning_masks(hdev); 3289 if (rc) 3290 return rc; 3291 3292 rc = gaudi2_set_xbar_edge_enable_mask(hdev, xbar_edge_iso_mask); 3293 if (rc) 3294 return rc; 3295 3296 3297 /* always initially set to full mask */ 3298 hdev->asic_prop.hmmu_hif_enabled_mask = GAUDI2_HIF_HMMU_FULL_MASK; 3299 3300 return 0; 3301} 3302 3303static int gaudi2_set_cluster_binning_masks(struct hl_device *hdev) 3304{ 3305 struct asic_fixed_properties *prop = &hdev->asic_prop; 3306 int rc; 3307 3308 rc = gaudi2_set_cluster_binning_masks_common(hdev, prop->cpucp_info.xbar_binning_mask); 3309 if (rc) 3310 return rc; 3311 3312 /* if we have DRAM binning reported by FW we should perform cluster config */ 3313 if (prop->faulty_dram_cluster_map) { 3314 u8 cluster_seq = __ffs((unsigned long)prop->faulty_dram_cluster_map); 3315 3316 prop->hmmu_hif_enabled_mask = cluster_hmmu_hif_enabled_mask[cluster_seq]; 3317 } 3318 3319 return 0; 3320} 3321 3322static int gaudi2_set_binning_masks(struct hl_device *hdev) 3323{ 3324 int rc; 3325 3326 rc = gaudi2_set_cluster_binning_masks(hdev); 3327 if (rc) 3328 return rc; 3329 3330 rc = gaudi2_set_tpc_binning_masks(hdev); 3331 if (rc) 3332 return rc; 3333 3334 rc = gaudi2_set_dec_binning_masks(hdev); 3335 if (rc) 3336 return rc; 3337 3338 return 0; 3339} 3340 3341static int gaudi2_cpucp_info_get(struct hl_device *hdev) 3342{ 3343 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3344 struct asic_fixed_properties *prop = &hdev->asic_prop; 3345 long max_power; 3346 u64 dram_size; 3347 int rc; 3348 3349 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 3350 return 0; 3351 3352 /* No point of asking this information again when not doing hard reset, as the device 3353 * CPU hasn't been reset 3354 */ 3355 if (hdev->reset_info.in_compute_reset) 3356 return 0; 3357 3358 rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_DEV_STS1, mmCPU_BOOT_ERR0, 3359 mmCPU_BOOT_ERR1); 3360 if (rc) 3361 return rc; 3362 3363 dram_size = le64_to_cpu(prop->cpucp_info.dram_size); 3364 if (dram_size) { 3365 /* we can have wither 5 or 6 HBMs. other values are invalid */ 3366 3367 if ((dram_size != ((GAUDI2_HBM_NUM - 1) * SZ_16G)) && 3368 (dram_size != (GAUDI2_HBM_NUM * SZ_16G))) { 3369 dev_err(hdev->dev, 3370 "F/W reported invalid DRAM size %llu. Trying to use default size %llu\n", 3371 dram_size, prop->dram_size); 3372 dram_size = prop->dram_size; 3373 } 3374 3375 prop->dram_size = dram_size; 3376 prop->dram_end_address = prop->dram_base_address + dram_size; 3377 } 3378 3379 if (!strlen(prop->cpucp_info.card_name)) 3380 strscpy_pad(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, 3381 CARD_NAME_MAX_LEN); 3382 3383 /* Overwrite binning masks with the actual binning values from F/W */ 3384 hdev->dram_binning = prop->cpucp_info.dram_binning_mask; 3385 hdev->edma_binning = prop->cpucp_info.edma_binning_mask; 3386 hdev->tpc_binning = le64_to_cpu(prop->cpucp_info.tpc_binning_mask); 3387 hdev->decoder_binning = lower_32_bits(le64_to_cpu(prop->cpucp_info.decoder_binning_mask)); 3388 3389 dev_dbg(hdev->dev, "Read binning masks: tpc: 0x%llx, dram: 0x%llx, edma: 0x%x, dec: 0x%x\n", 3390 hdev->tpc_binning, hdev->dram_binning, hdev->edma_binning, 3391 hdev->decoder_binning); 3392 3393 /* 3394 * at this point the DRAM parameters need to be updated according to data obtained 3395 * from the FW 3396 */ 3397 rc = hdev->asic_funcs->set_dram_properties(hdev); 3398 if (rc) 3399 return rc; 3400 3401 rc = hdev->asic_funcs->set_binning_masks(hdev); 3402 if (rc) 3403 return rc; 3404 3405 max_power = hl_fw_get_max_power(hdev); 3406 if (max_power < 0) 3407 return max_power; 3408 3409 prop->max_power_default = (u64) max_power; 3410 3411 return 0; 3412} 3413 3414static int gaudi2_fetch_psoc_frequency(struct hl_device *hdev) 3415{ 3416 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3417 u16 pll_freq_arr[HL_PLL_NUM_OUTPUTS]; 3418 int rc; 3419 3420 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 3421 return 0; 3422 3423 rc = hl_fw_cpucp_pll_info_get(hdev, HL_GAUDI2_CPU_PLL, pll_freq_arr); 3424 if (rc) 3425 return rc; 3426 3427 hdev->asic_prop.psoc_timestamp_frequency = pll_freq_arr[3]; 3428 3429 return 0; 3430} 3431 3432static int gaudi2_mmu_clear_pgt_range(struct hl_device *hdev) 3433{ 3434 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3435 struct asic_fixed_properties *prop = &hdev->asic_prop; 3436 int rc; 3437 3438 if (!(gaudi2->hw_cap_initialized & HW_CAP_MMU_MASK)) 3439 return 0; 3440 3441 if (prop->dmmu.host_resident) 3442 return 0; 3443 3444 rc = gaudi2_memset_device_memory(hdev, prop->mmu_pgt_addr, prop->dmmu.pgt_size, 0); 3445 if (rc) 3446 dev_err(hdev->dev, "Failed to clear mmu pgt"); 3447 3448 return rc; 3449} 3450 3451static int gaudi2_early_init(struct hl_device *hdev) 3452{ 3453 struct asic_fixed_properties *prop = &hdev->asic_prop; 3454 struct pci_dev *pdev = hdev->pdev; 3455 resource_size_t pci_bar_size; 3456 int rc; 3457 3458 rc = gaudi2_set_fixed_properties(hdev); 3459 if (rc) 3460 return rc; 3461 3462 /* Check BAR sizes */ 3463 pci_bar_size = pci_resource_len(pdev, SRAM_CFG_BAR_ID); 3464 3465 if (pci_bar_size != CFG_BAR_SIZE) { 3466 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", 3467 SRAM_CFG_BAR_ID, &pci_bar_size, CFG_BAR_SIZE); 3468 rc = -ENODEV; 3469 goto free_queue_props; 3470 } 3471 3472 pci_bar_size = pci_resource_len(pdev, MSIX_BAR_ID); 3473 if (pci_bar_size != MSIX_BAR_SIZE) { 3474 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", 3475 MSIX_BAR_ID, &pci_bar_size, MSIX_BAR_SIZE); 3476 rc = -ENODEV; 3477 goto free_queue_props; 3478 } 3479 3480 prop->dram_pci_bar_size = pci_resource_len(pdev, DRAM_BAR_ID); 3481 hdev->dram_pci_bar_start = pci_resource_start(pdev, DRAM_BAR_ID); 3482 3483 /* 3484 * Only in pldm driver config iATU 3485 */ 3486 if (hdev->pldm) 3487 hdev->asic_prop.iatu_done_by_fw = false; 3488 else 3489 hdev->asic_prop.iatu_done_by_fw = true; 3490 3491 rc = hl_pci_init(hdev); 3492 if (rc) 3493 goto free_queue_props; 3494 3495 /* Before continuing in the initialization, we need to read the preboot 3496 * version to determine whether we run with a security-enabled firmware 3497 */ 3498 rc = hl_fw_read_preboot_status(hdev); 3499 if (rc) { 3500 if (hdev->reset_on_preboot_fail) 3501 hdev->asic_funcs->hw_fini(hdev, true, false); 3502 goto pci_fini; 3503 } 3504 3505 if (gaudi2_get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) { 3506 dev_dbg(hdev->dev, "H/W state is dirty, must reset before initializing\n"); 3507 rc = hdev->asic_funcs->hw_fini(hdev, true, false); 3508 if (rc) { 3509 dev_err(hdev->dev, "failed to reset HW in dirty state (%d)\n", rc); 3510 goto pci_fini; 3511 } 3512 3513 rc = hl_fw_read_preboot_status(hdev); 3514 if (rc) { 3515 if (hdev->reset_on_preboot_fail) 3516 hdev->asic_funcs->hw_fini(hdev, true, false); 3517 goto pci_fini; 3518 } 3519 } 3520 3521 return 0; 3522 3523pci_fini: 3524 hl_pci_fini(hdev); 3525free_queue_props: 3526 kfree(hdev->asic_prop.hw_queues_props); 3527 return rc; 3528} 3529 3530static int gaudi2_early_fini(struct hl_device *hdev) 3531{ 3532 kfree(hdev->asic_prop.hw_queues_props); 3533 hl_pci_fini(hdev); 3534 3535 return 0; 3536} 3537 3538static bool gaudi2_is_arc_nic_owned(u64 arc_id) 3539{ 3540 switch (arc_id) { 3541 case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23: 3542 return true; 3543 default: 3544 return false; 3545 } 3546} 3547 3548static bool gaudi2_is_arc_tpc_owned(u64 arc_id) 3549{ 3550 switch (arc_id) { 3551 case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24: 3552 return true; 3553 default: 3554 return false; 3555 } 3556} 3557 3558static void gaudi2_init_arcs(struct hl_device *hdev) 3559{ 3560 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 3561 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3562 u64 arc_id; 3563 u32 i; 3564 3565 for (i = CPU_ID_SCHED_ARC0 ; i <= CPU_ID_SCHED_ARC3 ; i++) { 3566 if (gaudi2_is_arc_enabled(hdev, i)) 3567 continue; 3568 3569 gaudi2_set_arc_id_cap(hdev, i); 3570 } 3571 3572 for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ ; i += 4) { 3573 if (!gaudi2_is_queue_enabled(hdev, i)) 3574 continue; 3575 3576 arc_id = gaudi2_queue_id_to_arc_id[i]; 3577 if (gaudi2_is_arc_enabled(hdev, arc_id)) 3578 continue; 3579 3580 if (gaudi2_is_arc_nic_owned(arc_id) && 3581 !(hdev->nic_ports_mask & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0))) 3582 continue; 3583 3584 if (gaudi2_is_arc_tpc_owned(arc_id) && !(gaudi2->tpc_hw_cap_initialized & 3585 BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0))) 3586 continue; 3587 3588 gaudi2_set_arc_id_cap(hdev, arc_id); 3589 } 3590 3591 /* Fetch ARC scratchpad address */ 3592 hdev->asic_prop.engine_core_interrupt_reg_addr = 3593 CFG_BASE + le32_to_cpu(dyn_regs->eng_arc_irq_ctrl); 3594} 3595 3596static int gaudi2_scrub_arc_dccm(struct hl_device *hdev, u32 cpu_id) 3597{ 3598 u32 reg_base, reg_val; 3599 int rc; 3600 3601 switch (cpu_id) { 3602 case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC3: 3603 /* Each ARC scheduler has 2 consecutive DCCM blocks */ 3604 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id], 3605 ARC_DCCM_BLOCK_SIZE * 2, true); 3606 if (rc) 3607 return rc; 3608 break; 3609 case CPU_ID_SCHED_ARC4: 3610 case CPU_ID_SCHED_ARC5: 3611 case CPU_ID_MME_QMAN_ARC0: 3612 case CPU_ID_MME_QMAN_ARC1: 3613 reg_base = gaudi2_arc_blocks_bases[cpu_id]; 3614 3615 /* Scrub lower DCCM block */ 3616 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id], 3617 ARC_DCCM_BLOCK_SIZE, true); 3618 if (rc) 3619 return rc; 3620 3621 /* Switch to upper DCCM block */ 3622 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_MME_ARC_UPPER_DCCM_EN_VAL_MASK, 1); 3623 WREG32(reg_base + ARC_DCCM_UPPER_EN_OFFSET, reg_val); 3624 3625 /* Scrub upper DCCM block */ 3626 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id], 3627 ARC_DCCM_BLOCK_SIZE, true); 3628 if (rc) 3629 return rc; 3630 3631 /* Switch to lower DCCM block */ 3632 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_MME_ARC_UPPER_DCCM_EN_VAL_MASK, 0); 3633 WREG32(reg_base + ARC_DCCM_UPPER_EN_OFFSET, reg_val); 3634 break; 3635 default: 3636 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id], 3637 ARC_DCCM_BLOCK_SIZE, true); 3638 if (rc) 3639 return rc; 3640 } 3641 3642 return 0; 3643} 3644 3645static int gaudi2_scrub_arcs_dccm(struct hl_device *hdev) 3646{ 3647 u16 arc_id; 3648 int rc; 3649 3650 for (arc_id = CPU_ID_SCHED_ARC0 ; arc_id < CPU_ID_MAX ; arc_id++) { 3651 if (!gaudi2_is_arc_enabled(hdev, arc_id)) 3652 continue; 3653 3654 rc = gaudi2_scrub_arc_dccm(hdev, arc_id); 3655 if (rc) 3656 return rc; 3657 } 3658 3659 return 0; 3660} 3661 3662static int gaudi2_late_init(struct hl_device *hdev) 3663{ 3664 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3665 int rc; 3666 3667 rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_ENABLE_PCI_ACCESS, 3668 gaudi2->virt_msix_db_dma_addr); 3669 if (rc) 3670 return rc; 3671 3672 rc = gaudi2_fetch_psoc_frequency(hdev); 3673 if (rc) { 3674 dev_err(hdev->dev, "Failed to fetch psoc frequency\n"); 3675 goto disable_pci_access; 3676 } 3677 3678 rc = gaudi2_mmu_clear_pgt_range(hdev); 3679 if (rc) { 3680 dev_err(hdev->dev, "Failed to clear MMU page tables range\n"); 3681 goto disable_pci_access; 3682 } 3683 3684 gaudi2_init_arcs(hdev); 3685 3686 rc = gaudi2_scrub_arcs_dccm(hdev); 3687 if (rc) { 3688 dev_err(hdev->dev, "Failed to scrub arcs DCCM\n"); 3689 goto disable_pci_access; 3690 } 3691 3692 gaudi2_init_security(hdev); 3693 3694 return 0; 3695 3696disable_pci_access: 3697 hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0); 3698 3699 return rc; 3700} 3701 3702static void gaudi2_late_fini(struct hl_device *hdev) 3703{ 3704 hl_hwmon_release_resources(hdev); 3705} 3706 3707static void gaudi2_user_mapped_dec_init(struct gaudi2_device *gaudi2, u32 start_idx) 3708{ 3709 struct user_mapped_block *blocks = gaudi2->mapped_blocks; 3710 3711 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE0_DEC0_CMD_BASE, HL_BLOCK_SIZE); 3712 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE0_DEC1_CMD_BASE, HL_BLOCK_SIZE); 3713 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE1_DEC0_CMD_BASE, HL_BLOCK_SIZE); 3714 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE1_DEC1_CMD_BASE, HL_BLOCK_SIZE); 3715 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE2_DEC0_CMD_BASE, HL_BLOCK_SIZE); 3716 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE2_DEC1_CMD_BASE, HL_BLOCK_SIZE); 3717 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE3_DEC0_CMD_BASE, HL_BLOCK_SIZE); 3718 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE3_DEC1_CMD_BASE, HL_BLOCK_SIZE); 3719 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmPCIE_DEC0_CMD_BASE, HL_BLOCK_SIZE); 3720 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx], mmPCIE_DEC1_CMD_BASE, HL_BLOCK_SIZE); 3721} 3722 3723static void gaudi2_user_mapped_blocks_init(struct hl_device *hdev) 3724{ 3725 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3726 struct user_mapped_block *blocks = gaudi2->mapped_blocks; 3727 u32 block_size, umr_start_idx, num_umr_blocks; 3728 int i; 3729 3730 for (i = 0 ; i < NUM_ARC_CPUS ; i++) { 3731 if (i >= CPU_ID_SCHED_ARC0 && i <= CPU_ID_SCHED_ARC3) 3732 block_size = ARC_DCCM_BLOCK_SIZE * 2; 3733 else 3734 block_size = ARC_DCCM_BLOCK_SIZE; 3735 3736 blocks[i].address = gaudi2_arc_dccm_bases[i]; 3737 blocks[i].size = block_size; 3738 } 3739 3740 blocks[NUM_ARC_CPUS].address = mmARC_FARM_ARC0_ACP_ENG_BASE; 3741 blocks[NUM_ARC_CPUS].size = HL_BLOCK_SIZE; 3742 3743 blocks[NUM_ARC_CPUS + 1].address = mmARC_FARM_ARC1_ACP_ENG_BASE; 3744 blocks[NUM_ARC_CPUS + 1].size = HL_BLOCK_SIZE; 3745 3746 blocks[NUM_ARC_CPUS + 2].address = mmARC_FARM_ARC2_ACP_ENG_BASE; 3747 blocks[NUM_ARC_CPUS + 2].size = HL_BLOCK_SIZE; 3748 3749 blocks[NUM_ARC_CPUS + 3].address = mmARC_FARM_ARC3_ACP_ENG_BASE; 3750 blocks[NUM_ARC_CPUS + 3].size = HL_BLOCK_SIZE; 3751 3752 blocks[NUM_ARC_CPUS + 4].address = mmDCORE0_MME_QM_ARC_ACP_ENG_BASE; 3753 blocks[NUM_ARC_CPUS + 4].size = HL_BLOCK_SIZE; 3754 3755 blocks[NUM_ARC_CPUS + 5].address = mmDCORE1_MME_QM_ARC_ACP_ENG_BASE; 3756 blocks[NUM_ARC_CPUS + 5].size = HL_BLOCK_SIZE; 3757 3758 blocks[NUM_ARC_CPUS + 6].address = mmDCORE2_MME_QM_ARC_ACP_ENG_BASE; 3759 blocks[NUM_ARC_CPUS + 6].size = HL_BLOCK_SIZE; 3760 3761 blocks[NUM_ARC_CPUS + 7].address = mmDCORE3_MME_QM_ARC_ACP_ENG_BASE; 3762 blocks[NUM_ARC_CPUS + 7].size = HL_BLOCK_SIZE; 3763 3764 umr_start_idx = NUM_ARC_CPUS + NUM_OF_USER_ACP_BLOCKS; 3765 num_umr_blocks = NIC_NUMBER_OF_ENGINES * NUM_OF_USER_NIC_UMR_BLOCKS; 3766 for (i = 0 ; i < num_umr_blocks ; i++) { 3767 u8 nic_id, umr_block_id; 3768 3769 nic_id = i / NUM_OF_USER_NIC_UMR_BLOCKS; 3770 umr_block_id = i % NUM_OF_USER_NIC_UMR_BLOCKS; 3771 3772 blocks[umr_start_idx + i].address = 3773 mmNIC0_UMR0_0_UNSECURE_DOORBELL0_BASE + 3774 (nic_id / NIC_NUMBER_OF_QM_PER_MACRO) * NIC_OFFSET + 3775 (nic_id % NIC_NUMBER_OF_QM_PER_MACRO) * NIC_QM_OFFSET + 3776 umr_block_id * NIC_UMR_OFFSET; 3777 blocks[umr_start_idx + i].size = HL_BLOCK_SIZE; 3778 } 3779 3780 /* Expose decoder HW configuration block to user */ 3781 gaudi2_user_mapped_dec_init(gaudi2, USR_MAPPED_BLK_DEC_START_IDX); 3782 3783 for (i = 1; i < NUM_OF_DCORES; ++i) { 3784 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1)].size = SM_OBJS_BLOCK_SIZE; 3785 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1) + 1].size = HL_BLOCK_SIZE; 3786 3787 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1)].address = 3788 mmDCORE0_SYNC_MNGR_OBJS_BASE + i * DCORE_OFFSET; 3789 3790 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1) + 1].address = 3791 mmDCORE0_SYNC_MNGR_GLBL_BASE + i * DCORE_OFFSET; 3792 } 3793} 3794 3795static int gaudi2_alloc_cpu_accessible_dma_mem(struct hl_device *hdev) 3796{ 3797 dma_addr_t dma_addr_arr[GAUDI2_ALLOC_CPU_MEM_RETRY_CNT] = {}, end_addr; 3798 void *virt_addr_arr[GAUDI2_ALLOC_CPU_MEM_RETRY_CNT] = {}; 3799 int i, j, rc = 0; 3800 3801 /* The device ARC works with 32-bits addresses, and because there is a single HW register 3802 * that holds the extension bits (49..28), these bits must be identical in all the allocated 3803 * range. 3804 */ 3805 3806 for (i = 0 ; i < GAUDI2_ALLOC_CPU_MEM_RETRY_CNT ; i++) { 3807 virt_addr_arr[i] = hl_asic_dma_alloc_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, 3808 &dma_addr_arr[i], GFP_KERNEL | __GFP_ZERO); 3809 if (!virt_addr_arr[i]) { 3810 rc = -ENOMEM; 3811 goto free_dma_mem_arr; 3812 } 3813 3814 end_addr = dma_addr_arr[i] + HL_CPU_ACCESSIBLE_MEM_SIZE - 1; 3815 if (GAUDI2_ARC_PCI_MSB_ADDR(dma_addr_arr[i]) == GAUDI2_ARC_PCI_MSB_ADDR(end_addr)) 3816 break; 3817 } 3818 3819 if (i == GAUDI2_ALLOC_CPU_MEM_RETRY_CNT) { 3820 dev_err(hdev->dev, 3821 "MSB of ARC accessible DMA memory are not identical in all range\n"); 3822 rc = -EFAULT; 3823 goto free_dma_mem_arr; 3824 } 3825 3826 hdev->cpu_accessible_dma_mem = virt_addr_arr[i]; 3827 hdev->cpu_accessible_dma_address = dma_addr_arr[i]; 3828 3829free_dma_mem_arr: 3830 for (j = 0 ; j < i ; j++) 3831 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, virt_addr_arr[j], 3832 dma_addr_arr[j]); 3833 3834 return rc; 3835} 3836 3837static void gaudi2_set_pci_memory_regions(struct hl_device *hdev) 3838{ 3839 struct asic_fixed_properties *prop = &hdev->asic_prop; 3840 struct pci_mem_region *region; 3841 3842 /* CFG */ 3843 region = &hdev->pci_mem_region[PCI_REGION_CFG]; 3844 region->region_base = CFG_BASE; 3845 region->region_size = CFG_SIZE; 3846 region->offset_in_bar = CFG_BASE - STM_FLASH_BASE_ADDR; 3847 region->bar_size = CFG_BAR_SIZE; 3848 region->bar_id = SRAM_CFG_BAR_ID; 3849 region->used = 1; 3850 3851 /* SRAM */ 3852 region = &hdev->pci_mem_region[PCI_REGION_SRAM]; 3853 region->region_base = SRAM_BASE_ADDR; 3854 region->region_size = SRAM_SIZE; 3855 region->offset_in_bar = CFG_REGION_SIZE + BAR0_RSRVD_SIZE; 3856 region->bar_size = CFG_BAR_SIZE; 3857 region->bar_id = SRAM_CFG_BAR_ID; 3858 region->used = 1; 3859 3860 /* DRAM */ 3861 region = &hdev->pci_mem_region[PCI_REGION_DRAM]; 3862 region->region_base = DRAM_PHYS_BASE; 3863 region->region_size = hdev->asic_prop.dram_size; 3864 region->offset_in_bar = 0; 3865 region->bar_size = prop->dram_pci_bar_size; 3866 region->bar_id = DRAM_BAR_ID; 3867 region->used = 1; 3868} 3869 3870static void gaudi2_user_interrupt_setup(struct hl_device *hdev) 3871{ 3872 struct asic_fixed_properties *prop = &hdev->asic_prop; 3873 int i, j, k; 3874 3875 /* Initialize TPC interrupt */ 3876 HL_USR_INTR_STRUCT_INIT(hdev->tpc_interrupt, hdev, 0, HL_USR_INTERRUPT_TPC); 3877 3878 /* Initialize unexpected error interrupt */ 3879 HL_USR_INTR_STRUCT_INIT(hdev->unexpected_error_interrupt, hdev, 0, 3880 HL_USR_INTERRUPT_UNEXPECTED); 3881 3882 /* Initialize common user CQ interrupt */ 3883 HL_USR_INTR_STRUCT_INIT(hdev->common_user_cq_interrupt, hdev, 3884 HL_COMMON_USER_CQ_INTERRUPT_ID, HL_USR_INTERRUPT_CQ); 3885 3886 /* Initialize common decoder interrupt */ 3887 HL_USR_INTR_STRUCT_INIT(hdev->common_decoder_interrupt, hdev, 3888 HL_COMMON_DEC_INTERRUPT_ID, HL_USR_INTERRUPT_DECODER); 3889 3890 /* User interrupts structure holds both decoder and user interrupts from various engines. 3891 * We first initialize the decoder interrupts and then we add the user interrupts. 3892 * The only limitation is that the last decoder interrupt id must be smaller 3893 * then GAUDI2_IRQ_NUM_USER_FIRST. This is checked at compilation time. 3894 */ 3895 3896 /* Initialize decoder interrupts, expose only normal interrupts, 3897 * error interrupts to be handled by driver 3898 */ 3899 for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM, j = 0 ; i <= GAUDI2_IRQ_NUM_SHARED_DEC1_NRM; 3900 i += 2, j++) 3901 HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i, 3902 HL_USR_INTERRUPT_DECODER); 3903 3904 for (i = GAUDI2_IRQ_NUM_USER_FIRST, k = 0 ; k < prop->user_interrupt_count; i++, j++, k++) 3905 HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i, HL_USR_INTERRUPT_CQ); 3906} 3907 3908static inline int gaudi2_get_non_zero_random_int(void) 3909{ 3910 int rand = get_random_u32(); 3911 3912 return rand ? rand : 1; 3913} 3914 3915static void gaudi2_special_blocks_free(struct hl_device *hdev) 3916{ 3917 struct asic_fixed_properties *prop = &hdev->asic_prop; 3918 struct hl_skip_blocks_cfg *skip_special_blocks_cfg = 3919 &prop->skip_special_blocks_cfg; 3920 3921 kfree(prop->special_blocks); 3922 kfree(skip_special_blocks_cfg->block_types); 3923 kfree(skip_special_blocks_cfg->block_ranges); 3924} 3925 3926static void gaudi2_special_blocks_iterator_free(struct hl_device *hdev) 3927{ 3928 gaudi2_special_blocks_free(hdev); 3929} 3930 3931static bool gaudi2_special_block_skip(struct hl_device *hdev, 3932 struct hl_special_blocks_cfg *special_blocks_cfg, 3933 u32 blk_idx, u32 major, u32 minor, u32 sub_minor) 3934{ 3935 return false; 3936} 3937 3938static int gaudi2_special_blocks_config(struct hl_device *hdev) 3939{ 3940 struct asic_fixed_properties *prop = &hdev->asic_prop; 3941 int i, rc; 3942 3943 /* Configure Special blocks */ 3944 prop->glbl_err_max_cause_num = GAUDI2_GLBL_ERR_MAX_CAUSE_NUM; 3945 prop->num_of_special_blocks = ARRAY_SIZE(gaudi2_special_blocks); 3946 prop->special_blocks = kmalloc_array(prop->num_of_special_blocks, 3947 sizeof(*prop->special_blocks), GFP_KERNEL); 3948 if (!prop->special_blocks) 3949 return -ENOMEM; 3950 3951 for (i = 0 ; i < prop->num_of_special_blocks ; i++) 3952 memcpy(&prop->special_blocks[i], &gaudi2_special_blocks[i], 3953 sizeof(*prop->special_blocks)); 3954 3955 /* Configure when to skip Special blocks */ 3956 memset(&prop->skip_special_blocks_cfg, 0, sizeof(prop->skip_special_blocks_cfg)); 3957 prop->skip_special_blocks_cfg.skip_block_hook = gaudi2_special_block_skip; 3958 3959 if (ARRAY_SIZE(gaudi2_iterator_skip_block_types)) { 3960 prop->skip_special_blocks_cfg.block_types = 3961 kmalloc_array(ARRAY_SIZE(gaudi2_iterator_skip_block_types), 3962 sizeof(gaudi2_iterator_skip_block_types[0]), GFP_KERNEL); 3963 if (!prop->skip_special_blocks_cfg.block_types) { 3964 rc = -ENOMEM; 3965 goto free_special_blocks; 3966 } 3967 3968 memcpy(prop->skip_special_blocks_cfg.block_types, gaudi2_iterator_skip_block_types, 3969 sizeof(gaudi2_iterator_skip_block_types)); 3970 3971 prop->skip_special_blocks_cfg.block_types_len = 3972 ARRAY_SIZE(gaudi2_iterator_skip_block_types); 3973 } 3974 3975 if (ARRAY_SIZE(gaudi2_iterator_skip_block_ranges)) { 3976 prop->skip_special_blocks_cfg.block_ranges = 3977 kmalloc_array(ARRAY_SIZE(gaudi2_iterator_skip_block_ranges), 3978 sizeof(gaudi2_iterator_skip_block_ranges[0]), GFP_KERNEL); 3979 if (!prop->skip_special_blocks_cfg.block_ranges) { 3980 rc = -ENOMEM; 3981 goto free_skip_special_blocks_types; 3982 } 3983 3984 for (i = 0 ; i < ARRAY_SIZE(gaudi2_iterator_skip_block_ranges) ; i++) 3985 memcpy(&prop->skip_special_blocks_cfg.block_ranges[i], 3986 &gaudi2_iterator_skip_block_ranges[i], 3987 sizeof(struct range)); 3988 3989 prop->skip_special_blocks_cfg.block_ranges_len = 3990 ARRAY_SIZE(gaudi2_iterator_skip_block_ranges); 3991 } 3992 3993 return 0; 3994 3995free_skip_special_blocks_types: 3996 kfree(prop->skip_special_blocks_cfg.block_types); 3997free_special_blocks: 3998 kfree(prop->special_blocks); 3999 4000 return rc; 4001} 4002 4003static int gaudi2_special_blocks_iterator_config(struct hl_device *hdev) 4004{ 4005 return gaudi2_special_blocks_config(hdev); 4006} 4007 4008static void gaudi2_test_queues_msgs_free(struct hl_device *hdev) 4009{ 4010 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4011 struct gaudi2_queues_test_info *msg_info = gaudi2->queues_test_info; 4012 int i; 4013 4014 for (i = 0 ; i < GAUDI2_NUM_TESTED_QS ; i++) { 4015 /* bail-out if this is an allocation failure point */ 4016 if (!msg_info[i].kern_addr) 4017 break; 4018 4019 hl_asic_dma_pool_free(hdev, msg_info[i].kern_addr, msg_info[i].dma_addr); 4020 msg_info[i].kern_addr = NULL; 4021 } 4022} 4023 4024static int gaudi2_test_queues_msgs_alloc(struct hl_device *hdev) 4025{ 4026 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4027 struct gaudi2_queues_test_info *msg_info = gaudi2->queues_test_info; 4028 int i, rc; 4029 4030 /* allocate a message-short buf for each Q we intend to test */ 4031 for (i = 0 ; i < GAUDI2_NUM_TESTED_QS ; i++) { 4032 msg_info[i].kern_addr = 4033 (void *)hl_asic_dma_pool_zalloc(hdev, sizeof(struct packet_msg_short), 4034 GFP_KERNEL, &msg_info[i].dma_addr); 4035 if (!msg_info[i].kern_addr) { 4036 dev_err(hdev->dev, 4037 "Failed to allocate dma memory for H/W queue %d testing\n", i); 4038 rc = -ENOMEM; 4039 goto err_exit; 4040 } 4041 } 4042 4043 return 0; 4044 4045err_exit: 4046 gaudi2_test_queues_msgs_free(hdev); 4047 return rc; 4048} 4049 4050static int gaudi2_sw_init(struct hl_device *hdev) 4051{ 4052 struct asic_fixed_properties *prop = &hdev->asic_prop; 4053 struct gaudi2_device *gaudi2; 4054 int i, rc; 4055 4056 /* Allocate device structure */ 4057 gaudi2 = kzalloc(sizeof(*gaudi2), GFP_KERNEL); 4058 if (!gaudi2) 4059 return -ENOMEM; 4060 4061 for (i = 0 ; i < ARRAY_SIZE(gaudi2_irq_map_table) ; i++) { 4062 if (gaudi2_irq_map_table[i].msg || !gaudi2_irq_map_table[i].valid) 4063 continue; 4064 4065 if (gaudi2->num_of_valid_hw_events == GAUDI2_EVENT_SIZE) { 4066 dev_err(hdev->dev, "H/W events array exceeds the limit of %u events\n", 4067 GAUDI2_EVENT_SIZE); 4068 rc = -EINVAL; 4069 goto free_gaudi2_device; 4070 } 4071 4072 gaudi2->hw_events[gaudi2->num_of_valid_hw_events++] = gaudi2_irq_map_table[i].fc_id; 4073 } 4074 4075 for (i = 0 ; i < MME_NUM_OF_LFSR_SEEDS ; i++) 4076 gaudi2->lfsr_rand_seeds[i] = gaudi2_get_non_zero_random_int(); 4077 4078 gaudi2->cpucp_info_get = gaudi2_cpucp_info_get; 4079 4080 hdev->asic_specific = gaudi2; 4081 4082 /* Create DMA pool for small allocations. 4083 * Use DEVICE_CACHE_LINE_SIZE for alignment since the NIC memory-mapped 4084 * PI/CI registers allocated from this pool have this restriction 4085 */ 4086 hdev->dma_pool = dma_pool_create(dev_name(hdev->dev), &hdev->pdev->dev, 4087 GAUDI2_DMA_POOL_BLK_SIZE, DEVICE_CACHE_LINE_SIZE, 0); 4088 if (!hdev->dma_pool) { 4089 dev_err(hdev->dev, "failed to create DMA pool\n"); 4090 rc = -ENOMEM; 4091 goto free_gaudi2_device; 4092 } 4093 4094 rc = gaudi2_alloc_cpu_accessible_dma_mem(hdev); 4095 if (rc) 4096 goto free_dma_pool; 4097 4098 hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1); 4099 if (!hdev->cpu_accessible_dma_pool) { 4100 dev_err(hdev->dev, "Failed to create CPU accessible DMA pool\n"); 4101 rc = -ENOMEM; 4102 goto free_cpu_dma_mem; 4103 } 4104 4105 rc = gen_pool_add(hdev->cpu_accessible_dma_pool, (uintptr_t) hdev->cpu_accessible_dma_mem, 4106 HL_CPU_ACCESSIBLE_MEM_SIZE, -1); 4107 if (rc) { 4108 dev_err(hdev->dev, "Failed to add memory to CPU accessible DMA pool\n"); 4109 rc = -EFAULT; 4110 goto free_cpu_accessible_dma_pool; 4111 } 4112 4113 gaudi2->virt_msix_db_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, prop->pmmu.page_size, 4114 &gaudi2->virt_msix_db_dma_addr); 4115 if (!gaudi2->virt_msix_db_cpu_addr) { 4116 dev_err(hdev->dev, "Failed to allocate DMA memory for virtual MSI-X doorbell\n"); 4117 rc = -ENOMEM; 4118 goto free_cpu_accessible_dma_pool; 4119 } 4120 4121 spin_lock_init(&gaudi2->hw_queues_lock); 4122 4123 gaudi2->scratchpad_bus_address = prop->mmu_pgt_addr + HMMU_PAGE_TABLES_SIZE + EDMA_PQS_SIZE; 4124 4125 gaudi2_user_mapped_blocks_init(hdev); 4126 4127 /* Initialize user interrupts */ 4128 gaudi2_user_interrupt_setup(hdev); 4129 4130 hdev->supports_coresight = true; 4131 hdev->supports_sync_stream = true; 4132 hdev->supports_cb_mapping = true; 4133 hdev->supports_wait_for_multi_cs = false; 4134 4135 prop->supports_compute_reset = true; 4136 4137 /* Event queue sanity check added in FW version 1.11 */ 4138 if (hl_fw_version_cmp(hdev, 1, 11, 0) < 0) 4139 hdev->event_queue.check_eqe_index = false; 4140 else 4141 hdev->event_queue.check_eqe_index = true; 4142 4143 hdev->asic_funcs->set_pci_memory_regions(hdev); 4144 4145 rc = gaudi2_special_blocks_iterator_config(hdev); 4146 if (rc) 4147 goto free_virt_msix_db_mem; 4148 4149 rc = gaudi2_test_queues_msgs_alloc(hdev); 4150 if (rc) 4151 goto special_blocks_free; 4152 4153 hdev->heartbeat_debug_info.cpu_queue_id = GAUDI2_QUEUE_ID_CPU_PQ; 4154 4155 return 0; 4156 4157special_blocks_free: 4158 gaudi2_special_blocks_iterator_free(hdev); 4159free_virt_msix_db_mem: 4160 hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr); 4161free_cpu_accessible_dma_pool: 4162 gen_pool_destroy(hdev->cpu_accessible_dma_pool); 4163free_cpu_dma_mem: 4164 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, 4165 hdev->cpu_accessible_dma_address); 4166free_dma_pool: 4167 dma_pool_destroy(hdev->dma_pool); 4168free_gaudi2_device: 4169 kfree(gaudi2); 4170 return rc; 4171} 4172 4173static int gaudi2_sw_fini(struct hl_device *hdev) 4174{ 4175 struct asic_fixed_properties *prop = &hdev->asic_prop; 4176 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4177 4178 gaudi2_test_queues_msgs_free(hdev); 4179 4180 gaudi2_special_blocks_iterator_free(hdev); 4181 4182 hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr); 4183 4184 gen_pool_destroy(hdev->cpu_accessible_dma_pool); 4185 4186 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, 4187 hdev->cpu_accessible_dma_address); 4188 4189 dma_pool_destroy(hdev->dma_pool); 4190 4191 kfree(gaudi2); 4192 4193 return 0; 4194} 4195 4196static void gaudi2_stop_qman_common(struct hl_device *hdev, u32 reg_base) 4197{ 4198 WREG32(reg_base + QM_GLBL_CFG1_OFFSET, QM_GLBL_CFG1_PQF_STOP | 4199 QM_GLBL_CFG1_CQF_STOP | 4200 QM_GLBL_CFG1_CP_STOP); 4201 4202 /* stop also the ARC */ 4203 WREG32(reg_base + QM_GLBL_CFG2_OFFSET, QM_GLBL_CFG2_ARC_CQF_STOP); 4204} 4205 4206static void gaudi2_flush_qman_common(struct hl_device *hdev, u32 reg_base) 4207{ 4208 WREG32(reg_base + QM_GLBL_CFG1_OFFSET, QM_GLBL_CFG1_PQF_FLUSH | 4209 QM_GLBL_CFG1_CQF_FLUSH | 4210 QM_GLBL_CFG1_CP_FLUSH); 4211} 4212 4213static void gaudi2_flush_qman_arc_common(struct hl_device *hdev, u32 reg_base) 4214{ 4215 WREG32(reg_base + QM_GLBL_CFG2_OFFSET, QM_GLBL_CFG2_ARC_CQF_FLUSH); 4216} 4217 4218/** 4219 * gaudi2_clear_qm_fence_counters_common - clear QM's fence counters 4220 * 4221 * @hdev: pointer to the habanalabs device structure 4222 * @queue_id: queue to clear fence counters to 4223 * @skip_fence: if true set maximum fence value to all fence counters to avoid 4224 * getting stuck on any fence value. otherwise set all fence 4225 * counters to 0 (standard clear of fence counters) 4226 */ 4227static void gaudi2_clear_qm_fence_counters_common(struct hl_device *hdev, u32 queue_id, 4228 bool skip_fence) 4229{ 4230 u32 size, reg_base; 4231 u32 addr, val; 4232 4233 reg_base = gaudi2_qm_blocks_bases[queue_id]; 4234 4235 addr = reg_base + QM_CP_FENCE0_CNT_0_OFFSET; 4236 size = mmPDMA0_QM_CP_BARRIER_CFG - mmPDMA0_QM_CP_FENCE0_CNT_0; 4237 4238 /* 4239 * in case we want to make sure that QM that is stuck on a fence will 4240 * be released we should set the fence counter to a higher value that 4241 * the value the QM waiting for. to comply with any fence counter of 4242 * any value we set maximum fence value to all counters 4243 */ 4244 val = skip_fence ? U32_MAX : 0; 4245 gaudi2_memset_device_lbw(hdev, addr, size, val); 4246} 4247 4248static void gaudi2_qman_manual_flush_common(struct hl_device *hdev, u32 queue_id) 4249{ 4250 u32 reg_base = gaudi2_qm_blocks_bases[queue_id]; 4251 4252 gaudi2_clear_qm_fence_counters_common(hdev, queue_id, true); 4253 gaudi2_flush_qman_common(hdev, reg_base); 4254 gaudi2_flush_qman_arc_common(hdev, reg_base); 4255} 4256 4257static void gaudi2_stop_dma_qmans(struct hl_device *hdev) 4258{ 4259 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4260 int dcore, inst; 4261 4262 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) 4263 goto stop_edma_qmans; 4264 4265 /* Stop CPs of PDMA QMANs */ 4266 gaudi2_stop_qman_common(hdev, mmPDMA0_QM_BASE); 4267 gaudi2_stop_qman_common(hdev, mmPDMA1_QM_BASE); 4268 4269stop_edma_qmans: 4270 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) 4271 return; 4272 4273 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 4274 for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) { 4275 u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst; 4276 u32 qm_base; 4277 4278 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) 4279 continue; 4280 4281 qm_base = mmDCORE0_EDMA0_QM_BASE + dcore * DCORE_OFFSET + 4282 inst * DCORE_EDMA_OFFSET; 4283 4284 /* Stop CPs of EDMA QMANs */ 4285 gaudi2_stop_qman_common(hdev, qm_base); 4286 } 4287 } 4288} 4289 4290static void gaudi2_stop_mme_qmans(struct hl_device *hdev) 4291{ 4292 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4293 u32 offset, i; 4294 4295 offset = mmDCORE1_MME_QM_BASE - mmDCORE0_MME_QM_BASE; 4296 4297 for (i = 0 ; i < NUM_OF_DCORES ; i++) { 4298 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i))) 4299 continue; 4300 4301 gaudi2_stop_qman_common(hdev, mmDCORE0_MME_QM_BASE + (i * offset)); 4302 } 4303} 4304 4305static void gaudi2_stop_tpc_qmans(struct hl_device *hdev) 4306{ 4307 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4308 u32 reg_base; 4309 int i; 4310 4311 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) 4312 return; 4313 4314 for (i = 0 ; i < TPC_ID_SIZE ; i++) { 4315 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) 4316 continue; 4317 4318 reg_base = gaudi2_qm_blocks_bases[gaudi2_tpc_id_to_queue_id[i]]; 4319 gaudi2_stop_qman_common(hdev, reg_base); 4320 } 4321} 4322 4323static void gaudi2_stop_rot_qmans(struct hl_device *hdev) 4324{ 4325 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4326 u32 reg_base; 4327 int i; 4328 4329 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) 4330 return; 4331 4332 for (i = 0 ; i < ROTATOR_ID_SIZE ; i++) { 4333 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) 4334 continue; 4335 4336 reg_base = gaudi2_qm_blocks_bases[gaudi2_rot_id_to_queue_id[i]]; 4337 gaudi2_stop_qman_common(hdev, reg_base); 4338 } 4339} 4340 4341static void gaudi2_stop_nic_qmans(struct hl_device *hdev) 4342{ 4343 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4344 u32 reg_base, queue_id; 4345 int i; 4346 4347 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) 4348 return; 4349 4350 queue_id = GAUDI2_QUEUE_ID_NIC_0_0; 4351 4352 for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++, queue_id += NUM_OF_PQ_PER_QMAN) { 4353 if (!(hdev->nic_ports_mask & BIT(i))) 4354 continue; 4355 4356 reg_base = gaudi2_qm_blocks_bases[queue_id]; 4357 gaudi2_stop_qman_common(hdev, reg_base); 4358 } 4359} 4360 4361static void gaudi2_stall_dma_common(struct hl_device *hdev, u32 reg_base) 4362{ 4363 u32 reg_val; 4364 4365 reg_val = FIELD_PREP(PDMA0_CORE_CFG_1_HALT_MASK, 0x1); 4366 WREG32(reg_base + DMA_CORE_CFG_1_OFFSET, reg_val); 4367} 4368 4369static void gaudi2_dma_stall(struct hl_device *hdev) 4370{ 4371 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4372 int dcore, inst; 4373 4374 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) 4375 goto stall_edma; 4376 4377 gaudi2_stall_dma_common(hdev, mmPDMA0_CORE_BASE); 4378 gaudi2_stall_dma_common(hdev, mmPDMA1_CORE_BASE); 4379 4380stall_edma: 4381 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) 4382 return; 4383 4384 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 4385 for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) { 4386 u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst; 4387 u32 core_base; 4388 4389 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) 4390 continue; 4391 4392 core_base = mmDCORE0_EDMA0_CORE_BASE + dcore * DCORE_OFFSET + 4393 inst * DCORE_EDMA_OFFSET; 4394 4395 /* Stall CPs of EDMA QMANs */ 4396 gaudi2_stall_dma_common(hdev, core_base); 4397 } 4398 } 4399} 4400 4401static void gaudi2_mme_stall(struct hl_device *hdev) 4402{ 4403 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4404 u32 offset, i; 4405 4406 offset = mmDCORE1_MME_CTRL_LO_QM_STALL - mmDCORE0_MME_CTRL_LO_QM_STALL; 4407 4408 for (i = 0 ; i < NUM_OF_DCORES ; i++) 4409 if (gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i)) 4410 WREG32(mmDCORE0_MME_CTRL_LO_QM_STALL + (i * offset), 1); 4411} 4412 4413static void gaudi2_tpc_stall(struct hl_device *hdev) 4414{ 4415 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4416 u32 reg_base; 4417 int i; 4418 4419 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) 4420 return; 4421 4422 for (i = 0 ; i < TPC_ID_SIZE ; i++) { 4423 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) 4424 continue; 4425 4426 reg_base = gaudi2_tpc_cfg_blocks_bases[i]; 4427 WREG32(reg_base + TPC_CFG_STALL_OFFSET, 1); 4428 } 4429} 4430 4431static void gaudi2_rotator_stall(struct hl_device *hdev) 4432{ 4433 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4434 u32 reg_val; 4435 int i; 4436 4437 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) 4438 return; 4439 4440 reg_val = FIELD_PREP(ROT_MSS_HALT_WBC_MASK, 0x1) | 4441 FIELD_PREP(ROT_MSS_HALT_RSB_MASK, 0x1) | 4442 FIELD_PREP(ROT_MSS_HALT_MRSB_MASK, 0x1); 4443 4444 for (i = 0 ; i < ROTATOR_ID_SIZE ; i++) { 4445 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) 4446 continue; 4447 4448 WREG32(mmROT0_MSS_HALT + i * ROT_OFFSET, reg_val); 4449 } 4450} 4451 4452static void gaudi2_disable_qman_common(struct hl_device *hdev, u32 reg_base) 4453{ 4454 WREG32(reg_base + QM_GLBL_CFG0_OFFSET, 0); 4455} 4456 4457static void gaudi2_disable_dma_qmans(struct hl_device *hdev) 4458{ 4459 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4460 int dcore, inst; 4461 4462 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) 4463 goto stop_edma_qmans; 4464 4465 gaudi2_disable_qman_common(hdev, mmPDMA0_QM_BASE); 4466 gaudi2_disable_qman_common(hdev, mmPDMA1_QM_BASE); 4467 4468stop_edma_qmans: 4469 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) 4470 return; 4471 4472 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 4473 for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) { 4474 u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst; 4475 u32 qm_base; 4476 4477 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) 4478 continue; 4479 4480 qm_base = mmDCORE0_EDMA0_QM_BASE + dcore * DCORE_OFFSET + 4481 inst * DCORE_EDMA_OFFSET; 4482 4483 /* Disable CPs of EDMA QMANs */ 4484 gaudi2_disable_qman_common(hdev, qm_base); 4485 } 4486 } 4487} 4488 4489static void gaudi2_disable_mme_qmans(struct hl_device *hdev) 4490{ 4491 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4492 u32 offset, i; 4493 4494 offset = mmDCORE1_MME_QM_BASE - mmDCORE0_MME_QM_BASE; 4495 4496 for (i = 0 ; i < NUM_OF_DCORES ; i++) 4497 if (gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i)) 4498 gaudi2_disable_qman_common(hdev, mmDCORE0_MME_QM_BASE + (i * offset)); 4499} 4500 4501static void gaudi2_disable_tpc_qmans(struct hl_device *hdev) 4502{ 4503 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4504 u32 reg_base; 4505 int i; 4506 4507 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) 4508 return; 4509 4510 for (i = 0 ; i < TPC_ID_SIZE ; i++) { 4511 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) 4512 continue; 4513 4514 reg_base = gaudi2_qm_blocks_bases[gaudi2_tpc_id_to_queue_id[i]]; 4515 gaudi2_disable_qman_common(hdev, reg_base); 4516 } 4517} 4518 4519static void gaudi2_disable_rot_qmans(struct hl_device *hdev) 4520{ 4521 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4522 u32 reg_base; 4523 int i; 4524 4525 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) 4526 return; 4527 4528 for (i = 0 ; i < ROTATOR_ID_SIZE ; i++) { 4529 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) 4530 continue; 4531 4532 reg_base = gaudi2_qm_blocks_bases[gaudi2_rot_id_to_queue_id[i]]; 4533 gaudi2_disable_qman_common(hdev, reg_base); 4534 } 4535} 4536 4537static void gaudi2_disable_nic_qmans(struct hl_device *hdev) 4538{ 4539 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4540 u32 reg_base, queue_id; 4541 int i; 4542 4543 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) 4544 return; 4545 4546 queue_id = GAUDI2_QUEUE_ID_NIC_0_0; 4547 4548 for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++, queue_id += NUM_OF_PQ_PER_QMAN) { 4549 if (!(hdev->nic_ports_mask & BIT(i))) 4550 continue; 4551 4552 reg_base = gaudi2_qm_blocks_bases[queue_id]; 4553 gaudi2_disable_qman_common(hdev, reg_base); 4554 } 4555} 4556 4557static void gaudi2_enable_timestamp(struct hl_device *hdev) 4558{ 4559 /* Disable the timestamp counter */ 4560 WREG32(mmPSOC_TIMESTAMP_BASE, 0); 4561 4562 /* Zero the lower/upper parts of the 64-bit counter */ 4563 WREG32(mmPSOC_TIMESTAMP_BASE + 0xC, 0); 4564 WREG32(mmPSOC_TIMESTAMP_BASE + 0x8, 0); 4565 4566 /* Enable the counter */ 4567 WREG32(mmPSOC_TIMESTAMP_BASE, 1); 4568} 4569 4570static void gaudi2_disable_timestamp(struct hl_device *hdev) 4571{ 4572 /* Disable the timestamp counter */ 4573 WREG32(mmPSOC_TIMESTAMP_BASE, 0); 4574} 4575 4576static const char *gaudi2_irq_name(u16 irq_number) 4577{ 4578 switch (irq_number) { 4579 case GAUDI2_IRQ_NUM_EVENT_QUEUE: 4580 return "gaudi2 cpu eq"; 4581 case GAUDI2_IRQ_NUM_COMPLETION: 4582 return "gaudi2 completion"; 4583 case GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM ... GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM: 4584 return gaudi2_vdec_irq_name[irq_number - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM]; 4585 case GAUDI2_IRQ_NUM_TPC_ASSERT: 4586 return "gaudi2 tpc assert"; 4587 case GAUDI2_IRQ_NUM_UNEXPECTED_ERROR: 4588 return "gaudi2 unexpected error"; 4589 case GAUDI2_IRQ_NUM_USER_FIRST ... GAUDI2_IRQ_NUM_USER_LAST: 4590 return "gaudi2 user completion"; 4591 case GAUDI2_IRQ_NUM_EQ_ERROR: 4592 return "gaudi2 eq error"; 4593 default: 4594 return "invalid"; 4595 } 4596} 4597 4598static void gaudi2_dec_disable_msix(struct hl_device *hdev, u32 max_irq_num) 4599{ 4600 int i, irq, relative_idx; 4601 struct hl_dec *dec; 4602 4603 for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM ; i < max_irq_num ; i++) { 4604 irq = pci_irq_vector(hdev->pdev, i); 4605 relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM; 4606 4607 dec = hdev->dec + relative_idx / 2; 4608 4609 /* We pass different structures depending on the irq handler. For the abnormal 4610 * interrupt we pass hl_dec and for the regular interrupt we pass the relevant 4611 * user_interrupt entry 4612 */ 4613 free_irq(irq, ((relative_idx % 2) ? 4614 (void *) dec : 4615 (void *) &hdev->user_interrupt[dec->core_id])); 4616 } 4617} 4618 4619static int gaudi2_dec_enable_msix(struct hl_device *hdev) 4620{ 4621 int rc, i, irq_init_cnt, irq, relative_idx; 4622 struct hl_dec *dec; 4623 4624 for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM, irq_init_cnt = 0; 4625 i <= GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM; 4626 i++, irq_init_cnt++) { 4627 4628 irq = pci_irq_vector(hdev->pdev, i); 4629 relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM; 4630 4631 /* We pass different structures depending on the irq handler. For the abnormal 4632 * interrupt we pass hl_dec and for the regular interrupt we pass the relevant 4633 * user_interrupt entry 4634 * 4635 * TODO: change the dec abnrm to threaded irq 4636 */ 4637 4638 dec = hdev->dec + relative_idx / 2; 4639 if (relative_idx % 2) { 4640 rc = request_irq(irq, hl_irq_handler_dec_abnrm, 0, 4641 gaudi2_irq_name(i), (void *) dec); 4642 } else { 4643 rc = request_irq(irq, hl_irq_user_interrupt_handler, 0, gaudi2_irq_name(i), 4644 (void *) &hdev->user_interrupt[dec->core_id]); 4645 } 4646 4647 if (rc) { 4648 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4649 goto free_dec_irqs; 4650 } 4651 } 4652 4653 return 0; 4654 4655free_dec_irqs: 4656 gaudi2_dec_disable_msix(hdev, (GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + irq_init_cnt)); 4657 return rc; 4658} 4659 4660static int gaudi2_enable_msix(struct hl_device *hdev) 4661{ 4662 struct asic_fixed_properties *prop = &hdev->asic_prop; 4663 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4664 int rc, irq, i, j, user_irq_init_cnt; 4665 struct hl_cq *cq; 4666 4667 if (gaudi2->hw_cap_initialized & HW_CAP_MSIX) 4668 return 0; 4669 4670 hl_init_cpu_for_irq(hdev); 4671 4672 rc = pci_alloc_irq_vectors(hdev->pdev, GAUDI2_MSIX_ENTRIES, GAUDI2_MSIX_ENTRIES, 4673 PCI_IRQ_MSIX); 4674 if (rc < 0) { 4675 dev_err(hdev->dev, "MSI-X: Failed to enable support -- %d/%d\n", 4676 GAUDI2_MSIX_ENTRIES, rc); 4677 return rc; 4678 } 4679 4680 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); 4681 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION]; 4682 rc = request_irq(irq, hl_irq_handler_cq, 0, gaudi2_irq_name(GAUDI2_IRQ_NUM_COMPLETION), cq); 4683 if (rc) { 4684 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4685 goto free_irq_vectors; 4686 } 4687 4688 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); 4689 rc = request_irq(irq, hl_irq_handler_eq, 0, gaudi2_irq_name(GAUDI2_IRQ_NUM_EVENT_QUEUE), 4690 &hdev->event_queue); 4691 if (rc) { 4692 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4693 goto free_completion_irq; 4694 } 4695 4696 rc = gaudi2_dec_enable_msix(hdev); 4697 if (rc) { 4698 dev_err(hdev->dev, "Failed to enable decoder IRQ"); 4699 goto free_event_irq; 4700 } 4701 4702 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); 4703 rc = request_threaded_irq(irq, NULL, hl_irq_user_interrupt_thread_handler, IRQF_ONESHOT, 4704 gaudi2_irq_name(GAUDI2_IRQ_NUM_TPC_ASSERT), 4705 &hdev->tpc_interrupt); 4706 if (rc) { 4707 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4708 goto free_dec_irq; 4709 } 4710 4711 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); 4712 rc = request_threaded_irq(irq, NULL, hl_irq_user_interrupt_thread_handler, IRQF_ONESHOT, 4713 gaudi2_irq_name(GAUDI2_IRQ_NUM_UNEXPECTED_ERROR), 4714 &hdev->unexpected_error_interrupt); 4715 if (rc) { 4716 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4717 goto free_tpc_irq; 4718 } 4719 4720 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, user_irq_init_cnt = 0; 4721 user_irq_init_cnt < prop->user_interrupt_count; 4722 i++, j++, user_irq_init_cnt++) { 4723 4724 irq = pci_irq_vector(hdev->pdev, i); 4725 hl_set_irq_affinity(hdev, irq); 4726 rc = request_irq(irq, hl_irq_user_interrupt_handler, 0, gaudi2_irq_name(i), 4727 &hdev->user_interrupt[j]); 4728 if (rc) { 4729 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4730 goto free_user_irq; 4731 } 4732 } 4733 4734 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR); 4735 rc = request_threaded_irq(irq, NULL, hl_irq_eq_error_interrupt_thread_handler, 4736 IRQF_ONESHOT, gaudi2_irq_name(GAUDI2_IRQ_NUM_EQ_ERROR), 4737 hdev); 4738 if (rc) { 4739 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4740 goto free_user_irq; 4741 } 4742 4743 gaudi2->hw_cap_initialized |= HW_CAP_MSIX; 4744 4745 return 0; 4746 4747free_user_irq: 4748 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count; 4749 i < GAUDI2_IRQ_NUM_USER_FIRST + user_irq_init_cnt ; i++, j++) { 4750 4751 irq = pci_irq_vector(hdev->pdev, i); 4752 irq_set_affinity_and_hint(irq, NULL); 4753 free_irq(irq, &hdev->user_interrupt[j]); 4754 } 4755 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); 4756 free_irq(irq, &hdev->unexpected_error_interrupt); 4757free_tpc_irq: 4758 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); 4759 free_irq(irq, &hdev->tpc_interrupt); 4760free_dec_irq: 4761 gaudi2_dec_disable_msix(hdev, GAUDI2_IRQ_NUM_DEC_LAST + 1); 4762free_event_irq: 4763 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); 4764 free_irq(irq, cq); 4765 4766free_completion_irq: 4767 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); 4768 free_irq(irq, cq); 4769 4770free_irq_vectors: 4771 pci_free_irq_vectors(hdev->pdev); 4772 4773 return rc; 4774} 4775 4776static void gaudi2_sync_irqs(struct hl_device *hdev) 4777{ 4778 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4779 int i, j; 4780 int irq; 4781 4782 if (!(gaudi2->hw_cap_initialized & HW_CAP_MSIX)) 4783 return; 4784 4785 /* Wait for all pending IRQs to be finished */ 4786 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION)); 4787 4788 for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM ; i <= GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM ; i++) { 4789 irq = pci_irq_vector(hdev->pdev, i); 4790 synchronize_irq(irq); 4791 } 4792 4793 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT)); 4794 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR)); 4795 4796 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = 0 ; j < hdev->asic_prop.user_interrupt_count; 4797 i++, j++) { 4798 irq = pci_irq_vector(hdev->pdev, i); 4799 synchronize_irq(irq); 4800 } 4801 4802 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE)); 4803 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR)); 4804} 4805 4806static void gaudi2_disable_msix(struct hl_device *hdev) 4807{ 4808 struct asic_fixed_properties *prop = &hdev->asic_prop; 4809 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4810 struct hl_cq *cq; 4811 int irq, i, j, k; 4812 4813 if (!(gaudi2->hw_cap_initialized & HW_CAP_MSIX)) 4814 return; 4815 4816 gaudi2_sync_irqs(hdev); 4817 4818 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); 4819 free_irq(irq, &hdev->event_queue); 4820 4821 gaudi2_dec_disable_msix(hdev, GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM + 1); 4822 4823 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); 4824 free_irq(irq, &hdev->tpc_interrupt); 4825 4826 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); 4827 free_irq(irq, &hdev->unexpected_error_interrupt); 4828 4829 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, k = 0; 4830 k < hdev->asic_prop.user_interrupt_count ; i++, j++, k++) { 4831 4832 irq = pci_irq_vector(hdev->pdev, i); 4833 irq_set_affinity_and_hint(irq, NULL); 4834 free_irq(irq, &hdev->user_interrupt[j]); 4835 } 4836 4837 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); 4838 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION]; 4839 free_irq(irq, cq); 4840 4841 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR); 4842 free_irq(irq, hdev); 4843 4844 pci_free_irq_vectors(hdev->pdev); 4845 4846 gaudi2->hw_cap_initialized &= ~HW_CAP_MSIX; 4847} 4848 4849static void gaudi2_stop_dcore_dec(struct hl_device *hdev, int dcore_id) 4850{ 4851 u32 reg_val = FIELD_PREP(DCORE0_VDEC0_BRDG_CTRL_GRACEFUL_STOP_MASK, 0x1); 4852 u32 graceful_pend_mask = DCORE0_VDEC0_BRDG_CTRL_GRACEFUL_PEND_MASK; 4853 u32 timeout_usec, dec_id, dec_bit, offset, graceful; 4854 int rc; 4855 4856 if (hdev->pldm) 4857 timeout_usec = GAUDI2_PLDM_VDEC_TIMEOUT_USEC; 4858 else 4859 timeout_usec = GAUDI2_VDEC_TIMEOUT_USEC; 4860 4861 for (dec_id = 0 ; dec_id < NUM_OF_DEC_PER_DCORE ; dec_id++) { 4862 dec_bit = dcore_id * NUM_OF_DEC_PER_DCORE + dec_id; 4863 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) 4864 continue; 4865 4866 offset = dcore_id * DCORE_OFFSET + dec_id * DCORE_VDEC_OFFSET; 4867 4868 WREG32(mmDCORE0_DEC0_CMD_SWREG16 + offset, 0); 4869 4870 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_GRACEFUL + offset, reg_val); 4871 4872 /* Wait till all traffic from decoder stops 4873 * before apply core reset. 4874 */ 4875 rc = hl_poll_timeout( 4876 hdev, 4877 mmDCORE0_VDEC0_BRDG_CTRL_GRACEFUL + offset, 4878 graceful, 4879 (graceful & graceful_pend_mask), 4880 100, 4881 timeout_usec); 4882 if (rc) 4883 dev_err(hdev->dev, 4884 "Failed to stop traffic from DCORE%d Decoder %d\n", 4885 dcore_id, dec_id); 4886 } 4887} 4888 4889static void gaudi2_stop_pcie_dec(struct hl_device *hdev) 4890{ 4891 u32 reg_val = FIELD_PREP(DCORE0_VDEC0_BRDG_CTRL_GRACEFUL_STOP_MASK, 0x1); 4892 u32 graceful_pend_mask = PCIE_VDEC0_BRDG_CTRL_GRACEFUL_PEND_MASK; 4893 u32 timeout_usec, dec_id, dec_bit, offset, graceful; 4894 int rc; 4895 4896 if (hdev->pldm) 4897 timeout_usec = GAUDI2_PLDM_VDEC_TIMEOUT_USEC; 4898 else 4899 timeout_usec = GAUDI2_VDEC_TIMEOUT_USEC; 4900 4901 for (dec_id = 0 ; dec_id < NUM_OF_DEC_PER_DCORE ; dec_id++) { 4902 dec_bit = PCIE_DEC_SHIFT + dec_id; 4903 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) 4904 continue; 4905 4906 offset = dec_id * PCIE_VDEC_OFFSET; 4907 4908 WREG32(mmPCIE_DEC0_CMD_SWREG16 + offset, 0); 4909 4910 WREG32(mmPCIE_VDEC0_BRDG_CTRL_GRACEFUL + offset, reg_val); 4911 4912 /* Wait till all traffic from decoder stops 4913 * before apply core reset. 4914 */ 4915 rc = hl_poll_timeout( 4916 hdev, 4917 mmPCIE_VDEC0_BRDG_CTRL_GRACEFUL + offset, 4918 graceful, 4919 (graceful & graceful_pend_mask), 4920 100, 4921 timeout_usec); 4922 if (rc) 4923 dev_err(hdev->dev, 4924 "Failed to stop traffic from PCIe Decoder %d\n", 4925 dec_id); 4926 } 4927} 4928 4929static void gaudi2_stop_dec(struct hl_device *hdev) 4930{ 4931 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4932 int dcore_id; 4933 4934 if ((gaudi2->dec_hw_cap_initialized & HW_CAP_DEC_MASK) == 0) 4935 return; 4936 4937 for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) 4938 gaudi2_stop_dcore_dec(hdev, dcore_id); 4939 4940 gaudi2_stop_pcie_dec(hdev); 4941} 4942 4943static void gaudi2_set_arc_running_mode(struct hl_device *hdev, u32 cpu_id, u32 run_mode) 4944{ 4945 u32 reg_base, reg_val; 4946 4947 reg_base = gaudi2_arc_blocks_bases[cpu_id]; 4948 if (run_mode == HL_ENGINE_CORE_RUN) 4949 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_RUN_REQ_MASK, 1); 4950 else 4951 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_HALT_REQ_MASK, 1); 4952 4953 WREG32(reg_base + ARC_HALT_REQ_OFFSET, reg_val); 4954} 4955 4956static void gaudi2_halt_arcs(struct hl_device *hdev) 4957{ 4958 u16 arc_id; 4959 4960 for (arc_id = CPU_ID_SCHED_ARC0; arc_id < CPU_ID_MAX; arc_id++) { 4961 if (gaudi2_is_arc_enabled(hdev, arc_id)) 4962 gaudi2_set_arc_running_mode(hdev, arc_id, HL_ENGINE_CORE_HALT); 4963 } 4964} 4965 4966static int gaudi2_verify_arc_running_mode(struct hl_device *hdev, u32 cpu_id, u32 run_mode) 4967{ 4968 int rc; 4969 u32 reg_base, val, ack_mask, timeout_usec = 100000; 4970 4971 if (hdev->pldm) 4972 timeout_usec *= 100; 4973 4974 reg_base = gaudi2_arc_blocks_bases[cpu_id]; 4975 if (run_mode == HL_ENGINE_CORE_RUN) 4976 ack_mask = ARC_FARM_ARC0_AUX_RUN_HALT_ACK_RUN_ACK_MASK; 4977 else 4978 ack_mask = ARC_FARM_ARC0_AUX_RUN_HALT_ACK_HALT_ACK_MASK; 4979 4980 rc = hl_poll_timeout(hdev, reg_base + ARC_HALT_ACK_OFFSET, 4981 val, ((val & ack_mask) == ack_mask), 4982 1000, timeout_usec); 4983 4984 if (!rc) { 4985 /* Clear */ 4986 val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_RUN_REQ_MASK, 0); 4987 WREG32(reg_base + ARC_HALT_REQ_OFFSET, val); 4988 } 4989 4990 return rc; 4991} 4992 4993static void gaudi2_reset_arcs(struct hl_device *hdev) 4994{ 4995 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4996 u16 arc_id; 4997 4998 if (!gaudi2) 4999 return; 5000 5001 for (arc_id = CPU_ID_SCHED_ARC0; arc_id < CPU_ID_MAX; arc_id++) 5002 if (gaudi2_is_arc_enabled(hdev, arc_id)) 5003 gaudi2_clr_arc_id_cap(hdev, arc_id); 5004} 5005 5006static void gaudi2_nic_qmans_manual_flush(struct hl_device *hdev) 5007{ 5008 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5009 u32 queue_id; 5010 int i; 5011 5012 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) 5013 return; 5014 5015 queue_id = GAUDI2_QUEUE_ID_NIC_0_0; 5016 5017 for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++, queue_id += NUM_OF_PQ_PER_QMAN) { 5018 if (!(hdev->nic_ports_mask & BIT(i))) 5019 continue; 5020 5021 gaudi2_qman_manual_flush_common(hdev, queue_id); 5022 } 5023} 5024 5025static int gaudi2_set_engine_cores(struct hl_device *hdev, u32 *core_ids, 5026 u32 num_cores, u32 core_command) 5027{ 5028 int i, rc; 5029 5030 for (i = 0 ; i < num_cores ; i++) { 5031 if (gaudi2_is_arc_enabled(hdev, core_ids[i])) 5032 gaudi2_set_arc_running_mode(hdev, core_ids[i], core_command); 5033 } 5034 5035 for (i = 0 ; i < num_cores ; i++) { 5036 if (gaudi2_is_arc_enabled(hdev, core_ids[i])) { 5037 rc = gaudi2_verify_arc_running_mode(hdev, core_ids[i], core_command); 5038 5039 if (rc) { 5040 dev_err(hdev->dev, "failed to %s arc: %d\n", 5041 (core_command == HL_ENGINE_CORE_HALT) ? 5042 "HALT" : "RUN", core_ids[i]); 5043 return -1; 5044 } 5045 } 5046 } 5047 5048 return 0; 5049} 5050 5051static int gaudi2_set_tpc_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command) 5052{ 5053 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5054 u32 reg_base, reg_addr, reg_val, tpc_id; 5055 5056 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) 5057 return 0; 5058 5059 tpc_id = gaudi2_tpc_engine_id_to_tpc_id[engine_id]; 5060 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + tpc_id))) 5061 return 0; 5062 5063 reg_base = gaudi2_tpc_cfg_blocks_bases[tpc_id]; 5064 reg_addr = reg_base + TPC_CFG_STALL_OFFSET; 5065 reg_val = FIELD_PREP(DCORE0_TPC0_CFG_TPC_STALL_V_MASK, 5066 (engine_command == HL_ENGINE_STALL) ? 1 : 0); 5067 WREG32(reg_addr, reg_val); 5068 5069 if (engine_command == HL_ENGINE_RESUME) { 5070 reg_base = gaudi2_tpc_eml_cfg_blocks_bases[tpc_id]; 5071 reg_addr = reg_base + TPC_EML_CFG_DBG_CNT_OFFSET; 5072 RMWREG32(reg_addr, 0x1, DCORE0_TPC0_EML_CFG_DBG_CNT_DBG_EXIT_MASK); 5073 } 5074 5075 return 0; 5076} 5077 5078static int gaudi2_set_mme_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command) 5079{ 5080 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5081 u32 reg_base, reg_addr, reg_val, mme_id; 5082 5083 mme_id = gaudi2_mme_engine_id_to_mme_id[engine_id]; 5084 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + mme_id))) 5085 return 0; 5086 5087 reg_base = gaudi2_mme_ctrl_lo_blocks_bases[mme_id]; 5088 reg_addr = reg_base + MME_CTRL_LO_QM_STALL_OFFSET; 5089 reg_val = FIELD_PREP(DCORE0_MME_CTRL_LO_QM_STALL_V_MASK, 5090 (engine_command == HL_ENGINE_STALL) ? 1 : 0); 5091 WREG32(reg_addr, reg_val); 5092 5093 return 0; 5094} 5095 5096static int gaudi2_set_edma_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command) 5097{ 5098 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5099 u32 reg_base, reg_addr, reg_val, edma_id; 5100 5101 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) 5102 return 0; 5103 5104 edma_id = gaudi2_edma_engine_id_to_edma_id[engine_id]; 5105 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + edma_id))) 5106 return 0; 5107 5108 reg_base = gaudi2_dma_core_blocks_bases[edma_id]; 5109 reg_addr = reg_base + EDMA_CORE_CFG_STALL_OFFSET; 5110 reg_val = FIELD_PREP(DCORE0_EDMA0_CORE_CFG_1_HALT_MASK, 5111 (engine_command == HL_ENGINE_STALL) ? 1 : 0); 5112 WREG32(reg_addr, reg_val); 5113 5114 if (engine_command == HL_ENGINE_STALL) { 5115 reg_val = FIELD_PREP(DCORE0_EDMA0_CORE_CFG_1_HALT_MASK, 0x1) | 5116 FIELD_PREP(DCORE0_EDMA0_CORE_CFG_1_FLUSH_MASK, 0x1); 5117 WREG32(reg_addr, reg_val); 5118 } 5119 5120 return 0; 5121} 5122 5123static int gaudi2_set_engine_modes(struct hl_device *hdev, 5124 u32 *engine_ids, u32 num_engines, u32 engine_command) 5125{ 5126 int i, rc; 5127 5128 for (i = 0 ; i < num_engines ; ++i) { 5129 switch (engine_ids[i]) { 5130 case GAUDI2_DCORE0_ENGINE_ID_TPC_0 ... GAUDI2_DCORE0_ENGINE_ID_TPC_5: 5131 case GAUDI2_DCORE1_ENGINE_ID_TPC_0 ... GAUDI2_DCORE1_ENGINE_ID_TPC_5: 5132 case GAUDI2_DCORE2_ENGINE_ID_TPC_0 ... GAUDI2_DCORE2_ENGINE_ID_TPC_5: 5133 case GAUDI2_DCORE3_ENGINE_ID_TPC_0 ... GAUDI2_DCORE3_ENGINE_ID_TPC_5: 5134 rc = gaudi2_set_tpc_engine_mode(hdev, engine_ids[i], engine_command); 5135 if (rc) 5136 return rc; 5137 5138 break; 5139 case GAUDI2_DCORE0_ENGINE_ID_MME: 5140 case GAUDI2_DCORE1_ENGINE_ID_MME: 5141 case GAUDI2_DCORE2_ENGINE_ID_MME: 5142 case GAUDI2_DCORE3_ENGINE_ID_MME: 5143 rc = gaudi2_set_mme_engine_mode(hdev, engine_ids[i], engine_command); 5144 if (rc) 5145 return rc; 5146 5147 break; 5148 case GAUDI2_DCORE0_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE0_ENGINE_ID_EDMA_1: 5149 case GAUDI2_DCORE1_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE1_ENGINE_ID_EDMA_1: 5150 case GAUDI2_DCORE2_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE2_ENGINE_ID_EDMA_1: 5151 case GAUDI2_DCORE3_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE3_ENGINE_ID_EDMA_1: 5152 rc = gaudi2_set_edma_engine_mode(hdev, engine_ids[i], engine_command); 5153 if (rc) 5154 return rc; 5155 5156 break; 5157 default: 5158 dev_err(hdev->dev, "Invalid engine ID %u\n", engine_ids[i]); 5159 return -EINVAL; 5160 } 5161 } 5162 5163 return 0; 5164} 5165 5166static int gaudi2_set_engines(struct hl_device *hdev, u32 *engine_ids, 5167 u32 num_engines, u32 engine_command) 5168{ 5169 switch (engine_command) { 5170 case HL_ENGINE_CORE_HALT: 5171 case HL_ENGINE_CORE_RUN: 5172 return gaudi2_set_engine_cores(hdev, engine_ids, num_engines, engine_command); 5173 5174 case HL_ENGINE_STALL: 5175 case HL_ENGINE_RESUME: 5176 return gaudi2_set_engine_modes(hdev, engine_ids, num_engines, engine_command); 5177 5178 default: 5179 dev_err(hdev->dev, "failed to execute command id %u\n", engine_command); 5180 return -EINVAL; 5181 } 5182} 5183 5184static void gaudi2_halt_engines(struct hl_device *hdev, bool hard_reset, bool fw_reset) 5185{ 5186 u32 wait_timeout_ms; 5187 5188 if (hdev->pldm) 5189 wait_timeout_ms = GAUDI2_PLDM_RESET_WAIT_MSEC; 5190 else 5191 wait_timeout_ms = GAUDI2_RESET_WAIT_MSEC; 5192 5193 if (fw_reset || hdev->cpld_shutdown) 5194 goto skip_engines; 5195 5196 gaudi2_stop_dma_qmans(hdev); 5197 gaudi2_stop_mme_qmans(hdev); 5198 gaudi2_stop_tpc_qmans(hdev); 5199 gaudi2_stop_rot_qmans(hdev); 5200 gaudi2_stop_nic_qmans(hdev); 5201 msleep(wait_timeout_ms); 5202 5203 gaudi2_halt_arcs(hdev); 5204 gaudi2_dma_stall(hdev); 5205 gaudi2_mme_stall(hdev); 5206 gaudi2_tpc_stall(hdev); 5207 gaudi2_rotator_stall(hdev); 5208 5209 msleep(wait_timeout_ms); 5210 5211 gaudi2_stop_dec(hdev); 5212 5213 /* 5214 * in case of soft reset do a manual flush for QMANs (currently called 5215 * only for NIC QMANs 5216 */ 5217 if (!hard_reset) 5218 gaudi2_nic_qmans_manual_flush(hdev); 5219 5220 gaudi2_disable_dma_qmans(hdev); 5221 gaudi2_disable_mme_qmans(hdev); 5222 gaudi2_disable_tpc_qmans(hdev); 5223 gaudi2_disable_rot_qmans(hdev); 5224 gaudi2_disable_nic_qmans(hdev); 5225 gaudi2_disable_timestamp(hdev); 5226 5227skip_engines: 5228 if (hard_reset) { 5229 gaudi2_disable_msix(hdev); 5230 return; 5231 } 5232 5233 gaudi2_sync_irqs(hdev); 5234} 5235 5236static void gaudi2_init_firmware_preload_params(struct hl_device *hdev) 5237{ 5238 struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load; 5239 5240 pre_fw_load->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS; 5241 pre_fw_load->sts_boot_dev_sts0_reg = mmCPU_BOOT_DEV_STS0; 5242 pre_fw_load->sts_boot_dev_sts1_reg = mmCPU_BOOT_DEV_STS1; 5243 pre_fw_load->boot_err0_reg = mmCPU_BOOT_ERR0; 5244 pre_fw_load->boot_err1_reg = mmCPU_BOOT_ERR1; 5245 pre_fw_load->wait_for_preboot_timeout = GAUDI2_PREBOOT_REQ_TIMEOUT_USEC; 5246 pre_fw_load->wait_for_preboot_extended_timeout = 5247 GAUDI2_PREBOOT_EXTENDED_REQ_TIMEOUT_USEC; 5248} 5249 5250static void gaudi2_init_firmware_loader(struct hl_device *hdev) 5251{ 5252 struct fw_load_mgr *fw_loader = &hdev->fw_loader; 5253 struct dynamic_fw_load_mgr *dynamic_loader; 5254 struct cpu_dyn_regs *dyn_regs; 5255 5256 /* fill common fields */ 5257 fw_loader->fw_comp_loaded = FW_TYPE_NONE; 5258 fw_loader->boot_fit_img.image_name = GAUDI2_BOOT_FIT_FILE; 5259 fw_loader->linux_img.image_name = GAUDI2_LINUX_FW_FILE; 5260 fw_loader->boot_fit_timeout = GAUDI2_BOOT_FIT_REQ_TIMEOUT_USEC; 5261 fw_loader->skip_bmc = false; 5262 fw_loader->sram_bar_id = SRAM_CFG_BAR_ID; 5263 fw_loader->dram_bar_id = DRAM_BAR_ID; 5264 fw_loader->cpu_timeout = GAUDI2_CPU_TIMEOUT_USEC; 5265 5266 /* here we update initial values for few specific dynamic regs (as 5267 * before reading the first descriptor from FW those value has to be 5268 * hard-coded). in later stages of the protocol those values will be 5269 * updated automatically by reading the FW descriptor so data there 5270 * will always be up-to-date 5271 */ 5272 dynamic_loader = &hdev->fw_loader.dynamic_loader; 5273 dyn_regs = &dynamic_loader->comm_desc.cpu_dyn_regs; 5274 dyn_regs->kmd_msg_to_cpu = cpu_to_le32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU); 5275 dyn_regs->cpu_cmd_status_to_host = cpu_to_le32(mmCPU_CMD_STATUS_TO_HOST); 5276 dynamic_loader->wait_for_bl_timeout = GAUDI2_WAIT_FOR_BL_TIMEOUT_USEC; 5277} 5278 5279static int gaudi2_init_cpu(struct hl_device *hdev) 5280{ 5281 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5282 int rc; 5283 5284 if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU)) 5285 return 0; 5286 5287 if (gaudi2->hw_cap_initialized & HW_CAP_CPU) 5288 return 0; 5289 5290 rc = hl_fw_init_cpu(hdev); 5291 if (rc) 5292 return rc; 5293 5294 gaudi2->hw_cap_initialized |= HW_CAP_CPU; 5295 5296 return 0; 5297} 5298 5299static int gaudi2_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout) 5300{ 5301 struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; 5302 struct asic_fixed_properties *prop = &hdev->asic_prop; 5303 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5304 struct cpu_dyn_regs *dyn_regs; 5305 struct hl_eq *eq; 5306 u32 status; 5307 int err; 5308 5309 if (!hdev->cpu_queues_enable) 5310 return 0; 5311 5312 if (gaudi2->hw_cap_initialized & HW_CAP_CPU_Q) 5313 return 0; 5314 5315 eq = &hdev->event_queue; 5316 5317 dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 5318 5319 WREG32(mmCPU_IF_PQ_BASE_ADDR_LOW, lower_32_bits(cpu_pq->bus_address)); 5320 WREG32(mmCPU_IF_PQ_BASE_ADDR_HIGH, upper_32_bits(cpu_pq->bus_address)); 5321 5322 WREG32(mmCPU_IF_EQ_BASE_ADDR_LOW, lower_32_bits(eq->bus_address)); 5323 WREG32(mmCPU_IF_EQ_BASE_ADDR_HIGH, upper_32_bits(eq->bus_address)); 5324 5325 WREG32(mmCPU_IF_CQ_BASE_ADDR_LOW, lower_32_bits(hdev->cpu_accessible_dma_address)); 5326 WREG32(mmCPU_IF_CQ_BASE_ADDR_HIGH, upper_32_bits(hdev->cpu_accessible_dma_address)); 5327 5328 WREG32(mmCPU_IF_PQ_LENGTH, HL_QUEUE_SIZE_IN_BYTES); 5329 WREG32(mmCPU_IF_EQ_LENGTH, HL_EQ_SIZE_IN_BYTES); 5330 WREG32(mmCPU_IF_CQ_LENGTH, HL_CPU_ACCESSIBLE_MEM_SIZE); 5331 5332 /* Used for EQ CI */ 5333 WREG32(mmCPU_IF_EQ_RD_OFFS, 0); 5334 5335 WREG32(mmCPU_IF_PF_PQ_PI, 0); 5336 5337 WREG32(mmCPU_IF_QUEUE_INIT, PQ_INIT_STATUS_READY_FOR_CP); 5338 5339 /* Let the ARC know we are ready as it is now handling those queues */ 5340 5341 WREG32(le32_to_cpu(dyn_regs->gic_host_pi_upd_irq), 5342 gaudi2_irq_map_table[GAUDI2_EVENT_CPU_PI_UPDATE].cpu_id); 5343 5344 err = hl_poll_timeout( 5345 hdev, 5346 mmCPU_IF_QUEUE_INIT, 5347 status, 5348 (status == PQ_INIT_STATUS_READY_FOR_HOST), 5349 1000, 5350 cpu_timeout); 5351 5352 if (err) { 5353 dev_err(hdev->dev, "Failed to communicate with device CPU (timeout)\n"); 5354 return -EIO; 5355 } 5356 5357 /* update FW application security bits */ 5358 if (prop->fw_cpu_boot_dev_sts0_valid) 5359 prop->fw_app_cpu_boot_dev_sts0 = RREG32(mmCPU_BOOT_DEV_STS0); 5360 5361 if (prop->fw_cpu_boot_dev_sts1_valid) 5362 prop->fw_app_cpu_boot_dev_sts1 = RREG32(mmCPU_BOOT_DEV_STS1); 5363 5364 gaudi2->hw_cap_initialized |= HW_CAP_CPU_Q; 5365 return 0; 5366} 5367 5368static void gaudi2_init_qman_pq(struct hl_device *hdev, u32 reg_base, 5369 u32 queue_id_base) 5370{ 5371 struct hl_hw_queue *q; 5372 u32 pq_id, pq_offset; 5373 5374 for (pq_id = 0 ; pq_id < NUM_OF_PQ_PER_QMAN ; pq_id++) { 5375 q = &hdev->kernel_queues[queue_id_base + pq_id]; 5376 pq_offset = pq_id * 4; 5377 5378 if (q->dram_bd) { 5379 WREG32(reg_base + QM_PQ_BASE_LO_0_OFFSET + pq_offset, 5380 lower_32_bits(q->pq_dram_address)); 5381 WREG32(reg_base + QM_PQ_BASE_HI_0_OFFSET + pq_offset, 5382 upper_32_bits(q->pq_dram_address)); 5383 } else { 5384 WREG32(reg_base + QM_PQ_BASE_LO_0_OFFSET + pq_offset, 5385 lower_32_bits(q->bus_address)); 5386 WREG32(reg_base + QM_PQ_BASE_HI_0_OFFSET + pq_offset, 5387 upper_32_bits(q->bus_address)); 5388 } 5389 WREG32(reg_base + QM_PQ_SIZE_0_OFFSET + pq_offset, ilog2(HL_QUEUE_LENGTH)); 5390 WREG32(reg_base + QM_PQ_PI_0_OFFSET + pq_offset, 0); 5391 WREG32(reg_base + QM_PQ_CI_0_OFFSET + pq_offset, 0); 5392 } 5393} 5394 5395static void gaudi2_init_qman_cp(struct hl_device *hdev, u32 reg_base) 5396{ 5397 u32 cp_id, cp_offset, mtr_base_lo, mtr_base_hi, so_base_lo, so_base_hi; 5398 5399 mtr_base_lo = lower_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 5400 mtr_base_hi = upper_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 5401 so_base_lo = lower_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0); 5402 so_base_hi = upper_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0); 5403 5404 for (cp_id = 0 ; cp_id < NUM_OF_CP_PER_QMAN; cp_id++) { 5405 cp_offset = cp_id * 4; 5406 5407 WREG32(reg_base + QM_CP_MSG_BASE0_ADDR_LO_0_OFFSET + cp_offset, mtr_base_lo); 5408 WREG32(reg_base + QM_CP_MSG_BASE0_ADDR_HI_0_OFFSET + cp_offset, mtr_base_hi); 5409 WREG32(reg_base + QM_CP_MSG_BASE1_ADDR_LO_0_OFFSET + cp_offset, so_base_lo); 5410 WREG32(reg_base + QM_CP_MSG_BASE1_ADDR_HI_0_OFFSET + cp_offset, so_base_hi); 5411 } 5412 5413 /* allow QMANs to accept work from ARC CQF */ 5414 WREG32(reg_base + QM_CP_CFG_OFFSET, FIELD_PREP(PDMA0_QM_CP_CFG_SWITCH_EN_MASK, 0x1)); 5415} 5416 5417static void gaudi2_init_qman_pqc(struct hl_device *hdev, u32 reg_base, 5418 u32 queue_id_base) 5419{ 5420 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5421 u32 pq_id, pq_offset, so_base_lo, so_base_hi; 5422 5423 so_base_lo = lower_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0); 5424 so_base_hi = upper_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0); 5425 5426 for (pq_id = 0 ; pq_id < NUM_OF_PQ_PER_QMAN ; pq_id++) { 5427 pq_offset = pq_id * 4; 5428 5429 /* Configure QMAN HBW to scratchpad as it is not needed */ 5430 WREG32(reg_base + QM_PQC_HBW_BASE_LO_0_OFFSET + pq_offset, 5431 lower_32_bits(gaudi2->scratchpad_bus_address)); 5432 WREG32(reg_base + QM_PQC_HBW_BASE_HI_0_OFFSET + pq_offset, 5433 upper_32_bits(gaudi2->scratchpad_bus_address)); 5434 WREG32(reg_base + QM_PQC_SIZE_0_OFFSET + pq_offset, 5435 ilog2(PAGE_SIZE / sizeof(struct hl_cq_entry))); 5436 5437 WREG32(reg_base + QM_PQC_PI_0_OFFSET + pq_offset, 0); 5438 WREG32(reg_base + QM_PQC_LBW_WDATA_0_OFFSET + pq_offset, QM_PQC_LBW_WDATA); 5439 WREG32(reg_base + QM_PQC_LBW_BASE_LO_0_OFFSET + pq_offset, so_base_lo); 5440 WREG32(reg_base + QM_PQC_LBW_BASE_HI_0_OFFSET + pq_offset, so_base_hi); 5441 } 5442 5443 /* Enable QMAN H/W completion */ 5444 WREG32(reg_base + QM_PQC_CFG_OFFSET, 1 << PDMA0_QM_PQC_CFG_EN_SHIFT); 5445} 5446 5447static u32 gaudi2_get_dyn_sp_reg(struct hl_device *hdev, u32 queue_id_base) 5448{ 5449 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 5450 u32 sp_reg_addr; 5451 5452 switch (queue_id_base) { 5453 case GAUDI2_QUEUE_ID_PDMA_0_0...GAUDI2_QUEUE_ID_PDMA_1_3: 5454 fallthrough; 5455 case GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3: 5456 fallthrough; 5457 case GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3: 5458 fallthrough; 5459 case GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3: 5460 fallthrough; 5461 case GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3: 5462 sp_reg_addr = le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl); 5463 break; 5464 case GAUDI2_QUEUE_ID_DCORE0_MME_0_0...GAUDI2_QUEUE_ID_DCORE0_MME_0_3: 5465 fallthrough; 5466 case GAUDI2_QUEUE_ID_DCORE1_MME_0_0...GAUDI2_QUEUE_ID_DCORE1_MME_0_3: 5467 fallthrough; 5468 case GAUDI2_QUEUE_ID_DCORE2_MME_0_0...GAUDI2_QUEUE_ID_DCORE2_MME_0_3: 5469 fallthrough; 5470 case GAUDI2_QUEUE_ID_DCORE3_MME_0_0...GAUDI2_QUEUE_ID_DCORE3_MME_0_3: 5471 sp_reg_addr = le32_to_cpu(dyn_regs->gic_mme_qm_irq_ctrl); 5472 break; 5473 case GAUDI2_QUEUE_ID_DCORE0_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE0_TPC_6_3: 5474 fallthrough; 5475 case GAUDI2_QUEUE_ID_DCORE1_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE1_TPC_5_3: 5476 fallthrough; 5477 case GAUDI2_QUEUE_ID_DCORE2_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE2_TPC_5_3: 5478 fallthrough; 5479 case GAUDI2_QUEUE_ID_DCORE3_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE3_TPC_5_3: 5480 sp_reg_addr = le32_to_cpu(dyn_regs->gic_tpc_qm_irq_ctrl); 5481 break; 5482 case GAUDI2_QUEUE_ID_ROT_0_0...GAUDI2_QUEUE_ID_ROT_1_3: 5483 sp_reg_addr = le32_to_cpu(dyn_regs->gic_rot_qm_irq_ctrl); 5484 break; 5485 case GAUDI2_QUEUE_ID_NIC_0_0...GAUDI2_QUEUE_ID_NIC_23_3: 5486 sp_reg_addr = le32_to_cpu(dyn_regs->gic_nic_qm_irq_ctrl); 5487 break; 5488 default: 5489 dev_err(hdev->dev, "Unexpected h/w queue %d\n", queue_id_base); 5490 return 0; 5491 } 5492 5493 return sp_reg_addr; 5494} 5495 5496static void gaudi2_init_qman_common(struct hl_device *hdev, u32 reg_base, 5497 u32 queue_id_base) 5498{ 5499 u32 glbl_prot = QMAN_MAKE_TRUSTED, irq_handler_offset; 5500 int map_table_entry; 5501 5502 WREG32(reg_base + QM_GLBL_PROT_OFFSET, glbl_prot); 5503 5504 irq_handler_offset = gaudi2_get_dyn_sp_reg(hdev, queue_id_base); 5505 WREG32(reg_base + QM_GLBL_ERR_ADDR_LO_OFFSET, lower_32_bits(CFG_BASE + irq_handler_offset)); 5506 WREG32(reg_base + QM_GLBL_ERR_ADDR_HI_OFFSET, upper_32_bits(CFG_BASE + irq_handler_offset)); 5507 5508 map_table_entry = gaudi2_qman_async_event_id[queue_id_base]; 5509 WREG32(reg_base + QM_GLBL_ERR_WDATA_OFFSET, 5510 gaudi2_irq_map_table[map_table_entry].cpu_id); 5511 5512 WREG32(reg_base + QM_ARB_ERR_MSG_EN_OFFSET, QM_ARB_ERR_MSG_EN_MASK); 5513 5514 WREG32(reg_base + QM_ARB_SLV_CHOISE_WDT_OFFSET, GAUDI2_ARB_WDT_TIMEOUT); 5515 WREG32(reg_base + QM_GLBL_CFG1_OFFSET, 0); 5516 WREG32(reg_base + QM_GLBL_CFG2_OFFSET, 0); 5517 5518 /* Enable the QMAN channel. 5519 * PDMA QMAN configuration is different, as we do not allow user to 5520 * access some of the CPs. 5521 * PDMA0: CP2/3 are reserved for the ARC usage. 5522 * PDMA1: CP1/2/3 are reserved for the ARC usage. 5523 */ 5524 if (reg_base == gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_1_0]) 5525 WREG32(reg_base + QM_GLBL_CFG0_OFFSET, PDMA1_QMAN_ENABLE); 5526 else if (reg_base == gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_0_0]) 5527 WREG32(reg_base + QM_GLBL_CFG0_OFFSET, PDMA0_QMAN_ENABLE); 5528 else 5529 WREG32(reg_base + QM_GLBL_CFG0_OFFSET, QMAN_ENABLE); 5530} 5531 5532static void gaudi2_init_qman(struct hl_device *hdev, u32 reg_base, 5533 u32 queue_id_base) 5534{ 5535 u32 pq_id; 5536 5537 for (pq_id = 0 ; pq_id < NUM_OF_PQ_PER_QMAN ; pq_id++) 5538 hdev->kernel_queues[queue_id_base + pq_id].cq_id = GAUDI2_RESERVED_CQ_CS_COMPLETION; 5539 5540 gaudi2_init_qman_pq(hdev, reg_base, queue_id_base); 5541 gaudi2_init_qman_cp(hdev, reg_base); 5542 gaudi2_init_qman_pqc(hdev, reg_base, queue_id_base); 5543 gaudi2_init_qman_common(hdev, reg_base, queue_id_base); 5544} 5545 5546static void gaudi2_init_dma_core(struct hl_device *hdev, u32 reg_base, 5547 u32 dma_core_id, bool is_secure) 5548{ 5549 u32 prot, irq_handler_offset; 5550 struct cpu_dyn_regs *dyn_regs; 5551 int map_table_entry; 5552 5553 prot = 1 << ARC_FARM_KDMA_PROT_ERR_VAL_SHIFT; 5554 if (is_secure) 5555 prot |= 1 << ARC_FARM_KDMA_PROT_VAL_SHIFT; 5556 5557 WREG32(reg_base + DMA_CORE_PROT_OFFSET, prot); 5558 5559 dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 5560 irq_handler_offset = le32_to_cpu(dyn_regs->gic_dma_core_irq_ctrl); 5561 5562 WREG32(reg_base + DMA_CORE_ERRMSG_ADDR_LO_OFFSET, 5563 lower_32_bits(CFG_BASE + irq_handler_offset)); 5564 5565 WREG32(reg_base + DMA_CORE_ERRMSG_ADDR_HI_OFFSET, 5566 upper_32_bits(CFG_BASE + irq_handler_offset)); 5567 5568 map_table_entry = gaudi2_dma_core_async_event_id[dma_core_id]; 5569 WREG32(reg_base + DMA_CORE_ERRMSG_WDATA_OFFSET, 5570 gaudi2_irq_map_table[map_table_entry].cpu_id); 5571 5572 /* Enable the DMA channel */ 5573 WREG32(reg_base + DMA_CORE_CFG_0_OFFSET, 1 << ARC_FARM_KDMA_CFG_0_EN_SHIFT); 5574} 5575 5576static void gaudi2_init_kdma(struct hl_device *hdev) 5577{ 5578 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5579 u32 reg_base; 5580 5581 if ((gaudi2->hw_cap_initialized & HW_CAP_KDMA) == HW_CAP_KDMA) 5582 return; 5583 5584 reg_base = gaudi2_dma_core_blocks_bases[DMA_CORE_ID_KDMA]; 5585 5586 gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_KDMA, true); 5587 5588 gaudi2->hw_cap_initialized |= HW_CAP_KDMA; 5589} 5590 5591static void gaudi2_init_pdma(struct hl_device *hdev) 5592{ 5593 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5594 u32 reg_base; 5595 5596 if ((gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK) == HW_CAP_PDMA_MASK) 5597 return; 5598 5599 reg_base = gaudi2_dma_core_blocks_bases[DMA_CORE_ID_PDMA0]; 5600 gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_PDMA0, false); 5601 5602 reg_base = gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_0_0]; 5603 gaudi2_init_qman(hdev, reg_base, GAUDI2_QUEUE_ID_PDMA_0_0); 5604 5605 reg_base = gaudi2_dma_core_blocks_bases[DMA_CORE_ID_PDMA1]; 5606 gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_PDMA1, false); 5607 5608 reg_base = gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_1_0]; 5609 gaudi2_init_qman(hdev, reg_base, GAUDI2_QUEUE_ID_PDMA_1_0); 5610 5611 gaudi2->hw_cap_initialized |= HW_CAP_PDMA_MASK; 5612} 5613 5614static void gaudi2_init_edma_instance(struct hl_device *hdev, u8 seq) 5615{ 5616 u32 reg_base, base_edma_core_id, base_edma_qman_id; 5617 5618 base_edma_core_id = DMA_CORE_ID_EDMA0 + seq; 5619 base_edma_qman_id = edma_stream_base[seq]; 5620 5621 reg_base = gaudi2_dma_core_blocks_bases[base_edma_core_id]; 5622 gaudi2_init_dma_core(hdev, reg_base, base_edma_core_id, false); 5623 5624 reg_base = gaudi2_qm_blocks_bases[base_edma_qman_id]; 5625 gaudi2_init_qman(hdev, reg_base, base_edma_qman_id); 5626} 5627 5628static void gaudi2_init_edma(struct hl_device *hdev) 5629{ 5630 struct asic_fixed_properties *prop = &hdev->asic_prop; 5631 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5632 int dcore, inst; 5633 5634 if ((gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK) == HW_CAP_EDMA_MASK) 5635 return; 5636 5637 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 5638 for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) { 5639 u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst; 5640 5641 if (!(prop->edma_enabled_mask & BIT(seq))) 5642 continue; 5643 5644 gaudi2_init_edma_instance(hdev, seq); 5645 5646 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_EDMA_SHIFT + seq); 5647 } 5648 } 5649} 5650 5651/* 5652 * gaudi2_arm_monitors_for_virt_msix_db() - Arm monitors for writing to the virtual MSI-X doorbell. 5653 * @hdev: pointer to habanalabs device structure. 5654 * @sob_id: sync object ID. 5655 * @first_mon_id: ID of first monitor out of 3 consecutive monitors. 5656 * @interrupt_id: interrupt ID. 5657 * 5658 * Some initiators cannot have HBW address in their completion address registers, and thus cannot 5659 * write directly to the HBW host memory of the virtual MSI-X doorbell. 5660 * Instead, they are configured to LBW write to a sync object, and a monitor will do the HBW write. 5661 * 5662 * The mechanism in the sync manager block is composed of a master monitor with 3 messages. 5663 * In addition to the HBW write, the other 2 messages are for preparing the monitor to next 5664 * completion, by decrementing the sync object value and re-arming the monitor. 5665 */ 5666static void gaudi2_arm_monitors_for_virt_msix_db(struct hl_device *hdev, u32 sob_id, 5667 u32 first_mon_id, u32 interrupt_id) 5668{ 5669 u32 sob_offset, first_mon_offset, mon_offset, payload, sob_group, mode, arm, config; 5670 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5671 u64 addr; 5672 u8 mask; 5673 5674 /* Reset the SOB value */ 5675 sob_offset = sob_id * sizeof(u32); 5676 WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset, 0); 5677 5678 /* Configure 3 monitors: 5679 * 1. Write interrupt ID to the virtual MSI-X doorbell (master monitor) 5680 * 2. Decrement SOB value by 1. 5681 * 3. Re-arm the master monitor. 5682 */ 5683 5684 first_mon_offset = first_mon_id * sizeof(u32); 5685 5686 /* 2nd monitor: Decrement SOB value by 1 */ 5687 mon_offset = first_mon_offset + sizeof(u32); 5688 5689 addr = CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset; 5690 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, lower_32_bits(addr)); 5691 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_offset, upper_32_bits(addr)); 5692 5693 payload = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 0x7FFF) | /* "-1" */ 5694 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_SIGN_MASK, 1) | 5695 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1); 5696 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, payload); 5697 5698 /* 3rd monitor: Re-arm the master monitor */ 5699 mon_offset = first_mon_offset + 2 * sizeof(u32); 5700 5701 addr = CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + first_mon_offset; 5702 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, lower_32_bits(addr)); 5703 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_offset, upper_32_bits(addr)); 5704 5705 sob_group = sob_id / 8; 5706 mask = ~BIT(sob_id & 0x7); 5707 mode = 0; /* comparison mode is "greater than or equal to" */ 5708 arm = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SID_MASK, sob_group) | 5709 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_MASK_MASK, mask) | 5710 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOP_MASK, mode) | 5711 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOD_MASK, 1); 5712 5713 payload = arm; 5714 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, payload); 5715 5716 /* 1st monitor (master): Write interrupt ID to the virtual MSI-X doorbell */ 5717 mon_offset = first_mon_offset; 5718 5719 config = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_WR_NUM_MASK, 2); /* "2": 3 writes */ 5720 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + mon_offset, config); 5721 5722 addr = gaudi2->virt_msix_db_dma_addr; 5723 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, lower_32_bits(addr)); 5724 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_offset, upper_32_bits(addr)); 5725 5726 payload = interrupt_id; 5727 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, payload); 5728 5729 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + mon_offset, arm); 5730} 5731 5732static void gaudi2_prepare_sm_for_virt_msix_db(struct hl_device *hdev) 5733{ 5734 u32 decoder_id, sob_id, first_mon_id, interrupt_id; 5735 struct asic_fixed_properties *prop = &hdev->asic_prop; 5736 5737 /* Decoder normal/abnormal interrupts */ 5738 for (decoder_id = 0 ; decoder_id < NUMBER_OF_DEC ; ++decoder_id) { 5739 if (!(prop->decoder_enabled_mask & BIT(decoder_id))) 5740 continue; 5741 5742 sob_id = GAUDI2_RESERVED_SOB_DEC_NRM_FIRST + decoder_id; 5743 first_mon_id = GAUDI2_RESERVED_MON_DEC_NRM_FIRST + 3 * decoder_id; 5744 interrupt_id = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + 2 * decoder_id; 5745 gaudi2_arm_monitors_for_virt_msix_db(hdev, sob_id, first_mon_id, interrupt_id); 5746 5747 sob_id = GAUDI2_RESERVED_SOB_DEC_ABNRM_FIRST + decoder_id; 5748 first_mon_id = GAUDI2_RESERVED_MON_DEC_ABNRM_FIRST + 3 * decoder_id; 5749 interrupt_id += 1; 5750 gaudi2_arm_monitors_for_virt_msix_db(hdev, sob_id, first_mon_id, interrupt_id); 5751 } 5752} 5753 5754static void gaudi2_init_sm(struct hl_device *hdev) 5755{ 5756 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5757 u64 cq_address; 5758 u32 reg_val; 5759 int i; 5760 5761 /* Enable HBW/LBW CQ for completion monitors */ 5762 reg_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_CQ_EN_MASK, 1); 5763 reg_val |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_LBW_EN_MASK, 1); 5764 5765 for (i = 0 ; i < GAUDI2_MAX_PENDING_CS ; i++) 5766 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + (4 * i), reg_val); 5767 5768 /* Enable only HBW CQ for KDMA completion monitor */ 5769 reg_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_CQ_EN_MASK, 1); 5770 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + (4 * i), reg_val); 5771 5772 /* Init CQ0 DB - configure the monitor to trigger MSI-X interrupt */ 5773 WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0, lower_32_bits(gaudi2->virt_msix_db_dma_addr)); 5774 WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0, upper_32_bits(gaudi2->virt_msix_db_dma_addr)); 5775 WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_DATA_0, GAUDI2_IRQ_NUM_COMPLETION); 5776 5777 for (i = 0 ; i < GAUDI2_RESERVED_CQ_NUMBER ; i++) { 5778 cq_address = 5779 hdev->completion_queue[i].bus_address; 5780 5781 WREG32(mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_L_0 + (4 * i), 5782 lower_32_bits(cq_address)); 5783 WREG32(mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_H_0 + (4 * i), 5784 upper_32_bits(cq_address)); 5785 WREG32(mmDCORE0_SYNC_MNGR_GLBL_CQ_SIZE_LOG2_0 + (4 * i), 5786 ilog2(HL_CQ_SIZE_IN_BYTES)); 5787 } 5788 5789 /* Configure kernel ASID and MMU BP*/ 5790 WREG32(mmDCORE0_SYNC_MNGR_GLBL_ASID_SEC, 0x10000); 5791 WREG32(mmDCORE0_SYNC_MNGR_GLBL_ASID_NONE_SEC_PRIV, 0); 5792 5793 /* Initialize sync objects and monitors which are used for the virtual MSI-X doorbell */ 5794 gaudi2_prepare_sm_for_virt_msix_db(hdev); 5795} 5796 5797static void gaudi2_init_mme_acc(struct hl_device *hdev, u32 reg_base) 5798{ 5799 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5800 u32 reg_val; 5801 int i; 5802 5803 reg_val = FIELD_PREP(MME_ACC_INTR_MASK_WBC_ERR_RESP_MASK, 0); 5804 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_POS_INF_MASK, 1); 5805 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_NEG_INF_MASK, 1); 5806 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_NAN_MASK, 1); 5807 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_RESULT_POS_INF_MASK, 1); 5808 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_RESULT_NEG_INF_MASK, 1); 5809 5810 WREG32(reg_base + MME_ACC_INTR_MASK_OFFSET, reg_val); 5811 WREG32(reg_base + MME_ACC_AP_LFSR_POLY_OFFSET, 0x80DEADAF); 5812 5813 for (i = 0 ; i < MME_NUM_OF_LFSR_SEEDS ; i++) { 5814 WREG32(reg_base + MME_ACC_AP_LFSR_SEED_SEL_OFFSET, i); 5815 WREG32(reg_base + MME_ACC_AP_LFSR_SEED_WDATA_OFFSET, gaudi2->lfsr_rand_seeds[i]); 5816 } 5817} 5818 5819static void gaudi2_init_dcore_mme(struct hl_device *hdev, int dcore_id, 5820 bool config_qman_only) 5821{ 5822 u32 queue_id_base, reg_base; 5823 5824 switch (dcore_id) { 5825 case 0: 5826 queue_id_base = GAUDI2_QUEUE_ID_DCORE0_MME_0_0; 5827 break; 5828 case 1: 5829 queue_id_base = GAUDI2_QUEUE_ID_DCORE1_MME_0_0; 5830 break; 5831 case 2: 5832 queue_id_base = GAUDI2_QUEUE_ID_DCORE2_MME_0_0; 5833 break; 5834 case 3: 5835 queue_id_base = GAUDI2_QUEUE_ID_DCORE3_MME_0_0; 5836 break; 5837 default: 5838 dev_err(hdev->dev, "Invalid dcore id %u\n", dcore_id); 5839 return; 5840 } 5841 5842 if (!config_qman_only) { 5843 reg_base = gaudi2_mme_acc_blocks_bases[dcore_id]; 5844 gaudi2_init_mme_acc(hdev, reg_base); 5845 } 5846 5847 reg_base = gaudi2_qm_blocks_bases[queue_id_base]; 5848 gaudi2_init_qman(hdev, reg_base, queue_id_base); 5849} 5850 5851static void gaudi2_init_mme(struct hl_device *hdev) 5852{ 5853 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5854 int i; 5855 5856 if ((gaudi2->hw_cap_initialized & HW_CAP_MME_MASK) == HW_CAP_MME_MASK) 5857 return; 5858 5859 for (i = 0 ; i < NUM_OF_DCORES ; i++) { 5860 gaudi2_init_dcore_mme(hdev, i, false); 5861 5862 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_MME_SHIFT + i); 5863 } 5864} 5865 5866static void gaudi2_init_tpc_cfg(struct hl_device *hdev, u32 reg_base) 5867{ 5868 /* Mask arithmetic and QM interrupts in TPC */ 5869 WREG32(reg_base + TPC_CFG_TPC_INTR_MASK_OFFSET, 0x23FFFE); 5870 5871 /* Set 16 cache lines */ 5872 WREG32(reg_base + TPC_CFG_MSS_CONFIG_OFFSET, 5873 2 << DCORE0_TPC0_CFG_MSS_CONFIG_ICACHE_FETCH_LINE_NUM_SHIFT); 5874} 5875 5876struct gaudi2_tpc_init_cfg_data { 5877 enum gaudi2_queue_id dcore_tpc_qid_base[NUM_OF_DCORES]; 5878}; 5879 5880static void gaudi2_init_tpc_config(struct hl_device *hdev, int dcore, int inst, 5881 u32 offset, struct iterate_module_ctx *ctx) 5882{ 5883 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5884 struct gaudi2_tpc_init_cfg_data *cfg_data = ctx->data; 5885 u32 queue_id_base; 5886 u8 seq; 5887 5888 queue_id_base = cfg_data->dcore_tpc_qid_base[dcore] + (inst * NUM_OF_PQ_PER_QMAN); 5889 5890 if (dcore == 0 && inst == (NUM_DCORE0_TPC - 1)) 5891 /* gets last sequence number */ 5892 seq = NUM_OF_DCORES * NUM_OF_TPC_PER_DCORE; 5893 else 5894 seq = dcore * NUM_OF_TPC_PER_DCORE + inst; 5895 5896 gaudi2_init_tpc_cfg(hdev, mmDCORE0_TPC0_CFG_BASE + offset); 5897 gaudi2_init_qman(hdev, mmDCORE0_TPC0_QM_BASE + offset, queue_id_base); 5898 5899 gaudi2->tpc_hw_cap_initialized |= BIT_ULL(HW_CAP_TPC_SHIFT + seq); 5900} 5901 5902static void gaudi2_init_tpc(struct hl_device *hdev) 5903{ 5904 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5905 struct gaudi2_tpc_init_cfg_data init_cfg_data; 5906 struct iterate_module_ctx tpc_iter; 5907 5908 if (!hdev->asic_prop.tpc_enabled_mask) 5909 return; 5910 5911 if ((gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK) == HW_CAP_TPC_MASK) 5912 return; 5913 5914 init_cfg_data.dcore_tpc_qid_base[0] = GAUDI2_QUEUE_ID_DCORE0_TPC_0_0; 5915 init_cfg_data.dcore_tpc_qid_base[1] = GAUDI2_QUEUE_ID_DCORE1_TPC_0_0; 5916 init_cfg_data.dcore_tpc_qid_base[2] = GAUDI2_QUEUE_ID_DCORE2_TPC_0_0; 5917 init_cfg_data.dcore_tpc_qid_base[3] = GAUDI2_QUEUE_ID_DCORE3_TPC_0_0; 5918 tpc_iter.fn = &gaudi2_init_tpc_config; 5919 tpc_iter.data = &init_cfg_data; 5920 gaudi2_iterate_tpcs(hdev, &tpc_iter); 5921} 5922 5923static void gaudi2_init_rotator(struct hl_device *hdev) 5924{ 5925 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5926 u32 i, reg_base, queue_id; 5927 5928 queue_id = GAUDI2_QUEUE_ID_ROT_0_0; 5929 5930 for (i = 0 ; i < NUM_OF_ROT ; i++, queue_id += NUM_OF_PQ_PER_QMAN) { 5931 reg_base = gaudi2_qm_blocks_bases[queue_id]; 5932 gaudi2_init_qman(hdev, reg_base, queue_id); 5933 5934 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_ROT_SHIFT + i); 5935 } 5936} 5937 5938static void gaudi2_init_vdec_brdg_ctrl(struct hl_device *hdev, u64 base_addr, u32 decoder_id) 5939{ 5940 u32 sob_id; 5941 5942 /* VCMD normal interrupt */ 5943 sob_id = GAUDI2_RESERVED_SOB_DEC_NRM_FIRST + decoder_id; 5944 WREG32(base_addr + BRDG_CTRL_NRM_MSIX_LBW_AWADDR, 5945 mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_id * sizeof(u32)); 5946 WREG32(base_addr + BRDG_CTRL_NRM_MSIX_LBW_WDATA, GAUDI2_SOB_INCREMENT_BY_ONE); 5947 5948 /* VCMD abnormal interrupt */ 5949 sob_id = GAUDI2_RESERVED_SOB_DEC_ABNRM_FIRST + decoder_id; 5950 WREG32(base_addr + BRDG_CTRL_ABNRM_MSIX_LBW_AWADDR, 5951 mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_id * sizeof(u32)); 5952 WREG32(base_addr + BRDG_CTRL_ABNRM_MSIX_LBW_WDATA, GAUDI2_SOB_INCREMENT_BY_ONE); 5953} 5954 5955static void gaudi2_init_dec(struct hl_device *hdev) 5956{ 5957 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5958 u32 dcore_id, dec_id, dec_bit; 5959 u64 base_addr; 5960 5961 if (!hdev->asic_prop.decoder_enabled_mask) 5962 return; 5963 5964 if ((gaudi2->dec_hw_cap_initialized & HW_CAP_DEC_MASK) == HW_CAP_DEC_MASK) 5965 return; 5966 5967 for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) 5968 for (dec_id = 0 ; dec_id < NUM_OF_DEC_PER_DCORE ; dec_id++) { 5969 dec_bit = dcore_id * NUM_OF_DEC_PER_DCORE + dec_id; 5970 5971 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) 5972 continue; 5973 5974 base_addr = mmDCORE0_DEC0_CMD_BASE + 5975 BRDG_CTRL_BLOCK_OFFSET + 5976 dcore_id * DCORE_OFFSET + 5977 dec_id * DCORE_VDEC_OFFSET; 5978 5979 gaudi2_init_vdec_brdg_ctrl(hdev, base_addr, dec_bit); 5980 5981 gaudi2->dec_hw_cap_initialized |= BIT_ULL(HW_CAP_DEC_SHIFT + dec_bit); 5982 } 5983 5984 for (dec_id = 0 ; dec_id < NUM_OF_PCIE_VDEC ; dec_id++) { 5985 dec_bit = PCIE_DEC_SHIFT + dec_id; 5986 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) 5987 continue; 5988 5989 base_addr = mmPCIE_DEC0_CMD_BASE + BRDG_CTRL_BLOCK_OFFSET + 5990 dec_id * DCORE_VDEC_OFFSET; 5991 5992 gaudi2_init_vdec_brdg_ctrl(hdev, base_addr, dec_bit); 5993 5994 gaudi2->dec_hw_cap_initialized |= BIT_ULL(HW_CAP_DEC_SHIFT + dec_bit); 5995 } 5996} 5997 5998static int gaudi2_mmu_update_asid_hop0_addr(struct hl_device *hdev, 5999 u32 stlb_base, u32 asid, u64 phys_addr) 6000{ 6001 u32 status, timeout_usec; 6002 int rc; 6003 6004 if (hdev->pldm || !hdev->pdev) 6005 timeout_usec = GAUDI2_PLDM_MMU_TIMEOUT_USEC; 6006 else 6007 timeout_usec = MMU_CONFIG_TIMEOUT_USEC; 6008 6009 WREG32(stlb_base + STLB_ASID_OFFSET, asid); 6010 WREG32(stlb_base + STLB_HOP0_PA43_12_OFFSET, phys_addr >> MMU_HOP0_PA43_12_SHIFT); 6011 WREG32(stlb_base + STLB_HOP0_PA63_44_OFFSET, phys_addr >> MMU_HOP0_PA63_44_SHIFT); 6012 WREG32(stlb_base + STLB_BUSY_OFFSET, 0x80000000); 6013 6014 rc = hl_poll_timeout( 6015 hdev, 6016 stlb_base + STLB_BUSY_OFFSET, 6017 status, 6018 !(status & 0x80000000), 6019 1000, 6020 timeout_usec); 6021 6022 if (rc) { 6023 dev_err(hdev->dev, "Timeout during MMU hop0 config of asid %d\n", asid); 6024 return rc; 6025 } 6026 6027 return 0; 6028} 6029 6030static void gaudi2_mmu_send_invalidate_cache_cmd(struct hl_device *hdev, u32 stlb_base, 6031 u32 start_offset, u32 inv_start_val, 6032 u32 flags) 6033{ 6034 /* clear PMMU mem line cache (only needed in mmu range invalidation) */ 6035 if (flags & MMU_OP_CLEAR_MEMCACHE) 6036 WREG32(mmPMMU_HBW_STLB_MEM_CACHE_INVALIDATION, 0x1); 6037 6038 if (flags & MMU_OP_SKIP_LOW_CACHE_INV) 6039 return; 6040 6041 WREG32(stlb_base + start_offset, inv_start_val); 6042} 6043 6044static int gaudi2_mmu_invalidate_cache_status_poll(struct hl_device *hdev, u32 stlb_base, 6045 struct gaudi2_cache_invld_params *inv_params) 6046{ 6047 u32 status, timeout_usec, start_offset; 6048 int rc; 6049 6050 timeout_usec = (hdev->pldm) ? GAUDI2_PLDM_MMU_TIMEOUT_USEC : 6051 GAUDI2_MMU_CACHE_INV_TIMEOUT_USEC; 6052 6053 /* poll PMMU mem line cache (only needed in mmu range invalidation) */ 6054 if (inv_params->flags & MMU_OP_CLEAR_MEMCACHE) { 6055 rc = hl_poll_timeout( 6056 hdev, 6057 mmPMMU_HBW_STLB_MEM_CACHE_INV_STATUS, 6058 status, 6059 status & 0x1, 6060 1000, 6061 timeout_usec); 6062 6063 if (rc) 6064 return rc; 6065 6066 /* Need to manually reset the status to 0 */ 6067 WREG32(mmPMMU_HBW_STLB_MEM_CACHE_INV_STATUS, 0x0); 6068 } 6069 6070 /* Lower cache does not work with cache lines, hence we can skip its 6071 * invalidation upon map and invalidate only upon unmap 6072 */ 6073 if (inv_params->flags & MMU_OP_SKIP_LOW_CACHE_INV) 6074 return 0; 6075 6076 start_offset = inv_params->range_invalidation ? 6077 STLB_RANGE_CACHE_INVALIDATION_OFFSET : STLB_INV_ALL_START_OFFSET; 6078 6079 rc = hl_poll_timeout( 6080 hdev, 6081 stlb_base + start_offset, 6082 status, 6083 !(status & 0x1), 6084 1000, 6085 timeout_usec); 6086 6087 return rc; 6088} 6089 6090bool gaudi2_is_hmmu_enabled(struct hl_device *hdev, int dcore_id, int hmmu_id) 6091{ 6092 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6093 u32 hw_cap; 6094 6095 hw_cap = HW_CAP_DCORE0_DMMU0 << (NUM_OF_HMMU_PER_DCORE * dcore_id + hmmu_id); 6096 6097 if (gaudi2->hw_cap_initialized & hw_cap) 6098 return true; 6099 6100 return false; 6101} 6102 6103/* this function shall be called only for HMMUs for which capability bit is set */ 6104static inline u32 get_hmmu_stlb_base(int dcore_id, int hmmu_id) 6105{ 6106 u32 offset; 6107 6108 offset = (u32) (dcore_id * DCORE_OFFSET + hmmu_id * DCORE_HMMU_OFFSET); 6109 return (u32)(mmDCORE0_HMMU0_STLB_BASE + offset); 6110} 6111 6112static void gaudi2_mmu_invalidate_cache_trigger(struct hl_device *hdev, u32 stlb_base, 6113 struct gaudi2_cache_invld_params *inv_params) 6114{ 6115 u32 start_offset; 6116 6117 if (inv_params->range_invalidation) { 6118 /* Set the addresses range 6119 * Note: that the start address we set in register, is not included in 6120 * the range of the invalidation, by design. 6121 * that's why we need to set lower address than the one we actually 6122 * want to be included in the range invalidation. 6123 */ 6124 u64 start = inv_params->start_va - 1; 6125 6126 start_offset = STLB_RANGE_CACHE_INVALIDATION_OFFSET; 6127 6128 WREG32(stlb_base + STLB_RANGE_INV_START_LSB_OFFSET, 6129 start >> MMU_RANGE_INV_VA_LSB_SHIFT); 6130 6131 WREG32(stlb_base + STLB_RANGE_INV_START_MSB_OFFSET, 6132 start >> MMU_RANGE_INV_VA_MSB_SHIFT); 6133 6134 WREG32(stlb_base + STLB_RANGE_INV_END_LSB_OFFSET, 6135 inv_params->end_va >> MMU_RANGE_INV_VA_LSB_SHIFT); 6136 6137 WREG32(stlb_base + STLB_RANGE_INV_END_MSB_OFFSET, 6138 inv_params->end_va >> MMU_RANGE_INV_VA_MSB_SHIFT); 6139 } else { 6140 start_offset = STLB_INV_ALL_START_OFFSET; 6141 } 6142 6143 gaudi2_mmu_send_invalidate_cache_cmd(hdev, stlb_base, start_offset, 6144 inv_params->inv_start_val, inv_params->flags); 6145} 6146 6147static inline void gaudi2_hmmu_invalidate_cache_trigger(struct hl_device *hdev, 6148 int dcore_id, int hmmu_id, 6149 struct gaudi2_cache_invld_params *inv_params) 6150{ 6151 u32 stlb_base = get_hmmu_stlb_base(dcore_id, hmmu_id); 6152 6153 gaudi2_mmu_invalidate_cache_trigger(hdev, stlb_base, inv_params); 6154} 6155 6156static inline int gaudi2_hmmu_invalidate_cache_status_poll(struct hl_device *hdev, 6157 int dcore_id, int hmmu_id, 6158 struct gaudi2_cache_invld_params *inv_params) 6159{ 6160 u32 stlb_base = get_hmmu_stlb_base(dcore_id, hmmu_id); 6161 6162 return gaudi2_mmu_invalidate_cache_status_poll(hdev, stlb_base, inv_params); 6163} 6164 6165static int gaudi2_hmmus_invalidate_cache(struct hl_device *hdev, 6166 struct gaudi2_cache_invld_params *inv_params) 6167{ 6168 int dcore_id, hmmu_id; 6169 6170 /* first send all invalidation commands */ 6171 for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { 6172 for (hmmu_id = 0 ; hmmu_id < NUM_OF_HMMU_PER_DCORE ; hmmu_id++) { 6173 if (!gaudi2_is_hmmu_enabled(hdev, dcore_id, hmmu_id)) 6174 continue; 6175 6176 gaudi2_hmmu_invalidate_cache_trigger(hdev, dcore_id, hmmu_id, inv_params); 6177 } 6178 } 6179 6180 /* next, poll all invalidations status */ 6181 for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { 6182 for (hmmu_id = 0 ; hmmu_id < NUM_OF_HMMU_PER_DCORE ; hmmu_id++) { 6183 int rc; 6184 6185 if (!gaudi2_is_hmmu_enabled(hdev, dcore_id, hmmu_id)) 6186 continue; 6187 6188 rc = gaudi2_hmmu_invalidate_cache_status_poll(hdev, dcore_id, hmmu_id, 6189 inv_params); 6190 if (rc) 6191 return rc; 6192 } 6193 } 6194 6195 return 0; 6196} 6197 6198static int gaudi2_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, u32 flags) 6199{ 6200 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6201 struct gaudi2_cache_invld_params invld_params; 6202 int rc = 0; 6203 6204 if (hdev->reset_info.hard_reset_pending) 6205 return rc; 6206 6207 invld_params.range_invalidation = false; 6208 invld_params.inv_start_val = 1; 6209 6210 if ((flags & MMU_OP_USERPTR) && (gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { 6211 invld_params.flags = flags; 6212 gaudi2_mmu_invalidate_cache_trigger(hdev, mmPMMU_HBW_STLB_BASE, &invld_params); 6213 rc = gaudi2_mmu_invalidate_cache_status_poll(hdev, mmPMMU_HBW_STLB_BASE, 6214 &invld_params); 6215 } else if (flags & MMU_OP_PHYS_PACK) { 6216 invld_params.flags = 0; 6217 rc = gaudi2_hmmus_invalidate_cache(hdev, &invld_params); 6218 } 6219 6220 return rc; 6221} 6222 6223static int gaudi2_mmu_invalidate_cache_range(struct hl_device *hdev, bool is_hard, 6224 u32 flags, u32 asid, u64 va, u64 size) 6225{ 6226 struct gaudi2_cache_invld_params invld_params = {0}; 6227 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6228 u64 start_va, end_va; 6229 u32 inv_start_val; 6230 int rc = 0; 6231 6232 if (hdev->reset_info.hard_reset_pending) 6233 return 0; 6234 6235 inv_start_val = (1 << MMU_RANGE_INV_EN_SHIFT | 6236 1 << MMU_RANGE_INV_ASID_EN_SHIFT | 6237 asid << MMU_RANGE_INV_ASID_SHIFT); 6238 start_va = va; 6239 end_va = start_va + size; 6240 6241 if ((flags & MMU_OP_USERPTR) && (gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { 6242 /* As range invalidation does not support zero address we will 6243 * do full invalidation in this case 6244 */ 6245 if (start_va) { 6246 invld_params.range_invalidation = true; 6247 invld_params.start_va = start_va; 6248 invld_params.end_va = end_va; 6249 invld_params.inv_start_val = inv_start_val; 6250 invld_params.flags = flags | MMU_OP_CLEAR_MEMCACHE; 6251 } else { 6252 invld_params.range_invalidation = false; 6253 invld_params.inv_start_val = 1; 6254 invld_params.flags = flags; 6255 } 6256 6257 6258 gaudi2_mmu_invalidate_cache_trigger(hdev, mmPMMU_HBW_STLB_BASE, &invld_params); 6259 rc = gaudi2_mmu_invalidate_cache_status_poll(hdev, mmPMMU_HBW_STLB_BASE, 6260 &invld_params); 6261 if (rc) 6262 return rc; 6263 6264 } else if (flags & MMU_OP_PHYS_PACK) { 6265 invld_params.start_va = gaudi2_mmu_scramble_addr(hdev, start_va); 6266 invld_params.end_va = gaudi2_mmu_scramble_addr(hdev, end_va); 6267 invld_params.inv_start_val = inv_start_val; 6268 invld_params.flags = flags; 6269 rc = gaudi2_hmmus_invalidate_cache(hdev, &invld_params); 6270 } 6271 6272 return rc; 6273} 6274 6275static int gaudi2_mmu_update_hop0_addr(struct hl_device *hdev, u32 stlb_base, 6276 bool host_resident_pgt) 6277{ 6278 struct asic_fixed_properties *prop = &hdev->asic_prop; 6279 u64 hop0_addr; 6280 u32 asid, max_asid = prop->max_asid; 6281 int rc; 6282 6283 /* it takes too much time to init all of the ASIDs on palladium */ 6284 if (hdev->pldm) 6285 max_asid = min((u32) 8, max_asid); 6286 6287 for (asid = 0 ; asid < max_asid ; asid++) { 6288 if (host_resident_pgt) 6289 hop0_addr = hdev->mmu_priv.hr.mmu_asid_hop0[asid].phys_addr; 6290 else 6291 hop0_addr = prop->mmu_pgt_addr + (asid * prop->dmmu.hop_table_size); 6292 6293 rc = gaudi2_mmu_update_asid_hop0_addr(hdev, stlb_base, asid, hop0_addr); 6294 if (rc) { 6295 dev_err(hdev->dev, "failed to set hop0 addr for asid %d\n", asid); 6296 return rc; 6297 } 6298 } 6299 6300 return 0; 6301} 6302 6303static int gaudi2_mmu_init_common(struct hl_device *hdev, u32 mmu_base, u32 stlb_base, 6304 bool host_resident_pgt) 6305{ 6306 u32 status, timeout_usec; 6307 int rc; 6308 6309 if (hdev->pldm || !hdev->pdev) 6310 timeout_usec = GAUDI2_PLDM_MMU_TIMEOUT_USEC; 6311 else 6312 timeout_usec = GAUDI2_MMU_CACHE_INV_TIMEOUT_USEC; 6313 6314 WREG32(stlb_base + STLB_INV_ALL_START_OFFSET, 1); 6315 6316 rc = hl_poll_timeout( 6317 hdev, 6318 stlb_base + STLB_SRAM_INIT_OFFSET, 6319 status, 6320 !status, 6321 1000, 6322 timeout_usec); 6323 6324 if (rc) 6325 dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU SRAM init\n"); 6326 6327 rc = gaudi2_mmu_update_hop0_addr(hdev, stlb_base, host_resident_pgt); 6328 if (rc) 6329 return rc; 6330 6331 WREG32(mmu_base + MMU_BYPASS_OFFSET, 0); 6332 6333 rc = hl_poll_timeout( 6334 hdev, 6335 stlb_base + STLB_INV_ALL_START_OFFSET, 6336 status, 6337 !status, 6338 1000, 6339 timeout_usec); 6340 6341 if (rc) 6342 dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU invalidate all\n"); 6343 6344 WREG32(mmu_base + MMU_ENABLE_OFFSET, 1); 6345 6346 return rc; 6347} 6348 6349static int gaudi2_pci_mmu_init(struct hl_device *hdev) 6350{ 6351 struct asic_fixed_properties *prop = &hdev->asic_prop; 6352 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6353 u32 mmu_base, stlb_base; 6354 int rc; 6355 6356 if (gaudi2->hw_cap_initialized & HW_CAP_PMMU) 6357 return 0; 6358 6359 mmu_base = mmPMMU_HBW_MMU_BASE; 6360 stlb_base = mmPMMU_HBW_STLB_BASE; 6361 6362 RMWREG32_SHIFTED(stlb_base + STLB_HOP_CONFIGURATION_OFFSET, 6363 (0 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_HOP_SHIFT) | 6364 (5 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_SHIFT) | 6365 (4 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_SHIFT) | 6366 (5 << PMMU_HBW_STLB_HOP_CONFIGURATION_LAST_HOP_SHIFT) | 6367 (5 << PMMU_HBW_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_SHIFT), 6368 PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_HOP_MASK | 6369 PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_MASK | 6370 PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_MASK | 6371 PMMU_HBW_STLB_HOP_CONFIGURATION_LAST_HOP_MASK | 6372 PMMU_HBW_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_MASK); 6373 6374 WREG32(stlb_base + STLB_LL_LOOKUP_MASK_63_32_OFFSET, 0); 6375 6376 if (PAGE_SIZE == SZ_64K) { 6377 /* Set page sizes to 64K on hop5 and 16M on hop4 + enable 8 bit hops */ 6378 RMWREG32_SHIFTED(mmu_base + MMU_STATIC_MULTI_PAGE_SIZE_OFFSET, 6379 FIELD_PREP(DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP5_PAGE_SIZE_MASK, 4) | 6380 FIELD_PREP(DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP4_PAGE_SIZE_MASK, 3) | 6381 FIELD_PREP( 6382 DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_CFG_8_BITS_HOP_MODE_EN_MASK, 6383 1), 6384 DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP5_PAGE_SIZE_MASK | 6385 DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP4_PAGE_SIZE_MASK | 6386 DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_CFG_8_BITS_HOP_MODE_EN_MASK); 6387 } 6388 6389 WREG32(mmu_base + MMU_SPI_SEI_MASK_OFFSET, GAUDI2_PMMU_SPI_SEI_ENABLE_MASK); 6390 6391 rc = gaudi2_mmu_init_common(hdev, mmu_base, stlb_base, prop->pmmu.host_resident); 6392 if (rc) 6393 return rc; 6394 6395 gaudi2->hw_cap_initialized |= HW_CAP_PMMU; 6396 6397 return 0; 6398} 6399 6400static int gaudi2_dcore_hmmu_init(struct hl_device *hdev, int dcore_id, 6401 int hmmu_id) 6402{ 6403 struct asic_fixed_properties *prop = &hdev->asic_prop; 6404 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6405 u32 offset, mmu_base, stlb_base, hw_cap; 6406 u8 dmmu_seq; 6407 int rc; 6408 6409 dmmu_seq = NUM_OF_HMMU_PER_DCORE * dcore_id + hmmu_id; 6410 hw_cap = HW_CAP_DCORE0_DMMU0 << dmmu_seq; 6411 6412 /* 6413 * return if DMMU is already initialized or if it's not out of 6414 * isolation (due to cluster binning) 6415 */ 6416 if ((gaudi2->hw_cap_initialized & hw_cap) || !(prop->hmmu_hif_enabled_mask & BIT(dmmu_seq))) 6417 return 0; 6418 6419 offset = (u32) (dcore_id * DCORE_OFFSET + hmmu_id * DCORE_HMMU_OFFSET); 6420 mmu_base = mmDCORE0_HMMU0_MMU_BASE + offset; 6421 stlb_base = mmDCORE0_HMMU0_STLB_BASE + offset; 6422 6423 RMWREG32(mmu_base + MMU_STATIC_MULTI_PAGE_SIZE_OFFSET, 5 /* 64MB */, 6424 MMU_STATIC_MULTI_PAGE_SIZE_HOP4_PAGE_SIZE_MASK); 6425 6426 RMWREG32_SHIFTED(stlb_base + STLB_HOP_CONFIGURATION_OFFSET, 6427 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_HOP_MASK, 0) | 6428 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_MASK, 3) | 6429 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_MASK, 3) | 6430 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_LAST_HOP_MASK, 3) | 6431 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_MASK, 3), 6432 DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_HOP_MASK | 6433 DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_MASK | 6434 DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_MASK | 6435 DCORE0_HMMU0_STLB_HOP_CONFIGURATION_LAST_HOP_MASK | 6436 DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_MASK); 6437 6438 RMWREG32(stlb_base + STLB_HOP_CONFIGURATION_OFFSET, 1, 6439 STLB_HOP_CONFIGURATION_ONLY_LARGE_PAGE_MASK); 6440 6441 WREG32(mmu_base + MMU_SPI_SEI_MASK_OFFSET, GAUDI2_HMMU_SPI_SEI_ENABLE_MASK); 6442 6443 rc = gaudi2_mmu_init_common(hdev, mmu_base, stlb_base, prop->dmmu.host_resident); 6444 if (rc) 6445 return rc; 6446 6447 gaudi2->hw_cap_initialized |= hw_cap; 6448 6449 return 0; 6450} 6451 6452static int gaudi2_hbm_mmu_init(struct hl_device *hdev) 6453{ 6454 int rc, dcore_id, hmmu_id; 6455 6456 for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) 6457 for (hmmu_id = 0 ; hmmu_id < NUM_OF_HMMU_PER_DCORE; hmmu_id++) { 6458 rc = gaudi2_dcore_hmmu_init(hdev, dcore_id, hmmu_id); 6459 if (rc) 6460 return rc; 6461 } 6462 6463 return 0; 6464} 6465 6466static int gaudi2_mmu_init(struct hl_device *hdev) 6467{ 6468 int rc; 6469 6470 rc = gaudi2_pci_mmu_init(hdev); 6471 if (rc) 6472 return rc; 6473 6474 rc = gaudi2_hbm_mmu_init(hdev); 6475 if (rc) 6476 return rc; 6477 6478 return 0; 6479} 6480 6481static int gaudi2_hw_init(struct hl_device *hdev) 6482{ 6483 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6484 int rc; 6485 6486 /* Let's mark in the H/W that we have reached this point. We check 6487 * this value in the reset_before_init function to understand whether 6488 * we need to reset the chip before doing H/W init. This register is 6489 * cleared by the H/W upon H/W reset 6490 */ 6491 WREG32(mmHW_STATE, HL_DEVICE_HW_STATE_DIRTY); 6492 6493 /* Perform read from the device to make sure device is up */ 6494 RREG32(mmHW_STATE); 6495 6496 /* If iATU is done by FW, the HBM bar ALWAYS points to DRAM_PHYS_BASE. 6497 * So we set it here and if anyone tries to move it later to 6498 * a different address, there will be an error 6499 */ 6500 if (hdev->asic_prop.iatu_done_by_fw) 6501 gaudi2->dram_bar_cur_addr = DRAM_PHYS_BASE; 6502 6503 /* 6504 * Before pushing u-boot/linux to device, need to set the hbm bar to 6505 * base address of dram 6506 */ 6507 if (gaudi2_set_hbm_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) { 6508 dev_err(hdev->dev, "failed to map HBM bar to DRAM base address\n"); 6509 return -EIO; 6510 } 6511 6512 rc = gaudi2_init_cpu(hdev); 6513 if (rc) { 6514 dev_err(hdev->dev, "failed to initialize CPU\n"); 6515 return rc; 6516 } 6517 6518 gaudi2_init_scrambler_hbm(hdev); 6519 gaudi2_init_kdma(hdev); 6520 6521 rc = gaudi2_init_cpu_queues(hdev, GAUDI2_CPU_TIMEOUT_USEC); 6522 if (rc) { 6523 dev_err(hdev->dev, "failed to initialize CPU H/W queues %d\n", rc); 6524 return rc; 6525 } 6526 6527 rc = gaudi2->cpucp_info_get(hdev); 6528 if (rc) { 6529 dev_err(hdev->dev, "Failed to get cpucp info\n"); 6530 return rc; 6531 } 6532 6533 rc = gaudi2_mmu_init(hdev); 6534 if (rc) 6535 return rc; 6536 6537 gaudi2_init_pdma(hdev); 6538 gaudi2_init_edma(hdev); 6539 gaudi2_init_sm(hdev); 6540 gaudi2_init_tpc(hdev); 6541 gaudi2_init_mme(hdev); 6542 gaudi2_init_rotator(hdev); 6543 gaudi2_init_dec(hdev); 6544 gaudi2_enable_timestamp(hdev); 6545 6546 rc = gaudi2_coresight_init(hdev); 6547 if (rc) 6548 goto disable_queues; 6549 6550 rc = gaudi2_enable_msix(hdev); 6551 if (rc) 6552 goto disable_queues; 6553 6554 /* Perform read from the device to flush all configuration */ 6555 RREG32(mmHW_STATE); 6556 6557 return 0; 6558 6559disable_queues: 6560 gaudi2_disable_dma_qmans(hdev); 6561 gaudi2_disable_mme_qmans(hdev); 6562 gaudi2_disable_tpc_qmans(hdev); 6563 gaudi2_disable_rot_qmans(hdev); 6564 gaudi2_disable_nic_qmans(hdev); 6565 6566 gaudi2_disable_timestamp(hdev); 6567 6568 return rc; 6569} 6570 6571/** 6572 * gaudi2_send_hard_reset_cmd - common function to handle reset 6573 * 6574 * @hdev: pointer to the habanalabs device structure 6575 * 6576 * This function handles the various possible scenarios for reset. 6577 * It considers if reset is handled by driver\FW and what FW components are loaded 6578 */ 6579static void gaudi2_send_hard_reset_cmd(struct hl_device *hdev) 6580{ 6581 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 6582 bool heartbeat_reset, preboot_only, cpu_initialized = false; 6583 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6584 u32 cpu_boot_status; 6585 6586 preboot_only = (hdev->fw_loader.fw_comp_loaded == FW_TYPE_PREBOOT_CPU); 6587 heartbeat_reset = (hdev->reset_info.curr_reset_cause == HL_RESET_CAUSE_HEARTBEAT); 6588 6589 /* 6590 * Handle corner case where failure was at cpu management app load, 6591 * and driver didn't detect any failure while loading the FW, 6592 * then at such scenario driver will send only HALT_MACHINE 6593 * and no one will respond to this request since FW already back to preboot 6594 * and it cannot handle such cmd. 6595 * In this case next time the management app loads it'll check on events register 6596 * which will still have the halt indication, and will reboot the device. 6597 * The solution is to let preboot clear all relevant registers before next boot 6598 * once driver send COMMS_RST_DEV. 6599 */ 6600 cpu_boot_status = RREG32(mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS); 6601 6602 if (gaudi2 && (gaudi2->hw_cap_initialized & HW_CAP_CPU) && 6603 (cpu_boot_status == CPU_BOOT_STATUS_SRAM_AVAIL)) 6604 cpu_initialized = true; 6605 6606 /* 6607 * when Linux/Bootfit exist this write to the SP can be interpreted in 2 ways: 6608 * 1. FW reset: FW initiate the reset sequence 6609 * 2. driver reset: FW will start HALT sequence (the preparations for the 6610 * reset but not the reset itself as it is not implemented 6611 * on their part) and LKD will wait to let FW complete the 6612 * sequence before issuing the reset 6613 */ 6614 if (!preboot_only && cpu_initialized) { 6615 WREG32(le32_to_cpu(dyn_regs->gic_host_halt_irq), 6616 gaudi2_irq_map_table[GAUDI2_EVENT_CPU_HALT_MACHINE].cpu_id); 6617 6618 msleep(GAUDI2_CPU_RESET_WAIT_MSEC); 6619 } 6620 6621 /* 6622 * When working with preboot (without Linux/Boot fit) we can 6623 * communicate only using the COMMS commands to issue halt/reset. 6624 * 6625 * For the case in which we are working with Linux/Bootfit this is a hail-mary 6626 * attempt to revive the card in the small chance that the f/w has 6627 * experienced a watchdog event, which caused it to return back to preboot. 6628 * In that case, triggering reset through GIC won't help. We need to 6629 * trigger the reset as if Linux wasn't loaded. 6630 * 6631 * We do it only if the reset cause was HB, because that would be the 6632 * indication of such an event. 6633 * 6634 * In case watchdog hasn't expired but we still got HB, then this won't 6635 * do any damage. 6636 */ 6637 6638 if (heartbeat_reset || preboot_only || !cpu_initialized) { 6639 if (hdev->asic_prop.hard_reset_done_by_fw) 6640 hl_fw_ask_hard_reset_without_linux(hdev); 6641 else 6642 hl_fw_ask_halt_machine_without_linux(hdev); 6643 } 6644} 6645 6646/** 6647 * gaudi2_execute_hard_reset - execute hard reset by driver/FW 6648 * 6649 * @hdev: pointer to the habanalabs device structure 6650 * 6651 * This function executes hard reset based on if driver/FW should do the reset 6652 */ 6653static void gaudi2_execute_hard_reset(struct hl_device *hdev) 6654{ 6655 if (hdev->asic_prop.hard_reset_done_by_fw) { 6656 gaudi2_send_hard_reset_cmd(hdev); 6657 return; 6658 } 6659 6660 /* Set device to handle FLR by H/W as we will put the device 6661 * CPU to halt mode 6662 */ 6663 WREG32(mmPCIE_AUX_FLR_CTRL, 6664 (PCIE_AUX_FLR_CTRL_HW_CTRL_MASK | PCIE_AUX_FLR_CTRL_INT_MASK_MASK)); 6665 6666 gaudi2_send_hard_reset_cmd(hdev); 6667 6668 WREG32(mmPSOC_RESET_CONF_SW_ALL_RST, 1); 6669} 6670 6671/** 6672 * gaudi2_execute_soft_reset - execute soft reset by driver/FW 6673 * 6674 * @hdev: pointer to the habanalabs device structure 6675 * @driver_performs_reset: true if driver should perform reset instead of f/w. 6676 * @poll_timeout_us: time to wait for response from f/w. 6677 * 6678 * This function executes soft reset based on if driver/FW should do the reset 6679 */ 6680static int gaudi2_execute_soft_reset(struct hl_device *hdev, bool driver_performs_reset, 6681 u32 poll_timeout_us) 6682{ 6683 if (!driver_performs_reset) 6684 return hl_fw_send_soft_reset(hdev); 6685 6686 /* Block access to engines, QMANs and SM during reset, these 6687 * RRs will be reconfigured after soft reset. 6688 * PCIE_MSIX is left unsecured to allow NIC packets processing during the reset. 6689 */ 6690 gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 1, 6691 mmDCORE0_TPC0_QM_DCCM_BASE, mmPCIE_MSIX_BASE); 6692 6693 gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 2, 6694 mmPCIE_MSIX_BASE + HL_BLOCK_SIZE, 6695 mmPCIE_VDEC1_MSTR_IF_RR_SHRD_HBW_BASE + HL_BLOCK_SIZE); 6696 6697 WREG32(mmPSOC_RESET_CONF_SOFT_RST, 1); 6698 return 0; 6699} 6700 6701static void gaudi2_poll_btm_indication(struct hl_device *hdev, u32 poll_timeout_us) 6702{ 6703 int i, rc = 0; 6704 u32 reg_val; 6705 6706 /* We poll the BTM done indication multiple times after reset due to 6707 * a HW errata 'GAUDI2_0300' 6708 */ 6709 for (i = 0 ; i < GAUDI2_RESET_POLL_CNT ; i++) 6710 rc = hl_poll_timeout( 6711 hdev, 6712 mmPSOC_GLOBAL_CONF_BTM_FSM, 6713 reg_val, 6714 reg_val == 0, 6715 1000, 6716 poll_timeout_us); 6717 6718 if (rc) 6719 dev_err(hdev->dev, "Timeout while waiting for device to reset 0x%x\n", reg_val); 6720} 6721 6722static int gaudi2_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset) 6723{ 6724 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6725 u32 poll_timeout_us, reset_sleep_ms; 6726 bool driver_performs_reset = false; 6727 int rc; 6728 6729 if (hdev->pldm) { 6730 reset_sleep_ms = hard_reset ? GAUDI2_PLDM_HRESET_TIMEOUT_MSEC : 6731 GAUDI2_PLDM_SRESET_TIMEOUT_MSEC; 6732 poll_timeout_us = GAUDI2_PLDM_RESET_POLL_TIMEOUT_USEC; 6733 } else { 6734 reset_sleep_ms = GAUDI2_RESET_TIMEOUT_MSEC; 6735 poll_timeout_us = GAUDI2_RESET_POLL_TIMEOUT_USEC; 6736 } 6737 6738 if (fw_reset) 6739 goto skip_reset; 6740 6741 gaudi2_reset_arcs(hdev); 6742 6743 if (hard_reset) { 6744 driver_performs_reset = !hdev->asic_prop.hard_reset_done_by_fw; 6745 gaudi2_execute_hard_reset(hdev); 6746 } else { 6747 /* 6748 * As we have to support also work with preboot only (which does not supports 6749 * soft reset) we have to make sure that security is disabled before letting driver 6750 * do the reset. user shall control the BFE flags to avoid asking soft reset in 6751 * secured device with preboot only. 6752 */ 6753 driver_performs_reset = (hdev->fw_components == FW_TYPE_PREBOOT_CPU && 6754 !hdev->asic_prop.fw_security_enabled); 6755 rc = gaudi2_execute_soft_reset(hdev, driver_performs_reset, poll_timeout_us); 6756 if (rc) 6757 return rc; 6758 } 6759 6760skip_reset: 6761 if (driver_performs_reset || hard_reset) { 6762 /* 6763 * Instead of waiting for BTM indication we should wait for preboot ready: 6764 * Consider the below scenario: 6765 * 1. FW update is being triggered 6766 * - setting the dirty bit 6767 * 2. hard reset will be triggered due to the dirty bit 6768 * 3. FW initiates the reset: 6769 * - dirty bit cleared 6770 * - BTM indication cleared 6771 * - preboot ready indication cleared 6772 * 4. during hard reset: 6773 * - BTM indication will be set 6774 * - BIST test performed and another reset triggered 6775 * 5. only after this reset the preboot will set the preboot ready 6776 * 6777 * when polling on BTM indication alone we can lose sync with FW while trying to 6778 * communicate with FW that is during reset. 6779 * to overcome this we will always wait to preboot ready indication 6780 */ 6781 6782 /* without this sleep reset will not work */ 6783 msleep(reset_sleep_ms); 6784 6785 if (hdev->fw_components & FW_TYPE_PREBOOT_CPU) 6786 hl_fw_wait_preboot_ready(hdev); 6787 else 6788 gaudi2_poll_btm_indication(hdev, poll_timeout_us); 6789 } 6790 6791 if (!gaudi2) 6792 return 0; 6793 6794 gaudi2->dec_hw_cap_initialized &= ~(HW_CAP_DEC_MASK); 6795 gaudi2->tpc_hw_cap_initialized &= ~(HW_CAP_TPC_MASK); 6796 6797 /* 6798 * Clear NIC capability mask in order for driver to re-configure 6799 * NIC QMANs. NIC ports will not be re-configured during soft 6800 * reset as we call gaudi2_nic_init only during hard reset 6801 */ 6802 gaudi2->nic_hw_cap_initialized &= ~(HW_CAP_NIC_MASK); 6803 6804 if (hard_reset) { 6805 gaudi2->hw_cap_initialized &= 6806 ~(HW_CAP_DRAM | HW_CAP_CLK_GATE | HW_CAP_HBM_SCRAMBLER_MASK | 6807 HW_CAP_PMMU | HW_CAP_CPU | HW_CAP_CPU_Q | 6808 HW_CAP_SRAM_SCRAMBLER | HW_CAP_DMMU_MASK | 6809 HW_CAP_PDMA_MASK | HW_CAP_EDMA_MASK | HW_CAP_KDMA | 6810 HW_CAP_MME_MASK | HW_CAP_ROT_MASK); 6811 6812 memset(gaudi2->events_stat, 0, sizeof(gaudi2->events_stat)); 6813 } else { 6814 gaudi2->hw_cap_initialized &= 6815 ~(HW_CAP_CLK_GATE | HW_CAP_HBM_SCRAMBLER_SW_RESET | 6816 HW_CAP_PDMA_MASK | HW_CAP_EDMA_MASK | HW_CAP_MME_MASK | 6817 HW_CAP_ROT_MASK); 6818 } 6819 return 0; 6820} 6821 6822static int gaudi2_suspend(struct hl_device *hdev) 6823{ 6824 return hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0); 6825} 6826 6827static int gaudi2_resume(struct hl_device *hdev) 6828{ 6829 return gaudi2_init_iatu(hdev); 6830} 6831 6832static int gaudi2_mmap(struct hl_device *hdev, struct vm_area_struct *vma, 6833 void *cpu_addr, dma_addr_t dma_addr, size_t size) 6834{ 6835 int rc; 6836 6837 vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | 6838 VM_DONTCOPY | VM_NORESERVE); 6839 6840#ifdef _HAS_DMA_MMAP_COHERENT 6841 /* 6842 * If dma_alloc_coherent() returns a vmalloc address, set VM_MIXEDMAP 6843 * so vm_insert_page() can handle it safely. Without this, the kernel 6844 * may BUG_ON due to VM_PFNMAP. 6845 */ 6846 if (is_vmalloc_addr(cpu_addr)) 6847 vm_flags_set(vma, VM_MIXEDMAP); 6848 6849 rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr, dma_addr, size); 6850 if (rc) 6851 dev_err(hdev->dev, "dma_mmap_coherent error %d", rc); 6852 6853#else 6854 6855 rc = remap_pfn_range(vma, vma->vm_start, 6856 virt_to_phys(cpu_addr) >> PAGE_SHIFT, 6857 size, vma->vm_page_prot); 6858 if (rc) 6859 dev_err(hdev->dev, "remap_pfn_range error %d", rc); 6860 6861#endif 6862 6863 return rc; 6864} 6865 6866static bool gaudi2_is_queue_enabled(struct hl_device *hdev, u32 hw_queue_id) 6867{ 6868 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6869 u64 hw_cap_mask = 0; 6870 u64 hw_tpc_cap_bit = 0; 6871 u64 hw_nic_cap_bit = 0; 6872 u64 hw_test_cap_bit = 0; 6873 6874 switch (hw_queue_id) { 6875 case GAUDI2_QUEUE_ID_PDMA_0_0: 6876 case GAUDI2_QUEUE_ID_PDMA_0_1: 6877 case GAUDI2_QUEUE_ID_PDMA_1_0: 6878 hw_cap_mask = HW_CAP_PDMA_MASK; 6879 break; 6880 case GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3: 6881 hw_test_cap_bit = HW_CAP_EDMA_SHIFT + 6882 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0) >> 2); 6883 break; 6884 case GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3: 6885 hw_test_cap_bit = HW_CAP_EDMA_SHIFT + NUM_OF_EDMA_PER_DCORE + 6886 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0) >> 2); 6887 break; 6888 case GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3: 6889 hw_test_cap_bit = HW_CAP_EDMA_SHIFT + 2 * NUM_OF_EDMA_PER_DCORE + 6890 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0) >> 2); 6891 break; 6892 case GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3: 6893 hw_test_cap_bit = HW_CAP_EDMA_SHIFT + 3 * NUM_OF_EDMA_PER_DCORE + 6894 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0) >> 2); 6895 break; 6896 6897 case GAUDI2_QUEUE_ID_DCORE0_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE0_MME_0_3: 6898 hw_test_cap_bit = HW_CAP_MME_SHIFT; 6899 break; 6900 6901 case GAUDI2_QUEUE_ID_DCORE1_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE1_MME_0_3: 6902 hw_test_cap_bit = HW_CAP_MME_SHIFT + 1; 6903 break; 6904 6905 case GAUDI2_QUEUE_ID_DCORE2_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE2_MME_0_3: 6906 hw_test_cap_bit = HW_CAP_MME_SHIFT + 2; 6907 break; 6908 6909 case GAUDI2_QUEUE_ID_DCORE3_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE3_MME_0_3: 6910 hw_test_cap_bit = HW_CAP_MME_SHIFT + 3; 6911 break; 6912 6913 case GAUDI2_QUEUE_ID_DCORE0_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE0_TPC_5_3: 6914 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + 6915 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_TPC_0_0) >> 2); 6916 6917 /* special case where cap bit refers to the first queue id */ 6918 if (!hw_tpc_cap_bit) 6919 return !!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(0)); 6920 break; 6921 6922 case GAUDI2_QUEUE_ID_DCORE1_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE1_TPC_5_3: 6923 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + NUM_OF_TPC_PER_DCORE + 6924 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE1_TPC_0_0) >> 2); 6925 break; 6926 6927 case GAUDI2_QUEUE_ID_DCORE2_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE2_TPC_5_3: 6928 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + (2 * NUM_OF_TPC_PER_DCORE) + 6929 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE2_TPC_0_0) >> 2); 6930 break; 6931 6932 case GAUDI2_QUEUE_ID_DCORE3_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE3_TPC_5_3: 6933 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + (3 * NUM_OF_TPC_PER_DCORE) + 6934 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE3_TPC_0_0) >> 2); 6935 break; 6936 6937 case GAUDI2_QUEUE_ID_DCORE0_TPC_6_0 ... GAUDI2_QUEUE_ID_DCORE0_TPC_6_3: 6938 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + (4 * NUM_OF_TPC_PER_DCORE); 6939 break; 6940 6941 case GAUDI2_QUEUE_ID_ROT_0_0 ... GAUDI2_QUEUE_ID_ROT_1_3: 6942 hw_test_cap_bit = HW_CAP_ROT_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_ROT_0_0) >> 2); 6943 break; 6944 6945 case GAUDI2_QUEUE_ID_NIC_0_0 ... GAUDI2_QUEUE_ID_NIC_23_3: 6946 hw_nic_cap_bit = HW_CAP_NIC_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_NIC_0_0) >> 2); 6947 6948 /* special case where cap bit refers to the first queue id */ 6949 if (!hw_nic_cap_bit) 6950 return !!(gaudi2->nic_hw_cap_initialized & BIT_ULL(0)); 6951 break; 6952 6953 case GAUDI2_QUEUE_ID_CPU_PQ: 6954 return !!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q); 6955 6956 default: 6957 return false; 6958 } 6959 6960 if (hw_tpc_cap_bit) 6961 return !!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(hw_tpc_cap_bit)); 6962 6963 if (hw_nic_cap_bit) 6964 return !!(gaudi2->nic_hw_cap_initialized & BIT_ULL(hw_nic_cap_bit)); 6965 6966 if (hw_test_cap_bit) 6967 hw_cap_mask = BIT_ULL(hw_test_cap_bit); 6968 6969 return !!(gaudi2->hw_cap_initialized & hw_cap_mask); 6970} 6971 6972static bool gaudi2_is_arc_enabled(struct hl_device *hdev, u64 arc_id) 6973{ 6974 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6975 6976 switch (arc_id) { 6977 case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC5: 6978 case CPU_ID_MME_QMAN_ARC0...CPU_ID_ROT_QMAN_ARC1: 6979 return !!(gaudi2->active_hw_arc & BIT_ULL(arc_id)); 6980 6981 case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24: 6982 return !!(gaudi2->active_tpc_arc & BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0)); 6983 6984 case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23: 6985 return !!(gaudi2->active_nic_arc & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0)); 6986 6987 default: 6988 return false; 6989 } 6990} 6991 6992static void gaudi2_clr_arc_id_cap(struct hl_device *hdev, u64 arc_id) 6993{ 6994 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6995 6996 switch (arc_id) { 6997 case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC5: 6998 case CPU_ID_MME_QMAN_ARC0...CPU_ID_ROT_QMAN_ARC1: 6999 gaudi2->active_hw_arc &= ~(BIT_ULL(arc_id)); 7000 break; 7001 7002 case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24: 7003 gaudi2->active_tpc_arc &= ~(BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0)); 7004 break; 7005 7006 case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23: 7007 gaudi2->active_nic_arc &= ~(BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0)); 7008 break; 7009 7010 default: 7011 return; 7012 } 7013} 7014 7015static void gaudi2_set_arc_id_cap(struct hl_device *hdev, u64 arc_id) 7016{ 7017 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7018 7019 switch (arc_id) { 7020 case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC5: 7021 case CPU_ID_MME_QMAN_ARC0...CPU_ID_ROT_QMAN_ARC1: 7022 gaudi2->active_hw_arc |= BIT_ULL(arc_id); 7023 break; 7024 7025 case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24: 7026 gaudi2->active_tpc_arc |= BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0); 7027 break; 7028 7029 case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23: 7030 gaudi2->active_nic_arc |= BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0); 7031 break; 7032 7033 default: 7034 return; 7035 } 7036} 7037 7038static void gaudi2_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) 7039{ 7040 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 7041 u32 pq_offset, reg_base, db_reg_offset, db_value; 7042 7043 if (hw_queue_id != GAUDI2_QUEUE_ID_CPU_PQ) { 7044 /* 7045 * QMAN has 4 successive PQ_PI registers, 1 for each of the QMAN PQs. 7046 * Masking the H/W queue ID with 0x3 extracts the QMAN internal PQ 7047 * number. 7048 */ 7049 pq_offset = (hw_queue_id & 0x3) * 4; 7050 reg_base = gaudi2_qm_blocks_bases[hw_queue_id]; 7051 db_reg_offset = reg_base + QM_PQ_PI_0_OFFSET + pq_offset; 7052 } else { 7053 db_reg_offset = mmCPU_IF_PF_PQ_PI; 7054 } 7055 7056 db_value = pi; 7057 7058 /* ring the doorbell */ 7059 WREG32(db_reg_offset, db_value); 7060 7061 if (hw_queue_id == GAUDI2_QUEUE_ID_CPU_PQ) { 7062 /* make sure device CPU will read latest data from host */ 7063 mb(); 7064 WREG32(le32_to_cpu(dyn_regs->gic_host_pi_upd_irq), 7065 gaudi2_irq_map_table[GAUDI2_EVENT_CPU_PI_UPDATE].cpu_id); 7066 } 7067} 7068 7069static void gaudi2_pqe_write(struct hl_device *hdev, __le64 *pqe, struct hl_bd *bd) 7070{ 7071 __le64 *pbd = (__le64 *) bd; 7072 7073 /* The QMANs are on the host memory so a simple copy suffice */ 7074 pqe[0] = pbd[0]; 7075 pqe[1] = pbd[1]; 7076} 7077 7078static void *gaudi2_dma_alloc_coherent(struct hl_device *hdev, size_t size, 7079 dma_addr_t *dma_handle, gfp_t flags) 7080{ 7081 return dma_alloc_coherent(&hdev->pdev->dev, size, dma_handle, flags); 7082} 7083 7084static void gaudi2_dma_free_coherent(struct hl_device *hdev, size_t size, 7085 void *cpu_addr, dma_addr_t dma_handle) 7086{ 7087 dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, dma_handle); 7088} 7089 7090static int gaudi2_send_cpu_message(struct hl_device *hdev, u32 *msg, u16 len, 7091 u32 timeout, u64 *result) 7092{ 7093 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7094 7095 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) { 7096 if (result) 7097 *result = 0; 7098 return 0; 7099 } 7100 7101 if (!timeout) 7102 timeout = GAUDI2_MSG_TO_CPU_TIMEOUT_USEC; 7103 7104 return hl_fw_send_cpu_message(hdev, GAUDI2_QUEUE_ID_CPU_PQ, msg, len, timeout, result); 7105} 7106 7107static void *gaudi2_dma_pool_zalloc(struct hl_device *hdev, size_t size, 7108 gfp_t mem_flags, dma_addr_t *dma_handle) 7109{ 7110 if (size > GAUDI2_DMA_POOL_BLK_SIZE) 7111 return NULL; 7112 7113 return dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle); 7114} 7115 7116static void gaudi2_dma_pool_free(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr) 7117{ 7118 dma_pool_free(hdev->dma_pool, vaddr, dma_addr); 7119} 7120 7121static void *gaudi2_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, 7122 dma_addr_t *dma_handle) 7123{ 7124 return hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle); 7125} 7126 7127static void gaudi2_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr) 7128{ 7129 hl_fw_cpu_accessible_dma_pool_free(hdev, size, vaddr); 7130} 7131 7132static int gaudi2_validate_cb_address(struct hl_device *hdev, struct hl_cs_parser *parser) 7133{ 7134 struct asic_fixed_properties *asic_prop = &hdev->asic_prop; 7135 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7136 7137 if (!gaudi2_is_queue_enabled(hdev, parser->hw_queue_id)) { 7138 dev_err(hdev->dev, "h/w queue %s is disabled\n", 7139 GAUDI2_QUEUE_ID_TO_STR(parser->hw_queue_id)); 7140 return -EINVAL; 7141 } 7142 7143 /* Just check if CB address is valid */ 7144 7145 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, 7146 parser->user_cb_size, 7147 asic_prop->sram_user_base_address, 7148 asic_prop->sram_end_address)) 7149 return 0; 7150 7151 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, 7152 parser->user_cb_size, 7153 asic_prop->dram_user_base_address, 7154 asic_prop->dram_end_address)) 7155 return 0; 7156 7157 if ((gaudi2->hw_cap_initialized & HW_CAP_DMMU_MASK) && 7158 hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, 7159 parser->user_cb_size, 7160 asic_prop->dmmu.start_addr, 7161 asic_prop->dmmu.end_addr)) 7162 return 0; 7163 7164 if (gaudi2->hw_cap_initialized & HW_CAP_PMMU) { 7165 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, 7166 parser->user_cb_size, 7167 asic_prop->pmmu.start_addr, 7168 asic_prop->pmmu.end_addr) || 7169 hl_mem_area_inside_range( 7170 (u64) (uintptr_t) parser->user_cb, 7171 parser->user_cb_size, 7172 asic_prop->pmmu_huge.start_addr, 7173 asic_prop->pmmu_huge.end_addr)) 7174 return 0; 7175 7176 } else if (gaudi2_host_phys_addr_valid((u64) (uintptr_t) parser->user_cb)) { 7177 if (!hdev->pdev) 7178 return 0; 7179 7180 if (!device_iommu_mapped(&hdev->pdev->dev)) 7181 return 0; 7182 } 7183 7184 dev_err(hdev->dev, "CB address %p + 0x%x for internal QMAN is not valid\n", 7185 parser->user_cb, parser->user_cb_size); 7186 7187 return -EFAULT; 7188} 7189 7190static int gaudi2_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser) 7191{ 7192 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7193 7194 if (!parser->is_kernel_allocated_cb) 7195 return gaudi2_validate_cb_address(hdev, parser); 7196 7197 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { 7198 dev_err(hdev->dev, "PMMU not initialized - Unsupported mode in Gaudi2\n"); 7199 return -EINVAL; 7200 } 7201 7202 return 0; 7203} 7204 7205static int gaudi2_send_heartbeat(struct hl_device *hdev) 7206{ 7207 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7208 7209 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 7210 return 0; 7211 7212 return hl_fw_send_heartbeat(hdev); 7213} 7214 7215/* This is an internal helper function, used to update the KDMA mmu props. 7216 * Should be called with a proper kdma lock. 7217 */ 7218static void gaudi2_kdma_set_mmbp_asid(struct hl_device *hdev, 7219 bool mmu_bypass, u32 asid) 7220{ 7221 u32 rw_asid, rw_mmu_bp; 7222 7223 rw_asid = (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_SHIFT) | 7224 (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_SHIFT); 7225 7226 rw_mmu_bp = (!!mmu_bypass << ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_RD_SHIFT) | 7227 (!!mmu_bypass << ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_WR_SHIFT); 7228 7229 WREG32(mmARC_FARM_KDMA_CTX_AXUSER_HB_ASID, rw_asid); 7230 WREG32(mmARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP, rw_mmu_bp); 7231} 7232 7233static void gaudi2_arm_cq_monitor(struct hl_device *hdev, u32 sob_id, u32 mon_id, u32 cq_id, 7234 u32 mon_payload, u32 sync_value) 7235{ 7236 u32 sob_offset, mon_offset, sync_group_id, mode, mon_arm; 7237 u8 mask; 7238 7239 sob_offset = sob_id * 4; 7240 mon_offset = mon_id * 4; 7241 7242 /* Reset the SOB value */ 7243 WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset, 0); 7244 7245 /* Configure this address with CQ_ID 0 because CQ_EN is set */ 7246 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, cq_id); 7247 7248 /* Configure this address with CS index because CQ_EN is set */ 7249 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, mon_payload); 7250 7251 sync_group_id = sob_id / 8; 7252 mask = ~(1 << (sob_id & 0x7)); 7253 mode = 1; /* comparison mode is "equal to" */ 7254 7255 mon_arm = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOD_MASK, sync_value); 7256 mon_arm |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOP_MASK, mode); 7257 mon_arm |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_MASK_MASK, mask); 7258 mon_arm |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SID_MASK, sync_group_id); 7259 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + mon_offset, mon_arm); 7260} 7261 7262/* This is an internal helper function used by gaudi2_send_job_to_kdma only */ 7263static int gaudi2_send_job_to_kdma(struct hl_device *hdev, 7264 u64 src_addr, u64 dst_addr, 7265 u32 size, bool is_memset) 7266{ 7267 u32 comp_val, commit_mask, *polling_addr, timeout, status = 0; 7268 struct hl_cq_entry *cq_base; 7269 struct hl_cq *cq; 7270 u64 comp_addr; 7271 int rc; 7272 7273 gaudi2_arm_cq_monitor(hdev, GAUDI2_RESERVED_SOB_KDMA_COMPLETION, 7274 GAUDI2_RESERVED_MON_KDMA_COMPLETION, 7275 GAUDI2_RESERVED_CQ_KDMA_COMPLETION, 1, 1); 7276 7277 comp_addr = CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + 7278 (GAUDI2_RESERVED_SOB_KDMA_COMPLETION * sizeof(u32)); 7279 7280 comp_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1) | 7281 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 1); 7282 7283 WREG32(mmARC_FARM_KDMA_CTX_SRC_BASE_LO, lower_32_bits(src_addr)); 7284 WREG32(mmARC_FARM_KDMA_CTX_SRC_BASE_HI, upper_32_bits(src_addr)); 7285 WREG32(mmARC_FARM_KDMA_CTX_DST_BASE_LO, lower_32_bits(dst_addr)); 7286 WREG32(mmARC_FARM_KDMA_CTX_DST_BASE_HI, upper_32_bits(dst_addr)); 7287 WREG32(mmARC_FARM_KDMA_CTX_WR_COMP_ADDR_LO, lower_32_bits(comp_addr)); 7288 WREG32(mmARC_FARM_KDMA_CTX_WR_COMP_ADDR_HI, upper_32_bits(comp_addr)); 7289 WREG32(mmARC_FARM_KDMA_CTX_WR_COMP_WDATA, comp_val); 7290 WREG32(mmARC_FARM_KDMA_CTX_DST_TSIZE_0, size); 7291 7292 commit_mask = FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_LIN_MASK, 1) | 7293 FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_WR_COMP_EN_MASK, 1); 7294 7295 if (is_memset) 7296 commit_mask |= FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_MEM_SET_MASK, 1); 7297 7298 WREG32(mmARC_FARM_KDMA_CTX_COMMIT, commit_mask); 7299 7300 /* Wait for completion */ 7301 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_KDMA_COMPLETION]; 7302 cq_base = cq->kernel_address; 7303 polling_addr = (u32 *)&cq_base[cq->ci]; 7304 7305 if (hdev->pldm) 7306 /* for each 1MB 20 second of timeout */ 7307 timeout = ((size / SZ_1M) + 1) * USEC_PER_SEC * 20; 7308 else 7309 timeout = KDMA_TIMEOUT_USEC; 7310 7311 /* Polling */ 7312 rc = hl_poll_timeout_memory( 7313 hdev, 7314 polling_addr, 7315 status, 7316 (status == 1), 7317 1000, 7318 timeout, 7319 true); 7320 7321 *polling_addr = 0; 7322 7323 if (rc) { 7324 dev_err(hdev->dev, "Timeout while waiting for KDMA to be idle\n"); 7325 WREG32(mmARC_FARM_KDMA_CFG_1, 1 << ARC_FARM_KDMA_CFG_1_HALT_SHIFT); 7326 return rc; 7327 } 7328 7329 cq->ci = hl_cq_inc_ptr(cq->ci); 7330 7331 return 0; 7332} 7333 7334static void gaudi2_memset_device_lbw(struct hl_device *hdev, u32 addr, u32 size, u32 val) 7335{ 7336 u32 i; 7337 7338 for (i = 0 ; i < size ; i += sizeof(u32)) 7339 WREG32(addr + i, val); 7340} 7341 7342static void gaudi2_qman_set_test_mode(struct hl_device *hdev, u32 hw_queue_id, bool enable) 7343{ 7344 u32 reg_base = gaudi2_qm_blocks_bases[hw_queue_id]; 7345 7346 if (enable) { 7347 WREG32(reg_base + QM_GLBL_PROT_OFFSET, QMAN_MAKE_TRUSTED_TEST_MODE); 7348 WREG32(reg_base + QM_PQC_CFG_OFFSET, 0); 7349 } else { 7350 WREG32(reg_base + QM_GLBL_PROT_OFFSET, QMAN_MAKE_TRUSTED); 7351 WREG32(reg_base + QM_PQC_CFG_OFFSET, 1 << PDMA0_QM_PQC_CFG_EN_SHIFT); 7352 } 7353} 7354 7355static inline u32 gaudi2_test_queue_hw_queue_id_to_sob_id(struct hl_device *hdev, u32 hw_queue_id) 7356{ 7357 return hdev->asic_prop.first_available_user_sob[0] + 7358 hw_queue_id - GAUDI2_QUEUE_ID_PDMA_0_0; 7359} 7360 7361static void gaudi2_test_queue_clear(struct hl_device *hdev, u32 hw_queue_id) 7362{ 7363 u32 sob_offset = gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4; 7364 u32 sob_addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset; 7365 7366 /* Reset the SOB value */ 7367 WREG32(sob_addr, 0); 7368} 7369 7370static int gaudi2_test_queue_send_msg_short(struct hl_device *hdev, u32 hw_queue_id, u32 sob_val, 7371 struct gaudi2_queues_test_info *msg_info) 7372{ 7373 u32 sob_offset = gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4; 7374 u32 tmp, sob_base = 1; 7375 struct packet_msg_short *msg_short_pkt = msg_info->kern_addr; 7376 size_t pkt_size = sizeof(struct packet_msg_short); 7377 int rc; 7378 7379 tmp = (PACKET_MSG_SHORT << GAUDI2_PKT_CTL_OPCODE_SHIFT) | 7380 (1 << GAUDI2_PKT_CTL_EB_SHIFT) | 7381 (1 << GAUDI2_PKT_CTL_MB_SHIFT) | 7382 (sob_base << GAUDI2_PKT_SHORT_CTL_BASE_SHIFT) | 7383 (sob_offset << GAUDI2_PKT_SHORT_CTL_ADDR_SHIFT); 7384 7385 msg_short_pkt->value = cpu_to_le32(sob_val); 7386 msg_short_pkt->ctl = cpu_to_le32(tmp); 7387 7388 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, pkt_size, msg_info->dma_addr); 7389 if (rc) 7390 dev_err(hdev->dev, 7391 "Failed to send msg_short packet to H/W queue %s\n", 7392 GAUDI2_QUEUE_ID_TO_STR(hw_queue_id)); 7393 7394 return rc; 7395} 7396 7397static int gaudi2_test_queue_wait_completion(struct hl_device *hdev, u32 hw_queue_id, u32 sob_val) 7398{ 7399 u32 sob_offset = gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4; 7400 u32 sob_addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset; 7401 u32 timeout_usec, tmp; 7402 int rc; 7403 7404 if (hdev->pldm) 7405 timeout_usec = GAUDI2_PLDM_TEST_QUEUE_WAIT_USEC; 7406 else 7407 timeout_usec = GAUDI2_TEST_QUEUE_WAIT_USEC; 7408 7409 rc = hl_poll_timeout( 7410 hdev, 7411 sob_addr, 7412 tmp, 7413 (tmp == sob_val), 7414 1000, 7415 timeout_usec); 7416 7417 if (rc == -ETIMEDOUT) { 7418 dev_err(hdev->dev, "H/W queue %s test failed (SOB_OBJ_0 == 0x%x)\n", 7419 GAUDI2_QUEUE_ID_TO_STR(hw_queue_id), tmp); 7420 rc = -EIO; 7421 } 7422 7423 return rc; 7424} 7425 7426static int gaudi2_test_cpu_queue(struct hl_device *hdev) 7427{ 7428 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7429 7430 /* 7431 * check capability here as send_cpu_message() won't update the result 7432 * value if no capability 7433 */ 7434 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 7435 return 0; 7436 7437 return hl_fw_test_cpu_queue(hdev); 7438} 7439 7440static int gaudi2_test_queues(struct hl_device *hdev) 7441{ 7442 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7443 struct gaudi2_queues_test_info *msg_info; 7444 u32 sob_val = 0x5a5a; 7445 int i, rc; 7446 7447 /* send test message on all enabled Qs */ 7448 for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ; i++) { 7449 if (!gaudi2_is_queue_enabled(hdev, i) || gaudi2_is_edma_queue_id(i)) 7450 continue; 7451 7452 msg_info = &gaudi2->queues_test_info[i - GAUDI2_QUEUE_ID_PDMA_0_0]; 7453 gaudi2_qman_set_test_mode(hdev, i, true); 7454 gaudi2_test_queue_clear(hdev, i); 7455 rc = gaudi2_test_queue_send_msg_short(hdev, i, sob_val, msg_info); 7456 if (rc) 7457 goto done; 7458 } 7459 7460 rc = gaudi2_test_cpu_queue(hdev); 7461 if (rc) 7462 goto done; 7463 7464 /* verify that all messages were processed */ 7465 for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ; i++) { 7466 if (!gaudi2_is_queue_enabled(hdev, i) || gaudi2_is_edma_queue_id(i)) 7467 continue; 7468 7469 rc = gaudi2_test_queue_wait_completion(hdev, i, sob_val); 7470 if (rc) 7471 /* chip is not usable, no need for cleanups, just bail-out with error */ 7472 goto done; 7473 7474 gaudi2_test_queue_clear(hdev, i); 7475 gaudi2_qman_set_test_mode(hdev, i, false); 7476 } 7477 7478done: 7479 return rc; 7480} 7481 7482static int gaudi2_compute_reset_late_init(struct hl_device *hdev) 7483{ 7484 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7485 size_t irq_arr_size; 7486 int rc; 7487 7488 gaudi2_init_arcs(hdev); 7489 7490 rc = gaudi2_scrub_arcs_dccm(hdev); 7491 if (rc) { 7492 dev_err(hdev->dev, "Failed to scrub arcs DCCM\n"); 7493 return rc; 7494 } 7495 7496 gaudi2_init_security(hdev); 7497 7498 /* Unmask all IRQs since some could have been received during the soft reset */ 7499 irq_arr_size = gaudi2->num_of_valid_hw_events * sizeof(gaudi2->hw_events[0]); 7500 return hl_fw_unmask_irq_arr(hdev, gaudi2->hw_events, irq_arr_size); 7501} 7502 7503static bool gaudi2_get_edma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7504 struct engines_data *e) 7505{ 7506 u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, dma_core_sts0, dma_core_sts1; 7507 struct asic_fixed_properties *prop = &hdev->asic_prop; 7508 unsigned long *mask = (unsigned long *) mask_arr; 7509 const char *edma_fmt = "%-6d%-6d%-9s%#-14x%#-15x%#x\n"; 7510 bool is_idle = true, is_eng_idle; 7511 int engine_idx, i, j; 7512 u64 offset; 7513 7514 if (e) 7515 hl_engine_data_sprintf(e, 7516 "\nCORE EDMA is_idle QM_GLBL_STS0 DMA_CORE_STS0 DMA_CORE_STS1\n" 7517 "---- ---- ------- ------------ ------------- -------------\n"); 7518 7519 for (i = 0; i < NUM_OF_DCORES; i++) { 7520 for (j = 0 ; j < NUM_OF_EDMA_PER_DCORE ; j++) { 7521 int seq = i * NUM_OF_EDMA_PER_DCORE + j; 7522 7523 if (!(prop->edma_enabled_mask & BIT(seq))) 7524 continue; 7525 7526 engine_idx = GAUDI2_DCORE0_ENGINE_ID_EDMA_0 + 7527 i * GAUDI2_ENGINE_ID_DCORE_OFFSET + j; 7528 offset = i * DCORE_OFFSET + j * DCORE_EDMA_OFFSET; 7529 7530 dma_core_sts0 = RREG32(mmDCORE0_EDMA0_CORE_STS0 + offset); 7531 dma_core_sts1 = RREG32(mmDCORE0_EDMA0_CORE_STS1 + offset); 7532 7533 qm_glbl_sts0 = RREG32(mmDCORE0_EDMA0_QM_GLBL_STS0 + offset); 7534 qm_glbl_sts1 = RREG32(mmDCORE0_EDMA0_QM_GLBL_STS1 + offset); 7535 qm_cgm_sts = RREG32(mmDCORE0_EDMA0_QM_CGM_STS + offset); 7536 7537 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) && 7538 IS_DMA_IDLE(dma_core_sts0) && !IS_DMA_HALTED(dma_core_sts1); 7539 is_idle &= is_eng_idle; 7540 7541 if (mask && !is_eng_idle) 7542 set_bit(engine_idx, mask); 7543 7544 if (e) 7545 hl_engine_data_sprintf(e, edma_fmt, i, j, is_eng_idle ? "Y" : "N", 7546 qm_glbl_sts0, dma_core_sts0, dma_core_sts1); 7547 } 7548 } 7549 7550 return is_idle; 7551} 7552 7553static bool gaudi2_get_pdma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7554 struct engines_data *e) 7555{ 7556 u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, dma_core_sts0, dma_core_sts1; 7557 unsigned long *mask = (unsigned long *) mask_arr; 7558 const char *pdma_fmt = "%-6d%-9s%#-14x%#-15x%#x\n"; 7559 bool is_idle = true, is_eng_idle; 7560 int engine_idx, i; 7561 u64 offset; 7562 7563 if (e) 7564 hl_engine_data_sprintf(e, 7565 "\nPDMA is_idle QM_GLBL_STS0 DMA_CORE_STS0 DMA_CORE_STS1\n" 7566 "---- ------- ------------ ------------- -------------\n"); 7567 7568 for (i = 0 ; i < NUM_OF_PDMA ; i++) { 7569 engine_idx = GAUDI2_ENGINE_ID_PDMA_0 + i; 7570 offset = i * PDMA_OFFSET; 7571 dma_core_sts0 = RREG32(mmPDMA0_CORE_STS0 + offset); 7572 dma_core_sts1 = RREG32(mmPDMA0_CORE_STS1 + offset); 7573 7574 qm_glbl_sts0 = RREG32(mmPDMA0_QM_GLBL_STS0 + offset); 7575 qm_glbl_sts1 = RREG32(mmPDMA0_QM_GLBL_STS1 + offset); 7576 qm_cgm_sts = RREG32(mmPDMA0_QM_CGM_STS + offset); 7577 7578 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) && 7579 IS_DMA_IDLE(dma_core_sts0) && !IS_DMA_HALTED(dma_core_sts1); 7580 is_idle &= is_eng_idle; 7581 7582 if (mask && !is_eng_idle) 7583 set_bit(engine_idx, mask); 7584 7585 if (e) 7586 hl_engine_data_sprintf(e, pdma_fmt, i, is_eng_idle ? "Y" : "N", 7587 qm_glbl_sts0, dma_core_sts0, dma_core_sts1); 7588 } 7589 7590 return is_idle; 7591} 7592 7593static bool gaudi2_get_nic_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7594 struct engines_data *e) 7595{ 7596 unsigned long *mask = (unsigned long *) mask_arr; 7597 const char *nic_fmt = "%-5d%-9s%#-14x%#-12x\n"; 7598 u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts; 7599 bool is_idle = true, is_eng_idle; 7600 int engine_idx, i; 7601 u64 offset = 0; 7602 7603 /* NIC, twelve macros in Full chip */ 7604 if (e && hdev->nic_ports_mask) 7605 hl_engine_data_sprintf(e, 7606 "\nNIC is_idle QM_GLBL_STS0 QM_CGM_STS\n" 7607 "--- ------- ------------ ----------\n"); 7608 7609 for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++) { 7610 if (!(i & 1)) 7611 offset = i / 2 * NIC_OFFSET; 7612 else 7613 offset += NIC_QM_OFFSET; 7614 7615 if (!(hdev->nic_ports_mask & BIT(i))) 7616 continue; 7617 7618 engine_idx = GAUDI2_ENGINE_ID_NIC0_0 + i; 7619 7620 7621 qm_glbl_sts0 = RREG32(mmNIC0_QM0_GLBL_STS0 + offset); 7622 qm_glbl_sts1 = RREG32(mmNIC0_QM0_GLBL_STS1 + offset); 7623 qm_cgm_sts = RREG32(mmNIC0_QM0_CGM_STS + offset); 7624 7625 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts); 7626 is_idle &= is_eng_idle; 7627 7628 if (mask && !is_eng_idle) 7629 set_bit(engine_idx, mask); 7630 7631 if (e) 7632 hl_engine_data_sprintf(e, nic_fmt, i, is_eng_idle ? "Y" : "N", 7633 qm_glbl_sts0, qm_cgm_sts); 7634 } 7635 7636 return is_idle; 7637} 7638 7639static bool gaudi2_get_mme_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7640 struct engines_data *e) 7641{ 7642 u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, mme_arch_sts; 7643 unsigned long *mask = (unsigned long *) mask_arr; 7644 const char *mme_fmt = "%-5d%-6s%-9s%#-14x%#x\n"; 7645 bool is_idle = true, is_eng_idle; 7646 int engine_idx, i; 7647 u64 offset; 7648 7649 if (e) 7650 hl_engine_data_sprintf(e, 7651 "\nMME Stub is_idle QM_GLBL_STS0 MME_ARCH_STATUS\n" 7652 "--- ---- ------- ------------ ---------------\n"); 7653 /* MME, one per Dcore */ 7654 for (i = 0 ; i < NUM_OF_DCORES ; i++) { 7655 engine_idx = GAUDI2_DCORE0_ENGINE_ID_MME + i * GAUDI2_ENGINE_ID_DCORE_OFFSET; 7656 offset = i * DCORE_OFFSET; 7657 7658 qm_glbl_sts0 = RREG32(mmDCORE0_MME_QM_GLBL_STS0 + offset); 7659 qm_glbl_sts1 = RREG32(mmDCORE0_MME_QM_GLBL_STS1 + offset); 7660 qm_cgm_sts = RREG32(mmDCORE0_MME_QM_CGM_STS + offset); 7661 7662 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts); 7663 is_idle &= is_eng_idle; 7664 7665 mme_arch_sts = RREG32(mmDCORE0_MME_CTRL_LO_ARCH_STATUS + offset); 7666 is_eng_idle &= IS_MME_IDLE(mme_arch_sts); 7667 is_idle &= is_eng_idle; 7668 7669 if (e) 7670 hl_engine_data_sprintf(e, mme_fmt, i, "N", 7671 is_eng_idle ? "Y" : "N", 7672 qm_glbl_sts0, 7673 mme_arch_sts); 7674 7675 if (mask && !is_eng_idle) 7676 set_bit(engine_idx, mask); 7677 } 7678 7679 return is_idle; 7680} 7681 7682static void gaudi2_is_tpc_engine_idle(struct hl_device *hdev, int dcore, int inst, u32 offset, 7683 struct iterate_module_ctx *ctx) 7684{ 7685 struct gaudi2_tpc_idle_data *idle_data = ctx->data; 7686 u32 tpc_cfg_sts, qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts; 7687 bool is_eng_idle; 7688 int engine_idx; 7689 7690 if ((dcore == 0) && (inst == (NUM_DCORE0_TPC - 1))) 7691 engine_idx = GAUDI2_DCORE0_ENGINE_ID_TPC_6; 7692 else 7693 engine_idx = GAUDI2_DCORE0_ENGINE_ID_TPC_0 + 7694 dcore * GAUDI2_ENGINE_ID_DCORE_OFFSET + inst; 7695 7696 tpc_cfg_sts = RREG32(mmDCORE0_TPC0_CFG_STATUS + offset); 7697 qm_glbl_sts0 = RREG32(mmDCORE0_TPC0_QM_GLBL_STS0 + offset); 7698 qm_glbl_sts1 = RREG32(mmDCORE0_TPC0_QM_GLBL_STS1 + offset); 7699 qm_cgm_sts = RREG32(mmDCORE0_TPC0_QM_CGM_STS + offset); 7700 7701 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) && 7702 IS_TPC_IDLE(tpc_cfg_sts); 7703 *(idle_data->is_idle) &= is_eng_idle; 7704 7705 if (idle_data->mask && !is_eng_idle) 7706 set_bit(engine_idx, idle_data->mask); 7707 7708 if (idle_data->e) 7709 hl_engine_data_sprintf(idle_data->e, 7710 idle_data->tpc_fmt, dcore, inst, 7711 is_eng_idle ? "Y" : "N", 7712 qm_glbl_sts0, qm_cgm_sts, tpc_cfg_sts); 7713} 7714 7715static bool gaudi2_get_tpc_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7716 struct engines_data *e) 7717{ 7718 struct asic_fixed_properties *prop = &hdev->asic_prop; 7719 unsigned long *mask = (unsigned long *) mask_arr; 7720 bool is_idle = true; 7721 7722 struct gaudi2_tpc_idle_data tpc_idle_data = { 7723 .tpc_fmt = "%-6d%-5d%-9s%#-14x%#-12x%#x\n", 7724 .e = e, 7725 .mask = mask, 7726 .is_idle = &is_idle, 7727 }; 7728 struct iterate_module_ctx tpc_iter = { 7729 .fn = &gaudi2_is_tpc_engine_idle, 7730 .data = &tpc_idle_data, 7731 }; 7732 7733 if (e && prop->tpc_enabled_mask) 7734 hl_engine_data_sprintf(e, 7735 "\nCORE TPC is_idle QM_GLBL_STS0 QM_CGM_STS STATUS\n" 7736 "---- --- ------- ------------ ---------- ------\n"); 7737 7738 gaudi2_iterate_tpcs(hdev, &tpc_iter); 7739 7740 return *tpc_idle_data.is_idle; 7741} 7742 7743static bool gaudi2_get_decoder_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7744 struct engines_data *e) 7745{ 7746 struct asic_fixed_properties *prop = &hdev->asic_prop; 7747 unsigned long *mask = (unsigned long *) mask_arr; 7748 const char *pcie_dec_fmt = "%-10d%-9s%#x\n"; 7749 const char *dec_fmt = "%-6d%-5d%-9s%#x\n"; 7750 bool is_idle = true, is_eng_idle; 7751 u32 dec_swreg15, dec_enabled_bit; 7752 int engine_idx, i, j; 7753 u64 offset; 7754 7755 /* Decoders, two each Dcore and two shared PCIe decoders */ 7756 if (e && (prop->decoder_enabled_mask & (~PCIE_DEC_EN_MASK))) 7757 hl_engine_data_sprintf(e, 7758 "\nCORE DEC is_idle VSI_CMD_SWREG15\n" 7759 "---- --- ------- ---------------\n"); 7760 7761 for (i = 0 ; i < NUM_OF_DCORES ; i++) { 7762 for (j = 0 ; j < NUM_OF_DEC_PER_DCORE ; j++) { 7763 dec_enabled_bit = 1 << (i * NUM_OF_DEC_PER_DCORE + j); 7764 if (!(prop->decoder_enabled_mask & dec_enabled_bit)) 7765 continue; 7766 7767 engine_idx = GAUDI2_DCORE0_ENGINE_ID_DEC_0 + 7768 i * GAUDI2_ENGINE_ID_DCORE_OFFSET + j; 7769 offset = i * DCORE_OFFSET + j * DCORE_DEC_OFFSET; 7770 7771 dec_swreg15 = RREG32(mmDCORE0_DEC0_CMD_SWREG15 + offset); 7772 is_eng_idle = IS_DEC_IDLE(dec_swreg15); 7773 is_idle &= is_eng_idle; 7774 7775 if (mask && !is_eng_idle) 7776 set_bit(engine_idx, mask); 7777 7778 if (e) 7779 hl_engine_data_sprintf(e, dec_fmt, i, j, 7780 is_eng_idle ? "Y" : "N", dec_swreg15); 7781 } 7782 } 7783 7784 if (e && (prop->decoder_enabled_mask & PCIE_DEC_EN_MASK)) 7785 hl_engine_data_sprintf(e, 7786 "\nPCIe DEC is_idle VSI_CMD_SWREG15\n" 7787 "-------- ------- ---------------\n"); 7788 7789 /* Check shared(PCIe) decoders */ 7790 for (i = 0 ; i < NUM_OF_DEC_PER_DCORE ; i++) { 7791 dec_enabled_bit = PCIE_DEC_SHIFT + i; 7792 if (!(prop->decoder_enabled_mask & BIT(dec_enabled_bit))) 7793 continue; 7794 7795 engine_idx = GAUDI2_PCIE_ENGINE_ID_DEC_0 + i; 7796 offset = i * DCORE_DEC_OFFSET; 7797 dec_swreg15 = RREG32(mmPCIE_DEC0_CMD_SWREG15 + offset); 7798 is_eng_idle = IS_DEC_IDLE(dec_swreg15); 7799 is_idle &= is_eng_idle; 7800 7801 if (mask && !is_eng_idle) 7802 set_bit(engine_idx, mask); 7803 7804 if (e) 7805 hl_engine_data_sprintf(e, pcie_dec_fmt, i, 7806 is_eng_idle ? "Y" : "N", dec_swreg15); 7807 } 7808 7809 return is_idle; 7810} 7811 7812static bool gaudi2_get_rotator_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7813 struct engines_data *e) 7814{ 7815 const char *rot_fmt = "%-6d%-5d%-9s%#-14x%#-14x%#x\n"; 7816 unsigned long *mask = (unsigned long *) mask_arr; 7817 u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts; 7818 bool is_idle = true, is_eng_idle; 7819 int engine_idx, i; 7820 u64 offset; 7821 7822 if (e) 7823 hl_engine_data_sprintf(e, 7824 "\nCORE ROT is_idle QM_GLBL_STS0 QM_GLBL_STS1 QM_CGM_STS\n" 7825 "---- --- ------- ------------ ------------ ----------\n"); 7826 7827 for (i = 0 ; i < NUM_OF_ROT ; i++) { 7828 engine_idx = GAUDI2_ENGINE_ID_ROT_0 + i; 7829 7830 offset = i * ROT_OFFSET; 7831 7832 qm_glbl_sts0 = RREG32(mmROT0_QM_GLBL_STS0 + offset); 7833 qm_glbl_sts1 = RREG32(mmROT0_QM_GLBL_STS1 + offset); 7834 qm_cgm_sts = RREG32(mmROT0_QM_CGM_STS + offset); 7835 7836 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts); 7837 is_idle &= is_eng_idle; 7838 7839 if (mask && !is_eng_idle) 7840 set_bit(engine_idx, mask); 7841 7842 if (e) 7843 hl_engine_data_sprintf(e, rot_fmt, i, 0, is_eng_idle ? "Y" : "N", 7844 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts); 7845 } 7846 7847 return is_idle; 7848} 7849 7850static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7851 struct engines_data *e) 7852{ 7853 bool is_idle = true; 7854 7855 is_idle &= gaudi2_get_edma_idle_status(hdev, mask_arr, mask_len, e); 7856 is_idle &= gaudi2_get_pdma_idle_status(hdev, mask_arr, mask_len, e); 7857 is_idle &= gaudi2_get_nic_idle_status(hdev, mask_arr, mask_len, e); 7858 is_idle &= gaudi2_get_mme_idle_status(hdev, mask_arr, mask_len, e); 7859 is_idle &= gaudi2_get_tpc_idle_status(hdev, mask_arr, mask_len, e); 7860 is_idle &= gaudi2_get_decoder_idle_status(hdev, mask_arr, mask_len, e); 7861 is_idle &= gaudi2_get_rotator_idle_status(hdev, mask_arr, mask_len, e); 7862 7863 return is_idle; 7864} 7865 7866static void gaudi2_hw_queues_lock(struct hl_device *hdev) 7867 __acquires(&gaudi2->hw_queues_lock) 7868{ 7869 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7870 7871 spin_lock(&gaudi2->hw_queues_lock); 7872} 7873 7874static void gaudi2_hw_queues_unlock(struct hl_device *hdev) 7875 __releases(&gaudi2->hw_queues_lock) 7876{ 7877 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7878 7879 spin_unlock(&gaudi2->hw_queues_lock); 7880} 7881 7882static u32 gaudi2_get_pci_id(struct hl_device *hdev) 7883{ 7884 return hdev->pdev->device; 7885} 7886 7887static int gaudi2_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size) 7888{ 7889 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7890 7891 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 7892 return 0; 7893 7894 return hl_fw_get_eeprom_data(hdev, data, max_size); 7895} 7896 7897static void gaudi2_update_eq_ci(struct hl_device *hdev, u32 val) 7898{ 7899 WREG32(mmCPU_IF_EQ_RD_OFFS, val); 7900} 7901 7902static void *gaudi2_get_events_stat(struct hl_device *hdev, bool aggregate, u32 *size) 7903{ 7904 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7905 7906 if (aggregate) { 7907 *size = (u32) sizeof(gaudi2->events_stat_aggregate); 7908 return gaudi2->events_stat_aggregate; 7909 } 7910 7911 *size = (u32) sizeof(gaudi2->events_stat); 7912 return gaudi2->events_stat; 7913} 7914 7915static void gaudi2_mmu_vdec_dcore_prepare(struct hl_device *hdev, int dcore_id, 7916 int dcore_vdec_id, u32 rw_asid, u32 rw_mmu_bp) 7917{ 7918 u32 offset = (mmDCORE0_VDEC1_BRDG_CTRL_BASE - mmDCORE0_VDEC0_BRDG_CTRL_BASE) * 7919 dcore_vdec_id + DCORE_OFFSET * dcore_id; 7920 7921 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_MMU_BP + offset, rw_mmu_bp); 7922 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_ASID + offset, rw_asid); 7923 7924 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_MMU_BP + offset, rw_mmu_bp); 7925 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_ASID + offset, rw_asid); 7926 7927 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_MMU_BP + offset, rw_mmu_bp); 7928 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_ASID + offset, rw_asid); 7929 7930 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_MMU_BP + offset, rw_mmu_bp); 7931 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_ASID + offset, rw_asid); 7932 7933 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_MMU_BP + offset, rw_mmu_bp); 7934 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_ASID + offset, rw_asid); 7935} 7936 7937static void gaudi2_mmu_dcore_prepare(struct hl_device *hdev, int dcore_id, u32 asid) 7938{ 7939 u32 rw_asid = (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_SHIFT) | 7940 (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_SHIFT); 7941 struct asic_fixed_properties *prop = &hdev->asic_prop; 7942 u32 dcore_offset = dcore_id * DCORE_OFFSET; 7943 u32 vdec_id, i, ports_offset, reg_val; 7944 u8 edma_seq_base; 7945 7946 /* EDMA */ 7947 edma_seq_base = dcore_id * NUM_OF_EDMA_PER_DCORE; 7948 if (prop->edma_enabled_mask & BIT(edma_seq_base)) { 7949 WREG32(mmDCORE0_EDMA0_QM_AXUSER_NONSECURED_HB_MMU_BP + dcore_offset, 0); 7950 WREG32(mmDCORE0_EDMA0_QM_AXUSER_NONSECURED_HB_ASID + dcore_offset, rw_asid); 7951 WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP + dcore_offset, 0); 7952 WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_ASID + dcore_offset, rw_asid); 7953 } 7954 7955 if (prop->edma_enabled_mask & BIT(edma_seq_base + 1)) { 7956 WREG32(mmDCORE0_EDMA1_QM_AXUSER_NONSECURED_HB_MMU_BP + dcore_offset, 0); 7957 WREG32(mmDCORE0_EDMA1_QM_AXUSER_NONSECURED_HB_ASID + dcore_offset, rw_asid); 7958 WREG32(mmDCORE0_EDMA1_CORE_CTX_AXUSER_HB_ASID + dcore_offset, rw_asid); 7959 WREG32(mmDCORE0_EDMA1_CORE_CTX_AXUSER_HB_MMU_BP + dcore_offset, 0); 7960 } 7961 7962 /* Sync Mngr */ 7963 WREG32(mmDCORE0_SYNC_MNGR_GLBL_ASID_NONE_SEC_PRIV + dcore_offset, asid); 7964 /* 7965 * Sync Mngrs on dcores 1 - 3 are exposed to user, so must use user ASID 7966 * for any access type 7967 */ 7968 if (dcore_id > 0) { 7969 reg_val = (asid << DCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_ASID_RD_SHIFT) | 7970 (asid << DCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_ASID_WR_SHIFT); 7971 WREG32(mmDCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_ASID + dcore_offset, reg_val); 7972 WREG32(mmDCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_MMU_BP + dcore_offset, 0); 7973 } 7974 7975 WREG32(mmDCORE0_MME_CTRL_LO_MME_AXUSER_HB_MMU_BP + dcore_offset, 0); 7976 WREG32(mmDCORE0_MME_CTRL_LO_MME_AXUSER_HB_ASID + dcore_offset, rw_asid); 7977 7978 for (i = 0 ; i < NUM_OF_MME_SBTE_PORTS ; i++) { 7979 ports_offset = i * DCORE_MME_SBTE_OFFSET; 7980 WREG32(mmDCORE0_MME_SBTE0_MSTR_IF_AXUSER_HB_MMU_BP + 7981 dcore_offset + ports_offset, 0); 7982 WREG32(mmDCORE0_MME_SBTE0_MSTR_IF_AXUSER_HB_ASID + 7983 dcore_offset + ports_offset, rw_asid); 7984 } 7985 7986 for (i = 0 ; i < NUM_OF_MME_WB_PORTS ; i++) { 7987 ports_offset = i * DCORE_MME_WB_OFFSET; 7988 WREG32(mmDCORE0_MME_WB0_MSTR_IF_AXUSER_HB_MMU_BP + 7989 dcore_offset + ports_offset, 0); 7990 WREG32(mmDCORE0_MME_WB0_MSTR_IF_AXUSER_HB_ASID + 7991 dcore_offset + ports_offset, rw_asid); 7992 } 7993 7994 WREG32(mmDCORE0_MME_QM_AXUSER_NONSECURED_HB_MMU_BP + dcore_offset, 0); 7995 WREG32(mmDCORE0_MME_QM_AXUSER_NONSECURED_HB_ASID + dcore_offset, rw_asid); 7996 7997 /* 7998 * Decoders 7999 */ 8000 for (vdec_id = 0 ; vdec_id < NUM_OF_DEC_PER_DCORE ; vdec_id++) { 8001 if (prop->decoder_enabled_mask & BIT(dcore_id * NUM_OF_DEC_PER_DCORE + vdec_id)) 8002 gaudi2_mmu_vdec_dcore_prepare(hdev, dcore_id, vdec_id, rw_asid, 0); 8003 } 8004} 8005 8006static void gudi2_mmu_vdec_shared_prepare(struct hl_device *hdev, 8007 int shared_vdec_id, u32 rw_asid, u32 rw_mmu_bp) 8008{ 8009 u32 offset = (mmPCIE_VDEC1_BRDG_CTRL_BASE - mmPCIE_VDEC0_BRDG_CTRL_BASE) * shared_vdec_id; 8010 8011 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_MMU_BP + offset, rw_mmu_bp); 8012 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_ASID + offset, rw_asid); 8013 8014 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_MMU_BP + offset, rw_mmu_bp); 8015 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_ASID + offset, rw_asid); 8016 8017 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_MMU_BP + offset, rw_mmu_bp); 8018 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_ASID + offset, rw_asid); 8019 8020 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_MMU_BP + offset, rw_mmu_bp); 8021 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_ASID + offset, rw_asid); 8022 8023 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_MMU_BP + offset, rw_mmu_bp); 8024 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_ASID + offset, rw_asid); 8025} 8026 8027static void gudi2_mmu_arc_farm_arc_dup_eng_prepare(struct hl_device *hdev, int arc_farm_id, 8028 u32 rw_asid, u32 rw_mmu_bp) 8029{ 8030 u32 offset = (mmARC_FARM_ARC1_DUP_ENG_BASE - mmARC_FARM_ARC0_DUP_ENG_BASE) * arc_farm_id; 8031 8032 WREG32(mmARC_FARM_ARC0_DUP_ENG_AXUSER_HB_MMU_BP + offset, rw_mmu_bp); 8033 WREG32(mmARC_FARM_ARC0_DUP_ENG_AXUSER_HB_ASID + offset, rw_asid); 8034} 8035 8036static void gaudi2_arc_mmu_prepare(struct hl_device *hdev, u32 cpu_id, u32 asid) 8037{ 8038 u32 reg_base, reg_offset, reg_val = 0; 8039 8040 reg_base = gaudi2_arc_blocks_bases[cpu_id]; 8041 8042 /* Enable MMU and configure asid for all relevant ARC regions */ 8043 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_ARC_REGION_CFG_MMU_BP_MASK, 0); 8044 reg_val |= FIELD_PREP(ARC_FARM_ARC0_AUX_ARC_REGION_CFG_0_ASID_MASK, asid); 8045 8046 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION3_GENERAL); 8047 WREG32(reg_base + reg_offset, reg_val); 8048 8049 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION4_HBM0_FW); 8050 WREG32(reg_base + reg_offset, reg_val); 8051 8052 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION5_HBM1_GC_DATA); 8053 WREG32(reg_base + reg_offset, reg_val); 8054 8055 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION6_HBM2_GC_DATA); 8056 WREG32(reg_base + reg_offset, reg_val); 8057 8058 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION7_HBM3_GC_DATA); 8059 WREG32(reg_base + reg_offset, reg_val); 8060 8061 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION9_PCIE); 8062 WREG32(reg_base + reg_offset, reg_val); 8063 8064 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION10_GENERAL); 8065 WREG32(reg_base + reg_offset, reg_val); 8066 8067 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION11_GENERAL); 8068 WREG32(reg_base + reg_offset, reg_val); 8069 8070 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION12_GENERAL); 8071 WREG32(reg_base + reg_offset, reg_val); 8072 8073 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION13_GENERAL); 8074 WREG32(reg_base + reg_offset, reg_val); 8075 8076 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION14_GENERAL); 8077 WREG32(reg_base + reg_offset, reg_val); 8078} 8079 8080static int gaudi2_arc_mmu_prepare_all(struct hl_device *hdev, u32 asid) 8081{ 8082 int i; 8083 8084 if (hdev->fw_components & FW_TYPE_BOOT_CPU) 8085 return hl_fw_cpucp_engine_core_asid_set(hdev, asid); 8086 8087 for (i = CPU_ID_SCHED_ARC0 ; i < NUM_OF_ARC_FARMS_ARC ; i++) 8088 gaudi2_arc_mmu_prepare(hdev, i, asid); 8089 8090 for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ ; i += 4) { 8091 if (!gaudi2_is_queue_enabled(hdev, i)) 8092 continue; 8093 8094 gaudi2_arc_mmu_prepare(hdev, gaudi2_queue_id_to_arc_id[i], asid); 8095 } 8096 8097 return 0; 8098} 8099 8100static int gaudi2_mmu_shared_prepare(struct hl_device *hdev, u32 asid) 8101{ 8102 struct asic_fixed_properties *prop = &hdev->asic_prop; 8103 u32 rw_asid, offset; 8104 int rc, i; 8105 8106 rw_asid = FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_MASK, asid) | 8107 FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_MASK, asid); 8108 8109 WREG32(mmPDMA0_QM_AXUSER_NONSECURED_HB_ASID, rw_asid); 8110 WREG32(mmPDMA0_QM_AXUSER_NONSECURED_HB_MMU_BP, 0); 8111 WREG32(mmPDMA0_CORE_CTX_AXUSER_HB_ASID, rw_asid); 8112 WREG32(mmPDMA0_CORE_CTX_AXUSER_HB_MMU_BP, 0); 8113 8114 WREG32(mmPDMA1_QM_AXUSER_NONSECURED_HB_ASID, rw_asid); 8115 WREG32(mmPDMA1_QM_AXUSER_NONSECURED_HB_MMU_BP, 0); 8116 WREG32(mmPDMA1_CORE_CTX_AXUSER_HB_ASID, rw_asid); 8117 WREG32(mmPDMA1_CORE_CTX_AXUSER_HB_MMU_BP, 0); 8118 8119 /* ROT */ 8120 for (i = 0 ; i < NUM_OF_ROT ; i++) { 8121 offset = i * ROT_OFFSET; 8122 WREG32(mmROT0_QM_AXUSER_NONSECURED_HB_ASID + offset, rw_asid); 8123 WREG32(mmROT0_QM_AXUSER_NONSECURED_HB_MMU_BP + offset, 0); 8124 RMWREG32(mmROT0_CPL_QUEUE_AWUSER + offset, asid, MMUBP_ASID_MASK); 8125 RMWREG32(mmROT0_DESC_HBW_ARUSER_LO + offset, asid, MMUBP_ASID_MASK); 8126 RMWREG32(mmROT0_DESC_HBW_AWUSER_LO + offset, asid, MMUBP_ASID_MASK); 8127 } 8128 8129 /* Shared Decoders are the last bits in the decoders mask */ 8130 if (prop->decoder_enabled_mask & BIT(NUM_OF_DCORES * NUM_OF_DEC_PER_DCORE + 0)) 8131 gudi2_mmu_vdec_shared_prepare(hdev, 0, rw_asid, 0); 8132 8133 if (prop->decoder_enabled_mask & BIT(NUM_OF_DCORES * NUM_OF_DEC_PER_DCORE + 1)) 8134 gudi2_mmu_vdec_shared_prepare(hdev, 1, rw_asid, 0); 8135 8136 /* arc farm arc dup eng */ 8137 for (i = 0 ; i < NUM_OF_ARC_FARMS_ARC ; i++) 8138 gudi2_mmu_arc_farm_arc_dup_eng_prepare(hdev, i, rw_asid, 0); 8139 8140 rc = gaudi2_arc_mmu_prepare_all(hdev, asid); 8141 if (rc) 8142 return rc; 8143 8144 return 0; 8145} 8146 8147static void gaudi2_tpc_mmu_prepare(struct hl_device *hdev, int dcore, int inst, u32 offset, 8148 struct iterate_module_ctx *ctx) 8149{ 8150 struct gaudi2_tpc_mmu_data *mmu_data = ctx->data; 8151 8152 WREG32(mmDCORE0_TPC0_CFG_AXUSER_HB_MMU_BP + offset, 0); 8153 WREG32(mmDCORE0_TPC0_CFG_AXUSER_HB_ASID + offset, mmu_data->rw_asid); 8154 WREG32(mmDCORE0_TPC0_QM_AXUSER_NONSECURED_HB_MMU_BP + offset, 0); 8155 WREG32(mmDCORE0_TPC0_QM_AXUSER_NONSECURED_HB_ASID + offset, mmu_data->rw_asid); 8156} 8157 8158/* zero the MMUBP and set the ASID */ 8159static int gaudi2_mmu_prepare(struct hl_device *hdev, u32 asid) 8160{ 8161 struct gaudi2_device *gaudi2 = hdev->asic_specific; 8162 struct gaudi2_tpc_mmu_data tpc_mmu_data; 8163 struct iterate_module_ctx tpc_iter = { 8164 .fn = &gaudi2_tpc_mmu_prepare, 8165 .data = &tpc_mmu_data, 8166 }; 8167 int rc, i; 8168 8169 if (asid & ~DCORE0_HMMU0_STLB_ASID_ASID_MASK) { 8170 dev_crit(hdev->dev, "asid %u is too big\n", asid); 8171 return -EINVAL; 8172 } 8173 8174 if (!(gaudi2->hw_cap_initialized & HW_CAP_MMU_MASK)) 8175 return 0; 8176 8177 rc = gaudi2_mmu_shared_prepare(hdev, asid); 8178 if (rc) 8179 return rc; 8180 8181 /* configure DCORE MMUs */ 8182 tpc_mmu_data.rw_asid = (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_SHIFT) | 8183 (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_SHIFT); 8184 gaudi2_iterate_tpcs(hdev, &tpc_iter); 8185 for (i = 0 ; i < NUM_OF_DCORES ; i++) 8186 gaudi2_mmu_dcore_prepare(hdev, i, asid); 8187 8188 return 0; 8189} 8190 8191static inline bool is_info_event(u32 event) 8192{ 8193 switch (event) { 8194 case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_CAUSE: 8195 case GAUDI2_EVENT_CPU_FIX_POWER_ENV_S ... GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_E: 8196 case GAUDI2_EVENT_ARC_PWR_BRK_ENTRY ... GAUDI2_EVENT_ARC_PWR_RD_MODE3: 8197 8198 /* return in case of NIC status event - these events are received periodically and not as 8199 * an indication to an error. 8200 */ 8201 case GAUDI2_EVENT_CPU0_STATUS_NIC0_ENG0 ... GAUDI2_EVENT_CPU11_STATUS_NIC11_ENG1: 8202 case GAUDI2_EVENT_ARC_EQ_HEARTBEAT: 8203 return true; 8204 default: 8205 return false; 8206 } 8207} 8208 8209static void gaudi2_print_event(struct hl_device *hdev, u16 event_type, 8210 bool ratelimited, const char *fmt, ...) 8211{ 8212 struct va_format vaf; 8213 va_list args; 8214 8215 va_start(args, fmt); 8216 vaf.fmt = fmt; 8217 vaf.va = &args; 8218 8219 if (ratelimited) 8220 dev_err_ratelimited(hdev->dev, "%s: %pV\n", 8221 gaudi2_irq_map_table[event_type].valid ? 8222 gaudi2_irq_map_table[event_type].name : "N/A Event", &vaf); 8223 else 8224 dev_err(hdev->dev, "%s: %pV\n", 8225 gaudi2_irq_map_table[event_type].valid ? 8226 gaudi2_irq_map_table[event_type].name : "N/A Event", &vaf); 8227 8228 va_end(args); 8229} 8230 8231static bool gaudi2_handle_ecc_event(struct hl_device *hdev, u16 event_type, 8232 struct hl_eq_ecc_data *ecc_data) 8233{ 8234 u64 ecc_address = 0, ecc_syndrome = 0; 8235 u8 memory_wrapper_idx = 0; 8236 bool has_block_id = false; 8237 u16 block_id; 8238 8239 if (hl_fw_version_cmp(hdev, 1, 12, 0) >= 0) 8240 has_block_id = true; 8241 8242 ecc_address = le64_to_cpu(ecc_data->ecc_address); 8243 ecc_syndrome = le64_to_cpu(ecc_data->ecc_syndrom); 8244 memory_wrapper_idx = ecc_data->memory_wrapper_idx; 8245 8246 if (has_block_id) { 8247 block_id = le16_to_cpu(ecc_data->block_id); 8248 gaudi2_print_event(hdev, event_type, !ecc_data->is_critical, 8249 "ECC error detected. address: %#llx. Syndrome: %#llx. wrapper id %u. block id %#x. critical %u.", 8250 ecc_address, ecc_syndrome, memory_wrapper_idx, block_id, 8251 ecc_data->is_critical); 8252 } else { 8253 gaudi2_print_event(hdev, event_type, !ecc_data->is_critical, 8254 "ECC error detected. address: %#llx. Syndrome: %#llx. wrapper id %u. critical %u.", 8255 ecc_address, ecc_syndrome, memory_wrapper_idx, ecc_data->is_critical); 8256 } 8257 8258 return !!ecc_data->is_critical; 8259} 8260 8261static void handle_lower_qman_data_on_err(struct hl_device *hdev, u64 qman_base, u32 engine_id) 8262{ 8263 struct undefined_opcode_info *undef_opcode = &hdev->captured_err_info.undef_opcode; 8264 u64 cq_ptr, cp_current_inst; 8265 u32 lo, hi, cq_size, cp_sts; 8266 bool is_arc_cq; 8267 8268 cp_sts = RREG32(qman_base + QM_CP_STS_4_OFFSET); 8269 is_arc_cq = FIELD_GET(PDMA0_QM_CP_STS_CUR_CQ_MASK, cp_sts); /* 0 - legacy CQ, 1 - ARC_CQ */ 8270 8271 if (is_arc_cq) { 8272 lo = RREG32(qman_base + QM_ARC_CQ_PTR_LO_STS_OFFSET); 8273 hi = RREG32(qman_base + QM_ARC_CQ_PTR_HI_STS_OFFSET); 8274 cq_ptr = ((u64) hi) << 32 | lo; 8275 cq_size = RREG32(qman_base + QM_ARC_CQ_TSIZE_STS_OFFSET); 8276 } else { 8277 lo = RREG32(qman_base + QM_CQ_PTR_LO_STS_4_OFFSET); 8278 hi = RREG32(qman_base + QM_CQ_PTR_HI_STS_4_OFFSET); 8279 cq_ptr = ((u64) hi) << 32 | lo; 8280 cq_size = RREG32(qman_base + QM_CQ_TSIZE_STS_4_OFFSET); 8281 } 8282 8283 lo = RREG32(qman_base + QM_CP_CURRENT_INST_LO_4_OFFSET); 8284 hi = RREG32(qman_base + QM_CP_CURRENT_INST_HI_4_OFFSET); 8285 cp_current_inst = ((u64) hi) << 32 | lo; 8286 8287 dev_info(hdev->dev, 8288 "LowerQM. %sCQ: {ptr %#llx, size %u}, CP: {instruction %#018llx}\n", 8289 is_arc_cq ? "ARC_" : "", cq_ptr, cq_size, cp_current_inst); 8290 8291 if (undef_opcode->write_enable) { 8292 memset(undef_opcode, 0, sizeof(*undef_opcode)); 8293 undef_opcode->timestamp = ktime_get(); 8294 undef_opcode->cq_addr = cq_ptr; 8295 undef_opcode->cq_size = cq_size; 8296 undef_opcode->engine_id = engine_id; 8297 undef_opcode->stream_id = QMAN_STREAMS; 8298 undef_opcode->write_enable = 0; 8299 } 8300} 8301 8302static int gaudi2_handle_qman_err_generic(struct hl_device *hdev, u16 event_type, 8303 u64 qman_base, u32 qid_base, u64 *event_mask) 8304{ 8305 u32 i, j, glbl_sts_val, arb_err_val, num_error_causes, error_count = 0; 8306 u64 glbl_sts_addr, arb_err_addr; 8307 char reg_desc[32]; 8308 8309 glbl_sts_addr = qman_base + (mmDCORE0_TPC0_QM_GLBL_ERR_STS_0 - mmDCORE0_TPC0_QM_BASE); 8310 arb_err_addr = qman_base + (mmDCORE0_TPC0_QM_ARB_ERR_CAUSE - mmDCORE0_TPC0_QM_BASE); 8311 8312 /* Iterate through all stream GLBL_ERR_STS registers + Lower CP */ 8313 for (i = 0 ; i < QMAN_STREAMS + 1 ; i++) { 8314 glbl_sts_val = RREG32(glbl_sts_addr + 4 * i); 8315 8316 if (!glbl_sts_val) 8317 continue; 8318 8319 if (i == QMAN_STREAMS) { 8320 snprintf(reg_desc, ARRAY_SIZE(reg_desc), "LowerQM"); 8321 num_error_causes = GAUDI2_NUM_OF_LOWER_QM_ERR_CAUSE; 8322 } else { 8323 snprintf(reg_desc, ARRAY_SIZE(reg_desc), "stream%u", i); 8324 num_error_causes = GAUDI2_NUM_OF_QM_ERR_CAUSE; 8325 } 8326 8327 for (j = 0 ; j < num_error_causes ; j++) 8328 if (glbl_sts_val & BIT(j)) { 8329 gaudi2_print_event(hdev, event_type, true, 8330 "%s. err cause: %s", reg_desc, 8331 i == QMAN_STREAMS ? 8332 gaudi2_lower_qman_error_cause[j] : 8333 gaudi2_qman_error_cause[j]); 8334 error_count++; 8335 } 8336 8337 /* Check for undefined opcode error in lower QM */ 8338 if ((i == QMAN_STREAMS) && 8339 (glbl_sts_val & PDMA0_QM_GLBL_ERR_STS_CP_UNDEF_CMD_ERR_MASK)) { 8340 handle_lower_qman_data_on_err(hdev, qman_base, 8341 gaudi2_queue_id_to_engine_id[qid_base]); 8342 *event_mask |= HL_NOTIFIER_EVENT_UNDEFINED_OPCODE; 8343 } 8344 } 8345 8346 arb_err_val = RREG32(arb_err_addr); 8347 8348 if (!arb_err_val) 8349 goto out; 8350 8351 for (j = 0 ; j < GAUDI2_NUM_OF_QM_ARB_ERR_CAUSE ; j++) { 8352 if (arb_err_val & BIT(j)) { 8353 gaudi2_print_event(hdev, event_type, true, 8354 "ARB_ERR. err cause: %s", 8355 gaudi2_qman_arb_error_cause[j]); 8356 error_count++; 8357 } 8358 } 8359 8360out: 8361 return error_count; 8362} 8363 8364static void gaudi2_razwi_rr_hbw_shared_printf_info(struct hl_device *hdev, 8365 u64 rtr_mstr_if_base_addr, bool is_write, char *name, 8366 enum gaudi2_engine_id id, u64 *event_mask) 8367{ 8368 u32 razwi_hi, razwi_lo, razwi_xy; 8369 u16 eng_id = id; 8370 u8 rd_wr_flag; 8371 8372 if (is_write) { 8373 razwi_hi = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HI); 8374 razwi_lo = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_LO); 8375 razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_XY); 8376 rd_wr_flag = HL_RAZWI_WRITE; 8377 } else { 8378 razwi_hi = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HI); 8379 razwi_lo = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_LO); 8380 razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_XY); 8381 rd_wr_flag = HL_RAZWI_READ; 8382 } 8383 8384 hl_handle_razwi(hdev, (u64)razwi_hi << 32 | razwi_lo, &eng_id, 1, 8385 rd_wr_flag | HL_RAZWI_HBW, event_mask); 8386 8387 dev_err_ratelimited(hdev->dev, 8388 "%s-RAZWI SHARED RR HBW %s error, address %#llx, Initiator coordinates 0x%x\n", 8389 name, is_write ? "WR" : "RD", (u64)razwi_hi << 32 | razwi_lo, razwi_xy); 8390} 8391 8392static void gaudi2_razwi_rr_lbw_shared_printf_info(struct hl_device *hdev, 8393 u64 rtr_mstr_if_base_addr, bool is_write, char *name, 8394 enum gaudi2_engine_id id, u64 *event_mask) 8395{ 8396 u64 razwi_addr = CFG_BASE; 8397 u32 razwi_xy; 8398 u16 eng_id = id; 8399 u8 rd_wr_flag; 8400 8401 if (is_write) { 8402 razwi_addr += RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI); 8403 razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_XY); 8404 rd_wr_flag = HL_RAZWI_WRITE; 8405 } else { 8406 razwi_addr += RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI); 8407 razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_XY); 8408 rd_wr_flag = HL_RAZWI_READ; 8409 } 8410 8411 hl_handle_razwi(hdev, razwi_addr, &eng_id, 1, rd_wr_flag | HL_RAZWI_LBW, event_mask); 8412 dev_err_ratelimited(hdev->dev, 8413 "%s-RAZWI SHARED RR LBW %s error, mstr_if 0x%llx, captured address 0x%llX Initiator coordinates 0x%x\n", 8414 name, is_write ? "WR" : "RD", rtr_mstr_if_base_addr, razwi_addr, 8415 razwi_xy); 8416} 8417 8418static enum gaudi2_engine_id gaudi2_razwi_calc_engine_id(struct hl_device *hdev, 8419 enum razwi_event_sources module, u8 module_idx) 8420{ 8421 switch (module) { 8422 case RAZWI_TPC: 8423 if (module_idx == (NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES)) 8424 return GAUDI2_DCORE0_ENGINE_ID_TPC_6; 8425 return (((module_idx / NUM_OF_TPC_PER_DCORE) * ENGINE_ID_DCORE_OFFSET) + 8426 (module_idx % NUM_OF_TPC_PER_DCORE) + 8427 (GAUDI2_DCORE0_ENGINE_ID_TPC_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)); 8428 8429 case RAZWI_MME: 8430 return ((GAUDI2_DCORE0_ENGINE_ID_MME - GAUDI2_DCORE0_ENGINE_ID_EDMA_0) + 8431 (module_idx * ENGINE_ID_DCORE_OFFSET)); 8432 8433 case RAZWI_EDMA: 8434 return (((module_idx / NUM_OF_EDMA_PER_DCORE) * ENGINE_ID_DCORE_OFFSET) + 8435 (module_idx % NUM_OF_EDMA_PER_DCORE)); 8436 8437 case RAZWI_PDMA: 8438 return (GAUDI2_ENGINE_ID_PDMA_0 + module_idx); 8439 8440 case RAZWI_NIC: 8441 return (GAUDI2_ENGINE_ID_NIC0_0 + (NIC_NUMBER_OF_QM_PER_MACRO * module_idx)); 8442 8443 case RAZWI_DEC: 8444 if (module_idx == 8) 8445 return GAUDI2_PCIE_ENGINE_ID_DEC_0; 8446 8447 if (module_idx == 9) 8448 return GAUDI2_PCIE_ENGINE_ID_DEC_1; 8449 ; 8450 return (((module_idx / NUM_OF_DEC_PER_DCORE) * ENGINE_ID_DCORE_OFFSET) + 8451 (module_idx % NUM_OF_DEC_PER_DCORE) + 8452 (GAUDI2_DCORE0_ENGINE_ID_DEC_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)); 8453 8454 case RAZWI_ROT: 8455 return GAUDI2_ENGINE_ID_ROT_0 + module_idx; 8456 8457 case RAZWI_ARC_FARM: 8458 return GAUDI2_ENGINE_ID_ARC_FARM; 8459 8460 default: 8461 return GAUDI2_ENGINE_ID_SIZE; 8462 } 8463} 8464 8465/* 8466 * This function handles RR(Range register) hit events. 8467 * raised be initiators not PSOC RAZWI. 8468 */ 8469static void gaudi2_ack_module_razwi_event_handler(struct hl_device *hdev, 8470 enum razwi_event_sources module, u8 module_idx, 8471 u8 module_sub_idx, u64 *event_mask) 8472{ 8473 bool via_sft = false; 8474 u32 hbw_rtr_id, lbw_rtr_id, dcore_id, dcore_rtr_id, eng_id, binned_idx; 8475 u64 hbw_rtr_mstr_if_base_addr, lbw_rtr_mstr_if_base_addr; 8476 u32 hbw_shrd_aw = 0, hbw_shrd_ar = 0; 8477 u32 lbw_shrd_aw = 0, lbw_shrd_ar = 0; 8478 char initiator_name[64]; 8479 8480 switch (module) { 8481 case RAZWI_TPC: 8482 sprintf(initiator_name, "TPC_%u", module_idx); 8483 if (hdev->tpc_binning) { 8484 binned_idx = __ffs(hdev->tpc_binning); 8485 if (binned_idx == module_idx) 8486 module_idx = TPC_ID_DCORE0_TPC6; 8487 } 8488 8489 hbw_rtr_id = gaudi2_tpc_initiator_hbw_rtr_id[module_idx]; 8490 lbw_rtr_id = gaudi2_tpc_initiator_lbw_rtr_id[module_idx]; 8491 break; 8492 case RAZWI_MME: 8493 sprintf(initiator_name, "MME_%u", module_idx); 8494 switch (module_sub_idx) { 8495 case MME_WAP0: 8496 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].wap0; 8497 break; 8498 case MME_WAP1: 8499 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].wap1; 8500 break; 8501 case MME_WRITE: 8502 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].write; 8503 break; 8504 case MME_READ: 8505 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].read; 8506 break; 8507 case MME_SBTE0: 8508 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte0; 8509 break; 8510 case MME_SBTE1: 8511 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte1; 8512 break; 8513 case MME_SBTE2: 8514 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte2; 8515 break; 8516 case MME_SBTE3: 8517 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte3; 8518 break; 8519 case MME_SBTE4: 8520 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte4; 8521 break; 8522 default: 8523 return; 8524 } 8525 lbw_rtr_id = hbw_rtr_id; 8526 break; 8527 case RAZWI_EDMA: 8528 hbw_rtr_mstr_if_base_addr = gaudi2_edma_initiator_hbw_sft[module_idx]; 8529 dcore_id = module_idx / NUM_OF_EDMA_PER_DCORE; 8530 /* SFT has separate MSTR_IF for LBW, only there we can 8531 * read the LBW razwi related registers 8532 */ 8533 lbw_rtr_mstr_if_base_addr = mmSFT0_LBW_RTR_IF_MSTR_IF_RR_SHRD_HBW_BASE + 8534 dcore_id * SFT_DCORE_OFFSET; 8535 via_sft = true; 8536 sprintf(initiator_name, "EDMA_%u", module_idx); 8537 break; 8538 case RAZWI_PDMA: 8539 hbw_rtr_id = gaudi2_pdma_initiator_hbw_rtr_id[module_idx]; 8540 lbw_rtr_id = gaudi2_pdma_initiator_lbw_rtr_id[module_idx]; 8541 sprintf(initiator_name, "PDMA_%u", module_idx); 8542 break; 8543 case RAZWI_NIC: 8544 hbw_rtr_id = gaudi2_nic_initiator_hbw_rtr_id[module_idx]; 8545 lbw_rtr_id = gaudi2_nic_initiator_lbw_rtr_id[module_idx]; 8546 sprintf(initiator_name, "NIC_%u", module_idx); 8547 break; 8548 case RAZWI_DEC: 8549 sprintf(initiator_name, "DEC_%u", module_idx); 8550 if (hdev->decoder_binning) { 8551 binned_idx = __ffs(hdev->decoder_binning); 8552 if (binned_idx == module_idx) 8553 module_idx = DEC_ID_PCIE_VDEC1; 8554 } 8555 hbw_rtr_id = gaudi2_dec_initiator_hbw_rtr_id[module_idx]; 8556 lbw_rtr_id = gaudi2_dec_initiator_lbw_rtr_id[module_idx]; 8557 break; 8558 case RAZWI_ROT: 8559 hbw_rtr_id = gaudi2_rot_initiator_hbw_rtr_id[module_idx]; 8560 lbw_rtr_id = gaudi2_rot_initiator_lbw_rtr_id[module_idx]; 8561 sprintf(initiator_name, "ROT_%u", module_idx); 8562 break; 8563 case RAZWI_ARC_FARM: 8564 lbw_rtr_id = DCORE1_RTR5; 8565 hbw_rtr_id = DCORE1_RTR7; 8566 sprintf(initiator_name, "ARC_FARM_%u", module_idx); 8567 break; 8568 default: 8569 return; 8570 } 8571 8572 /* Find router mstr_if register base */ 8573 if (!via_sft) { 8574 dcore_id = hbw_rtr_id / NUM_OF_RTR_PER_DCORE; 8575 dcore_rtr_id = hbw_rtr_id % NUM_OF_RTR_PER_DCORE; 8576 hbw_rtr_mstr_if_base_addr = mmDCORE0_RTR0_CTRL_BASE + 8577 dcore_id * DCORE_OFFSET + 8578 dcore_rtr_id * DCORE_RTR_OFFSET + 8579 RTR_MSTR_IF_OFFSET; 8580 lbw_rtr_mstr_if_base_addr = hbw_rtr_mstr_if_base_addr + 8581 (((s32)lbw_rtr_id - hbw_rtr_id) * DCORE_RTR_OFFSET); 8582 } 8583 8584 /* Find out event cause by reading "RAZWI_HAPPENED" registers */ 8585 hbw_shrd_aw = RREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HAPPENED); 8586 hbw_shrd_ar = RREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HAPPENED); 8587 lbw_shrd_aw = RREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_HAPPENED); 8588 lbw_shrd_ar = RREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_HAPPENED); 8589 8590 eng_id = gaudi2_razwi_calc_engine_id(hdev, module, module_idx); 8591 if (hbw_shrd_aw) { 8592 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, hbw_rtr_mstr_if_base_addr, true, 8593 initiator_name, eng_id, event_mask); 8594 8595 /* Clear event indication */ 8596 WREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HAPPENED, hbw_shrd_aw); 8597 } 8598 8599 if (hbw_shrd_ar) { 8600 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, hbw_rtr_mstr_if_base_addr, false, 8601 initiator_name, eng_id, event_mask); 8602 8603 /* Clear event indication */ 8604 WREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HAPPENED, hbw_shrd_ar); 8605 } 8606 8607 if (lbw_shrd_aw) { 8608 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, lbw_rtr_mstr_if_base_addr, true, 8609 initiator_name, eng_id, event_mask); 8610 8611 /* Clear event indication */ 8612 WREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_HAPPENED, lbw_shrd_aw); 8613 } 8614 8615 if (lbw_shrd_ar) { 8616 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, lbw_rtr_mstr_if_base_addr, false, 8617 initiator_name, eng_id, event_mask); 8618 8619 /* Clear event indication */ 8620 WREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_HAPPENED, lbw_shrd_ar); 8621 } 8622} 8623 8624static void gaudi2_check_if_razwi_happened(struct hl_device *hdev) 8625{ 8626 struct asic_fixed_properties *prop = &hdev->asic_prop; 8627 u8 mod_idx, sub_mod; 8628 8629 /* check all TPCs */ 8630 for (mod_idx = 0 ; mod_idx < (NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1) ; mod_idx++) { 8631 if (prop->tpc_enabled_mask & BIT(mod_idx)) 8632 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_TPC, mod_idx, 0, NULL); 8633 } 8634 8635 /* check all MMEs */ 8636 for (mod_idx = 0 ; mod_idx < (NUM_OF_MME_PER_DCORE * NUM_OF_DCORES) ; mod_idx++) 8637 for (sub_mod = MME_WAP0 ; sub_mod < MME_INITIATORS_MAX ; sub_mod++) 8638 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mod_idx, 8639 sub_mod, NULL); 8640 8641 /* check all EDMAs */ 8642 for (mod_idx = 0 ; mod_idx < (NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES) ; mod_idx++) 8643 if (prop->edma_enabled_mask & BIT(mod_idx)) 8644 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_EDMA, mod_idx, 0, NULL); 8645 8646 /* check all PDMAs */ 8647 for (mod_idx = 0 ; mod_idx < NUM_OF_PDMA ; mod_idx++) 8648 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_PDMA, mod_idx, 0, NULL); 8649 8650 /* check all NICs */ 8651 for (mod_idx = 0 ; mod_idx < NIC_NUMBER_OF_PORTS ; mod_idx++) 8652 if (hdev->nic_ports_mask & BIT(mod_idx)) 8653 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_NIC, mod_idx >> 1, 0, 8654 NULL); 8655 8656 /* check all DECs */ 8657 for (mod_idx = 0 ; mod_idx < NUMBER_OF_DEC ; mod_idx++) 8658 if (prop->decoder_enabled_mask & BIT(mod_idx)) 8659 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_DEC, mod_idx, 0, NULL); 8660 8661 /* check all ROTs */ 8662 for (mod_idx = 0 ; mod_idx < NUM_OF_ROT ; mod_idx++) 8663 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ROT, mod_idx, 0, NULL); 8664} 8665 8666static int gaudi2_psoc_razwi_get_engines(struct gaudi2_razwi_info *razwi_info, u32 array_size, 8667 u32 axuser_xy, u32 *base, u16 *eng_id, 8668 char *eng_name) 8669{ 8670 8671 int i, num_of_eng = 0; 8672 u16 str_size = 0; 8673 8674 for (i = 0 ; i < array_size ; i++) { 8675 if (axuser_xy != razwi_info[i].axuser_xy) 8676 continue; 8677 8678 eng_id[num_of_eng] = razwi_info[i].eng_id; 8679 base[num_of_eng] = razwi_info[i].rtr_ctrl; 8680 if (!num_of_eng) 8681 str_size += scnprintf(eng_name + str_size, 8682 PSOC_RAZWI_ENG_STR_SIZE - str_size, "%s", 8683 razwi_info[i].eng_name); 8684 else 8685 str_size += scnprintf(eng_name + str_size, 8686 PSOC_RAZWI_ENG_STR_SIZE - str_size, " or %s", 8687 razwi_info[i].eng_name); 8688 num_of_eng++; 8689 } 8690 8691 return num_of_eng; 8692} 8693 8694static bool gaudi2_handle_psoc_razwi_happened(struct hl_device *hdev, u32 razwi_reg, 8695 u64 *event_mask) 8696{ 8697 u32 axuser_xy = RAZWI_GET_AXUSER_XY(razwi_reg), addr_hi = 0, addr_lo = 0; 8698 u32 base[PSOC_RAZWI_MAX_ENG_PER_RTR]; 8699 u16 num_of_eng, eng_id[PSOC_RAZWI_MAX_ENG_PER_RTR]; 8700 char eng_name_str[PSOC_RAZWI_ENG_STR_SIZE]; 8701 bool razwi_happened = false; 8702 u64 addr; 8703 int i; 8704 8705 num_of_eng = gaudi2_psoc_razwi_get_engines(common_razwi_info, ARRAY_SIZE(common_razwi_info), 8706 axuser_xy, base, eng_id, eng_name_str); 8707 8708 /* If no match for XY coordinates, try to find it in MME razwi table */ 8709 if (!num_of_eng) { 8710 axuser_xy = RAZWI_GET_AXUSER_LOW_XY(razwi_reg); 8711 num_of_eng = gaudi2_psoc_razwi_get_engines(mme_razwi_info, 8712 ARRAY_SIZE(mme_razwi_info), 8713 axuser_xy, base, eng_id, 8714 eng_name_str); 8715 } 8716 8717 for (i = 0 ; i < num_of_eng ; i++) { 8718 if (RREG32(base[i] + DEC_RAZWI_HBW_AW_SET)) { 8719 addr_hi = RREG32(base[i] + DEC_RAZWI_HBW_AW_ADDR_HI); 8720 addr_lo = RREG32(base[i] + DEC_RAZWI_HBW_AW_ADDR_LO); 8721 addr = ((u64)addr_hi << 32) + addr_lo; 8722 if (addr) { 8723 dev_err(hdev->dev, 8724 "PSOC HBW AW RAZWI: %s, address (aligned to 128 byte): 0x%llX\n", 8725 eng_name_str, addr); 8726 hl_handle_razwi(hdev, addr, &eng_id[0], 8727 num_of_eng, HL_RAZWI_HBW | HL_RAZWI_WRITE, event_mask); 8728 razwi_happened = true; 8729 } 8730 } 8731 8732 if (RREG32(base[i] + DEC_RAZWI_HBW_AR_SET)) { 8733 addr_hi = RREG32(base[i] + DEC_RAZWI_HBW_AR_ADDR_HI); 8734 addr_lo = RREG32(base[i] + DEC_RAZWI_HBW_AR_ADDR_LO); 8735 addr = ((u64)addr_hi << 32) + addr_lo; 8736 if (addr) { 8737 dev_err(hdev->dev, 8738 "PSOC HBW AR RAZWI: %s, address (aligned to 128 byte): 0x%llX\n", 8739 eng_name_str, addr); 8740 hl_handle_razwi(hdev, addr, &eng_id[0], 8741 num_of_eng, HL_RAZWI_HBW | HL_RAZWI_READ, event_mask); 8742 razwi_happened = true; 8743 } 8744 } 8745 8746 if (RREG32(base[i] + DEC_RAZWI_LBW_AW_SET)) { 8747 addr_lo = RREG32(base[i] + DEC_RAZWI_LBW_AW_ADDR); 8748 if (addr_lo) { 8749 dev_err(hdev->dev, 8750 "PSOC LBW AW RAZWI: %s, address (aligned to 128 byte): 0x%X\n", 8751 eng_name_str, addr_lo); 8752 hl_handle_razwi(hdev, addr_lo, &eng_id[0], 8753 num_of_eng, HL_RAZWI_LBW | HL_RAZWI_WRITE, event_mask); 8754 razwi_happened = true; 8755 } 8756 } 8757 8758 if (RREG32(base[i] + DEC_RAZWI_LBW_AR_SET)) { 8759 addr_lo = RREG32(base[i] + DEC_RAZWI_LBW_AR_ADDR); 8760 if (addr_lo) { 8761 dev_err(hdev->dev, 8762 "PSOC LBW AR RAZWI: %s, address (aligned to 128 byte): 0x%X\n", 8763 eng_name_str, addr_lo); 8764 hl_handle_razwi(hdev, addr_lo, &eng_id[0], 8765 num_of_eng, HL_RAZWI_LBW | HL_RAZWI_READ, event_mask); 8766 razwi_happened = true; 8767 } 8768 } 8769 /* In common case the loop will break, when there is only one engine id, or 8770 * several engines with the same router. The exceptional case is with psoc razwi 8771 * from EDMA, where it's possible to get axuser id which fits 2 routers (2 8772 * interfaces of sft router). In this case, maybe the first router won't hold info 8773 * and we will need to iterate on the other router. 8774 */ 8775 if (razwi_happened) 8776 break; 8777 } 8778 8779 return razwi_happened; 8780} 8781 8782/* PSOC RAZWI interrupt occurs only when trying to access a bad address */ 8783static int gaudi2_ack_psoc_razwi_event_handler(struct hl_device *hdev, u64 *event_mask) 8784{ 8785 u32 razwi_mask_info, razwi_intr = 0, error_count = 0; 8786 8787 if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX)) { 8788 razwi_intr = RREG32(mmPSOC_GLOBAL_CONF_RAZWI_INTERRUPT); 8789 if (!razwi_intr) 8790 return 0; 8791 } 8792 8793 razwi_mask_info = RREG32(mmPSOC_GLOBAL_CONF_RAZWI_MASK_INFO); 8794 8795 dev_err_ratelimited(hdev->dev, 8796 "PSOC RAZWI interrupt: Mask %d, AR %d, AW %d, AXUSER_L 0x%x AXUSER_H 0x%x\n", 8797 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_MASK_MASK, razwi_mask_info), 8798 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_WAS_AR_MASK, razwi_mask_info), 8799 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_WAS_AW_MASK, razwi_mask_info), 8800 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_L_MASK, razwi_mask_info), 8801 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_H_MASK, razwi_mask_info)); 8802 8803 if (gaudi2_handle_psoc_razwi_happened(hdev, razwi_mask_info, event_mask)) 8804 error_count++; 8805 else 8806 dev_err_ratelimited(hdev->dev, 8807 "PSOC RAZWI interrupt: invalid razwi info (0x%x)\n", 8808 razwi_mask_info); 8809 8810 /* Clear Interrupts only on pldm or if f/w doesn't handle interrupts */ 8811 if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX)) 8812 WREG32(mmPSOC_GLOBAL_CONF_RAZWI_INTERRUPT, razwi_intr); 8813 8814 return error_count; 8815} 8816 8817static int _gaudi2_handle_qm_sei_err(struct hl_device *hdev, u64 qman_base, u16 event_type) 8818{ 8819 u32 i, sts_val, sts_clr_val = 0, error_count = 0; 8820 8821 sts_val = RREG32(qman_base + QM_SEI_STATUS_OFFSET); 8822 8823 for (i = 0 ; i < GAUDI2_NUM_OF_QM_SEI_ERR_CAUSE ; i++) { 8824 if (sts_val & BIT(i)) { 8825 gaudi2_print_event(hdev, event_type, true, 8826 "err cause: %s", gaudi2_qm_sei_error_cause[i]); 8827 sts_clr_val |= BIT(i); 8828 error_count++; 8829 } 8830 } 8831 8832 WREG32(qman_base + QM_SEI_STATUS_OFFSET, sts_clr_val); 8833 8834 return error_count; 8835} 8836 8837static int gaudi2_handle_qm_sei_err(struct hl_device *hdev, u16 event_type, 8838 bool extended_err_check, u64 *event_mask) 8839{ 8840 enum razwi_event_sources module; 8841 u32 error_count = 0; 8842 u64 qman_base; 8843 u8 index; 8844 8845 switch (event_type) { 8846 case GAUDI2_EVENT_TPC0_AXI_ERR_RSP ... GAUDI2_EVENT_TPC23_AXI_ERR_RSP: 8847 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP; 8848 qman_base = mmDCORE0_TPC0_QM_BASE + 8849 (index / NUM_OF_TPC_PER_DCORE) * DCORE_OFFSET + 8850 (index % NUM_OF_TPC_PER_DCORE) * DCORE_TPC_OFFSET; 8851 module = RAZWI_TPC; 8852 break; 8853 case GAUDI2_EVENT_TPC24_AXI_ERR_RSP: 8854 qman_base = mmDCORE0_TPC6_QM_BASE; 8855 module = RAZWI_TPC; 8856 break; 8857 case GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE: 8858 case GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE: 8859 case GAUDI2_EVENT_MME2_CTRL_AXI_ERROR_RESPONSE: 8860 case GAUDI2_EVENT_MME3_CTRL_AXI_ERROR_RESPONSE: 8861 index = (event_type - GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE) / 8862 (GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE - 8863 GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE); 8864 qman_base = mmDCORE0_MME_QM_BASE + index * DCORE_OFFSET; 8865 module = RAZWI_MME; 8866 break; 8867 case GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP: 8868 case GAUDI2_EVENT_PDMA_CH1_AXI_ERR_RSP: 8869 index = event_type - GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP; 8870 qman_base = mmPDMA0_QM_BASE + index * PDMA_OFFSET; 8871 module = RAZWI_PDMA; 8872 break; 8873 case GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE: 8874 case GAUDI2_EVENT_ROTATOR1_AXI_ERROR_RESPONSE: 8875 index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE; 8876 qman_base = mmROT0_QM_BASE + index * ROT_OFFSET; 8877 module = RAZWI_ROT; 8878 break; 8879 default: 8880 return 0; 8881 } 8882 8883 error_count = _gaudi2_handle_qm_sei_err(hdev, qman_base, event_type); 8884 8885 /* There is a single event per NIC macro, so should check its both QMAN blocks */ 8886 if (event_type >= GAUDI2_EVENT_NIC0_AXI_ERROR_RESPONSE && 8887 event_type <= GAUDI2_EVENT_NIC11_AXI_ERROR_RESPONSE) 8888 error_count += _gaudi2_handle_qm_sei_err(hdev, 8889 qman_base + NIC_QM_OFFSET, event_type); 8890 8891 if (extended_err_check) { 8892 /* check if RAZWI happened */ 8893 gaudi2_ack_module_razwi_event_handler(hdev, module, 0, 0, event_mask); 8894 hl_check_for_glbl_errors(hdev); 8895 } 8896 8897 return error_count; 8898} 8899 8900static int gaudi2_handle_qman_err(struct hl_device *hdev, u16 event_type, u64 *event_mask) 8901{ 8902 u32 qid_base, error_count = 0; 8903 u64 qman_base; 8904 u8 index = 0; 8905 8906 switch (event_type) { 8907 case GAUDI2_EVENT_TPC0_QM ... GAUDI2_EVENT_TPC5_QM: 8908 index = event_type - GAUDI2_EVENT_TPC0_QM; 8909 qid_base = GAUDI2_QUEUE_ID_DCORE0_TPC_0_0 + index * QMAN_STREAMS; 8910 qman_base = mmDCORE0_TPC0_QM_BASE + index * DCORE_TPC_OFFSET; 8911 break; 8912 case GAUDI2_EVENT_TPC6_QM ... GAUDI2_EVENT_TPC11_QM: 8913 index = event_type - GAUDI2_EVENT_TPC6_QM; 8914 qid_base = GAUDI2_QUEUE_ID_DCORE1_TPC_0_0 + index * QMAN_STREAMS; 8915 qman_base = mmDCORE1_TPC0_QM_BASE + index * DCORE_TPC_OFFSET; 8916 break; 8917 case GAUDI2_EVENT_TPC12_QM ... GAUDI2_EVENT_TPC17_QM: 8918 index = event_type - GAUDI2_EVENT_TPC12_QM; 8919 qid_base = GAUDI2_QUEUE_ID_DCORE2_TPC_0_0 + index * QMAN_STREAMS; 8920 qman_base = mmDCORE2_TPC0_QM_BASE + index * DCORE_TPC_OFFSET; 8921 break; 8922 case GAUDI2_EVENT_TPC18_QM ... GAUDI2_EVENT_TPC23_QM: 8923 index = event_type - GAUDI2_EVENT_TPC18_QM; 8924 qid_base = GAUDI2_QUEUE_ID_DCORE3_TPC_0_0 + index * QMAN_STREAMS; 8925 qman_base = mmDCORE3_TPC0_QM_BASE + index * DCORE_TPC_OFFSET; 8926 break; 8927 case GAUDI2_EVENT_TPC24_QM: 8928 qid_base = GAUDI2_QUEUE_ID_DCORE0_TPC_6_0; 8929 qman_base = mmDCORE0_TPC6_QM_BASE; 8930 break; 8931 case GAUDI2_EVENT_MME0_QM: 8932 qid_base = GAUDI2_QUEUE_ID_DCORE0_MME_0_0; 8933 qman_base = mmDCORE0_MME_QM_BASE; 8934 break; 8935 case GAUDI2_EVENT_MME1_QM: 8936 qid_base = GAUDI2_QUEUE_ID_DCORE1_MME_0_0; 8937 qman_base = mmDCORE1_MME_QM_BASE; 8938 break; 8939 case GAUDI2_EVENT_MME2_QM: 8940 qid_base = GAUDI2_QUEUE_ID_DCORE2_MME_0_0; 8941 qman_base = mmDCORE2_MME_QM_BASE; 8942 break; 8943 case GAUDI2_EVENT_MME3_QM: 8944 qid_base = GAUDI2_QUEUE_ID_DCORE3_MME_0_0; 8945 qman_base = mmDCORE3_MME_QM_BASE; 8946 break; 8947 case GAUDI2_EVENT_HDMA0_QM: 8948 index = 0; 8949 qid_base = GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0; 8950 qman_base = mmDCORE0_EDMA0_QM_BASE; 8951 break; 8952 case GAUDI2_EVENT_HDMA1_QM: 8953 index = 1; 8954 qid_base = GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0; 8955 qman_base = mmDCORE0_EDMA1_QM_BASE; 8956 break; 8957 case GAUDI2_EVENT_HDMA2_QM: 8958 index = 2; 8959 qid_base = GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0; 8960 qman_base = mmDCORE1_EDMA0_QM_BASE; 8961 break; 8962 case GAUDI2_EVENT_HDMA3_QM: 8963 index = 3; 8964 qid_base = GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0; 8965 qman_base = mmDCORE1_EDMA1_QM_BASE; 8966 break; 8967 case GAUDI2_EVENT_HDMA4_QM: 8968 index = 4; 8969 qid_base = GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0; 8970 qman_base = mmDCORE2_EDMA0_QM_BASE; 8971 break; 8972 case GAUDI2_EVENT_HDMA5_QM: 8973 index = 5; 8974 qid_base = GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0; 8975 qman_base = mmDCORE2_EDMA1_QM_BASE; 8976 break; 8977 case GAUDI2_EVENT_HDMA6_QM: 8978 index = 6; 8979 qid_base = GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0; 8980 qman_base = mmDCORE3_EDMA0_QM_BASE; 8981 break; 8982 case GAUDI2_EVENT_HDMA7_QM: 8983 index = 7; 8984 qid_base = GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0; 8985 qman_base = mmDCORE3_EDMA1_QM_BASE; 8986 break; 8987 case GAUDI2_EVENT_PDMA0_QM: 8988 qid_base = GAUDI2_QUEUE_ID_PDMA_0_0; 8989 qman_base = mmPDMA0_QM_BASE; 8990 break; 8991 case GAUDI2_EVENT_PDMA1_QM: 8992 qid_base = GAUDI2_QUEUE_ID_PDMA_1_0; 8993 qman_base = mmPDMA1_QM_BASE; 8994 break; 8995 case GAUDI2_EVENT_ROTATOR0_ROT0_QM: 8996 qid_base = GAUDI2_QUEUE_ID_ROT_0_0; 8997 qman_base = mmROT0_QM_BASE; 8998 break; 8999 case GAUDI2_EVENT_ROTATOR1_ROT1_QM: 9000 qid_base = GAUDI2_QUEUE_ID_ROT_1_0; 9001 qman_base = mmROT1_QM_BASE; 9002 break; 9003 default: 9004 return 0; 9005 } 9006 9007 error_count = gaudi2_handle_qman_err_generic(hdev, event_type, qman_base, 9008 qid_base, event_mask); 9009 9010 /* Handle EDMA QM SEI here because there is no AXI error response event for EDMA */ 9011 if (event_type >= GAUDI2_EVENT_HDMA2_QM && event_type <= GAUDI2_EVENT_HDMA5_QM) { 9012 error_count += _gaudi2_handle_qm_sei_err(hdev, qman_base, event_type); 9013 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_EDMA, index, 0, event_mask); 9014 } 9015 9016 hl_check_for_glbl_errors(hdev); 9017 9018 return error_count; 9019} 9020 9021static int gaudi2_handle_arc_farm_sei_err(struct hl_device *hdev, u16 event_type, u64 *event_mask) 9022{ 9023 u32 i, sts_val, sts_clr_val, error_count = 0, arc_farm; 9024 9025 for (arc_farm = 0 ; arc_farm < NUM_OF_ARC_FARMS_ARC ; arc_farm++) { 9026 sts_clr_val = 0; 9027 sts_val = RREG32(mmARC_FARM_ARC0_AUX_ARC_SEI_INTR_STS + 9028 (arc_farm * ARC_FARM_OFFSET)); 9029 9030 for (i = 0 ; i < GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE ; i++) { 9031 if (sts_val & BIT(i)) { 9032 gaudi2_print_event(hdev, event_type, true, 9033 "ARC FARM ARC %u err cause: %s", 9034 arc_farm, gaudi2_arc_sei_error_cause[i]); 9035 sts_clr_val |= BIT(i); 9036 error_count++; 9037 } 9038 } 9039 WREG32(mmARC_FARM_ARC0_AUX_ARC_SEI_INTR_CLR + (arc_farm * ARC_FARM_OFFSET), 9040 sts_clr_val); 9041 } 9042 9043 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ARC_FARM, 0, 0, event_mask); 9044 hl_check_for_glbl_errors(hdev); 9045 9046 return error_count; 9047} 9048 9049static int gaudi2_handle_cpu_sei_err(struct hl_device *hdev, u16 event_type) 9050{ 9051 u32 i, sts_val, sts_clr_val = 0, error_count = 0; 9052 9053 sts_val = RREG32(mmCPU_IF_CPU_SEI_INTR_STS); 9054 9055 for (i = 0 ; i < GAUDI2_NUM_OF_CPU_SEI_ERR_CAUSE ; i++) { 9056 if (sts_val & BIT(i)) { 9057 gaudi2_print_event(hdev, event_type, true, 9058 "err cause: %s", gaudi2_cpu_sei_error_cause[i]); 9059 sts_clr_val |= BIT(i); 9060 error_count++; 9061 } 9062 } 9063 9064 hl_check_for_glbl_errors(hdev); 9065 9066 WREG32(mmCPU_IF_CPU_SEI_INTR_CLR, sts_clr_val); 9067 9068 return error_count; 9069} 9070 9071static int gaudi2_handle_rot_err(struct hl_device *hdev, u8 rot_index, u16 event_type, 9072 struct hl_eq_razwi_with_intr_cause *razwi_with_intr_cause, 9073 u64 *event_mask) 9074{ 9075 u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data); 9076 u32 error_count = 0; 9077 int i; 9078 9079 for (i = 0 ; i < GAUDI2_NUM_OF_ROT_ERR_CAUSE ; i++) 9080 if (intr_cause_data & BIT(i)) { 9081 gaudi2_print_event(hdev, event_type, true, 9082 "err cause: %s", guadi2_rot_error_cause[i]); 9083 error_count++; 9084 } 9085 9086 /* check if RAZWI happened */ 9087 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ROT, rot_index, 0, event_mask); 9088 hl_check_for_glbl_errors(hdev); 9089 9090 return error_count; 9091} 9092 9093static int gaudi2_tpc_ack_interrupts(struct hl_device *hdev, u8 tpc_index, u16 event_type, 9094 struct hl_eq_razwi_with_intr_cause *razwi_with_intr_cause, 9095 u64 *event_mask) 9096{ 9097 u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data); 9098 u32 error_count = 0; 9099 int i; 9100 9101 for (i = 0 ; i < GAUDI2_NUM_OF_TPC_INTR_CAUSE ; i++) 9102 if (intr_cause_data & BIT(i)) { 9103 gaudi2_print_event(hdev, event_type, true, 9104 "interrupt cause: %s", gaudi2_tpc_interrupts_cause[i]); 9105 error_count++; 9106 } 9107 9108 /* check if RAZWI happened */ 9109 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_TPC, tpc_index, 0, event_mask); 9110 hl_check_for_glbl_errors(hdev); 9111 9112 return error_count; 9113} 9114 9115static int gaudi2_handle_dec_err(struct hl_device *hdev, u8 dec_index, u16 event_type, 9116 u64 *event_mask) 9117{ 9118 u32 sts_addr, sts_val, sts_clr_val = 0, error_count = 0; 9119 int i; 9120 9121 if (dec_index < NUM_OF_VDEC_PER_DCORE * NUM_OF_DCORES) 9122 /* DCORE DEC */ 9123 sts_addr = mmDCORE0_VDEC0_BRDG_CTRL_CAUSE_INTR + 9124 DCORE_OFFSET * (dec_index / NUM_OF_DEC_PER_DCORE) + 9125 DCORE_VDEC_OFFSET * (dec_index % NUM_OF_DEC_PER_DCORE); 9126 else 9127 /* PCIE DEC */ 9128 sts_addr = mmPCIE_VDEC0_BRDG_CTRL_CAUSE_INTR + PCIE_VDEC_OFFSET * 9129 (dec_index - NUM_OF_VDEC_PER_DCORE * NUM_OF_DCORES); 9130 9131 sts_val = RREG32(sts_addr); 9132 9133 for (i = 0 ; i < GAUDI2_NUM_OF_DEC_ERR_CAUSE ; i++) { 9134 if (sts_val & BIT(i)) { 9135 gaudi2_print_event(hdev, event_type, true, 9136 "err cause: %s", gaudi2_dec_error_cause[i]); 9137 sts_clr_val |= BIT(i); 9138 error_count++; 9139 } 9140 } 9141 9142 /* check if RAZWI happened */ 9143 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_DEC, dec_index, 0, event_mask); 9144 hl_check_for_glbl_errors(hdev); 9145 9146 /* Write 1 clear errors */ 9147 WREG32(sts_addr, sts_clr_val); 9148 9149 return error_count; 9150} 9151 9152static int gaudi2_handle_mme_err(struct hl_device *hdev, u8 mme_index, u16 event_type, 9153 u64 *event_mask) 9154{ 9155 u32 sts_addr, sts_val, sts_clr_addr, sts_clr_val = 0, error_count = 0; 9156 int i; 9157 9158 sts_addr = mmDCORE0_MME_CTRL_LO_INTR_CAUSE + DCORE_OFFSET * mme_index; 9159 sts_clr_addr = mmDCORE0_MME_CTRL_LO_INTR_CLEAR + DCORE_OFFSET * mme_index; 9160 9161 sts_val = RREG32(sts_addr); 9162 9163 for (i = 0 ; i < GAUDI2_NUM_OF_MME_ERR_CAUSE ; i++) { 9164 if (sts_val & BIT(i)) { 9165 gaudi2_print_event(hdev, event_type, true, 9166 "err cause: %s", guadi2_mme_error_cause[i]); 9167 sts_clr_val |= BIT(i); 9168 error_count++; 9169 } 9170 } 9171 9172 /* check if RAZWI happened */ 9173 for (i = MME_WRITE ; i < MME_INITIATORS_MAX ; i++) 9174 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, i, event_mask); 9175 9176 hl_check_for_glbl_errors(hdev); 9177 9178 WREG32(sts_clr_addr, sts_clr_val); 9179 9180 return error_count; 9181} 9182 9183static int gaudi2_handle_mme_sbte_err(struct hl_device *hdev, u16 event_type) 9184{ 9185 /* 9186 * We have a single error cause here but the report mechanism is 9187 * buggy. Hence there is no good reason to fetch the cause so we 9188 * just check for glbl_errors and exit. 9189 */ 9190 hl_check_for_glbl_errors(hdev); 9191 9192 return GAUDI2_NA_EVENT_CAUSE; 9193} 9194 9195static int gaudi2_handle_mme_wap_err(struct hl_device *hdev, u8 mme_index, u16 event_type, 9196 u64 *event_mask) 9197{ 9198 u32 sts_addr, sts_val, sts_clr_addr, sts_clr_val = 0, error_count = 0; 9199 int i; 9200 9201 sts_addr = mmDCORE0_MME_ACC_INTR_CAUSE + DCORE_OFFSET * mme_index; 9202 sts_clr_addr = mmDCORE0_MME_ACC_INTR_CLEAR + DCORE_OFFSET * mme_index; 9203 9204 sts_val = RREG32(sts_addr); 9205 9206 for (i = 0 ; i < GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE ; i++) { 9207 if (sts_val & BIT(i)) { 9208 gaudi2_print_event(hdev, event_type, true, 9209 "err cause: %s", guadi2_mme_wap_error_cause[i]); 9210 sts_clr_val |= BIT(i); 9211 error_count++; 9212 } 9213 } 9214 9215 /* check if RAZWI happened on WAP0/1 */ 9216 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, MME_WAP0, event_mask); 9217 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, MME_WAP1, event_mask); 9218 hl_check_for_glbl_errors(hdev); 9219 9220 WREG32(sts_clr_addr, sts_clr_val); 9221 9222 return error_count; 9223} 9224 9225static int gaudi2_handle_kdma_core_event(struct hl_device *hdev, u16 event_type, 9226 u64 intr_cause_data) 9227{ 9228 u32 error_count = 0; 9229 int i; 9230 9231 /* If an AXI read or write error is received, an error is reported and 9232 * interrupt message is sent. Due to an HW errata, when reading the cause 9233 * register of the KDMA engine, the reported error is always HBW even if 9234 * the actual error caused by a LBW KDMA transaction. 9235 */ 9236 for (i = 0 ; i < GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE ; i++) 9237 if (intr_cause_data & BIT(i)) { 9238 gaudi2_print_event(hdev, event_type, true, 9239 "err cause: %s", gaudi2_kdma_core_interrupts_cause[i]); 9240 error_count++; 9241 } 9242 9243 hl_check_for_glbl_errors(hdev); 9244 9245 return error_count; 9246} 9247 9248static int gaudi2_handle_dma_core_event(struct hl_device *hdev, u16 event_type, u64 intr_cause) 9249{ 9250 u32 error_count = 0; 9251 int i; 9252 9253 for (i = 0 ; i < GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE ; i++) 9254 if (intr_cause & BIT(i)) { 9255 gaudi2_print_event(hdev, event_type, true, 9256 "err cause: %s", gaudi2_dma_core_interrupts_cause[i]); 9257 error_count++; 9258 } 9259 9260 hl_check_for_glbl_errors(hdev); 9261 9262 return error_count; 9263} 9264 9265static void gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info(struct hl_device *hdev, u64 *event_mask) 9266{ 9267 u32 mstr_if_base_addr = mmPCIE_MSTR_RR_MSTR_IF_RR_SHRD_HBW_BASE, razwi_happened_addr; 9268 9269 razwi_happened_addr = mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HAPPENED; 9270 if (RREG32(razwi_happened_addr)) { 9271 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, mstr_if_base_addr, true, "PCIE", 9272 GAUDI2_ENGINE_ID_PCIE, event_mask); 9273 WREG32(razwi_happened_addr, 0x1); 9274 } 9275 9276 razwi_happened_addr = mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HAPPENED; 9277 if (RREG32(razwi_happened_addr)) { 9278 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, mstr_if_base_addr, false, "PCIE", 9279 GAUDI2_ENGINE_ID_PCIE, event_mask); 9280 WREG32(razwi_happened_addr, 0x1); 9281 } 9282 9283 razwi_happened_addr = mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_HAPPENED; 9284 if (RREG32(razwi_happened_addr)) { 9285 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, mstr_if_base_addr, true, "PCIE", 9286 GAUDI2_ENGINE_ID_PCIE, event_mask); 9287 WREG32(razwi_happened_addr, 0x1); 9288 } 9289 9290 razwi_happened_addr = mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_HAPPENED; 9291 if (RREG32(razwi_happened_addr)) { 9292 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, mstr_if_base_addr, false, "PCIE", 9293 GAUDI2_ENGINE_ID_PCIE, event_mask); 9294 WREG32(razwi_happened_addr, 0x1); 9295 } 9296} 9297 9298static int gaudi2_print_pcie_addr_dec_info(struct hl_device *hdev, u16 event_type, 9299 u64 intr_cause_data, u64 *event_mask) 9300{ 9301 u32 error_count = 0; 9302 int i; 9303 9304 for (i = 0 ; i < GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE ; i++) { 9305 if (!(intr_cause_data & BIT_ULL(i))) 9306 continue; 9307 9308 gaudi2_print_event(hdev, event_type, true, 9309 "err cause: %s", gaudi2_pcie_addr_dec_error_cause[i]); 9310 error_count++; 9311 9312 switch (intr_cause_data & BIT_ULL(i)) { 9313 case PCIE_WRAP_PCIE_IC_SEI_INTR_IND_AXI_LBW_ERR_INTR_MASK: 9314 hl_check_for_glbl_errors(hdev); 9315 break; 9316 case PCIE_WRAP_PCIE_IC_SEI_INTR_IND_BAD_ACCESS_INTR_MASK: 9317 gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info(hdev, event_mask); 9318 break; 9319 } 9320 } 9321 9322 return error_count; 9323} 9324 9325static int gaudi2_handle_pif_fatal(struct hl_device *hdev, u16 event_type, 9326 u64 intr_cause_data) 9327 9328{ 9329 u32 error_count = 0; 9330 int i; 9331 9332 for (i = 0 ; i < GAUDI2_NUM_OF_PMMU_FATAL_ERR_CAUSE ; i++) { 9333 if (intr_cause_data & BIT_ULL(i)) { 9334 gaudi2_print_event(hdev, event_type, true, 9335 "err cause: %s", gaudi2_pmmu_fatal_interrupts_cause[i]); 9336 error_count++; 9337 } 9338 } 9339 9340 return error_count; 9341} 9342 9343static int gaudi2_handle_hif_fatal(struct hl_device *hdev, u16 event_type, u64 intr_cause_data) 9344{ 9345 u32 error_count = 0; 9346 int i; 9347 9348 for (i = 0 ; i < GAUDI2_NUM_OF_HIF_FATAL_ERR_CAUSE ; i++) { 9349 if (intr_cause_data & BIT_ULL(i)) { 9350 gaudi2_print_event(hdev, event_type, true, 9351 "err cause: %s", gaudi2_hif_fatal_interrupts_cause[i]); 9352 error_count++; 9353 } 9354 } 9355 9356 return error_count; 9357} 9358 9359static void gaudi2_handle_page_error(struct hl_device *hdev, u64 mmu_base, bool is_pmmu, 9360 u64 *event_mask) 9361{ 9362 u32 valid, val; 9363 u64 addr; 9364 9365 valid = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID)); 9366 9367 if (!(valid & DCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID_PAGE_ERR_VALID_ENTRY_MASK)) 9368 return; 9369 9370 val = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE)); 9371 addr = val & DCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE_VA_63_32_MASK; 9372 addr <<= 32; 9373 addr |= RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE_VA)); 9374 9375 if (is_pmmu) { 9376 dev_err_ratelimited(hdev->dev, "PMMU page fault on va 0x%llx\n", addr); 9377 } else { 9378 addr = gaudi2_mmu_descramble_addr(hdev, addr); 9379 addr &= HW_UNSCRAMBLED_BITS_MASK; 9380 dev_err_ratelimited(hdev->dev, "HMMU page fault on va range 0x%llx - 0x%llx\n", 9381 addr, addr + ~HW_UNSCRAMBLED_BITS_MASK); 9382 } 9383 9384 hl_handle_page_fault(hdev, addr, 0, is_pmmu, event_mask); 9385 9386 WREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID), 0); 9387} 9388 9389static void gaudi2_handle_access_error(struct hl_device *hdev, u64 mmu_base, bool is_pmmu) 9390{ 9391 u32 valid, val; 9392 u64 addr; 9393 9394 valid = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID)); 9395 9396 if (!(valid & DCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID_ACCESS_ERR_VALID_ENTRY_MASK)) 9397 return; 9398 9399 val = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE)); 9400 addr = val & DCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE_VA_63_32_MASK; 9401 addr <<= 32; 9402 addr |= RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE_VA)); 9403 9404 if (!is_pmmu) 9405 addr = gaudi2_mmu_descramble_addr(hdev, addr); 9406 9407 dev_err_ratelimited(hdev->dev, "%s access error on va 0x%llx\n", 9408 is_pmmu ? "PMMU" : "HMMU", addr); 9409 WREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID), 0); 9410} 9411 9412static int gaudi2_handle_mmu_spi_sei_generic(struct hl_device *hdev, u16 event_type, 9413 u64 mmu_base, bool is_pmmu, u64 *event_mask) 9414{ 9415 u32 spi_sei_cause, interrupt_clr = 0x0, error_count = 0; 9416 int i; 9417 9418 spi_sei_cause = RREG32(mmu_base + MMU_SPI_SEI_CAUSE_OFFSET); 9419 9420 for (i = 0 ; i < GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE ; i++) { 9421 if (spi_sei_cause & BIT(i)) { 9422 gaudi2_print_event(hdev, event_type, true, 9423 "err cause: %s", gaudi2_mmu_spi_sei[i].cause); 9424 9425 if (i == 0) 9426 gaudi2_handle_page_error(hdev, mmu_base, is_pmmu, event_mask); 9427 else if (i == 1) 9428 gaudi2_handle_access_error(hdev, mmu_base, is_pmmu); 9429 9430 if (gaudi2_mmu_spi_sei[i].clear_bit >= 0) 9431 interrupt_clr |= BIT(gaudi2_mmu_spi_sei[i].clear_bit); 9432 9433 error_count++; 9434 } 9435 } 9436 9437 /* Clear cause */ 9438 WREG32_AND(mmu_base + MMU_SPI_SEI_CAUSE_OFFSET, ~spi_sei_cause); 9439 9440 /* Clear interrupt */ 9441 WREG32(mmu_base + MMU_INTERRUPT_CLR_OFFSET, interrupt_clr); 9442 9443 return error_count; 9444} 9445 9446static int gaudi2_handle_sm_err(struct hl_device *hdev, u16 event_type, u8 sm_index) 9447{ 9448 u32 sei_cause_addr, sei_cause_val, sei_cause_cause, sei_cause_log, 9449 cq_intr_addr, cq_intr_val, cq_intr_queue_index, error_count = 0; 9450 int i; 9451 9452 sei_cause_addr = mmDCORE0_SYNC_MNGR_GLBL_SM_SEI_CAUSE + DCORE_OFFSET * sm_index; 9453 cq_intr_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_INTR + DCORE_OFFSET * sm_index; 9454 9455 sei_cause_val = RREG32(sei_cause_addr); 9456 sei_cause_cause = FIELD_GET(DCORE0_SYNC_MNGR_GLBL_SM_SEI_CAUSE_CAUSE_MASK, sei_cause_val); 9457 cq_intr_val = RREG32(cq_intr_addr); 9458 9459 /* SEI interrupt */ 9460 if (sei_cause_cause) { 9461 /* There are corresponding SEI_CAUSE_log bits for every SEI_CAUSE_cause bit */ 9462 sei_cause_log = FIELD_GET(DCORE0_SYNC_MNGR_GLBL_SM_SEI_CAUSE_LOG_MASK, 9463 sei_cause_val); 9464 9465 for (i = 0 ; i < GAUDI2_NUM_OF_SM_SEI_ERR_CAUSE ; i++) { 9466 if (!(sei_cause_cause & BIT(i))) 9467 continue; 9468 9469 gaudi2_print_event(hdev, event_type, true, 9470 "err cause: %s. %s: 0x%X", 9471 gaudi2_sm_sei_cause[i].cause_name, 9472 gaudi2_sm_sei_cause[i].log_name, 9473 sei_cause_log); 9474 error_count++; 9475 break; 9476 } 9477 9478 /* Clear SM_SEI_CAUSE */ 9479 WREG32(sei_cause_addr, 0); 9480 } 9481 9482 /* CQ interrupt */ 9483 if (cq_intr_val & DCORE0_SYNC_MNGR_GLBL_CQ_INTR_CQ_SEC_INTR_MASK) { 9484 cq_intr_queue_index = 9485 FIELD_GET(DCORE0_SYNC_MNGR_GLBL_CQ_INTR_CQ_INTR_QUEUE_INDEX_MASK, 9486 cq_intr_val); 9487 9488 dev_err_ratelimited(hdev->dev, "SM%u err. err cause: CQ_INTR. queue index: %u\n", 9489 sm_index, cq_intr_queue_index); 9490 error_count++; 9491 9492 /* Clear CQ_INTR */ 9493 WREG32(cq_intr_addr, 0); 9494 } 9495 9496 hl_check_for_glbl_errors(hdev); 9497 9498 return error_count; 9499} 9500 9501static u64 get_hmmu_base(u16 event_type) 9502{ 9503 u8 dcore, index_in_dcore; 9504 9505 switch (event_type) { 9506 case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP: 9507 case GAUDI2_EVENT_HMMU0_SPI_BASE ... GAUDI2_EVENT_HMMU0_SECURITY_ERROR: 9508 dcore = 0; 9509 index_in_dcore = 0; 9510 break; 9511 case GAUDI2_EVENT_HMMU_1_AXI_ERR_RSP: 9512 case GAUDI2_EVENT_HMMU1_SPI_BASE ... GAUDI2_EVENT_HMMU1_SECURITY_ERROR: 9513 dcore = 1; 9514 index_in_dcore = 0; 9515 break; 9516 case GAUDI2_EVENT_HMMU_2_AXI_ERR_RSP: 9517 case GAUDI2_EVENT_HMMU2_SPI_BASE ... GAUDI2_EVENT_HMMU2_SECURITY_ERROR: 9518 dcore = 0; 9519 index_in_dcore = 1; 9520 break; 9521 case GAUDI2_EVENT_HMMU_3_AXI_ERR_RSP: 9522 case GAUDI2_EVENT_HMMU3_SPI_BASE ... GAUDI2_EVENT_HMMU3_SECURITY_ERROR: 9523 dcore = 1; 9524 index_in_dcore = 1; 9525 break; 9526 case GAUDI2_EVENT_HMMU_4_AXI_ERR_RSP: 9527 case GAUDI2_EVENT_HMMU4_SPI_BASE ... GAUDI2_EVENT_HMMU4_SECURITY_ERROR: 9528 dcore = 3; 9529 index_in_dcore = 2; 9530 break; 9531 case GAUDI2_EVENT_HMMU_5_AXI_ERR_RSP: 9532 case GAUDI2_EVENT_HMMU5_SPI_BASE ... GAUDI2_EVENT_HMMU5_SECURITY_ERROR: 9533 dcore = 2; 9534 index_in_dcore = 2; 9535 break; 9536 case GAUDI2_EVENT_HMMU_6_AXI_ERR_RSP: 9537 case GAUDI2_EVENT_HMMU6_SPI_BASE ... GAUDI2_EVENT_HMMU6_SECURITY_ERROR: 9538 dcore = 3; 9539 index_in_dcore = 3; 9540 break; 9541 case GAUDI2_EVENT_HMMU_7_AXI_ERR_RSP: 9542 case GAUDI2_EVENT_HMMU7_SPI_BASE ... GAUDI2_EVENT_HMMU7_SECURITY_ERROR: 9543 dcore = 2; 9544 index_in_dcore = 3; 9545 break; 9546 case GAUDI2_EVENT_HMMU_8_AXI_ERR_RSP: 9547 case GAUDI2_EVENT_HMMU8_SPI_BASE ... GAUDI2_EVENT_HMMU8_SECURITY_ERROR: 9548 dcore = 0; 9549 index_in_dcore = 2; 9550 break; 9551 case GAUDI2_EVENT_HMMU_9_AXI_ERR_RSP: 9552 case GAUDI2_EVENT_HMMU9_SPI_BASE ... GAUDI2_EVENT_HMMU9_SECURITY_ERROR: 9553 dcore = 1; 9554 index_in_dcore = 2; 9555 break; 9556 case GAUDI2_EVENT_HMMU_10_AXI_ERR_RSP: 9557 case GAUDI2_EVENT_HMMU10_SPI_BASE ... GAUDI2_EVENT_HMMU10_SECURITY_ERROR: 9558 dcore = 0; 9559 index_in_dcore = 3; 9560 break; 9561 case GAUDI2_EVENT_HMMU_11_AXI_ERR_RSP: 9562 case GAUDI2_EVENT_HMMU11_SPI_BASE ... GAUDI2_EVENT_HMMU11_SECURITY_ERROR: 9563 dcore = 1; 9564 index_in_dcore = 3; 9565 break; 9566 case GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP: 9567 case GAUDI2_EVENT_HMMU12_SPI_BASE ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR: 9568 dcore = 3; 9569 index_in_dcore = 0; 9570 break; 9571 case GAUDI2_EVENT_HMMU_13_AXI_ERR_RSP: 9572 case GAUDI2_EVENT_HMMU13_SPI_BASE ... GAUDI2_EVENT_HMMU13_SECURITY_ERROR: 9573 dcore = 2; 9574 index_in_dcore = 0; 9575 break; 9576 case GAUDI2_EVENT_HMMU_14_AXI_ERR_RSP: 9577 case GAUDI2_EVENT_HMMU14_SPI_BASE ... GAUDI2_EVENT_HMMU14_SECURITY_ERROR: 9578 dcore = 3; 9579 index_in_dcore = 1; 9580 break; 9581 case GAUDI2_EVENT_HMMU_15_AXI_ERR_RSP: 9582 case GAUDI2_EVENT_HMMU15_SPI_BASE ... GAUDI2_EVENT_HMMU15_SECURITY_ERROR: 9583 dcore = 2; 9584 index_in_dcore = 1; 9585 break; 9586 default: 9587 return ULONG_MAX; 9588 } 9589 9590 return mmDCORE0_HMMU0_MMU_BASE + dcore * DCORE_OFFSET + index_in_dcore * DCORE_HMMU_OFFSET; 9591} 9592 9593static int gaudi2_handle_mmu_spi_sei_err(struct hl_device *hdev, u16 event_type, u64 *event_mask) 9594{ 9595 bool is_pmmu = false; 9596 u32 error_count = 0; 9597 u64 mmu_base; 9598 9599 switch (event_type) { 9600 case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP ... GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP: 9601 case GAUDI2_EVENT_HMMU0_SPI_BASE ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR: 9602 mmu_base = get_hmmu_base(event_type); 9603 break; 9604 9605 case GAUDI2_EVENT_PMMU0_PAGE_FAULT_WR_PERM ... GAUDI2_EVENT_PMMU0_SECURITY_ERROR: 9606 case GAUDI2_EVENT_PMMU_AXI_ERR_RSP_0: 9607 is_pmmu = true; 9608 mmu_base = mmPMMU_HBW_MMU_BASE; 9609 break; 9610 default: 9611 return 0; 9612 } 9613 9614 if (mmu_base == ULONG_MAX) 9615 return 0; 9616 9617 error_count = gaudi2_handle_mmu_spi_sei_generic(hdev, event_type, mmu_base, 9618 is_pmmu, event_mask); 9619 hl_check_for_glbl_errors(hdev); 9620 9621 return error_count; 9622} 9623 9624 9625/* returns true if hard reset is required (ECC DERR or Read parity), false otherwise (ECC SERR) */ 9626static bool gaudi2_hbm_sei_handle_read_err(struct hl_device *hdev, 9627 struct hl_eq_hbm_sei_read_err_intr_info *rd_err_data, u32 err_cnt) 9628{ 9629 bool require_hard_reset = false; 9630 u32 addr, beat, beat_shift; 9631 9632 dev_err_ratelimited(hdev->dev, 9633 "READ ERROR count: ECC SERR: %d, ECC DERR: %d, RD_PARITY: %d\n", 9634 FIELD_GET(HBM_ECC_SERR_CNTR_MASK, err_cnt), 9635 FIELD_GET(HBM_ECC_DERR_CNTR_MASK, err_cnt), 9636 FIELD_GET(HBM_RD_PARITY_CNTR_MASK, err_cnt)); 9637 9638 addr = le32_to_cpu(rd_err_data->dbg_rd_err_addr.rd_addr_val); 9639 dev_err_ratelimited(hdev->dev, 9640 "READ ERROR address: sid(%u), bg(%u), ba(%u), col(%u), row(%u)\n", 9641 FIELD_GET(HBM_RD_ADDR_SID_MASK, addr), 9642 FIELD_GET(HBM_RD_ADDR_BG_MASK, addr), 9643 FIELD_GET(HBM_RD_ADDR_BA_MASK, addr), 9644 FIELD_GET(HBM_RD_ADDR_COL_MASK, addr), 9645 FIELD_GET(HBM_RD_ADDR_ROW_MASK, addr)); 9646 9647 /* For each beat (RDQS edge), look for possible errors and print relevant info */ 9648 for (beat = 0 ; beat < 4 ; beat++) { 9649 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & 9650 (HBM_RD_ERR_SERR_BEAT0_MASK << beat)) 9651 dev_err_ratelimited(hdev->dev, "Beat%d ECC SERR: DM: %#x, Syndrome: %#x\n", 9652 beat, 9653 le32_to_cpu(rd_err_data->dbg_rd_err_dm), 9654 le32_to_cpu(rd_err_data->dbg_rd_err_syndrome)); 9655 9656 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & 9657 (HBM_RD_ERR_DERR_BEAT0_MASK << beat)) { 9658 dev_err_ratelimited(hdev->dev, "Beat%d ECC DERR: DM: %#x, Syndrome: %#x\n", 9659 beat, 9660 le32_to_cpu(rd_err_data->dbg_rd_err_dm), 9661 le32_to_cpu(rd_err_data->dbg_rd_err_syndrome)); 9662 require_hard_reset = true; 9663 } 9664 9665 beat_shift = beat * HBM_RD_ERR_BEAT_SHIFT; 9666 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & 9667 (HBM_RD_ERR_PAR_ERR_BEAT0_MASK << beat_shift)) { 9668 dev_err_ratelimited(hdev->dev, 9669 "Beat%d read PARITY: DM: %#x, PAR data: %#x\n", 9670 beat, 9671 le32_to_cpu(rd_err_data->dbg_rd_err_dm), 9672 (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & 9673 (HBM_RD_ERR_PAR_DATA_BEAT0_MASK << beat_shift)) >> 9674 (HBM_RD_ERR_PAR_DATA_BEAT0_SHIFT + beat_shift)); 9675 require_hard_reset = true; 9676 } 9677 9678 dev_err_ratelimited(hdev->dev, "Beat%d DQ data:\n", beat); 9679 dev_err_ratelimited(hdev->dev, "\t0x%08x\n", 9680 le32_to_cpu(rd_err_data->dbg_rd_err_data[beat * 2])); 9681 dev_err_ratelimited(hdev->dev, "\t0x%08x\n", 9682 le32_to_cpu(rd_err_data->dbg_rd_err_data[beat * 2 + 1])); 9683 } 9684 9685 return require_hard_reset; 9686} 9687 9688static void gaudi2_hbm_sei_print_wr_par_info(struct hl_device *hdev, 9689 struct hl_eq_hbm_sei_wr_par_intr_info *wr_par_err_data, u32 err_cnt) 9690{ 9691 struct hbm_sei_wr_cmd_address *wr_cmd_addr = wr_par_err_data->dbg_last_wr_cmds; 9692 u32 i, curr_addr, derr = wr_par_err_data->dbg_derr; 9693 9694 dev_err_ratelimited(hdev->dev, "WRITE PARITY ERROR count: %d\n", err_cnt); 9695 9696 dev_err_ratelimited(hdev->dev, "CK-0 DERR: 0x%02x, CK-1 DERR: 0x%02x\n", 9697 derr & 0x3, derr & 0xc); 9698 9699 /* JIRA H6-3286 - the following prints may not be valid */ 9700 dev_err_ratelimited(hdev->dev, "Last latched write commands addresses:\n"); 9701 for (i = 0 ; i < HBM_WR_PAR_CMD_LIFO_LEN ; i++) { 9702 curr_addr = le32_to_cpu(wr_cmd_addr[i].dbg_wr_cmd_addr); 9703 dev_err_ratelimited(hdev->dev, 9704 "\twrite cmd[%u]: Address: SID(%u) BG(%u) BA(%u) COL(%u).\n", 9705 i, 9706 FIELD_GET(WR_PAR_LAST_CMD_SID_MASK, curr_addr), 9707 FIELD_GET(WR_PAR_LAST_CMD_BG_MASK, curr_addr), 9708 FIELD_GET(WR_PAR_LAST_CMD_BA_MASK, curr_addr), 9709 FIELD_GET(WR_PAR_LAST_CMD_COL_MASK, curr_addr)); 9710 } 9711} 9712 9713static void gaudi2_hbm_sei_print_ca_par_info(struct hl_device *hdev, 9714 struct hl_eq_hbm_sei_ca_par_intr_info *ca_par_err_data, u32 err_cnt) 9715{ 9716 __le32 *col_cmd = ca_par_err_data->dbg_col; 9717 __le16 *row_cmd = ca_par_err_data->dbg_row; 9718 u32 i; 9719 9720 dev_err_ratelimited(hdev->dev, "CA ERROR count: %d\n", err_cnt); 9721 9722 dev_err_ratelimited(hdev->dev, "Last latched C&R bus commands:\n"); 9723 for (i = 0 ; i < HBM_CA_ERR_CMD_LIFO_LEN ; i++) 9724 dev_err_ratelimited(hdev->dev, "cmd%u: ROW(0x%04x) COL(0x%05x)\n", i, 9725 le16_to_cpu(row_cmd[i]) & (u16)GENMASK(13, 0), 9726 le32_to_cpu(col_cmd[i]) & (u32)GENMASK(17, 0)); 9727} 9728 9729/* Returns true if hard reset is needed or false otherwise */ 9730static bool gaudi2_handle_hbm_mc_sei_err(struct hl_device *hdev, u16 event_type, 9731 struct hl_eq_hbm_sei_data *sei_data) 9732{ 9733 bool require_hard_reset = false; 9734 u32 hbm_id, mc_id, cause_idx; 9735 9736 hbm_id = (event_type - GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE) / 4; 9737 mc_id = ((event_type - GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE) / 2) % 2; 9738 9739 cause_idx = sei_data->hdr.sei_cause; 9740 if (cause_idx > GAUDI2_NUM_OF_HBM_SEI_CAUSE - 1) { 9741 gaudi2_print_event(hdev, event_type, true, 9742 "err cause: %s", 9743 "Invalid HBM SEI event cause (%d) provided by FW", cause_idx); 9744 return true; 9745 } 9746 9747 gaudi2_print_event(hdev, event_type, !sei_data->hdr.is_critical, 9748 "System %s Error Interrupt - HBM(%u) MC(%u) MC_CH(%u) MC_PC(%u). Error cause: %s", 9749 sei_data->hdr.is_critical ? "Critical" : "Non-critical", 9750 hbm_id, mc_id, sei_data->hdr.mc_channel, sei_data->hdr.mc_pseudo_channel, 9751 hbm_mc_sei_cause[cause_idx]); 9752 9753 /* Print error-specific info */ 9754 switch (cause_idx) { 9755 case HBM_SEI_CATTRIP: 9756 require_hard_reset = true; 9757 break; 9758 9759 case HBM_SEI_CMD_PARITY_EVEN: 9760 gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_even_info, 9761 le32_to_cpu(sei_data->hdr.cnt)); 9762 require_hard_reset = true; 9763 break; 9764 9765 case HBM_SEI_CMD_PARITY_ODD: 9766 gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_odd_info, 9767 le32_to_cpu(sei_data->hdr.cnt)); 9768 require_hard_reset = true; 9769 break; 9770 9771 case HBM_SEI_WRITE_DATA_PARITY_ERR: 9772 gaudi2_hbm_sei_print_wr_par_info(hdev, &sei_data->wr_parity_info, 9773 le32_to_cpu(sei_data->hdr.cnt)); 9774 require_hard_reset = true; 9775 break; 9776 9777 case HBM_SEI_READ_ERR: 9778 /* Unlike other SEI events, read error requires further processing of the 9779 * raw data in order to determine the root cause. 9780 */ 9781 require_hard_reset = gaudi2_hbm_sei_handle_read_err(hdev, 9782 &sei_data->read_err_info, 9783 le32_to_cpu(sei_data->hdr.cnt)); 9784 break; 9785 9786 default: 9787 break; 9788 } 9789 9790 require_hard_reset |= !!sei_data->hdr.is_critical; 9791 9792 return require_hard_reset; 9793} 9794 9795static int gaudi2_handle_hbm_cattrip(struct hl_device *hdev, u16 event_type, 9796 u64 intr_cause_data) 9797{ 9798 if (intr_cause_data) { 9799 gaudi2_print_event(hdev, event_type, true, 9800 "temperature error cause: %#llx", intr_cause_data); 9801 return 1; 9802 } 9803 9804 return 0; 9805} 9806 9807static int gaudi2_handle_hbm_mc_spi(struct hl_device *hdev, u64 intr_cause_data) 9808{ 9809 u32 i, error_count = 0; 9810 9811 for (i = 0 ; i < GAUDI2_NUM_OF_HBM_MC_SPI_CAUSE ; i++) 9812 if (intr_cause_data & hbm_mc_spi[i].mask) { 9813 dev_dbg(hdev->dev, "HBM spi event: notification cause(%s)\n", 9814 hbm_mc_spi[i].cause); 9815 error_count++; 9816 } 9817 9818 return error_count; 9819} 9820 9821static void gaudi2_print_clk_change_info(struct hl_device *hdev, u16 event_type, u64 *event_mask) 9822{ 9823 ktime_t zero_time = ktime_set(0, 0); 9824 9825 mutex_lock(&hdev->clk_throttling.lock); 9826 9827 switch (event_type) { 9828 case GAUDI2_EVENT_CPU_FIX_POWER_ENV_S: 9829 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_POWER; 9830 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_POWER; 9831 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].start = ktime_get(); 9832 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = zero_time; 9833 dev_dbg_ratelimited(hdev->dev, "Clock throttling due to power consumption\n"); 9834 break; 9835 9836 case GAUDI2_EVENT_CPU_FIX_POWER_ENV_E: 9837 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_POWER; 9838 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = ktime_get(); 9839 dev_dbg_ratelimited(hdev->dev, "Power envelop is safe, back to optimal clock\n"); 9840 break; 9841 9842 case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_S: 9843 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_THERMAL; 9844 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_THERMAL; 9845 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].start = ktime_get(); 9846 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = zero_time; 9847 *event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9848 dev_info_ratelimited(hdev->dev, "Clock throttling due to overheating\n"); 9849 break; 9850 9851 case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_E: 9852 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_THERMAL; 9853 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = ktime_get(); 9854 *event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9855 dev_info_ratelimited(hdev->dev, "Thermal envelop is safe, back to optimal clock\n"); 9856 break; 9857 9858 default: 9859 dev_err(hdev->dev, "Received invalid clock change event %d\n", event_type); 9860 break; 9861 } 9862 9863 mutex_unlock(&hdev->clk_throttling.lock); 9864} 9865 9866static void gaudi2_print_out_of_sync_info(struct hl_device *hdev, u16 event_type, 9867 struct cpucp_pkt_sync_err *sync_err) 9868{ 9869 struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; 9870 9871 gaudi2_print_event(hdev, event_type, false, 9872 "FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d", 9873 le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), 9874 q->pi, atomic_read(&q->ci)); 9875} 9876 9877static int gaudi2_handle_pcie_p2p_msix(struct hl_device *hdev, u16 event_type) 9878{ 9879 u32 p2p_intr, msix_gw_intr, error_count = 0; 9880 9881 p2p_intr = RREG32(mmPCIE_WRAP_P2P_INTR); 9882 msix_gw_intr = RREG32(mmPCIE_WRAP_MSIX_GW_INTR); 9883 9884 if (p2p_intr) { 9885 gaudi2_print_event(hdev, event_type, true, 9886 "pcie p2p transaction terminated due to security, req_id(0x%x)", 9887 RREG32(mmPCIE_WRAP_P2P_REQ_ID)); 9888 9889 WREG32(mmPCIE_WRAP_P2P_INTR, 0x1); 9890 error_count++; 9891 } 9892 9893 if (msix_gw_intr) { 9894 gaudi2_print_event(hdev, event_type, true, 9895 "pcie msi-x gen denied due to vector num check failure, vec(0x%X)", 9896 RREG32(mmPCIE_WRAP_MSIX_GW_VEC)); 9897 9898 WREG32(mmPCIE_WRAP_MSIX_GW_INTR, 0x1); 9899 error_count++; 9900 } 9901 9902 return error_count; 9903} 9904 9905static int gaudi2_handle_pcie_drain(struct hl_device *hdev, 9906 struct hl_eq_pcie_drain_ind_data *drain_data) 9907{ 9908 u64 cause, error_count = 0; 9909 9910 cause = le64_to_cpu(drain_data->intr_cause.intr_cause_data); 9911 9912 if (cause & BIT_ULL(0)) { 9913 dev_err_ratelimited(hdev->dev, "PCIE AXI drain LBW completed\n"); 9914 error_count++; 9915 } 9916 9917 if (cause & BIT_ULL(1)) { 9918 dev_err_ratelimited(hdev->dev, "PCIE AXI drain HBW completed\n"); 9919 error_count++; 9920 } 9921 9922 return error_count; 9923} 9924 9925static int gaudi2_handle_psoc_drain(struct hl_device *hdev, u64 intr_cause_data) 9926{ 9927 u32 error_count = 0; 9928 int i; 9929 9930 for (i = 0 ; i < GAUDI2_NUM_OF_AXI_DRAIN_ERR_CAUSE ; i++) { 9931 if (intr_cause_data & BIT_ULL(i)) { 9932 dev_err_ratelimited(hdev->dev, "PSOC %s completed\n", 9933 gaudi2_psoc_axi_drain_interrupts_cause[i]); 9934 error_count++; 9935 } 9936 } 9937 9938 hl_check_for_glbl_errors(hdev); 9939 9940 return error_count; 9941} 9942 9943static void gaudi2_print_cpu_pkt_failure_info(struct hl_device *hdev, u16 event_type, 9944 struct cpucp_pkt_sync_err *sync_err) 9945{ 9946 struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; 9947 9948 gaudi2_print_event(hdev, event_type, false, 9949 "FW reported sanity check failure, FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d", 9950 le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), q->pi, atomic_read(&q->ci)); 9951} 9952 9953static int hl_arc_event_handle(struct hl_device *hdev, u16 event_type, 9954 struct hl_eq_engine_arc_intr_data *data) 9955{ 9956 struct hl_engine_arc_dccm_queue_full_irq *q; 9957 u32 intr_type, engine_id; 9958 u64 payload; 9959 9960 intr_type = le32_to_cpu(data->intr_type); 9961 engine_id = le32_to_cpu(data->engine_id); 9962 payload = le64_to_cpu(data->payload); 9963 9964 switch (intr_type) { 9965 case ENGINE_ARC_DCCM_QUEUE_FULL_IRQ: 9966 q = (struct hl_engine_arc_dccm_queue_full_irq *) &payload; 9967 9968 gaudi2_print_event(hdev, event_type, true, 9969 "ARC DCCM Full event: Eng: %s, Intr_type: %u, Qidx: %u", 9970 GAUDI2_ENG_ID_TO_STR(engine_id), intr_type, q->queue_index); 9971 return 1; 9972 default: 9973 gaudi2_print_event(hdev, event_type, true, "Unknown ARC event type"); 9974 return 0; 9975 } 9976} 9977 9978static u16 event_id_to_engine_id(struct hl_device *hdev, u16 event_type) 9979{ 9980 enum gaudi2_block_types type = GAUDI2_BLOCK_TYPE_MAX; 9981 u16 index; 9982 9983 switch (event_type) { 9984 case GAUDI2_EVENT_TPC0_AXI_ERR_RSP ... GAUDI2_EVENT_TPC24_AXI_ERR_RSP: 9985 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP; 9986 type = GAUDI2_BLOCK_TYPE_TPC; 9987 break; 9988 case GAUDI2_EVENT_TPC0_QM ... GAUDI2_EVENT_TPC24_QM: 9989 index = event_type - GAUDI2_EVENT_TPC0_QM; 9990 type = GAUDI2_BLOCK_TYPE_TPC; 9991 break; 9992 case GAUDI2_EVENT_MME0_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE: 9993 case GAUDI2_EVENT_MME0_SPI_BASE ... GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID: 9994 case GAUDI2_EVENT_MME0_QM: 9995 index = 0; 9996 type = GAUDI2_BLOCK_TYPE_MME; 9997 break; 9998 case GAUDI2_EVENT_MME1_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE: 9999 case GAUDI2_EVENT_MME1_SPI_BASE ... GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID: 10000 case GAUDI2_EVENT_MME1_QM: 10001 index = 1; 10002 type = GAUDI2_BLOCK_TYPE_MME; 10003 break; 10004 case GAUDI2_EVENT_MME2_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME2_CTRL_AXI_ERROR_RESPONSE: 10005 case GAUDI2_EVENT_MME2_SPI_BASE ... GAUDI2_EVENT_MME2_WAP_SOURCE_RESULT_INVALID: 10006 case GAUDI2_EVENT_MME2_QM: 10007 index = 2; 10008 type = GAUDI2_BLOCK_TYPE_MME; 10009 break; 10010 case GAUDI2_EVENT_MME3_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME3_CTRL_AXI_ERROR_RESPONSE: 10011 case GAUDI2_EVENT_MME3_SPI_BASE ... GAUDI2_EVENT_MME3_WAP_SOURCE_RESULT_INVALID: 10012 case GAUDI2_EVENT_MME3_QM: 10013 index = 3; 10014 type = GAUDI2_BLOCK_TYPE_MME; 10015 break; 10016 case GAUDI2_EVENT_KDMA_CH0_AXI_ERR_RSP: 10017 case GAUDI2_EVENT_KDMA_BM_SPMU: 10018 case GAUDI2_EVENT_KDMA0_CORE: 10019 return GAUDI2_ENGINE_ID_KDMA; 10020 case GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP: 10021 case GAUDI2_EVENT_PDMA0_CORE: 10022 case GAUDI2_EVENT_PDMA0_BM_SPMU: 10023 case GAUDI2_EVENT_PDMA0_QM: 10024 return GAUDI2_ENGINE_ID_PDMA_0; 10025 case GAUDI2_EVENT_PDMA_CH1_AXI_ERR_RSP: 10026 case GAUDI2_EVENT_PDMA1_CORE: 10027 case GAUDI2_EVENT_PDMA1_BM_SPMU: 10028 case GAUDI2_EVENT_PDMA1_QM: 10029 return GAUDI2_ENGINE_ID_PDMA_1; 10030 case GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE ... GAUDI2_EVENT_DEC9_AXI_ERR_RSPONSE: 10031 index = event_type - GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE; 10032 type = GAUDI2_BLOCK_TYPE_DEC; 10033 break; 10034 case GAUDI2_EVENT_DEC0_SPI ... GAUDI2_EVENT_DEC9_BMON_SPMU: 10035 index = (event_type - GAUDI2_EVENT_DEC0_SPI) >> 1; 10036 type = GAUDI2_BLOCK_TYPE_DEC; 10037 break; 10038 case GAUDI2_EVENT_NIC0_AXI_ERROR_RESPONSE ... GAUDI2_EVENT_NIC11_AXI_ERROR_RESPONSE: 10039 index = event_type - GAUDI2_EVENT_NIC0_AXI_ERROR_RESPONSE; 10040 return GAUDI2_ENGINE_ID_NIC0_0 + (index * 2); 10041 case GAUDI2_EVENT_NIC0_QM0 ... GAUDI2_EVENT_NIC11_QM1: 10042 index = event_type - GAUDI2_EVENT_NIC0_QM0; 10043 return GAUDI2_ENGINE_ID_NIC0_0 + index; 10044 case GAUDI2_EVENT_NIC0_BMON_SPMU ... GAUDI2_EVENT_NIC11_SW_ERROR: 10045 index = event_type - GAUDI2_EVENT_NIC0_BMON_SPMU; 10046 return GAUDI2_ENGINE_ID_NIC0_0 + (index * 2); 10047 case GAUDI2_EVENT_TPC0_BMON_SPMU ... GAUDI2_EVENT_TPC24_KERNEL_ERR: 10048 index = (event_type - GAUDI2_EVENT_TPC0_BMON_SPMU) >> 1; 10049 type = GAUDI2_BLOCK_TYPE_TPC; 10050 break; 10051 case GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE: 10052 case GAUDI2_EVENT_ROTATOR0_BMON_SPMU: 10053 case GAUDI2_EVENT_ROTATOR0_ROT0_QM: 10054 return GAUDI2_ENGINE_ID_ROT_0; 10055 case GAUDI2_EVENT_ROTATOR1_AXI_ERROR_RESPONSE: 10056 case GAUDI2_EVENT_ROTATOR1_BMON_SPMU: 10057 case GAUDI2_EVENT_ROTATOR1_ROT1_QM: 10058 return GAUDI2_ENGINE_ID_ROT_1; 10059 case GAUDI2_EVENT_HDMA0_BM_SPMU: 10060 case GAUDI2_EVENT_HDMA0_QM: 10061 case GAUDI2_EVENT_HDMA0_CORE: 10062 return GAUDI2_DCORE0_ENGINE_ID_EDMA_0; 10063 case GAUDI2_EVENT_HDMA1_BM_SPMU: 10064 case GAUDI2_EVENT_HDMA1_QM: 10065 case GAUDI2_EVENT_HDMA1_CORE: 10066 return GAUDI2_DCORE0_ENGINE_ID_EDMA_1; 10067 case GAUDI2_EVENT_HDMA2_BM_SPMU: 10068 case GAUDI2_EVENT_HDMA2_QM: 10069 case GAUDI2_EVENT_HDMA2_CORE: 10070 return GAUDI2_DCORE1_ENGINE_ID_EDMA_0; 10071 case GAUDI2_EVENT_HDMA3_BM_SPMU: 10072 case GAUDI2_EVENT_HDMA3_QM: 10073 case GAUDI2_EVENT_HDMA3_CORE: 10074 return GAUDI2_DCORE1_ENGINE_ID_EDMA_1; 10075 case GAUDI2_EVENT_HDMA4_BM_SPMU: 10076 case GAUDI2_EVENT_HDMA4_QM: 10077 case GAUDI2_EVENT_HDMA4_CORE: 10078 return GAUDI2_DCORE2_ENGINE_ID_EDMA_0; 10079 case GAUDI2_EVENT_HDMA5_BM_SPMU: 10080 case GAUDI2_EVENT_HDMA5_QM: 10081 case GAUDI2_EVENT_HDMA5_CORE: 10082 return GAUDI2_DCORE2_ENGINE_ID_EDMA_1; 10083 case GAUDI2_EVENT_HDMA6_BM_SPMU: 10084 case GAUDI2_EVENT_HDMA6_QM: 10085 case GAUDI2_EVENT_HDMA6_CORE: 10086 return GAUDI2_DCORE3_ENGINE_ID_EDMA_0; 10087 case GAUDI2_EVENT_HDMA7_BM_SPMU: 10088 case GAUDI2_EVENT_HDMA7_QM: 10089 case GAUDI2_EVENT_HDMA7_CORE: 10090 return GAUDI2_DCORE3_ENGINE_ID_EDMA_1; 10091 default: 10092 break; 10093 } 10094 10095 switch (type) { 10096 case GAUDI2_BLOCK_TYPE_TPC: 10097 switch (index) { 10098 case TPC_ID_DCORE0_TPC0 ... TPC_ID_DCORE0_TPC5: 10099 return GAUDI2_DCORE0_ENGINE_ID_TPC_0 + index; 10100 case TPC_ID_DCORE1_TPC0 ... TPC_ID_DCORE1_TPC5: 10101 return GAUDI2_DCORE1_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE1_TPC0; 10102 case TPC_ID_DCORE2_TPC0 ... TPC_ID_DCORE2_TPC5: 10103 return GAUDI2_DCORE2_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE2_TPC0; 10104 case TPC_ID_DCORE3_TPC0 ... TPC_ID_DCORE3_TPC5: 10105 return GAUDI2_DCORE3_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE3_TPC0; 10106 default: 10107 break; 10108 } 10109 break; 10110 case GAUDI2_BLOCK_TYPE_MME: 10111 switch (index) { 10112 case MME_ID_DCORE0: return GAUDI2_DCORE0_ENGINE_ID_MME; 10113 case MME_ID_DCORE1: return GAUDI2_DCORE1_ENGINE_ID_MME; 10114 case MME_ID_DCORE2: return GAUDI2_DCORE2_ENGINE_ID_MME; 10115 case MME_ID_DCORE3: return GAUDI2_DCORE3_ENGINE_ID_MME; 10116 default: 10117 break; 10118 } 10119 break; 10120 case GAUDI2_BLOCK_TYPE_DEC: 10121 switch (index) { 10122 case DEC_ID_DCORE0_DEC0: return GAUDI2_DCORE0_ENGINE_ID_DEC_0; 10123 case DEC_ID_DCORE0_DEC1: return GAUDI2_DCORE0_ENGINE_ID_DEC_1; 10124 case DEC_ID_DCORE1_DEC0: return GAUDI2_DCORE1_ENGINE_ID_DEC_0; 10125 case DEC_ID_DCORE1_DEC1: return GAUDI2_DCORE1_ENGINE_ID_DEC_1; 10126 case DEC_ID_DCORE2_DEC0: return GAUDI2_DCORE2_ENGINE_ID_DEC_0; 10127 case DEC_ID_DCORE2_DEC1: return GAUDI2_DCORE2_ENGINE_ID_DEC_1; 10128 case DEC_ID_DCORE3_DEC0: return GAUDI2_DCORE3_ENGINE_ID_DEC_0; 10129 case DEC_ID_DCORE3_DEC1: return GAUDI2_DCORE3_ENGINE_ID_DEC_1; 10130 case DEC_ID_PCIE_VDEC0: return GAUDI2_PCIE_ENGINE_ID_DEC_0; 10131 case DEC_ID_PCIE_VDEC1: return GAUDI2_PCIE_ENGINE_ID_DEC_1; 10132 default: 10133 break; 10134 } 10135 break; 10136 default: 10137 break; 10138 } 10139 10140 return U16_MAX; 10141} 10142 10143static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry) 10144{ 10145 struct gaudi2_device *gaudi2 = hdev->asic_specific; 10146 bool reset_required = false, is_critical = false; 10147 u32 index, ctl, reset_flags = 0, error_count = 0; 10148 u64 event_mask = 0; 10149 u16 event_type; 10150 10151 ctl = le32_to_cpu(eq_entry->hdr.ctl); 10152 event_type = ((ctl & EQ_CTL_EVENT_TYPE_MASK) >> EQ_CTL_EVENT_TYPE_SHIFT); 10153 10154 if (event_type >= GAUDI2_EVENT_SIZE) { 10155 dev_err(hdev->dev, "Event type %u exceeds maximum of %u", 10156 event_type, GAUDI2_EVENT_SIZE - 1); 10157 return; 10158 } 10159 10160 gaudi2->events_stat[event_type]++; 10161 gaudi2->events_stat_aggregate[event_type]++; 10162 10163 switch (event_type) { 10164 case GAUDI2_EVENT_PCIE_CORE_SERR ... GAUDI2_EVENT_ARC0_ECC_DERR: 10165 fallthrough; 10166 case GAUDI2_EVENT_ROTATOR0_SERR ... GAUDI2_EVENT_ROTATOR1_DERR: 10167 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10168 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10169 reset_required = gaudi2_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data); 10170 is_critical = eq_entry->ecc_data.is_critical; 10171 error_count++; 10172 break; 10173 10174 case GAUDI2_EVENT_TPC0_QM ... GAUDI2_EVENT_PDMA1_QM: 10175 fallthrough; 10176 case GAUDI2_EVENT_ROTATOR0_ROT0_QM ... GAUDI2_EVENT_ROTATOR1_ROT1_QM: 10177 fallthrough; 10178 case GAUDI2_EVENT_NIC0_QM0 ... GAUDI2_EVENT_NIC11_QM1: 10179 error_count = gaudi2_handle_qman_err(hdev, event_type, &event_mask); 10180 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10181 break; 10182 10183 case GAUDI2_EVENT_ARC_AXI_ERROR_RESPONSE_0: 10184 error_count = gaudi2_handle_arc_farm_sei_err(hdev, event_type, &event_mask); 10185 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10186 break; 10187 10188 case GAUDI2_EVENT_CPU_AXI_ERR_RSP: 10189 error_count = gaudi2_handle_cpu_sei_err(hdev, event_type); 10190 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10191 event_mask |= HL_NOTIFIER_EVENT_CRITICL_FW_ERR; 10192 break; 10193 10194 case GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP: 10195 case GAUDI2_EVENT_PDMA_CH1_AXI_ERR_RSP: 10196 error_count = gaudi2_handle_qm_sei_err(hdev, event_type, true, &event_mask); 10197 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10198 break; 10199 10200 case GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE: 10201 case GAUDI2_EVENT_ROTATOR1_AXI_ERROR_RESPONSE: 10202 index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE; 10203 error_count = gaudi2_handle_rot_err(hdev, index, event_type, 10204 &eq_entry->razwi_with_intr_cause, &event_mask); 10205 error_count += gaudi2_handle_qm_sei_err(hdev, event_type, false, &event_mask); 10206 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10207 break; 10208 10209 case GAUDI2_EVENT_TPC0_AXI_ERR_RSP ... GAUDI2_EVENT_TPC24_AXI_ERR_RSP: 10210 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP; 10211 error_count = gaudi2_tpc_ack_interrupts(hdev, index, event_type, 10212 &eq_entry->razwi_with_intr_cause, &event_mask); 10213 error_count += gaudi2_handle_qm_sei_err(hdev, event_type, false, &event_mask); 10214 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10215 break; 10216 10217 case GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE ... GAUDI2_EVENT_DEC9_AXI_ERR_RSPONSE: 10218 index = event_type - GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE; 10219 error_count = gaudi2_handle_dec_err(hdev, index, event_type, &event_mask); 10220 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10221 break; 10222 10223 case GAUDI2_EVENT_TPC0_KERNEL_ERR: 10224 case GAUDI2_EVENT_TPC1_KERNEL_ERR: 10225 case GAUDI2_EVENT_TPC2_KERNEL_ERR: 10226 case GAUDI2_EVENT_TPC3_KERNEL_ERR: 10227 case GAUDI2_EVENT_TPC4_KERNEL_ERR: 10228 case GAUDI2_EVENT_TPC5_KERNEL_ERR: 10229 case GAUDI2_EVENT_TPC6_KERNEL_ERR: 10230 case GAUDI2_EVENT_TPC7_KERNEL_ERR: 10231 case GAUDI2_EVENT_TPC8_KERNEL_ERR: 10232 case GAUDI2_EVENT_TPC9_KERNEL_ERR: 10233 case GAUDI2_EVENT_TPC10_KERNEL_ERR: 10234 case GAUDI2_EVENT_TPC11_KERNEL_ERR: 10235 case GAUDI2_EVENT_TPC12_KERNEL_ERR: 10236 case GAUDI2_EVENT_TPC13_KERNEL_ERR: 10237 case GAUDI2_EVENT_TPC14_KERNEL_ERR: 10238 case GAUDI2_EVENT_TPC15_KERNEL_ERR: 10239 case GAUDI2_EVENT_TPC16_KERNEL_ERR: 10240 case GAUDI2_EVENT_TPC17_KERNEL_ERR: 10241 case GAUDI2_EVENT_TPC18_KERNEL_ERR: 10242 case GAUDI2_EVENT_TPC19_KERNEL_ERR: 10243 case GAUDI2_EVENT_TPC20_KERNEL_ERR: 10244 case GAUDI2_EVENT_TPC21_KERNEL_ERR: 10245 case GAUDI2_EVENT_TPC22_KERNEL_ERR: 10246 case GAUDI2_EVENT_TPC23_KERNEL_ERR: 10247 case GAUDI2_EVENT_TPC24_KERNEL_ERR: 10248 index = (event_type - GAUDI2_EVENT_TPC0_KERNEL_ERR) / 10249 (GAUDI2_EVENT_TPC1_KERNEL_ERR - GAUDI2_EVENT_TPC0_KERNEL_ERR); 10250 error_count = gaudi2_tpc_ack_interrupts(hdev, index, event_type, 10251 &eq_entry->razwi_with_intr_cause, &event_mask); 10252 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10253 break; 10254 10255 case GAUDI2_EVENT_DEC0_SPI: 10256 case GAUDI2_EVENT_DEC1_SPI: 10257 case GAUDI2_EVENT_DEC2_SPI: 10258 case GAUDI2_EVENT_DEC3_SPI: 10259 case GAUDI2_EVENT_DEC4_SPI: 10260 case GAUDI2_EVENT_DEC5_SPI: 10261 case GAUDI2_EVENT_DEC6_SPI: 10262 case GAUDI2_EVENT_DEC7_SPI: 10263 case GAUDI2_EVENT_DEC8_SPI: 10264 case GAUDI2_EVENT_DEC9_SPI: 10265 index = (event_type - GAUDI2_EVENT_DEC0_SPI) / 10266 (GAUDI2_EVENT_DEC1_SPI - GAUDI2_EVENT_DEC0_SPI); 10267 error_count = gaudi2_handle_dec_err(hdev, index, event_type, &event_mask); 10268 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10269 break; 10270 10271 case GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE: 10272 case GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE: 10273 case GAUDI2_EVENT_MME2_CTRL_AXI_ERROR_RESPONSE: 10274 case GAUDI2_EVENT_MME3_CTRL_AXI_ERROR_RESPONSE: 10275 index = (event_type - GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE) / 10276 (GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE - 10277 GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE); 10278 error_count = gaudi2_handle_mme_err(hdev, index, event_type, &event_mask); 10279 error_count += gaudi2_handle_qm_sei_err(hdev, event_type, false, &event_mask); 10280 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10281 break; 10282 10283 case GAUDI2_EVENT_MME0_QMAN_SW_ERROR: 10284 case GAUDI2_EVENT_MME1_QMAN_SW_ERROR: 10285 case GAUDI2_EVENT_MME2_QMAN_SW_ERROR: 10286 case GAUDI2_EVENT_MME3_QMAN_SW_ERROR: 10287 index = (event_type - GAUDI2_EVENT_MME0_QMAN_SW_ERROR) / 10288 (GAUDI2_EVENT_MME1_QMAN_SW_ERROR - 10289 GAUDI2_EVENT_MME0_QMAN_SW_ERROR); 10290 error_count = gaudi2_handle_mme_err(hdev, index, event_type, &event_mask); 10291 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10292 break; 10293 10294 case GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID: 10295 case GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID: 10296 case GAUDI2_EVENT_MME2_WAP_SOURCE_RESULT_INVALID: 10297 case GAUDI2_EVENT_MME3_WAP_SOURCE_RESULT_INVALID: 10298 index = (event_type - GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID) / 10299 (GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID - 10300 GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID); 10301 error_count = gaudi2_handle_mme_wap_err(hdev, index, event_type, &event_mask); 10302 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10303 break; 10304 10305 case GAUDI2_EVENT_KDMA_CH0_AXI_ERR_RSP: 10306 case GAUDI2_EVENT_KDMA0_CORE: 10307 error_count = gaudi2_handle_kdma_core_event(hdev, event_type, 10308 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 10309 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10310 break; 10311 10312 case GAUDI2_EVENT_HDMA2_CORE ... GAUDI2_EVENT_HDMA5_CORE: 10313 error_count = gaudi2_handle_dma_core_event(hdev, event_type, 10314 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 10315 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10316 break; 10317 10318 case GAUDI2_EVENT_PDMA0_CORE ... GAUDI2_EVENT_PDMA1_CORE: 10319 error_count = gaudi2_handle_dma_core_event(hdev, event_type, 10320 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 10321 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10322 break; 10323 10324 case GAUDI2_EVENT_PCIE_ADDR_DEC_ERR: 10325 error_count = gaudi2_print_pcie_addr_dec_info(hdev, event_type, 10326 le64_to_cpu(eq_entry->intr_cause.intr_cause_data), &event_mask); 10327 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10328 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10329 break; 10330 10331 case GAUDI2_EVENT_HMMU0_PAGE_FAULT_OR_WR_PERM ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR: 10332 case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP ... GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP: 10333 case GAUDI2_EVENT_PMMU0_PAGE_FAULT_WR_PERM ... GAUDI2_EVENT_PMMU0_SECURITY_ERROR: 10334 case GAUDI2_EVENT_PMMU_AXI_ERR_RSP_0: 10335 error_count = gaudi2_handle_mmu_spi_sei_err(hdev, event_type, &event_mask); 10336 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10337 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10338 break; 10339 10340 case GAUDI2_EVENT_HIF0_FATAL ... GAUDI2_EVENT_HIF12_FATAL: 10341 error_count = gaudi2_handle_hif_fatal(hdev, event_type, 10342 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 10343 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10344 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10345 break; 10346 10347 case GAUDI2_EVENT_PMMU_FATAL_0: 10348 error_count = gaudi2_handle_pif_fatal(hdev, event_type, 10349 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 10350 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10351 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10352 break; 10353 10354 case GAUDI2_EVENT_PSOC63_RAZWI_OR_PID_MIN_MAX_INTERRUPT: 10355 error_count = gaudi2_ack_psoc_razwi_event_handler(hdev, &event_mask); 10356 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10357 break; 10358 10359 case GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE ... GAUDI2_EVENT_HBM5_MC1_SEI_NON_SEVERE: 10360 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10361 if (gaudi2_handle_hbm_mc_sei_err(hdev, event_type, &eq_entry->sei_data)) { 10362 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10363 reset_required = true; 10364 is_critical = eq_entry->sei_data.hdr.is_critical; 10365 } 10366 error_count++; 10367 break; 10368 10369 case GAUDI2_EVENT_HBM_CATTRIP_0 ... GAUDI2_EVENT_HBM_CATTRIP_5: 10370 error_count = gaudi2_handle_hbm_cattrip(hdev, event_type, 10371 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 10372 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10373 break; 10374 10375 case GAUDI2_EVENT_HBM0_MC0_SPI ... GAUDI2_EVENT_HBM5_MC1_SPI: 10376 error_count = gaudi2_handle_hbm_mc_spi(hdev, 10377 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 10378 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10379 break; 10380 10381 case GAUDI2_EVENT_PCIE_DRAIN_COMPLETE: 10382 error_count = gaudi2_handle_pcie_drain(hdev, &eq_entry->pcie_drain_ind_data); 10383 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10384 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10385 if (hl_fw_version_cmp(hdev, 1, 13, 0) >= 0) 10386 is_critical = true; 10387 break; 10388 10389 case GAUDI2_EVENT_PSOC59_RPM_ERROR_OR_DRAIN: 10390 error_count = gaudi2_handle_psoc_drain(hdev, 10391 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 10392 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10393 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10394 break; 10395 10396 case GAUDI2_EVENT_CPU_AXI_ECC: 10397 error_count = GAUDI2_NA_EVENT_CAUSE; 10398 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10399 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10400 break; 10401 case GAUDI2_EVENT_CPU_L2_RAM_ECC: 10402 error_count = GAUDI2_NA_EVENT_CAUSE; 10403 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10404 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10405 break; 10406 case GAUDI2_EVENT_MME0_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME0_SBTE4_AXI_ERR_RSP: 10407 case GAUDI2_EVENT_MME1_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME1_SBTE4_AXI_ERR_RSP: 10408 case GAUDI2_EVENT_MME2_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME2_SBTE4_AXI_ERR_RSP: 10409 case GAUDI2_EVENT_MME3_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME3_SBTE4_AXI_ERR_RSP: 10410 error_count = gaudi2_handle_mme_sbte_err(hdev, event_type); 10411 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10412 break; 10413 case GAUDI2_EVENT_VM0_ALARM_A ... GAUDI2_EVENT_VM3_ALARM_B: 10414 error_count = GAUDI2_NA_EVENT_CAUSE; 10415 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10416 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10417 break; 10418 case GAUDI2_EVENT_PSOC_AXI_ERR_RSP: 10419 error_count = GAUDI2_NA_EVENT_CAUSE; 10420 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10421 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10422 break; 10423 case GAUDI2_EVENT_PSOC_PRSTN_FALL: 10424 error_count = GAUDI2_NA_EVENT_CAUSE; 10425 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10426 break; 10427 case GAUDI2_EVENT_PCIE_APB_TIMEOUT: 10428 error_count = GAUDI2_NA_EVENT_CAUSE; 10429 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10430 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10431 break; 10432 case GAUDI2_EVENT_PCIE_FATAL_ERR: 10433 error_count = GAUDI2_NA_EVENT_CAUSE; 10434 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10435 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10436 break; 10437 case GAUDI2_EVENT_TPC0_BMON_SPMU: 10438 case GAUDI2_EVENT_TPC1_BMON_SPMU: 10439 case GAUDI2_EVENT_TPC2_BMON_SPMU: 10440 case GAUDI2_EVENT_TPC3_BMON_SPMU: 10441 case GAUDI2_EVENT_TPC4_BMON_SPMU: 10442 case GAUDI2_EVENT_TPC5_BMON_SPMU: 10443 case GAUDI2_EVENT_TPC6_BMON_SPMU: 10444 case GAUDI2_EVENT_TPC7_BMON_SPMU: 10445 case GAUDI2_EVENT_TPC8_BMON_SPMU: 10446 case GAUDI2_EVENT_TPC9_BMON_SPMU: 10447 case GAUDI2_EVENT_TPC10_BMON_SPMU: 10448 case GAUDI2_EVENT_TPC11_BMON_SPMU: 10449 case GAUDI2_EVENT_TPC12_BMON_SPMU: 10450 case GAUDI2_EVENT_TPC13_BMON_SPMU: 10451 case GAUDI2_EVENT_TPC14_BMON_SPMU: 10452 case GAUDI2_EVENT_TPC15_BMON_SPMU: 10453 case GAUDI2_EVENT_TPC16_BMON_SPMU: 10454 case GAUDI2_EVENT_TPC17_BMON_SPMU: 10455 case GAUDI2_EVENT_TPC18_BMON_SPMU: 10456 case GAUDI2_EVENT_TPC19_BMON_SPMU: 10457 case GAUDI2_EVENT_TPC20_BMON_SPMU: 10458 case GAUDI2_EVENT_TPC21_BMON_SPMU: 10459 case GAUDI2_EVENT_TPC22_BMON_SPMU: 10460 case GAUDI2_EVENT_TPC23_BMON_SPMU: 10461 case GAUDI2_EVENT_TPC24_BMON_SPMU: 10462 case GAUDI2_EVENT_MME0_CTRL_BMON_SPMU: 10463 case GAUDI2_EVENT_MME0_SBTE_BMON_SPMU: 10464 case GAUDI2_EVENT_MME0_WAP_BMON_SPMU: 10465 case GAUDI2_EVENT_MME1_CTRL_BMON_SPMU: 10466 case GAUDI2_EVENT_MME1_SBTE_BMON_SPMU: 10467 case GAUDI2_EVENT_MME1_WAP_BMON_SPMU: 10468 case GAUDI2_EVENT_MME2_CTRL_BMON_SPMU: 10469 case GAUDI2_EVENT_MME2_SBTE_BMON_SPMU: 10470 case GAUDI2_EVENT_MME2_WAP_BMON_SPMU: 10471 case GAUDI2_EVENT_MME3_CTRL_BMON_SPMU: 10472 case GAUDI2_EVENT_MME3_SBTE_BMON_SPMU: 10473 case GAUDI2_EVENT_MME3_WAP_BMON_SPMU: 10474 case GAUDI2_EVENT_HDMA2_BM_SPMU ... GAUDI2_EVENT_PDMA1_BM_SPMU: 10475 fallthrough; 10476 case GAUDI2_EVENT_DEC0_BMON_SPMU: 10477 case GAUDI2_EVENT_DEC1_BMON_SPMU: 10478 case GAUDI2_EVENT_DEC2_BMON_SPMU: 10479 case GAUDI2_EVENT_DEC3_BMON_SPMU: 10480 case GAUDI2_EVENT_DEC4_BMON_SPMU: 10481 case GAUDI2_EVENT_DEC5_BMON_SPMU: 10482 case GAUDI2_EVENT_DEC6_BMON_SPMU: 10483 case GAUDI2_EVENT_DEC7_BMON_SPMU: 10484 case GAUDI2_EVENT_DEC8_BMON_SPMU: 10485 case GAUDI2_EVENT_DEC9_BMON_SPMU: 10486 case GAUDI2_EVENT_ROTATOR0_BMON_SPMU ... GAUDI2_EVENT_SM3_BMON_SPMU: 10487 error_count = GAUDI2_NA_EVENT_CAUSE; 10488 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10489 break; 10490 10491 case GAUDI2_EVENT_CPU_FIX_POWER_ENV_S: 10492 case GAUDI2_EVENT_CPU_FIX_POWER_ENV_E: 10493 case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_S: 10494 case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_E: 10495 gaudi2_print_clk_change_info(hdev, event_type, &event_mask); 10496 error_count = GAUDI2_NA_EVENT_CAUSE; 10497 break; 10498 10499 case GAUDI2_EVENT_CPU_PKT_QUEUE_OUT_SYNC: 10500 gaudi2_print_out_of_sync_info(hdev, event_type, &eq_entry->pkt_sync_err); 10501 error_count = GAUDI2_NA_EVENT_CAUSE; 10502 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10503 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10504 break; 10505 10506 case GAUDI2_EVENT_PCIE_FLR_REQUESTED: 10507 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10508 error_count = GAUDI2_NA_EVENT_CAUSE; 10509 /* Do nothing- FW will handle it */ 10510 break; 10511 10512 case GAUDI2_EVENT_PCIE_P2P_MSIX: 10513 error_count = gaudi2_handle_pcie_p2p_msix(hdev, event_type); 10514 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10515 break; 10516 10517 case GAUDI2_EVENT_SM0_AXI_ERROR_RESPONSE ... GAUDI2_EVENT_SM3_AXI_ERROR_RESPONSE: 10518 index = event_type - GAUDI2_EVENT_SM0_AXI_ERROR_RESPONSE; 10519 error_count = gaudi2_handle_sm_err(hdev, event_type, index); 10520 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10521 break; 10522 10523 case GAUDI2_EVENT_PSOC_MME_PLL_LOCK_ERR ... GAUDI2_EVENT_DCORE2_HBM_PLL_LOCK_ERR: 10524 error_count = GAUDI2_NA_EVENT_CAUSE; 10525 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10526 break; 10527 10528 case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_CAUSE: 10529 dev_info(hdev->dev, "CPLD shutdown cause, reset reason: 0x%llx\n", 10530 le64_to_cpu(eq_entry->data[0])); 10531 error_count = GAUDI2_NA_EVENT_CAUSE; 10532 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10533 break; 10534 case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_EVENT: 10535 dev_err(hdev->dev, "CPLD shutdown event, reset reason: 0x%llx\n", 10536 le64_to_cpu(eq_entry->data[0])); 10537 error_count = GAUDI2_NA_EVENT_CAUSE; 10538 hl_eq_cpld_shutdown_event_handle(hdev, event_type, &event_mask); 10539 break; 10540 10541 case GAUDI2_EVENT_CPU_PKT_SANITY_FAILED: 10542 gaudi2_print_cpu_pkt_failure_info(hdev, event_type, &eq_entry->pkt_sync_err); 10543 error_count = GAUDI2_NA_EVENT_CAUSE; 10544 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10545 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10546 break; 10547 10548 case GAUDI2_EVENT_ARC_DCCM_FULL: 10549 error_count = hl_arc_event_handle(hdev, event_type, &eq_entry->arc_data); 10550 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10551 break; 10552 10553 case GAUDI2_EVENT_CPU_FP32_NOT_SUPPORTED: 10554 case GAUDI2_EVENT_CPU_DEV_RESET_REQ: 10555 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10556 error_count = GAUDI2_NA_EVENT_CAUSE; 10557 is_critical = true; 10558 break; 10559 10560 case GAUDI2_EVENT_ARC_PWR_BRK_ENTRY: 10561 case GAUDI2_EVENT_ARC_PWR_BRK_EXT: 10562 case GAUDI2_EVENT_ARC_PWR_RD_MODE0: 10563 case GAUDI2_EVENT_ARC_PWR_RD_MODE1: 10564 case GAUDI2_EVENT_ARC_PWR_RD_MODE2: 10565 case GAUDI2_EVENT_ARC_PWR_RD_MODE3: 10566 error_count = GAUDI2_NA_EVENT_CAUSE; 10567 dev_info_ratelimited(hdev->dev, "%s event received\n", 10568 gaudi2_irq_map_table[event_type].name); 10569 break; 10570 10571 case GAUDI2_EVENT_ARC_EQ_HEARTBEAT: 10572 hl_eq_heartbeat_event_handle(hdev); 10573 error_count = GAUDI2_NA_EVENT_CAUSE; 10574 break; 10575 default: 10576 if (gaudi2_irq_map_table[event_type].valid) { 10577 dev_err_ratelimited(hdev->dev, "Cannot find handler for event %d\n", 10578 event_type); 10579 error_count = GAUDI2_NA_EVENT_CAUSE; 10580 } 10581 } 10582 10583 if (event_mask & HL_NOTIFIER_EVENT_USER_ENGINE_ERR) 10584 hl_capture_engine_err(hdev, event_id_to_engine_id(hdev, event_type), error_count); 10585 10586 /* Make sure to dump an error in case no error cause was printed so far. 10587 * Note that although we have counted the errors, we use this number as 10588 * a boolean. 10589 */ 10590 if (error_count == GAUDI2_NA_EVENT_CAUSE && !is_info_event(event_type)) 10591 gaudi2_print_event(hdev, event_type, true, "%d", event_type); 10592 else if (error_count == 0) 10593 gaudi2_print_event(hdev, event_type, true, 10594 "No error cause for H/W event %u", event_type); 10595 10596 if ((gaudi2_irq_map_table[event_type].reset != EVENT_RESET_TYPE_NONE) || reset_required) { 10597 if (reset_required || 10598 (gaudi2_irq_map_table[event_type].reset == EVENT_RESET_TYPE_HARD)) 10599 reset_flags |= HL_DRV_RESET_HARD; 10600 10601 if (hdev->hard_reset_on_fw_events || 10602 (hdev->asic_prop.fw_security_enabled && is_critical)) 10603 goto reset_device; 10604 } 10605 10606 /* Send unmask irq only for interrupts not classified as MSG */ 10607 if (!gaudi2_irq_map_table[event_type].msg) 10608 hl_fw_unmask_irq(hdev, event_type); 10609 10610 if (event_mask) 10611 hl_notifier_event_send_all(hdev, event_mask); 10612 10613 return; 10614 10615reset_device: 10616 if (hdev->asic_prop.fw_security_enabled && is_critical) { 10617 reset_flags |= HL_DRV_RESET_BYPASS_REQ_TO_FW; 10618 event_mask |= HL_NOTIFIER_EVENT_DEVICE_UNAVAILABLE; 10619 } else { 10620 reset_flags |= HL_DRV_RESET_DELAY; 10621 } 10622 /* escalate general hw errors to critical/fatal error */ 10623 if (event_mask & HL_NOTIFIER_EVENT_GENERAL_HW_ERR) 10624 hl_handle_critical_hw_err(hdev, event_type, &event_mask); 10625 10626 hl_debugfs_cfg_access_history_dump(hdev); 10627 event_mask |= HL_NOTIFIER_EVENT_DEVICE_RESET; 10628 hl_device_cond_reset(hdev, reset_flags, event_mask); 10629} 10630 10631static int gaudi2_memset_memory_chunk_using_edma_qm(struct hl_device *hdev, 10632 struct packet_lin_dma *lin_dma_pkt, 10633 u64 phys_addr, u32 hw_queue_id, u32 size, u64 addr, u32 val) 10634{ 10635 u32 ctl, pkt_size; 10636 int rc = 0, i; 10637 10638 ctl = FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_LIN_DMA); 10639 ctl |= FIELD_PREP(GAUDI2_PKT_LIN_DMA_CTL_MEMSET_MASK, 1); 10640 ctl |= FIELD_PREP(GAUDI2_PKT_LIN_DMA_CTL_WRCOMP_MASK, 1); 10641 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 1); 10642 10643 lin_dma_pkt->ctl = cpu_to_le32(ctl); 10644 lin_dma_pkt->src_addr = cpu_to_le64(val); 10645 lin_dma_pkt->dst_addr = cpu_to_le64(addr); 10646 lin_dma_pkt->tsize = cpu_to_le32(size); 10647 10648 pkt_size = sizeof(struct packet_lin_dma); 10649 10650 for (i = 0; i < 3; i++) { 10651 rc = hdev->asic_funcs->access_dev_mem(hdev, PCI_REGION_DRAM, 10652 phys_addr + (i * sizeof(u64)), 10653 ((u64 *)(lin_dma_pkt)) + i, DEBUGFS_WRITE64); 10654 if (rc) { 10655 dev_err(hdev->dev, "Failed to copy lin_dma packet to HBM (%#llx)\n", 10656 phys_addr); 10657 return rc; 10658 } 10659 } 10660 10661 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, pkt_size, phys_addr); 10662 if (rc) 10663 dev_err(hdev->dev, "Failed to send lin_dma packet to H/W queue %s\n", 10664 GAUDI2_QUEUE_ID_TO_STR(hw_queue_id)); 10665 10666 return rc; 10667} 10668 10669static int gaudi2_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, u64 val) 10670{ 10671 u32 edma_queues_id[] = {GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0, 10672 GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0, 10673 GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0, 10674 GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0}; 10675 u32 chunk_size, dcore, edma_idx, sob_offset, sob_addr, comp_val, 10676 old_mmubp, mmubp, num_of_pkts, busy, pkt_size, cb_len; 10677 u64 comp_addr, cur_addr = addr, end_addr = addr + size; 10678 struct asic_fixed_properties *prop = &hdev->asic_prop; 10679 int rc = 0, dma_num = 0, i; 10680 void *lin_dma_pkts_arr; 10681 10682 if (prop->edma_enabled_mask == 0) { 10683 dev_info(hdev->dev, "non of the EDMA engines is enabled - skip dram scrubbing\n"); 10684 return -EIO; 10685 } 10686 10687 sob_offset = hdev->asic_prop.first_available_user_sob[0] * 4; 10688 sob_addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset; 10689 comp_addr = CFG_BASE + sob_addr; 10690 comp_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1) | 10691 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 1); 10692 mmubp = FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_WR_MASK, 1) | 10693 FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_RD_MASK, 1); 10694 10695 /* Calculate how many lin dma pkts we'll need */ 10696 num_of_pkts = div64_u64(round_up(size, SZ_2G), SZ_2G); 10697 pkt_size = sizeof(struct packet_lin_dma); 10698 cb_len = pkt_size * num_of_pkts; 10699 10700 /* 10701 * if we're not scrubing HMMU or NIC reserved sections in hbm, 10702 * then it the scrubing of the user section, as we use the start of the user section 10703 * to store the CB of the EDMA QM, so shift the start address of the scrubbing accordingly 10704 * and scrub the CB section before leaving this function. 10705 */ 10706 if ((addr >= prop->dram_user_base_address) && 10707 (addr < prop->dram_user_base_address + cb_len)) 10708 cur_addr += (prop->dram_user_base_address + cb_len) - addr; 10709 10710 lin_dma_pkts_arr = kvcalloc(num_of_pkts, pkt_size, GFP_KERNEL); 10711 if (!lin_dma_pkts_arr) 10712 return -ENOMEM; 10713 10714 /* 10715 * set mmu bypass for the scrubbing - all ddmas are configured the same so save 10716 * only the first one to restore later 10717 * also set the sob addr for all edma cores for completion. 10718 * set QM as trusted to allow it to access physical address with MMU bp. 10719 */ 10720 old_mmubp = RREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP); 10721 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 10722 for (edma_idx = 0 ; edma_idx < NUM_OF_EDMA_PER_DCORE ; edma_idx++) { 10723 u32 edma_offset = dcore * DCORE_OFFSET + edma_idx * DCORE_EDMA_OFFSET; 10724 u32 edma_bit = dcore * NUM_OF_EDMA_PER_DCORE + edma_idx; 10725 10726 if (!(prop->edma_enabled_mask & BIT(edma_bit))) 10727 continue; 10728 10729 WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP + 10730 edma_offset, mmubp); 10731 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_LO + edma_offset, 10732 lower_32_bits(comp_addr)); 10733 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_HI + edma_offset, 10734 upper_32_bits(comp_addr)); 10735 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_WDATA + edma_offset, 10736 comp_val); 10737 gaudi2_qman_set_test_mode(hdev, 10738 edma_queues_id[dcore] + 4 * edma_idx, true); 10739 } 10740 } 10741 10742 WREG32(sob_addr, 0); 10743 10744 while (cur_addr < end_addr) { 10745 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 10746 for (edma_idx = 0 ; edma_idx < NUM_OF_EDMA_PER_DCORE ; edma_idx++) { 10747 u32 edma_bit = dcore * NUM_OF_EDMA_PER_DCORE + edma_idx; 10748 10749 if (!(prop->edma_enabled_mask & BIT(edma_bit))) 10750 continue; 10751 10752 chunk_size = min_t(u64, SZ_2G, end_addr - cur_addr); 10753 10754 rc = gaudi2_memset_memory_chunk_using_edma_qm(hdev, 10755 (struct packet_lin_dma *)lin_dma_pkts_arr + dma_num, 10756 prop->dram_user_base_address + (dma_num * pkt_size), 10757 edma_queues_id[dcore] + edma_idx * 4, 10758 chunk_size, cur_addr, val); 10759 if (rc) 10760 goto end; 10761 10762 dma_num++; 10763 cur_addr += chunk_size; 10764 if (cur_addr == end_addr) 10765 goto edma_wait; 10766 } 10767 } 10768 } 10769 10770edma_wait: 10771 rc = hl_poll_timeout(hdev, sob_addr, busy, (busy == dma_num), 1000, 1000000); 10772 if (rc) { 10773 dev_err(hdev->dev, "DMA Timeout during HBM scrubbing(sob: 0x%x, dma_num: 0x%x)\n", 10774 busy, dma_num); 10775 goto end; 10776 } 10777end: 10778 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 10779 for (edma_idx = 0 ; edma_idx < NUM_OF_EDMA_PER_DCORE ; edma_idx++) { 10780 u32 edma_offset = dcore * DCORE_OFFSET + edma_idx * DCORE_EDMA_OFFSET; 10781 u32 edma_bit = dcore * NUM_OF_EDMA_PER_DCORE + edma_idx; 10782 10783 if (!(prop->edma_enabled_mask & BIT(edma_bit))) 10784 continue; 10785 10786 WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP + edma_offset, old_mmubp); 10787 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_LO + edma_offset, 0); 10788 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_HI + edma_offset, 0); 10789 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_WDATA + edma_offset, 0); 10790 gaudi2_qman_set_test_mode(hdev, 10791 edma_queues_id[dcore] + 4 * edma_idx, false); 10792 } 10793 } 10794 10795 memset(lin_dma_pkts_arr, 0, sizeof(u64)); 10796 10797 /* Zero the HBM area where we copied the CB */ 10798 for (i = 0; i < cb_len / sizeof(u64); i += sizeof(u64)) 10799 rc = hdev->asic_funcs->access_dev_mem(hdev, PCI_REGION_DRAM, 10800 prop->dram_user_base_address + i, 10801 (u64 *)(lin_dma_pkts_arr), DEBUGFS_WRITE64); 10802 WREG32(sob_addr, 0); 10803 10804 kvfree(lin_dma_pkts_arr); 10805 10806 return rc; 10807} 10808 10809static int gaudi2_scrub_device_dram(struct hl_device *hdev, u64 val) 10810{ 10811 int rc; 10812 struct asic_fixed_properties *prop = &hdev->asic_prop; 10813 u64 size = prop->dram_end_address - prop->dram_user_base_address; 10814 10815 rc = gaudi2_memset_device_memory(hdev, prop->dram_user_base_address, size, val); 10816 10817 if (rc) 10818 dev_err(hdev->dev, "Failed to scrub dram, address: 0x%llx size: %llu\n", 10819 prop->dram_user_base_address, size); 10820 return rc; 10821} 10822 10823static int gaudi2_scrub_device_mem(struct hl_device *hdev) 10824{ 10825 int rc; 10826 struct asic_fixed_properties *prop = &hdev->asic_prop; 10827 u64 val = hdev->memory_scrub_val; 10828 u64 addr, size; 10829 10830 if (!hdev->memory_scrub) 10831 return 0; 10832 10833 /* scrub SRAM */ 10834 addr = prop->sram_user_base_address; 10835 size = hdev->pldm ? 0x10000 : (prop->sram_size - SRAM_USER_BASE_OFFSET); 10836 dev_dbg(hdev->dev, "Scrubbing SRAM: 0x%09llx - 0x%09llx, val: 0x%llx\n", 10837 addr, addr + size, val); 10838 rc = gaudi2_memset_device_memory(hdev, addr, size, val); 10839 if (rc) { 10840 dev_err(hdev->dev, "scrubbing SRAM failed (%d)\n", rc); 10841 return rc; 10842 } 10843 10844 /* scrub DRAM */ 10845 rc = gaudi2_scrub_device_dram(hdev, val); 10846 if (rc) { 10847 dev_err(hdev->dev, "scrubbing DRAM failed (%d)\n", rc); 10848 return rc; 10849 } 10850 return 0; 10851} 10852 10853static void gaudi2_restore_user_sm_registers(struct hl_device *hdev) 10854{ 10855 u64 addr, mon_sts_addr, mon_cfg_addr, cq_lbw_l_addr, cq_lbw_h_addr, 10856 cq_lbw_data_addr, cq_base_l_addr, cq_base_h_addr, cq_size_addr; 10857 u32 val, size, offset; 10858 int dcore_id; 10859 10860 offset = hdev->asic_prop.first_available_cq[0] * 4; 10861 cq_lbw_l_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0 + offset; 10862 cq_lbw_h_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 + offset; 10863 cq_lbw_data_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_DATA_0 + offset; 10864 cq_base_l_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_L_0 + offset; 10865 cq_base_h_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_H_0 + offset; 10866 cq_size_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_SIZE_LOG2_0 + offset; 10867 size = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 - 10868 (mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0 + offset); 10869 10870 /* memset dcore0 CQ registers */ 10871 gaudi2_memset_device_lbw(hdev, cq_lbw_l_addr, size, 0); 10872 gaudi2_memset_device_lbw(hdev, cq_lbw_h_addr, size, 0); 10873 gaudi2_memset_device_lbw(hdev, cq_lbw_data_addr, size, 0); 10874 gaudi2_memset_device_lbw(hdev, cq_base_l_addr, size, 0); 10875 gaudi2_memset_device_lbw(hdev, cq_base_h_addr, size, 0); 10876 gaudi2_memset_device_lbw(hdev, cq_size_addr, size, 0); 10877 10878 cq_lbw_l_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0 + DCORE_OFFSET; 10879 cq_lbw_h_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 + DCORE_OFFSET; 10880 cq_lbw_data_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_DATA_0 + DCORE_OFFSET; 10881 cq_base_l_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_L_0 + DCORE_OFFSET; 10882 cq_base_h_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_H_0 + DCORE_OFFSET; 10883 cq_size_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_SIZE_LOG2_0 + DCORE_OFFSET; 10884 size = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 - mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0; 10885 10886 for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { 10887 gaudi2_memset_device_lbw(hdev, cq_lbw_l_addr, size, 0); 10888 gaudi2_memset_device_lbw(hdev, cq_lbw_h_addr, size, 0); 10889 gaudi2_memset_device_lbw(hdev, cq_lbw_data_addr, size, 0); 10890 gaudi2_memset_device_lbw(hdev, cq_base_l_addr, size, 0); 10891 gaudi2_memset_device_lbw(hdev, cq_base_h_addr, size, 0); 10892 gaudi2_memset_device_lbw(hdev, cq_size_addr, size, 0); 10893 10894 cq_lbw_l_addr += DCORE_OFFSET; 10895 cq_lbw_h_addr += DCORE_OFFSET; 10896 cq_lbw_data_addr += DCORE_OFFSET; 10897 cq_base_l_addr += DCORE_OFFSET; 10898 cq_base_h_addr += DCORE_OFFSET; 10899 cq_size_addr += DCORE_OFFSET; 10900 } 10901 10902 offset = hdev->asic_prop.first_available_user_mon[0] * 4; 10903 addr = mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + offset; 10904 val = 1 << DCORE0_SYNC_MNGR_OBJS_MON_STATUS_PROT_SHIFT; 10905 size = mmDCORE0_SYNC_MNGR_OBJS_SM_SEC_0 - (mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + offset); 10906 10907 /* memset dcore0 monitors */ 10908 gaudi2_memset_device_lbw(hdev, addr, size, val); 10909 10910 addr = mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + offset; 10911 gaudi2_memset_device_lbw(hdev, addr, size, 0); 10912 10913 mon_sts_addr = mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + DCORE_OFFSET; 10914 mon_cfg_addr = mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + DCORE_OFFSET; 10915 size = mmDCORE0_SYNC_MNGR_OBJS_SM_SEC_0 - mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0; 10916 10917 for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { 10918 gaudi2_memset_device_lbw(hdev, mon_sts_addr, size, val); 10919 gaudi2_memset_device_lbw(hdev, mon_cfg_addr, size, 0); 10920 mon_sts_addr += DCORE_OFFSET; 10921 mon_cfg_addr += DCORE_OFFSET; 10922 } 10923 10924 offset = hdev->asic_prop.first_available_user_sob[0] * 4; 10925 addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + offset; 10926 val = 0; 10927 size = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 - 10928 (mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + offset); 10929 10930 /* memset dcore0 sobs */ 10931 gaudi2_memset_device_lbw(hdev, addr, size, val); 10932 10933 addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + DCORE_OFFSET; 10934 size = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 - mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0; 10935 10936 for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { 10937 gaudi2_memset_device_lbw(hdev, addr, size, val); 10938 addr += DCORE_OFFSET; 10939 } 10940 10941 /* Flush all WREG to prevent race */ 10942 val = RREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + offset); 10943} 10944 10945static void gaudi2_restore_user_qm_registers(struct hl_device *hdev) 10946{ 10947 u32 reg_base, hw_queue_id; 10948 10949 for (hw_queue_id = GAUDI2_QUEUE_ID_PDMA_0_0 ; hw_queue_id <= GAUDI2_QUEUE_ID_ROT_1_0; 10950 hw_queue_id += NUM_OF_PQ_PER_QMAN) { 10951 if (!gaudi2_is_queue_enabled(hdev, hw_queue_id)) 10952 continue; 10953 10954 gaudi2_clear_qm_fence_counters_common(hdev, hw_queue_id, false); 10955 10956 reg_base = gaudi2_qm_blocks_bases[hw_queue_id]; 10957 WREG32(reg_base + QM_ARB_CFG_0_OFFSET, 0); 10958 } 10959 10960 /* Flush all WREG to prevent race */ 10961 RREG32(mmPDMA0_QM_ARB_CFG_0); 10962} 10963 10964static void gaudi2_restore_nic_qm_registers(struct hl_device *hdev) 10965{ 10966 u32 reg_base, hw_queue_id; 10967 10968 for (hw_queue_id = GAUDI2_QUEUE_ID_NIC_0_0 ; hw_queue_id <= GAUDI2_QUEUE_ID_NIC_23_3; 10969 hw_queue_id += NUM_OF_PQ_PER_QMAN) { 10970 if (!gaudi2_is_queue_enabled(hdev, hw_queue_id)) 10971 continue; 10972 10973 gaudi2_clear_qm_fence_counters_common(hdev, hw_queue_id, false); 10974 10975 reg_base = gaudi2_qm_blocks_bases[hw_queue_id]; 10976 WREG32(reg_base + QM_ARB_CFG_0_OFFSET, 0); 10977 } 10978 10979 /* Flush all WREG to prevent race */ 10980 RREG32(mmPDMA0_QM_ARB_CFG_0); 10981} 10982 10983static int gaudi2_context_switch(struct hl_device *hdev, u32 asid) 10984{ 10985 return 0; 10986} 10987 10988static void gaudi2_restore_phase_topology(struct hl_device *hdev) 10989{ 10990} 10991 10992static void gaudi2_init_block_instances(struct hl_device *hdev, u32 block_idx, 10993 struct dup_block_ctx *cfg_ctx) 10994{ 10995 u64 block_base = cfg_ctx->base + block_idx * cfg_ctx->block_off; 10996 u8 seq; 10997 int i; 10998 10999 for (i = 0 ; i < cfg_ctx->instances ; i++) { 11000 seq = block_idx * cfg_ctx->instances + i; 11001 11002 /* skip disabled instance */ 11003 if (!(cfg_ctx->enabled_mask & BIT_ULL(seq))) 11004 continue; 11005 11006 cfg_ctx->instance_cfg_fn(hdev, block_base + i * cfg_ctx->instance_off, 11007 cfg_ctx->data); 11008 } 11009} 11010 11011static void gaudi2_init_blocks_with_mask(struct hl_device *hdev, struct dup_block_ctx *cfg_ctx, 11012 u64 mask) 11013{ 11014 int i; 11015 11016 cfg_ctx->enabled_mask = mask; 11017 11018 for (i = 0 ; i < cfg_ctx->blocks ; i++) 11019 gaudi2_init_block_instances(hdev, i, cfg_ctx); 11020} 11021 11022void gaudi2_init_blocks(struct hl_device *hdev, struct dup_block_ctx *cfg_ctx) 11023{ 11024 gaudi2_init_blocks_with_mask(hdev, cfg_ctx, U64_MAX); 11025} 11026 11027static int gaudi2_debugfs_read_dma(struct hl_device *hdev, u64 addr, u32 size, void *blob_addr) 11028{ 11029 void *host_mem_virtual_addr; 11030 dma_addr_t host_mem_dma_addr; 11031 u64 reserved_va_base; 11032 u32 pos, size_left, size_to_dma; 11033 struct hl_ctx *ctx; 11034 int rc = 0; 11035 11036 /* Fetch the ctx */ 11037 ctx = hl_get_compute_ctx(hdev); 11038 if (!ctx) { 11039 dev_err(hdev->dev, "No ctx available\n"); 11040 return -EINVAL; 11041 } 11042 11043 /* Allocate buffers for read and for poll */ 11044 host_mem_virtual_addr = hl_asic_dma_alloc_coherent(hdev, SZ_2M, &host_mem_dma_addr, 11045 GFP_KERNEL | __GFP_ZERO); 11046 if (host_mem_virtual_addr == NULL) { 11047 dev_err(hdev->dev, "Failed to allocate memory for KDMA read\n"); 11048 rc = -ENOMEM; 11049 goto put_ctx; 11050 } 11051 11052 /* Reserve VM region on asic side */ 11053 reserved_va_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST, SZ_2M, 11054 HL_MMU_VA_ALIGNMENT_NOT_NEEDED); 11055 if (!reserved_va_base) { 11056 dev_err(hdev->dev, "Failed to reserve vmem on asic\n"); 11057 rc = -ENOMEM; 11058 goto free_data_buffer; 11059 } 11060 11061 /* Create mapping on asic side */ 11062 mutex_lock(&hdev->mmu_lock); 11063 11064 rc = hl_mmu_map_contiguous(ctx, reserved_va_base, host_mem_dma_addr, SZ_2M); 11065 if (rc) { 11066 dev_err(hdev->dev, "Failed to create mapping on asic mmu\n"); 11067 goto unreserve_va; 11068 } 11069 11070 rc = hl_mmu_invalidate_cache_range(hdev, false, 11071 MMU_OP_USERPTR | MMU_OP_SKIP_LOW_CACHE_INV, 11072 ctx->asid, reserved_va_base, SZ_2M); 11073 if (rc) { 11074 hl_mmu_unmap_contiguous(ctx, reserved_va_base, SZ_2M); 11075 goto unreserve_va; 11076 } 11077 11078 mutex_unlock(&hdev->mmu_lock); 11079 11080 /* Enable MMU on KDMA */ 11081 gaudi2_kdma_set_mmbp_asid(hdev, false, ctx->asid); 11082 11083 pos = 0; 11084 size_left = size; 11085 size_to_dma = SZ_2M; 11086 11087 while (size_left > 0) { 11088 if (size_left < SZ_2M) 11089 size_to_dma = size_left; 11090 11091 rc = gaudi2_send_job_to_kdma(hdev, addr, reserved_va_base, size_to_dma, false); 11092 if (rc) 11093 break; 11094 11095 memcpy(blob_addr + pos, host_mem_virtual_addr, size_to_dma); 11096 11097 if (size_left <= SZ_2M) 11098 break; 11099 11100 pos += SZ_2M; 11101 addr += SZ_2M; 11102 size_left -= SZ_2M; 11103 } 11104 11105 gaudi2_kdma_set_mmbp_asid(hdev, true, HL_KERNEL_ASID_ID); 11106 11107 mutex_lock(&hdev->mmu_lock); 11108 11109 rc = hl_mmu_unmap_contiguous(ctx, reserved_va_base, SZ_2M); 11110 if (rc) 11111 goto unreserve_va; 11112 11113 rc = hl_mmu_invalidate_cache_range(hdev, false, MMU_OP_USERPTR, 11114 ctx->asid, reserved_va_base, SZ_2M); 11115 11116unreserve_va: 11117 mutex_unlock(&hdev->mmu_lock); 11118 hl_unreserve_va_block(hdev, ctx, reserved_va_base, SZ_2M); 11119free_data_buffer: 11120 hl_asic_dma_free_coherent(hdev, SZ_2M, host_mem_virtual_addr, host_mem_dma_addr); 11121put_ctx: 11122 hl_ctx_put(ctx); 11123 11124 return rc; 11125} 11126 11127static int gaudi2_internal_cb_pool_init(struct hl_device *hdev, struct hl_ctx *ctx) 11128{ 11129 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11130 int min_alloc_order, rc; 11131 11132 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) 11133 return 0; 11134 11135 hdev->internal_cb_pool_virt_addr = hl_asic_dma_alloc_coherent(hdev, 11136 HOST_SPACE_INTERNAL_CB_SZ, 11137 &hdev->internal_cb_pool_dma_addr, 11138 GFP_KERNEL | __GFP_ZERO); 11139 11140 if (!hdev->internal_cb_pool_virt_addr) 11141 return -ENOMEM; 11142 11143 min_alloc_order = ilog2(min(gaudi2_get_signal_cb_size(hdev), 11144 gaudi2_get_wait_cb_size(hdev))); 11145 11146 hdev->internal_cb_pool = gen_pool_create(min_alloc_order, -1); 11147 if (!hdev->internal_cb_pool) { 11148 dev_err(hdev->dev, "Failed to create internal CB pool\n"); 11149 rc = -ENOMEM; 11150 goto free_internal_cb_pool; 11151 } 11152 11153 rc = gen_pool_add(hdev->internal_cb_pool, (uintptr_t) hdev->internal_cb_pool_virt_addr, 11154 HOST_SPACE_INTERNAL_CB_SZ, -1); 11155 if (rc) { 11156 dev_err(hdev->dev, "Failed to add memory to internal CB pool\n"); 11157 rc = -EFAULT; 11158 goto destroy_internal_cb_pool; 11159 } 11160 11161 hdev->internal_cb_va_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST, 11162 HOST_SPACE_INTERNAL_CB_SZ, HL_MMU_VA_ALIGNMENT_NOT_NEEDED); 11163 11164 if (!hdev->internal_cb_va_base) { 11165 rc = -ENOMEM; 11166 goto destroy_internal_cb_pool; 11167 } 11168 11169 mutex_lock(&hdev->mmu_lock); 11170 11171 rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base, hdev->internal_cb_pool_dma_addr, 11172 HOST_SPACE_INTERNAL_CB_SZ); 11173 if (rc) 11174 goto unreserve_internal_cb_pool; 11175 11176 rc = hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR); 11177 if (rc) 11178 goto unmap_internal_cb_pool; 11179 11180 mutex_unlock(&hdev->mmu_lock); 11181 11182 return 0; 11183 11184unmap_internal_cb_pool: 11185 hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); 11186unreserve_internal_cb_pool: 11187 mutex_unlock(&hdev->mmu_lock); 11188 hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); 11189destroy_internal_cb_pool: 11190 gen_pool_destroy(hdev->internal_cb_pool); 11191free_internal_cb_pool: 11192 hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr, 11193 hdev->internal_cb_pool_dma_addr); 11194 11195 return rc; 11196} 11197 11198static void gaudi2_internal_cb_pool_fini(struct hl_device *hdev, struct hl_ctx *ctx) 11199{ 11200 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11201 11202 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) 11203 return; 11204 11205 mutex_lock(&hdev->mmu_lock); 11206 hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); 11207 hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); 11208 hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR); 11209 mutex_unlock(&hdev->mmu_lock); 11210 11211 gen_pool_destroy(hdev->internal_cb_pool); 11212 11213 hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr, 11214 hdev->internal_cb_pool_dma_addr); 11215} 11216 11217static void gaudi2_restore_user_registers(struct hl_device *hdev) 11218{ 11219 gaudi2_restore_user_sm_registers(hdev); 11220 gaudi2_restore_user_qm_registers(hdev); 11221} 11222 11223static int gaudi2_map_virtual_msix_doorbell_memory(struct hl_ctx *ctx) 11224{ 11225 struct hl_device *hdev = ctx->hdev; 11226 struct asic_fixed_properties *prop = &hdev->asic_prop; 11227 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11228 int rc; 11229 11230 rc = hl_mmu_map_page(ctx, RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START, 11231 gaudi2->virt_msix_db_dma_addr, prop->pmmu.page_size, true); 11232 if (rc) 11233 dev_err(hdev->dev, "Failed to map VA %#llx for virtual MSI-X doorbell memory\n", 11234 RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START); 11235 11236 return rc; 11237} 11238 11239static void gaudi2_unmap_virtual_msix_doorbell_memory(struct hl_ctx *ctx) 11240{ 11241 struct hl_device *hdev = ctx->hdev; 11242 struct asic_fixed_properties *prop = &hdev->asic_prop; 11243 int rc; 11244 11245 rc = hl_mmu_unmap_page(ctx, RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START, 11246 prop->pmmu.page_size, true); 11247 if (rc) 11248 dev_err(hdev->dev, "Failed to unmap VA %#llx of virtual MSI-X doorbell memory\n", 11249 RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START); 11250} 11251 11252static int gaudi2_ctx_init(struct hl_ctx *ctx) 11253{ 11254 int rc; 11255 11256 if (ctx->asid == HL_KERNEL_ASID_ID) 11257 return 0; 11258 11259 rc = gaudi2_mmu_prepare(ctx->hdev, ctx->asid); 11260 if (rc) 11261 return rc; 11262 11263 /* No need to clear user registers if the device has just 11264 * performed reset, we restore only nic qm registers 11265 */ 11266 if (ctx->hdev->reset_upon_device_release) 11267 gaudi2_restore_nic_qm_registers(ctx->hdev); 11268 else 11269 gaudi2_restore_user_registers(ctx->hdev); 11270 11271 rc = gaudi2_internal_cb_pool_init(ctx->hdev, ctx); 11272 if (rc) 11273 return rc; 11274 11275 rc = gaudi2_map_virtual_msix_doorbell_memory(ctx); 11276 if (rc) 11277 gaudi2_internal_cb_pool_fini(ctx->hdev, ctx); 11278 11279 return rc; 11280} 11281 11282static void gaudi2_ctx_fini(struct hl_ctx *ctx) 11283{ 11284 if (ctx->asid == HL_KERNEL_ASID_ID) 11285 return; 11286 11287 gaudi2_internal_cb_pool_fini(ctx->hdev, ctx); 11288 11289 gaudi2_unmap_virtual_msix_doorbell_memory(ctx); 11290} 11291 11292static int gaudi2_pre_schedule_cs(struct hl_cs *cs) 11293{ 11294 struct hl_device *hdev = cs->ctx->hdev; 11295 int index = cs->sequence & (hdev->asic_prop.max_pending_cs - 1); 11296 u32 mon_payload, sob_id, mon_id; 11297 11298 if (!cs_needs_completion(cs)) 11299 return 0; 11300 11301 /* 11302 * First 64 SOB/MON are reserved for driver for QMAN auto completion 11303 * mechanism. Each SOB/MON pair are used for a pending CS with the same 11304 * cyclic index. The SOB value is increased when each of the CS jobs is 11305 * completed. When the SOB reaches the number of CS jobs, the monitor 11306 * generates MSI-X interrupt. 11307 */ 11308 11309 sob_id = mon_id = index; 11310 mon_payload = (1 << CQ_ENTRY_SHADOW_INDEX_VALID_SHIFT) | 11311 (1 << CQ_ENTRY_READY_SHIFT) | index; 11312 11313 gaudi2_arm_cq_monitor(hdev, sob_id, mon_id, GAUDI2_RESERVED_CQ_CS_COMPLETION, mon_payload, 11314 cs->jobs_cnt); 11315 11316 return 0; 11317} 11318 11319static u32 gaudi2_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx) 11320{ 11321 return HL_INVALID_QUEUE; 11322} 11323 11324static u32 gaudi2_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id, u32 size, bool eb) 11325{ 11326 struct hl_cb *cb = data; 11327 struct packet_msg_short *pkt; 11328 u32 value, ctl, pkt_size = sizeof(*pkt); 11329 11330 pkt = (struct packet_msg_short *) (uintptr_t) (cb->kernel_address + size); 11331 memset(pkt, 0, pkt_size); 11332 11333 /* Inc by 1, Mode ADD */ 11334 value = FIELD_PREP(GAUDI2_PKT_SHORT_VAL_SOB_SYNC_VAL_MASK, 1); 11335 value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_SOB_MOD_MASK, 1); 11336 11337 ctl = FIELD_PREP(GAUDI2_PKT_SHORT_CTL_ADDR_MASK, sob_id * 4); 11338 ctl |= FIELD_PREP(GAUDI2_PKT_SHORT_CTL_BASE_MASK, 1); /* SOB base */ 11339 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); 11340 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, eb); 11341 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1); 11342 11343 pkt->value = cpu_to_le32(value); 11344 pkt->ctl = cpu_to_le32(ctl); 11345 11346 return size + pkt_size; 11347} 11348 11349static u32 gaudi2_add_mon_msg_short(struct packet_msg_short *pkt, u32 value, u16 addr) 11350{ 11351 u32 ctl, pkt_size = sizeof(*pkt); 11352 11353 memset(pkt, 0, pkt_size); 11354 11355 ctl = FIELD_PREP(GAUDI2_PKT_SHORT_CTL_ADDR_MASK, addr); 11356 ctl |= FIELD_PREP(GAUDI2_PKT_SHORT_CTL_BASE_MASK, 0); /* MON base */ 11357 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); 11358 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 0); 11359 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 0); 11360 11361 pkt->value = cpu_to_le32(value); 11362 pkt->ctl = cpu_to_le32(ctl); 11363 11364 return pkt_size; 11365} 11366 11367static u32 gaudi2_add_arm_monitor_pkt(struct hl_device *hdev, struct packet_msg_short *pkt, 11368 u16 sob_base, u8 sob_mask, u16 sob_val, u16 addr) 11369{ 11370 u32 ctl, value, pkt_size = sizeof(*pkt); 11371 u8 mask; 11372 11373 if (hl_gen_sob_mask(sob_base, sob_mask, &mask)) { 11374 dev_err(hdev->dev, "sob_base %u (mask %#x) is not valid\n", sob_base, sob_mask); 11375 return 0; 11376 } 11377 11378 memset(pkt, 0, pkt_size); 11379 11380 value = FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_SYNC_GID_MASK, sob_base / 8); 11381 value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_SYNC_VAL_MASK, sob_val); 11382 value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_MODE_MASK, 0); /* GREATER OR EQUAL*/ 11383 value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_MASK_MASK, mask); 11384 11385 ctl = FIELD_PREP(GAUDI2_PKT_SHORT_CTL_ADDR_MASK, addr); 11386 ctl |= FIELD_PREP(GAUDI2_PKT_SHORT_CTL_BASE_MASK, 0); /* MON base */ 11387 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); 11388 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 0); 11389 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1); 11390 11391 pkt->value = cpu_to_le32(value); 11392 pkt->ctl = cpu_to_le32(ctl); 11393 11394 return pkt_size; 11395} 11396 11397static u32 gaudi2_add_fence_pkt(struct packet_fence *pkt) 11398{ 11399 u32 ctl, cfg, pkt_size = sizeof(*pkt); 11400 11401 memset(pkt, 0, pkt_size); 11402 11403 cfg = FIELD_PREP(GAUDI2_PKT_FENCE_CFG_DEC_VAL_MASK, 1); 11404 cfg |= FIELD_PREP(GAUDI2_PKT_FENCE_CFG_TARGET_VAL_MASK, 1); 11405 cfg |= FIELD_PREP(GAUDI2_PKT_FENCE_CFG_ID_MASK, 2); 11406 11407 ctl = FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_FENCE); 11408 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 0); 11409 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1); 11410 11411 pkt->cfg = cpu_to_le32(cfg); 11412 pkt->ctl = cpu_to_le32(ctl); 11413 11414 return pkt_size; 11415} 11416 11417static u32 gaudi2_gen_wait_cb(struct hl_device *hdev, struct hl_gen_wait_properties *prop) 11418{ 11419 struct hl_cb *cb = prop->data; 11420 void *buf = (void *) (uintptr_t) (cb->kernel_address); 11421 11422 u64 monitor_base, fence_addr = 0; 11423 u32 stream_index, size = prop->size; 11424 u16 msg_addr_offset; 11425 11426 stream_index = prop->q_idx % 4; 11427 fence_addr = CFG_BASE + gaudi2_qm_blocks_bases[prop->q_idx] + 11428 QM_FENCE2_OFFSET + stream_index * 4; 11429 11430 /* 11431 * monitor_base should be the content of the base0 address registers, 11432 * so it will be added to the msg short offsets 11433 */ 11434 monitor_base = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0; 11435 11436 /* First monitor config packet: low address of the sync */ 11437 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + prop->mon_id * 4) - 11438 monitor_base; 11439 11440 size += gaudi2_add_mon_msg_short(buf + size, (u32) fence_addr, msg_addr_offset); 11441 11442 /* Second monitor config packet: high address of the sync */ 11443 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + prop->mon_id * 4) - 11444 monitor_base; 11445 11446 size += gaudi2_add_mon_msg_short(buf + size, (u32) (fence_addr >> 32), msg_addr_offset); 11447 11448 /* 11449 * Third monitor config packet: the payload, i.e. what to write when the 11450 * sync triggers 11451 */ 11452 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + prop->mon_id * 4) - 11453 monitor_base; 11454 11455 size += gaudi2_add_mon_msg_short(buf + size, 1, msg_addr_offset); 11456 11457 /* Fourth monitor config packet: bind the monitor to a sync object */ 11458 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + prop->mon_id * 4) - monitor_base; 11459 11460 size += gaudi2_add_arm_monitor_pkt(hdev, buf + size, prop->sob_base, prop->sob_mask, 11461 prop->sob_val, msg_addr_offset); 11462 11463 /* Fence packet */ 11464 size += gaudi2_add_fence_pkt(buf + size); 11465 11466 return size; 11467} 11468 11469static void gaudi2_reset_sob(struct hl_device *hdev, void *data) 11470{ 11471 struct hl_hw_sob *hw_sob = data; 11472 11473 dev_dbg(hdev->dev, "reset SOB, q_idx: %d, sob_id: %d\n", hw_sob->q_idx, hw_sob->sob_id); 11474 11475 WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + hw_sob->sob_id * 4, 0); 11476 11477 kref_init(&hw_sob->kref); 11478} 11479 11480static void gaudi2_reset_sob_group(struct hl_device *hdev, u16 sob_group) 11481{ 11482} 11483 11484static u64 gaudi2_get_device_time(struct hl_device *hdev) 11485{ 11486 u64 device_time = ((u64) RREG32(mmPSOC_TIMESTAMP_CNTCVU)) << 32; 11487 11488 return device_time | RREG32(mmPSOC_TIMESTAMP_CNTCVL); 11489} 11490 11491static int gaudi2_collective_wait_init_cs(struct hl_cs *cs) 11492{ 11493 return 0; 11494} 11495 11496static int gaudi2_collective_wait_create_jobs(struct hl_device *hdev, struct hl_ctx *ctx, 11497 struct hl_cs *cs, u32 wait_queue_id, 11498 u32 collective_engine_id, u32 encaps_signal_offset) 11499{ 11500 return -EINVAL; 11501} 11502 11503/* 11504 * hl_mmu_scramble - converts a dram (non power of 2) page-size aligned address 11505 * to DMMU page-size address (64MB) before mapping it in 11506 * the MMU. 11507 * The operation is performed on both the virtual and physical addresses. 11508 * for device with 6 HBMs the scramble is: 11509 * (addr[47:0] / 48M) * 64M + addr % 48M + addr[63:48] 11510 * 11511 * Example: 11512 * ============================================================================= 11513 * Allocated DRAM Reserved VA scrambled VA for MMU mapping Scrambled PA 11514 * Phys address in MMU last 11515 * HOP 11516 * ============================================================================= 11517 * PA1 0x3000000 VA1 0x9C000000 SVA1= (VA1/48M)*64M 0xD0000000 <- PA1/48M 0x1 11518 * PA2 0x9000000 VA2 0x9F000000 SVA2= (VA2/48M)*64M 0xD4000000 <- PA2/48M 0x3 11519 * ============================================================================= 11520 */ 11521static u64 gaudi2_mmu_scramble_addr(struct hl_device *hdev, u64 raw_addr) 11522{ 11523 struct asic_fixed_properties *prop = &hdev->asic_prop; 11524 u32 divisor, mod_va; 11525 u64 div_va; 11526 11527 /* accept any address in the DRAM address space */ 11528 if (hl_mem_area_inside_range(raw_addr, sizeof(raw_addr), DRAM_PHYS_BASE, 11529 VA_HBM_SPACE_END)) { 11530 11531 divisor = prop->num_functional_hbms * GAUDI2_HBM_MMU_SCRM_MEM_SIZE; 11532 div_va = div_u64_rem(raw_addr & GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK, divisor, &mod_va); 11533 return (raw_addr & ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK) | 11534 (div_va << GAUDI2_HBM_MMU_SCRM_DIV_SHIFT) | 11535 (mod_va << GAUDI2_HBM_MMU_SCRM_MOD_SHIFT); 11536 } 11537 11538 return raw_addr; 11539} 11540 11541static u64 gaudi2_mmu_descramble_addr(struct hl_device *hdev, u64 scrambled_addr) 11542{ 11543 struct asic_fixed_properties *prop = &hdev->asic_prop; 11544 u32 divisor, mod_va; 11545 u64 div_va; 11546 11547 /* accept any address in the DRAM address space */ 11548 if (hl_mem_area_inside_range(scrambled_addr, sizeof(scrambled_addr), DRAM_PHYS_BASE, 11549 VA_HBM_SPACE_END)) { 11550 11551 divisor = prop->num_functional_hbms * GAUDI2_HBM_MMU_SCRM_MEM_SIZE; 11552 div_va = div_u64_rem(scrambled_addr & GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK, 11553 PAGE_SIZE_64MB, &mod_va); 11554 11555 return ((scrambled_addr & ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK) + 11556 (div_va * divisor + mod_va)); 11557 } 11558 11559 return scrambled_addr; 11560} 11561 11562static u32 gaudi2_get_dec_base_addr(struct hl_device *hdev, u32 core_id) 11563{ 11564 u32 base = 0, dcore_id, dec_id; 11565 11566 if (core_id >= NUMBER_OF_DEC) { 11567 dev_err(hdev->dev, "Unexpected core number %d for DEC\n", core_id); 11568 goto out; 11569 } 11570 11571 if (core_id < 8) { 11572 dcore_id = core_id / NUM_OF_DEC_PER_DCORE; 11573 dec_id = core_id % NUM_OF_DEC_PER_DCORE; 11574 11575 base = mmDCORE0_DEC0_CMD_BASE + dcore_id * DCORE_OFFSET + 11576 dec_id * DCORE_VDEC_OFFSET; 11577 } else { 11578 /* PCIe Shared Decoder */ 11579 base = mmPCIE_DEC0_CMD_BASE + ((core_id % 8) * PCIE_VDEC_OFFSET); 11580 } 11581out: 11582 return base; 11583} 11584 11585static int gaudi2_get_hw_block_id(struct hl_device *hdev, u64 block_addr, 11586 u32 *block_size, u32 *block_id) 11587{ 11588 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11589 int i; 11590 11591 for (i = 0 ; i < NUM_USER_MAPPED_BLOCKS ; i++) { 11592 if (block_addr == CFG_BASE + gaudi2->mapped_blocks[i].address) { 11593 *block_id = i; 11594 if (block_size) 11595 *block_size = gaudi2->mapped_blocks[i].size; 11596 return 0; 11597 } 11598 } 11599 11600 dev_err(hdev->dev, "Invalid block address %#llx", block_addr); 11601 11602 return -EINVAL; 11603} 11604 11605static int gaudi2_block_mmap(struct hl_device *hdev, struct vm_area_struct *vma, 11606 u32 block_id, u32 block_size) 11607{ 11608 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11609 u64 offset_in_bar; 11610 u64 address; 11611 int rc; 11612 11613 if (block_id >= NUM_USER_MAPPED_BLOCKS) { 11614 dev_err(hdev->dev, "Invalid block id %u", block_id); 11615 return -EINVAL; 11616 } 11617 11618 /* we allow mapping only an entire block */ 11619 if (block_size != gaudi2->mapped_blocks[block_id].size) { 11620 dev_err(hdev->dev, "Invalid block size %u", block_size); 11621 return -EINVAL; 11622 } 11623 11624 offset_in_bar = CFG_BASE + gaudi2->mapped_blocks[block_id].address - STM_FLASH_BASE_ADDR; 11625 11626 address = pci_resource_start(hdev->pdev, SRAM_CFG_BAR_ID) + offset_in_bar; 11627 11628 vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | 11629 VM_DONTCOPY | VM_NORESERVE); 11630 11631 rc = remap_pfn_range(vma, vma->vm_start, address >> PAGE_SHIFT, 11632 block_size, vma->vm_page_prot); 11633 if (rc) 11634 dev_err(hdev->dev, "remap_pfn_range error %d", rc); 11635 11636 return rc; 11637} 11638 11639static void gaudi2_enable_events_from_fw(struct hl_device *hdev) 11640{ 11641 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11642 11643 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 11644 u32 irq_handler_offset = le32_to_cpu(dyn_regs->gic_host_ints_irq); 11645 11646 if (gaudi2->hw_cap_initialized & HW_CAP_CPU_Q) 11647 WREG32(irq_handler_offset, 11648 gaudi2_irq_map_table[GAUDI2_EVENT_CPU_INTS_REGISTER].cpu_id); 11649} 11650 11651static int gaudi2_get_mmu_base(struct hl_device *hdev, u64 mmu_id, u32 *mmu_base) 11652{ 11653 switch (mmu_id) { 11654 case HW_CAP_DCORE0_DMMU0: 11655 *mmu_base = mmDCORE0_HMMU0_MMU_BASE; 11656 break; 11657 case HW_CAP_DCORE0_DMMU1: 11658 *mmu_base = mmDCORE0_HMMU1_MMU_BASE; 11659 break; 11660 case HW_CAP_DCORE0_DMMU2: 11661 *mmu_base = mmDCORE0_HMMU2_MMU_BASE; 11662 break; 11663 case HW_CAP_DCORE0_DMMU3: 11664 *mmu_base = mmDCORE0_HMMU3_MMU_BASE; 11665 break; 11666 case HW_CAP_DCORE1_DMMU0: 11667 *mmu_base = mmDCORE1_HMMU0_MMU_BASE; 11668 break; 11669 case HW_CAP_DCORE1_DMMU1: 11670 *mmu_base = mmDCORE1_HMMU1_MMU_BASE; 11671 break; 11672 case HW_CAP_DCORE1_DMMU2: 11673 *mmu_base = mmDCORE1_HMMU2_MMU_BASE; 11674 break; 11675 case HW_CAP_DCORE1_DMMU3: 11676 *mmu_base = mmDCORE1_HMMU3_MMU_BASE; 11677 break; 11678 case HW_CAP_DCORE2_DMMU0: 11679 *mmu_base = mmDCORE2_HMMU0_MMU_BASE; 11680 break; 11681 case HW_CAP_DCORE2_DMMU1: 11682 *mmu_base = mmDCORE2_HMMU1_MMU_BASE; 11683 break; 11684 case HW_CAP_DCORE2_DMMU2: 11685 *mmu_base = mmDCORE2_HMMU2_MMU_BASE; 11686 break; 11687 case HW_CAP_DCORE2_DMMU3: 11688 *mmu_base = mmDCORE2_HMMU3_MMU_BASE; 11689 break; 11690 case HW_CAP_DCORE3_DMMU0: 11691 *mmu_base = mmDCORE3_HMMU0_MMU_BASE; 11692 break; 11693 case HW_CAP_DCORE3_DMMU1: 11694 *mmu_base = mmDCORE3_HMMU1_MMU_BASE; 11695 break; 11696 case HW_CAP_DCORE3_DMMU2: 11697 *mmu_base = mmDCORE3_HMMU2_MMU_BASE; 11698 break; 11699 case HW_CAP_DCORE3_DMMU3: 11700 *mmu_base = mmDCORE3_HMMU3_MMU_BASE; 11701 break; 11702 case HW_CAP_PMMU: 11703 *mmu_base = mmPMMU_HBW_MMU_BASE; 11704 break; 11705 default: 11706 return -EINVAL; 11707 } 11708 11709 return 0; 11710} 11711 11712static void gaudi2_ack_mmu_error(struct hl_device *hdev, u64 mmu_id) 11713{ 11714 bool is_pmmu = (mmu_id == HW_CAP_PMMU); 11715 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11716 u32 mmu_base; 11717 11718 if (!(gaudi2->hw_cap_initialized & mmu_id)) 11719 return; 11720 11721 if (gaudi2_get_mmu_base(hdev, mmu_id, &mmu_base)) 11722 return; 11723 11724 gaudi2_handle_page_error(hdev, mmu_base, is_pmmu, NULL); 11725 gaudi2_handle_access_error(hdev, mmu_base, is_pmmu); 11726} 11727 11728static int gaudi2_ack_mmu_page_fault_or_access_error(struct hl_device *hdev, u64 mmu_cap_mask) 11729{ 11730 u32 i, mmu_id, num_of_hmmus = NUM_OF_HMMU_PER_DCORE * NUM_OF_DCORES; 11731 11732 /* check all HMMUs */ 11733 for (i = 0 ; i < num_of_hmmus ; i++) { 11734 mmu_id = HW_CAP_DCORE0_DMMU0 << i; 11735 11736 if (mmu_cap_mask & mmu_id) 11737 gaudi2_ack_mmu_error(hdev, mmu_id); 11738 } 11739 11740 /* check PMMU */ 11741 if (mmu_cap_mask & HW_CAP_PMMU) 11742 gaudi2_ack_mmu_error(hdev, HW_CAP_PMMU); 11743 11744 return 0; 11745} 11746 11747static void gaudi2_get_msi_info(__le32 *table) 11748{ 11749 table[CPUCP_EVENT_QUEUE_MSI_TYPE] = cpu_to_le32(GAUDI2_EVENT_QUEUE_MSIX_IDX); 11750 table[CPUCP_EVENT_QUEUE_ERR_MSI_TYPE] = cpu_to_le32(GAUDI2_IRQ_NUM_EQ_ERROR); 11751} 11752 11753static int gaudi2_map_pll_idx_to_fw_idx(u32 pll_idx) 11754{ 11755 switch (pll_idx) { 11756 case HL_GAUDI2_CPU_PLL: return CPU_PLL; 11757 case HL_GAUDI2_PCI_PLL: return PCI_PLL; 11758 case HL_GAUDI2_NIC_PLL: return NIC_PLL; 11759 case HL_GAUDI2_DMA_PLL: return DMA_PLL; 11760 case HL_GAUDI2_MESH_PLL: return MESH_PLL; 11761 case HL_GAUDI2_MME_PLL: return MME_PLL; 11762 case HL_GAUDI2_TPC_PLL: return TPC_PLL; 11763 case HL_GAUDI2_IF_PLL: return IF_PLL; 11764 case HL_GAUDI2_SRAM_PLL: return SRAM_PLL; 11765 case HL_GAUDI2_HBM_PLL: return HBM_PLL; 11766 case HL_GAUDI2_VID_PLL: return VID_PLL; 11767 case HL_GAUDI2_MSS_PLL: return MSS_PLL; 11768 default: return -EINVAL; 11769 } 11770} 11771 11772static int gaudi2_gen_sync_to_engine_map(struct hl_device *hdev, struct hl_sync_to_engine_map *map) 11773{ 11774 /* Not implemented */ 11775 return 0; 11776} 11777 11778static int gaudi2_monitor_valid(struct hl_mon_state_dump *mon) 11779{ 11780 /* Not implemented */ 11781 return 0; 11782} 11783 11784static int gaudi2_print_single_monitor(char **buf, size_t *size, size_t *offset, 11785 struct hl_device *hdev, struct hl_mon_state_dump *mon) 11786{ 11787 /* Not implemented */ 11788 return 0; 11789} 11790 11791 11792static int gaudi2_print_fences_single_engine(struct hl_device *hdev, u64 base_offset, 11793 u64 status_base_offset, enum hl_sync_engine_type engine_type, 11794 u32 engine_id, char **buf, size_t *size, size_t *offset) 11795{ 11796 /* Not implemented */ 11797 return 0; 11798} 11799 11800 11801static struct hl_state_dump_specs_funcs gaudi2_state_dump_funcs = { 11802 .monitor_valid = gaudi2_monitor_valid, 11803 .print_single_monitor = gaudi2_print_single_monitor, 11804 .gen_sync_to_engine_map = gaudi2_gen_sync_to_engine_map, 11805 .print_fences_single_engine = gaudi2_print_fences_single_engine, 11806}; 11807 11808static void gaudi2_state_dump_init(struct hl_device *hdev) 11809{ 11810 /* Not implemented */ 11811 hdev->state_dump_specs.props = gaudi2_state_dump_specs_props; 11812 hdev->state_dump_specs.funcs = gaudi2_state_dump_funcs; 11813} 11814 11815static u32 gaudi2_get_sob_addr(struct hl_device *hdev, u32 sob_id) 11816{ 11817 return 0; 11818} 11819 11820static u32 *gaudi2_get_stream_master_qid_arr(void) 11821{ 11822 return NULL; 11823} 11824 11825static void gaudi2_add_device_attr(struct hl_device *hdev, struct attribute_group *dev_clk_attr_grp, 11826 struct attribute_group *dev_vrm_attr_grp) 11827{ 11828 hl_sysfs_add_dev_clk_attr(hdev, dev_clk_attr_grp); 11829 hl_sysfs_add_dev_vrm_attr(hdev, dev_vrm_attr_grp); 11830} 11831 11832static int gaudi2_mmu_get_real_page_size(struct hl_device *hdev, struct hl_mmu_properties *mmu_prop, 11833 u32 page_size, u32 *real_page_size, bool is_dram_addr) 11834{ 11835 struct asic_fixed_properties *prop = &hdev->asic_prop; 11836 11837 /* for host pages the page size must be */ 11838 if (!is_dram_addr) { 11839 if (page_size % mmu_prop->page_size) 11840 goto page_size_err; 11841 11842 *real_page_size = mmu_prop->page_size; 11843 return 0; 11844 } 11845 11846 if ((page_size % prop->dram_page_size) || (prop->dram_page_size > mmu_prop->page_size)) 11847 goto page_size_err; 11848 11849 /* 11850 * MMU page size is different from DRAM page size (more precisely, DMMU page is greater 11851 * than DRAM page size). 11852 * for this reason work with the DRAM page size and let the MMU scrambling routine handle 11853 * this mismatch when calculating the address to place in the MMU page table. 11854 * (in that case also make sure that the dram_page_size is not greater than the 11855 * mmu page size) 11856 */ 11857 *real_page_size = prop->dram_page_size; 11858 11859 return 0; 11860 11861page_size_err: 11862 dev_err(hdev->dev, "page size of 0x%X is not 0x%X aligned, can't map\n", 11863 page_size, mmu_prop->page_size >> 10); 11864 return -EFAULT; 11865} 11866 11867static int gaudi2_get_monitor_dump(struct hl_device *hdev, void *data) 11868{ 11869 return -EOPNOTSUPP; 11870} 11871 11872int gaudi2_send_device_activity(struct hl_device *hdev, bool open) 11873{ 11874 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11875 11876 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 11877 return 0; 11878 11879 return hl_fw_send_device_activity(hdev, open); 11880} 11881 11882static u64 gaudi2_read_pte(struct hl_device *hdev, u64 addr) 11883{ 11884 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11885 u64 val; 11886 11887 if (hdev->reset_info.hard_reset_pending) 11888 return U64_MAX; 11889 11890 val = readq(hdev->pcie_bar[DRAM_BAR_ID] + (addr - gaudi2->dram_bar_cur_addr)); 11891 11892 return val; 11893} 11894 11895static void gaudi2_write_pte(struct hl_device *hdev, u64 addr, u64 val) 11896{ 11897 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11898 11899 if (hdev->reset_info.hard_reset_pending) 11900 return; 11901 11902 writeq(val, hdev->pcie_bar[DRAM_BAR_ID] + (addr - gaudi2->dram_bar_cur_addr)); 11903} 11904 11905static const struct hl_asic_funcs gaudi2_funcs = { 11906 .early_init = gaudi2_early_init, 11907 .early_fini = gaudi2_early_fini, 11908 .late_init = gaudi2_late_init, 11909 .late_fini = gaudi2_late_fini, 11910 .sw_init = gaudi2_sw_init, 11911 .sw_fini = gaudi2_sw_fini, 11912 .hw_init = gaudi2_hw_init, 11913 .hw_fini = gaudi2_hw_fini, 11914 .halt_engines = gaudi2_halt_engines, 11915 .suspend = gaudi2_suspend, 11916 .resume = gaudi2_resume, 11917 .mmap = gaudi2_mmap, 11918 .ring_doorbell = gaudi2_ring_doorbell, 11919 .pqe_write = gaudi2_pqe_write, 11920 .asic_dma_alloc_coherent = gaudi2_dma_alloc_coherent, 11921 .asic_dma_free_coherent = gaudi2_dma_free_coherent, 11922 .scrub_device_mem = gaudi2_scrub_device_mem, 11923 .scrub_device_dram = gaudi2_scrub_device_dram, 11924 .get_int_queue_base = NULL, 11925 .test_queues = gaudi2_test_queues, 11926 .asic_dma_pool_zalloc = gaudi2_dma_pool_zalloc, 11927 .asic_dma_pool_free = gaudi2_dma_pool_free, 11928 .cpu_accessible_dma_pool_alloc = gaudi2_cpu_accessible_dma_pool_alloc, 11929 .cpu_accessible_dma_pool_free = gaudi2_cpu_accessible_dma_pool_free, 11930 .dma_unmap_sgtable = hl_asic_dma_unmap_sgtable, 11931 .cs_parser = gaudi2_cs_parser, 11932 .dma_map_sgtable = hl_asic_dma_map_sgtable, 11933 .add_end_of_cb_packets = NULL, 11934 .update_eq_ci = gaudi2_update_eq_ci, 11935 .context_switch = gaudi2_context_switch, 11936 .restore_phase_topology = gaudi2_restore_phase_topology, 11937 .debugfs_read_dma = gaudi2_debugfs_read_dma, 11938 .add_device_attr = gaudi2_add_device_attr, 11939 .handle_eqe = gaudi2_handle_eqe, 11940 .get_events_stat = gaudi2_get_events_stat, 11941 .read_pte = gaudi2_read_pte, 11942 .write_pte = gaudi2_write_pte, 11943 .mmu_invalidate_cache = gaudi2_mmu_invalidate_cache, 11944 .mmu_invalidate_cache_range = gaudi2_mmu_invalidate_cache_range, 11945 .mmu_prefetch_cache_range = NULL, 11946 .send_heartbeat = gaudi2_send_heartbeat, 11947 .debug_coresight = gaudi2_debug_coresight, 11948 .is_device_idle = gaudi2_is_device_idle, 11949 .compute_reset_late_init = gaudi2_compute_reset_late_init, 11950 .hw_queues_lock = gaudi2_hw_queues_lock, 11951 .hw_queues_unlock = gaudi2_hw_queues_unlock, 11952 .get_pci_id = gaudi2_get_pci_id, 11953 .get_eeprom_data = gaudi2_get_eeprom_data, 11954 .get_monitor_dump = gaudi2_get_monitor_dump, 11955 .send_cpu_message = gaudi2_send_cpu_message, 11956 .pci_bars_map = gaudi2_pci_bars_map, 11957 .init_iatu = gaudi2_init_iatu, 11958 .rreg = hl_rreg, 11959 .wreg = hl_wreg, 11960 .halt_coresight = gaudi2_halt_coresight, 11961 .ctx_init = gaudi2_ctx_init, 11962 .ctx_fini = gaudi2_ctx_fini, 11963 .pre_schedule_cs = gaudi2_pre_schedule_cs, 11964 .get_queue_id_for_cq = gaudi2_get_queue_id_for_cq, 11965 .load_firmware_to_device = NULL, 11966 .load_boot_fit_to_device = NULL, 11967 .get_signal_cb_size = gaudi2_get_signal_cb_size, 11968 .get_wait_cb_size = gaudi2_get_wait_cb_size, 11969 .gen_signal_cb = gaudi2_gen_signal_cb, 11970 .gen_wait_cb = gaudi2_gen_wait_cb, 11971 .reset_sob = gaudi2_reset_sob, 11972 .reset_sob_group = gaudi2_reset_sob_group, 11973 .get_device_time = gaudi2_get_device_time, 11974 .pb_print_security_errors = gaudi2_pb_print_security_errors, 11975 .collective_wait_init_cs = gaudi2_collective_wait_init_cs, 11976 .collective_wait_create_jobs = gaudi2_collective_wait_create_jobs, 11977 .get_dec_base_addr = gaudi2_get_dec_base_addr, 11978 .scramble_addr = gaudi2_mmu_scramble_addr, 11979 .descramble_addr = gaudi2_mmu_descramble_addr, 11980 .ack_protection_bits_errors = gaudi2_ack_protection_bits_errors, 11981 .get_hw_block_id = gaudi2_get_hw_block_id, 11982 .hw_block_mmap = gaudi2_block_mmap, 11983 .enable_events_from_fw = gaudi2_enable_events_from_fw, 11984 .ack_mmu_errors = gaudi2_ack_mmu_page_fault_or_access_error, 11985 .get_msi_info = gaudi2_get_msi_info, 11986 .map_pll_idx_to_fw_idx = gaudi2_map_pll_idx_to_fw_idx, 11987 .init_firmware_preload_params = gaudi2_init_firmware_preload_params, 11988 .init_firmware_loader = gaudi2_init_firmware_loader, 11989 .init_cpu_scrambler_dram = gaudi2_init_scrambler_hbm, 11990 .state_dump_init = gaudi2_state_dump_init, 11991 .get_sob_addr = &gaudi2_get_sob_addr, 11992 .set_pci_memory_regions = gaudi2_set_pci_memory_regions, 11993 .get_stream_master_qid_arr = gaudi2_get_stream_master_qid_arr, 11994 .check_if_razwi_happened = gaudi2_check_if_razwi_happened, 11995 .mmu_get_real_page_size = gaudi2_mmu_get_real_page_size, 11996 .access_dev_mem = hl_access_dev_mem, 11997 .set_dram_bar_base = gaudi2_set_hbm_bar_base, 11998 .set_engine_cores = gaudi2_set_engine_cores, 11999 .set_engines = gaudi2_set_engines, 12000 .send_device_activity = gaudi2_send_device_activity, 12001 .set_dram_properties = gaudi2_set_dram_properties, 12002 .set_binning_masks = gaudi2_set_binning_masks, 12003}; 12004 12005void gaudi2_set_asic_funcs(struct hl_device *hdev) 12006{ 12007 hdev->asic_funcs = &gaudi2_funcs; 12008}
~/
Read-only file preview.