FILE
habanalabs.h
File preview with syntax highlighting.
linux-6.19.14/drivers/accel/habanalabs/common/habanalabs.h
1/* SPDX-License-Identifier: GPL-2.0 2 * 3 * Copyright 2016-2023 HabanaLabs, Ltd. 4 * All Rights Reserved. 5 * 6 */ 7 8#ifndef HABANALABSP_H_ 9#define HABANALABSP_H_ 10 11#include <linux/habanalabs/cpucp_if.h> 12#include "../include/common/qman_if.h" 13#include "../include/hw_ip/mmu/mmu_general.h" 14#include <uapi/drm/habanalabs_accel.h> 15 16#include <linux/cdev.h> 17#include <linux/iopoll.h> 18#include <linux/irqreturn.h> 19#include <linux/dma-direction.h> 20#include <linux/scatterlist.h> 21#include <linux/hashtable.h> 22#include <linux/debugfs.h> 23#include <linux/rwsem.h> 24#include <linux/eventfd.h> 25#include <linux/bitfield.h> 26#include <linux/genalloc.h> 27#include <linux/sched/signal.h> 28#include <linux/io-64-nonatomic-lo-hi.h> 29#include <linux/coresight.h> 30#include <linux/dma-buf.h> 31 32#include <drm/drm_device.h> 33#include <drm/drm_file.h> 34 35#include "security.h" 36 37#define HL_NAME "habanalabs" 38 39struct hl_device; 40struct hl_fpriv; 41 42#define PCI_VENDOR_ID_HABANALABS 0x1da3 43 44/* Use upper bits of mmap offset to store habana driver specific information. 45 * bits[63:59] - Encode mmap type 46 * bits[45:0] - mmap offset value 47 * 48 * NOTE: struct vm_area_struct.vm_pgoff uses offset in pages. Hence, these 49 * defines are w.r.t to PAGE_SIZE 50 */ 51#define HL_MMAP_TYPE_SHIFT (59 - PAGE_SHIFT) 52#define HL_MMAP_TYPE_MASK (0x1full << HL_MMAP_TYPE_SHIFT) 53#define HL_MMAP_TYPE_TS_BUFF (0x10ull << HL_MMAP_TYPE_SHIFT) 54#define HL_MMAP_TYPE_BLOCK (0x4ull << HL_MMAP_TYPE_SHIFT) 55#define HL_MMAP_TYPE_CB (0x2ull << HL_MMAP_TYPE_SHIFT) 56 57#define HL_MMAP_OFFSET_VALUE_MASK (0x1FFFFFFFFFFFull >> PAGE_SHIFT) 58#define HL_MMAP_OFFSET_VALUE_GET(off) (off & HL_MMAP_OFFSET_VALUE_MASK) 59 60#define HL_PENDING_RESET_PER_SEC 10 61#define HL_PENDING_RESET_MAX_TRIALS 60 /* 10 minutes */ 62#define HL_PENDING_RESET_LONG_SEC 60 63/* 64 * In device fini, wait 10 minutes for user processes to be terminated after we kill them. 65 * This is needed to prevent situation of clearing resources while user processes are still alive. 66 */ 67#define HL_WAIT_PROCESS_KILL_ON_DEVICE_FINI 600 68 69#define HL_HARD_RESET_MAX_TIMEOUT 120 70#define HL_PLDM_HARD_RESET_MAX_TIMEOUT (HL_HARD_RESET_MAX_TIMEOUT * 3) 71 72#define HL_DEVICE_TIMEOUT_USEC 1000000 /* 1 s */ 73 74#define HL_HEARTBEAT_PER_USEC 10000000 /* 10 s */ 75 76#define HL_PLL_LOW_JOB_FREQ_USEC 5000000 /* 5 s */ 77 78#define HL_CPUCP_INFO_TIMEOUT_USEC 10000000 /* 10s */ 79#define HL_CPUCP_EEPROM_TIMEOUT_USEC 10000000 /* 10s */ 80#define HL_CPUCP_MON_DUMP_TIMEOUT_USEC 10000000 /* 10s */ 81#define HL_CPUCP_SEC_ATTEST_INFO_TINEOUT_USEC 10000000 /* 10s */ 82 83#define HL_FW_STATUS_POLL_INTERVAL_USEC 10000 /* 10ms */ 84#define HL_FW_COMMS_STATUS_PLDM_POLL_INTERVAL_USEC 1000000 /* 1s */ 85 86#define HL_PCI_ELBI_TIMEOUT_MSEC 10 /* 10ms */ 87 88#define HL_INVALID_QUEUE UINT_MAX 89 90#define HL_COMMON_USER_CQ_INTERRUPT_ID 0xFFF 91#define HL_COMMON_DEC_INTERRUPT_ID 0xFFE 92 93#define HL_STATE_DUMP_HIST_LEN 5 94#define HL_DBGFS_CFG_ACCESS_HIST_LEN 20 95#define HL_DBGFS_CFG_ACCESS_HIST_TIMEOUT_SEC 2 /* 2s */ 96 97/* Default value for device reset trigger , an invalid value */ 98#define HL_RESET_TRIGGER_DEFAULT 0xFF 99 100#define OBJ_NAMES_HASH_TABLE_BITS 7 /* 1 << 7 buckets */ 101#define SYNC_TO_ENGINE_HASH_TABLE_BITS 7 /* 1 << 7 buckets */ 102 103/* Memory */ 104#define MEM_HASH_TABLE_BITS 7 /* 1 << 7 buckets */ 105 106/* MMU */ 107#define MMU_HASH_TABLE_BITS 7 /* 1 << 7 buckets */ 108 109#define TIMESTAMP_FREE_NODES_NUM 512 110 111/** 112 * enum hl_mmu_page_table_location - mmu page table location 113 * @MMU_DR_PGT: page-table is located on device DRAM. 114 * @MMU_HR_PGT: page-table is located on host memory. 115 * @MMU_NUM_PGT_LOCATIONS: number of page-table locations currently supported. 116 */ 117enum hl_mmu_page_table_location { 118 MMU_DR_PGT = 0, /* device-dram-resident MMU PGT */ 119 MMU_HR_PGT, /* host resident MMU PGT */ 120 MMU_NUM_PGT_LOCATIONS /* num of PGT locations */ 121}; 122 123/* 124 * HL_RSVD_SOBS 'sync stream' reserved sync objects per QMAN stream 125 * HL_RSVD_MONS 'sync stream' reserved monitors per QMAN stream 126 */ 127#define HL_RSVD_SOBS 2 128#define HL_RSVD_MONS 1 129 130/* 131 * HL_COLLECTIVE_RSVD_MSTR_MONS 'collective' reserved monitors per QMAN stream 132 */ 133#define HL_COLLECTIVE_RSVD_MSTR_MONS 2 134 135#define HL_MAX_SOB_VAL (1 << 15) 136 137#define IS_POWER_OF_2(n) (n != 0 && ((n & (n - 1)) == 0)) 138#define IS_MAX_PENDING_CS_VALID(n) (IS_POWER_OF_2(n) && (n > 1)) 139 140#define HL_PCI_NUM_BARS 6 141 142/* Completion queue entry relates to completed job */ 143#define HL_COMPLETION_MODE_JOB 0 144/* Completion queue entry relates to completed command submission */ 145#define HL_COMPLETION_MODE_CS 1 146 147#define HL_MAX_DCORES 8 148 149/* DMA alloc/free wrappers */ 150#define hl_asic_dma_alloc_coherent(hdev, size, dma_handle, flags) \ 151 hl_asic_dma_alloc_coherent_caller(hdev, size, dma_handle, flags, __func__) 152 153#define hl_asic_dma_pool_zalloc(hdev, size, mem_flags, dma_handle) \ 154 hl_asic_dma_pool_zalloc_caller(hdev, size, mem_flags, dma_handle, __func__) 155 156#define hl_asic_dma_free_coherent(hdev, size, cpu_addr, dma_handle) \ 157 hl_asic_dma_free_coherent_caller(hdev, size, cpu_addr, dma_handle, __func__) 158 159#define hl_asic_dma_pool_free(hdev, vaddr, dma_addr) \ 160 hl_asic_dma_pool_free_caller(hdev, vaddr, dma_addr, __func__) 161 162#define hl_dma_map_sgtable(hdev, sgt, dir) \ 163 hl_dma_map_sgtable_caller(hdev, sgt, dir, __func__) 164#define hl_dma_unmap_sgtable(hdev, sgt, dir) \ 165 hl_dma_unmap_sgtable_caller(hdev, sgt, dir, __func__) 166 167/* 168 * Reset Flags 169 * 170 * - HL_DRV_RESET_HARD 171 * If set do hard reset to all engines. If not set reset just 172 * compute/DMA engines. 173 * 174 * - HL_DRV_RESET_FROM_RESET_THR 175 * Set if the caller is the hard-reset thread 176 * 177 * - HL_DRV_RESET_HEARTBEAT 178 * Set if reset is due to heartbeat 179 * 180 * - HL_DRV_RESET_TDR 181 * Set if reset is due to TDR 182 * 183 * - HL_DRV_RESET_DEV_RELEASE 184 * Set if reset is due to device release 185 * 186 * - HL_DRV_RESET_BYPASS_REQ_TO_FW 187 * F/W will perform the reset. No need to ask it to reset the device. This is relevant 188 * only when running with secured f/w 189 * 190 * - HL_DRV_RESET_FW_FATAL_ERR 191 * Set if reset is due to a fatal error from FW 192 * 193 * - HL_DRV_RESET_DELAY 194 * Set if a delay should be added before the reset 195 * 196 * - HL_DRV_RESET_FROM_WD_THR 197 * Set if the caller is the device release watchdog thread 198 */ 199 200#define HL_DRV_RESET_HARD (1 << 0) 201#define HL_DRV_RESET_FROM_RESET_THR (1 << 1) 202#define HL_DRV_RESET_HEARTBEAT (1 << 2) 203#define HL_DRV_RESET_TDR (1 << 3) 204#define HL_DRV_RESET_DEV_RELEASE (1 << 4) 205#define HL_DRV_RESET_BYPASS_REQ_TO_FW (1 << 5) 206#define HL_DRV_RESET_FW_FATAL_ERR (1 << 6) 207#define HL_DRV_RESET_DELAY (1 << 7) 208#define HL_DRV_RESET_FROM_WD_THR (1 << 8) 209 210/* 211 * Security 212 */ 213 214#define HL_PB_SHARED 1 215#define HL_PB_NA 0 216#define HL_PB_SINGLE_INSTANCE 1 217#define HL_BLOCK_SIZE 0x1000 218#define HL_BLOCK_GLBL_ERR_MASK 0xF40 219#define HL_BLOCK_GLBL_ERR_ADDR 0xF44 220#define HL_BLOCK_GLBL_ERR_CAUSE 0xF48 221#define HL_BLOCK_GLBL_SEC_OFFS 0xF80 222#define HL_BLOCK_GLBL_SEC_SIZE (HL_BLOCK_SIZE - HL_BLOCK_GLBL_SEC_OFFS) 223#define HL_BLOCK_GLBL_SEC_LEN (HL_BLOCK_GLBL_SEC_SIZE / sizeof(u32)) 224#define UNSET_GLBL_SEC_BIT(array, b) ((array)[((b) / 32)] |= (1 << ((b) % 32))) 225 226enum hl_protection_levels { 227 SECURED_LVL, 228 PRIVILEGED_LVL, 229 NON_SECURED_LVL 230}; 231 232/** 233 * struct iterate_module_ctx - HW module iterator 234 * @fn: function to apply to each HW module instance 235 * @data: optional internal data to the function iterator 236 * @rc: return code for optional use of iterator/iterator-caller 237 */ 238struct iterate_module_ctx { 239 /* 240 * callback for the HW module iterator 241 * @hdev: pointer to the habanalabs device structure 242 * @block: block (ASIC specific definition can be dcore/hdcore) 243 * @inst: HW module instance within the block 244 * @offset: current HW module instance offset from the 1-st HW module instance 245 * in the 1-st block 246 * @ctx: the iterator context. 247 */ 248 void (*fn)(struct hl_device *hdev, int block, int inst, u32 offset, 249 struct iterate_module_ctx *ctx); 250 void *data; 251 int rc; 252}; 253 254struct hl_block_glbl_sec { 255 u32 sec_array[HL_BLOCK_GLBL_SEC_LEN]; 256}; 257 258#define HL_MAX_SOBS_PER_MONITOR 8 259 260/** 261 * struct hl_gen_wait_properties - properties for generating a wait CB 262 * @data: command buffer 263 * @q_idx: queue id is used to extract fence register address 264 * @size: offset in command buffer 265 * @sob_base: SOB base to use in this wait CB 266 * @sob_val: SOB value to wait for 267 * @mon_id: monitor to use in this wait CB 268 * @sob_mask: each bit represents a SOB offset from sob_base to be used 269 */ 270struct hl_gen_wait_properties { 271 void *data; 272 u32 q_idx; 273 u32 size; 274 u16 sob_base; 275 u16 sob_val; 276 u16 mon_id; 277 u8 sob_mask; 278}; 279 280/** 281 * struct pgt_info - MMU hop page info. 282 * @node: hash linked-list node for the pgts on host (shadow pgts for device resident MMU and 283 * actual pgts for host resident MMU). 284 * @phys_addr: physical address of the pgt. 285 * @virt_addr: host virtual address of the pgt (see above device/host resident). 286 * @shadow_addr: shadow hop in the host for device resident MMU. 287 * @ctx: pointer to the owner ctx. 288 * @num_of_ptes: indicates how many ptes are used in the pgt. used only for dynamically 289 * allocated HOPs (all HOPs but HOP0) 290 * 291 * The MMU page tables hierarchy can be placed either on the device's DRAM (in which case shadow 292 * pgts will be stored on host memory) or on host memory (in which case no shadow is required). 293 * 294 * When a new level (hop) is needed during mapping this structure will be used to describe 295 * the newly allocated hop as well as to track number of PTEs in it. 296 * During unmapping, if no valid PTEs remained in the page of a newly allocated hop, it is 297 * freed with its pgt_info structure. 298 */ 299struct pgt_info { 300 struct hlist_node node; 301 u64 phys_addr; 302 u64 virt_addr; 303 u64 shadow_addr; 304 struct hl_ctx *ctx; 305 int num_of_ptes; 306}; 307 308/** 309 * enum hl_pci_match_mode - pci match mode per region 310 * @PCI_ADDRESS_MATCH_MODE: address match mode 311 * @PCI_BAR_MATCH_MODE: bar match mode 312 */ 313enum hl_pci_match_mode { 314 PCI_ADDRESS_MATCH_MODE, 315 PCI_BAR_MATCH_MODE 316}; 317 318/** 319 * enum hl_fw_component - F/W components to read version through registers. 320 * @FW_COMP_BOOT_FIT: boot fit. 321 * @FW_COMP_PREBOOT: preboot. 322 * @FW_COMP_LINUX: linux. 323 */ 324enum hl_fw_component { 325 FW_COMP_BOOT_FIT, 326 FW_COMP_PREBOOT, 327 FW_COMP_LINUX, 328}; 329 330/** 331 * enum hl_fw_types - F/W types present in the system 332 * @FW_TYPE_NONE: no FW component indication 333 * @FW_TYPE_LINUX: Linux image for device CPU 334 * @FW_TYPE_BOOT_CPU: Boot image for device CPU 335 * @FW_TYPE_PREBOOT_CPU: Indicates pre-loaded CPUs are present in the system 336 * (preboot, ppboot etc...) 337 * @FW_TYPE_ALL_TYPES: Mask for all types 338 */ 339enum hl_fw_types { 340 FW_TYPE_NONE = 0x0, 341 FW_TYPE_LINUX = 0x1, 342 FW_TYPE_BOOT_CPU = 0x2, 343 FW_TYPE_PREBOOT_CPU = 0x4, 344 FW_TYPE_ALL_TYPES = 345 (FW_TYPE_LINUX | FW_TYPE_BOOT_CPU | FW_TYPE_PREBOOT_CPU) 346}; 347 348/** 349 * enum hl_queue_type - Supported QUEUE types. 350 * @QUEUE_TYPE_NA: queue is not available. 351 * @QUEUE_TYPE_EXT: external queue which is a DMA channel that may access the 352 * host. 353 * @QUEUE_TYPE_INT: internal queue that performs DMA inside the device's 354 * memories and/or operates the compute engines. 355 * @QUEUE_TYPE_CPU: S/W queue for communication with the device's CPU. 356 * @QUEUE_TYPE_HW: queue of DMA and compute engines jobs, for which completion 357 * notifications are sent by H/W. 358 */ 359enum hl_queue_type { 360 QUEUE_TYPE_NA, 361 QUEUE_TYPE_EXT, 362 QUEUE_TYPE_INT, 363 QUEUE_TYPE_CPU, 364 QUEUE_TYPE_HW 365}; 366 367enum hl_cs_type { 368 CS_TYPE_DEFAULT, 369 CS_TYPE_SIGNAL, 370 CS_TYPE_WAIT, 371 CS_TYPE_COLLECTIVE_WAIT, 372 CS_RESERVE_SIGNALS, 373 CS_UNRESERVE_SIGNALS, 374 CS_TYPE_ENGINE_CORE, 375 CS_TYPE_ENGINES, 376 CS_TYPE_FLUSH_PCI_HBW_WRITES, 377}; 378 379/* 380 * struct hl_inbound_pci_region - inbound region descriptor 381 * @mode: pci match mode for this region 382 * @addr: region target address 383 * @size: region size in bytes 384 * @offset_in_bar: offset within bar (address match mode) 385 * @bar: bar id 386 */ 387struct hl_inbound_pci_region { 388 enum hl_pci_match_mode mode; 389 u64 addr; 390 u64 size; 391 u64 offset_in_bar; 392 u8 bar; 393}; 394 395/* 396 * struct hl_outbound_pci_region - outbound region descriptor 397 * @addr: region target address 398 * @size: region size in bytes 399 */ 400struct hl_outbound_pci_region { 401 u64 addr; 402 u64 size; 403}; 404 405/* 406 * enum queue_cb_alloc_flags - Indicates queue support for CBs that 407 * allocated by Kernel or by User 408 * @CB_ALLOC_KERNEL: support only CBs that allocated by Kernel 409 * @CB_ALLOC_USER: support only CBs that allocated by User 410 */ 411enum queue_cb_alloc_flags { 412 CB_ALLOC_KERNEL = 0x1, 413 CB_ALLOC_USER = 0x2 414}; 415 416/* 417 * struct hl_hw_sob - H/W SOB info. 418 * @hdev: habanalabs device structure. 419 * @kref: refcount of this SOB. The SOB will reset once the refcount is zero. 420 * @sob_id: id of this SOB. 421 * @sob_addr: the sob offset from the base address. 422 * @q_idx: the H/W queue that uses this SOB. 423 * @need_reset: reset indication set when switching to the other sob. 424 */ 425struct hl_hw_sob { 426 struct hl_device *hdev; 427 struct kref kref; 428 u32 sob_id; 429 u32 sob_addr; 430 u32 q_idx; 431 bool need_reset; 432}; 433 434enum hl_collective_mode { 435 HL_COLLECTIVE_NOT_SUPPORTED = 0x0, 436 HL_COLLECTIVE_MASTER = 0x1, 437 HL_COLLECTIVE_SLAVE = 0x2 438}; 439 440/** 441 * struct hw_queue_properties - queue information. 442 * @type: queue type. 443 * @cb_alloc_flags: bitmap which indicates if the hw queue supports CB 444 * that allocated by the Kernel driver and therefore, 445 * a CB handle can be provided for jobs on this queue. 446 * Otherwise, a CB address must be provided. 447 * @collective_mode: collective mode of current queue 448 * @q_dram_bd_address: PQ dram address, used when PQ need to reside in DRAM. 449 * @driver_only: true if only the driver is allowed to send a job to this queue, 450 * false otherwise. 451 * @binned: True if the queue is binned out and should not be used 452 * @supports_sync_stream: True if queue supports sync stream 453 * @dram_bd: True if the bd should be copied to dram, needed for PQ which has been allocated on dram 454 */ 455struct hw_queue_properties { 456 enum hl_queue_type type; 457 enum queue_cb_alloc_flags cb_alloc_flags; 458 enum hl_collective_mode collective_mode; 459 u64 q_dram_bd_address; 460 u8 driver_only; 461 u8 binned; 462 u8 supports_sync_stream; 463 u8 dram_bd; 464}; 465 466/** 467 * enum vm_type - virtual memory mapping request information. 468 * @VM_TYPE_USERPTR: mapping of user memory to device virtual address. 469 * @VM_TYPE_PHYS_PACK: mapping of DRAM memory to device virtual address. 470 */ 471enum vm_type { 472 VM_TYPE_USERPTR = 0x1, 473 VM_TYPE_PHYS_PACK = 0x2 474}; 475 476/** 477 * enum mmu_op_flags - mmu operation relevant information. 478 * @MMU_OP_USERPTR: operation on user memory (host resident). 479 * @MMU_OP_PHYS_PACK: operation on DRAM (device resident). 480 * @MMU_OP_CLEAR_MEMCACHE: operation has to clear memcache. 481 * @MMU_OP_SKIP_LOW_CACHE_INV: operation is allowed to skip parts of cache invalidation. 482 */ 483enum mmu_op_flags { 484 MMU_OP_USERPTR = 0x1, 485 MMU_OP_PHYS_PACK = 0x2, 486 MMU_OP_CLEAR_MEMCACHE = 0x4, 487 MMU_OP_SKIP_LOW_CACHE_INV = 0x8, 488}; 489 490 491/** 492 * enum hl_device_hw_state - H/W device state. use this to understand whether 493 * to do reset before hw_init or not 494 * @HL_DEVICE_HW_STATE_CLEAN: H/W state is clean. i.e. after hard reset 495 * @HL_DEVICE_HW_STATE_DIRTY: H/W state is dirty. i.e. we started to execute 496 * hw_init 497 */ 498enum hl_device_hw_state { 499 HL_DEVICE_HW_STATE_CLEAN = 0, 500 HL_DEVICE_HW_STATE_DIRTY 501}; 502 503#define HL_MMU_VA_ALIGNMENT_NOT_NEEDED 0 504 505/** 506 * struct hl_mmu_properties - ASIC specific MMU address translation properties. 507 * @start_addr: virtual start address of the memory region. 508 * @end_addr: virtual end address of the memory region. 509 * @hop_shifts: array holds HOPs shifts. 510 * @hop_masks: array holds HOPs masks. 511 * @last_mask: mask to get the bit indicating this is the last hop. 512 * @pgt_size: size for page tables. 513 * @supported_pages_mask: bitmask for supported page size (relevant only for MMUs 514 * supporting multiple page size). 515 * @page_size: default page size used to allocate memory. 516 * @num_hops: The amount of hops supported by the translation table. 517 * @hop_table_size: HOP table size. 518 * @hop0_tables_total_size: total size for all HOP0 tables. 519 * @host_resident: Should the MMU page table reside in host memory or in the 520 * device DRAM. 521 */ 522struct hl_mmu_properties { 523 u64 start_addr; 524 u64 end_addr; 525 u64 hop_shifts[MMU_HOP_MAX]; 526 u64 hop_masks[MMU_HOP_MAX]; 527 u64 last_mask; 528 u64 pgt_size; 529 u64 supported_pages_mask; 530 u32 page_size; 531 u32 num_hops; 532 u32 hop_table_size; 533 u32 hop0_tables_total_size; 534 u8 host_resident; 535}; 536 537/** 538 * struct hl_hints_range - hint addresses reserved va range. 539 * @start_addr: start address of the va range. 540 * @end_addr: end address of the va range. 541 */ 542struct hl_hints_range { 543 u64 start_addr; 544 u64 end_addr; 545}; 546 547/** 548 * struct asic_fixed_properties - ASIC specific immutable properties. 549 * @hw_queues_props: H/W queues properties. 550 * @special_blocks: points to an array containing special blocks info. 551 * @skip_special_blocks_cfg: special blocks skip configs. 552 * @cpucp_info: received various information from CPU-CP regarding the H/W, e.g. 553 * available sensors. 554 * @uboot_ver: F/W U-boot version. 555 * @preboot_ver: F/W Preboot version. 556 * @dmmu: DRAM MMU address translation properties. 557 * @pmmu: PCI (host) MMU address translation properties. 558 * @pmmu_huge: PCI (host) MMU address translation properties for memory 559 * allocated with huge pages. 560 * @hints_dram_reserved_va_range: dram hint addresses reserved range. 561 * @hints_host_reserved_va_range: host hint addresses reserved range. 562 * @hints_host_hpage_reserved_va_range: host huge page hint addresses reserved range. 563 * @sram_base_address: SRAM physical start address. 564 * @sram_end_address: SRAM physical end address. 565 * @sram_user_base_address - SRAM physical start address for user access. 566 * @dram_base_address: DRAM physical start address. 567 * @dram_end_address: DRAM physical end address. 568 * @dram_user_base_address: DRAM physical start address for user access. 569 * @dram_size: DRAM total size. 570 * @dram_pci_bar_size: size of PCI bar towards DRAM. 571 * @max_power_default: max power of the device after reset. 572 * @dc_power_default: power consumed by the device in mode idle. 573 * @dram_size_for_default_page_mapping: DRAM size needed to map to avoid page 574 * fault. 575 * @pcie_dbi_base_address: Base address of the PCIE_DBI block. 576 * @pcie_aux_dbi_reg_addr: Address of the PCIE_AUX DBI register. 577 * @mmu_pgt_addr: base physical address in DRAM of MMU page tables. 578 * @mmu_dram_default_page_addr: DRAM default page physical address. 579 * @tpc_enabled_mask: which TPCs are enabled. 580 * @tpc_binning_mask: which TPCs are binned. 0 means usable and 1 means binned. 581 * @dram_enabled_mask: which DRAMs are enabled. 582 * @dram_binning_mask: which DRAMs are binned. 0 means usable, 1 means binned. 583 * @dram_hints_align_mask: dram va hint addresses alignment mask which is used 584 * for hints validity check. 585 * @cfg_base_address: config space base address. 586 * @mmu_cache_mng_addr: address of the MMU cache. 587 * @mmu_cache_mng_size: size of the MMU cache. 588 * @device_dma_offset_for_host_access: the offset to add to host DMA addresses 589 * to enable the device to access them. 590 * @host_base_address: host physical start address for host DMA from device 591 * @host_end_address: host physical end address for host DMA from device 592 * @max_freq_value: current max clk frequency. 593 * @engine_core_interrupt_reg_addr: interrupt register address for engine core to use 594 * in order to raise events toward FW. 595 * @clk_pll_index: clock PLL index that specify which PLL determines the clock 596 * we display to the user 597 * @mmu_pgt_size: MMU page tables total size. 598 * @mmu_pte_size: PTE size in MMU page tables. 599 * @dram_page_size: The DRAM physical page size. 600 * @cfg_size: configuration space size on SRAM. 601 * @sram_size: total size of SRAM. 602 * @max_asid: maximum number of open contexts (ASIDs). 603 * @num_of_events: number of possible internal H/W IRQs. 604 * @psoc_pci_pll_nr: PCI PLL NR value. 605 * @psoc_pci_pll_nf: PCI PLL NF value. 606 * @psoc_pci_pll_od: PCI PLL OD value. 607 * @psoc_pci_pll_div_factor: PCI PLL DIV FACTOR 1 value. 608 * @psoc_timestamp_frequency: frequency of the psoc timestamp clock. 609 * @high_pll: high PLL frequency used by the device. 610 * @cb_pool_cb_cnt: number of CBs in the CB pool. 611 * @cb_pool_cb_size: size of each CB in the CB pool. 612 * @decoder_enabled_mask: which decoders are enabled. 613 * @decoder_binning_mask: which decoders are binned, 0 means usable and 1 means binned. 614 * @rotator_enabled_mask: which rotators are enabled. 615 * @edma_enabled_mask: which EDMAs are enabled. 616 * @edma_binning_mask: which EDMAs are binned, 0 means usable and 1 means 617 * binned (at most one binned DMA). 618 * @max_pending_cs: maximum of concurrent pending command submissions 619 * @max_queues: maximum amount of queues in the system 620 * @fw_preboot_cpu_boot_dev_sts0: bitmap representation of preboot cpu 621 * capabilities reported by FW, bit description 622 * can be found in CPU_BOOT_DEV_STS0 623 * @fw_preboot_cpu_boot_dev_sts1: bitmap representation of preboot cpu 624 * capabilities reported by FW, bit description 625 * can be found in CPU_BOOT_DEV_STS1 626 * @fw_bootfit_cpu_boot_dev_sts0: bitmap representation of boot cpu security 627 * status reported by FW, bit description can be 628 * found in CPU_BOOT_DEV_STS0 629 * @fw_bootfit_cpu_boot_dev_sts1: bitmap representation of boot cpu security 630 * status reported by FW, bit description can be 631 * found in CPU_BOOT_DEV_STS1 632 * @fw_app_cpu_boot_dev_sts0: bitmap representation of application security 633 * status reported by FW, bit description can be 634 * found in CPU_BOOT_DEV_STS0 635 * @fw_app_cpu_boot_dev_sts1: bitmap representation of application security 636 * status reported by FW, bit description can be 637 * found in CPU_BOOT_DEV_STS1 638 * @max_dec: maximum number of decoders 639 * @hmmu_hif_enabled_mask: mask of HMMUs/HIFs that are not isolated (enabled) 640 * 1- enabled, 0- isolated. 641 * @faulty_dram_cluster_map: mask of faulty DRAM cluster. 642 * 1- faulty cluster, 0- good cluster. 643 * @xbar_edge_enabled_mask: mask of XBAR_EDGEs that are not isolated (enabled) 644 * 1- enabled, 0- isolated. 645 * @device_mem_alloc_default_page_size: may be different than dram_page_size only for ASICs for 646 * which the property supports_user_set_page_size is true 647 * (i.e. the DRAM supports multiple page sizes), otherwise 648 * it will shall be equal to dram_page_size. 649 * @num_engine_cores: number of engine cpu cores. 650 * @max_num_of_engines: maximum number of all engines in the ASIC. 651 * @num_of_special_blocks: special_blocks array size. 652 * @glbl_err_max_cause_num: global err max cause number. 653 * @hbw_flush_reg: register to read to generate HBW flush. value of 0 means HBW flush is 654 * not supported. 655 * @reserved_fw_mem_size: size of dram memory reserved for FW. 656 * @fw_event_queue_size: queue size for events from CPU-CP. 657 * A value of 0 means using the default HL_EQ_SIZE_IN_BYTES value. 658 * @collective_first_sob: first sync object available for collective use 659 * @collective_first_mon: first monitor available for collective use 660 * @sync_stream_first_sob: first sync object available for sync stream use 661 * @sync_stream_first_mon: first monitor available for sync stream use 662 * @first_available_user_sob: first sob available for the user 663 * @first_available_user_mon: first monitor available for the user 664 * @first_available_user_interrupt: first available interrupt reserved for the user 665 * @first_available_cq: first available CQ for the user. 666 * @user_interrupt_count: number of user interrupts. 667 * @user_dec_intr_count: number of decoder interrupts exposed to user. 668 * @tpc_interrupt_id: interrupt id for TPC to use in order to raise events towards the host. 669 * @eq_interrupt_id: interrupt id for EQ, uses to synchronize EQ interrupts in hard-reset. 670 * @cache_line_size: device cache line size. 671 * @server_type: Server type that the ASIC is currently installed in. 672 * The value is according to enum hl_server_type in uapi file. 673 * @completion_queues_count: number of completion queues. 674 * @completion_mode: 0 - job based completion, 1 - cs based completion 675 * @mme_master_slave_mode: 0 - Each MME works independently, 1 - MME works 676 * in Master/Slave mode 677 * @fw_security_enabled: true if security measures are enabled in firmware, 678 * false otherwise 679 * @fw_cpu_boot_dev_sts0_valid: status bits are valid and can be fetched from 680 * BOOT_DEV_STS0 681 * @fw_cpu_boot_dev_sts1_valid: status bits are valid and can be fetched from 682 * BOOT_DEV_STS1 683 * @dram_supports_virtual_memory: is there an MMU towards the DRAM 684 * @hard_reset_done_by_fw: true if firmware is handling hard reset flow 685 * @num_functional_hbms: number of functional HBMs in each DCORE. 686 * @hints_range_reservation: device support hint addresses range reservation. 687 * @iatu_done_by_fw: true if iATU configuration is being done by FW. 688 * @dynamic_fw_load: is dynamic FW load is supported. 689 * @gic_interrupts_enable: true if FW is not blocking GIC controller, 690 * false otherwise. 691 * @use_get_power_for_reset_history: To support backward compatibility for Goya 692 * and Gaudi 693 * @supports_compute_reset: is a reset which is not a hard-reset supported by this asic. 694 * @allow_inference_soft_reset: true if the ASIC supports soft reset that is 695 * initiated by user or TDR. This is only true 696 * in inference ASICs, as there is no real-world 697 * use-case of doing soft-reset in training (due 698 * to the fact that training runs on multiple 699 * devices) 700 * @configurable_stop_on_err: is stop-on-error option configurable via debugfs. 701 * @set_max_power_on_device_init: true if need to set max power in F/W on device init. 702 * @supports_user_set_page_size: true if user can set the allocation page size. 703 * @dma_mask: the dma mask to be set for this device. 704 * @supports_advanced_cpucp_rc: true if new cpucp opcodes are supported. 705 * @supports_engine_modes: true if changing engines/engine_cores modes is supported. 706 * @support_dynamic_resereved_fw_size: true if we support dynamic reserved size for fw. 707 * @supports_nvme: indicates whether the asic supports NVMe P2P DMA. 708 */ 709struct asic_fixed_properties { 710 struct hw_queue_properties *hw_queues_props; 711 struct hl_special_block_info *special_blocks; 712 struct hl_skip_blocks_cfg skip_special_blocks_cfg; 713 struct cpucp_info cpucp_info; 714 char uboot_ver[VERSION_MAX_LEN]; 715 char preboot_ver[VERSION_MAX_LEN]; 716 struct hl_mmu_properties dmmu; 717 struct hl_mmu_properties pmmu; 718 struct hl_mmu_properties pmmu_huge; 719 struct hl_hints_range hints_dram_reserved_va_range; 720 struct hl_hints_range hints_host_reserved_va_range; 721 struct hl_hints_range hints_host_hpage_reserved_va_range; 722 u64 sram_base_address; 723 u64 sram_end_address; 724 u64 sram_user_base_address; 725 u64 dram_base_address; 726 u64 dram_end_address; 727 u64 dram_user_base_address; 728 u64 dram_size; 729 u64 dram_pci_bar_size; 730 u64 max_power_default; 731 u64 dc_power_default; 732 u64 dram_size_for_default_page_mapping; 733 u64 pcie_dbi_base_address; 734 u64 pcie_aux_dbi_reg_addr; 735 u64 mmu_pgt_addr; 736 u64 mmu_dram_default_page_addr; 737 u64 tpc_enabled_mask; 738 u64 tpc_binning_mask; 739 u64 dram_enabled_mask; 740 u64 dram_binning_mask; 741 u64 dram_hints_align_mask; 742 u64 cfg_base_address; 743 u64 mmu_cache_mng_addr; 744 u64 mmu_cache_mng_size; 745 u64 device_dma_offset_for_host_access; 746 u64 host_base_address; 747 u64 host_end_address; 748 u64 max_freq_value; 749 u64 engine_core_interrupt_reg_addr; 750 u32 clk_pll_index; 751 u32 mmu_pgt_size; 752 u32 mmu_pte_size; 753 u32 dram_page_size; 754 u32 cfg_size; 755 u32 sram_size; 756 u32 max_asid; 757 u32 num_of_events; 758 u32 psoc_pci_pll_nr; 759 u32 psoc_pci_pll_nf; 760 u32 psoc_pci_pll_od; 761 u32 psoc_pci_pll_div_factor; 762 u32 psoc_timestamp_frequency; 763 u32 high_pll; 764 u32 cb_pool_cb_cnt; 765 u32 cb_pool_cb_size; 766 u32 decoder_enabled_mask; 767 u32 decoder_binning_mask; 768 u32 rotator_enabled_mask; 769 u32 edma_enabled_mask; 770 u32 edma_binning_mask; 771 u32 max_pending_cs; 772 u32 max_queues; 773 u32 fw_preboot_cpu_boot_dev_sts0; 774 u32 fw_preboot_cpu_boot_dev_sts1; 775 u32 fw_bootfit_cpu_boot_dev_sts0; 776 u32 fw_bootfit_cpu_boot_dev_sts1; 777 u32 fw_app_cpu_boot_dev_sts0; 778 u32 fw_app_cpu_boot_dev_sts1; 779 u32 max_dec; 780 u32 hmmu_hif_enabled_mask; 781 u32 faulty_dram_cluster_map; 782 u32 xbar_edge_enabled_mask; 783 u32 device_mem_alloc_default_page_size; 784 u32 num_engine_cores; 785 u32 max_num_of_engines; 786 u32 num_of_special_blocks; 787 u32 glbl_err_max_cause_num; 788 u32 hbw_flush_reg; 789 u32 reserved_fw_mem_size; 790 u32 fw_event_queue_size; 791 u16 collective_first_sob; 792 u16 collective_first_mon; 793 u16 sync_stream_first_sob; 794 u16 sync_stream_first_mon; 795 u16 first_available_user_sob[HL_MAX_DCORES]; 796 u16 first_available_user_mon[HL_MAX_DCORES]; 797 u16 first_available_user_interrupt; 798 u16 first_available_cq[HL_MAX_DCORES]; 799 u16 user_interrupt_count; 800 u16 user_dec_intr_count; 801 u16 tpc_interrupt_id; 802 u16 eq_interrupt_id; 803 u16 cache_line_size; 804 u16 server_type; 805 u8 completion_queues_count; 806 u8 completion_mode; 807 u8 mme_master_slave_mode; 808 u8 fw_security_enabled; 809 u8 fw_cpu_boot_dev_sts0_valid; 810 u8 fw_cpu_boot_dev_sts1_valid; 811 u8 dram_supports_virtual_memory; 812 u8 hard_reset_done_by_fw; 813 u8 num_functional_hbms; 814 u8 hints_range_reservation; 815 u8 iatu_done_by_fw; 816 u8 dynamic_fw_load; 817 u8 gic_interrupts_enable; 818 u8 use_get_power_for_reset_history; 819 u8 supports_compute_reset; 820 u8 allow_inference_soft_reset; 821 u8 configurable_stop_on_err; 822 u8 set_max_power_on_device_init; 823 u8 supports_user_set_page_size; 824 u8 dma_mask; 825 u8 supports_advanced_cpucp_rc; 826 u8 supports_engine_modes; 827 u8 support_dynamic_resereved_fw_size; 828 u8 supports_nvme; 829}; 830 831/** 832 * struct hl_fence - software synchronization primitive 833 * @completion: fence is implemented using completion 834 * @refcount: refcount for this fence 835 * @cs_sequence: sequence of the corresponding command submission 836 * @stream_master_qid_map: streams masters QID bitmap to represent all streams 837 * masters QIDs that multi cs is waiting on 838 * @error: mark this fence with error 839 * @timestamp: timestamp upon completion 840 * @mcs_handling_done: indicates that corresponding command submission has 841 * finished msc handling, this does not mean it was part 842 * of the mcs 843 */ 844struct hl_fence { 845 struct completion completion; 846 struct kref refcount; 847 u64 cs_sequence; 848 u32 stream_master_qid_map; 849 int error; 850 ktime_t timestamp; 851 u8 mcs_handling_done; 852}; 853 854/** 855 * struct hl_cs_compl - command submission completion object. 856 * @base_fence: hl fence object. 857 * @lock: spinlock to protect fence. 858 * @hdev: habanalabs device structure. 859 * @hw_sob: the H/W SOB used in this signal/wait CS. 860 * @encaps_sig_hdl: encaps signals handler. 861 * @cs_seq: command submission sequence number. 862 * @type: type of the CS - signal/wait. 863 * @sob_val: the SOB value that is used in this signal/wait CS. 864 * @sob_group: the SOB group that is used in this collective wait CS. 865 * @encaps_signals: indication whether it's a completion object of cs with 866 * encaps signals or not. 867 */ 868struct hl_cs_compl { 869 struct hl_fence base_fence; 870 spinlock_t lock; 871 struct hl_device *hdev; 872 struct hl_hw_sob *hw_sob; 873 struct hl_cs_encaps_sig_handle *encaps_sig_hdl; 874 u64 cs_seq; 875 enum hl_cs_type type; 876 u16 sob_val; 877 u16 sob_group; 878 bool encaps_signals; 879}; 880 881/* 882 * Command Buffers 883 */ 884 885/** 886 * struct hl_ts_buff - describes a timestamp buffer. 887 * @kernel_buff_address: Holds the internal buffer's kernel virtual address. 888 * @user_buff_address: Holds the user buffer's kernel virtual address. 889 * @kernel_buff_size: Holds the internal kernel buffer size. 890 */ 891struct hl_ts_buff { 892 void *kernel_buff_address; 893 void *user_buff_address; 894 u32 kernel_buff_size; 895}; 896 897struct hl_mmap_mem_buf; 898 899/** 900 * struct hl_mem_mgr - describes unified memory manager for mappable memory chunks. 901 * @dev: back pointer to the owning device 902 * @lock: protects handles 903 * @handles: an idr holding all active handles to the memory buffers in the system. 904 */ 905struct hl_mem_mgr { 906 struct device *dev; 907 spinlock_t lock; 908 struct idr handles; 909}; 910 911/** 912 * struct hl_mem_mgr_fini_stats - describes statistics returned during memory manager teardown. 913 * @n_busy_cb: the amount of CB handles that could not be removed 914 * @n_busy_ts: the amount of TS handles that could not be removed 915 * @n_busy_other: the amount of any other type of handles that could not be removed 916 */ 917struct hl_mem_mgr_fini_stats { 918 u32 n_busy_cb; 919 u32 n_busy_ts; 920 u32 n_busy_other; 921}; 922 923/** 924 * struct hl_mmap_mem_buf_behavior - describes unified memory manager buffer behavior 925 * @topic: string identifier used for logging 926 * @mem_id: memory type identifier, embedded in the handle and used to identify 927 * the memory type by handle. 928 * @alloc: callback executed on buffer allocation, shall allocate the memory, 929 * set it under buffer private, and set mappable size. 930 * @mmap: callback executed on mmap, must map the buffer to vma 931 * @release: callback executed on release, must free the resources used by the buffer 932 */ 933struct hl_mmap_mem_buf_behavior { 934 const char *topic; 935 u64 mem_id; 936 937 int (*alloc)(struct hl_mmap_mem_buf *buf, gfp_t gfp, void *args); 938 int (*mmap)(struct hl_mmap_mem_buf *buf, struct vm_area_struct *vma, void *args); 939 void (*release)(struct hl_mmap_mem_buf *buf); 940}; 941 942/** 943 * struct hl_mmap_mem_buf - describes a single unified memory buffer 944 * @behavior: buffer behavior 945 * @mmg: back pointer to the unified memory manager 946 * @refcount: reference counter for buffer users 947 * @private: pointer to buffer behavior private data 948 * @mmap: atomic boolean indicating whether or not the buffer is mapped right now 949 * @real_mapped_size: the actual size of buffer mapped, after part of it may be released, 950 * may change at runtime. 951 * @mappable_size: the original mappable size of the buffer, does not change after 952 * the allocation. 953 * @handle: the buffer id in mmg handles store 954 */ 955struct hl_mmap_mem_buf { 956 struct hl_mmap_mem_buf_behavior *behavior; 957 struct hl_mem_mgr *mmg; 958 struct kref refcount; 959 void *private; 960 atomic_t mmap; 961 u64 real_mapped_size; 962 u64 mappable_size; 963 u64 handle; 964}; 965 966/** 967 * struct hl_cb - describes a Command Buffer. 968 * @hdev: pointer to device this CB belongs to. 969 * @ctx: pointer to the CB owner's context. 970 * @buf: back pointer to the parent mappable memory buffer 971 * @debugfs_list: node in debugfs list of command buffers. 972 * @pool_list: node in pool list of command buffers. 973 * @kernel_address: Holds the CB's kernel virtual address. 974 * @virtual_addr: Holds the CB's virtual address. 975 * @bus_address: Holds the CB's DMA address. 976 * @size: holds the CB's size. 977 * @roundup_size: holds the cb size after roundup to page size. 978 * @cs_cnt: holds number of CS that this CB participates in. 979 * @is_handle_destroyed: atomic boolean indicating whether or not the CB handle was destroyed. 980 * @is_pool: true if CB was acquired from the pool, false otherwise. 981 * @is_internal: internally allocated 982 * @is_mmu_mapped: true if the CB is mapped to the device's MMU. 983 */ 984struct hl_cb { 985 struct hl_device *hdev; 986 struct hl_ctx *ctx; 987 struct hl_mmap_mem_buf *buf; 988 struct list_head debugfs_list; 989 struct list_head pool_list; 990 void *kernel_address; 991 u64 virtual_addr; 992 dma_addr_t bus_address; 993 u32 size; 994 u32 roundup_size; 995 atomic_t cs_cnt; 996 atomic_t is_handle_destroyed; 997 u8 is_pool; 998 u8 is_internal; 999 u8 is_mmu_mapped; 1000}; 1001 1002 1003/* 1004 * QUEUES 1005 */ 1006 1007struct hl_cs_job; 1008 1009/* Queue length of external and HW queues */ 1010#define HL_QUEUE_LENGTH 4096 1011#define HL_QUEUE_SIZE_IN_BYTES (HL_QUEUE_LENGTH * HL_BD_SIZE) 1012 1013#if (HL_MAX_JOBS_PER_CS > HL_QUEUE_LENGTH) 1014#error "HL_QUEUE_LENGTH must be greater than HL_MAX_JOBS_PER_CS" 1015#endif 1016 1017/* HL_CQ_LENGTH is in units of struct hl_cq_entry */ 1018#define HL_CQ_LENGTH HL_QUEUE_LENGTH 1019#define HL_CQ_SIZE_IN_BYTES (HL_CQ_LENGTH * HL_CQ_ENTRY_SIZE) 1020 1021/* Must be power of 2 */ 1022#define HL_EQ_LENGTH 64 1023#define HL_EQ_SIZE_IN_BYTES (HL_EQ_LENGTH * HL_EQ_ENTRY_SIZE) 1024 1025/* Host <-> CPU-CP shared memory size */ 1026#define HL_CPU_ACCESSIBLE_MEM_SIZE SZ_2M 1027 1028/** 1029 * struct hl_sync_stream_properties - 1030 * describes a H/W queue sync stream properties 1031 * @hw_sob: array of the used H/W SOBs by this H/W queue. 1032 * @next_sob_val: the next value to use for the currently used SOB. 1033 * @base_sob_id: the base SOB id of the SOBs used by this queue. 1034 * @base_mon_id: the base MON id of the MONs used by this queue. 1035 * @collective_mstr_mon_id: the MON ids of the MONs used by this master queue 1036 * in order to sync with all slave queues. 1037 * @collective_slave_mon_id: the MON id used by this slave queue in order to 1038 * sync with its master queue. 1039 * @collective_sob_id: current SOB id used by this collective slave queue 1040 * to signal its collective master queue upon completion. 1041 * @curr_sob_offset: the id offset to the currently used SOB from the 1042 * HL_RSVD_SOBS that are being used by this queue. 1043 */ 1044struct hl_sync_stream_properties { 1045 struct hl_hw_sob hw_sob[HL_RSVD_SOBS]; 1046 u16 next_sob_val; 1047 u16 base_sob_id; 1048 u16 base_mon_id; 1049 u16 collective_mstr_mon_id[HL_COLLECTIVE_RSVD_MSTR_MONS]; 1050 u16 collective_slave_mon_id; 1051 u16 collective_sob_id; 1052 u8 curr_sob_offset; 1053}; 1054 1055/** 1056 * struct hl_encaps_signals_mgr - describes sync stream encapsulated signals 1057 * handlers manager 1058 * @lock: protects handles. 1059 * @handles: an idr to hold all encapsulated signals handles. 1060 */ 1061struct hl_encaps_signals_mgr { 1062 spinlock_t lock; 1063 struct idr handles; 1064}; 1065 1066/** 1067 * struct hl_hw_queue - describes a H/W transport queue. 1068 * @shadow_queue: pointer to a shadow queue that holds pointers to jobs. 1069 * @sync_stream_prop: sync stream queue properties 1070 * @queue_type: type of queue. 1071 * @collective_mode: collective mode of current queue 1072 * @kernel_address: holds the queue's kernel virtual address. 1073 * @bus_address: holds the queue's DMA address. 1074 * @pq_dram_address: hold the dram address when the PQ is allocated, used when dram_bd is true in 1075 * queue properites. 1076 * @pi: holds the queue's pi value. 1077 * @ci: holds the queue's ci value, AS CALCULATED BY THE DRIVER (not real ci). 1078 * @hw_queue_id: the id of the H/W queue. 1079 * @cq_id: the id for the corresponding CQ for this H/W queue. 1080 * @msi_vec: the IRQ number of the H/W queue. 1081 * @int_queue_len: length of internal queue (number of entries). 1082 * @valid: is the queue valid (we have array of 32 queues, not all of them 1083 * exist). 1084 * @supports_sync_stream: True if queue supports sync stream 1085 * @dram_bd: True if the bd should be copied to dram, needed for PQ which has been allocated on dram 1086 */ 1087struct hl_hw_queue { 1088 struct hl_cs_job **shadow_queue; 1089 struct hl_sync_stream_properties sync_stream_prop; 1090 enum hl_queue_type queue_type; 1091 enum hl_collective_mode collective_mode; 1092 void *kernel_address; 1093 dma_addr_t bus_address; 1094 u64 pq_dram_address; 1095 u32 pi; 1096 atomic_t ci; 1097 u32 hw_queue_id; 1098 u32 cq_id; 1099 u32 msi_vec; 1100 u16 int_queue_len; 1101 u8 valid; 1102 u8 supports_sync_stream; 1103 u8 dram_bd; 1104}; 1105 1106/** 1107 * struct hl_cq - describes a completion queue 1108 * @hdev: pointer to the device structure 1109 * @kernel_address: holds the queue's kernel virtual address 1110 * @bus_address: holds the queue's DMA address 1111 * @cq_idx: completion queue index in array 1112 * @hw_queue_id: the id of the matching H/W queue 1113 * @ci: ci inside the queue 1114 * @pi: pi inside the queue 1115 * @free_slots_cnt: counter of free slots in queue 1116 */ 1117struct hl_cq { 1118 struct hl_device *hdev; 1119 void *kernel_address; 1120 dma_addr_t bus_address; 1121 u32 cq_idx; 1122 u32 hw_queue_id; 1123 u32 ci; 1124 u32 pi; 1125 atomic_t free_slots_cnt; 1126}; 1127 1128enum hl_user_interrupt_type { 1129 HL_USR_INTERRUPT_CQ = 0, 1130 HL_USR_INTERRUPT_DECODER, 1131 HL_USR_INTERRUPT_TPC, 1132 HL_USR_INTERRUPT_UNEXPECTED 1133}; 1134 1135/** 1136 * struct hl_ts_free_jobs - holds user interrupt ts free nodes related data 1137 * @free_nodes_pool: pool of nodes to be used for free timestamp jobs 1138 * @free_nodes_length: number of nodes in free_nodes_pool 1139 * @next_avail_free_node_idx: index of the next free node in the pool 1140 * 1141 * the free nodes pool must be protected by the user interrupt lock 1142 * to avoid race between different interrupts which are using the same 1143 * ts buffer with different offsets. 1144 */ 1145struct hl_ts_free_jobs { 1146 struct timestamp_reg_free_node *free_nodes_pool; 1147 u32 free_nodes_length; 1148 u32 next_avail_free_node_idx; 1149}; 1150 1151/** 1152 * struct hl_user_interrupt - holds user interrupt information 1153 * @hdev: pointer to the device structure 1154 * @ts_free_jobs_data: timestamp free jobs related data 1155 * @type: user interrupt type 1156 * @wait_list_head: head to the list of user threads pending on this interrupt 1157 * @ts_list_head: head to the list of timestamp records 1158 * @wait_list_lock: protects wait_list_head 1159 * @ts_list_lock: protects ts_list_head 1160 * @timestamp: last timestamp taken upon interrupt 1161 * @interrupt_id: msix interrupt id 1162 */ 1163struct hl_user_interrupt { 1164 struct hl_device *hdev; 1165 struct hl_ts_free_jobs ts_free_jobs_data; 1166 enum hl_user_interrupt_type type; 1167 struct list_head wait_list_head; 1168 struct list_head ts_list_head; 1169 spinlock_t wait_list_lock; 1170 spinlock_t ts_list_lock; 1171 ktime_t timestamp; 1172 u32 interrupt_id; 1173}; 1174 1175/** 1176 * struct timestamp_reg_free_node - holds the timestamp registration free objects node 1177 * @free_objects_node: node in the list free_obj_jobs 1178 * @cq_cb: pointer to cq command buffer to be freed 1179 * @buf: pointer to timestamp buffer to be freed 1180 * @in_use: indicates whether the node still in use in workqueue thread. 1181 * @dynamic_alloc: indicates whether the node was allocated dynamically in the interrupt handler 1182 */ 1183struct timestamp_reg_free_node { 1184 struct list_head free_objects_node; 1185 struct hl_cb *cq_cb; 1186 struct hl_mmap_mem_buf *buf; 1187 atomic_t in_use; 1188 u8 dynamic_alloc; 1189}; 1190 1191/* struct timestamp_reg_work_obj - holds the timestamp registration free objects job 1192 * the job will be to pass over the free_obj_jobs list and put refcount to objects 1193 * in each node of the list 1194 * @free_obj: workqueue object to free timestamp registration node objects 1195 * @hdev: pointer to the device structure 1196 * @free_obj_head: list of free jobs nodes (node type timestamp_reg_free_node) 1197 * @dynamic_alloc_free_obj_head: list of free jobs nodes which were dynamically allocated in the 1198 * interrupt handler. 1199 */ 1200struct timestamp_reg_work_obj { 1201 struct work_struct free_obj; 1202 struct hl_device *hdev; 1203 struct list_head *free_obj_head; 1204 struct list_head *dynamic_alloc_free_obj_head; 1205}; 1206 1207/* struct timestamp_reg_info - holds the timestamp registration related data. 1208 * @buf: pointer to the timestamp buffer which include both user/kernel buffers. 1209 * relevant only when doing timestamps records registration. 1210 * @cq_cb: pointer to CQ counter CB. 1211 * @interrupt: interrupt that the node hanged on it's wait list. 1212 * @timestamp_kernel_addr: timestamp handle address, where to set timestamp 1213 * relevant only when doing timestamps records 1214 * registration. 1215 * @in_use: indicates if the node already in use. relevant only when doing 1216 * timestamps records registration, since in this case the driver 1217 * will have it's own buffer which serve as a records pool instead of 1218 * allocating records dynamically. 1219 */ 1220struct timestamp_reg_info { 1221 struct hl_mmap_mem_buf *buf; 1222 struct hl_cb *cq_cb; 1223 struct hl_user_interrupt *interrupt; 1224 u64 *timestamp_kernel_addr; 1225 bool in_use; 1226}; 1227 1228/** 1229 * struct hl_user_pending_interrupt - holds a context to a user thread 1230 * pending on an interrupt 1231 * @ts_reg_info: holds the timestamps registration nodes info 1232 * @list_node: node in the list of user threads pending on an interrupt or timestamp 1233 * @fence: hl fence object for interrupt completion 1234 * @cq_target_value: CQ target value 1235 * @cq_kernel_addr: CQ kernel address, to be used in the cq interrupt 1236 * handler for target value comparison 1237 */ 1238struct hl_user_pending_interrupt { 1239 struct timestamp_reg_info ts_reg_info; 1240 struct list_head list_node; 1241 struct hl_fence fence; 1242 u64 cq_target_value; 1243 u64 *cq_kernel_addr; 1244}; 1245 1246/** 1247 * struct hl_eq - describes the event queue (single one per device) 1248 * @hdev: pointer to the device structure 1249 * @kernel_address: holds the queue's kernel virtual address 1250 * @bus_address: holds the queue's DMA address 1251 * @size: the event queue size 1252 * @ci: ci inside the queue 1253 * @prev_eqe_index: the index of the previous event queue entry. The index of 1254 * the current entry's index must be +1 of the previous one. 1255 * @check_eqe_index: do we need to check the index of the current entry vs. the 1256 * previous one. This is for backward compatibility with older 1257 * firmwares 1258 */ 1259struct hl_eq { 1260 struct hl_device *hdev; 1261 void *kernel_address; 1262 dma_addr_t bus_address; 1263 u32 size; 1264 u32 ci; 1265 u32 prev_eqe_index; 1266 bool check_eqe_index; 1267}; 1268 1269/** 1270 * struct hl_dec - describes a decoder sw instance. 1271 * @hdev: pointer to the device structure. 1272 * @abnrm_intr_work: workqueue work item to run when decoder generates an error interrupt. 1273 * @core_id: ID of the decoder. 1274 * @base_addr: base address of the decoder. 1275 */ 1276struct hl_dec { 1277 struct hl_device *hdev; 1278 struct work_struct abnrm_intr_work; 1279 u32 core_id; 1280 u32 base_addr; 1281}; 1282 1283/** 1284 * enum hl_asic_type - supported ASIC types. 1285 * @ASIC_INVALID: Invalid ASIC type. 1286 * @ASIC_GOYA: Goya device (HL-1000). 1287 * @ASIC_GAUDI: Gaudi device (HL-2000). 1288 * @ASIC_GAUDI_SEC: Gaudi secured device (HL-2000). 1289 * @ASIC_GAUDI2: Gaudi2 device. 1290 * @ASIC_GAUDI2B: Gaudi2B device. 1291 * @ASIC_GAUDI2C: Gaudi2C device. 1292 * @ASIC_GAUDI2D: Gaudi2D device. 1293 */ 1294enum hl_asic_type { 1295 ASIC_INVALID, 1296 1297 ASIC_GOYA, 1298 ASIC_GAUDI, 1299 ASIC_GAUDI_SEC, 1300 ASIC_GAUDI2, 1301 ASIC_GAUDI2B, 1302 ASIC_GAUDI2C, 1303 ASIC_GAUDI2D, 1304}; 1305 1306struct hl_cs_parser; 1307 1308/** 1309 * enum hl_pm_mng_profile - power management profile. 1310 * @PM_AUTO: internal clock is set by the Linux driver. 1311 * @PM_MANUAL: internal clock is set by the user. 1312 * @PM_LAST: last power management type. 1313 */ 1314enum hl_pm_mng_profile { 1315 PM_AUTO = 1, 1316 PM_MANUAL, 1317 PM_LAST 1318}; 1319 1320/** 1321 * enum hl_pll_frequency - PLL frequency. 1322 * @PLL_HIGH: high frequency. 1323 * @PLL_LOW: low frequency. 1324 * @PLL_LAST: last frequency values that were configured by the user. 1325 */ 1326enum hl_pll_frequency { 1327 PLL_HIGH = 1, 1328 PLL_LOW, 1329 PLL_LAST 1330}; 1331 1332#define PLL_REF_CLK 50 1333 1334enum div_select_defs { 1335 DIV_SEL_REF_CLK = 0, 1336 DIV_SEL_PLL_CLK = 1, 1337 DIV_SEL_DIVIDED_REF = 2, 1338 DIV_SEL_DIVIDED_PLL = 3, 1339}; 1340 1341enum debugfs_access_type { 1342 DEBUGFS_READ8, 1343 DEBUGFS_WRITE8, 1344 DEBUGFS_READ32, 1345 DEBUGFS_WRITE32, 1346 DEBUGFS_READ64, 1347 DEBUGFS_WRITE64, 1348}; 1349 1350enum pci_region { 1351 PCI_REGION_CFG, 1352 PCI_REGION_SRAM, 1353 PCI_REGION_DRAM, 1354 PCI_REGION_SP_SRAM, 1355 PCI_REGION_NUMBER, 1356}; 1357 1358/** 1359 * struct pci_mem_region - describe memory region in a PCI bar 1360 * @region_base: region base address 1361 * @region_size: region size 1362 * @bar_size: size of the BAR 1363 * @offset_in_bar: region offset into the bar 1364 * @bar_id: bar ID of the region 1365 * @used: if used 1, otherwise 0 1366 */ 1367struct pci_mem_region { 1368 u64 region_base; 1369 u64 region_size; 1370 u64 bar_size; 1371 u64 offset_in_bar; 1372 u8 bar_id; 1373 u8 used; 1374}; 1375 1376/** 1377 * struct static_fw_load_mgr - static FW load manager 1378 * @preboot_version_max_off: max offset to preboot version 1379 * @boot_fit_version_max_off: max offset to boot fit version 1380 * @kmd_msg_to_cpu_reg: register address for KDM->CPU messages 1381 * @cpu_cmd_status_to_host_reg: register address for CPU command status response 1382 * @cpu_boot_status_reg: boot status register 1383 * @cpu_boot_dev_status0_reg: boot device status register 0 1384 * @cpu_boot_dev_status1_reg: boot device status register 1 1385 * @boot_err0_reg: boot error register 0 1386 * @boot_err1_reg: boot error register 1 1387 * @preboot_version_offset_reg: SRAM offset to preboot version register 1388 * @boot_fit_version_offset_reg: SRAM offset to boot fit version register 1389 * @sram_offset_mask: mask for getting offset into the SRAM 1390 * @cpu_reset_wait_msec: used when setting WFE via kmd_msg_to_cpu_reg 1391 */ 1392struct static_fw_load_mgr { 1393 u64 preboot_version_max_off; 1394 u64 boot_fit_version_max_off; 1395 u32 kmd_msg_to_cpu_reg; 1396 u32 cpu_cmd_status_to_host_reg; 1397 u32 cpu_boot_status_reg; 1398 u32 cpu_boot_dev_status0_reg; 1399 u32 cpu_boot_dev_status1_reg; 1400 u32 boot_err0_reg; 1401 u32 boot_err1_reg; 1402 u32 preboot_version_offset_reg; 1403 u32 boot_fit_version_offset_reg; 1404 u32 sram_offset_mask; 1405 u32 cpu_reset_wait_msec; 1406}; 1407 1408/** 1409 * struct fw_response - FW response to LKD command 1410 * @ram_offset: descriptor offset into the RAM 1411 * @ram_type: RAM type containing the descriptor (SRAM/DRAM) 1412 * @status: command status 1413 */ 1414struct fw_response { 1415 u32 ram_offset; 1416 u8 ram_type; 1417 u8 status; 1418}; 1419 1420/** 1421 * struct dynamic_fw_load_mgr - dynamic FW load manager 1422 * @response: FW to LKD response 1423 * @comm_desc: the communication descriptor with FW 1424 * @image_region: region to copy the FW image to 1425 * @fw_image_size: size of FW image to load 1426 * @wait_for_bl_timeout: timeout for waiting for boot loader to respond 1427 * @fw_desc_valid: true if FW descriptor has been validated and hence the data can be used 1428 */ 1429struct dynamic_fw_load_mgr { 1430 struct fw_response response; 1431 struct lkd_fw_comms_desc comm_desc; 1432 struct pci_mem_region *image_region; 1433 size_t fw_image_size; 1434 u32 wait_for_bl_timeout; 1435 bool fw_desc_valid; 1436}; 1437 1438/** 1439 * struct pre_fw_load_props - needed properties for pre-FW load 1440 * @cpu_boot_status_reg: cpu_boot_status register address 1441 * @sts_boot_dev_sts0_reg: sts_boot_dev_sts0 register address 1442 * @sts_boot_dev_sts1_reg: sts_boot_dev_sts1 register address 1443 * @boot_err0_reg: boot_err0 register address 1444 * @boot_err1_reg: boot_err1 register address 1445 * @wait_for_preboot_timeout: timeout to poll for preboot ready 1446 * @wait_for_preboot_extended_timeout: timeout to pull for preboot ready in case where we know 1447 * preboot needs longer time. 1448 */ 1449struct pre_fw_load_props { 1450 u32 cpu_boot_status_reg; 1451 u32 sts_boot_dev_sts0_reg; 1452 u32 sts_boot_dev_sts1_reg; 1453 u32 boot_err0_reg; 1454 u32 boot_err1_reg; 1455 u32 wait_for_preboot_timeout; 1456 u32 wait_for_preboot_extended_timeout; 1457}; 1458 1459/** 1460 * struct fw_image_props - properties of FW image 1461 * @image_name: name of the image 1462 * @src_off: offset in src FW to copy from 1463 * @copy_size: amount of bytes to copy (0 to copy the whole binary) 1464 */ 1465struct fw_image_props { 1466 char *image_name; 1467 u32 src_off; 1468 u32 copy_size; 1469}; 1470 1471/** 1472 * struct fw_load_mgr - manager FW loading process 1473 * @dynamic_loader: specific structure for dynamic load 1474 * @static_loader: specific structure for static load 1475 * @pre_fw_load_props: parameter for pre FW load 1476 * @boot_fit_img: boot fit image properties 1477 * @linux_img: linux image properties 1478 * @cpu_timeout: CPU response timeout in usec 1479 * @boot_fit_timeout: Boot fit load timeout in usec 1480 * @skip_bmc: should BMC be skipped 1481 * @sram_bar_id: SRAM bar ID 1482 * @dram_bar_id: DRAM bar ID 1483 * @fw_comp_loaded: bitmask of loaded FW components. set bit meaning loaded 1484 * component. values are set according to enum hl_fw_types. 1485 */ 1486struct fw_load_mgr { 1487 union { 1488 struct dynamic_fw_load_mgr dynamic_loader; 1489 struct static_fw_load_mgr static_loader; 1490 }; 1491 struct pre_fw_load_props pre_fw_load; 1492 struct fw_image_props boot_fit_img; 1493 struct fw_image_props linux_img; 1494 u32 cpu_timeout; 1495 u32 boot_fit_timeout; 1496 u8 skip_bmc; 1497 u8 sram_bar_id; 1498 u8 dram_bar_id; 1499 u8 fw_comp_loaded; 1500}; 1501 1502struct hl_cs; 1503 1504/** 1505 * struct engines_data - asic engines data 1506 * @buf: buffer for engines data in ascii 1507 * @actual_size: actual size of data that was written by the driver to the allocated buffer 1508 * @allocated_buf_size: total size of allocated buffer 1509 */ 1510struct engines_data { 1511 char *buf; 1512 int actual_size; 1513 u32 allocated_buf_size; 1514}; 1515 1516/** 1517 * struct hl_asic_funcs - ASIC specific functions that are can be called from 1518 * common code. 1519 * @early_init: sets up early driver state (pre sw_init), doesn't configure H/W. 1520 * @early_fini: tears down what was done in early_init. 1521 * @late_init: sets up late driver/hw state (post hw_init) - Optional. 1522 * @late_fini: tears down what was done in late_init (pre hw_fini) - Optional. 1523 * @sw_init: sets up driver state, does not configure H/W. 1524 * @sw_fini: tears down driver state, does not configure H/W. 1525 * @hw_init: sets up the H/W state. 1526 * @hw_fini: tears down the H/W state. 1527 * @halt_engines: halt engines, needed for reset sequence. This also disables 1528 * interrupts from the device. Should be called before 1529 * hw_fini and before CS rollback. 1530 * @suspend: handles IP specific H/W or SW changes for suspend. 1531 * @resume: handles IP specific H/W or SW changes for resume. 1532 * @mmap: maps a memory. 1533 * @ring_doorbell: increment PI on a given QMAN. 1534 * @pqe_write: Write the PQ entry to the PQ. This is ASIC-specific 1535 * function because the PQs are located in different memory areas 1536 * per ASIC (SRAM, DRAM, Host memory) and therefore, the method of 1537 * writing the PQE must match the destination memory area 1538 * properties. 1539 * @asic_dma_alloc_coherent: Allocate coherent DMA memory by calling 1540 * dma_alloc_coherent(). This is ASIC function because 1541 * its implementation is not trivial when the driver 1542 * is loaded in simulation mode (not upstreamed). 1543 * @asic_dma_free_coherent: Free coherent DMA memory by calling 1544 * dma_free_coherent(). This is ASIC function because 1545 * its implementation is not trivial when the driver 1546 * is loaded in simulation mode (not upstreamed). 1547 * @scrub_device_mem: Scrub the entire SRAM and DRAM. 1548 * @scrub_device_dram: Scrub the dram memory of the device. 1549 * @get_int_queue_base: get the internal queue base address. 1550 * @test_queues: run simple test on all queues for sanity check. 1551 * @asic_dma_pool_zalloc: small DMA allocation of coherent memory from DMA pool. 1552 * size of allocation is HL_DMA_POOL_BLK_SIZE. 1553 * @asic_dma_pool_free: free small DMA allocation from pool. 1554 * @cpu_accessible_dma_pool_alloc: allocate CPU PQ packet from DMA pool. 1555 * @cpu_accessible_dma_pool_free: free CPU PQ packet from DMA pool. 1556 * @dma_unmap_sgtable: DMA unmap scatter-gather table. 1557 * @dma_map_sgtable: DMA map scatter-gather table. 1558 * @cs_parser: parse Command Submission. 1559 * @add_end_of_cb_packets: Add packets to the end of CB, if device requires it. 1560 * @update_eq_ci: update event queue CI. 1561 * @context_switch: called upon ASID context switch. 1562 * @restore_phase_topology: clear all SOBs amd MONs. 1563 * @debugfs_read_dma: debug interface for reading up to 2MB from the device's 1564 * internal memory via DMA engine. 1565 * @add_device_attr: add ASIC specific device attributes. 1566 * @handle_eqe: handle event queue entry (IRQ) from CPU-CP. 1567 * @get_events_stat: retrieve event queue entries histogram. 1568 * @read_pte: read MMU page table entry from DRAM. 1569 * @write_pte: write MMU page table entry to DRAM. 1570 * @mmu_invalidate_cache: flush MMU STLB host/DRAM cache, either with soft 1571 * (L1 only) or hard (L0 & L1) flush. 1572 * @mmu_invalidate_cache_range: flush specific MMU STLB cache lines with ASID-VA-size mask. 1573 * @mmu_prefetch_cache_range: pre-fetch specific MMU STLB cache lines with ASID-VA-size mask. 1574 * @send_heartbeat: send is-alive packet to CPU-CP and verify response. 1575 * @debug_coresight: perform certain actions on Coresight for debugging. 1576 * @is_device_idle: return true if device is idle, false otherwise. 1577 * @compute_reset_late_init: perform certain actions needed after a compute reset 1578 * @hw_queues_lock: acquire H/W queues lock. 1579 * @hw_queues_unlock: release H/W queues lock. 1580 * @get_pci_id: retrieve PCI ID. 1581 * @get_eeprom_data: retrieve EEPROM data from F/W. 1582 * @get_monitor_dump: retrieve monitor registers dump from F/W. 1583 * @send_cpu_message: send message to F/W. If the message is timedout, the 1584 * driver will eventually reset the device. The timeout can 1585 * be determined by the calling function or it can be 0 and 1586 * then the timeout is the default timeout for the specific 1587 * ASIC 1588 * @get_hw_state: retrieve the H/W state 1589 * @pci_bars_map: Map PCI BARs. 1590 * @init_iatu: Initialize the iATU unit inside the PCI controller. 1591 * @rreg: Read a register. Needed for simulator support. 1592 * @wreg: Write a register. Needed for simulator support. 1593 * @halt_coresight: stop the ETF and ETR traces. 1594 * @ctx_init: context dependent initialization. 1595 * @ctx_fini: context dependent cleanup. 1596 * @pre_schedule_cs: Perform pre-CS-scheduling operations. 1597 * @get_queue_id_for_cq: Get the H/W queue id related to the given CQ index. 1598 * @load_firmware_to_device: load the firmware to the device's memory 1599 * @load_boot_fit_to_device: load boot fit to device's memory 1600 * @get_signal_cb_size: Get signal CB size. 1601 * @get_wait_cb_size: Get wait CB size. 1602 * @gen_signal_cb: Generate a signal CB. 1603 * @gen_wait_cb: Generate a wait CB. 1604 * @reset_sob: Reset a SOB. 1605 * @reset_sob_group: Reset SOB group 1606 * @get_device_time: Get the device time. 1607 * @pb_print_security_errors: print security errors according block and cause 1608 * @collective_wait_init_cs: Generate collective master/slave packets 1609 * and place them in the relevant cs jobs 1610 * @collective_wait_create_jobs: allocate collective wait cs jobs 1611 * @get_dec_base_addr: get the base address of a given decoder. 1612 * @scramble_addr: Routine to scramble the address prior of mapping it 1613 * in the MMU. 1614 * @descramble_addr: Routine to de-scramble the address prior of 1615 * showing it to users. 1616 * @ack_protection_bits_errors: ack and dump all security violations 1617 * @get_hw_block_id: retrieve a HW block id to be used by the user to mmap it. 1618 * also returns the size of the block if caller supplies 1619 * a valid pointer for it 1620 * @hw_block_mmap: mmap a HW block with a given id. 1621 * @enable_events_from_fw: send interrupt to firmware to notify them the 1622 * driver is ready to receive asynchronous events. This 1623 * function should be called during the first init and 1624 * after every hard-reset of the device 1625 * @ack_mmu_errors: check and ack mmu errors, page fault, access violation. 1626 * @get_msi_info: Retrieve asic-specific MSI ID of the f/w async event 1627 * @map_pll_idx_to_fw_idx: convert driver specific per asic PLL index to 1628 * generic f/w compatible PLL Indexes 1629 * @init_firmware_preload_params: initialize pre FW-load parameters. 1630 * @init_firmware_loader: initialize data for FW loader. 1631 * @init_cpu_scrambler_dram: Enable CPU specific DRAM scrambling 1632 * @state_dump_init: initialize constants required for state dump 1633 * @get_sob_addr: get SOB base address offset. 1634 * @set_pci_memory_regions: setting properties of PCI memory regions 1635 * @get_stream_master_qid_arr: get pointer to stream masters QID array 1636 * @check_if_razwi_happened: check if there was a razwi due to RR violation. 1637 * @access_dev_mem: access device memory 1638 * @set_dram_bar_base: set the base of the DRAM BAR 1639 * @set_engine_cores: set a config command to engine cores 1640 * @set_engines: set a config command to user engines 1641 * @send_device_activity: indication to FW about device availability 1642 * @set_dram_properties: set DRAM related properties. 1643 * @set_binning_masks: set binning/enable masks for all relevant components. 1644 */ 1645struct hl_asic_funcs { 1646 int (*early_init)(struct hl_device *hdev); 1647 int (*early_fini)(struct hl_device *hdev); 1648 int (*late_init)(struct hl_device *hdev); 1649 void (*late_fini)(struct hl_device *hdev); 1650 int (*sw_init)(struct hl_device *hdev); 1651 int (*sw_fini)(struct hl_device *hdev); 1652 int (*hw_init)(struct hl_device *hdev); 1653 int (*hw_fini)(struct hl_device *hdev, bool hard_reset, bool fw_reset); 1654 void (*halt_engines)(struct hl_device *hdev, bool hard_reset, bool fw_reset); 1655 int (*suspend)(struct hl_device *hdev); 1656 int (*resume)(struct hl_device *hdev); 1657 int (*mmap)(struct hl_device *hdev, struct vm_area_struct *vma, 1658 void *cpu_addr, dma_addr_t dma_addr, size_t size); 1659 void (*ring_doorbell)(struct hl_device *hdev, u32 hw_queue_id, u32 pi); 1660 void (*pqe_write)(struct hl_device *hdev, __le64 *pqe, 1661 struct hl_bd *bd); 1662 void* (*asic_dma_alloc_coherent)(struct hl_device *hdev, size_t size, 1663 dma_addr_t *dma_handle, gfp_t flag); 1664 void (*asic_dma_free_coherent)(struct hl_device *hdev, size_t size, 1665 void *cpu_addr, dma_addr_t dma_handle); 1666 int (*scrub_device_mem)(struct hl_device *hdev); 1667 int (*scrub_device_dram)(struct hl_device *hdev, u64 val); 1668 void* (*get_int_queue_base)(struct hl_device *hdev, u32 queue_id, 1669 dma_addr_t *dma_handle, u16 *queue_len); 1670 int (*test_queues)(struct hl_device *hdev); 1671 void* (*asic_dma_pool_zalloc)(struct hl_device *hdev, size_t size, 1672 gfp_t mem_flags, dma_addr_t *dma_handle); 1673 void (*asic_dma_pool_free)(struct hl_device *hdev, void *vaddr, 1674 dma_addr_t dma_addr); 1675 void* (*cpu_accessible_dma_pool_alloc)(struct hl_device *hdev, 1676 size_t size, dma_addr_t *dma_handle); 1677 void (*cpu_accessible_dma_pool_free)(struct hl_device *hdev, 1678 size_t size, void *vaddr); 1679 void (*dma_unmap_sgtable)(struct hl_device *hdev, struct sg_table *sgt, 1680 enum dma_data_direction dir); 1681 int (*dma_map_sgtable)(struct hl_device *hdev, struct sg_table *sgt, 1682 enum dma_data_direction dir); 1683 int (*cs_parser)(struct hl_device *hdev, struct hl_cs_parser *parser); 1684 void (*add_end_of_cb_packets)(struct hl_device *hdev, 1685 void *kernel_address, u32 len, 1686 u32 original_len, 1687 u64 cq_addr, u32 cq_val, u32 msix_num, 1688 bool eb); 1689 void (*update_eq_ci)(struct hl_device *hdev, u32 val); 1690 int (*context_switch)(struct hl_device *hdev, u32 asid); 1691 void (*restore_phase_topology)(struct hl_device *hdev); 1692 int (*debugfs_read_dma)(struct hl_device *hdev, u64 addr, u32 size, 1693 void *blob_addr); 1694 void (*add_device_attr)(struct hl_device *hdev, struct attribute_group *dev_clk_attr_grp, 1695 struct attribute_group *dev_vrm_attr_grp); 1696 void (*handle_eqe)(struct hl_device *hdev, 1697 struct hl_eq_entry *eq_entry); 1698 void* (*get_events_stat)(struct hl_device *hdev, bool aggregate, 1699 u32 *size); 1700 u64 (*read_pte)(struct hl_device *hdev, u64 addr); 1701 void (*write_pte)(struct hl_device *hdev, u64 addr, u64 val); 1702 int (*mmu_invalidate_cache)(struct hl_device *hdev, bool is_hard, 1703 u32 flags); 1704 int (*mmu_invalidate_cache_range)(struct hl_device *hdev, bool is_hard, 1705 u32 flags, u32 asid, u64 va, u64 size); 1706 int (*mmu_prefetch_cache_range)(struct hl_ctx *ctx, u32 flags, u32 asid, u64 va, u64 size); 1707 int (*send_heartbeat)(struct hl_device *hdev); 1708 int (*debug_coresight)(struct hl_device *hdev, struct hl_ctx *ctx, void *data); 1709 bool (*is_device_idle)(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 1710 struct engines_data *e); 1711 int (*compute_reset_late_init)(struct hl_device *hdev); 1712 void (*hw_queues_lock)(struct hl_device *hdev); 1713 void (*hw_queues_unlock)(struct hl_device *hdev); 1714 u32 (*get_pci_id)(struct hl_device *hdev); 1715 int (*get_eeprom_data)(struct hl_device *hdev, void *data, size_t max_size); 1716 int (*get_monitor_dump)(struct hl_device *hdev, void *data); 1717 int (*send_cpu_message)(struct hl_device *hdev, u32 *msg, 1718 u16 len, u32 timeout, u64 *result); 1719 int (*pci_bars_map)(struct hl_device *hdev); 1720 int (*init_iatu)(struct hl_device *hdev); 1721 u32 (*rreg)(struct hl_device *hdev, u32 reg); 1722 void (*wreg)(struct hl_device *hdev, u32 reg, u32 val); 1723 void (*halt_coresight)(struct hl_device *hdev, struct hl_ctx *ctx); 1724 int (*ctx_init)(struct hl_ctx *ctx); 1725 void (*ctx_fini)(struct hl_ctx *ctx); 1726 int (*pre_schedule_cs)(struct hl_cs *cs); 1727 u32 (*get_queue_id_for_cq)(struct hl_device *hdev, u32 cq_idx); 1728 int (*load_firmware_to_device)(struct hl_device *hdev); 1729 int (*load_boot_fit_to_device)(struct hl_device *hdev); 1730 u32 (*get_signal_cb_size)(struct hl_device *hdev); 1731 u32 (*get_wait_cb_size)(struct hl_device *hdev); 1732 u32 (*gen_signal_cb)(struct hl_device *hdev, void *data, u16 sob_id, 1733 u32 size, bool eb); 1734 u32 (*gen_wait_cb)(struct hl_device *hdev, 1735 struct hl_gen_wait_properties *prop); 1736 void (*reset_sob)(struct hl_device *hdev, void *data); 1737 void (*reset_sob_group)(struct hl_device *hdev, u16 sob_group); 1738 u64 (*get_device_time)(struct hl_device *hdev); 1739 void (*pb_print_security_errors)(struct hl_device *hdev, 1740 u32 block_addr, u32 cause, u32 offended_addr); 1741 int (*collective_wait_init_cs)(struct hl_cs *cs); 1742 int (*collective_wait_create_jobs)(struct hl_device *hdev, 1743 struct hl_ctx *ctx, struct hl_cs *cs, 1744 u32 wait_queue_id, u32 collective_engine_id, 1745 u32 encaps_signal_offset); 1746 u32 (*get_dec_base_addr)(struct hl_device *hdev, u32 core_id); 1747 u64 (*scramble_addr)(struct hl_device *hdev, u64 addr); 1748 u64 (*descramble_addr)(struct hl_device *hdev, u64 addr); 1749 void (*ack_protection_bits_errors)(struct hl_device *hdev); 1750 int (*get_hw_block_id)(struct hl_device *hdev, u64 block_addr, 1751 u32 *block_size, u32 *block_id); 1752 int (*hw_block_mmap)(struct hl_device *hdev, struct vm_area_struct *vma, 1753 u32 block_id, u32 block_size); 1754 void (*enable_events_from_fw)(struct hl_device *hdev); 1755 int (*ack_mmu_errors)(struct hl_device *hdev, u64 mmu_cap_mask); 1756 void (*get_msi_info)(__le32 *table); 1757 int (*map_pll_idx_to_fw_idx)(u32 pll_idx); 1758 void (*init_firmware_preload_params)(struct hl_device *hdev); 1759 void (*init_firmware_loader)(struct hl_device *hdev); 1760 void (*init_cpu_scrambler_dram)(struct hl_device *hdev); 1761 void (*state_dump_init)(struct hl_device *hdev); 1762 u32 (*get_sob_addr)(struct hl_device *hdev, u32 sob_id); 1763 void (*set_pci_memory_regions)(struct hl_device *hdev); 1764 u32* (*get_stream_master_qid_arr)(void); 1765 void (*check_if_razwi_happened)(struct hl_device *hdev); 1766 int (*mmu_get_real_page_size)(struct hl_device *hdev, struct hl_mmu_properties *mmu_prop, 1767 u32 page_size, u32 *real_page_size, bool is_dram_addr); 1768 int (*access_dev_mem)(struct hl_device *hdev, enum pci_region region_type, 1769 u64 addr, u64 *val, enum debugfs_access_type acc_type); 1770 u64 (*set_dram_bar_base)(struct hl_device *hdev, u64 addr); 1771 int (*set_engine_cores)(struct hl_device *hdev, u32 *core_ids, 1772 u32 num_cores, u32 core_command); 1773 int (*set_engines)(struct hl_device *hdev, u32 *engine_ids, 1774 u32 num_engines, u32 engine_command); 1775 int (*send_device_activity)(struct hl_device *hdev, bool open); 1776 int (*set_dram_properties)(struct hl_device *hdev); 1777 int (*set_binning_masks)(struct hl_device *hdev); 1778}; 1779 1780 1781/* 1782 * CONTEXTS 1783 */ 1784 1785#define HL_KERNEL_ASID_ID 0 1786 1787/** 1788 * enum hl_va_range_type - virtual address range type. 1789 * @HL_VA_RANGE_TYPE_HOST: range type of host pages 1790 * @HL_VA_RANGE_TYPE_HOST_HUGE: range type of host huge pages 1791 * @HL_VA_RANGE_TYPE_DRAM: range type of dram pages 1792 */ 1793enum hl_va_range_type { 1794 HL_VA_RANGE_TYPE_HOST, 1795 HL_VA_RANGE_TYPE_HOST_HUGE, 1796 HL_VA_RANGE_TYPE_DRAM, 1797 HL_VA_RANGE_TYPE_MAX 1798}; 1799 1800/** 1801 * struct hl_va_range - virtual addresses range. 1802 * @lock: protects the virtual addresses list. 1803 * @list: list of virtual addresses blocks available for mappings. 1804 * @start_addr: range start address. 1805 * @end_addr: range end address. 1806 * @page_size: page size of this va range. 1807 */ 1808struct hl_va_range { 1809 struct mutex lock; 1810 struct list_head list; 1811 u64 start_addr; 1812 u64 end_addr; 1813 u32 page_size; 1814}; 1815 1816/** 1817 * struct hl_cs_counters_atomic - command submission counters 1818 * @out_of_mem_drop_cnt: dropped due to memory allocation issue 1819 * @parsing_drop_cnt: dropped due to error in packet parsing 1820 * @queue_full_drop_cnt: dropped due to queue full 1821 * @device_in_reset_drop_cnt: dropped due to device in reset 1822 * @max_cs_in_flight_drop_cnt: dropped due to maximum CS in-flight 1823 * @validation_drop_cnt: dropped due to error in validation 1824 */ 1825struct hl_cs_counters_atomic { 1826 atomic64_t out_of_mem_drop_cnt; 1827 atomic64_t parsing_drop_cnt; 1828 atomic64_t queue_full_drop_cnt; 1829 atomic64_t device_in_reset_drop_cnt; 1830 atomic64_t max_cs_in_flight_drop_cnt; 1831 atomic64_t validation_drop_cnt; 1832}; 1833 1834/** 1835 * struct hl_dmabuf_priv - a dma-buf private object. 1836 * @dmabuf: pointer to dma-buf object. 1837 * @ctx: pointer to the dma-buf owner's context. 1838 * @phys_pg_pack: pointer to physical page pack if the dma-buf was exported 1839 * where virtual memory is supported. 1840 * @memhash_hnode: pointer to the memhash node. this object holds the export count. 1841 * @offset: the offset into the buffer from which the memory is exported. 1842 * Relevant only if virtual memory is supported and phys_pg_pack is being used. 1843 * device_phys_addr: physical address of the device's memory. Relevant only 1844 * if phys_pg_pack is NULL (dma-buf was exported from address). 1845 * The total size can be taken from the dmabuf object. 1846 */ 1847struct hl_dmabuf_priv { 1848 struct dma_buf *dmabuf; 1849 struct hl_ctx *ctx; 1850 struct hl_vm_phys_pg_pack *phys_pg_pack; 1851 struct hl_vm_hash_node *memhash_hnode; 1852 u64 offset; 1853 u64 device_phys_addr; 1854}; 1855 1856#define HL_CS_OUTCOME_HISTORY_LEN 256 1857 1858/** 1859 * struct hl_cs_outcome - represents a single completed CS outcome 1860 * @list_link: link to either container's used list or free list 1861 * @map_link: list to the container hash map 1862 * @ts: completion ts 1863 * @seq: the original cs sequence 1864 * @error: error code cs completed with, if any 1865 */ 1866struct hl_cs_outcome { 1867 struct list_head list_link; 1868 struct hlist_node map_link; 1869 ktime_t ts; 1870 u64 seq; 1871 int error; 1872}; 1873 1874/** 1875 * struct hl_cs_outcome_store - represents a limited store of completed CS outcomes 1876 * @outcome_map: index of completed CS searchable by sequence number 1877 * @used_list: list of outcome objects currently in use 1878 * @free_list: list of outcome objects currently not in use 1879 * @nodes_pool: a static pool of pre-allocated outcome objects 1880 * @db_lock: any operation on the store must take this lock 1881 */ 1882struct hl_cs_outcome_store { 1883 DECLARE_HASHTABLE(outcome_map, 8); 1884 struct list_head used_list; 1885 struct list_head free_list; 1886 struct hl_cs_outcome nodes_pool[HL_CS_OUTCOME_HISTORY_LEN]; 1887 spinlock_t db_lock; 1888}; 1889 1890/** 1891 * struct hl_ctx - user/kernel context. 1892 * @mem_hash: holds mapping from virtual address to virtual memory area 1893 * descriptor (hl_vm_phys_pg_list or hl_userptr). 1894 * @mmu_shadow_hash: holds a mapping from shadow address to pgt_info structure. 1895 * @hr_mmu_phys_hash: if host-resident MMU is used, holds a mapping from 1896 * MMU-hop-page physical address to its host-resident 1897 * pgt_info structure. 1898 * @hpriv: pointer to the private (Kernel Driver) data of the process (fd). 1899 * @hdev: pointer to the device structure. 1900 * @refcount: reference counter for the context. Context is released only when 1901 * this hits 0. It is incremented on CS and CS_WAIT. 1902 * @cs_pending: array of hl fence objects representing pending CS. 1903 * @outcome_store: storage data structure used to remember outcomes of completed 1904 * command submissions for a long time after CS id wraparound. 1905 * @va_range: holds available virtual addresses for host and dram mappings. 1906 * @mem_hash_lock: protects the mem_hash. 1907 * @hw_block_list_lock: protects the HW block memory list. 1908 * @ts_reg_lock: timestamp registration ioctls lock. 1909 * @debugfs_list: node in debugfs list of contexts. 1910 * @hw_block_mem_list: list of HW block virtual mapped addresses. 1911 * @cs_counters: context command submission counters. 1912 * @cb_va_pool: device VA pool for command buffers which are mapped to the 1913 * device's MMU. 1914 * @sig_mgr: encaps signals handle manager. 1915 * @cb_va_pool_base: the base address for the device VA pool 1916 * @cs_sequence: sequence number for CS. Value is assigned to a CS and passed 1917 * to user so user could inquire about CS. It is used as 1918 * index to cs_pending array. 1919 * @dram_default_hops: array that holds all hops addresses needed for default 1920 * DRAM mapping. 1921 * @cs_lock: spinlock to protect cs_sequence. 1922 * @dram_phys_mem: amount of used physical DRAM memory by this context. 1923 * @thread_ctx_switch_token: token to prevent multiple threads of the same 1924 * context from running the context switch phase. 1925 * Only a single thread should run it. 1926 * @thread_ctx_switch_wait_token: token to prevent the threads that didn't run 1927 * the context switch phase from moving to their 1928 * execution phase before the context switch phase 1929 * has finished. 1930 * @asid: context's unique address space ID in the device's MMU. 1931 * @handle: context's opaque handle for user 1932 */ 1933struct hl_ctx { 1934 DECLARE_HASHTABLE(mem_hash, MEM_HASH_TABLE_BITS); 1935 DECLARE_HASHTABLE(mmu_shadow_hash, MMU_HASH_TABLE_BITS); 1936 DECLARE_HASHTABLE(hr_mmu_phys_hash, MMU_HASH_TABLE_BITS); 1937 struct hl_fpriv *hpriv; 1938 struct hl_device *hdev; 1939 struct kref refcount; 1940 struct hl_fence **cs_pending; 1941 struct hl_cs_outcome_store outcome_store; 1942 struct hl_va_range *va_range[HL_VA_RANGE_TYPE_MAX]; 1943 struct mutex mem_hash_lock; 1944 struct mutex hw_block_list_lock; 1945 struct mutex ts_reg_lock; 1946 struct list_head debugfs_list; 1947 struct list_head hw_block_mem_list; 1948 struct hl_cs_counters_atomic cs_counters; 1949 struct gen_pool *cb_va_pool; 1950 struct hl_encaps_signals_mgr sig_mgr; 1951 u64 cb_va_pool_base; 1952 u64 cs_sequence; 1953 u64 *dram_default_hops; 1954 spinlock_t cs_lock; 1955 atomic64_t dram_phys_mem; 1956 atomic_t thread_ctx_switch_token; 1957 u32 thread_ctx_switch_wait_token; 1958 u32 asid; 1959 u32 handle; 1960}; 1961 1962/** 1963 * struct hl_ctx_mgr - for handling multiple contexts. 1964 * @lock: protects ctx_handles. 1965 * @handles: idr to hold all ctx handles. 1966 */ 1967struct hl_ctx_mgr { 1968 struct mutex lock; 1969 struct idr handles; 1970}; 1971 1972 1973/* 1974 * COMMAND SUBMISSIONS 1975 */ 1976 1977/** 1978 * struct hl_userptr - memory mapping chunk information 1979 * @vm_type: type of the VM. 1980 * @job_node: linked-list node for hanging the object on the Job's list. 1981 * @pages: pointer to struct page array 1982 * @npages: size of @pages array 1983 * @sgt: pointer to the scatter-gather table that holds the pages. 1984 * @dir: for DMA unmapping, the direction must be supplied, so save it. 1985 * @debugfs_list: node in debugfs list of command submissions. 1986 * @pid: the pid of the user process owning the memory 1987 * @addr: user-space virtual address of the start of the memory area. 1988 * @size: size of the memory area to pin & map. 1989 * @dma_mapped: true if the SG was mapped to DMA addresses, false otherwise. 1990 */ 1991struct hl_userptr { 1992 enum vm_type vm_type; /* must be first */ 1993 struct list_head job_node; 1994 struct page **pages; 1995 unsigned int npages; 1996 struct sg_table *sgt; 1997 enum dma_data_direction dir; 1998 struct list_head debugfs_list; 1999 pid_t pid; 2000 u64 addr; 2001 u64 size; 2002 u8 dma_mapped; 2003}; 2004 2005/** 2006 * struct hl_cs - command submission. 2007 * @jobs_in_queue_cnt: per each queue, maintain counter of submitted jobs. 2008 * @ctx: the context this CS belongs to. 2009 * @job_list: list of the CS's jobs in the various queues. 2010 * @job_lock: spinlock for the CS's jobs list. Needed for free_job. 2011 * @refcount: reference counter for usage of the CS. 2012 * @fence: pointer to the fence object of this CS. 2013 * @signal_fence: pointer to the fence object of the signal CS (used by wait 2014 * CS only). 2015 * @finish_work: workqueue object to run when CS is completed by H/W. 2016 * @work_tdr: delayed work node for TDR. 2017 * @mirror_node : node in device mirror list of command submissions. 2018 * @staged_cs_node: node in the staged cs list. 2019 * @debugfs_list: node in debugfs list of command submissions. 2020 * @encaps_sig_hdl: holds the encaps signals handle. 2021 * @sequence: the sequence number of this CS. 2022 * @staged_sequence: the sequence of the staged submission this CS is part of, 2023 * relevant only if staged_cs is set. 2024 * @timeout_jiffies: cs timeout in jiffies. 2025 * @submission_time_jiffies: submission time of the cs 2026 * @type: CS_TYPE_*. 2027 * @jobs_cnt: counter of submitted jobs on all queues. 2028 * @encaps_sig_hdl_id: encaps signals handle id, set for the first staged cs. 2029 * @completion_timestamp: timestamp of the last completed cs job. 2030 * @sob_addr_offset: sob offset from the configuration base address. 2031 * @initial_sob_count: count of completed signals in SOB before current submission of signal or 2032 * cs with encaps signals. 2033 * @submitted: true if CS was submitted to H/W. 2034 * @completed: true if CS was completed by device. 2035 * @timedout : true if CS was timedout. 2036 * @tdr_active: true if TDR was activated for this CS (to prevent 2037 * double TDR activation). 2038 * @aborted: true if CS was aborted due to some device error. 2039 * @timestamp: true if a timestamp must be captured upon completion. 2040 * @staged_last: true if this is the last staged CS and needs completion. 2041 * @staged_first: true if this is the first staged CS and we need to receive 2042 * timeout for this CS. 2043 * @staged_cs: true if this CS is part of a staged submission. 2044 * @skip_reset_on_timeout: true if we shall not reset the device in case 2045 * timeout occurs (debug scenario). 2046 * @encaps_signals: true if this CS has encaps reserved signals. 2047 */ 2048struct hl_cs { 2049 u16 *jobs_in_queue_cnt; 2050 struct hl_ctx *ctx; 2051 struct list_head job_list; 2052 spinlock_t job_lock; 2053 struct kref refcount; 2054 struct hl_fence *fence; 2055 struct hl_fence *signal_fence; 2056 struct work_struct finish_work; 2057 struct delayed_work work_tdr; 2058 struct list_head mirror_node; 2059 struct list_head staged_cs_node; 2060 struct list_head debugfs_list; 2061 struct hl_cs_encaps_sig_handle *encaps_sig_hdl; 2062 ktime_t completion_timestamp; 2063 u64 sequence; 2064 u64 staged_sequence; 2065 u64 timeout_jiffies; 2066 u64 submission_time_jiffies; 2067 enum hl_cs_type type; 2068 u32 jobs_cnt; 2069 u32 encaps_sig_hdl_id; 2070 u32 sob_addr_offset; 2071 u16 initial_sob_count; 2072 u8 submitted; 2073 u8 completed; 2074 u8 timedout; 2075 u8 tdr_active; 2076 u8 aborted; 2077 u8 timestamp; 2078 u8 staged_last; 2079 u8 staged_first; 2080 u8 staged_cs; 2081 u8 skip_reset_on_timeout; 2082 u8 encaps_signals; 2083}; 2084 2085/** 2086 * struct hl_cs_job - command submission job. 2087 * @cs_node: the node to hang on the CS jobs list. 2088 * @cs: the CS this job belongs to. 2089 * @user_cb: the CB we got from the user. 2090 * @patched_cb: in case of patching, this is internal CB which is submitted on 2091 * the queue instead of the CB we got from the IOCTL. 2092 * @finish_work: workqueue object to run when job is completed. 2093 * @userptr_list: linked-list of userptr mappings that belong to this job and 2094 * wait for completion. 2095 * @debugfs_list: node in debugfs list of command submission jobs. 2096 * @refcount: reference counter for usage of the CS job. 2097 * @queue_type: the type of the H/W queue this job is submitted to. 2098 * @timestamp: timestamp upon job completion 2099 * @id: the id of this job inside a CS. 2100 * @hw_queue_id: the id of the H/W queue this job is submitted to. 2101 * @user_cb_size: the actual size of the CB we got from the user. 2102 * @job_cb_size: the actual size of the CB that we put on the queue. 2103 * @encaps_sig_wait_offset: encapsulated signals offset, which allow user 2104 * to wait on part of the reserved signals. 2105 * @is_kernel_allocated_cb: true if the CB handle we got from the user holds a 2106 * handle to a kernel-allocated CB object, false 2107 * otherwise (SRAM/DRAM/host address). 2108 * @contains_dma_pkt: whether the JOB contains at least one DMA packet. This 2109 * info is needed later, when adding the 2xMSG_PROT at the 2110 * end of the JOB, to know which barriers to put in the 2111 * MSG_PROT packets. Relevant only for GAUDI as GOYA doesn't 2112 * have streams so the engine can't be busy by another 2113 * stream. 2114 */ 2115struct hl_cs_job { 2116 struct list_head cs_node; 2117 struct hl_cs *cs; 2118 struct hl_cb *user_cb; 2119 struct hl_cb *patched_cb; 2120 struct work_struct finish_work; 2121 struct list_head userptr_list; 2122 struct list_head debugfs_list; 2123 struct kref refcount; 2124 enum hl_queue_type queue_type; 2125 ktime_t timestamp; 2126 u32 id; 2127 u32 hw_queue_id; 2128 u32 user_cb_size; 2129 u32 job_cb_size; 2130 u32 encaps_sig_wait_offset; 2131 u8 is_kernel_allocated_cb; 2132 u8 contains_dma_pkt; 2133}; 2134 2135/** 2136 * struct hl_cs_parser - command submission parser properties. 2137 * @user_cb: the CB we got from the user. 2138 * @patched_cb: in case of patching, this is internal CB which is submitted on 2139 * the queue instead of the CB we got from the IOCTL. 2140 * @job_userptr_list: linked-list of userptr mappings that belong to the related 2141 * job and wait for completion. 2142 * @cs_sequence: the sequence number of the related CS. 2143 * @queue_type: the type of the H/W queue this job is submitted to. 2144 * @ctx_id: the ID of the context the related CS belongs to. 2145 * @hw_queue_id: the id of the H/W queue this job is submitted to. 2146 * @user_cb_size: the actual size of the CB we got from the user. 2147 * @patched_cb_size: the size of the CB after parsing. 2148 * @job_id: the id of the related job inside the related CS. 2149 * @is_kernel_allocated_cb: true if the CB handle we got from the user holds a 2150 * handle to a kernel-allocated CB object, false 2151 * otherwise (SRAM/DRAM/host address). 2152 * @contains_dma_pkt: whether the JOB contains at least one DMA packet. This 2153 * info is needed later, when adding the 2xMSG_PROT at the 2154 * end of the JOB, to know which barriers to put in the 2155 * MSG_PROT packets. Relevant only for GAUDI as GOYA doesn't 2156 * have streams so the engine can't be busy by another 2157 * stream. 2158 * @completion: true if we need completion for this CS. 2159 */ 2160struct hl_cs_parser { 2161 struct hl_cb *user_cb; 2162 struct hl_cb *patched_cb; 2163 struct list_head *job_userptr_list; 2164 u64 cs_sequence; 2165 enum hl_queue_type queue_type; 2166 u32 ctx_id; 2167 u32 hw_queue_id; 2168 u32 user_cb_size; 2169 u32 patched_cb_size; 2170 u8 job_id; 2171 u8 is_kernel_allocated_cb; 2172 u8 contains_dma_pkt; 2173 u8 completion; 2174}; 2175 2176/* 2177 * MEMORY STRUCTURE 2178 */ 2179 2180/** 2181 * struct hl_vm_hash_node - hash element from virtual address to virtual 2182 * memory area descriptor (hl_vm_phys_pg_list or 2183 * hl_userptr). 2184 * @node: node to hang on the hash table in context object. 2185 * @vaddr: key virtual address. 2186 * @handle: memory handle for device memory allocation. 2187 * @ptr: value pointer (hl_vm_phys_pg_list or hl_userptr). 2188 * @export_cnt: number of exports from within the VA block. 2189 */ 2190struct hl_vm_hash_node { 2191 struct hlist_node node; 2192 u64 vaddr; 2193 u64 handle; 2194 void *ptr; 2195 int export_cnt; 2196}; 2197 2198/** 2199 * struct hl_vm_hw_block_list_node - list element from user virtual address to 2200 * HW block id. 2201 * @node: node to hang on the list in context object. 2202 * @ctx: the context this node belongs to. 2203 * @vaddr: virtual address of the HW block. 2204 * @block_size: size of the block. 2205 * @mapped_size: size of the block which is mapped. May change if partial un-mappings are done. 2206 * @id: HW block id (handle). 2207 */ 2208struct hl_vm_hw_block_list_node { 2209 struct list_head node; 2210 struct hl_ctx *ctx; 2211 unsigned long vaddr; 2212 u32 block_size; 2213 u32 mapped_size; 2214 u32 id; 2215}; 2216 2217/** 2218 * struct hl_vm_phys_pg_pack - physical page pack. 2219 * @vm_type: describes the type of the virtual area descriptor. 2220 * @pages: the physical page array. 2221 * @npages: num physical pages in the pack. 2222 * @total_size: total size of all the pages in this list. 2223 * @node: used to attach to deletion list that is used when all the allocations are cleared 2224 * at the teardown of the context. 2225 * @mapping_cnt: number of shared mappings. 2226 * @asid: the context related to this list. 2227 * @page_size: size of each page in the pack. 2228 * @flags: HL_MEM_* flags related to this list. 2229 * @handle: the provided handle related to this list. 2230 * @offset: offset from the first page. 2231 * @contiguous: is contiguous physical memory. 2232 * @created_from_userptr: is product of host virtual address. 2233 */ 2234struct hl_vm_phys_pg_pack { 2235 enum vm_type vm_type; /* must be first */ 2236 u64 *pages; 2237 u64 npages; 2238 u64 total_size; 2239 struct list_head node; 2240 atomic_t mapping_cnt; 2241 u32 asid; 2242 u32 page_size; 2243 u32 flags; 2244 u32 handle; 2245 u32 offset; 2246 u8 contiguous; 2247 u8 created_from_userptr; 2248}; 2249 2250/** 2251 * struct hl_vm_va_block - virtual range block information. 2252 * @node: node to hang on the virtual range list in context object. 2253 * @start: virtual range start address. 2254 * @end: virtual range end address. 2255 * @size: virtual range size. 2256 */ 2257struct hl_vm_va_block { 2258 struct list_head node; 2259 u64 start; 2260 u64 end; 2261 u64 size; 2262}; 2263 2264/** 2265 * struct hl_vm - virtual memory manager for MMU. 2266 * @dram_pg_pool: pool for DRAM physical pages of 2MB. 2267 * @dram_pg_pool_refcount: reference counter for the pool usage. 2268 * @idr_lock: protects the phys_pg_list_handles. 2269 * @phys_pg_pack_handles: idr to hold all device allocations handles. 2270 * @init_done: whether initialization was done. We need this because VM 2271 * initialization might be skipped during device initialization. 2272 */ 2273struct hl_vm { 2274 struct gen_pool *dram_pg_pool; 2275 struct kref dram_pg_pool_refcount; 2276 spinlock_t idr_lock; 2277 struct idr phys_pg_pack_handles; 2278 u8 init_done; 2279}; 2280 2281#ifdef CONFIG_HL_HLDIO 2282#include "hldio.h" 2283#endif 2284 2285/* 2286 * DEBUG, PROFILING STRUCTURE 2287 */ 2288 2289/** 2290 * struct hl_debug_params - Coresight debug parameters. 2291 * @input: pointer to component specific input parameters. 2292 * @output: pointer to component specific output parameters. 2293 * @output_size: size of output buffer. 2294 * @reg_idx: relevant register ID. 2295 * @op: component operation to execute. 2296 * @enable: true if to enable component debugging, false otherwise. 2297 */ 2298struct hl_debug_params { 2299 void *input; 2300 void *output; 2301 u32 output_size; 2302 u32 reg_idx; 2303 u32 op; 2304 bool enable; 2305}; 2306 2307/** 2308 * struct hl_notifier_event - holds the notifier data structure 2309 * @eventfd: the event file descriptor to raise the notifications 2310 * @lock: mutex lock to protect the notifier data flows 2311 * @events_mask: indicates the bitmap events 2312 */ 2313struct hl_notifier_event { 2314 struct eventfd_ctx *eventfd; 2315 struct mutex lock; 2316 u64 events_mask; 2317}; 2318 2319/* 2320 * FILE PRIVATE STRUCTURE 2321 */ 2322 2323/** 2324 * struct hl_fpriv - process information stored in FD private data. 2325 * @hdev: habanalabs device structure. 2326 * @file_priv: pointer to the DRM file private data structure. 2327 * @taskpid: current process ID. 2328 * @ctx: current executing context. TODO: remove for multiple ctx per process 2329 * @ctx_mgr: context manager to handle multiple context for this FD. 2330 * @mem_mgr: manager descriptor for memory exportable via mmap 2331 * @notifier_event: notifier eventfd towards user process 2332 * @debugfs_list: list of relevant ASIC debugfs. 2333 * @dev_node: node in the device list of file private data 2334 * @refcount: number of related contexts. 2335 * @restore_phase_mutex: lock for context switch and restore phase. 2336 * @ctx_lock: protects the pointer to current executing context pointer. TODO: remove for multiple 2337 * ctx per process. 2338 */ 2339struct hl_fpriv { 2340 struct hl_device *hdev; 2341 struct drm_file *file_priv; 2342 struct pid *taskpid; 2343 struct hl_ctx *ctx; 2344 struct hl_ctx_mgr ctx_mgr; 2345 struct hl_mem_mgr mem_mgr; 2346 struct hl_notifier_event notifier_event; 2347 struct list_head debugfs_list; 2348 struct list_head dev_node; 2349 struct kref refcount; 2350 struct mutex restore_phase_mutex; 2351 struct mutex ctx_lock; 2352}; 2353 2354/* 2355 * DebugFS 2356 */ 2357 2358/** 2359 * struct hl_info_list - debugfs file ops. 2360 * @name: file name. 2361 * @show: function to output information. 2362 * @write: function to write to the file. 2363 */ 2364struct hl_info_list { 2365 const char *name; 2366 int (*show)(struct seq_file *s, void *data); 2367 ssize_t (*write)(struct file *file, const char __user *buf, 2368 size_t count, loff_t *f_pos); 2369}; 2370 2371/** 2372 * struct hl_debugfs_entry - debugfs dentry wrapper. 2373 * @info_ent: dentry related ops. 2374 * @dev_entry: ASIC specific debugfs manager. 2375 */ 2376struct hl_debugfs_entry { 2377 const struct hl_info_list *info_ent; 2378 struct hl_dbg_device_entry *dev_entry; 2379}; 2380 2381 2382/** 2383 * struct hl_dbg_device_entry - ASIC specific debugfs manager. 2384 * @root: root dentry. 2385 * @hdev: habanalabs device structure. 2386 * @entry_arr: array of available hl_debugfs_entry. 2387 * @file_list: list of available debugfs files. 2388 * @file_mutex: protects file_list. 2389 * @cb_list: list of available CBs. 2390 * @cb_spinlock: protects cb_list. 2391 * @cs_list: list of available CSs. 2392 * @cs_spinlock: protects cs_list. 2393 * @cs_job_list: list of available CB jobs. 2394 * @cs_job_spinlock: protects cs_job_list. 2395 * @userptr_list: list of available userptrs (virtual memory chunk descriptor). 2396 * @userptr_spinlock: protects userptr_list. 2397 * @ctx_mem_hash_list: list of available contexts with MMU mappings. 2398 * @ctx_mem_hash_mutex: protects list of available contexts with MMU mappings. 2399 * @data_dma_blob_desc: data DMA descriptor of blob. 2400 * @mon_dump_blob_desc: monitor dump descriptor of blob. 2401 * @state_dump: data of the system states in case of a bad cs. 2402 * @state_dump_sem: protects state_dump. 2403 * @addr: next address to read/write from/to in read/write32. 2404 * @mmu_addr: next virtual address to translate to physical address in mmu_show. 2405 * @mmu_cap_mask: mmu hw capability mask, to be used in mmu_ack_error. 2406 * @userptr_lookup: the target user ptr to look up for on demand. 2407 * @mmu_asid: ASID to use while translating in mmu_show. 2408 * @state_dump_head: index of the latest state dump 2409 * @i2c_bus: generic u8 debugfs file for bus value to use in i2c_data_read. 2410 * @i2c_addr: generic u8 debugfs file for address value to use in i2c_data_read. 2411 * @i2c_reg: generic u8 debugfs file for register value to use in i2c_data_read. 2412 * @i2c_len: generic u8 debugfs file for length value to use in i2c_data_read. 2413 * @dio_stats: Direct I/O statistics 2414 */ 2415struct hl_dbg_device_entry { 2416 struct dentry *root; 2417 struct hl_device *hdev; 2418 struct hl_debugfs_entry *entry_arr; 2419 struct list_head file_list; 2420 struct mutex file_mutex; 2421 struct list_head cb_list; 2422 spinlock_t cb_spinlock; 2423 struct list_head cs_list; 2424 spinlock_t cs_spinlock; 2425 struct list_head cs_job_list; 2426 spinlock_t cs_job_spinlock; 2427 struct list_head userptr_list; 2428 spinlock_t userptr_spinlock; 2429 struct list_head ctx_mem_hash_list; 2430 struct mutex ctx_mem_hash_mutex; 2431 struct debugfs_blob_wrapper data_dma_blob_desc; 2432 struct debugfs_blob_wrapper mon_dump_blob_desc; 2433 char *state_dump[HL_STATE_DUMP_HIST_LEN]; 2434 struct rw_semaphore state_dump_sem; 2435 u64 addr; 2436 u64 mmu_addr; 2437 u64 mmu_cap_mask; 2438 u64 userptr_lookup; 2439 u32 mmu_asid; 2440 u32 state_dump_head; 2441 u8 i2c_bus; 2442 u8 i2c_addr; 2443 u8 i2c_reg; 2444 u8 i2c_len; 2445#ifdef CONFIG_HL_HLDIO 2446 struct hl_dio_stats dio_stats; 2447#endif 2448}; 2449 2450/** 2451 * struct hl_debugfs_cfg_access_entry - single debugfs config access object, member of 2452 * hl_debugfs_cfg_access. 2453 * @seconds_since_epoch: seconds since January 1, 1970, used for time comparisons. 2454 * @debugfs_type: the debugfs operation requested, can be READ32, WRITE32, READ64 or WRITE64. 2455 * @addr: the requested address to access. 2456 * @valid: if set, this entry has valid data for dumping at interrupt time. 2457 */ 2458struct hl_debugfs_cfg_access_entry { 2459 ktime_t seconds_since_epoch; 2460 enum debugfs_access_type debugfs_type; 2461 u64 addr; 2462 bool valid; 2463}; 2464 2465/** 2466 * struct hl_debugfs_cfg_access - saves debugfs config region access requests history. 2467 * @cfg_access_list: list of objects describing config region access requests. 2468 * @head: next valid index to add new entry to in cfg_access_list. 2469 */ 2470struct hl_debugfs_cfg_access { 2471 struct hl_debugfs_cfg_access_entry cfg_access_list[HL_DBGFS_CFG_ACCESS_HIST_LEN]; 2472 u32 head; 2473 spinlock_t lock; /* protects head and entries */ 2474}; 2475 2476/** 2477 * struct hl_hw_obj_name_entry - single hw object name, member of 2478 * hl_state_dump_specs 2479 * @node: link to the containing hash table 2480 * @name: hw object name 2481 * @id: object identifier 2482 */ 2483struct hl_hw_obj_name_entry { 2484 struct hlist_node node; 2485 const char *name; 2486 u32 id; 2487}; 2488 2489enum hl_state_dump_specs_props { 2490 SP_SYNC_OBJ_BASE_ADDR, 2491 SP_NEXT_SYNC_OBJ_ADDR, 2492 SP_SYNC_OBJ_AMOUNT, 2493 SP_MON_OBJ_WR_ADDR_LOW, 2494 SP_MON_OBJ_WR_ADDR_HIGH, 2495 SP_MON_OBJ_WR_DATA, 2496 SP_MON_OBJ_ARM_DATA, 2497 SP_MON_OBJ_STATUS, 2498 SP_MONITORS_AMOUNT, 2499 SP_TPC0_CMDQ, 2500 SP_TPC0_CFG_SO, 2501 SP_NEXT_TPC, 2502 SP_MME_CMDQ, 2503 SP_MME_CFG_SO, 2504 SP_NEXT_MME, 2505 SP_DMA_CMDQ, 2506 SP_DMA_CFG_SO, 2507 SP_DMA_QUEUES_OFFSET, 2508 SP_NUM_OF_MME_ENGINES, 2509 SP_SUB_MME_ENG_NUM, 2510 SP_NUM_OF_DMA_ENGINES, 2511 SP_NUM_OF_TPC_ENGINES, 2512 SP_ENGINE_NUM_OF_QUEUES, 2513 SP_ENGINE_NUM_OF_STREAMS, 2514 SP_ENGINE_NUM_OF_FENCES, 2515 SP_FENCE0_CNT_OFFSET, 2516 SP_FENCE0_RDATA_OFFSET, 2517 SP_CP_STS_OFFSET, 2518 SP_NUM_CORES, 2519 2520 SP_MAX 2521}; 2522 2523enum hl_sync_engine_type { 2524 ENGINE_TPC, 2525 ENGINE_DMA, 2526 ENGINE_MME, 2527}; 2528 2529/** 2530 * struct hl_mon_state_dump - represents a state dump of a single monitor 2531 * @id: monitor id 2532 * @wr_addr_low: address monitor will write to, low bits 2533 * @wr_addr_high: address monitor will write to, high bits 2534 * @wr_data: data monitor will write 2535 * @arm_data: register value containing monitor configuration 2536 * @status: monitor status 2537 */ 2538struct hl_mon_state_dump { 2539 u32 id; 2540 u32 wr_addr_low; 2541 u32 wr_addr_high; 2542 u32 wr_data; 2543 u32 arm_data; 2544 u32 status; 2545}; 2546 2547/** 2548 * struct hl_sync_to_engine_map_entry - sync object id to engine mapping entry 2549 * @engine_type: type of the engine 2550 * @engine_id: id of the engine 2551 * @sync_id: id of the sync object 2552 */ 2553struct hl_sync_to_engine_map_entry { 2554 struct hlist_node node; 2555 enum hl_sync_engine_type engine_type; 2556 u32 engine_id; 2557 u32 sync_id; 2558}; 2559 2560/** 2561 * struct hl_sync_to_engine_map - maps sync object id to associated engine id 2562 * @tb: hash table containing the mapping, each element is of type 2563 * struct hl_sync_to_engine_map_entry 2564 */ 2565struct hl_sync_to_engine_map { 2566 DECLARE_HASHTABLE(tb, SYNC_TO_ENGINE_HASH_TABLE_BITS); 2567}; 2568 2569/** 2570 * struct hl_state_dump_specs_funcs - virtual functions used by the state dump 2571 * @gen_sync_to_engine_map: generate a hash map from sync obj id to its engine 2572 * @print_single_monitor: format monitor data as string 2573 * @monitor_valid: return true if given monitor dump is valid 2574 * @print_fences_single_engine: format fences data as string 2575 */ 2576struct hl_state_dump_specs_funcs { 2577 int (*gen_sync_to_engine_map)(struct hl_device *hdev, 2578 struct hl_sync_to_engine_map *map); 2579 int (*print_single_monitor)(char **buf, size_t *size, size_t *offset, 2580 struct hl_device *hdev, 2581 struct hl_mon_state_dump *mon); 2582 int (*monitor_valid)(struct hl_mon_state_dump *mon); 2583 int (*print_fences_single_engine)(struct hl_device *hdev, 2584 u64 base_offset, 2585 u64 status_base_offset, 2586 enum hl_sync_engine_type engine_type, 2587 u32 engine_id, char **buf, 2588 size_t *size, size_t *offset); 2589}; 2590 2591/** 2592 * struct hl_state_dump_specs - defines ASIC known hw objects names 2593 * @so_id_to_str_tb: sync objects names index table 2594 * @monitor_id_to_str_tb: monitors names index table 2595 * @funcs: virtual functions used for state dump 2596 * @sync_namager_names: readable names for sync manager if available (ex: N_E) 2597 * @props: pointer to a per asic const props array required for state dump 2598 */ 2599struct hl_state_dump_specs { 2600 DECLARE_HASHTABLE(so_id_to_str_tb, OBJ_NAMES_HASH_TABLE_BITS); 2601 DECLARE_HASHTABLE(monitor_id_to_str_tb, OBJ_NAMES_HASH_TABLE_BITS); 2602 struct hl_state_dump_specs_funcs funcs; 2603 const char * const *sync_namager_names; 2604 s64 *props; 2605}; 2606 2607 2608/* 2609 * DEVICES 2610 */ 2611 2612#define HL_STR_MAX 64 2613 2614#define HL_DEV_STS_MAX (HL_DEVICE_STATUS_LAST + 1) 2615 2616/* Theoretical limit only. A single host can only contain up to 4 or 8 PCIe 2617 * x16 cards. In extreme cases, there are hosts that can accommodate 16 cards. 2618 */ 2619#define HL_MAX_MINORS 256 2620 2621/* 2622 * Registers read & write functions. 2623 */ 2624 2625u32 hl_rreg(struct hl_device *hdev, u32 reg); 2626void hl_wreg(struct hl_device *hdev, u32 reg, u32 val); 2627 2628#define RREG32(reg) hdev->asic_funcs->rreg(hdev, (reg)) 2629#define WREG32(reg, v) hdev->asic_funcs->wreg(hdev, (reg), (v)) 2630#define DREG32(reg) pr_info("REGISTER: " #reg " : 0x%08X\n", \ 2631 hdev->asic_funcs->rreg(hdev, (reg))) 2632 2633#define WREG32_P(reg, val, mask) \ 2634 do { \ 2635 u32 tmp_ = RREG32(reg); \ 2636 tmp_ &= (mask); \ 2637 tmp_ |= ((val) & ~(mask)); \ 2638 WREG32(reg, tmp_); \ 2639 } while (0) 2640#define WREG32_AND(reg, and) WREG32_P(reg, 0, and) 2641#define WREG32_OR(reg, or) WREG32_P(reg, or, ~(or)) 2642 2643#define RMWREG32_SHIFTED(reg, val, mask) WREG32_P(reg, val, ~(mask)) 2644 2645#define RMWREG32(reg, val, mask) RMWREG32_SHIFTED(reg, (val) << __ffs(mask), mask) 2646 2647#define RREG32_MASK(reg, mask) ((RREG32(reg) & mask) >> __ffs(mask)) 2648 2649#define REG_FIELD_SHIFT(reg, field) reg##_##field##_SHIFT 2650#define REG_FIELD_MASK(reg, field) reg##_##field##_MASK 2651#define WREG32_FIELD(reg, offset, field, val) \ 2652 WREG32(mm##reg + offset, (RREG32(mm##reg + offset) & \ 2653 ~REG_FIELD_MASK(reg, field)) | \ 2654 (val) << REG_FIELD_SHIFT(reg, field)) 2655 2656/* Timeout should be longer when working with simulator but cap the 2657 * increased timeout to some maximum 2658 */ 2659#define hl_poll_timeout_common(hdev, addr, val, cond, sleep_us, timeout_us, elbi) \ 2660({ \ 2661 ktime_t __timeout; \ 2662 u32 __elbi_read; \ 2663 int __rc = 0; \ 2664 __timeout = ktime_add_us(ktime_get(), timeout_us); \ 2665 might_sleep_if(sleep_us); \ 2666 for (;;) { \ 2667 if (elbi) { \ 2668 __rc = hl_pci_elbi_read(hdev, addr, &__elbi_read); \ 2669 if (__rc) \ 2670 break; \ 2671 (val) = __elbi_read; \ 2672 } else {\ 2673 (val) = RREG32(lower_32_bits(addr)); \ 2674 } \ 2675 if (cond) \ 2676 break; \ 2677 if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) { \ 2678 if (elbi) { \ 2679 __rc = hl_pci_elbi_read(hdev, addr, &__elbi_read); \ 2680 if (__rc) \ 2681 break; \ 2682 (val) = __elbi_read; \ 2683 } else {\ 2684 (val) = RREG32(lower_32_bits(addr)); \ 2685 } \ 2686 break; \ 2687 } \ 2688 if (sleep_us) \ 2689 usleep_range((sleep_us >> 2) + 1, sleep_us); \ 2690 } \ 2691 __rc ? __rc : ((cond) ? 0 : -ETIMEDOUT); \ 2692}) 2693 2694#define hl_poll_timeout(hdev, addr, val, cond, sleep_us, timeout_us) \ 2695 hl_poll_timeout_common(hdev, addr, val, cond, sleep_us, timeout_us, false) 2696 2697#define hl_poll_timeout_elbi(hdev, addr, val, cond, sleep_us, timeout_us) \ 2698 hl_poll_timeout_common(hdev, addr, val, cond, sleep_us, timeout_us, true) 2699 2700/* 2701 * poll array of register addresses. 2702 * condition is satisfied if all registers values match the expected value. 2703 * once some register in the array satisfies the condition it will not be polled again, 2704 * this is done both for efficiency and due to some registers are "clear on read". 2705 * TODO: use read from PCI bar in other places in the code (SW-91406) 2706 */ 2707#define hl_poll_reg_array_timeout_common(hdev, addr_arr, arr_size, expected_val, sleep_us, \ 2708 timeout_us, elbi) \ 2709({ \ 2710 ktime_t __timeout; \ 2711 u64 __elem_bitmask; \ 2712 u32 __read_val; \ 2713 u8 __arr_idx; \ 2714 int __rc = 0; \ 2715 \ 2716 __timeout = ktime_add_us(ktime_get(), timeout_us); \ 2717 might_sleep_if(sleep_us); \ 2718 if (arr_size >= 64) \ 2719 __rc = -EINVAL; \ 2720 else \ 2721 __elem_bitmask = BIT_ULL(arr_size) - 1; \ 2722 for (;;) { \ 2723 if (__rc) \ 2724 break; \ 2725 for (__arr_idx = 0; __arr_idx < (arr_size); __arr_idx++) { \ 2726 if (!(__elem_bitmask & BIT_ULL(__arr_idx))) \ 2727 continue; \ 2728 if (elbi) { \ 2729 __rc = hl_pci_elbi_read(hdev, (addr_arr)[__arr_idx], &__read_val); \ 2730 if (__rc) \ 2731 break; \ 2732 } else { \ 2733 __read_val = RREG32(lower_32_bits(addr_arr[__arr_idx])); \ 2734 } \ 2735 if (__read_val == (expected_val)) \ 2736 __elem_bitmask &= ~BIT_ULL(__arr_idx); \ 2737 } \ 2738 if (__rc || (__elem_bitmask == 0)) \ 2739 break; \ 2740 if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) \ 2741 break; \ 2742 if (sleep_us) \ 2743 usleep_range((sleep_us >> 2) + 1, sleep_us); \ 2744 } \ 2745 __rc ? __rc : ((__elem_bitmask == 0) ? 0 : -ETIMEDOUT); \ 2746}) 2747 2748#define hl_poll_reg_array_timeout(hdev, addr_arr, arr_size, expected_val, sleep_us, \ 2749 timeout_us) \ 2750 hl_poll_reg_array_timeout_common(hdev, addr_arr, arr_size, expected_val, sleep_us, \ 2751 timeout_us, false) 2752 2753#define hl_poll_reg_array_timeout_elbi(hdev, addr_arr, arr_size, expected_val, sleep_us, \ 2754 timeout_us) \ 2755 hl_poll_reg_array_timeout_common(hdev, addr_arr, arr_size, expected_val, sleep_us, \ 2756 timeout_us, true) 2757 2758/* 2759 * address in this macro points always to a memory location in the 2760 * host's (server's) memory. That location is updated asynchronously 2761 * either by the direct access of the device or by another core. 2762 * 2763 * To work both in LE and BE architectures, we need to distinguish between the 2764 * two states (device or another core updates the memory location). Therefore, 2765 * if mem_written_by_device is true, the host memory being polled will be 2766 * updated directly by the device. If false, the host memory being polled will 2767 * be updated by host CPU. Required so host knows whether or not the memory 2768 * might need to be byte-swapped before returning value to caller. 2769 * 2770 * On the first 4 polling iterations the macro goes to sleep for short period of 2771 * time that gradually increases and reaches sleep_us on the fifth iteration. 2772 */ 2773#define hl_poll_timeout_memory(hdev, addr, val, cond, sleep_us, timeout_us, \ 2774 mem_written_by_device) \ 2775({ \ 2776 u64 __sleep_step_us; \ 2777 ktime_t __timeout; \ 2778 u8 __step = 8; \ 2779 \ 2780 __timeout = ktime_add_us(ktime_get(), timeout_us); \ 2781 might_sleep_if(sleep_us); \ 2782 for (;;) { \ 2783 /* Verify we read updates done by other cores or by device */ \ 2784 mb(); \ 2785 (val) = *((u32 *)(addr)); \ 2786 if (mem_written_by_device) \ 2787 (val) = le32_to_cpu(*(__le32 *) &(val)); \ 2788 if (cond) \ 2789 break; \ 2790 if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) { \ 2791 (val) = *((u32 *)(addr)); \ 2792 if (mem_written_by_device) \ 2793 (val) = le32_to_cpu(*(__le32 *) &(val)); \ 2794 break; \ 2795 } \ 2796 __sleep_step_us = sleep_us >> __step; \ 2797 if (__sleep_step_us) \ 2798 usleep_range((__sleep_step_us >> 2) + 1, __sleep_step_us); \ 2799 __step >>= 1; \ 2800 } \ 2801 (cond) ? 0 : -ETIMEDOUT; \ 2802}) 2803 2804#define HL_USR_MAPPED_BLK_INIT(blk, base, sz) \ 2805({ \ 2806 struct user_mapped_block *p = blk; \ 2807\ 2808 p->address = base; \ 2809 p->size = sz; \ 2810}) 2811 2812#define HL_USR_INTR_STRUCT_INIT(usr_intr, hdev, intr_id, intr_type) \ 2813({ \ 2814 usr_intr.hdev = hdev; \ 2815 usr_intr.interrupt_id = intr_id; \ 2816 usr_intr.type = intr_type; \ 2817 INIT_LIST_HEAD(&usr_intr.wait_list_head); \ 2818 spin_lock_init(&usr_intr.wait_list_lock); \ 2819 INIT_LIST_HEAD(&usr_intr.ts_list_head); \ 2820 spin_lock_init(&usr_intr.ts_list_lock); \ 2821}) 2822 2823struct hwmon_chip_info; 2824 2825/** 2826 * struct hl_device_reset_work - reset work wrapper. 2827 * @reset_work: reset work to be done. 2828 * @hdev: habanalabs device structure. 2829 * @flags: reset flags. 2830 */ 2831struct hl_device_reset_work { 2832 struct delayed_work reset_work; 2833 struct hl_device *hdev; 2834 u32 flags; 2835}; 2836 2837/** 2838 * struct hl_mmu_hr_pgt_priv - used for holding per-device mmu host-resident 2839 * page-table internal information. 2840 * @mmu_pgt_pool: pool of page tables used by a host-resident MMU for 2841 * allocating hops. 2842 * @mmu_asid_hop0: per-ASID array of host-resident hop0 tables. 2843 */ 2844struct hl_mmu_hr_priv { 2845 struct gen_pool *mmu_pgt_pool; 2846 struct pgt_info *mmu_asid_hop0; 2847}; 2848 2849/** 2850 * struct hl_mmu_dr_pgt_priv - used for holding per-device mmu device-resident 2851 * page-table internal information. 2852 * @mmu_pgt_pool: pool of page tables used by MMU for allocating hops. 2853 * @mmu_shadow_hop0: shadow array of hop0 tables. 2854 */ 2855struct hl_mmu_dr_priv { 2856 struct gen_pool *mmu_pgt_pool; 2857 void *mmu_shadow_hop0; 2858}; 2859 2860/** 2861 * struct hl_mmu_priv - used for holding per-device mmu internal information. 2862 * @dr: information on the device-resident MMU, when exists. 2863 * @hr: information on the host-resident MMU, when exists. 2864 */ 2865struct hl_mmu_priv { 2866 struct hl_mmu_dr_priv dr; 2867 struct hl_mmu_hr_priv hr; 2868}; 2869 2870/** 2871 * struct hl_mmu_per_hop_info - A structure describing one TLB HOP and its entry 2872 * that was created in order to translate a virtual address to a 2873 * physical one. 2874 * @hop_addr: The address of the hop. 2875 * @hop_pte_addr: The address of the hop entry. 2876 * @hop_pte_val: The value in the hop entry. 2877 */ 2878struct hl_mmu_per_hop_info { 2879 u64 hop_addr; 2880 u64 hop_pte_addr; 2881 u64 hop_pte_val; 2882}; 2883 2884/** 2885 * struct hl_mmu_hop_info - A structure describing the TLB hops and their 2886 * hop-entries that were created in order to translate a virtual address to a 2887 * physical one. 2888 * @scrambled_vaddr: The value of the virtual address after scrambling. This 2889 * address replaces the original virtual-address when mapped 2890 * in the MMU tables. 2891 * @unscrambled_paddr: The un-scrambled physical address. 2892 * @hop_info: Array holding the per-hop information used for the translation. 2893 * @used_hops: The number of hops used for the translation. 2894 * @range_type: virtual address range type. 2895 */ 2896struct hl_mmu_hop_info { 2897 u64 scrambled_vaddr; 2898 u64 unscrambled_paddr; 2899 struct hl_mmu_per_hop_info hop_info[MMU_ARCH_6_HOPS]; 2900 u32 used_hops; 2901 enum hl_va_range_type range_type; 2902}; 2903 2904/** 2905 * struct hl_hr_mmu_funcs - Device related host resident MMU functions. 2906 * @get_hop0_pgt_info: get page table info structure for HOP0. 2907 * @get_pgt_info: get page table info structure for HOP other than HOP0. 2908 * @add_pgt_info: add page table info structure to hash. 2909 * @get_tlb_mapping_params: get mapping parameters needed for getting TLB info for specific mapping. 2910 */ 2911struct hl_hr_mmu_funcs { 2912 struct pgt_info *(*get_hop0_pgt_info)(struct hl_ctx *ctx); 2913 struct pgt_info *(*get_pgt_info)(struct hl_ctx *ctx, u64 phys_hop_addr); 2914 void (*add_pgt_info)(struct hl_ctx *ctx, struct pgt_info *pgt_info, dma_addr_t phys_addr); 2915 int (*get_tlb_mapping_params)(struct hl_device *hdev, struct hl_mmu_properties **mmu_prop, 2916 struct hl_mmu_hop_info *hops, 2917 u64 virt_addr, bool *is_huge); 2918}; 2919 2920/** 2921 * struct hl_mmu_funcs - Device related MMU functions. 2922 * @init: initialize the MMU module. 2923 * @fini: release the MMU module. 2924 * @ctx_init: Initialize a context for using the MMU module. 2925 * @ctx_fini: disable a ctx from using the mmu module. 2926 * @map: maps a virtual address to physical address for a context. 2927 * @unmap: unmap a virtual address of a context. 2928 * @flush: flush all writes from all cores to reach device MMU. 2929 * @swap_out: marks all mapping of the given context as swapped out. 2930 * @swap_in: marks all mapping of the given context as swapped in. 2931 * @get_tlb_info: returns the list of hops and hop-entries used that were 2932 * created in order to translate the giver virtual address to a 2933 * physical one. 2934 * @hr_funcs: functions specific to host resident MMU. 2935 */ 2936struct hl_mmu_funcs { 2937 int (*init)(struct hl_device *hdev); 2938 void (*fini)(struct hl_device *hdev); 2939 int (*ctx_init)(struct hl_ctx *ctx); 2940 void (*ctx_fini)(struct hl_ctx *ctx); 2941 int (*map)(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr, u32 page_size, 2942 bool is_dram_addr); 2943 int (*unmap)(struct hl_ctx *ctx, u64 virt_addr, bool is_dram_addr); 2944 void (*flush)(struct hl_ctx *ctx); 2945 void (*swap_out)(struct hl_ctx *ctx); 2946 void (*swap_in)(struct hl_ctx *ctx); 2947 int (*get_tlb_info)(struct hl_ctx *ctx, u64 virt_addr, struct hl_mmu_hop_info *hops); 2948 struct hl_hr_mmu_funcs hr_funcs; 2949}; 2950 2951/** 2952 * struct hl_prefetch_work - prefetch work structure handler 2953 * @prefetch_work: actual work struct. 2954 * @ctx: compute context. 2955 * @va: virtual address to pre-fetch. 2956 * @size: pre-fetch size. 2957 * @flags: operation flags. 2958 * @asid: ASID for maintenance operation. 2959 */ 2960struct hl_prefetch_work { 2961 struct work_struct prefetch_work; 2962 struct hl_ctx *ctx; 2963 u64 va; 2964 u64 size; 2965 u32 flags; 2966 u32 asid; 2967}; 2968 2969/* 2970 * number of user contexts allowed to call wait_for_multi_cs ioctl in 2971 * parallel 2972 */ 2973#define MULTI_CS_MAX_USER_CTX 2 2974 2975/** 2976 * struct multi_cs_completion - multi CS wait completion. 2977 * @completion: completion of any of the CS in the list 2978 * @lock: spinlock for the completion structure 2979 * @timestamp: timestamp for the multi-CS completion 2980 * @stream_master_qid_map: bitmap of all stream masters on which the multi-CS 2981 * is waiting 2982 * @used: 1 if in use, otherwise 0 2983 */ 2984struct multi_cs_completion { 2985 struct completion completion; 2986 spinlock_t lock; 2987 s64 timestamp; 2988 u32 stream_master_qid_map; 2989 u8 used; 2990}; 2991 2992/** 2993 * struct multi_cs_data - internal data for multi CS call 2994 * @ctx: pointer to the context structure 2995 * @fence_arr: array of fences of all CSs 2996 * @seq_arr: array of CS sequence numbers 2997 * @timeout_jiffies: timeout in jiffies for waiting for CS to complete 2998 * @timestamp: timestamp of first completed CS 2999 * @wait_status: wait for CS status 3000 * @completion_bitmap: bitmap of completed CSs (1- completed, otherwise 0) 3001 * @arr_len: fence_arr and seq_arr array length 3002 * @gone_cs: indication of gone CS (1- there was gone CS, otherwise 0) 3003 * @update_ts: update timestamp. 1- update the timestamp, otherwise 0. 3004 */ 3005struct multi_cs_data { 3006 struct hl_ctx *ctx; 3007 struct hl_fence **fence_arr; 3008 u64 *seq_arr; 3009 s64 timeout_jiffies; 3010 s64 timestamp; 3011 long wait_status; 3012 u32 completion_bitmap; 3013 u8 arr_len; 3014 u8 gone_cs; 3015 u8 update_ts; 3016}; 3017 3018/** 3019 * struct hl_clk_throttle_timestamp - current/last clock throttling timestamp 3020 * @start: timestamp taken when 'start' event is received in driver 3021 * @end: timestamp taken when 'end' event is received in driver 3022 */ 3023struct hl_clk_throttle_timestamp { 3024 ktime_t start; 3025 ktime_t end; 3026}; 3027 3028/** 3029 * struct hl_clk_throttle - keeps current/last clock throttling timestamps 3030 * @timestamp: timestamp taken by driver and firmware, index 0 refers to POWER 3031 * index 1 refers to THERMAL 3032 * @lock: protects this structure as it can be accessed from both event queue 3033 * context and info_ioctl context 3034 * @current_reason: bitmask represents the current clk throttling reasons 3035 * @aggregated_reason: bitmask represents aggregated clk throttling reasons since driver load 3036 */ 3037struct hl_clk_throttle { 3038 struct hl_clk_throttle_timestamp timestamp[HL_CLK_THROTTLE_TYPE_MAX]; 3039 struct mutex lock; 3040 u32 current_reason; 3041 u32 aggregated_reason; 3042}; 3043 3044/** 3045 * struct user_mapped_block - describes a hw block allowed to be mmapped by user 3046 * @address: physical HW block address 3047 * @size: allowed size for mmap 3048 */ 3049struct user_mapped_block { 3050 u32 address; 3051 u32 size; 3052}; 3053 3054/** 3055 * struct cs_timeout_info - info of last CS timeout occurred. 3056 * @timestamp: CS timeout timestamp. 3057 * @write_enable: if set writing to CS parameters in the structure is enabled. otherwise - disabled, 3058 * so the first (root cause) CS timeout will not be overwritten. 3059 * @seq: CS timeout sequence number. 3060 */ 3061struct cs_timeout_info { 3062 ktime_t timestamp; 3063 atomic_t write_enable; 3064 u64 seq; 3065}; 3066 3067#define MAX_QMAN_STREAMS_INFO 4 3068#define OPCODE_INFO_MAX_ADDR_SIZE 8 3069/** 3070 * struct undefined_opcode_info - info about last undefined opcode error 3071 * @timestamp: timestamp of the undefined opcode error 3072 * @cb_addr_streams: CB addresses (per stream) that are currently exists in the PQ 3073 * entries. In case all streams array entries are 3074 * filled with values, it means the execution was in Lower-CP. 3075 * @cq_addr: the address of the current handled command buffer 3076 * @cq_size: the size of the current handled command buffer 3077 * @cb_addr_streams_len: num of streams - actual len of cb_addr_streams array. 3078 * should be equal to 1 in case of undefined opcode 3079 * in Upper-CP (specific stream) and equal to 4 in case 3080 * of undefined opcode in Lower-CP. 3081 * @engine_id: engine-id that the error occurred on 3082 * @stream_id: the stream id the error occurred on. In case the stream equals to 3083 * MAX_QMAN_STREAMS_INFO it means the error occurred on a Lower-CP. 3084 * @write_enable: if set, writing to undefined opcode parameters in the structure 3085 * is enable so the first (root cause) undefined opcode will not be 3086 * overwritten. 3087 */ 3088struct undefined_opcode_info { 3089 ktime_t timestamp; 3090 u64 cb_addr_streams[MAX_QMAN_STREAMS_INFO][OPCODE_INFO_MAX_ADDR_SIZE]; 3091 u64 cq_addr; 3092 u32 cq_size; 3093 u32 cb_addr_streams_len; 3094 u32 engine_id; 3095 u32 stream_id; 3096 bool write_enable; 3097}; 3098 3099/** 3100 * struct page_fault_info - page fault information. 3101 * @page_fault: holds information collected during a page fault. 3102 * @user_mappings: buffer containing user mappings. 3103 * @num_of_user_mappings: number of user mappings. 3104 * @page_fault_detected: if set as 1, then a page-fault was discovered for the 3105 * first time after the driver has finished booting-up. 3106 * Since we're looking for the page-fault's root cause, 3107 * we don't care of the others that might follow it- 3108 * so once changed to 1, it will remain that way. 3109 * @page_fault_info_available: indicates that a page fault info is now available. 3110 */ 3111struct page_fault_info { 3112 struct hl_page_fault_info page_fault; 3113 struct hl_user_mapping *user_mappings; 3114 u64 num_of_user_mappings; 3115 atomic_t page_fault_detected; 3116 bool page_fault_info_available; 3117}; 3118 3119/** 3120 * struct razwi_info - RAZWI information. 3121 * @razwi: holds information collected during a RAZWI 3122 * @razwi_detected: if set as 1, then a RAZWI was discovered for the 3123 * first time after the driver has finished booting-up. 3124 * Since we're looking for the RAZWI's root cause, 3125 * we don't care of the others that might follow it- 3126 * so once changed to 1, it will remain that way. 3127 * @razwi_info_available: indicates that a RAZWI info is now available. 3128 */ 3129struct razwi_info { 3130 struct hl_info_razwi_event razwi; 3131 atomic_t razwi_detected; 3132 bool razwi_info_available; 3133}; 3134 3135/** 3136 * struct hw_err_info - HW error information. 3137 * @event: holds information on the event. 3138 * @event_detected: if set as 1, then a HW event was discovered for the 3139 * first time after the driver has finished booting-up. 3140 * currently we assume that only fatal events (that require hard-reset) are 3141 * reported so we don't care of the others that might follow it. 3142 * so once changed to 1, it will remain that way. 3143 * TODO: support multiple events. 3144 * @event_info_available: indicates that a HW event info is now available. 3145 */ 3146struct hw_err_info { 3147 struct hl_info_hw_err_event event; 3148 atomic_t event_detected; 3149 bool event_info_available; 3150}; 3151 3152/** 3153 * struct fw_err_info - FW error information. 3154 * @event: holds information on the event. 3155 * @event_detected: if set as 1, then a FW event was discovered for the 3156 * first time after the driver has finished booting-up. 3157 * currently we assume that only fatal events (that require hard-reset) are 3158 * reported so we don't care of the others that might follow it. 3159 * so once changed to 1, it will remain that way. 3160 * TODO: support multiple events. 3161 * @event_info_available: indicates that a HW event info is now available. 3162 */ 3163struct fw_err_info { 3164 struct hl_info_fw_err_event event; 3165 atomic_t event_detected; 3166 bool event_info_available; 3167}; 3168 3169/** 3170 * struct engine_err_info - engine error information. 3171 * @event: holds information on the event. 3172 * @event_detected: if set as 1, then an engine event was discovered for the 3173 * first time after the driver has finished booting-up. 3174 * @event_info_available: indicates that an engine event info is now available. 3175 */ 3176struct engine_err_info { 3177 struct hl_info_engine_err_event event; 3178 atomic_t event_detected; 3179 bool event_info_available; 3180}; 3181 3182 3183/** 3184 * struct hl_error_info - holds information collected during an error. 3185 * @cs_timeout: CS timeout error information. 3186 * @razwi_info: RAZWI information. 3187 * @undef_opcode: undefined opcode information. 3188 * @page_fault_info: page fault information. 3189 * @hw_err: (fatal) hardware error information. 3190 * @fw_err: firmware error information. 3191 * @engine_err: engine error information. 3192 */ 3193struct hl_error_info { 3194 struct cs_timeout_info cs_timeout; 3195 struct razwi_info razwi_info; 3196 struct undefined_opcode_info undef_opcode; 3197 struct page_fault_info page_fault_info; 3198 struct hw_err_info hw_err; 3199 struct fw_err_info fw_err; 3200 struct engine_err_info engine_err; 3201}; 3202 3203/** 3204 * struct hl_reset_info - holds current device reset information. 3205 * @lock: lock to protect critical reset flows. 3206 * @compute_reset_cnt: number of compute resets since the driver was loaded. 3207 * @hard_reset_cnt: number of hard resets since the driver was loaded. 3208 * @hard_reset_schedule_flags: hard reset is scheduled to after current compute reset, 3209 * here we hold the hard reset flags. 3210 * @in_reset: is device in reset flow. 3211 * @in_compute_reset: Device is currently in reset but not in hard-reset. 3212 * @needs_reset: true if reset_on_lockup is false and device should be reset 3213 * due to lockup. 3214 * @hard_reset_pending: is there a hard reset work pending. 3215 * @curr_reset_cause: saves an enumerated reset cause when a hard reset is 3216 * triggered, and cleared after it is shared with preboot. 3217 * @prev_reset_trigger: saves the previous trigger which caused a reset, overridden 3218 * with a new value on next reset 3219 * @reset_trigger_repeated: set if device reset is triggered more than once with 3220 * same cause. 3221 * @skip_reset_on_timeout: Skip device reset if CS has timed out, wait for it to 3222 * complete instead. 3223 * @watchdog_active: true if a device release watchdog work is scheduled. 3224 */ 3225struct hl_reset_info { 3226 spinlock_t lock; 3227 u32 compute_reset_cnt; 3228 u32 hard_reset_cnt; 3229 u32 hard_reset_schedule_flags; 3230 u8 in_reset; 3231 u8 in_compute_reset; 3232 u8 needs_reset; 3233 u8 hard_reset_pending; 3234 u8 curr_reset_cause; 3235 u8 prev_reset_trigger; 3236 u8 reset_trigger_repeated; 3237 u8 skip_reset_on_timeout; 3238 u8 watchdog_active; 3239}; 3240 3241/** 3242 * struct eq_heartbeat_debug_info - stores debug info to be used upon heartbeat failure. 3243 * @last_pq_heartbeat_ts: timestamp of the last test packet that was sent to FW. 3244 * This packet is the trigger in FW to send the EQ heartbeat event. 3245 * @last_eq_heartbeat_ts: timestamp of the last EQ heartbeat event that was received from FW. 3246 * @heartbeat_event_counter: number of heartbeat events received. 3247 * @cpu_queue_id: used to read the queue pi/ci 3248 */ 3249struct eq_heartbeat_debug_info { 3250 time64_t last_pq_heartbeat_ts; 3251 time64_t last_eq_heartbeat_ts; 3252 u32 heartbeat_event_counter; 3253 u32 cpu_queue_id; 3254}; 3255 3256/** 3257 * struct hl_device - habanalabs device structure. 3258 * @pdev: pointer to PCI device, can be NULL in case of simulator device. 3259 * @pcie_bar_phys: array of available PCIe bars physical addresses. 3260 * (required only for PCI address match mode) 3261 * @pcie_bar: array of available PCIe bars virtual addresses. 3262 * @rmmio: configuration area address on SRAM. 3263 * @drm: related DRM device. 3264 * @cdev_ctrl: char device for control operations only (INFO IOCTL) 3265 * @dev: related kernel basic device structure. 3266 * @dev_ctrl: related kernel device structure for the control device 3267 * @work_heartbeat: delayed work for CPU-CP is-alive check. 3268 * @device_reset_work: delayed work which performs hard reset 3269 * @device_release_watchdog_work: watchdog work that performs hard reset if user doesn't release 3270 * device upon certain error cases. 3271 * @asic_name: ASIC specific name. 3272 * @asic_type: ASIC specific type. 3273 * @completion_queue: array of hl_cq. 3274 * @user_interrupt: array of hl_user_interrupt. upon the corresponding user 3275 * interrupt, driver will monitor the list of fences 3276 * registered to this interrupt. 3277 * @tpc_interrupt: single TPC interrupt for all TPCs. 3278 * @unexpected_error_interrupt: single interrupt for unexpected user error indication. 3279 * @common_user_cq_interrupt: common user CQ interrupt for all user CQ interrupts. 3280 * upon any user CQ interrupt, driver will monitor the 3281 * list of fences registered to this common structure. 3282 * @common_decoder_interrupt: common decoder interrupt for all user decoder interrupts. 3283 * @shadow_cs_queue: pointer to a shadow queue that holds pointers to 3284 * outstanding command submissions. 3285 * @cq_wq: work queues of completion queues for executing work in process 3286 * context. 3287 * @eq_wq: work queue of event queue for executing work in process context. 3288 * @cs_cmplt_wq: work queue of CS completions for executing work in process 3289 * context. 3290 * @ts_free_obj_wq: work queue for timestamp registration objects release. 3291 * @prefetch_wq: work queue for MMU pre-fetch operations. 3292 * @reset_wq: work queue for device reset procedure. 3293 * @kernel_ctx: Kernel driver context structure. 3294 * @kernel_queues: array of hl_hw_queue. 3295 * @cs_mirror_list: CS mirror list for TDR. 3296 * @cs_mirror_lock: protects cs_mirror_list. 3297 * @kernel_mem_mgr: memory manager for memory buffers with lifespan of driver. 3298 * @event_queue: event queue for IRQ from CPU-CP. 3299 * @dma_pool: DMA pool for small allocations. 3300 * @cpu_accessible_dma_mem: Host <-> CPU-CP shared memory CPU address. 3301 * @cpu_accessible_dma_address: Host <-> CPU-CP shared memory DMA address. 3302 * @cpu_accessible_dma_pool: Host <-> CPU-CP shared memory pool. 3303 * @asid_bitmap: holds used/available ASIDs. 3304 * @asid_mutex: protects asid_bitmap. 3305 * @send_cpu_message_lock: enforces only one message in Host <-> CPU-CP queue. 3306 * @debug_lock: protects critical section of setting debug mode for device 3307 * @mmu_lock: protects the MMU page tables and invalidation h/w. Although the 3308 * page tables are per context, the invalidation h/w is per MMU. 3309 * Therefore, we can't allow multiple contexts (we only have two, 3310 * user and kernel) to access the invalidation h/w at the same time. 3311 * In addition, any change to the PGT, modifying the MMU hash or 3312 * walking the PGT requires talking this lock. 3313 * @asic_prop: ASIC specific immutable properties. 3314 * @asic_funcs: ASIC specific functions. 3315 * @asic_specific: ASIC specific information to use only from ASIC files. 3316 * @vm: virtual memory manager for MMU. 3317 * @hwmon_dev: H/W monitor device. 3318 * @hl_chip_info: ASIC's sensors information. 3319 * @device_status_description: device status description. 3320 * @hl_debugfs: device's debugfs manager. 3321 * @debugfs_cfg_accesses: list of last debugfs config region accesses. 3322 * @cb_pool: list of pre allocated CBs. 3323 * @cb_pool_lock: protects the CB pool. 3324 * @internal_cb_pool_virt_addr: internal command buffer pool virtual address. 3325 * @internal_cb_pool_dma_addr: internal command buffer pool dma address. 3326 * @internal_cb_pool: internal command buffer memory pool. 3327 * @internal_cb_va_base: internal cb pool mmu virtual address base 3328 * @fpriv_list: list of file private data structures. Each structure is created 3329 * when a user opens the device 3330 * @fpriv_ctrl_list: list of file private data structures. Each structure is created 3331 * when a user opens the control device 3332 * @fpriv_list_lock: protects the fpriv_list 3333 * @fpriv_ctrl_list_lock: protects the fpriv_ctrl_list 3334 * @aggregated_cs_counters: aggregated cs counters among all contexts 3335 * @mmu_priv: device-specific MMU data. 3336 * @mmu_func: device-related MMU functions. 3337 * @dec: list of decoder sw instance 3338 * @fw_loader: FW loader manager. 3339 * @pci_mem_region: array of memory regions in the PCI 3340 * @state_dump_specs: constants and dictionaries needed to dump system state. 3341 * @multi_cs_completion: array of multi-CS completion. 3342 * @clk_throttling: holds information about current/previous clock throttling events 3343 * @captured_err_info: holds information about errors. 3344 * @reset_info: holds current device reset information. 3345 * @heartbeat_debug_info: counters used to debug heartbeat failures. 3346 * @hldio: describes habanalabs direct storage interaction interface. 3347 * @irq_affinity_mask: mask of available CPU cores for user and decoder interrupt handling. 3348 * @stream_master_qid_arr: pointer to array with QIDs of master streams. 3349 * @fw_inner_major_ver: the major of current loaded preboot inner version. 3350 * @fw_inner_minor_ver: the minor of current loaded preboot inner version. 3351 * @fw_sw_major_ver: the major of current loaded preboot SW version. 3352 * @fw_sw_minor_ver: the minor of current loaded preboot SW version. 3353 * @fw_sw_sub_minor_ver: the sub-minor of current loaded preboot SW version. 3354 * @dram_used_mem: current DRAM memory consumption. 3355 * @memory_scrub_val: the value to which the dram will be scrubbed to using cb scrub_device_dram 3356 * @timeout_jiffies: device CS timeout value. 3357 * @max_power: the max power of the device, as configured by the sysadmin. This 3358 * value is saved so in case of hard-reset, the driver will restore 3359 * this value and update the F/W after the re-initialization 3360 * @boot_error_status_mask: contains a mask of the device boot error status. 3361 * Each bit represents a different error, according to 3362 * the defines in hl_boot_if.h. If the bit is cleared, 3363 * the error will be ignored by the driver during 3364 * device initialization. Mainly used to debug and 3365 * workaround firmware bugs 3366 * @dram_pci_bar_start: start bus address of PCIe bar towards DRAM. 3367 * @last_successful_open_ktime: timestamp (ktime) of the last successful device open. 3368 * @last_successful_open_jif: timestamp (jiffies) of the last successful 3369 * device open. 3370 * @last_open_session_duration_jif: duration (jiffies) of the last device open 3371 * session. 3372 * @open_counter: number of successful device open operations. 3373 * @fw_poll_interval_usec: FW status poll interval in usec. 3374 * used for CPU boot status 3375 * @fw_comms_poll_interval_usec: FW comms/protocol poll interval in usec. 3376 * used for COMMs protocols cmds(COMMS_STS_*) 3377 * @dram_binning: contains mask of drams that is received from the f/w which indicates which 3378 * drams are binned-out 3379 * @tpc_binning: contains mask of tpc engines that is received from the f/w which indicates which 3380 * tpc engines are binned-out 3381 * @dmabuf_export_cnt: number of dma-buf exporting. 3382 * @card_type: Various ASICs have several card types. This indicates the card 3383 * type of the current device. 3384 * @major: habanalabs kernel driver major. 3385 * @high_pll: high PLL profile frequency. 3386 * @decoder_binning: contains mask of decoder engines that is received from the f/w which 3387 * indicates which decoder engines are binned-out 3388 * @edma_binning: contains mask of edma engines that is received from the f/w which 3389 * indicates which edma engines are binned-out 3390 * @device_release_watchdog_timeout_sec: device release watchdog timeout value in seconds. 3391 * @rotator_binning: contains mask of rotators engines that is received from the f/w 3392 * which indicates which rotator engines are binned-out(Gaudi3 and above). 3393 * @id: device minor. 3394 * @cdev_idx: char device index. 3395 * @cpu_pci_msb_addr: 50-bit extension bits for the device CPU's 40-bit 3396 * addresses. 3397 * @is_in_dram_scrub: true if dram scrub operation is on going. 3398 * @disabled: is device disabled. 3399 * @cpld_shutdown: is cpld shutdown. 3400 * @late_init_done: is late init stage was done during initialization. 3401 * @hwmon_initialized: is H/W monitor sensors was initialized. 3402 * @reset_on_lockup: true if a reset should be done in case of stuck CS, false 3403 * otherwise. 3404 * @dram_default_page_mapping: is DRAM default page mapping enabled. 3405 * @memory_scrub: true to perform device memory scrub in various locations, 3406 * such as context-switch, context close, page free, etc. 3407 * @pmmu_huge_range: is a different virtual addresses range used for PMMU with 3408 * huge pages. 3409 * @init_done: is the initialization of the device done. 3410 * @device_cpu_disabled: is the device CPU disabled (due to timeouts) 3411 * @in_debug: whether the device is in a state where the profiling/tracing infrastructure 3412 * can be used. This indication is needed because in some ASICs we need to do 3413 * specific operations to enable that infrastructure. 3414 * @cdev_sysfs_debugfs_created: were char devices and sysfs/debugfs files created. 3415 * @stop_on_err: true if engines should stop on error. 3416 * @supports_sync_stream: is sync stream supported. 3417 * @sync_stream_queue_idx: helper index for sync stream queues initialization. 3418 * @collective_mon_idx: helper index for collective initialization 3419 * @supports_coresight: is CoreSight supported. 3420 * @supports_cb_mapping: is mapping a CB to the device's MMU supported. 3421 * @process_kill_trial_cnt: number of trials reset thread tried killing 3422 * user processes 3423 * @device_fini_pending: true if device_fini was called and might be 3424 * waiting for the reset thread to finish 3425 * @supports_staged_submission: true if staged submissions are supported 3426 * @device_cpu_is_halted: Flag to indicate whether the device CPU was already 3427 * halted. We can't halt it again because the COMMS 3428 * protocol will throw an error. Relevant only for 3429 * cases where Linux was not loaded to device CPU 3430 * @supports_wait_for_multi_cs: true if wait for multi CS is supported 3431 * @is_compute_ctx_active: Whether there is an active compute context executing. 3432 * @compute_ctx_in_release: true if the current compute context is being released. 3433 * @supports_mmu_prefetch: true if prefetch is supported, otherwise false. 3434 * @reset_upon_device_release: reset the device when the user closes the file descriptor of the 3435 * device. 3436 * @supports_ctx_switch: true if a ctx switch is required upon first submission. 3437 * @support_preboot_binning: true if we support read binning info from preboot. 3438 * @eq_heartbeat_received: indication that eq heartbeat event has received from FW. 3439 * @nic_ports_mask: Controls which NIC ports are enabled. Used only for testing. 3440 * @fw_components: Controls which f/w components to load to the device. There are multiple f/w 3441 * stages and sometimes we want to stop at a certain stage. Used only for testing. 3442 * @mmu_disable: Disable the device MMU(s). Used only for testing. 3443 * @cpu_queues_enable: Whether to enable queues communication vs. the f/w. Used only for testing. 3444 * @pldm: Whether we are running in Palladium environment. Used only for testing. 3445 * @hard_reset_on_fw_events: Whether to do device hard-reset when a fatal event is received from 3446 * the f/w. Used only for testing. 3447 * @bmc_enable: Whether we are running in a box with BMC. Used only for testing. 3448 * @reset_on_preboot_fail: Whether to reset the device if preboot f/w fails to load. 3449 * Used only for testing. 3450 * @heartbeat: Controls if we want to enable the heartbeat mechanism vs. the f/w, which verifies 3451 * that the f/w is always alive. Used only for testing. 3452 */ 3453struct hl_device { 3454 struct pci_dev *pdev; 3455 u64 pcie_bar_phys[HL_PCI_NUM_BARS]; 3456 void __iomem *pcie_bar[HL_PCI_NUM_BARS]; 3457 void __iomem *rmmio; 3458 struct drm_device drm; 3459 struct cdev cdev_ctrl; 3460 struct device *dev; 3461 struct device *dev_ctrl; 3462 struct delayed_work work_heartbeat; 3463 struct hl_device_reset_work device_reset_work; 3464 struct hl_device_reset_work device_release_watchdog_work; 3465 char asic_name[HL_STR_MAX]; 3466 char status[HL_DEV_STS_MAX][HL_STR_MAX]; 3467 enum hl_asic_type asic_type; 3468 struct hl_cq *completion_queue; 3469 struct hl_user_interrupt *user_interrupt; 3470 struct hl_user_interrupt tpc_interrupt; 3471 struct hl_user_interrupt unexpected_error_interrupt; 3472 struct hl_user_interrupt common_user_cq_interrupt; 3473 struct hl_user_interrupt common_decoder_interrupt; 3474 struct hl_cs **shadow_cs_queue; 3475 struct workqueue_struct **cq_wq; 3476 struct workqueue_struct *eq_wq; 3477 struct workqueue_struct *cs_cmplt_wq; 3478 struct workqueue_struct *ts_free_obj_wq; 3479 struct workqueue_struct *prefetch_wq; 3480 struct workqueue_struct *reset_wq; 3481 struct hl_ctx *kernel_ctx; 3482 struct hl_hw_queue *kernel_queues; 3483 struct list_head cs_mirror_list; 3484 spinlock_t cs_mirror_lock; 3485 struct hl_mem_mgr kernel_mem_mgr; 3486 struct hl_eq event_queue; 3487 struct dma_pool *dma_pool; 3488 void *cpu_accessible_dma_mem; 3489 dma_addr_t cpu_accessible_dma_address; 3490 struct gen_pool *cpu_accessible_dma_pool; 3491 unsigned long *asid_bitmap; 3492 struct mutex asid_mutex; 3493 struct mutex send_cpu_message_lock; 3494 struct mutex debug_lock; 3495 struct mutex mmu_lock; 3496 struct asic_fixed_properties asic_prop; 3497 const struct hl_asic_funcs *asic_funcs; 3498 void *asic_specific; 3499 struct hl_vm vm; 3500 struct device *hwmon_dev; 3501 struct hwmon_chip_info *hl_chip_info; 3502 3503 struct hl_dbg_device_entry hl_debugfs; 3504 struct hl_debugfs_cfg_access debugfs_cfg_accesses; 3505 3506 struct list_head cb_pool; 3507 spinlock_t cb_pool_lock; 3508 3509 void *internal_cb_pool_virt_addr; 3510 dma_addr_t internal_cb_pool_dma_addr; 3511 struct gen_pool *internal_cb_pool; 3512 u64 internal_cb_va_base; 3513 3514 struct list_head fpriv_list; 3515 struct list_head fpriv_ctrl_list; 3516 struct mutex fpriv_list_lock; 3517 struct mutex fpriv_ctrl_list_lock; 3518 3519 struct hl_cs_counters_atomic aggregated_cs_counters; 3520 3521 struct hl_mmu_priv mmu_priv; 3522 struct hl_mmu_funcs mmu_func[MMU_NUM_PGT_LOCATIONS]; 3523 3524 struct hl_dec *dec; 3525 3526 struct fw_load_mgr fw_loader; 3527 3528 struct pci_mem_region pci_mem_region[PCI_REGION_NUMBER]; 3529 3530 struct hl_state_dump_specs state_dump_specs; 3531 3532 struct multi_cs_completion multi_cs_completion[ 3533 MULTI_CS_MAX_USER_CTX]; 3534 struct hl_clk_throttle clk_throttling; 3535 struct hl_error_info captured_err_info; 3536 3537 struct hl_reset_info reset_info; 3538 3539 struct eq_heartbeat_debug_info heartbeat_debug_info; 3540#ifdef CONFIG_HL_HLDIO 3541 struct hl_dio hldio; 3542#endif 3543 cpumask_t irq_affinity_mask; 3544 3545 u32 *stream_master_qid_arr; 3546 u32 fw_inner_major_ver; 3547 u32 fw_inner_minor_ver; 3548 u32 fw_sw_major_ver; 3549 u32 fw_sw_minor_ver; 3550 u32 fw_sw_sub_minor_ver; 3551 atomic64_t dram_used_mem; 3552 u64 memory_scrub_val; 3553 u64 timeout_jiffies; 3554 u64 max_power; 3555 u64 boot_error_status_mask; 3556 u64 dram_pci_bar_start; 3557 u64 last_successful_open_jif; 3558 u64 last_open_session_duration_jif; 3559 u64 open_counter; 3560 u64 fw_poll_interval_usec; 3561 ktime_t last_successful_open_ktime; 3562 u64 fw_comms_poll_interval_usec; 3563 u64 dram_binning; 3564 u64 tpc_binning; 3565 atomic_t dmabuf_export_cnt; 3566 enum cpucp_card_types card_type; 3567 u32 major; 3568 u32 high_pll; 3569 u32 decoder_binning; 3570 u32 edma_binning; 3571 u32 device_release_watchdog_timeout_sec; 3572 u32 rotator_binning; 3573 u16 id; 3574 u16 cdev_idx; 3575 u16 cpu_pci_msb_addr; 3576 u8 is_in_dram_scrub; 3577 u8 disabled; 3578 u8 cpld_shutdown; 3579 u8 late_init_done; 3580 u8 hwmon_initialized; 3581 u8 reset_on_lockup; 3582 u8 dram_default_page_mapping; 3583 u8 memory_scrub; 3584 u8 pmmu_huge_range; 3585 u8 init_done; 3586 u8 device_cpu_disabled; 3587 u8 in_debug; 3588 u8 cdev_sysfs_debugfs_created; 3589 u8 stop_on_err; 3590 u8 supports_sync_stream; 3591 u8 sync_stream_queue_idx; 3592 u8 collective_mon_idx; 3593 u8 supports_coresight; 3594 u8 supports_cb_mapping; 3595 u8 process_kill_trial_cnt; 3596 u8 device_fini_pending; 3597 u8 supports_staged_submission; 3598 u8 device_cpu_is_halted; 3599 u8 supports_wait_for_multi_cs; 3600 u8 stream_master_qid_arr_size; 3601 u8 is_compute_ctx_active; 3602 u8 compute_ctx_in_release; 3603 u8 supports_mmu_prefetch; 3604 u8 reset_upon_device_release; 3605 u8 supports_ctx_switch; 3606 u8 support_preboot_binning; 3607 u8 eq_heartbeat_received; 3608 3609 /* Parameters for bring-up to be upstreamed */ 3610 u64 nic_ports_mask; 3611 u64 fw_components; 3612 u8 mmu_disable; 3613 u8 cpu_queues_enable; 3614 u8 pldm; 3615 u8 hard_reset_on_fw_events; 3616 u8 bmc_enable; 3617 u8 reset_on_preboot_fail; 3618 u8 heartbeat; 3619}; 3620 3621/* Retrieve PCI device name in case of a PCI device or dev name in simulator */ 3622#define HL_DEV_NAME(hdev) \ 3623 ((hdev)->pdev ? dev_name(&(hdev)->pdev->dev) : "NA-DEVICE") 3624 3625/** 3626 * struct hl_cs_encaps_sig_handle - encapsulated signals handle structure 3627 * @refcount: refcount used to protect removing this id when several 3628 * wait cs are used to wait of the reserved encaps signals. 3629 * @hdev: pointer to habanalabs device structure. 3630 * @hw_sob: pointer to H/W SOB used in the reservation. 3631 * @ctx: pointer to the user's context data structure 3632 * @cs_seq: staged cs sequence which contains encapsulated signals 3633 * @id: idr handler id to be used to fetch the handler info 3634 * @q_idx: stream queue index 3635 * @pre_sob_val: current SOB value before reservation 3636 * @count: signals number 3637 */ 3638struct hl_cs_encaps_sig_handle { 3639 struct kref refcount; 3640 struct hl_device *hdev; 3641 struct hl_hw_sob *hw_sob; 3642 struct hl_ctx *ctx; 3643 u64 cs_seq; 3644 u32 id; 3645 u32 q_idx; 3646 u32 pre_sob_val; 3647 u32 count; 3648}; 3649 3650/** 3651 * struct hl_info_fw_err_info - firmware error information structure 3652 * @err_type: The type of error detected (or reported). 3653 * @event_mask: Pointer to the event mask to be modified with the detected error flag 3654 * (can be NULL) 3655 * @event_id: The id of the event that reported the error 3656 * (applicable when err_type is HL_INFO_FW_REPORTED_ERR). 3657 */ 3658struct hl_info_fw_err_info { 3659 enum hl_info_fw_err_type err_type; 3660 u64 *event_mask; 3661 u16 event_id; 3662}; 3663 3664/* 3665 * IOCTLs 3666 */ 3667 3668/** 3669 * typedef hl_ioctl_t - typedef for ioctl function in the driver 3670 * @hpriv: pointer to the FD's private data, which contains state of 3671 * user process 3672 * @data: pointer to the input/output arguments structure of the IOCTL 3673 * 3674 * Return: 0 for success, negative value for error 3675 */ 3676typedef int hl_ioctl_t(struct hl_fpriv *hpriv, void *data); 3677 3678/** 3679 * struct hl_ioctl_desc - describes an IOCTL entry of the driver. 3680 * @cmd: the IOCTL code as created by the kernel macros. 3681 * @func: pointer to the driver's function that should be called for this IOCTL. 3682 */ 3683struct hl_ioctl_desc { 3684 unsigned int cmd; 3685 hl_ioctl_t *func; 3686}; 3687 3688/* 3689 * Kernel module functions that can be accessed by entire module 3690 */ 3691 3692/** 3693 * hl_get_sg_info() - get number of pages and the DMA address from SG list. 3694 * @sg: the SG list. 3695 * @dma_addr: pointer to DMA address to return. 3696 * 3697 * Calculate the number of consecutive pages described by the SG list. Take the 3698 * offset of the address in the first page, add to it the length and round it up 3699 * to the number of needed pages. 3700 */ 3701static inline u32 hl_get_sg_info(struct scatterlist *sg, dma_addr_t *dma_addr) 3702{ 3703 *dma_addr = sg_dma_address(sg); 3704 3705 return ((((*dma_addr) & (PAGE_SIZE - 1)) + sg_dma_len(sg)) + 3706 (PAGE_SIZE - 1)) >> PAGE_SHIFT; 3707} 3708 3709/** 3710 * hl_mem_area_inside_range() - Checks whether address+size are inside a range. 3711 * @address: The start address of the area we want to validate. 3712 * @size: The size in bytes of the area we want to validate. 3713 * @range_start_address: The start address of the valid range. 3714 * @range_end_address: The end address of the valid range. 3715 * 3716 * Return: true if the area is inside the valid range, false otherwise. 3717 */ 3718static inline bool hl_mem_area_inside_range(u64 address, u64 size, 3719 u64 range_start_address, u64 range_end_address) 3720{ 3721 u64 end_address = address + size; 3722 3723 if ((address >= range_start_address) && 3724 (end_address <= range_end_address) && 3725 (end_address > address)) 3726 return true; 3727 3728 return false; 3729} 3730 3731static inline struct hl_device *to_hl_device(struct drm_device *ddev) 3732{ 3733 return container_of(ddev, struct hl_device, drm); 3734} 3735 3736/** 3737 * hl_mem_area_crosses_range() - Checks whether address+size crossing a range. 3738 * @address: The start address of the area we want to validate. 3739 * @size: The size in bytes of the area we want to validate. 3740 * @range_start_address: The start address of the valid range. 3741 * @range_end_address: The end address of the valid range. 3742 * 3743 * Return: true if the area overlaps part or all of the valid range, 3744 * false otherwise. 3745 */ 3746static inline bool hl_mem_area_crosses_range(u64 address, u32 size, 3747 u64 range_start_address, u64 range_end_address) 3748{ 3749 u64 end_address = address + size - 1; 3750 3751 return ((address <= range_end_address) && (range_start_address <= end_address)); 3752} 3753 3754uint64_t hl_set_dram_bar_default(struct hl_device *hdev, u64 addr); 3755void *hl_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle); 3756void hl_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr); 3757void *hl_asic_dma_alloc_coherent_caller(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle, 3758 gfp_t flag, const char *caller); 3759void hl_asic_dma_free_coherent_caller(struct hl_device *hdev, size_t size, void *cpu_addr, 3760 dma_addr_t dma_handle, const char *caller); 3761void *hl_asic_dma_pool_zalloc_caller(struct hl_device *hdev, size_t size, gfp_t mem_flags, 3762 dma_addr_t *dma_handle, const char *caller); 3763void hl_asic_dma_pool_free_caller(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr, 3764 const char *caller); 3765int hl_dma_map_sgtable_caller(struct hl_device *hdev, struct sg_table *sgt, 3766 enum dma_data_direction dir, const char *caller); 3767void hl_dma_unmap_sgtable_caller(struct hl_device *hdev, struct sg_table *sgt, 3768 enum dma_data_direction dir, const char *caller); 3769int hl_asic_dma_map_sgtable(struct hl_device *hdev, struct sg_table *sgt, 3770 enum dma_data_direction dir); 3771void hl_asic_dma_unmap_sgtable(struct hl_device *hdev, struct sg_table *sgt, 3772 enum dma_data_direction dir); 3773int hl_access_sram_dram_region(struct hl_device *hdev, u64 addr, u64 *val, 3774 enum debugfs_access_type acc_type, enum pci_region region_type, bool set_dram_bar); 3775int hl_access_cfg_region(struct hl_device *hdev, u64 addr, u64 *val, 3776 enum debugfs_access_type acc_type); 3777int hl_access_dev_mem(struct hl_device *hdev, enum pci_region region_type, 3778 u64 addr, u64 *val, enum debugfs_access_type acc_type); 3779 3780int hl_mmap(struct file *filp, struct vm_area_struct *vma); 3781 3782int hl_device_open(struct drm_device *drm, struct drm_file *file_priv); 3783void hl_device_release(struct drm_device *ddev, struct drm_file *file_priv); 3784 3785int hl_device_open_ctrl(struct inode *inode, struct file *filp); 3786bool hl_device_operational(struct hl_device *hdev, 3787 enum hl_device_status *status); 3788bool hl_ctrl_device_operational(struct hl_device *hdev, 3789 enum hl_device_status *status); 3790enum hl_device_status hl_device_status(struct hl_device *hdev); 3791int hl_device_set_debug_mode(struct hl_device *hdev, struct hl_ctx *ctx, bool enable); 3792int hl_hw_queues_create(struct hl_device *hdev); 3793void hl_hw_queues_destroy(struct hl_device *hdev); 3794int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id, 3795 u32 cb_size, u64 cb_ptr); 3796void hl_hw_queue_submit_bd(struct hl_device *hdev, struct hl_hw_queue *q, 3797 u32 ctl, u32 len, u64 ptr); 3798int hl_hw_queue_schedule_cs(struct hl_cs *cs); 3799u32 hl_hw_queue_add_ptr(u32 ptr, u16 val); 3800void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id); 3801void hl_hw_queue_update_ci(struct hl_cs *cs); 3802void hl_hw_queue_reset(struct hl_device *hdev, bool hard_reset); 3803 3804#define hl_queue_inc_ptr(p) hl_hw_queue_add_ptr(p, 1) 3805#define hl_pi_2_offset(pi) ((pi) & (HL_QUEUE_LENGTH - 1)) 3806 3807int hl_cq_init(struct hl_device *hdev, struct hl_cq *q, u32 hw_queue_id); 3808void hl_cq_fini(struct hl_device *hdev, struct hl_cq *q); 3809int hl_eq_init(struct hl_device *hdev, struct hl_eq *q); 3810void hl_eq_fini(struct hl_device *hdev, struct hl_eq *q); 3811void hl_cq_reset(struct hl_device *hdev, struct hl_cq *q); 3812void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q); 3813void hl_eq_dump(struct hl_device *hdev, struct hl_eq *q); 3814irqreturn_t hl_irq_handler_cq(int irq, void *arg); 3815irqreturn_t hl_irq_handler_eq(int irq, void *arg); 3816irqreturn_t hl_irq_handler_dec_abnrm(int irq, void *arg); 3817irqreturn_t hl_irq_user_interrupt_handler(int irq, void *arg); 3818irqreturn_t hl_irq_user_interrupt_thread_handler(int irq, void *arg); 3819irqreturn_t hl_irq_eq_error_interrupt_thread_handler(int irq, void *arg); 3820u32 hl_cq_inc_ptr(u32 ptr); 3821 3822int hl_asid_init(struct hl_device *hdev); 3823void hl_asid_fini(struct hl_device *hdev); 3824unsigned long hl_asid_alloc(struct hl_device *hdev); 3825void hl_asid_free(struct hl_device *hdev, unsigned long asid); 3826 3827int hl_ctx_create(struct hl_device *hdev, struct hl_fpriv *hpriv); 3828void hl_ctx_free(struct hl_device *hdev, struct hl_ctx *ctx); 3829int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx); 3830void hl_ctx_do_release(struct kref *ref); 3831void hl_ctx_get(struct hl_ctx *ctx); 3832int hl_ctx_put(struct hl_ctx *ctx); 3833struct hl_ctx *hl_get_compute_ctx(struct hl_device *hdev); 3834struct hl_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq); 3835int hl_ctx_get_fences(struct hl_ctx *ctx, u64 *seq_arr, 3836 struct hl_fence **fence, u32 arr_len); 3837void hl_ctx_mgr_init(struct hl_ctx_mgr *mgr); 3838void hl_ctx_mgr_fini(struct hl_device *hdev, struct hl_ctx_mgr *mgr); 3839 3840int hl_device_init(struct hl_device *hdev); 3841void hl_device_fini(struct hl_device *hdev); 3842int hl_device_suspend(struct hl_device *hdev); 3843int hl_device_resume(struct hl_device *hdev); 3844int hl_device_reset(struct hl_device *hdev, u32 flags); 3845int hl_device_cond_reset(struct hl_device *hdev, u32 flags, u64 event_mask); 3846void hl_hpriv_get(struct hl_fpriv *hpriv); 3847int hl_hpriv_put(struct hl_fpriv *hpriv); 3848int hl_device_utilization(struct hl_device *hdev, u32 *utilization); 3849 3850int hl_build_hwmon_channel_info(struct hl_device *hdev, 3851 struct cpucp_sensor *sensors_arr); 3852 3853void hl_notifier_event_send_all(struct hl_device *hdev, u64 event_mask); 3854 3855int hl_sysfs_init(struct hl_device *hdev); 3856void hl_sysfs_fini(struct hl_device *hdev); 3857 3858int hl_hwmon_init(struct hl_device *hdev); 3859void hl_hwmon_fini(struct hl_device *hdev); 3860void hl_hwmon_release_resources(struct hl_device *hdev); 3861 3862int hl_cb_create(struct hl_device *hdev, struct hl_mem_mgr *mmg, 3863 struct hl_ctx *ctx, u32 cb_size, bool internal_cb, 3864 bool map_cb, u64 *handle); 3865int hl_cb_destroy(struct hl_mem_mgr *mmg, u64 cb_handle); 3866int hl_hw_block_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma); 3867struct hl_cb *hl_cb_get(struct hl_mem_mgr *mmg, u64 handle); 3868void hl_cb_put(struct hl_cb *cb); 3869struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size, 3870 bool internal_cb); 3871int hl_cb_pool_init(struct hl_device *hdev); 3872int hl_cb_pool_fini(struct hl_device *hdev); 3873int hl_cb_va_pool_init(struct hl_ctx *ctx); 3874void hl_cb_va_pool_fini(struct hl_ctx *ctx); 3875 3876void hl_cs_rollback_all(struct hl_device *hdev, bool skip_wq_flush); 3877struct hl_cs_job *hl_cs_allocate_job(struct hl_device *hdev, 3878 enum hl_queue_type queue_type, bool is_kernel_allocated_cb); 3879void hl_sob_reset_error(struct kref *ref); 3880int hl_gen_sob_mask(u16 sob_base, u8 sob_mask, u8 *mask); 3881void hl_fence_put(struct hl_fence *fence); 3882void hl_fences_put(struct hl_fence **fence, int len); 3883void hl_fence_get(struct hl_fence *fence); 3884void cs_get(struct hl_cs *cs); 3885bool cs_needs_completion(struct hl_cs *cs); 3886bool cs_needs_timeout(struct hl_cs *cs); 3887bool is_staged_cs_last_exists(struct hl_device *hdev, struct hl_cs *cs); 3888struct hl_cs *hl_staged_cs_find_first(struct hl_device *hdev, u64 cs_seq); 3889void hl_multi_cs_completion_init(struct hl_device *hdev); 3890u32 hl_get_active_cs_num(struct hl_device *hdev); 3891 3892void goya_set_asic_funcs(struct hl_device *hdev); 3893void gaudi_set_asic_funcs(struct hl_device *hdev); 3894void gaudi2_set_asic_funcs(struct hl_device *hdev); 3895 3896int hl_vm_ctx_init(struct hl_ctx *ctx); 3897void hl_vm_ctx_fini(struct hl_ctx *ctx); 3898 3899int hl_vm_init(struct hl_device *hdev); 3900void hl_vm_fini(struct hl_device *hdev); 3901 3902void hl_hw_block_mem_init(struct hl_ctx *ctx); 3903void hl_hw_block_mem_fini(struct hl_ctx *ctx); 3904 3905u64 hl_reserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx, 3906 enum hl_va_range_type type, u64 size, u32 alignment); 3907int hl_unreserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx, 3908 u64 start_addr, u64 size); 3909int hl_pin_host_memory(struct hl_device *hdev, u64 addr, u64 size, 3910 struct hl_userptr *userptr); 3911void hl_unpin_host_memory(struct hl_device *hdev, struct hl_userptr *userptr); 3912void hl_userptr_delete_list(struct hl_device *hdev, 3913 struct list_head *userptr_list); 3914bool hl_userptr_is_pinned(struct hl_device *hdev, u64 addr, u32 size, 3915 struct list_head *userptr_list, 3916 struct hl_userptr **userptr); 3917 3918int hl_mmu_init(struct hl_device *hdev); 3919void hl_mmu_fini(struct hl_device *hdev); 3920int hl_mmu_ctx_init(struct hl_ctx *ctx); 3921void hl_mmu_ctx_fini(struct hl_ctx *ctx); 3922int hl_mmu_map_page(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr, 3923 u32 page_size, bool flush_pte); 3924int hl_mmu_get_real_page_size(struct hl_device *hdev, struct hl_mmu_properties *mmu_prop, 3925 u32 page_size, u32 *real_page_size, bool is_dram_addr); 3926int hl_mmu_unmap_page(struct hl_ctx *ctx, u64 virt_addr, u32 page_size, 3927 bool flush_pte); 3928int hl_mmu_map_contiguous(struct hl_ctx *ctx, u64 virt_addr, 3929 u64 phys_addr, u32 size); 3930int hl_mmu_unmap_contiguous(struct hl_ctx *ctx, u64 virt_addr, u32 size); 3931int hl_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, u32 flags); 3932int hl_mmu_invalidate_cache_range(struct hl_device *hdev, bool is_hard, 3933 u32 flags, u32 asid, u64 va, u64 size); 3934int hl_mmu_prefetch_cache_range(struct hl_ctx *ctx, u32 flags, u32 asid, u64 va, u64 size); 3935u64 hl_mmu_get_next_hop_addr(struct hl_ctx *ctx, u64 curr_pte); 3936u64 hl_mmu_get_hop_pte_phys_addr(struct hl_ctx *ctx, struct hl_mmu_properties *mmu_prop, 3937 u8 hop_idx, u64 hop_addr, u64 virt_addr); 3938void hl_mmu_hr_flush(struct hl_ctx *ctx); 3939int hl_mmu_hr_init(struct hl_device *hdev, struct hl_mmu_hr_priv *hr_priv, u32 hop_table_size, 3940 u64 pgt_size); 3941void hl_mmu_hr_fini(struct hl_device *hdev, struct hl_mmu_hr_priv *hr_priv, u32 hop_table_size); 3942void hl_mmu_hr_free_hop_remove_pgt(struct pgt_info *pgt_info, struct hl_mmu_hr_priv *hr_priv, 3943 u32 hop_table_size); 3944u64 hl_mmu_hr_pte_phys_to_virt(struct hl_ctx *ctx, struct pgt_info *pgt, u64 phys_pte_addr, 3945 u32 hop_table_size); 3946void hl_mmu_hr_write_pte(struct hl_ctx *ctx, struct pgt_info *pgt_info, u64 phys_pte_addr, 3947 u64 val, u32 hop_table_size); 3948void hl_mmu_hr_clear_pte(struct hl_ctx *ctx, struct pgt_info *pgt_info, u64 phys_pte_addr, 3949 u32 hop_table_size); 3950int hl_mmu_hr_put_pte(struct hl_ctx *ctx, struct pgt_info *pgt_info, struct hl_mmu_hr_priv *hr_priv, 3951 u32 hop_table_size); 3952void hl_mmu_hr_get_pte(struct hl_ctx *ctx, struct hl_hr_mmu_funcs *hr_func, u64 phys_hop_addr); 3953struct pgt_info *hl_mmu_hr_get_next_hop_pgt_info(struct hl_ctx *ctx, 3954 struct hl_hr_mmu_funcs *hr_func, 3955 u64 curr_pte); 3956struct pgt_info *hl_mmu_hr_alloc_hop(struct hl_ctx *ctx, struct hl_mmu_hr_priv *hr_priv, 3957 struct hl_hr_mmu_funcs *hr_func, 3958 struct hl_mmu_properties *mmu_prop); 3959struct pgt_info *hl_mmu_hr_get_alloc_next_hop(struct hl_ctx *ctx, 3960 struct hl_mmu_hr_priv *hr_priv, 3961 struct hl_hr_mmu_funcs *hr_func, 3962 struct hl_mmu_properties *mmu_prop, 3963 u64 curr_pte, bool *is_new_hop); 3964int hl_mmu_hr_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr, struct hl_mmu_hop_info *hops, 3965 struct hl_hr_mmu_funcs *hr_func); 3966int hl_mmu_if_set_funcs(struct hl_device *hdev); 3967void hl_mmu_v1_set_funcs(struct hl_device *hdev, struct hl_mmu_funcs *mmu); 3968void hl_mmu_v2_set_funcs(struct hl_device *hdev, struct hl_mmu_funcs *mmu); 3969void hl_mmu_v2_hr_set_funcs(struct hl_device *hdev, struct hl_mmu_funcs *mmu); 3970int hl_mmu_va_to_pa(struct hl_ctx *ctx, u64 virt_addr, u64 *phys_addr); 3971int hl_mmu_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr, 3972 struct hl_mmu_hop_info *hops); 3973u64 hl_mmu_scramble_addr(struct hl_device *hdev, u64 addr); 3974u64 hl_mmu_descramble_addr(struct hl_device *hdev, u64 addr); 3975bool hl_is_dram_va(struct hl_device *hdev, u64 virt_addr); 3976struct pgt_info *hl_mmu_dr_get_pgt_info(struct hl_ctx *ctx, u64 hop_addr); 3977void hl_mmu_dr_free_hop(struct hl_ctx *ctx, u64 hop_addr); 3978void hl_mmu_dr_free_pgt_node(struct hl_ctx *ctx, struct pgt_info *pgt_info); 3979u64 hl_mmu_dr_get_phys_hop0_addr(struct hl_ctx *ctx); 3980u64 hl_mmu_dr_get_hop0_addr(struct hl_ctx *ctx); 3981void hl_mmu_dr_write_pte(struct hl_ctx *ctx, u64 shadow_pte_addr, u64 val); 3982void hl_mmu_dr_write_final_pte(struct hl_ctx *ctx, u64 shadow_pte_addr, u64 val); 3983void hl_mmu_dr_clear_pte(struct hl_ctx *ctx, u64 pte_addr); 3984u64 hl_mmu_dr_get_phys_addr(struct hl_ctx *ctx, u64 shadow_addr); 3985void hl_mmu_dr_get_pte(struct hl_ctx *ctx, u64 hop_addr); 3986int hl_mmu_dr_put_pte(struct hl_ctx *ctx, u64 hop_addr); 3987u64 hl_mmu_dr_get_alloc_next_hop_addr(struct hl_ctx *ctx, u64 curr_pte, bool *is_new_hop); 3988u64 hl_mmu_dr_alloc_hop(struct hl_ctx *ctx); 3989void hl_mmu_dr_flush(struct hl_ctx *ctx); 3990int hl_mmu_dr_init(struct hl_device *hdev); 3991void hl_mmu_dr_fini(struct hl_device *hdev); 3992 3993int hl_fw_version_cmp(struct hl_device *hdev, u32 major, u32 minor, u32 subminor); 3994int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name, 3995 void __iomem *dst, u32 src_offset, u32 size); 3996int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode, u64 value); 3997int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg, 3998 u16 len, u32 timeout, u64 *result); 3999int hl_fw_unmask_irq(struct hl_device *hdev, u16 event_type); 4000int hl_fw_unmask_irq_arr(struct hl_device *hdev, const u32 *irq_arr, 4001 size_t irq_arr_size); 4002int hl_fw_test_cpu_queue(struct hl_device *hdev); 4003void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, 4004 dma_addr_t *dma_handle); 4005void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, 4006 void *vaddr); 4007int hl_fw_send_heartbeat(struct hl_device *hdev); 4008int hl_fw_cpucp_info_get(struct hl_device *hdev, 4009 u32 sts_boot_dev_sts0_reg, 4010 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg, 4011 u32 boot_err1_reg); 4012int hl_fw_cpucp_handshake(struct hl_device *hdev, 4013 u32 sts_boot_dev_sts0_reg, 4014 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg, 4015 u32 boot_err1_reg); 4016int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size); 4017int hl_fw_get_monitor_dump(struct hl_device *hdev, void *data); 4018int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev, 4019 struct hl_info_pci_counters *counters); 4020int hl_fw_cpucp_total_energy_get(struct hl_device *hdev, 4021 u64 *total_energy); 4022int get_used_pll_index(struct hl_device *hdev, u32 input_pll_index, 4023 enum pll_index *pll_index); 4024int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u32 pll_index, 4025 u16 *pll_freq_arr); 4026int hl_fw_cpucp_power_get(struct hl_device *hdev, u64 *power); 4027void hl_fw_ask_hard_reset_without_linux(struct hl_device *hdev); 4028void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev); 4029int hl_fw_init_cpu(struct hl_device *hdev); 4030int hl_fw_wait_preboot_ready(struct hl_device *hdev); 4031int hl_fw_read_preboot_status(struct hl_device *hdev); 4032int hl_fw_dynamic_send_protocol_cmd(struct hl_device *hdev, 4033 struct fw_load_mgr *fw_loader, 4034 enum comms_cmd cmd, unsigned int size, 4035 bool wait_ok, u32 timeout); 4036int hl_fw_dram_replaced_row_get(struct hl_device *hdev, 4037 struct cpucp_hbm_row_info *info); 4038int hl_fw_dram_pending_row_get(struct hl_device *hdev, u32 *pend_rows_num); 4039int hl_fw_cpucp_engine_core_asid_set(struct hl_device *hdev, u32 asid); 4040int hl_fw_send_device_activity(struct hl_device *hdev, bool open); 4041int hl_fw_send_soft_reset(struct hl_device *hdev); 4042int hl_pci_bars_map(struct hl_device *hdev, const char * const name[3], 4043 bool is_wc[3]); 4044int hl_pci_elbi_read(struct hl_device *hdev, u64 addr, u32 *data); 4045int hl_pci_iatu_write(struct hl_device *hdev, u32 addr, u32 data); 4046int hl_pci_set_inbound_region(struct hl_device *hdev, u8 region, 4047 struct hl_inbound_pci_region *pci_region); 4048int hl_pci_set_outbound_region(struct hl_device *hdev, 4049 struct hl_outbound_pci_region *pci_region); 4050enum pci_region hl_get_pci_memory_region(struct hl_device *hdev, u64 addr); 4051int hl_pci_init(struct hl_device *hdev); 4052void hl_pci_fini(struct hl_device *hdev); 4053 4054long hl_fw_get_frequency(struct hl_device *hdev, u32 pll_index, bool curr); 4055void hl_fw_set_frequency(struct hl_device *hdev, u32 pll_index, u64 freq); 4056int hl_get_temperature(struct hl_device *hdev, int sensor_index, u32 attr, long *value); 4057int hl_set_temperature(struct hl_device *hdev, int sensor_index, u32 attr, long value); 4058int hl_get_voltage(struct hl_device *hdev, int sensor_index, u32 attr, long *value); 4059int hl_get_current(struct hl_device *hdev, int sensor_index, u32 attr, long *value); 4060int hl_get_fan_speed(struct hl_device *hdev, int sensor_index, u32 attr, long *value); 4061int hl_get_pwm_info(struct hl_device *hdev, int sensor_index, u32 attr, long *value); 4062void hl_set_pwm_info(struct hl_device *hdev, int sensor_index, u32 attr, long value); 4063long hl_fw_get_max_power(struct hl_device *hdev); 4064void hl_fw_set_max_power(struct hl_device *hdev); 4065int hl_fw_get_sec_attest_info(struct hl_device *hdev, struct cpucp_sec_attest_info *sec_attest_info, 4066 u32 nonce); 4067int hl_fw_get_dev_info_signed(struct hl_device *hdev, 4068 struct cpucp_dev_info_signed *dev_info_signed, u32 nonce); 4069int hl_set_voltage(struct hl_device *hdev, int sensor_index, u32 attr, long value); 4070int hl_set_current(struct hl_device *hdev, int sensor_index, u32 attr, long value); 4071int hl_set_power(struct hl_device *hdev, int sensor_index, u32 attr, long value); 4072int hl_get_power(struct hl_device *hdev, int sensor_index, u32 attr, long *value); 4073int hl_fw_get_clk_rate(struct hl_device *hdev, u32 *cur_clk, u32 *max_clk); 4074void hl_fw_set_pll_profile(struct hl_device *hdev); 4075void hl_sysfs_add_dev_clk_attr(struct hl_device *hdev, struct attribute_group *dev_clk_attr_grp); 4076void hl_sysfs_add_dev_vrm_attr(struct hl_device *hdev, struct attribute_group *dev_vrm_attr_grp); 4077int hl_fw_send_generic_request(struct hl_device *hdev, enum hl_passthrough_type sub_opcode, 4078 dma_addr_t buff, u32 *size); 4079 4080void hw_sob_get(struct hl_hw_sob *hw_sob); 4081void hw_sob_put(struct hl_hw_sob *hw_sob); 4082void hl_encaps_release_handle_and_put_ctx(struct kref *ref); 4083void hl_encaps_release_handle_and_put_sob_ctx(struct kref *ref); 4084void hl_hw_queue_encaps_sig_set_sob_info(struct hl_device *hdev, 4085 struct hl_cs *cs, struct hl_cs_job *job, 4086 struct hl_cs_compl *cs_cmpl); 4087 4088int hl_dec_init(struct hl_device *hdev); 4089void hl_dec_fini(struct hl_device *hdev); 4090void hl_dec_ctx_fini(struct hl_ctx *ctx); 4091 4092void hl_release_pending_user_interrupts(struct hl_device *hdev); 4093void hl_abort_waiting_for_cs_completions(struct hl_device *hdev); 4094int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx, 4095 struct hl_hw_sob **hw_sob, u32 count, bool encaps_sig); 4096 4097int hl_state_dump(struct hl_device *hdev); 4098const char *hl_state_dump_get_sync_name(struct hl_device *hdev, u32 sync_id); 4099const char *hl_state_dump_get_monitor_name(struct hl_device *hdev, 4100 struct hl_mon_state_dump *mon); 4101void hl_state_dump_free_sync_to_engine_map(struct hl_sync_to_engine_map *map); 4102__printf(4, 5) int hl_snprintf_resize(char **buf, size_t *size, size_t *offset, 4103 const char *format, ...); 4104char *hl_format_as_binary(char *buf, size_t buf_len, u32 n); 4105const char *hl_sync_engine_to_string(enum hl_sync_engine_type engine_type); 4106 4107void hl_mem_mgr_init(struct device *dev, struct hl_mem_mgr *mmg); 4108void hl_mem_mgr_fini(struct hl_mem_mgr *mmg, struct hl_mem_mgr_fini_stats *stats); 4109void hl_mem_mgr_idr_destroy(struct hl_mem_mgr *mmg); 4110int hl_mem_mgr_mmap(struct hl_mem_mgr *mmg, struct vm_area_struct *vma, 4111 void *args); 4112struct hl_mmap_mem_buf *hl_mmap_mem_buf_get(struct hl_mem_mgr *mmg, 4113 u64 handle); 4114int hl_mmap_mem_buf_put_handle(struct hl_mem_mgr *mmg, u64 handle); 4115int hl_mmap_mem_buf_put(struct hl_mmap_mem_buf *buf); 4116struct hl_mmap_mem_buf * 4117hl_mmap_mem_buf_alloc(struct hl_mem_mgr *mmg, 4118 struct hl_mmap_mem_buf_behavior *behavior, gfp_t gfp, 4119 void *args); 4120__printf(2, 3) void hl_engine_data_sprintf(struct engines_data *e, const char *fmt, ...); 4121void hl_capture_razwi(struct hl_device *hdev, u64 addr, u16 *engine_id, u16 num_of_engines, 4122 u8 flags); 4123void hl_handle_razwi(struct hl_device *hdev, u64 addr, u16 *engine_id, u16 num_of_engines, 4124 u8 flags, u64 *event_mask); 4125void hl_capture_page_fault(struct hl_device *hdev, u64 addr, u16 eng_id, bool is_pmmu); 4126void hl_handle_page_fault(struct hl_device *hdev, u64 addr, u16 eng_id, bool is_pmmu, 4127 u64 *event_mask); 4128void hl_handle_critical_hw_err(struct hl_device *hdev, u16 event_id, u64 *event_mask); 4129void hl_handle_fw_err(struct hl_device *hdev, struct hl_info_fw_err_info *info); 4130void hl_capture_engine_err(struct hl_device *hdev, u16 engine_id, u16 error_count); 4131void hl_enable_err_info_capture(struct hl_error_info *captured_err_info); 4132void hl_init_cpu_for_irq(struct hl_device *hdev); 4133void hl_set_irq_affinity(struct hl_device *hdev, int irq); 4134void hl_eq_heartbeat_event_handle(struct hl_device *hdev); 4135void hl_handle_clk_change_event(struct hl_device *hdev, u16 event_type, u64 *event_mask); 4136void hl_eq_cpld_shutdown_event_handle(struct hl_device *hdev, u16 event_id, u64 *event_mask); 4137 4138#ifdef CONFIG_DEBUG_FS 4139 4140int hl_debugfs_device_init(struct hl_device *hdev); 4141void hl_debugfs_device_fini(struct hl_device *hdev); 4142void hl_debugfs_add_device(struct hl_device *hdev); 4143void hl_debugfs_add_file(struct hl_fpriv *hpriv); 4144void hl_debugfs_remove_file(struct hl_fpriv *hpriv); 4145void hl_debugfs_add_cb(struct hl_cb *cb); 4146void hl_debugfs_remove_cb(struct hl_cb *cb); 4147void hl_debugfs_add_cs(struct hl_cs *cs); 4148void hl_debugfs_remove_cs(struct hl_cs *cs); 4149void hl_debugfs_add_job(struct hl_device *hdev, struct hl_cs_job *job); 4150void hl_debugfs_remove_job(struct hl_device *hdev, struct hl_cs_job *job); 4151void hl_debugfs_add_userptr(struct hl_device *hdev, struct hl_userptr *userptr); 4152void hl_debugfs_remove_userptr(struct hl_device *hdev, 4153 struct hl_userptr *userptr); 4154void hl_debugfs_add_ctx_mem_hash(struct hl_device *hdev, struct hl_ctx *ctx); 4155void hl_debugfs_remove_ctx_mem_hash(struct hl_device *hdev, struct hl_ctx *ctx); 4156void hl_debugfs_set_state_dump(struct hl_device *hdev, char *data, 4157 unsigned long length); 4158void hl_debugfs_cfg_access_history_dump(struct hl_device *hdev); 4159 4160#else 4161 4162static inline int hl_debugfs_device_init(struct hl_device *hdev) 4163{ 4164 return 0; 4165} 4166 4167static inline void hl_debugfs_device_fini(struct hl_device *hdev) 4168{ 4169} 4170 4171static inline void hl_debugfs_add_device(struct hl_device *hdev) 4172{ 4173} 4174 4175static inline void hl_debugfs_add_file(struct hl_fpriv *hpriv) 4176{ 4177} 4178 4179static inline void hl_debugfs_remove_file(struct hl_fpriv *hpriv) 4180{ 4181} 4182 4183static inline void hl_debugfs_add_cb(struct hl_cb *cb) 4184{ 4185} 4186 4187static inline void hl_debugfs_remove_cb(struct hl_cb *cb) 4188{ 4189} 4190 4191static inline void hl_debugfs_add_cs(struct hl_cs *cs) 4192{ 4193} 4194 4195static inline void hl_debugfs_remove_cs(struct hl_cs *cs) 4196{ 4197} 4198 4199static inline void hl_debugfs_add_job(struct hl_device *hdev, 4200 struct hl_cs_job *job) 4201{ 4202} 4203 4204static inline void hl_debugfs_remove_job(struct hl_device *hdev, 4205 struct hl_cs_job *job) 4206{ 4207} 4208 4209static inline void hl_debugfs_add_userptr(struct hl_device *hdev, 4210 struct hl_userptr *userptr) 4211{ 4212} 4213 4214static inline void hl_debugfs_remove_userptr(struct hl_device *hdev, 4215 struct hl_userptr *userptr) 4216{ 4217} 4218 4219static inline void hl_debugfs_add_ctx_mem_hash(struct hl_device *hdev, 4220 struct hl_ctx *ctx) 4221{ 4222} 4223 4224static inline void hl_debugfs_remove_ctx_mem_hash(struct hl_device *hdev, 4225 struct hl_ctx *ctx) 4226{ 4227} 4228 4229static inline void hl_debugfs_set_state_dump(struct hl_device *hdev, 4230 char *data, unsigned long length) 4231{ 4232} 4233 4234static inline void hl_debugfs_cfg_access_history_dump(struct hl_device *hdev) 4235{ 4236} 4237 4238#endif 4239 4240/* Security */ 4241int hl_unsecure_register(struct hl_device *hdev, u32 mm_reg_addr, int offset, 4242 const u32 pb_blocks[], struct hl_block_glbl_sec sgs_array[], 4243 int array_size); 4244int hl_unsecure_registers(struct hl_device *hdev, const u32 mm_reg_array[], 4245 int mm_array_size, int offset, const u32 pb_blocks[], 4246 struct hl_block_glbl_sec sgs_array[], int blocks_array_size); 4247void hl_config_glbl_sec(struct hl_device *hdev, const u32 pb_blocks[], 4248 struct hl_block_glbl_sec sgs_array[], u32 block_offset, 4249 int array_size); 4250void hl_secure_block(struct hl_device *hdev, 4251 struct hl_block_glbl_sec sgs_array[], int array_size); 4252int hl_init_pb_with_mask(struct hl_device *hdev, u32 num_dcores, 4253 u32 dcore_offset, u32 num_instances, u32 instance_offset, 4254 const u32 pb_blocks[], u32 blocks_array_size, 4255 const u32 *regs_array, u32 regs_array_size, u64 mask); 4256int hl_init_pb(struct hl_device *hdev, u32 num_dcores, u32 dcore_offset, 4257 u32 num_instances, u32 instance_offset, 4258 const u32 pb_blocks[], u32 blocks_array_size, 4259 const u32 *regs_array, u32 regs_array_size); 4260int hl_init_pb_ranges_with_mask(struct hl_device *hdev, u32 num_dcores, 4261 u32 dcore_offset, u32 num_instances, u32 instance_offset, 4262 const u32 pb_blocks[], u32 blocks_array_size, 4263 const struct range *regs_range_array, u32 regs_range_array_size, 4264 u64 mask); 4265int hl_init_pb_ranges(struct hl_device *hdev, u32 num_dcores, 4266 u32 dcore_offset, u32 num_instances, u32 instance_offset, 4267 const u32 pb_blocks[], u32 blocks_array_size, 4268 const struct range *regs_range_array, 4269 u32 regs_range_array_size); 4270int hl_init_pb_single_dcore(struct hl_device *hdev, u32 dcore_offset, 4271 u32 num_instances, u32 instance_offset, 4272 const u32 pb_blocks[], u32 blocks_array_size, 4273 const u32 *regs_array, u32 regs_array_size); 4274int hl_init_pb_ranges_single_dcore(struct hl_device *hdev, u32 dcore_offset, 4275 u32 num_instances, u32 instance_offset, 4276 const u32 pb_blocks[], u32 blocks_array_size, 4277 const struct range *regs_range_array, 4278 u32 regs_range_array_size); 4279void hl_ack_pb(struct hl_device *hdev, u32 num_dcores, u32 dcore_offset, 4280 u32 num_instances, u32 instance_offset, 4281 const u32 pb_blocks[], u32 blocks_array_size); 4282void hl_ack_pb_with_mask(struct hl_device *hdev, u32 num_dcores, 4283 u32 dcore_offset, u32 num_instances, u32 instance_offset, 4284 const u32 pb_blocks[], u32 blocks_array_size, u64 mask); 4285void hl_ack_pb_single_dcore(struct hl_device *hdev, u32 dcore_offset, 4286 u32 num_instances, u32 instance_offset, 4287 const u32 pb_blocks[], u32 blocks_array_size); 4288 4289/* IOCTLs */ 4290long hl_ioctl_control(struct file *filep, unsigned int cmd, unsigned long arg); 4291int hl_info_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv); 4292int hl_cb_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv); 4293int hl_cs_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv); 4294int hl_wait_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv); 4295int hl_mem_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv); 4296int hl_debug_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv); 4297 4298#endif /* HABANALABSP_H_ */
~/
Read-only file preview.