memory.c

   1// SPDX-License-Identifier: GPL-2.0
   2
   3/*
   4 * Copyright 2016-2022 HabanaLabs, Ltd.
   5 * All Rights Reserved.
   6 */
   7
   8#include <uapi/drm/habanalabs_accel.h>
   9#include "habanalabs.h"
  10#include "../include/hw_ip/mmu/mmu_general.h"
  11
  12#include <linux/uaccess.h>
  13#include <linux/slab.h>
  14#include <linux/vmalloc.h>
  15#include <linux/pci-p2pdma.h>
  16
  17MODULE_IMPORT_NS("DMA_BUF");
  18
  19#define HL_MMU_DEBUG	0
  20
  21/* use small pages for supporting non-pow2 (32M/40M/48M) DRAM phys page sizes */
  22#define DRAM_POOL_PAGE_SIZE	SZ_8M
  23
  24#define MEM_HANDLE_INVALID	ULONG_MAX
  25
  26static int allocate_timestamps_buffers(struct hl_fpriv *hpriv,
  27			struct hl_mem_in *args, u64 *handle);
  28
  29static int set_alloc_page_size(struct hl_device *hdev, struct hl_mem_in *args, u32 *page_size)
  30{
  31	struct asic_fixed_properties *prop = &hdev->asic_prop;
  32	u64 psize;
  33
  34	/*
  35	 * for ASIC that supports setting the allocation page size by user we will address
  36	 * user's choice only if it is not 0 (as 0 means taking the default page size)
  37	 */
  38	if (prop->supports_user_set_page_size && args->alloc.page_size) {
  39		psize = args->alloc.page_size;
  40
  41		if (!is_power_of_2(psize)) {
  42			dev_err(hdev->dev, "user page size (%#llx) is not power of 2\n", psize);
  43			return -EINVAL;
  44		}
  45	} else {
  46		psize = prop->device_mem_alloc_default_page_size;
  47	}
  48
  49	*page_size = psize;
  50
  51	return 0;
  52}
  53
  54/*
  55 * The va ranges in context object contain a list with the available chunks of
  56 * device virtual memory.
  57 * There is one range for host allocations and one for DRAM allocations.
  58 *
  59 * On initialization each range contains one chunk of all of its available
  60 * virtual range which is a half of the total device virtual range.
  61 *
  62 * On each mapping of physical pages, a suitable virtual range chunk (with a
  63 * minimum size) is selected from the list. If the chunk size equals the
  64 * requested size, the chunk is returned. Otherwise, the chunk is split into
  65 * two chunks - one to return as result and a remainder to stay in the list.
  66 *
  67 * On each Unmapping of a virtual address, the relevant virtual chunk is
  68 * returned to the list. The chunk is added to the list and if its edges match
  69 * the edges of the adjacent chunks (means a contiguous chunk can be created),
  70 * the chunks are merged.
  71 *
  72 * On finish, the list is checked to have only one chunk of all the relevant
  73 * virtual range (which is a half of the device total virtual range).
  74 * If not (means not all mappings were unmapped), a warning is printed.
  75 */
  76
  77/*
  78 * alloc_device_memory() - allocate device memory.
  79 * @ctx: pointer to the context structure.
  80 * @args: host parameters containing the requested size.
  81 * @ret_handle: result handle.
  82 *
  83 * This function does the following:
  84 * - Allocate the requested size rounded up to 'dram_page_size' pages.
  85 * - Return unique handle for later map/unmap/free.
  86 */
  87static int alloc_device_memory(struct hl_ctx *ctx, struct hl_mem_in *args,
  88				u32 *ret_handle)
  89{
  90	struct hl_device *hdev = ctx->hdev;
  91	struct hl_vm *vm = &hdev->vm;
  92	struct hl_vm_phys_pg_pack *phys_pg_pack;
  93	u64 paddr = 0, total_size, num_pgs, i;
  94	u32 num_curr_pgs, page_size;
  95	bool contiguous;
  96	int handle, rc;
  97
  98	num_curr_pgs = 0;
  99
 100	rc = set_alloc_page_size(hdev, args, &page_size);
 101	if (rc)
 102		return rc;
 103
 104	num_pgs = DIV_ROUND_UP_ULL(args->alloc.mem_size, page_size);
 105	total_size = num_pgs * page_size;
 106
 107	if (!total_size) {
 108		dev_err(hdev->dev, "Cannot allocate 0 bytes\n");
 109		return -EINVAL;
 110	}
 111
 112	contiguous = args->flags & HL_MEM_CONTIGUOUS;
 113
 114	if (contiguous) {
 115		if (is_power_of_2(page_size))
 116			paddr = (uintptr_t) gen_pool_dma_alloc_align(vm->dram_pg_pool,
 117								     total_size, NULL, page_size);
 118		else
 119			paddr = gen_pool_alloc(vm->dram_pg_pool, total_size);
 120		if (!paddr) {
 121			dev_err(hdev->dev,
 122				"Cannot allocate %llu contiguous pages with total size of %llu\n",
 123				num_pgs, total_size);
 124			return -ENOMEM;
 125		}
 126	}
 127
 128	phys_pg_pack = kzalloc(sizeof(*phys_pg_pack), GFP_KERNEL);
 129	if (!phys_pg_pack) {
 130		rc = -ENOMEM;
 131		goto pages_pack_err;
 132	}
 133
 134	phys_pg_pack->vm_type = VM_TYPE_PHYS_PACK;
 135	phys_pg_pack->asid = ctx->asid;
 136	phys_pg_pack->npages = num_pgs;
 137	phys_pg_pack->page_size = page_size;
 138	phys_pg_pack->total_size = total_size;
 139	phys_pg_pack->flags = args->flags;
 140	phys_pg_pack->contiguous = contiguous;
 141
 142	phys_pg_pack->pages = kvmalloc_array(num_pgs, sizeof(u64), GFP_KERNEL);
 143	if (ZERO_OR_NULL_PTR(phys_pg_pack->pages)) {
 144		rc = -ENOMEM;
 145		goto pages_arr_err;
 146	}
 147
 148	if (phys_pg_pack->contiguous) {
 149		for (i = 0 ; i < num_pgs ; i++)
 150			phys_pg_pack->pages[i] = paddr + i * page_size;
 151	} else {
 152		for (i = 0 ; i < num_pgs ; i++) {
 153			if (is_power_of_2(page_size))
 154				phys_pg_pack->pages[i] =
 155					(uintptr_t)gen_pool_dma_alloc_align(vm->dram_pg_pool,
 156									    page_size, NULL,
 157									    page_size);
 158			else
 159				phys_pg_pack->pages[i] = gen_pool_alloc(vm->dram_pg_pool,
 160									page_size);
 161
 162			if (!phys_pg_pack->pages[i]) {
 163				dev_err(hdev->dev,
 164					"Cannot allocate device memory (out of memory)\n");
 165				rc = -ENOMEM;
 166				goto page_err;
 167			}
 168
 169			num_curr_pgs++;
 170		}
 171	}
 172
 173	spin_lock(&vm->idr_lock);
 174	handle = idr_alloc(&vm->phys_pg_pack_handles, phys_pg_pack, 1, 0,
 175				GFP_ATOMIC);
 176	spin_unlock(&vm->idr_lock);
 177
 178	if (handle < 0) {
 179		dev_err(hdev->dev, "Failed to get handle for page\n");
 180		rc = -EFAULT;
 181		goto idr_err;
 182	}
 183
 184	for (i = 0 ; i < num_pgs ; i++)
 185		kref_get(&vm->dram_pg_pool_refcount);
 186
 187	phys_pg_pack->handle = handle;
 188
 189	atomic64_add(phys_pg_pack->total_size, &ctx->dram_phys_mem);
 190	atomic64_add(phys_pg_pack->total_size, &hdev->dram_used_mem);
 191
 192	*ret_handle = handle;
 193
 194	return 0;
 195
 196idr_err:
 197page_err:
 198	if (!phys_pg_pack->contiguous)
 199		for (i = 0 ; i < num_curr_pgs ; i++)
 200			gen_pool_free(vm->dram_pg_pool, phys_pg_pack->pages[i],
 201					page_size);
 202
 203	kvfree(phys_pg_pack->pages);
 204pages_arr_err:
 205	kfree(phys_pg_pack);
 206pages_pack_err:
 207	if (contiguous)
 208		gen_pool_free(vm->dram_pg_pool, paddr, total_size);
 209
 210	return rc;
 211}
 212
 213/**
 214 * dma_map_host_va() - DMA mapping of the given host virtual address.
 215 * @hdev: habanalabs device structure.
 216 * @addr: the host virtual address of the memory area.
 217 * @size: the size of the memory area.
 218 * @p_userptr: pointer to result userptr structure.
 219 *
 220 * This function does the following:
 221 * - Allocate userptr structure.
 222 * - Pin the given host memory using the userptr structure.
 223 * - Perform DMA mapping to have the DMA addresses of the pages.
 224 */
 225static int dma_map_host_va(struct hl_device *hdev, u64 addr, u64 size,
 226				struct hl_userptr **p_userptr)
 227{
 228	struct hl_userptr *userptr;
 229	int rc;
 230
 231	userptr = kzalloc(sizeof(*userptr), GFP_KERNEL);
 232	if (!userptr) {
 233		rc = -ENOMEM;
 234		goto userptr_err;
 235	}
 236
 237	rc = hl_pin_host_memory(hdev, addr, size, userptr);
 238	if (rc)
 239		goto pin_err;
 240
 241	userptr->dma_mapped = true;
 242	userptr->dir = DMA_BIDIRECTIONAL;
 243	userptr->vm_type = VM_TYPE_USERPTR;
 244
 245	*p_userptr = userptr;
 246
 247	rc = hl_dma_map_sgtable(hdev, userptr->sgt, DMA_BIDIRECTIONAL);
 248	if (rc) {
 249		dev_err(hdev->dev, "failed to map sgt with DMA region\n");
 250		goto dma_map_err;
 251	}
 252
 253	return 0;
 254
 255dma_map_err:
 256	hl_unpin_host_memory(hdev, userptr);
 257pin_err:
 258	kfree(userptr);
 259userptr_err:
 260
 261	return rc;
 262}
 263
 264/**
 265 * dma_unmap_host_va() - DMA unmapping of the given host virtual address.
 266 * @hdev: habanalabs device structure.
 267 * @userptr: userptr to free.
 268 *
 269 * This function does the following:
 270 * - Unpins the physical pages.
 271 * - Frees the userptr structure.
 272 */
 273static void dma_unmap_host_va(struct hl_device *hdev,
 274				struct hl_userptr *userptr)
 275{
 276	hl_unpin_host_memory(hdev, userptr);
 277	kfree(userptr);
 278}
 279
 280/**
 281 * dram_pg_pool_do_release() - free DRAM pages pool
 282 * @ref: pointer to reference object.
 283 *
 284 * This function does the following:
 285 * - Frees the idr structure of physical pages handles.
 286 * - Frees the generic pool of DRAM physical pages.
 287 */
 288static void dram_pg_pool_do_release(struct kref *ref)
 289{
 290	struct hl_vm *vm = container_of(ref, struct hl_vm,
 291			dram_pg_pool_refcount);
 292
 293	/*
 294	 * free the idr here as only here we know for sure that there are no
 295	 * allocated physical pages and hence there are no handles in use
 296	 */
 297	idr_destroy(&vm->phys_pg_pack_handles);
 298	gen_pool_destroy(vm->dram_pg_pool);
 299}
 300
 301/**
 302 * free_phys_pg_pack() - free physical page pack.
 303 * @hdev: habanalabs device structure.
 304 * @phys_pg_pack: physical page pack to free.
 305 *
 306 * This function does the following:
 307 * - For DRAM memory only
 308 *   - iterate over the pack, free each physical block structure by
 309 *     returning it to the general pool.
 310 * - Free the hl_vm_phys_pg_pack structure.
 311 */
 312static void free_phys_pg_pack(struct hl_device *hdev,
 313				struct hl_vm_phys_pg_pack *phys_pg_pack)
 314{
 315	struct hl_vm *vm = &hdev->vm;
 316	u64 i;
 317
 318	if (phys_pg_pack->created_from_userptr)
 319		goto end;
 320
 321	if (phys_pg_pack->contiguous) {
 322		gen_pool_free(vm->dram_pg_pool, phys_pg_pack->pages[0],
 323			phys_pg_pack->total_size);
 324
 325		for (i = 0; i < phys_pg_pack->npages ; i++)
 326			kref_put(&vm->dram_pg_pool_refcount,
 327				dram_pg_pool_do_release);
 328	} else {
 329		for (i = 0 ; i < phys_pg_pack->npages ; i++) {
 330			gen_pool_free(vm->dram_pg_pool,
 331				phys_pg_pack->pages[i],
 332				phys_pg_pack->page_size);
 333			kref_put(&vm->dram_pg_pool_refcount,
 334				dram_pg_pool_do_release);
 335		}
 336	}
 337
 338end:
 339	kvfree(phys_pg_pack->pages);
 340	kfree(phys_pg_pack);
 341
 342	return;
 343}
 344
 345/**
 346 * free_device_memory() - free device memory.
 347 * @ctx: pointer to the context structure.
 348 * @args: host parameters containing the requested size.
 349 *
 350 * This function does the following:
 351 * - Free the device memory related to the given handle.
 352 */
 353static int free_device_memory(struct hl_ctx *ctx, struct hl_mem_in *args)
 354{
 355	struct hl_device *hdev = ctx->hdev;
 356	struct hl_vm *vm = &hdev->vm;
 357	struct hl_vm_phys_pg_pack *phys_pg_pack;
 358	u32 handle = args->free.handle;
 359
 360	spin_lock(&vm->idr_lock);
 361	phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle);
 362	if (!phys_pg_pack) {
 363		spin_unlock(&vm->idr_lock);
 364		dev_err(hdev->dev, "free device memory failed, no match for handle %u\n", handle);
 365		return -EINVAL;
 366	}
 367
 368	if (atomic_read(&phys_pg_pack->mapping_cnt) > 0) {
 369		spin_unlock(&vm->idr_lock);
 370		dev_err(hdev->dev, "handle %u is mapped, cannot free\n", handle);
 371		return -EINVAL;
 372	}
 373
 374	/* must remove from idr before the freeing of the physical pages as the refcount of the pool
 375	 * is also the trigger of the idr destroy
 376	 */
 377	idr_remove(&vm->phys_pg_pack_handles, handle);
 378	spin_unlock(&vm->idr_lock);
 379
 380	atomic64_sub(phys_pg_pack->total_size, &ctx->dram_phys_mem);
 381	atomic64_sub(phys_pg_pack->total_size, &hdev->dram_used_mem);
 382
 383	free_phys_pg_pack(hdev, phys_pg_pack);
 384
 385	return 0;
 386}
 387
 388/**
 389 * clear_va_list_locked() - free virtual addresses list.
 390 * @hdev: habanalabs device structure.
 391 * @va_list: list of virtual addresses to free.
 392 *
 393 * This function does the following:
 394 * - Iterate over the list and free each virtual addresses block.
 395 *
 396 * This function should be called only when va_list lock is taken.
 397 */
 398static void clear_va_list_locked(struct hl_device *hdev,
 399		struct list_head *va_list)
 400{
 401	struct hl_vm_va_block *va_block, *tmp;
 402
 403	list_for_each_entry_safe(va_block, tmp, va_list, node) {
 404		list_del(&va_block->node);
 405		kfree(va_block);
 406	}
 407}
 408
 409/**
 410 * print_va_list_locked() - print virtual addresses list.
 411 * @hdev: habanalabs device structure.
 412 * @va_list: list of virtual addresses to print.
 413 *
 414 * This function does the following:
 415 * - Iterate over the list and print each virtual addresses block.
 416 *
 417 * This function should be called only when va_list lock is taken.
 418 */
 419static void print_va_list_locked(struct hl_device *hdev,
 420		struct list_head *va_list)
 421{
 422#if HL_MMU_DEBUG
 423	struct hl_vm_va_block *va_block;
 424
 425	dev_dbg(hdev->dev, "print va list:\n");
 426
 427	list_for_each_entry(va_block, va_list, node)
 428		dev_dbg(hdev->dev,
 429			"va block, start: 0x%llx, end: 0x%llx, size: %llu\n",
 430			va_block->start, va_block->end, va_block->size);
 431#endif
 432}
 433
 434/**
 435 * merge_va_blocks_locked() - merge a virtual block if possible.
 436 * @hdev: pointer to the habanalabs device structure.
 437 * @va_list: pointer to the virtual addresses block list.
 438 * @va_block: virtual block to merge with adjacent blocks.
 439 *
 440 * This function does the following:
 441 * - Merge the given blocks with the adjacent blocks if their virtual ranges
 442 *   create a contiguous virtual range.
 443 *
 444 * This Function should be called only when va_list lock is taken.
 445 */
 446static void merge_va_blocks_locked(struct hl_device *hdev,
 447		struct list_head *va_list, struct hl_vm_va_block *va_block)
 448{
 449	struct hl_vm_va_block *prev, *next;
 450
 451	prev = list_prev_entry(va_block, node);
 452	if (&prev->node != va_list && prev->end + 1 == va_block->start) {
 453		prev->end = va_block->end;
 454		prev->size = prev->end - prev->start + 1;
 455		list_del(&va_block->node);
 456		kfree(va_block);
 457		va_block = prev;
 458	}
 459
 460	next = list_next_entry(va_block, node);
 461	if (&next->node != va_list && va_block->end + 1 == next->start) {
 462		next->start = va_block->start;
 463		next->size = next->end - next->start + 1;
 464		list_del(&va_block->node);
 465		kfree(va_block);
 466	}
 467}
 468
 469/**
 470 * add_va_block_locked() - add a virtual block to the virtual addresses list.
 471 * @hdev: pointer to the habanalabs device structure.
 472 * @va_list: pointer to the virtual addresses block list.
 473 * @start: start virtual address.
 474 * @end: end virtual address.
 475 *
 476 * This function does the following:
 477 * - Add the given block to the virtual blocks list and merge with other blocks
 478 *   if a contiguous virtual block can be created.
 479 *
 480 * This Function should be called only when va_list lock is taken.
 481 */
 482static int add_va_block_locked(struct hl_device *hdev,
 483		struct list_head *va_list, u64 start, u64 end)
 484{
 485	struct hl_vm_va_block *va_block, *res = NULL;
 486	u64 size = end - start + 1;
 487
 488	print_va_list_locked(hdev, va_list);
 489
 490	list_for_each_entry(va_block, va_list, node) {
 491		/* TODO: remove upon matureness */
 492		if (hl_mem_area_crosses_range(start, size, va_block->start,
 493				va_block->end)) {
 494			dev_err(hdev->dev,
 495				"block crossing ranges at start 0x%llx, end 0x%llx\n",
 496				va_block->start, va_block->end);
 497			return -EINVAL;
 498		}
 499
 500		if (va_block->end < start)
 501			res = va_block;
 502	}
 503
 504	va_block = kmalloc(sizeof(*va_block), GFP_KERNEL);
 505	if (!va_block)
 506		return -ENOMEM;
 507
 508	va_block->start = start;
 509	va_block->end = end;
 510	va_block->size = size;
 511
 512	if (!res)
 513		list_add(&va_block->node, va_list);
 514	else
 515		list_add(&va_block->node, &res->node);
 516
 517	merge_va_blocks_locked(hdev, va_list, va_block);
 518
 519	print_va_list_locked(hdev, va_list);
 520
 521	return 0;
 522}
 523
 524/**
 525 * add_va_block() - wrapper for add_va_block_locked.
 526 * @hdev: pointer to the habanalabs device structure.
 527 * @va_range: pointer to the virtual addresses range object.
 528 * @start: start virtual address.
 529 * @end: end virtual address.
 530 *
 531 * This function does the following:
 532 * - Takes the list lock and calls add_va_block_locked.
 533 */
 534static inline int add_va_block(struct hl_device *hdev,
 535		struct hl_va_range *va_range, u64 start, u64 end)
 536{
 537	int rc;
 538
 539	mutex_lock(&va_range->lock);
 540	rc = add_va_block_locked(hdev, &va_range->list, start, end);
 541	mutex_unlock(&va_range->lock);
 542
 543	return rc;
 544}
 545
 546/**
 547 * is_hint_crossing_range() - check if hint address crossing specified reserved.
 548 * @range_type: virtual space range type.
 549 * @start_addr: start virtual address.
 550 * @size: block size.
 551 * @prop: asic properties structure to retrieve reserved ranges from.
 552 */
 553static inline bool is_hint_crossing_range(enum hl_va_range_type range_type,
 554		u64 start_addr, u32 size, struct asic_fixed_properties *prop) {
 555	bool range_cross;
 556
 557	if (range_type == HL_VA_RANGE_TYPE_DRAM)
 558		range_cross =
 559			hl_mem_area_crosses_range(start_addr, size,
 560			prop->hints_dram_reserved_va_range.start_addr,
 561			prop->hints_dram_reserved_va_range.end_addr);
 562	else if (range_type == HL_VA_RANGE_TYPE_HOST)
 563		range_cross =
 564			hl_mem_area_crosses_range(start_addr,	size,
 565			prop->hints_host_reserved_va_range.start_addr,
 566			prop->hints_host_reserved_va_range.end_addr);
 567	else
 568		range_cross =
 569			hl_mem_area_crosses_range(start_addr, size,
 570			prop->hints_host_hpage_reserved_va_range.start_addr,
 571			prop->hints_host_hpage_reserved_va_range.end_addr);
 572
 573	return range_cross;
 574}
 575
 576/**
 577 * get_va_block() - get a virtual block for the given size and alignment.
 578 *
 579 * @hdev: pointer to the habanalabs device structure.
 580 * @va_range: pointer to the virtual addresses range.
 581 * @size: requested block size.
 582 * @hint_addr: hint for requested address by the user.
 583 * @va_block_align: required alignment of the virtual block start address.
 584 * @range_type: va range type (host, dram)
 585 * @flags: additional memory flags, currently only uses HL_MEM_FORCE_HINT
 586 *
 587 * This function does the following:
 588 * - Iterate on the virtual block list to find a suitable virtual block for the
 589 *   given size, hint address and alignment.
 590 * - Reserve the requested block and update the list.
 591 * - Return the start address of the virtual block.
 592 */
 593static u64 get_va_block(struct hl_device *hdev,
 594				struct hl_va_range *va_range,
 595				u64 size, u64 hint_addr, u32 va_block_align,
 596				enum hl_va_range_type range_type,
 597				u32 flags)
 598{
 599	struct hl_vm_va_block *va_block, *new_va_block = NULL;
 600	struct asic_fixed_properties *prop = &hdev->asic_prop;
 601	u64 tmp_hint_addr, valid_start, valid_size, prev_start, prev_end,
 602		align_mask, reserved_valid_start = 0, reserved_valid_size = 0,
 603		dram_hint_mask = prop->dram_hints_align_mask;
 604	bool add_prev = false;
 605	bool is_align_pow_2  = is_power_of_2(va_range->page_size);
 606	bool is_hint_dram_addr = hl_is_dram_va(hdev, hint_addr);
 607	bool force_hint = flags & HL_MEM_FORCE_HINT;
 608	int rc;
 609
 610	if (is_align_pow_2)
 611		align_mask = ~((u64)va_block_align - 1);
 612	else
 613		/*
 614		 * with non-power-of-2 range we work only with page granularity
 615		 * and the start address is page aligned,
 616		 * so no need for alignment checking.
 617		 */
 618		size = DIV_ROUND_UP_ULL(size, va_range->page_size) *
 619							va_range->page_size;
 620
 621	tmp_hint_addr = hint_addr & ~dram_hint_mask;
 622
 623	/* Check if we need to ignore hint address */
 624	if ((is_align_pow_2 && (hint_addr & (va_block_align - 1))) ||
 625			(!is_align_pow_2 && is_hint_dram_addr &&
 626			do_div(tmp_hint_addr, va_range->page_size))) {
 627
 628		if (force_hint) {
 629			/* Hint must be respected, so here we just fail */
 630			dev_err(hdev->dev,
 631				"Hint address 0x%llx is not page aligned - cannot be respected\n",
 632				hint_addr);
 633			return 0;
 634		}
 635
 636		dev_dbg(hdev->dev,
 637			"Hint address 0x%llx will be ignored because it is not aligned\n",
 638			hint_addr);
 639		hint_addr = 0;
 640	}
 641
 642	mutex_lock(&va_range->lock);
 643
 644	print_va_list_locked(hdev, &va_range->list);
 645
 646	list_for_each_entry(va_block, &va_range->list, node) {
 647		/* Calc the first possible aligned addr */
 648		valid_start = va_block->start;
 649
 650		if (is_align_pow_2 && (valid_start & (va_block_align - 1))) {
 651			valid_start &= align_mask;
 652			valid_start += va_block_align;
 653			if (valid_start > va_block->end)
 654				continue;
 655		}
 656
 657		valid_size = va_block->end - valid_start + 1;
 658		if (valid_size < size)
 659			continue;
 660
 661		/*
 662		 * In case hint address is 0, and hints_range_reservation
 663		 * property enabled, then avoid allocating va blocks from the
 664		 * range reserved for hint addresses
 665		 */
 666		if (prop->hints_range_reservation && !hint_addr)
 667			if (is_hint_crossing_range(range_type, valid_start,
 668					size, prop))
 669				continue;
 670
 671		/* Pick the minimal length block which has the required size */
 672		if (!new_va_block || (valid_size < reserved_valid_size)) {
 673			new_va_block = va_block;
 674			reserved_valid_start = valid_start;
 675			reserved_valid_size = valid_size;
 676		}
 677
 678		if (hint_addr && hint_addr >= valid_start &&
 679					(hint_addr + size) <= va_block->end) {
 680			new_va_block = va_block;
 681			reserved_valid_start = hint_addr;
 682			reserved_valid_size = valid_size;
 683			break;
 684		}
 685	}
 686
 687	if (!new_va_block) {
 688		dev_err(hdev->dev, "no available va block for size %llu\n",
 689								size);
 690		goto out;
 691	}
 692
 693	if (force_hint && reserved_valid_start != hint_addr) {
 694		/* Hint address must be respected. If we are here - this means
 695		 * we could not respect it.
 696		 */
 697		dev_err(hdev->dev,
 698			"Hint address 0x%llx could not be respected\n",
 699			hint_addr);
 700		reserved_valid_start = 0;
 701		goto out;
 702	}
 703
 704	/*
 705	 * Check if there is some leftover range due to reserving the new
 706	 * va block, then return it to the main virtual addresses list.
 707	 */
 708	if (reserved_valid_start > new_va_block->start) {
 709		prev_start = new_va_block->start;
 710		prev_end = reserved_valid_start - 1;
 711
 712		new_va_block->start = reserved_valid_start;
 713		new_va_block->size = reserved_valid_size;
 714
 715		add_prev = true;
 716	}
 717
 718	if (new_va_block->size > size) {
 719		new_va_block->start += size;
 720		new_va_block->size = new_va_block->end - new_va_block->start + 1;
 721	} else {
 722		list_del(&new_va_block->node);
 723		kfree(new_va_block);
 724	}
 725
 726	if (add_prev) {
 727		rc = add_va_block_locked(hdev, &va_range->list, prev_start, prev_end);
 728		if (rc) {
 729			reserved_valid_start = 0;
 730			goto out;
 731		}
 732	}
 733
 734	print_va_list_locked(hdev, &va_range->list);
 735out:
 736	mutex_unlock(&va_range->lock);
 737
 738	return reserved_valid_start;
 739}
 740
 741/*
 742 * hl_reserve_va_block() - reserve a virtual block of a given size.
 743 * @hdev: pointer to the habanalabs device structure.
 744 * @ctx: current context
 745 * @type: virtual addresses range type.
 746 * @size: requested block size.
 747 * @alignment: required alignment in bytes of the virtual block start address,
 748 *             0 means no alignment.
 749 *
 750 * This function does the following:
 751 * - Iterate on the virtual block list to find a suitable virtual block for the
 752 *   given size and alignment.
 753 * - Reserve the requested block and update the list.
 754 * - Return the start address of the virtual block.
 755 */
 756u64 hl_reserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx,
 757		enum hl_va_range_type type, u64 size, u32 alignment)
 758{
 759	return get_va_block(hdev, ctx->va_range[type], size, 0,
 760			max(alignment, ctx->va_range[type]->page_size),
 761			type, 0);
 762}
 763
 764/**
 765 * hl_get_va_range_type() - get va_range type for the given address and size.
 766 * @ctx: context to fetch va_range from.
 767 * @address: the start address of the area we want to validate.
 768 * @size: the size in bytes of the area we want to validate.
 769 * @type: returned va_range type.
 770 *
 771 * Return: true if the area is inside a valid range, false otherwise.
 772 */
 773static int hl_get_va_range_type(struct hl_ctx *ctx, u64 address, u64 size,
 774			enum hl_va_range_type *type)
 775{
 776	int i;
 777
 778	for (i = 0 ; i < HL_VA_RANGE_TYPE_MAX; i++) {
 779		if (hl_mem_area_inside_range(address, size,
 780				ctx->va_range[i]->start_addr,
 781				ctx->va_range[i]->end_addr)) {
 782			*type = i;
 783			return 0;
 784		}
 785	}
 786
 787	return -EINVAL;
 788}
 789
 790/**
 791 * hl_unreserve_va_block() - wrapper for add_va_block to unreserve a va block.
 792 * @hdev: pointer to the habanalabs device structure
 793 * @ctx: pointer to the context structure.
 794 * @start_addr: start virtual address.
 795 * @size: number of bytes to unreserve.
 796 *
 797 * This function does the following:
 798 * - Takes the list lock and calls add_va_block_locked.
 799 */
 800int hl_unreserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx,
 801		u64 start_addr, u64 size)
 802{
 803	enum hl_va_range_type type;
 804	int rc;
 805
 806	rc = hl_get_va_range_type(ctx, start_addr, size, &type);
 807	if (rc) {
 808		dev_err(hdev->dev,
 809			"cannot find va_range for va %#llx size %llu",
 810			start_addr, size);
 811		return rc;
 812	}
 813
 814	rc = add_va_block(hdev, ctx->va_range[type], start_addr,
 815						start_addr + size - 1);
 816	if (rc)
 817		dev_warn(hdev->dev,
 818			"add va block failed for vaddr: 0x%llx\n", start_addr);
 819
 820	return rc;
 821}
 822
 823/**
 824 * init_phys_pg_pack_from_userptr() - initialize physical page pack from host
 825 *                                    memory
 826 * @ctx: pointer to the context structure.
 827 * @userptr: userptr to initialize from.
 828 * @pphys_pg_pack: result pointer.
 829 * @force_regular_page: tell the function to ignore huge page optimization,
 830 *                      even if possible. Needed for cases where the device VA
 831 *                      is allocated before we know the composition of the
 832 *                      physical pages
 833 *
 834 * This function does the following:
 835 * - Create a physical page pack from the physical pages related to the given
 836 *   virtual block.
 837 */
 838static int init_phys_pg_pack_from_userptr(struct hl_ctx *ctx,
 839				struct hl_userptr *userptr,
 840				struct hl_vm_phys_pg_pack **pphys_pg_pack,
 841				bool force_regular_page)
 842{
 843	u32 npages, page_size = PAGE_SIZE,
 844		huge_page_size = ctx->hdev->asic_prop.pmmu_huge.page_size;
 845	u32 pgs_in_huge_page = huge_page_size >> __ffs(page_size);
 846	struct hl_vm_phys_pg_pack *phys_pg_pack;
 847	bool first = true, is_huge_page_opt;
 848	u64 page_mask, total_npages;
 849	struct scatterlist *sg;
 850	dma_addr_t dma_addr;
 851	int rc, i, j;
 852
 853	phys_pg_pack = kzalloc(sizeof(*phys_pg_pack), GFP_KERNEL);
 854	if (!phys_pg_pack)
 855		return -ENOMEM;
 856
 857	phys_pg_pack->vm_type = userptr->vm_type;
 858	phys_pg_pack->created_from_userptr = true;
 859	phys_pg_pack->asid = ctx->asid;
 860	atomic_set(&phys_pg_pack->mapping_cnt, 1);
 861
 862	is_huge_page_opt = (force_regular_page ? false : true);
 863
 864	/* Only if all dma_addrs are aligned to 2MB and their
 865	 * sizes is at least 2MB, we can use huge page mapping.
 866	 * We limit the 2MB optimization to this condition,
 867	 * since later on we acquire the related VA range as one
 868	 * consecutive block.
 869	 */
 870	total_npages = 0;
 871	for_each_sgtable_dma_sg(userptr->sgt, sg, i) {
 872		npages = hl_get_sg_info(sg, &dma_addr);
 873
 874		total_npages += npages;
 875
 876		if ((npages % pgs_in_huge_page) ||
 877					(dma_addr & (huge_page_size - 1)))
 878			is_huge_page_opt = false;
 879	}
 880
 881	if (is_huge_page_opt) {
 882		page_size = huge_page_size;
 883		do_div(total_npages, pgs_in_huge_page);
 884	}
 885
 886	page_mask = ~(((u64) page_size) - 1);
 887
 888	phys_pg_pack->pages = kvmalloc_array(total_npages, sizeof(u64),
 889						GFP_KERNEL);
 890	if (ZERO_OR_NULL_PTR(phys_pg_pack->pages)) {
 891		rc = -ENOMEM;
 892		goto page_pack_arr_mem_err;
 893	}
 894
 895	phys_pg_pack->npages = total_npages;
 896	phys_pg_pack->page_size = page_size;
 897	phys_pg_pack->total_size = total_npages * page_size;
 898
 899	j = 0;
 900	for_each_sgtable_dma_sg(userptr->sgt, sg, i) {
 901		npages = hl_get_sg_info(sg, &dma_addr);
 902
 903		/* align down to physical page size and save the offset */
 904		if (first) {
 905			first = false;
 906			phys_pg_pack->offset = dma_addr & (page_size - 1);
 907			dma_addr &= page_mask;
 908		}
 909
 910		while (npages) {
 911			phys_pg_pack->pages[j++] = dma_addr;
 912			dma_addr += page_size;
 913
 914			if (is_huge_page_opt)
 915				npages -= pgs_in_huge_page;
 916			else
 917				npages--;
 918		}
 919	}
 920
 921	*pphys_pg_pack = phys_pg_pack;
 922
 923	return 0;
 924
 925page_pack_arr_mem_err:
 926	kfree(phys_pg_pack);
 927
 928	return rc;
 929}
 930
 931/**
 932 * map_phys_pg_pack() - maps the physical page pack..
 933 * @ctx: pointer to the context structure.
 934 * @vaddr: start address of the virtual area to map from.
 935 * @phys_pg_pack: the pack of physical pages to map to.
 936 *
 937 * This function does the following:
 938 * - Maps each chunk of virtual memory to matching physical chunk.
 939 * - Stores number of successful mappings in the given argument.
 940 * - Returns 0 on success, error code otherwise.
 941 */
 942static int map_phys_pg_pack(struct hl_ctx *ctx, u64 vaddr,
 943				struct hl_vm_phys_pg_pack *phys_pg_pack)
 944{
 945	struct hl_device *hdev = ctx->hdev;
 946	u64 next_vaddr = vaddr, paddr, mapped_pg_cnt = 0, i;
 947	u32 page_size = phys_pg_pack->page_size;
 948	int rc = 0;
 949	bool is_host_addr;
 950
 951	for (i = 0 ; i < phys_pg_pack->npages ; i++) {
 952		paddr = phys_pg_pack->pages[i];
 953
 954		rc = hl_mmu_map_page(ctx, next_vaddr, paddr, page_size,
 955				(i + 1) == phys_pg_pack->npages);
 956		if (rc) {
 957			dev_err(hdev->dev,
 958				"map failed (%d) for handle %u, npages: %llu, mapped: %llu\n",
 959				rc, phys_pg_pack->handle, phys_pg_pack->npages,
 960				mapped_pg_cnt);
 961			goto err;
 962		}
 963
 964		mapped_pg_cnt++;
 965		next_vaddr += page_size;
 966	}
 967
 968	return 0;
 969
 970err:
 971	is_host_addr = !hl_is_dram_va(hdev, vaddr);
 972
 973	next_vaddr = vaddr;
 974	for (i = 0 ; i < mapped_pg_cnt ; i++) {
 975		if (hl_mmu_unmap_page(ctx, next_vaddr, page_size,
 976					(i + 1) == mapped_pg_cnt))
 977			dev_warn_ratelimited(hdev->dev,
 978				"failed to unmap handle %u, va: 0x%llx, pa: 0x%llx, page size: %u\n",
 979					phys_pg_pack->handle, next_vaddr,
 980					phys_pg_pack->pages[i], page_size);
 981
 982		next_vaddr += page_size;
 983
 984		/*
 985		 * unmapping on Palladium can be really long, so avoid a CPU
 986		 * soft lockup bug by sleeping a little between unmapping pages
 987		 *
 988		 * In addition, on host num of pages could be huge,
 989		 * because page size could be 4KB, so when unmapping host
 990		 * pages sleep every 32K pages to avoid soft lockup
 991		 */
 992		if (hdev->pldm || (is_host_addr && (i & 0x7FFF) == 0))
 993			usleep_range(50, 200);
 994	}
 995
 996	return rc;
 997}
 998
 999/**
1000 * unmap_phys_pg_pack() - unmaps the physical page pack.
1001 * @ctx: pointer to the context structure.
1002 * @vaddr: start address of the virtual area to unmap.
1003 * @phys_pg_pack: the pack of physical pages to unmap.
1004 */
1005static void unmap_phys_pg_pack(struct hl_ctx *ctx, u64 vaddr,
1006				struct hl_vm_phys_pg_pack *phys_pg_pack)
1007{
1008	struct hl_device *hdev = ctx->hdev;
1009	u64 next_vaddr, i;
1010	bool is_host_addr;
1011	u32 page_size;
1012
1013	is_host_addr = !hl_is_dram_va(hdev, vaddr);
1014	page_size = phys_pg_pack->page_size;
1015	next_vaddr = vaddr;
1016
1017	for (i = 0 ; i < phys_pg_pack->npages ; i++, next_vaddr += page_size) {
1018		if (hl_mmu_unmap_page(ctx, next_vaddr, page_size,
1019				       (i + 1) == phys_pg_pack->npages))
1020			dev_warn_ratelimited(hdev->dev,
1021			"unmap failed for vaddr: 0x%llx\n", next_vaddr);
1022
1023		/*
1024		 * unmapping on Palladium can be really long, so avoid a CPU
1025		 * soft lockup bug by sleeping a little between unmapping pages
1026		 *
1027		 * In addition, on host num of pages could be huge,
1028		 * because page size could be 4KB, so when unmapping host
1029		 * pages sleep every 32K pages to avoid soft lockup
1030		 */
1031		if (hdev->pldm || (is_host_addr && (i & 0x7FFF) == 0))
1032			usleep_range(50, 200);
1033	}
1034}
1035
1036/**
1037 * map_device_va() - map the given memory.
1038 * @ctx: pointer to the context structure.
1039 * @args: host parameters with handle/host virtual address.
1040 * @device_addr: pointer to result device virtual address.
1041 *
1042 * This function does the following:
1043 * - If given a physical device memory handle, map to a device virtual block
1044 *   and return the start address of this block.
1045 * - If given a host virtual address and size, find the related physical pages,
1046 *   map a device virtual block to this pages and return the start address of
1047 *   this block.
1048 */
1049static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, u64 *device_addr)
1050{
1051	struct hl_vm_phys_pg_pack *phys_pg_pack;
1052	enum hl_va_range_type va_range_type = 0;
1053	struct hl_device *hdev = ctx->hdev;
1054	struct hl_userptr *userptr = NULL;
1055	u32 handle = 0, va_block_align;
1056	struct hl_vm_hash_node *hnode;
1057	struct hl_vm *vm = &hdev->vm;
1058	struct hl_va_range *va_range;
1059	bool is_userptr, do_prefetch;
1060	u64 ret_vaddr, hint_addr;
1061	enum vm_type *vm_type;
1062	int rc;
1063
1064	/* set map flags */
1065	is_userptr = args->flags & HL_MEM_USERPTR;
1066	do_prefetch = hdev->supports_mmu_prefetch && (args->flags & HL_MEM_PREFETCH);
1067
1068	/* Assume failure */
1069	*device_addr = 0;
1070
1071	if (is_userptr) {
1072		u64 addr = args->map_host.host_virt_addr,
1073			size = args->map_host.mem_size;
1074		u32 page_size = hdev->asic_prop.pmmu.page_size,
1075			huge_page_size = hdev->asic_prop.pmmu_huge.page_size;
1076
1077		rc = dma_map_host_va(hdev, addr, size, &userptr);
1078		if (rc)
1079			return rc;
1080
1081		rc = init_phys_pg_pack_from_userptr(ctx, userptr,
1082				&phys_pg_pack, false);
1083		if (rc) {
1084			dev_err(hdev->dev,
1085				"unable to init page pack for vaddr 0x%llx\n",
1086				addr);
1087			goto init_page_pack_err;
1088		}
1089
1090		vm_type = (enum vm_type *) userptr;
1091		hint_addr = args->map_host.hint_addr;
1092		handle = phys_pg_pack->handle;
1093
1094		/* get required alignment */
1095		if (phys_pg_pack->page_size == page_size) {
1096			va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST];
1097			va_range_type = HL_VA_RANGE_TYPE_HOST;
1098			/*
1099			 * huge page alignment may be needed in case of regular
1100			 * page mapping, depending on the host VA alignment
1101			 */
1102			if (addr & (huge_page_size - 1))
1103				va_block_align = page_size;
1104			else
1105				va_block_align = huge_page_size;
1106		} else {
1107			/*
1108			 * huge page alignment is needed in case of huge page
1109			 * mapping
1110			 */
1111			va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE];
1112			va_range_type = HL_VA_RANGE_TYPE_HOST_HUGE;
1113			va_block_align = huge_page_size;
1114		}
1115	} else {
1116		handle = lower_32_bits(args->map_device.handle);
1117
1118		spin_lock(&vm->idr_lock);
1119		phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle);
1120		if (!phys_pg_pack) {
1121			spin_unlock(&vm->idr_lock);
1122			dev_err(hdev->dev,
1123				"no match for handle %u\n", handle);
1124			return -EINVAL;
1125		}
1126
1127		/* increment now to avoid freeing device memory while mapping */
1128		atomic_inc(&phys_pg_pack->mapping_cnt);
1129
1130		spin_unlock(&vm->idr_lock);
1131
1132		vm_type = (enum vm_type *) phys_pg_pack;
1133
1134		hint_addr = args->map_device.hint_addr;
1135
1136		/* DRAM VA alignment is the same as the MMU page size */
1137		va_range = ctx->va_range[HL_VA_RANGE_TYPE_DRAM];
1138		va_range_type = HL_VA_RANGE_TYPE_DRAM;
1139		va_block_align = hdev->asic_prop.dmmu.page_size;
1140	}
1141
1142	/*
1143	 * relevant for mapping device physical memory only, as host memory is
1144	 * implicitly shared
1145	 */
1146	if (!is_userptr && !(phys_pg_pack->flags & HL_MEM_SHARED) &&
1147			phys_pg_pack->asid != ctx->asid) {
1148		dev_err(hdev->dev,
1149			"Failed to map memory, handle %u is not shared\n",
1150			handle);
1151		rc = -EPERM;
1152		goto shared_err;
1153	}
1154
1155	hnode = kzalloc(sizeof(*hnode), GFP_KERNEL);
1156	if (!hnode) {
1157		rc = -ENOMEM;
1158		goto hnode_err;
1159	}
1160
1161	if (hint_addr && phys_pg_pack->offset) {
1162		if (args->flags & HL_MEM_FORCE_HINT) {
1163			/* Fail if hint must be respected but it can't be */
1164			dev_err(hdev->dev,
1165				"Hint address 0x%llx cannot be respected because source memory is not aligned 0x%x\n",
1166				hint_addr, phys_pg_pack->offset);
1167			rc = -EINVAL;
1168			goto va_block_err;
1169		}
1170		dev_dbg(hdev->dev,
1171			"Hint address 0x%llx will be ignored because source memory is not aligned 0x%x\n",
1172			hint_addr, phys_pg_pack->offset);
1173	}
1174
1175	ret_vaddr = get_va_block(hdev, va_range, phys_pg_pack->total_size,
1176					hint_addr, va_block_align,
1177					va_range_type, args->flags);
1178	if (!ret_vaddr) {
1179		dev_err(hdev->dev, "no available va block for handle %u\n",
1180				handle);
1181		rc = -ENOMEM;
1182		goto va_block_err;
1183	}
1184
1185	mutex_lock(&hdev->mmu_lock);
1186
1187	rc = map_phys_pg_pack(ctx, ret_vaddr, phys_pg_pack);
1188	if (rc) {
1189		dev_err(hdev->dev, "mapping page pack failed (%d) for handle %u\n",
1190			rc, handle);
1191		mutex_unlock(&hdev->mmu_lock);
1192		goto map_err;
1193	}
1194
1195	rc = hl_mmu_invalidate_cache_range(hdev, false, *vm_type | MMU_OP_SKIP_LOW_CACHE_INV,
1196				ctx->asid, ret_vaddr, phys_pg_pack->total_size);
1197	mutex_unlock(&hdev->mmu_lock);
1198	if (rc)
1199		goto map_err;
1200
1201	/*
1202	 * prefetch is done upon user's request. it is performed in WQ as and so can
1203	 * be outside the MMU lock. the operation itself is already protected by the mmu lock
1204	 */
1205	if (do_prefetch) {
1206		rc = hl_mmu_prefetch_cache_range(ctx, *vm_type, ctx->asid, ret_vaddr,
1207							phys_pg_pack->total_size);
1208		if (rc)
1209			goto map_err;
1210	}
1211
1212	ret_vaddr += phys_pg_pack->offset;
1213
1214	hnode->ptr = vm_type;
1215	hnode->vaddr = ret_vaddr;
1216	hnode->handle = is_userptr ? MEM_HANDLE_INVALID : handle;
1217
1218	mutex_lock(&ctx->mem_hash_lock);
1219	hash_add(ctx->mem_hash, &hnode->node, ret_vaddr);
1220	mutex_unlock(&ctx->mem_hash_lock);
1221
1222	*device_addr = ret_vaddr;
1223
1224	if (is_userptr)
1225		free_phys_pg_pack(hdev, phys_pg_pack);
1226
1227	return rc;
1228
1229map_err:
1230	if (add_va_block(hdev, va_range, ret_vaddr,
1231				ret_vaddr + phys_pg_pack->total_size - 1))
1232		dev_warn(hdev->dev,
1233			"release va block failed for handle 0x%x, vaddr: 0x%llx\n",
1234				handle, ret_vaddr);
1235
1236va_block_err:
1237	kfree(hnode);
1238hnode_err:
1239shared_err:
1240	atomic_dec(&phys_pg_pack->mapping_cnt);
1241	if (is_userptr)
1242		free_phys_pg_pack(hdev, phys_pg_pack);
1243init_page_pack_err:
1244	if (is_userptr)
1245		dma_unmap_host_va(hdev, userptr);
1246
1247	return rc;
1248}
1249
1250/* Should be called while the context's mem_hash_lock is taken */
1251static struct hl_vm_hash_node *get_vm_hash_node_locked(struct hl_ctx *ctx, u64 vaddr)
1252{
1253	struct hl_vm_hash_node *hnode;
1254
1255	hash_for_each_possible(ctx->mem_hash, hnode, node, vaddr)
1256		if (vaddr == hnode->vaddr)
1257			return hnode;
1258
1259	return NULL;
1260}
1261
1262/**
1263 * unmap_device_va() - unmap the given device virtual address.
1264 * @ctx: pointer to the context structure.
1265 * @args: host parameters with device virtual address to unmap.
1266 * @ctx_free: true if in context free flow, false otherwise.
1267 *
1268 * This function does the following:
1269 * - unmap the physical pages related to the given virtual address.
1270 * - return the device virtual block to the virtual block list.
1271 */
1272static int unmap_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
1273				bool ctx_free)
1274{
1275	struct hl_vm_phys_pg_pack *phys_pg_pack = NULL;
1276	u64 vaddr = args->unmap.device_virt_addr;
1277	struct asic_fixed_properties *prop;
1278	struct hl_device *hdev = ctx->hdev;
1279	struct hl_userptr *userptr = NULL;
1280	struct hl_vm_hash_node *hnode;
1281	struct hl_va_range *va_range;
1282	enum vm_type *vm_type;
1283	bool is_userptr;
1284	int rc = 0;
1285
1286	prop = &hdev->asic_prop;
1287
1288	/* protect from double entrance */
1289	mutex_lock(&ctx->mem_hash_lock);
1290	hnode = get_vm_hash_node_locked(ctx, vaddr);
1291	if (!hnode) {
1292		mutex_unlock(&ctx->mem_hash_lock);
1293		dev_err(hdev->dev, "unmap failed, no mem hnode for vaddr 0x%llx\n", vaddr);
1294		return -EINVAL;
1295	}
1296
1297	if (hnode->export_cnt) {
1298		mutex_unlock(&ctx->mem_hash_lock);
1299		dev_err(hdev->dev, "failed to unmap %#llx, memory is exported\n", vaddr);
1300		return -EINVAL;
1301	}
1302
1303	hash_del(&hnode->node);
1304	mutex_unlock(&ctx->mem_hash_lock);
1305
1306	vm_type = hnode->ptr;
1307
1308	if (*vm_type == VM_TYPE_USERPTR) {
1309		is_userptr = true;
1310		userptr = hnode->ptr;
1311
1312		rc = init_phys_pg_pack_from_userptr(ctx, userptr, &phys_pg_pack,
1313							false);
1314		if (rc) {
1315			dev_err(hdev->dev,
1316				"unable to init page pack for vaddr 0x%llx\n",
1317				vaddr);
1318			goto vm_type_err;
1319		}
1320
1321		if (phys_pg_pack->page_size ==
1322					hdev->asic_prop.pmmu.page_size)
1323			va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST];
1324		else
1325			va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE];
1326	} else if (*vm_type == VM_TYPE_PHYS_PACK) {
1327		is_userptr = false;
1328		va_range = ctx->va_range[HL_VA_RANGE_TYPE_DRAM];
1329		phys_pg_pack = hnode->ptr;
1330	} else {
1331		dev_warn(hdev->dev,
1332			"unmap failed, unknown vm desc for vaddr 0x%llx\n",
1333				vaddr);
1334		rc = -EFAULT;
1335		goto vm_type_err;
1336	}
1337
1338	if (atomic_read(&phys_pg_pack->mapping_cnt) == 0) {
1339		dev_err(hdev->dev, "vaddr 0x%llx is not mapped\n", vaddr);
1340		rc = -EINVAL;
1341		goto mapping_cnt_err;
1342	}
1343
1344	if (!is_userptr && !is_power_of_2(phys_pg_pack->page_size))
1345		vaddr = prop->dram_base_address +
1346			DIV_ROUND_DOWN_ULL(vaddr - prop->dram_base_address,
1347						phys_pg_pack->page_size) *
1348							phys_pg_pack->page_size;
1349	else
1350		vaddr &= ~(((u64) phys_pg_pack->page_size) - 1);
1351
1352	mutex_lock(&hdev->mmu_lock);
1353
1354	unmap_phys_pg_pack(ctx, vaddr, phys_pg_pack);
1355
1356	/*
1357	 * During context free this function is called in a loop to clean all
1358	 * the context mappings. Hence the cache invalidation can be called once
1359	 * at the loop end rather than for each iteration
1360	 */
1361	if (!ctx_free)
1362		rc = hl_mmu_invalidate_cache_range(hdev, true, *vm_type, ctx->asid, vaddr,
1363							phys_pg_pack->total_size);
1364
1365	mutex_unlock(&hdev->mmu_lock);
1366
1367	/*
1368	 * If the context is closing we don't need to check for the MMU cache
1369	 * invalidation return code and update the VA free list as in this flow
1370	 * we invalidate the MMU cache outside of this unmap function and the VA
1371	 * free list will be freed anyway.
1372	 */
1373	if (!ctx_free) {
1374		int tmp_rc;
1375
1376		tmp_rc = add_va_block(hdev, va_range, vaddr,
1377					vaddr + phys_pg_pack->total_size - 1);
1378		if (tmp_rc) {
1379			dev_warn(hdev->dev,
1380					"add va block failed for vaddr: 0x%llx\n",
1381					vaddr);
1382			if (!rc)
1383				rc = tmp_rc;
1384		}
1385	}
1386
1387	atomic_dec(&phys_pg_pack->mapping_cnt);
1388	kfree(hnode);
1389
1390	if (is_userptr) {
1391		free_phys_pg_pack(hdev, phys_pg_pack);
1392		dma_unmap_host_va(hdev, userptr);
1393	}
1394
1395	return rc;
1396
1397mapping_cnt_err:
1398	if (is_userptr)
1399		free_phys_pg_pack(hdev, phys_pg_pack);
1400vm_type_err:
1401	mutex_lock(&ctx->mem_hash_lock);
1402	hash_add(ctx->mem_hash, &hnode->node, vaddr);
1403	mutex_unlock(&ctx->mem_hash_lock);
1404
1405	return rc;
1406}
1407
1408static int map_block(struct hl_device *hdev, u64 address, u64 *handle, u32 *size)
1409{
1410	u32 block_id;
1411	int rc;
1412
1413	*handle = 0;
1414	if (size)
1415		*size = 0;
1416
1417	rc = hdev->asic_funcs->get_hw_block_id(hdev, address, size, &block_id);
1418	if (rc)
1419		return rc;
1420
1421	*handle = block_id | HL_MMAP_TYPE_BLOCK;
1422	*handle <<= PAGE_SHIFT;
1423
1424	return 0;
1425}
1426
1427static void hw_block_vm_close(struct vm_area_struct *vma)
1428{
1429	struct hl_vm_hw_block_list_node *lnode =
1430		(struct hl_vm_hw_block_list_node *) vma->vm_private_data;
1431	struct hl_ctx *ctx = lnode->ctx;
1432	long new_mmap_size;
1433
1434	new_mmap_size = lnode->mapped_size - (vma->vm_end - vma->vm_start);
1435	if (new_mmap_size > 0) {
1436		lnode->mapped_size = new_mmap_size;
1437		return;
1438	}
1439
1440	mutex_lock(&ctx->hw_block_list_lock);
1441	list_del(&lnode->node);
1442	mutex_unlock(&ctx->hw_block_list_lock);
1443	hl_ctx_put(ctx);
1444	kfree(lnode);
1445	vma->vm_private_data = NULL;
1446}
1447
1448static const struct vm_operations_struct hw_block_vm_ops = {
1449	.close = hw_block_vm_close
1450};
1451
1452/**
1453 * hl_hw_block_mmap() - mmap a hw block to user.
1454 * @hpriv: pointer to the private data of the fd
1455 * @vma: pointer to vm_area_struct of the process
1456 *
1457 * Driver increments context reference for every HW block mapped in order
1458 * to prevent user from closing FD without unmapping first
1459 */
1460int hl_hw_block_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma)
1461{
1462	struct hl_vm_hw_block_list_node *lnode;
1463	struct hl_device *hdev = hpriv->hdev;
1464	struct hl_ctx *ctx = hpriv->ctx;
1465	u32 block_id, block_size;
1466	int rc;
1467
1468	/* We use the page offset to hold the block id and thus we need to clear
1469	 * it before doing the mmap itself
1470	 */
1471	block_id = vma->vm_pgoff;
1472	vma->vm_pgoff = 0;
1473
1474	/* Driver only allows mapping of a complete HW block */
1475	block_size = vma->vm_end - vma->vm_start;
1476
1477	if (!access_ok((void __user *) (uintptr_t) vma->vm_start, block_size)) {
1478		dev_err(hdev->dev,
1479			"user pointer is invalid - 0x%lx\n",
1480			vma->vm_start);
1481
1482		return -EINVAL;
1483	}
1484
1485	lnode = kzalloc(sizeof(*lnode), GFP_KERNEL);
1486	if (!lnode)
1487		return -ENOMEM;
1488
1489	rc = hdev->asic_funcs->hw_block_mmap(hdev, vma, block_id, block_size);
1490	if (rc) {
1491		kfree(lnode);
1492		return rc;
1493	}
1494
1495	hl_ctx_get(ctx);
1496
1497	lnode->ctx = ctx;
1498	lnode->vaddr = vma->vm_start;
1499	lnode->block_size = block_size;
1500	lnode->mapped_size = lnode->block_size;
1501	lnode->id = block_id;
1502
1503	vma->vm_private_data = lnode;
1504	vma->vm_ops = &hw_block_vm_ops;
1505
1506	mutex_lock(&ctx->hw_block_list_lock);
1507	list_add_tail(&lnode->node, &ctx->hw_block_mem_list);
1508	mutex_unlock(&ctx->hw_block_list_lock);
1509
1510	vma->vm_pgoff = block_id;
1511
1512	return 0;
1513}
1514
1515static int set_dma_sg(struct scatterlist *sg, u64 bar_address, u64 chunk_size,
1516			struct device *dev, enum dma_data_direction dir)
1517{
1518	dma_addr_t addr;
1519	int rc;
1520
1521	addr = dma_map_resource(dev, bar_address, chunk_size, dir,
1522				DMA_ATTR_SKIP_CPU_SYNC);
1523	rc = dma_mapping_error(dev, addr);
1524	if (rc)
1525		return rc;
1526
1527	sg_set_page(sg, NULL, chunk_size, 0);
1528	sg_dma_address(sg) = addr;
1529	sg_dma_len(sg) = chunk_size;
1530
1531	return 0;
1532}
1533
1534static struct sg_table *alloc_sgt_from_device_pages(struct hl_device *hdev, u64 *pages, u64 npages,
1535						u64 page_size, u64 exported_size, u64 offset,
1536						struct device *dev, enum dma_data_direction dir)
1537{
1538	u64 dma_max_seg_size, curr_page, size, chunk_size, left_size_to_export, left_size_in_page,
1539		left_size_in_dma_seg, device_address, bar_address, start_page;
1540	struct asic_fixed_properties *prop = &hdev->asic_prop;
1541	struct scatterlist *sg;
1542	unsigned int nents, i;
1543	struct sg_table *sgt;
1544	bool next_sg_entry;
1545	int rc;
1546
1547	/* Align max segment size to PAGE_SIZE to fit the minimal IOMMU mapping granularity */
1548	dma_max_seg_size = ALIGN_DOWN(dma_get_max_seg_size(dev), PAGE_SIZE);
1549	if (dma_max_seg_size < PAGE_SIZE) {
1550		dev_err_ratelimited(hdev->dev,
1551				"dma_max_seg_size %llu can't be smaller than PAGE_SIZE\n",
1552				dma_max_seg_size);
1553		return ERR_PTR(-EINVAL);
1554	}
1555
1556	sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
1557	if (!sgt)
1558		return ERR_PTR(-ENOMEM);
1559
1560	/* Use the offset to move to the actual first page that is exported */
1561	for (start_page = 0 ; start_page < npages ; ++start_page) {
1562		if (offset < page_size)
1563			break;
1564
1565		/* The offset value was validated so there can't be an underflow */
1566		offset -= page_size;
1567	}
1568
1569	/* Calculate the required number of entries for the SG table */
1570	curr_page = start_page;
1571	nents = 1;
1572	left_size_to_export = exported_size;
1573	left_size_in_page = page_size - offset;
1574	left_size_in_dma_seg = dma_max_seg_size;
1575	next_sg_entry = false;
1576
1577	while (true) {
1578		size = min3(left_size_to_export, left_size_in_page, left_size_in_dma_seg);
1579		left_size_to_export -= size;
1580		left_size_in_page -= size;
1581		left_size_in_dma_seg -= size;
1582
1583		if (!left_size_to_export)
1584			break;
1585
1586		if (!left_size_in_page) {
1587			/* left_size_to_export is not zero so there must be another page */
1588			if (pages[curr_page] + page_size != pages[curr_page + 1])
1589				next_sg_entry = true;
1590
1591			++curr_page;
1592			left_size_in_page = page_size;
1593		}
1594
1595		if (!left_size_in_dma_seg) {
1596			next_sg_entry = true;
1597			left_size_in_dma_seg = dma_max_seg_size;
1598		}
1599
1600		if (next_sg_entry) {
1601			++nents;
1602			next_sg_entry = false;
1603		}
1604	}
1605
1606	rc = sg_alloc_table(sgt, nents, GFP_KERNEL | __GFP_ZERO);
1607	if (rc)
1608		goto err_free_sgt;
1609
1610	/* Prepare the SG table entries */
1611	curr_page = start_page;
1612	device_address = pages[curr_page] + offset;
1613	left_size_to_export = exported_size;
1614	left_size_in_page = page_size - offset;
1615	left_size_in_dma_seg = dma_max_seg_size;
1616	next_sg_entry = false;
1617
1618	for_each_sgtable_dma_sg(sgt, sg, i) {
1619		bar_address = hdev->dram_pci_bar_start + (device_address - prop->dram_base_address);
1620		chunk_size = 0;
1621
1622		for ( ; curr_page < npages ; ++curr_page) {
1623			size = min3(left_size_to_export, left_size_in_page, left_size_in_dma_seg);
1624			chunk_size += size;
1625			left_size_to_export -= size;
1626			left_size_in_page -= size;
1627			left_size_in_dma_seg -= size;
1628
1629			if (!left_size_to_export)
1630				break;
1631
1632			if (!left_size_in_page) {
1633				/* left_size_to_export is not zero so there must be another page */
1634				if (pages[curr_page] + page_size != pages[curr_page + 1]) {
1635					device_address = pages[curr_page + 1];
1636					next_sg_entry = true;
1637				}
1638
1639				left_size_in_page = page_size;
1640			}
1641
1642			if (!left_size_in_dma_seg) {
1643				/*
1644				 * Skip setting a new device address if already moving to a page
1645				 * which is not contiguous with the current page.
1646				 */
1647				if (!next_sg_entry) {
1648					device_address += chunk_size;
1649					next_sg_entry = true;
1650				}
1651
1652				left_size_in_dma_seg = dma_max_seg_size;
1653			}
1654
1655			if (next_sg_entry) {
1656				next_sg_entry = false;
1657				break;
1658			}
1659		}
1660
1661		rc = set_dma_sg(sg, bar_address, chunk_size, dev, dir);
1662		if (rc)
1663			goto err_unmap;
1664	}
1665
1666	/* There should be nothing left to export exactly after looping over all SG elements */
1667	if (left_size_to_export) {
1668		dev_err(hdev->dev,
1669			"left size to export %#llx after initializing %u SG elements\n",
1670			left_size_to_export, sgt->nents);
1671		rc = -ENOMEM;
1672		goto err_unmap;
1673	}
1674
1675	/*
1676	 * Because we are not going to include a CPU list, we want to have some chance that other
1677	 * users will detect this when going over SG table, by setting the orig_nents to 0 and using
1678	 * only nents (length of DMA list).
1679	 */
1680	sgt->orig_nents = 0;
1681
1682	dev_dbg(hdev->dev, "prepared SG table with %u entries for importer %s\n",
1683		nents, dev_name(dev));
1684	for_each_sgtable_dma_sg(sgt, sg, i)
1685		dev_dbg(hdev->dev,
1686			"SG entry %d: address %#llx, length %#x\n",
1687			i, sg_dma_address(sg), sg_dma_len(sg));
1688
1689	return sgt;
1690
1691err_unmap:
1692	for_each_sgtable_dma_sg(sgt, sg, i) {
1693		if (!sg_dma_len(sg))
1694			continue;
1695
1696		dma_unmap_resource(dev, sg_dma_address(sg), sg_dma_len(sg), dir,
1697					DMA_ATTR_SKIP_CPU_SYNC);
1698	}
1699
1700	sg_free_table(sgt);
1701
1702err_free_sgt:
1703	kfree(sgt);
1704	return ERR_PTR(rc);
1705}
1706
1707static int hl_dmabuf_attach(struct dma_buf *dmabuf,
1708				struct dma_buf_attachment *attachment)
1709{
1710	struct hl_dmabuf_priv *hl_dmabuf;
1711	struct hl_device *hdev;
1712	int rc;
1713
1714	hl_dmabuf = dmabuf->priv;
1715	hdev = hl_dmabuf->ctx->hdev;
1716
1717	rc = pci_p2pdma_distance(hdev->pdev, attachment->dev, true);
1718
1719	if (rc < 0)
1720		attachment->peer2peer = false;
1721	return 0;
1722}
1723
1724static struct sg_table *hl_map_dmabuf(struct dma_buf_attachment *attachment,
1725					enum dma_data_direction dir)
1726{
1727	u64 *pages, npages, page_size, exported_size, offset;
1728	struct dma_buf *dma_buf = attachment->dmabuf;
1729	struct hl_vm_phys_pg_pack *phys_pg_pack;
1730	struct hl_dmabuf_priv *hl_dmabuf;
1731	struct hl_device *hdev;
1732	struct sg_table *sgt;
1733
1734	hl_dmabuf = dma_buf->priv;
1735	hdev = hl_dmabuf->ctx->hdev;
1736
1737	if (!attachment->peer2peer) {
1738		dev_dbg(hdev->dev, "Failed to map dmabuf because p2p is disabled\n");
1739		return ERR_PTR(-EPERM);
1740	}
1741
1742	exported_size = hl_dmabuf->dmabuf->size;
1743	offset = hl_dmabuf->offset;
1744	phys_pg_pack = hl_dmabuf->phys_pg_pack;
1745
1746	if (phys_pg_pack) {
1747		pages = phys_pg_pack->pages;
1748		npages = phys_pg_pack->npages;
1749		page_size = phys_pg_pack->page_size;
1750	} else {
1751		pages = &hl_dmabuf->device_phys_addr;
1752		npages = 1;
1753		page_size = hl_dmabuf->dmabuf->size;
1754	}
1755
1756	sgt = alloc_sgt_from_device_pages(hdev, pages, npages, page_size, exported_size, offset,
1757						attachment->dev, dir);
1758	if (IS_ERR(sgt))
1759		dev_err(hdev->dev, "failed (%ld) to initialize sgt for dmabuf\n", PTR_ERR(sgt));
1760
1761	return sgt;
1762}
1763
1764static void hl_unmap_dmabuf(struct dma_buf_attachment *attachment,
1765				  struct sg_table *sgt,
1766				  enum dma_data_direction dir)
1767{
1768	struct scatterlist *sg;
1769	int i;
1770
1771	/* The memory behind the dma-buf has *always* resided on the device itself, i.e. it lives
1772	 * only in the 'device' domain (after all, it maps a PCI bar address which points to the
1773	 * device memory).
1774	 *
1775	 * Therefore, it was never in the 'CPU' domain and hence, there is no need to perform
1776	 * a sync of the memory to the CPU's cache, as it never resided inside that cache.
1777	 */
1778	for_each_sgtable_dma_sg(sgt, sg, i)
1779		dma_unmap_resource(attachment->dev, sg_dma_address(sg),
1780					sg_dma_len(sg), dir,
1781					DMA_ATTR_SKIP_CPU_SYNC);
1782
1783	/* Need to restore orig_nents because sg_free_table use that field */
1784	sgt->orig_nents = sgt->nents;
1785	sg_free_table(sgt);
1786	kfree(sgt);
1787}
1788
1789static struct hl_vm_hash_node *memhash_node_export_get(struct hl_ctx *ctx, u64 addr)
1790{
1791	struct hl_device *hdev = ctx->hdev;
1792	struct hl_vm_hash_node *hnode;
1793
1794	/* get the memory handle */
1795	mutex_lock(&ctx->mem_hash_lock);
1796	hnode = get_vm_hash_node_locked(ctx, addr);
1797	if (!hnode) {
1798		mutex_unlock(&ctx->mem_hash_lock);
1799		dev_dbg(hdev->dev, "map address %#llx not found\n", addr);
1800		return ERR_PTR(-EINVAL);
1801	}
1802
1803	if (upper_32_bits(hnode->handle)) {
1804		mutex_unlock(&ctx->mem_hash_lock);
1805		dev_dbg(hdev->dev, "invalid handle %#llx for map address %#llx\n",
1806				hnode->handle, addr);
1807		return ERR_PTR(-EINVAL);
1808	}
1809
1810	/*
1811	 * node found, increase export count so this memory cannot be unmapped
1812	 * and the hash node cannot be deleted.
1813	 */
1814	hnode->export_cnt++;
1815	mutex_unlock(&ctx->mem_hash_lock);
1816
1817	return hnode;
1818}
1819
1820static void memhash_node_export_put(struct hl_ctx *ctx, struct hl_vm_hash_node *hnode)
1821{
1822	mutex_lock(&ctx->mem_hash_lock);
1823	hnode->export_cnt--;
1824	mutex_unlock(&ctx->mem_hash_lock);
1825}
1826
1827static void hl_release_dmabuf(struct dma_buf *dmabuf)
1828{
1829	struct hl_dmabuf_priv *hl_dmabuf = dmabuf->priv;
1830	struct hl_ctx *ctx;
1831
1832	ctx = hl_dmabuf->ctx;
1833
1834	if (hl_dmabuf->memhash_hnode)
1835		memhash_node_export_put(ctx, hl_dmabuf->memhash_hnode);
1836
1837	atomic_dec(&ctx->hdev->dmabuf_export_cnt);
1838	hl_ctx_put(ctx);
1839
1840	/*
1841	 * Paired with get_file() in export_dmabuf().
1842	 * 'ctx' can be still used here to get the file pointer, even after hl_ctx_put() was called,
1843	 * because releasing the compute device file involves another reference decrement, and it
1844	 * would be possible only after calling fput().
1845	 */
1846	fput(ctx->hpriv->file_priv->filp);
1847
1848	kfree(hl_dmabuf);
1849}
1850
1851static const struct dma_buf_ops habanalabs_dmabuf_ops = {
1852	.attach = hl_dmabuf_attach,
1853	.map_dma_buf = hl_map_dmabuf,
1854	.unmap_dma_buf = hl_unmap_dmabuf,
1855	.release = hl_release_dmabuf,
1856};
1857
1858static int export_dmabuf(struct hl_ctx *ctx,
1859				struct hl_dmabuf_priv *hl_dmabuf,
1860				u64 total_size, int flags, int *dmabuf_fd)
1861{
1862	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
1863	struct hl_device *hdev = ctx->hdev;
1864	CLASS(get_unused_fd, fd)(flags);
1865
1866	if (fd < 0) {
1867		dev_err(hdev->dev, "failed to get a file descriptor for a dma-buf, %d\n", fd);
1868		return fd;
1869	}
1870
1871	exp_info.ops = &habanalabs_dmabuf_ops;
1872	exp_info.size = total_size;
1873	exp_info.flags = flags;
1874	exp_info.priv = hl_dmabuf;
1875
1876	hl_dmabuf->dmabuf = dma_buf_export(&exp_info);
1877	if (IS_ERR(hl_dmabuf->dmabuf)) {
1878		dev_err(hdev->dev, "failed to export dma-buf\n");
1879		return PTR_ERR(hl_dmabuf->dmabuf);
1880	}
1881
1882	hl_dmabuf->ctx = ctx;
1883	hl_ctx_get(hl_dmabuf->ctx);
1884	atomic_inc(&ctx->hdev->dmabuf_export_cnt);
1885
1886	/* Get compute device file to enforce release order, such that all exported dma-buf will be
1887	 * released first and only then the compute device.
1888	 * Paired with fput() in hl_release_dmabuf().
1889	 */
1890	get_file(ctx->hpriv->file_priv->filp);
1891
1892	*dmabuf_fd = fd;
1893	fd_install(take_fd(fd), hl_dmabuf->dmabuf->file);
1894
1895	return 0;
1896}
1897
1898static int validate_export_params_common(struct hl_device *hdev, u64 addr, u64 size, u64 offset)
1899{
1900	if (!PAGE_ALIGNED(addr)) {
1901		dev_dbg(hdev->dev,
1902			"exported device memory address 0x%llx should be aligned to PAGE_SIZE 0x%lx\n",
1903			addr, PAGE_SIZE);
1904		return -EINVAL;
1905	}
1906
1907	if (!size || !PAGE_ALIGNED(size)) {
1908		dev_dbg(hdev->dev,
1909			"exported device memory size %llu should be a multiple of PAGE_SIZE %lu\n",
1910			size, PAGE_SIZE);
1911		return -EINVAL;
1912	}
1913
1914	if (!PAGE_ALIGNED(offset)) {
1915		dev_dbg(hdev->dev,
1916			"exported device memory offset %llu should be a multiple of PAGE_SIZE %lu\n",
1917			offset, PAGE_SIZE);
1918		return -EINVAL;
1919	}
1920
1921	return 0;
1922}
1923
1924static int validate_export_params_no_mmu(struct hl_device *hdev, u64 device_addr, u64 size)
1925{
1926	struct asic_fixed_properties *prop = &hdev->asic_prop;
1927	u64 bar_address;
1928	int rc;
1929
1930	rc = validate_export_params_common(hdev, device_addr, size, 0);
1931	if (rc)
1932		return rc;
1933
1934	if (device_addr < prop->dram_user_base_address ||
1935			(device_addr + size) > prop->dram_end_address ||
1936			(device_addr + size) < device_addr) {
1937		dev_dbg(hdev->dev,
1938			"DRAM memory range 0x%llx (+0x%llx) is outside of DRAM boundaries\n",
1939			device_addr, size);
1940		return -EINVAL;
1941	}
1942
1943	bar_address = hdev->dram_pci_bar_start + (device_addr - prop->dram_base_address);
1944
1945	if ((bar_address + size) > (hdev->dram_pci_bar_start + prop->dram_pci_bar_size) ||
1946			(bar_address + size) < bar_address) {
1947		dev_dbg(hdev->dev,
1948			"DRAM memory range 0x%llx (+0x%llx) is outside of PCI BAR boundaries\n",
1949			device_addr, size);
1950		return -EINVAL;
1951	}
1952
1953	return 0;
1954}
1955
1956static int validate_export_params(struct hl_device *hdev, u64 device_addr, u64 size, u64 offset,
1957					struct hl_vm_phys_pg_pack *phys_pg_pack)
1958{
1959	struct asic_fixed_properties *prop = &hdev->asic_prop;
1960	u64 bar_address;
1961	int i, rc;
1962
1963	rc = validate_export_params_common(hdev, device_addr, size, offset);
1964	if (rc)
1965		return rc;
1966
1967	if ((offset + size) > phys_pg_pack->total_size) {
1968		dev_dbg(hdev->dev, "offset %#llx and size %#llx exceed total map size %#llx\n",
1969			offset, size, phys_pg_pack->total_size);
1970		return -EINVAL;
1971	}
1972
1973	for (i = 0 ; i < phys_pg_pack->npages ; i++) {
1974		bar_address = hdev->dram_pci_bar_start +
1975				(phys_pg_pack->pages[i] - prop->dram_base_address);
1976
1977		if ((bar_address + phys_pg_pack->page_size) >
1978				(hdev->dram_pci_bar_start + prop->dram_pci_bar_size) ||
1979				(bar_address + phys_pg_pack->page_size) < bar_address) {
1980			dev_dbg(hdev->dev,
1981				"DRAM memory range 0x%llx (+0x%x) is outside of PCI BAR boundaries\n",
1982				phys_pg_pack->pages[i], phys_pg_pack->page_size);
1983			return -EINVAL;
1984		}
1985	}
1986
1987	return 0;
1988}
1989
1990static struct hl_vm_phys_pg_pack *get_phys_pg_pack_from_hash_node(struct hl_device *hdev,
1991							struct hl_vm_hash_node *hnode)
1992{
1993	struct hl_vm_phys_pg_pack *phys_pg_pack;
1994	struct hl_vm *vm = &hdev->vm;
1995
1996	spin_lock(&vm->idr_lock);
1997	phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, (u32) hnode->handle);
1998	if (!phys_pg_pack) {
1999		spin_unlock(&vm->idr_lock);
2000		dev_dbg(hdev->dev, "no match for handle 0x%x\n", (u32) hnode->handle);
2001		return ERR_PTR(-EINVAL);
2002	}
2003
2004	spin_unlock(&vm->idr_lock);
2005
2006	if (phys_pg_pack->vm_type != VM_TYPE_PHYS_PACK) {
2007		dev_dbg(hdev->dev, "handle 0x%llx does not represent DRAM memory\n", hnode->handle);
2008		return ERR_PTR(-EINVAL);
2009	}
2010
2011	return phys_pg_pack;
2012}
2013
2014/**
2015 * export_dmabuf_from_addr() - export a dma-buf object for the given memory
2016 *                             address and size.
2017 * @ctx: pointer to the context structure.
2018 * @addr: device address.
2019 * @size: size of device memory to export.
2020 * @offset: the offset into the buffer from which to start exporting
2021 * @flags: DMA-BUF file/FD flags.
2022 * @dmabuf_fd: pointer to result FD that represents the dma-buf object.
2023 *
2024 * Create and export a dma-buf object for an existing memory allocation inside
2025 * the device memory, and return a FD which is associated with the dma-buf
2026 * object.
2027 *
2028 * Return: 0 on success, non-zero for failure.
2029 */
2030static int export_dmabuf_from_addr(struct hl_ctx *ctx, u64 addr, u64 size, u64 offset,
2031					int flags, int *dmabuf_fd)
2032{
2033	struct hl_vm_phys_pg_pack *phys_pg_pack = NULL;
2034	struct hl_vm_hash_node *hnode = NULL;
2035	struct asic_fixed_properties *prop;
2036	struct hl_dmabuf_priv *hl_dmabuf;
2037	struct hl_device *hdev;
2038	int rc;
2039
2040	hdev = ctx->hdev;
2041	prop = &hdev->asic_prop;
2042
2043	/* offset must be 0 in devices without virtual memory support */
2044	if (!prop->dram_supports_virtual_memory && offset) {
2045		dev_dbg(hdev->dev, "offset is not allowed in device without virtual memory\n");
2046		return -EINVAL;
2047	}
2048
2049	hl_dmabuf = kzalloc(sizeof(*hl_dmabuf), GFP_KERNEL);
2050	if (!hl_dmabuf)
2051		return -ENOMEM;
2052
2053	if (prop->dram_supports_virtual_memory) {
2054		hnode = memhash_node_export_get(ctx, addr);
2055		if (IS_ERR(hnode)) {
2056			rc = PTR_ERR(hnode);
2057			goto err_free_dmabuf_wrapper;
2058		}
2059		phys_pg_pack = get_phys_pg_pack_from_hash_node(hdev, hnode);
2060		if (IS_ERR(phys_pg_pack)) {
2061			rc = PTR_ERR(phys_pg_pack);
2062			goto dec_memhash_export_cnt;
2063		}
2064		rc = validate_export_params(hdev, addr, size, offset, phys_pg_pack);
2065		if (rc)
2066			goto dec_memhash_export_cnt;
2067
2068		hl_dmabuf->phys_pg_pack = phys_pg_pack;
2069		hl_dmabuf->memhash_hnode = hnode;
2070		hl_dmabuf->offset = offset;
2071	} else {
2072		rc = validate_export_params_no_mmu(hdev, addr, size);
2073		if (rc)
2074			goto err_free_dmabuf_wrapper;
2075
2076		hl_dmabuf->device_phys_addr = addr;
2077	}
2078
2079	rc = export_dmabuf(ctx, hl_dmabuf, size, flags, dmabuf_fd);
2080	if (rc)
2081		goto dec_memhash_export_cnt;
2082
2083	return 0;
2084
2085dec_memhash_export_cnt:
2086	if (prop->dram_supports_virtual_memory)
2087		memhash_node_export_put(ctx, hnode);
2088err_free_dmabuf_wrapper:
2089	kfree(hl_dmabuf);
2090	return rc;
2091}
2092
2093static void ts_buff_release(struct hl_mmap_mem_buf *buf)
2094{
2095	struct hl_ts_buff *ts_buff = buf->private;
2096
2097	vfree(ts_buff->kernel_buff_address);
2098	vfree(ts_buff->user_buff_address);
2099	kfree(ts_buff);
2100}
2101
2102static int hl_ts_mmap(struct hl_mmap_mem_buf *buf, struct vm_area_struct *vma, void *args)
2103{
2104	struct hl_ts_buff *ts_buff = buf->private;
2105
2106	vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP | VM_DONTCOPY | VM_NORESERVE);
2107	return remap_vmalloc_range(vma, ts_buff->user_buff_address, 0);
2108}
2109
2110static int hl_ts_alloc_buf(struct hl_mmap_mem_buf *buf, gfp_t gfp, void *args)
2111{
2112	struct hl_ts_buff *ts_buff = NULL;
2113	u32 num_elements;
2114	size_t size;
2115	void *p;
2116
2117	num_elements = *(u32 *)args;
2118
2119	ts_buff = kzalloc(sizeof(*ts_buff), gfp);
2120	if (!ts_buff)
2121		return -ENOMEM;
2122
2123	/* Allocate the user buffer */
2124	size = num_elements * sizeof(u64);
2125	p = vmalloc_user(size);
2126	if (!p)
2127		goto free_mem;
2128
2129	ts_buff->user_buff_address = p;
2130	buf->mappable_size = size;
2131
2132	/* Allocate the internal kernel buffer */
2133	size = num_elements * sizeof(struct hl_user_pending_interrupt);
2134	p = vzalloc(size);
2135	if (!p)
2136		goto free_user_buff;
2137
2138	ts_buff->kernel_buff_address = p;
2139	ts_buff->kernel_buff_size = size;
2140
2141	buf->private = ts_buff;
2142
2143	return 0;
2144
2145free_user_buff:
2146	vfree(ts_buff->user_buff_address);
2147free_mem:
2148	kfree(ts_buff);
2149	return -ENOMEM;
2150}
2151
2152static struct hl_mmap_mem_buf_behavior hl_ts_behavior = {
2153	.topic = "TS",
2154	.mem_id = HL_MMAP_TYPE_TS_BUFF,
2155	.mmap = hl_ts_mmap,
2156	.alloc = hl_ts_alloc_buf,
2157	.release = ts_buff_release,
2158};
2159
2160/**
2161 * allocate_timestamps_buffers() - allocate timestamps buffers
2162 * This function will allocate ts buffer that will later on be mapped to the user
2163 * in order to be able to read the timestamp.
2164 * in addition it'll allocate an extra buffer for registration management.
2165 * since we cannot fail during registration for out-of-memory situation, so
2166 * we'll prepare a pool which will be used as user interrupt nodes and instead
2167 * of dynamically allocating nodes while registration we'll pick the node from
2168 * this pool. in addition it'll add node to the mapping hash which will be used
2169 * to map user ts buffer to the internal kernel ts buffer.
2170 * @hpriv: pointer to the private data of the fd
2171 * @args: ioctl input
2172 * @handle: user timestamp buffer handle as an output
2173 */
2174static int allocate_timestamps_buffers(struct hl_fpriv *hpriv, struct hl_mem_in *args, u64 *handle)
2175{
2176	struct hl_mem_mgr *mmg = &hpriv->mem_mgr;
2177	struct hl_mmap_mem_buf *buf;
2178
2179	if (args->num_of_elements > TS_MAX_ELEMENTS_NUM) {
2180		dev_err(mmg->dev, "Num of elements exceeds Max allowed number (0x%x > 0x%x)\n",
2181				args->num_of_elements, TS_MAX_ELEMENTS_NUM);
2182		return -EINVAL;
2183	}
2184
2185	buf = hl_mmap_mem_buf_alloc(mmg, &hl_ts_behavior, GFP_KERNEL, &args->num_of_elements);
2186	if (!buf)
2187		return -ENOMEM;
2188
2189	*handle = buf->handle;
2190
2191	return 0;
2192}
2193
2194int hl_mem_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv)
2195{
2196	struct hl_fpriv *hpriv = file_priv->driver_priv;
2197	enum hl_device_status status;
2198	union hl_mem_args *args = data;
2199	struct hl_device *hdev = hpriv->hdev;
2200	struct hl_ctx *ctx = hpriv->ctx;
2201	u64 block_handle, device_addr = 0;
2202	u32 handle = 0, block_size;
2203	int rc, dmabuf_fd = -EBADF;
2204
2205	if (!hl_device_operational(hdev, &status)) {
2206		dev_dbg_ratelimited(hdev->dev,
2207			"Device is %s. Can't execute MEMORY IOCTL\n",
2208			hdev->status[status]);
2209		return -EBUSY;
2210	}
2211
2212	switch (args->in.op) {
2213	case HL_MEM_OP_ALLOC:
2214		if (args->in.alloc.mem_size == 0) {
2215			dev_err(hdev->dev,
2216				"alloc size must be larger than 0\n");
2217			rc = -EINVAL;
2218			goto out;
2219		}
2220
2221		/* If DRAM does not support virtual memory the driver won't
2222		 * handle the allocation/freeing of that memory. However, for
2223		 * system administration/monitoring purposes, the driver will
2224		 * keep track of the amount of DRAM memory that is allocated
2225		 * and freed by the user. Because this code totally relies on
2226		 * the user's input, the driver can't ensure the validity
2227		 * of this accounting.
2228		 */
2229		if (!hdev->asic_prop.dram_supports_virtual_memory) {
2230			atomic64_add(args->in.alloc.mem_size,
2231					&ctx->dram_phys_mem);
2232			atomic64_add(args->in.alloc.mem_size,
2233					&hdev->dram_used_mem);
2234
2235			dev_dbg(hdev->dev, "DRAM alloc is not supported\n");
2236			rc = 0;
2237
2238			memset(args, 0, sizeof(*args));
2239			args->out.handle = 0;
2240			goto out;
2241		}
2242
2243		rc = alloc_device_memory(ctx, &args->in, &handle);
2244
2245		memset(args, 0, sizeof(*args));
2246		args->out.handle = (__u64) handle;
2247		break;
2248
2249	case HL_MEM_OP_FREE:
2250		/* If DRAM does not support virtual memory the driver won't
2251		 * handle the allocation/freeing of that memory. However, for
2252		 * system administration/monitoring purposes, the driver will
2253		 * keep track of the amount of DRAM memory that is allocated
2254		 * and freed by the user. Because this code totally relies on
2255		 * the user's input, the driver can't ensure the validity
2256		 * of this accounting.
2257		 */
2258		if (!hdev->asic_prop.dram_supports_virtual_memory) {
2259			atomic64_sub(args->in.alloc.mem_size,
2260					&ctx->dram_phys_mem);
2261			atomic64_sub(args->in.alloc.mem_size,
2262					&hdev->dram_used_mem);
2263
2264			dev_dbg(hdev->dev, "DRAM alloc is not supported\n");
2265			rc = 0;
2266
2267			goto out;
2268		}
2269
2270		rc = free_device_memory(ctx, &args->in);
2271		break;
2272
2273	case HL_MEM_OP_MAP:
2274		rc = map_device_va(ctx, &args->in, &device_addr);
2275
2276		memset(args, 0, sizeof(*args));
2277		args->out.device_virt_addr = device_addr;
2278		break;
2279
2280	case HL_MEM_OP_UNMAP:
2281		rc = unmap_device_va(ctx, &args->in, false);
2282		break;
2283
2284	case HL_MEM_OP_MAP_BLOCK:
2285		rc = map_block(hdev, args->in.map_block.block_addr,
2286				&block_handle, &block_size);
2287		args->out.block_handle = block_handle;
2288		args->out.block_size = block_size;
2289		break;
2290
2291	case HL_MEM_OP_EXPORT_DMABUF_FD:
2292		rc = export_dmabuf_from_addr(ctx,
2293				args->in.export_dmabuf_fd.addr,
2294				args->in.export_dmabuf_fd.mem_size,
2295				args->in.export_dmabuf_fd.offset,
2296				args->in.flags,
2297				&dmabuf_fd);
2298		memset(args, 0, sizeof(*args));
2299		args->out.fd = dmabuf_fd;
2300		break;
2301
2302	case HL_MEM_OP_TS_ALLOC:
2303		rc = allocate_timestamps_buffers(hpriv, &args->in, &args->out.handle);
2304		break;
2305	default:
2306		dev_err(hdev->dev, "Unknown opcode for memory IOCTL\n");
2307		rc = -EINVAL;
2308		break;
2309	}
2310
2311out:
2312	return rc;
2313}
2314
2315static int get_user_memory(struct hl_device *hdev, u64 addr, u64 size,
2316				u32 npages, u64 start, u32 offset,
2317				struct hl_userptr *userptr)
2318{
2319	int rc;
2320
2321	if (!access_ok((void __user *) (uintptr_t) addr, size)) {
2322		dev_err(hdev->dev, "user pointer is invalid - 0x%llx\n", addr);
2323		return -EFAULT;
2324	}
2325
2326	userptr->pages = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
2327	if (!userptr->pages)
2328		return -ENOMEM;
2329
2330	rc = pin_user_pages_fast(start, npages, FOLL_WRITE | FOLL_LONGTERM,
2331				 userptr->pages);
2332
2333	if (rc != npages) {
2334		dev_err(hdev->dev,
2335			"Failed (%d) to pin host memory with user ptr 0x%llx, size 0x%llx, npages %d\n",
2336			rc, addr, size, npages);
2337		if (rc < 0)
2338			goto destroy_pages;
2339		npages = rc;
2340		rc = -ENOMEM;
2341		goto put_pages;
2342	}
2343	userptr->npages = npages;
2344
2345	rc = sg_alloc_table_from_pages(userptr->sgt,
2346				       userptr->pages,
2347				       npages, offset, size, GFP_KERNEL);
2348	if (rc < 0) {
2349		dev_err(hdev->dev, "failed to create SG table from pages\n");
2350		goto put_pages;
2351	}
2352
2353	return 0;
2354
2355put_pages:
2356	unpin_user_pages(userptr->pages, npages);
2357destroy_pages:
2358	kvfree(userptr->pages);
2359	return rc;
2360}
2361
2362/**
2363 * hl_pin_host_memory() - pins a chunk of host memory.
2364 * @hdev: pointer to the habanalabs device structure.
2365 * @addr: the host virtual address of the memory area.
2366 * @size: the size of the memory area.
2367 * @userptr: pointer to hl_userptr structure.
2368 *
2369 * This function does the following:
2370 * - Pins the physical pages.
2371 * - Create an SG list from those pages.
2372 */
2373int hl_pin_host_memory(struct hl_device *hdev, u64 addr, u64 size,
2374					struct hl_userptr *userptr)
2375{
2376	u64 start, end;
2377	u32 npages, offset;
2378	int rc;
2379
2380	if (!size) {
2381		dev_err(hdev->dev, "size to pin is invalid - %llu\n", size);
2382		return -EINVAL;
2383	}
2384
2385	/*
2386	 * If the combination of the address and size requested for this memory
2387	 * region causes an integer overflow, return error.
2388	 */
2389	if (((addr + size) < addr) ||
2390			PAGE_ALIGN(addr + size) < (addr + size)) {
2391		dev_err(hdev->dev,
2392			"user pointer 0x%llx + %llu causes integer overflow\n",
2393			addr, size);
2394		return -EINVAL;
2395	}
2396
2397	userptr->pid = current->pid;
2398	userptr->sgt = kzalloc(sizeof(*userptr->sgt), GFP_KERNEL);
2399	if (!userptr->sgt)
2400		return -ENOMEM;
2401
2402	start = addr & PAGE_MASK;
2403	offset = addr & ~PAGE_MASK;
2404	end = PAGE_ALIGN(addr + size);
2405	npages = (end - start) >> PAGE_SHIFT;
2406
2407	userptr->size = size;
2408	userptr->addr = addr;
2409	userptr->dma_mapped = false;
2410	INIT_LIST_HEAD(&userptr->job_node);
2411
2412	rc = get_user_memory(hdev, addr, size, npages, start, offset,
2413				userptr);
2414	if (rc) {
2415		dev_err(hdev->dev,
2416			"failed to get user memory for address 0x%llx\n",
2417			addr);
2418		goto free_sgt;
2419	}
2420
2421	hl_debugfs_add_userptr(hdev, userptr);
2422
2423	return 0;
2424
2425free_sgt:
2426	kfree(userptr->sgt);
2427	return rc;
2428}
2429
2430/*
2431 * hl_unpin_host_memory - unpins a chunk of host memory.
2432 * @hdev: pointer to the habanalabs device structure
2433 * @userptr: pointer to hl_userptr structure
2434 *
2435 * This function does the following:
2436 * - Unpins the physical pages related to the host memory
2437 * - Free the SG list
2438 */
2439void hl_unpin_host_memory(struct hl_device *hdev, struct hl_userptr *userptr)
2440{
2441	hl_debugfs_remove_userptr(hdev, userptr);
2442
2443	if (userptr->dma_mapped)
2444		hl_dma_unmap_sgtable(hdev, userptr->sgt, userptr->dir);
2445
2446	unpin_user_pages_dirty_lock(userptr->pages, userptr->npages, true);
2447	kvfree(userptr->pages);
2448
2449	list_del(&userptr->job_node);
2450
2451	sg_free_table(userptr->sgt);
2452	kfree(userptr->sgt);
2453}
2454
2455/**
2456 * hl_userptr_delete_list() - clear userptr list.
2457 * @hdev: pointer to the habanalabs device structure.
2458 * @userptr_list: pointer to the list to clear.
2459 *
2460 * This function does the following:
2461 * - Iterates over the list and unpins the host memory and frees the userptr
2462 *   structure.
2463 */
2464void hl_userptr_delete_list(struct hl_device *hdev,
2465				struct list_head *userptr_list)
2466{
2467	struct hl_userptr *userptr, *tmp;
2468
2469	list_for_each_entry_safe(userptr, tmp, userptr_list, job_node) {
2470		hl_unpin_host_memory(hdev, userptr);
2471		kfree(userptr);
2472	}
2473
2474	INIT_LIST_HEAD(userptr_list);
2475}
2476
2477/**
2478 * hl_userptr_is_pinned() - returns whether the given userptr is pinned.
2479 * @hdev: pointer to the habanalabs device structure.
2480 * @addr: user address to check.
2481 * @size: user block size to check.
2482 * @userptr_list: pointer to the list to clear.
2483 * @userptr: pointer to userptr to check.
2484 *
2485 * This function does the following:
2486 * - Iterates over the list and checks if the given userptr is in it, means is
2487 *   pinned. If so, returns true, otherwise returns false.
2488 */
2489bool hl_userptr_is_pinned(struct hl_device *hdev, u64 addr,
2490				u32 size, struct list_head *userptr_list,
2491				struct hl_userptr **userptr)
2492{
2493	list_for_each_entry((*userptr), userptr_list, job_node) {
2494		if ((addr == (*userptr)->addr) && (size == (*userptr)->size))
2495			return true;
2496	}
2497
2498	return false;
2499}
2500
2501/**
2502 * va_range_init() - initialize virtual addresses range.
2503 * @hdev: pointer to the habanalabs device structure.
2504 * @va_ranges: pointer to va_ranges array.
2505 * @range_type: virtual address range type.
2506 * @start: range start address, inclusive.
2507 * @end: range end address, inclusive.
2508 * @page_size: page size for this va_range.
2509 *
2510 * This function does the following:
2511 * - Initializes the virtual addresses list of the given range with the given
2512 *   addresses.
2513 */
2514static int va_range_init(struct hl_device *hdev, struct hl_va_range **va_ranges,
2515				enum hl_va_range_type range_type, u64 start,
2516				u64 end, u32 page_size)
2517{
2518	struct hl_va_range *va_range = va_ranges[range_type];
2519	int rc;
2520
2521	INIT_LIST_HEAD(&va_range->list);
2522
2523	/*
2524	 * PAGE_SIZE alignment
2525	 * it is the caller's responsibility to align the addresses if the
2526	 * page size is not a power of 2
2527	 */
2528
2529	if (is_power_of_2(page_size)) {
2530		start = round_up(start, page_size);
2531
2532		/*
2533		 * The end of the range is inclusive, hence we need to align it
2534		 * to the end of the last full page in the range. For example if
2535		 * end = 0x3ff5 with page size 0x1000, we need to align it to
2536		 * 0x2fff. The remaining 0xff5 bytes do not form a full page.
2537		 */
2538		end = round_down(end + 1, page_size) - 1;
2539	}
2540
2541	if (start >= end) {
2542		dev_err(hdev->dev, "too small vm range for va list\n");
2543		return -EFAULT;
2544	}
2545
2546	rc = add_va_block(hdev, va_range, start, end);
2547
2548	if (rc) {
2549		dev_err(hdev->dev, "Failed to init host va list\n");
2550		return rc;
2551	}
2552
2553	va_range->start_addr = start;
2554	va_range->end_addr = end;
2555	va_range->page_size = page_size;
2556
2557	return 0;
2558}
2559
2560/**
2561 * va_range_fini() - clear a virtual addresses range.
2562 * @hdev: pointer to the habanalabs structure.
2563 * @va_range: pointer to virtual addresses range.
2564 *
2565 * This function does the following:
2566 * - Frees the virtual addresses block list and its lock.
2567 */
2568static void va_range_fini(struct hl_device *hdev, struct hl_va_range *va_range)
2569{
2570	mutex_lock(&va_range->lock);
2571	clear_va_list_locked(hdev, &va_range->list);
2572	mutex_unlock(&va_range->lock);
2573
2574	mutex_destroy(&va_range->lock);
2575	kfree(va_range);
2576}
2577
2578/**
2579 * vm_ctx_init_with_ranges() - initialize virtual memory for context.
2580 * @ctx: pointer to the habanalabs context structure.
2581 * @host_range_start: host virtual addresses range start.
2582 * @host_range_end: host virtual addresses range end.
2583 * @host_page_size: host page size.
2584 * @host_huge_range_start: host virtual addresses range start for memory
2585 *                         allocated with huge pages.
2586 * @host_huge_range_end: host virtual addresses range end for memory allocated
2587 *                        with huge pages.
2588 * @host_huge_page_size: host huge page size.
2589 * @dram_range_start: dram virtual addresses range start.
2590 * @dram_range_end: dram virtual addresses range end.
2591 * @dram_page_size: dram page size.
2592 *
2593 * This function initializes the following:
2594 * - MMU for context.
2595 * - Virtual address to area descriptor hashtable.
2596 * - Virtual block list of available virtual memory.
2597 */
2598static int vm_ctx_init_with_ranges(struct hl_ctx *ctx,
2599					u64 host_range_start,
2600					u64 host_range_end,
2601					u32 host_page_size,
2602					u64 host_huge_range_start,
2603					u64 host_huge_range_end,
2604					u32 host_huge_page_size,
2605					u64 dram_range_start,
2606					u64 dram_range_end,
2607					u32 dram_page_size)
2608{
2609	struct hl_device *hdev = ctx->hdev;
2610	int i, rc;
2611
2612	for (i = 0 ; i < HL_VA_RANGE_TYPE_MAX ; i++) {
2613		ctx->va_range[i] =
2614			kzalloc(sizeof(struct hl_va_range), GFP_KERNEL);
2615		if (!ctx->va_range[i]) {
2616			rc = -ENOMEM;
2617			goto free_va_range;
2618		}
2619	}
2620
2621	rc = hl_mmu_ctx_init(ctx);
2622	if (rc) {
2623		dev_err(hdev->dev, "failed to init context %d\n", ctx->asid);
2624		goto free_va_range;
2625	}
2626
2627	mutex_init(&ctx->mem_hash_lock);
2628	hash_init(ctx->mem_hash);
2629
2630	mutex_init(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock);
2631
2632	rc = va_range_init(hdev, ctx->va_range, HL_VA_RANGE_TYPE_HOST,
2633			host_range_start, host_range_end, host_page_size);
2634	if (rc) {
2635		dev_err(hdev->dev, "failed to init host vm range\n");
2636		goto mmu_ctx_fini;
2637	}
2638
2639	if (hdev->pmmu_huge_range) {
2640		mutex_init(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock);
2641
2642		rc = va_range_init(hdev,
2643			ctx->va_range, HL_VA_RANGE_TYPE_HOST_HUGE,
2644			host_huge_range_start, host_huge_range_end,
2645			host_huge_page_size);
2646		if (rc) {
2647			dev_err(hdev->dev,
2648				"failed to init host huge vm range\n");
2649			goto clear_host_va_range;
2650		}
2651	} else {
2652		kfree(ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]);
2653		ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE] =
2654				ctx->va_range[HL_VA_RANGE_TYPE_HOST];
2655	}
2656
2657	mutex_init(&ctx->va_range[HL_VA_RANGE_TYPE_DRAM]->lock);
2658
2659	rc = va_range_init(hdev, ctx->va_range, HL_VA_RANGE_TYPE_DRAM,
2660			dram_range_start, dram_range_end, dram_page_size);
2661	if (rc) {
2662		dev_err(hdev->dev, "failed to init dram vm range\n");
2663		goto clear_host_huge_va_range;
2664	}
2665
2666	hl_debugfs_add_ctx_mem_hash(hdev, ctx);
2667
2668	return 0;
2669
2670clear_host_huge_va_range:
2671	mutex_destroy(&ctx->va_range[HL_VA_RANGE_TYPE_DRAM]->lock);
2672
2673	if (hdev->pmmu_huge_range) {
2674		mutex_lock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock);
2675		clear_va_list_locked(hdev,
2676			&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->list);
2677		mutex_unlock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock);
2678	}
2679clear_host_va_range:
2680	if (hdev->pmmu_huge_range)
2681		mutex_destroy(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock);
2682	mutex_lock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock);
2683	clear_va_list_locked(hdev, &ctx->va_range[HL_VA_RANGE_TYPE_HOST]->list);
2684	mutex_unlock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock);
2685mmu_ctx_fini:
2686	mutex_destroy(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock);
2687	mutex_destroy(&ctx->mem_hash_lock);
2688	hl_mmu_ctx_fini(ctx);
2689free_va_range:
2690	for (i = 0 ; i < HL_VA_RANGE_TYPE_MAX ; i++)
2691		kfree(ctx->va_range[i]);
2692
2693	return rc;
2694}
2695
2696int hl_vm_ctx_init(struct hl_ctx *ctx)
2697{
2698	struct asic_fixed_properties *prop = &ctx->hdev->asic_prop;
2699	u64 host_range_start, host_range_end, host_huge_range_start,
2700		host_huge_range_end, dram_range_start, dram_range_end;
2701	u32 host_page_size, host_huge_page_size, dram_page_size;
2702
2703	atomic64_set(&ctx->dram_phys_mem, 0);
2704
2705	/*
2706	 *   In case of DRAM mapping, the returned address is the physical
2707	 *   address of the memory related to the given handle.
2708	 */
2709	if (ctx->hdev->mmu_disable)
2710		return 0;
2711
2712	dram_range_start = prop->dmmu.start_addr;
2713	dram_range_end = prop->dmmu.end_addr - 1;
2714	dram_page_size = prop->dram_page_size ?
2715				prop->dram_page_size : prop->dmmu.page_size;
2716	host_range_start = prop->pmmu.start_addr;
2717	host_range_end = prop->pmmu.end_addr - 1;
2718	host_page_size = prop->pmmu.page_size;
2719	host_huge_range_start = prop->pmmu_huge.start_addr;
2720	host_huge_range_end = prop->pmmu_huge.end_addr - 1;
2721	host_huge_page_size = prop->pmmu_huge.page_size;
2722
2723	return vm_ctx_init_with_ranges(ctx, host_range_start, host_range_end,
2724			host_page_size, host_huge_range_start,
2725			host_huge_range_end, host_huge_page_size,
2726			dram_range_start, dram_range_end, dram_page_size);
2727}
2728
2729/**
2730 * hl_vm_ctx_fini() - virtual memory teardown of context.
2731 * @ctx: pointer to the habanalabs context structure.
2732 *
2733 * This function perform teardown the following:
2734 * - Virtual block list of available virtual memory.
2735 * - Virtual address to area descriptor hashtable.
2736 * - MMU for context.
2737 *
2738 * In addition this function does the following:
2739 * - Unmaps the existing hashtable nodes if the hashtable is not empty. The
2740 *   hashtable should be empty as no valid mappings should exist at this
2741 *   point.
2742 * - Frees any existing physical page list from the idr which relates to the
2743 *   current context asid.
2744 * - This function checks the virtual block list for correctness. At this point
2745 *   the list should contain one element which describes the whole virtual
2746 *   memory range of the context. Otherwise, a warning is printed.
2747 */
2748void hl_vm_ctx_fini(struct hl_ctx *ctx)
2749{
2750	struct hl_vm_phys_pg_pack *phys_pg_list, *tmp_phys_node;
2751	struct hl_device *hdev = ctx->hdev;
2752	struct hl_vm_hash_node *hnode;
2753	struct hl_vm *vm = &hdev->vm;
2754	struct hlist_node *tmp_node;
2755	struct list_head free_list;
2756	struct hl_mem_in args;
2757	int i;
2758
2759	if (hdev->mmu_disable)
2760		return;
2761
2762	hl_debugfs_remove_ctx_mem_hash(hdev, ctx);
2763
2764	/*
2765	 * Clearly something went wrong on hard reset so no point in printing
2766	 * another side effect error
2767	 */
2768	if (!hdev->reset_info.hard_reset_pending && !hash_empty(ctx->mem_hash))
2769		dev_dbg(hdev->dev,
2770			"user released device without removing its memory mappings\n");
2771
2772	hash_for_each_safe(ctx->mem_hash, i, tmp_node, hnode, node) {
2773		dev_dbg(hdev->dev,
2774			"hl_mem_hash_node of vaddr 0x%llx of asid %d is still alive\n",
2775			hnode->vaddr, ctx->asid);
2776		args.unmap.device_virt_addr = hnode->vaddr;
2777		unmap_device_va(ctx, &args, true);
2778	}
2779
2780	mutex_lock(&hdev->mmu_lock);
2781
2782	/* invalidate the cache once after the unmapping loop */
2783	hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR);
2784	hl_mmu_invalidate_cache(hdev, true, MMU_OP_PHYS_PACK);
2785
2786	mutex_unlock(&hdev->mmu_lock);
2787
2788	INIT_LIST_HEAD(&free_list);
2789
2790	spin_lock(&vm->idr_lock);
2791	idr_for_each_entry(&vm->phys_pg_pack_handles, phys_pg_list, i)
2792		if (phys_pg_list->asid == ctx->asid) {
2793			dev_dbg(hdev->dev,
2794				"page list 0x%px of asid %d is still alive\n",
2795				phys_pg_list, ctx->asid);
2796
2797			atomic64_sub(phys_pg_list->total_size, &hdev->dram_used_mem);
2798			idr_remove(&vm->phys_pg_pack_handles, i);
2799			list_add(&phys_pg_list->node, &free_list);
2800		}
2801	spin_unlock(&vm->idr_lock);
2802
2803	list_for_each_entry_safe(phys_pg_list, tmp_phys_node, &free_list, node)
2804		free_phys_pg_pack(hdev, phys_pg_list);
2805
2806	va_range_fini(hdev, ctx->va_range[HL_VA_RANGE_TYPE_DRAM]);
2807	va_range_fini(hdev, ctx->va_range[HL_VA_RANGE_TYPE_HOST]);
2808
2809	if (hdev->pmmu_huge_range)
2810		va_range_fini(hdev, ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]);
2811
2812	mutex_destroy(&ctx->mem_hash_lock);
2813	hl_mmu_ctx_fini(ctx);
2814
2815	/* In this case we need to clear the global accounting of DRAM usage
2816	 * because the user notifies us on allocations. If the user is no more,
2817	 * all DRAM is available
2818	 */
2819	if (ctx->asid != HL_KERNEL_ASID_ID &&
2820			!hdev->asic_prop.dram_supports_virtual_memory)
2821		atomic64_set(&hdev->dram_used_mem, 0);
2822}
2823
2824/**
2825 * hl_vm_init() - initialize virtual memory module.
2826 * @hdev: pointer to the habanalabs device structure.
2827 *
2828 * This function initializes the following:
2829 * - MMU module.
2830 * - DRAM physical pages pool of 2MB.
2831 * - Idr for device memory allocation handles.
2832 */
2833int hl_vm_init(struct hl_device *hdev)
2834{
2835	struct asic_fixed_properties *prop = &hdev->asic_prop;
2836	struct hl_vm *vm = &hdev->vm;
2837	int rc;
2838
2839	if (is_power_of_2(prop->dram_page_size))
2840		vm->dram_pg_pool =
2841			gen_pool_create(__ffs(prop->dram_page_size), -1);
2842	else
2843		vm->dram_pg_pool =
2844			gen_pool_create(__ffs(DRAM_POOL_PAGE_SIZE), -1);
2845
2846	if (!vm->dram_pg_pool) {
2847		dev_err(hdev->dev, "Failed to create dram page pool\n");
2848		return -ENOMEM;
2849	}
2850
2851	kref_init(&vm->dram_pg_pool_refcount);
2852
2853	rc = gen_pool_add(vm->dram_pg_pool, prop->dram_user_base_address,
2854			prop->dram_end_address - prop->dram_user_base_address,
2855			-1);
2856
2857	if (rc) {
2858		dev_err(hdev->dev,
2859			"Failed to add memory to dram page pool %d\n", rc);
2860		goto pool_add_err;
2861	}
2862
2863	spin_lock_init(&vm->idr_lock);
2864	idr_init(&vm->phys_pg_pack_handles);
2865
2866	atomic64_set(&hdev->dram_used_mem, 0);
2867
2868	vm->init_done = true;
2869
2870	return 0;
2871
2872pool_add_err:
2873	gen_pool_destroy(vm->dram_pg_pool);
2874
2875	return rc;
2876}
2877
2878/**
2879 * hl_vm_fini() - virtual memory module teardown.
2880 * @hdev: pointer to the habanalabs device structure.
2881 *
2882 * This function perform teardown to the following:
2883 * - Idr for device memory allocation handles.
2884 * - DRAM physical pages pool of 2MB.
2885 * - MMU module.
2886 */
2887void hl_vm_fini(struct hl_device *hdev)
2888{
2889	struct hl_vm *vm = &hdev->vm;
2890
2891	if (!vm->init_done)
2892		return;
2893
2894	/*
2895	 * At this point all the contexts should be freed and hence no DRAM
2896	 * memory should be in use. Hence the DRAM pool should be freed here.
2897	 */
2898	if (kref_put(&vm->dram_pg_pool_refcount, dram_pg_pool_do_release) != 1)
2899		dev_warn(hdev->dev, "dram_pg_pool was not destroyed on %s\n",
2900				__func__);
2901
2902	vm->init_done = false;
2903}
2904
2905/**
2906 * hl_hw_block_mem_init() - HW block memory initialization.
2907 * @ctx: pointer to the habanalabs context structure.
2908 *
2909 * This function initializes the HW block virtual mapped addresses list and
2910 * it's lock.
2911 */
2912void hl_hw_block_mem_init(struct hl_ctx *ctx)
2913{
2914	mutex_init(&ctx->hw_block_list_lock);
2915	INIT_LIST_HEAD(&ctx->hw_block_mem_list);
2916}
2917
2918/**
2919 * hl_hw_block_mem_fini() - HW block memory teardown.
2920 * @ctx: pointer to the habanalabs context structure.
2921 *
2922 * This function clears the HW block virtual mapped addresses list and destroys
2923 * it's lock.
2924 */
2925void hl_hw_block_mem_fini(struct hl_ctx *ctx)
2926{
2927	struct hl_vm_hw_block_list_node *lnode, *tmp;
2928
2929	if (!list_empty(&ctx->hw_block_mem_list))
2930		dev_crit(ctx->hdev->dev, "HW block mem list isn't empty\n");
2931
2932	list_for_each_entry_safe(lnode, tmp, &ctx->hw_block_mem_list, node) {
2933		list_del(&lnode->node);
2934		kfree(lnode);
2935	}
2936
2937	mutex_destroy(&ctx->hw_block_list_lock);
2938}