aie2_pci.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (C) 2023-2024, Advanced Micro Devices, Inc.
   4 */
   5
   6#include <drm/amdxdna_accel.h>
   7#include <drm/drm_device.h>
   8#include <drm/drm_drv.h>
   9#include <drm/drm_gem_shmem_helper.h>
  10#include <drm/drm_managed.h>
  11#include <drm/drm_print.h>
  12#include <drm/gpu_scheduler.h>
  13#include <linux/cleanup.h>
  14#include <linux/errno.h>
  15#include <linux/firmware.h>
  16#include <linux/iommu.h>
  17#include <linux/iopoll.h>
  18#include <linux/pci.h>
  19#include <linux/xarray.h>
  20#include <asm/hypervisor.h>
  21
  22#include "aie2_msg_priv.h"
  23#include "aie2_pci.h"
  24#include "aie2_solver.h"
  25#include "amdxdna_ctx.h"
  26#include "amdxdna_gem.h"
  27#include "amdxdna_mailbox.h"
  28#include "amdxdna_pci_drv.h"
  29#include "amdxdna_pm.h"
  30
  31static int aie2_max_col = XRS_MAX_COL;
  32module_param(aie2_max_col, uint, 0600);
  33MODULE_PARM_DESC(aie2_max_col, "Maximum column could be used");
  34
  35/*
  36 * The management mailbox channel is allocated by firmware.
  37 * The related register and ring buffer information is on SRAM BAR.
  38 * This struct is the register layout.
  39 */
  40#define MGMT_MBOX_MAGIC 0x55504e5f /* _NPU */
  41struct mgmt_mbox_chann_info {
  42	__u32	x2i_tail;
  43	__u32	x2i_head;
  44	__u32	x2i_buf;
  45	__u32	x2i_buf_sz;
  46	__u32	i2x_tail;
  47	__u32	i2x_head;
  48	__u32	i2x_buf;
  49	__u32	i2x_buf_sz;
  50	__u32	magic;
  51	__u32	msi_id;
  52	__u32	prot_major;
  53	__u32	prot_minor;
  54	__u32	rsvd[4];
  55};
  56
  57static int aie2_check_protocol(struct amdxdna_dev_hdl *ndev, u32 fw_major, u32 fw_minor)
  58{
  59	const struct aie2_fw_feature_tbl *feature;
  60	struct amdxdna_dev *xdna = ndev->xdna;
  61
  62	/*
  63	 * The driver supported mailbox behavior is defined by
  64	 * ndev->priv->protocol_major and protocol_minor.
  65	 *
  66	 * When protocol_major and fw_major are different, it means driver
  67	 * and firmware are incompatible.
  68	 */
  69	if (ndev->priv->protocol_major != fw_major) {
  70		XDNA_ERR(xdna, "Incompatible firmware protocol major %d minor %d",
  71			 fw_major, fw_minor);
  72		return -EINVAL;
  73	}
  74
  75	/*
  76	 * When protocol_minor is greater then fw_minor, that means driver
  77	 * relies on operation the installed firmware does not support.
  78	 */
  79	if (ndev->priv->protocol_minor > fw_minor) {
  80		XDNA_ERR(xdna, "Firmware minor version smaller than supported");
  81		return -EINVAL;
  82	}
  83
  84	for (feature = ndev->priv->fw_feature_tbl; feature && feature->min_minor;
  85	     feature++) {
  86		if (fw_minor < feature->min_minor)
  87			continue;
  88		if (feature->max_minor > 0 && fw_minor > feature->max_minor)
  89			continue;
  90
  91		set_bit(feature->feature, &ndev->feature_mask);
  92	}
  93
  94	return 0;
  95}
  96
  97static void aie2_dump_chann_info_debug(struct amdxdna_dev_hdl *ndev)
  98{
  99	struct amdxdna_dev *xdna = ndev->xdna;
 100
 101	XDNA_DBG(xdna, "i2x tail    0x%x", ndev->mgmt_i2x.mb_tail_ptr_reg);
 102	XDNA_DBG(xdna, "i2x head    0x%x", ndev->mgmt_i2x.mb_head_ptr_reg);
 103	XDNA_DBG(xdna, "i2x ringbuf 0x%x", ndev->mgmt_i2x.rb_start_addr);
 104	XDNA_DBG(xdna, "i2x rsize   0x%x", ndev->mgmt_i2x.rb_size);
 105	XDNA_DBG(xdna, "x2i tail    0x%x", ndev->mgmt_x2i.mb_tail_ptr_reg);
 106	XDNA_DBG(xdna, "x2i head    0x%x", ndev->mgmt_x2i.mb_head_ptr_reg);
 107	XDNA_DBG(xdna, "x2i ringbuf 0x%x", ndev->mgmt_x2i.rb_start_addr);
 108	XDNA_DBG(xdna, "x2i rsize   0x%x", ndev->mgmt_x2i.rb_size);
 109	XDNA_DBG(xdna, "x2i chann index 0x%x", ndev->mgmt_chan_idx);
 110	XDNA_DBG(xdna, "mailbox protocol major 0x%x", ndev->mgmt_prot_major);
 111	XDNA_DBG(xdna, "mailbox protocol minor 0x%x", ndev->mgmt_prot_minor);
 112}
 113
 114static int aie2_get_mgmt_chann_info(struct amdxdna_dev_hdl *ndev)
 115{
 116	struct mgmt_mbox_chann_info info_regs;
 117	struct xdna_mailbox_chann_res *i2x;
 118	struct xdna_mailbox_chann_res *x2i;
 119	u32 addr, off;
 120	u32 *reg;
 121	int ret;
 122	int i;
 123
 124	/*
 125	 * Once firmware is alive, it will write management channel
 126	 * information in SRAM BAR and write the address of that information
 127	 * at FW_ALIVE_OFF offset in SRMA BAR.
 128	 *
 129	 * Read a non-zero value from FW_ALIVE_OFF implies that firmware
 130	 * is alive.
 131	 */
 132	ret = readx_poll_timeout(readl, SRAM_GET_ADDR(ndev, FW_ALIVE_OFF),
 133				 addr, addr, AIE2_INTERVAL, AIE2_TIMEOUT);
 134	if (ret || !addr)
 135		return -ETIME;
 136
 137	off = AIE2_SRAM_OFF(ndev, addr);
 138	reg = (u32 *)&info_regs;
 139	for (i = 0; i < sizeof(info_regs) / sizeof(u32); i++)
 140		reg[i] = readl(ndev->sram_base + off + i * sizeof(u32));
 141
 142	if (info_regs.magic != MGMT_MBOX_MAGIC) {
 143		XDNA_ERR(ndev->xdna, "Invalid mbox magic 0x%x", info_regs.magic);
 144		ret = -EINVAL;
 145		goto done;
 146	}
 147
 148	i2x = &ndev->mgmt_i2x;
 149	x2i = &ndev->mgmt_x2i;
 150
 151	i2x->mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.i2x_head);
 152	i2x->mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.i2x_tail);
 153	i2x->rb_start_addr   = AIE2_SRAM_OFF(ndev, info_regs.i2x_buf);
 154	i2x->rb_size         = info_regs.i2x_buf_sz;
 155
 156	x2i->mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.x2i_head);
 157	x2i->mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.x2i_tail);
 158	x2i->rb_start_addr   = AIE2_SRAM_OFF(ndev, info_regs.x2i_buf);
 159	x2i->rb_size         = info_regs.x2i_buf_sz;
 160
 161	ndev->mgmt_chan_idx  = info_regs.msi_id;
 162	ndev->mgmt_prot_major = info_regs.prot_major;
 163	ndev->mgmt_prot_minor = info_regs.prot_minor;
 164
 165	ret = aie2_check_protocol(ndev, ndev->mgmt_prot_major, ndev->mgmt_prot_minor);
 166
 167done:
 168	aie2_dump_chann_info_debug(ndev);
 169
 170	/* Must clear address at FW_ALIVE_OFF */
 171	writel(0, SRAM_GET_ADDR(ndev, FW_ALIVE_OFF));
 172
 173	return ret;
 174}
 175
 176int aie2_runtime_cfg(struct amdxdna_dev_hdl *ndev,
 177		     enum rt_config_category category, u32 *val)
 178{
 179	const struct rt_config *cfg;
 180	u32 value;
 181	int ret;
 182
 183	for (cfg = ndev->priv->rt_config; cfg->type; cfg++) {
 184		if (cfg->category != category)
 185			continue;
 186
 187		if (cfg->feature_mask &&
 188		    bitmap_subset(&cfg->feature_mask, &ndev->feature_mask, AIE2_FEATURE_MAX))
 189			continue;
 190
 191		value = val ? *val : cfg->value;
 192		ret = aie2_set_runtime_cfg(ndev, cfg->type, value);
 193		if (ret) {
 194			XDNA_ERR(ndev->xdna, "Set type %d value %d failed",
 195				 cfg->type, value);
 196			return ret;
 197		}
 198	}
 199
 200	return 0;
 201}
 202
 203static int aie2_xdna_reset(struct amdxdna_dev_hdl *ndev)
 204{
 205	int ret;
 206
 207	ret = aie2_suspend_fw(ndev);
 208	if (ret) {
 209		XDNA_ERR(ndev->xdna, "Suspend firmware failed");
 210		return ret;
 211	}
 212
 213	ret = aie2_resume_fw(ndev);
 214	if (ret) {
 215		XDNA_ERR(ndev->xdna, "Resume firmware failed");
 216		return ret;
 217	}
 218
 219	return 0;
 220}
 221
 222static int aie2_mgmt_fw_init(struct amdxdna_dev_hdl *ndev)
 223{
 224	int ret;
 225
 226	ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_INIT, NULL);
 227	if (ret) {
 228		XDNA_ERR(ndev->xdna, "Runtime config failed");
 229		return ret;
 230	}
 231
 232	ret = aie2_assign_mgmt_pasid(ndev, 0);
 233	if (ret) {
 234		XDNA_ERR(ndev->xdna, "Can not assign PASID");
 235		return ret;
 236	}
 237
 238	ret = aie2_xdna_reset(ndev);
 239	if (ret) {
 240		XDNA_ERR(ndev->xdna, "Reset firmware failed");
 241		return ret;
 242	}
 243
 244	return 0;
 245}
 246
 247static int aie2_mgmt_fw_query(struct amdxdna_dev_hdl *ndev)
 248{
 249	int ret;
 250
 251	ret = aie2_query_firmware_version(ndev, &ndev->xdna->fw_ver);
 252	if (ret) {
 253		XDNA_ERR(ndev->xdna, "query firmware version failed");
 254		return ret;
 255	}
 256
 257	ret = aie2_query_aie_version(ndev, &ndev->version);
 258	if (ret) {
 259		XDNA_ERR(ndev->xdna, "Query AIE version failed");
 260		return ret;
 261	}
 262
 263	ret = aie2_query_aie_metadata(ndev, &ndev->metadata);
 264	if (ret) {
 265		XDNA_ERR(ndev->xdna, "Query AIE metadata failed");
 266		return ret;
 267	}
 268
 269	ndev->total_col = min(aie2_max_col, ndev->metadata.cols);
 270
 271	return 0;
 272}
 273
 274static void aie2_mgmt_fw_fini(struct amdxdna_dev_hdl *ndev)
 275{
 276	if (aie2_suspend_fw(ndev))
 277		XDNA_ERR(ndev->xdna, "Suspend_fw failed");
 278	XDNA_DBG(ndev->xdna, "Firmware suspended");
 279}
 280
 281static int aie2_xrs_load(void *cb_arg, struct xrs_action_load *action)
 282{
 283	struct amdxdna_hwctx *hwctx = cb_arg;
 284	struct amdxdna_dev *xdna;
 285	int ret;
 286
 287	xdna = hwctx->client->xdna;
 288
 289	hwctx->start_col = action->part.start_col;
 290	hwctx->num_col = action->part.ncols;
 291	ret = aie2_create_context(xdna->dev_handle, hwctx);
 292	if (ret)
 293		XDNA_ERR(xdna, "create context failed, ret %d", ret);
 294
 295	return ret;
 296}
 297
 298static int aie2_xrs_unload(void *cb_arg)
 299{
 300	struct amdxdna_hwctx *hwctx = cb_arg;
 301	struct amdxdna_dev *xdna;
 302	int ret;
 303
 304	xdna = hwctx->client->xdna;
 305
 306	ret = aie2_destroy_context(xdna->dev_handle, hwctx);
 307	if (ret)
 308		XDNA_ERR(xdna, "destroy context failed, ret %d", ret);
 309
 310	return ret;
 311}
 312
 313static int aie2_xrs_set_dft_dpm_level(struct drm_device *ddev, u32 dpm_level)
 314{
 315	struct amdxdna_dev *xdna = to_xdna_dev(ddev);
 316	struct amdxdna_dev_hdl *ndev;
 317
 318	drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
 319
 320	ndev = xdna->dev_handle;
 321	ndev->dft_dpm_level = dpm_level;
 322	if (ndev->pw_mode != POWER_MODE_DEFAULT || ndev->dpm_level == dpm_level)
 323		return 0;
 324
 325	return aie2_pm_set_dpm(ndev, dpm_level);
 326}
 327
 328static struct xrs_action_ops aie2_xrs_actions = {
 329	.load = aie2_xrs_load,
 330	.unload = aie2_xrs_unload,
 331	.set_dft_dpm_level = aie2_xrs_set_dft_dpm_level,
 332};
 333
 334static void aie2_hw_stop(struct amdxdna_dev *xdna)
 335{
 336	struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
 337	struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
 338
 339	if (ndev->dev_status <= AIE2_DEV_INIT) {
 340		XDNA_ERR(xdna, "device is already stopped");
 341		return;
 342	}
 343
 344	aie2_runtime_cfg(ndev, AIE2_RT_CFG_CLK_GATING, NULL);
 345	aie2_mgmt_fw_fini(ndev);
 346	aie2_destroy_mgmt_chann(ndev);
 347	drmm_kfree(&xdna->ddev, ndev->mbox);
 348	ndev->mbox = NULL;
 349	aie2_psp_stop(ndev->psp_hdl);
 350	aie2_smu_fini(ndev);
 351	aie2_error_async_events_free(ndev);
 352	pci_disable_device(pdev);
 353
 354	ndev->dev_status = AIE2_DEV_INIT;
 355}
 356
 357static int aie2_hw_start(struct amdxdna_dev *xdna)
 358{
 359	struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
 360	struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
 361	struct xdna_mailbox_res mbox_res;
 362	u32 xdna_mailbox_intr_reg;
 363	int mgmt_mb_irq, ret;
 364
 365	if (ndev->dev_status >= AIE2_DEV_START) {
 366		XDNA_INFO(xdna, "device is already started");
 367		return 0;
 368	}
 369
 370	ret = pci_enable_device(pdev);
 371	if (ret) {
 372		XDNA_ERR(xdna, "failed to enable device, ret %d", ret);
 373		return ret;
 374	}
 375	pci_set_master(pdev);
 376
 377	ret = aie2_smu_init(ndev);
 378	if (ret) {
 379		XDNA_ERR(xdna, "failed to init smu, ret %d", ret);
 380		goto disable_dev;
 381	}
 382
 383	ret = aie2_psp_start(ndev->psp_hdl);
 384	if (ret) {
 385		XDNA_ERR(xdna, "failed to start psp, ret %d", ret);
 386		goto fini_smu;
 387	}
 388
 389	ret = aie2_get_mgmt_chann_info(ndev);
 390	if (ret) {
 391		XDNA_ERR(xdna, "firmware is not alive");
 392		goto stop_psp;
 393	}
 394
 395	mbox_res.ringbuf_base = ndev->sram_base;
 396	mbox_res.ringbuf_size = pci_resource_len(pdev, xdna->dev_info->sram_bar);
 397	mbox_res.mbox_base = ndev->mbox_base;
 398	mbox_res.mbox_size = MBOX_SIZE(ndev);
 399	mbox_res.name = "xdna_mailbox";
 400	ndev->mbox = xdnam_mailbox_create(&xdna->ddev, &mbox_res);
 401	if (!ndev->mbox) {
 402		XDNA_ERR(xdna, "failed to create mailbox device");
 403		ret = -ENODEV;
 404		goto stop_psp;
 405	}
 406
 407	mgmt_mb_irq = pci_irq_vector(pdev, ndev->mgmt_chan_idx);
 408	if (mgmt_mb_irq < 0) {
 409		ret = mgmt_mb_irq;
 410		XDNA_ERR(xdna, "failed to alloc irq vector, ret %d", ret);
 411		goto stop_psp;
 412	}
 413
 414	xdna_mailbox_intr_reg = ndev->mgmt_i2x.mb_head_ptr_reg + 4;
 415	ndev->mgmt_chann = xdna_mailbox_create_channel(ndev->mbox,
 416						       &ndev->mgmt_x2i,
 417						       &ndev->mgmt_i2x,
 418						       xdna_mailbox_intr_reg,
 419						       mgmt_mb_irq);
 420	if (!ndev->mgmt_chann) {
 421		XDNA_ERR(xdna, "failed to create management mailbox channel");
 422		ret = -EINVAL;
 423		goto stop_psp;
 424	}
 425
 426	ret = aie2_mgmt_fw_init(ndev);
 427	if (ret) {
 428		XDNA_ERR(xdna, "initial mgmt firmware failed, ret %d", ret);
 429		goto destroy_mgmt_chann;
 430	}
 431
 432	ret = aie2_pm_init(ndev);
 433	if (ret) {
 434		XDNA_ERR(xdna, "failed to init pm, ret %d", ret);
 435		goto destroy_mgmt_chann;
 436	}
 437
 438	ret = aie2_mgmt_fw_query(ndev);
 439	if (ret) {
 440		XDNA_ERR(xdna, "failed to query fw, ret %d", ret);
 441		goto destroy_mgmt_chann;
 442	}
 443
 444	ret = aie2_error_async_events_alloc(ndev);
 445	if (ret) {
 446		XDNA_ERR(xdna, "Allocate async events failed, ret %d", ret);
 447		goto destroy_mgmt_chann;
 448	}
 449
 450	ndev->dev_status = AIE2_DEV_START;
 451
 452	return 0;
 453
 454destroy_mgmt_chann:
 455	aie2_destroy_mgmt_chann(ndev);
 456stop_psp:
 457	aie2_psp_stop(ndev->psp_hdl);
 458fini_smu:
 459	aie2_smu_fini(ndev);
 460disable_dev:
 461	pci_disable_device(pdev);
 462
 463	return ret;
 464}
 465
 466static int aie2_hw_suspend(struct amdxdna_dev *xdna)
 467{
 468	struct amdxdna_client *client;
 469
 470	list_for_each_entry(client, &xdna->client_list, node)
 471		aie2_hwctx_suspend(client);
 472
 473	aie2_hw_stop(xdna);
 474
 475	return 0;
 476}
 477
 478static int aie2_hw_resume(struct amdxdna_dev *xdna)
 479{
 480	struct amdxdna_client *client;
 481	int ret;
 482
 483	ret = aie2_hw_start(xdna);
 484	if (ret) {
 485		XDNA_ERR(xdna, "Start hardware failed, %d", ret);
 486		return ret;
 487	}
 488
 489	list_for_each_entry(client, &xdna->client_list, node) {
 490		ret = aie2_hwctx_resume(client);
 491		if (ret)
 492			break;
 493	}
 494
 495	return ret;
 496}
 497
 498static int aie2_init(struct amdxdna_dev *xdna)
 499{
 500	struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
 501	void __iomem *tbl[PCI_NUM_RESOURCES] = {0};
 502	struct init_config xrs_cfg = { 0 };
 503	struct amdxdna_dev_hdl *ndev;
 504	struct psp_config psp_conf;
 505	const struct firmware *fw;
 506	unsigned long bars = 0;
 507	int i, nvec, ret;
 508
 509	if (!hypervisor_is_type(X86_HYPER_NATIVE)) {
 510		XDNA_ERR(xdna, "Running under hypervisor not supported");
 511		return -EINVAL;
 512	}
 513
 514	ndev = drmm_kzalloc(&xdna->ddev, sizeof(*ndev), GFP_KERNEL);
 515	if (!ndev)
 516		return -ENOMEM;
 517
 518	ndev->priv = xdna->dev_info->dev_priv;
 519	ndev->xdna = xdna;
 520
 521	ret = request_firmware(&fw, ndev->priv->fw_path, &pdev->dev);
 522	if (ret) {
 523		XDNA_ERR(xdna, "failed to request_firmware %s, ret %d",
 524			 ndev->priv->fw_path, ret);
 525		return ret;
 526	}
 527
 528	ret = pcim_enable_device(pdev);
 529	if (ret) {
 530		XDNA_ERR(xdna, "pcim enable device failed, ret %d", ret);
 531		goto release_fw;
 532	}
 533
 534	for (i = 0; i < PSP_MAX_REGS; i++)
 535		set_bit(PSP_REG_BAR(ndev, i), &bars);
 536
 537	set_bit(xdna->dev_info->sram_bar, &bars);
 538	set_bit(xdna->dev_info->smu_bar, &bars);
 539	set_bit(xdna->dev_info->mbox_bar, &bars);
 540
 541	for (i = 0; i < PCI_NUM_RESOURCES; i++) {
 542		if (!test_bit(i, &bars))
 543			continue;
 544		tbl[i] = pcim_iomap(pdev, i, 0);
 545		if (!tbl[i]) {
 546			XDNA_ERR(xdna, "map bar %d failed", i);
 547			ret = -ENOMEM;
 548			goto release_fw;
 549		}
 550	}
 551
 552	ndev->sram_base = tbl[xdna->dev_info->sram_bar];
 553	ndev->smu_base = tbl[xdna->dev_info->smu_bar];
 554	ndev->mbox_base = tbl[xdna->dev_info->mbox_bar];
 555
 556	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
 557	if (ret) {
 558		XDNA_ERR(xdna, "Failed to set DMA mask: %d", ret);
 559		goto release_fw;
 560	}
 561
 562	nvec = pci_msix_vec_count(pdev);
 563	if (nvec <= 0) {
 564		XDNA_ERR(xdna, "does not get number of interrupt vector");
 565		ret = -EINVAL;
 566		goto release_fw;
 567	}
 568
 569	ret = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
 570	if (ret < 0) {
 571		XDNA_ERR(xdna, "failed to alloc irq vectors, ret %d", ret);
 572		goto release_fw;
 573	}
 574
 575	psp_conf.fw_size = fw->size;
 576	psp_conf.fw_buf = fw->data;
 577	for (i = 0; i < PSP_MAX_REGS; i++)
 578		psp_conf.psp_regs[i] = tbl[PSP_REG_BAR(ndev, i)] + PSP_REG_OFF(ndev, i);
 579	ndev->psp_hdl = aie2m_psp_create(&xdna->ddev, &psp_conf);
 580	if (!ndev->psp_hdl) {
 581		XDNA_ERR(xdna, "failed to create psp");
 582		ret = -ENOMEM;
 583		goto release_fw;
 584	}
 585	xdna->dev_handle = ndev;
 586
 587	ret = aie2_hw_start(xdna);
 588	if (ret) {
 589		XDNA_ERR(xdna, "start npu failed, ret %d", ret);
 590		goto release_fw;
 591	}
 592
 593	xrs_cfg.clk_list.num_levels = ndev->max_dpm_level + 1;
 594	for (i = 0; i < xrs_cfg.clk_list.num_levels; i++)
 595		xrs_cfg.clk_list.cu_clk_list[i] = ndev->priv->dpm_clk_tbl[i].hclk;
 596	xrs_cfg.sys_eff_factor = 1;
 597	xrs_cfg.ddev = &xdna->ddev;
 598	xrs_cfg.actions = &aie2_xrs_actions;
 599	xrs_cfg.total_col = ndev->total_col;
 600
 601	xdna->xrs_hdl = xrsm_init(&xrs_cfg);
 602	if (!xdna->xrs_hdl) {
 603		XDNA_ERR(xdna, "Initialize resolver failed");
 604		ret = -EINVAL;
 605		goto stop_hw;
 606	}
 607
 608	release_firmware(fw);
 609	aie2_msg_init(ndev);
 610	amdxdna_pm_init(xdna);
 611	return 0;
 612
 613stop_hw:
 614	aie2_hw_stop(xdna);
 615release_fw:
 616	release_firmware(fw);
 617
 618	return ret;
 619}
 620
 621static void aie2_fini(struct amdxdna_dev *xdna)
 622{
 623	amdxdna_pm_fini(xdna);
 624	aie2_hw_stop(xdna);
 625}
 626
 627static int aie2_get_aie_status(struct amdxdna_client *client,
 628			       struct amdxdna_drm_get_info *args)
 629{
 630	struct amdxdna_drm_query_aie_status status;
 631	struct amdxdna_dev *xdna = client->xdna;
 632	struct amdxdna_dev_hdl *ndev;
 633	int ret;
 634
 635	ndev = xdna->dev_handle;
 636	if (copy_from_user(&status, u64_to_user_ptr(args->buffer), sizeof(status))) {
 637		XDNA_ERR(xdna, "Failed to copy AIE request into kernel");
 638		return -EFAULT;
 639	}
 640
 641	if (ndev->metadata.cols * ndev->metadata.size < status.buffer_size) {
 642		XDNA_ERR(xdna, "Invalid buffer size. Given Size: %u. Need Size: %u.",
 643			 status.buffer_size, ndev->metadata.cols * ndev->metadata.size);
 644		return -EINVAL;
 645	}
 646
 647	ret = aie2_query_status(ndev, u64_to_user_ptr(status.buffer),
 648				status.buffer_size, &status.cols_filled);
 649	if (ret) {
 650		XDNA_ERR(xdna, "Failed to get AIE status info. Ret: %d", ret);
 651		return ret;
 652	}
 653
 654	if (copy_to_user(u64_to_user_ptr(args->buffer), &status, sizeof(status))) {
 655		XDNA_ERR(xdna, "Failed to copy AIE request info to user space");
 656		return -EFAULT;
 657	}
 658
 659	return 0;
 660}
 661
 662static int aie2_get_aie_metadata(struct amdxdna_client *client,
 663				 struct amdxdna_drm_get_info *args)
 664{
 665	struct amdxdna_drm_query_aie_metadata *meta;
 666	struct amdxdna_dev *xdna = client->xdna;
 667	struct amdxdna_dev_hdl *ndev;
 668	int ret = 0;
 669
 670	ndev = xdna->dev_handle;
 671	meta = kzalloc(sizeof(*meta), GFP_KERNEL);
 672	if (!meta)
 673		return -ENOMEM;
 674
 675	meta->col_size = ndev->metadata.size;
 676	meta->cols = ndev->metadata.cols;
 677	meta->rows = ndev->metadata.rows;
 678
 679	meta->version.major = ndev->metadata.version.major;
 680	meta->version.minor = ndev->metadata.version.minor;
 681
 682	meta->core.row_count = ndev->metadata.core.row_count;
 683	meta->core.row_start = ndev->metadata.core.row_start;
 684	meta->core.dma_channel_count = ndev->metadata.core.dma_channel_count;
 685	meta->core.lock_count = ndev->metadata.core.lock_count;
 686	meta->core.event_reg_count = ndev->metadata.core.event_reg_count;
 687
 688	meta->mem.row_count = ndev->metadata.mem.row_count;
 689	meta->mem.row_start = ndev->metadata.mem.row_start;
 690	meta->mem.dma_channel_count = ndev->metadata.mem.dma_channel_count;
 691	meta->mem.lock_count = ndev->metadata.mem.lock_count;
 692	meta->mem.event_reg_count = ndev->metadata.mem.event_reg_count;
 693
 694	meta->shim.row_count = ndev->metadata.shim.row_count;
 695	meta->shim.row_start = ndev->metadata.shim.row_start;
 696	meta->shim.dma_channel_count = ndev->metadata.shim.dma_channel_count;
 697	meta->shim.lock_count = ndev->metadata.shim.lock_count;
 698	meta->shim.event_reg_count = ndev->metadata.shim.event_reg_count;
 699
 700	if (copy_to_user(u64_to_user_ptr(args->buffer), meta, sizeof(*meta)))
 701		ret = -EFAULT;
 702
 703	kfree(meta);
 704	return ret;
 705}
 706
 707static int aie2_get_aie_version(struct amdxdna_client *client,
 708				struct amdxdna_drm_get_info *args)
 709{
 710	struct amdxdna_drm_query_aie_version version;
 711	struct amdxdna_dev *xdna = client->xdna;
 712	struct amdxdna_dev_hdl *ndev;
 713
 714	ndev = xdna->dev_handle;
 715	version.major = ndev->version.major;
 716	version.minor = ndev->version.minor;
 717
 718	if (copy_to_user(u64_to_user_ptr(args->buffer), &version, sizeof(version)))
 719		return -EFAULT;
 720
 721	return 0;
 722}
 723
 724static int aie2_get_firmware_version(struct amdxdna_client *client,
 725				     struct amdxdna_drm_get_info *args)
 726{
 727	struct amdxdna_drm_query_firmware_version version;
 728	struct amdxdna_dev *xdna = client->xdna;
 729
 730	version.major = xdna->fw_ver.major;
 731	version.minor = xdna->fw_ver.minor;
 732	version.patch = xdna->fw_ver.sub;
 733	version.build = xdna->fw_ver.build;
 734
 735	if (copy_to_user(u64_to_user_ptr(args->buffer), &version, sizeof(version)))
 736		return -EFAULT;
 737
 738	return 0;
 739}
 740
 741static int aie2_get_power_mode(struct amdxdna_client *client,
 742			       struct amdxdna_drm_get_info *args)
 743{
 744	struct amdxdna_drm_get_power_mode mode = {};
 745	struct amdxdna_dev *xdna = client->xdna;
 746	struct amdxdna_dev_hdl *ndev;
 747
 748	ndev = xdna->dev_handle;
 749	mode.power_mode = ndev->pw_mode;
 750
 751	if (copy_to_user(u64_to_user_ptr(args->buffer), &mode, sizeof(mode)))
 752		return -EFAULT;
 753
 754	return 0;
 755}
 756
 757static int aie2_get_clock_metadata(struct amdxdna_client *client,
 758				   struct amdxdna_drm_get_info *args)
 759{
 760	struct amdxdna_drm_query_clock_metadata *clock;
 761	struct amdxdna_dev *xdna = client->xdna;
 762	struct amdxdna_dev_hdl *ndev;
 763	int ret = 0;
 764
 765	ndev = xdna->dev_handle;
 766	clock = kzalloc(sizeof(*clock), GFP_KERNEL);
 767	if (!clock)
 768		return -ENOMEM;
 769
 770	snprintf(clock->mp_npu_clock.name, sizeof(clock->mp_npu_clock.name),
 771		 "MP-NPU Clock");
 772	clock->mp_npu_clock.freq_mhz = ndev->npuclk_freq;
 773	snprintf(clock->h_clock.name, sizeof(clock->h_clock.name), "H Clock");
 774	clock->h_clock.freq_mhz = ndev->hclk_freq;
 775
 776	if (copy_to_user(u64_to_user_ptr(args->buffer), clock, sizeof(*clock)))
 777		ret = -EFAULT;
 778
 779	kfree(clock);
 780	return ret;
 781}
 782
 783static int aie2_hwctx_status_cb(struct amdxdna_hwctx *hwctx, void *arg)
 784{
 785	struct amdxdna_drm_hwctx_entry *tmp __free(kfree) = NULL;
 786	struct amdxdna_drm_get_array *array_args = arg;
 787	struct amdxdna_drm_hwctx_entry __user *buf;
 788	u32 size;
 789
 790	if (!array_args->num_element)
 791		return -EINVAL;
 792
 793	tmp = kzalloc(sizeof(*tmp), GFP_KERNEL);
 794	if (!tmp)
 795		return -ENOMEM;
 796
 797	tmp->pid = hwctx->client->pid;
 798	tmp->context_id = hwctx->id;
 799	tmp->start_col = hwctx->start_col;
 800	tmp->num_col = hwctx->num_col;
 801	tmp->command_submissions = hwctx->priv->seq;
 802	tmp->command_completions = hwctx->priv->completed;
 803	tmp->pasid = hwctx->client->pasid;
 804	tmp->priority = hwctx->qos.priority;
 805	tmp->gops = hwctx->qos.gops;
 806	tmp->fps = hwctx->qos.fps;
 807	tmp->dma_bandwidth = hwctx->qos.dma_bandwidth;
 808	tmp->latency = hwctx->qos.latency;
 809	tmp->frame_exec_time = hwctx->qos.frame_exec_time;
 810	tmp->state = AMDXDNA_HWCTX_STATE_ACTIVE;
 811
 812	buf = u64_to_user_ptr(array_args->buffer);
 813	size = min(sizeof(*tmp), array_args->element_size);
 814
 815	if (copy_to_user(buf, tmp, size))
 816		return -EFAULT;
 817
 818	array_args->buffer += size;
 819	array_args->num_element--;
 820
 821	return 0;
 822}
 823
 824static int aie2_get_hwctx_status(struct amdxdna_client *client,
 825				 struct amdxdna_drm_get_info *args)
 826{
 827	struct amdxdna_drm_get_array array_args;
 828	struct amdxdna_dev *xdna = client->xdna;
 829	struct amdxdna_client *tmp_client;
 830	int ret;
 831
 832	drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
 833
 834	array_args.element_size = sizeof(struct amdxdna_drm_query_hwctx);
 835	array_args.buffer = args->buffer;
 836	array_args.num_element = args->buffer_size / array_args.element_size;
 837	list_for_each_entry(tmp_client, &xdna->client_list, node) {
 838		ret = amdxdna_hwctx_walk(tmp_client, &array_args,
 839					 aie2_hwctx_status_cb);
 840		if (ret)
 841			break;
 842	}
 843
 844	args->buffer_size -= (u32)(array_args.buffer - args->buffer);
 845	return 0;
 846}
 847
 848static int aie2_query_resource_info(struct amdxdna_client *client,
 849				    struct amdxdna_drm_get_info *args)
 850{
 851	struct amdxdna_drm_get_resource_info res_info;
 852	const struct amdxdna_dev_priv *priv;
 853	struct amdxdna_dev_hdl *ndev;
 854	struct amdxdna_dev *xdna;
 855
 856	xdna = client->xdna;
 857	ndev = xdna->dev_handle;
 858	priv = ndev->priv;
 859
 860	res_info.npu_clk_max = priv->dpm_clk_tbl[ndev->max_dpm_level].hclk;
 861	res_info.npu_tops_max = ndev->max_tops;
 862	res_info.npu_task_max = priv->hwctx_limit;
 863	res_info.npu_tops_curr = ndev->curr_tops;
 864	res_info.npu_task_curr = ndev->hwctx_num;
 865
 866	if (copy_to_user(u64_to_user_ptr(args->buffer), &res_info, sizeof(res_info)))
 867		return -EFAULT;
 868
 869	return 0;
 870}
 871
 872static int aie2_fill_hwctx_map(struct amdxdna_hwctx *hwctx, void *arg)
 873{
 874	struct amdxdna_dev *xdna = hwctx->client->xdna;
 875	u32 *map = arg;
 876
 877	if (hwctx->fw_ctx_id >= xdna->dev_handle->priv->hwctx_limit) {
 878		XDNA_ERR(xdna, "Invalid fw ctx id %d/%d ", hwctx->fw_ctx_id,
 879			 xdna->dev_handle->priv->hwctx_limit);
 880		return -EINVAL;
 881	}
 882
 883	map[hwctx->fw_ctx_id] = hwctx->id;
 884	return 0;
 885}
 886
 887static int aie2_get_telemetry(struct amdxdna_client *client,
 888			      struct amdxdna_drm_get_info *args)
 889{
 890	struct amdxdna_drm_query_telemetry_header *header __free(kfree) = NULL;
 891	u32 telemetry_data_sz, header_sz, elem_num;
 892	struct amdxdna_dev *xdna = client->xdna;
 893	struct amdxdna_client *tmp_client;
 894	int ret;
 895
 896	elem_num = xdna->dev_handle->priv->hwctx_limit;
 897	header_sz = struct_size(header, map, elem_num);
 898	if (args->buffer_size <= header_sz) {
 899		XDNA_ERR(xdna, "Invalid buffer size");
 900		return -EINVAL;
 901	}
 902
 903	telemetry_data_sz = args->buffer_size - header_sz;
 904	if (telemetry_data_sz > SZ_4M) {
 905		XDNA_ERR(xdna, "Buffer size is too big, %d", telemetry_data_sz);
 906		return -EINVAL;
 907	}
 908
 909	header = kzalloc(header_sz, GFP_KERNEL);
 910	if (!header)
 911		return -ENOMEM;
 912
 913	if (copy_from_user(header, u64_to_user_ptr(args->buffer), sizeof(*header))) {
 914		XDNA_ERR(xdna, "Failed to copy telemetry header from user");
 915		return -EFAULT;
 916	}
 917
 918	header->map_num_elements = elem_num;
 919	list_for_each_entry(tmp_client, &xdna->client_list, node) {
 920		ret = amdxdna_hwctx_walk(tmp_client, &header->map,
 921					 aie2_fill_hwctx_map);
 922		if (ret)
 923			return ret;
 924	}
 925
 926	ret = aie2_query_telemetry(xdna->dev_handle,
 927				   u64_to_user_ptr(args->buffer + header_sz),
 928				   telemetry_data_sz, header);
 929	if (ret) {
 930		XDNA_ERR(xdna, "Query telemetry failed ret %d", ret);
 931		return ret;
 932	}
 933
 934	if (copy_to_user(u64_to_user_ptr(args->buffer), header, header_sz)) {
 935		XDNA_ERR(xdna, "Copy header failed");
 936		return -EFAULT;
 937	}
 938
 939	return 0;
 940}
 941
 942static int aie2_get_preempt_state(struct amdxdna_client *client,
 943				  struct amdxdna_drm_get_info *args)
 944{
 945	struct amdxdna_drm_attribute_state state = {};
 946	struct amdxdna_dev *xdna = client->xdna;
 947	struct amdxdna_dev_hdl *ndev;
 948
 949	ndev = xdna->dev_handle;
 950	if (args->param == DRM_AMDXDNA_GET_FORCE_PREEMPT_STATE)
 951		state.state = ndev->force_preempt_enabled;
 952	else if (args->param == DRM_AMDXDNA_GET_FRAME_BOUNDARY_PREEMPT_STATE)
 953		state.state = ndev->frame_boundary_preempt;
 954
 955	if (copy_to_user(u64_to_user_ptr(args->buffer), &state, sizeof(state)))
 956		return -EFAULT;
 957
 958	return 0;
 959}
 960
 961static int aie2_get_info(struct amdxdna_client *client, struct amdxdna_drm_get_info *args)
 962{
 963	struct amdxdna_dev *xdna = client->xdna;
 964	int ret, idx;
 965
 966	if (!drm_dev_enter(&xdna->ddev, &idx))
 967		return -ENODEV;
 968
 969	ret = amdxdna_pm_resume_get_locked(xdna);
 970	if (ret)
 971		goto dev_exit;
 972
 973	switch (args->param) {
 974	case DRM_AMDXDNA_QUERY_AIE_STATUS:
 975		ret = aie2_get_aie_status(client, args);
 976		break;
 977	case DRM_AMDXDNA_QUERY_AIE_METADATA:
 978		ret = aie2_get_aie_metadata(client, args);
 979		break;
 980	case DRM_AMDXDNA_QUERY_AIE_VERSION:
 981		ret = aie2_get_aie_version(client, args);
 982		break;
 983	case DRM_AMDXDNA_QUERY_CLOCK_METADATA:
 984		ret = aie2_get_clock_metadata(client, args);
 985		break;
 986	case DRM_AMDXDNA_QUERY_HW_CONTEXTS:
 987		ret = aie2_get_hwctx_status(client, args);
 988		break;
 989	case DRM_AMDXDNA_QUERY_FIRMWARE_VERSION:
 990		ret = aie2_get_firmware_version(client, args);
 991		break;
 992	case DRM_AMDXDNA_GET_POWER_MODE:
 993		ret = aie2_get_power_mode(client, args);
 994		break;
 995	case DRM_AMDXDNA_QUERY_TELEMETRY:
 996		ret = aie2_get_telemetry(client, args);
 997		break;
 998	case DRM_AMDXDNA_QUERY_RESOURCE_INFO:
 999		ret = aie2_query_resource_info(client, args);
1000		break;
1001	case DRM_AMDXDNA_GET_FORCE_PREEMPT_STATE:
1002	case DRM_AMDXDNA_GET_FRAME_BOUNDARY_PREEMPT_STATE:
1003		ret = aie2_get_preempt_state(client, args);
1004		break;
1005	default:
1006		XDNA_ERR(xdna, "Not supported request parameter %u", args->param);
1007		ret = -EOPNOTSUPP;
1008	}
1009
1010	amdxdna_pm_suspend_put(xdna);
1011	XDNA_DBG(xdna, "Got param %d", args->param);
1012
1013dev_exit:
1014	drm_dev_exit(idx);
1015	return ret;
1016}
1017
1018static int aie2_query_ctx_status_array(struct amdxdna_client *client,
1019				       struct amdxdna_drm_get_array *args)
1020{
1021	struct amdxdna_drm_get_array array_args;
1022	struct amdxdna_dev *xdna = client->xdna;
1023	struct amdxdna_client *tmp_client;
1024	int ret;
1025
1026	drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
1027
1028	if (args->element_size > SZ_4K || args->num_element > SZ_1K) {
1029		XDNA_DBG(xdna, "Invalid element size %d or number of element %d",
1030			 args->element_size, args->num_element);
1031		return -EINVAL;
1032	}
1033
1034	array_args.element_size = min(args->element_size,
1035				      sizeof(struct amdxdna_drm_hwctx_entry));
1036	array_args.buffer = args->buffer;
1037	array_args.num_element = args->num_element * args->element_size /
1038				array_args.element_size;
1039	list_for_each_entry(tmp_client, &xdna->client_list, node) {
1040		ret = amdxdna_hwctx_walk(tmp_client, &array_args,
1041					 aie2_hwctx_status_cb);
1042		if (ret)
1043			break;
1044	}
1045
1046	args->element_size = array_args.element_size;
1047	args->num_element = (u32)((array_args.buffer - args->buffer) /
1048				  args->element_size);
1049
1050	return 0;
1051}
1052
1053static int aie2_get_array(struct amdxdna_client *client,
1054			  struct amdxdna_drm_get_array *args)
1055{
1056	struct amdxdna_dev *xdna = client->xdna;
1057	int ret, idx;
1058
1059	if (!drm_dev_enter(&xdna->ddev, &idx))
1060		return -ENODEV;
1061
1062	ret = amdxdna_pm_resume_get_locked(xdna);
1063	if (ret)
1064		goto dev_exit;
1065
1066	switch (args->param) {
1067	case DRM_AMDXDNA_HW_CONTEXT_ALL:
1068		ret = aie2_query_ctx_status_array(client, args);
1069		break;
1070	case DRM_AMDXDNA_HW_LAST_ASYNC_ERR:
1071		ret = aie2_get_array_async_error(xdna->dev_handle, args);
1072		break;
1073	default:
1074		XDNA_ERR(xdna, "Not supported request parameter %u", args->param);
1075		ret = -EOPNOTSUPP;
1076	}
1077
1078	amdxdna_pm_suspend_put(xdna);
1079	XDNA_DBG(xdna, "Got param %d", args->param);
1080
1081dev_exit:
1082	drm_dev_exit(idx);
1083	return ret;
1084}
1085
1086static int aie2_set_power_mode(struct amdxdna_client *client,
1087			       struct amdxdna_drm_set_state *args)
1088{
1089	struct amdxdna_drm_set_power_mode power_state;
1090	enum amdxdna_power_mode_type power_mode;
1091	struct amdxdna_dev *xdna = client->xdna;
1092
1093	if (copy_from_user(&power_state, u64_to_user_ptr(args->buffer),
1094			   sizeof(power_state))) {
1095		XDNA_ERR(xdna, "Failed to copy power mode request into kernel");
1096		return -EFAULT;
1097	}
1098
1099	if (XDNA_MBZ_DBG(xdna, power_state.pad, sizeof(power_state.pad)))
1100		return -EINVAL;
1101
1102	power_mode = power_state.power_mode;
1103	if (power_mode > POWER_MODE_TURBO) {
1104		XDNA_ERR(xdna, "Invalid power mode %d", power_mode);
1105		return -EINVAL;
1106	}
1107
1108	return aie2_pm_set_mode(xdna->dev_handle, power_mode);
1109}
1110
1111static int aie2_set_preempt_state(struct amdxdna_client *client,
1112				  struct amdxdna_drm_set_state *args)
1113{
1114	struct amdxdna_dev_hdl *ndev = client->xdna->dev_handle;
1115	struct amdxdna_drm_attribute_state state;
1116	u32 val;
1117	int ret;
1118
1119	if (copy_from_user(&state, u64_to_user_ptr(args->buffer), sizeof(state)))
1120		return -EFAULT;
1121
1122	if (state.state > 1)
1123		return -EINVAL;
1124
1125	if (XDNA_MBZ_DBG(client->xdna, state.pad, sizeof(state.pad)))
1126		return -EINVAL;
1127
1128	if (args->param == DRM_AMDXDNA_SET_FORCE_PREEMPT) {
1129		ndev->force_preempt_enabled = state.state;
1130	} else if (args->param == DRM_AMDXDNA_SET_FRAME_BOUNDARY_PREEMPT) {
1131		val = state.state;
1132		ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_FRAME_BOUNDARY_PREEMPT,
1133				       &val);
1134		if (ret)
1135			return ret;
1136
1137		ndev->frame_boundary_preempt = state.state;
1138	}
1139
1140	return 0;
1141}
1142
1143static int aie2_set_state(struct amdxdna_client *client,
1144			  struct amdxdna_drm_set_state *args)
1145{
1146	struct amdxdna_dev *xdna = client->xdna;
1147	int ret, idx;
1148
1149	if (!drm_dev_enter(&xdna->ddev, &idx))
1150		return -ENODEV;
1151
1152	ret = amdxdna_pm_resume_get_locked(xdna);
1153	if (ret)
1154		goto dev_exit;
1155
1156	switch (args->param) {
1157	case DRM_AMDXDNA_SET_POWER_MODE:
1158		ret = aie2_set_power_mode(client, args);
1159		break;
1160	case DRM_AMDXDNA_SET_FORCE_PREEMPT:
1161	case DRM_AMDXDNA_SET_FRAME_BOUNDARY_PREEMPT:
1162		ret = aie2_set_preempt_state(client, args);
1163		break;
1164	default:
1165		XDNA_ERR(xdna, "Not supported request parameter %u", args->param);
1166		ret = -EOPNOTSUPP;
1167		break;
1168	}
1169
1170	amdxdna_pm_suspend_put(xdna);
1171dev_exit:
1172	drm_dev_exit(idx);
1173	return ret;
1174}
1175
1176const struct amdxdna_dev_ops aie2_ops = {
1177	.init = aie2_init,
1178	.fini = aie2_fini,
1179	.resume = aie2_hw_resume,
1180	.suspend = aie2_hw_suspend,
1181	.get_aie_info = aie2_get_info,
1182	.set_aie_state = aie2_set_state,
1183	.hwctx_init = aie2_hwctx_init,
1184	.hwctx_fini = aie2_hwctx_fini,
1185	.hwctx_config = aie2_hwctx_config,
1186	.hwctx_sync_debug_bo = aie2_hwctx_sync_debug_bo,
1187	.cmd_submit = aie2_cmd_submit,
1188	.hmm_invalidate = aie2_hmm_invalidate,
1189	.get_array = aie2_get_array,
1190};