aie2_message.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (C) 2023-2024, Advanced Micro Devices, Inc.
   4 */
   5
   6#include <drm/amdxdna_accel.h>
   7#include <drm/drm_cache.h>
   8#include <drm/drm_device.h>
   9#include <drm/drm_gem.h>
  10#include <drm/drm_gem_shmem_helper.h>
  11#include <drm/drm_print.h>
  12#include <drm/gpu_scheduler.h>
  13#include <linux/bitfield.h>
  14#include <linux/errno.h>
  15#include <linux/pci.h>
  16#include <linux/types.h>
  17#include <linux/xarray.h>
  18
  19#include "aie2_msg_priv.h"
  20#include "aie2_pci.h"
  21#include "amdxdna_ctx.h"
  22#include "amdxdna_gem.h"
  23#include "amdxdna_mailbox.h"
  24#include "amdxdna_mailbox_helper.h"
  25#include "amdxdna_pci_drv.h"
  26
  27#define DECLARE_AIE2_MSG(name, op) \
  28	DECLARE_XDNA_MSG_COMMON(name, op, MAX_AIE2_STATUS_CODE)
  29
  30#define EXEC_MSG_OPS(xdna)	((xdna)->dev_handle->exec_msg_ops)
  31
  32static int aie2_send_mgmt_msg_wait(struct amdxdna_dev_hdl *ndev,
  33				   struct xdna_mailbox_msg *msg)
  34{
  35	struct amdxdna_dev *xdna = ndev->xdna;
  36	struct xdna_notify *hdl = msg->handle;
  37	int ret;
  38
  39	if (!ndev->mgmt_chann)
  40		return -ENODEV;
  41
  42	ret = xdna_send_msg_wait(xdna, ndev->mgmt_chann, msg);
  43	if (ret == -ETIME)
  44		aie2_destroy_mgmt_chann(ndev);
  45
  46	if (!ret && *hdl->status != AIE2_STATUS_SUCCESS) {
  47		XDNA_ERR(xdna, "command opcode 0x%x failed, status 0x%x",
  48			 msg->opcode, *hdl->data);
  49		ret = -EINVAL;
  50	}
  51
  52	return ret;
  53}
  54
  55int aie2_suspend_fw(struct amdxdna_dev_hdl *ndev)
  56{
  57	DECLARE_AIE2_MSG(suspend, MSG_OP_SUSPEND);
  58
  59	return aie2_send_mgmt_msg_wait(ndev, &msg);
  60}
  61
  62int aie2_resume_fw(struct amdxdna_dev_hdl *ndev)
  63{
  64	DECLARE_AIE2_MSG(suspend, MSG_OP_RESUME);
  65
  66	return aie2_send_mgmt_msg_wait(ndev, &msg);
  67}
  68
  69int aie2_set_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 value)
  70{
  71	DECLARE_AIE2_MSG(set_runtime_cfg, MSG_OP_SET_RUNTIME_CONFIG);
  72	int ret;
  73
  74	req.type = type;
  75	req.value = value;
  76
  77	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
  78	if (ret) {
  79		XDNA_ERR(ndev->xdna, "Failed to set runtime config, ret %d", ret);
  80		return ret;
  81	}
  82
  83	return 0;
  84}
  85
  86int aie2_get_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 *value)
  87{
  88	DECLARE_AIE2_MSG(get_runtime_cfg, MSG_OP_GET_RUNTIME_CONFIG);
  89	int ret;
  90
  91	req.type = type;
  92	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
  93	if (ret) {
  94		XDNA_ERR(ndev->xdna, "Failed to get runtime config, ret %d", ret);
  95		return ret;
  96	}
  97
  98	*value = resp.value;
  99	return 0;
 100}
 101
 102int aie2_assign_mgmt_pasid(struct amdxdna_dev_hdl *ndev, u16 pasid)
 103{
 104	DECLARE_AIE2_MSG(assign_mgmt_pasid, MSG_OP_ASSIGN_MGMT_PASID);
 105
 106	req.pasid = pasid;
 107
 108	return aie2_send_mgmt_msg_wait(ndev, &msg);
 109}
 110
 111int aie2_query_aie_version(struct amdxdna_dev_hdl *ndev, struct aie_version *version)
 112{
 113	DECLARE_AIE2_MSG(aie_version_info, MSG_OP_QUERY_AIE_VERSION);
 114	struct amdxdna_dev *xdna = ndev->xdna;
 115	int ret;
 116
 117	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
 118	if (ret)
 119		return ret;
 120
 121	XDNA_DBG(xdna, "Query AIE version - major: %u minor: %u completed",
 122		 resp.major, resp.minor);
 123
 124	version->major = resp.major;
 125	version->minor = resp.minor;
 126
 127	return 0;
 128}
 129
 130int aie2_query_aie_metadata(struct amdxdna_dev_hdl *ndev, struct aie_metadata *metadata)
 131{
 132	DECLARE_AIE2_MSG(aie_tile_info, MSG_OP_QUERY_AIE_TILE_INFO);
 133	int ret;
 134
 135	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
 136	if (ret)
 137		return ret;
 138
 139	metadata->size = resp.info.size;
 140	metadata->cols = resp.info.cols;
 141	metadata->rows = resp.info.rows;
 142
 143	metadata->version.major = resp.info.major;
 144	metadata->version.minor = resp.info.minor;
 145
 146	metadata->core.row_count = resp.info.core_rows;
 147	metadata->core.row_start = resp.info.core_row_start;
 148	metadata->core.dma_channel_count = resp.info.core_dma_channels;
 149	metadata->core.lock_count = resp.info.core_locks;
 150	metadata->core.event_reg_count = resp.info.core_events;
 151
 152	metadata->mem.row_count = resp.info.mem_rows;
 153	metadata->mem.row_start = resp.info.mem_row_start;
 154	metadata->mem.dma_channel_count = resp.info.mem_dma_channels;
 155	metadata->mem.lock_count = resp.info.mem_locks;
 156	metadata->mem.event_reg_count = resp.info.mem_events;
 157
 158	metadata->shim.row_count = resp.info.shim_rows;
 159	metadata->shim.row_start = resp.info.shim_row_start;
 160	metadata->shim.dma_channel_count = resp.info.shim_dma_channels;
 161	metadata->shim.lock_count = resp.info.shim_locks;
 162	metadata->shim.event_reg_count = resp.info.shim_events;
 163
 164	return 0;
 165}
 166
 167int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev,
 168				struct amdxdna_fw_ver *fw_ver)
 169{
 170	DECLARE_AIE2_MSG(firmware_version, MSG_OP_GET_FIRMWARE_VERSION);
 171	int ret;
 172
 173	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
 174	if (ret)
 175		return ret;
 176
 177	fw_ver->major = resp.major;
 178	fw_ver->minor = resp.minor;
 179	fw_ver->sub = resp.sub;
 180	fw_ver->build = resp.build;
 181
 182	return 0;
 183}
 184
 185static int aie2_destroy_context_req(struct amdxdna_dev_hdl *ndev, u32 id)
 186{
 187	DECLARE_AIE2_MSG(destroy_ctx, MSG_OP_DESTROY_CONTEXT);
 188	struct amdxdna_dev *xdna = ndev->xdna;
 189	int ret;
 190
 191	req.context_id = id;
 192	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
 193	if (ret && ret != -ENODEV)
 194		XDNA_WARN(xdna, "Destroy context failed, ret %d", ret);
 195	else if (ret == -ENODEV)
 196		XDNA_DBG(xdna, "Destroy context: device already stopped");
 197
 198	return ret;
 199}
 200int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx)
 201{
 202	DECLARE_AIE2_MSG(create_ctx, MSG_OP_CREATE_CONTEXT);
 203	struct amdxdna_dev *xdna = ndev->xdna;
 204	struct xdna_mailbox_chann_res x2i;
 205	struct xdna_mailbox_chann_res i2x;
 206	struct cq_pair *cq_pair;
 207	u32 intr_reg;
 208	int ret;
 209
 210	req.aie_type = 1;
 211	req.start_col = hwctx->start_col;
 212	req.num_col = hwctx->num_col;
 213	req.num_unused_col = hwctx->num_unused_col;
 214	req.num_cq_pairs_requested = 1;
 215	req.pasid = hwctx->client->pasid;
 216	req.context_priority = 2;
 217
 218	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
 219	if (ret)
 220		return ret;
 221
 222	hwctx->fw_ctx_id = resp.context_id;
 223	if (WARN_ON_ONCE(hwctx->fw_ctx_id == -1))
 224		return -EINVAL;
 225
 226	if (ndev->force_preempt_enabled) {
 227		ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_FORCE_PREEMPT, &hwctx->fw_ctx_id);
 228		if (ret) {
 229			XDNA_ERR(xdna, "failed to enable force preempt %d", ret);
 230			goto del_ctx_req;
 231		}
 232	}
 233
 234	cq_pair = &resp.cq_pair[0];
 235	x2i.mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->x2i_q.head_addr);
 236	x2i.mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->x2i_q.tail_addr);
 237	x2i.rb_start_addr   = AIE2_SRAM_OFF(ndev, cq_pair->x2i_q.buf_addr);
 238	x2i.rb_size	    = cq_pair->x2i_q.buf_size;
 239
 240	i2x.mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->i2x_q.head_addr);
 241	i2x.mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->i2x_q.tail_addr);
 242	i2x.rb_start_addr   = AIE2_SRAM_OFF(ndev, cq_pair->i2x_q.buf_addr);
 243	i2x.rb_size	    = cq_pair->i2x_q.buf_size;
 244
 245	ret = pci_irq_vector(to_pci_dev(xdna->ddev.dev), resp.msix_id);
 246	if (ret == -EINVAL) {
 247		XDNA_ERR(xdna, "Alloc IRQ failed %d", ret);
 248		goto del_ctx_req;
 249	}
 250
 251	intr_reg = i2x.mb_head_ptr_reg + 4;
 252	hwctx->priv->mbox_chann = xdna_mailbox_create_channel(ndev->mbox, &x2i, &i2x,
 253							      intr_reg, ret);
 254	if (!hwctx->priv->mbox_chann) {
 255		XDNA_ERR(xdna, "Not able to create channel");
 256		ret = -EINVAL;
 257		goto del_ctx_req;
 258	}
 259	ndev->hwctx_num++;
 260
 261	XDNA_DBG(xdna, "Mailbox channel irq: %d, msix_id: %d", ret, resp.msix_id);
 262	XDNA_DBG(xdna, "Created fw ctx %d pasid %d", hwctx->fw_ctx_id, hwctx->client->pasid);
 263
 264	return 0;
 265
 266del_ctx_req:
 267	aie2_destroy_context_req(ndev, hwctx->fw_ctx_id);
 268	return ret;
 269}
 270
 271int aie2_destroy_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx)
 272{
 273	struct amdxdna_dev *xdna = ndev->xdna;
 274	int ret;
 275
 276	if (!hwctx->priv->mbox_chann)
 277		return 0;
 278
 279	xdna_mailbox_stop_channel(hwctx->priv->mbox_chann);
 280	ret = aie2_destroy_context_req(ndev, hwctx->fw_ctx_id);
 281	xdna_mailbox_destroy_channel(hwctx->priv->mbox_chann);
 282	XDNA_DBG(xdna, "Destroyed fw ctx %d", hwctx->fw_ctx_id);
 283	hwctx->priv->mbox_chann = NULL;
 284	hwctx->fw_ctx_id = -1;
 285	ndev->hwctx_num--;
 286
 287	return ret;
 288}
 289
 290int aie2_map_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size)
 291{
 292	DECLARE_AIE2_MSG(map_host_buffer, MSG_OP_MAP_HOST_BUFFER);
 293	struct amdxdna_dev *xdna = ndev->xdna;
 294	int ret;
 295
 296	req.context_id = context_id;
 297	req.buf_addr = addr;
 298	req.buf_size = size;
 299	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
 300	if (ret)
 301		return ret;
 302
 303	XDNA_DBG(xdna, "fw ctx %d map host buf addr 0x%llx size 0x%llx",
 304		 context_id, addr, size);
 305
 306	return 0;
 307}
 308
 309static int amdxdna_hwctx_col_map(struct amdxdna_hwctx *hwctx, void *arg)
 310{
 311	u32 *bitmap = arg;
 312
 313	*bitmap |= GENMASK(hwctx->start_col + hwctx->num_col - 1, hwctx->start_col);
 314
 315	return 0;
 316}
 317
 318int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf,
 319		      u32 size, u32 *cols_filled)
 320{
 321	DECLARE_AIE2_MSG(aie_column_info, MSG_OP_QUERY_COL_STATUS);
 322	struct amdxdna_dev *xdna = ndev->xdna;
 323	struct amdxdna_client *client;
 324	dma_addr_t dma_addr;
 325	u32 aie_bitmap = 0;
 326	u8 *buff_addr;
 327	int ret;
 328
 329	buff_addr = dma_alloc_noncoherent(xdna->ddev.dev, size, &dma_addr,
 330					  DMA_FROM_DEVICE, GFP_KERNEL);
 331	if (!buff_addr)
 332		return -ENOMEM;
 333
 334	/* Go through each hardware context and mark the AIE columns that are active */
 335	list_for_each_entry(client, &xdna->client_list, node)
 336		amdxdna_hwctx_walk(client, &aie_bitmap, amdxdna_hwctx_col_map);
 337
 338	*cols_filled = 0;
 339	req.dump_buff_addr = dma_addr;
 340	req.dump_buff_size = size;
 341	req.num_cols = hweight32(aie_bitmap);
 342	req.aie_bitmap = aie_bitmap;
 343
 344	drm_clflush_virt_range(buff_addr, size); /* device can access */
 345	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
 346	if (ret) {
 347		XDNA_ERR(xdna, "Error during NPU query, status %d", ret);
 348		goto fail;
 349	}
 350
 351	XDNA_DBG(xdna, "Query NPU status completed");
 352
 353	if (size < resp.size) {
 354		ret = -EINVAL;
 355		XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", size, resp.size);
 356		goto fail;
 357	}
 358
 359	if (copy_to_user(buf, buff_addr, resp.size)) {
 360		ret = -EFAULT;
 361		XDNA_ERR(xdna, "Failed to copy NPU status to user space");
 362		goto fail;
 363	}
 364
 365	*cols_filled = aie_bitmap;
 366
 367fail:
 368	dma_free_noncoherent(xdna->ddev.dev, size, buff_addr, dma_addr, DMA_FROM_DEVICE);
 369	return ret;
 370}
 371
 372int aie2_query_telemetry(struct amdxdna_dev_hdl *ndev,
 373			 char __user *buf, u32 size,
 374			 struct amdxdna_drm_query_telemetry_header *header)
 375{
 376	DECLARE_AIE2_MSG(get_telemetry, MSG_OP_GET_TELEMETRY);
 377	struct amdxdna_dev *xdna = ndev->xdna;
 378	dma_addr_t dma_addr;
 379	u8 *addr;
 380	int ret;
 381
 382	if (header->type >= MAX_TELEMETRY_TYPE)
 383		return -EINVAL;
 384
 385	addr = dma_alloc_noncoherent(xdna->ddev.dev, size, &dma_addr,
 386				     DMA_FROM_DEVICE, GFP_KERNEL);
 387	if (!addr)
 388		return -ENOMEM;
 389
 390	req.buf_addr = dma_addr;
 391	req.buf_size = size;
 392	req.type = header->type;
 393
 394	drm_clflush_virt_range(addr, size); /* device can access */
 395	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
 396	if (ret) {
 397		XDNA_ERR(xdna, "Query telemetry failed, status %d", ret);
 398		goto free_buf;
 399	}
 400
 401	if (size < resp.size) {
 402		ret = -EINVAL;
 403		XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", size, resp.size);
 404		goto free_buf;
 405	}
 406
 407	if (copy_to_user(buf, addr, resp.size)) {
 408		ret = -EFAULT;
 409		XDNA_ERR(xdna, "Failed to copy telemetry to user space");
 410		goto free_buf;
 411	}
 412
 413	header->major = resp.major;
 414	header->minor = resp.minor;
 415
 416free_buf:
 417	dma_free_noncoherent(xdna->ddev.dev, size, addr, dma_addr, DMA_FROM_DEVICE);
 418	return ret;
 419}
 420
 421int aie2_register_asyn_event_msg(struct amdxdna_dev_hdl *ndev, dma_addr_t addr, u32 size,
 422				 void *handle, int (*cb)(void*, void __iomem *, size_t))
 423{
 424	struct async_event_msg_req req = { 0 };
 425	struct xdna_mailbox_msg msg = {
 426		.send_data = (u8 *)&req,
 427		.send_size = sizeof(req),
 428		.handle = handle,
 429		.opcode = MSG_OP_REGISTER_ASYNC_EVENT_MSG,
 430		.notify_cb = cb,
 431	};
 432
 433	req.buf_addr = addr;
 434	req.buf_size = size;
 435
 436	XDNA_DBG(ndev->xdna, "Register addr 0x%llx size 0x%x", addr, size);
 437	return xdna_mailbox_send_msg(ndev->mgmt_chann, &msg, TX_TIMEOUT);
 438}
 439
 440int aie2_config_cu(struct amdxdna_hwctx *hwctx,
 441		   int (*notify_cb)(void *, void __iomem *, size_t))
 442{
 443	struct mailbox_channel *chann = hwctx->priv->mbox_chann;
 444	struct amdxdna_dev *xdna = hwctx->client->xdna;
 445	u32 shift = xdna->dev_info->dev_mem_buf_shift;
 446	struct config_cu_req req = { 0 };
 447	struct xdna_mailbox_msg msg;
 448	struct drm_gem_object *gobj;
 449	struct amdxdna_gem_obj *abo;
 450	int i;
 451
 452	if (!chann)
 453		return -ENODEV;
 454
 455	if (!hwctx->cus)
 456		return 0;
 457
 458	if (hwctx->cus->num_cus > MAX_NUM_CUS) {
 459		XDNA_DBG(xdna, "Exceed maximum CU %d", MAX_NUM_CUS);
 460		return -EINVAL;
 461	}
 462
 463	for (i = 0; i < hwctx->cus->num_cus; i++) {
 464		struct amdxdna_cu_config *cu = &hwctx->cus->cu_configs[i];
 465
 466		if (XDNA_MBZ_DBG(xdna, cu->pad, sizeof(cu->pad)))
 467			return -EINVAL;
 468
 469		gobj = drm_gem_object_lookup(hwctx->client->filp, cu->cu_bo);
 470		if (!gobj) {
 471			XDNA_ERR(xdna, "Lookup GEM object failed");
 472			return -EINVAL;
 473		}
 474		abo = to_xdna_obj(gobj);
 475
 476		if (abo->type != AMDXDNA_BO_DEV) {
 477			drm_gem_object_put(gobj);
 478			XDNA_ERR(xdna, "Invalid BO type");
 479			return -EINVAL;
 480		}
 481
 482		req.cfgs[i] = FIELD_PREP(AIE2_MSG_CFG_CU_PDI_ADDR,
 483					 abo->mem.dev_addr >> shift);
 484		req.cfgs[i] |= FIELD_PREP(AIE2_MSG_CFG_CU_FUNC, cu->cu_func);
 485		XDNA_DBG(xdna, "CU %d full addr 0x%llx, cfg 0x%x", i,
 486			 abo->mem.dev_addr, req.cfgs[i]);
 487		drm_gem_object_put(gobj);
 488	}
 489	req.num_cus = hwctx->cus->num_cus;
 490
 491	msg.send_data = (u8 *)&req;
 492	msg.send_size = sizeof(req);
 493	msg.handle = hwctx;
 494	msg.opcode = MSG_OP_CONFIG_CU;
 495	msg.notify_cb = notify_cb;
 496	return xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
 497}
 498
 499static int aie2_init_exec_cu_req(struct amdxdna_gem_obj *cmd_bo, void *req,
 500				 size_t *size, u32 *msg_op)
 501{
 502	struct execute_buffer_req *cu_req = req;
 503	u32 cmd_len;
 504	void *cmd;
 505
 506	cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
 507	if (cmd_len > sizeof(cu_req->payload))
 508		return -EINVAL;
 509
 510	cu_req->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
 511	if (cu_req->cu_idx == INVALID_CU_IDX)
 512		return -EINVAL;
 513
 514	memcpy(cu_req->payload, cmd, cmd_len);
 515
 516	*size = sizeof(*cu_req);
 517	*msg_op = MSG_OP_EXECUTE_BUFFER_CF;
 518	return 0;
 519}
 520
 521static int aie2_init_exec_dpu_req(struct amdxdna_gem_obj *cmd_bo, void *req,
 522				  size_t *size, u32 *msg_op)
 523{
 524	struct exec_dpu_req *dpu_req = req;
 525	struct amdxdna_cmd_start_npu *sn;
 526	u32 cmd_len;
 527
 528	sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
 529	if (cmd_len - sizeof(*sn) > sizeof(dpu_req->payload))
 530		return -EINVAL;
 531
 532	dpu_req->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
 533	if (dpu_req->cu_idx == INVALID_CU_IDX)
 534		return -EINVAL;
 535
 536	dpu_req->inst_buf_addr = sn->buffer;
 537	dpu_req->inst_size = sn->buffer_size;
 538	dpu_req->inst_prop_cnt = sn->prop_count;
 539	memcpy(dpu_req->payload, sn->prop_args, cmd_len - sizeof(*sn));
 540
 541	*size = sizeof(*dpu_req);
 542	*msg_op = MSG_OP_EXEC_DPU;
 543	return 0;
 544}
 545
 546static void aie2_init_exec_chain_req(void *req, u64 slot_addr, size_t size, u32 cmd_cnt)
 547{
 548	struct cmd_chain_req *chain_req = req;
 549
 550	chain_req->buf_addr = slot_addr;
 551	chain_req->buf_size = size;
 552	chain_req->count = cmd_cnt;
 553}
 554
 555static void aie2_init_npu_chain_req(void *req, u64 slot_addr, size_t size, u32 cmd_cnt)
 556{
 557	struct cmd_chain_npu_req *npu_chain_req = req;
 558
 559	npu_chain_req->flags = 0;
 560	npu_chain_req->reserved = 0;
 561	npu_chain_req->buf_addr = slot_addr;
 562	npu_chain_req->buf_size = size;
 563	npu_chain_req->count = cmd_cnt;
 564}
 565
 566static int
 567aie2_cmdlist_fill_cf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
 568{
 569	struct cmd_chain_slot_execbuf_cf *cf_slot = slot;
 570	u32 cmd_len;
 571	void *cmd;
 572
 573	cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
 574	if (*size < sizeof(*cf_slot) + cmd_len)
 575		return -EINVAL;
 576
 577	cf_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
 578	if (cf_slot->cu_idx == INVALID_CU_IDX)
 579		return -EINVAL;
 580
 581	cf_slot->arg_cnt = cmd_len / sizeof(u32);
 582	memcpy(cf_slot->args, cmd, cmd_len);
 583	/* Accurate slot size to hint firmware to do necessary copy */
 584	*size = sizeof(*cf_slot) + cmd_len;
 585	return 0;
 586}
 587
 588static int
 589aie2_cmdlist_fill_dpu(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
 590{
 591	struct cmd_chain_slot_dpu *dpu_slot = slot;
 592	struct amdxdna_cmd_start_npu *sn;
 593	u32 cmd_len;
 594	u32 arg_sz;
 595
 596	sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
 597	arg_sz = cmd_len - sizeof(*sn);
 598	if (cmd_len < sizeof(*sn) || arg_sz > MAX_DPU_ARGS_SIZE)
 599		return -EINVAL;
 600
 601	if (*size < sizeof(*dpu_slot) + arg_sz)
 602		return -EINVAL;
 603
 604	dpu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
 605	if (dpu_slot->cu_idx == INVALID_CU_IDX)
 606		return -EINVAL;
 607
 608	dpu_slot->inst_buf_addr = sn->buffer;
 609	dpu_slot->inst_size = sn->buffer_size;
 610	dpu_slot->inst_prop_cnt = sn->prop_count;
 611	dpu_slot->arg_cnt = arg_sz / sizeof(u32);
 612	memcpy(dpu_slot->args, sn->prop_args, arg_sz);
 613
 614	/* Accurate slot size to hint firmware to do necessary copy */
 615	*size = sizeof(*dpu_slot) + arg_sz;
 616	return 0;
 617}
 618
 619static int aie2_cmdlist_unsupp(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
 620{
 621	return -EOPNOTSUPP;
 622}
 623
 624static u32 aie2_get_chain_msg_op(u32 cmd_op)
 625{
 626	switch (cmd_op) {
 627	case ERT_START_CU:
 628		return MSG_OP_CHAIN_EXEC_BUFFER_CF;
 629	case ERT_START_NPU:
 630		return MSG_OP_CHAIN_EXEC_DPU;
 631	default:
 632		break;
 633	}
 634
 635	return MSG_OP_MAX_OPCODE;
 636}
 637
 638static struct aie2_exec_msg_ops legacy_exec_message_ops = {
 639	.init_cu_req = aie2_init_exec_cu_req,
 640	.init_dpu_req = aie2_init_exec_dpu_req,
 641	.init_chain_req = aie2_init_exec_chain_req,
 642	.fill_cf_slot = aie2_cmdlist_fill_cf,
 643	.fill_dpu_slot = aie2_cmdlist_fill_dpu,
 644	.fill_preempt_slot = aie2_cmdlist_unsupp,
 645	.fill_elf_slot = aie2_cmdlist_unsupp,
 646	.get_chain_msg_op = aie2_get_chain_msg_op,
 647};
 648
 649static int
 650aie2_cmdlist_fill_npu_cf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
 651{
 652	struct cmd_chain_slot_npu *npu_slot = slot;
 653	u32 cmd_len;
 654	void *cmd;
 655
 656	cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
 657	if (*size < sizeof(*npu_slot) + cmd_len)
 658		return -EINVAL;
 659
 660	memset(npu_slot, 0, sizeof(*npu_slot));
 661	npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
 662	if (npu_slot->cu_idx == INVALID_CU_IDX)
 663		return -EINVAL;
 664
 665	npu_slot->type = EXEC_NPU_TYPE_NON_ELF;
 666	npu_slot->arg_cnt = cmd_len / sizeof(u32);
 667	memcpy(npu_slot->args, cmd, cmd_len);
 668
 669	*size = sizeof(*npu_slot) + cmd_len;
 670	return 0;
 671}
 672
 673static int
 674aie2_cmdlist_fill_npu_dpu(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
 675{
 676	struct cmd_chain_slot_npu *npu_slot = slot;
 677	struct amdxdna_cmd_start_npu *sn;
 678	u32 cmd_len;
 679	u32 arg_sz;
 680
 681	sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
 682	arg_sz = cmd_len - sizeof(*sn);
 683	if (cmd_len < sizeof(*sn) || arg_sz > MAX_NPU_ARGS_SIZE)
 684		return -EINVAL;
 685
 686	if (*size < sizeof(*npu_slot) + arg_sz)
 687		return -EINVAL;
 688
 689	memset(npu_slot, 0, sizeof(*npu_slot));
 690	npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
 691	if (npu_slot->cu_idx == INVALID_CU_IDX)
 692		return -EINVAL;
 693
 694	npu_slot->type = EXEC_NPU_TYPE_PARTIAL_ELF;
 695	npu_slot->inst_buf_addr = sn->buffer;
 696	npu_slot->inst_size = sn->buffer_size;
 697	npu_slot->inst_prop_cnt = sn->prop_count;
 698	npu_slot->arg_cnt = arg_sz / sizeof(u32);
 699	memcpy(npu_slot->args, sn->prop_args, arg_sz);
 700
 701	*size = sizeof(*npu_slot) + arg_sz;
 702	return 0;
 703}
 704
 705static int
 706aie2_cmdlist_fill_npu_preempt(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
 707{
 708	struct cmd_chain_slot_npu *npu_slot = slot;
 709	struct amdxdna_cmd_preempt_data *pd;
 710	u32 cmd_len;
 711	u32 arg_sz;
 712
 713	pd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
 714	arg_sz = cmd_len - sizeof(*pd);
 715	if (cmd_len < sizeof(*pd) || arg_sz > MAX_NPU_ARGS_SIZE)
 716		return -EINVAL;
 717
 718	if (*size < sizeof(*npu_slot) + arg_sz)
 719		return -EINVAL;
 720
 721	memset(npu_slot, 0, sizeof(*npu_slot));
 722	npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
 723	if (npu_slot->cu_idx == INVALID_CU_IDX)
 724		return -EINVAL;
 725
 726	npu_slot->type = EXEC_NPU_TYPE_PREEMPT;
 727	npu_slot->inst_buf_addr = pd->inst_buf;
 728	npu_slot->save_buf_addr = pd->save_buf;
 729	npu_slot->restore_buf_addr = pd->restore_buf;
 730	npu_slot->inst_size = pd->inst_size;
 731	npu_slot->save_size = pd->save_size;
 732	npu_slot->restore_size = pd->restore_size;
 733	npu_slot->inst_prop_cnt = pd->inst_prop_cnt;
 734	npu_slot->arg_cnt = arg_sz / sizeof(u32);
 735	memcpy(npu_slot->args, pd->prop_args, arg_sz);
 736
 737	*size = sizeof(*npu_slot) + arg_sz;
 738	return 0;
 739}
 740
 741static int
 742aie2_cmdlist_fill_npu_elf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
 743{
 744	struct cmd_chain_slot_npu *npu_slot = slot;
 745	struct amdxdna_cmd_preempt_data *pd;
 746	u32 cmd_len;
 747	u32 arg_sz;
 748
 749	pd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
 750	arg_sz = cmd_len - sizeof(*pd);
 751	if (cmd_len < sizeof(*pd) || arg_sz > MAX_NPU_ARGS_SIZE)
 752		return -EINVAL;
 753
 754	if (*size < sizeof(*npu_slot) + arg_sz)
 755		return -EINVAL;
 756
 757	memset(npu_slot, 0, sizeof(*npu_slot));
 758	npu_slot->type = EXEC_NPU_TYPE_ELF;
 759	npu_slot->inst_buf_addr = pd->inst_buf;
 760	npu_slot->save_buf_addr = pd->save_buf;
 761	npu_slot->restore_buf_addr = pd->restore_buf;
 762	npu_slot->inst_size = pd->inst_size;
 763	npu_slot->save_size = pd->save_size;
 764	npu_slot->restore_size = pd->restore_size;
 765	npu_slot->inst_prop_cnt = pd->inst_prop_cnt;
 766	npu_slot->arg_cnt = 1;
 767	npu_slot->args[0] = AIE2_EXEC_BUFFER_KERNEL_OP_TXN;
 768
 769	*size = struct_size(npu_slot, args, npu_slot->arg_cnt);
 770	return 0;
 771}
 772
 773static u32 aie2_get_npu_chain_msg_op(u32 cmd_op)
 774{
 775	return MSG_OP_CHAIN_EXEC_NPU;
 776}
 777
 778static struct aie2_exec_msg_ops npu_exec_message_ops = {
 779	.init_cu_req = aie2_init_exec_cu_req,
 780	.init_dpu_req = aie2_init_exec_dpu_req,
 781	.init_chain_req = aie2_init_npu_chain_req,
 782	.fill_cf_slot = aie2_cmdlist_fill_npu_cf,
 783	.fill_dpu_slot = aie2_cmdlist_fill_npu_dpu,
 784	.fill_preempt_slot = aie2_cmdlist_fill_npu_preempt,
 785	.fill_elf_slot = aie2_cmdlist_fill_npu_elf,
 786	.get_chain_msg_op = aie2_get_npu_chain_msg_op,
 787};
 788
 789static int aie2_init_exec_req(void *req, struct amdxdna_gem_obj *cmd_abo,
 790			      size_t *size, u32 *msg_op)
 791{
 792	struct amdxdna_dev *xdna = cmd_abo->client->xdna;
 793	int ret;
 794	u32 op;
 795
 796
 797	op = amdxdna_cmd_get_op(cmd_abo);
 798	switch (op) {
 799	case ERT_START_CU:
 800		ret = EXEC_MSG_OPS(xdna)->init_cu_req(cmd_abo, req, size, msg_op);
 801		if (ret) {
 802			XDNA_DBG(xdna, "Init CU req failed ret %d", ret);
 803			return ret;
 804		}
 805		break;
 806	case ERT_START_NPU:
 807		ret = EXEC_MSG_OPS(xdna)->init_dpu_req(cmd_abo, req, size, msg_op);
 808		if (ret) {
 809			XDNA_DBG(xdna, "Init DPU req failed ret %d", ret);
 810			return ret;
 811		}
 812
 813		break;
 814	default:
 815		XDNA_ERR(xdna, "Unsupported op %d", op);
 816		ret = -EOPNOTSUPP;
 817		break;
 818	}
 819
 820	return ret;
 821}
 822
 823static int
 824aie2_cmdlist_fill_slot(void *slot, struct amdxdna_gem_obj *cmd_abo,
 825		       size_t *size, u32 *cmd_op)
 826{
 827	struct amdxdna_dev *xdna = cmd_abo->client->xdna;
 828	int ret;
 829	u32 op;
 830
 831	op = amdxdna_cmd_get_op(cmd_abo);
 832	if (*cmd_op == ERT_INVALID_CMD)
 833		*cmd_op = op;
 834	else if (op != *cmd_op)
 835		return -EINVAL;
 836
 837	switch (op) {
 838	case ERT_START_CU:
 839		ret = EXEC_MSG_OPS(xdna)->fill_cf_slot(cmd_abo, slot, size);
 840		break;
 841	case ERT_START_NPU:
 842		ret = EXEC_MSG_OPS(xdna)->fill_dpu_slot(cmd_abo, slot, size);
 843		break;
 844	case ERT_START_NPU_PREEMPT:
 845		if (!AIE2_FEATURE_ON(xdna->dev_handle, AIE2_PREEMPT))
 846			return -EOPNOTSUPP;
 847		ret = EXEC_MSG_OPS(xdna)->fill_preempt_slot(cmd_abo, slot, size);
 848		break;
 849	case ERT_START_NPU_PREEMPT_ELF:
 850		if (!AIE2_FEATURE_ON(xdna->dev_handle, AIE2_PREEMPT))
 851			return -EOPNOTSUPP;
 852		ret = EXEC_MSG_OPS(xdna)->fill_elf_slot(cmd_abo, slot, size);
 853		break;
 854	default:
 855		XDNA_INFO(xdna, "Unsupported op %d", op);
 856		ret = -EOPNOTSUPP;
 857		break;
 858	}
 859
 860	return ret;
 861}
 862
 863void aie2_msg_init(struct amdxdna_dev_hdl *ndev)
 864{
 865	if (AIE2_FEATURE_ON(ndev, AIE2_NPU_COMMAND))
 866		ndev->exec_msg_ops = &npu_exec_message_ops;
 867	else
 868		ndev->exec_msg_ops = &legacy_exec_message_ops;
 869}
 870
 871void aie2_destroy_mgmt_chann(struct amdxdna_dev_hdl *ndev)
 872{
 873	struct amdxdna_dev *xdna = ndev->xdna;
 874
 875	drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
 876
 877	if (!ndev->mgmt_chann)
 878		return;
 879
 880	xdna_mailbox_stop_channel(ndev->mgmt_chann);
 881	xdna_mailbox_destroy_channel(ndev->mgmt_chann);
 882	ndev->mgmt_chann = NULL;
 883}
 884
 885static inline struct amdxdna_gem_obj *
 886aie2_cmdlist_get_cmd_buf(struct amdxdna_sched_job *job)
 887{
 888	int idx = get_job_idx(job->seq);
 889
 890	return job->hwctx->priv->cmd_buf[idx];
 891}
 892
 893int aie2_execbuf(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job,
 894		 int (*notify_cb)(void *, void __iomem *, size_t))
 895{
 896	struct mailbox_channel *chann = hwctx->priv->mbox_chann;
 897	struct amdxdna_dev *xdna = hwctx->client->xdna;
 898	struct amdxdna_gem_obj *cmd_abo = job->cmd_bo;
 899	struct xdna_mailbox_msg msg;
 900	union exec_req req;
 901	int ret;
 902
 903	if (!chann)
 904		return -ENODEV;
 905
 906	ret = aie2_init_exec_req(&req, cmd_abo, &msg.send_size, &msg.opcode);
 907	if (ret)
 908		return ret;
 909
 910	msg.handle = job;
 911	msg.notify_cb = notify_cb;
 912	msg.send_data = (u8 *)&req;
 913	print_hex_dump_debug("cmd: ", DUMP_PREFIX_OFFSET, 16, 4, &req,
 914			     0x40, false);
 915
 916	ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
 917	if (ret) {
 918		XDNA_ERR(xdna, "Send message failed");
 919		return ret;
 920	}
 921
 922	return 0;
 923}
 924
 925int aie2_cmdlist_multi_execbuf(struct amdxdna_hwctx *hwctx,
 926			       struct amdxdna_sched_job *job,
 927			       int (*notify_cb)(void *, void __iomem *, size_t))
 928{
 929	struct amdxdna_gem_obj *cmdbuf_abo = aie2_cmdlist_get_cmd_buf(job);
 930	struct mailbox_channel *chann = hwctx->priv->mbox_chann;
 931	struct amdxdna_client *client = hwctx->client;
 932	struct amdxdna_gem_obj *cmd_abo = job->cmd_bo;
 933	struct amdxdna_dev *xdna = client->xdna;
 934	struct amdxdna_cmd_chain *payload;
 935	struct xdna_mailbox_msg msg;
 936	union exec_chain_req req;
 937	u32 payload_len;
 938	u32 offset = 0;
 939	size_t size;
 940	int ret;
 941	u32 op;
 942	u32 i;
 943
 944	op = amdxdna_cmd_get_op(cmd_abo);
 945	payload = amdxdna_cmd_get_payload(cmd_abo, &payload_len);
 946	if (op != ERT_CMD_CHAIN || !payload ||
 947	    payload_len < struct_size(payload, data, payload->command_count))
 948		return -EINVAL;
 949
 950	op = ERT_INVALID_CMD;
 951	for (i = 0; i < payload->command_count; i++) {
 952		u32 boh = (u32)(payload->data[i]);
 953		struct amdxdna_gem_obj *abo;
 954
 955		abo = amdxdna_gem_get_obj(client, boh, AMDXDNA_BO_CMD);
 956		if (!abo) {
 957			XDNA_ERR(xdna, "Failed to find cmd BO %d", boh);
 958			return -ENOENT;
 959		}
 960
 961		size = cmdbuf_abo->mem.size - offset;
 962		ret = aie2_cmdlist_fill_slot(cmdbuf_abo->mem.kva + offset,
 963					     abo, &size, &op);
 964		amdxdna_gem_put_obj(abo);
 965		if (ret)
 966			return ret;
 967
 968		offset += size;
 969	}
 970	msg.opcode = EXEC_MSG_OPS(xdna)->get_chain_msg_op(op);
 971	if (msg.opcode == MSG_OP_MAX_OPCODE)
 972		return -EOPNOTSUPP;
 973
 974	/* The offset is the accumulated total size of the cmd buffer */
 975	EXEC_MSG_OPS(xdna)->init_chain_req(&req, cmdbuf_abo->mem.dev_addr,
 976					   offset, payload->command_count);
 977	drm_clflush_virt_range(cmdbuf_abo->mem.kva, offset);
 978
 979	msg.handle = job;
 980	msg.notify_cb = notify_cb;
 981	msg.send_data = (u8 *)&req;
 982	msg.send_size = sizeof(req);
 983	ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
 984	if (ret) {
 985		XDNA_ERR(xdna, "Send message failed");
 986		return ret;
 987	}
 988
 989	return 0;
 990}
 991
 992int aie2_cmdlist_single_execbuf(struct amdxdna_hwctx *hwctx,
 993				struct amdxdna_sched_job *job,
 994				int (*notify_cb)(void *, void __iomem *, size_t))
 995{
 996	struct amdxdna_gem_obj *cmdbuf_abo = aie2_cmdlist_get_cmd_buf(job);
 997	struct mailbox_channel *chann = hwctx->priv->mbox_chann;
 998	struct amdxdna_dev *xdna = hwctx->client->xdna;
 999	struct amdxdna_gem_obj *cmd_abo = job->cmd_bo;
1000	struct xdna_mailbox_msg msg;
1001	union exec_chain_req req;
1002	u32 op = ERT_INVALID_CMD;
1003	size_t size;
1004	int ret;
1005
1006	size = cmdbuf_abo->mem.size;
1007	ret = aie2_cmdlist_fill_slot(cmdbuf_abo->mem.kva, cmd_abo, &size, &op);
1008	if (ret)
1009		return ret;
1010
1011	msg.opcode = EXEC_MSG_OPS(xdna)->get_chain_msg_op(op);
1012	if (msg.opcode == MSG_OP_MAX_OPCODE)
1013		return -EOPNOTSUPP;
1014
1015	EXEC_MSG_OPS(xdna)->init_chain_req(&req, cmdbuf_abo->mem.dev_addr,
1016					   size, 1);
1017	drm_clflush_virt_range(cmdbuf_abo->mem.kva, size);
1018
1019	msg.handle = job;
1020	msg.notify_cb = notify_cb;
1021	msg.send_data = (u8 *)&req;
1022	msg.send_size = sizeof(req);
1023	ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
1024	if (ret) {
1025		XDNA_ERR(hwctx->client->xdna, "Send message failed");
1026		return ret;
1027	}
1028
1029	return 0;
1030}
1031
1032int aie2_sync_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job,
1033		 int (*notify_cb)(void *, void __iomem *, size_t))
1034{
1035	struct mailbox_channel *chann = hwctx->priv->mbox_chann;
1036	struct amdxdna_gem_obj *abo = to_xdna_obj(job->bos[0]);
1037	struct amdxdna_dev *xdna = hwctx->client->xdna;
1038	struct xdna_mailbox_msg msg;
1039	struct sync_bo_req req;
1040	int ret = 0;
1041
1042	req.src_addr = 0;
1043	req.dst_addr = amdxdna_dev_bo_offset(abo);
1044	req.size = abo->mem.size;
1045
1046	/* Device to Host */
1047	req.type = FIELD_PREP(AIE2_MSG_SYNC_BO_SRC_TYPE, SYNC_BO_DEV_MEM) |
1048		FIELD_PREP(AIE2_MSG_SYNC_BO_DST_TYPE, SYNC_BO_HOST_MEM);
1049
1050	XDNA_DBG(xdna, "sync %d bytes src(0x%llx) to dst(0x%llx) completed",
1051		 req.size, req.src_addr, req.dst_addr);
1052
1053	msg.handle = job;
1054	msg.notify_cb = notify_cb;
1055	msg.send_data = (u8 *)&req;
1056	msg.send_size = sizeof(req);
1057	msg.opcode = MSG_OP_SYNC_BO;
1058
1059	ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
1060	if (ret) {
1061		XDNA_ERR(xdna, "Send message failed");
1062		return ret;
1063	}
1064
1065	return 0;
1066}
1067
1068int aie2_config_debug_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job,
1069			 int (*notify_cb)(void *, void __iomem *, size_t))
1070{
1071	struct mailbox_channel *chann = hwctx->priv->mbox_chann;
1072	struct amdxdna_gem_obj *abo = to_xdna_obj(job->bos[0]);
1073	struct amdxdna_dev *xdna = hwctx->client->xdna;
1074	struct config_debug_bo_req req;
1075	struct xdna_mailbox_msg msg;
1076
1077	if (job->drv_cmd->opcode == ATTACH_DEBUG_BO)
1078		req.config = DEBUG_BO_REGISTER;
1079	else
1080		req.config = DEBUG_BO_UNREGISTER;
1081
1082	req.offset = amdxdna_dev_bo_offset(abo);
1083	req.size = abo->mem.size;
1084
1085	XDNA_DBG(xdna, "offset 0x%llx size 0x%llx config %d",
1086		 req.offset, req.size, req.config);
1087
1088	msg.handle = job;
1089	msg.notify_cb = notify_cb;
1090	msg.send_data = (u8 *)&req;
1091	msg.send_size = sizeof(req);
1092	msg.opcode = MSG_OP_CONFIG_DEBUG_BO;
1093
1094	return xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
1095}