aie2_pci.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2/*
  3 * Copyright (C) 2023-2024, Advanced Micro Devices, Inc.
  4 */
  5
  6#ifndef _AIE2_PCI_H_
  7#define _AIE2_PCI_H_
  8
  9#include <drm/amdxdna_accel.h>
 10#include <linux/semaphore.h>
 11
 12#include "amdxdna_mailbox.h"
 13
 14#define AIE2_INTERVAL	20000	/* us */
 15#define AIE2_TIMEOUT	1000000	/* us */
 16
 17/* Firmware determines device memory base address and size */
 18#define AIE2_DEVM_BASE	0x4000000
 19#define AIE2_DEVM_SIZE	SZ_64M
 20
 21#define NDEV2PDEV(ndev) (to_pci_dev((ndev)->xdna->ddev.dev))
 22
 23#define AIE2_SRAM_OFF(ndev, addr) ((addr) - (ndev)->priv->sram_dev_addr)
 24#define AIE2_MBOX_OFF(ndev, addr) ((addr) - (ndev)->priv->mbox_dev_addr)
 25
 26#define PSP_REG_BAR(ndev, idx) ((ndev)->priv->psp_regs_off[(idx)].bar_idx)
 27#define PSP_REG_OFF(ndev, idx) ((ndev)->priv->psp_regs_off[(idx)].offset)
 28#define SRAM_REG_OFF(ndev, idx) ((ndev)->priv->sram_offs[(idx)].offset)
 29
 30#define SMU_REG(ndev, idx) \
 31({ \
 32	typeof(ndev) _ndev = ndev; \
 33	((_ndev)->smu_base + (_ndev)->priv->smu_regs_off[(idx)].offset); \
 34})
 35#define SRAM_GET_ADDR(ndev, idx) \
 36({ \
 37	typeof(ndev) _ndev = ndev; \
 38	((_ndev)->sram_base + SRAM_REG_OFF((_ndev), (idx))); \
 39})
 40
 41#define CHAN_SLOT_SZ SZ_8K
 42#define MBOX_SIZE(ndev) \
 43({ \
 44	typeof(ndev) _ndev = (ndev); \
 45	((_ndev)->priv->mbox_size) ? (_ndev)->priv->mbox_size : \
 46	pci_resource_len(NDEV2PDEV(_ndev), (_ndev)->xdna->dev_info->mbox_bar); \
 47})
 48
 49enum aie2_smu_reg_idx {
 50	SMU_CMD_REG = 0,
 51	SMU_ARG_REG,
 52	SMU_INTR_REG,
 53	SMU_RESP_REG,
 54	SMU_OUT_REG,
 55	SMU_MAX_REGS /* Keep this at the end */
 56};
 57
 58enum aie2_sram_reg_idx {
 59	MBOX_CHANN_OFF = 0,
 60	FW_ALIVE_OFF,
 61	SRAM_MAX_INDEX /* Keep this at the end */
 62};
 63
 64enum psp_reg_idx {
 65	PSP_CMD_REG = 0,
 66	PSP_ARG0_REG,
 67	PSP_ARG1_REG,
 68	PSP_ARG2_REG,
 69	PSP_NUM_IN_REGS, /* number of input registers */
 70	PSP_INTR_REG = PSP_NUM_IN_REGS,
 71	PSP_STATUS_REG,
 72	PSP_RESP_REG,
 73	PSP_MAX_REGS /* Keep this at the end */
 74};
 75
 76struct amdxdna_client;
 77struct amdxdna_fw_ver;
 78struct amdxdna_hwctx;
 79struct amdxdna_sched_job;
 80
 81struct psp_config {
 82	const void	*fw_buf;
 83	u32		fw_size;
 84	void __iomem	*psp_regs[PSP_MAX_REGS];
 85};
 86
 87struct aie_version {
 88	u16 major;
 89	u16 minor;
 90};
 91
 92struct aie_tile_metadata {
 93	u16 row_count;
 94	u16 row_start;
 95	u16 dma_channel_count;
 96	u16 lock_count;
 97	u16 event_reg_count;
 98};
 99
100struct aie_metadata {
101	u32 size;
102	u16 cols;
103	u16 rows;
104	struct aie_version version;
105	struct aie_tile_metadata core;
106	struct aie_tile_metadata mem;
107	struct aie_tile_metadata shim;
108};
109
110enum rt_config_category {
111	AIE2_RT_CFG_INIT,
112	AIE2_RT_CFG_CLK_GATING,
113	AIE2_RT_CFG_FORCE_PREEMPT,
114	AIE2_RT_CFG_FRAME_BOUNDARY_PREEMPT,
115};
116
117struct rt_config {
118	u32	type;
119	u32	value;
120	u32	category;
121	unsigned long feature_mask;
122};
123
124struct dpm_clk_freq {
125	u32	npuclk;
126	u32	hclk;
127};
128
129/*
130 * Define the maximum number of pending commands in a hardware context.
131 * Must be power of 2!
132 */
133#define HWCTX_MAX_CMDS		4
134#define get_job_idx(seq) ((seq) & (HWCTX_MAX_CMDS - 1))
135struct amdxdna_hwctx_priv {
136	struct amdxdna_gem_obj		*heap;
137	void				*mbox_chann;
138
139	struct drm_gpu_scheduler	sched;
140	struct drm_sched_entity		entity;
141
142	struct mutex			io_lock; /* protect seq and cmd order */
143	struct wait_queue_head		job_free_wq;
144	u32				num_pending;
145	u64				seq;
146	struct semaphore		job_sem;
147	bool				job_done;
148
149	/* Completed job counter */
150	u64				completed;
151
152	struct amdxdna_gem_obj		*cmd_buf[HWCTX_MAX_CMDS];
153	struct drm_syncobj		*syncobj;
154};
155
156enum aie2_dev_status {
157	AIE2_DEV_UNINIT,
158	AIE2_DEV_INIT,
159	AIE2_DEV_START,
160};
161
162struct aie2_exec_msg_ops {
163	int (*init_cu_req)(struct amdxdna_gem_obj *cmd_bo, void *req,
164			   size_t *size, u32 *msg_op);
165	int (*init_dpu_req)(struct amdxdna_gem_obj *cmd_bo, void *req,
166			    size_t *size, u32 *msg_op);
167	void (*init_chain_req)(void *req, u64 slot_addr, size_t size, u32 cmd_cnt);
168	int (*fill_cf_slot)(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size);
169	int (*fill_dpu_slot)(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size);
170	int (*fill_preempt_slot)(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size);
171	int (*fill_elf_slot)(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size);
172	u32 (*get_chain_msg_op)(u32 cmd_op);
173};
174
175struct amdxdna_dev_hdl {
176	struct amdxdna_dev		*xdna;
177	const struct amdxdna_dev_priv	*priv;
178	void			__iomem *sram_base;
179	void			__iomem *smu_base;
180	void			__iomem *mbox_base;
181	struct psp_device		*psp_hdl;
182
183	struct xdna_mailbox_chann_res	mgmt_x2i;
184	struct xdna_mailbox_chann_res	mgmt_i2x;
185	u32				mgmt_chan_idx;
186	u32				mgmt_prot_major;
187	u32				mgmt_prot_minor;
188
189	u32				total_col;
190	struct aie_version		version;
191	struct aie_metadata		metadata;
192	unsigned long			feature_mask;
193	struct aie2_exec_msg_ops	*exec_msg_ops;
194
195	/* power management and clock*/
196	enum amdxdna_power_mode_type	pw_mode;
197	u32				dpm_level;
198	u32				dft_dpm_level;
199	u32				max_dpm_level;
200	u32				clk_gating;
201	u32				npuclk_freq;
202	u32				hclk_freq;
203	u32				max_tops;
204	u32				curr_tops;
205	u32				force_preempt_enabled;
206	u32				frame_boundary_preempt;
207
208	/* Mailbox and the management channel */
209	struct mailbox			*mbox;
210	struct mailbox_channel		*mgmt_chann;
211	struct async_events		*async_events;
212
213	enum aie2_dev_status		dev_status;
214	u32				hwctx_num;
215
216	struct amdxdna_async_error	last_async_err;
217};
218
219#define DEFINE_BAR_OFFSET(reg_name, bar, reg_addr) \
220	[reg_name] = {bar##_BAR_INDEX, (reg_addr) - bar##_BAR_BASE}
221
222struct aie2_bar_off_pair {
223	int	bar_idx;
224	u32	offset;
225};
226
227struct aie2_hw_ops {
228	int (*set_dpm)(struct amdxdna_dev_hdl *ndev, u32 dpm_level);
229};
230
231enum aie2_fw_feature {
232	AIE2_NPU_COMMAND,
233	AIE2_PREEMPT,
234	AIE2_TEMPORAL_ONLY,
235	AIE2_FEATURE_MAX
236};
237
238struct aie2_fw_feature_tbl {
239	enum aie2_fw_feature feature;
240	u32 max_minor;
241	u32 min_minor;
242};
243
244#define AIE2_FEATURE_ON(ndev, feature)	test_bit(feature, &(ndev)->feature_mask)
245
246struct amdxdna_dev_priv {
247	const char			*fw_path;
248	u64				protocol_major;
249	u64				protocol_minor;
250	const struct rt_config		*rt_config;
251	const struct dpm_clk_freq	*dpm_clk_tbl;
252	const struct aie2_fw_feature_tbl *fw_feature_tbl;
253
254#define COL_ALIGN_NONE   0
255#define COL_ALIGN_NATURE 1
256	u32				col_align;
257	u32				mbox_dev_addr;
258	/* If mbox_size is 0, use BAR size. See MBOX_SIZE macro */
259	u32				mbox_size;
260	u32				hwctx_limit;
261	u32				sram_dev_addr;
262	struct aie2_bar_off_pair	sram_offs[SRAM_MAX_INDEX];
263	struct aie2_bar_off_pair	psp_regs_off[PSP_MAX_REGS];
264	struct aie2_bar_off_pair	smu_regs_off[SMU_MAX_REGS];
265	struct aie2_hw_ops		hw_ops;
266};
267
268extern const struct amdxdna_dev_ops aie2_ops;
269
270int aie2_runtime_cfg(struct amdxdna_dev_hdl *ndev,
271		     enum rt_config_category category, u32 *val);
272
273/* aie2 npu hw config */
274extern const struct dpm_clk_freq npu1_dpm_clk_table[];
275extern const struct dpm_clk_freq npu4_dpm_clk_table[];
276extern const struct rt_config npu1_default_rt_cfg[];
277extern const struct rt_config npu4_default_rt_cfg[];
278extern const struct aie2_fw_feature_tbl npu4_fw_feature_table[];
279
280/* aie2_smu.c */
281int aie2_smu_init(struct amdxdna_dev_hdl *ndev);
282void aie2_smu_fini(struct amdxdna_dev_hdl *ndev);
283int npu1_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level);
284int npu4_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level);
285
286/* aie2_pm.c */
287int aie2_pm_init(struct amdxdna_dev_hdl *ndev);
288int aie2_pm_set_mode(struct amdxdna_dev_hdl *ndev, enum amdxdna_power_mode_type target);
289int aie2_pm_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level);
290
291/* aie2_psp.c */
292struct psp_device *aie2m_psp_create(struct drm_device *ddev, struct psp_config *conf);
293int aie2_psp_start(struct psp_device *psp);
294void aie2_psp_stop(struct psp_device *psp);
295
296/* aie2_error.c */
297int aie2_error_async_events_alloc(struct amdxdna_dev_hdl *ndev);
298void aie2_error_async_events_free(struct amdxdna_dev_hdl *ndev);
299int aie2_error_async_msg_thread(void *data);
300int aie2_get_array_async_error(struct amdxdna_dev_hdl *ndev,
301			       struct amdxdna_drm_get_array *args);
302
303/* aie2_message.c */
304void aie2_msg_init(struct amdxdna_dev_hdl *ndev);
305void aie2_destroy_mgmt_chann(struct amdxdna_dev_hdl *ndev);
306int aie2_suspend_fw(struct amdxdna_dev_hdl *ndev);
307int aie2_resume_fw(struct amdxdna_dev_hdl *ndev);
308int aie2_set_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 value);
309int aie2_get_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 *value);
310int aie2_assign_mgmt_pasid(struct amdxdna_dev_hdl *ndev, u16 pasid);
311int aie2_query_aie_version(struct amdxdna_dev_hdl *ndev, struct aie_version *version);
312int aie2_query_aie_metadata(struct amdxdna_dev_hdl *ndev, struct aie_metadata *metadata);
313int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev,
314				struct amdxdna_fw_ver *fw_ver);
315int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx);
316int aie2_destroy_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx);
317int aie2_map_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size);
318int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf, u32 size, u32 *cols_filled);
319int aie2_query_telemetry(struct amdxdna_dev_hdl *ndev,
320			 char __user *buf, u32 size,
321			 struct amdxdna_drm_query_telemetry_header *header);
322int aie2_register_asyn_event_msg(struct amdxdna_dev_hdl *ndev, dma_addr_t addr, u32 size,
323				 void *handle, int (*cb)(void*, void __iomem *, size_t));
324int aie2_config_cu(struct amdxdna_hwctx *hwctx,
325		   int (*notify_cb)(void *, void __iomem *, size_t));
326int aie2_execbuf(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job,
327		 int (*notify_cb)(void *, void __iomem *, size_t));
328int aie2_cmdlist_single_execbuf(struct amdxdna_hwctx *hwctx,
329				struct amdxdna_sched_job *job,
330				int (*notify_cb)(void *, void __iomem *, size_t));
331int aie2_cmdlist_multi_execbuf(struct amdxdna_hwctx *hwctx,
332			       struct amdxdna_sched_job *job,
333			       int (*notify_cb)(void *, void __iomem *, size_t));
334int aie2_sync_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job,
335		 int (*notify_cb)(void *, void __iomem *, size_t));
336int aie2_config_debug_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job,
337			 int (*notify_cb)(void *, void __iomem *, size_t));
338
339/* aie2_hwctx.c */
340int aie2_hwctx_init(struct amdxdna_hwctx *hwctx);
341void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx);
342int aie2_hwctx_config(struct amdxdna_hwctx *hwctx, u32 type, u64 value, void *buf, u32 size);
343int aie2_hwctx_sync_debug_bo(struct amdxdna_hwctx *hwctx, u32 debug_bo_hdl);
344void aie2_hwctx_suspend(struct amdxdna_client *client);
345int aie2_hwctx_resume(struct amdxdna_client *client);
346int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, u64 *seq);
347void aie2_hmm_invalidate(struct amdxdna_gem_obj *abo, unsigned long cur_seq);
348
349#endif /* _AIE2_PCI_H_ */