registers.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
  3#ifndef _IDXD_REGISTERS_H_
  4#define _IDXD_REGISTERS_H_
  5
  6#ifdef __KERNEL__
  7#include <uapi/linux/idxd.h>
  8#else
  9#include <linux/idxd.h>
 10#endif
 11
 12/* PCI Config */
 13#define PCI_DEVICE_ID_INTEL_DSA_GNRD	0x11fb
 14#define PCI_DEVICE_ID_INTEL_DSA_DMR	0x1212
 15#define PCI_DEVICE_ID_INTEL_IAA_DMR	0x1216
 16#define PCI_DEVICE_ID_INTEL_IAA_PTL	0xb02d
 17#define PCI_DEVICE_ID_INTEL_IAA_WCL	0xfd2d
 18
 19#define DEVICE_VERSION_1		0x100
 20#define DEVICE_VERSION_2		0x200
 21
 22#define IDXD_MMIO_BAR		0
 23#define IDXD_WQ_BAR		2
 24#define IDXD_PORTAL_SIZE	PAGE_SIZE
 25
 26/* MMIO Device BAR0 Registers */
 27#define IDXD_VER_OFFSET			0x00
 28#define IDXD_VER_MAJOR_MASK		0xf0
 29#define IDXD_VER_MINOR_MASK		0x0f
 30#define GET_IDXD_VER_MAJOR(x)		(((x) & IDXD_VER_MAJOR_MASK) >> 4)
 31#define GET_IDXD_VER_MINOR(x)		((x) & IDXD_VER_MINOR_MASK)
 32
 33union gen_cap_reg {
 34	struct {
 35		u64 block_on_fault:1;
 36		u64 overlap_copy:1;
 37		u64 cache_control_mem:1;
 38		u64 cache_control_cache:1;
 39		u64 cmd_cap:1;
 40		u64 rsvd:3;
 41		u64 dest_readback:1;
 42		u64 drain_readback:1;
 43		u64 rsvd2:3;
 44		u64 evl_support:2;
 45		u64 batch_continuation:1;
 46		u64 max_xfer_shift:5;
 47		u64 max_batch_shift:4;
 48		u64 max_ims_mult:6;
 49		u64 config_en:1;
 50		u64 rsvd3:32;
 51	};
 52	u64 bits;
 53};
 54#define IDXD_GENCAP_OFFSET		0x10
 55
 56union wq_cap_reg {
 57	struct {
 58		u64 total_wq_size:16;
 59		u64 num_wqs:8;
 60		u64 wqcfg_size:4;
 61		u64 rsvd:20;
 62		u64 shared_mode:1;
 63		u64 dedicated_mode:1;
 64		u64 wq_ats_support:1;
 65		u64 priority:1;
 66		u64 occupancy:1;
 67		u64 occupancy_int:1;
 68		u64 op_config:1;
 69		u64 wq_prs_support:1;
 70		u64 rsvd4:8;
 71	};
 72	u64 bits;
 73};
 74#define IDXD_WQCAP_OFFSET		0x20
 75#define IDXD_WQCFG_MIN			5
 76
 77union group_cap_reg {
 78	struct {
 79		u64 num_groups:8;
 80		u64 total_rdbufs:8;	/* formerly total_tokens */
 81		u64 rdbuf_ctrl:1;	/* formerly token_en */
 82		u64 rdbuf_limit:1;	/* formerly token_limit */
 83		u64 progress_limit:1;	/* descriptor and batch descriptor */
 84		u64 rsvd:45;
 85	};
 86	u64 bits;
 87};
 88#define IDXD_GRPCAP_OFFSET		0x30
 89
 90union engine_cap_reg {
 91	struct {
 92		u64 num_engines:8;
 93		u64 rsvd:56;
 94	};
 95	u64 bits;
 96};
 97
 98#define IDXD_ENGCAP_OFFSET		0x38
 99
100#define IDXD_OPCAP_NOOP			0x0001
101#define IDXD_OPCAP_BATCH			0x0002
102#define IDXD_OPCAP_MEMMOVE		0x0008
103struct opcap {
104	u64 bits[4];
105};
106
107#define IDXD_MAX_OPCAP_BITS		256U
108
109#define IDXD_OPCAP_OFFSET		0x40
110
111#define IDXD_TABLE_OFFSET		0x60
112union offsets_reg {
113	struct {
114		u64 grpcfg:16;
115		u64 wqcfg:16;
116		u64 msix_perm:16;
117		u64 ims:16;
118		u64 perfmon:16;
119		u64 rsvd:48;
120	};
121	u64 bits[2];
122};
123
124#define IDXD_TABLE_MULT			0x100
125
126#define IDXD_GENCFG_OFFSET		0x80
127union gencfg_reg {
128	struct {
129		u32 rdbuf_limit:8;
130		u32 rsvd:4;
131		u32 user_int_en:1;
132		u32 evl_en:1;
133		u32 rsvd2:18;
134	};
135	u32 bits;
136};
137
138#define IDXD_GENCTRL_OFFSET		0x88
139union genctrl_reg {
140	struct {
141		u32 softerr_int_en:1;
142		u32 halt_int_en:1;
143		u32 evl_int_en:1;
144		u32 rsvd:29;
145	};
146	u32 bits;
147};
148
149#define IDXD_GENSTATS_OFFSET		0x90
150union gensts_reg {
151	struct {
152		u32 state:2;
153		u32 reset_type:2;
154		u32 rsvd:28;
155	};
156	u32 bits;
157};
158
159enum idxd_device_status_state {
160	IDXD_DEVICE_STATE_DISABLED = 0,
161	IDXD_DEVICE_STATE_ENABLED,
162	IDXD_DEVICE_STATE_DRAIN,
163	IDXD_DEVICE_STATE_HALT,
164};
165
166enum idxd_device_reset_type {
167	IDXD_DEVICE_RESET_SOFTWARE = 0,
168	IDXD_DEVICE_RESET_FLR,
169	IDXD_DEVICE_RESET_WARM,
170	IDXD_DEVICE_RESET_COLD,
171};
172
173#define IDXD_INTCAUSE_OFFSET		0x98
174#define IDXD_INTC_ERR			0x01
175#define IDXD_INTC_CMD			0x02
176#define IDXD_INTC_OCCUPY			0x04
177#define IDXD_INTC_PERFMON_OVFL		0x08
178#define IDXD_INTC_HALT_STATE		0x10
179#define IDXD_INTC_EVL			0x20
180#define IDXD_INTC_INT_HANDLE_REVOKED	0x80000000
181
182#define IDXD_CMD_OFFSET			0xa0
183union idxd_command_reg {
184	struct {
185		u32 operand:20;
186		u32 cmd:5;
187		u32 rsvd:6;
188		u32 int_req:1;
189	};
190	u32 bits;
191};
192
193enum idxd_cmd {
194	IDXD_CMD_ENABLE_DEVICE = 1,
195	IDXD_CMD_DISABLE_DEVICE,
196	IDXD_CMD_DRAIN_ALL,
197	IDXD_CMD_ABORT_ALL,
198	IDXD_CMD_RESET_DEVICE,
199	IDXD_CMD_ENABLE_WQ,
200	IDXD_CMD_DISABLE_WQ,
201	IDXD_CMD_DRAIN_WQ,
202	IDXD_CMD_ABORT_WQ,
203	IDXD_CMD_RESET_WQ,
204	IDXD_CMD_DRAIN_PASID,
205	IDXD_CMD_ABORT_PASID,
206	IDXD_CMD_REQUEST_INT_HANDLE,
207	IDXD_CMD_RELEASE_INT_HANDLE,
208};
209
210#define CMD_INT_HANDLE_IMS		0x10000
211
212#define IDXD_CMDSTS_OFFSET		0xa8
213union cmdsts_reg {
214	struct {
215		u8 err;
216		u16 result;
217		u8 rsvd:7;
218		u8 active:1;
219	};
220	u32 bits;
221};
222#define IDXD_CMDSTS_ACTIVE		0x80000000
223#define IDXD_CMDSTS_ERR_MASK		0xff
224#define IDXD_CMDSTS_RES_SHIFT		8
225
226enum idxd_cmdsts_err {
227	IDXD_CMDSTS_SUCCESS = 0,
228	IDXD_CMDSTS_INVAL_CMD,
229	IDXD_CMDSTS_INVAL_WQIDX,
230	IDXD_CMDSTS_HW_ERR,
231	/* enable device errors */
232	IDXD_CMDSTS_ERR_DEV_ENABLED = 0x10,
233	IDXD_CMDSTS_ERR_CONFIG,
234	IDXD_CMDSTS_ERR_BUSMASTER_EN,
235	IDXD_CMDSTS_ERR_PASID_INVAL,
236	IDXD_CMDSTS_ERR_WQ_SIZE_ERANGE,
237	IDXD_CMDSTS_ERR_GRP_CONFIG,
238	IDXD_CMDSTS_ERR_GRP_CONFIG2,
239	IDXD_CMDSTS_ERR_GRP_CONFIG3,
240	IDXD_CMDSTS_ERR_GRP_CONFIG4,
241	/* enable wq errors */
242	IDXD_CMDSTS_ERR_DEV_NOTEN = 0x20,
243	IDXD_CMDSTS_ERR_WQ_ENABLED,
244	IDXD_CMDSTS_ERR_WQ_SIZE,
245	IDXD_CMDSTS_ERR_WQ_PRIOR,
246	IDXD_CMDSTS_ERR_WQ_MODE,
247	IDXD_CMDSTS_ERR_BOF_EN,
248	IDXD_CMDSTS_ERR_PASID_EN,
249	IDXD_CMDSTS_ERR_MAX_BATCH_SIZE,
250	IDXD_CMDSTS_ERR_MAX_XFER_SIZE,
251	/* disable device errors */
252	IDXD_CMDSTS_ERR_DIS_DEV_EN = 0x31,
253	/* disable WQ, drain WQ, abort WQ, reset WQ */
254	IDXD_CMDSTS_ERR_DEV_NOT_EN,
255	/* request interrupt handle */
256	IDXD_CMDSTS_ERR_INVAL_INT_IDX = 0x41,
257	IDXD_CMDSTS_ERR_NO_HANDLE,
258};
259
260#define IDXD_CMDCAP_OFFSET		0xb0
261
262#define IDXD_SWERR_OFFSET		0xc0
263#define IDXD_SWERR_VALID		0x00000001
264#define IDXD_SWERR_OVERFLOW		0x00000002
265#define IDXD_SWERR_ACK			(IDXD_SWERR_VALID | IDXD_SWERR_OVERFLOW)
266union sw_err_reg {
267	struct {
268		u64 valid:1;
269		u64 overflow:1;
270		u64 desc_valid:1;
271		u64 wq_idx_valid:1;
272		u64 batch:1;
273		u64 fault_rw:1;
274		u64 priv:1;
275		u64 rsvd:1;
276		u64 error:8;
277		u64 wq_idx:8;
278		u64 rsvd2:8;
279		u64 operation:8;
280		u64 pasid:20;
281		u64 rsvd3:4;
282
283		u64 batch_idx:16;
284		u64 rsvd4:16;
285		u64 invalid_flags:32;
286
287		u64 fault_addr;
288
289		u64 rsvd5;
290	};
291	u64 bits[4];
292};
293
294union iaa_cap_reg {
295	struct {
296		u64 dec_aecs_format_ver:1;
297		u64 drop_init_bits:1;
298		u64 chaining:1;
299		u64 force_array_output_mod:1;
300		u64 load_part_aecs:1;
301		u64 comp_early_abort:1;
302		u64 nested_comp:1;
303		u64 diction_comp:1;
304		u64 header_gen:1;
305		u64 crypto_gcm:1;
306		u64 crypto_cfb:1;
307		u64 crypto_xts:1;
308		u64 rsvd:52;
309	};
310	u64 bits;
311};
312
313#define IDXD_IAACAP_OFFSET	0x180
314
315#define IDXD_EVLCFG_OFFSET	0xe0
316union evlcfg_reg {
317	struct {
318		u64 pasid_en:1;
319		u64 priv:1;
320		u64 rsvd:10;
321		u64 base_addr:52;
322
323		u64 size:16;
324		u64 pasid:20;
325		u64 rsvd2:28;
326	};
327	u64 bits[2];
328};
329
330#define IDXD_EVL_SIZE_MIN	0x0040
331#define IDXD_EVL_SIZE_MAX	0xffff
332
333union msix_perm {
334	struct {
335		u32 rsvd:2;
336		u32 ignore:1;
337		u32 pasid_en:1;
338		u32 rsvd2:8;
339		u32 pasid:20;
340	};
341	u32 bits;
342};
343
344union group_flags {
345	struct {
346		u64 tc_a:3;
347		u64 tc_b:3;
348		u64 rsvd:1;
349		u64 use_rdbuf_limit:1;
350		u64 rdbufs_reserved:8;
351		u64 rsvd2:4;
352		u64 rdbufs_allowed:8;
353		u64 rsvd3:4;
354		u64 desc_progress_limit:2;
355		u64 rsvd4:2;
356		u64 batch_progress_limit:2;
357		u64 rsvd5:26;
358	};
359	u64 bits;
360};
361
362struct grpcfg {
363	u64 wqs[4];
364	u64 engines;
365	union group_flags flags;
366};
367
368union wqcfg {
369	struct {
370		/* bytes 0-3 */
371		u16 wq_size;
372		u16 rsvd;
373
374		/* bytes 4-7 */
375		u16 wq_thresh;
376		u16 rsvd1;
377
378		/* bytes 8-11 */
379		u32 mode:1;	/* shared or dedicated */
380		u32 bof:1;	/* block on fault */
381		u32 wq_ats_disable:1;
382		u32 wq_prs_disable:1;
383		u32 priority:4;
384		u32 pasid:20;
385		u32 pasid_en:1;
386		u32 priv:1;
387		u32 rsvd3:2;
388
389		/* bytes 12-15 */
390		u32 max_xfer_shift:5;
391		u32 max_batch_shift:4;
392		u32 rsvd4:23;
393
394		/* bytes 16-19 */
395		u16 occupancy_inth;
396		u16 occupancy_table_sel:1;
397		u16 rsvd5:15;
398
399		/* bytes 20-23 */
400		u16 occupancy_limit;
401		u16 occupancy_int_en:1;
402		u16 rsvd6:15;
403
404		/* bytes 24-27 */
405		u16 occupancy;
406		u16 occupancy_int:1;
407		u16 rsvd7:12;
408		u16 mode_support:1;
409		u16 wq_state:2;
410
411		/* bytes 28-31 */
412		u32 rsvd8;
413
414		/* bytes 32-63 */
415		u64 op_config[4];
416	};
417	u32 bits[16];
418};
419
420#define WQCFG_PASID_IDX                2
421#define WQCFG_PRIVL_IDX		2
422#define WQCFG_OCCUP_IDX		6
423
424#define WQCFG_OCCUP_MASK	0xffff
425
426/*
427 * This macro calculates the offset into the WQCFG register
428 * idxd - struct idxd *
429 * n - wq id
430 * ofs - the index of the 32b dword for the config register
431 *
432 * The WQCFG register block is divided into groups per each wq. The n index
433 * allows us to move to the register group that's for that particular wq.
434 * Each register is 32bits. The ofs gives us the number of register to access.
435 */
436#define WQCFG_OFFSET(_idxd_dev, n, ofs) \
437({\
438	typeof(_idxd_dev) __idxd_dev = (_idxd_dev);	\
439	(__idxd_dev)->wqcfg_offset + (n) * (__idxd_dev)->wqcfg_size + sizeof(u32) * (ofs);	\
440})
441
442#define WQCFG_STRIDES(_idxd_dev) ((_idxd_dev)->wqcfg_size / sizeof(u32))
443
444#define GRPCFG_SIZE		64
445#define GRPWQCFG_STRIDES	4
446
447/*
448 * This macro calculates the offset into the GRPCFG register
449 * idxd - struct idxd *
450 * n - group id
451 * ofs - the index of the 64b qword for the config register
452 *
453 * The GRPCFG register block is divided into three sub-registers, which
454 * are GRPWQCFG, GRPENGCFG and GRPFLGCFG. The n index allows us to move
455 * to the register block that contains the three sub-registers.
456 * Each register block is 64bits. And the ofs gives us the offset
457 * within the GRPWQCFG register to access.
458 */
459#define GRPWQCFG_OFFSET(idxd_dev, n, ofs) ((idxd_dev)->grpcfg_offset +\
460					   (n) * GRPCFG_SIZE + sizeof(u64) * (ofs))
461#define GRPENGCFG_OFFSET(idxd_dev, n) ((idxd_dev)->grpcfg_offset + (n) * GRPCFG_SIZE + 32)
462#define GRPFLGCFG_OFFSET(idxd_dev, n) ((idxd_dev)->grpcfg_offset + (n) * GRPCFG_SIZE + 40)
463
464/* Following is performance monitor registers */
465#define IDXD_PERFCAP_OFFSET		0x0
466union idxd_perfcap {
467	struct {
468		u64 num_perf_counter:6;
469		u64 rsvd1:2;
470		u64 counter_width:8;
471		u64 num_event_category:4;
472		u64 global_event_category:16;
473		u64 filter:8;
474		u64 rsvd2:8;
475		u64 cap_per_counter:1;
476		u64 writeable_counter:1;
477		u64 counter_freeze:1;
478		u64 overflow_interrupt:1;
479		u64 rsvd3:8;
480	};
481	u64 bits;
482};
483
484#define IDXD_EVNTCAP_OFFSET		0x80
485union idxd_evntcap {
486	struct {
487		u64 events:28;
488		u64 rsvd:36;
489	};
490	u64 bits;
491};
492
493struct idxd_event {
494	union {
495		struct {
496			u32 event_category:4;
497			u32 events:28;
498		};
499		u32 val;
500	};
501};
502
503#define IDXD_CNTRCAP_OFFSET		0x800
504struct idxd_cntrcap {
505	union {
506		struct {
507			u32 counter_width:8;
508			u32 rsvd:20;
509			u32 num_events:4;
510		};
511		u32 val;
512	};
513	struct idxd_event events[];
514};
515
516#define IDXD_PERFRST_OFFSET		0x10
517union idxd_perfrst {
518	struct {
519		u32 perfrst_config:1;
520		u32 perfrst_counter:1;
521		u32 rsvd:30;
522	};
523	u32 val;
524};
525
526#define IDXD_OVFSTATUS_OFFSET		0x30
527#define IDXD_PERFFRZ_OFFSET		0x20
528#define IDXD_CNTRCFG_OFFSET		0x100
529union idxd_cntrcfg {
530	struct {
531		u64 enable:1;
532		u64 interrupt_ovf:1;
533		u64 global_freeze_ovf:1;
534		u64 rsvd1:5;
535		u64 event_category:4;
536		u64 rsvd2:20;
537		u64 events:28;
538		u64 rsvd3:4;
539	};
540	u64 val;
541};
542
543#define IDXD_FLTCFG_OFFSET		0x300
544
545#define IDXD_CNTRDATA_OFFSET		0x200
546union idxd_cntrdata {
547	struct {
548		u64 event_count_value;
549	};
550	u64 val;
551};
552
553union event_cfg {
554	struct {
555		u64 event_cat:4;
556		u64 event_enc:28;
557	};
558	u64 val;
559};
560
561union filter_cfg {
562	struct {
563		u64 wq:32;
564		u64 tc:8;
565		u64 pg_sz:4;
566		u64 xfer_sz:8;
567		u64 eng:8;
568	};
569	u64 val;
570};
571
572#define IDXD_EVLSTATUS_OFFSET		0xf0
573
574union evl_status_reg {
575	struct {
576		u32 head:16;
577		u32 rsvd:16;
578		u32 tail:16;
579		u32 rsvd2:14;
580		u32 int_pending:1;
581		u32 rsvd3:1;
582	};
583	struct {
584		u32 bits_lower32;
585		u32 bits_upper32;
586	};
587	u64 bits;
588};
589
590#define IDXD_MAX_BATCH_IDENT	256
591
592struct __evl_entry {
593	u64 rsvd:2;
594	u64 desc_valid:1;
595	u64 wq_idx_valid:1;
596	u64 batch:1;
597	u64 fault_rw:1;
598	u64 priv:1;
599	u64 err_info_valid:1;
600	u64 error:8;
601	u64 wq_idx:8;
602	u64 batch_id:8;
603	u64 operation:8;
604	u64 pasid:20;
605	u64 rsvd2:4;
606
607	u16 batch_idx;
608	u16 rsvd3;
609	union {
610		/* Invalid Flags 0x11 */
611		u32 invalid_flags;
612		/* Invalid Int Handle 0x19 */
613		/* Page fault 0x1a */
614		/* Page fault 0x06, 0x1f, only operand_id */
615		/* Page fault before drain or in batch, 0x26, 0x27 */
616		struct {
617			u16 int_handle;
618			u16 rci:1;
619			u16 ims:1;
620			u16 rcr:1;
621			u16 first_err_in_batch:1;
622			u16 rsvd4_2:9;
623			u16 operand_id:3;
624		};
625	};
626	u64 fault_addr;
627	u64 rsvd5;
628};
629
630struct dsa_evl_entry {
631	struct __evl_entry e;
632	struct dsa_completion_record cr;
633};
634
635struct iax_evl_entry {
636	struct __evl_entry e;
637	u64 rsvd[4];
638	struct iax_completion_record cr;
639};
640
641#endif