gic_v3_its.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Guest ITS library, generously donated by drivers/irqchip/irq-gic-v3-its.c
  4 * over in the kernel tree.
  5 */
  6
  7#include <linux/kvm.h>
  8#include <linux/sizes.h>
  9#include <asm/kvm_para.h>
 10#include <asm/kvm.h>
 11
 12#include "kvm_util.h"
 13#include "vgic.h"
 14#include "gic.h"
 15#include "gic_v3.h"
 16#include "processor.h"
 17
 18#define GITS_COLLECTION_TARGET_SHIFT 16
 19
 20static u64 its_read_u64(unsigned long offset)
 21{
 22	return readq_relaxed(GITS_BASE_GVA + offset);
 23}
 24
 25static void its_write_u64(unsigned long offset, u64 val)
 26{
 27	writeq_relaxed(val, GITS_BASE_GVA + offset);
 28}
 29
 30static u32 its_read_u32(unsigned long offset)
 31{
 32	return readl_relaxed(GITS_BASE_GVA + offset);
 33}
 34
 35static void its_write_u32(unsigned long offset, u32 val)
 36{
 37	writel_relaxed(val, GITS_BASE_GVA + offset);
 38}
 39
 40static unsigned long its_find_baser(unsigned int type)
 41{
 42	int i;
 43
 44	for (i = 0; i < GITS_BASER_NR_REGS; i++) {
 45		u64 baser;
 46		unsigned long offset = GITS_BASER + (i * sizeof(baser));
 47
 48		baser = its_read_u64(offset);
 49		if (GITS_BASER_TYPE(baser) == type)
 50			return offset;
 51	}
 52
 53	GUEST_FAIL("Couldn't find an ITS BASER of type %u", type);
 54	return -1;
 55}
 56
 57static void its_install_table(unsigned int type, vm_paddr_t base, size_t size)
 58{
 59	unsigned long offset = its_find_baser(type);
 60	u64 baser;
 61
 62	baser = ((size / SZ_64K) - 1) |
 63		GITS_BASER_PAGE_SIZE_64K |
 64		GITS_BASER_InnerShareable |
 65		base |
 66		GITS_BASER_RaWaWb |
 67		GITS_BASER_VALID;
 68
 69	its_write_u64(offset, baser);
 70}
 71
 72static void its_install_cmdq(vm_paddr_t base, size_t size)
 73{
 74	u64 cbaser;
 75
 76	cbaser = ((size / SZ_4K) - 1) |
 77		 GITS_CBASER_InnerShareable |
 78		 base |
 79		 GITS_CBASER_RaWaWb |
 80		 GITS_CBASER_VALID;
 81
 82	its_write_u64(GITS_CBASER, cbaser);
 83}
 84
 85void its_init(vm_paddr_t coll_tbl, size_t coll_tbl_sz,
 86	      vm_paddr_t device_tbl, size_t device_tbl_sz,
 87	      vm_paddr_t cmdq, size_t cmdq_size)
 88{
 89	u32 ctlr;
 90
 91	its_install_table(GITS_BASER_TYPE_COLLECTION, coll_tbl, coll_tbl_sz);
 92	its_install_table(GITS_BASER_TYPE_DEVICE, device_tbl, device_tbl_sz);
 93	its_install_cmdq(cmdq, cmdq_size);
 94
 95	ctlr = its_read_u32(GITS_CTLR);
 96	ctlr |= GITS_CTLR_ENABLE;
 97	its_write_u32(GITS_CTLR, ctlr);
 98}
 99
100struct its_cmd_block {
101	union {
102		u64	raw_cmd[4];
103		__le64	raw_cmd_le[4];
104	};
105};
106
107static inline void its_fixup_cmd(struct its_cmd_block *cmd)
108{
109	/* Let's fixup BE commands */
110	cmd->raw_cmd_le[0] = cpu_to_le64(cmd->raw_cmd[0]);
111	cmd->raw_cmd_le[1] = cpu_to_le64(cmd->raw_cmd[1]);
112	cmd->raw_cmd_le[2] = cpu_to_le64(cmd->raw_cmd[2]);
113	cmd->raw_cmd_le[3] = cpu_to_le64(cmd->raw_cmd[3]);
114}
115
116static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l)
117{
118	u64 mask = GENMASK_ULL(h, l);
119	*raw_cmd &= ~mask;
120	*raw_cmd |= (val << l) & mask;
121}
122
123static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr)
124{
125	its_mask_encode(&cmd->raw_cmd[0], cmd_nr, 7, 0);
126}
127
128static void its_encode_devid(struct its_cmd_block *cmd, u32 devid)
129{
130	its_mask_encode(&cmd->raw_cmd[0], devid, 63, 32);
131}
132
133static void its_encode_event_id(struct its_cmd_block *cmd, u32 id)
134{
135	its_mask_encode(&cmd->raw_cmd[1], id, 31, 0);
136}
137
138static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id)
139{
140	its_mask_encode(&cmd->raw_cmd[1], phys_id, 63, 32);
141}
142
143static void its_encode_size(struct its_cmd_block *cmd, u8 size)
144{
145	its_mask_encode(&cmd->raw_cmd[1], size, 4, 0);
146}
147
148static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr)
149{
150	its_mask_encode(&cmd->raw_cmd[2], itt_addr >> 8, 51, 8);
151}
152
153static void its_encode_valid(struct its_cmd_block *cmd, int valid)
154{
155	its_mask_encode(&cmd->raw_cmd[2], !!valid, 63, 63);
156}
157
158static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr)
159{
160	its_mask_encode(&cmd->raw_cmd[2], target_addr >> 16, 51, 16);
161}
162
163static void its_encode_collection(struct its_cmd_block *cmd, u16 col)
164{
165	its_mask_encode(&cmd->raw_cmd[2], col, 15, 0);
166}
167
168static u64 procnum_to_rdbase(u32 vcpu_id)
169{
170	return vcpu_id << GITS_COLLECTION_TARGET_SHIFT;
171}
172
173#define GITS_CMDQ_POLL_ITERATIONS	0
174
175static void its_send_cmd(void *cmdq_base, struct its_cmd_block *cmd)
176{
177	u64 cwriter = its_read_u64(GITS_CWRITER);
178	struct its_cmd_block *dst = cmdq_base + cwriter;
179	u64 cbaser = its_read_u64(GITS_CBASER);
180	size_t cmdq_size;
181	u64 next;
182	int i;
183
184	cmdq_size = ((cbaser & 0xFF) + 1) * SZ_4K;
185
186	its_fixup_cmd(cmd);
187
188	WRITE_ONCE(*dst, *cmd);
189	dsb(ishst);
190	next = (cwriter + sizeof(*cmd)) % cmdq_size;
191	its_write_u64(GITS_CWRITER, next);
192
193	/*
194	 * Polling isn't necessary considering KVM's ITS emulation at the time
195	 * of writing this, as the CMDQ is processed synchronously after a write
196	 * to CWRITER.
197	 */
198	for (i = 0; its_read_u64(GITS_CREADR) != next; i++) {
199		__GUEST_ASSERT(i < GITS_CMDQ_POLL_ITERATIONS,
200			       "ITS didn't process command at offset %lu after %d iterations\n",
201			       cwriter, i);
202
203		cpu_relax();
204	}
205}
206
207void its_send_mapd_cmd(void *cmdq_base, u32 device_id, vm_paddr_t itt_base,
208		       size_t itt_size, bool valid)
209{
210	struct its_cmd_block cmd = {};
211
212	its_encode_cmd(&cmd, GITS_CMD_MAPD);
213	its_encode_devid(&cmd, device_id);
214	its_encode_size(&cmd, ilog2(itt_size) - 1);
215	its_encode_itt(&cmd, itt_base);
216	its_encode_valid(&cmd, valid);
217
218	its_send_cmd(cmdq_base, &cmd);
219}
220
221void its_send_mapc_cmd(void *cmdq_base, u32 vcpu_id, u32 collection_id, bool valid)
222{
223	struct its_cmd_block cmd = {};
224
225	its_encode_cmd(&cmd, GITS_CMD_MAPC);
226	its_encode_collection(&cmd, collection_id);
227	its_encode_target(&cmd, procnum_to_rdbase(vcpu_id));
228	its_encode_valid(&cmd, valid);
229
230	its_send_cmd(cmdq_base, &cmd);
231}
232
233void its_send_mapti_cmd(void *cmdq_base, u32 device_id, u32 event_id,
234			u32 collection_id, u32 intid)
235{
236	struct its_cmd_block cmd = {};
237
238	its_encode_cmd(&cmd, GITS_CMD_MAPTI);
239	its_encode_devid(&cmd, device_id);
240	its_encode_event_id(&cmd, event_id);
241	its_encode_phys_id(&cmd, intid);
242	its_encode_collection(&cmd, collection_id);
243
244	its_send_cmd(cmdq_base, &cmd);
245}
246
247void its_send_invall_cmd(void *cmdq_base, u32 collection_id)
248{
249	struct its_cmd_block cmd = {};
250
251	its_encode_cmd(&cmd, GITS_CMD_INVALL);
252	its_encode_collection(&cmd, collection_id);
253
254	its_send_cmd(cmdq_base, &cmd);
255}
256
257void its_send_sync_cmd(void *cmdq_base, u32 vcpu_id)
258{
259	struct its_cmd_block cmd = {};
260
261	its_encode_cmd(&cmd, GITS_CMD_SYNC);
262	its_encode_target(&cmd, procnum_to_rdbase(vcpu_id));
263
264	its_send_cmd(cmdq_base, &cmd);
265}