xe_sriov_packet.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2025 Intel Corporation
  4 */
  5
  6#include "xe_bo.h"
  7#include "xe_device.h"
  8#include "xe_guc_klv_helpers.h"
  9#include "xe_printk.h"
 10#include "xe_sriov_packet.h"
 11#include "xe_sriov_packet_types.h"
 12#include "xe_sriov_pf_helpers.h"
 13#include "xe_sriov_pf_migration.h"
 14#include "xe_sriov_printk.h"
 15
 16static struct mutex *pf_migration_mutex(struct xe_device *xe, unsigned int vfid)
 17{
 18	xe_assert(xe, IS_SRIOV_PF(xe));
 19	xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe));
 20
 21	return &xe->sriov.pf.vfs[vfid].migration.lock;
 22}
 23
 24static struct xe_sriov_packet **pf_pick_pending(struct xe_device *xe, unsigned int vfid)
 25{
 26	xe_assert(xe, IS_SRIOV_PF(xe));
 27	xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe));
 28	lockdep_assert_held(pf_migration_mutex(xe, vfid));
 29
 30	return &xe->sriov.pf.vfs[vfid].migration.pending;
 31}
 32
 33static struct xe_sriov_packet **
 34pf_pick_descriptor(struct xe_device *xe, unsigned int vfid)
 35{
 36	xe_assert(xe, IS_SRIOV_PF(xe));
 37	xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe));
 38	lockdep_assert_held(pf_migration_mutex(xe, vfid));
 39
 40	return &xe->sriov.pf.vfs[vfid].migration.descriptor;
 41}
 42
 43static struct xe_sriov_packet **pf_pick_trailer(struct xe_device *xe, unsigned int vfid)
 44{
 45	xe_assert(xe, IS_SRIOV_PF(xe));
 46	xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe));
 47	lockdep_assert_held(pf_migration_mutex(xe, vfid));
 48
 49	return &xe->sriov.pf.vfs[vfid].migration.trailer;
 50}
 51
 52static struct xe_sriov_packet **pf_pick_read_packet(struct xe_device *xe,
 53						    unsigned int vfid)
 54{
 55	struct xe_sriov_packet **data;
 56
 57	data = pf_pick_descriptor(xe, vfid);
 58	if (*data)
 59		return data;
 60
 61	data = pf_pick_pending(xe, vfid);
 62	if (!*data)
 63		*data = xe_sriov_pf_migration_save_consume(xe, vfid);
 64	if (*data)
 65		return data;
 66
 67	data = pf_pick_trailer(xe, vfid);
 68	if (*data)
 69		return data;
 70
 71	return NULL;
 72}
 73
 74static bool pkt_needs_bo(struct xe_sriov_packet *data)
 75{
 76	return data->hdr.type == XE_SRIOV_PACKET_TYPE_VRAM;
 77}
 78
 79/**
 80 * xe_sriov_packet_alloc() - Allocate migration data packet
 81 * @xe: the &xe_device
 82 *
 83 * Only allocates the "outer" structure, without initializing the migration
 84 * data backing storage.
 85 *
 86 * Return: Pointer to &xe_sriov_packet on success,
 87 *         NULL in case of error.
 88 */
 89struct xe_sriov_packet *xe_sriov_packet_alloc(struct xe_device *xe)
 90{
 91	struct xe_sriov_packet *data;
 92
 93	data = kzalloc(sizeof(*data), GFP_KERNEL);
 94	if (!data)
 95		return NULL;
 96
 97	data->xe = xe;
 98	data->hdr_remaining = sizeof(data->hdr);
 99
100	return data;
101}
102
103/**
104 * xe_sriov_packet_free() - Free migration data packet.
105 * @data: the &xe_sriov_packet
106 */
107void xe_sriov_packet_free(struct xe_sriov_packet *data)
108{
109	if (IS_ERR_OR_NULL(data))
110		return;
111
112	if (pkt_needs_bo(data))
113		xe_bo_unpin_map_no_vm(data->bo);
114	else
115		kvfree(data->buff);
116
117	kfree(data);
118}
119
120static int pkt_init(struct xe_sriov_packet *data)
121{
122	struct xe_gt *gt = xe_device_get_gt(data->xe, data->hdr.gt_id);
123
124	if (!gt)
125		return -EINVAL;
126
127	if (data->hdr.size == 0)
128		return 0;
129
130	if (pkt_needs_bo(data)) {
131		struct xe_bo *bo;
132
133		bo = xe_bo_create_pin_map_novm(data->xe, gt->tile, PAGE_ALIGN(data->hdr.size),
134					       ttm_bo_type_kernel,
135					       XE_BO_FLAG_SYSTEM | XE_BO_FLAG_PINNED, false);
136		if (IS_ERR(bo))
137			return PTR_ERR(bo);
138
139		data->bo = bo;
140		data->vaddr = bo->vmap.vaddr;
141	} else {
142		void *buff = kvzalloc(data->hdr.size, GFP_KERNEL);
143
144		if (!buff)
145			return -ENOMEM;
146
147		data->buff = buff;
148		data->vaddr = buff;
149	}
150
151	return 0;
152}
153
154#define XE_SRIOV_PACKET_SUPPORTED_VERSION 1
155
156/**
157 * xe_sriov_packet_init() - Initialize migration packet header and backing storage.
158 * @data: the &xe_sriov_packet
159 * @tile_id: tile identifier
160 * @gt_id: GT identifier
161 * @type: &xe_sriov_packet_type
162 * @offset: offset of data packet payload (within wider resource)
163 * @size: size of data packet payload
164 *
165 * Return: 0 on success or a negative error code on failure.
166 */
167int xe_sriov_packet_init(struct xe_sriov_packet *data, u8 tile_id, u8 gt_id,
168			 enum xe_sriov_packet_type type, loff_t offset, size_t size)
169{
170	data->hdr.version = XE_SRIOV_PACKET_SUPPORTED_VERSION;
171	data->hdr.type = type;
172	data->hdr.tile_id = tile_id;
173	data->hdr.gt_id = gt_id;
174	data->hdr.offset = offset;
175	data->hdr.size = size;
176	data->remaining = size;
177
178	return pkt_init(data);
179}
180
181/**
182 * xe_sriov_packet_init_from_hdr() - Initialize migration packet backing storage based on header.
183 * @data: the &xe_sriov_packet
184 *
185 * Header data is expected to be filled prior to calling this function.
186 *
187 * Return: 0 on success or a negative error code on failure.
188 */
189int xe_sriov_packet_init_from_hdr(struct xe_sriov_packet *data)
190{
191	xe_assert(data->xe, !data->hdr_remaining);
192
193	if (data->hdr.version != XE_SRIOV_PACKET_SUPPORTED_VERSION)
194		return -EINVAL;
195
196	data->remaining = data->hdr.size;
197
198	return pkt_init(data);
199}
200
201static ssize_t pkt_hdr_read(struct xe_sriov_packet *data,
202			    char __user *buf, size_t len)
203{
204	loff_t offset = sizeof(data->hdr) - data->hdr_remaining;
205
206	if (!data->hdr_remaining)
207		return -EINVAL;
208
209	if (len > data->hdr_remaining)
210		len = data->hdr_remaining;
211
212	if (copy_to_user(buf, (void *)&data->hdr + offset, len))
213		return -EFAULT;
214
215	data->hdr_remaining -= len;
216
217	return len;
218}
219
220static ssize_t pkt_data_read(struct xe_sriov_packet *data,
221			     char __user *buf, size_t len)
222{
223	if (len > data->remaining)
224		len = data->remaining;
225
226	if (copy_to_user(buf, data->vaddr + (data->hdr.size - data->remaining), len))
227		return -EFAULT;
228
229	data->remaining -= len;
230
231	return len;
232}
233
234static ssize_t pkt_read_single(struct xe_sriov_packet **data,
235			       unsigned int vfid, char __user *buf, size_t len)
236{
237	ssize_t copied = 0;
238
239	if ((*data)->hdr_remaining)
240		copied = pkt_hdr_read(*data, buf, len);
241	else
242		copied = pkt_data_read(*data, buf, len);
243
244	if ((*data)->remaining == 0 && (*data)->hdr_remaining == 0) {
245		xe_sriov_packet_free(*data);
246		*data = NULL;
247	}
248
249	return copied;
250}
251
252/**
253 * xe_sriov_packet_read_single() - Read migration data from a single packet.
254 * @xe: the &xe_device
255 * @vfid: the VF identifier
256 * @buf: start address of userspace buffer
257 * @len: requested read size from userspace
258 *
259 * Return: number of bytes that has been successfully read,
260 *	   0 if no more migration data is available,
261 *	   -errno on failure.
262 */
263ssize_t xe_sriov_packet_read_single(struct xe_device *xe, unsigned int vfid,
264				    char __user *buf, size_t len)
265{
266	struct xe_sriov_packet **data = pf_pick_read_packet(xe, vfid);
267
268	if (!data)
269		return -ENODATA;
270	if (IS_ERR(*data))
271		return PTR_ERR(*data);
272
273	return pkt_read_single(data, vfid, buf, len);
274}
275
276static ssize_t pkt_hdr_write(struct xe_sriov_packet *data,
277			     const char __user *buf, size_t len)
278{
279	loff_t offset = sizeof(data->hdr) - data->hdr_remaining;
280	int ret;
281
282	if (len > data->hdr_remaining)
283		len = data->hdr_remaining;
284
285	if (copy_from_user((void *)&data->hdr + offset, buf, len))
286		return -EFAULT;
287
288	data->hdr_remaining -= len;
289
290	if (!data->hdr_remaining) {
291		ret = xe_sriov_packet_init_from_hdr(data);
292		if (ret)
293			return ret;
294	}
295
296	return len;
297}
298
299static ssize_t pkt_data_write(struct xe_sriov_packet *data,
300			      const char __user *buf, size_t len)
301{
302	if (len > data->remaining)
303		len = data->remaining;
304
305	if (copy_from_user(data->vaddr + (data->hdr.size - data->remaining), buf, len))
306		return -EFAULT;
307
308	data->remaining -= len;
309
310	return len;
311}
312
313/**
314 * xe_sriov_packet_write_single() - Write migration data to a single packet.
315 * @xe: the &xe_device
316 * @vfid: the VF identifier
317 * @buf: start address of userspace buffer
318 * @len: requested write size from userspace
319 *
320 * Return: number of bytes that has been successfully written,
321 *	   -errno on failure.
322 */
323ssize_t xe_sriov_packet_write_single(struct xe_device *xe, unsigned int vfid,
324				     const char __user *buf, size_t len)
325{
326	struct xe_sriov_packet **data = pf_pick_pending(xe, vfid);
327	int ret;
328	ssize_t copied;
329
330	if (IS_ERR_OR_NULL(*data)) {
331		*data = xe_sriov_packet_alloc(xe);
332		if (!*data)
333			return -ENOMEM;
334	}
335
336	if ((*data)->hdr_remaining)
337		copied = pkt_hdr_write(*data, buf, len);
338	else
339		copied = pkt_data_write(*data, buf, len);
340
341	if ((*data)->hdr_remaining == 0 && (*data)->remaining == 0) {
342		ret = xe_sriov_pf_migration_restore_produce(xe, vfid, *data);
343		if (ret) {
344			xe_sriov_packet_free(*data);
345			*data = NULL;
346
347			return ret;
348		}
349
350		*data = NULL;
351	}
352
353	return copied;
354}
355
356#define MIGRATION_KLV_DEVICE_DEVID_KEY	0xf001u
357#define MIGRATION_KLV_DEVICE_DEVID_LEN	1u
358#define MIGRATION_KLV_DEVICE_REVID_KEY	0xf002u
359#define MIGRATION_KLV_DEVICE_REVID_LEN	1u
360
361#define MIGRATION_DESCRIPTOR_DWORDS	(GUC_KLV_LEN_MIN + MIGRATION_KLV_DEVICE_DEVID_LEN + \
362					 GUC_KLV_LEN_MIN + MIGRATION_KLV_DEVICE_REVID_LEN)
363static size_t pf_descriptor_init(struct xe_device *xe, unsigned int vfid)
364{
365	struct xe_sriov_packet **desc = pf_pick_descriptor(xe, vfid);
366	struct xe_sriov_packet *data;
367	unsigned int len = 0;
368	u32 *klvs;
369	int ret;
370
371	data = xe_sriov_packet_alloc(xe);
372	if (!data)
373		return -ENOMEM;
374
375	ret = xe_sriov_packet_init(data, 0, 0, XE_SRIOV_PACKET_TYPE_DESCRIPTOR,
376				   0, MIGRATION_DESCRIPTOR_DWORDS * sizeof(u32));
377	if (ret) {
378		xe_sriov_packet_free(data);
379		return ret;
380	}
381
382	klvs = data->vaddr;
383	klvs[len++] = PREP_GUC_KLV_CONST(MIGRATION_KLV_DEVICE_DEVID_KEY,
384					 MIGRATION_KLV_DEVICE_DEVID_LEN);
385	klvs[len++] = xe->info.devid;
386	klvs[len++] = PREP_GUC_KLV_CONST(MIGRATION_KLV_DEVICE_REVID_KEY,
387					 MIGRATION_KLV_DEVICE_REVID_LEN);
388	klvs[len++] = xe->info.revid;
389
390	xe_assert(xe, len == MIGRATION_DESCRIPTOR_DWORDS);
391
392	*desc = data;
393
394	return 0;
395}
396
397/**
398 * xe_sriov_packet_process_descriptor() - Process migration data descriptor packet.
399 * @xe: the &xe_device
400 * @vfid: the VF identifier
401 * @data: the &xe_sriov_packet containing the descriptor
402 *
403 * The descriptor uses the same KLV format as GuC, and contains metadata used for
404 * checking migration data compatibility.
405 *
406 * Return: 0 on success, -errno on failure.
407 */
408int xe_sriov_packet_process_descriptor(struct xe_device *xe, unsigned int vfid,
409				       struct xe_sriov_packet *data)
410{
411	u32 num_dwords = data->hdr.size / sizeof(u32);
412	u32 *klvs = data->vaddr;
413
414	xe_assert(xe, data->hdr.type == XE_SRIOV_PACKET_TYPE_DESCRIPTOR);
415
416	if (data->hdr.size % sizeof(u32)) {
417		xe_sriov_warn(xe, "Aborting migration, descriptor not in KLV format (size=%llu)\n",
418			      data->hdr.size);
419		return -EINVAL;
420	}
421
422	while (num_dwords >= GUC_KLV_LEN_MIN) {
423		u32 key = FIELD_GET(GUC_KLV_0_KEY, klvs[0]);
424		u32 len = FIELD_GET(GUC_KLV_0_LEN, klvs[0]);
425
426		klvs += GUC_KLV_LEN_MIN;
427		num_dwords -= GUC_KLV_LEN_MIN;
428
429		if (len > num_dwords) {
430			xe_sriov_warn(xe, "Aborting migration, truncated KLV %#x, len %u\n",
431				      key, len);
432			return -EINVAL;
433		}
434
435		switch (key) {
436		case MIGRATION_KLV_DEVICE_DEVID_KEY:
437			if (*klvs != xe->info.devid) {
438				xe_sriov_warn(xe,
439					      "Aborting migration, devid mismatch %#06x!=%#06x\n",
440					      *klvs, xe->info.devid);
441				return -ENODEV;
442			}
443			break;
444		case MIGRATION_KLV_DEVICE_REVID_KEY:
445			if (*klvs != xe->info.revid) {
446				xe_sriov_warn(xe,
447					      "Aborting migration, revid mismatch %#06x!=%#06x\n",
448					      *klvs, xe->info.revid);
449				return -ENODEV;
450			}
451			break;
452		default:
453			xe_sriov_dbg(xe,
454				     "Skipping unknown migration KLV %#x, len=%u\n",
455				     key, len);
456			print_hex_dump_bytes("desc: ", DUMP_PREFIX_OFFSET, klvs,
457					     min(SZ_64, len * sizeof(u32)));
458			break;
459		}
460
461		klvs += len;
462		num_dwords -= len;
463	}
464
465	return 0;
466}
467
468static void pf_pending_init(struct xe_device *xe, unsigned int vfid)
469{
470	struct xe_sriov_packet **data = pf_pick_pending(xe, vfid);
471
472	*data = NULL;
473}
474
475#define MIGRATION_TRAILER_SIZE 0
476static int pf_trailer_init(struct xe_device *xe, unsigned int vfid)
477{
478	struct xe_sriov_packet **trailer = pf_pick_trailer(xe, vfid);
479	struct xe_sriov_packet *data;
480	int ret;
481
482	data = xe_sriov_packet_alloc(xe);
483	if (!data)
484		return -ENOMEM;
485
486	ret = xe_sriov_packet_init(data, 0, 0, XE_SRIOV_PACKET_TYPE_TRAILER,
487				   0, MIGRATION_TRAILER_SIZE);
488	if (ret) {
489		xe_sriov_packet_free(data);
490		return ret;
491	}
492
493	*trailer = data;
494
495	return 0;
496}
497
498/**
499 * xe_sriov_packet_save_init() - Initialize the pending save migration packets.
500 * @xe: the &xe_device
501 * @vfid: the VF identifier
502 *
503 * Return: 0 on success, -errno on failure.
504 */
505int xe_sriov_packet_save_init(struct xe_device *xe, unsigned int vfid)
506{
507	int ret;
508
509	scoped_cond_guard(mutex_intr, return -EINTR, pf_migration_mutex(xe, vfid)) {
510		ret = pf_descriptor_init(xe, vfid);
511		if (ret)
512			return ret;
513
514		ret = pf_trailer_init(xe, vfid);
515		if (ret)
516			return ret;
517
518		pf_pending_init(xe, vfid);
519	}
520
521	return 0;
522}