rpc.c

  1/*
  2 * Copyright 2023 Red Hat Inc.
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice shall be included in
 12 * all copies or substantial portions of the Software.
 13 *
 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 20 * OTHER DEALINGS IN THE SOFTWARE.
 21 */
 22#include <rm/rpc.h>
 23
 24#include "nvrm/rpcfn.h"
 25
 26#define GSP_MSG_MIN_SIZE GSP_PAGE_SIZE
 27#define GSP_MSG_MAX_SIZE (GSP_MSG_MIN_SIZE * 16)
 28
 29/**
 30 * DOC: GSP message queue element
 31 *
 32 * https://github.com/NVIDIA/open-gpu-kernel-modules/blob/535/src/nvidia/inc/kernel/gpu/gsp/message_queue_priv.h
 33 *
 34 * The GSP command queue and status queue are message queues for the
 35 * communication between software and GSP. The software submits the GSP
 36 * RPC via the GSP command queue, GSP writes the status of the submitted
 37 * RPC in the status queue.
 38 *
 39 * A GSP message queue element consists of three parts:
 40 *
 41 * - message element header (struct r535_gsp_msg), which mostly maintains
 42 *   the metadata for queuing the element.
 43 *
 44 * - RPC message header (struct nvfw_gsp_rpc), which maintains the info
 45 *   of the RPC. E.g., the RPC function number.
 46 *
 47 * - The payload, where the RPC message stays. E.g. the params of a
 48 *   specific RPC function. Some RPC functions also have their headers
 49 *   in the payload. E.g. rm_alloc, rm_control.
 50 *
 51 * The memory layout of a GSP message element can be illustrated below::
 52 *
 53 *    +------------------------+
 54 *    | Message Element Header |
 55 *    |    (r535_gsp_msg)      |
 56 *    |                        |
 57 *    | (r535_gsp_msg.data)    |
 58 *    |          |             |
 59 *    |----------V-------------|
 60 *    |    GSP RPC Header      |
 61 *    |    (nvfw_gsp_rpc)      |
 62 *    |                        |
 63 *    | (nvfw_gsp_rpc.data)    |
 64 *    |          |             |
 65 *    |----------V-------------|
 66 *    |       Payload          |
 67 *    |                        |
 68 *    |   header(optional)     |
 69 *    |        params          |
 70 *    +------------------------+
 71 *
 72 * The max size of a message queue element is 16 pages (including the
 73 * headers). When a GSP message to be sent is larger than 16 pages, the
 74 * message should be split into multiple elements and sent accordingly.
 75 *
 76 * In the bunch of the split elements, the first element has the expected
 77 * function number, while the rest of the elements are sent with the
 78 * function number NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD.
 79 *
 80 * GSP consumes the elements from the cmdq and always writes the result
 81 * back to the msgq. The result is also formed as split elements.
 82 *
 83 * Terminology:
 84 *
 85 * - gsp_msg(msg): GSP message element (element header + GSP RPC header +
 86 *   payload)
 87 * - gsp_rpc(rpc): GSP RPC (RPC header + payload)
 88 * - gsp_rpc_buf: buffer for (GSP RPC header + payload)
 89 * - gsp_rpc_len: size of (GSP RPC header + payload)
 90 * - params_size: size of params in the payload
 91 * - payload_size: size of (header if exists + params) in the payload
 92 */
 93
 94struct r535_gsp_msg {
 95	u8 auth_tag_buffer[16];
 96	u8 aad_buffer[16];
 97	u32 checksum;
 98	u32 sequence;
 99	u32 elem_count;
100	u32 pad;
101	u8  data[];
102};
103
104struct nvfw_gsp_rpc {
105	u32 header_version;
106	u32 signature;
107	u32 length;
108	u32 function;
109	u32 rpc_result;
110	u32 rpc_result_private;
111	u32 sequence;
112	union {
113		u32 spare;
114		u32 cpuRmGfid;
115	};
116	u8  data[];
117};
118
119#define GSP_MSG_HDR_SIZE offsetof(struct r535_gsp_msg, data)
120
121#define to_gsp_hdr(p, header) \
122	container_of((void *)p, typeof(*header), data)
123
124#define to_payload_hdr(p, header) \
125	container_of((void *)p, typeof(*header), params)
126
127int
128r535_rpc_status_to_errno(uint32_t rpc_status)
129{
130	switch (rpc_status) {
131	case 0x55: /* NV_ERR_NOT_READY */
132	case 0x66: /* NV_ERR_TIMEOUT_RETRY */
133		return -EBUSY;
134	case 0x51: /* NV_ERR_NO_MEMORY */
135		return -ENOMEM;
136	default:
137		return -EINVAL;
138	}
139}
140
141static int
142r535_gsp_msgq_wait(struct nvkm_gsp *gsp, u32 gsp_rpc_len, int *ptime)
143{
144	u32 size, rptr = *gsp->msgq.rptr;
145	int used;
146
147	size = DIV_ROUND_UP(GSP_MSG_HDR_SIZE + gsp_rpc_len,
148			    GSP_PAGE_SIZE);
149	if (WARN_ON(!size || size >= gsp->msgq.cnt))
150		return -EINVAL;
151
152	do {
153		u32 wptr = *gsp->msgq.wptr;
154
155		used = wptr + gsp->msgq.cnt - rptr;
156		if (used >= gsp->msgq.cnt)
157			used -= gsp->msgq.cnt;
158		if (used >= size)
159			break;
160
161		usleep_range(1, 2);
162	} while (--(*ptime));
163
164	if (WARN_ON(!*ptime))
165		return -ETIMEDOUT;
166
167	return used;
168}
169
170static struct r535_gsp_msg *
171r535_gsp_msgq_get_entry(struct nvkm_gsp *gsp)
172{
173	u32 rptr = *gsp->msgq.rptr;
174
175	/* Skip the first page, which is the message queue info */
176	return (void *)((u8 *)gsp->shm.msgq.ptr + GSP_PAGE_SIZE +
177	       rptr * GSP_PAGE_SIZE);
178}
179
180/**
181 * DOC: Receive a GSP message queue element
182 *
183 * Receiving a GSP message queue element from the message queue consists of
184 * the following steps:
185 *
186 * - Peek the element from the queue: r535_gsp_msgq_peek().
187 *   Peek the first page of the element to determine the total size of the
188 *   message before allocating the proper memory.
189 *
190 * - Allocate memory for the message.
191 *   Once the total size of the message is determined from the GSP message
192 *   queue element, the caller of r535_gsp_msgq_recv() allocates the
193 *   required memory.
194 *
195 * - Receive the message: r535_gsp_msgq_recv().
196 *   Copy the message into the allocated memory. Advance the read pointer.
197 *   If the message is a large GSP message, r535_gsp_msgq_recv() calls
198 *   r535_gsp_msgq_recv_one_elem() repeatedly to receive continuation parts
199 *   until the complete message is received.
200 *   r535_gsp_msgq_recv() assembles the payloads of cotinuation parts into
201 *   the return of the large GSP message.
202 *
203 * - Free the allocated memory: r535_gsp_msg_done().
204 *   The user is responsible for freeing the memory allocated for the GSP
205 *   message pages after they have been processed.
206 */
207static void *
208r535_gsp_msgq_peek(struct nvkm_gsp *gsp, u32 gsp_rpc_len, int *retries)
209{
210	struct r535_gsp_msg *mqe;
211	int ret;
212
213	ret = r535_gsp_msgq_wait(gsp, gsp_rpc_len, retries);
214	if (ret < 0)
215		return ERR_PTR(ret);
216
217	mqe = r535_gsp_msgq_get_entry(gsp);
218
219	return mqe->data;
220}
221
222struct r535_gsp_msg_info {
223	int *retries;
224	u32 gsp_rpc_len;
225	void *gsp_rpc_buf;
226	bool continuation;
227};
228
229static void
230r535_gsp_msg_dump(struct nvkm_gsp *gsp, struct nvfw_gsp_rpc *msg, int lvl);
231
232static void *
233r535_gsp_msgq_recv_one_elem(struct nvkm_gsp *gsp,
234			    struct r535_gsp_msg_info *info)
235{
236	u8 *buf = info->gsp_rpc_buf;
237	u32 rptr = *gsp->msgq.rptr;
238	struct r535_gsp_msg *mqe;
239	u32 size, expected, len;
240	int ret;
241
242	expected = info->gsp_rpc_len;
243
244	ret = r535_gsp_msgq_wait(gsp, expected, info->retries);
245	if (ret < 0)
246		return ERR_PTR(ret);
247
248	mqe = r535_gsp_msgq_get_entry(gsp);
249
250	if (info->continuation) {
251		struct nvfw_gsp_rpc *rpc = (struct nvfw_gsp_rpc *)mqe->data;
252
253		if (rpc->function != NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD) {
254			nvkm_error(&gsp->subdev,
255				   "Not a continuation of a large RPC\n");
256			r535_gsp_msg_dump(gsp, rpc, NV_DBG_ERROR);
257			return ERR_PTR(-EIO);
258		}
259	}
260
261	size = ALIGN(expected + GSP_MSG_HDR_SIZE, GSP_PAGE_SIZE);
262
263	len = ((gsp->msgq.cnt - rptr) * GSP_PAGE_SIZE) - sizeof(*mqe);
264	len = min_t(u32, expected, len);
265
266	if (info->continuation)
267		memcpy(buf, mqe->data + sizeof(struct nvfw_gsp_rpc),
268		       len - sizeof(struct nvfw_gsp_rpc));
269	else
270		memcpy(buf, mqe->data, len);
271
272	expected -= len;
273
274	if (expected) {
275		mqe = (void *)((u8 *)gsp->shm.msgq.ptr + 0x1000 + 0 * 0x1000);
276		memcpy(buf + len, mqe, expected);
277	}
278
279	rptr = (rptr + DIV_ROUND_UP(size, GSP_PAGE_SIZE)) % gsp->msgq.cnt;
280
281	mb();
282	(*gsp->msgq.rptr) = rptr;
283	return buf;
284}
285
286static void *
287r535_gsp_msgq_recv(struct nvkm_gsp *gsp, u32 gsp_rpc_len, int *retries)
288{
289	struct r535_gsp_msg *mqe;
290	const u32 max_rpc_size = GSP_MSG_MAX_SIZE - sizeof(*mqe);
291	struct nvfw_gsp_rpc *rpc;
292	struct r535_gsp_msg_info info = {0};
293	u32 expected = gsp_rpc_len;
294	void *buf;
295
296	mqe = r535_gsp_msgq_get_entry(gsp);
297	rpc = (struct nvfw_gsp_rpc *)mqe->data;
298
299	if (WARN_ON(rpc->length > max_rpc_size))
300		return NULL;
301
302	buf = kvmalloc(max_t(u32, rpc->length, expected), GFP_KERNEL);
303	if (!buf)
304		return ERR_PTR(-ENOMEM);
305
306	info.gsp_rpc_buf = buf;
307	info.retries = retries;
308	info.gsp_rpc_len = rpc->length;
309
310	buf = r535_gsp_msgq_recv_one_elem(gsp, &info);
311	if (IS_ERR(buf)) {
312		kvfree(info.gsp_rpc_buf);
313		info.gsp_rpc_buf = NULL;
314		return buf;
315	}
316
317	if (expected <= max_rpc_size)
318		return buf;
319
320	info.gsp_rpc_buf += info.gsp_rpc_len;
321	expected -= info.gsp_rpc_len;
322
323	while (expected) {
324		u32 size;
325
326		rpc = r535_gsp_msgq_peek(gsp, sizeof(*rpc), info.retries);
327		if (IS_ERR_OR_NULL(rpc)) {
328			kvfree(buf);
329			return rpc;
330		}
331
332		info.gsp_rpc_len = rpc->length;
333		info.continuation = true;
334
335		rpc = r535_gsp_msgq_recv_one_elem(gsp, &info);
336		if (IS_ERR_OR_NULL(rpc)) {
337			kvfree(buf);
338			return rpc;
339		}
340
341		size = info.gsp_rpc_len - sizeof(*rpc);
342		expected -= size;
343		info.gsp_rpc_buf += size;
344	}
345
346	rpc = buf;
347	rpc->length = gsp_rpc_len;
348	return buf;
349}
350
351static int
352r535_gsp_cmdq_push(struct nvkm_gsp *gsp, void *rpc)
353{
354	struct r535_gsp_msg *msg = to_gsp_hdr(rpc, msg);
355	struct r535_gsp_msg *cqe;
356	u32 gsp_rpc_len = msg->checksum;
357	u64 *ptr = (void *)msg;
358	u64 *end;
359	u64 csum = 0;
360	int free, time = 1000000;
361	u32 wptr, size, step, len;
362	u32 off = 0;
363
364	len = ALIGN(GSP_MSG_HDR_SIZE + gsp_rpc_len, GSP_PAGE_SIZE);
365
366	end = (u64 *)((char *)ptr + len);
367	msg->pad = 0;
368	msg->checksum = 0;
369	msg->sequence = gsp->cmdq.seq++;
370	msg->elem_count = DIV_ROUND_UP(len, 0x1000);
371
372	while (ptr < end)
373		csum ^= *ptr++;
374
375	msg->checksum = upper_32_bits(csum) ^ lower_32_bits(csum);
376
377	wptr = *gsp->cmdq.wptr;
378	do {
379		do {
380			free = *gsp->cmdq.rptr + gsp->cmdq.cnt - wptr - 1;
381			if (free >= gsp->cmdq.cnt)
382				free -= gsp->cmdq.cnt;
383			if (free >= 1)
384				break;
385
386			usleep_range(1, 2);
387		} while(--time);
388
389		if (WARN_ON(!time)) {
390			kvfree(msg);
391			return -ETIMEDOUT;
392		}
393
394		cqe = (void *)((u8 *)gsp->shm.cmdq.ptr + 0x1000 + wptr * 0x1000);
395		step = min_t(u32, free, (gsp->cmdq.cnt - wptr));
396		size = min_t(u32, len, step * GSP_PAGE_SIZE);
397
398		memcpy(cqe, (u8 *)msg + off, size);
399
400		wptr += DIV_ROUND_UP(size, 0x1000);
401		if (wptr == gsp->cmdq.cnt)
402			wptr = 0;
403
404		off  += size;
405		len -= size;
406	} while (len);
407
408	nvkm_trace(&gsp->subdev, "cmdq: wptr %d\n", wptr);
409	wmb();
410	(*gsp->cmdq.wptr) = wptr;
411	mb();
412
413	nvkm_falcon_wr32(&gsp->falcon, 0xc00, 0x00000000);
414
415	kvfree(msg);
416	return 0;
417}
418
419static void *
420r535_gsp_cmdq_get(struct nvkm_gsp *gsp, u32 gsp_rpc_len)
421{
422	struct r535_gsp_msg *msg;
423	u32 size = GSP_MSG_HDR_SIZE + gsp_rpc_len;
424
425	size = ALIGN(size, GSP_MSG_MIN_SIZE);
426	msg = kvzalloc(size, GFP_KERNEL);
427	if (!msg)
428		return ERR_PTR(-ENOMEM);
429
430	msg->checksum = gsp_rpc_len;
431	return msg->data;
432}
433
434static void
435r535_gsp_msg_done(struct nvkm_gsp *gsp, struct nvfw_gsp_rpc *msg)
436{
437	kvfree(msg);
438}
439
440static void
441r535_gsp_msg_dump(struct nvkm_gsp *gsp, struct nvfw_gsp_rpc *msg, int lvl)
442{
443	if (gsp->subdev.debug >= lvl) {
444		nvkm_printk__(&gsp->subdev, lvl, info,
445			      "msg fn:%d len:0x%x/0x%zx res:0x%x resp:0x%x\n",
446			      msg->function, msg->length, msg->length - sizeof(*msg),
447			      msg->rpc_result, msg->rpc_result_private);
448		print_hex_dump(KERN_INFO, "msg: ", DUMP_PREFIX_OFFSET, 16, 1,
449			       msg->data, msg->length - sizeof(*msg), true);
450	}
451}
452
453struct nvfw_gsp_rpc *
454r535_gsp_msg_recv(struct nvkm_gsp *gsp, int fn, u32 gsp_rpc_len)
455{
456	struct nvkm_subdev *subdev = &gsp->subdev;
457	struct nvfw_gsp_rpc *rpc;
458	int retries = 4000000, i;
459
460retry:
461	rpc = r535_gsp_msgq_peek(gsp, sizeof(*rpc), &retries);
462	if (IS_ERR_OR_NULL(rpc))
463		return rpc;
464
465	rpc = r535_gsp_msgq_recv(gsp, gsp_rpc_len, &retries);
466	if (IS_ERR_OR_NULL(rpc))
467		return rpc;
468
469	if (rpc->rpc_result) {
470		r535_gsp_msg_dump(gsp, rpc, NV_DBG_ERROR);
471		r535_gsp_msg_done(gsp, rpc);
472		return ERR_PTR(-EINVAL);
473	}
474
475	r535_gsp_msg_dump(gsp, rpc, NV_DBG_TRACE);
476
477	if (fn && rpc->function == fn) {
478		if (gsp_rpc_len) {
479			if (rpc->length < gsp_rpc_len) {
480				nvkm_error(subdev, "rpc len %d < %d\n",
481					   rpc->length, gsp_rpc_len);
482				r535_gsp_msg_dump(gsp, rpc, NV_DBG_ERROR);
483				r535_gsp_msg_done(gsp, rpc);
484				return ERR_PTR(-EIO);
485			}
486
487			return rpc;
488		}
489
490		r535_gsp_msg_done(gsp, rpc);
491		return NULL;
492	}
493
494	for (i = 0; i < gsp->msgq.ntfy_nr; i++) {
495		struct nvkm_gsp_msgq_ntfy *ntfy = &gsp->msgq.ntfy[i];
496
497		if (ntfy->fn == rpc->function) {
498			if (ntfy->func)
499				ntfy->func(ntfy->priv, ntfy->fn, rpc->data,
500					   rpc->length - sizeof(*rpc));
501			break;
502		}
503	}
504
505	if (i == gsp->msgq.ntfy_nr)
506		r535_gsp_msg_dump(gsp, rpc, NV_DBG_WARN);
507
508	r535_gsp_msg_done(gsp, rpc);
509	if (fn)
510		goto retry;
511
512	if (*gsp->msgq.rptr != *gsp->msgq.wptr)
513		goto retry;
514
515	return NULL;
516}
517
518int
519r535_gsp_msg_ntfy_add(struct nvkm_gsp *gsp, u32 fn, nvkm_gsp_msg_ntfy_func func, void *priv)
520{
521	int ret = 0;
522
523	mutex_lock(&gsp->msgq.mutex);
524	if (WARN_ON(gsp->msgq.ntfy_nr >= ARRAY_SIZE(gsp->msgq.ntfy))) {
525		ret = -ENOSPC;
526	} else {
527		gsp->msgq.ntfy[gsp->msgq.ntfy_nr].fn = fn;
528		gsp->msgq.ntfy[gsp->msgq.ntfy_nr].func = func;
529		gsp->msgq.ntfy[gsp->msgq.ntfy_nr].priv = priv;
530		gsp->msgq.ntfy_nr++;
531	}
532	mutex_unlock(&gsp->msgq.mutex);
533	return ret;
534}
535
536int
537r535_gsp_rpc_poll(struct nvkm_gsp *gsp, u32 fn)
538{
539	void *repv;
540
541	mutex_lock(&gsp->cmdq.mutex);
542	repv = r535_gsp_msg_recv(gsp, fn, 0);
543	mutex_unlock(&gsp->cmdq.mutex);
544	if (IS_ERR(repv))
545		return PTR_ERR(repv);
546
547	return 0;
548}
549
550static void *
551r535_gsp_rpc_handle_reply(struct nvkm_gsp *gsp, u32 fn,
552			  enum nvkm_gsp_rpc_reply_policy policy,
553			  u32 gsp_rpc_len)
554{
555	struct nvfw_gsp_rpc *reply;
556	void *repv = NULL;
557
558	switch (policy) {
559	case NVKM_GSP_RPC_REPLY_NOWAIT:
560	case NVKM_GSP_RPC_REPLY_NOSEQ:
561		break;
562	case NVKM_GSP_RPC_REPLY_RECV:
563		reply = r535_gsp_msg_recv(gsp, fn, gsp_rpc_len);
564		if (!IS_ERR_OR_NULL(reply))
565			repv = reply->data;
566		else
567			repv = reply;
568		break;
569	case NVKM_GSP_RPC_REPLY_POLL:
570		repv = r535_gsp_msg_recv(gsp, fn, 0);
571		break;
572	}
573
574	return repv;
575}
576
577static void *
578r535_gsp_rpc_send(struct nvkm_gsp *gsp, void *payload,
579		  enum nvkm_gsp_rpc_reply_policy policy, u32 gsp_rpc_len)
580{
581	struct nvfw_gsp_rpc *rpc = to_gsp_hdr(payload, rpc);
582	u32 fn = rpc->function;
583	int ret;
584
585	if (gsp->subdev.debug >= NV_DBG_TRACE) {
586		nvkm_trace(&gsp->subdev, "rpc fn:%d len:0x%x/0x%zx\n", rpc->function,
587			   rpc->length, rpc->length - sizeof(*rpc));
588		print_hex_dump(KERN_INFO, "rpc: ", DUMP_PREFIX_OFFSET, 16, 1,
589			       rpc->data, rpc->length - sizeof(*rpc), true);
590	}
591
592	if (policy == NVKM_GSP_RPC_REPLY_NOSEQ)
593		rpc->sequence = 0;
594	else
595		rpc->sequence = gsp->rpc_seq++;
596
597	ret = r535_gsp_cmdq_push(gsp, rpc);
598	if (ret)
599		return ERR_PTR(ret);
600
601	return r535_gsp_rpc_handle_reply(gsp, fn, policy, gsp_rpc_len);
602}
603
604static void
605r535_gsp_rpc_done(struct nvkm_gsp *gsp, void *repv)
606{
607	struct nvfw_gsp_rpc *rpc = container_of(repv, typeof(*rpc), data);
608
609	r535_gsp_msg_done(gsp, rpc);
610}
611
612static void *
613r535_gsp_rpc_get(struct nvkm_gsp *gsp, u32 fn, u32 payload_size)
614{
615	struct nvfw_gsp_rpc *rpc;
616
617	rpc = r535_gsp_cmdq_get(gsp, ALIGN(sizeof(*rpc) + payload_size,
618					   sizeof(u64)));
619	if (IS_ERR(rpc))
620		return ERR_CAST(rpc);
621
622	rpc->header_version = 0x03000000;
623	rpc->signature = ('C' << 24) | ('P' << 16) | ('R' << 8) | 'V';
624	rpc->function = fn;
625	rpc->rpc_result = 0xffffffff;
626	rpc->rpc_result_private = 0xffffffff;
627	rpc->length = sizeof(*rpc) + payload_size;
628	return rpc->data;
629}
630
631static void *
632r535_gsp_rpc_push(struct nvkm_gsp *gsp, void *payload,
633		  enum nvkm_gsp_rpc_reply_policy policy, u32 gsp_rpc_len)
634{
635	struct nvfw_gsp_rpc *rpc = to_gsp_hdr(payload, rpc);
636	struct r535_gsp_msg *msg = to_gsp_hdr(rpc, msg);
637	const u32 max_rpc_size = GSP_MSG_MAX_SIZE - sizeof(*msg);
638	const u32 max_payload_size = max_rpc_size - sizeof(*rpc);
639	u32 payload_size = rpc->length - sizeof(*rpc);
640	void *repv;
641
642	mutex_lock(&gsp->cmdq.mutex);
643	if (payload_size > max_payload_size) {
644		const u32 fn = rpc->function;
645		u32 remain_payload_size = payload_size;
646		void *next;
647
648		/* Send initial RPC. */
649		next = r535_gsp_rpc_get(gsp, fn, max_payload_size);
650		if (IS_ERR(next)) {
651			repv = next;
652			goto done;
653		}
654
655		memcpy(next, payload, max_payload_size);
656
657		repv = r535_gsp_rpc_send(gsp, next, NVKM_GSP_RPC_REPLY_NOWAIT, 0);
658		if (IS_ERR(repv))
659			goto done;
660
661		payload += max_payload_size;
662		remain_payload_size -= max_payload_size;
663
664		/* Remaining chunks sent as CONTINUATION_RECORD RPCs. */
665		while (remain_payload_size) {
666			u32 size = min(remain_payload_size,
667				       max_payload_size);
668
669			next = r535_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD, size);
670			if (IS_ERR(next)) {
671				repv = next;
672				goto done;
673			}
674
675			memcpy(next, payload, size);
676
677			repv = r535_gsp_rpc_send(gsp, next, NVKM_GSP_RPC_REPLY_NOWAIT, 0);
678			if (IS_ERR(repv))
679				goto done;
680
681			payload += size;
682			remain_payload_size -= size;
683		}
684
685		/* Wait for reply. */
686		repv = r535_gsp_rpc_handle_reply(gsp, fn, policy, payload_size +
687						 sizeof(*rpc));
688		if (!IS_ERR(repv))
689			kvfree(msg);
690	} else {
691		repv = r535_gsp_rpc_send(gsp, payload, policy, gsp_rpc_len);
692	}
693
694done:
695	mutex_unlock(&gsp->cmdq.mutex);
696	return repv;
697}
698
699const struct nvkm_rm_api_rpc
700r535_rpc = {
701	.get = r535_gsp_rpc_get,
702	.push = r535_gsp_rpc_push,
703	.done = r535_gsp_rpc_done,
704};