ivpu_ipc.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) 2020-2024 Intel Corporation
  4 */
  5
  6#include <linux/genalloc.h>
  7#include <linux/highmem.h>
  8#include <linux/pm_runtime.h>
  9#include <linux/wait.h>
 10
 11#include "ivpu_drv.h"
 12#include "ivpu_gem.h"
 13#include "ivpu_hw.h"
 14#include "ivpu_hw_reg_io.h"
 15#include "ivpu_ipc.h"
 16#include "ivpu_jsm_msg.h"
 17#include "ivpu_pm.h"
 18#include "ivpu_trace.h"
 19
 20#define IPC_MAX_RX_MSG	128
 21
 22struct ivpu_ipc_tx_buf {
 23	struct ivpu_ipc_hdr ipc;
 24	struct vpu_jsm_msg jsm;
 25};
 26
 27static void ivpu_ipc_msg_dump(struct ivpu_device *vdev, char *c,
 28			      struct ivpu_ipc_hdr *ipc_hdr, u32 vpu_addr)
 29{
 30	ivpu_dbg(vdev, IPC,
 31		 "%s: vpu:0x%x (data_addr:0x%08x, data_size:0x%x, channel:0x%x, src_node:0x%x, dst_node:0x%x, status:0x%x)",
 32		 c, vpu_addr, ipc_hdr->data_addr, ipc_hdr->data_size, ipc_hdr->channel,
 33		 ipc_hdr->src_node, ipc_hdr->dst_node, ipc_hdr->status);
 34}
 35
 36static void ivpu_jsm_msg_dump(struct ivpu_device *vdev, char *c,
 37			      struct vpu_jsm_msg *jsm_msg, u32 vpu_addr)
 38{
 39	u32 *payload = (u32 *)&jsm_msg->payload;
 40
 41	ivpu_dbg(vdev, JSM,
 42		 "%s: vpu:0x%08x (type:%s, status:0x%x, id: 0x%x, result: 0x%x, payload:0x%x 0x%x 0x%x 0x%x 0x%x)\n",
 43		 c, vpu_addr, ivpu_jsm_msg_type_to_str(jsm_msg->type),
 44		 jsm_msg->status, jsm_msg->request_id, jsm_msg->result,
 45		 payload[0], payload[1], payload[2], payload[3], payload[4]);
 46}
 47
 48static void
 49ivpu_ipc_rx_mark_free(struct ivpu_device *vdev, struct ivpu_ipc_hdr *ipc_hdr,
 50		      struct vpu_jsm_msg *jsm_msg)
 51{
 52	ipc_hdr->status = IVPU_IPC_HDR_FREE;
 53	if (jsm_msg)
 54		jsm_msg->status = VPU_JSM_MSG_FREE;
 55	wmb(); /* Flush WC buffers for message statuses */
 56}
 57
 58static void ivpu_ipc_mem_fini(struct ivpu_device *vdev)
 59{
 60	struct ivpu_ipc_info *ipc = vdev->ipc;
 61
 62	ivpu_bo_free(ipc->mem_rx);
 63	ivpu_bo_free(ipc->mem_tx);
 64}
 65
 66static int
 67ivpu_ipc_tx_prepare(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
 68		    struct vpu_jsm_msg *req)
 69{
 70	struct ivpu_ipc_info *ipc = vdev->ipc;
 71	struct ivpu_ipc_tx_buf *tx_buf;
 72	u32 tx_buf_vpu_addr;
 73	u32 jsm_vpu_addr;
 74
 75	tx_buf_vpu_addr = gen_pool_alloc(ipc->mm_tx, sizeof(*tx_buf));
 76	if (!tx_buf_vpu_addr) {
 77		ivpu_err_ratelimited(vdev, "Failed to reserve IPC buffer, size %ld\n",
 78				     sizeof(*tx_buf));
 79		return -ENOMEM;
 80	}
 81
 82	tx_buf = ivpu_to_cpu_addr(ipc->mem_tx, tx_buf_vpu_addr);
 83	if (drm_WARN_ON(&vdev->drm, !tx_buf)) {
 84		gen_pool_free(ipc->mm_tx, tx_buf_vpu_addr, sizeof(*tx_buf));
 85		return -EIO;
 86	}
 87
 88	jsm_vpu_addr = tx_buf_vpu_addr + offsetof(struct ivpu_ipc_tx_buf, jsm);
 89
 90	if (tx_buf->ipc.status != IVPU_IPC_HDR_FREE)
 91		ivpu_warn_ratelimited(vdev, "IPC message vpu:0x%x not released by firmware\n",
 92				      tx_buf_vpu_addr);
 93
 94	if (tx_buf->jsm.status != VPU_JSM_MSG_FREE)
 95		ivpu_warn_ratelimited(vdev, "JSM message vpu:0x%x not released by firmware\n",
 96				      jsm_vpu_addr);
 97
 98	memset(tx_buf, 0, sizeof(*tx_buf));
 99	tx_buf->ipc.data_addr = jsm_vpu_addr;
100	/* TODO: Set data_size to actual JSM message size, not union of all messages */
101	tx_buf->ipc.data_size = sizeof(*req);
102	tx_buf->ipc.channel = cons->channel;
103	tx_buf->ipc.src_node = 0;
104	tx_buf->ipc.dst_node = 1;
105	tx_buf->ipc.status = IVPU_IPC_HDR_ALLOCATED;
106	tx_buf->jsm.type = req->type;
107	tx_buf->jsm.status = VPU_JSM_MSG_ALLOCATED;
108	tx_buf->jsm.payload = req->payload;
109
110	req->request_id = atomic_inc_return(&ipc->request_id);
111	tx_buf->jsm.request_id = req->request_id;
112	cons->request_id = req->request_id;
113	wmb(); /* Flush WC buffers for IPC, JSM msgs */
114
115	cons->tx_vpu_addr = tx_buf_vpu_addr;
116
117	ivpu_jsm_msg_dump(vdev, "TX", &tx_buf->jsm, jsm_vpu_addr);
118	ivpu_ipc_msg_dump(vdev, "TX", &tx_buf->ipc, tx_buf_vpu_addr);
119
120	return 0;
121}
122
123static void ivpu_ipc_tx_release(struct ivpu_device *vdev, u32 vpu_addr)
124{
125	struct ivpu_ipc_info *ipc = vdev->ipc;
126
127	if (vpu_addr)
128		gen_pool_free(ipc->mm_tx, vpu_addr, sizeof(struct ivpu_ipc_tx_buf));
129}
130
131static void ivpu_ipc_tx(struct ivpu_device *vdev, u32 vpu_addr)
132{
133	ivpu_hw_ipc_tx_set(vdev, vpu_addr);
134}
135
136static void
137ivpu_ipc_rx_msg_add(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
138		    struct ivpu_ipc_hdr *ipc_hdr, struct vpu_jsm_msg *jsm_msg)
139{
140	struct ivpu_ipc_info *ipc = vdev->ipc;
141	struct ivpu_ipc_rx_msg *rx_msg;
142
143	lockdep_assert_held(&ipc->cons_lock);
144
145	rx_msg = kzalloc(sizeof(*rx_msg), GFP_ATOMIC);
146	if (!rx_msg) {
147		ivpu_ipc_rx_mark_free(vdev, ipc_hdr, jsm_msg);
148		return;
149	}
150
151	atomic_inc(&ipc->rx_msg_count);
152
153	rx_msg->ipc_hdr = ipc_hdr;
154	rx_msg->jsm_msg = jsm_msg;
155	rx_msg->callback = cons->rx_callback;
156
157	if (rx_msg->callback) {
158		list_add_tail(&rx_msg->link, &ipc->cb_msg_list);
159	} else {
160		spin_lock(&cons->rx_lock);
161		list_add_tail(&rx_msg->link, &cons->rx_msg_list);
162		spin_unlock(&cons->rx_lock);
163		wake_up(&cons->rx_msg_wq);
164	}
165}
166
167static void
168ivpu_ipc_rx_msg_del(struct ivpu_device *vdev, struct ivpu_ipc_rx_msg *rx_msg)
169{
170	list_del(&rx_msg->link);
171	ivpu_ipc_rx_mark_free(vdev, rx_msg->ipc_hdr, rx_msg->jsm_msg);
172	atomic_dec(&vdev->ipc->rx_msg_count);
173	kfree(rx_msg);
174}
175
176void ivpu_ipc_consumer_add(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
177			   u32 channel, ivpu_ipc_rx_callback_t rx_callback)
178{
179	struct ivpu_ipc_info *ipc = vdev->ipc;
180
181	INIT_LIST_HEAD(&cons->link);
182	cons->channel = channel;
183	cons->tx_vpu_addr = 0;
184	cons->request_id = 0;
185	cons->aborted = false;
186	cons->rx_callback = rx_callback;
187	spin_lock_init(&cons->rx_lock);
188	INIT_LIST_HEAD(&cons->rx_msg_list);
189	init_waitqueue_head(&cons->rx_msg_wq);
190
191	spin_lock_irq(&ipc->cons_lock);
192	list_add_tail(&cons->link, &ipc->cons_list);
193	spin_unlock_irq(&ipc->cons_lock);
194}
195
196void ivpu_ipc_consumer_del(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons)
197{
198	struct ivpu_ipc_info *ipc = vdev->ipc;
199	struct ivpu_ipc_rx_msg *rx_msg, *r;
200
201	spin_lock_irq(&ipc->cons_lock);
202	list_del(&cons->link);
203	spin_unlock_irq(&ipc->cons_lock);
204
205	spin_lock_irq(&cons->rx_lock);
206	list_for_each_entry_safe(rx_msg, r, &cons->rx_msg_list, link)
207		ivpu_ipc_rx_msg_del(vdev, rx_msg);
208	spin_unlock_irq(&cons->rx_lock);
209
210	ivpu_ipc_tx_release(vdev, cons->tx_vpu_addr);
211}
212
213int ivpu_ipc_send(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons, struct vpu_jsm_msg *req)
214{
215	struct ivpu_ipc_info *ipc = vdev->ipc;
216	int ret;
217
218	mutex_lock(&ipc->lock);
219
220	if (!ipc->on) {
221		ret = -EAGAIN;
222		goto unlock;
223	}
224
225	ret = ivpu_ipc_tx_prepare(vdev, cons, req);
226	if (ret)
227		goto unlock;
228
229	ivpu_ipc_tx(vdev, cons->tx_vpu_addr);
230	trace_jsm("[tx]", req);
231
232unlock:
233	mutex_unlock(&ipc->lock);
234	return ret;
235}
236
237static bool ivpu_ipc_rx_need_wakeup(struct ivpu_ipc_consumer *cons)
238{
239	bool ret;
240
241	spin_lock_irq(&cons->rx_lock);
242	ret = !list_empty(&cons->rx_msg_list) || cons->aborted;
243	spin_unlock_irq(&cons->rx_lock);
244
245	return ret;
246}
247
248int ivpu_ipc_receive(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
249		     struct ivpu_ipc_hdr *ipc_buf,
250		     struct vpu_jsm_msg *jsm_msg, unsigned long timeout_ms)
251{
252	struct ivpu_ipc_rx_msg *rx_msg;
253	int wait_ret, ret = 0;
254
255	if (drm_WARN_ONCE(&vdev->drm, cons->rx_callback, "Consumer works only in async mode\n"))
256		return -EINVAL;
257
258	wait_ret = wait_event_timeout(cons->rx_msg_wq,
259				      ivpu_ipc_rx_need_wakeup(cons),
260				      msecs_to_jiffies(timeout_ms));
261
262	if (wait_ret == 0)
263		return -ETIMEDOUT;
264
265	spin_lock_irq(&cons->rx_lock);
266	if (cons->aborted) {
267		spin_unlock_irq(&cons->rx_lock);
268		return -ECANCELED;
269	}
270	rx_msg = list_first_entry_or_null(&cons->rx_msg_list, struct ivpu_ipc_rx_msg, link);
271	if (!rx_msg) {
272		spin_unlock_irq(&cons->rx_lock);
273		return -EAGAIN;
274	}
275
276	if (ipc_buf)
277		memcpy(ipc_buf, rx_msg->ipc_hdr, sizeof(*ipc_buf));
278	if (rx_msg->jsm_msg) {
279		u32 size = min_t(int, rx_msg->ipc_hdr->data_size, sizeof(*jsm_msg));
280
281		if (rx_msg->jsm_msg->result != VPU_JSM_STATUS_SUCCESS) {
282			ivpu_err(vdev, "IPC resp result error: %d\n", rx_msg->jsm_msg->result);
283			ret = -EBADMSG;
284		}
285
286		if (jsm_msg)
287			memcpy(jsm_msg, rx_msg->jsm_msg, size);
288		trace_jsm("[rx]", rx_msg->jsm_msg);
289	}
290
291	ivpu_ipc_rx_msg_del(vdev, rx_msg);
292	spin_unlock_irq(&cons->rx_lock);
293	return ret;
294}
295
296int
297ivpu_ipc_send_receive_internal(struct ivpu_device *vdev, struct vpu_jsm_msg *req,
298			       enum vpu_ipc_msg_type expected_resp_type,
299			       struct vpu_jsm_msg *resp, u32 channel, unsigned long timeout_ms)
300{
301	struct ivpu_ipc_consumer cons;
302	int ret;
303
304	drm_WARN_ON(&vdev->drm, pm_runtime_status_suspended(vdev->drm.dev) &&
305		    pm_runtime_enabled(vdev->drm.dev));
306
307	ivpu_ipc_consumer_add(vdev, &cons, channel, NULL);
308
309	ret = ivpu_ipc_send(vdev, &cons, req);
310	if (ret) {
311		ivpu_warn_ratelimited(vdev, "IPC send failed: %d\n", ret);
312		goto consumer_del;
313	}
314
315	ret = ivpu_ipc_receive(vdev, &cons, NULL, resp, timeout_ms);
316	if (ret) {
317		ivpu_warn_ratelimited(vdev, "IPC receive failed: type %s, ret %d\n",
318				      ivpu_jsm_msg_type_to_str(req->type), ret);
319		goto consumer_del;
320	}
321
322	if (resp->type != expected_resp_type) {
323		ivpu_warn_ratelimited(vdev, "Invalid JSM response type: 0x%x\n", resp->type);
324		ret = -EBADE;
325	}
326
327consumer_del:
328	ivpu_ipc_consumer_del(vdev, &cons);
329	return ret;
330}
331
332int ivpu_ipc_send_receive(struct ivpu_device *vdev, struct vpu_jsm_msg *req,
333			  enum vpu_ipc_msg_type expected_resp, struct vpu_jsm_msg *resp,
334			  u32 channel, unsigned long timeout_ms)
335{
336	struct vpu_jsm_msg hb_req = { .type = VPU_JSM_MSG_QUERY_ENGINE_HB };
337	struct vpu_jsm_msg hb_resp;
338	int ret, hb_ret;
339
340	ret = ivpu_rpm_get(vdev);
341	if (ret < 0)
342		return ret;
343
344	ret = ivpu_ipc_send_receive_internal(vdev, req, expected_resp, resp, channel, timeout_ms);
345	if (ret != -ETIMEDOUT)
346		goto rpm_put;
347
348	hb_ret = ivpu_ipc_send_receive_internal(vdev, &hb_req, VPU_JSM_MSG_QUERY_ENGINE_HB_DONE,
349						&hb_resp, VPU_IPC_CHAN_ASYNC_CMD,
350						vdev->timeout.jsm);
351	if (hb_ret == -ETIMEDOUT)
352		ivpu_pm_trigger_recovery(vdev, "IPC timeout");
353
354rpm_put:
355	ivpu_rpm_put(vdev);
356	return ret;
357}
358
359int ivpu_ipc_send_and_wait(struct ivpu_device *vdev, struct vpu_jsm_msg *req,
360			   u32 channel, unsigned long timeout_ms)
361{
362	struct ivpu_ipc_consumer cons;
363	int ret;
364
365	ret = ivpu_rpm_get(vdev);
366	if (ret < 0)
367		return ret;
368
369	ivpu_ipc_consumer_add(vdev, &cons, channel, NULL);
370
371	ret = ivpu_ipc_send(vdev, &cons, req);
372	if (ret) {
373		ivpu_warn_ratelimited(vdev, "IPC send failed: %d\n", ret);
374		goto consumer_del;
375	}
376
377	msleep(timeout_ms);
378
379consumer_del:
380	ivpu_ipc_consumer_del(vdev, &cons);
381	ivpu_rpm_put(vdev);
382	return ret;
383}
384
385static bool
386ivpu_ipc_match_consumer(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
387			struct ivpu_ipc_hdr *ipc_hdr, struct vpu_jsm_msg *jsm_msg)
388{
389	if (cons->channel != ipc_hdr->channel)
390		return false;
391
392	if (!jsm_msg || jsm_msg->request_id == cons->request_id)
393		return true;
394
395	return false;
396}
397
398void ivpu_ipc_irq_handler(struct ivpu_device *vdev)
399{
400	struct ivpu_ipc_info *ipc = vdev->ipc;
401	struct ivpu_ipc_consumer *cons;
402	struct ivpu_ipc_hdr *ipc_hdr;
403	struct vpu_jsm_msg *jsm_msg;
404	unsigned long flags;
405	bool dispatched;
406	u32 vpu_addr;
407
408	/*
409	 * Driver needs to purge all messages from IPC FIFO to clear IPC interrupt.
410	 * Without purge IPC FIFO to 0 next IPC interrupts won't be generated.
411	 */
412	while (ivpu_hw_ipc_rx_count_get(vdev)) {
413		vpu_addr = ivpu_hw_ipc_rx_addr_get(vdev);
414		if (vpu_addr == REG_IO_ERROR) {
415			ivpu_err_ratelimited(vdev, "Failed to read IPC rx addr register\n");
416			return;
417		}
418
419		ipc_hdr = ivpu_to_cpu_addr(ipc->mem_rx, vpu_addr);
420		if (!ipc_hdr) {
421			ivpu_warn_ratelimited(vdev, "IPC msg 0x%x out of range\n", vpu_addr);
422			continue;
423		}
424		ivpu_ipc_msg_dump(vdev, "RX", ipc_hdr, vpu_addr);
425
426		jsm_msg = NULL;
427		if (ipc_hdr->channel != IVPU_IPC_CHAN_BOOT_MSG) {
428			jsm_msg = ivpu_to_cpu_addr(ipc->mem_rx, ipc_hdr->data_addr);
429			if (!jsm_msg) {
430				ivpu_warn_ratelimited(vdev, "JSM msg 0x%x out of range\n",
431						      ipc_hdr->data_addr);
432				ivpu_ipc_rx_mark_free(vdev, ipc_hdr, NULL);
433				continue;
434			}
435			ivpu_jsm_msg_dump(vdev, "RX", jsm_msg, ipc_hdr->data_addr);
436		}
437
438		if (atomic_read(&ipc->rx_msg_count) > IPC_MAX_RX_MSG) {
439			ivpu_warn_ratelimited(vdev, "IPC RX msg dropped, msg count %d\n",
440					      IPC_MAX_RX_MSG);
441			ivpu_ipc_rx_mark_free(vdev, ipc_hdr, jsm_msg);
442			continue;
443		}
444
445		dispatched = false;
446		spin_lock_irqsave(&ipc->cons_lock, flags);
447		list_for_each_entry(cons, &ipc->cons_list, link) {
448			if (ivpu_ipc_match_consumer(vdev, cons, ipc_hdr, jsm_msg)) {
449				ivpu_ipc_rx_msg_add(vdev, cons, ipc_hdr, jsm_msg);
450				dispatched = true;
451				break;
452			}
453		}
454		spin_unlock_irqrestore(&ipc->cons_lock, flags);
455
456		if (!dispatched) {
457			ivpu_dbg(vdev, IPC, "IPC RX msg 0x%x dropped (no consumer)\n", vpu_addr);
458			ivpu_ipc_rx_mark_free(vdev, ipc_hdr, jsm_msg);
459		}
460	}
461
462	queue_work(system_percpu_wq, &vdev->irq_ipc_work);
463}
464
465void ivpu_ipc_irq_work_fn(struct work_struct *work)
466{
467	struct ivpu_device *vdev = container_of(work, struct ivpu_device, irq_ipc_work);
468	struct ivpu_ipc_info *ipc = vdev->ipc;
469	struct ivpu_ipc_rx_msg *rx_msg, *r;
470	struct list_head cb_msg_list;
471
472	INIT_LIST_HEAD(&cb_msg_list);
473
474	spin_lock_irq(&ipc->cons_lock);
475	list_splice_tail_init(&ipc->cb_msg_list, &cb_msg_list);
476	spin_unlock_irq(&ipc->cons_lock);
477
478	list_for_each_entry_safe(rx_msg, r, &cb_msg_list, link) {
479		rx_msg->callback(vdev, rx_msg->ipc_hdr, rx_msg->jsm_msg);
480		ivpu_ipc_rx_msg_del(vdev, rx_msg);
481	}
482}
483
484int ivpu_ipc_init(struct ivpu_device *vdev)
485{
486	struct ivpu_ipc_info *ipc = vdev->ipc;
487	int ret;
488
489	ipc->mem_tx = ivpu_bo_create_global(vdev, SZ_16K, DRM_IVPU_BO_WC | DRM_IVPU_BO_MAPPABLE);
490	if (!ipc->mem_tx) {
491		ivpu_err(vdev, "Failed to allocate mem_tx\n");
492		return -ENOMEM;
493	}
494
495	ipc->mem_rx = ivpu_bo_create_global(vdev, SZ_16K, DRM_IVPU_BO_WC | DRM_IVPU_BO_MAPPABLE);
496	if (!ipc->mem_rx) {
497		ivpu_err(vdev, "Failed to allocate mem_rx\n");
498		ret = -ENOMEM;
499		goto err_free_tx;
500	}
501
502	ipc->mm_tx = devm_gen_pool_create(vdev->drm.dev, __ffs(IVPU_IPC_ALIGNMENT),
503					  -1, "TX_IPC_JSM");
504	if (IS_ERR(ipc->mm_tx)) {
505		ret = PTR_ERR(ipc->mm_tx);
506		ivpu_err(vdev, "Failed to create gen pool, %pe\n", ipc->mm_tx);
507		goto err_free_rx;
508	}
509
510	ret = gen_pool_add(ipc->mm_tx, ipc->mem_tx->vpu_addr, ivpu_bo_size(ipc->mem_tx), -1);
511	if (ret) {
512		ivpu_err(vdev, "gen_pool_add failed, ret %d\n", ret);
513		goto err_free_rx;
514	}
515
516	spin_lock_init(&ipc->cons_lock);
517	INIT_LIST_HEAD(&ipc->cons_list);
518	INIT_LIST_HEAD(&ipc->cb_msg_list);
519	ret = drmm_mutex_init(&vdev->drm, &ipc->lock);
520	if (ret) {
521		ivpu_err(vdev, "Failed to initialize ipc->lock, ret %d\n", ret);
522		goto err_free_rx;
523	}
524	ivpu_ipc_reset(vdev);
525	return 0;
526
527err_free_rx:
528	ivpu_bo_free(ipc->mem_rx);
529err_free_tx:
530	ivpu_bo_free(ipc->mem_tx);
531	return ret;
532}
533
534void ivpu_ipc_fini(struct ivpu_device *vdev)
535{
536	struct ivpu_ipc_info *ipc = vdev->ipc;
537
538	drm_WARN_ON(&vdev->drm, !list_empty(&ipc->cons_list));
539	drm_WARN_ON(&vdev->drm, !list_empty(&ipc->cb_msg_list));
540	drm_WARN_ON(&vdev->drm, atomic_read(&ipc->rx_msg_count) > 0);
541
542	ivpu_ipc_mem_fini(vdev);
543}
544
545void ivpu_ipc_enable(struct ivpu_device *vdev)
546{
547	struct ivpu_ipc_info *ipc = vdev->ipc;
548
549	mutex_lock(&ipc->lock);
550	ipc->on = true;
551	mutex_unlock(&ipc->lock);
552}
553
554void ivpu_ipc_disable(struct ivpu_device *vdev)
555{
556	struct ivpu_ipc_info *ipc = vdev->ipc;
557	struct ivpu_ipc_consumer *cons, *c;
558	struct ivpu_ipc_rx_msg *rx_msg, *r;
559
560	drm_WARN_ON(&vdev->drm, !list_empty(&ipc->cb_msg_list));
561
562	mutex_lock(&ipc->lock);
563	ipc->on = false;
564	mutex_unlock(&ipc->lock);
565
566	spin_lock_irq(&ipc->cons_lock);
567	list_for_each_entry_safe(cons, c, &ipc->cons_list, link) {
568		spin_lock(&cons->rx_lock);
569		if (!cons->rx_callback)
570			cons->aborted = true;
571		list_for_each_entry_safe(rx_msg, r, &cons->rx_msg_list, link)
572			ivpu_ipc_rx_msg_del(vdev, rx_msg);
573		spin_unlock(&cons->rx_lock);
574		wake_up(&cons->rx_msg_wq);
575	}
576	spin_unlock_irq(&ipc->cons_lock);
577
578	drm_WARN_ON(&vdev->drm, atomic_read(&ipc->rx_msg_count) > 0);
579}
580
581void ivpu_ipc_reset(struct ivpu_device *vdev)
582{
583	struct ivpu_ipc_info *ipc = vdev->ipc;
584
585	mutex_lock(&ipc->lock);
586	drm_WARN_ON(&vdev->drm, ipc->on);
587
588	memset(ivpu_bo_vaddr(ipc->mem_tx), 0, ivpu_bo_size(ipc->mem_tx));
589	memset(ivpu_bo_vaddr(ipc->mem_rx), 0, ivpu_bo_size(ipc->mem_rx));
590	wmb(); /* Flush WC buffers for TX and RX rings */
591
592	mutex_unlock(&ipc->lock);
593}