ipc4.c

  1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
  2//
  3// This file is provided under a dual BSD/GPLv2 license.  When using or
  4// redistributing this file, you may do so under either license.
  5//
  6// Copyright(c) 2022 Intel Corporation
  7//
  8// Authors: Rander Wang <rander.wang@linux.intel.com>
  9//	    Peter Ujfalusi <peter.ujfalusi@linux.intel.com>
 10//
 11#include <linux/firmware.h>
 12#include <sound/sof/header.h>
 13#include <sound/sof/ipc4/header.h>
 14#include "sof-priv.h"
 15#include "sof-audio.h"
 16#include "ipc4-fw-reg.h"
 17#include "ipc4-priv.h"
 18#include "ipc4-topology.h"
 19#include "ipc4-telemetry.h"
 20#include "ops.h"
 21
 22static const struct sof_ipc4_fw_status {
 23	int status;
 24	char *msg;
 25} ipc4_status[] = {
 26	{0, "The operation was successful"},
 27	{1, "Invalid parameter specified"},
 28	{2, "Unknown message type specified"},
 29	{3, "Not enough space in the IPC reply buffer to complete the request"},
 30	{4, "The system or resource is busy"},
 31	{5, "Replaced ADSP IPC PENDING (unused)"},
 32	{6, "Unknown error while processing the request"},
 33	{7, "Unsupported operation requested"},
 34	{8, "Reserved (ADSP_STAGE_UNINITIALIZED removed)"},
 35	{9, "Specified resource not found"},
 36	{10, "A resource's ID requested to be created is already assigned"},
 37	{11, "Reserved (ADSP_IPC_OUT_OF_MIPS removed)"},
 38	{12, "Required resource is in invalid state"},
 39	{13, "Requested power transition failed to complete"},
 40	{14, "Manifest of the library being loaded is invalid"},
 41	{15, "Requested service or data is unavailable on the target platform"},
 42	{42, "Library target address is out of storage memory range"},
 43	{43, "Reserved"},
 44	{44, "Image verification by CSE failed"},
 45	{100, "General module management error"},
 46	{101, "Module loading failed"},
 47	{102, "Integrity check of the loaded module content failed"},
 48	{103, "Attempt to unload code of the module in use"},
 49	{104, "Other failure of module instance initialization request"},
 50	{105, "Reserved (ADSP_IPC_OUT_OF_MIPS removed)"},
 51	{106, "Reserved (ADSP_IPC_CONFIG_GET_ERROR removed)"},
 52	{107, "Reserved (ADSP_IPC_CONFIG_SET_ERROR removed)"},
 53	{108, "Reserved (ADSP_IPC_LARGE_CONFIG_GET_ERROR removed)"},
 54	{109, "Reserved (ADSP_IPC_LARGE_CONFIG_SET_ERROR removed)"},
 55	{110, "Invalid (out of range) module ID provided"},
 56	{111, "Invalid module instance ID provided"},
 57	{112, "Invalid queue (pin) ID provided"},
 58	{113, "Invalid destination queue (pin) ID provided"},
 59	{114, "Reserved (ADSP_IPC_BIND_UNBIND_DST_SINK_UNSUPPORTED removed)"},
 60	{115, "Reserved (ADSP_IPC_UNLOAD_INST_EXISTS removed)"},
 61	{116, "Invalid target code ID provided"},
 62	{117, "Injection DMA buffer is too small for probing the input pin"},
 63	{118, "Extraction DMA buffer is too small for probing the output pin"},
 64	{120, "Invalid ID of configuration item provided in TLV list"},
 65	{121, "Invalid length of configuration item provided in TLV list"},
 66	{122, "Invalid structure of configuration item provided"},
 67	{140, "Initialization of DMA Gateway failed"},
 68	{141, "Invalid ID of gateway provided"},
 69	{142, "Setting state of DMA Gateway failed"},
 70	{143, "DMA_CONTROL message targeting gateway not allocated yet"},
 71	{150, "Attempt to configure SCLK while I2S port is running"},
 72	{151, "Attempt to configure MCLK while I2S port is running"},
 73	{152, "Attempt to stop SCLK that is not running"},
 74	{153, "Attempt to stop MCLK that is not running"},
 75	{160, "Reserved (ADSP_IPC_PIPELINE_NOT_INITIALIZED removed)"},
 76	{161, "Reserved (ADSP_IPC_PIPELINE_NOT_EXIST removed)"},
 77	{162, "Reserved (ADSP_IPC_PIPELINE_SAVE_FAILED removed)"},
 78	{163, "Reserved (ADSP_IPC_PIPELINE_RESTORE_FAILED removed)"},
 79	{165, "Reserved (ADSP_IPC_PIPELINE_ALREADY_EXISTS removed)"},
 80};
 81
 82typedef void (*ipc4_notification_handler)(struct snd_sof_dev *sdev,
 83					  struct sof_ipc4_msg *msg);
 84
 85static int sof_ipc4_check_reply_status(struct snd_sof_dev *sdev, u32 status)
 86{
 87	int i, ret;
 88
 89	status &= SOF_IPC4_REPLY_STATUS;
 90
 91	if (!status)
 92		return 0;
 93
 94	for (i = 0; i < ARRAY_SIZE(ipc4_status); i++) {
 95		if (ipc4_status[i].status == status) {
 96			dev_err(sdev->dev, "FW reported error: %u - %s\n",
 97				status, ipc4_status[i].msg);
 98			goto to_errno;
 99		}
100	}
101
102	if (i == ARRAY_SIZE(ipc4_status))
103		dev_err(sdev->dev, "FW reported error: %u - Unknown\n", status);
104
105to_errno:
106	switch (status) {
107	case 2:
108	case 15:
109		ret = -EOPNOTSUPP;
110		break;
111	case 8:
112	case 11:
113	case 105 ... 109:
114	case 114 ... 115:
115	case 160 ... 163:
116	case 165:
117		ret = -ENOENT;
118		break;
119	case 4:
120	case 150:
121	case 151:
122		ret = -EBUSY;
123		break;
124	default:
125		ret = -EINVAL;
126		break;
127	}
128
129	return ret;
130}
131
132#if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_VERBOSE_IPC)
133#define DBG_IPC4_MSG_TYPE_ENTRY(type)	[SOF_IPC4_##type] = #type
134static const char * const ipc4_dbg_mod_msg_type[] = {
135	DBG_IPC4_MSG_TYPE_ENTRY(MOD_INIT_INSTANCE),
136	DBG_IPC4_MSG_TYPE_ENTRY(MOD_CONFIG_GET),
137	DBG_IPC4_MSG_TYPE_ENTRY(MOD_CONFIG_SET),
138	DBG_IPC4_MSG_TYPE_ENTRY(MOD_LARGE_CONFIG_GET),
139	DBG_IPC4_MSG_TYPE_ENTRY(MOD_LARGE_CONFIG_SET),
140	DBG_IPC4_MSG_TYPE_ENTRY(MOD_BIND),
141	DBG_IPC4_MSG_TYPE_ENTRY(MOD_UNBIND),
142	DBG_IPC4_MSG_TYPE_ENTRY(MOD_SET_DX),
143	DBG_IPC4_MSG_TYPE_ENTRY(MOD_SET_D0IX),
144	DBG_IPC4_MSG_TYPE_ENTRY(MOD_ENTER_MODULE_RESTORE),
145	DBG_IPC4_MSG_TYPE_ENTRY(MOD_EXIT_MODULE_RESTORE),
146	DBG_IPC4_MSG_TYPE_ENTRY(MOD_DELETE_INSTANCE),
147};
148
149static const char * const ipc4_dbg_glb_msg_type[] = {
150	DBG_IPC4_MSG_TYPE_ENTRY(GLB_BOOT_CONFIG),
151	DBG_IPC4_MSG_TYPE_ENTRY(GLB_ROM_CONTROL),
152	DBG_IPC4_MSG_TYPE_ENTRY(GLB_IPCGATEWAY_CMD),
153	DBG_IPC4_MSG_TYPE_ENTRY(GLB_PERF_MEASUREMENTS_CMD),
154	DBG_IPC4_MSG_TYPE_ENTRY(GLB_CHAIN_DMA),
155	DBG_IPC4_MSG_TYPE_ENTRY(GLB_LOAD_MULTIPLE_MODULES),
156	DBG_IPC4_MSG_TYPE_ENTRY(GLB_UNLOAD_MULTIPLE_MODULES),
157	DBG_IPC4_MSG_TYPE_ENTRY(GLB_CREATE_PIPELINE),
158	DBG_IPC4_MSG_TYPE_ENTRY(GLB_DELETE_PIPELINE),
159	DBG_IPC4_MSG_TYPE_ENTRY(GLB_SET_PIPELINE_STATE),
160	DBG_IPC4_MSG_TYPE_ENTRY(GLB_GET_PIPELINE_STATE),
161	DBG_IPC4_MSG_TYPE_ENTRY(GLB_GET_PIPELINE_CONTEXT_SIZE),
162	DBG_IPC4_MSG_TYPE_ENTRY(GLB_SAVE_PIPELINE),
163	DBG_IPC4_MSG_TYPE_ENTRY(GLB_RESTORE_PIPELINE),
164	DBG_IPC4_MSG_TYPE_ENTRY(GLB_LOAD_LIBRARY),
165	DBG_IPC4_MSG_TYPE_ENTRY(GLB_LOAD_LIBRARY_PREPARE),
166	DBG_IPC4_MSG_TYPE_ENTRY(GLB_INTERNAL_MESSAGE),
167	DBG_IPC4_MSG_TYPE_ENTRY(GLB_NOTIFICATION),
168};
169
170#define DBG_IPC4_NOTIFICATION_TYPE_ENTRY(type)	[SOF_IPC4_NOTIFY_##type] = #type
171static const char * const ipc4_dbg_notification_type[] = {
172	DBG_IPC4_NOTIFICATION_TYPE_ENTRY(PHRASE_DETECTED),
173	DBG_IPC4_NOTIFICATION_TYPE_ENTRY(RESOURCE_EVENT),
174	DBG_IPC4_NOTIFICATION_TYPE_ENTRY(LOG_BUFFER_STATUS),
175	DBG_IPC4_NOTIFICATION_TYPE_ENTRY(TIMESTAMP_CAPTURED),
176	DBG_IPC4_NOTIFICATION_TYPE_ENTRY(FW_READY),
177	DBG_IPC4_NOTIFICATION_TYPE_ENTRY(FW_AUD_CLASS_RESULT),
178	DBG_IPC4_NOTIFICATION_TYPE_ENTRY(EXCEPTION_CAUGHT),
179	DBG_IPC4_NOTIFICATION_TYPE_ENTRY(MODULE_NOTIFICATION),
180	DBG_IPC4_NOTIFICATION_TYPE_ENTRY(PROBE_DATA_AVAILABLE),
181	DBG_IPC4_NOTIFICATION_TYPE_ENTRY(ASYNC_MSG_SRVC_MESSAGE),
182};
183
184static void sof_ipc4_log_header(struct device *dev, u8 *text, struct sof_ipc4_msg *msg,
185				bool data_size_valid)
186{
187	u32 val, type;
188	const u8 *str2 = NULL;
189	const u8 *str = NULL;
190
191	val = msg->primary & SOF_IPC4_MSG_TARGET_MASK;
192	type = SOF_IPC4_MSG_TYPE_GET(msg->primary);
193
194	if (val == SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG)) {
195		/* Module message */
196		if (type < SOF_IPC4_MOD_TYPE_LAST)
197			str = ipc4_dbg_mod_msg_type[type];
198		if (!str)
199			str = "Unknown Module message type";
200	} else {
201		/* Global FW message */
202		if (type < SOF_IPC4_GLB_TYPE_LAST)
203			str = ipc4_dbg_glb_msg_type[type];
204		if (!str)
205			str = "Unknown Global message type";
206
207		if (type == SOF_IPC4_GLB_NOTIFICATION) {
208			/* Notification message */
209			u32 notif = SOF_IPC4_NOTIFICATION_TYPE_GET(msg->primary);
210
211			/* Do not print log buffer notification if not desired */
212			if (notif == SOF_IPC4_NOTIFY_LOG_BUFFER_STATUS &&
213			    !sof_debug_check_flag(SOF_DBG_PRINT_DMA_POSITION_UPDATE_LOGS))
214				return;
215
216			if (notif < SOF_IPC4_NOTIFY_TYPE_LAST)
217				str2 = ipc4_dbg_notification_type[notif];
218			if (!str2)
219				str2 = "Unknown Global notification";
220		}
221	}
222
223	if (str2) {
224		if (data_size_valid && msg->data_size)
225			dev_dbg(dev, "%s: %#x|%#x: %s|%s [data size: %zu]\n",
226				text, msg->primary, msg->extension, str, str2,
227				msg->data_size);
228		else
229			dev_dbg(dev, "%s: %#x|%#x: %s|%s\n", text, msg->primary,
230				msg->extension, str, str2);
231	} else {
232		if (data_size_valid && msg->data_size)
233			dev_dbg(dev, "%s: %#x|%#x: %s [data size: %zu]\n",
234				text, msg->primary, msg->extension, str,
235				msg->data_size);
236		else
237			dev_dbg(dev, "%s: %#x|%#x: %s\n", text, msg->primary,
238				msg->extension, str);
239	}
240}
241
242const char *sof_ipc4_pipeline_state_str(enum sof_ipc4_pipeline_state state)
243{
244	switch (state) {
245	case SOF_IPC4_PIPE_INVALID_STATE:
246		return " (INVALID_STATE)";
247	case SOF_IPC4_PIPE_UNINITIALIZED:
248		return " (UNINITIALIZED)";
249	case SOF_IPC4_PIPE_RESET:
250		return " (RESET)";
251	case SOF_IPC4_PIPE_PAUSED:
252		return " (PAUSED)";
253	case SOF_IPC4_PIPE_RUNNING:
254		return " (RUNNING)";
255	case SOF_IPC4_PIPE_EOS:
256		return " (EOS)";
257	default:
258		return " (<unknown>)";
259	}
260}
261#else /* CONFIG_SND_SOC_SOF_DEBUG_VERBOSE_IPC */
262static void sof_ipc4_log_header(struct device *dev, u8 *text, struct sof_ipc4_msg *msg,
263				bool data_size_valid)
264{
265	/* Do not print log buffer notification if not desired */
266	if (!sof_debug_check_flag(SOF_DBG_PRINT_DMA_POSITION_UPDATE_LOGS) &&
267	    !SOF_IPC4_MSG_IS_MODULE_MSG(msg->primary) &&
268	    SOF_IPC4_MSG_TYPE_GET(msg->primary) == SOF_IPC4_GLB_NOTIFICATION &&
269	    SOF_IPC4_NOTIFICATION_TYPE_GET(msg->primary) == SOF_IPC4_NOTIFY_LOG_BUFFER_STATUS)
270		return;
271
272	if (data_size_valid && msg->data_size)
273		dev_dbg(dev, "%s: %#x|%#x [data size: %zu]\n", text,
274			msg->primary, msg->extension, msg->data_size);
275	else
276		dev_dbg(dev, "%s: %#x|%#x\n", text, msg->primary, msg->extension);
277}
278
279const char *sof_ipc4_pipeline_state_str(enum sof_ipc4_pipeline_state state)
280{
281	return "";
282}
283#endif
284
285static void sof_ipc4_dump_payload(struct snd_sof_dev *sdev,
286				  void *ipc_data, size_t size)
287{
288	print_hex_dump_debug("Message payload: ", DUMP_PREFIX_OFFSET,
289			     16, 4, ipc_data, size, false);
290}
291
292static int sof_ipc4_get_reply(struct snd_sof_dev *sdev)
293{
294	struct snd_sof_ipc_msg *msg = sdev->msg;
295	struct sof_ipc4_msg *ipc4_reply;
296	int ret;
297
298	/* get the generic reply */
299	ipc4_reply = msg->reply_data;
300
301	sof_ipc4_log_header(sdev->dev, "ipc tx reply", ipc4_reply, false);
302
303	ret = sof_ipc4_check_reply_status(sdev, ipc4_reply->primary);
304	if (ret)
305		return ret;
306
307	/* No other information is expected for non large config get replies */
308	if (!msg->reply_size || !SOF_IPC4_MSG_IS_MODULE_MSG(ipc4_reply->primary) ||
309	    (SOF_IPC4_MSG_TYPE_GET(ipc4_reply->primary) != SOF_IPC4_MOD_LARGE_CONFIG_GET))
310		return 0;
311
312	/* Read the requested payload */
313	snd_sof_dsp_mailbox_read(sdev, sdev->dsp_box.offset, ipc4_reply->data_ptr,
314				 msg->reply_size);
315
316	return 0;
317}
318
319/* wait for IPC message reply */
320static int ipc4_wait_tx_done(struct snd_sof_ipc *ipc, void *reply_data)
321{
322	struct snd_sof_ipc_msg *msg = &ipc->msg;
323	struct sof_ipc4_msg *ipc4_msg = msg->msg_data;
324	struct snd_sof_dev *sdev = ipc->sdev;
325	int ret;
326
327	/* wait for DSP IPC completion */
328	ret = wait_event_timeout(msg->waitq, msg->ipc_complete,
329				 msecs_to_jiffies(sdev->ipc_timeout));
330	if (ret == 0) {
331		dev_err(sdev->dev, "ipc timed out for %#x|%#x\n",
332			ipc4_msg->primary, ipc4_msg->extension);
333		snd_sof_handle_fw_exception(ipc->sdev, "IPC timeout");
334		return -ETIMEDOUT;
335	}
336
337	if (msg->reply_error) {
338		dev_err(sdev->dev, "ipc error for msg %#x|%#x\n",
339			ipc4_msg->primary, ipc4_msg->extension);
340		ret =  msg->reply_error;
341	} else {
342		if (reply_data) {
343			struct sof_ipc4_msg *ipc4_reply = msg->reply_data;
344			struct sof_ipc4_msg *ipc4_reply_data = reply_data;
345
346			/* Copy the header */
347			ipc4_reply_data->header_u64 = ipc4_reply->header_u64;
348			if (msg->reply_size && ipc4_reply_data->data_ptr) {
349				/* copy the payload returned from DSP */
350				memcpy(ipc4_reply_data->data_ptr, ipc4_reply->data_ptr,
351				       msg->reply_size);
352				ipc4_reply_data->data_size = msg->reply_size;
353			}
354		}
355
356		ret = 0;
357		sof_ipc4_log_header(sdev->dev, "ipc tx done ", ipc4_msg, true);
358	}
359
360	/* re-enable dumps after successful IPC tx */
361	if (sdev->ipc_dump_printed) {
362		sdev->dbg_dump_printed = false;
363		sdev->ipc_dump_printed = false;
364	}
365
366	return ret;
367}
368
369static int ipc4_tx_msg_unlocked(struct snd_sof_ipc *ipc,
370				void *msg_data, size_t msg_bytes,
371				void *reply_data, size_t reply_bytes)
372{
373	struct sof_ipc4_msg *ipc4_msg = msg_data;
374	struct snd_sof_dev *sdev = ipc->sdev;
375	int ret;
376
377	if (msg_bytes > ipc->max_payload_size || reply_bytes > ipc->max_payload_size)
378		return -EINVAL;
379
380	sof_ipc4_log_header(sdev->dev, "ipc tx      ", msg_data, true);
381
382	ret = sof_ipc_send_msg(sdev, msg_data, msg_bytes, reply_bytes);
383	if (ret) {
384		dev_err_ratelimited(sdev->dev,
385				    "%s: ipc message send for %#x|%#x failed: %d\n",
386				    __func__, ipc4_msg->primary, ipc4_msg->extension, ret);
387		return ret;
388	}
389
390	/* now wait for completion */
391	return ipc4_wait_tx_done(ipc, reply_data);
392}
393
394static int sof_ipc4_tx_msg(struct snd_sof_dev *sdev, void *msg_data, size_t msg_bytes,
395			   void *reply_data, size_t reply_bytes, bool no_pm)
396{
397	struct snd_sof_ipc *ipc = sdev->ipc;
398	int ret;
399
400	if (!msg_data)
401		return -EINVAL;
402
403	if (!no_pm) {
404		const struct sof_dsp_power_state target_state = {
405			.state = SOF_DSP_PM_D0,
406		};
407
408		/* ensure the DSP is in D0i0 before sending a new IPC */
409		ret = snd_sof_dsp_set_power_state(sdev, &target_state);
410		if (ret < 0)
411			return ret;
412	}
413
414	/* Serialise IPC TX */
415	mutex_lock(&ipc->tx_mutex);
416
417	ret = ipc4_tx_msg_unlocked(ipc, msg_data, msg_bytes, reply_data, reply_bytes);
418
419	if (sof_debug_check_flag(SOF_DBG_DUMP_IPC_MESSAGE_PAYLOAD)) {
420		struct sof_ipc4_msg *msg = NULL;
421
422		/* payload is indicated by non zero msg/reply_bytes */
423		if (msg_bytes)
424			msg = msg_data;
425		else if (reply_bytes)
426			msg = reply_data;
427
428		if (msg)
429			sof_ipc4_dump_payload(sdev, msg->data_ptr, msg->data_size);
430	}
431
432	mutex_unlock(&ipc->tx_mutex);
433
434	return ret;
435}
436
437static bool sof_ipc4_tx_payload_for_get_data(struct sof_ipc4_msg *tx)
438{
439	/*
440	 * Messages that require TX payload with LARGE_CONFIG_GET.
441	 * The TX payload is placed into the IPC message data section by caller,
442	 * which needs to be copied to temporary buffer since the received data
443	 * will overwrite it.
444	 */
445	switch (tx->extension & SOF_IPC4_MOD_EXT_MSG_PARAM_ID_MASK) {
446	case SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_SWITCH_CONTROL_PARAM_ID):
447	case SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_ENUM_CONTROL_PARAM_ID):
448		return true;
449	default:
450		return false;
451	}
452}
453
454static int sof_ipc4_set_get_data(struct snd_sof_dev *sdev, void *data,
455				 size_t payload_bytes, bool set)
456{
457	const struct sof_dsp_power_state target_state = {
458			.state = SOF_DSP_PM_D0,
459	};
460	size_t payload_limit = sdev->ipc->max_payload_size;
461	struct sof_ipc4_msg *ipc4_msg = data;
462	struct sof_ipc4_msg tx = {{ 0 }};
463	struct sof_ipc4_msg rx = {{ 0 }};
464	size_t remaining = payload_bytes;
465	void *tx_payload_for_get = NULL;
466	size_t tx_data_size = 0;
467	size_t offset = 0;
468	size_t chunk_size;
469	int ret;
470
471	if (!data)
472		return -EINVAL;
473
474	if ((ipc4_msg->primary & SOF_IPC4_MSG_TARGET_MASK) !=
475	    SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG))
476		return -EINVAL;
477
478	ipc4_msg->primary &= ~SOF_IPC4_MSG_TYPE_MASK;
479	tx.primary = ipc4_msg->primary;
480	tx.extension = ipc4_msg->extension;
481
482	if (set)
483		tx.primary |= SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_MOD_LARGE_CONFIG_SET);
484	else
485		tx.primary |= SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_MOD_LARGE_CONFIG_GET);
486
487	tx.extension &= ~SOF_IPC4_MOD_EXT_MSG_SIZE_MASK;
488	tx.extension |= SOF_IPC4_MOD_EXT_MSG_SIZE(payload_bytes);
489
490	tx.extension |= SOF_IPC4_MOD_EXT_MSG_FIRST_BLOCK(1);
491
492	if (sof_ipc4_tx_payload_for_get_data(&tx)) {
493		tx_data_size = min(ipc4_msg->data_size, payload_limit);
494		tx_payload_for_get = kmemdup(ipc4_msg->data_ptr, tx_data_size,
495					     GFP_KERNEL);
496		if (!tx_payload_for_get)
497			return -ENOMEM;
498	}
499
500	/* ensure the DSP is in D0i0 before sending IPC */
501	ret = snd_sof_dsp_set_power_state(sdev, &target_state);
502	if (ret < 0) {
503		kfree(tx_payload_for_get);
504		return ret;
505	}
506
507	/* Serialise IPC TX */
508	mutex_lock(&sdev->ipc->tx_mutex);
509
510	do {
511		size_t tx_size, rx_size;
512
513		if (remaining > payload_limit) {
514			chunk_size = payload_limit;
515		} else {
516			chunk_size = remaining;
517			if (set)
518				tx.extension |= SOF_IPC4_MOD_EXT_MSG_LAST_BLOCK(1);
519		}
520
521		if (offset) {
522			tx.extension &= ~SOF_IPC4_MOD_EXT_MSG_FIRST_BLOCK_MASK;
523			tx.extension &= ~SOF_IPC4_MOD_EXT_MSG_SIZE_MASK;
524			tx.extension |= SOF_IPC4_MOD_EXT_MSG_SIZE(offset);
525		}
526
527		if (set) {
528			tx.data_size = chunk_size;
529			tx.data_ptr = ipc4_msg->data_ptr + offset;
530
531			tx_size = chunk_size;
532			rx_size = 0;
533		} else {
534			rx.primary = 0;
535			rx.extension = 0;
536			rx.data_size = chunk_size;
537			rx.data_ptr = ipc4_msg->data_ptr + offset;
538
539			if (tx_payload_for_get) {
540				tx_size = tx_data_size;
541				tx.data_size = tx_size;
542				tx.data_ptr = tx_payload_for_get;
543			} else {
544				tx_size = 0;
545				tx.data_size = 0;
546				tx.data_ptr = NULL;
547			}
548			rx_size = chunk_size;
549		}
550
551		/* Send the message for the current chunk */
552		ret = ipc4_tx_msg_unlocked(sdev->ipc, &tx, tx_size, &rx, rx_size);
553		if (ret < 0) {
554			dev_err(sdev->dev,
555				"%s: large config %s failed at offset %zu: %d\n",
556				__func__, set ? "set" : "get", offset, ret);
557			goto out;
558		}
559
560		if (!set && rx.extension & SOF_IPC4_MOD_EXT_MSG_FIRST_BLOCK_MASK) {
561			/* Verify the firmware reported total payload size */
562			rx_size = rx.extension & SOF_IPC4_MOD_EXT_MSG_SIZE_MASK;
563
564			if (rx_size > payload_bytes) {
565				dev_err(sdev->dev,
566					"%s: Receive buffer (%zu) is too small for %zu\n",
567					__func__, payload_bytes, rx_size);
568				ret = -ENOMEM;
569				goto out;
570			}
571
572			if (rx_size < chunk_size) {
573				chunk_size = rx_size;
574				remaining = rx_size;
575			} else if (rx_size < payload_bytes) {
576				remaining = rx_size;
577			}
578		}
579
580		offset += chunk_size;
581		remaining -= chunk_size;
582	} while (remaining);
583
584	/* Adjust the received data size if needed */
585	if (!set && payload_bytes != offset)
586		ipc4_msg->data_size = offset;
587
588out:
589	if (sof_debug_check_flag(SOF_DBG_DUMP_IPC_MESSAGE_PAYLOAD))
590		sof_ipc4_dump_payload(sdev, ipc4_msg->data_ptr, ipc4_msg->data_size);
591
592	mutex_unlock(&sdev->ipc->tx_mutex);
593
594	kfree(tx_payload_for_get);
595
596	return ret;
597}
598
599static int sof_ipc4_init_msg_memory(struct snd_sof_dev *sdev)
600{
601	struct sof_ipc4_msg *ipc4_msg;
602	struct snd_sof_ipc_msg *msg = &sdev->ipc->msg;
603
604	/* TODO: get max_payload_size from firmware */
605	sdev->ipc->max_payload_size = SOF_IPC4_MSG_MAX_SIZE;
606
607	/* Allocate memory for the ipc4 container and the maximum payload */
608	msg->reply_data = devm_kzalloc(sdev->dev, sdev->ipc->max_payload_size +
609				       sizeof(struct sof_ipc4_msg), GFP_KERNEL);
610	if (!msg->reply_data)
611		return -ENOMEM;
612
613	ipc4_msg = msg->reply_data;
614	ipc4_msg->data_ptr = msg->reply_data + sizeof(struct sof_ipc4_msg);
615
616	return 0;
617}
618
619size_t sof_ipc4_find_debug_slot_offset_by_type(struct snd_sof_dev *sdev,
620					       u32 slot_type)
621{
622	size_t slot_desc_type_offset;
623	u32 type;
624	int i;
625
626	/* The type is the second u32 in the slot descriptor */
627	slot_desc_type_offset = sdev->debug_box.offset + sizeof(u32);
628	for (i = 0; i < SOF_IPC4_MAX_DEBUG_SLOTS; i++) {
629		sof_mailbox_read(sdev, slot_desc_type_offset, &type, sizeof(type));
630
631		if (type == slot_type)
632			return sdev->debug_box.offset + (i + 1) * SOF_IPC4_DEBUG_SLOT_SIZE;
633
634		slot_desc_type_offset += SOF_IPC4_DEBUG_DESCRIPTOR_SIZE;
635	}
636
637	dev_dbg(sdev->dev, "Slot type %#x is not available in debug window\n", slot_type);
638	return 0;
639}
640EXPORT_SYMBOL(sof_ipc4_find_debug_slot_offset_by_type);
641
642static int ipc4_fw_ready(struct snd_sof_dev *sdev, struct sof_ipc4_msg *ipc4_msg)
643{
644	if (!sdev->first_boot) {
645		struct sof_ipc4_fw_data *ipc4_data = sdev->private;
646
647		/*
648		 * After the initial boot only check if the libraries have been
649		 * restored when full context save is not enabled
650		 */
651		if (!ipc4_data->fw_context_save)
652			ipc4_data->libraries_restored = !!(ipc4_msg->primary &
653							   SOF_IPC4_FW_READY_LIB_RESTORED);
654
655		return 0;
656	}
657
658	sof_ipc4_create_exception_debugfs_node(sdev);
659
660	return sof_ipc4_init_msg_memory(sdev);
661}
662
663static void sof_ipc4_module_notification_handler(struct snd_sof_dev *sdev,
664						 struct sof_ipc4_msg *ipc4_msg)
665{
666	struct sof_ipc4_notify_module_data *data = ipc4_msg->data_ptr;
667
668	/*
669	 * If the notification includes additional, module specific data, then
670	 * we need to re-allocate the buffer and re-read the whole payload,
671	 * including the event_data
672	 */
673	if (data->event_data_size) {
674		void *new;
675		int ret;
676
677		ipc4_msg->data_size += data->event_data_size;
678
679		new = krealloc(ipc4_msg->data_ptr, ipc4_msg->data_size, GFP_KERNEL);
680		if (!new) {
681			ipc4_msg->data_size -= data->event_data_size;
682			return;
683		}
684
685		/* re-read the whole payload */
686		ipc4_msg->data_ptr = new;
687		ret = snd_sof_ipc_msg_data(sdev, NULL, ipc4_msg->data_ptr,
688					   ipc4_msg->data_size);
689		if (ret < 0) {
690			dev_err(sdev->dev,
691				"Failed to read the full module notification: %d\n",
692				ret);
693			return;
694		}
695		data = ipc4_msg->data_ptr;
696	}
697
698	/* Handle ALSA kcontrol notification */
699	if ((data->event_id & SOF_IPC4_NOTIFY_MODULE_EVENTID_ALSA_MAGIC_MASK) ==
700	    SOF_IPC4_NOTIFY_MODULE_EVENTID_ALSA_MAGIC_VAL) {
701		const struct sof_ipc_tplg_ops *tplg_ops = sdev->ipc->ops->tplg;
702
703		if (tplg_ops->control->update)
704			tplg_ops->control->update(sdev, ipc4_msg);
705	}
706}
707
708static void sof_ipc4_rx_msg(struct snd_sof_dev *sdev)
709{
710	struct sof_ipc4_msg *ipc4_msg = sdev->ipc->msg.rx_data;
711	ipc4_notification_handler handler_func = NULL;
712	size_t data_size = 0;
713	int err;
714
715	if (!ipc4_msg || !SOF_IPC4_MSG_IS_NOTIFICATION(ipc4_msg->primary))
716		return;
717
718	ipc4_msg->data_ptr = NULL;
719	ipc4_msg->data_size = 0;
720
721	sof_ipc4_log_header(sdev->dev, "ipc rx      ", ipc4_msg, false);
722
723	switch (SOF_IPC4_NOTIFICATION_TYPE_GET(ipc4_msg->primary)) {
724	case SOF_IPC4_NOTIFY_FW_READY:
725		/* check for FW boot completion */
726		if (sdev->fw_state == SOF_FW_BOOT_IN_PROGRESS) {
727			err = ipc4_fw_ready(sdev, ipc4_msg);
728			if (err < 0)
729				sof_set_fw_state(sdev, SOF_FW_BOOT_READY_FAILED);
730			else
731				sof_set_fw_state(sdev, SOF_FW_BOOT_READY_OK);
732
733			/* wake up firmware loader */
734			wake_up(&sdev->boot_wait);
735		}
736
737		break;
738	case SOF_IPC4_NOTIFY_RESOURCE_EVENT:
739		data_size = sizeof(struct sof_ipc4_notify_resource_data);
740		break;
741	case SOF_IPC4_NOTIFY_LOG_BUFFER_STATUS:
742		sof_ipc4_mtrace_update_pos(sdev, SOF_IPC4_LOG_CORE_GET(ipc4_msg->primary));
743		break;
744	case SOF_IPC4_NOTIFY_EXCEPTION_CAUGHT:
745		snd_sof_dsp_panic(sdev, 0, true);
746		break;
747	case SOF_IPC4_NOTIFY_MODULE_NOTIFICATION:
748		data_size = sizeof(struct sof_ipc4_notify_module_data);
749		handler_func = sof_ipc4_module_notification_handler;
750		break;
751	default:
752		dev_dbg(sdev->dev, "Unhandled DSP message: %#x|%#x\n",
753			ipc4_msg->primary, ipc4_msg->extension);
754		break;
755	}
756
757	if (data_size) {
758		ipc4_msg->data_ptr = kmalloc(data_size, GFP_KERNEL);
759		if (!ipc4_msg->data_ptr)
760			return;
761
762		ipc4_msg->data_size = data_size;
763		err = snd_sof_ipc_msg_data(sdev, NULL, ipc4_msg->data_ptr, ipc4_msg->data_size);
764		if (err < 0) {
765			dev_err(sdev->dev, "failed to read IPC notification data: %d\n", err);
766			kfree(ipc4_msg->data_ptr);
767			ipc4_msg->data_ptr = NULL;
768			ipc4_msg->data_size = 0;
769			return;
770		}
771	}
772
773	/* Handle notifications with payload */
774	if (handler_func)
775		handler_func(sdev, ipc4_msg);
776
777	sof_ipc4_log_header(sdev->dev, "ipc rx done ", ipc4_msg, true);
778
779	if (data_size) {
780		if (sof_debug_check_flag(SOF_DBG_DUMP_IPC_MESSAGE_PAYLOAD))
781			sof_ipc4_dump_payload(sdev, ipc4_msg->data_ptr,
782					      ipc4_msg->data_size);
783
784		kfree(ipc4_msg->data_ptr);
785		ipc4_msg->data_ptr = NULL;
786		ipc4_msg->data_size = 0;
787	}
788}
789
790static int sof_ipc4_set_core_state(struct snd_sof_dev *sdev, int core_idx, bool on)
791{
792	struct sof_ipc4_dx_state_info dx_state;
793	struct sof_ipc4_msg msg;
794
795	dx_state.core_mask = BIT(core_idx);
796	if (on)
797		dx_state.dx_mask = BIT(core_idx);
798	else
799		dx_state.dx_mask = 0;
800
801	msg.primary = SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_MOD_SET_DX);
802	msg.primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
803	msg.primary |= SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG);
804	msg.extension = 0;
805	msg.data_ptr = &dx_state;
806	msg.data_size = sizeof(dx_state);
807
808	return sof_ipc4_tx_msg(sdev, &msg, msg.data_size, NULL, 0, false);
809}
810
811/*
812 * The context save callback is used to send a message to the firmware notifying
813 * it that the primary core is going to be turned off, which is used as an
814 * indication to prepare for a full power down, thus preparing for IMR boot
815 * (when supported)
816 *
817 * Note: in IPC4 there is no message used to restore context, thus no context
818 * restore callback is implemented
819 */
820static int sof_ipc4_ctx_save(struct snd_sof_dev *sdev)
821{
822	return sof_ipc4_set_core_state(sdev, SOF_DSP_PRIMARY_CORE, false);
823}
824
825static int sof_ipc4_set_pm_gate(struct snd_sof_dev *sdev, u32 flags)
826{
827	struct sof_ipc4_msg msg = {{0}};
828
829	msg.primary = SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_MOD_SET_D0IX);
830	msg.primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
831	msg.primary |= SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG);
832	msg.extension = flags;
833
834	return sof_ipc4_tx_msg(sdev, &msg, 0, NULL, 0, true);
835}
836
837static const struct sof_ipc_pm_ops ipc4_pm_ops = {
838	.ctx_save = sof_ipc4_ctx_save,
839	.set_core_state = sof_ipc4_set_core_state,
840	.set_pm_gate = sof_ipc4_set_pm_gate,
841};
842
843static int sof_ipc4_init(struct snd_sof_dev *sdev)
844{
845	struct sof_ipc4_fw_data *ipc4_data = sdev->private;
846	int inbox_offset;
847
848	mutex_init(&ipc4_data->pipeline_state_mutex);
849
850	xa_init_flags(&ipc4_data->fw_lib_xa, XA_FLAGS_ALLOC);
851
852	/* Set up the windows for IPC communication */
853	inbox_offset = snd_sof_dsp_get_mailbox_offset(sdev);
854	if (inbox_offset < 0) {
855		dev_err(sdev->dev, "%s: No mailbox offset\n", __func__);
856		return inbox_offset;
857	}
858
859	sdev->dsp_box.offset = inbox_offset;
860	sdev->dsp_box.size = SOF_IPC4_MSG_MAX_SIZE;
861	sdev->host_box.offset = snd_sof_dsp_get_window_offset(sdev,
862							SOF_IPC4_OUTBOX_WINDOW_IDX);
863	sdev->host_box.size = SOF_IPC4_MSG_MAX_SIZE;
864
865	sdev->debug_box.offset = snd_sof_dsp_get_window_offset(sdev,
866							SOF_IPC4_DEBUG_WINDOW_IDX);
867
868	sdev->fw_info_box.offset = snd_sof_dsp_get_window_offset(sdev,
869							SOF_IPC4_INBOX_WINDOW_IDX);
870	sdev->fw_info_box.size = sizeof(struct sof_ipc4_fw_registers);
871
872	dev_dbg(sdev->dev, "mailbox upstream %#x - size %#x\n",
873		sdev->dsp_box.offset, SOF_IPC4_MSG_MAX_SIZE);
874	dev_dbg(sdev->dev, "mailbox downstream %#x - size %#x\n",
875		sdev->host_box.offset, SOF_IPC4_MSG_MAX_SIZE);
876	dev_dbg(sdev->dev, "debug box %#x\n", sdev->debug_box.offset);
877
878	return 0;
879}
880
881static void sof_ipc4_exit(struct snd_sof_dev *sdev)
882{
883	struct sof_ipc4_fw_data *ipc4_data = sdev->private;
884	struct sof_ipc4_fw_library *fw_lib;
885	unsigned long lib_id;
886
887	xa_for_each(&ipc4_data->fw_lib_xa, lib_id, fw_lib) {
888		/*
889		 * The basefw (ID == 0) is handled by generic code, it is not
890		 * loaded by IPC4 code.
891		 */
892		if (lib_id != 0)
893			release_firmware(fw_lib->sof_fw.fw);
894
895		fw_lib->sof_fw.fw = NULL;
896	}
897
898	xa_destroy(&ipc4_data->fw_lib_xa);
899}
900
901static int sof_ipc4_post_boot(struct snd_sof_dev *sdev)
902{
903	if (sdev->first_boot) {
904		int  ret = sof_ipc4_complete_split_release(sdev);
905
906		if (ret)
907			return ret;
908
909		return sof_ipc4_query_fw_configuration(sdev);
910	}
911
912	return sof_ipc4_reload_fw_libraries(sdev);
913}
914
915const struct sof_ipc_ops ipc4_ops = {
916	.init = sof_ipc4_init,
917	.exit = sof_ipc4_exit,
918	.post_fw_boot = sof_ipc4_post_boot,
919	.tx_msg = sof_ipc4_tx_msg,
920	.rx_msg = sof_ipc4_rx_msg,
921	.set_get_data = sof_ipc4_set_get_data,
922	.get_reply = sof_ipc4_get_reply,
923	.pm = &ipc4_pm_ops,
924	.fw_loader = &ipc4_loader_ops,
925	.tplg = &ipc4_tplg_ops,
926	.pcm = &ipc4_pcm_ops,
927	.fw_tracing = &ipc4_mtrace_ops,
928};
929
930void sof_ipc4_mic_privacy_state_change(struct snd_sof_dev *sdev, bool state)
931{
932	struct sof_ipc4_msg msg;
933	u32 data = state;
934
935	msg.primary = SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG);
936	msg.primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
937	msg.primary |= SOF_IPC4_MOD_ID(SOF_IPC4_MOD_INIT_BASEFW_MOD_ID);
938	msg.primary |= SOF_IPC4_MOD_INSTANCE(SOF_IPC4_MOD_INIT_BASEFW_INSTANCE_ID);
939	msg.extension = SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_FW_PARAM_MIC_PRIVACY_STATE_CHANGE);
940
941	msg.data_size = sizeof(data);
942	msg.data_ptr = &data;
943
944	sof_ipc4_set_get_data(sdev, &msg, msg.data_size, true);
945}
946EXPORT_SYMBOL(sof_ipc4_mic_privacy_state_change);