rtkit.c

  1// SPDX-License-Identifier: GPL-2.0-only OR MIT
  2/*
  3 * Apple RTKit IPC library
  4 * Copyright (C) The Asahi Linux Contributors
  5 */
  6
  7#include "rtkit-internal.h"
  8
  9enum {
 10	APPLE_RTKIT_PWR_STATE_OFF = 0x00, /* power off, cannot be restarted */
 11	APPLE_RTKIT_PWR_STATE_SLEEP = 0x01, /* sleeping, can be restarted */
 12	APPLE_RTKIT_PWR_STATE_IDLE = 0x201, /* sleeping, retain state */
 13	APPLE_RTKIT_PWR_STATE_QUIESCED = 0x10, /* running but no communication */
 14	APPLE_RTKIT_PWR_STATE_ON = 0x20, /* normal operating state */
 15	APPLE_RTKIT_PWR_STATE_INIT = 0x220, /* init after starting the coproc */
 16};
 17
 18enum {
 19	APPLE_RTKIT_EP_MGMT = 0,
 20	APPLE_RTKIT_EP_CRASHLOG = 1,
 21	APPLE_RTKIT_EP_SYSLOG = 2,
 22	APPLE_RTKIT_EP_DEBUG = 3,
 23	APPLE_RTKIT_EP_IOREPORT = 4,
 24	APPLE_RTKIT_EP_OSLOG = 8,
 25};
 26
 27#define APPLE_RTKIT_MGMT_TYPE GENMASK_ULL(59, 52)
 28
 29enum {
 30	APPLE_RTKIT_MGMT_HELLO = 1,
 31	APPLE_RTKIT_MGMT_HELLO_REPLY = 2,
 32	APPLE_RTKIT_MGMT_STARTEP = 5,
 33	APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE = 6,
 34	APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE_ACK = 7,
 35	APPLE_RTKIT_MGMT_EPMAP = 8,
 36	APPLE_RTKIT_MGMT_EPMAP_REPLY = 8,
 37	APPLE_RTKIT_MGMT_SET_AP_PWR_STATE = 0xb,
 38	APPLE_RTKIT_MGMT_SET_AP_PWR_STATE_ACK = 0xb,
 39};
 40
 41#define APPLE_RTKIT_MGMT_HELLO_MINVER GENMASK_ULL(15, 0)
 42#define APPLE_RTKIT_MGMT_HELLO_MAXVER GENMASK_ULL(31, 16)
 43
 44#define APPLE_RTKIT_MGMT_EPMAP_LAST   BIT_ULL(51)
 45#define APPLE_RTKIT_MGMT_EPMAP_BASE   GENMASK_ULL(34, 32)
 46#define APPLE_RTKIT_MGMT_EPMAP_BITMAP GENMASK_ULL(31, 0)
 47
 48#define APPLE_RTKIT_MGMT_EPMAP_REPLY_MORE BIT_ULL(0)
 49
 50#define APPLE_RTKIT_MGMT_STARTEP_EP   GENMASK_ULL(39, 32)
 51#define APPLE_RTKIT_MGMT_STARTEP_FLAG BIT_ULL(1)
 52
 53#define APPLE_RTKIT_MGMT_PWR_STATE GENMASK_ULL(15, 0)
 54
 55#define APPLE_RTKIT_CRASHLOG_CRASH 1
 56
 57#define APPLE_RTKIT_BUFFER_REQUEST	1
 58#define APPLE_RTKIT_BUFFER_REQUEST_SIZE GENMASK_ULL(51, 44)
 59#define APPLE_RTKIT_BUFFER_REQUEST_IOVA GENMASK_ULL(43, 0)
 60
 61#define APPLE_RTKIT_SYSLOG_TYPE GENMASK_ULL(59, 52)
 62
 63#define APPLE_RTKIT_SYSLOG_LOG 5
 64
 65#define APPLE_RTKIT_SYSLOG_INIT	     8
 66#define APPLE_RTKIT_SYSLOG_N_ENTRIES GENMASK_ULL(7, 0)
 67#define APPLE_RTKIT_SYSLOG_MSG_SIZE  GENMASK_ULL(31, 24)
 68
 69#define APPLE_RTKIT_OSLOG_TYPE GENMASK_ULL(63, 56)
 70#define APPLE_RTKIT_OSLOG_BUFFER_REQUEST 1
 71#define APPLE_RTKIT_OSLOG_SIZE GENMASK_ULL(55, 36)
 72#define APPLE_RTKIT_OSLOG_IOVA GENMASK_ULL(35, 0)
 73
 74#define APPLE_RTKIT_MIN_SUPPORTED_VERSION 11
 75#define APPLE_RTKIT_MAX_SUPPORTED_VERSION 12
 76
 77struct apple_rtkit_rx_work {
 78	struct apple_rtkit *rtk;
 79	u8 ep;
 80	u64 msg;
 81	struct work_struct work;
 82};
 83
 84bool apple_rtkit_is_running(struct apple_rtkit *rtk)
 85{
 86	if (rtk->crashed)
 87		return false;
 88	if ((rtk->iop_power_state & 0xff) != APPLE_RTKIT_PWR_STATE_ON)
 89		return false;
 90	if ((rtk->ap_power_state & 0xff) != APPLE_RTKIT_PWR_STATE_ON)
 91		return false;
 92	return true;
 93}
 94EXPORT_SYMBOL_GPL(apple_rtkit_is_running);
 95
 96bool apple_rtkit_is_crashed(struct apple_rtkit *rtk)
 97{
 98	return rtk->crashed;
 99}
100EXPORT_SYMBOL_GPL(apple_rtkit_is_crashed);
101
102static int apple_rtkit_management_send(struct apple_rtkit *rtk, u8 type,
103					u64 msg)
104{
105	int ret;
106
107	msg &= ~APPLE_RTKIT_MGMT_TYPE;
108	msg |= FIELD_PREP(APPLE_RTKIT_MGMT_TYPE, type);
109	ret = apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_MGMT, msg, NULL, false);
110
111	if (ret)
112		dev_err(rtk->dev, "RTKit: Failed to send management message: %d\n", ret);
113
114	return ret;
115}
116
117static void apple_rtkit_management_rx_hello(struct apple_rtkit *rtk, u64 msg)
118{
119	u64 reply;
120
121	int min_ver = FIELD_GET(APPLE_RTKIT_MGMT_HELLO_MINVER, msg);
122	int max_ver = FIELD_GET(APPLE_RTKIT_MGMT_HELLO_MAXVER, msg);
123	int want_ver = min(APPLE_RTKIT_MAX_SUPPORTED_VERSION, max_ver);
124
125	dev_dbg(rtk->dev, "RTKit: Min ver %d, max ver %d\n", min_ver, max_ver);
126
127	if (min_ver > APPLE_RTKIT_MAX_SUPPORTED_VERSION) {
128		dev_err(rtk->dev, "RTKit: Firmware min version %d is too new\n",
129			min_ver);
130		goto abort_boot;
131	}
132
133	if (max_ver < APPLE_RTKIT_MIN_SUPPORTED_VERSION) {
134		dev_err(rtk->dev, "RTKit: Firmware max version %d is too old\n",
135			max_ver);
136		goto abort_boot;
137	}
138
139	dev_info(rtk->dev, "RTKit: Initializing (protocol version %d)\n",
140		 want_ver);
141	rtk->version = want_ver;
142
143	reply = FIELD_PREP(APPLE_RTKIT_MGMT_HELLO_MINVER, want_ver);
144	reply |= FIELD_PREP(APPLE_RTKIT_MGMT_HELLO_MAXVER, want_ver);
145	apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_HELLO_REPLY, reply);
146
147	return;
148
149abort_boot:
150	rtk->boot_result = -EINVAL;
151	complete_all(&rtk->epmap_completion);
152}
153
154static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg)
155{
156	int i, ep;
157	u64 reply;
158	unsigned long bitmap = FIELD_GET(APPLE_RTKIT_MGMT_EPMAP_BITMAP, msg);
159	u32 base = FIELD_GET(APPLE_RTKIT_MGMT_EPMAP_BASE, msg);
160
161	dev_dbg(rtk->dev,
162		"RTKit: received endpoint bitmap 0x%lx with base 0x%x\n",
163		bitmap, base);
164
165	for_each_set_bit(i, &bitmap, 32) {
166		ep = 32 * base + i;
167		dev_dbg(rtk->dev, "RTKit: Discovered endpoint 0x%02x\n", ep);
168		set_bit(ep, rtk->endpoints);
169	}
170
171	reply = FIELD_PREP(APPLE_RTKIT_MGMT_EPMAP_BASE, base);
172	if (msg & APPLE_RTKIT_MGMT_EPMAP_LAST)
173		reply |= APPLE_RTKIT_MGMT_EPMAP_LAST;
174	else
175		reply |= APPLE_RTKIT_MGMT_EPMAP_REPLY_MORE;
176
177	apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_EPMAP_REPLY, reply);
178
179	if (!(msg & APPLE_RTKIT_MGMT_EPMAP_LAST))
180		return;
181
182	for_each_set_bit(ep, rtk->endpoints, APPLE_RTKIT_APP_ENDPOINT_START) {
183		switch (ep) {
184		/* the management endpoint is started by default */
185		case APPLE_RTKIT_EP_MGMT:
186			break;
187
188		/* without starting these RTKit refuses to boot */
189		case APPLE_RTKIT_EP_SYSLOG:
190		case APPLE_RTKIT_EP_CRASHLOG:
191		case APPLE_RTKIT_EP_DEBUG:
192		case APPLE_RTKIT_EP_IOREPORT:
193		case APPLE_RTKIT_EP_OSLOG:
194			dev_dbg(rtk->dev,
195				"RTKit: Starting system endpoint 0x%02x\n", ep);
196			apple_rtkit_start_ep(rtk, ep);
197			break;
198
199		default:
200			dev_warn(rtk->dev,
201				 "RTKit: Unknown system endpoint: 0x%02x\n",
202				 ep);
203		}
204	}
205
206	rtk->boot_result = 0;
207	complete_all(&rtk->epmap_completion);
208}
209
210static void apple_rtkit_management_rx_iop_pwr_ack(struct apple_rtkit *rtk,
211						  u64 msg)
212{
213	unsigned int new_state = FIELD_GET(APPLE_RTKIT_MGMT_PWR_STATE, msg);
214
215	dev_dbg(rtk->dev, "RTKit: IOP power state transition: 0x%x -> 0x%x\n",
216		rtk->iop_power_state, new_state);
217	rtk->iop_power_state = new_state;
218
219	complete_all(&rtk->iop_pwr_ack_completion);
220}
221
222static void apple_rtkit_management_rx_ap_pwr_ack(struct apple_rtkit *rtk,
223						 u64 msg)
224{
225	unsigned int new_state = FIELD_GET(APPLE_RTKIT_MGMT_PWR_STATE, msg);
226
227	dev_dbg(rtk->dev, "RTKit: AP power state transition: 0x%x -> 0x%x\n",
228		rtk->ap_power_state, new_state);
229	rtk->ap_power_state = new_state;
230
231	complete_all(&rtk->ap_pwr_ack_completion);
232}
233
234static void apple_rtkit_management_rx(struct apple_rtkit *rtk, u64 msg)
235{
236	u8 type = FIELD_GET(APPLE_RTKIT_MGMT_TYPE, msg);
237
238	switch (type) {
239	case APPLE_RTKIT_MGMT_HELLO:
240		apple_rtkit_management_rx_hello(rtk, msg);
241		break;
242	case APPLE_RTKIT_MGMT_EPMAP:
243		apple_rtkit_management_rx_epmap(rtk, msg);
244		break;
245	case APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE_ACK:
246		apple_rtkit_management_rx_iop_pwr_ack(rtk, msg);
247		break;
248	case APPLE_RTKIT_MGMT_SET_AP_PWR_STATE_ACK:
249		apple_rtkit_management_rx_ap_pwr_ack(rtk, msg);
250		break;
251	default:
252		dev_warn(
253			rtk->dev,
254			"RTKit: unknown management message: 0x%llx (type: 0x%02x)\n",
255			msg, type);
256	}
257}
258
259static int apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk,
260					    struct apple_rtkit_shmem *buffer,
261					    u8 ep, u64 msg)
262{
263	u64 reply;
264	int err;
265
266	/* The different size vs. IOVA shifts look odd but are indeed correct this way */
267	if (ep == APPLE_RTKIT_EP_OSLOG) {
268		buffer->size = FIELD_GET(APPLE_RTKIT_OSLOG_SIZE, msg);
269		buffer->iova = FIELD_GET(APPLE_RTKIT_OSLOG_IOVA, msg) << 12;
270	} else {
271		buffer->size = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_SIZE, msg) << 12;
272		buffer->iova = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg);
273	}
274
275	buffer->buffer = NULL;
276	buffer->iomem = NULL;
277	buffer->is_mapped = false;
278
279	dev_dbg(rtk->dev, "RTKit: buffer request for 0x%zx bytes at %pad\n",
280		buffer->size, &buffer->iova);
281
282	if (buffer->iova && !rtk->ops->shmem_setup) {
283		err = -EINVAL;
284		goto error;
285	}
286
287	if (rtk->ops->shmem_setup) {
288		err = rtk->ops->shmem_setup(rtk->cookie, buffer);
289		if (err)
290			goto error;
291	} else {
292		buffer->buffer = dma_alloc_coherent(rtk->dev, buffer->size,
293						    &buffer->iova, GFP_KERNEL);
294		if (!buffer->buffer) {
295			err = -ENOMEM;
296			goto error;
297		}
298	}
299
300	if (!buffer->is_mapped) {
301		/* oslog uses different fields and needs a shifted IOVA instead of size */
302		if (ep == APPLE_RTKIT_EP_OSLOG) {
303			reply = FIELD_PREP(APPLE_RTKIT_OSLOG_TYPE,
304					   APPLE_RTKIT_OSLOG_BUFFER_REQUEST);
305			reply |= FIELD_PREP(APPLE_RTKIT_OSLOG_SIZE, buffer->size);
306			reply |= FIELD_PREP(APPLE_RTKIT_OSLOG_IOVA,
307					    buffer->iova >> 12);
308		} else {
309			reply = FIELD_PREP(APPLE_RTKIT_SYSLOG_TYPE,
310					   APPLE_RTKIT_BUFFER_REQUEST);
311			reply |= FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_SIZE,
312					    buffer->size >> 12);
313			reply |= FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_IOVA,
314					    buffer->iova);
315		}
316		apple_rtkit_send_message(rtk, ep, reply, NULL, false);
317	}
318
319	return 0;
320
321error:
322	dev_err(rtk->dev, "RTKit: failed buffer request for 0x%zx bytes (%d)\n",
323		buffer->size, err);
324
325	buffer->buffer = NULL;
326	buffer->iomem = NULL;
327	buffer->iova = 0;
328	buffer->size = 0;
329	buffer->is_mapped = false;
330	return err;
331}
332
333static void apple_rtkit_free_buffer(struct apple_rtkit *rtk,
334				    struct apple_rtkit_shmem *bfr)
335{
336	if (bfr->size == 0)
337		return;
338
339	if (rtk->ops->shmem_destroy)
340		rtk->ops->shmem_destroy(rtk->cookie, bfr);
341	else if (bfr->buffer)
342		dma_free_coherent(rtk->dev, bfr->size, bfr->buffer, bfr->iova);
343
344	bfr->buffer = NULL;
345	bfr->iomem = NULL;
346	bfr->iova = 0;
347	bfr->size = 0;
348	bfr->is_mapped = false;
349}
350
351static void apple_rtkit_memcpy(struct apple_rtkit *rtk, void *dst,
352			       struct apple_rtkit_shmem *bfr, size_t offset,
353			       size_t len)
354{
355	if (bfr->iomem)
356		memcpy_fromio(dst, bfr->iomem + offset, len);
357	else
358		memcpy(dst, bfr->buffer + offset, len);
359}
360
361static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg)
362{
363	u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg);
364	u8 *bfr;
365
366	if (type != APPLE_RTKIT_CRASHLOG_CRASH) {
367		dev_warn(rtk->dev, "RTKit: Unknown crashlog message: %llx\n",
368			 msg);
369		return;
370	}
371
372	if (!rtk->crashlog_buffer.size) {
373		apple_rtkit_common_rx_get_buffer(rtk, &rtk->crashlog_buffer,
374						 APPLE_RTKIT_EP_CRASHLOG, msg);
375		return;
376	}
377
378	dev_err(rtk->dev, "RTKit: co-processor has crashed\n");
379
380	/*
381	 * create a shadow copy here to make sure the co-processor isn't able
382	 * to change the log while we're dumping it. this also ensures
383	 * the buffer is in normal memory and not iomem for e.g. the SMC
384	 */
385	bfr = kzalloc(rtk->crashlog_buffer.size, GFP_KERNEL);
386	if (bfr) {
387		apple_rtkit_memcpy(rtk, bfr, &rtk->crashlog_buffer, 0,
388				   rtk->crashlog_buffer.size);
389		apple_rtkit_crashlog_dump(rtk, bfr, rtk->crashlog_buffer.size);
390	} else {
391		dev_err(rtk->dev,
392			"RTKit: Couldn't allocate crashlog shadow buffer\n");
393	}
394
395	rtk->crashed = true;
396	if (rtk->ops->crashed)
397		rtk->ops->crashed(rtk->cookie, bfr, rtk->crashlog_buffer.size);
398
399	kfree(bfr);
400}
401
402static void apple_rtkit_ioreport_rx(struct apple_rtkit *rtk, u64 msg)
403{
404	u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg);
405
406	switch (type) {
407	case APPLE_RTKIT_BUFFER_REQUEST:
408		apple_rtkit_common_rx_get_buffer(rtk, &rtk->ioreport_buffer,
409						 APPLE_RTKIT_EP_IOREPORT, msg);
410		break;
411	/* unknown, must be ACKed or the co-processor will hang */
412	case 0x8:
413	case 0xc:
414		apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_IOREPORT, msg,
415					 NULL, false);
416		break;
417	default:
418		dev_warn(rtk->dev, "RTKit: Unknown ioreport message: %llx\n",
419			 msg);
420	}
421}
422
423static void apple_rtkit_syslog_rx_init(struct apple_rtkit *rtk, u64 msg)
424{
425	rtk->syslog_n_entries = FIELD_GET(APPLE_RTKIT_SYSLOG_N_ENTRIES, msg);
426	rtk->syslog_msg_size = FIELD_GET(APPLE_RTKIT_SYSLOG_MSG_SIZE, msg);
427
428	rtk->syslog_msg_buffer = kzalloc(rtk->syslog_msg_size, GFP_KERNEL);
429
430	dev_dbg(rtk->dev,
431		"RTKit: syslog initialized: entries: %zd, msg_size: %zd\n",
432		rtk->syslog_n_entries, rtk->syslog_msg_size);
433}
434
435static bool should_crop_syslog_char(char c)
436{
437	return c == '\n' || c == '\r' || c == ' ' || c == '\0';
438}
439
440static void apple_rtkit_syslog_rx_log(struct apple_rtkit *rtk, u64 msg)
441{
442	u8 idx = msg & 0xff;
443	char log_context[24];
444	size_t entry_size = 0x20 + rtk->syslog_msg_size;
445	int msglen;
446
447	if (!rtk->syslog_msg_buffer) {
448		dev_warn(
449			rtk->dev,
450			"RTKit: received syslog message but no syslog_msg_buffer\n");
451		goto done;
452	}
453	if (!rtk->syslog_buffer.size) {
454		dev_warn(
455			rtk->dev,
456			"RTKit: received syslog message but syslog_buffer.size is zero\n");
457		goto done;
458	}
459	if (!rtk->syslog_buffer.buffer && !rtk->syslog_buffer.iomem) {
460		dev_warn(
461			rtk->dev,
462			"RTKit: received syslog message but no syslog_buffer.buffer or syslog_buffer.iomem\n");
463		goto done;
464	}
465	if (idx > rtk->syslog_n_entries) {
466		dev_warn(rtk->dev, "RTKit: syslog index %d out of range\n",
467			 idx);
468		goto done;
469	}
470
471	apple_rtkit_memcpy(rtk, log_context, &rtk->syslog_buffer,
472			   idx * entry_size + 8, sizeof(log_context));
473	apple_rtkit_memcpy(rtk, rtk->syslog_msg_buffer, &rtk->syslog_buffer,
474			   idx * entry_size + 8 + sizeof(log_context),
475			   rtk->syslog_msg_size);
476
477	log_context[sizeof(log_context) - 1] = 0;
478
479	msglen = strnlen(rtk->syslog_msg_buffer, rtk->syslog_msg_size - 1);
480	while (msglen > 0 &&
481		   should_crop_syslog_char(rtk->syslog_msg_buffer[msglen - 1]))
482		msglen--;
483
484	rtk->syslog_msg_buffer[msglen] = 0;
485	dev_info(rtk->dev, "RTKit: syslog message: %s: %s\n", log_context,
486		 rtk->syslog_msg_buffer);
487
488done:
489	apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_SYSLOG, msg, NULL, false);
490}
491
492static void apple_rtkit_syslog_rx(struct apple_rtkit *rtk, u64 msg)
493{
494	u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg);
495
496	switch (type) {
497	case APPLE_RTKIT_BUFFER_REQUEST:
498		apple_rtkit_common_rx_get_buffer(rtk, &rtk->syslog_buffer,
499						 APPLE_RTKIT_EP_SYSLOG, msg);
500		break;
501	case APPLE_RTKIT_SYSLOG_INIT:
502		apple_rtkit_syslog_rx_init(rtk, msg);
503		break;
504	case APPLE_RTKIT_SYSLOG_LOG:
505		apple_rtkit_syslog_rx_log(rtk, msg);
506		break;
507	default:
508		dev_warn(rtk->dev, "RTKit: Unknown syslog message: %llx\n",
509			 msg);
510	}
511}
512
513static void apple_rtkit_oslog_rx(struct apple_rtkit *rtk, u64 msg)
514{
515	u8 type = FIELD_GET(APPLE_RTKIT_OSLOG_TYPE, msg);
516
517	switch (type) {
518	case APPLE_RTKIT_OSLOG_BUFFER_REQUEST:
519		apple_rtkit_common_rx_get_buffer(rtk, &rtk->oslog_buffer,
520						 APPLE_RTKIT_EP_OSLOG, msg);
521		break;
522	default:
523		dev_warn(rtk->dev, "RTKit: Unknown oslog message: %llx\n",
524			 msg);
525	}
526}
527
528static void apple_rtkit_rx_work(struct work_struct *work)
529{
530	struct apple_rtkit_rx_work *rtk_work =
531		container_of(work, struct apple_rtkit_rx_work, work);
532	struct apple_rtkit *rtk = rtk_work->rtk;
533
534	switch (rtk_work->ep) {
535	case APPLE_RTKIT_EP_MGMT:
536		apple_rtkit_management_rx(rtk, rtk_work->msg);
537		break;
538	case APPLE_RTKIT_EP_CRASHLOG:
539		apple_rtkit_crashlog_rx(rtk, rtk_work->msg);
540		break;
541	case APPLE_RTKIT_EP_SYSLOG:
542		apple_rtkit_syslog_rx(rtk, rtk_work->msg);
543		break;
544	case APPLE_RTKIT_EP_IOREPORT:
545		apple_rtkit_ioreport_rx(rtk, rtk_work->msg);
546		break;
547	case APPLE_RTKIT_EP_OSLOG:
548		apple_rtkit_oslog_rx(rtk, rtk_work->msg);
549		break;
550	case APPLE_RTKIT_APP_ENDPOINT_START ... 0xff:
551		if (rtk->ops->recv_message)
552			rtk->ops->recv_message(rtk->cookie, rtk_work->ep,
553					       rtk_work->msg);
554		else
555			dev_warn(
556				rtk->dev,
557				"Received unexpected message to EP%02d: %llx\n",
558				rtk_work->ep, rtk_work->msg);
559		break;
560	default:
561		dev_warn(rtk->dev,
562			 "RTKit: message to unknown endpoint %02x: %llx\n",
563			 rtk_work->ep, rtk_work->msg);
564	}
565
566	kfree(rtk_work);
567}
568
569static void apple_rtkit_rx(struct apple_mbox *mbox, struct apple_mbox_msg msg,
570			   void *cookie)
571{
572	struct apple_rtkit *rtk = cookie;
573	struct apple_rtkit_rx_work *work;
574	u8 ep = msg.msg1;
575
576	/*
577	 * The message was read from a MMIO FIFO and we have to make
578	 * sure all reads from buffers sent with that message happen
579	 * afterwards.
580	 */
581	dma_rmb();
582
583	if (!test_bit(ep, rtk->endpoints))
584		dev_warn(rtk->dev,
585			 "RTKit: Message to undiscovered endpoint 0x%02x\n",
586			 ep);
587
588	if (ep >= APPLE_RTKIT_APP_ENDPOINT_START &&
589	    rtk->ops->recv_message_early &&
590	    rtk->ops->recv_message_early(rtk->cookie, ep, msg.msg0))
591		return;
592
593	work = kzalloc(sizeof(*work), GFP_ATOMIC);
594	if (!work)
595		return;
596
597	work->rtk = rtk;
598	work->ep = ep;
599	work->msg = msg.msg0;
600	INIT_WORK(&work->work, apple_rtkit_rx_work);
601	queue_work(rtk->wq, &work->work);
602}
603
604int apple_rtkit_send_message(struct apple_rtkit *rtk, u8 ep, u64 message,
605			     struct completion *completion, bool atomic)
606{
607	struct apple_mbox_msg msg = {
608		.msg0 = message,
609		.msg1 = ep,
610	};
611
612	if (rtk->crashed) {
613		dev_warn(rtk->dev,
614			 "RTKit: Device is crashed, cannot send message\n");
615		return -EINVAL;
616	}
617
618	if (ep >= APPLE_RTKIT_APP_ENDPOINT_START &&
619	    !apple_rtkit_is_running(rtk)) {
620		dev_warn(rtk->dev,
621			 "RTKit: Endpoint 0x%02x is not running, cannot send message\n", ep);
622		return -EINVAL;
623	}
624
625	/*
626	 * The message will be sent with a MMIO write. We need the barrier
627	 * here to ensure any previous writes to buffers are visible to the
628	 * device before that MMIO write happens.
629	 */
630	dma_wmb();
631
632	return apple_mbox_send(rtk->mbox, msg, atomic);
633}
634EXPORT_SYMBOL_GPL(apple_rtkit_send_message);
635
636int apple_rtkit_poll(struct apple_rtkit *rtk)
637{
638	return apple_mbox_poll(rtk->mbox);
639}
640EXPORT_SYMBOL_GPL(apple_rtkit_poll);
641
642int apple_rtkit_start_ep(struct apple_rtkit *rtk, u8 endpoint)
643{
644	u64 msg;
645
646	if (!test_bit(endpoint, rtk->endpoints))
647		return -EINVAL;
648	if (endpoint >= APPLE_RTKIT_APP_ENDPOINT_START &&
649	    !apple_rtkit_is_running(rtk))
650		return -EINVAL;
651
652	msg = FIELD_PREP(APPLE_RTKIT_MGMT_STARTEP_EP, endpoint);
653	msg |= APPLE_RTKIT_MGMT_STARTEP_FLAG;
654	apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_STARTEP, msg);
655
656	return 0;
657}
658EXPORT_SYMBOL_GPL(apple_rtkit_start_ep);
659
660struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie,
661					    const char *mbox_name, int mbox_idx,
662					    const struct apple_rtkit_ops *ops)
663{
664	struct apple_rtkit *rtk;
665	int ret;
666
667	if (!ops)
668		return ERR_PTR(-EINVAL);
669
670	rtk = kzalloc(sizeof(*rtk), GFP_KERNEL);
671	if (!rtk)
672		return ERR_PTR(-ENOMEM);
673
674	rtk->dev = dev;
675	rtk->cookie = cookie;
676	rtk->ops = ops;
677
678	init_completion(&rtk->epmap_completion);
679	init_completion(&rtk->iop_pwr_ack_completion);
680	init_completion(&rtk->ap_pwr_ack_completion);
681
682	bitmap_zero(rtk->endpoints, APPLE_RTKIT_MAX_ENDPOINTS);
683	set_bit(APPLE_RTKIT_EP_MGMT, rtk->endpoints);
684
685	if (mbox_name)
686		rtk->mbox = apple_mbox_get_byname(dev, mbox_name);
687	else
688		rtk->mbox = apple_mbox_get(dev, mbox_idx);
689
690	if (IS_ERR(rtk->mbox)) {
691		ret = PTR_ERR(rtk->mbox);
692		goto free_rtk;
693	}
694
695	rtk->mbox->rx = apple_rtkit_rx;
696	rtk->mbox->cookie = rtk;
697
698	rtk->wq = alloc_ordered_workqueue("rtkit-%s", WQ_HIGHPRI | WQ_MEM_RECLAIM,
699					  dev_name(rtk->dev));
700	if (!rtk->wq) {
701		ret = -ENOMEM;
702		goto free_rtk;
703	}
704
705	ret = apple_mbox_start(rtk->mbox);
706	if (ret)
707		goto destroy_wq;
708
709	return rtk;
710
711destroy_wq:
712	destroy_workqueue(rtk->wq);
713free_rtk:
714	kfree(rtk);
715	return ERR_PTR(ret);
716}
717EXPORT_SYMBOL_GPL(apple_rtkit_init);
718
719static int apple_rtkit_wait_for_completion(struct completion *c)
720{
721	long t;
722
723	t = wait_for_completion_interruptible_timeout(c,
724						      msecs_to_jiffies(1000));
725	if (t < 0)
726		return t;
727	else if (t == 0)
728		return -ETIME;
729	else
730		return 0;
731}
732
733int apple_rtkit_reinit(struct apple_rtkit *rtk)
734{
735	/* make sure we don't handle any messages while reinitializing */
736	apple_mbox_stop(rtk->mbox);
737	flush_workqueue(rtk->wq);
738
739	apple_rtkit_free_buffer(rtk, &rtk->ioreport_buffer);
740	apple_rtkit_free_buffer(rtk, &rtk->crashlog_buffer);
741	apple_rtkit_free_buffer(rtk, &rtk->oslog_buffer);
742	apple_rtkit_free_buffer(rtk, &rtk->syslog_buffer);
743
744	kfree(rtk->syslog_msg_buffer);
745
746	rtk->syslog_msg_buffer = NULL;
747	rtk->syslog_n_entries = 0;
748	rtk->syslog_msg_size = 0;
749
750	bitmap_zero(rtk->endpoints, APPLE_RTKIT_MAX_ENDPOINTS);
751	set_bit(APPLE_RTKIT_EP_MGMT, rtk->endpoints);
752
753	reinit_completion(&rtk->epmap_completion);
754	reinit_completion(&rtk->iop_pwr_ack_completion);
755	reinit_completion(&rtk->ap_pwr_ack_completion);
756
757	rtk->crashed = false;
758	rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_OFF;
759	rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_OFF;
760
761	return apple_mbox_start(rtk->mbox);
762}
763EXPORT_SYMBOL_GPL(apple_rtkit_reinit);
764
765static int apple_rtkit_set_ap_power_state(struct apple_rtkit *rtk,
766					  unsigned int state)
767{
768	u64 msg;
769	int ret;
770
771	reinit_completion(&rtk->ap_pwr_ack_completion);
772
773	msg = FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, state);
774	ret = apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE,
775					  msg);
776	if (ret)
777		return ret;
778
779	ret = apple_rtkit_wait_for_completion(&rtk->ap_pwr_ack_completion);
780	if (ret)
781		return ret;
782
783	if (rtk->ap_power_state != state)
784		return -EINVAL;
785	return 0;
786}
787
788static int apple_rtkit_set_iop_power_state(struct apple_rtkit *rtk,
789					   unsigned int state)
790{
791	u64 msg;
792	int ret;
793
794	reinit_completion(&rtk->iop_pwr_ack_completion);
795
796	msg = FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, state);
797	ret = apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE,
798					  msg);
799	if (ret)
800		return ret;
801
802	ret = apple_rtkit_wait_for_completion(&rtk->iop_pwr_ack_completion);
803	if (ret)
804		return ret;
805
806	if (rtk->iop_power_state != state)
807		return -EINVAL;
808	return 0;
809}
810
811int apple_rtkit_boot(struct apple_rtkit *rtk)
812{
813	int ret;
814
815	if (apple_rtkit_is_running(rtk))
816		return 0;
817	if (rtk->crashed)
818		return -EINVAL;
819
820	dev_dbg(rtk->dev, "RTKit: waiting for boot to finish\n");
821	ret = apple_rtkit_wait_for_completion(&rtk->epmap_completion);
822	if (ret)
823		return ret;
824	if (rtk->boot_result)
825		return rtk->boot_result;
826
827	dev_dbg(rtk->dev, "RTKit: waiting for IOP power state ACK\n");
828	ret = apple_rtkit_wait_for_completion(&rtk->iop_pwr_ack_completion);
829	if (ret)
830		return ret;
831
832	return apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON);
833}
834EXPORT_SYMBOL_GPL(apple_rtkit_boot);
835
836int apple_rtkit_shutdown(struct apple_rtkit *rtk)
837{
838	int ret;
839
840	/* if OFF is used here the co-processor will not wake up again */
841	ret = apple_rtkit_set_ap_power_state(rtk,
842					     APPLE_RTKIT_PWR_STATE_QUIESCED);
843	if (ret)
844		return ret;
845
846	ret = apple_rtkit_set_iop_power_state(rtk, APPLE_RTKIT_PWR_STATE_SLEEP);
847	if (ret)
848		return ret;
849
850	return apple_rtkit_reinit(rtk);
851}
852EXPORT_SYMBOL_GPL(apple_rtkit_shutdown);
853
854int apple_rtkit_idle(struct apple_rtkit *rtk)
855{
856	int ret;
857
858	/* if OFF is used here the co-processor will not wake up again */
859	ret = apple_rtkit_set_ap_power_state(rtk,
860					     APPLE_RTKIT_PWR_STATE_IDLE);
861	if (ret)
862		return ret;
863
864	ret = apple_rtkit_set_iop_power_state(rtk, APPLE_RTKIT_PWR_STATE_IDLE);
865	if (ret)
866		return ret;
867
868	rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_IDLE;
869	rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_IDLE;
870	return 0;
871}
872EXPORT_SYMBOL_GPL(apple_rtkit_idle);
873
874int apple_rtkit_quiesce(struct apple_rtkit *rtk)
875{
876	int ret;
877
878	ret = apple_rtkit_set_ap_power_state(rtk,
879					     APPLE_RTKIT_PWR_STATE_QUIESCED);
880	if (ret)
881		return ret;
882
883	ret = apple_rtkit_set_iop_power_state(rtk,
884					      APPLE_RTKIT_PWR_STATE_QUIESCED);
885	if (ret)
886		return ret;
887
888	ret = apple_rtkit_reinit(rtk);
889	if (ret)
890		return ret;
891
892	rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_QUIESCED;
893	rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_QUIESCED;
894	return 0;
895}
896EXPORT_SYMBOL_GPL(apple_rtkit_quiesce);
897
898int apple_rtkit_wake(struct apple_rtkit *rtk)
899{
900	u64 msg;
901	int ret;
902
903	if (apple_rtkit_is_running(rtk))
904		return -EINVAL;
905
906	reinit_completion(&rtk->iop_pwr_ack_completion);
907
908	/*
909	 * Use open-coded apple_rtkit_set_iop_power_state since apple_rtkit_boot
910	 * will wait for the completion anyway.
911	 */
912	msg = FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, APPLE_RTKIT_PWR_STATE_INIT);
913	ret = apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE,
914					  msg);
915	if (ret)
916		return ret;
917
918	return apple_rtkit_boot(rtk);
919}
920EXPORT_SYMBOL_GPL(apple_rtkit_wake);
921
922void apple_rtkit_free(struct apple_rtkit *rtk)
923{
924	apple_mbox_stop(rtk->mbox);
925	destroy_workqueue(rtk->wq);
926
927	apple_rtkit_free_buffer(rtk, &rtk->ioreport_buffer);
928	apple_rtkit_free_buffer(rtk, &rtk->crashlog_buffer);
929	apple_rtkit_free_buffer(rtk, &rtk->oslog_buffer);
930	apple_rtkit_free_buffer(rtk, &rtk->syslog_buffer);
931
932	kfree(rtk->syslog_msg_buffer);
933	kfree(rtk);
934}
935EXPORT_SYMBOL_GPL(apple_rtkit_free);
936
937static void apple_rtkit_free_wrapper(void *data)
938{
939	apple_rtkit_free(data);
940}
941
942struct apple_rtkit *devm_apple_rtkit_init(struct device *dev, void *cookie,
943					  const char *mbox_name, int mbox_idx,
944					  const struct apple_rtkit_ops *ops)
945{
946	struct apple_rtkit *rtk;
947	int ret;
948
949	rtk = apple_rtkit_init(dev, cookie, mbox_name, mbox_idx, ops);
950	if (IS_ERR(rtk))
951		return rtk;
952
953	ret = devm_add_action_or_reset(dev, apple_rtkit_free_wrapper, rtk);
954	if (ret)
955		return ERR_PTR(ret);
956
957	return rtk;
958}
959EXPORT_SYMBOL_GPL(devm_apple_rtkit_init);
960
961MODULE_LICENSE("Dual MIT/GPL");
962MODULE_AUTHOR("Sven Peter <sven@svenpeter.dev>");
963MODULE_DESCRIPTION("Apple RTKit driver");