ffa.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * FF-A v1.0 proxy to filter out invalid memory-sharing SMC calls issued by
  4 * the host. FF-A is a slightly more palatable abbreviation of "Arm Firmware
  5 * Framework for Arm A-profile", which is specified by Arm in document
  6 * number DEN0077.
  7 *
  8 * Copyright (C) 2022 - Google LLC
  9 * Author: Andrew Walbran <qwandor@google.com>
 10 *
 11 * This driver hooks into the SMC trapping logic for the host and intercepts
 12 * all calls falling within the FF-A range. Each call is either:
 13 *
 14 *	- Forwarded on unmodified to the SPMD at EL3
 15 *	- Rejected as "unsupported"
 16 *	- Accompanied by a host stage-2 page-table check/update and reissued
 17 *
 18 * Consequently, any attempts by the host to make guest memory pages
 19 * accessible to the secure world using FF-A will be detected either here
 20 * (in the case that the memory is already owned by the guest) or during
 21 * donation to the guest (in the case that the memory was previously shared
 22 * with the secure world).
 23 *
 24 * To allow the rolling-back of page-table updates and FF-A calls in the
 25 * event of failure, operations involving the RXTX buffers are locked for
 26 * the duration and are therefore serialised.
 27 */
 28
 29#include <linux/arm-smccc.h>
 30#include <linux/arm_ffa.h>
 31#include <asm/kvm_pkvm.h>
 32
 33#include <nvhe/ffa.h>
 34#include <nvhe/mem_protect.h>
 35#include <nvhe/memory.h>
 36#include <nvhe/trap_handler.h>
 37#include <nvhe/spinlock.h>
 38
 39/*
 40 * "ID value 0 must be returned at the Non-secure physical FF-A instance"
 41 * We share this ID with the host.
 42 */
 43#define HOST_FFA_ID	0
 44
 45/*
 46 * A buffer to hold the maximum descriptor size we can see from the host,
 47 * which is required when the SPMD returns a fragmented FFA_MEM_RETRIEVE_RESP
 48 * when resolving the handle on the reclaim path.
 49 */
 50struct kvm_ffa_descriptor_buffer {
 51	void	*buf;
 52	size_t	len;
 53};
 54
 55static struct kvm_ffa_descriptor_buffer ffa_desc_buf;
 56
 57struct kvm_ffa_buffers {
 58	hyp_spinlock_t lock;
 59	void *tx;
 60	void *rx;
 61};
 62
 63/*
 64 * Note that we don't currently lock these buffers explicitly, instead
 65 * relying on the locking of the host FFA buffers as we only have one
 66 * client.
 67 */
 68static struct kvm_ffa_buffers hyp_buffers;
 69static struct kvm_ffa_buffers host_buffers;
 70static u32 hyp_ffa_version;
 71static bool has_version_negotiated;
 72static hyp_spinlock_t version_lock;
 73
 74static void ffa_to_smccc_error(struct arm_smccc_1_2_regs *res, u64 ffa_errno)
 75{
 76	*res = (struct arm_smccc_1_2_regs) {
 77		.a0	= FFA_ERROR,
 78		.a2	= ffa_errno,
 79	};
 80}
 81
 82static void ffa_to_smccc_res_prop(struct arm_smccc_1_2_regs *res, int ret, u64 prop)
 83{
 84	if (ret == FFA_RET_SUCCESS) {
 85		*res = (struct arm_smccc_1_2_regs) { .a0 = FFA_SUCCESS,
 86						      .a2 = prop };
 87	} else {
 88		ffa_to_smccc_error(res, ret);
 89	}
 90}
 91
 92static void ffa_to_smccc_res(struct arm_smccc_1_2_regs *res, int ret)
 93{
 94	ffa_to_smccc_res_prop(res, ret, 0);
 95}
 96
 97static void ffa_set_retval(struct kvm_cpu_context *ctxt,
 98			   struct arm_smccc_1_2_regs *res)
 99{
100	cpu_reg(ctxt, 0) = res->a0;
101	cpu_reg(ctxt, 1) = res->a1;
102	cpu_reg(ctxt, 2) = res->a2;
103	cpu_reg(ctxt, 3) = res->a3;
104	cpu_reg(ctxt, 4) = res->a4;
105	cpu_reg(ctxt, 5) = res->a5;
106	cpu_reg(ctxt, 6) = res->a6;
107	cpu_reg(ctxt, 7) = res->a7;
108
109	/*
110	 * DEN0028C 2.6: SMC32/HVC32 call from aarch64 must preserve x8-x30.
111	 *
112	 * In FF-A 1.2, we cannot rely on the function ID sent by the caller to
113	 * detect 32-bit calls because the CPU cycle management interfaces (e.g.
114	 * FFA_MSG_WAIT, FFA_RUN) are 32-bit only but can have 64-bit responses.
115	 *
116	 * FFA-1.3 introduces 64-bit variants of the CPU cycle management
117	 * interfaces. Moreover, FF-A 1.3 clarifies that SMC32 direct requests
118	 * complete with SMC32 direct responses which *should* allow us use the
119	 * function ID sent by the caller to determine whether to return x8-x17.
120	 *
121	 * Note that we also cannot rely on function IDs in the response.
122	 *
123	 * Given the above, assume SMC64 and send back x0-x17 unconditionally
124	 * as the passthrough code (__kvm_hyp_host_forward_smc) does the same.
125	 */
126	cpu_reg(ctxt, 8) = res->a8;
127	cpu_reg(ctxt, 9) = res->a9;
128	cpu_reg(ctxt, 10) = res->a10;
129	cpu_reg(ctxt, 11) = res->a11;
130	cpu_reg(ctxt, 12) = res->a12;
131	cpu_reg(ctxt, 13) = res->a13;
132	cpu_reg(ctxt, 14) = res->a14;
133	cpu_reg(ctxt, 15) = res->a15;
134	cpu_reg(ctxt, 16) = res->a16;
135	cpu_reg(ctxt, 17) = res->a17;
136}
137
138static bool is_ffa_call(u64 func_id)
139{
140	return ARM_SMCCC_IS_FAST_CALL(func_id) &&
141	       ARM_SMCCC_OWNER_NUM(func_id) == ARM_SMCCC_OWNER_STANDARD &&
142	       ARM_SMCCC_FUNC_NUM(func_id) >= FFA_MIN_FUNC_NUM &&
143	       ARM_SMCCC_FUNC_NUM(func_id) <= FFA_MAX_FUNC_NUM;
144}
145
146static int ffa_map_hyp_buffers(u64 ffa_page_count)
147{
148	struct arm_smccc_1_2_regs res;
149
150	arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
151		.a0 = FFA_FN64_RXTX_MAP,
152		.a1 = hyp_virt_to_phys(hyp_buffers.tx),
153		.a2 = hyp_virt_to_phys(hyp_buffers.rx),
154		.a3 = ffa_page_count,
155	}, &res);
156
157	return res.a0 == FFA_SUCCESS ? FFA_RET_SUCCESS : res.a2;
158}
159
160static int ffa_unmap_hyp_buffers(void)
161{
162	struct arm_smccc_1_2_regs res;
163
164	arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
165		.a0 = FFA_RXTX_UNMAP,
166		.a1 = HOST_FFA_ID,
167	}, &res);
168
169	return res.a0 == FFA_SUCCESS ? FFA_RET_SUCCESS : res.a2;
170}
171
172static void ffa_mem_frag_tx(struct arm_smccc_1_2_regs *res, u32 handle_lo,
173			     u32 handle_hi, u32 fraglen, u32 endpoint_id)
174{
175	arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
176		.a0 = FFA_MEM_FRAG_TX,
177		.a1 = handle_lo,
178		.a2 = handle_hi,
179		.a3 = fraglen,
180		.a4 = endpoint_id,
181	}, res);
182}
183
184static void ffa_mem_frag_rx(struct arm_smccc_1_2_regs *res, u32 handle_lo,
185			     u32 handle_hi, u32 fragoff)
186{
187	arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
188		.a0 = FFA_MEM_FRAG_RX,
189		.a1 = handle_lo,
190		.a2 = handle_hi,
191		.a3 = fragoff,
192		.a4 = HOST_FFA_ID,
193	}, res);
194}
195
196static void ffa_mem_xfer(struct arm_smccc_1_2_regs *res, u64 func_id, u32 len,
197			  u32 fraglen)
198{
199	arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
200		.a0 = func_id,
201		.a1 = len,
202		.a2 = fraglen,
203	}, res);
204}
205
206static void ffa_mem_reclaim(struct arm_smccc_1_2_regs *res, u32 handle_lo,
207			     u32 handle_hi, u32 flags)
208{
209	arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
210		.a0 = FFA_MEM_RECLAIM,
211		.a1 = handle_lo,
212		.a2 = handle_hi,
213		.a3 = flags,
214	}, res);
215}
216
217static void ffa_retrieve_req(struct arm_smccc_1_2_regs *res, u32 len)
218{
219	arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
220		.a0 = FFA_FN64_MEM_RETRIEVE_REQ,
221		.a1 = len,
222		.a2 = len,
223	}, res);
224}
225
226static void ffa_rx_release(struct arm_smccc_1_2_regs *res)
227{
228	arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
229		.a0 = FFA_RX_RELEASE,
230	}, res);
231}
232
233static void do_ffa_rxtx_map(struct arm_smccc_1_2_regs *res,
234			    struct kvm_cpu_context *ctxt)
235{
236	DECLARE_REG(phys_addr_t, tx, ctxt, 1);
237	DECLARE_REG(phys_addr_t, rx, ctxt, 2);
238	DECLARE_REG(u32, npages, ctxt, 3);
239	int ret = 0;
240	void *rx_virt, *tx_virt;
241
242	if (npages != (KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) / FFA_PAGE_SIZE) {
243		ret = FFA_RET_INVALID_PARAMETERS;
244		goto out;
245	}
246
247	if (!PAGE_ALIGNED(tx) || !PAGE_ALIGNED(rx)) {
248		ret = FFA_RET_INVALID_PARAMETERS;
249		goto out;
250	}
251
252	hyp_spin_lock(&host_buffers.lock);
253	if (host_buffers.tx) {
254		ret = FFA_RET_DENIED;
255		goto out_unlock;
256	}
257
258	/*
259	 * Map our hypervisor buffers into the SPMD before mapping and
260	 * pinning the host buffers in our own address space.
261	 */
262	ret = ffa_map_hyp_buffers(npages);
263	if (ret)
264		goto out_unlock;
265
266	ret = __pkvm_host_share_hyp(hyp_phys_to_pfn(tx));
267	if (ret) {
268		ret = FFA_RET_INVALID_PARAMETERS;
269		goto err_unmap;
270	}
271
272	ret = __pkvm_host_share_hyp(hyp_phys_to_pfn(rx));
273	if (ret) {
274		ret = FFA_RET_INVALID_PARAMETERS;
275		goto err_unshare_tx;
276	}
277
278	tx_virt = hyp_phys_to_virt(tx);
279	ret = hyp_pin_shared_mem(tx_virt, tx_virt + 1);
280	if (ret) {
281		ret = FFA_RET_INVALID_PARAMETERS;
282		goto err_unshare_rx;
283	}
284
285	rx_virt = hyp_phys_to_virt(rx);
286	ret = hyp_pin_shared_mem(rx_virt, rx_virt + 1);
287	if (ret) {
288		ret = FFA_RET_INVALID_PARAMETERS;
289		goto err_unpin_tx;
290	}
291
292	host_buffers.tx = tx_virt;
293	host_buffers.rx = rx_virt;
294
295out_unlock:
296	hyp_spin_unlock(&host_buffers.lock);
297out:
298	ffa_to_smccc_res(res, ret);
299	return;
300
301err_unpin_tx:
302	hyp_unpin_shared_mem(tx_virt, tx_virt + 1);
303err_unshare_rx:
304	__pkvm_host_unshare_hyp(hyp_phys_to_pfn(rx));
305err_unshare_tx:
306	__pkvm_host_unshare_hyp(hyp_phys_to_pfn(tx));
307err_unmap:
308	ffa_unmap_hyp_buffers();
309	goto out_unlock;
310}
311
312static void do_ffa_rxtx_unmap(struct arm_smccc_1_2_regs *res,
313			      struct kvm_cpu_context *ctxt)
314{
315	DECLARE_REG(u32, id, ctxt, 1);
316	int ret = 0;
317
318	if (id != HOST_FFA_ID) {
319		ret = FFA_RET_INVALID_PARAMETERS;
320		goto out;
321	}
322
323	hyp_spin_lock(&host_buffers.lock);
324	if (!host_buffers.tx) {
325		ret = FFA_RET_INVALID_PARAMETERS;
326		goto out_unlock;
327	}
328
329	hyp_unpin_shared_mem(host_buffers.tx, host_buffers.tx + 1);
330	WARN_ON(__pkvm_host_unshare_hyp(hyp_virt_to_pfn(host_buffers.tx)));
331	host_buffers.tx = NULL;
332
333	hyp_unpin_shared_mem(host_buffers.rx, host_buffers.rx + 1);
334	WARN_ON(__pkvm_host_unshare_hyp(hyp_virt_to_pfn(host_buffers.rx)));
335	host_buffers.rx = NULL;
336
337	ffa_unmap_hyp_buffers();
338
339out_unlock:
340	hyp_spin_unlock(&host_buffers.lock);
341out:
342	ffa_to_smccc_res(res, ret);
343}
344
345static u32 __ffa_host_share_ranges(struct ffa_mem_region_addr_range *ranges,
346				   u32 nranges)
347{
348	u32 i;
349
350	for (i = 0; i < nranges; ++i) {
351		struct ffa_mem_region_addr_range *range = &ranges[i];
352		u64 sz = (u64)range->pg_cnt * FFA_PAGE_SIZE;
353		u64 pfn = hyp_phys_to_pfn(range->address);
354
355		if (!PAGE_ALIGNED(sz))
356			break;
357
358		if (__pkvm_host_share_ffa(pfn, sz / PAGE_SIZE))
359			break;
360	}
361
362	return i;
363}
364
365static u32 __ffa_host_unshare_ranges(struct ffa_mem_region_addr_range *ranges,
366				     u32 nranges)
367{
368	u32 i;
369
370	for (i = 0; i < nranges; ++i) {
371		struct ffa_mem_region_addr_range *range = &ranges[i];
372		u64 sz = (u64)range->pg_cnt * FFA_PAGE_SIZE;
373		u64 pfn = hyp_phys_to_pfn(range->address);
374
375		if (!PAGE_ALIGNED(sz))
376			break;
377
378		if (__pkvm_host_unshare_ffa(pfn, sz / PAGE_SIZE))
379			break;
380	}
381
382	return i;
383}
384
385static int ffa_host_share_ranges(struct ffa_mem_region_addr_range *ranges,
386				 u32 nranges)
387{
388	u32 nshared = __ffa_host_share_ranges(ranges, nranges);
389	int ret = 0;
390
391	if (nshared != nranges) {
392		WARN_ON(__ffa_host_unshare_ranges(ranges, nshared) != nshared);
393		ret = FFA_RET_DENIED;
394	}
395
396	return ret;
397}
398
399static int ffa_host_unshare_ranges(struct ffa_mem_region_addr_range *ranges,
400				   u32 nranges)
401{
402	u32 nunshared = __ffa_host_unshare_ranges(ranges, nranges);
403	int ret = 0;
404
405	if (nunshared != nranges) {
406		WARN_ON(__ffa_host_share_ranges(ranges, nunshared) != nunshared);
407		ret = FFA_RET_DENIED;
408	}
409
410	return ret;
411}
412
413static void do_ffa_mem_frag_tx(struct arm_smccc_1_2_regs *res,
414			       struct kvm_cpu_context *ctxt)
415{
416	DECLARE_REG(u32, handle_lo, ctxt, 1);
417	DECLARE_REG(u32, handle_hi, ctxt, 2);
418	DECLARE_REG(u32, fraglen, ctxt, 3);
419	DECLARE_REG(u32, endpoint_id, ctxt, 4);
420	struct ffa_mem_region_addr_range *buf;
421	int ret = FFA_RET_INVALID_PARAMETERS;
422	u32 nr_ranges;
423
424	if (fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE)
425		goto out;
426
427	if (fraglen % sizeof(*buf))
428		goto out;
429
430	hyp_spin_lock(&host_buffers.lock);
431	if (!host_buffers.tx)
432		goto out_unlock;
433
434	buf = hyp_buffers.tx;
435	memcpy(buf, host_buffers.tx, fraglen);
436	nr_ranges = fraglen / sizeof(*buf);
437
438	ret = ffa_host_share_ranges(buf, nr_ranges);
439	if (ret) {
440		/*
441		 * We're effectively aborting the transaction, so we need
442		 * to restore the global state back to what it was prior to
443		 * transmission of the first fragment.
444		 */
445		ffa_mem_reclaim(res, handle_lo, handle_hi, 0);
446		WARN_ON(res->a0 != FFA_SUCCESS);
447		goto out_unlock;
448	}
449
450	ffa_mem_frag_tx(res, handle_lo, handle_hi, fraglen, endpoint_id);
451	if (res->a0 != FFA_SUCCESS && res->a0 != FFA_MEM_FRAG_RX)
452		WARN_ON(ffa_host_unshare_ranges(buf, nr_ranges));
453
454out_unlock:
455	hyp_spin_unlock(&host_buffers.lock);
456out:
457	if (ret)
458		ffa_to_smccc_res(res, ret);
459
460	/*
461	 * If for any reason this did not succeed, we're in trouble as we have
462	 * now lost the content of the previous fragments and we can't rollback
463	 * the host stage-2 changes. The pages previously marked as shared will
464	 * remain stuck in that state forever, hence preventing the host from
465	 * sharing/donating them again and may possibly lead to subsequent
466	 * failures, but this will not compromise confidentiality.
467	 */
468	return;
469}
470
471static void __do_ffa_mem_xfer(const u64 func_id,
472			      struct arm_smccc_1_2_regs *res,
473			      struct kvm_cpu_context *ctxt)
474{
475	DECLARE_REG(u32, len, ctxt, 1);
476	DECLARE_REG(u32, fraglen, ctxt, 2);
477	DECLARE_REG(u64, addr_mbz, ctxt, 3);
478	DECLARE_REG(u32, npages_mbz, ctxt, 4);
479	struct ffa_mem_region_attributes *ep_mem_access;
480	struct ffa_composite_mem_region *reg;
481	struct ffa_mem_region *buf;
482	u32 offset, nr_ranges, checked_offset;
483	int ret = 0;
484
485	if (addr_mbz || npages_mbz || fraglen > len ||
486	    fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) {
487		ret = FFA_RET_INVALID_PARAMETERS;
488		goto out;
489	}
490
491	if (fraglen < sizeof(struct ffa_mem_region) +
492		      sizeof(struct ffa_mem_region_attributes)) {
493		ret = FFA_RET_INVALID_PARAMETERS;
494		goto out;
495	}
496
497	hyp_spin_lock(&host_buffers.lock);
498	if (!host_buffers.tx) {
499		ret = FFA_RET_INVALID_PARAMETERS;
500		goto out_unlock;
501	}
502
503	if (len > ffa_desc_buf.len) {
504		ret = FFA_RET_NO_MEMORY;
505		goto out_unlock;
506	}
507
508	buf = hyp_buffers.tx;
509	memcpy(buf, host_buffers.tx, fraglen);
510
511	ep_mem_access = (void *)buf +
512			ffa_mem_desc_offset(buf, 0, hyp_ffa_version);
513	offset = ep_mem_access->composite_off;
514	if (!offset || buf->ep_count != 1 || buf->sender_id != HOST_FFA_ID) {
515		ret = FFA_RET_INVALID_PARAMETERS;
516		goto out_unlock;
517	}
518
519	if (check_add_overflow(offset, sizeof(struct ffa_composite_mem_region), &checked_offset)) {
520		ret = FFA_RET_INVALID_PARAMETERS;
521		goto out_unlock;
522	}
523
524	if (fraglen < checked_offset) {
525		ret = FFA_RET_INVALID_PARAMETERS;
526		goto out_unlock;
527	}
528
529	reg = (void *)buf + offset;
530	nr_ranges = ((void *)buf + fraglen) - (void *)reg->constituents;
531	if (nr_ranges % sizeof(reg->constituents[0])) {
532		ret = FFA_RET_INVALID_PARAMETERS;
533		goto out_unlock;
534	}
535
536	nr_ranges /= sizeof(reg->constituents[0]);
537	ret = ffa_host_share_ranges(reg->constituents, nr_ranges);
538	if (ret)
539		goto out_unlock;
540
541	ffa_mem_xfer(res, func_id, len, fraglen);
542	if (fraglen != len) {
543		if (res->a0 != FFA_MEM_FRAG_RX)
544			goto err_unshare;
545
546		if (res->a3 != fraglen)
547			goto err_unshare;
548	} else if (res->a0 != FFA_SUCCESS) {
549		goto err_unshare;
550	}
551
552out_unlock:
553	hyp_spin_unlock(&host_buffers.lock);
554out:
555	if (ret)
556		ffa_to_smccc_res(res, ret);
557	return;
558
559err_unshare:
560	WARN_ON(ffa_host_unshare_ranges(reg->constituents, nr_ranges));
561	goto out_unlock;
562}
563
564#define do_ffa_mem_xfer(fid, res, ctxt)				\
565	do {							\
566		BUILD_BUG_ON((fid) != FFA_FN64_MEM_SHARE &&	\
567			     (fid) != FFA_FN64_MEM_LEND);	\
568		__do_ffa_mem_xfer((fid), (res), (ctxt));	\
569	} while (0);
570
571static void do_ffa_mem_reclaim(struct arm_smccc_1_2_regs *res,
572			       struct kvm_cpu_context *ctxt)
573{
574	DECLARE_REG(u32, handle_lo, ctxt, 1);
575	DECLARE_REG(u32, handle_hi, ctxt, 2);
576	DECLARE_REG(u32, flags, ctxt, 3);
577	struct ffa_mem_region_attributes *ep_mem_access;
578	struct ffa_composite_mem_region *reg;
579	u32 offset, len, fraglen, fragoff;
580	struct ffa_mem_region *buf;
581	int ret = 0;
582	u64 handle;
583
584	handle = PACK_HANDLE(handle_lo, handle_hi);
585
586	hyp_spin_lock(&host_buffers.lock);
587
588	buf = hyp_buffers.tx;
589	*buf = (struct ffa_mem_region) {
590		.sender_id	= HOST_FFA_ID,
591		.handle		= handle,
592	};
593
594	ffa_retrieve_req(res, sizeof(*buf));
595	buf = hyp_buffers.rx;
596	if (res->a0 != FFA_MEM_RETRIEVE_RESP)
597		goto out_unlock;
598
599	len = res->a1;
600	fraglen = res->a2;
601
602	ep_mem_access = (void *)buf +
603			ffa_mem_desc_offset(buf, 0, hyp_ffa_version);
604	offset = ep_mem_access->composite_off;
605	/*
606	 * We can trust the SPMD to get this right, but let's at least
607	 * check that we end up with something that doesn't look _completely_
608	 * bogus.
609	 */
610	if (WARN_ON(offset > len ||
611		    fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE)) {
612		ret = FFA_RET_ABORTED;
613		ffa_rx_release(res);
614		goto out_unlock;
615	}
616
617	if (len > ffa_desc_buf.len) {
618		ret = FFA_RET_NO_MEMORY;
619		ffa_rx_release(res);
620		goto out_unlock;
621	}
622
623	buf = ffa_desc_buf.buf;
624	memcpy(buf, hyp_buffers.rx, fraglen);
625	ffa_rx_release(res);
626
627	for (fragoff = fraglen; fragoff < len; fragoff += fraglen) {
628		ffa_mem_frag_rx(res, handle_lo, handle_hi, fragoff);
629		if (res->a0 != FFA_MEM_FRAG_TX) {
630			ret = FFA_RET_INVALID_PARAMETERS;
631			goto out_unlock;
632		}
633
634		fraglen = res->a3;
635		memcpy((void *)buf + fragoff, hyp_buffers.rx, fraglen);
636		ffa_rx_release(res);
637	}
638
639	ffa_mem_reclaim(res, handle_lo, handle_hi, flags);
640	if (res->a0 != FFA_SUCCESS)
641		goto out_unlock;
642
643	reg = (void *)buf + offset;
644	/* If the SPMD was happy, then we should be too. */
645	WARN_ON(ffa_host_unshare_ranges(reg->constituents,
646					reg->addr_range_cnt));
647out_unlock:
648	hyp_spin_unlock(&host_buffers.lock);
649
650	if (ret)
651		ffa_to_smccc_res(res, ret);
652}
653
654/*
655 * Is a given FFA function supported, either by forwarding on directly
656 * or by handling at EL2?
657 */
658static bool ffa_call_supported(u64 func_id)
659{
660	switch (func_id) {
661	/* Unsupported memory management calls */
662	case FFA_FN64_MEM_RETRIEVE_REQ:
663	case FFA_MEM_RETRIEVE_RESP:
664	case FFA_MEM_RELINQUISH:
665	case FFA_MEM_OP_PAUSE:
666	case FFA_MEM_OP_RESUME:
667	case FFA_MEM_FRAG_RX:
668	case FFA_FN64_MEM_DONATE:
669	/* Indirect message passing via RX/TX buffers */
670	case FFA_MSG_SEND:
671	case FFA_MSG_POLL:
672	case FFA_MSG_WAIT:
673	/* 32-bit variants of 64-bit calls */
674	case FFA_MSG_SEND_DIRECT_RESP:
675	case FFA_RXTX_MAP:
676	case FFA_MEM_DONATE:
677	case FFA_MEM_RETRIEVE_REQ:
678       /* Optional notification interfaces added in FF-A 1.1 */
679	case FFA_NOTIFICATION_BITMAP_CREATE:
680	case FFA_NOTIFICATION_BITMAP_DESTROY:
681	case FFA_NOTIFICATION_BIND:
682	case FFA_NOTIFICATION_UNBIND:
683	case FFA_NOTIFICATION_SET:
684	case FFA_NOTIFICATION_GET:
685	case FFA_NOTIFICATION_INFO_GET:
686	/* Optional interfaces added in FF-A 1.2 */
687	case FFA_MSG_SEND_DIRECT_REQ2:		/* Optional per 7.5.1 */
688	case FFA_MSG_SEND_DIRECT_RESP2:		/* Optional per 7.5.1 */
689	case FFA_CONSOLE_LOG:			/* Optional per 13.1: not in Table 13.1 */
690	case FFA_PARTITION_INFO_GET_REGS:	/* Optional for virtual instances per 13.1 */
691		return false;
692	}
693
694	return true;
695}
696
697static bool do_ffa_features(struct arm_smccc_1_2_regs *res,
698			    struct kvm_cpu_context *ctxt)
699{
700	DECLARE_REG(u32, id, ctxt, 1);
701	u64 prop = 0;
702	int ret = 0;
703
704	if (!ffa_call_supported(id)) {
705		ret = FFA_RET_NOT_SUPPORTED;
706		goto out_handled;
707	}
708
709	switch (id) {
710	case FFA_MEM_SHARE:
711	case FFA_FN64_MEM_SHARE:
712	case FFA_MEM_LEND:
713	case FFA_FN64_MEM_LEND:
714		ret = FFA_RET_SUCCESS;
715		prop = 0; /* No support for dynamic buffers */
716		goto out_handled;
717	default:
718		return false;
719	}
720
721out_handled:
722	ffa_to_smccc_res_prop(res, ret, prop);
723	return true;
724}
725
726static int hyp_ffa_post_init(void)
727{
728	size_t min_rxtx_sz;
729	struct arm_smccc_1_2_regs res;
730
731	arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs){
732		.a0 = FFA_ID_GET,
733	}, &res);
734	if (res.a0 != FFA_SUCCESS)
735		return -EOPNOTSUPP;
736
737	if (res.a2 != HOST_FFA_ID)
738		return -EINVAL;
739
740	arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs){
741		.a0 = FFA_FEATURES,
742		.a1 = FFA_FN64_RXTX_MAP,
743	}, &res);
744	if (res.a0 != FFA_SUCCESS)
745		return -EOPNOTSUPP;
746
747	switch (res.a2 & FFA_FEAT_RXTX_MIN_SZ_MASK) {
748	case FFA_FEAT_RXTX_MIN_SZ_4K:
749		min_rxtx_sz = SZ_4K;
750		break;
751	case FFA_FEAT_RXTX_MIN_SZ_16K:
752		min_rxtx_sz = SZ_16K;
753		break;
754	case FFA_FEAT_RXTX_MIN_SZ_64K:
755		min_rxtx_sz = SZ_64K;
756		break;
757	default:
758		return -EINVAL;
759	}
760
761	if (min_rxtx_sz > PAGE_SIZE)
762		return -EOPNOTSUPP;
763
764	return 0;
765}
766
767static void do_ffa_version(struct arm_smccc_1_2_regs *res,
768			   struct kvm_cpu_context *ctxt)
769{
770	DECLARE_REG(u32, ffa_req_version, ctxt, 1);
771
772	if (FFA_MAJOR_VERSION(ffa_req_version) != 1) {
773		res->a0 = FFA_RET_NOT_SUPPORTED;
774		return;
775	}
776
777	hyp_spin_lock(&version_lock);
778	if (has_version_negotiated) {
779		if (FFA_MINOR_VERSION(ffa_req_version) < FFA_MINOR_VERSION(hyp_ffa_version))
780			res->a0 = FFA_RET_NOT_SUPPORTED;
781		else
782			res->a0 = hyp_ffa_version;
783		goto unlock;
784	}
785
786	/*
787	 * If the client driver tries to downgrade the version, we need to ask
788	 * first if TEE supports it.
789	 */
790	if (FFA_MINOR_VERSION(ffa_req_version) < FFA_MINOR_VERSION(hyp_ffa_version)) {
791		arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
792			.a0 = FFA_VERSION,
793			.a1 = ffa_req_version,
794		}, res);
795		if (res->a0 == FFA_RET_NOT_SUPPORTED)
796			goto unlock;
797
798		hyp_ffa_version = ffa_req_version;
799	}
800
801	if (hyp_ffa_post_init()) {
802		res->a0 = FFA_RET_NOT_SUPPORTED;
803	} else {
804		smp_store_release(&has_version_negotiated, true);
805		res->a0 = hyp_ffa_version;
806	}
807unlock:
808	hyp_spin_unlock(&version_lock);
809}
810
811static void do_ffa_part_get(struct arm_smccc_1_2_regs *res,
812			    struct kvm_cpu_context *ctxt)
813{
814	DECLARE_REG(u32, uuid0, ctxt, 1);
815	DECLARE_REG(u32, uuid1, ctxt, 2);
816	DECLARE_REG(u32, uuid2, ctxt, 3);
817	DECLARE_REG(u32, uuid3, ctxt, 4);
818	DECLARE_REG(u32, flags, ctxt, 5);
819	u32 count, partition_sz, copy_sz;
820
821	hyp_spin_lock(&host_buffers.lock);
822	if (!host_buffers.rx) {
823		ffa_to_smccc_res(res, FFA_RET_BUSY);
824		goto out_unlock;
825	}
826
827	arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
828		.a0 = FFA_PARTITION_INFO_GET,
829		.a1 = uuid0,
830		.a2 = uuid1,
831		.a3 = uuid2,
832		.a4 = uuid3,
833		.a5 = flags,
834	}, res);
835
836	if (res->a0 != FFA_SUCCESS)
837		goto out_unlock;
838
839	count = res->a2;
840	if (!count)
841		goto out_unlock;
842
843	if (hyp_ffa_version > FFA_VERSION_1_0) {
844		/* Get the number of partitions deployed in the system */
845		if (flags & 0x1)
846			goto out_unlock;
847
848		partition_sz  = res->a3;
849	} else {
850		/* FFA_VERSION_1_0 lacks the size in the response */
851		partition_sz = FFA_1_0_PARTITON_INFO_SZ;
852	}
853
854	copy_sz = partition_sz * count;
855	if (copy_sz > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) {
856		ffa_to_smccc_res(res, FFA_RET_ABORTED);
857		goto out_unlock;
858	}
859
860	memcpy(host_buffers.rx, hyp_buffers.rx, copy_sz);
861out_unlock:
862	hyp_spin_unlock(&host_buffers.lock);
863}
864
865bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id)
866{
867	struct arm_smccc_1_2_regs res;
868
869	/*
870	 * There's no way we can tell what a non-standard SMC call might
871	 * be up to. Ideally, we would terminate these here and return
872	 * an error to the host, but sadly devices make use of custom
873	 * firmware calls for things like power management, debugging,
874	 * RNG access and crash reporting.
875	 *
876	 * Given that the architecture requires us to trust EL3 anyway,
877	 * we forward unrecognised calls on under the assumption that
878	 * the firmware doesn't expose a mechanism to access arbitrary
879	 * non-secure memory. Short of a per-device table of SMCs, this
880	 * is the best we can do.
881	 */
882	if (!is_ffa_call(func_id))
883		return false;
884
885	if (func_id != FFA_VERSION &&
886	    !smp_load_acquire(&has_version_negotiated)) {
887		ffa_to_smccc_error(&res, FFA_RET_INVALID_PARAMETERS);
888		goto out_handled;
889	}
890
891	switch (func_id) {
892	case FFA_FEATURES:
893		if (!do_ffa_features(&res, host_ctxt))
894			return false;
895		goto out_handled;
896	/* Memory management */
897	case FFA_FN64_RXTX_MAP:
898		do_ffa_rxtx_map(&res, host_ctxt);
899		goto out_handled;
900	case FFA_RXTX_UNMAP:
901		do_ffa_rxtx_unmap(&res, host_ctxt);
902		goto out_handled;
903	case FFA_MEM_SHARE:
904	case FFA_FN64_MEM_SHARE:
905		do_ffa_mem_xfer(FFA_FN64_MEM_SHARE, &res, host_ctxt);
906		goto out_handled;
907	case FFA_MEM_RECLAIM:
908		do_ffa_mem_reclaim(&res, host_ctxt);
909		goto out_handled;
910	case FFA_MEM_LEND:
911	case FFA_FN64_MEM_LEND:
912		do_ffa_mem_xfer(FFA_FN64_MEM_LEND, &res, host_ctxt);
913		goto out_handled;
914	case FFA_MEM_FRAG_TX:
915		do_ffa_mem_frag_tx(&res, host_ctxt);
916		goto out_handled;
917	case FFA_VERSION:
918		do_ffa_version(&res, host_ctxt);
919		goto out_handled;
920	case FFA_PARTITION_INFO_GET:
921		do_ffa_part_get(&res, host_ctxt);
922		goto out_handled;
923	}
924
925	if (ffa_call_supported(func_id))
926		return false; /* Pass through */
927
928	ffa_to_smccc_error(&res, FFA_RET_NOT_SUPPORTED);
929out_handled:
930	ffa_set_retval(host_ctxt, &res);
931	return true;
932}
933
934int hyp_ffa_init(void *pages)
935{
936	struct arm_smccc_1_2_regs res;
937	void *tx, *rx;
938
939	if (kvm_host_psci_config.smccc_version < ARM_SMCCC_VERSION_1_2)
940		return 0;
941
942	arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
943		.a0 = FFA_VERSION,
944		.a1 = FFA_VERSION_1_2,
945	}, &res);
946	if (res.a0 == FFA_RET_NOT_SUPPORTED)
947		return 0;
948
949	/*
950	 * Firmware returns the maximum supported version of the FF-A
951	 * implementation. Check that the returned version is
952	 * backwards-compatible with the hyp according to the rules in DEN0077A
953	 * v1.1 REL0 13.2.1.
954	 *
955	 * Of course, things are never simple when dealing with firmware. v1.1
956	 * broke ABI with v1.0 on several structures, which is itself
957	 * incompatible with the aforementioned versioning scheme. The
958	 * expectation is that v1.x implementations that do not support the v1.0
959	 * ABI return NOT_SUPPORTED rather than a version number, according to
960	 * DEN0077A v1.1 REL0 18.6.4.
961	 */
962	if (FFA_MAJOR_VERSION(res.a0) != 1)
963		return -EOPNOTSUPP;
964
965	if (FFA_MINOR_VERSION(res.a0) < FFA_MINOR_VERSION(FFA_VERSION_1_2))
966		hyp_ffa_version = res.a0;
967	else
968		hyp_ffa_version = FFA_VERSION_1_2;
969
970	tx = pages;
971	pages += KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE;
972	rx = pages;
973	pages += KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE;
974
975	ffa_desc_buf = (struct kvm_ffa_descriptor_buffer) {
976		.buf	= pages,
977		.len	= PAGE_SIZE *
978			  (hyp_ffa_proxy_pages() - (2 * KVM_FFA_MBOX_NR_PAGES)),
979	};
980
981	hyp_buffers = (struct kvm_ffa_buffers) {
982		.lock	= __HYP_SPIN_LOCK_UNLOCKED,
983		.tx	= tx,
984		.rx	= rx,
985	};
986
987	host_buffers = (struct kvm_ffa_buffers) {
988		.lock	= __HYP_SPIN_LOCK_UNLOCKED,
989	};
990
991	version_lock = __HYP_SPIN_LOCK_UNLOCKED;
992	return 0;
993}