ffa_abi.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (c) 2021, 2023 Linaro Limited
   4 */
   5
   6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   7
   8#include <linux/arm_ffa.h>
   9#include <linux/errno.h>
  10#include <linux/rpmb.h>
  11#include <linux/scatterlist.h>
  12#include <linux/sched.h>
  13#include <linux/slab.h>
  14#include <linux/string.h>
  15#include <linux/tee_core.h>
  16#include <linux/types.h>
  17#include "optee_private.h"
  18#include "optee_ffa.h"
  19#include "optee_rpc_cmd.h"
  20
  21/*
  22 * This file implement the FF-A ABI used when communicating with secure world
  23 * OP-TEE OS via FF-A.
  24 * This file is divided into the following sections:
  25 * 1. Maintain a hash table for lookup of a global FF-A memory handle
  26 * 2. Convert between struct tee_param and struct optee_msg_param
  27 * 3. Low level support functions to register shared memory in secure world
  28 * 4. Dynamic shared memory pool based on alloc_pages()
  29 * 5. Do a normal scheduled call into secure world
  30 * 6. Driver initialization.
  31 */
  32
  33/*
  34 * 1. Maintain a hash table for lookup of a global FF-A memory handle
  35 *
  36 * FF-A assigns a global memory handle for each piece shared memory.
  37 * This handle is then used when communicating with secure world.
  38 *
  39 * Main functions are optee_shm_add_ffa_handle() and optee_shm_rem_ffa_handle()
  40 */
  41struct shm_rhash {
  42	struct tee_shm *shm;
  43	u64 global_id;
  44	struct rhash_head linkage;
  45};
  46
  47static void rh_free_fn(void *ptr, void *arg)
  48{
  49	kfree(ptr);
  50}
  51
  52static const struct rhashtable_params shm_rhash_params = {
  53	.head_offset = offsetof(struct shm_rhash, linkage),
  54	.key_len     = sizeof(u64),
  55	.key_offset  = offsetof(struct shm_rhash, global_id),
  56	.automatic_shrinking = true,
  57};
  58
  59static struct tee_shm *optee_shm_from_ffa_handle(struct optee *optee,
  60						 u64 global_id)
  61{
  62	struct tee_shm *shm = NULL;
  63	struct shm_rhash *r;
  64
  65	mutex_lock(&optee->ffa.mutex);
  66	r = rhashtable_lookup_fast(&optee->ffa.global_ids, &global_id,
  67				   shm_rhash_params);
  68	if (r)
  69		shm = r->shm;
  70	mutex_unlock(&optee->ffa.mutex);
  71
  72	return shm;
  73}
  74
  75static int optee_shm_add_ffa_handle(struct optee *optee, struct tee_shm *shm,
  76				    u64 global_id)
  77{
  78	struct shm_rhash *r;
  79	int rc;
  80
  81	r = kmalloc(sizeof(*r), GFP_KERNEL);
  82	if (!r)
  83		return -ENOMEM;
  84	r->shm = shm;
  85	r->global_id = global_id;
  86
  87	mutex_lock(&optee->ffa.mutex);
  88	rc = rhashtable_lookup_insert_fast(&optee->ffa.global_ids, &r->linkage,
  89					   shm_rhash_params);
  90	mutex_unlock(&optee->ffa.mutex);
  91
  92	if (rc)
  93		kfree(r);
  94
  95	return rc;
  96}
  97
  98static int optee_shm_rem_ffa_handle(struct optee *optee, u64 global_id)
  99{
 100	struct shm_rhash *r;
 101	int rc = -ENOENT;
 102
 103	mutex_lock(&optee->ffa.mutex);
 104	r = rhashtable_lookup_fast(&optee->ffa.global_ids, &global_id,
 105				   shm_rhash_params);
 106	if (r)
 107		rc = rhashtable_remove_fast(&optee->ffa.global_ids,
 108					    &r->linkage, shm_rhash_params);
 109	mutex_unlock(&optee->ffa.mutex);
 110
 111	if (!rc)
 112		kfree(r);
 113
 114	return rc;
 115}
 116
 117/*
 118 * 2. Convert between struct tee_param and struct optee_msg_param
 119 *
 120 * optee_ffa_from_msg_param() and optee_ffa_to_msg_param() are the main
 121 * functions.
 122 */
 123
 124static void from_msg_param_ffa_mem(struct optee *optee, struct tee_param *p,
 125				   u32 attr, const struct optee_msg_param *mp)
 126{
 127	struct tee_shm *shm = NULL;
 128	u64 offs_high = 0;
 129	u64 offs_low = 0;
 130
 131	p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT +
 132		  attr - OPTEE_MSG_ATTR_TYPE_FMEM_INPUT;
 133	p->u.memref.size = mp->u.fmem.size;
 134
 135	if (mp->u.fmem.global_id != OPTEE_MSG_FMEM_INVALID_GLOBAL_ID)
 136		shm = optee_shm_from_ffa_handle(optee, mp->u.fmem.global_id);
 137	p->u.memref.shm = shm;
 138
 139	if (shm) {
 140		offs_low = mp->u.fmem.offs_low;
 141		offs_high = mp->u.fmem.offs_high;
 142	}
 143	p->u.memref.shm_offs = offs_low | offs_high << 32;
 144}
 145
 146/**
 147 * optee_ffa_from_msg_param() - convert from OPTEE_MSG parameters to
 148 *				struct tee_param
 149 * @optee:	main service struct
 150 * @params:	subsystem internal parameter representation
 151 * @num_params:	number of elements in the parameter arrays
 152 * @msg_params:	OPTEE_MSG parameters
 153 *
 154 * Returns 0 on success or <0 on failure
 155 */
 156static int optee_ffa_from_msg_param(struct optee *optee,
 157				    struct tee_param *params, size_t num_params,
 158				    const struct optee_msg_param *msg_params)
 159{
 160	size_t n;
 161
 162	for (n = 0; n < num_params; n++) {
 163		struct tee_param *p = params + n;
 164		const struct optee_msg_param *mp = msg_params + n;
 165		u32 attr = mp->attr & OPTEE_MSG_ATTR_TYPE_MASK;
 166
 167		switch (attr) {
 168		case OPTEE_MSG_ATTR_TYPE_NONE:
 169			p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_NONE;
 170			memset(&p->u, 0, sizeof(p->u));
 171			break;
 172		case OPTEE_MSG_ATTR_TYPE_VALUE_INPUT:
 173		case OPTEE_MSG_ATTR_TYPE_VALUE_OUTPUT:
 174		case OPTEE_MSG_ATTR_TYPE_VALUE_INOUT:
 175			optee_from_msg_param_value(p, attr, mp);
 176			break;
 177		case OPTEE_MSG_ATTR_TYPE_FMEM_INPUT:
 178		case OPTEE_MSG_ATTR_TYPE_FMEM_OUTPUT:
 179		case OPTEE_MSG_ATTR_TYPE_FMEM_INOUT:
 180			from_msg_param_ffa_mem(optee, p, attr, mp);
 181			break;
 182		default:
 183			return -EINVAL;
 184		}
 185	}
 186
 187	return 0;
 188}
 189
 190static int to_msg_param_ffa_mem(struct optee_msg_param *mp,
 191				const struct tee_param *p)
 192{
 193	struct tee_shm *shm = p->u.memref.shm;
 194
 195	mp->attr = OPTEE_MSG_ATTR_TYPE_FMEM_INPUT + p->attr -
 196		   TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT;
 197
 198	if (shm) {
 199		u64 shm_offs = p->u.memref.shm_offs;
 200
 201		mp->u.fmem.internal_offs = shm->offset;
 202
 203		mp->u.fmem.offs_low = shm_offs;
 204		mp->u.fmem.offs_high = shm_offs >> 32;
 205		/* Check that the entire offset could be stored. */
 206		if (mp->u.fmem.offs_high != shm_offs >> 32)
 207			return -EINVAL;
 208
 209		mp->u.fmem.global_id = shm->sec_world_id;
 210	} else {
 211		memset(&mp->u, 0, sizeof(mp->u));
 212		mp->u.fmem.global_id = OPTEE_MSG_FMEM_INVALID_GLOBAL_ID;
 213	}
 214	mp->u.fmem.size = p->u.memref.size;
 215
 216	return 0;
 217}
 218
 219/**
 220 * optee_ffa_to_msg_param() - convert from struct tee_params to OPTEE_MSG
 221 *			      parameters
 222 * @optee:	main service struct
 223 * @msg_params:	OPTEE_MSG parameters
 224 * @num_params:	number of elements in the parameter arrays
 225 * @params:	subsystem itnernal parameter representation
 226 * Returns 0 on success or <0 on failure
 227 */
 228static int optee_ffa_to_msg_param(struct optee *optee,
 229				  struct optee_msg_param *msg_params,
 230				  size_t num_params,
 231				  const struct tee_param *params)
 232{
 233	size_t n;
 234
 235	for (n = 0; n < num_params; n++) {
 236		const struct tee_param *p = params + n;
 237		struct optee_msg_param *mp = msg_params + n;
 238
 239		switch (p->attr) {
 240		case TEE_IOCTL_PARAM_ATTR_TYPE_NONE:
 241			mp->attr = TEE_IOCTL_PARAM_ATTR_TYPE_NONE;
 242			memset(&mp->u, 0, sizeof(mp->u));
 243			break;
 244		case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT:
 245		case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_OUTPUT:
 246		case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT:
 247			optee_to_msg_param_value(mp, p);
 248			break;
 249		case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT:
 250		case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT:
 251		case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT:
 252			if (to_msg_param_ffa_mem(mp, p))
 253				return -EINVAL;
 254			break;
 255		default:
 256			return -EINVAL;
 257		}
 258	}
 259
 260	return 0;
 261}
 262
 263/*
 264 * 3. Low level support functions to register shared memory in secure world
 265 *
 266 * Functions to register and unregister shared memory both for normal
 267 * clients and for tee-supplicant.
 268 */
 269
 270static int optee_ffa_shm_register(struct tee_context *ctx, struct tee_shm *shm,
 271				  struct page **pages, size_t num_pages,
 272				  unsigned long start)
 273{
 274	struct optee *optee = tee_get_drvdata(ctx->teedev);
 275	struct ffa_device *ffa_dev = optee->ffa.ffa_dev;
 276	const struct ffa_mem_ops *mem_ops = ffa_dev->ops->mem_ops;
 277	struct ffa_mem_region_attributes mem_attr = {
 278		.receiver = ffa_dev->vm_id,
 279		.attrs = FFA_MEM_RW,
 280	};
 281	struct ffa_mem_ops_args args = {
 282		.use_txbuf = true,
 283		.attrs = &mem_attr,
 284		.nattrs = 1,
 285	};
 286	struct sg_table sgt;
 287	int rc;
 288
 289	rc = optee_check_mem_type(start, num_pages);
 290	if (rc)
 291		return rc;
 292
 293	rc = sg_alloc_table_from_pages(&sgt, pages, num_pages, 0,
 294				       num_pages * PAGE_SIZE, GFP_KERNEL);
 295	if (rc)
 296		return rc;
 297	args.sg = sgt.sgl;
 298	rc = mem_ops->memory_share(&args);
 299	sg_free_table(&sgt);
 300	if (rc)
 301		return rc;
 302
 303	rc = optee_shm_add_ffa_handle(optee, shm, args.g_handle);
 304	if (rc) {
 305		mem_ops->memory_reclaim(args.g_handle, 0);
 306		return rc;
 307	}
 308
 309	shm->sec_world_id = args.g_handle;
 310
 311	return 0;
 312}
 313
 314static int optee_ffa_shm_unregister(struct tee_context *ctx,
 315				    struct tee_shm *shm)
 316{
 317	struct optee *optee = tee_get_drvdata(ctx->teedev);
 318	struct ffa_device *ffa_dev = optee->ffa.ffa_dev;
 319	const struct ffa_msg_ops *msg_ops = ffa_dev->ops->msg_ops;
 320	const struct ffa_mem_ops *mem_ops = ffa_dev->ops->mem_ops;
 321	u64 global_handle = shm->sec_world_id;
 322	struct ffa_send_direct_data data = {
 323		.data0 = OPTEE_FFA_UNREGISTER_SHM,
 324		.data1 = (u32)global_handle,
 325		.data2 = (u32)(global_handle >> 32)
 326	};
 327	int rc;
 328
 329	optee_shm_rem_ffa_handle(optee, global_handle);
 330	shm->sec_world_id = 0;
 331
 332	rc = msg_ops->sync_send_receive(ffa_dev, &data);
 333	if (rc)
 334		pr_err("Unregister SHM id 0x%llx rc %d\n", global_handle, rc);
 335
 336	rc = mem_ops->memory_reclaim(global_handle, 0);
 337	if (rc)
 338		pr_err("mem_reclaim: 0x%llx %d", global_handle, rc);
 339
 340	return rc;
 341}
 342
 343static int optee_ffa_shm_unregister_supp(struct tee_context *ctx,
 344					 struct tee_shm *shm)
 345{
 346	struct optee *optee = tee_get_drvdata(ctx->teedev);
 347	const struct ffa_mem_ops *mem_ops;
 348	u64 global_handle = shm->sec_world_id;
 349	int rc;
 350
 351	/*
 352	 * We're skipping the OPTEE_FFA_YIELDING_CALL_UNREGISTER_SHM call
 353	 * since this is OP-TEE freeing via RPC so it has already retired
 354	 * this ID.
 355	 */
 356
 357	optee_shm_rem_ffa_handle(optee, global_handle);
 358	mem_ops = optee->ffa.ffa_dev->ops->mem_ops;
 359	rc = mem_ops->memory_reclaim(global_handle, 0);
 360	if (rc)
 361		pr_err("mem_reclaim: 0x%llx %d", global_handle, rc);
 362
 363	shm->sec_world_id = 0;
 364
 365	return rc;
 366}
 367
 368/*
 369 * 4. Dynamic shared memory pool based on alloc_pages()
 370 *
 371 * Implements an OP-TEE specific shared memory pool.
 372 * The main function is optee_ffa_shm_pool_alloc_pages().
 373 */
 374
 375static int pool_ffa_op_alloc(struct tee_shm_pool *pool,
 376			     struct tee_shm *shm, size_t size, size_t align)
 377{
 378	return tee_dyn_shm_alloc_helper(shm, size, align,
 379					optee_ffa_shm_register);
 380}
 381
 382static void pool_ffa_op_free(struct tee_shm_pool *pool,
 383			     struct tee_shm *shm)
 384{
 385	tee_dyn_shm_free_helper(shm, optee_ffa_shm_unregister);
 386}
 387
 388static void pool_ffa_op_destroy_pool(struct tee_shm_pool *pool)
 389{
 390	kfree(pool);
 391}
 392
 393static const struct tee_shm_pool_ops pool_ffa_ops = {
 394	.alloc = pool_ffa_op_alloc,
 395	.free = pool_ffa_op_free,
 396	.destroy_pool = pool_ffa_op_destroy_pool,
 397};
 398
 399/**
 400 * optee_ffa_shm_pool_alloc_pages() - create page-based allocator pool
 401 *
 402 * This pool is used with OP-TEE over FF-A. In this case command buffers
 403 * and such are allocated from kernel's own memory.
 404 */
 405static struct tee_shm_pool *optee_ffa_shm_pool_alloc_pages(void)
 406{
 407	struct tee_shm_pool *pool = kzalloc(sizeof(*pool), GFP_KERNEL);
 408
 409	if (!pool)
 410		return ERR_PTR(-ENOMEM);
 411
 412	pool->ops = &pool_ffa_ops;
 413
 414	return pool;
 415}
 416
 417/*
 418 * 5. Do a normal scheduled call into secure world
 419 *
 420 * The function optee_ffa_do_call_with_arg() performs a normal scheduled
 421 * call into secure world. During this call may normal world request help
 422 * from normal world using RPCs, Remote Procedure Calls. This includes
 423 * delivery of non-secure interrupts to for instance allow rescheduling of
 424 * the current task.
 425 */
 426
 427static void handle_ffa_rpc_func_cmd_shm_alloc(struct tee_context *ctx,
 428					      struct optee *optee,
 429					      struct optee_msg_arg *arg)
 430{
 431	struct tee_shm *shm;
 432
 433	if (arg->num_params != 1 ||
 434	    arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) {
 435		arg->ret = TEEC_ERROR_BAD_PARAMETERS;
 436		return;
 437	}
 438
 439	switch (arg->params[0].u.value.a) {
 440	case OPTEE_RPC_SHM_TYPE_APPL:
 441		shm = optee_rpc_cmd_alloc_suppl(ctx, arg->params[0].u.value.b);
 442		break;
 443	case OPTEE_RPC_SHM_TYPE_KERNEL:
 444		shm = tee_shm_alloc_priv_buf(optee->ctx,
 445					     arg->params[0].u.value.b);
 446		break;
 447	default:
 448		arg->ret = TEEC_ERROR_BAD_PARAMETERS;
 449		return;
 450	}
 451
 452	if (IS_ERR(shm)) {
 453		arg->ret = TEEC_ERROR_OUT_OF_MEMORY;
 454		return;
 455	}
 456
 457	arg->params[0] = (struct optee_msg_param){
 458		.attr = OPTEE_MSG_ATTR_TYPE_FMEM_OUTPUT,
 459		.u.fmem.size = tee_shm_get_size(shm),
 460		.u.fmem.global_id = shm->sec_world_id,
 461		.u.fmem.internal_offs = shm->offset,
 462	};
 463
 464	arg->ret = TEEC_SUCCESS;
 465}
 466
 467static void handle_ffa_rpc_func_cmd_shm_free(struct tee_context *ctx,
 468					     struct optee *optee,
 469					     struct optee_msg_arg *arg)
 470{
 471	struct tee_shm *shm;
 472
 473	if (arg->num_params != 1 ||
 474	    arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT)
 475		goto err_bad_param;
 476
 477	shm = optee_shm_from_ffa_handle(optee, arg->params[0].u.value.b);
 478	if (!shm)
 479		goto err_bad_param;
 480	switch (arg->params[0].u.value.a) {
 481	case OPTEE_RPC_SHM_TYPE_APPL:
 482		optee_rpc_cmd_free_suppl(ctx, shm);
 483		break;
 484	case OPTEE_RPC_SHM_TYPE_KERNEL:
 485		tee_shm_free(shm);
 486		break;
 487	default:
 488		goto err_bad_param;
 489	}
 490	arg->ret = TEEC_SUCCESS;
 491	return;
 492
 493err_bad_param:
 494	arg->ret = TEEC_ERROR_BAD_PARAMETERS;
 495}
 496
 497static void handle_ffa_rpc_func_cmd(struct tee_context *ctx,
 498				    struct optee *optee,
 499				    struct optee_msg_arg *arg)
 500{
 501	arg->ret_origin = TEEC_ORIGIN_COMMS;
 502	switch (arg->cmd) {
 503	case OPTEE_RPC_CMD_SHM_ALLOC:
 504		handle_ffa_rpc_func_cmd_shm_alloc(ctx, optee, arg);
 505		break;
 506	case OPTEE_RPC_CMD_SHM_FREE:
 507		handle_ffa_rpc_func_cmd_shm_free(ctx, optee, arg);
 508		break;
 509	default:
 510		optee_rpc_cmd(ctx, optee, arg);
 511	}
 512}
 513
 514static void optee_handle_ffa_rpc(struct tee_context *ctx, struct optee *optee,
 515				 u32 cmd, struct optee_msg_arg *arg)
 516{
 517	switch (cmd) {
 518	case OPTEE_FFA_YIELDING_CALL_RETURN_RPC_CMD:
 519		handle_ffa_rpc_func_cmd(ctx, optee, arg);
 520		break;
 521	case OPTEE_FFA_YIELDING_CALL_RETURN_INTERRUPT:
 522		/* Interrupt delivered by now */
 523		break;
 524	default:
 525		pr_warn("Unknown RPC func 0x%x\n", cmd);
 526		break;
 527	}
 528}
 529
 530static int optee_ffa_yielding_call(struct tee_context *ctx,
 531				   struct ffa_send_direct_data *data,
 532				   struct optee_msg_arg *rpc_arg,
 533				   bool system_thread)
 534{
 535	struct optee *optee = tee_get_drvdata(ctx->teedev);
 536	struct ffa_device *ffa_dev = optee->ffa.ffa_dev;
 537	const struct ffa_msg_ops *msg_ops = ffa_dev->ops->msg_ops;
 538	struct optee_call_waiter w;
 539	u32 cmd = data->data0;
 540	u32 w4 = data->data1;
 541	u32 w5 = data->data2;
 542	u32 w6 = data->data3;
 543	int rc;
 544
 545	/* Initialize waiter */
 546	optee_cq_wait_init(&optee->call_queue, &w, system_thread);
 547	while (true) {
 548		rc = msg_ops->sync_send_receive(ffa_dev, data);
 549		if (rc)
 550			goto done;
 551
 552		switch ((int)data->data0) {
 553		case TEEC_SUCCESS:
 554			break;
 555		case TEEC_ERROR_BUSY:
 556			if (cmd == OPTEE_FFA_YIELDING_CALL_RESUME) {
 557				rc = -EIO;
 558				goto done;
 559			}
 560
 561			/*
 562			 * Out of threads in secure world, wait for a thread
 563			 * become available.
 564			 */
 565			optee_cq_wait_for_completion(&optee->call_queue, &w);
 566			data->data0 = cmd;
 567			data->data1 = w4;
 568			data->data2 = w5;
 569			data->data3 = w6;
 570			continue;
 571		default:
 572			rc = -EIO;
 573			goto done;
 574		}
 575
 576		if (data->data1 == OPTEE_FFA_YIELDING_CALL_RETURN_DONE)
 577			goto done;
 578
 579		/*
 580		 * OP-TEE has returned with a RPC request.
 581		 *
 582		 * Note that data->data4 (passed in register w7) is already
 583		 * filled in by ffa_mem_ops->sync_send_receive() returning
 584		 * above.
 585		 */
 586		cond_resched();
 587		optee_handle_ffa_rpc(ctx, optee, data->data1, rpc_arg);
 588		cmd = OPTEE_FFA_YIELDING_CALL_RESUME;
 589		data->data0 = cmd;
 590		data->data1 = 0;
 591		data->data2 = 0;
 592		data->data3 = 0;
 593	}
 594done:
 595	/*
 596	 * We're done with our thread in secure world, if there's any
 597	 * thread waiters wake up one.
 598	 */
 599	optee_cq_wait_final(&optee->call_queue, &w);
 600
 601	return rc;
 602}
 603
 604/**
 605 * optee_ffa_do_call_with_arg() - Do a FF-A call to enter OP-TEE in secure world
 606 * @ctx:	calling context
 607 * @shm:	shared memory holding the message to pass to secure world
 608 * @offs:	offset of the message in @shm
 609 * @system_thread: true if caller requests TEE system thread support
 610 *
 611 * Does a FF-A call to OP-TEE in secure world and handles eventual resulting
 612 * Remote Procedure Calls (RPC) from OP-TEE.
 613 *
 614 * Returns return code from FF-A, 0 is OK
 615 */
 616
 617static int optee_ffa_do_call_with_arg(struct tee_context *ctx,
 618				      struct tee_shm *shm, u_int offs,
 619				      bool system_thread)
 620{
 621	struct ffa_send_direct_data data = {
 622		.data0 = OPTEE_FFA_YIELDING_CALL_WITH_ARG,
 623		.data1 = (u32)shm->sec_world_id,
 624		.data2 = (u32)(shm->sec_world_id >> 32),
 625		.data3 = offs,
 626	};
 627	struct optee_msg_arg *arg;
 628	unsigned int rpc_arg_offs;
 629	struct optee_msg_arg *rpc_arg;
 630
 631	/*
 632	 * The shared memory object has to start on a page when passed as
 633	 * an argument struct. This is also what the shm pool allocator
 634	 * returns, but check this before calling secure world to catch
 635	 * eventual errors early in case something changes.
 636	 */
 637	if (shm->offset)
 638		return -EINVAL;
 639
 640	arg = tee_shm_get_va(shm, offs);
 641	if (IS_ERR(arg))
 642		return PTR_ERR(arg);
 643
 644	rpc_arg_offs = OPTEE_MSG_GET_ARG_SIZE(arg->num_params);
 645	rpc_arg = tee_shm_get_va(shm, offs + rpc_arg_offs);
 646	if (IS_ERR(rpc_arg))
 647		return PTR_ERR(rpc_arg);
 648
 649	return optee_ffa_yielding_call(ctx, &data, rpc_arg, system_thread);
 650}
 651
 652static int do_call_lend_protmem(struct optee *optee, u64 cookie, u32 use_case)
 653{
 654	struct optee_shm_arg_entry *entry;
 655	struct optee_msg_arg *msg_arg;
 656	struct tee_shm *shm;
 657	u_int offs;
 658	int rc;
 659
 660	msg_arg = optee_get_msg_arg(optee->ctx, 1, &entry, &shm, &offs);
 661	if (IS_ERR(msg_arg))
 662		return PTR_ERR(msg_arg);
 663
 664	msg_arg->cmd = OPTEE_MSG_CMD_ASSIGN_PROTMEM;
 665	msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_VALUE_INPUT;
 666	msg_arg->params[0].u.value.a = cookie;
 667	msg_arg->params[0].u.value.b = use_case;
 668
 669	rc = optee->ops->do_call_with_arg(optee->ctx, shm, offs, false);
 670	if (rc)
 671		goto out;
 672	if (msg_arg->ret != TEEC_SUCCESS) {
 673		rc = -EINVAL;
 674		goto out;
 675	}
 676
 677out:
 678	optee_free_msg_arg(optee->ctx, entry, offs);
 679	return rc;
 680}
 681
 682static int optee_ffa_lend_protmem(struct optee *optee, struct tee_shm *protmem,
 683				  u32 *mem_attrs, unsigned int ma_count,
 684				  u32 use_case)
 685{
 686	struct ffa_device *ffa_dev = optee->ffa.ffa_dev;
 687	const struct ffa_mem_ops *mem_ops = ffa_dev->ops->mem_ops;
 688	const struct ffa_msg_ops *msg_ops = ffa_dev->ops->msg_ops;
 689	struct ffa_send_direct_data data;
 690	struct ffa_mem_region_attributes *mem_attr;
 691	struct ffa_mem_ops_args args = {
 692		.use_txbuf = true,
 693		.tag = use_case,
 694	};
 695	struct page *page;
 696	struct scatterlist sgl;
 697	unsigned int n;
 698	int rc;
 699
 700	mem_attr = kcalloc(ma_count, sizeof(*mem_attr), GFP_KERNEL);
 701	for (n = 0; n < ma_count; n++) {
 702		mem_attr[n].receiver = mem_attrs[n] & U16_MAX;
 703		mem_attr[n].attrs = mem_attrs[n] >> 16;
 704	}
 705	args.attrs = mem_attr;
 706	args.nattrs = ma_count;
 707
 708	page = phys_to_page(protmem->paddr);
 709	sg_init_table(&sgl, 1);
 710	sg_set_page(&sgl, page, protmem->size, 0);
 711
 712	args.sg = &sgl;
 713	rc = mem_ops->memory_lend(&args);
 714	kfree(mem_attr);
 715	if (rc)
 716		return rc;
 717
 718	rc = do_call_lend_protmem(optee, args.g_handle, use_case);
 719	if (rc)
 720		goto err_reclaim;
 721
 722	rc = optee_shm_add_ffa_handle(optee, protmem, args.g_handle);
 723	if (rc)
 724		goto err_unreg;
 725
 726	protmem->sec_world_id = args.g_handle;
 727
 728	return 0;
 729
 730err_unreg:
 731	data = (struct ffa_send_direct_data){
 732		.data0 = OPTEE_FFA_RELEASE_PROTMEM,
 733		.data1 = (u32)args.g_handle,
 734		.data2 = (u32)(args.g_handle >> 32),
 735	};
 736	msg_ops->sync_send_receive(ffa_dev, &data);
 737err_reclaim:
 738	mem_ops->memory_reclaim(args.g_handle, 0);
 739	return rc;
 740}
 741
 742static int optee_ffa_reclaim_protmem(struct optee *optee,
 743				     struct tee_shm *protmem)
 744{
 745	struct ffa_device *ffa_dev = optee->ffa.ffa_dev;
 746	const struct ffa_msg_ops *msg_ops = ffa_dev->ops->msg_ops;
 747	const struct ffa_mem_ops *mem_ops = ffa_dev->ops->mem_ops;
 748	u64 global_handle = protmem->sec_world_id;
 749	struct ffa_send_direct_data data = {
 750		.data0 = OPTEE_FFA_RELEASE_PROTMEM,
 751		.data1 = (u32)global_handle,
 752		.data2 = (u32)(global_handle >> 32)
 753	};
 754	int rc;
 755
 756	optee_shm_rem_ffa_handle(optee, global_handle);
 757	protmem->sec_world_id = 0;
 758
 759	rc = msg_ops->sync_send_receive(ffa_dev, &data);
 760	if (rc)
 761		pr_err("Release SHM id 0x%llx rc %d\n", global_handle, rc);
 762
 763	rc = mem_ops->memory_reclaim(global_handle, 0);
 764	if (rc)
 765		pr_err("mem_reclaim: 0x%llx %d", global_handle, rc);
 766
 767	return rc;
 768}
 769
 770/*
 771 * 6. Driver initialization
 772 *
 773 * During driver inititialization is the OP-TEE Secure Partition is probed
 774 * to find out which features it supports so the driver can be initialized
 775 * with a matching configuration.
 776 */
 777
 778static bool optee_ffa_api_is_compatible(struct ffa_device *ffa_dev,
 779					const struct ffa_ops *ops)
 780{
 781	const struct ffa_msg_ops *msg_ops = ops->msg_ops;
 782	struct ffa_send_direct_data data = {
 783		.data0 = OPTEE_FFA_GET_API_VERSION,
 784	};
 785	int rc;
 786
 787	msg_ops->mode_32bit_set(ffa_dev);
 788
 789	rc = msg_ops->sync_send_receive(ffa_dev, &data);
 790	if (rc) {
 791		pr_err("Unexpected error %d\n", rc);
 792		return false;
 793	}
 794	if (data.data0 != OPTEE_FFA_VERSION_MAJOR ||
 795	    data.data1 < OPTEE_FFA_VERSION_MINOR) {
 796		pr_err("Incompatible OP-TEE API version %lu.%lu",
 797		       data.data0, data.data1);
 798		return false;
 799	}
 800
 801	data = (struct ffa_send_direct_data){
 802		.data0 = OPTEE_FFA_GET_OS_VERSION,
 803	};
 804	rc = msg_ops->sync_send_receive(ffa_dev, &data);
 805	if (rc) {
 806		pr_err("Unexpected error %d\n", rc);
 807		return false;
 808	}
 809	if (data.data2)
 810		pr_info("revision %lu.%lu (%08lx)",
 811			data.data0, data.data1, data.data2);
 812	else
 813		pr_info("revision %lu.%lu", data.data0, data.data1);
 814
 815	return true;
 816}
 817
 818static bool optee_ffa_exchange_caps(struct ffa_device *ffa_dev,
 819				    const struct ffa_ops *ops,
 820				    u32 *sec_caps,
 821				    unsigned int *rpc_param_count,
 822				    unsigned int *max_notif_value)
 823{
 824	struct ffa_send_direct_data data = {
 825		.data0 = OPTEE_FFA_EXCHANGE_CAPABILITIES,
 826	};
 827	int rc;
 828
 829	rc = ops->msg_ops->sync_send_receive(ffa_dev, &data);
 830	if (rc) {
 831		pr_err("Unexpected error %d", rc);
 832		return false;
 833	}
 834	if (data.data0) {
 835		pr_err("Unexpected exchange error %lu", data.data0);
 836		return false;
 837	}
 838
 839	*rpc_param_count = (u8)data.data1;
 840	*sec_caps = data.data2;
 841	if (data.data3)
 842		*max_notif_value = data.data3;
 843	else
 844		*max_notif_value = OPTEE_DEFAULT_MAX_NOTIF_VALUE;
 845
 846	return true;
 847}
 848
 849static void notif_work_fn(struct work_struct *work)
 850{
 851	struct optee_ffa *optee_ffa = container_of(work, struct optee_ffa,
 852						   notif_work);
 853	struct optee *optee = container_of(optee_ffa, struct optee, ffa);
 854
 855	optee_do_bottom_half(optee->ctx);
 856}
 857
 858static void notif_callback(int notify_id, void *cb_data)
 859{
 860	struct optee *optee = cb_data;
 861
 862	if (notify_id == optee->ffa.bottom_half_value)
 863		queue_work(optee->ffa.notif_wq, &optee->ffa.notif_work);
 864	else
 865		optee_notif_send(optee, notify_id);
 866}
 867
 868static int enable_async_notif(struct optee *optee)
 869{
 870	struct ffa_device *ffa_dev = optee->ffa.ffa_dev;
 871	struct ffa_send_direct_data data = {
 872		.data0 = OPTEE_FFA_ENABLE_ASYNC_NOTIF,
 873		.data1 = optee->ffa.bottom_half_value,
 874	};
 875	int rc;
 876
 877	rc = ffa_dev->ops->msg_ops->sync_send_receive(ffa_dev, &data);
 878	if (rc)
 879		return rc;
 880	return data.data0;
 881}
 882
 883static void optee_ffa_get_version(struct tee_device *teedev,
 884				  struct tee_ioctl_version_data *vers)
 885{
 886	struct tee_ioctl_version_data v = {
 887		.impl_id = TEE_IMPL_ID_OPTEE,
 888		.impl_caps = TEE_OPTEE_CAP_TZ,
 889		.gen_caps = TEE_GEN_CAP_GP | TEE_GEN_CAP_REG_MEM |
 890			    TEE_GEN_CAP_MEMREF_NULL,
 891	};
 892
 893	*vers = v;
 894}
 895
 896static int optee_ffa_open(struct tee_context *ctx)
 897{
 898	return optee_open(ctx, true);
 899}
 900
 901static const struct tee_driver_ops optee_ffa_clnt_ops = {
 902	.get_version = optee_ffa_get_version,
 903	.open = optee_ffa_open,
 904	.release = optee_release,
 905	.open_session = optee_open_session,
 906	.close_session = optee_close_session,
 907	.invoke_func = optee_invoke_func,
 908	.cancel_req = optee_cancel_req,
 909	.shm_register = optee_ffa_shm_register,
 910	.shm_unregister = optee_ffa_shm_unregister,
 911};
 912
 913static const struct tee_desc optee_ffa_clnt_desc = {
 914	.name = DRIVER_NAME "-ffa-clnt",
 915	.ops = &optee_ffa_clnt_ops,
 916	.owner = THIS_MODULE,
 917};
 918
 919static const struct tee_driver_ops optee_ffa_supp_ops = {
 920	.get_version = optee_ffa_get_version,
 921	.open = optee_ffa_open,
 922	.release = optee_release_supp,
 923	.supp_recv = optee_supp_recv,
 924	.supp_send = optee_supp_send,
 925	.shm_register = optee_ffa_shm_register, /* same as for clnt ops */
 926	.shm_unregister = optee_ffa_shm_unregister_supp,
 927};
 928
 929static const struct tee_desc optee_ffa_supp_desc = {
 930	.name = DRIVER_NAME "-ffa-supp",
 931	.ops = &optee_ffa_supp_ops,
 932	.owner = THIS_MODULE,
 933	.flags = TEE_DESC_PRIVILEGED,
 934};
 935
 936static const struct optee_ops optee_ffa_ops = {
 937	.do_call_with_arg = optee_ffa_do_call_with_arg,
 938	.to_msg_param = optee_ffa_to_msg_param,
 939	.from_msg_param = optee_ffa_from_msg_param,
 940	.lend_protmem = optee_ffa_lend_protmem,
 941	.reclaim_protmem = optee_ffa_reclaim_protmem,
 942};
 943
 944static void optee_ffa_remove(struct ffa_device *ffa_dev)
 945{
 946	struct optee *optee = ffa_dev_get_drvdata(ffa_dev);
 947	u32 bottom_half_id = optee->ffa.bottom_half_value;
 948
 949	if (bottom_half_id != U32_MAX) {
 950		ffa_dev->ops->notifier_ops->notify_relinquish(ffa_dev,
 951							      bottom_half_id);
 952		destroy_workqueue(optee->ffa.notif_wq);
 953	}
 954	optee_remove_common(optee);
 955
 956	mutex_destroy(&optee->ffa.mutex);
 957	rhashtable_free_and_destroy(&optee->ffa.global_ids, rh_free_fn, NULL);
 958
 959	kfree(optee);
 960}
 961
 962static int optee_ffa_async_notif_init(struct ffa_device *ffa_dev,
 963				      struct optee *optee)
 964{
 965	bool is_per_vcpu = false;
 966	u32 notif_id = 0;
 967	int rc;
 968
 969	INIT_WORK(&optee->ffa.notif_work, notif_work_fn);
 970	optee->ffa.notif_wq = create_workqueue("optee_notification");
 971	if (!optee->ffa.notif_wq) {
 972		rc = -EINVAL;
 973		goto err;
 974	}
 975
 976	while (true) {
 977		rc = ffa_dev->ops->notifier_ops->notify_request(ffa_dev,
 978								is_per_vcpu,
 979								notif_callback,
 980								optee,
 981								notif_id);
 982		if (!rc)
 983			break;
 984		/*
 985		 * -EACCES means that the notification ID was
 986		 * already bound, try the next one as long as we
 987		 * haven't reached the max. Any other error is a
 988		 * permanent error, so skip asynchronous
 989		 * notifications in that case.
 990		 */
 991		if (rc != -EACCES)
 992			goto err_wq;
 993		notif_id++;
 994		if (notif_id >= OPTEE_FFA_MAX_ASYNC_NOTIF_VALUE)
 995			goto err_wq;
 996	}
 997	optee->ffa.bottom_half_value = notif_id;
 998
 999	rc = enable_async_notif(optee);
1000	if (rc < 0)
1001		goto err_rel;
1002
1003	return 0;
1004err_rel:
1005	ffa_dev->ops->notifier_ops->notify_relinquish(ffa_dev, notif_id);
1006err_wq:
1007	destroy_workqueue(optee->ffa.notif_wq);
1008err:
1009	optee->ffa.bottom_half_value = U32_MAX;
1010
1011	return rc;
1012}
1013
1014static int optee_ffa_protmem_pool_init(struct optee *optee, u32 sec_caps)
1015{
1016	enum tee_dma_heap_id id = TEE_DMA_HEAP_SECURE_VIDEO_PLAY;
1017	struct tee_protmem_pool *pool;
1018	int rc = 0;
1019
1020	if (sec_caps & OPTEE_FFA_SEC_CAP_PROTMEM) {
1021		pool = optee_protmem_alloc_dyn_pool(optee, id);
1022		if (IS_ERR(pool))
1023			return PTR_ERR(pool);
1024
1025		rc = tee_device_register_dma_heap(optee->teedev, id, pool);
1026		if (rc)
1027			pool->ops->destroy_pool(pool);
1028	}
1029
1030	return rc;
1031}
1032
1033static int optee_ffa_probe(struct ffa_device *ffa_dev)
1034{
1035	const struct ffa_notifier_ops *notif_ops;
1036	const struct ffa_ops *ffa_ops;
1037	unsigned int max_notif_value;
1038	unsigned int rpc_param_count;
1039	struct tee_shm_pool *pool;
1040	struct tee_device *teedev;
1041	struct tee_context *ctx;
1042	u32 arg_cache_flags = 0;
1043	struct optee *optee;
1044	u32 sec_caps;
1045	int rc;
1046
1047	ffa_ops = ffa_dev->ops;
1048	notif_ops = ffa_ops->notifier_ops;
1049
1050	if (!optee_ffa_api_is_compatible(ffa_dev, ffa_ops))
1051		return -EINVAL;
1052
1053	if (!optee_ffa_exchange_caps(ffa_dev, ffa_ops, &sec_caps,
1054				     &rpc_param_count, &max_notif_value))
1055		return -EINVAL;
1056	if (sec_caps & OPTEE_FFA_SEC_CAP_ARG_OFFSET)
1057		arg_cache_flags |= OPTEE_SHM_ARG_SHARED;
1058
1059	optee = kzalloc(sizeof(*optee), GFP_KERNEL);
1060	if (!optee)
1061		return -ENOMEM;
1062
1063	pool = optee_ffa_shm_pool_alloc_pages();
1064	if (IS_ERR(pool)) {
1065		rc = PTR_ERR(pool);
1066		goto err_free_optee;
1067	}
1068	optee->pool = pool;
1069
1070	optee->ops = &optee_ffa_ops;
1071	optee->ffa.ffa_dev = ffa_dev;
1072	optee->ffa.bottom_half_value = U32_MAX;
1073	optee->rpc_param_count = rpc_param_count;
1074
1075	if (IS_REACHABLE(CONFIG_RPMB) &&
1076	    (sec_caps & OPTEE_FFA_SEC_CAP_RPMB_PROBE))
1077		optee->in_kernel_rpmb_routing = true;
1078
1079	teedev = tee_device_alloc(&optee_ffa_clnt_desc, NULL, optee->pool,
1080				  optee);
1081	if (IS_ERR(teedev)) {
1082		rc = PTR_ERR(teedev);
1083		goto err_free_shm_pool;
1084	}
1085	optee->teedev = teedev;
1086
1087	teedev = tee_device_alloc(&optee_ffa_supp_desc, NULL, optee->pool,
1088				  optee);
1089	if (IS_ERR(teedev)) {
1090		rc = PTR_ERR(teedev);
1091		goto err_unreg_teedev;
1092	}
1093	optee->supp_teedev = teedev;
1094
1095	optee_set_dev_group(optee);
1096
1097	rc = tee_device_register(optee->teedev);
1098	if (rc)
1099		goto err_unreg_supp_teedev;
1100
1101	rc = tee_device_register(optee->supp_teedev);
1102	if (rc)
1103		goto err_unreg_supp_teedev;
1104
1105	rc = rhashtable_init(&optee->ffa.global_ids, &shm_rhash_params);
1106	if (rc)
1107		goto err_unreg_supp_teedev;
1108	mutex_init(&optee->ffa.mutex);
1109	optee_cq_init(&optee->call_queue, 0);
1110	optee_supp_init(&optee->supp);
1111	optee_shm_arg_cache_init(optee, arg_cache_flags);
1112	mutex_init(&optee->rpmb_dev_mutex);
1113	ffa_dev_set_drvdata(ffa_dev, optee);
1114	ctx = teedev_open(optee->teedev);
1115	if (IS_ERR(ctx)) {
1116		rc = PTR_ERR(ctx);
1117		goto err_rhashtable_free;
1118	}
1119	optee->ctx = ctx;
1120	rc = optee_notif_init(optee, OPTEE_DEFAULT_MAX_NOTIF_VALUE);
1121	if (rc)
1122		goto err_close_ctx;
1123	if (sec_caps & OPTEE_FFA_SEC_CAP_ASYNC_NOTIF) {
1124		rc = optee_ffa_async_notif_init(ffa_dev, optee);
1125		if (rc < 0)
1126			pr_err("Failed to initialize async notifications: %d",
1127			       rc);
1128	}
1129
1130	if (optee_ffa_protmem_pool_init(optee, sec_caps))
1131		pr_info("Protected memory service not available\n");
1132
1133	rc = optee_enumerate_devices(PTA_CMD_GET_DEVICES);
1134	if (rc)
1135		goto err_unregister_devices;
1136
1137	INIT_WORK(&optee->rpmb_scan_bus_work, optee_bus_scan_rpmb);
1138	optee->rpmb_intf.notifier_call = optee_rpmb_intf_rdev;
1139	blocking_notifier_chain_register(&optee_rpmb_intf_added,
1140					 &optee->rpmb_intf);
1141	pr_info("initialized driver\n");
1142	return 0;
1143
1144err_unregister_devices:
1145	optee_unregister_devices();
1146	if (optee->ffa.bottom_half_value != U32_MAX)
1147		notif_ops->notify_relinquish(ffa_dev,
1148					     optee->ffa.bottom_half_value);
1149	optee_notif_uninit(optee);
1150err_close_ctx:
1151	teedev_close_context(ctx);
1152err_rhashtable_free:
1153	rhashtable_free_and_destroy(&optee->ffa.global_ids, rh_free_fn, NULL);
1154	rpmb_dev_put(optee->rpmb_dev);
1155	mutex_destroy(&optee->rpmb_dev_mutex);
1156	optee_supp_uninit(&optee->supp);
1157	mutex_destroy(&optee->call_queue.mutex);
1158	mutex_destroy(&optee->ffa.mutex);
1159err_unreg_supp_teedev:
1160	tee_device_unregister(optee->supp_teedev);
1161err_unreg_teedev:
1162	tee_device_unregister(optee->teedev);
1163err_free_shm_pool:
1164	tee_shm_pool_free(pool);
1165err_free_optee:
1166	kfree(optee);
1167	return rc;
1168}
1169
1170static const struct ffa_device_id optee_ffa_device_id[] = {
1171	/* 486178e0-e7f8-11e3-bc5e0002a5d5c51b */
1172	{ UUID_INIT(0x486178e0, 0xe7f8, 0x11e3,
1173		    0xbc, 0x5e, 0x00, 0x02, 0xa5, 0xd5, 0xc5, 0x1b) },
1174	{}
1175};
1176
1177static struct ffa_driver optee_ffa_driver = {
1178	.name = "optee",
1179	.probe = optee_ffa_probe,
1180	.remove = optee_ffa_remove,
1181	.id_table = optee_ffa_device_id,
1182};
1183
1184int optee_ffa_abi_register(void)
1185{
1186	if (IS_REACHABLE(CONFIG_ARM_FFA_TRANSPORT))
1187		return ffa_register(&optee_ffa_driver);
1188	else
1189		return -EOPNOTSUPP;
1190}
1191
1192void optee_ffa_abi_unregister(void)
1193{
1194	if (IS_REACHABLE(CONFIG_ARM_FFA_TRANSPORT))
1195		ffa_unregister(&optee_ffa_driver);
1196}