File Index
Files and directories
Sign in
FILE

smc_abi.c

File preview with syntax highlighting.

linux-6.19.14/drivers/tee/optee/smc_abi.c C 
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (c) 2015-2021, 2023 Linaro Limited
   4 * Copyright (c) 2016, EPAM Systems
   5 */
   6
   7#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   8
   9#include <linux/arm-smccc.h>
  10#include <linux/cpuhotplug.h>
  11#include <linux/errno.h>
  12#include <linux/firmware.h>
  13#include <linux/interrupt.h>
  14#include <linux/io.h>
  15#include <linux/irqdomain.h>
  16#include <linux/kernel.h>
  17#include <linux/mm.h>
  18#include <linux/module.h>
  19#include <linux/of.h>
  20#include <linux/of_irq.h>
  21#include <linux/of_platform.h>
  22#include <linux/platform_device.h>
  23#include <linux/rpmb.h>
  24#include <linux/sched.h>
  25#include <linux/slab.h>
  26#include <linux/string.h>
  27#include <linux/tee_core.h>
  28#include <linux/types.h>
  29#include <linux/workqueue.h>
  30#include "optee_private.h"
  31#include "optee_smc.h"
  32#include "optee_rpc_cmd.h"
  33#include <linux/kmemleak.h>
  34#define CREATE_TRACE_POINTS
  35#include "optee_trace.h"
  36
  37/*
  38 * This file implement the SMC ABI used when communicating with secure world
  39 * OP-TEE OS via raw SMCs.
  40 * This file is divided into the following sections:
  41 * 1. Convert between struct tee_param and struct optee_msg_param
  42 * 2. Low level support functions to register shared memory in secure world
  43 * 3. Dynamic shared memory pool based on alloc_pages()
  44 * 4. Do a normal scheduled call into secure world
  45 * 5. Asynchronous notification
  46 * 6. Driver initialization.
  47 */
  48
  49/*
  50 * A typical OP-TEE private shm allocation is 224 bytes (argument struct
  51 * with 6 parameters, needed for open session). So with an alignment of 512
  52 * we'll waste a bit more than 50%. However, it's only expected that we'll
  53 * have a handful of these structs allocated at a time. Most memory will
  54 * be allocated aligned to the page size, So all in all this should scale
  55 * up and down quite well.
  56 */
  57#define OPTEE_MIN_STATIC_POOL_ALIGN    9 /* 512 bytes aligned */
  58
  59/* SMC ABI considers at most a single TEE firmware */
  60static unsigned int pcpu_irq_num;
  61
  62static int optee_cpuhp_enable_pcpu_irq(unsigned int cpu)
  63{
  64	enable_percpu_irq(pcpu_irq_num, IRQ_TYPE_NONE);
  65
  66	return 0;
  67}
  68
  69static int optee_cpuhp_disable_pcpu_irq(unsigned int cpu)
  70{
  71	disable_percpu_irq(pcpu_irq_num);
  72
  73	return 0;
  74}
  75
  76/*
  77 * 1. Convert between struct tee_param and struct optee_msg_param
  78 *
  79 * optee_from_msg_param() and optee_to_msg_param() are the main
  80 * functions.
  81 */
  82
  83static int from_msg_param_tmp_mem(struct tee_param *p, u32 attr,
  84				  const struct optee_msg_param *mp)
  85{
  86	struct tee_shm *shm;
  87	phys_addr_t pa;
  88	int rc;
  89
  90	p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT +
  91		  attr - OPTEE_MSG_ATTR_TYPE_TMEM_INPUT;
  92	p->u.memref.size = mp->u.tmem.size;
  93	shm = (struct tee_shm *)(unsigned long)mp->u.tmem.shm_ref;
  94	if (!shm) {
  95		p->u.memref.shm_offs = 0;
  96		p->u.memref.shm = NULL;
  97		return 0;
  98	}
  99
 100	rc = tee_shm_get_pa(shm, 0, &pa);
 101	if (rc)
 102		return rc;
 103
 104	p->u.memref.shm_offs = mp->u.tmem.buf_ptr - pa;
 105	p->u.memref.shm = shm;
 106
 107	return 0;
 108}
 109
 110static void from_msg_param_reg_mem(struct tee_param *p, u32 attr,
 111				   const struct optee_msg_param *mp)
 112{
 113	struct tee_shm *shm;
 114
 115	p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT +
 116		  attr - OPTEE_MSG_ATTR_TYPE_RMEM_INPUT;
 117	p->u.memref.size = mp->u.rmem.size;
 118	shm = (struct tee_shm *)(unsigned long)mp->u.rmem.shm_ref;
 119
 120	if (shm) {
 121		p->u.memref.shm_offs = mp->u.rmem.offs;
 122		p->u.memref.shm = shm;
 123	} else {
 124		p->u.memref.shm_offs = 0;
 125		p->u.memref.shm = NULL;
 126	}
 127}
 128
 129/**
 130 * optee_from_msg_param() - convert from OPTEE_MSG parameters to
 131 *			    struct tee_param
 132 * @optee:	main service struct
 133 * @params:	subsystem internal parameter representation
 134 * @num_params:	number of elements in the parameter arrays
 135 * @msg_params:	OPTEE_MSG parameters
 136 * Returns 0 on success or <0 on failure
 137 */
 138static int optee_from_msg_param(struct optee *optee, struct tee_param *params,
 139				size_t num_params,
 140				const struct optee_msg_param *msg_params)
 141{
 142	int rc;
 143	size_t n;
 144
 145	for (n = 0; n < num_params; n++) {
 146		struct tee_param *p = params + n;
 147		const struct optee_msg_param *mp = msg_params + n;
 148		u32 attr = mp->attr & OPTEE_MSG_ATTR_TYPE_MASK;
 149
 150		switch (attr) {
 151		case OPTEE_MSG_ATTR_TYPE_NONE:
 152			p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_NONE;
 153			memset(&p->u, 0, sizeof(p->u));
 154			break;
 155		case OPTEE_MSG_ATTR_TYPE_VALUE_INPUT:
 156		case OPTEE_MSG_ATTR_TYPE_VALUE_OUTPUT:
 157		case OPTEE_MSG_ATTR_TYPE_VALUE_INOUT:
 158			optee_from_msg_param_value(p, attr, mp);
 159			break;
 160		case OPTEE_MSG_ATTR_TYPE_TMEM_INPUT:
 161		case OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT:
 162		case OPTEE_MSG_ATTR_TYPE_TMEM_INOUT:
 163			rc = from_msg_param_tmp_mem(p, attr, mp);
 164			if (rc)
 165				return rc;
 166			break;
 167		case OPTEE_MSG_ATTR_TYPE_RMEM_INPUT:
 168		case OPTEE_MSG_ATTR_TYPE_RMEM_OUTPUT:
 169		case OPTEE_MSG_ATTR_TYPE_RMEM_INOUT:
 170			from_msg_param_reg_mem(p, attr, mp);
 171			break;
 172
 173		default:
 174			return -EINVAL;
 175		}
 176	}
 177	return 0;
 178}
 179
 180static int to_msg_param_tmp_mem(struct optee_msg_param *mp,
 181				const struct tee_param *p)
 182{
 183	int rc;
 184	phys_addr_t pa;
 185
 186	mp->attr = OPTEE_MSG_ATTR_TYPE_TMEM_INPUT + p->attr -
 187		   TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT;
 188
 189	mp->u.tmem.shm_ref = (unsigned long)p->u.memref.shm;
 190	mp->u.tmem.size = p->u.memref.size;
 191
 192	if (!p->u.memref.shm) {
 193		mp->u.tmem.buf_ptr = 0;
 194		return 0;
 195	}
 196
 197	rc = tee_shm_get_pa(p->u.memref.shm, p->u.memref.shm_offs, &pa);
 198	if (rc)
 199		return rc;
 200
 201	mp->u.tmem.buf_ptr = pa;
 202	mp->attr |= OPTEE_MSG_ATTR_CACHE_PREDEFINED <<
 203		    OPTEE_MSG_ATTR_CACHE_SHIFT;
 204
 205	return 0;
 206}
 207
 208static int to_msg_param_reg_mem(struct optee_msg_param *mp,
 209				const struct tee_param *p)
 210{
 211	mp->attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT + p->attr -
 212		   TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT;
 213
 214	mp->u.rmem.shm_ref = (unsigned long)p->u.memref.shm;
 215	mp->u.rmem.size = p->u.memref.size;
 216	mp->u.rmem.offs = p->u.memref.shm_offs;
 217	return 0;
 218}
 219
 220/**
 221 * optee_to_msg_param() - convert from struct tee_params to OPTEE_MSG parameters
 222 * @optee:	main service struct
 223 * @msg_params:	OPTEE_MSG parameters
 224 * @num_params:	number of elements in the parameter arrays
 225 * @params:	subsystem itnernal parameter representation
 226 * Returns 0 on success or <0 on failure
 227 */
 228static int optee_to_msg_param(struct optee *optee,
 229			      struct optee_msg_param *msg_params,
 230			      size_t num_params, const struct tee_param *params)
 231{
 232	int rc;
 233	size_t n;
 234
 235	for (n = 0; n < num_params; n++) {
 236		const struct tee_param *p = params + n;
 237		struct optee_msg_param *mp = msg_params + n;
 238
 239		switch (p->attr) {
 240		case TEE_IOCTL_PARAM_ATTR_TYPE_NONE:
 241			mp->attr = TEE_IOCTL_PARAM_ATTR_TYPE_NONE;
 242			memset(&mp->u, 0, sizeof(mp->u));
 243			break;
 244		case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT:
 245		case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_OUTPUT:
 246		case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT:
 247			optee_to_msg_param_value(mp, p);
 248			break;
 249		case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT:
 250		case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT:
 251		case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT:
 252			if (tee_shm_is_dynamic(p->u.memref.shm))
 253				rc = to_msg_param_reg_mem(mp, p);
 254			else
 255				rc = to_msg_param_tmp_mem(mp, p);
 256			if (rc)
 257				return rc;
 258			break;
 259		default:
 260			return -EINVAL;
 261		}
 262	}
 263	return 0;
 264}
 265
 266/*
 267 * 2. Low level support functions to register shared memory in secure world
 268 *
 269 * Functions to enable/disable shared memory caching in secure world, that
 270 * is, lazy freeing of previously allocated shared memory. Freeing is
 271 * performed when a request has been compled.
 272 *
 273 * Functions to register and unregister shared memory both for normal
 274 * clients and for tee-supplicant.
 275 */
 276
 277/**
 278 * optee_enable_shm_cache() - Enables caching of some shared memory allocation
 279 *			      in OP-TEE
 280 * @optee:	main service struct
 281 */
 282static void optee_enable_shm_cache(struct optee *optee)
 283{
 284	struct optee_call_waiter w;
 285
 286	/* We need to retry until secure world isn't busy. */
 287	optee_cq_wait_init(&optee->call_queue, &w, false);
 288	while (true) {
 289		struct arm_smccc_res res;
 290
 291		optee->smc.invoke_fn(OPTEE_SMC_ENABLE_SHM_CACHE,
 292				     0, 0, 0, 0, 0, 0, 0, &res);
 293		if (res.a0 == OPTEE_SMC_RETURN_OK)
 294			break;
 295		optee_cq_wait_for_completion(&optee->call_queue, &w);
 296	}
 297	optee_cq_wait_final(&optee->call_queue, &w);
 298}
 299
 300/**
 301 * __optee_disable_shm_cache() - Disables caching of some shared memory
 302 *				 allocation in OP-TEE
 303 * @optee:	main service struct
 304 * @is_mapped:	true if the cached shared memory addresses were mapped by this
 305 *		kernel, are safe to dereference, and should be freed
 306 */
 307static void __optee_disable_shm_cache(struct optee *optee, bool is_mapped)
 308{
 309	struct optee_call_waiter w;
 310
 311	/* We need to retry until secure world isn't busy. */
 312	optee_cq_wait_init(&optee->call_queue, &w, false);
 313	while (true) {
 314		union {
 315			struct arm_smccc_res smccc;
 316			struct optee_smc_disable_shm_cache_result result;
 317		} res;
 318
 319		optee->smc.invoke_fn(OPTEE_SMC_DISABLE_SHM_CACHE,
 320				     0, 0, 0, 0, 0, 0, 0, &res.smccc);
 321		if (res.result.status == OPTEE_SMC_RETURN_ENOTAVAIL)
 322			break; /* All shm's freed */
 323		if (res.result.status == OPTEE_SMC_RETURN_OK) {
 324			struct tee_shm *shm;
 325
 326			/*
 327			 * Shared memory references that were not mapped by
 328			 * this kernel must be ignored to prevent a crash.
 329			 */
 330			if (!is_mapped)
 331				continue;
 332
 333			shm = reg_pair_to_ptr(res.result.shm_upper32,
 334					      res.result.shm_lower32);
 335			tee_shm_free(shm);
 336		} else {
 337			optee_cq_wait_for_completion(&optee->call_queue, &w);
 338		}
 339	}
 340	optee_cq_wait_final(&optee->call_queue, &w);
 341}
 342
 343/**
 344 * optee_disable_shm_cache() - Disables caching of mapped shared memory
 345 *			       allocations in OP-TEE
 346 * @optee:	main service struct
 347 */
 348static void optee_disable_shm_cache(struct optee *optee)
 349{
 350	return __optee_disable_shm_cache(optee, true);
 351}
 352
 353/**
 354 * optee_disable_unmapped_shm_cache() - Disables caching of shared memory
 355 *					allocations in OP-TEE which are not
 356 *					currently mapped
 357 * @optee:	main service struct
 358 */
 359static void optee_disable_unmapped_shm_cache(struct optee *optee)
 360{
 361	return __optee_disable_shm_cache(optee, false);
 362}
 363
 364#define PAGELIST_ENTRIES_PER_PAGE				\
 365	((OPTEE_MSG_NONCONTIG_PAGE_SIZE / sizeof(u64)) - 1)
 366
 367/*
 368 * The final entry in each pagelist page is a pointer to the next
 369 * pagelist page.
 370 */
 371static size_t get_pages_list_size(size_t num_entries)
 372{
 373	int pages = DIV_ROUND_UP(num_entries, PAGELIST_ENTRIES_PER_PAGE);
 374
 375	return pages * OPTEE_MSG_NONCONTIG_PAGE_SIZE;
 376}
 377
 378static u64 *optee_allocate_pages_list(size_t num_entries)
 379{
 380	return alloc_pages_exact(get_pages_list_size(num_entries), GFP_KERNEL);
 381}
 382
 383static void optee_free_pages_list(void *list, size_t num_entries)
 384{
 385	free_pages_exact(list, get_pages_list_size(num_entries));
 386}
 387
 388/**
 389 * optee_fill_pages_list() - write list of user pages to given shared
 390 * buffer.
 391 *
 392 * @dst: page-aligned buffer where list of pages will be stored
 393 * @pages: array of pages that represents shared buffer
 394 * @num_pages: number of entries in @pages
 395 * @page_offset: offset of user buffer from page start
 396 *
 397 * @dst should be big enough to hold list of user page addresses and
 398 *	links to the next pages of buffer
 399 */
 400static void optee_fill_pages_list(u64 *dst, struct page **pages, int num_pages,
 401				  size_t page_offset)
 402{
 403	int n = 0;
 404	phys_addr_t optee_page;
 405	/*
 406	 * Refer to OPTEE_MSG_ATTR_NONCONTIG description in optee_msg.h
 407	 * for details.
 408	 */
 409	struct {
 410		u64 pages_list[PAGELIST_ENTRIES_PER_PAGE];
 411		u64 next_page_data;
 412	} *pages_data;
 413
 414	/*
 415	 * Currently OP-TEE uses 4k page size and it does not looks
 416	 * like this will change in the future.  On other hand, there are
 417	 * no know ARM architectures with page size < 4k.
 418	 * Thus the next built assert looks redundant. But the following
 419	 * code heavily relies on this assumption, so it is better be
 420	 * safe than sorry.
 421	 */
 422	BUILD_BUG_ON(PAGE_SIZE < OPTEE_MSG_NONCONTIG_PAGE_SIZE);
 423
 424	pages_data = (void *)dst;
 425	/*
 426	 * If linux page is bigger than 4k, and user buffer offset is
 427	 * larger than 4k/8k/12k/etc this will skip first 4k pages,
 428	 * because they bear no value data for OP-TEE.
 429	 */
 430	optee_page = page_to_phys(*pages) +
 431		round_down(page_offset, OPTEE_MSG_NONCONTIG_PAGE_SIZE);
 432
 433	while (true) {
 434		pages_data->pages_list[n++] = optee_page;
 435
 436		if (n == PAGELIST_ENTRIES_PER_PAGE) {
 437			pages_data->next_page_data =
 438				virt_to_phys(pages_data + 1);
 439			pages_data++;
 440			n = 0;
 441		}
 442
 443		optee_page += OPTEE_MSG_NONCONTIG_PAGE_SIZE;
 444		if (!(optee_page & ~PAGE_MASK)) {
 445			if (!--num_pages)
 446				break;
 447			pages++;
 448			optee_page = page_to_phys(*pages);
 449		}
 450	}
 451}
 452
 453static int optee_shm_register(struct tee_context *ctx, struct tee_shm *shm,
 454			      struct page **pages, size_t num_pages,
 455			      unsigned long start)
 456{
 457	struct optee *optee = tee_get_drvdata(ctx->teedev);
 458	struct optee_msg_arg *msg_arg;
 459	struct tee_shm *shm_arg;
 460	u64 *pages_list;
 461	size_t sz;
 462	int rc;
 463
 464	if (!num_pages)
 465		return -EINVAL;
 466
 467	rc = optee_check_mem_type(start, num_pages);
 468	if (rc)
 469		return rc;
 470
 471	pages_list = optee_allocate_pages_list(num_pages);
 472	if (!pages_list)
 473		return -ENOMEM;
 474
 475	/*
 476	 * We're about to register shared memory we can't register shared
 477	 * memory for this request or there's a catch-22.
 478	 *
 479	 * So in this we'll have to do the good old temporary private
 480	 * allocation instead of using optee_get_msg_arg().
 481	 */
 482	sz = optee_msg_arg_size(optee->rpc_param_count);
 483	shm_arg = tee_shm_alloc_priv_buf(ctx, sz);
 484	if (IS_ERR(shm_arg)) {
 485		rc = PTR_ERR(shm_arg);
 486		goto out;
 487	}
 488	msg_arg = tee_shm_get_va(shm_arg, 0);
 489	if (IS_ERR(msg_arg)) {
 490		rc = PTR_ERR(msg_arg);
 491		goto out;
 492	}
 493
 494	optee_fill_pages_list(pages_list, pages, num_pages,
 495			      tee_shm_get_page_offset(shm));
 496
 497	memset(msg_arg, 0, OPTEE_MSG_GET_ARG_SIZE(1));
 498	msg_arg->num_params = 1;
 499	msg_arg->cmd = OPTEE_MSG_CMD_REGISTER_SHM;
 500	msg_arg->params->attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT |
 501				OPTEE_MSG_ATTR_NONCONTIG;
 502	msg_arg->params->u.tmem.shm_ref = (unsigned long)shm;
 503	msg_arg->params->u.tmem.size = tee_shm_get_size(shm);
 504	/*
 505	 * In the least bits of msg_arg->params->u.tmem.buf_ptr we
 506	 * store buffer offset from 4k page, as described in OP-TEE ABI.
 507	 */
 508	msg_arg->params->u.tmem.buf_ptr = virt_to_phys(pages_list) |
 509	  (tee_shm_get_page_offset(shm) & (OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1));
 510
 511	if (optee->ops->do_call_with_arg(ctx, shm_arg, 0, false) ||
 512	    msg_arg->ret != TEEC_SUCCESS)
 513		rc = -EINVAL;
 514
 515	tee_shm_free(shm_arg);
 516out:
 517	optee_free_pages_list(pages_list, num_pages);
 518	return rc;
 519}
 520
 521static int optee_shm_unregister(struct tee_context *ctx, struct tee_shm *shm)
 522{
 523	struct optee *optee = tee_get_drvdata(ctx->teedev);
 524	struct optee_msg_arg *msg_arg;
 525	struct tee_shm *shm_arg;
 526	int rc = 0;
 527	size_t sz;
 528
 529	/*
 530	 * We're about to unregister shared memory and we may not be able
 531	 * register shared memory for this request in case we're called
 532	 * from optee_shm_arg_cache_uninit().
 533	 *
 534	 * So in order to keep things simple in this function just as in
 535	 * optee_shm_register() we'll use temporary private allocation
 536	 * instead of using optee_get_msg_arg().
 537	 */
 538	sz = optee_msg_arg_size(optee->rpc_param_count);
 539	shm_arg = tee_shm_alloc_priv_buf(ctx, sz);
 540	if (IS_ERR(shm_arg))
 541		return PTR_ERR(shm_arg);
 542	msg_arg = tee_shm_get_va(shm_arg, 0);
 543	if (IS_ERR(msg_arg)) {
 544		rc = PTR_ERR(msg_arg);
 545		goto out;
 546	}
 547
 548	memset(msg_arg, 0, sz);
 549	msg_arg->num_params = 1;
 550	msg_arg->cmd = OPTEE_MSG_CMD_UNREGISTER_SHM;
 551	msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT;
 552	msg_arg->params[0].u.rmem.shm_ref = (unsigned long)shm;
 553
 554	if (optee->ops->do_call_with_arg(ctx, shm_arg, 0, false) ||
 555	    msg_arg->ret != TEEC_SUCCESS)
 556		rc = -EINVAL;
 557out:
 558	tee_shm_free(shm_arg);
 559	return rc;
 560}
 561
 562static int optee_shm_register_supp(struct tee_context *ctx, struct tee_shm *shm,
 563				   struct page **pages, size_t num_pages,
 564				   unsigned long start)
 565{
 566	/*
 567	 * We don't want to register supplicant memory in OP-TEE.
 568	 * Instead information about it will be passed in RPC code.
 569	 */
 570	return optee_check_mem_type(start, num_pages);
 571}
 572
 573static int optee_shm_unregister_supp(struct tee_context *ctx,
 574				     struct tee_shm *shm)
 575{
 576	return 0;
 577}
 578
 579/*
 580 * 3. Dynamic shared memory pool based on alloc_pages()
 581 *
 582 * Implements an OP-TEE specific shared memory pool which is used
 583 * when dynamic shared memory is supported by secure world.
 584 *
 585 * The main function is optee_shm_pool_alloc_pages().
 586 */
 587
 588static int pool_op_alloc(struct tee_shm_pool *pool,
 589			 struct tee_shm *shm, size_t size, size_t align)
 590{
 591	/*
 592	 * Shared memory private to the OP-TEE driver doesn't need
 593	 * to be registered with OP-TEE.
 594	 */
 595	if (shm->flags & TEE_SHM_PRIV)
 596		return tee_dyn_shm_alloc_helper(shm, size, align, NULL);
 597
 598	return tee_dyn_shm_alloc_helper(shm, size, align, optee_shm_register);
 599}
 600
 601static void pool_op_free(struct tee_shm_pool *pool,
 602			 struct tee_shm *shm)
 603{
 604	if (!(shm->flags & TEE_SHM_PRIV))
 605		tee_dyn_shm_free_helper(shm, optee_shm_unregister);
 606	else
 607		tee_dyn_shm_free_helper(shm, NULL);
 608}
 609
 610static void pool_op_destroy_pool(struct tee_shm_pool *pool)
 611{
 612	kfree(pool);
 613}
 614
 615static const struct tee_shm_pool_ops pool_ops = {
 616	.alloc = pool_op_alloc,
 617	.free = pool_op_free,
 618	.destroy_pool = pool_op_destroy_pool,
 619};
 620
 621/**
 622 * optee_shm_pool_alloc_pages() - create page-based allocator pool
 623 *
 624 * This pool is used when OP-TEE supports dymanic SHM. In this case
 625 * command buffers and such are allocated from kernel's own memory.
 626 */
 627static struct tee_shm_pool *optee_shm_pool_alloc_pages(void)
 628{
 629	struct tee_shm_pool *pool = kzalloc(sizeof(*pool), GFP_KERNEL);
 630
 631	if (!pool)
 632		return ERR_PTR(-ENOMEM);
 633
 634	pool->ops = &pool_ops;
 635
 636	return pool;
 637}
 638
 639/*
 640 * 4. Do a normal scheduled call into secure world
 641 *
 642 * The function optee_smc_do_call_with_arg() performs a normal scheduled
 643 * call into secure world. During this call may normal world request help
 644 * from normal world using RPCs, Remote Procedure Calls. This includes
 645 * delivery of non-secure interrupts to for instance allow rescheduling of
 646 * the current task.
 647 */
 648
 649static void handle_rpc_func_cmd_shm_free(struct tee_context *ctx,
 650					 struct optee_msg_arg *arg)
 651{
 652	struct tee_shm *shm;
 653
 654	arg->ret_origin = TEEC_ORIGIN_COMMS;
 655
 656	if (arg->num_params != 1 ||
 657	    arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) {
 658		arg->ret = TEEC_ERROR_BAD_PARAMETERS;
 659		return;
 660	}
 661
 662	shm = (struct tee_shm *)(unsigned long)arg->params[0].u.value.b;
 663	switch (arg->params[0].u.value.a) {
 664	case OPTEE_RPC_SHM_TYPE_APPL:
 665		optee_rpc_cmd_free_suppl(ctx, shm);
 666		break;
 667	case OPTEE_RPC_SHM_TYPE_KERNEL:
 668		tee_shm_free(shm);
 669		break;
 670	default:
 671		arg->ret = TEEC_ERROR_BAD_PARAMETERS;
 672	}
 673	arg->ret = TEEC_SUCCESS;
 674}
 675
 676static void handle_rpc_func_cmd_shm_alloc(struct tee_context *ctx,
 677					  struct optee *optee,
 678					  struct optee_msg_arg *arg,
 679					  struct optee_call_ctx *call_ctx)
 680{
 681	struct tee_shm *shm;
 682	size_t sz;
 683	size_t n;
 684	struct page **pages;
 685	size_t page_count;
 686
 687	arg->ret_origin = TEEC_ORIGIN_COMMS;
 688
 689	if (!arg->num_params ||
 690	    arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) {
 691		arg->ret = TEEC_ERROR_BAD_PARAMETERS;
 692		return;
 693	}
 694
 695	for (n = 1; n < arg->num_params; n++) {
 696		if (arg->params[n].attr != OPTEE_MSG_ATTR_TYPE_NONE) {
 697			arg->ret = TEEC_ERROR_BAD_PARAMETERS;
 698			return;
 699		}
 700	}
 701
 702	sz = arg->params[0].u.value.b;
 703	switch (arg->params[0].u.value.a) {
 704	case OPTEE_RPC_SHM_TYPE_APPL:
 705		shm = optee_rpc_cmd_alloc_suppl(ctx, sz);
 706		break;
 707	case OPTEE_RPC_SHM_TYPE_KERNEL:
 708		shm = tee_shm_alloc_priv_buf(optee->ctx, sz);
 709		break;
 710	default:
 711		arg->ret = TEEC_ERROR_BAD_PARAMETERS;
 712		return;
 713	}
 714
 715	if (IS_ERR(shm)) {
 716		arg->ret = TEEC_ERROR_OUT_OF_MEMORY;
 717		return;
 718	}
 719
 720	/*
 721	 * If there are pages it's dynamically allocated shared memory (not
 722	 * from the reserved shared memory pool) and needs to be
 723	 * registered.
 724	 */
 725	pages = tee_shm_get_pages(shm, &page_count);
 726	if (pages) {
 727		u64 *pages_list;
 728
 729		pages_list = optee_allocate_pages_list(page_count);
 730		if (!pages_list) {
 731			arg->ret = TEEC_ERROR_OUT_OF_MEMORY;
 732			goto bad;
 733		}
 734
 735		call_ctx->pages_list = pages_list;
 736		call_ctx->num_entries = page_count;
 737
 738		arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT |
 739				      OPTEE_MSG_ATTR_NONCONTIG;
 740		/*
 741		 * In the least bits of u.tmem.buf_ptr we store buffer offset
 742		 * from 4k page, as described in OP-TEE ABI.
 743		 */
 744		arg->params[0].u.tmem.buf_ptr = virt_to_phys(pages_list) |
 745			(tee_shm_get_page_offset(shm) &
 746			 (OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1));
 747
 748		optee_fill_pages_list(pages_list, pages, page_count,
 749				      tee_shm_get_page_offset(shm));
 750	} else {
 751		phys_addr_t pa;
 752
 753		if (tee_shm_get_pa(shm, 0, &pa)) {
 754			arg->ret = TEEC_ERROR_BAD_PARAMETERS;
 755			goto bad;
 756		}
 757
 758		arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT;
 759		arg->params[0].u.tmem.buf_ptr = pa;
 760	}
 761	arg->params[0].u.tmem.size = tee_shm_get_size(shm);
 762	arg->params[0].u.tmem.shm_ref = (unsigned long)shm;
 763
 764	arg->ret = TEEC_SUCCESS;
 765	return;
 766bad:
 767	tee_shm_free(shm);
 768}
 769
 770static void free_pages_list(struct optee_call_ctx *call_ctx)
 771{
 772	if (call_ctx->pages_list) {
 773		optee_free_pages_list(call_ctx->pages_list,
 774				      call_ctx->num_entries);
 775		call_ctx->pages_list = NULL;
 776		call_ctx->num_entries = 0;
 777	}
 778}
 779
 780static void optee_rpc_finalize_call(struct optee_call_ctx *call_ctx)
 781{
 782	free_pages_list(call_ctx);
 783}
 784
 785static void handle_rpc_func_cmd(struct tee_context *ctx, struct optee *optee,
 786				struct optee_msg_arg *arg,
 787				struct optee_call_ctx *call_ctx)
 788{
 789
 790	switch (arg->cmd) {
 791	case OPTEE_RPC_CMD_SHM_ALLOC:
 792		free_pages_list(call_ctx);
 793		handle_rpc_func_cmd_shm_alloc(ctx, optee, arg, call_ctx);
 794		break;
 795	case OPTEE_RPC_CMD_SHM_FREE:
 796		handle_rpc_func_cmd_shm_free(ctx, arg);
 797		break;
 798	default:
 799		optee_rpc_cmd(ctx, optee, arg);
 800	}
 801}
 802
 803/**
 804 * optee_handle_rpc() - handle RPC from secure world
 805 * @ctx:	context doing the RPC
 806 * @rpc_arg:	pointer to RPC arguments if any, or NULL if none
 807 * @param:	value of registers for the RPC
 808 * @call_ctx:	call context. Preserved during one OP-TEE invocation
 809 *
 810 * Result of RPC is written back into @param.
 811 */
 812static void optee_handle_rpc(struct tee_context *ctx,
 813			     struct optee_msg_arg *rpc_arg,
 814			     struct optee_rpc_param *param,
 815			     struct optee_call_ctx *call_ctx)
 816{
 817	struct tee_device *teedev = ctx->teedev;
 818	struct optee *optee = tee_get_drvdata(teedev);
 819	struct optee_msg_arg *arg;
 820	struct tee_shm *shm;
 821	phys_addr_t pa;
 822
 823	switch (OPTEE_SMC_RETURN_GET_RPC_FUNC(param->a0)) {
 824	case OPTEE_SMC_RPC_FUNC_ALLOC:
 825		shm = tee_shm_alloc_priv_buf(optee->ctx, param->a1);
 826		if (!IS_ERR(shm) && !tee_shm_get_pa(shm, 0, &pa)) {
 827			reg_pair_from_64(&param->a1, &param->a2, pa);
 828			reg_pair_from_64(&param->a4, &param->a5,
 829					 (unsigned long)shm);
 830		} else {
 831			param->a1 = 0;
 832			param->a2 = 0;
 833			param->a4 = 0;
 834			param->a5 = 0;
 835		}
 836		kmemleak_not_leak(shm);
 837		break;
 838	case OPTEE_SMC_RPC_FUNC_FREE:
 839		shm = reg_pair_to_ptr(param->a1, param->a2);
 840		tee_shm_free(shm);
 841		break;
 842	case OPTEE_SMC_RPC_FUNC_FOREIGN_INTR:
 843		/*
 844		 * A foreign interrupt was raised while secure world was
 845		 * executing, since they are handled in Linux a dummy RPC is
 846		 * performed to let Linux take the interrupt through the normal
 847		 * vector.
 848		 */
 849		break;
 850	case OPTEE_SMC_RPC_FUNC_CMD:
 851		if (rpc_arg) {
 852			arg = rpc_arg;
 853		} else {
 854			shm = reg_pair_to_ptr(param->a1, param->a2);
 855			arg = tee_shm_get_va(shm, 0);
 856			if (IS_ERR(arg)) {
 857				pr_err("%s: tee_shm_get_va %p failed\n",
 858				       __func__, shm);
 859				break;
 860			}
 861		}
 862
 863		handle_rpc_func_cmd(ctx, optee, arg, call_ctx);
 864		break;
 865	default:
 866		pr_warn("Unknown RPC func 0x%x\n",
 867			(u32)OPTEE_SMC_RETURN_GET_RPC_FUNC(param->a0));
 868		break;
 869	}
 870
 871	param->a0 = OPTEE_SMC_CALL_RETURN_FROM_RPC;
 872}
 873
 874/**
 875 * optee_smc_do_call_with_arg() - Do an SMC to OP-TEE in secure world
 876 * @ctx:	calling context
 877 * @shm:	shared memory holding the message to pass to secure world
 878 * @offs:	offset of the message in @shm
 879 * @system_thread: true if caller requests TEE system thread support
 880 *
 881 * Does and SMC to OP-TEE in secure world and handles eventual resulting
 882 * Remote Procedure Calls (RPC) from OP-TEE.
 883 *
 884 * Returns return code from secure world, 0 is OK
 885 */
 886static int optee_smc_do_call_with_arg(struct tee_context *ctx,
 887				      struct tee_shm *shm, u_int offs,
 888				      bool system_thread)
 889{
 890	struct optee *optee = tee_get_drvdata(ctx->teedev);
 891	struct optee_call_waiter w;
 892	struct optee_rpc_param param = { };
 893	struct optee_call_ctx call_ctx = { };
 894	struct optee_msg_arg *rpc_arg = NULL;
 895	int rc;
 896
 897	if (optee->rpc_param_count) {
 898		struct optee_msg_arg *arg;
 899		unsigned int rpc_arg_offs;
 900
 901		arg = tee_shm_get_va(shm, offs);
 902		if (IS_ERR(arg))
 903			return PTR_ERR(arg);
 904
 905		rpc_arg_offs = OPTEE_MSG_GET_ARG_SIZE(arg->num_params);
 906		rpc_arg = tee_shm_get_va(shm, offs + rpc_arg_offs);
 907		if (IS_ERR(rpc_arg))
 908			return PTR_ERR(rpc_arg);
 909	}
 910
 911	if  (rpc_arg && tee_shm_is_dynamic(shm)) {
 912		param.a0 = OPTEE_SMC_CALL_WITH_REGD_ARG;
 913		reg_pair_from_64(&param.a1, &param.a2, (u_long)shm);
 914		param.a3 = offs;
 915	} else {
 916		phys_addr_t parg;
 917
 918		rc = tee_shm_get_pa(shm, offs, &parg);
 919		if (rc)
 920			return rc;
 921
 922		if (rpc_arg)
 923			param.a0 = OPTEE_SMC_CALL_WITH_RPC_ARG;
 924		else
 925			param.a0 = OPTEE_SMC_CALL_WITH_ARG;
 926		reg_pair_from_64(&param.a1, &param.a2, parg);
 927	}
 928	/* Initialize waiter */
 929	optee_cq_wait_init(&optee->call_queue, &w, system_thread);
 930	while (true) {
 931		struct arm_smccc_res res;
 932
 933		trace_optee_invoke_fn_begin(&param);
 934		optee->smc.invoke_fn(param.a0, param.a1, param.a2, param.a3,
 935				     param.a4, param.a5, param.a6, param.a7,
 936				     &res);
 937		trace_optee_invoke_fn_end(&param, &res);
 938
 939		if (res.a0 == OPTEE_SMC_RETURN_ETHREAD_LIMIT) {
 940			/*
 941			 * Out of threads in secure world, wait for a thread
 942			 * become available.
 943			 */
 944			optee_cq_wait_for_completion(&optee->call_queue, &w);
 945		} else if (OPTEE_SMC_RETURN_IS_RPC(res.a0)) {
 946			cond_resched();
 947			param.a0 = res.a0;
 948			param.a1 = res.a1;
 949			param.a2 = res.a2;
 950			param.a3 = res.a3;
 951			optee_handle_rpc(ctx, rpc_arg, &param, &call_ctx);
 952		} else {
 953			rc = res.a0;
 954			break;
 955		}
 956	}
 957
 958	optee_rpc_finalize_call(&call_ctx);
 959	/*
 960	 * We're done with our thread in secure world, if there's any
 961	 * thread waiters wake up one.
 962	 */
 963	optee_cq_wait_final(&optee->call_queue, &w);
 964
 965	return rc;
 966}
 967
 968static int optee_smc_lend_protmem(struct optee *optee, struct tee_shm *protmem,
 969				  u32 *mem_attrs, unsigned int ma_count,
 970				  u32 use_case)
 971{
 972	struct optee_shm_arg_entry *entry;
 973	struct optee_msg_arg *msg_arg;
 974	struct tee_shm *shm;
 975	u_int offs;
 976	int rc;
 977
 978	msg_arg = optee_get_msg_arg(optee->ctx, 2, &entry, &shm, &offs);
 979	if (IS_ERR(msg_arg))
 980		return PTR_ERR(msg_arg);
 981
 982	msg_arg->cmd = OPTEE_MSG_CMD_LEND_PROTMEM;
 983	msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_VALUE_INPUT;
 984	msg_arg->params[0].u.value.a = use_case;
 985	msg_arg->params[1].attr = OPTEE_MSG_ATTR_TYPE_TMEM_INPUT;
 986	msg_arg->params[1].u.tmem.buf_ptr = protmem->paddr;
 987	msg_arg->params[1].u.tmem.size = protmem->size;
 988	msg_arg->params[1].u.tmem.shm_ref = (u_long)protmem;
 989
 990	rc = optee->ops->do_call_with_arg(optee->ctx, shm, offs, false);
 991	if (rc)
 992		goto out;
 993	if (msg_arg->ret != TEEC_SUCCESS) {
 994		rc = -EINVAL;
 995		goto out;
 996	}
 997	protmem->sec_world_id = (u_long)protmem;
 998
 999out:
1000	optee_free_msg_arg(optee->ctx, entry, offs);
1001	return rc;
1002}
1003
1004static int optee_smc_reclaim_protmem(struct optee *optee,
1005				     struct tee_shm *protmem)
1006{
1007	struct optee_shm_arg_entry *entry;
1008	struct optee_msg_arg *msg_arg;
1009	struct tee_shm *shm;
1010	u_int offs;
1011	int rc;
1012
1013	msg_arg = optee_get_msg_arg(optee->ctx, 1, &entry, &shm, &offs);
1014	if (IS_ERR(msg_arg))
1015		return PTR_ERR(msg_arg);
1016
1017	msg_arg->cmd = OPTEE_MSG_CMD_RECLAIM_PROTMEM;
1018	msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT;
1019	msg_arg->params[0].u.rmem.shm_ref = (u_long)protmem;
1020
1021	rc = optee->ops->do_call_with_arg(optee->ctx, shm, offs, false);
1022	if (rc)
1023		goto out;
1024	if (msg_arg->ret != TEEC_SUCCESS)
1025		rc = -EINVAL;
1026
1027out:
1028	optee_free_msg_arg(optee->ctx, entry, offs);
1029	return rc;
1030}
1031
1032/*
1033 * 5. Asynchronous notification
1034 */
1035
1036static u32 get_async_notif_value(optee_invoke_fn *invoke_fn, bool *value_valid,
1037				 bool *value_pending)
1038{
1039	struct arm_smccc_res res;
1040
1041	invoke_fn(OPTEE_SMC_GET_ASYNC_NOTIF_VALUE, 0, 0, 0, 0, 0, 0, 0, &res);
1042
1043	if (res.a0) {
1044		*value_valid = false;
1045		return 0;
1046	}
1047	*value_valid = (res.a2 & OPTEE_SMC_ASYNC_NOTIF_VALUE_VALID);
1048	*value_pending = (res.a2 & OPTEE_SMC_ASYNC_NOTIF_VALUE_PENDING);
1049	return res.a1;
1050}
1051
1052static irqreturn_t irq_handler(struct optee *optee)
1053{
1054	bool do_bottom_half = false;
1055	bool value_valid;
1056	bool value_pending;
1057	u32 value;
1058
1059	do {
1060		value = get_async_notif_value(optee->smc.invoke_fn,
1061					      &value_valid, &value_pending);
1062		if (!value_valid)
1063			break;
1064
1065		if (value == OPTEE_SMC_ASYNC_NOTIF_VALUE_DO_BOTTOM_HALF)
1066			do_bottom_half = true;
1067		else
1068			optee_notif_send(optee, value);
1069	} while (value_pending);
1070
1071	if (do_bottom_half)
1072		return IRQ_WAKE_THREAD;
1073	return IRQ_HANDLED;
1074}
1075
1076static irqreturn_t notif_irq_handler(int irq, void *dev_id)
1077{
1078	struct optee *optee = dev_id;
1079
1080	return irq_handler(optee);
1081}
1082
1083static irqreturn_t notif_irq_thread_fn(int irq, void *dev_id)
1084{
1085	struct optee *optee = dev_id;
1086
1087	optee_do_bottom_half(optee->ctx);
1088
1089	return IRQ_HANDLED;
1090}
1091
1092static int init_irq(struct optee *optee, u_int irq)
1093{
1094	int rc;
1095
1096	rc = request_threaded_irq(irq, notif_irq_handler,
1097				  notif_irq_thread_fn,
1098				  0, "optee_notification", optee);
1099	if (rc)
1100		return rc;
1101
1102	optee->smc.notif_irq = irq;
1103
1104	return 0;
1105}
1106
1107static irqreturn_t notif_pcpu_irq_handler(int irq, void *dev_id)
1108{
1109	struct optee_pcpu *pcpu = dev_id;
1110	struct optee *optee = pcpu->optee;
1111
1112	if (irq_handler(optee) == IRQ_WAKE_THREAD)
1113		queue_work(optee->smc.notif_pcpu_wq,
1114			   &optee->smc.notif_pcpu_work);
1115
1116	return IRQ_HANDLED;
1117}
1118
1119static void notif_pcpu_irq_work_fn(struct work_struct *work)
1120{
1121	struct optee_smc *optee_smc = container_of(work, struct optee_smc,
1122						   notif_pcpu_work);
1123	struct optee *optee = container_of(optee_smc, struct optee, smc);
1124
1125	optee_do_bottom_half(optee->ctx);
1126}
1127
1128static int init_pcpu_irq(struct optee *optee, u_int irq)
1129{
1130	struct optee_pcpu __percpu *optee_pcpu;
1131	int cpu, rc;
1132
1133	optee_pcpu = alloc_percpu(struct optee_pcpu);
1134	if (!optee_pcpu)
1135		return -ENOMEM;
1136
1137	for_each_present_cpu(cpu)
1138		per_cpu_ptr(optee_pcpu, cpu)->optee = optee;
1139
1140	rc = request_percpu_irq(irq, notif_pcpu_irq_handler,
1141				"optee_pcpu_notification", optee_pcpu);
1142	if (rc)
1143		goto err_free_pcpu;
1144
1145	INIT_WORK(&optee->smc.notif_pcpu_work, notif_pcpu_irq_work_fn);
1146	optee->smc.notif_pcpu_wq = create_workqueue("optee_pcpu_notification");
1147	if (!optee->smc.notif_pcpu_wq) {
1148		rc = -EINVAL;
1149		goto err_free_pcpu_irq;
1150	}
1151
1152	optee->smc.optee_pcpu = optee_pcpu;
1153	optee->smc.notif_irq = irq;
1154
1155	pcpu_irq_num = irq;
1156	rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "optee/pcpu-notif:starting",
1157			       optee_cpuhp_enable_pcpu_irq,
1158			       optee_cpuhp_disable_pcpu_irq);
1159	if (!rc)
1160		rc = -EINVAL;
1161	if (rc < 0)
1162		goto err_free_pcpu_irq;
1163
1164	optee->smc.notif_cpuhp_state = rc;
1165
1166	return 0;
1167
1168err_free_pcpu_irq:
1169	free_percpu_irq(irq, optee_pcpu);
1170err_free_pcpu:
1171	free_percpu(optee_pcpu);
1172
1173	return rc;
1174}
1175
1176static int optee_smc_notif_init_irq(struct optee *optee, u_int irq)
1177{
1178	if (irq_is_percpu_devid(irq))
1179		return init_pcpu_irq(optee, irq);
1180	else
1181		return init_irq(optee, irq);
1182}
1183
1184static void uninit_pcpu_irq(struct optee *optee)
1185{
1186	cpuhp_remove_state(optee->smc.notif_cpuhp_state);
1187
1188	destroy_workqueue(optee->smc.notif_pcpu_wq);
1189
1190	free_percpu_irq(optee->smc.notif_irq, optee->smc.optee_pcpu);
1191	free_percpu(optee->smc.optee_pcpu);
1192}
1193
1194static void optee_smc_notif_uninit_irq(struct optee *optee)
1195{
1196	if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF) {
1197		optee_stop_async_notif(optee->ctx);
1198		if (optee->smc.notif_irq) {
1199			if (irq_is_percpu_devid(optee->smc.notif_irq))
1200				uninit_pcpu_irq(optee);
1201			else
1202				free_irq(optee->smc.notif_irq, optee);
1203
1204			irq_dispose_mapping(optee->smc.notif_irq);
1205		}
1206	}
1207}
1208
1209/*
1210 * 6. Driver initialization
1211 *
1212 * During driver initialization is secure world probed to find out which
1213 * features it supports so the driver can be initialized with a matching
1214 * configuration. This involves for instance support for dynamic shared
1215 * memory instead of a static memory carvout.
1216 */
1217
1218static void optee_get_version(struct tee_device *teedev,
1219			      struct tee_ioctl_version_data *vers)
1220{
1221	struct tee_ioctl_version_data v = {
1222		.impl_id = TEE_IMPL_ID_OPTEE,
1223		.impl_caps = TEE_OPTEE_CAP_TZ,
1224		.gen_caps = TEE_GEN_CAP_GP,
1225	};
1226	struct optee *optee = tee_get_drvdata(teedev);
1227
1228	if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM)
1229		v.gen_caps |= TEE_GEN_CAP_REG_MEM;
1230	if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_MEMREF_NULL)
1231		v.gen_caps |= TEE_GEN_CAP_MEMREF_NULL;
1232	*vers = v;
1233}
1234
1235static int optee_smc_open(struct tee_context *ctx)
1236{
1237	struct optee *optee = tee_get_drvdata(ctx->teedev);
1238	u32 sec_caps = optee->smc.sec_caps;
1239
1240	return optee_open(ctx, sec_caps & OPTEE_SMC_SEC_CAP_MEMREF_NULL);
1241}
1242
1243static const struct tee_driver_ops optee_clnt_ops = {
1244	.get_version = optee_get_version,
1245	.open = optee_smc_open,
1246	.release = optee_release,
1247	.open_session = optee_open_session,
1248	.close_session = optee_close_session,
1249	.system_session = optee_system_session,
1250	.invoke_func = optee_invoke_func,
1251	.cancel_req = optee_cancel_req,
1252	.shm_register = optee_shm_register,
1253	.shm_unregister = optee_shm_unregister,
1254};
1255
1256static const struct tee_desc optee_clnt_desc = {
1257	.name = DRIVER_NAME "-clnt",
1258	.ops = &optee_clnt_ops,
1259	.owner = THIS_MODULE,
1260};
1261
1262static const struct tee_driver_ops optee_supp_ops = {
1263	.get_version = optee_get_version,
1264	.open = optee_smc_open,
1265	.release = optee_release_supp,
1266	.supp_recv = optee_supp_recv,
1267	.supp_send = optee_supp_send,
1268	.shm_register = optee_shm_register_supp,
1269	.shm_unregister = optee_shm_unregister_supp,
1270};
1271
1272static const struct tee_desc optee_supp_desc = {
1273	.name = DRIVER_NAME "-supp",
1274	.ops = &optee_supp_ops,
1275	.owner = THIS_MODULE,
1276	.flags = TEE_DESC_PRIVILEGED,
1277};
1278
1279static const struct optee_ops optee_ops = {
1280	.do_call_with_arg = optee_smc_do_call_with_arg,
1281	.to_msg_param = optee_to_msg_param,
1282	.from_msg_param = optee_from_msg_param,
1283	.lend_protmem = optee_smc_lend_protmem,
1284	.reclaim_protmem = optee_smc_reclaim_protmem,
1285};
1286
1287static int enable_async_notif(optee_invoke_fn *invoke_fn)
1288{
1289	struct arm_smccc_res res;
1290
1291	invoke_fn(OPTEE_SMC_ENABLE_ASYNC_NOTIF, 0, 0, 0, 0, 0, 0, 0, &res);
1292
1293	if (res.a0)
1294		return -EINVAL;
1295	return 0;
1296}
1297
1298static bool optee_msg_api_uid_is_optee_api(optee_invoke_fn *invoke_fn)
1299{
1300	struct arm_smccc_res res;
1301
1302	invoke_fn(OPTEE_SMC_CALLS_UID, 0, 0, 0, 0, 0, 0, 0, &res);
1303
1304	if (res.a0 == OPTEE_MSG_UID_0 && res.a1 == OPTEE_MSG_UID_1 &&
1305	    res.a2 == OPTEE_MSG_UID_2 && res.a3 == OPTEE_MSG_UID_3)
1306		return true;
1307	return false;
1308}
1309
1310#ifdef CONFIG_OPTEE_INSECURE_LOAD_IMAGE
1311static bool optee_msg_api_uid_is_optee_image_load(optee_invoke_fn *invoke_fn)
1312{
1313	struct arm_smccc_res res;
1314
1315	invoke_fn(OPTEE_SMC_CALLS_UID, 0, 0, 0, 0, 0, 0, 0, &res);
1316
1317	if (res.a0 == OPTEE_MSG_IMAGE_LOAD_UID_0 &&
1318	    res.a1 == OPTEE_MSG_IMAGE_LOAD_UID_1 &&
1319	    res.a2 == OPTEE_MSG_IMAGE_LOAD_UID_2 &&
1320	    res.a3 == OPTEE_MSG_IMAGE_LOAD_UID_3)
1321		return true;
1322	return false;
1323}
1324#endif
1325
1326static void optee_msg_get_os_revision(optee_invoke_fn *invoke_fn)
1327{
1328	union {
1329		struct arm_smccc_res smccc;
1330		struct optee_smc_call_get_os_revision_result result;
1331	} res = {
1332		.result = {
1333			.build_id = 0
1334		}
1335	};
1336
1337	invoke_fn(OPTEE_SMC_CALL_GET_OS_REVISION, 0, 0, 0, 0, 0, 0, 0,
1338		  &res.smccc);
1339
1340	if (res.result.build_id)
1341		pr_info("revision %lu.%lu (%0*lx)", res.result.major,
1342			res.result.minor, (int)sizeof(res.result.build_id) * 2,
1343			res.result.build_id);
1344	else
1345		pr_info("revision %lu.%lu", res.result.major, res.result.minor);
1346}
1347
1348static bool optee_msg_api_revision_is_compatible(optee_invoke_fn *invoke_fn)
1349{
1350	union {
1351		struct arm_smccc_res smccc;
1352		struct optee_smc_calls_revision_result result;
1353	} res;
1354
1355	invoke_fn(OPTEE_SMC_CALLS_REVISION, 0, 0, 0, 0, 0, 0, 0, &res.smccc);
1356
1357	if (res.result.major == OPTEE_MSG_REVISION_MAJOR &&
1358	    (int)res.result.minor >= OPTEE_MSG_REVISION_MINOR)
1359		return true;
1360	return false;
1361}
1362
1363static bool optee_msg_exchange_capabilities(optee_invoke_fn *invoke_fn,
1364					    u32 *sec_caps, u32 *max_notif_value,
1365					    unsigned int *rpc_param_count)
1366{
1367	union {
1368		struct arm_smccc_res smccc;
1369		struct optee_smc_exchange_capabilities_result result;
1370	} res;
1371	u32 a1 = 0;
1372
1373	/*
1374	 * TODO This isn't enough to tell if it's UP system (from kernel
1375	 * point of view) or not, is_smp() returns the information
1376	 * needed, but can't be called directly from here.
1377	 */
1378	if (!IS_ENABLED(CONFIG_SMP) || nr_cpu_ids == 1)
1379		a1 |= OPTEE_SMC_NSEC_CAP_UNIPROCESSOR;
1380
1381	invoke_fn(OPTEE_SMC_EXCHANGE_CAPABILITIES, a1, 0, 0, 0, 0, 0, 0,
1382		  &res.smccc);
1383
1384	if (res.result.status != OPTEE_SMC_RETURN_OK)
1385		return false;
1386
1387	*sec_caps = res.result.capabilities;
1388	if (*sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF)
1389		*max_notif_value = res.result.max_notif_value;
1390	else
1391		*max_notif_value = OPTEE_DEFAULT_MAX_NOTIF_VALUE;
1392	if (*sec_caps & OPTEE_SMC_SEC_CAP_RPC_ARG)
1393		*rpc_param_count = (u8)res.result.data;
1394	else
1395		*rpc_param_count = 0;
1396
1397	return true;
1398}
1399
1400static unsigned int optee_msg_get_thread_count(optee_invoke_fn *invoke_fn)
1401{
1402	struct arm_smccc_res res;
1403
1404	invoke_fn(OPTEE_SMC_GET_THREAD_COUNT, 0, 0, 0, 0, 0, 0, 0, &res);
1405	if (res.a0)
1406		return 0;
1407	return res.a1;
1408}
1409
1410static struct tee_shm_pool *
1411optee_config_shm_memremap(optee_invoke_fn *invoke_fn, void **memremaped_shm)
1412{
1413	union {
1414		struct arm_smccc_res smccc;
1415		struct optee_smc_get_shm_config_result result;
1416	} res;
1417	unsigned long vaddr;
1418	phys_addr_t paddr;
1419	size_t size;
1420	phys_addr_t begin;
1421	phys_addr_t end;
1422	void *va;
1423	void *rc;
1424
1425	invoke_fn(OPTEE_SMC_GET_SHM_CONFIG, 0, 0, 0, 0, 0, 0, 0, &res.smccc);
1426	if (res.result.status != OPTEE_SMC_RETURN_OK) {
1427		pr_err("static shm service not available\n");
1428		return ERR_PTR(-ENOENT);
1429	}
1430
1431	if (res.result.settings != OPTEE_SMC_SHM_CACHED) {
1432		pr_err("only normal cached shared memory supported\n");
1433		return ERR_PTR(-EINVAL);
1434	}
1435
1436	begin = roundup(res.result.start, PAGE_SIZE);
1437	end = rounddown(res.result.start + res.result.size, PAGE_SIZE);
1438	paddr = begin;
1439	size = end - begin;
1440
1441	va = memremap(paddr, size, MEMREMAP_WB);
1442	if (!va) {
1443		pr_err("shared memory ioremap failed\n");
1444		return ERR_PTR(-EINVAL);
1445	}
1446	vaddr = (unsigned long)va;
1447
1448	rc = tee_shm_pool_alloc_res_mem(vaddr, paddr, size,
1449					OPTEE_MIN_STATIC_POOL_ALIGN);
1450	if (IS_ERR(rc))
1451		memunmap(va);
1452	else
1453		*memremaped_shm = va;
1454
1455	return rc;
1456}
1457
1458/* Simple wrapper functions to be able to use a function pointer */
1459static void optee_smccc_smc(unsigned long a0, unsigned long a1,
1460			    unsigned long a2, unsigned long a3,
1461			    unsigned long a4, unsigned long a5,
1462			    unsigned long a6, unsigned long a7,
1463			    struct arm_smccc_res *res)
1464{
1465	arm_smccc_smc(a0, a1, a2, a3, a4, a5, a6, a7, res);
1466}
1467
1468static void optee_smccc_hvc(unsigned long a0, unsigned long a1,
1469			    unsigned long a2, unsigned long a3,
1470			    unsigned long a4, unsigned long a5,
1471			    unsigned long a6, unsigned long a7,
1472			    struct arm_smccc_res *res)
1473{
1474	arm_smccc_hvc(a0, a1, a2, a3, a4, a5, a6, a7, res);
1475}
1476
1477static optee_invoke_fn *get_invoke_func(struct device *dev)
1478{
1479	const char *method;
1480
1481	pr_info("probing for conduit method.\n");
1482
1483	if (device_property_read_string(dev, "method", &method)) {
1484		pr_warn("missing \"method\" property\n");
1485		return ERR_PTR(-ENXIO);
1486	}
1487
1488	if (!strcmp("hvc", method))
1489		return optee_smccc_hvc;
1490	else if (!strcmp("smc", method))
1491		return optee_smccc_smc;
1492
1493	pr_warn("invalid \"method\" property: %s\n", method);
1494	return ERR_PTR(-EINVAL);
1495}
1496
1497/* optee_remove - Device Removal Routine
1498 * @pdev: platform device information struct
1499 *
1500 * optee_remove is called by platform subsystem to alert the driver
1501 * that it should release the device
1502 */
1503static void optee_smc_remove(struct platform_device *pdev)
1504{
1505	struct optee *optee = platform_get_drvdata(pdev);
1506
1507	/*
1508	 * Ask OP-TEE to free all cached shared memory objects to decrease
1509	 * reference counters and also avoid wild pointers in secure world
1510	 * into the old shared memory range.
1511	 */
1512	if (!optee->rpc_param_count)
1513		optee_disable_shm_cache(optee);
1514
1515	optee_smc_notif_uninit_irq(optee);
1516
1517	optee_remove_common(optee);
1518
1519	if (optee->smc.memremaped_shm)
1520		memunmap(optee->smc.memremaped_shm);
1521
1522	kfree(optee);
1523}
1524
1525/* optee_shutdown - Device Removal Routine
1526 * @pdev: platform device information struct
1527 *
1528 * platform_shutdown is called by the platform subsystem to alert
1529 * the driver that a shutdown, reboot, or kexec is happening and
1530 * device must be disabled.
1531 */
1532static void optee_shutdown(struct platform_device *pdev)
1533{
1534	struct optee *optee = platform_get_drvdata(pdev);
1535
1536	if (!optee->rpc_param_count)
1537		optee_disable_shm_cache(optee);
1538}
1539
1540#ifdef CONFIG_OPTEE_INSECURE_LOAD_IMAGE
1541
1542#define OPTEE_FW_IMAGE "optee/tee.bin"
1543
1544static optee_invoke_fn *cpuhp_invoke_fn;
1545
1546static int optee_cpuhp_probe(unsigned int cpu)
1547{
1548	/*
1549	 * Invoking a call on a CPU will cause OP-TEE to perform the required
1550	 * setup for that CPU. Just invoke the call to get the UID since that
1551	 * has no side effects.
1552	 */
1553	if (optee_msg_api_uid_is_optee_api(cpuhp_invoke_fn))
1554		return 0;
1555	else
1556		return -EINVAL;
1557}
1558
1559static int optee_load_fw(struct platform_device *pdev,
1560			 optee_invoke_fn *invoke_fn)
1561{
1562	const struct firmware *fw = NULL;
1563	struct arm_smccc_res res;
1564	phys_addr_t data_pa;
1565	u8 *data_buf = NULL;
1566	u64 data_size;
1567	u32 data_pa_high, data_pa_low;
1568	u32 data_size_high, data_size_low;
1569	int rc;
1570	int hp_state;
1571
1572	if (!optee_msg_api_uid_is_optee_image_load(invoke_fn))
1573		return 0;
1574
1575	rc = request_firmware(&fw, OPTEE_FW_IMAGE, &pdev->dev);
1576	if (rc) {
1577		/*
1578		 * The firmware in the rootfs will not be accessible until we
1579		 * are in the SYSTEM_RUNNING state, so return EPROBE_DEFER until
1580		 * that point.
1581		 */
1582		if (system_state < SYSTEM_RUNNING)
1583			return -EPROBE_DEFER;
1584		goto fw_err;
1585	}
1586
1587	data_size = fw->size;
1588	/*
1589	 * This uses the GFP_DMA flag to ensure we are allocated memory in the
1590	 * 32-bit space since TF-A cannot map memory beyond the 32-bit boundary.
1591	 */
1592	data_buf = kmemdup(fw->data, fw->size, GFP_KERNEL | GFP_DMA);
1593	if (!data_buf) {
1594		rc = -ENOMEM;
1595		goto fw_err;
1596	}
1597	data_pa = virt_to_phys(data_buf);
1598	reg_pair_from_64(&data_pa_high, &data_pa_low, data_pa);
1599	reg_pair_from_64(&data_size_high, &data_size_low, data_size);
1600	goto fw_load;
1601
1602fw_err:
1603	pr_warn("image loading failed\n");
1604	data_pa_high = 0;
1605	data_pa_low = 0;
1606	data_size_high = 0;
1607	data_size_low = 0;
1608
1609fw_load:
1610	/*
1611	 * Always invoke the SMC, even if loading the image fails, to indicate
1612	 * to EL3 that we have passed the point where it should allow invoking
1613	 * this SMC.
1614	 */
1615	pr_warn("OP-TEE image loaded from kernel, this can be insecure");
1616	invoke_fn(OPTEE_SMC_CALL_LOAD_IMAGE, data_size_high, data_size_low,
1617		  data_pa_high, data_pa_low, 0, 0, 0, &res);
1618	if (!rc)
1619		rc = res.a0;
1620	release_firmware(fw);
1621	kfree(data_buf);
1622
1623	if (!rc) {
1624		/*
1625		 * We need to initialize OP-TEE on all other running cores as
1626		 * well. Any cores that aren't running yet will get initialized
1627		 * when they are brought up by the power management functions in
1628		 * TF-A which are registered by the OP-TEE SPD. Due to that we
1629		 * can un-register the callback right after registering it.
1630		 */
1631		cpuhp_invoke_fn = invoke_fn;
1632		hp_state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "optee:probe",
1633					     optee_cpuhp_probe, NULL);
1634		if (hp_state < 0) {
1635			pr_warn("Failed with CPU hotplug setup for OP-TEE");
1636			return -EINVAL;
1637		}
1638		cpuhp_remove_state(hp_state);
1639		cpuhp_invoke_fn = NULL;
1640	}
1641
1642	return rc;
1643}
1644#else
1645static inline int optee_load_fw(struct platform_device *pdev,
1646				optee_invoke_fn *invoke_fn)
1647{
1648	return 0;
1649}
1650#endif
1651
1652static struct tee_protmem_pool *static_protmem_pool_init(struct optee *optee)
1653{
1654#if IS_ENABLED(CONFIG_OPTEE_STATIC_PROTMEM_POOL)
1655	union {
1656		struct arm_smccc_res smccc;
1657		struct optee_smc_get_protmem_config_result result;
1658	} res;
1659	struct tee_protmem_pool *pool;
1660	void *p;
1661	int rc;
1662
1663	optee->smc.invoke_fn(OPTEE_SMC_GET_PROTMEM_CONFIG, 0, 0, 0, 0,
1664			     0, 0, 0, &res.smccc);
1665	if (res.result.status != OPTEE_SMC_RETURN_OK)
1666		return ERR_PTR(-EINVAL);
1667
1668	rc = optee_set_dma_mask(optee, res.result.pa_width);
1669	if (rc)
1670		return ERR_PTR(rc);
1671
1672	/*
1673	 * Map the memory as uncached to make sure the kernel can work with
1674	 * __pfn_to_page() and friends since that's needed when passing the
1675	 * protected DMA-buf to a device. The memory should otherwise not
1676	 * be touched by the kernel since it's likely to cause an external
1677	 * abort due to the protection status.
1678	 */
1679	p = devm_memremap(&optee->teedev->dev, res.result.start,
1680			  res.result.size, MEMREMAP_WC);
1681	if (IS_ERR(p))
1682		return p;
1683
1684	pool = tee_protmem_static_pool_alloc(res.result.start, res.result.size);
1685	if (IS_ERR(pool))
1686		devm_memunmap(&optee->teedev->dev, p);
1687
1688	return pool;
1689#else
1690	return ERR_PTR(-EINVAL);
1691#endif
1692}
1693
1694static int optee_protmem_pool_init(struct optee *optee)
1695{
1696	bool protm = optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_PROTMEM;
1697	bool dyn_protm = optee->smc.sec_caps &
1698			 OPTEE_SMC_SEC_CAP_DYNAMIC_PROTMEM;
1699	enum tee_dma_heap_id heap_id = TEE_DMA_HEAP_SECURE_VIDEO_PLAY;
1700	struct tee_protmem_pool *pool = ERR_PTR(-EINVAL);
1701	int rc = -EINVAL;
1702
1703	if (!protm && !dyn_protm)
1704		return 0;
1705
1706	if (protm)
1707		pool = static_protmem_pool_init(optee);
1708	if (dyn_protm && IS_ERR(pool))
1709		pool = optee_protmem_alloc_dyn_pool(optee, heap_id);
1710	if (IS_ERR(pool))
1711		return PTR_ERR(pool);
1712
1713	rc = tee_device_register_dma_heap(optee->teedev, heap_id, pool);
1714	if (rc)
1715		pool->ops->destroy_pool(pool);
1716
1717	return rc;
1718}
1719
1720static int optee_probe(struct platform_device *pdev)
1721{
1722	optee_invoke_fn *invoke_fn;
1723	struct tee_shm_pool *pool = ERR_PTR(-EINVAL);
1724	struct optee *optee = NULL;
1725	void *memremaped_shm = NULL;
1726	unsigned int rpc_param_count;
1727	unsigned int thread_count;
1728	struct tee_device *teedev;
1729	struct tee_context *ctx;
1730	u32 max_notif_value;
1731	u32 arg_cache_flags;
1732	u32 sec_caps;
1733	int rc;
1734
1735	invoke_fn = get_invoke_func(&pdev->dev);
1736	if (IS_ERR(invoke_fn))
1737		return PTR_ERR(invoke_fn);
1738
1739	rc = optee_load_fw(pdev, invoke_fn);
1740	if (rc)
1741		return rc;
1742
1743	if (!optee_msg_api_uid_is_optee_api(invoke_fn)) {
1744		pr_warn("api uid mismatch\n");
1745		return -EINVAL;
1746	}
1747
1748	optee_msg_get_os_revision(invoke_fn);
1749
1750	if (!optee_msg_api_revision_is_compatible(invoke_fn)) {
1751		pr_warn("api revision mismatch\n");
1752		return -EINVAL;
1753	}
1754
1755	thread_count = optee_msg_get_thread_count(invoke_fn);
1756	if (!optee_msg_exchange_capabilities(invoke_fn, &sec_caps,
1757					     &max_notif_value,
1758					     &rpc_param_count)) {
1759		pr_warn("capabilities mismatch\n");
1760		return -EINVAL;
1761	}
1762
1763	/*
1764	 * Try to use dynamic shared memory if possible
1765	 */
1766	if (sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM) {
1767		/*
1768		 * If we have OPTEE_SMC_SEC_CAP_RPC_ARG we can ask
1769		 * optee_get_msg_arg() to pre-register (by having
1770		 * OPTEE_SHM_ARG_ALLOC_PRIV cleared) the page used to pass
1771		 * an argument struct.
1772		 *
1773		 * With the page is pre-registered we can use a non-zero
1774		 * offset for argument struct, this is indicated with
1775		 * OPTEE_SHM_ARG_SHARED.
1776		 *
1777		 * This means that optee_smc_do_call_with_arg() will use
1778		 * OPTEE_SMC_CALL_WITH_REGD_ARG for pre-registered pages.
1779		 */
1780		if (sec_caps & OPTEE_SMC_SEC_CAP_RPC_ARG)
1781			arg_cache_flags = OPTEE_SHM_ARG_SHARED;
1782		else
1783			arg_cache_flags = OPTEE_SHM_ARG_ALLOC_PRIV;
1784
1785		pool = optee_shm_pool_alloc_pages();
1786	}
1787
1788	/*
1789	 * If dynamic shared memory is not available or failed - try static one
1790	 */
1791	if (IS_ERR(pool) && (sec_caps & OPTEE_SMC_SEC_CAP_HAVE_RESERVED_SHM)) {
1792		/*
1793		 * The static memory pool can use non-zero page offsets so
1794		 * let optee_get_msg_arg() know that with OPTEE_SHM_ARG_SHARED.
1795		 *
1796		 * optee_get_msg_arg() should not pre-register the
1797		 * allocated page used to pass an argument struct, this is
1798		 * indicated with OPTEE_SHM_ARG_ALLOC_PRIV.
1799		 *
1800		 * This means that optee_smc_do_call_with_arg() will use
1801		 * OPTEE_SMC_CALL_WITH_ARG if rpc_param_count is 0, else
1802		 * OPTEE_SMC_CALL_WITH_RPC_ARG.
1803		 */
1804		arg_cache_flags = OPTEE_SHM_ARG_SHARED |
1805				  OPTEE_SHM_ARG_ALLOC_PRIV;
1806		pool = optee_config_shm_memremap(invoke_fn, &memremaped_shm);
1807	}
1808
1809	if (IS_ERR(pool))
1810		return PTR_ERR(pool);
1811
1812	optee = kzalloc(sizeof(*optee), GFP_KERNEL);
1813	if (!optee) {
1814		rc = -ENOMEM;
1815		goto err_free_shm_pool;
1816	}
1817
1818	optee->ops = &optee_ops;
1819	optee->smc.invoke_fn = invoke_fn;
1820	optee->smc.sec_caps = sec_caps;
1821	optee->rpc_param_count = rpc_param_count;
1822
1823	if (IS_REACHABLE(CONFIG_RPMB) &&
1824	    (sec_caps & OPTEE_SMC_SEC_CAP_RPMB_PROBE))
1825		optee->in_kernel_rpmb_routing = true;
1826
1827	teedev = tee_device_alloc(&optee_clnt_desc, NULL, pool, optee);
1828	if (IS_ERR(teedev)) {
1829		rc = PTR_ERR(teedev);
1830		goto err_free_optee;
1831	}
1832	optee->teedev = teedev;
1833
1834	teedev = tee_device_alloc(&optee_supp_desc, NULL, pool, optee);
1835	if (IS_ERR(teedev)) {
1836		rc = PTR_ERR(teedev);
1837		goto err_unreg_teedev;
1838	}
1839	optee->supp_teedev = teedev;
1840
1841	optee_set_dev_group(optee);
1842
1843	rc = tee_device_register(optee->teedev);
1844	if (rc)
1845		goto err_unreg_supp_teedev;
1846
1847	rc = tee_device_register(optee->supp_teedev);
1848	if (rc)
1849		goto err_unreg_supp_teedev;
1850
1851	optee_cq_init(&optee->call_queue, thread_count);
1852	optee_supp_init(&optee->supp);
1853	optee->smc.memremaped_shm = memremaped_shm;
1854	optee->pool = pool;
1855	optee_shm_arg_cache_init(optee, arg_cache_flags);
1856	mutex_init(&optee->rpmb_dev_mutex);
1857
1858	platform_set_drvdata(pdev, optee);
1859	ctx = teedev_open(optee->teedev);
1860	if (IS_ERR(ctx)) {
1861		rc = PTR_ERR(ctx);
1862		goto err_supp_uninit;
1863	}
1864	optee->ctx = ctx;
1865	rc = optee_notif_init(optee, max_notif_value);
1866	if (rc)
1867		goto err_close_ctx;
1868
1869	if (sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF) {
1870		unsigned int irq;
1871
1872		rc = platform_get_irq(pdev, 0);
1873		if (rc < 0) {
1874			pr_err("platform_get_irq: ret %d\n", rc);
1875			goto err_notif_uninit;
1876		}
1877		irq = rc;
1878
1879		rc = optee_smc_notif_init_irq(optee, irq);
1880		if (rc) {
1881			irq_dispose_mapping(irq);
1882			goto err_notif_uninit;
1883		}
1884		enable_async_notif(optee->smc.invoke_fn);
1885		pr_info("Asynchronous notifications enabled\n");
1886	}
1887
1888	if (optee_protmem_pool_init(optee))
1889		pr_info("Protected memory service not available\n");
1890
1891	/*
1892	 * Ensure that there are no pre-existing shm objects before enabling
1893	 * the shm cache so that there's no chance of receiving an invalid
1894	 * address during shutdown. This could occur, for example, if we're
1895	 * kexec booting from an older kernel that did not properly cleanup the
1896	 * shm cache.
1897	 */
1898	optee_disable_unmapped_shm_cache(optee);
1899
1900	/*
1901	 * Only enable the shm cache in case we're not able to pass the RPC
1902	 * arg struct right after the normal arg struct.
1903	 */
1904	if (!optee->rpc_param_count)
1905		optee_enable_shm_cache(optee);
1906
1907	if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM)
1908		pr_info("dynamic shared memory is enabled\n");
1909
1910	rc = optee_enumerate_devices(PTA_CMD_GET_DEVICES);
1911	if (rc)
1912		goto err_disable_shm_cache;
1913
1914	INIT_WORK(&optee->rpmb_scan_bus_work, optee_bus_scan_rpmb);
1915	optee->rpmb_intf.notifier_call = optee_rpmb_intf_rdev;
1916	blocking_notifier_chain_register(&optee_rpmb_intf_added,
1917					 &optee->rpmb_intf);
1918	pr_info("initialized driver\n");
1919	return 0;
1920
1921err_disable_shm_cache:
1922	if (!optee->rpc_param_count)
1923		optee_disable_shm_cache(optee);
1924	optee_smc_notif_uninit_irq(optee);
1925	optee_unregister_devices();
1926err_notif_uninit:
1927	optee_notif_uninit(optee);
1928err_close_ctx:
1929	teedev_close_context(ctx);
1930err_supp_uninit:
1931	rpmb_dev_put(optee->rpmb_dev);
1932	mutex_destroy(&optee->rpmb_dev_mutex);
1933	optee_shm_arg_cache_uninit(optee);
1934	optee_supp_uninit(&optee->supp);
1935	mutex_destroy(&optee->call_queue.mutex);
1936err_unreg_supp_teedev:
1937	tee_device_unregister(optee->supp_teedev);
1938err_unreg_teedev:
1939	tee_device_unregister(optee->teedev);
1940err_free_optee:
1941	kfree(optee);
1942err_free_shm_pool:
1943	tee_shm_pool_free(pool);
1944	if (memremaped_shm)
1945		memunmap(memremaped_shm);
1946	return rc;
1947}
1948
1949static const struct of_device_id optee_dt_match[] = {
1950	{ .compatible = "linaro,optee-tz" },
1951	{},
1952};
1953MODULE_DEVICE_TABLE(of, optee_dt_match);
1954
1955static struct platform_driver optee_driver = {
1956	.probe  = optee_probe,
1957	.remove = optee_smc_remove,
1958	.shutdown = optee_shutdown,
1959	.driver = {
1960		.name = "optee",
1961		.of_match_table = optee_dt_match,
1962	},
1963};
1964
1965int optee_smc_abi_register(void)
1966{
1967	return platform_driver_register(&optee_driver);
1968}
1969
1970void optee_smc_abi_unregister(void)
1971{
1972	platform_driver_unregister(&optee_driver);
1973}
~/ Read-only file preview.