qaic_control.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2
   3/* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */
   4/* Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */
   5
   6#include <asm/byteorder.h>
   7#include <linux/completion.h>
   8#include <linux/crc32.h>
   9#include <linux/delay.h>
  10#include <linux/dma-mapping.h>
  11#include <linux/kref.h>
  12#include <linux/list.h>
  13#include <linux/mhi.h>
  14#include <linux/mm.h>
  15#include <linux/moduleparam.h>
  16#include <linux/mutex.h>
  17#include <linux/overflow.h>
  18#include <linux/pci.h>
  19#include <linux/scatterlist.h>
  20#include <linux/sched/signal.h>
  21#include <linux/types.h>
  22#include <linux/uaccess.h>
  23#include <linux/workqueue.h>
  24#include <linux/wait.h>
  25#include <drm/drm_device.h>
  26#include <drm/drm_file.h>
  27#include <uapi/drm/qaic_accel.h>
  28
  29#include "qaic.h"
  30
  31#define MANAGE_MAGIC_NUMBER		((__force __le32)0x43494151) /* "QAIC" in little endian */
  32#define QAIC_DBC_Q_GAP			SZ_256
  33#define QAIC_DBC_Q_BUF_ALIGN		SZ_4K
  34#define QAIC_MANAGE_WIRE_MSG_LENGTH	SZ_64K /* Max DMA message length */
  35#define QAIC_WRAPPER_MAX_SIZE		SZ_4K
  36#define QAIC_MHI_RETRY_WAIT_MS		100
  37#define QAIC_MHI_RETRY_MAX		20
  38
  39static unsigned int control_resp_timeout_s = 60; /* 60 sec default */
  40module_param(control_resp_timeout_s, uint, 0600);
  41MODULE_PARM_DESC(control_resp_timeout_s, "Timeout for NNC responses from QSM");
  42
  43struct manage_msg {
  44	u32 len;
  45	u32 count;
  46	u8 data[];
  47};
  48
  49/*
  50 * wire encoding structures for the manage protocol.
  51 * All fields are little endian on the wire
  52 */
  53struct wire_msg_hdr {
  54	__le32 crc32; /* crc of everything following this field in the message */
  55	__le32 magic_number;
  56	__le32 sequence_number;
  57	__le32 len; /* length of this message */
  58	__le32 count; /* number of transactions in this message */
  59	__le32 handle; /* unique id to track the resources consumed */
  60	__le32 partition_id; /* partition id for the request (signed) */
  61	__le32 padding; /* must be 0 */
  62} __packed;
  63
  64struct wire_msg {
  65	struct wire_msg_hdr hdr;
  66	u8 data[];
  67} __packed;
  68
  69struct wire_trans_hdr {
  70	__le32 type;
  71	__le32 len;
  72} __packed;
  73
  74/* Each message sent from driver to device are organized in a list of wrapper_msg */
  75struct wrapper_msg {
  76	struct list_head list;
  77	struct kref ref_count;
  78	u32 len; /* length of data to transfer */
  79	struct wrapper_list *head;
  80	union {
  81		struct wire_msg msg;
  82		struct wire_trans_hdr trans;
  83	};
  84};
  85
  86struct wrapper_list {
  87	struct list_head list;
  88	spinlock_t lock; /* Protects the list state during additions and removals */
  89};
  90
  91struct wire_trans_passthrough {
  92	struct wire_trans_hdr hdr;
  93	u8 data[];
  94} __packed;
  95
  96struct wire_addr_size_pair {
  97	__le64 addr;
  98	__le64 size;
  99} __packed;
 100
 101struct wire_trans_dma_xfer {
 102	struct wire_trans_hdr hdr;
 103	__le32 tag;
 104	__le32 count;
 105	__le32 dma_chunk_id;
 106	__le32 padding;
 107	struct wire_addr_size_pair data[];
 108} __packed;
 109
 110/* Initiated by device to continue the DMA xfer of a large piece of data */
 111struct wire_trans_dma_xfer_cont {
 112	struct wire_trans_hdr hdr;
 113	__le32 dma_chunk_id;
 114	__le32 padding;
 115	__le64 xferred_size;
 116} __packed;
 117
 118struct wire_trans_activate_to_dev {
 119	struct wire_trans_hdr hdr;
 120	__le64 req_q_addr;
 121	__le64 rsp_q_addr;
 122	__le32 req_q_size;
 123	__le32 rsp_q_size;
 124	__le32 buf_len;
 125	__le32 options; /* unused, but BIT(16) has meaning to the device */
 126} __packed;
 127
 128struct wire_trans_activate_from_dev {
 129	struct wire_trans_hdr hdr;
 130	__le32 status;
 131	__le32 dbc_id;
 132	__le64 options; /* unused */
 133} __packed;
 134
 135struct wire_trans_deactivate_from_dev {
 136	struct wire_trans_hdr hdr;
 137	__le32 status;
 138	__le32 dbc_id;
 139} __packed;
 140
 141struct wire_trans_terminate_to_dev {
 142	struct wire_trans_hdr hdr;
 143	__le32 handle;
 144	__le32 padding;
 145} __packed;
 146
 147struct wire_trans_terminate_from_dev {
 148	struct wire_trans_hdr hdr;
 149	__le32 status;
 150	__le32 padding;
 151} __packed;
 152
 153struct wire_trans_status_to_dev {
 154	struct wire_trans_hdr hdr;
 155} __packed;
 156
 157struct wire_trans_status_from_dev {
 158	struct wire_trans_hdr hdr;
 159	__le16 major;
 160	__le16 minor;
 161	__le32 status;
 162	__le64 status_flags;
 163} __packed;
 164
 165struct wire_trans_validate_part_to_dev {
 166	struct wire_trans_hdr hdr;
 167	__le32 part_id;
 168	__le32 padding;
 169} __packed;
 170
 171struct wire_trans_validate_part_from_dev {
 172	struct wire_trans_hdr hdr;
 173	__le32 status;
 174	__le32 padding;
 175} __packed;
 176
 177struct xfer_queue_elem {
 178	/*
 179	 * Node in list of ongoing transfer request on control channel.
 180	 * Maintained by root device struct.
 181	 */
 182	struct list_head list;
 183	/* Sequence number of this transfer request */
 184	u32 seq_num;
 185	/* This is used to wait on until completion of transfer request */
 186	struct completion xfer_done;
 187	/* Received data from device */
 188	void *buf;
 189};
 190
 191struct dma_xfer {
 192	/* Node in list of DMA transfers which is used for cleanup */
 193	struct list_head list;
 194	/* SG table of memory used for DMA */
 195	struct sg_table *sgt;
 196	/* Array pages used for DMA */
 197	struct page **page_list;
 198	/* Number of pages used for DMA */
 199	unsigned long nr_pages;
 200};
 201
 202struct ioctl_resources {
 203	/* List of all DMA transfers which is used later for cleanup */
 204	struct list_head dma_xfers;
 205	/* Base address of request queue which belongs to a DBC */
 206	void *buf;
 207	/*
 208	 * Base bus address of request queue which belongs to a DBC. Response
 209	 * queue base bus address can be calculated by adding size of request
 210	 * queue to base bus address of request queue.
 211	 */
 212	dma_addr_t dma_addr;
 213	/* Total size of request queue and response queue in byte */
 214	u32 total_size;
 215	/* Total number of elements that can be queued in each of request and response queue */
 216	u32 nelem;
 217	/* Base address of response queue which belongs to a DBC */
 218	void *rsp_q_base;
 219	/* Status of the NNC message received */
 220	u32 status;
 221	/* DBC id of the DBC received from device */
 222	u32 dbc_id;
 223	/*
 224	 * DMA transfer request messages can be big in size and it may not be
 225	 * possible to send them in one shot. In such cases the messages are
 226	 * broken into chunks, this field stores ID of such chunks.
 227	 */
 228	u32 dma_chunk_id;
 229	/* Total number of bytes transferred for a DMA xfer request */
 230	u64 xferred_dma_size;
 231	/* Header of transaction message received from user. Used during DMA xfer request. */
 232	void *trans_hdr;
 233};
 234
 235struct resp_work {
 236	struct work_struct work;
 237	struct qaic_device *qdev;
 238	void *buf;
 239};
 240
 241/*
 242 * Since we're working with little endian messages, its useful to be able to
 243 * increment without filling a whole line with conversions back and forth just
 244 * to add one(1) to a message count.
 245 */
 246static __le32 incr_le32(__le32 val)
 247{
 248	return cpu_to_le32(le32_to_cpu(val) + 1);
 249}
 250
 251static u32 gen_crc(void *msg)
 252{
 253	struct wrapper_list *wrappers = msg;
 254	struct wrapper_msg *w;
 255	u32 crc = ~0;
 256
 257	list_for_each_entry(w, &wrappers->list, list)
 258		crc = crc32(crc, &w->msg, w->len);
 259
 260	return crc ^ ~0;
 261}
 262
 263static u32 gen_crc_stub(void *msg)
 264{
 265	return 0;
 266}
 267
 268static bool valid_crc(void *msg)
 269{
 270	struct wire_msg_hdr *hdr = msg;
 271	bool ret;
 272	u32 crc;
 273
 274	/*
 275	 * The output of this algorithm is always converted to the native
 276	 * endianness.
 277	 */
 278	crc = le32_to_cpu(hdr->crc32);
 279	hdr->crc32 = 0;
 280	ret = (crc32(~0, msg, le32_to_cpu(hdr->len)) ^ ~0) == crc;
 281	hdr->crc32 = cpu_to_le32(crc);
 282	return ret;
 283}
 284
 285static bool valid_crc_stub(void *msg)
 286{
 287	return true;
 288}
 289
 290static void free_wrapper(struct kref *ref)
 291{
 292	struct wrapper_msg *wrapper = container_of(ref, struct wrapper_msg, ref_count);
 293
 294	list_del(&wrapper->list);
 295	kfree(wrapper);
 296}
 297
 298static void save_dbc_buf(struct qaic_device *qdev, struct ioctl_resources *resources,
 299			 struct qaic_user *usr)
 300{
 301	u32 dbc_id = resources->dbc_id;
 302
 303	if (resources->buf) {
 304		wait_event_interruptible(qdev->dbc[dbc_id].dbc_release, !qdev->dbc[dbc_id].in_use);
 305		qdev->dbc[dbc_id].req_q_base = resources->buf;
 306		qdev->dbc[dbc_id].rsp_q_base = resources->rsp_q_base;
 307		qdev->dbc[dbc_id].dma_addr = resources->dma_addr;
 308		qdev->dbc[dbc_id].total_size = resources->total_size;
 309		qdev->dbc[dbc_id].nelem = resources->nelem;
 310		enable_dbc(qdev, dbc_id, usr);
 311		qdev->dbc[dbc_id].in_use = true;
 312		resources->buf = NULL;
 313		set_dbc_state(qdev, dbc_id, DBC_STATE_ASSIGNED);
 314	}
 315}
 316
 317static void free_dbc_buf(struct qaic_device *qdev, struct ioctl_resources *resources)
 318{
 319	if (resources->buf)
 320		dma_free_coherent(&qdev->pdev->dev, resources->total_size, resources->buf,
 321				  resources->dma_addr);
 322	resources->buf = NULL;
 323}
 324
 325static void free_dma_xfers(struct qaic_device *qdev, struct ioctl_resources *resources)
 326{
 327	struct dma_xfer *xfer;
 328	struct dma_xfer *x;
 329	int i;
 330
 331	list_for_each_entry_safe(xfer, x, &resources->dma_xfers, list) {
 332		dma_unmap_sgtable(&qdev->pdev->dev, xfer->sgt, DMA_TO_DEVICE, 0);
 333		sg_free_table(xfer->sgt);
 334		kfree(xfer->sgt);
 335		for (i = 0; i < xfer->nr_pages; ++i)
 336			put_page(xfer->page_list[i]);
 337		kfree(xfer->page_list);
 338		list_del(&xfer->list);
 339		kfree(xfer);
 340	}
 341}
 342
 343static struct wrapper_msg *add_wrapper(struct wrapper_list *wrappers, u32 size)
 344{
 345	struct wrapper_msg *w = kzalloc(size, GFP_KERNEL);
 346
 347	if (!w)
 348		return NULL;
 349	list_add_tail(&w->list, &wrappers->list);
 350	kref_init(&w->ref_count);
 351	w->head = wrappers;
 352	return w;
 353}
 354
 355static int encode_passthrough(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,
 356			      u32 *user_len)
 357{
 358	struct qaic_manage_trans_passthrough *in_trans = trans;
 359	struct wire_trans_passthrough *out_trans;
 360	struct wrapper_msg *trans_wrapper;
 361	struct wrapper_msg *wrapper;
 362	struct wire_msg *msg;
 363	u32 msg_hdr_len;
 364
 365	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
 366	msg = &wrapper->msg;
 367	msg_hdr_len = le32_to_cpu(msg->hdr.len);
 368
 369	if (in_trans->hdr.len % 8 != 0)
 370		return -EINVAL;
 371
 372	if (size_add(msg_hdr_len, in_trans->hdr.len) > QAIC_MANAGE_WIRE_MSG_LENGTH)
 373		return -ENOSPC;
 374
 375	trans_wrapper = add_wrapper(wrappers,
 376				    offsetof(struct wrapper_msg, trans) + in_trans->hdr.len);
 377	if (!trans_wrapper)
 378		return -ENOMEM;
 379	trans_wrapper->len = in_trans->hdr.len;
 380	out_trans = (struct wire_trans_passthrough *)&trans_wrapper->trans;
 381
 382	memcpy(out_trans->data, in_trans->data, in_trans->hdr.len - sizeof(in_trans->hdr));
 383	msg->hdr.len = cpu_to_le32(msg_hdr_len + in_trans->hdr.len);
 384	msg->hdr.count = incr_le32(msg->hdr.count);
 385	*user_len += in_trans->hdr.len;
 386	out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_PASSTHROUGH_TO_DEV);
 387	out_trans->hdr.len = cpu_to_le32(in_trans->hdr.len);
 388
 389	return 0;
 390}
 391
 392/* returns error code for failure, 0 if enough pages alloc'd, 1 if dma_cont is needed */
 393static int find_and_map_user_pages(struct qaic_device *qdev,
 394				   struct qaic_manage_trans_dma_xfer *in_trans,
 395				   struct ioctl_resources *resources, struct dma_xfer *xfer)
 396{
 397	u64 xfer_start_addr, remaining, end, total;
 398	unsigned long need_pages;
 399	struct page **page_list;
 400	unsigned long nr_pages;
 401	struct sg_table *sgt;
 402	int ret;
 403	int i;
 404
 405	if (check_add_overflow(in_trans->addr, resources->xferred_dma_size, &xfer_start_addr))
 406		return -EINVAL;
 407
 408	if (in_trans->size < resources->xferred_dma_size)
 409		return -EINVAL;
 410	remaining = in_trans->size - resources->xferred_dma_size;
 411	if (remaining == 0)
 412		return -EINVAL;
 413
 414	if (check_add_overflow(xfer_start_addr, remaining, &end))
 415		return -EINVAL;
 416
 417	total = remaining + offset_in_page(xfer_start_addr);
 418	if (total >= SIZE_MAX)
 419		return -EINVAL;
 420
 421	need_pages = DIV_ROUND_UP(total, PAGE_SIZE);
 422
 423	nr_pages = need_pages;
 424
 425	while (1) {
 426		page_list = kmalloc_array(nr_pages, sizeof(*page_list), GFP_KERNEL | __GFP_NOWARN);
 427		if (!page_list) {
 428			nr_pages = nr_pages / 2;
 429			if (!nr_pages)
 430				return -ENOMEM;
 431		} else {
 432			break;
 433		}
 434	}
 435
 436	ret = get_user_pages_fast(xfer_start_addr, nr_pages, 0, page_list);
 437	if (ret < 0)
 438		goto free_page_list;
 439	if (ret != nr_pages) {
 440		nr_pages = ret;
 441		ret = -EFAULT;
 442		goto put_pages;
 443	}
 444
 445	sgt = kmalloc(sizeof(*sgt), GFP_KERNEL);
 446	if (!sgt) {
 447		ret = -ENOMEM;
 448		goto put_pages;
 449	}
 450
 451	ret = sg_alloc_table_from_pages(sgt, page_list, nr_pages,
 452					offset_in_page(xfer_start_addr),
 453					remaining, GFP_KERNEL);
 454	if (ret) {
 455		ret = -ENOMEM;
 456		goto free_sgt;
 457	}
 458
 459	ret = dma_map_sgtable(&qdev->pdev->dev, sgt, DMA_TO_DEVICE, 0);
 460	if (ret)
 461		goto free_table;
 462
 463	xfer->sgt = sgt;
 464	xfer->page_list = page_list;
 465	xfer->nr_pages = nr_pages;
 466
 467	return need_pages > nr_pages ? 1 : 0;
 468
 469free_table:
 470	sg_free_table(sgt);
 471free_sgt:
 472	kfree(sgt);
 473put_pages:
 474	for (i = 0; i < nr_pages; ++i)
 475		put_page(page_list[i]);
 476free_page_list:
 477	kfree(page_list);
 478	return ret;
 479}
 480
 481/* returns error code for failure, 0 if everything was encoded, 1 if dma_cont is needed */
 482static int encode_addr_size_pairs(struct dma_xfer *xfer, struct wrapper_list *wrappers,
 483				  struct ioctl_resources *resources, u32 msg_hdr_len, u32 *size,
 484				  struct wire_trans_dma_xfer **out_trans)
 485{
 486	struct wrapper_msg *trans_wrapper;
 487	struct sg_table *sgt = xfer->sgt;
 488	struct wire_addr_size_pair *asp;
 489	struct scatterlist *sg;
 490	struct wrapper_msg *w;
 491	unsigned int dma_len;
 492	u64 dma_chunk_len;
 493	void *boundary;
 494	int nents_dma;
 495	int nents;
 496	int i;
 497
 498	nents = sgt->nents;
 499	nents_dma = nents;
 500	*size = QAIC_MANAGE_WIRE_MSG_LENGTH - msg_hdr_len - sizeof(**out_trans);
 501	for_each_sgtable_dma_sg(sgt, sg, i) {
 502		*size -= sizeof(*asp);
 503		/* Save 1K for possible follow-up transactions. */
 504		if (*size < SZ_1K) {
 505			nents_dma = i;
 506			break;
 507		}
 508	}
 509
 510	trans_wrapper = add_wrapper(wrappers, QAIC_WRAPPER_MAX_SIZE);
 511	if (!trans_wrapper)
 512		return -ENOMEM;
 513	*out_trans = (struct wire_trans_dma_xfer *)&trans_wrapper->trans;
 514
 515	asp = (*out_trans)->data;
 516	boundary = (void *)trans_wrapper + QAIC_WRAPPER_MAX_SIZE;
 517	*size = 0;
 518
 519	dma_len = 0;
 520	w = trans_wrapper;
 521	dma_chunk_len = 0;
 522	for_each_sg(sgt->sgl, sg, nents_dma, i) {
 523		asp->size = cpu_to_le64(dma_len);
 524		dma_chunk_len += dma_len;
 525		if (dma_len) {
 526			asp++;
 527			if ((void *)asp + sizeof(*asp) > boundary) {
 528				w->len = (void *)asp - (void *)&w->msg;
 529				*size += w->len;
 530				w = add_wrapper(wrappers, QAIC_WRAPPER_MAX_SIZE);
 531				if (!w)
 532					return -ENOMEM;
 533				boundary = (void *)w + QAIC_WRAPPER_MAX_SIZE;
 534				asp = (struct wire_addr_size_pair *)&w->msg;
 535			}
 536		}
 537		asp->addr = cpu_to_le64(sg_dma_address(sg));
 538		dma_len = sg_dma_len(sg);
 539	}
 540	/* finalize the last segment */
 541	asp->size = cpu_to_le64(dma_len);
 542	w->len = (void *)asp + sizeof(*asp) - (void *)&w->msg;
 543	*size += w->len;
 544	dma_chunk_len += dma_len;
 545	resources->xferred_dma_size += dma_chunk_len;
 546
 547	return nents_dma < nents ? 1 : 0;
 548}
 549
 550static void cleanup_xfer(struct qaic_device *qdev, struct dma_xfer *xfer)
 551{
 552	int i;
 553
 554	dma_unmap_sgtable(&qdev->pdev->dev, xfer->sgt, DMA_TO_DEVICE, 0);
 555	sg_free_table(xfer->sgt);
 556	kfree(xfer->sgt);
 557	for (i = 0; i < xfer->nr_pages; ++i)
 558		put_page(xfer->page_list[i]);
 559	kfree(xfer->page_list);
 560}
 561
 562static int encode_dma(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,
 563		      u32 *user_len, struct ioctl_resources *resources, struct qaic_user *usr)
 564{
 565	struct qaic_manage_trans_dma_xfer *in_trans = trans;
 566	struct wire_trans_dma_xfer *out_trans;
 567	struct wrapper_msg *wrapper;
 568	struct dma_xfer *xfer;
 569	struct wire_msg *msg;
 570	bool need_cont_dma;
 571	u32 msg_hdr_len;
 572	u32 size;
 573	int ret;
 574
 575	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
 576	msg = &wrapper->msg;
 577	msg_hdr_len = le32_to_cpu(msg->hdr.len);
 578
 579	/* There should be enough space to hold at least one ASP entry. */
 580	if (size_add(msg_hdr_len, sizeof(*out_trans) + sizeof(struct wire_addr_size_pair)) >
 581	    QAIC_MANAGE_WIRE_MSG_LENGTH)
 582		return -ENOMEM;
 583
 584	xfer = kmalloc(sizeof(*xfer), GFP_KERNEL);
 585	if (!xfer)
 586		return -ENOMEM;
 587
 588	ret = find_and_map_user_pages(qdev, in_trans, resources, xfer);
 589	if (ret < 0)
 590		goto free_xfer;
 591
 592	need_cont_dma = (bool)ret;
 593
 594	ret = encode_addr_size_pairs(xfer, wrappers, resources, msg_hdr_len, &size, &out_trans);
 595	if (ret < 0)
 596		goto cleanup_xfer;
 597
 598	need_cont_dma = need_cont_dma || (bool)ret;
 599
 600	msg->hdr.len = cpu_to_le32(msg_hdr_len + size);
 601	msg->hdr.count = incr_le32(msg->hdr.count);
 602
 603	out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_DMA_XFER_TO_DEV);
 604	out_trans->hdr.len = cpu_to_le32(size);
 605	out_trans->tag = cpu_to_le32(in_trans->tag);
 606	out_trans->count = cpu_to_le32((size - sizeof(*out_trans)) /
 607								sizeof(struct wire_addr_size_pair));
 608
 609	*user_len += in_trans->hdr.len;
 610
 611	if (resources->dma_chunk_id) {
 612		out_trans->dma_chunk_id = cpu_to_le32(resources->dma_chunk_id);
 613	} else if (need_cont_dma) {
 614		while (resources->dma_chunk_id == 0)
 615			resources->dma_chunk_id = atomic_inc_return(&usr->chunk_id);
 616
 617		out_trans->dma_chunk_id = cpu_to_le32(resources->dma_chunk_id);
 618	}
 619	resources->trans_hdr = trans;
 620
 621	list_add(&xfer->list, &resources->dma_xfers);
 622	return 0;
 623
 624cleanup_xfer:
 625	cleanup_xfer(qdev, xfer);
 626free_xfer:
 627	kfree(xfer);
 628	return ret;
 629}
 630
 631static int encode_activate(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,
 632			   u32 *user_len, struct ioctl_resources *resources)
 633{
 634	struct qaic_manage_trans_activate_to_dev *in_trans = trans;
 635	struct wire_trans_activate_to_dev *out_trans;
 636	struct wrapper_msg *trans_wrapper;
 637	struct wrapper_msg *wrapper;
 638	struct wire_msg *msg;
 639	dma_addr_t dma_addr;
 640	u32 msg_hdr_len;
 641	void *buf;
 642	u32 nelem;
 643	u32 size;
 644	int ret;
 645
 646	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
 647	msg = &wrapper->msg;
 648	msg_hdr_len = le32_to_cpu(msg->hdr.len);
 649
 650	if (size_add(msg_hdr_len, sizeof(*out_trans)) > QAIC_MANAGE_WIRE_MSG_LENGTH)
 651		return -ENOSPC;
 652
 653	if (!in_trans->queue_size)
 654		return -EINVAL;
 655
 656	if (in_trans->pad)
 657		return -EINVAL;
 658
 659	nelem = in_trans->queue_size;
 660	if (check_mul_overflow((u32)(get_dbc_req_elem_size() + get_dbc_rsp_elem_size()),
 661			       nelem,
 662			       &size))
 663		return -EINVAL;
 664
 665	if (size + QAIC_DBC_Q_GAP + QAIC_DBC_Q_BUF_ALIGN < size)
 666		return -EINVAL;
 667
 668	size = ALIGN((size + QAIC_DBC_Q_GAP), QAIC_DBC_Q_BUF_ALIGN);
 669
 670	buf = dma_alloc_coherent(&qdev->pdev->dev, size, &dma_addr, GFP_KERNEL);
 671	if (!buf)
 672		return -ENOMEM;
 673
 674	trans_wrapper = add_wrapper(wrappers,
 675				    offsetof(struct wrapper_msg, trans) + sizeof(*out_trans));
 676	if (!trans_wrapper) {
 677		ret = -ENOMEM;
 678		goto free_dma;
 679	}
 680	trans_wrapper->len = sizeof(*out_trans);
 681	out_trans = (struct wire_trans_activate_to_dev *)&trans_wrapper->trans;
 682
 683	out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_ACTIVATE_TO_DEV);
 684	out_trans->hdr.len = cpu_to_le32(sizeof(*out_trans));
 685	out_trans->buf_len = cpu_to_le32(size);
 686	out_trans->req_q_addr = cpu_to_le64(dma_addr);
 687	out_trans->req_q_size = cpu_to_le32(nelem);
 688	out_trans->rsp_q_addr = cpu_to_le64(dma_addr + size - nelem * get_dbc_rsp_elem_size());
 689	out_trans->rsp_q_size = cpu_to_le32(nelem);
 690	out_trans->options = cpu_to_le32(in_trans->options);
 691
 692	*user_len += in_trans->hdr.len;
 693	msg->hdr.len = cpu_to_le32(msg_hdr_len + sizeof(*out_trans));
 694	msg->hdr.count = incr_le32(msg->hdr.count);
 695
 696	resources->buf = buf;
 697	resources->dma_addr = dma_addr;
 698	resources->total_size = size;
 699	resources->nelem = nelem;
 700	resources->rsp_q_base = buf + size - nelem * get_dbc_rsp_elem_size();
 701	return 0;
 702
 703free_dma:
 704	dma_free_coherent(&qdev->pdev->dev, size, buf, dma_addr);
 705	return ret;
 706}
 707
 708static int encode_deactivate(struct qaic_device *qdev, void *trans,
 709			     u32 *user_len, struct qaic_user *usr)
 710{
 711	struct qaic_manage_trans_deactivate *in_trans = trans;
 712
 713	if (in_trans->dbc_id >= qdev->num_dbc || in_trans->pad)
 714		return -EINVAL;
 715
 716	*user_len += in_trans->hdr.len;
 717
 718	return disable_dbc(qdev, in_trans->dbc_id, usr);
 719}
 720
 721static int encode_status(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,
 722			 u32 *user_len)
 723{
 724	struct qaic_manage_trans_status_to_dev *in_trans = trans;
 725	struct wire_trans_status_to_dev *out_trans;
 726	struct wrapper_msg *trans_wrapper;
 727	struct wrapper_msg *wrapper;
 728	struct wire_msg *msg;
 729	u32 msg_hdr_len;
 730
 731	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
 732	msg = &wrapper->msg;
 733	msg_hdr_len = le32_to_cpu(msg->hdr.len);
 734
 735	if (size_add(msg_hdr_len, in_trans->hdr.len) > QAIC_MANAGE_WIRE_MSG_LENGTH)
 736		return -ENOSPC;
 737
 738	trans_wrapper = add_wrapper(wrappers, sizeof(*trans_wrapper));
 739	if (!trans_wrapper)
 740		return -ENOMEM;
 741
 742	trans_wrapper->len = sizeof(*out_trans);
 743	out_trans = (struct wire_trans_status_to_dev *)&trans_wrapper->trans;
 744
 745	out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_STATUS_TO_DEV);
 746	out_trans->hdr.len = cpu_to_le32(in_trans->hdr.len);
 747	msg->hdr.len = cpu_to_le32(msg_hdr_len + in_trans->hdr.len);
 748	msg->hdr.count = incr_le32(msg->hdr.count);
 749	*user_len += in_trans->hdr.len;
 750
 751	return 0;
 752}
 753
 754static int encode_message(struct qaic_device *qdev, struct manage_msg *user_msg,
 755			  struct wrapper_list *wrappers, struct ioctl_resources *resources,
 756			  struct qaic_user *usr)
 757{
 758	struct qaic_manage_trans_hdr *trans_hdr;
 759	struct wrapper_msg *wrapper;
 760	struct wire_msg *msg;
 761	u32 user_len = 0;
 762	int ret;
 763	int i;
 764
 765	if (!user_msg->count ||
 766	    user_msg->len < sizeof(*trans_hdr)) {
 767		ret = -EINVAL;
 768		goto out;
 769	}
 770
 771	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
 772	msg = &wrapper->msg;
 773
 774	msg->hdr.len = cpu_to_le32(sizeof(msg->hdr));
 775
 776	if (resources->dma_chunk_id) {
 777		ret = encode_dma(qdev, resources->trans_hdr, wrappers, &user_len, resources, usr);
 778		msg->hdr.count = cpu_to_le32(1);
 779		goto out;
 780	}
 781
 782	for (i = 0; i < user_msg->count; ++i) {
 783		if (user_len > user_msg->len - sizeof(*trans_hdr)) {
 784			ret = -EINVAL;
 785			break;
 786		}
 787		trans_hdr = (struct qaic_manage_trans_hdr *)(user_msg->data + user_len);
 788		if (trans_hdr->len < sizeof(trans_hdr) ||
 789		    size_add(user_len, trans_hdr->len) > user_msg->len) {
 790			ret = -EINVAL;
 791			break;
 792		}
 793
 794		switch (trans_hdr->type) {
 795		case QAIC_TRANS_PASSTHROUGH_FROM_USR:
 796			ret = encode_passthrough(qdev, trans_hdr, wrappers, &user_len);
 797			break;
 798		case QAIC_TRANS_DMA_XFER_FROM_USR:
 799			ret = encode_dma(qdev, trans_hdr, wrappers, &user_len, resources, usr);
 800			break;
 801		case QAIC_TRANS_ACTIVATE_FROM_USR:
 802			ret = encode_activate(qdev, trans_hdr, wrappers, &user_len, resources);
 803			break;
 804		case QAIC_TRANS_DEACTIVATE_FROM_USR:
 805			ret = encode_deactivate(qdev, trans_hdr, &user_len, usr);
 806			break;
 807		case QAIC_TRANS_STATUS_FROM_USR:
 808			ret = encode_status(qdev, trans_hdr, wrappers, &user_len);
 809			break;
 810		default:
 811			ret = -EINVAL;
 812			break;
 813		}
 814
 815		if (ret)
 816			goto out;
 817	}
 818
 819	if (user_len != user_msg->len)
 820		ret = -EINVAL;
 821out:
 822	if (ret) {
 823		free_dma_xfers(qdev, resources);
 824		free_dbc_buf(qdev, resources);
 825		return ret;
 826	}
 827
 828	return 0;
 829}
 830
 831static int decode_passthrough(struct qaic_device *qdev, void *trans, struct manage_msg *user_msg,
 832			      u32 *msg_len)
 833{
 834	struct qaic_manage_trans_passthrough *out_trans;
 835	struct wire_trans_passthrough *in_trans = trans;
 836	u32 len;
 837
 838	out_trans = (void *)user_msg->data + user_msg->len;
 839
 840	len = le32_to_cpu(in_trans->hdr.len);
 841	if (len % 8 != 0)
 842		return -EINVAL;
 843
 844	if (user_msg->len + len > QAIC_MANAGE_MAX_MSG_LENGTH)
 845		return -ENOSPC;
 846
 847	memcpy(out_trans->data, in_trans->data, len - sizeof(in_trans->hdr));
 848	user_msg->len += len;
 849	*msg_len += len;
 850	out_trans->hdr.type = le32_to_cpu(in_trans->hdr.type);
 851	out_trans->hdr.len = len;
 852
 853	return 0;
 854}
 855
 856static int decode_activate(struct qaic_device *qdev, void *trans, struct manage_msg *user_msg,
 857			   u32 *msg_len, struct ioctl_resources *resources, struct qaic_user *usr)
 858{
 859	struct qaic_manage_trans_activate_from_dev *out_trans;
 860	struct wire_trans_activate_from_dev *in_trans = trans;
 861	u32 len;
 862
 863	out_trans = (void *)user_msg->data + user_msg->len;
 864
 865	len = le32_to_cpu(in_trans->hdr.len);
 866	if (user_msg->len + len > QAIC_MANAGE_MAX_MSG_LENGTH)
 867		return -ENOSPC;
 868
 869	user_msg->len += len;
 870	*msg_len += len;
 871	out_trans->hdr.type = le32_to_cpu(in_trans->hdr.type);
 872	out_trans->hdr.len = len;
 873	out_trans->status = le32_to_cpu(in_trans->status);
 874	out_trans->dbc_id = le32_to_cpu(in_trans->dbc_id);
 875	out_trans->options = le64_to_cpu(in_trans->options);
 876
 877	if (!resources->buf)
 878		/* how did we get an activate response without a request? */
 879		return -EINVAL;
 880
 881	if (out_trans->dbc_id >= qdev->num_dbc)
 882		/*
 883		 * The device assigned an invalid resource, which should never
 884		 * happen. Return an error so the user can try to recover.
 885		 */
 886		return -ENODEV;
 887
 888	if (out_trans->status)
 889		/*
 890		 * Allocating resources failed on device side. This is not an
 891		 * expected behaviour, user is expected to handle this situation.
 892		 */
 893		return -ECANCELED;
 894
 895	resources->status = out_trans->status;
 896	resources->dbc_id = out_trans->dbc_id;
 897	save_dbc_buf(qdev, resources, usr);
 898
 899	return 0;
 900}
 901
 902static int decode_deactivate(struct qaic_device *qdev, void *trans, u32 *msg_len,
 903			     struct qaic_user *usr)
 904{
 905	struct wire_trans_deactivate_from_dev *in_trans = trans;
 906	u32 dbc_id = le32_to_cpu(in_trans->dbc_id);
 907	u32 status = le32_to_cpu(in_trans->status);
 908
 909	if (dbc_id >= qdev->num_dbc)
 910		/*
 911		 * The device assigned an invalid resource, which should never
 912		 * happen. Inject an error so the user can try to recover.
 913		 */
 914		return -ENODEV;
 915
 916	if (usr && status) {
 917		/*
 918		 * Releasing resources failed on the device side, which puts
 919		 * us in a bind since they may still be in use, so enable the
 920		 * dbc. User is expected to retry deactivation.
 921		 */
 922		enable_dbc(qdev, dbc_id, usr);
 923		return -ECANCELED;
 924	}
 925
 926	release_dbc(qdev, dbc_id);
 927	set_dbc_state(qdev, dbc_id, DBC_STATE_IDLE);
 928	*msg_len += sizeof(*in_trans);
 929
 930	return 0;
 931}
 932
 933static int decode_status(struct qaic_device *qdev, void *trans, struct manage_msg *user_msg,
 934			 u32 *user_len, struct wire_msg *msg)
 935{
 936	struct qaic_manage_trans_status_from_dev *out_trans;
 937	struct wire_trans_status_from_dev *in_trans = trans;
 938	u32 len;
 939
 940	out_trans = (void *)user_msg->data + user_msg->len;
 941
 942	len = le32_to_cpu(in_trans->hdr.len);
 943	if (user_msg->len + len > QAIC_MANAGE_MAX_MSG_LENGTH)
 944		return -ENOSPC;
 945
 946	out_trans->hdr.type = QAIC_TRANS_STATUS_FROM_DEV;
 947	out_trans->hdr.len = len;
 948	out_trans->major = le16_to_cpu(in_trans->major);
 949	out_trans->minor = le16_to_cpu(in_trans->minor);
 950	out_trans->status_flags = le64_to_cpu(in_trans->status_flags);
 951	out_trans->status = le32_to_cpu(in_trans->status);
 952	*user_len += le32_to_cpu(in_trans->hdr.len);
 953	user_msg->len += len;
 954
 955	if (out_trans->status)
 956		return -ECANCELED;
 957	if (out_trans->status_flags & BIT(0) && !valid_crc(msg))
 958		return -EPIPE;
 959
 960	return 0;
 961}
 962
 963static int decode_message(struct qaic_device *qdev, struct manage_msg *user_msg,
 964			  struct wire_msg *msg, struct ioctl_resources *resources,
 965			  struct qaic_user *usr)
 966{
 967	u32 msg_hdr_len = le32_to_cpu(msg->hdr.len);
 968	struct wire_trans_hdr *trans_hdr;
 969	u32 msg_len = 0;
 970	int ret;
 971	int i;
 972
 973	if (msg_hdr_len < sizeof(*trans_hdr) ||
 974	    msg_hdr_len > QAIC_MANAGE_MAX_MSG_LENGTH)
 975		return -EINVAL;
 976
 977	user_msg->len = 0;
 978	user_msg->count = le32_to_cpu(msg->hdr.count);
 979
 980	for (i = 0; i < user_msg->count; ++i) {
 981		u32 hdr_len;
 982
 983		if (msg_len > msg_hdr_len - sizeof(*trans_hdr))
 984			return -EINVAL;
 985
 986		trans_hdr = (struct wire_trans_hdr *)(msg->data + msg_len);
 987		hdr_len = le32_to_cpu(trans_hdr->len);
 988		if (hdr_len < sizeof(*trans_hdr) ||
 989		    size_add(msg_len, hdr_len) > msg_hdr_len)
 990			return -EINVAL;
 991
 992		switch (le32_to_cpu(trans_hdr->type)) {
 993		case QAIC_TRANS_PASSTHROUGH_FROM_DEV:
 994			ret = decode_passthrough(qdev, trans_hdr, user_msg, &msg_len);
 995			break;
 996		case QAIC_TRANS_ACTIVATE_FROM_DEV:
 997			ret = decode_activate(qdev, trans_hdr, user_msg, &msg_len, resources, usr);
 998			break;
 999		case QAIC_TRANS_DEACTIVATE_FROM_DEV:
1000			ret = decode_deactivate(qdev, trans_hdr, &msg_len, usr);
1001			break;
1002		case QAIC_TRANS_STATUS_FROM_DEV:
1003			ret = decode_status(qdev, trans_hdr, user_msg, &msg_len, msg);
1004			break;
1005		default:
1006			return -EINVAL;
1007		}
1008
1009		if (ret)
1010			return ret;
1011	}
1012
1013	if (msg_len != (msg_hdr_len - sizeof(msg->hdr)))
1014		return -EINVAL;
1015
1016	return 0;
1017}
1018
1019static void *msg_xfer(struct qaic_device *qdev, struct wrapper_list *wrappers, u32 seq_num,
1020		      bool ignore_signal)
1021{
1022	struct xfer_queue_elem elem;
1023	struct wire_msg *out_buf;
1024	struct wrapper_msg *w;
1025	long ret = -EAGAIN;
1026	int xfer_count = 0;
1027	int retry_count;
1028
1029	/* Allow QAIC_BOOT state since we need to check control protocol version */
1030	if (qdev->dev_state == QAIC_OFFLINE) {
1031		mutex_unlock(&qdev->cntl_mutex);
1032		return ERR_PTR(-ENODEV);
1033	}
1034
1035	/* Attempt to avoid a partial commit of a message */
1036	list_for_each_entry(w, &wrappers->list, list)
1037		xfer_count++;
1038
1039	for (retry_count = 0; retry_count < QAIC_MHI_RETRY_MAX; retry_count++) {
1040		if (xfer_count <= mhi_get_free_desc_count(qdev->cntl_ch, DMA_TO_DEVICE)) {
1041			ret = 0;
1042			break;
1043		}
1044		msleep_interruptible(QAIC_MHI_RETRY_WAIT_MS);
1045		if (signal_pending(current))
1046			break;
1047	}
1048
1049	if (ret) {
1050		mutex_unlock(&qdev->cntl_mutex);
1051		return ERR_PTR(ret);
1052	}
1053
1054	elem.seq_num = seq_num;
1055	elem.buf = NULL;
1056	init_completion(&elem.xfer_done);
1057	if (likely(!qdev->cntl_lost_buf)) {
1058		/*
1059		 * The max size of request to device is QAIC_MANAGE_WIRE_MSG_LENGTH.
1060		 * The max size of response from device is QAIC_MANAGE_MAX_MSG_LENGTH.
1061		 */
1062		out_buf = kmalloc(QAIC_MANAGE_MAX_MSG_LENGTH, GFP_KERNEL);
1063		if (!out_buf) {
1064			mutex_unlock(&qdev->cntl_mutex);
1065			return ERR_PTR(-ENOMEM);
1066		}
1067
1068		ret = mhi_queue_buf(qdev->cntl_ch, DMA_FROM_DEVICE, out_buf,
1069				    QAIC_MANAGE_MAX_MSG_LENGTH, MHI_EOT);
1070		if (ret) {
1071			mutex_unlock(&qdev->cntl_mutex);
1072			return ERR_PTR(ret);
1073		}
1074	} else {
1075		/*
1076		 * we lost a buffer because we queued a recv buf, but then
1077		 * queuing the corresponding tx buf failed. To try to avoid
1078		 * a memory leak, lets reclaim it and use it for this
1079		 * transaction.
1080		 */
1081		qdev->cntl_lost_buf = false;
1082	}
1083
1084	list_for_each_entry(w, &wrappers->list, list) {
1085		kref_get(&w->ref_count);
1086		ret = mhi_queue_buf(qdev->cntl_ch, DMA_TO_DEVICE, &w->msg, w->len,
1087				    list_is_last(&w->list, &wrappers->list) ? MHI_EOT : MHI_CHAIN);
1088		if (ret) {
1089			qdev->cntl_lost_buf = true;
1090			kref_put(&w->ref_count, free_wrapper);
1091			mutex_unlock(&qdev->cntl_mutex);
1092			return ERR_PTR(ret);
1093		}
1094	}
1095
1096	list_add_tail(&elem.list, &qdev->cntl_xfer_list);
1097	mutex_unlock(&qdev->cntl_mutex);
1098
1099	if (ignore_signal)
1100		ret = wait_for_completion_timeout(&elem.xfer_done, control_resp_timeout_s * HZ);
1101	else
1102		ret = wait_for_completion_interruptible_timeout(&elem.xfer_done,
1103								control_resp_timeout_s * HZ);
1104	/*
1105	 * not using _interruptable because we have to cleanup or we'll
1106	 * likely cause memory corruption
1107	 */
1108	mutex_lock(&qdev->cntl_mutex);
1109	if (!list_empty(&elem.list))
1110		list_del(&elem.list);
1111	/* resp_worker() processed the response but the wait was interrupted */
1112	else if (ret == -ERESTARTSYS)
1113		ret = 0;
1114	if (!ret && !elem.buf)
1115		ret = -ETIMEDOUT;
1116	else if (ret > 0 && !elem.buf)
1117		ret = -EIO;
1118	mutex_unlock(&qdev->cntl_mutex);
1119
1120	if (ret < 0) {
1121		kfree(elem.buf);
1122		return ERR_PTR(ret);
1123	} else if (!qdev->valid_crc(elem.buf)) {
1124		kfree(elem.buf);
1125		return ERR_PTR(-EPIPE);
1126	}
1127
1128	return elem.buf;
1129}
1130
1131/* Add a transaction to abort the outstanding DMA continuation */
1132static int abort_dma_cont(struct qaic_device *qdev, struct wrapper_list *wrappers, u32 dma_chunk_id)
1133{
1134	struct wire_trans_dma_xfer *out_trans;
1135	u32 size = sizeof(*out_trans);
1136	struct wrapper_msg *wrapper;
1137	struct wrapper_msg *w;
1138	struct wire_msg *msg;
1139
1140	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
1141	msg = &wrapper->msg;
1142
1143	/* Remove all but the first wrapper which has the msg header */
1144	list_for_each_entry_safe(wrapper, w, &wrappers->list, list)
1145		if (!list_is_first(&wrapper->list, &wrappers->list))
1146			kref_put(&wrapper->ref_count, free_wrapper);
1147
1148	wrapper = add_wrapper(wrappers, sizeof(*wrapper));
1149
1150	if (!wrapper)
1151		return -ENOMEM;
1152
1153	out_trans = (struct wire_trans_dma_xfer *)&wrapper->trans;
1154	out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_DMA_XFER_TO_DEV);
1155	out_trans->hdr.len = cpu_to_le32(size);
1156	out_trans->tag = cpu_to_le32(0);
1157	out_trans->count = cpu_to_le32(0);
1158	out_trans->dma_chunk_id = cpu_to_le32(dma_chunk_id);
1159
1160	msg->hdr.len = cpu_to_le32(size + sizeof(*msg));
1161	msg->hdr.count = cpu_to_le32(1);
1162	wrapper->len = size;
1163
1164	return 0;
1165}
1166
1167static struct wrapper_list *alloc_wrapper_list(void)
1168{
1169	struct wrapper_list *wrappers;
1170
1171	wrappers = kmalloc(sizeof(*wrappers), GFP_KERNEL);
1172	if (!wrappers)
1173		return NULL;
1174	INIT_LIST_HEAD(&wrappers->list);
1175	spin_lock_init(&wrappers->lock);
1176
1177	return wrappers;
1178}
1179
1180static int qaic_manage_msg_xfer(struct qaic_device *qdev, struct qaic_user *usr,
1181				struct manage_msg *user_msg, struct ioctl_resources *resources,
1182				struct wire_msg **rsp)
1183{
1184	struct wrapper_list *wrappers;
1185	struct wrapper_msg *wrapper;
1186	struct wrapper_msg *w;
1187	bool all_done = false;
1188	struct wire_msg *msg;
1189	int ret;
1190
1191	wrappers = alloc_wrapper_list();
1192	if (!wrappers)
1193		return -ENOMEM;
1194
1195	wrapper = add_wrapper(wrappers, sizeof(*wrapper));
1196	if (!wrapper) {
1197		kfree(wrappers);
1198		return -ENOMEM;
1199	}
1200
1201	msg = &wrapper->msg;
1202	wrapper->len = sizeof(*msg);
1203
1204	ret = encode_message(qdev, user_msg, wrappers, resources, usr);
1205	if (ret && resources->dma_chunk_id)
1206		ret = abort_dma_cont(qdev, wrappers, resources->dma_chunk_id);
1207	if (ret)
1208		goto encode_failed;
1209
1210	ret = mutex_lock_interruptible(&qdev->cntl_mutex);
1211	if (ret)
1212		goto lock_failed;
1213
1214	msg->hdr.magic_number = MANAGE_MAGIC_NUMBER;
1215	msg->hdr.sequence_number = cpu_to_le32(qdev->next_seq_num++);
1216
1217	if (usr) {
1218		msg->hdr.handle = cpu_to_le32(usr->handle);
1219		msg->hdr.partition_id = cpu_to_le32(usr->qddev->partition_id);
1220	} else {
1221		msg->hdr.handle = 0;
1222		msg->hdr.partition_id = cpu_to_le32(QAIC_NO_PARTITION);
1223	}
1224
1225	msg->hdr.padding = cpu_to_le32(0);
1226	msg->hdr.crc32 = cpu_to_le32(qdev->gen_crc(wrappers));
1227
1228	/* msg_xfer releases the mutex */
1229	*rsp = msg_xfer(qdev, wrappers, qdev->next_seq_num - 1, false);
1230	if (IS_ERR(*rsp))
1231		ret = PTR_ERR(*rsp);
1232
1233lock_failed:
1234	free_dma_xfers(qdev, resources);
1235encode_failed:
1236	spin_lock(&wrappers->lock);
1237	list_for_each_entry_safe(wrapper, w, &wrappers->list, list)
1238		kref_put(&wrapper->ref_count, free_wrapper);
1239	all_done = list_empty(&wrappers->list);
1240	spin_unlock(&wrappers->lock);
1241	if (all_done)
1242		kfree(wrappers);
1243
1244	return ret;
1245}
1246
1247static int qaic_manage(struct qaic_device *qdev, struct qaic_user *usr, struct manage_msg *user_msg)
1248{
1249	struct wire_trans_dma_xfer_cont *dma_cont = NULL;
1250	struct ioctl_resources resources;
1251	struct wire_msg *rsp = NULL;
1252	int ret;
1253
1254	memset(&resources, 0, sizeof(struct ioctl_resources));
1255
1256	INIT_LIST_HEAD(&resources.dma_xfers);
1257
1258	if (user_msg->len > QAIC_MANAGE_MAX_MSG_LENGTH ||
1259	    user_msg->count > QAIC_MANAGE_MAX_MSG_LENGTH / sizeof(struct qaic_manage_trans_hdr))
1260		return -EINVAL;
1261
1262dma_xfer_continue:
1263	ret = qaic_manage_msg_xfer(qdev, usr, user_msg, &resources, &rsp);
1264	if (ret)
1265		return ret;
1266	/* dma_cont should be the only transaction if present */
1267	if (le32_to_cpu(rsp->hdr.count) == 1) {
1268		dma_cont = (struct wire_trans_dma_xfer_cont *)rsp->data;
1269		if (le32_to_cpu(dma_cont->hdr.type) != QAIC_TRANS_DMA_XFER_CONT)
1270			dma_cont = NULL;
1271	}
1272	if (dma_cont) {
1273		if (le32_to_cpu(dma_cont->dma_chunk_id) == resources.dma_chunk_id &&
1274		    le64_to_cpu(dma_cont->xferred_size) == resources.xferred_dma_size) {
1275			kfree(rsp);
1276			goto dma_xfer_continue;
1277		}
1278
1279		ret = -EINVAL;
1280		goto dma_cont_failed;
1281	}
1282
1283	ret = decode_message(qdev, user_msg, rsp, &resources, usr);
1284
1285dma_cont_failed:
1286	free_dbc_buf(qdev, &resources);
1287	kfree(rsp);
1288	return ret;
1289}
1290
1291int qaic_manage_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
1292{
1293	struct qaic_manage_msg *user_msg = data;
1294	struct qaic_device *qdev;
1295	struct manage_msg *msg;
1296	struct qaic_user *usr;
1297	u8 __user *user_data;
1298	int qdev_rcu_id;
1299	int usr_rcu_id;
1300	int ret;
1301
1302	if (user_msg->len > QAIC_MANAGE_MAX_MSG_LENGTH)
1303		return -EINVAL;
1304
1305	usr = file_priv->driver_priv;
1306
1307	usr_rcu_id = srcu_read_lock(&usr->qddev_lock);
1308	if (!usr->qddev) {
1309		srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
1310		return -ENODEV;
1311	}
1312
1313	qdev = usr->qddev->qdev;
1314
1315	qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);
1316	if (qdev->dev_state != QAIC_ONLINE) {
1317		srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
1318		srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
1319		return -ENODEV;
1320	}
1321
1322	msg = kzalloc(QAIC_MANAGE_MAX_MSG_LENGTH + sizeof(*msg), GFP_KERNEL);
1323	if (!msg) {
1324		ret = -ENOMEM;
1325		goto out;
1326	}
1327
1328	msg->len = user_msg->len;
1329	msg->count = user_msg->count;
1330
1331	user_data = u64_to_user_ptr(user_msg->data);
1332
1333	if (copy_from_user(msg->data, user_data, user_msg->len)) {
1334		ret = -EFAULT;
1335		goto free_msg;
1336	}
1337
1338	ret = qaic_manage(qdev, usr, msg);
1339
1340	/*
1341	 * If the qaic_manage() is successful then we copy the message onto
1342	 * userspace memory but we have an exception for -ECANCELED.
1343	 * For -ECANCELED, it means that device has NACKed the message with a
1344	 * status error code which userspace would like to know.
1345	 */
1346	if (ret == -ECANCELED || !ret) {
1347		if (copy_to_user(user_data, msg->data, msg->len)) {
1348			ret = -EFAULT;
1349		} else {
1350			user_msg->len = msg->len;
1351			user_msg->count = msg->count;
1352		}
1353	}
1354
1355free_msg:
1356	kfree(msg);
1357out:
1358	srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
1359	srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
1360	return ret;
1361}
1362
1363int get_cntl_version(struct qaic_device *qdev, struct qaic_user *usr, u16 *major, u16 *minor)
1364{
1365	struct qaic_manage_trans_status_from_dev *status_result;
1366	struct qaic_manage_trans_status_to_dev *status_query;
1367	struct manage_msg *user_msg;
1368	int ret;
1369
1370	user_msg = kmalloc(sizeof(*user_msg) + sizeof(*status_result), GFP_KERNEL);
1371	if (!user_msg) {
1372		ret = -ENOMEM;
1373		goto out;
1374	}
1375	user_msg->len = sizeof(*status_query);
1376	user_msg->count = 1;
1377
1378	status_query = (struct qaic_manage_trans_status_to_dev *)user_msg->data;
1379	status_query->hdr.type = QAIC_TRANS_STATUS_FROM_USR;
1380	status_query->hdr.len = sizeof(status_query->hdr);
1381
1382	ret = qaic_manage(qdev, usr, user_msg);
1383	if (ret)
1384		goto kfree_user_msg;
1385	status_result = (struct qaic_manage_trans_status_from_dev *)user_msg->data;
1386	*major = status_result->major;
1387	*minor = status_result->minor;
1388
1389	if (status_result->status_flags & BIT(0)) { /* device is using CRC */
1390		/* By default qdev->gen_crc is programmed to generate CRC */
1391		qdev->valid_crc = valid_crc;
1392	} else {
1393		/* By default qdev->valid_crc is programmed to bypass CRC */
1394		qdev->gen_crc = gen_crc_stub;
1395	}
1396
1397kfree_user_msg:
1398	kfree(user_msg);
1399out:
1400	return ret;
1401}
1402
1403static void resp_worker(struct work_struct *work)
1404{
1405	struct resp_work *resp = container_of(work, struct resp_work, work);
1406	struct qaic_device *qdev = resp->qdev;
1407	struct wire_msg *msg = resp->buf;
1408	struct xfer_queue_elem *elem;
1409	struct xfer_queue_elem *i;
1410	bool found = false;
1411
1412	mutex_lock(&qdev->cntl_mutex);
1413	list_for_each_entry_safe(elem, i, &qdev->cntl_xfer_list, list) {
1414		if (elem->seq_num == le32_to_cpu(msg->hdr.sequence_number)) {
1415			found = true;
1416			list_del_init(&elem->list);
1417			elem->buf = msg;
1418			complete_all(&elem->xfer_done);
1419			break;
1420		}
1421	}
1422	mutex_unlock(&qdev->cntl_mutex);
1423
1424	if (!found) {
1425		/*
1426		 * The user might have gone away at this point without waiting
1427		 * for QAIC_TRANS_DEACTIVATE_FROM_DEV transaction coming from
1428		 * the device. If this is not handled correctly, the host will
1429		 * not know that the DBC[n] has been freed on the device.
1430		 * Due to this failure in synchronization between the device and
1431		 * the host, if another user requests to activate a network, and
1432		 * the device assigns DBC[n] again, save_dbc_buf() will hang,
1433		 * waiting for dbc[n]->in_use to be set to false, which will not
1434		 * happen unless the qaic_dev_reset_clean_local_state() gets
1435		 * called by resetting the device (or re-inserting the module).
1436		 *
1437		 * As a solution, we look for QAIC_TRANS_DEACTIVATE_FROM_DEV
1438		 * transactions in the message before disposing of it, then
1439		 * handle releasing the DBC resources.
1440		 *
1441		 * Since the user has gone away, if the device could not
1442		 * deactivate the network (status != 0), there is no way to
1443		 * enable and reassign the DBC to the user. We can put trust in
1444		 * the device that it will release all the active DBCs in
1445		 * response to the QAIC_TRANS_TERMINATE_TO_DEV transaction,
1446		 * otherwise, the user can issue an soc_reset to the device.
1447		 */
1448		u32 msg_count = le32_to_cpu(msg->hdr.count);
1449		u32 msg_len = le32_to_cpu(msg->hdr.len);
1450		u32 len = 0;
1451		int j;
1452
1453		for (j = 0; j < msg_count && len < msg_len; ++j) {
1454			struct wire_trans_hdr *trans_hdr;
1455
1456			trans_hdr = (struct wire_trans_hdr *)(msg->data + len);
1457			if (le32_to_cpu(trans_hdr->type) == QAIC_TRANS_DEACTIVATE_FROM_DEV) {
1458				if (decode_deactivate(qdev, trans_hdr, &len, NULL))
1459					len += le32_to_cpu(trans_hdr->len);
1460			} else {
1461				len += le32_to_cpu(trans_hdr->len);
1462			}
1463		}
1464		/* request must have timed out, drop packet */
1465		kfree(msg);
1466	}
1467
1468	kfree(resp);
1469}
1470
1471static void free_wrapper_from_list(struct wrapper_list *wrappers, struct wrapper_msg *wrapper)
1472{
1473	bool all_done = false;
1474
1475	spin_lock(&wrappers->lock);
1476	kref_put(&wrapper->ref_count, free_wrapper);
1477	all_done = list_empty(&wrappers->list);
1478	spin_unlock(&wrappers->lock);
1479
1480	if (all_done)
1481		kfree(wrappers);
1482}
1483
1484void qaic_mhi_ul_xfer_cb(struct mhi_device *mhi_dev, struct mhi_result *mhi_result)
1485{
1486	struct wire_msg *msg = mhi_result->buf_addr;
1487	struct wrapper_msg *wrapper = container_of(msg, struct wrapper_msg, msg);
1488
1489	free_wrapper_from_list(wrapper->head, wrapper);
1490}
1491
1492void qaic_mhi_dl_xfer_cb(struct mhi_device *mhi_dev, struct mhi_result *mhi_result)
1493{
1494	struct qaic_device *qdev = dev_get_drvdata(&mhi_dev->dev);
1495	struct wire_msg *msg = mhi_result->buf_addr;
1496	struct resp_work *resp;
1497
1498	if (mhi_result->transaction_status || msg->hdr.magic_number != MANAGE_MAGIC_NUMBER) {
1499		kfree(msg);
1500		return;
1501	}
1502
1503	resp = kmalloc(sizeof(*resp), GFP_ATOMIC);
1504	if (!resp) {
1505		kfree(msg);
1506		return;
1507	}
1508
1509	INIT_WORK(&resp->work, resp_worker);
1510	resp->qdev = qdev;
1511	resp->buf = msg;
1512	queue_work(qdev->cntl_wq, &resp->work);
1513}
1514
1515int qaic_control_open(struct qaic_device *qdev)
1516{
1517	if (!qdev->cntl_ch)
1518		return -ENODEV;
1519
1520	qdev->cntl_lost_buf = false;
1521	/*
1522	 * By default qaic should assume that device has CRC enabled.
1523	 * Qaic comes to know if device has CRC enabled or disabled during the
1524	 * device status transaction, which is the first transaction performed
1525	 * on control channel.
1526	 *
1527	 * So CRC validation of first device status transaction response is
1528	 * ignored (by calling valid_crc_stub) and is done later during decoding
1529	 * if device has CRC enabled.
1530	 * Now that qaic knows whether device has CRC enabled or not it acts
1531	 * accordingly.
1532	 */
1533	qdev->gen_crc = gen_crc;
1534	qdev->valid_crc = valid_crc_stub;
1535
1536	return mhi_prepare_for_transfer(qdev->cntl_ch);
1537}
1538
1539void qaic_control_close(struct qaic_device *qdev)
1540{
1541	mhi_unprepare_from_transfer(qdev->cntl_ch);
1542}
1543
1544void qaic_release_usr(struct qaic_device *qdev, struct qaic_user *usr)
1545{
1546	struct wire_trans_terminate_to_dev *trans;
1547	struct wrapper_list *wrappers;
1548	struct wrapper_msg *wrapper;
1549	struct wire_msg *msg;
1550	struct wire_msg *rsp;
1551
1552	wrappers = alloc_wrapper_list();
1553	if (!wrappers)
1554		return;
1555
1556	wrapper = add_wrapper(wrappers, sizeof(*wrapper) + sizeof(*msg) + sizeof(*trans));
1557	if (!wrapper)
1558		return;
1559
1560	msg = &wrapper->msg;
1561
1562	trans = (struct wire_trans_terminate_to_dev *)msg->data;
1563
1564	trans->hdr.type = cpu_to_le32(QAIC_TRANS_TERMINATE_TO_DEV);
1565	trans->hdr.len = cpu_to_le32(sizeof(*trans));
1566	trans->handle = cpu_to_le32(usr->handle);
1567
1568	mutex_lock(&qdev->cntl_mutex);
1569	wrapper->len = sizeof(msg->hdr) + sizeof(*trans);
1570	msg->hdr.magic_number = MANAGE_MAGIC_NUMBER;
1571	msg->hdr.sequence_number = cpu_to_le32(qdev->next_seq_num++);
1572	msg->hdr.len = cpu_to_le32(wrapper->len);
1573	msg->hdr.count = cpu_to_le32(1);
1574	msg->hdr.handle = cpu_to_le32(usr->handle);
1575	msg->hdr.padding = cpu_to_le32(0);
1576	msg->hdr.crc32 = cpu_to_le32(qdev->gen_crc(wrappers));
1577
1578	/*
1579	 * msg_xfer releases the mutex
1580	 * We don't care about the return of msg_xfer since we will not do
1581	 * anything different based on what happens.
1582	 * We ignore pending signals since one will be set if the user is
1583	 * killed, and we need give the device a chance to cleanup, otherwise
1584	 * DMA may still be in progress when we return.
1585	 */
1586	rsp = msg_xfer(qdev, wrappers, qdev->next_seq_num - 1, true);
1587	if (!IS_ERR(rsp))
1588		kfree(rsp);
1589	free_wrapper_from_list(wrappers, wrapper);
1590}
1591
1592void wake_all_cntl(struct qaic_device *qdev)
1593{
1594	struct xfer_queue_elem *elem;
1595	struct xfer_queue_elem *i;
1596
1597	mutex_lock(&qdev->cntl_mutex);
1598	list_for_each_entry_safe(elem, i, &qdev->cntl_xfer_list, list) {
1599		list_del_init(&elem->list);
1600		complete_all(&elem->xfer_done);
1601	}
1602	mutex_unlock(&qdev->cntl_mutex);
1603}