ce.c

   1// SPDX-License-Identifier: BSD-3-Clause-Clear
   2/*
   3 * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
   4 * Copyright (c) 2021-2022, 2024-2025 Qualcomm Innovation Center, Inc. All rights reserved.
   5 */
   6
   7#include "dp_rx.h"
   8#include "debug.h"
   9#include "hif.h"
  10
  11const struct ce_attr ath12k_host_ce_config_qcn9274[] = {
  12	/* CE0: host->target HTC control and raw streams */
  13	{
  14		.flags = CE_ATTR_FLAGS,
  15		.src_nentries = 16,
  16		.src_sz_max = 2048,
  17		.dest_nentries = 0,
  18	},
  19
  20	/* CE1: target->host HTT + HTC control */
  21	{
  22		.flags = CE_ATTR_FLAGS,
  23		.src_nentries = 0,
  24		.src_sz_max = 2048,
  25		.dest_nentries = 512,
  26		.recv_cb = ath12k_htc_rx_completion_handler,
  27	},
  28
  29	/* CE2: target->host WMI */
  30	{
  31		.flags = CE_ATTR_FLAGS,
  32		.src_nentries = 0,
  33		.src_sz_max = 2048,
  34		.dest_nentries = 128,
  35		.recv_cb = ath12k_htc_rx_completion_handler,
  36	},
  37
  38	/* CE3: host->target WMI (mac0) */
  39	{
  40		.flags = CE_ATTR_FLAGS,
  41		.src_nentries = 32,
  42		.src_sz_max = 2048,
  43		.dest_nentries = 0,
  44	},
  45
  46	/* CE4: host->target HTT */
  47	{
  48		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
  49		.src_nentries = 2048,
  50		.src_sz_max = 256,
  51		.dest_nentries = 0,
  52	},
  53
  54	/* CE5: target->host pktlog */
  55	{
  56		.flags = CE_ATTR_FLAGS,
  57		.src_nentries = 0,
  58		.src_sz_max = 2048,
  59		.dest_nentries = 512,
  60		.recv_cb = ath12k_dp_htt_htc_t2h_msg_handler,
  61	},
  62
  63	/* CE6: target autonomous hif_memcpy */
  64	{
  65		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
  66		.src_nentries = 0,
  67		.src_sz_max = 0,
  68		.dest_nentries = 0,
  69	},
  70
  71	/* CE7: host->target WMI (mac1) */
  72	{
  73		.flags = CE_ATTR_FLAGS,
  74		.src_nentries = 32,
  75		.src_sz_max = 2048,
  76		.dest_nentries = 0,
  77	},
  78
  79	/* CE8: target autonomous hif_memcpy */
  80	{
  81		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
  82		.src_nentries = 0,
  83		.src_sz_max = 0,
  84		.dest_nentries = 0,
  85	},
  86
  87	/* CE9: MHI */
  88	{
  89		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
  90		.src_nentries = 0,
  91		.src_sz_max = 0,
  92		.dest_nentries = 0,
  93	},
  94
  95	/* CE10: MHI */
  96	{
  97		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
  98		.src_nentries = 0,
  99		.src_sz_max = 0,
 100		.dest_nentries = 0,
 101	},
 102
 103	/* CE11: MHI */
 104	{
 105		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
 106		.src_nentries = 0,
 107		.src_sz_max = 0,
 108		.dest_nentries = 0,
 109	},
 110
 111	/* CE12: CV Prefetch */
 112	{
 113		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
 114		.src_nentries = 0,
 115		.src_sz_max = 0,
 116		.dest_nentries = 0,
 117	},
 118
 119	/* CE13: CV Prefetch */
 120	{
 121		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
 122		.src_nentries = 0,
 123		.src_sz_max = 0,
 124		.dest_nentries = 0,
 125	},
 126
 127	/* CE14: target->host dbg log */
 128	{
 129		.flags = CE_ATTR_FLAGS,
 130		.src_nentries = 0,
 131		.src_sz_max = 2048,
 132		.dest_nentries = 512,
 133		.recv_cb = ath12k_htc_rx_completion_handler,
 134	},
 135
 136	/* CE15: reserved for future use */
 137	{
 138		.flags = (CE_ATTR_FLAGS | CE_ATTR_DIS_INTR),
 139		.src_nentries = 0,
 140		.src_sz_max = 0,
 141		.dest_nentries = 0,
 142	},
 143};
 144
 145const struct ce_attr ath12k_host_ce_config_wcn7850[] = {
 146	/* CE0: host->target HTC control and raw streams */
 147	{
 148		.flags = CE_ATTR_FLAGS,
 149		.src_nentries = 16,
 150		.src_sz_max = 2048,
 151		.dest_nentries = 0,
 152	},
 153
 154	/* CE1: target->host HTT + HTC control */
 155	{
 156		.flags = CE_ATTR_FLAGS,
 157		.src_nentries = 0,
 158		.src_sz_max = 2048,
 159		.dest_nentries = 512,
 160		.recv_cb = ath12k_htc_rx_completion_handler,
 161	},
 162
 163	/* CE2: target->host WMI */
 164	{
 165		.flags = CE_ATTR_FLAGS,
 166		.src_nentries = 0,
 167		.src_sz_max = 2048,
 168		.dest_nentries = 64,
 169		.recv_cb = ath12k_htc_rx_completion_handler,
 170	},
 171
 172	/* CE3: host->target WMI (mac0) */
 173	{
 174		.flags = CE_ATTR_FLAGS,
 175		.src_nentries = 32,
 176		.src_sz_max = 2048,
 177		.dest_nentries = 0,
 178	},
 179
 180	/* CE4: host->target HTT */
 181	{
 182		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
 183		.src_nentries = 2048,
 184		.src_sz_max = 256,
 185		.dest_nentries = 0,
 186	},
 187
 188	/* CE5: target->host pktlog */
 189	{
 190		.flags = CE_ATTR_FLAGS,
 191		.src_nentries = 0,
 192		.src_sz_max = 0,
 193		.dest_nentries = 0,
 194	},
 195
 196	/* CE6: target autonomous hif_memcpy */
 197	{
 198		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
 199		.src_nentries = 0,
 200		.src_sz_max = 0,
 201		.dest_nentries = 0,
 202	},
 203
 204	/* CE7: host->target WMI (mac1) */
 205	{
 206		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
 207		.src_nentries = 0,
 208		.src_sz_max = 2048,
 209		.dest_nentries = 0,
 210	},
 211
 212	/* CE8: target autonomous hif_memcpy */
 213	{
 214		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
 215		.src_nentries = 0,
 216		.src_sz_max = 0,
 217		.dest_nentries = 0,
 218	},
 219
 220};
 221
 222const struct ce_attr ath12k_host_ce_config_ipq5332[] = {
 223	/* CE0: host->target HTC control and raw streams */
 224	{
 225		.flags = CE_ATTR_FLAGS,
 226		.src_nentries = 16,
 227		.src_sz_max = 2048,
 228		.dest_nentries = 0,
 229	},
 230	/* CE1: target->host HTT + HTC control */
 231	{
 232		.flags = CE_ATTR_FLAGS,
 233		.src_nentries = 0,
 234		.src_sz_max = 2048,
 235		.dest_nentries = 512,
 236		.recv_cb = ath12k_htc_rx_completion_handler,
 237	},
 238	/* CE2: target->host WMI */
 239	{
 240		.flags = CE_ATTR_FLAGS,
 241		.src_nentries = 0,
 242		.src_sz_max = 2048,
 243		.dest_nentries = 128,
 244		.recv_cb = ath12k_htc_rx_completion_handler,
 245	},
 246	/* CE3: host->target WMI */
 247	{
 248		.flags = CE_ATTR_FLAGS,
 249		.src_nentries = 32,
 250		.src_sz_max = 2048,
 251		.dest_nentries = 0,
 252	},
 253	/* CE4: host->target HTT */
 254	{
 255		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
 256		.src_nentries = 2048,
 257		.src_sz_max = 256,
 258		.dest_nentries = 0,
 259	},
 260	/* CE5: target -> host PKTLOG */
 261	{
 262		.flags = CE_ATTR_FLAGS,
 263		.src_nentries = 0,
 264		.src_sz_max = 2048,
 265		.dest_nentries = 512,
 266		.recv_cb = ath12k_dp_htt_htc_t2h_msg_handler,
 267	},
 268	/* CE6: Target autonomous HIF_memcpy */
 269	{
 270		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
 271		.src_nentries = 0,
 272		.src_sz_max = 0,
 273		.dest_nentries = 0,
 274	},
 275	/* CE7: CV Prefetch */
 276	{
 277		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
 278		.src_nentries = 0,
 279		.src_sz_max = 0,
 280		.dest_nentries = 0,
 281	},
 282	/* CE8: Target HIF memcpy (Generic HIF memcypy) */
 283	{
 284		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
 285		.src_nentries = 0,
 286		.src_sz_max = 0,
 287		.dest_nentries = 0,
 288	},
 289	/* CE9: WMI logging/CFR/Spectral/Radar */
 290	{
 291		.flags = CE_ATTR_FLAGS,
 292		.src_nentries = 0,
 293		.src_sz_max = 2048,
 294		.dest_nentries = 128,
 295	},
 296	/* CE10: Unused */
 297	{
 298		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
 299		.src_nentries = 0,
 300		.src_sz_max = 0,
 301		.dest_nentries = 0,
 302	},
 303	/* CE11: Unused */
 304	{
 305		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
 306		.src_nentries = 0,
 307		.src_sz_max = 0,
 308		.dest_nentries = 0,
 309	},
 310};
 311
 312static int ath12k_ce_rx_buf_enqueue_pipe(struct ath12k_ce_pipe *pipe,
 313					 struct sk_buff *skb, dma_addr_t paddr)
 314{
 315	struct ath12k_base *ab = pipe->ab;
 316	struct ath12k_ce_ring *ring = pipe->dest_ring;
 317	struct hal_srng *srng;
 318	unsigned int write_index;
 319	unsigned int nentries_mask = ring->nentries_mask;
 320	struct hal_ce_srng_dest_desc *desc;
 321	int ret;
 322
 323	lockdep_assert_held(&ab->ce.ce_lock);
 324
 325	write_index = ring->write_index;
 326
 327	srng = &ab->hal.srng_list[ring->hal_ring_id];
 328
 329	spin_lock_bh(&srng->lock);
 330
 331	ath12k_hal_srng_access_begin(ab, srng);
 332
 333	if (unlikely(ath12k_hal_srng_src_num_free(ab, srng, false) < 1)) {
 334		ret = -ENOSPC;
 335		goto exit;
 336	}
 337
 338	desc = ath12k_hal_srng_src_get_next_entry(ab, srng);
 339	if (!desc) {
 340		ret = -ENOSPC;
 341		goto exit;
 342	}
 343
 344	ath12k_hal_ce_dst_set_desc(desc, paddr);
 345
 346	ring->skb[write_index] = skb;
 347	write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
 348	ring->write_index = write_index;
 349
 350	pipe->rx_buf_needed--;
 351
 352	ret = 0;
 353exit:
 354	ath12k_hal_srng_access_end(ab, srng);
 355
 356	spin_unlock_bh(&srng->lock);
 357
 358	return ret;
 359}
 360
 361static int ath12k_ce_rx_post_pipe(struct ath12k_ce_pipe *pipe)
 362{
 363	struct ath12k_base *ab = pipe->ab;
 364	struct sk_buff *skb;
 365	dma_addr_t paddr;
 366	int ret = 0;
 367
 368	if (!(pipe->dest_ring || pipe->status_ring))
 369		return 0;
 370
 371	spin_lock_bh(&ab->ce.ce_lock);
 372	while (pipe->rx_buf_needed) {
 373		skb = dev_alloc_skb(pipe->buf_sz);
 374		if (!skb) {
 375			ret = -ENOMEM;
 376			goto exit;
 377		}
 378
 379		WARN_ON_ONCE(!IS_ALIGNED((unsigned long)skb->data, 4));
 380
 381		paddr = dma_map_single(ab->dev, skb->data,
 382				       skb->len + skb_tailroom(skb),
 383				       DMA_FROM_DEVICE);
 384		if (unlikely(dma_mapping_error(ab->dev, paddr))) {
 385			ath12k_warn(ab, "failed to dma map ce rx buf\n");
 386			dev_kfree_skb_any(skb);
 387			ret = -EIO;
 388			goto exit;
 389		}
 390
 391		ATH12K_SKB_RXCB(skb)->paddr = paddr;
 392
 393		ret = ath12k_ce_rx_buf_enqueue_pipe(pipe, skb, paddr);
 394		if (ret) {
 395			ath12k_dbg(ab, ATH12K_DBG_CE, "failed to enqueue rx buf: %d\n",
 396				   ret);
 397			dma_unmap_single(ab->dev, paddr,
 398					 skb->len + skb_tailroom(skb),
 399					 DMA_FROM_DEVICE);
 400			dev_kfree_skb_any(skb);
 401			goto exit;
 402		}
 403	}
 404
 405exit:
 406	spin_unlock_bh(&ab->ce.ce_lock);
 407	return ret;
 408}
 409
 410static int ath12k_ce_completed_recv_next(struct ath12k_ce_pipe *pipe,
 411					 struct sk_buff **skb, int *nbytes)
 412{
 413	struct ath12k_base *ab = pipe->ab;
 414	struct hal_ce_srng_dst_status_desc *desc;
 415	struct hal_srng *srng;
 416	unsigned int sw_index;
 417	unsigned int nentries_mask;
 418	int ret = 0;
 419
 420	spin_lock_bh(&ab->ce.ce_lock);
 421
 422	sw_index = pipe->dest_ring->sw_index;
 423	nentries_mask = pipe->dest_ring->nentries_mask;
 424
 425	srng = &ab->hal.srng_list[pipe->status_ring->hal_ring_id];
 426
 427	spin_lock_bh(&srng->lock);
 428
 429	ath12k_hal_srng_access_begin(ab, srng);
 430
 431	desc = ath12k_hal_srng_dst_get_next_entry(ab, srng);
 432	if (!desc) {
 433		ret = -EIO;
 434		goto err;
 435	}
 436
 437	*nbytes = ath12k_hal_ce_dst_status_get_length(desc);
 438
 439	*skb = pipe->dest_ring->skb[sw_index];
 440	pipe->dest_ring->skb[sw_index] = NULL;
 441
 442	sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
 443	pipe->dest_ring->sw_index = sw_index;
 444
 445	pipe->rx_buf_needed++;
 446err:
 447	ath12k_hal_srng_access_end(ab, srng);
 448
 449	spin_unlock_bh(&srng->lock);
 450
 451	spin_unlock_bh(&ab->ce.ce_lock);
 452
 453	return ret;
 454}
 455
 456static void ath12k_ce_recv_process_cb(struct ath12k_ce_pipe *pipe)
 457{
 458	struct ath12k_base *ab = pipe->ab;
 459	struct sk_buff *skb;
 460	struct sk_buff_head list;
 461	unsigned int nbytes, max_nbytes;
 462	int ret;
 463
 464	__skb_queue_head_init(&list);
 465	while (ath12k_ce_completed_recv_next(pipe, &skb, &nbytes) == 0) {
 466		max_nbytes = skb->len + skb_tailroom(skb);
 467		dma_unmap_single(ab->dev, ATH12K_SKB_RXCB(skb)->paddr,
 468				 max_nbytes, DMA_FROM_DEVICE);
 469
 470		if (unlikely(max_nbytes < nbytes || nbytes == 0)) {
 471			ath12k_warn(ab, "unexpected rx length (nbytes %d, max %d)",
 472				    nbytes, max_nbytes);
 473			dev_kfree_skb_any(skb);
 474			continue;
 475		}
 476
 477		skb_put(skb, nbytes);
 478		__skb_queue_tail(&list, skb);
 479	}
 480
 481	while ((skb = __skb_dequeue(&list))) {
 482		ath12k_dbg(ab, ATH12K_DBG_CE, "rx ce pipe %d len %d\n",
 483			   pipe->pipe_num, skb->len);
 484		pipe->recv_cb(ab, skb);
 485	}
 486
 487	ret = ath12k_ce_rx_post_pipe(pipe);
 488	if (ret && ret != -ENOSPC) {
 489		ath12k_warn(ab, "failed to post rx buf to pipe: %d err: %d\n",
 490			    pipe->pipe_num, ret);
 491		mod_timer(&ab->rx_replenish_retry,
 492			  jiffies + ATH12K_CE_RX_POST_RETRY_JIFFIES);
 493	}
 494}
 495
 496static struct sk_buff *ath12k_ce_completed_send_next(struct ath12k_ce_pipe *pipe)
 497{
 498	struct ath12k_base *ab = pipe->ab;
 499	struct hal_ce_srng_src_desc *desc;
 500	struct hal_srng *srng;
 501	unsigned int sw_index;
 502	unsigned int nentries_mask;
 503	struct sk_buff *skb;
 504
 505	spin_lock_bh(&ab->ce.ce_lock);
 506
 507	sw_index = pipe->src_ring->sw_index;
 508	nentries_mask = pipe->src_ring->nentries_mask;
 509
 510	srng = &ab->hal.srng_list[pipe->src_ring->hal_ring_id];
 511
 512	spin_lock_bh(&srng->lock);
 513
 514	ath12k_hal_srng_access_begin(ab, srng);
 515
 516	desc = ath12k_hal_srng_src_reap_next(ab, srng);
 517	if (!desc) {
 518		skb = ERR_PTR(-EIO);
 519		goto err_unlock;
 520	}
 521
 522	skb = pipe->src_ring->skb[sw_index];
 523
 524	pipe->src_ring->skb[sw_index] = NULL;
 525
 526	sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
 527	pipe->src_ring->sw_index = sw_index;
 528
 529err_unlock:
 530	spin_unlock_bh(&srng->lock);
 531
 532	spin_unlock_bh(&ab->ce.ce_lock);
 533
 534	return skb;
 535}
 536
 537static void ath12k_ce_send_done_cb(struct ath12k_ce_pipe *pipe)
 538{
 539	struct ath12k_base *ab = pipe->ab;
 540	struct sk_buff *skb;
 541
 542	while (!IS_ERR(skb = ath12k_ce_completed_send_next(pipe))) {
 543		if (!skb)
 544			continue;
 545
 546		dma_unmap_single(ab->dev, ATH12K_SKB_CB(skb)->paddr, skb->len,
 547				 DMA_TO_DEVICE);
 548		dev_kfree_skb_any(skb);
 549	}
 550}
 551
 552static void ath12k_ce_srng_msi_ring_params_setup(struct ath12k_base *ab, u32 ce_id,
 553						 struct hal_srng_params *ring_params)
 554{
 555	u32 msi_data_start;
 556	u32 msi_data_count, msi_data_idx;
 557	u32 msi_irq_start;
 558	u32 addr_lo;
 559	u32 addr_hi;
 560	int ret;
 561
 562	ret = ath12k_hif_get_user_msi_vector(ab, "CE",
 563					     &msi_data_count, &msi_data_start,
 564					     &msi_irq_start);
 565
 566	if (ret)
 567		return;
 568
 569	ath12k_hif_get_msi_address(ab, &addr_lo, &addr_hi);
 570	ath12k_hif_get_ce_msi_idx(ab, ce_id, &msi_data_idx);
 571
 572	ring_params->msi_addr = addr_lo;
 573	ring_params->msi_addr |= (dma_addr_t)(((uint64_t)addr_hi) << 32);
 574	ring_params->msi_data = (msi_data_idx % msi_data_count) + msi_data_start;
 575	ring_params->flags |= HAL_SRNG_FLAGS_MSI_INTR;
 576}
 577
 578static int ath12k_ce_init_ring(struct ath12k_base *ab,
 579			       struct ath12k_ce_ring *ce_ring,
 580			       int ce_id, enum hal_ring_type type)
 581{
 582	struct hal_srng_params params = {};
 583	int ret;
 584
 585	params.ring_base_paddr = ce_ring->base_addr_ce_space;
 586	params.ring_base_vaddr = ce_ring->base_addr_owner_space;
 587	params.num_entries = ce_ring->nentries;
 588
 589	if (!(CE_ATTR_DIS_INTR & ab->hw_params->host_ce_config[ce_id].flags))
 590		ath12k_ce_srng_msi_ring_params_setup(ab, ce_id, &params);
 591
 592	switch (type) {
 593	case HAL_CE_SRC:
 594		if (!(CE_ATTR_DIS_INTR & ab->hw_params->host_ce_config[ce_id].flags))
 595			params.intr_batch_cntr_thres_entries = 1;
 596		break;
 597	case HAL_CE_DST:
 598		params.max_buffer_len = ab->hw_params->host_ce_config[ce_id].src_sz_max;
 599		if (!(ab->hw_params->host_ce_config[ce_id].flags & CE_ATTR_DIS_INTR)) {
 600			params.intr_timer_thres_us = 1024;
 601			params.flags |= HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN;
 602			params.low_threshold = ce_ring->nentries - 3;
 603		}
 604		break;
 605	case HAL_CE_DST_STATUS:
 606		if (!(ab->hw_params->host_ce_config[ce_id].flags & CE_ATTR_DIS_INTR)) {
 607			params.intr_batch_cntr_thres_entries = 1;
 608			params.intr_timer_thres_us = 0x1000;
 609		}
 610		break;
 611	default:
 612		ath12k_warn(ab, "Invalid CE ring type %d\n", type);
 613		return -EINVAL;
 614	}
 615
 616	/* TODO: Init other params needed by HAL to init the ring */
 617
 618	ret = ath12k_hal_srng_setup(ab, type, ce_id, 0, &params);
 619	if (ret < 0) {
 620		ath12k_warn(ab, "failed to setup srng: %d ring_id %d\n",
 621			    ret, ce_id);
 622		return ret;
 623	}
 624
 625	ce_ring->hal_ring_id = ret;
 626
 627	return 0;
 628}
 629
 630static struct ath12k_ce_ring *
 631ath12k_ce_alloc_ring(struct ath12k_base *ab, int nentries, int desc_sz)
 632{
 633	struct ath12k_ce_ring *ce_ring;
 634	dma_addr_t base_addr;
 635
 636	ce_ring = kzalloc(struct_size(ce_ring, skb, nentries), GFP_KERNEL);
 637	if (!ce_ring)
 638		return ERR_PTR(-ENOMEM);
 639
 640	ce_ring->nentries = nentries;
 641	ce_ring->nentries_mask = nentries - 1;
 642
 643	/* Legacy platforms that do not support cache
 644	 * coherent DMA are unsupported
 645	 */
 646	ce_ring->base_addr_owner_space_unaligned =
 647		dma_alloc_coherent(ab->dev,
 648				   nentries * desc_sz + CE_DESC_RING_ALIGN,
 649				   &base_addr, GFP_KERNEL);
 650	if (!ce_ring->base_addr_owner_space_unaligned) {
 651		kfree(ce_ring);
 652		return ERR_PTR(-ENOMEM);
 653	}
 654
 655	ce_ring->base_addr_ce_space_unaligned = base_addr;
 656
 657	ce_ring->base_addr_owner_space =
 658		PTR_ALIGN(ce_ring->base_addr_owner_space_unaligned,
 659			  CE_DESC_RING_ALIGN);
 660
 661	ce_ring->base_addr_ce_space = ALIGN(ce_ring->base_addr_ce_space_unaligned,
 662					    CE_DESC_RING_ALIGN);
 663
 664	return ce_ring;
 665}
 666
 667static int ath12k_ce_alloc_pipe(struct ath12k_base *ab, int ce_id)
 668{
 669	struct ath12k_ce_pipe *pipe = &ab->ce.ce_pipe[ce_id];
 670	const struct ce_attr *attr = &ab->hw_params->host_ce_config[ce_id];
 671	struct ath12k_ce_ring *ring;
 672	int nentries;
 673	int desc_sz;
 674
 675	pipe->attr_flags = attr->flags;
 676
 677	if (attr->src_nentries) {
 678		pipe->send_cb = ath12k_ce_send_done_cb;
 679		nentries = roundup_pow_of_two(attr->src_nentries);
 680		desc_sz = ath12k_hal_ce_get_desc_size(HAL_CE_DESC_SRC);
 681		ring = ath12k_ce_alloc_ring(ab, nentries, desc_sz);
 682		if (IS_ERR(ring))
 683			return PTR_ERR(ring);
 684		pipe->src_ring = ring;
 685	}
 686
 687	if (attr->dest_nentries) {
 688		pipe->recv_cb = attr->recv_cb;
 689		nentries = roundup_pow_of_two(attr->dest_nentries);
 690		desc_sz = ath12k_hal_ce_get_desc_size(HAL_CE_DESC_DST);
 691		ring = ath12k_ce_alloc_ring(ab, nentries, desc_sz);
 692		if (IS_ERR(ring))
 693			return PTR_ERR(ring);
 694		pipe->dest_ring = ring;
 695
 696		desc_sz = ath12k_hal_ce_get_desc_size(HAL_CE_DESC_DST_STATUS);
 697		ring = ath12k_ce_alloc_ring(ab, nentries, desc_sz);
 698		if (IS_ERR(ring))
 699			return PTR_ERR(ring);
 700		pipe->status_ring = ring;
 701	}
 702
 703	return 0;
 704}
 705
 706void ath12k_ce_per_engine_service(struct ath12k_base *ab, u16 ce_id)
 707{
 708	struct ath12k_ce_pipe *pipe = &ab->ce.ce_pipe[ce_id];
 709
 710	if (pipe->send_cb)
 711		pipe->send_cb(pipe);
 712
 713	if (pipe->recv_cb)
 714		ath12k_ce_recv_process_cb(pipe);
 715}
 716
 717void ath12k_ce_poll_send_completed(struct ath12k_base *ab, u8 pipe_id)
 718{
 719	struct ath12k_ce_pipe *pipe = &ab->ce.ce_pipe[pipe_id];
 720
 721	if ((pipe->attr_flags & CE_ATTR_DIS_INTR) && pipe->send_cb)
 722		pipe->send_cb(pipe);
 723}
 724
 725int ath12k_ce_send(struct ath12k_base *ab, struct sk_buff *skb, u8 pipe_id,
 726		   u16 transfer_id)
 727{
 728	struct ath12k_ce_pipe *pipe = &ab->ce.ce_pipe[pipe_id];
 729	struct hal_ce_srng_src_desc *desc;
 730	struct hal_srng *srng;
 731	unsigned int write_index, sw_index;
 732	unsigned int nentries_mask;
 733	int ret = 0;
 734	u8 byte_swap_data = 0;
 735	int num_used;
 736
 737	/* Check if some entries could be regained by handling tx completion if
 738	 * the CE has interrupts disabled and the used entries is more than the
 739	 * defined usage threshold.
 740	 */
 741	if (pipe->attr_flags & CE_ATTR_DIS_INTR) {
 742		spin_lock_bh(&ab->ce.ce_lock);
 743		write_index = pipe->src_ring->write_index;
 744
 745		sw_index = pipe->src_ring->sw_index;
 746
 747		if (write_index >= sw_index)
 748			num_used = write_index - sw_index;
 749		else
 750			num_used = pipe->src_ring->nentries - sw_index +
 751				   write_index;
 752
 753		spin_unlock_bh(&ab->ce.ce_lock);
 754
 755		if (num_used > ATH12K_CE_USAGE_THRESHOLD)
 756			ath12k_ce_poll_send_completed(ab, pipe->pipe_num);
 757	}
 758
 759	if (test_bit(ATH12K_FLAG_CRASH_FLUSH, &ab->dev_flags))
 760		return -ESHUTDOWN;
 761
 762	spin_lock_bh(&ab->ce.ce_lock);
 763
 764	write_index = pipe->src_ring->write_index;
 765	nentries_mask = pipe->src_ring->nentries_mask;
 766
 767	srng = &ab->hal.srng_list[pipe->src_ring->hal_ring_id];
 768
 769	spin_lock_bh(&srng->lock);
 770
 771	ath12k_hal_srng_access_begin(ab, srng);
 772
 773	if (unlikely(ath12k_hal_srng_src_num_free(ab, srng, false) < 1)) {
 774		ath12k_hal_srng_access_end(ab, srng);
 775		ret = -ENOBUFS;
 776		goto unlock;
 777	}
 778
 779	desc = ath12k_hal_srng_src_get_next_reaped(ab, srng);
 780	if (!desc) {
 781		ath12k_hal_srng_access_end(ab, srng);
 782		ret = -ENOBUFS;
 783		goto unlock;
 784	}
 785
 786	if (pipe->attr_flags & CE_ATTR_BYTE_SWAP_DATA)
 787		byte_swap_data = 1;
 788
 789	ath12k_hal_ce_src_set_desc(desc, ATH12K_SKB_CB(skb)->paddr,
 790				   skb->len, transfer_id, byte_swap_data);
 791
 792	pipe->src_ring->skb[write_index] = skb;
 793	pipe->src_ring->write_index = CE_RING_IDX_INCR(nentries_mask,
 794						       write_index);
 795
 796	ath12k_hal_srng_access_end(ab, srng);
 797
 798unlock:
 799	spin_unlock_bh(&srng->lock);
 800
 801	spin_unlock_bh(&ab->ce.ce_lock);
 802
 803	return ret;
 804}
 805
 806static void ath12k_ce_rx_pipe_cleanup(struct ath12k_ce_pipe *pipe)
 807{
 808	struct ath12k_base *ab = pipe->ab;
 809	struct ath12k_ce_ring *ring = pipe->dest_ring;
 810	struct sk_buff *skb;
 811	int i;
 812
 813	if (!(ring && pipe->buf_sz))
 814		return;
 815
 816	for (i = 0; i < ring->nentries; i++) {
 817		skb = ring->skb[i];
 818		if (!skb)
 819			continue;
 820
 821		ring->skb[i] = NULL;
 822		dma_unmap_single(ab->dev, ATH12K_SKB_RXCB(skb)->paddr,
 823				 skb->len + skb_tailroom(skb), DMA_FROM_DEVICE);
 824		dev_kfree_skb_any(skb);
 825	}
 826}
 827
 828void ath12k_ce_cleanup_pipes(struct ath12k_base *ab)
 829{
 830	struct ath12k_ce_pipe *pipe;
 831	int pipe_num;
 832
 833	for (pipe_num = 0; pipe_num < ab->hw_params->ce_count; pipe_num++) {
 834		pipe = &ab->ce.ce_pipe[pipe_num];
 835		ath12k_ce_rx_pipe_cleanup(pipe);
 836
 837		/* Cleanup any src CE's which have interrupts disabled */
 838		ath12k_ce_poll_send_completed(ab, pipe_num);
 839
 840		/* NOTE: Should we also clean up tx buffer in all pipes? */
 841	}
 842}
 843
 844void ath12k_ce_rx_post_buf(struct ath12k_base *ab)
 845{
 846	struct ath12k_ce_pipe *pipe;
 847	int i;
 848	int ret;
 849
 850	for (i = 0; i < ab->hw_params->ce_count; i++) {
 851		pipe = &ab->ce.ce_pipe[i];
 852		ret = ath12k_ce_rx_post_pipe(pipe);
 853		if (ret) {
 854			if (ret == -ENOSPC)
 855				continue;
 856
 857			ath12k_warn(ab, "failed to post rx buf to pipe: %d err: %d\n",
 858				    i, ret);
 859			mod_timer(&ab->rx_replenish_retry,
 860				  jiffies + ATH12K_CE_RX_POST_RETRY_JIFFIES);
 861
 862			return;
 863		}
 864	}
 865}
 866
 867void ath12k_ce_rx_replenish_retry(struct timer_list *t)
 868{
 869	struct ath12k_base *ab = timer_container_of(ab, t, rx_replenish_retry);
 870
 871	ath12k_ce_rx_post_buf(ab);
 872}
 873
 874static void ath12k_ce_shadow_config(struct ath12k_base *ab)
 875{
 876	int i;
 877
 878	for (i = 0; i < ab->hw_params->ce_count; i++) {
 879		if (ab->hw_params->host_ce_config[i].src_nentries)
 880			ath12k_hal_srng_update_shadow_config(ab, HAL_CE_SRC, i);
 881
 882		if (ab->hw_params->host_ce_config[i].dest_nentries) {
 883			ath12k_hal_srng_update_shadow_config(ab, HAL_CE_DST, i);
 884			ath12k_hal_srng_update_shadow_config(ab, HAL_CE_DST_STATUS, i);
 885		}
 886	}
 887}
 888
 889void ath12k_ce_get_shadow_config(struct ath12k_base *ab,
 890				 u32 **shadow_cfg, u32 *shadow_cfg_len)
 891{
 892	if (!ab->hw_params->supports_shadow_regs)
 893		return;
 894
 895	ath12k_hal_srng_get_shadow_config(ab, shadow_cfg, shadow_cfg_len);
 896
 897	/* shadow is already configured */
 898	if (*shadow_cfg_len)
 899		return;
 900
 901	/* shadow isn't configured yet, configure now.
 902	 * non-CE srngs are configured firstly, then
 903	 * all CE srngs.
 904	 */
 905	ath12k_hal_srng_shadow_config(ab);
 906	ath12k_ce_shadow_config(ab);
 907
 908	/* get the shadow configuration */
 909	ath12k_hal_srng_get_shadow_config(ab, shadow_cfg, shadow_cfg_len);
 910}
 911
 912int ath12k_ce_init_pipes(struct ath12k_base *ab)
 913{
 914	struct ath12k_ce_pipe *pipe;
 915	int i;
 916	int ret;
 917
 918	ath12k_ce_get_shadow_config(ab, &ab->qmi.ce_cfg.shadow_reg_v3,
 919				    &ab->qmi.ce_cfg.shadow_reg_v3_len);
 920
 921	for (i = 0; i < ab->hw_params->ce_count; i++) {
 922		pipe = &ab->ce.ce_pipe[i];
 923
 924		if (pipe->src_ring) {
 925			ret = ath12k_ce_init_ring(ab, pipe->src_ring, i,
 926						  HAL_CE_SRC);
 927			if (ret) {
 928				ath12k_warn(ab, "failed to init src ring: %d\n",
 929					    ret);
 930				/* Should we clear any partial init */
 931				return ret;
 932			}
 933
 934			pipe->src_ring->write_index = 0;
 935			pipe->src_ring->sw_index = 0;
 936		}
 937
 938		if (pipe->dest_ring) {
 939			ret = ath12k_ce_init_ring(ab, pipe->dest_ring, i,
 940						  HAL_CE_DST);
 941			if (ret) {
 942				ath12k_warn(ab, "failed to init dest ring: %d\n",
 943					    ret);
 944				/* Should we clear any partial init */
 945				return ret;
 946			}
 947
 948			pipe->rx_buf_needed = pipe->dest_ring->nentries ?
 949					      pipe->dest_ring->nentries - 2 : 0;
 950
 951			pipe->dest_ring->write_index = 0;
 952			pipe->dest_ring->sw_index = 0;
 953		}
 954
 955		if (pipe->status_ring) {
 956			ret = ath12k_ce_init_ring(ab, pipe->status_ring, i,
 957						  HAL_CE_DST_STATUS);
 958			if (ret) {
 959				ath12k_warn(ab, "failed to init dest status ing: %d\n",
 960					    ret);
 961				/* Should we clear any partial init */
 962				return ret;
 963			}
 964
 965			pipe->status_ring->write_index = 0;
 966			pipe->status_ring->sw_index = 0;
 967		}
 968	}
 969
 970	return 0;
 971}
 972
 973void ath12k_ce_free_pipes(struct ath12k_base *ab)
 974{
 975	struct ath12k_ce_pipe *pipe;
 976	int desc_sz;
 977	int i;
 978
 979	for (i = 0; i < ab->hw_params->ce_count; i++) {
 980		pipe = &ab->ce.ce_pipe[i];
 981
 982		if (pipe->src_ring) {
 983			desc_sz = ath12k_hal_ce_get_desc_size(HAL_CE_DESC_SRC);
 984			dma_free_coherent(ab->dev,
 985					  pipe->src_ring->nentries * desc_sz +
 986					  CE_DESC_RING_ALIGN,
 987					  pipe->src_ring->base_addr_owner_space_unaligned,
 988					  pipe->src_ring->base_addr_ce_space_unaligned);
 989			kfree(pipe->src_ring);
 990			pipe->src_ring = NULL;
 991		}
 992
 993		if (pipe->dest_ring) {
 994			desc_sz = ath12k_hal_ce_get_desc_size(HAL_CE_DESC_DST);
 995			dma_free_coherent(ab->dev,
 996					  pipe->dest_ring->nentries * desc_sz +
 997					  CE_DESC_RING_ALIGN,
 998					  pipe->dest_ring->base_addr_owner_space_unaligned,
 999					  pipe->dest_ring->base_addr_ce_space_unaligned);
1000			kfree(pipe->dest_ring);
1001			pipe->dest_ring = NULL;
1002		}
1003
1004		if (pipe->status_ring) {
1005			desc_sz =
1006			  ath12k_hal_ce_get_desc_size(HAL_CE_DESC_DST_STATUS);
1007			dma_free_coherent(ab->dev,
1008					  pipe->status_ring->nentries * desc_sz +
1009					  CE_DESC_RING_ALIGN,
1010					  pipe->status_ring->base_addr_owner_space_unaligned,
1011					  pipe->status_ring->base_addr_ce_space_unaligned);
1012			kfree(pipe->status_ring);
1013			pipe->status_ring = NULL;
1014		}
1015	}
1016}
1017
1018int ath12k_ce_alloc_pipes(struct ath12k_base *ab)
1019{
1020	struct ath12k_ce_pipe *pipe;
1021	int i;
1022	int ret;
1023	const struct ce_attr *attr;
1024
1025	spin_lock_init(&ab->ce.ce_lock);
1026
1027	for (i = 0; i < ab->hw_params->ce_count; i++) {
1028		attr = &ab->hw_params->host_ce_config[i];
1029		pipe = &ab->ce.ce_pipe[i];
1030		pipe->pipe_num = i;
1031		pipe->ab = ab;
1032		pipe->buf_sz = attr->src_sz_max;
1033
1034		ret = ath12k_ce_alloc_pipe(ab, i);
1035		if (ret) {
1036			/* Free any partial successful allocation */
1037			ath12k_ce_free_pipes(ab);
1038			return ret;
1039		}
1040	}
1041
1042	return 0;
1043}
1044
1045int ath12k_ce_get_attr_flags(struct ath12k_base *ab, int ce_id)
1046{
1047	if (ce_id >= ab->hw_params->ce_count)
1048		return -EINVAL;
1049
1050	return ab->hw_params->host_ce_config[ce_id].flags;
1051}