hal.c

   1// SPDX-License-Identifier: BSD-3-Clause-Clear
   2/*
   3 * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved.
   4 * Copyright (c) 2021-2024 Qualcomm Innovation Center, Inc. All rights reserved.
   5 * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
   6 */
   7#include <linux/dma-mapping.h>
   8#include <linux/export.h>
   9#include "hal_tx.h"
  10#include "debug.h"
  11#include "hal_desc.h"
  12#include "hif.h"
  13
  14static const struct hal_srng_config hw_srng_config_template[] = {
  15	/* TODO: max_rings can populated by querying HW capabilities */
  16	{ /* REO_DST */
  17		.start_ring_id = HAL_SRNG_RING_ID_REO2SW1,
  18		.max_rings = 4,
  19		.entry_size = sizeof(struct hal_reo_dest_ring) >> 2,
  20		.lmac_ring = false,
  21		.ring_dir = HAL_SRNG_DIR_DST,
  22		.max_size = HAL_REO_REO2SW1_RING_BASE_MSB_RING_SIZE,
  23	},
  24	{ /* REO_EXCEPTION */
  25		/* Designating REO2TCL ring as exception ring. This ring is
  26		 * similar to other REO2SW rings though it is named as REO2TCL.
  27		 * Any of theREO2SW rings can be used as exception ring.
  28		 */
  29		.start_ring_id = HAL_SRNG_RING_ID_REO2TCL,
  30		.max_rings = 1,
  31		.entry_size = sizeof(struct hal_reo_dest_ring) >> 2,
  32		.lmac_ring = false,
  33		.ring_dir = HAL_SRNG_DIR_DST,
  34		.max_size = HAL_REO_REO2TCL_RING_BASE_MSB_RING_SIZE,
  35	},
  36	{ /* REO_REINJECT */
  37		.start_ring_id = HAL_SRNG_RING_ID_SW2REO,
  38		.max_rings = 1,
  39		.entry_size = sizeof(struct hal_reo_entrance_ring) >> 2,
  40		.lmac_ring = false,
  41		.ring_dir = HAL_SRNG_DIR_SRC,
  42		.max_size = HAL_REO_SW2REO_RING_BASE_MSB_RING_SIZE,
  43	},
  44	{ /* REO_CMD */
  45		.start_ring_id = HAL_SRNG_RING_ID_REO_CMD,
  46		.max_rings = 1,
  47		.entry_size = (sizeof(struct hal_tlv_hdr) +
  48			sizeof(struct hal_reo_get_queue_stats)) >> 2,
  49		.lmac_ring = false,
  50		.ring_dir = HAL_SRNG_DIR_SRC,
  51		.max_size = HAL_REO_CMD_RING_BASE_MSB_RING_SIZE,
  52	},
  53	{ /* REO_STATUS */
  54		.start_ring_id = HAL_SRNG_RING_ID_REO_STATUS,
  55		.max_rings = 1,
  56		.entry_size = (sizeof(struct hal_tlv_hdr) +
  57			sizeof(struct hal_reo_get_queue_stats_status)) >> 2,
  58		.lmac_ring = false,
  59		.ring_dir = HAL_SRNG_DIR_DST,
  60		.max_size = HAL_REO_STATUS_RING_BASE_MSB_RING_SIZE,
  61	},
  62	{ /* TCL_DATA */
  63		.start_ring_id = HAL_SRNG_RING_ID_SW2TCL1,
  64		.max_rings = 3,
  65		.entry_size = (sizeof(struct hal_tlv_hdr) +
  66			     sizeof(struct hal_tcl_data_cmd)) >> 2,
  67		.lmac_ring = false,
  68		.ring_dir = HAL_SRNG_DIR_SRC,
  69		.max_size = HAL_SW2TCL1_RING_BASE_MSB_RING_SIZE,
  70	},
  71	{ /* TCL_CMD */
  72		.start_ring_id = HAL_SRNG_RING_ID_SW2TCL_CMD,
  73		.max_rings = 1,
  74		.entry_size = (sizeof(struct hal_tlv_hdr) +
  75			     sizeof(struct hal_tcl_gse_cmd)) >> 2,
  76		.lmac_ring =  false,
  77		.ring_dir = HAL_SRNG_DIR_SRC,
  78		.max_size = HAL_SW2TCL1_CMD_RING_BASE_MSB_RING_SIZE,
  79	},
  80	{ /* TCL_STATUS */
  81		.start_ring_id = HAL_SRNG_RING_ID_TCL_STATUS,
  82		.max_rings = 1,
  83		.entry_size = (sizeof(struct hal_tlv_hdr) +
  84			     sizeof(struct hal_tcl_status_ring)) >> 2,
  85		.lmac_ring = false,
  86		.ring_dir = HAL_SRNG_DIR_DST,
  87		.max_size = HAL_TCL_STATUS_RING_BASE_MSB_RING_SIZE,
  88	},
  89	{ /* CE_SRC */
  90		.start_ring_id = HAL_SRNG_RING_ID_CE0_SRC,
  91		.max_rings = 12,
  92		.entry_size = sizeof(struct hal_ce_srng_src_desc) >> 2,
  93		.lmac_ring = false,
  94		.ring_dir = HAL_SRNG_DIR_SRC,
  95		.max_size = HAL_CE_SRC_RING_BASE_MSB_RING_SIZE,
  96	},
  97	{ /* CE_DST */
  98		.start_ring_id = HAL_SRNG_RING_ID_CE0_DST,
  99		.max_rings = 12,
 100		.entry_size = sizeof(struct hal_ce_srng_dest_desc) >> 2,
 101		.lmac_ring = false,
 102		.ring_dir = HAL_SRNG_DIR_SRC,
 103		.max_size = HAL_CE_DST_RING_BASE_MSB_RING_SIZE,
 104	},
 105	{ /* CE_DST_STATUS */
 106		.start_ring_id = HAL_SRNG_RING_ID_CE0_DST_STATUS,
 107		.max_rings = 12,
 108		.entry_size = sizeof(struct hal_ce_srng_dst_status_desc) >> 2,
 109		.lmac_ring = false,
 110		.ring_dir = HAL_SRNG_DIR_DST,
 111		.max_size = HAL_CE_DST_STATUS_RING_BASE_MSB_RING_SIZE,
 112	},
 113	{ /* WBM_IDLE_LINK */
 114		.start_ring_id = HAL_SRNG_RING_ID_WBM_IDLE_LINK,
 115		.max_rings = 1,
 116		.entry_size = sizeof(struct hal_wbm_link_desc) >> 2,
 117		.lmac_ring = false,
 118		.ring_dir = HAL_SRNG_DIR_SRC,
 119		.max_size = HAL_WBM_IDLE_LINK_RING_BASE_MSB_RING_SIZE,
 120	},
 121	{ /* SW2WBM_RELEASE */
 122		.start_ring_id = HAL_SRNG_RING_ID_WBM_SW_RELEASE,
 123		.max_rings = 1,
 124		.entry_size = sizeof(struct hal_wbm_release_ring) >> 2,
 125		.lmac_ring = false,
 126		.ring_dir = HAL_SRNG_DIR_SRC,
 127		.max_size = HAL_SW2WBM_RELEASE_RING_BASE_MSB_RING_SIZE,
 128	},
 129	{ /* WBM2SW_RELEASE */
 130		.start_ring_id = HAL_SRNG_RING_ID_WBM2SW0_RELEASE,
 131		.max_rings = 5,
 132		.entry_size = sizeof(struct hal_wbm_release_ring) >> 2,
 133		.lmac_ring = false,
 134		.ring_dir = HAL_SRNG_DIR_DST,
 135		.max_size = HAL_WBM2SW_RELEASE_RING_BASE_MSB_RING_SIZE,
 136	},
 137	{ /* RXDMA_BUF */
 138		.start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2RXDMA0_BUF,
 139		.max_rings = 2,
 140		.entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2,
 141		.lmac_ring = true,
 142		.ring_dir = HAL_SRNG_DIR_SRC,
 143		.max_size = HAL_RXDMA_RING_MAX_SIZE,
 144	},
 145	{ /* RXDMA_DST */
 146		.start_ring_id = HAL_SRNG_RING_ID_WMAC1_RXDMA2SW0,
 147		.max_rings = 1,
 148		.entry_size = sizeof(struct hal_reo_entrance_ring) >> 2,
 149		.lmac_ring = true,
 150		.ring_dir = HAL_SRNG_DIR_DST,
 151		.max_size = HAL_RXDMA_RING_MAX_SIZE,
 152	},
 153	{ /* RXDMA_MONITOR_BUF */
 154		.start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2RXDMA2_BUF,
 155		.max_rings = 1,
 156		.entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2,
 157		.lmac_ring = true,
 158		.ring_dir = HAL_SRNG_DIR_SRC,
 159		.max_size = HAL_RXDMA_RING_MAX_SIZE,
 160	},
 161	{ /* RXDMA_MONITOR_STATUS */
 162		.start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2RXDMA1_STATBUF,
 163		.max_rings = 1,
 164		.entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2,
 165		.lmac_ring = true,
 166		.ring_dir = HAL_SRNG_DIR_SRC,
 167		.max_size = HAL_RXDMA_RING_MAX_SIZE,
 168	},
 169	{ /* RXDMA_MONITOR_DST */
 170		.start_ring_id = HAL_SRNG_RING_ID_WMAC1_RXDMA2SW1,
 171		.max_rings = 1,
 172		.entry_size = sizeof(struct hal_reo_entrance_ring) >> 2,
 173		.lmac_ring = true,
 174		.ring_dir = HAL_SRNG_DIR_DST,
 175		.max_size = HAL_RXDMA_RING_MAX_SIZE,
 176	},
 177	{ /* RXDMA_MONITOR_DESC */
 178		.start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2RXDMA1_DESC,
 179		.max_rings = 1,
 180		.entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2,
 181		.lmac_ring = true,
 182		.ring_dir = HAL_SRNG_DIR_SRC,
 183		.max_size = HAL_RXDMA_RING_MAX_SIZE,
 184	},
 185	{ /* RXDMA DIR BUF */
 186		.start_ring_id = HAL_SRNG_RING_ID_RXDMA_DIR_BUF,
 187		.max_rings = 1,
 188		.entry_size = 8 >> 2, /* TODO: Define the struct */
 189		.lmac_ring = true,
 190		.ring_dir = HAL_SRNG_DIR_SRC,
 191		.max_size = HAL_RXDMA_RING_MAX_SIZE,
 192	},
 193};
 194
 195static int ath11k_hal_alloc_cont_rdp(struct ath11k_base *ab)
 196{
 197	struct ath11k_hal *hal = &ab->hal;
 198	size_t size;
 199
 200	size = sizeof(u32) * HAL_SRNG_RING_ID_MAX;
 201	hal->rdp.vaddr = dma_alloc_coherent(ab->dev, size, &hal->rdp.paddr,
 202					    GFP_KERNEL);
 203	if (!hal->rdp.vaddr)
 204		return -ENOMEM;
 205
 206	return 0;
 207}
 208
 209static void ath11k_hal_free_cont_rdp(struct ath11k_base *ab)
 210{
 211	struct ath11k_hal *hal = &ab->hal;
 212	size_t size;
 213
 214	if (!hal->rdp.vaddr)
 215		return;
 216
 217	size = sizeof(u32) * HAL_SRNG_RING_ID_MAX;
 218	dma_free_coherent(ab->dev, size,
 219			  hal->rdp.vaddr, hal->rdp.paddr);
 220	hal->rdp.vaddr = NULL;
 221}
 222
 223static int ath11k_hal_alloc_cont_wrp(struct ath11k_base *ab)
 224{
 225	struct ath11k_hal *hal = &ab->hal;
 226	size_t size;
 227
 228	size = sizeof(u32) * HAL_SRNG_NUM_LMAC_RINGS;
 229	hal->wrp.vaddr = dma_alloc_coherent(ab->dev, size, &hal->wrp.paddr,
 230					    GFP_KERNEL);
 231	if (!hal->wrp.vaddr)
 232		return -ENOMEM;
 233
 234	return 0;
 235}
 236
 237static void ath11k_hal_free_cont_wrp(struct ath11k_base *ab)
 238{
 239	struct ath11k_hal *hal = &ab->hal;
 240	size_t size;
 241
 242	if (!hal->wrp.vaddr)
 243		return;
 244
 245	size = sizeof(u32) * HAL_SRNG_NUM_LMAC_RINGS;
 246	dma_free_coherent(ab->dev, size,
 247			  hal->wrp.vaddr, hal->wrp.paddr);
 248	hal->wrp.vaddr = NULL;
 249}
 250
 251static void ath11k_hal_ce_dst_setup(struct ath11k_base *ab,
 252				    struct hal_srng *srng, int ring_num)
 253{
 254	struct hal_srng_config *srng_config = &ab->hal.srng_config[HAL_CE_DST];
 255	u32 addr;
 256	u32 val;
 257
 258	addr = HAL_CE_DST_RING_CTRL +
 259	       srng_config->reg_start[HAL_SRNG_REG_GRP_R0] +
 260	       ring_num * srng_config->reg_size[HAL_SRNG_REG_GRP_R0];
 261
 262	val = ath11k_hif_read32(ab, addr);
 263	val &= ~HAL_CE_DST_R0_DEST_CTRL_MAX_LEN;
 264	val |= FIELD_PREP(HAL_CE_DST_R0_DEST_CTRL_MAX_LEN,
 265			  srng->u.dst_ring.max_buffer_length);
 266	ath11k_hif_write32(ab, addr, val);
 267}
 268
 269static void ath11k_hal_srng_dst_hw_init(struct ath11k_base *ab,
 270					struct hal_srng *srng)
 271{
 272	struct ath11k_hal *hal = &ab->hal;
 273	u32 val;
 274	u64 hp_addr;
 275	u32 reg_base;
 276
 277	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0];
 278
 279	if (srng->flags & HAL_SRNG_FLAGS_MSI_INTR) {
 280		ath11k_hif_write32(ab, reg_base +
 281				   HAL_REO1_RING_MSI1_BASE_LSB_OFFSET(ab),
 282				   srng->msi_addr);
 283
 284		val = FIELD_PREP(HAL_REO1_RING_MSI1_BASE_MSB_ADDR,
 285				 ((u64)srng->msi_addr >>
 286				  HAL_ADDR_MSB_REG_SHIFT)) |
 287		      HAL_REO1_RING_MSI1_BASE_MSB_MSI1_ENABLE;
 288		ath11k_hif_write32(ab, reg_base +
 289				       HAL_REO1_RING_MSI1_BASE_MSB_OFFSET(ab), val);
 290
 291		ath11k_hif_write32(ab,
 292				   reg_base + HAL_REO1_RING_MSI1_DATA_OFFSET(ab),
 293				   srng->msi_data);
 294	}
 295
 296	ath11k_hif_write32(ab, reg_base, srng->ring_base_paddr);
 297
 298	val = FIELD_PREP(HAL_REO1_RING_BASE_MSB_RING_BASE_ADDR_MSB,
 299			 ((u64)srng->ring_base_paddr >>
 300			  HAL_ADDR_MSB_REG_SHIFT)) |
 301	      FIELD_PREP(HAL_REO1_RING_BASE_MSB_RING_SIZE,
 302			 (srng->entry_size * srng->num_entries));
 303	ath11k_hif_write32(ab, reg_base + HAL_REO1_RING_BASE_MSB_OFFSET(ab), val);
 304
 305	val = FIELD_PREP(HAL_REO1_RING_ID_RING_ID, srng->ring_id) |
 306	      FIELD_PREP(HAL_REO1_RING_ID_ENTRY_SIZE, srng->entry_size);
 307	ath11k_hif_write32(ab, reg_base + HAL_REO1_RING_ID_OFFSET(ab), val);
 308
 309	/* interrupt setup */
 310	val = FIELD_PREP(HAL_REO1_RING_PRDR_INT_SETUP_INTR_TMR_THOLD,
 311			 (srng->intr_timer_thres_us >> 3));
 312
 313	val |= FIELD_PREP(HAL_REO1_RING_PRDR_INT_SETUP_BATCH_COUNTER_THOLD,
 314			  (srng->intr_batch_cntr_thres_entries *
 315			   srng->entry_size));
 316
 317	ath11k_hif_write32(ab,
 318			   reg_base + HAL_REO1_RING_PRODUCER_INT_SETUP_OFFSET(ab),
 319			   val);
 320
 321	hp_addr = hal->rdp.paddr +
 322		  ((unsigned long)srng->u.dst_ring.hp_addr -
 323		   (unsigned long)hal->rdp.vaddr);
 324	ath11k_hif_write32(ab, reg_base + HAL_REO1_RING_HP_ADDR_LSB_OFFSET(ab),
 325			   hp_addr & HAL_ADDR_LSB_REG_MASK);
 326	ath11k_hif_write32(ab, reg_base + HAL_REO1_RING_HP_ADDR_MSB_OFFSET(ab),
 327			   hp_addr >> HAL_ADDR_MSB_REG_SHIFT);
 328
 329	/* Initialize head and tail pointers to indicate ring is empty */
 330	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R2];
 331	ath11k_hif_write32(ab, reg_base, 0);
 332	ath11k_hif_write32(ab, reg_base + HAL_REO1_RING_TP_OFFSET(ab), 0);
 333	*srng->u.dst_ring.hp_addr = 0;
 334
 335	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0];
 336	val = 0;
 337	if (srng->flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP)
 338		val |= HAL_REO1_RING_MISC_DATA_TLV_SWAP;
 339	if (srng->flags & HAL_SRNG_FLAGS_RING_PTR_SWAP)
 340		val |= HAL_REO1_RING_MISC_HOST_FW_SWAP;
 341	if (srng->flags & HAL_SRNG_FLAGS_MSI_SWAP)
 342		val |= HAL_REO1_RING_MISC_MSI_SWAP;
 343	val |= HAL_REO1_RING_MISC_SRNG_ENABLE;
 344
 345	ath11k_hif_write32(ab, reg_base + HAL_REO1_RING_MISC_OFFSET(ab), val);
 346}
 347
 348static void ath11k_hal_srng_src_hw_init(struct ath11k_base *ab,
 349					struct hal_srng *srng)
 350{
 351	struct ath11k_hal *hal = &ab->hal;
 352	u32 val;
 353	u64 tp_addr;
 354	u32 reg_base;
 355
 356	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0];
 357
 358	if (srng->flags & HAL_SRNG_FLAGS_MSI_INTR) {
 359		ath11k_hif_write32(ab, reg_base +
 360				   HAL_TCL1_RING_MSI1_BASE_LSB_OFFSET(ab),
 361				   srng->msi_addr);
 362
 363		val = FIELD_PREP(HAL_TCL1_RING_MSI1_BASE_MSB_ADDR,
 364				 ((u64)srng->msi_addr >>
 365				  HAL_ADDR_MSB_REG_SHIFT)) |
 366		      HAL_TCL1_RING_MSI1_BASE_MSB_MSI1_ENABLE;
 367		ath11k_hif_write32(ab, reg_base +
 368				       HAL_TCL1_RING_MSI1_BASE_MSB_OFFSET(ab),
 369				   val);
 370
 371		ath11k_hif_write32(ab, reg_base +
 372				       HAL_TCL1_RING_MSI1_DATA_OFFSET(ab),
 373				   srng->msi_data);
 374	}
 375
 376	ath11k_hif_write32(ab, reg_base, srng->ring_base_paddr);
 377
 378	val = FIELD_PREP(HAL_TCL1_RING_BASE_MSB_RING_BASE_ADDR_MSB,
 379			 ((u64)srng->ring_base_paddr >>
 380			  HAL_ADDR_MSB_REG_SHIFT)) |
 381	      FIELD_PREP(HAL_TCL1_RING_BASE_MSB_RING_SIZE,
 382			 (srng->entry_size * srng->num_entries));
 383	ath11k_hif_write32(ab, reg_base + HAL_TCL1_RING_BASE_MSB_OFFSET(ab), val);
 384
 385	val = FIELD_PREP(HAL_REO1_RING_ID_ENTRY_SIZE, srng->entry_size);
 386	ath11k_hif_write32(ab, reg_base + HAL_TCL1_RING_ID_OFFSET(ab), val);
 387
 388	if (srng->ring_id == HAL_SRNG_RING_ID_WBM_IDLE_LINK) {
 389		ath11k_hif_write32(ab, reg_base, (u32)srng->ring_base_paddr);
 390		val = FIELD_PREP(HAL_TCL1_RING_BASE_MSB_RING_BASE_ADDR_MSB,
 391				 ((u64)srng->ring_base_paddr >>
 392				 HAL_ADDR_MSB_REG_SHIFT)) |
 393			FIELD_PREP(HAL_TCL1_RING_BASE_MSB_RING_SIZE,
 394				   (srng->entry_size * srng->num_entries));
 395		ath11k_hif_write32(ab, reg_base + HAL_TCL1_RING_BASE_MSB_OFFSET(ab), val);
 396	}
 397
 398	/* interrupt setup */
 399	/* NOTE: IPQ8074 v2 requires the interrupt timer threshold in the
 400	 * unit of 8 usecs instead of 1 usec (as required by v1).
 401	 */
 402	val = FIELD_PREP(HAL_TCL1_RING_CONSR_INT_SETUP_IX0_INTR_TMR_THOLD,
 403			 srng->intr_timer_thres_us);
 404
 405	val |= FIELD_PREP(HAL_TCL1_RING_CONSR_INT_SETUP_IX0_BATCH_COUNTER_THOLD,
 406			  (srng->intr_batch_cntr_thres_entries *
 407			   srng->entry_size));
 408
 409	ath11k_hif_write32(ab,
 410			   reg_base + HAL_TCL1_RING_CONSR_INT_SETUP_IX0_OFFSET(ab),
 411			   val);
 412
 413	val = 0;
 414	if (srng->flags & HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN) {
 415		val |= FIELD_PREP(HAL_TCL1_RING_CONSR_INT_SETUP_IX1_LOW_THOLD,
 416				  srng->u.src_ring.low_threshold);
 417	}
 418	ath11k_hif_write32(ab,
 419			   reg_base + HAL_TCL1_RING_CONSR_INT_SETUP_IX1_OFFSET(ab),
 420			   val);
 421
 422	if (srng->ring_id != HAL_SRNG_RING_ID_WBM_IDLE_LINK) {
 423		tp_addr = hal->rdp.paddr +
 424			  ((unsigned long)srng->u.src_ring.tp_addr -
 425			   (unsigned long)hal->rdp.vaddr);
 426		ath11k_hif_write32(ab,
 427				   reg_base + HAL_TCL1_RING_TP_ADDR_LSB_OFFSET(ab),
 428				   tp_addr & HAL_ADDR_LSB_REG_MASK);
 429		ath11k_hif_write32(ab,
 430				   reg_base + HAL_TCL1_RING_TP_ADDR_MSB_OFFSET(ab),
 431				   tp_addr >> HAL_ADDR_MSB_REG_SHIFT);
 432	}
 433
 434	/* Initialize head and tail pointers to indicate ring is empty */
 435	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R2];
 436	ath11k_hif_write32(ab, reg_base, 0);
 437	ath11k_hif_write32(ab, reg_base + HAL_TCL1_RING_TP_OFFSET, 0);
 438	*srng->u.src_ring.tp_addr = 0;
 439
 440	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0];
 441	val = 0;
 442	if (srng->flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP)
 443		val |= HAL_TCL1_RING_MISC_DATA_TLV_SWAP;
 444	if (srng->flags & HAL_SRNG_FLAGS_RING_PTR_SWAP)
 445		val |= HAL_TCL1_RING_MISC_HOST_FW_SWAP;
 446	if (srng->flags & HAL_SRNG_FLAGS_MSI_SWAP)
 447		val |= HAL_TCL1_RING_MISC_MSI_SWAP;
 448
 449	/* Loop count is not used for SRC rings */
 450	val |= HAL_TCL1_RING_MISC_MSI_LOOPCNT_DISABLE;
 451
 452	val |= HAL_TCL1_RING_MISC_SRNG_ENABLE;
 453
 454	ath11k_hif_write32(ab, reg_base + HAL_TCL1_RING_MISC_OFFSET(ab), val);
 455}
 456
 457static void ath11k_hal_srng_hw_init(struct ath11k_base *ab,
 458				    struct hal_srng *srng)
 459{
 460	if (srng->ring_dir == HAL_SRNG_DIR_SRC)
 461		ath11k_hal_srng_src_hw_init(ab, srng);
 462	else
 463		ath11k_hal_srng_dst_hw_init(ab, srng);
 464}
 465
 466static int ath11k_hal_srng_get_ring_id(struct ath11k_base *ab,
 467				       enum hal_ring_type type,
 468				       int ring_num, int mac_id)
 469{
 470	struct hal_srng_config *srng_config = &ab->hal.srng_config[type];
 471	int ring_id;
 472
 473	if (ring_num >= srng_config->max_rings) {
 474		ath11k_warn(ab, "invalid ring number :%d\n", ring_num);
 475		return -EINVAL;
 476	}
 477
 478	ring_id = srng_config->start_ring_id + ring_num;
 479	if (srng_config->lmac_ring)
 480		ring_id += mac_id * HAL_SRNG_RINGS_PER_LMAC;
 481
 482	if (WARN_ON(ring_id >= HAL_SRNG_RING_ID_MAX))
 483		return -EINVAL;
 484
 485	return ring_id;
 486}
 487
 488int ath11k_hal_srng_get_entrysize(struct ath11k_base *ab, u32 ring_type)
 489{
 490	struct hal_srng_config *srng_config;
 491
 492	if (WARN_ON(ring_type >= HAL_MAX_RING_TYPES))
 493		return -EINVAL;
 494
 495	srng_config = &ab->hal.srng_config[ring_type];
 496
 497	return (srng_config->entry_size << 2);
 498}
 499
 500int ath11k_hal_srng_get_max_entries(struct ath11k_base *ab, u32 ring_type)
 501{
 502	struct hal_srng_config *srng_config;
 503
 504	if (WARN_ON(ring_type >= HAL_MAX_RING_TYPES))
 505		return -EINVAL;
 506
 507	srng_config = &ab->hal.srng_config[ring_type];
 508
 509	return (srng_config->max_size / srng_config->entry_size);
 510}
 511
 512void ath11k_hal_srng_get_params(struct ath11k_base *ab, struct hal_srng *srng,
 513				struct hal_srng_params *params)
 514{
 515	params->ring_base_paddr = srng->ring_base_paddr;
 516	params->ring_base_vaddr = srng->ring_base_vaddr;
 517	params->num_entries = srng->num_entries;
 518	params->intr_timer_thres_us = srng->intr_timer_thres_us;
 519	params->intr_batch_cntr_thres_entries =
 520		srng->intr_batch_cntr_thres_entries;
 521	params->low_threshold = srng->u.src_ring.low_threshold;
 522	params->msi_addr = srng->msi_addr;
 523	params->msi_data = srng->msi_data;
 524	params->flags = srng->flags;
 525}
 526
 527dma_addr_t ath11k_hal_srng_get_hp_addr(struct ath11k_base *ab,
 528				       struct hal_srng *srng)
 529{
 530	if (!(srng->flags & HAL_SRNG_FLAGS_LMAC_RING))
 531		return 0;
 532
 533	if (srng->ring_dir == HAL_SRNG_DIR_SRC)
 534		return ab->hal.wrp.paddr +
 535		       ((unsigned long)srng->u.src_ring.hp_addr -
 536			(unsigned long)ab->hal.wrp.vaddr);
 537	else
 538		return ab->hal.rdp.paddr +
 539		       ((unsigned long)srng->u.dst_ring.hp_addr -
 540			 (unsigned long)ab->hal.rdp.vaddr);
 541}
 542
 543dma_addr_t ath11k_hal_srng_get_tp_addr(struct ath11k_base *ab,
 544				       struct hal_srng *srng)
 545{
 546	if (!(srng->flags & HAL_SRNG_FLAGS_LMAC_RING))
 547		return 0;
 548
 549	if (srng->ring_dir == HAL_SRNG_DIR_SRC)
 550		return ab->hal.rdp.paddr +
 551		       ((unsigned long)srng->u.src_ring.tp_addr -
 552			(unsigned long)ab->hal.rdp.vaddr);
 553	else
 554		return ab->hal.wrp.paddr +
 555		       ((unsigned long)srng->u.dst_ring.tp_addr -
 556			(unsigned long)ab->hal.wrp.vaddr);
 557}
 558
 559u32 ath11k_hal_ce_get_desc_size(enum hal_ce_desc type)
 560{
 561	switch (type) {
 562	case HAL_CE_DESC_SRC:
 563		return sizeof(struct hal_ce_srng_src_desc);
 564	case HAL_CE_DESC_DST:
 565		return sizeof(struct hal_ce_srng_dest_desc);
 566	case HAL_CE_DESC_DST_STATUS:
 567		return sizeof(struct hal_ce_srng_dst_status_desc);
 568	}
 569
 570	return 0;
 571}
 572
 573void ath11k_hal_ce_src_set_desc(void *buf, dma_addr_t paddr, u32 len, u32 id,
 574				u8 byte_swap_data)
 575{
 576	struct hal_ce_srng_src_desc *desc = buf;
 577
 578	desc->buffer_addr_low = paddr & HAL_ADDR_LSB_REG_MASK;
 579	desc->buffer_addr_info =
 580		FIELD_PREP(HAL_CE_SRC_DESC_ADDR_INFO_ADDR_HI,
 581			   ((u64)paddr >> HAL_ADDR_MSB_REG_SHIFT)) |
 582		FIELD_PREP(HAL_CE_SRC_DESC_ADDR_INFO_BYTE_SWAP,
 583			   byte_swap_data) |
 584		FIELD_PREP(HAL_CE_SRC_DESC_ADDR_INFO_GATHER, 0) |
 585		FIELD_PREP(HAL_CE_SRC_DESC_ADDR_INFO_LEN, len);
 586	desc->meta_info = FIELD_PREP(HAL_CE_SRC_DESC_META_INFO_DATA, id);
 587}
 588
 589void ath11k_hal_ce_dst_set_desc(void *buf, dma_addr_t paddr)
 590{
 591	struct hal_ce_srng_dest_desc *desc = buf;
 592
 593	desc->buffer_addr_low = paddr & HAL_ADDR_LSB_REG_MASK;
 594	desc->buffer_addr_info =
 595		FIELD_PREP(HAL_CE_DEST_DESC_ADDR_INFO_ADDR_HI,
 596			   ((u64)paddr >> HAL_ADDR_MSB_REG_SHIFT));
 597}
 598
 599u32 ath11k_hal_ce_dst_status_get_length(void *buf)
 600{
 601	struct hal_ce_srng_dst_status_desc *desc = buf;
 602	u32 len;
 603
 604	len = FIELD_GET(HAL_CE_DST_STATUS_DESC_FLAGS_LEN, desc->flags);
 605	desc->flags &= ~HAL_CE_DST_STATUS_DESC_FLAGS_LEN;
 606
 607	return len;
 608}
 609
 610void ath11k_hal_set_link_desc_addr(struct hal_wbm_link_desc *desc, u32 cookie,
 611				   dma_addr_t paddr)
 612{
 613	desc->buf_addr_info.info0 = FIELD_PREP(BUFFER_ADDR_INFO0_ADDR,
 614					       (paddr & HAL_ADDR_LSB_REG_MASK));
 615	desc->buf_addr_info.info1 = FIELD_PREP(BUFFER_ADDR_INFO1_ADDR,
 616					       ((u64)paddr >> HAL_ADDR_MSB_REG_SHIFT)) |
 617				    FIELD_PREP(BUFFER_ADDR_INFO1_RET_BUF_MGR, 1) |
 618				    FIELD_PREP(BUFFER_ADDR_INFO1_SW_COOKIE, cookie);
 619}
 620
 621u32 *ath11k_hal_srng_dst_peek(struct ath11k_base *ab, struct hal_srng *srng)
 622{
 623	lockdep_assert_held(&srng->lock);
 624
 625	if (srng->u.dst_ring.tp != srng->u.dst_ring.cached_hp)
 626		return (srng->ring_base_vaddr + srng->u.dst_ring.tp);
 627
 628	return NULL;
 629}
 630
 631static u32 *ath11k_hal_srng_dst_peek_with_dma(struct ath11k_base *ab,
 632					      struct hal_srng *srng, dma_addr_t *paddr)
 633{
 634	lockdep_assert_held(&srng->lock);
 635
 636	if (srng->u.dst_ring.tp != srng->u.dst_ring.cached_hp) {
 637		*paddr = srng->ring_base_paddr +
 638			  sizeof(*srng->ring_base_vaddr) * srng->u.dst_ring.tp;
 639		return srng->ring_base_vaddr + srng->u.dst_ring.tp;
 640	}
 641
 642	return NULL;
 643}
 644
 645static void ath11k_hal_srng_prefetch_desc(struct ath11k_base *ab,
 646					  struct hal_srng *srng)
 647{
 648	dma_addr_t desc_paddr;
 649	u32 *desc;
 650
 651	/* prefetch only if desc is available */
 652	desc = ath11k_hal_srng_dst_peek_with_dma(ab, srng, &desc_paddr);
 653	if (likely(desc)) {
 654		dma_sync_single_for_cpu(ab->dev, desc_paddr,
 655					(srng->entry_size * sizeof(u32)),
 656					DMA_FROM_DEVICE);
 657		prefetch(desc);
 658	}
 659}
 660
 661u32 *ath11k_hal_srng_dst_get_next_entry(struct ath11k_base *ab,
 662					struct hal_srng *srng)
 663{
 664	u32 *desc;
 665
 666	lockdep_assert_held(&srng->lock);
 667
 668	if (srng->u.dst_ring.tp == srng->u.dst_ring.cached_hp)
 669		return NULL;
 670
 671	desc = srng->ring_base_vaddr + srng->u.dst_ring.tp;
 672
 673	srng->u.dst_ring.tp += srng->entry_size;
 674
 675	/* wrap around to start of ring*/
 676	if (srng->u.dst_ring.tp == srng->ring_size)
 677		srng->u.dst_ring.tp = 0;
 678
 679	/* Try to prefetch the next descriptor in the ring */
 680	if (srng->flags & HAL_SRNG_FLAGS_CACHED)
 681		ath11k_hal_srng_prefetch_desc(ab, srng);
 682
 683	return desc;
 684}
 685
 686int ath11k_hal_srng_dst_num_free(struct ath11k_base *ab, struct hal_srng *srng,
 687				 bool sync_hw_ptr)
 688{
 689	u32 tp, hp;
 690
 691	lockdep_assert_held(&srng->lock);
 692
 693	tp = srng->u.dst_ring.tp;
 694
 695	if (sync_hw_ptr) {
 696		hp = *srng->u.dst_ring.hp_addr;
 697		srng->u.dst_ring.cached_hp = hp;
 698	} else {
 699		hp = srng->u.dst_ring.cached_hp;
 700	}
 701
 702	if (hp >= tp)
 703		return (hp - tp) / srng->entry_size;
 704	else
 705		return (srng->ring_size - tp + hp) / srng->entry_size;
 706}
 707
 708/* Returns number of available entries in src ring */
 709int ath11k_hal_srng_src_num_free(struct ath11k_base *ab, struct hal_srng *srng,
 710				 bool sync_hw_ptr)
 711{
 712	u32 tp, hp;
 713
 714	lockdep_assert_held(&srng->lock);
 715
 716	hp = srng->u.src_ring.hp;
 717
 718	if (sync_hw_ptr) {
 719		tp = *srng->u.src_ring.tp_addr;
 720		srng->u.src_ring.cached_tp = tp;
 721	} else {
 722		tp = srng->u.src_ring.cached_tp;
 723	}
 724
 725	if (tp > hp)
 726		return ((tp - hp) / srng->entry_size) - 1;
 727	else
 728		return ((srng->ring_size - hp + tp) / srng->entry_size) - 1;
 729}
 730
 731u32 *ath11k_hal_srng_src_get_next_entry(struct ath11k_base *ab,
 732					struct hal_srng *srng)
 733{
 734	u32 *desc;
 735	u32 next_hp;
 736
 737	lockdep_assert_held(&srng->lock);
 738
 739	/* TODO: Using % is expensive, but we have to do this since size of some
 740	 * SRNG rings is not power of 2 (due to descriptor sizes). Need to see
 741	 * if separate function is defined for rings having power of 2 ring size
 742	 * (TCL2SW, REO2SW, SW2RXDMA and CE rings) so that we can avoid the
 743	 * overhead of % by using mask (with &).
 744	 */
 745	next_hp = (srng->u.src_ring.hp + srng->entry_size) % srng->ring_size;
 746
 747	if (next_hp == srng->u.src_ring.cached_tp)
 748		return NULL;
 749
 750	desc = srng->ring_base_vaddr + srng->u.src_ring.hp;
 751	srng->u.src_ring.hp = next_hp;
 752
 753	/* TODO: Reap functionality is not used by all rings. If particular
 754	 * ring does not use reap functionality, we need not update reap_hp
 755	 * with next_hp pointer. Need to make sure a separate function is used
 756	 * before doing any optimization by removing below code updating
 757	 * reap_hp.
 758	 */
 759	srng->u.src_ring.reap_hp = next_hp;
 760
 761	return desc;
 762}
 763
 764u32 *ath11k_hal_srng_src_reap_next(struct ath11k_base *ab,
 765				   struct hal_srng *srng)
 766{
 767	u32 *desc;
 768	u32 next_reap_hp;
 769
 770	lockdep_assert_held(&srng->lock);
 771
 772	next_reap_hp = (srng->u.src_ring.reap_hp + srng->entry_size) %
 773		       srng->ring_size;
 774
 775	if (next_reap_hp == srng->u.src_ring.cached_tp)
 776		return NULL;
 777
 778	desc = srng->ring_base_vaddr + next_reap_hp;
 779	srng->u.src_ring.reap_hp = next_reap_hp;
 780
 781	return desc;
 782}
 783
 784u32 *ath11k_hal_srng_src_get_next_reaped(struct ath11k_base *ab,
 785					 struct hal_srng *srng)
 786{
 787	u32 *desc;
 788
 789	lockdep_assert_held(&srng->lock);
 790
 791	if (srng->u.src_ring.hp == srng->u.src_ring.reap_hp)
 792		return NULL;
 793
 794	desc = srng->ring_base_vaddr + srng->u.src_ring.hp;
 795	srng->u.src_ring.hp = (srng->u.src_ring.hp + srng->entry_size) %
 796			      srng->ring_size;
 797
 798	return desc;
 799}
 800
 801u32 *ath11k_hal_srng_src_next_peek(struct ath11k_base *ab, struct hal_srng *srng)
 802{
 803	u32 next_hp;
 804
 805	lockdep_assert_held(&srng->lock);
 806
 807	next_hp = (srng->u.src_ring.hp + srng->entry_size) % srng->ring_size;
 808
 809	if (next_hp != srng->u.src_ring.cached_tp)
 810		return srng->ring_base_vaddr + next_hp;
 811
 812	return NULL;
 813}
 814
 815u32 *ath11k_hal_srng_src_peek(struct ath11k_base *ab, struct hal_srng *srng)
 816{
 817	lockdep_assert_held(&srng->lock);
 818
 819	if (((srng->u.src_ring.hp + srng->entry_size) % srng->ring_size) ==
 820	    srng->u.src_ring.cached_tp)
 821		return NULL;
 822
 823	return srng->ring_base_vaddr + srng->u.src_ring.hp;
 824}
 825
 826void ath11k_hal_srng_access_begin(struct ath11k_base *ab, struct hal_srng *srng)
 827{
 828	u32 hp;
 829
 830	lockdep_assert_held(&srng->lock);
 831
 832	if (srng->ring_dir == HAL_SRNG_DIR_SRC) {
 833		srng->u.src_ring.cached_tp =
 834			*(volatile u32 *)srng->u.src_ring.tp_addr;
 835	} else {
 836		hp = READ_ONCE(*srng->u.dst_ring.hp_addr);
 837
 838		if (hp != srng->u.dst_ring.cached_hp) {
 839			srng->u.dst_ring.cached_hp = hp;
 840			/* Make sure descriptor is read after the head
 841			 * pointer.
 842			 */
 843			dma_rmb();
 844		}
 845
 846		/* Try to prefetch the next descriptor in the ring */
 847		if (srng->flags & HAL_SRNG_FLAGS_CACHED)
 848			ath11k_hal_srng_prefetch_desc(ab, srng);
 849	}
 850}
 851
 852/* Update cached ring head/tail pointers to HW. ath11k_hal_srng_access_begin()
 853 * should have been called before this.
 854 */
 855void ath11k_hal_srng_access_end(struct ath11k_base *ab, struct hal_srng *srng)
 856{
 857	lockdep_assert_held(&srng->lock);
 858
 859	if (srng->flags & HAL_SRNG_FLAGS_LMAC_RING) {
 860		/* For LMAC rings, ring pointer updates are done through FW and
 861		 * hence written to a shared memory location that is read by FW
 862		 */
 863		if (srng->ring_dir == HAL_SRNG_DIR_SRC) {
 864			srng->u.src_ring.last_tp =
 865				*(volatile u32 *)srng->u.src_ring.tp_addr;
 866			/* Make sure descriptor is written before updating the
 867			 * head pointer.
 868			 */
 869			dma_wmb();
 870			WRITE_ONCE(*srng->u.src_ring.hp_addr, srng->u.src_ring.hp);
 871		} else {
 872			srng->u.dst_ring.last_hp = *srng->u.dst_ring.hp_addr;
 873			/* Make sure descriptor is read before updating the
 874			 * tail pointer.
 875			 */
 876			dma_mb();
 877			WRITE_ONCE(*srng->u.dst_ring.tp_addr, srng->u.dst_ring.tp);
 878		}
 879	} else {
 880		if (srng->ring_dir == HAL_SRNG_DIR_SRC) {
 881			srng->u.src_ring.last_tp =
 882				*(volatile u32 *)srng->u.src_ring.tp_addr;
 883			/* Assume implementation use an MMIO write accessor
 884			 * which has the required wmb() so that the descriptor
 885			 * is written before the updating the head pointer.
 886			 */
 887			ath11k_hif_write32(ab,
 888					   (unsigned long)srng->u.src_ring.hp_addr -
 889					   (unsigned long)ab->mem,
 890					   srng->u.src_ring.hp);
 891		} else {
 892			srng->u.dst_ring.last_hp = *srng->u.dst_ring.hp_addr;
 893			/* Make sure descriptor is read before updating the
 894			 * tail pointer.
 895			 */
 896			mb();
 897			ath11k_hif_write32(ab,
 898					   (unsigned long)srng->u.dst_ring.tp_addr -
 899					   (unsigned long)ab->mem,
 900					   srng->u.dst_ring.tp);
 901		}
 902	}
 903
 904	srng->timestamp = jiffies;
 905}
 906
 907void ath11k_hal_setup_link_idle_list(struct ath11k_base *ab,
 908				     struct hal_wbm_idle_scatter_list *sbuf,
 909				     u32 nsbufs, u32 tot_link_desc,
 910				     u32 end_offset)
 911{
 912	struct ath11k_buffer_addr *link_addr;
 913	int i;
 914	u32 reg_scatter_buf_sz = HAL_WBM_IDLE_SCATTER_BUF_SIZE / 64;
 915
 916	link_addr = (void *)sbuf[0].vaddr + HAL_WBM_IDLE_SCATTER_BUF_SIZE;
 917
 918	for (i = 1; i < nsbufs; i++) {
 919		link_addr->info0 = sbuf[i].paddr & HAL_ADDR_LSB_REG_MASK;
 920		link_addr->info1 = FIELD_PREP(
 921				HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32,
 922				(u64)sbuf[i].paddr >> HAL_ADDR_MSB_REG_SHIFT) |
 923				FIELD_PREP(
 924				HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_MATCH_TAG,
 925				BASE_ADDR_MATCH_TAG_VAL);
 926
 927		link_addr = (void *)sbuf[i].vaddr +
 928			     HAL_WBM_IDLE_SCATTER_BUF_SIZE;
 929	}
 930
 931	ath11k_hif_write32(ab,
 932			   HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_R0_IDLE_LIST_CONTROL_ADDR,
 933			   FIELD_PREP(HAL_WBM_SCATTER_BUFFER_SIZE, reg_scatter_buf_sz) |
 934			   FIELD_PREP(HAL_WBM_LINK_DESC_IDLE_LIST_MODE, 0x1));
 935	ath11k_hif_write32(ab,
 936			   HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_R0_IDLE_LIST_SIZE_ADDR,
 937			   FIELD_PREP(HAL_WBM_SCATTER_RING_SIZE_OF_IDLE_LINK_DESC_LIST,
 938				      reg_scatter_buf_sz * nsbufs));
 939	ath11k_hif_write32(ab,
 940			   HAL_SEQ_WCSS_UMAC_WBM_REG +
 941			   HAL_WBM_SCATTERED_RING_BASE_LSB,
 942			   FIELD_PREP(BUFFER_ADDR_INFO0_ADDR,
 943				      sbuf[0].paddr & HAL_ADDR_LSB_REG_MASK));
 944	ath11k_hif_write32(ab,
 945			   HAL_SEQ_WCSS_UMAC_WBM_REG +
 946			   HAL_WBM_SCATTERED_RING_BASE_MSB,
 947			   FIELD_PREP(
 948				HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32,
 949				(u64)sbuf[0].paddr >> HAL_ADDR_MSB_REG_SHIFT) |
 950				FIELD_PREP(
 951				HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_MATCH_TAG,
 952				BASE_ADDR_MATCH_TAG_VAL));
 953
 954	/* Setup head and tail pointers for the idle list */
 955	ath11k_hif_write32(ab,
 956			   HAL_SEQ_WCSS_UMAC_WBM_REG +
 957			   HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX0,
 958			   FIELD_PREP(BUFFER_ADDR_INFO0_ADDR,
 959				      sbuf[nsbufs - 1].paddr));
 960	ath11k_hif_write32(ab,
 961			   HAL_SEQ_WCSS_UMAC_WBM_REG +
 962			   HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX1,
 963			   FIELD_PREP(
 964				HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32,
 965				((u64)sbuf[nsbufs - 1].paddr >>
 966				 HAL_ADDR_MSB_REG_SHIFT)) |
 967			   FIELD_PREP(HAL_WBM_SCATTERED_DESC_HEAD_P_OFFSET_IX1,
 968				      (end_offset >> 2)));
 969	ath11k_hif_write32(ab,
 970			   HAL_SEQ_WCSS_UMAC_WBM_REG +
 971			   HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX0,
 972			   FIELD_PREP(BUFFER_ADDR_INFO0_ADDR,
 973				      sbuf[0].paddr));
 974
 975	ath11k_hif_write32(ab,
 976			   HAL_SEQ_WCSS_UMAC_WBM_REG +
 977			   HAL_WBM_SCATTERED_DESC_PTR_TAIL_INFO_IX0,
 978			   FIELD_PREP(BUFFER_ADDR_INFO0_ADDR,
 979				      sbuf[0].paddr));
 980	ath11k_hif_write32(ab,
 981			   HAL_SEQ_WCSS_UMAC_WBM_REG +
 982			   HAL_WBM_SCATTERED_DESC_PTR_TAIL_INFO_IX1,
 983			   FIELD_PREP(
 984				HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32,
 985				((u64)sbuf[0].paddr >> HAL_ADDR_MSB_REG_SHIFT)) |
 986			   FIELD_PREP(HAL_WBM_SCATTERED_DESC_TAIL_P_OFFSET_IX1,
 987				      0));
 988	ath11k_hif_write32(ab,
 989			   HAL_SEQ_WCSS_UMAC_WBM_REG +
 990			   HAL_WBM_SCATTERED_DESC_PTR_HP_ADDR,
 991			   2 * tot_link_desc);
 992
 993	/* Enable the SRNG */
 994	ath11k_hif_write32(ab,
 995			   HAL_SEQ_WCSS_UMAC_WBM_REG +
 996			   HAL_WBM_IDLE_LINK_RING_MISC_ADDR(ab), 0x40);
 997}
 998
 999int ath11k_hal_srng_setup(struct ath11k_base *ab, enum hal_ring_type type,
1000			  int ring_num, int mac_id,
1001			  struct hal_srng_params *params)
1002{
1003	struct ath11k_hal *hal = &ab->hal;
1004	struct hal_srng_config *srng_config = &ab->hal.srng_config[type];
1005	struct hal_srng *srng;
1006	int ring_id;
1007	u32 lmac_idx;
1008	int i;
1009	u32 reg_base;
1010
1011	ring_id = ath11k_hal_srng_get_ring_id(ab, type, ring_num, mac_id);
1012	if (ring_id < 0)
1013		return ring_id;
1014
1015	srng = &hal->srng_list[ring_id];
1016
1017	srng->ring_id = ring_id;
1018	srng->ring_dir = srng_config->ring_dir;
1019	srng->ring_base_paddr = params->ring_base_paddr;
1020	srng->ring_base_vaddr = params->ring_base_vaddr;
1021	srng->entry_size = srng_config->entry_size;
1022	srng->num_entries = params->num_entries;
1023	srng->ring_size = srng->entry_size * srng->num_entries;
1024	srng->intr_batch_cntr_thres_entries =
1025				params->intr_batch_cntr_thres_entries;
1026	srng->intr_timer_thres_us = params->intr_timer_thres_us;
1027	srng->flags = params->flags;
1028	srng->msi_addr = params->msi_addr;
1029	srng->msi_data = params->msi_data;
1030	srng->initialized = 1;
1031	spin_lock_init(&srng->lock);
1032	lockdep_set_class(&srng->lock, hal->srng_key + ring_id);
1033
1034	for (i = 0; i < HAL_SRNG_NUM_REG_GRP; i++) {
1035		srng->hwreg_base[i] = srng_config->reg_start[i] +
1036				      (ring_num * srng_config->reg_size[i]);
1037	}
1038
1039	memset(srng->ring_base_vaddr, 0,
1040	       (srng->entry_size * srng->num_entries) << 2);
1041
1042	/* TODO: Add comments on these swap configurations */
1043	if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
1044		srng->flags |= HAL_SRNG_FLAGS_MSI_SWAP | HAL_SRNG_FLAGS_DATA_TLV_SWAP |
1045			       HAL_SRNG_FLAGS_RING_PTR_SWAP;
1046
1047	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R2];
1048
1049	if (srng->ring_dir == HAL_SRNG_DIR_SRC) {
1050		srng->u.src_ring.hp = 0;
1051		srng->u.src_ring.cached_tp = 0;
1052		srng->u.src_ring.reap_hp = srng->ring_size - srng->entry_size;
1053		srng->u.src_ring.tp_addr = (void *)(hal->rdp.vaddr + ring_id);
1054		srng->u.src_ring.low_threshold = params->low_threshold *
1055						 srng->entry_size;
1056		if (srng_config->lmac_ring) {
1057			lmac_idx = ring_id - HAL_SRNG_RING_ID_LMAC1_ID_START;
1058			srng->u.src_ring.hp_addr = (void *)(hal->wrp.vaddr +
1059						   lmac_idx);
1060			srng->flags |= HAL_SRNG_FLAGS_LMAC_RING;
1061		} else {
1062			if (!ab->hw_params.supports_shadow_regs)
1063				srng->u.src_ring.hp_addr =
1064				(u32 *)((unsigned long)ab->mem + reg_base);
1065			else
1066				ath11k_dbg(ab, ATH11K_DBG_HAL,
1067					   "type %d ring_num %d reg_base 0x%x shadow 0x%lx\n",
1068					   type, ring_num,
1069					   reg_base,
1070					   (unsigned long)srng->u.src_ring.hp_addr -
1071					   (unsigned long)ab->mem);
1072		}
1073	} else {
1074		/* During initialization loop count in all the descriptors
1075		 * will be set to zero, and HW will set it to 1 on completing
1076		 * descriptor update in first loop, and increments it by 1 on
1077		 * subsequent loops (loop count wraps around after reaching
1078		 * 0xffff). The 'loop_cnt' in SW ring state is the expected
1079		 * loop count in descriptors updated by HW (to be processed
1080		 * by SW).
1081		 */
1082		srng->u.dst_ring.loop_cnt = 1;
1083		srng->u.dst_ring.tp = 0;
1084		srng->u.dst_ring.cached_hp = 0;
1085		srng->u.dst_ring.hp_addr = (void *)(hal->rdp.vaddr + ring_id);
1086		if (srng_config->lmac_ring) {
1087			/* For LMAC rings, tail pointer updates will be done
1088			 * through FW by writing to a shared memory location
1089			 */
1090			lmac_idx = ring_id - HAL_SRNG_RING_ID_LMAC1_ID_START;
1091			srng->u.dst_ring.tp_addr = (void *)(hal->wrp.vaddr +
1092						   lmac_idx);
1093			srng->flags |= HAL_SRNG_FLAGS_LMAC_RING;
1094		} else {
1095			if (!ab->hw_params.supports_shadow_regs)
1096				srng->u.dst_ring.tp_addr =
1097				(u32 *)((unsigned long)ab->mem + reg_base +
1098					(HAL_REO1_RING_TP(ab) - HAL_REO1_RING_HP(ab)));
1099			else
1100				ath11k_dbg(ab, ATH11K_DBG_HAL,
1101					   "type %d ring_num %d target_reg 0x%x shadow 0x%lx\n",
1102					   type, ring_num,
1103					   reg_base + (HAL_REO1_RING_TP(ab) -
1104						       HAL_REO1_RING_HP(ab)),
1105					   (unsigned long)srng->u.dst_ring.tp_addr -
1106					   (unsigned long)ab->mem);
1107		}
1108	}
1109
1110	if (srng_config->lmac_ring)
1111		return ring_id;
1112
1113	ath11k_hal_srng_hw_init(ab, srng);
1114
1115	if (type == HAL_CE_DST) {
1116		srng->u.dst_ring.max_buffer_length = params->max_buffer_len;
1117		ath11k_hal_ce_dst_setup(ab, srng, ring_num);
1118	}
1119
1120	return ring_id;
1121}
1122
1123static void ath11k_hal_srng_update_hp_tp_addr(struct ath11k_base *ab,
1124					      int shadow_cfg_idx,
1125					  enum hal_ring_type ring_type,
1126					  int ring_num)
1127{
1128	struct hal_srng *srng;
1129	struct ath11k_hal *hal = &ab->hal;
1130	int ring_id;
1131	struct hal_srng_config *srng_config = &hal->srng_config[ring_type];
1132
1133	ring_id = ath11k_hal_srng_get_ring_id(ab, ring_type, ring_num, 0);
1134	if (ring_id < 0)
1135		return;
1136
1137	srng = &hal->srng_list[ring_id];
1138
1139	if (srng_config->ring_dir == HAL_SRNG_DIR_DST)
1140		srng->u.dst_ring.tp_addr = (u32 *)(HAL_SHADOW_REG(ab, shadow_cfg_idx) +
1141						   (unsigned long)ab->mem);
1142	else
1143		srng->u.src_ring.hp_addr = (u32 *)(HAL_SHADOW_REG(ab, shadow_cfg_idx) +
1144						   (unsigned long)ab->mem);
1145}
1146
1147int ath11k_hal_srng_update_shadow_config(struct ath11k_base *ab,
1148					 enum hal_ring_type ring_type,
1149					 int ring_num)
1150{
1151	struct ath11k_hal *hal = &ab->hal;
1152	struct hal_srng_config *srng_config = &hal->srng_config[ring_type];
1153	int shadow_cfg_idx = hal->num_shadow_reg_configured;
1154	u32 target_reg;
1155
1156	if (shadow_cfg_idx >= HAL_SHADOW_NUM_REGS)
1157		return -EINVAL;
1158
1159	hal->num_shadow_reg_configured++;
1160
1161	target_reg = srng_config->reg_start[HAL_HP_OFFSET_IN_REG_START];
1162	target_reg += srng_config->reg_size[HAL_HP_OFFSET_IN_REG_START] *
1163		ring_num;
1164
1165	/* For destination ring, shadow the TP */
1166	if (srng_config->ring_dir == HAL_SRNG_DIR_DST)
1167		target_reg += HAL_OFFSET_FROM_HP_TO_TP;
1168
1169	hal->shadow_reg_addr[shadow_cfg_idx] = target_reg;
1170
1171	/* update hp/tp addr to hal structure*/
1172	ath11k_hal_srng_update_hp_tp_addr(ab, shadow_cfg_idx, ring_type,
1173					  ring_num);
1174
1175	ath11k_dbg(ab, ATH11K_DBG_HAL,
1176		   "update shadow config target_reg %x shadow reg 0x%x shadow_idx 0x%x ring_type %d ring num %d",
1177		  target_reg,
1178		  HAL_SHADOW_REG(ab, shadow_cfg_idx),
1179		  shadow_cfg_idx,
1180		  ring_type, ring_num);
1181
1182	return 0;
1183}
1184
1185void ath11k_hal_srng_shadow_config(struct ath11k_base *ab)
1186{
1187	struct ath11k_hal *hal = &ab->hal;
1188	int ring_type, ring_num;
1189
1190	/* update all the non-CE srngs. */
1191	for (ring_type = 0; ring_type < HAL_MAX_RING_TYPES; ring_type++) {
1192		struct hal_srng_config *srng_config = &hal->srng_config[ring_type];
1193
1194		if (ring_type == HAL_CE_SRC ||
1195		    ring_type == HAL_CE_DST ||
1196			ring_type == HAL_CE_DST_STATUS)
1197			continue;
1198
1199		if (srng_config->lmac_ring)
1200			continue;
1201
1202		for (ring_num = 0; ring_num < srng_config->max_rings; ring_num++)
1203			ath11k_hal_srng_update_shadow_config(ab, ring_type, ring_num);
1204	}
1205}
1206
1207void ath11k_hal_srng_get_shadow_config(struct ath11k_base *ab,
1208				       u32 **cfg, u32 *len)
1209{
1210	struct ath11k_hal *hal = &ab->hal;
1211
1212	*len = hal->num_shadow_reg_configured;
1213	*cfg = hal->shadow_reg_addr;
1214}
1215
1216void ath11k_hal_srng_shadow_update_hp_tp(struct ath11k_base *ab,
1217					 struct hal_srng *srng)
1218{
1219	lockdep_assert_held(&srng->lock);
1220
1221	/* check whether the ring is empty. Update the shadow
1222	 * HP only when then ring isn't empty.
1223	 */
1224	if (srng->ring_dir == HAL_SRNG_DIR_SRC &&
1225	    *srng->u.src_ring.tp_addr != srng->u.src_ring.hp)
1226		ath11k_hal_srng_access_end(ab, srng);
1227}
1228
1229static int ath11k_hal_srng_create_config(struct ath11k_base *ab)
1230{
1231	struct ath11k_hal *hal = &ab->hal;
1232	struct hal_srng_config *s;
1233
1234	hal->srng_config = kmemdup(hw_srng_config_template,
1235				   sizeof(hw_srng_config_template),
1236				   GFP_KERNEL);
1237	if (!hal->srng_config)
1238		return -ENOMEM;
1239
1240	s = &hal->srng_config[HAL_REO_DST];
1241	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_RING_BASE_LSB(ab);
1242	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_RING_HP(ab);
1243	s->reg_size[0] = HAL_REO2_RING_BASE_LSB(ab) - HAL_REO1_RING_BASE_LSB(ab);
1244	s->reg_size[1] = HAL_REO2_RING_HP(ab) - HAL_REO1_RING_HP(ab);
1245
1246	s = &hal->srng_config[HAL_REO_EXCEPTION];
1247	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_TCL_RING_BASE_LSB(ab);
1248	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_TCL_RING_HP(ab);
1249
1250	s = &hal->srng_config[HAL_REO_REINJECT];
1251	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_SW2REO_RING_BASE_LSB(ab);
1252	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_SW2REO_RING_HP(ab);
1253
1254	s = &hal->srng_config[HAL_REO_CMD];
1255	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_CMD_RING_BASE_LSB(ab);
1256	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_CMD_HP(ab);
1257
1258	s = &hal->srng_config[HAL_REO_STATUS];
1259	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_STATUS_RING_BASE_LSB(ab);
1260	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_STATUS_HP(ab);
1261
1262	s = &hal->srng_config[HAL_TCL_DATA];
1263	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL1_RING_BASE_LSB(ab);
1264	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL1_RING_HP;
1265	s->reg_size[0] = HAL_TCL2_RING_BASE_LSB(ab) - HAL_TCL1_RING_BASE_LSB(ab);
1266	s->reg_size[1] = HAL_TCL2_RING_HP - HAL_TCL1_RING_HP;
1267
1268	s = &hal->srng_config[HAL_TCL_CMD];
1269	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_RING_BASE_LSB(ab);
1270	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_RING_HP;
1271
1272	s = &hal->srng_config[HAL_TCL_STATUS];
1273	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_STATUS_RING_BASE_LSB(ab);
1274	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_STATUS_RING_HP;
1275
1276	s = &hal->srng_config[HAL_CE_SRC];
1277	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab) + HAL_CE_DST_RING_BASE_LSB +
1278		ATH11K_CE_OFFSET(ab);
1279	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab) + HAL_CE_DST_RING_HP +
1280		ATH11K_CE_OFFSET(ab);
1281	s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(ab) -
1282		HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab);
1283	s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(ab) -
1284		HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab);
1285
1286	s = &hal->srng_config[HAL_CE_DST];
1287	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + HAL_CE_DST_RING_BASE_LSB +
1288		ATH11K_CE_OFFSET(ab);
1289	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + HAL_CE_DST_RING_HP +
1290		ATH11K_CE_OFFSET(ab);
1291	s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) -
1292		HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab);
1293	s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) -
1294		HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab);
1295
1296	s = &hal->srng_config[HAL_CE_DST_STATUS];
1297	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) +
1298		HAL_CE_DST_STATUS_RING_BASE_LSB + ATH11K_CE_OFFSET(ab);
1299	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + HAL_CE_DST_STATUS_RING_HP +
1300		ATH11K_CE_OFFSET(ab);
1301	s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) -
1302		HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab);
1303	s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) -
1304		HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab);
1305
1306	s = &hal->srng_config[HAL_WBM_IDLE_LINK];
1307	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_IDLE_LINK_RING_BASE_LSB(ab);
1308	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_IDLE_LINK_RING_HP;
1309
1310	s = &hal->srng_config[HAL_SW2WBM_RELEASE];
1311	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_RELEASE_RING_BASE_LSB(ab);
1312	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_RELEASE_RING_HP;
1313
1314	s = &hal->srng_config[HAL_WBM2SW_RELEASE];
1315	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM0_RELEASE_RING_BASE_LSB(ab);
1316	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM0_RELEASE_RING_HP;
1317	s->reg_size[0] = HAL_WBM1_RELEASE_RING_BASE_LSB(ab) -
1318		HAL_WBM0_RELEASE_RING_BASE_LSB(ab);
1319	s->reg_size[1] = HAL_WBM1_RELEASE_RING_HP - HAL_WBM0_RELEASE_RING_HP;
1320
1321	return 0;
1322}
1323
1324static void ath11k_hal_register_srng_key(struct ath11k_base *ab)
1325{
1326	struct ath11k_hal *hal = &ab->hal;
1327	u32 ring_id;
1328
1329	for (ring_id = 0; ring_id < HAL_SRNG_RING_ID_MAX; ring_id++)
1330		lockdep_register_key(hal->srng_key + ring_id);
1331}
1332
1333static void ath11k_hal_unregister_srng_key(struct ath11k_base *ab)
1334{
1335	struct ath11k_hal *hal = &ab->hal;
1336	u32 ring_id;
1337
1338	for (ring_id = 0; ring_id < HAL_SRNG_RING_ID_MAX; ring_id++)
1339		lockdep_unregister_key(hal->srng_key + ring_id);
1340}
1341
1342int ath11k_hal_srng_init(struct ath11k_base *ab)
1343{
1344	struct ath11k_hal *hal = &ab->hal;
1345	int ret;
1346
1347	memset(hal, 0, sizeof(*hal));
1348
1349	ret = ath11k_hal_srng_create_config(ab);
1350	if (ret)
1351		goto err_hal;
1352
1353	ret = ath11k_hal_alloc_cont_rdp(ab);
1354	if (ret)
1355		goto err_hal;
1356
1357	ret = ath11k_hal_alloc_cont_wrp(ab);
1358	if (ret)
1359		goto err_free_cont_rdp;
1360
1361	ath11k_hal_register_srng_key(ab);
1362
1363	return 0;
1364
1365err_free_cont_rdp:
1366	ath11k_hal_free_cont_rdp(ab);
1367
1368err_hal:
1369	return ret;
1370}
1371EXPORT_SYMBOL(ath11k_hal_srng_init);
1372
1373void ath11k_hal_srng_deinit(struct ath11k_base *ab)
1374{
1375	struct ath11k_hal *hal = &ab->hal;
1376	int i;
1377
1378	for (i = 0; i < HAL_SRNG_RING_ID_MAX; i++)
1379		ab->hal.srng_list[i].initialized = 0;
1380
1381	ath11k_hal_unregister_srng_key(ab);
1382	ath11k_hal_free_cont_rdp(ab);
1383	ath11k_hal_free_cont_wrp(ab);
1384	kfree(hal->srng_config);
1385	hal->srng_config = NULL;
1386}
1387EXPORT_SYMBOL(ath11k_hal_srng_deinit);
1388
1389void ath11k_hal_srng_clear(struct ath11k_base *ab)
1390{
1391	/* No need to memset rdp and wrp memory since each individual
1392	 * segment would get cleared in ath11k_hal_srng_src_hw_init()
1393	 * and ath11k_hal_srng_dst_hw_init().
1394	 */
1395	memset(ab->hal.srng_list, 0,
1396	       sizeof(ab->hal.srng_list));
1397	memset(ab->hal.shadow_reg_addr, 0,
1398	       sizeof(ab->hal.shadow_reg_addr));
1399	ab->hal.avail_blk_resource = 0;
1400	ab->hal.current_blk_index = 0;
1401	ab->hal.num_shadow_reg_configured = 0;
1402}
1403EXPORT_SYMBOL(ath11k_hal_srng_clear);
1404
1405void ath11k_hal_dump_srng_stats(struct ath11k_base *ab)
1406{
1407	struct hal_srng *srng;
1408	struct ath11k_ext_irq_grp *irq_grp;
1409	struct ath11k_ce_pipe *ce_pipe;
1410	int i;
1411
1412	ath11k_err(ab, "Last interrupt received for each CE:\n");
1413	for (i = 0; i < ab->hw_params.ce_count; i++) {
1414		ce_pipe = &ab->ce.ce_pipe[i];
1415
1416		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
1417			continue;
1418
1419		ath11k_err(ab, "CE_id %d pipe_num %d %ums before\n",
1420			   i, ce_pipe->pipe_num,
1421			   jiffies_to_msecs(jiffies - ce_pipe->timestamp));
1422	}
1423
1424	ath11k_err(ab, "\nLast interrupt received for each group:\n");
1425	for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) {
1426		irq_grp = &ab->ext_irq_grp[i];
1427		ath11k_err(ab, "group_id %d %ums before\n",
1428			   irq_grp->grp_id,
1429			   jiffies_to_msecs(jiffies - irq_grp->timestamp));
1430	}
1431
1432	for (i = 0; i < HAL_SRNG_RING_ID_MAX; i++) {
1433		srng = &ab->hal.srng_list[i];
1434
1435		if (!srng->initialized)
1436			continue;
1437
1438		if (srng->ring_dir == HAL_SRNG_DIR_SRC)
1439			ath11k_err(ab,
1440				   "src srng id %u hp %u, reap_hp %u, cur tp %u, cached tp %u last tp %u napi processed before %ums\n",
1441				   srng->ring_id, srng->u.src_ring.hp,
1442				   srng->u.src_ring.reap_hp,
1443				   *srng->u.src_ring.tp_addr, srng->u.src_ring.cached_tp,
1444				   srng->u.src_ring.last_tp,
1445				   jiffies_to_msecs(jiffies - srng->timestamp));
1446		else if (srng->ring_dir == HAL_SRNG_DIR_DST)
1447			ath11k_err(ab,
1448				   "dst srng id %u tp %u, cur hp %u, cached hp %u last hp %u napi processed before %ums\n",
1449				   srng->ring_id, srng->u.dst_ring.tp,
1450				   *srng->u.dst_ring.hp_addr,
1451				   srng->u.dst_ring.cached_hp,
1452				   srng->u.dst_ring.last_hp,
1453				   jiffies_to_msecs(jiffies - srng->timestamp));
1454	}
1455}