gve_adminq.c

   1// SPDX-License-Identifier: (GPL-2.0 OR MIT)
   2/* Google virtual Ethernet (gve) driver
   3 *
   4 * Copyright (C) 2015-2021 Google, Inc.
   5 */
   6
   7#include <linux/etherdevice.h>
   8#include <linux/pci.h>
   9#include "gve.h"
  10#include "gve_adminq.h"
  11#include "gve_register.h"
  12
  13#define GVE_MAX_ADMINQ_RELEASE_CHECK	500
  14#define GVE_ADMINQ_SLEEP_LEN		20
  15#define GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK	100
  16
  17#define GVE_DEVICE_OPTION_ERROR_FMT "%s option error:\n" \
  18"Expected: length=%d, feature_mask=%x.\n" \
  19"Actual: length=%d, feature_mask=%x.\n"
  20
  21#define GVE_DEVICE_OPTION_TOO_BIG_FMT "Length of %s option larger than expected. Possible older version of guest driver.\n"
  22
  23static
  24struct gve_device_option *gve_get_next_option(struct gve_device_descriptor *descriptor,
  25					      struct gve_device_option *option)
  26{
  27	void *option_end, *descriptor_end;
  28
  29	option_end = (void *)(option + 1) + be16_to_cpu(option->option_length);
  30	descriptor_end = (void *)descriptor + be16_to_cpu(descriptor->total_length);
  31
  32	return option_end > descriptor_end ? NULL : (struct gve_device_option *)option_end;
  33}
  34
  35#define GVE_DEVICE_OPTION_NO_MIN_RING_SIZE	8
  36
  37static
  38void gve_parse_device_option(struct gve_priv *priv,
  39			     struct gve_device_descriptor *device_descriptor,
  40			     struct gve_device_option *option,
  41			     struct gve_device_option_gqi_rda **dev_op_gqi_rda,
  42			     struct gve_device_option_gqi_qpl **dev_op_gqi_qpl,
  43			     struct gve_device_option_dqo_rda **dev_op_dqo_rda,
  44			     struct gve_device_option_jumbo_frames **dev_op_jumbo_frames,
  45			     struct gve_device_option_dqo_qpl **dev_op_dqo_qpl,
  46			     struct gve_device_option_buffer_sizes **dev_op_buffer_sizes,
  47			     struct gve_device_option_flow_steering **dev_op_flow_steering,
  48			     struct gve_device_option_rss_config **dev_op_rss_config,
  49			     struct gve_device_option_nic_timestamp **dev_op_nic_timestamp,
  50			     struct gve_device_option_modify_ring **dev_op_modify_ring)
  51{
  52	u32 req_feat_mask = be32_to_cpu(option->required_features_mask);
  53	u16 option_length = be16_to_cpu(option->option_length);
  54	u16 option_id = be16_to_cpu(option->option_id);
  55
  56	/* If the length or feature mask doesn't match, continue without
  57	 * enabling the feature.
  58	 */
  59	switch (option_id) {
  60	case GVE_DEV_OPT_ID_GQI_RAW_ADDRESSING:
  61		if (option_length != GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING ||
  62		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING) {
  63			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
  64				 "Raw Addressing",
  65				 GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING,
  66				 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING,
  67				 option_length, req_feat_mask);
  68			break;
  69		}
  70
  71		dev_info(&priv->pdev->dev,
  72			 "Gqi raw addressing device option enabled.\n");
  73		priv->queue_format = GVE_GQI_RDA_FORMAT;
  74		break;
  75	case GVE_DEV_OPT_ID_GQI_RDA:
  76		if (option_length < sizeof(**dev_op_gqi_rda) ||
  77		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA) {
  78			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
  79				 "GQI RDA", (int)sizeof(**dev_op_gqi_rda),
  80				 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA,
  81				 option_length, req_feat_mask);
  82			break;
  83		}
  84
  85		if (option_length > sizeof(**dev_op_gqi_rda)) {
  86			dev_warn(&priv->pdev->dev,
  87				 GVE_DEVICE_OPTION_TOO_BIG_FMT, "GQI RDA");
  88		}
  89		*dev_op_gqi_rda = (void *)(option + 1);
  90		break;
  91	case GVE_DEV_OPT_ID_GQI_QPL:
  92		if (option_length < sizeof(**dev_op_gqi_qpl) ||
  93		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL) {
  94			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
  95				 "GQI QPL", (int)sizeof(**dev_op_gqi_qpl),
  96				 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL,
  97				 option_length, req_feat_mask);
  98			break;
  99		}
 100
 101		if (option_length > sizeof(**dev_op_gqi_qpl)) {
 102			dev_warn(&priv->pdev->dev,
 103				 GVE_DEVICE_OPTION_TOO_BIG_FMT, "GQI QPL");
 104		}
 105		*dev_op_gqi_qpl = (void *)(option + 1);
 106		break;
 107	case GVE_DEV_OPT_ID_DQO_RDA:
 108		if (option_length < sizeof(**dev_op_dqo_rda) ||
 109		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA) {
 110			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
 111				 "DQO RDA", (int)sizeof(**dev_op_dqo_rda),
 112				 GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA,
 113				 option_length, req_feat_mask);
 114			break;
 115		}
 116
 117		if (option_length > sizeof(**dev_op_dqo_rda)) {
 118			dev_warn(&priv->pdev->dev,
 119				 GVE_DEVICE_OPTION_TOO_BIG_FMT, "DQO RDA");
 120		}
 121		*dev_op_dqo_rda = (void *)(option + 1);
 122		break;
 123	case GVE_DEV_OPT_ID_DQO_QPL:
 124		if (option_length < sizeof(**dev_op_dqo_qpl) ||
 125		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_DQO_QPL) {
 126			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
 127				 "DQO QPL", (int)sizeof(**dev_op_dqo_qpl),
 128				 GVE_DEV_OPT_REQ_FEAT_MASK_DQO_QPL,
 129				 option_length, req_feat_mask);
 130			break;
 131		}
 132
 133		if (option_length > sizeof(**dev_op_dqo_qpl)) {
 134			dev_warn(&priv->pdev->dev,
 135				 GVE_DEVICE_OPTION_TOO_BIG_FMT, "DQO QPL");
 136		}
 137		*dev_op_dqo_qpl = (void *)(option + 1);
 138		break;
 139	case GVE_DEV_OPT_ID_JUMBO_FRAMES:
 140		if (option_length < sizeof(**dev_op_jumbo_frames) ||
 141		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES) {
 142			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
 143				 "Jumbo Frames",
 144				 (int)sizeof(**dev_op_jumbo_frames),
 145				 GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES,
 146				 option_length, req_feat_mask);
 147			break;
 148		}
 149
 150		if (option_length > sizeof(**dev_op_jumbo_frames)) {
 151			dev_warn(&priv->pdev->dev,
 152				 GVE_DEVICE_OPTION_TOO_BIG_FMT,
 153				 "Jumbo Frames");
 154		}
 155		*dev_op_jumbo_frames = (void *)(option + 1);
 156		break;
 157	case GVE_DEV_OPT_ID_BUFFER_SIZES:
 158		if (option_length < sizeof(**dev_op_buffer_sizes) ||
 159		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_BUFFER_SIZES) {
 160			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
 161				 "Buffer Sizes",
 162				 (int)sizeof(**dev_op_buffer_sizes),
 163				 GVE_DEV_OPT_REQ_FEAT_MASK_BUFFER_SIZES,
 164				 option_length, req_feat_mask);
 165			break;
 166		}
 167
 168		if (option_length > sizeof(**dev_op_buffer_sizes))
 169			dev_warn(&priv->pdev->dev,
 170				 GVE_DEVICE_OPTION_TOO_BIG_FMT,
 171				 "Buffer Sizes");
 172		*dev_op_buffer_sizes = (void *)(option + 1);
 173		break;
 174	case GVE_DEV_OPT_ID_MODIFY_RING:
 175		if (option_length < GVE_DEVICE_OPTION_NO_MIN_RING_SIZE ||
 176		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_MODIFY_RING) {
 177			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
 178				 "Modify Ring", (int)sizeof(**dev_op_modify_ring),
 179				 GVE_DEV_OPT_REQ_FEAT_MASK_MODIFY_RING,
 180				 option_length, req_feat_mask);
 181			break;
 182		}
 183
 184		if (option_length > sizeof(**dev_op_modify_ring)) {
 185			dev_warn(&priv->pdev->dev,
 186				 GVE_DEVICE_OPTION_TOO_BIG_FMT, "Modify Ring");
 187		}
 188
 189		*dev_op_modify_ring = (void *)(option + 1);
 190
 191		/* device has not provided min ring size */
 192		if (option_length == GVE_DEVICE_OPTION_NO_MIN_RING_SIZE)
 193			priv->default_min_ring_size = true;
 194		break;
 195	case GVE_DEV_OPT_ID_FLOW_STEERING:
 196		if (option_length < sizeof(**dev_op_flow_steering) ||
 197		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_FLOW_STEERING) {
 198			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
 199				 "Flow Steering",
 200				 (int)sizeof(**dev_op_flow_steering),
 201				 GVE_DEV_OPT_REQ_FEAT_MASK_FLOW_STEERING,
 202				 option_length, req_feat_mask);
 203			break;
 204		}
 205
 206		if (option_length > sizeof(**dev_op_flow_steering))
 207			dev_warn(&priv->pdev->dev,
 208				 GVE_DEVICE_OPTION_TOO_BIG_FMT,
 209				 "Flow Steering");
 210		*dev_op_flow_steering = (void *)(option + 1);
 211		break;
 212	case GVE_DEV_OPT_ID_RSS_CONFIG:
 213		if (option_length < sizeof(**dev_op_rss_config) ||
 214		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_RSS_CONFIG) {
 215			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
 216				 "RSS config",
 217				 (int)sizeof(**dev_op_rss_config),
 218				 GVE_DEV_OPT_REQ_FEAT_MASK_RSS_CONFIG,
 219				 option_length, req_feat_mask);
 220			break;
 221		}
 222
 223		if (option_length > sizeof(**dev_op_rss_config))
 224			dev_warn(&priv->pdev->dev,
 225				 GVE_DEVICE_OPTION_TOO_BIG_FMT,
 226				 "RSS config");
 227		*dev_op_rss_config = (void *)(option + 1);
 228		break;
 229	case GVE_DEV_OPT_ID_NIC_TIMESTAMP:
 230		if (option_length < sizeof(**dev_op_nic_timestamp) ||
 231		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_NIC_TIMESTAMP) {
 232			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
 233				 "Nic Timestamp",
 234				 (int)sizeof(**dev_op_nic_timestamp),
 235				 GVE_DEV_OPT_REQ_FEAT_MASK_NIC_TIMESTAMP,
 236				 option_length, req_feat_mask);
 237			break;
 238		}
 239
 240		if (option_length > sizeof(**dev_op_nic_timestamp))
 241			dev_warn(&priv->pdev->dev,
 242				 GVE_DEVICE_OPTION_TOO_BIG_FMT,
 243				 "Nic Timestamp");
 244		*dev_op_nic_timestamp = (void *)(option + 1);
 245		break;
 246	default:
 247		/* If we don't recognize the option just continue
 248		 * without doing anything.
 249		 */
 250		dev_dbg(&priv->pdev->dev, "Unrecognized device option 0x%hx not enabled.\n",
 251			option_id);
 252	}
 253}
 254
 255/* Process all device options for a given describe device call. */
 256static int
 257gve_process_device_options(struct gve_priv *priv,
 258			   struct gve_device_descriptor *descriptor,
 259			   struct gve_device_option_gqi_rda **dev_op_gqi_rda,
 260			   struct gve_device_option_gqi_qpl **dev_op_gqi_qpl,
 261			   struct gve_device_option_dqo_rda **dev_op_dqo_rda,
 262			   struct gve_device_option_jumbo_frames **dev_op_jumbo_frames,
 263			   struct gve_device_option_dqo_qpl **dev_op_dqo_qpl,
 264			   struct gve_device_option_buffer_sizes **dev_op_buffer_sizes,
 265			   struct gve_device_option_flow_steering **dev_op_flow_steering,
 266			   struct gve_device_option_rss_config **dev_op_rss_config,
 267			   struct gve_device_option_nic_timestamp **dev_op_nic_timestamp,
 268			   struct gve_device_option_modify_ring **dev_op_modify_ring)
 269{
 270	const int num_options = be16_to_cpu(descriptor->num_device_options);
 271	struct gve_device_option *dev_opt;
 272	int i;
 273
 274	/* The options struct directly follows the device descriptor. */
 275	dev_opt = (void *)(descriptor + 1);
 276	for (i = 0; i < num_options; i++) {
 277		struct gve_device_option *next_opt;
 278
 279		next_opt = gve_get_next_option(descriptor, dev_opt);
 280		if (!next_opt) {
 281			dev_err(&priv->dev->dev,
 282				"options exceed device_descriptor's total length.\n");
 283			return -EINVAL;
 284		}
 285
 286		gve_parse_device_option(priv, descriptor, dev_opt,
 287					dev_op_gqi_rda, dev_op_gqi_qpl,
 288					dev_op_dqo_rda, dev_op_jumbo_frames,
 289					dev_op_dqo_qpl, dev_op_buffer_sizes,
 290					dev_op_flow_steering, dev_op_rss_config,
 291					dev_op_nic_timestamp,
 292					dev_op_modify_ring);
 293		dev_opt = next_opt;
 294	}
 295
 296	return 0;
 297}
 298
 299int gve_adminq_alloc(struct device *dev, struct gve_priv *priv)
 300{
 301	priv->adminq_pool = dma_pool_create("adminq_pool", dev,
 302					    GVE_ADMINQ_BUFFER_SIZE, 0, 0);
 303	if (unlikely(!priv->adminq_pool))
 304		return -ENOMEM;
 305	priv->adminq = dma_pool_alloc(priv->adminq_pool, GFP_KERNEL,
 306				      &priv->adminq_bus_addr);
 307	if (unlikely(!priv->adminq)) {
 308		dma_pool_destroy(priv->adminq_pool);
 309		return -ENOMEM;
 310	}
 311
 312	priv->adminq_mask =
 313		(GVE_ADMINQ_BUFFER_SIZE / sizeof(union gve_adminq_command)) - 1;
 314	priv->adminq_prod_cnt = 0;
 315	priv->adminq_cmd_fail = 0;
 316	priv->adminq_timeouts = 0;
 317	priv->adminq_describe_device_cnt = 0;
 318	priv->adminq_cfg_device_resources_cnt = 0;
 319	priv->adminq_register_page_list_cnt = 0;
 320	priv->adminq_unregister_page_list_cnt = 0;
 321	priv->adminq_create_tx_queue_cnt = 0;
 322	priv->adminq_create_rx_queue_cnt = 0;
 323	priv->adminq_destroy_tx_queue_cnt = 0;
 324	priv->adminq_destroy_rx_queue_cnt = 0;
 325	priv->adminq_dcfg_device_resources_cnt = 0;
 326	priv->adminq_set_driver_parameter_cnt = 0;
 327	priv->adminq_report_stats_cnt = 0;
 328	priv->adminq_report_link_speed_cnt = 0;
 329	priv->adminq_report_nic_timestamp_cnt = 0;
 330	priv->adminq_get_ptype_map_cnt = 0;
 331	priv->adminq_query_flow_rules_cnt = 0;
 332	priv->adminq_cfg_flow_rule_cnt = 0;
 333	priv->adminq_cfg_rss_cnt = 0;
 334	priv->adminq_query_rss_cnt = 0;
 335
 336	/* Setup Admin queue with the device */
 337	if (priv->pdev->revision < 0x1) {
 338		iowrite32be(priv->adminq_bus_addr / PAGE_SIZE,
 339			    &priv->reg_bar0->adminq_pfn);
 340	} else {
 341		iowrite16be(GVE_ADMINQ_BUFFER_SIZE,
 342			    &priv->reg_bar0->adminq_length);
 343#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
 344		iowrite32be(priv->adminq_bus_addr >> 32,
 345			    &priv->reg_bar0->adminq_base_address_hi);
 346#endif
 347		iowrite32be(priv->adminq_bus_addr,
 348			    &priv->reg_bar0->adminq_base_address_lo);
 349		iowrite32be(GVE_DRIVER_STATUS_RUN_MASK, &priv->reg_bar0->driver_status);
 350	}
 351	mutex_init(&priv->adminq_lock);
 352	gve_set_admin_queue_ok(priv);
 353	return 0;
 354}
 355
 356void gve_adminq_release(struct gve_priv *priv)
 357{
 358	int i = 0;
 359
 360	/* Tell the device the adminq is leaving */
 361	if (priv->pdev->revision < 0x1) {
 362		iowrite32be(0x0, &priv->reg_bar0->adminq_pfn);
 363		while (ioread32be(&priv->reg_bar0->adminq_pfn)) {
 364			/* If this is reached the device is unrecoverable and still
 365			 * holding memory. Continue looping to avoid memory corruption,
 366			 * but WARN so it is visible what is going on.
 367			 */
 368			if (i == GVE_MAX_ADMINQ_RELEASE_CHECK)
 369				WARN(1, "Unrecoverable platform error!");
 370			i++;
 371			msleep(GVE_ADMINQ_SLEEP_LEN);
 372		}
 373	} else {
 374		iowrite32be(GVE_DRIVER_STATUS_RESET_MASK, &priv->reg_bar0->driver_status);
 375		while (!(ioread32be(&priv->reg_bar0->device_status)
 376				& GVE_DEVICE_STATUS_DEVICE_IS_RESET)) {
 377			if (i == GVE_MAX_ADMINQ_RELEASE_CHECK)
 378				WARN(1, "Unrecoverable platform error!");
 379			i++;
 380			msleep(GVE_ADMINQ_SLEEP_LEN);
 381		}
 382	}
 383	gve_clear_device_rings_ok(priv);
 384	gve_clear_device_resources_ok(priv);
 385	gve_clear_admin_queue_ok(priv);
 386}
 387
 388void gve_adminq_free(struct device *dev, struct gve_priv *priv)
 389{
 390	if (!gve_get_admin_queue_ok(priv))
 391		return;
 392	gve_adminq_release(priv);
 393	dma_pool_free(priv->adminq_pool, priv->adminq, priv->adminq_bus_addr);
 394	dma_pool_destroy(priv->adminq_pool);
 395	gve_clear_admin_queue_ok(priv);
 396}
 397
 398static void gve_adminq_kick_cmd(struct gve_priv *priv, u32 prod_cnt)
 399{
 400	iowrite32be(prod_cnt, &priv->reg_bar0->adminq_doorbell);
 401}
 402
 403static bool gve_adminq_wait_for_cmd(struct gve_priv *priv, u32 prod_cnt)
 404{
 405	int i;
 406
 407	for (i = 0; i < GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK; i++) {
 408		if (ioread32be(&priv->reg_bar0->adminq_event_counter)
 409		    == prod_cnt)
 410			return true;
 411		msleep(GVE_ADMINQ_SLEEP_LEN);
 412	}
 413
 414	return false;
 415}
 416
 417static int gve_adminq_parse_err(struct gve_priv *priv, u32 status)
 418{
 419	if (status != GVE_ADMINQ_COMMAND_PASSED &&
 420	    status != GVE_ADMINQ_COMMAND_UNSET) {
 421		dev_err(&priv->pdev->dev, "AQ command failed with status %d\n", status);
 422		priv->adminq_cmd_fail++;
 423	}
 424	switch (status) {
 425	case GVE_ADMINQ_COMMAND_PASSED:
 426		return 0;
 427	case GVE_ADMINQ_COMMAND_UNSET:
 428		dev_err(&priv->pdev->dev, "parse_aq_err: err and status both unset, this should not be possible.\n");
 429		return -EINVAL;
 430	case GVE_ADMINQ_COMMAND_ERROR_ABORTED:
 431	case GVE_ADMINQ_COMMAND_ERROR_CANCELLED:
 432	case GVE_ADMINQ_COMMAND_ERROR_DATALOSS:
 433	case GVE_ADMINQ_COMMAND_ERROR_FAILED_PRECONDITION:
 434	case GVE_ADMINQ_COMMAND_ERROR_UNAVAILABLE:
 435		return -EAGAIN;
 436	case GVE_ADMINQ_COMMAND_ERROR_ALREADY_EXISTS:
 437	case GVE_ADMINQ_COMMAND_ERROR_INTERNAL_ERROR:
 438	case GVE_ADMINQ_COMMAND_ERROR_INVALID_ARGUMENT:
 439	case GVE_ADMINQ_COMMAND_ERROR_NOT_FOUND:
 440	case GVE_ADMINQ_COMMAND_ERROR_OUT_OF_RANGE:
 441	case GVE_ADMINQ_COMMAND_ERROR_UNKNOWN_ERROR:
 442		return -EINVAL;
 443	case GVE_ADMINQ_COMMAND_ERROR_DEADLINE_EXCEEDED:
 444		return -ETIME;
 445	case GVE_ADMINQ_COMMAND_ERROR_PERMISSION_DENIED:
 446	case GVE_ADMINQ_COMMAND_ERROR_UNAUTHENTICATED:
 447		return -EACCES;
 448	case GVE_ADMINQ_COMMAND_ERROR_RESOURCE_EXHAUSTED:
 449		return -ENOMEM;
 450	case GVE_ADMINQ_COMMAND_ERROR_UNIMPLEMENTED:
 451		return -EOPNOTSUPP;
 452	default:
 453		dev_err(&priv->pdev->dev, "parse_aq_err: unknown status code %d\n", status);
 454		return -EINVAL;
 455	}
 456}
 457
 458/* Flushes all AQ commands currently queued and waits for them to complete.
 459 * If there are failures, it will return the first error.
 460 */
 461static int gve_adminq_kick_and_wait(struct gve_priv *priv)
 462{
 463	int tail, head;
 464	int i;
 465
 466	lockdep_assert_held(&priv->adminq_lock);
 467
 468	tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
 469	head = priv->adminq_prod_cnt;
 470
 471	gve_adminq_kick_cmd(priv, head);
 472	if (!gve_adminq_wait_for_cmd(priv, head)) {
 473		dev_err(&priv->pdev->dev, "AQ commands timed out, need to reset AQ\n");
 474		priv->adminq_timeouts++;
 475		return -ENOTRECOVERABLE;
 476	}
 477
 478	for (i = tail; i < head; i++) {
 479		union gve_adminq_command *cmd;
 480		u32 status, err;
 481
 482		cmd = &priv->adminq[i & priv->adminq_mask];
 483		status = be32_to_cpu(READ_ONCE(cmd->status));
 484		err = gve_adminq_parse_err(priv, status);
 485		if (err)
 486			// Return the first error if we failed.
 487			return err;
 488	}
 489
 490	return 0;
 491}
 492
 493static int gve_adminq_issue_cmd(struct gve_priv *priv,
 494				union gve_adminq_command *cmd_orig)
 495{
 496	union gve_adminq_command *cmd;
 497	u32 opcode;
 498	u32 tail;
 499
 500	lockdep_assert_held(&priv->adminq_lock);
 501
 502	tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
 503
 504	// Check if next command will overflow the buffer.
 505	if (((priv->adminq_prod_cnt + 1) & priv->adminq_mask) ==
 506	    (tail & priv->adminq_mask)) {
 507		int err;
 508
 509		// Flush existing commands to make room.
 510		err = gve_adminq_kick_and_wait(priv);
 511		if (err)
 512			return err;
 513
 514		// Retry.
 515		tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
 516		if (((priv->adminq_prod_cnt + 1) & priv->adminq_mask) ==
 517		    (tail & priv->adminq_mask)) {
 518			// This should never happen. We just flushed the
 519			// command queue so there should be enough space.
 520			return -ENOMEM;
 521		}
 522	}
 523
 524	cmd = &priv->adminq[priv->adminq_prod_cnt & priv->adminq_mask];
 525	priv->adminq_prod_cnt++;
 526
 527	memcpy(cmd, cmd_orig, sizeof(*cmd_orig));
 528	opcode = be32_to_cpu(READ_ONCE(cmd->opcode));
 529	if (opcode == GVE_ADMINQ_EXTENDED_COMMAND)
 530		opcode = be32_to_cpu(cmd->extended_command.inner_opcode);
 531
 532	switch (opcode) {
 533	case GVE_ADMINQ_DESCRIBE_DEVICE:
 534		priv->adminq_describe_device_cnt++;
 535		break;
 536	case GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES:
 537		priv->adminq_cfg_device_resources_cnt++;
 538		break;
 539	case GVE_ADMINQ_REGISTER_PAGE_LIST:
 540		priv->adminq_register_page_list_cnt++;
 541		break;
 542	case GVE_ADMINQ_UNREGISTER_PAGE_LIST:
 543		priv->adminq_unregister_page_list_cnt++;
 544		break;
 545	case GVE_ADMINQ_CREATE_TX_QUEUE:
 546		priv->adminq_create_tx_queue_cnt++;
 547		break;
 548	case GVE_ADMINQ_CREATE_RX_QUEUE:
 549		priv->adminq_create_rx_queue_cnt++;
 550		break;
 551	case GVE_ADMINQ_DESTROY_TX_QUEUE:
 552		priv->adminq_destroy_tx_queue_cnt++;
 553		break;
 554	case GVE_ADMINQ_DESTROY_RX_QUEUE:
 555		priv->adminq_destroy_rx_queue_cnt++;
 556		break;
 557	case GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES:
 558		priv->adminq_dcfg_device_resources_cnt++;
 559		break;
 560	case GVE_ADMINQ_SET_DRIVER_PARAMETER:
 561		priv->adminq_set_driver_parameter_cnt++;
 562		break;
 563	case GVE_ADMINQ_REPORT_STATS:
 564		priv->adminq_report_stats_cnt++;
 565		break;
 566	case GVE_ADMINQ_REPORT_LINK_SPEED:
 567		priv->adminq_report_link_speed_cnt++;
 568		break;
 569	case GVE_ADMINQ_REPORT_NIC_TIMESTAMP:
 570		priv->adminq_report_nic_timestamp_cnt++;
 571		break;
 572	case GVE_ADMINQ_GET_PTYPE_MAP:
 573		priv->adminq_get_ptype_map_cnt++;
 574		break;
 575	case GVE_ADMINQ_VERIFY_DRIVER_COMPATIBILITY:
 576		priv->adminq_verify_driver_compatibility_cnt++;
 577		break;
 578	case GVE_ADMINQ_QUERY_FLOW_RULES:
 579		priv->adminq_query_flow_rules_cnt++;
 580		break;
 581	case GVE_ADMINQ_CONFIGURE_FLOW_RULE:
 582		priv->adminq_cfg_flow_rule_cnt++;
 583		break;
 584	case GVE_ADMINQ_CONFIGURE_RSS:
 585		priv->adminq_cfg_rss_cnt++;
 586		break;
 587	case GVE_ADMINQ_QUERY_RSS:
 588		priv->adminq_query_rss_cnt++;
 589		break;
 590	default:
 591		dev_err(&priv->pdev->dev, "unknown AQ command opcode %d\n", opcode);
 592		return -EINVAL;
 593	}
 594
 595	return 0;
 596}
 597
 598static int gve_adminq_execute_cmd(struct gve_priv *priv,
 599				  union gve_adminq_command *cmd_orig)
 600{
 601	u32 tail, head;
 602	int err;
 603
 604	mutex_lock(&priv->adminq_lock);
 605	tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
 606	head = priv->adminq_prod_cnt;
 607	if (tail != head) {
 608		err = -EINVAL;
 609		goto out;
 610	}
 611
 612	err = gve_adminq_issue_cmd(priv, cmd_orig);
 613	if (err)
 614		goto out;
 615
 616	err = gve_adminq_kick_and_wait(priv);
 617
 618out:
 619	mutex_unlock(&priv->adminq_lock);
 620	return err;
 621}
 622
 623static int gve_adminq_execute_extended_cmd(struct gve_priv *priv, u32 opcode,
 624					   size_t cmd_size, void *cmd_orig)
 625{
 626	union gve_adminq_command cmd;
 627	dma_addr_t inner_cmd_bus;
 628	void *inner_cmd;
 629	int err;
 630
 631	inner_cmd = dma_alloc_coherent(&priv->pdev->dev, cmd_size,
 632				       &inner_cmd_bus, GFP_KERNEL);
 633	if (!inner_cmd)
 634		return -ENOMEM;
 635
 636	memcpy(inner_cmd, cmd_orig, cmd_size);
 637
 638	memset(&cmd, 0, sizeof(cmd));
 639	cmd.opcode = cpu_to_be32(GVE_ADMINQ_EXTENDED_COMMAND);
 640	cmd.extended_command = (struct gve_adminq_extended_command) {
 641		.inner_opcode = cpu_to_be32(opcode),
 642		.inner_length = cpu_to_be32(cmd_size),
 643		.inner_command_addr = cpu_to_be64(inner_cmd_bus),
 644	};
 645
 646	err = gve_adminq_execute_cmd(priv, &cmd);
 647
 648	dma_free_coherent(&priv->pdev->dev, cmd_size, inner_cmd, inner_cmd_bus);
 649	return err;
 650}
 651
 652/* The device specifies that the management vector can either be the first irq
 653 * or the last irq. ntfy_blk_msix_base_idx indicates the first irq assigned to
 654 * the ntfy blks. If it is 0 then the management vector is last, if it is 1 then
 655 * the management vector is first.
 656 *
 657 * gve arranges the msix vectors so that the management vector is last.
 658 */
 659#define GVE_NTFY_BLK_BASE_MSIX_IDX	0
 660int gve_adminq_configure_device_resources(struct gve_priv *priv,
 661					  dma_addr_t counter_array_bus_addr,
 662					  u32 num_counters,
 663					  dma_addr_t db_array_bus_addr,
 664					  u32 num_ntfy_blks)
 665{
 666	union gve_adminq_command cmd;
 667
 668	memset(&cmd, 0, sizeof(cmd));
 669	cmd.opcode = cpu_to_be32(GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES);
 670	cmd.configure_device_resources =
 671		(struct gve_adminq_configure_device_resources) {
 672		.counter_array = cpu_to_be64(counter_array_bus_addr),
 673		.num_counters = cpu_to_be32(num_counters),
 674		.irq_db_addr = cpu_to_be64(db_array_bus_addr),
 675		.num_irq_dbs = cpu_to_be32(num_ntfy_blks),
 676		.irq_db_stride = cpu_to_be32(sizeof(*priv->irq_db_indices)),
 677		.ntfy_blk_msix_base_idx =
 678					cpu_to_be32(GVE_NTFY_BLK_BASE_MSIX_IDX),
 679		.queue_format = priv->queue_format,
 680	};
 681
 682	return gve_adminq_execute_cmd(priv, &cmd);
 683}
 684
 685int gve_adminq_deconfigure_device_resources(struct gve_priv *priv)
 686{
 687	union gve_adminq_command cmd;
 688
 689	memset(&cmd, 0, sizeof(cmd));
 690	cmd.opcode = cpu_to_be32(GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES);
 691
 692	return gve_adminq_execute_cmd(priv, &cmd);
 693}
 694
 695static int gve_adminq_create_tx_queue(struct gve_priv *priv, u32 queue_index)
 696{
 697	struct gve_tx_ring *tx = &priv->tx[queue_index];
 698	union gve_adminq_command cmd;
 699
 700	memset(&cmd, 0, sizeof(cmd));
 701	cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_TX_QUEUE);
 702	cmd.create_tx_queue = (struct gve_adminq_create_tx_queue) {
 703		.queue_id = cpu_to_be32(queue_index),
 704		.queue_resources_addr =
 705			cpu_to_be64(tx->q_resources_bus),
 706		.tx_ring_addr = cpu_to_be64(tx->bus),
 707		.ntfy_id = cpu_to_be32(tx->ntfy_id),
 708		.tx_ring_size = cpu_to_be16(priv->tx_desc_cnt),
 709	};
 710
 711	if (gve_is_gqi(priv)) {
 712		u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ?
 713			GVE_RAW_ADDRESSING_QPL_ID : tx->tx_fifo.qpl->id;
 714
 715		cmd.create_tx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
 716	} else {
 717		u32 qpl_id = 0;
 718
 719		if (priv->queue_format == GVE_DQO_RDA_FORMAT)
 720			qpl_id = GVE_RAW_ADDRESSING_QPL_ID;
 721		else
 722			qpl_id = tx->dqo.qpl->id;
 723		cmd.create_tx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
 724		cmd.create_tx_queue.tx_comp_ring_addr =
 725			cpu_to_be64(tx->complq_bus_dqo);
 726		cmd.create_tx_queue.tx_comp_ring_size =
 727			cpu_to_be16(priv->tx_desc_cnt);
 728	}
 729
 730	return gve_adminq_issue_cmd(priv, &cmd);
 731}
 732
 733int gve_adminq_create_tx_queues(struct gve_priv *priv, u32 start_id, u32 num_queues)
 734{
 735	int err;
 736	int i;
 737
 738	mutex_lock(&priv->adminq_lock);
 739
 740	for (i = start_id; i < start_id + num_queues; i++) {
 741		err = gve_adminq_create_tx_queue(priv, i);
 742		if (err)
 743			goto out;
 744	}
 745
 746	err = gve_adminq_kick_and_wait(priv);
 747
 748out:
 749	mutex_unlock(&priv->adminq_lock);
 750	return err;
 751}
 752
 753static void gve_adminq_get_create_rx_queue_cmd(struct gve_priv *priv,
 754					       union gve_adminq_command *cmd,
 755					       u32 queue_index)
 756{
 757	struct gve_rx_ring *rx = &priv->rx[queue_index];
 758
 759	memset(cmd, 0, sizeof(*cmd));
 760	cmd->opcode = cpu_to_be32(GVE_ADMINQ_CREATE_RX_QUEUE);
 761	cmd->create_rx_queue = (struct gve_adminq_create_rx_queue) {
 762		.queue_id = cpu_to_be32(queue_index),
 763		.ntfy_id = cpu_to_be32(rx->ntfy_id),
 764		.queue_resources_addr = cpu_to_be64(rx->q_resources_bus),
 765		.rx_ring_size = cpu_to_be16(priv->rx_desc_cnt),
 766		.packet_buffer_size = cpu_to_be16(rx->packet_buffer_size),
 767	};
 768
 769	if (gve_is_gqi(priv)) {
 770		u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ?
 771			GVE_RAW_ADDRESSING_QPL_ID : rx->data.qpl->id;
 772
 773		cmd->create_rx_queue.rx_desc_ring_addr =
 774			cpu_to_be64(rx->desc.bus);
 775		cmd->create_rx_queue.rx_data_ring_addr =
 776			cpu_to_be64(rx->data.data_bus);
 777		cmd->create_rx_queue.index = cpu_to_be32(queue_index);
 778		cmd->create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
 779	} else {
 780		u32 qpl_id = 0;
 781
 782		if (priv->queue_format == GVE_DQO_RDA_FORMAT)
 783			qpl_id = GVE_RAW_ADDRESSING_QPL_ID;
 784		else
 785			qpl_id = rx->dqo.qpl->id;
 786		cmd->create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
 787		cmd->create_rx_queue.rx_desc_ring_addr =
 788			cpu_to_be64(rx->dqo.complq.bus);
 789		cmd->create_rx_queue.rx_data_ring_addr =
 790			cpu_to_be64(rx->dqo.bufq.bus);
 791		cmd->create_rx_queue.rx_buff_ring_size =
 792			cpu_to_be16(priv->rx_desc_cnt);
 793		cmd->create_rx_queue.enable_rsc =
 794			!!(priv->dev->features & NETIF_F_LRO);
 795		if (priv->header_split_enabled)
 796			cmd->create_rx_queue.header_buffer_size =
 797				cpu_to_be16(priv->header_buf_size);
 798	}
 799}
 800
 801static int gve_adminq_create_rx_queue(struct gve_priv *priv, u32 queue_index)
 802{
 803	union gve_adminq_command cmd;
 804
 805	gve_adminq_get_create_rx_queue_cmd(priv, &cmd, queue_index);
 806	return gve_adminq_issue_cmd(priv, &cmd);
 807}
 808
 809/* Unlike gve_adminq_create_rx_queue, this actually rings the doorbell */
 810int gve_adminq_create_single_rx_queue(struct gve_priv *priv, u32 queue_index)
 811{
 812	union gve_adminq_command cmd;
 813
 814	gve_adminq_get_create_rx_queue_cmd(priv, &cmd, queue_index);
 815	return gve_adminq_execute_cmd(priv, &cmd);
 816}
 817
 818int gve_adminq_create_rx_queues(struct gve_priv *priv, u32 num_queues)
 819{
 820	int err;
 821	int i;
 822
 823	mutex_lock(&priv->adminq_lock);
 824
 825	for (i = 0; i < num_queues; i++) {
 826		err = gve_adminq_create_rx_queue(priv, i);
 827		if (err)
 828			goto out;
 829	}
 830
 831	err = gve_adminq_kick_and_wait(priv);
 832
 833out:
 834	mutex_unlock(&priv->adminq_lock);
 835	return err;
 836}
 837
 838static int gve_adminq_destroy_tx_queue(struct gve_priv *priv, u32 queue_index)
 839{
 840	union gve_adminq_command cmd;
 841	int err;
 842
 843	memset(&cmd, 0, sizeof(cmd));
 844	cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_TX_QUEUE);
 845	cmd.destroy_tx_queue = (struct gve_adminq_destroy_tx_queue) {
 846		.queue_id = cpu_to_be32(queue_index),
 847	};
 848
 849	err = gve_adminq_issue_cmd(priv, &cmd);
 850	if (err)
 851		return err;
 852
 853	return 0;
 854}
 855
 856int gve_adminq_destroy_tx_queues(struct gve_priv *priv, u32 start_id, u32 num_queues)
 857{
 858	int err;
 859	int i;
 860
 861	mutex_lock(&priv->adminq_lock);
 862
 863	for (i = start_id; i < start_id + num_queues; i++) {
 864		err = gve_adminq_destroy_tx_queue(priv, i);
 865		if (err)
 866			goto out;
 867	}
 868
 869	err = gve_adminq_kick_and_wait(priv);
 870
 871out:
 872	mutex_unlock(&priv->adminq_lock);
 873	return err;
 874}
 875
 876static void gve_adminq_make_destroy_rx_queue_cmd(union gve_adminq_command *cmd,
 877						 u32 queue_index)
 878{
 879	memset(cmd, 0, sizeof(*cmd));
 880	cmd->opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_RX_QUEUE);
 881	cmd->destroy_rx_queue = (struct gve_adminq_destroy_rx_queue) {
 882		.queue_id = cpu_to_be32(queue_index),
 883	};
 884}
 885
 886static int gve_adminq_destroy_rx_queue(struct gve_priv *priv, u32 queue_index)
 887{
 888	union gve_adminq_command cmd;
 889
 890	gve_adminq_make_destroy_rx_queue_cmd(&cmd, queue_index);
 891	return gve_adminq_issue_cmd(priv, &cmd);
 892}
 893
 894/* Unlike gve_adminq_destroy_rx_queue, this actually rings the doorbell */
 895int gve_adminq_destroy_single_rx_queue(struct gve_priv *priv, u32 queue_index)
 896{
 897	union gve_adminq_command cmd;
 898
 899	gve_adminq_make_destroy_rx_queue_cmd(&cmd, queue_index);
 900	return gve_adminq_execute_cmd(priv, &cmd);
 901}
 902
 903int gve_adminq_destroy_rx_queues(struct gve_priv *priv, u32 num_queues)
 904{
 905	int err;
 906	int i;
 907
 908	mutex_lock(&priv->adminq_lock);
 909
 910	for (i = 0; i < num_queues; i++) {
 911		err = gve_adminq_destroy_rx_queue(priv, i);
 912		if (err)
 913			goto out;
 914	}
 915
 916	err = gve_adminq_kick_and_wait(priv);
 917
 918out:
 919	mutex_unlock(&priv->adminq_lock);
 920	return err;
 921}
 922
 923static void gve_set_default_desc_cnt(struct gve_priv *priv,
 924			const struct gve_device_descriptor *descriptor)
 925{
 926	priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries);
 927	priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries);
 928
 929	/* set default ranges */
 930	priv->max_tx_desc_cnt = priv->tx_desc_cnt;
 931	priv->max_rx_desc_cnt = priv->rx_desc_cnt;
 932	priv->min_tx_desc_cnt = priv->tx_desc_cnt;
 933	priv->min_rx_desc_cnt = priv->rx_desc_cnt;
 934}
 935
 936static void gve_set_default_rss_sizes(struct gve_priv *priv)
 937{
 938	if (!gve_is_gqi(priv)) {
 939		priv->rss_key_size = GVE_RSS_KEY_SIZE;
 940		priv->rss_lut_size = GVE_RSS_INDIR_SIZE;
 941		priv->cache_rss_config = true;
 942	}
 943}
 944
 945static void gve_enable_supported_features(struct gve_priv *priv,
 946					  u32 supported_features_mask,
 947					  const struct gve_device_option_jumbo_frames
 948					  *dev_op_jumbo_frames,
 949					  const struct gve_device_option_dqo_qpl
 950					  *dev_op_dqo_qpl,
 951					  const struct gve_device_option_buffer_sizes
 952					  *dev_op_buffer_sizes,
 953					  const struct gve_device_option_flow_steering
 954					  *dev_op_flow_steering,
 955					  const struct gve_device_option_rss_config
 956					  *dev_op_rss_config,
 957					  const struct gve_device_option_nic_timestamp
 958					  *dev_op_nic_timestamp,
 959					  const struct gve_device_option_modify_ring
 960					  *dev_op_modify_ring)
 961{
 962	/* Before control reaches this point, the page-size-capped max MTU from
 963	 * the gve_device_descriptor field has already been stored in
 964	 * priv->dev->max_mtu. We overwrite it with the true max MTU below.
 965	 */
 966	if (dev_op_jumbo_frames &&
 967	    (supported_features_mask & GVE_SUP_JUMBO_FRAMES_MASK)) {
 968		dev_info(&priv->pdev->dev,
 969			 "JUMBO FRAMES device option enabled.\n");
 970		priv->dev->max_mtu = be16_to_cpu(dev_op_jumbo_frames->max_mtu);
 971	}
 972
 973	/* Override pages for qpl for DQO-QPL */
 974	if (dev_op_dqo_qpl) {
 975		priv->tx_pages_per_qpl =
 976			be16_to_cpu(dev_op_dqo_qpl->tx_pages_per_qpl);
 977		if (priv->tx_pages_per_qpl == 0)
 978			priv->tx_pages_per_qpl = DQO_QPL_DEFAULT_TX_PAGES;
 979	}
 980
 981	if (dev_op_buffer_sizes &&
 982	    (supported_features_mask & GVE_SUP_BUFFER_SIZES_MASK)) {
 983		priv->max_rx_buffer_size =
 984			be16_to_cpu(dev_op_buffer_sizes->packet_buffer_size);
 985		priv->header_buf_size =
 986			be16_to_cpu(dev_op_buffer_sizes->header_buffer_size);
 987		dev_info(&priv->pdev->dev,
 988			 "BUFFER SIZES device option enabled with max_rx_buffer_size of %u, header_buf_size of %u.\n",
 989			 priv->max_rx_buffer_size, priv->header_buf_size);
 990		if (gve_is_dqo(priv) &&
 991		    priv->max_rx_buffer_size > GVE_DEFAULT_RX_BUFFER_SIZE)
 992			priv->rx_cfg.packet_buffer_size =
 993				priv->max_rx_buffer_size;
 994	}
 995
 996	/* Read and store ring size ranges given by device */
 997	if (dev_op_modify_ring &&
 998	    (supported_features_mask & GVE_SUP_MODIFY_RING_MASK)) {
 999		priv->modify_ring_size_enabled = true;
1000
1001		/* max ring size for DQO QPL should not be overwritten because of device limit */
1002		if (priv->queue_format != GVE_DQO_QPL_FORMAT) {
1003			priv->max_rx_desc_cnt = be16_to_cpu(dev_op_modify_ring->max_rx_ring_size);
1004			priv->max_tx_desc_cnt = be16_to_cpu(dev_op_modify_ring->max_tx_ring_size);
1005		}
1006		if (priv->default_min_ring_size) {
1007			/* If device hasn't provided minimums, use default minimums */
1008			priv->min_tx_desc_cnt = GVE_DEFAULT_MIN_TX_RING_SIZE;
1009			priv->min_rx_desc_cnt = GVE_DEFAULT_MIN_RX_RING_SIZE;
1010		} else {
1011			priv->min_rx_desc_cnt = be16_to_cpu(dev_op_modify_ring->min_rx_ring_size);
1012			priv->min_tx_desc_cnt = be16_to_cpu(dev_op_modify_ring->min_tx_ring_size);
1013		}
1014	}
1015
1016	if (dev_op_flow_steering &&
1017	    (supported_features_mask & GVE_SUP_FLOW_STEERING_MASK)) {
1018		if (dev_op_flow_steering->max_flow_rules) {
1019			priv->max_flow_rules =
1020				be32_to_cpu(dev_op_flow_steering->max_flow_rules);
1021			priv->dev->hw_features |= NETIF_F_NTUPLE;
1022			dev_info(&priv->pdev->dev,
1023				 "FLOW STEERING device option enabled with max rule limit of %u.\n",
1024				 priv->max_flow_rules);
1025		}
1026	}
1027
1028	if (dev_op_rss_config &&
1029	    (supported_features_mask & GVE_SUP_RSS_CONFIG_MASK)) {
1030		priv->rss_key_size =
1031			be16_to_cpu(dev_op_rss_config->hash_key_size);
1032		priv->rss_lut_size =
1033			be16_to_cpu(dev_op_rss_config->hash_lut_size);
1034		priv->cache_rss_config = false;
1035		dev_dbg(&priv->pdev->dev,
1036			"RSS device option enabled with key size of %u, lut size of %u.\n",
1037			priv->rss_key_size, priv->rss_lut_size);
1038	}
1039
1040	if (dev_op_nic_timestamp &&
1041	    (supported_features_mask & GVE_SUP_NIC_TIMESTAMP_MASK))
1042		priv->nic_timestamp_supported = true;
1043}
1044
1045int gve_adminq_describe_device(struct gve_priv *priv)
1046{
1047	struct gve_device_option_nic_timestamp *dev_op_nic_timestamp = NULL;
1048	struct gve_device_option_flow_steering *dev_op_flow_steering = NULL;
1049	struct gve_device_option_buffer_sizes *dev_op_buffer_sizes = NULL;
1050	struct gve_device_option_jumbo_frames *dev_op_jumbo_frames = NULL;
1051	struct gve_device_option_modify_ring *dev_op_modify_ring = NULL;
1052	struct gve_device_option_rss_config *dev_op_rss_config = NULL;
1053	struct gve_device_option_gqi_rda *dev_op_gqi_rda = NULL;
1054	struct gve_device_option_gqi_qpl *dev_op_gqi_qpl = NULL;
1055	struct gve_device_option_dqo_rda *dev_op_dqo_rda = NULL;
1056	struct gve_device_option_dqo_qpl *dev_op_dqo_qpl = NULL;
1057	struct gve_device_descriptor *descriptor;
1058	u32 supported_features_mask = 0;
1059	union gve_adminq_command cmd;
1060	dma_addr_t descriptor_bus;
1061	int err = 0;
1062	u8 *mac;
1063	u16 mtu;
1064
1065	memset(&cmd, 0, sizeof(cmd));
1066	descriptor = dma_pool_alloc(priv->adminq_pool, GFP_KERNEL,
1067				    &descriptor_bus);
1068	if (!descriptor)
1069		return -ENOMEM;
1070	cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESCRIBE_DEVICE);
1071	cmd.describe_device.device_descriptor_addr =
1072						cpu_to_be64(descriptor_bus);
1073	cmd.describe_device.device_descriptor_version =
1074			cpu_to_be32(GVE_ADMINQ_DEVICE_DESCRIPTOR_VERSION);
1075	cmd.describe_device.available_length =
1076		cpu_to_be32(GVE_ADMINQ_BUFFER_SIZE);
1077
1078	err = gve_adminq_execute_cmd(priv, &cmd);
1079	if (err)
1080		goto free_device_descriptor;
1081
1082	err = gve_process_device_options(priv, descriptor, &dev_op_gqi_rda,
1083					 &dev_op_gqi_qpl, &dev_op_dqo_rda,
1084					 &dev_op_jumbo_frames, &dev_op_dqo_qpl,
1085					 &dev_op_buffer_sizes,
1086					 &dev_op_flow_steering,
1087					 &dev_op_rss_config,
1088					 &dev_op_nic_timestamp,
1089					 &dev_op_modify_ring);
1090	if (err)
1091		goto free_device_descriptor;
1092
1093	/* If the GQI_RAW_ADDRESSING option is not enabled and the queue format
1094	 * is not set to GqiRda, choose the queue format in a priority order:
1095	 * DqoRda, DqoQpl, GqiRda, GqiQpl. Use GqiQpl as default.
1096	 */
1097	if (dev_op_dqo_rda) {
1098		priv->queue_format = GVE_DQO_RDA_FORMAT;
1099		dev_info(&priv->pdev->dev,
1100			 "Driver is running with DQO RDA queue format.\n");
1101		supported_features_mask =
1102			be32_to_cpu(dev_op_dqo_rda->supported_features_mask);
1103	} else if (dev_op_dqo_qpl) {
1104		priv->queue_format = GVE_DQO_QPL_FORMAT;
1105		supported_features_mask =
1106			be32_to_cpu(dev_op_dqo_qpl->supported_features_mask);
1107	}  else if (dev_op_gqi_rda) {
1108		priv->queue_format = GVE_GQI_RDA_FORMAT;
1109		dev_info(&priv->pdev->dev,
1110			 "Driver is running with GQI RDA queue format.\n");
1111		supported_features_mask =
1112			be32_to_cpu(dev_op_gqi_rda->supported_features_mask);
1113	} else if (priv->queue_format == GVE_GQI_RDA_FORMAT) {
1114		dev_info(&priv->pdev->dev,
1115			 "Driver is running with GQI RDA queue format.\n");
1116	} else {
1117		priv->queue_format = GVE_GQI_QPL_FORMAT;
1118		if (dev_op_gqi_qpl)
1119			supported_features_mask =
1120				be32_to_cpu(dev_op_gqi_qpl->supported_features_mask);
1121		dev_info(&priv->pdev->dev,
1122			 "Driver is running with GQI QPL queue format.\n");
1123	}
1124
1125	/* set default descriptor counts */
1126	gve_set_default_desc_cnt(priv, descriptor);
1127
1128	gve_set_default_rss_sizes(priv);
1129
1130	/* DQO supports LRO. */
1131	if (!gve_is_gqi(priv))
1132		priv->dev->hw_features |= NETIF_F_LRO;
1133
1134	priv->max_registered_pages =
1135				be64_to_cpu(descriptor->max_registered_pages);
1136	mtu = be16_to_cpu(descriptor->mtu);
1137	if (mtu < ETH_MIN_MTU) {
1138		dev_err(&priv->pdev->dev, "MTU %d below minimum MTU\n", mtu);
1139		err = -EINVAL;
1140		goto free_device_descriptor;
1141	}
1142	priv->dev->max_mtu = mtu;
1143	priv->num_event_counters = be16_to_cpu(descriptor->counters);
1144	eth_hw_addr_set(priv->dev, descriptor->mac);
1145	mac = descriptor->mac;
1146	dev_info(&priv->pdev->dev, "MAC addr: %pM\n", mac);
1147	priv->tx_pages_per_qpl = be16_to_cpu(descriptor->tx_pages_per_qpl);
1148	priv->default_num_queues = be16_to_cpu(descriptor->default_num_queues);
1149
1150	gve_enable_supported_features(priv, supported_features_mask,
1151				      dev_op_jumbo_frames, dev_op_dqo_qpl,
1152				      dev_op_buffer_sizes, dev_op_flow_steering,
1153				      dev_op_rss_config, dev_op_nic_timestamp,
1154				      dev_op_modify_ring);
1155
1156free_device_descriptor:
1157	dma_pool_free(priv->adminq_pool, descriptor, descriptor_bus);
1158	return err;
1159}
1160
1161int gve_adminq_register_page_list(struct gve_priv *priv,
1162				  struct gve_queue_page_list *qpl)
1163{
1164	struct device *hdev = &priv->pdev->dev;
1165	u32 num_entries = qpl->num_entries;
1166	u32 size = num_entries * sizeof(qpl->page_buses[0]);
1167	union gve_adminq_command cmd;
1168	dma_addr_t page_list_bus;
1169	__be64 *page_list;
1170	int err;
1171	int i;
1172
1173	memset(&cmd, 0, sizeof(cmd));
1174	page_list = dma_alloc_coherent(hdev, size, &page_list_bus, GFP_KERNEL);
1175	if (!page_list)
1176		return -ENOMEM;
1177
1178	for (i = 0; i < num_entries; i++)
1179		page_list[i] = cpu_to_be64(qpl->page_buses[i]);
1180
1181	cmd.opcode = cpu_to_be32(GVE_ADMINQ_REGISTER_PAGE_LIST);
1182	cmd.reg_page_list = (struct gve_adminq_register_page_list) {
1183		.page_list_id = cpu_to_be32(qpl->id),
1184		.num_pages = cpu_to_be32(num_entries),
1185		.page_address_list_addr = cpu_to_be64(page_list_bus),
1186		.page_size = cpu_to_be64(PAGE_SIZE),
1187	};
1188
1189	err = gve_adminq_execute_cmd(priv, &cmd);
1190	dma_free_coherent(hdev, size, page_list, page_list_bus);
1191	return err;
1192}
1193
1194int gve_adminq_unregister_page_list(struct gve_priv *priv, u32 page_list_id)
1195{
1196	union gve_adminq_command cmd;
1197
1198	memset(&cmd, 0, sizeof(cmd));
1199	cmd.opcode = cpu_to_be32(GVE_ADMINQ_UNREGISTER_PAGE_LIST);
1200	cmd.unreg_page_list = (struct gve_adminq_unregister_page_list) {
1201		.page_list_id = cpu_to_be32(page_list_id),
1202	};
1203
1204	return gve_adminq_execute_cmd(priv, &cmd);
1205}
1206
1207int gve_adminq_report_stats(struct gve_priv *priv, u64 stats_report_len,
1208			    dma_addr_t stats_report_addr, u64 interval)
1209{
1210	union gve_adminq_command cmd;
1211
1212	memset(&cmd, 0, sizeof(cmd));
1213	cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_STATS);
1214	cmd.report_stats = (struct gve_adminq_report_stats) {
1215		.stats_report_len = cpu_to_be64(stats_report_len),
1216		.stats_report_addr = cpu_to_be64(stats_report_addr),
1217		.interval = cpu_to_be64(interval),
1218	};
1219
1220	return gve_adminq_execute_cmd(priv, &cmd);
1221}
1222
1223int gve_adminq_verify_driver_compatibility(struct gve_priv *priv,
1224					   u64 driver_info_len,
1225					   dma_addr_t driver_info_addr)
1226{
1227	union gve_adminq_command cmd;
1228
1229	memset(&cmd, 0, sizeof(cmd));
1230	cmd.opcode = cpu_to_be32(GVE_ADMINQ_VERIFY_DRIVER_COMPATIBILITY);
1231	cmd.verify_driver_compatibility = (struct gve_adminq_verify_driver_compatibility) {
1232		.driver_info_len = cpu_to_be64(driver_info_len),
1233		.driver_info_addr = cpu_to_be64(driver_info_addr),
1234	};
1235
1236	return gve_adminq_execute_cmd(priv, &cmd);
1237}
1238
1239int gve_adminq_report_link_speed(struct gve_priv *priv)
1240{
1241	union gve_adminq_command gvnic_cmd;
1242	dma_addr_t link_speed_region_bus;
1243	__be64 *link_speed_region;
1244	int err;
1245
1246	link_speed_region =
1247		dma_alloc_coherent(&priv->pdev->dev, sizeof(*link_speed_region),
1248				   &link_speed_region_bus, GFP_KERNEL);
1249
1250	if (!link_speed_region)
1251		return -ENOMEM;
1252
1253	memset(&gvnic_cmd, 0, sizeof(gvnic_cmd));
1254	gvnic_cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_LINK_SPEED);
1255	gvnic_cmd.report_link_speed.link_speed_address =
1256		cpu_to_be64(link_speed_region_bus);
1257
1258	err = gve_adminq_execute_cmd(priv, &gvnic_cmd);
1259
1260	priv->link_speed = be64_to_cpu(*link_speed_region);
1261	dma_free_coherent(&priv->pdev->dev, sizeof(*link_speed_region), link_speed_region,
1262			  link_speed_region_bus);
1263	return err;
1264}
1265
1266int gve_adminq_report_nic_ts(struct gve_priv *priv,
1267			     dma_addr_t nic_ts_report_addr)
1268{
1269	union gve_adminq_command cmd;
1270
1271	memset(&cmd, 0, sizeof(cmd));
1272	cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_NIC_TIMESTAMP);
1273	cmd.report_nic_ts = (struct gve_adminq_report_nic_ts) {
1274		.nic_ts_report_len =
1275			cpu_to_be64(sizeof(struct gve_nic_ts_report)),
1276		.nic_ts_report_addr = cpu_to_be64(nic_ts_report_addr),
1277	};
1278
1279	return gve_adminq_execute_cmd(priv, &cmd);
1280}
1281
1282int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv,
1283				 struct gve_ptype_lut *ptype_lut)
1284{
1285	struct gve_ptype_map *ptype_map;
1286	union gve_adminq_command cmd;
1287	dma_addr_t ptype_map_bus;
1288	int err = 0;
1289	int i;
1290
1291	memset(&cmd, 0, sizeof(cmd));
1292	ptype_map = dma_alloc_coherent(&priv->pdev->dev, sizeof(*ptype_map),
1293				       &ptype_map_bus, GFP_KERNEL);
1294	if (!ptype_map)
1295		return -ENOMEM;
1296
1297	cmd.opcode = cpu_to_be32(GVE_ADMINQ_GET_PTYPE_MAP);
1298	cmd.get_ptype_map = (struct gve_adminq_get_ptype_map) {
1299		.ptype_map_len = cpu_to_be64(sizeof(*ptype_map)),
1300		.ptype_map_addr = cpu_to_be64(ptype_map_bus),
1301	};
1302
1303	err = gve_adminq_execute_cmd(priv, &cmd);
1304	if (err)
1305		goto err;
1306
1307	/* Populate ptype_lut. */
1308	for (i = 0; i < GVE_NUM_PTYPES; i++) {
1309		ptype_lut->ptypes[i].l3_type =
1310			ptype_map->ptypes[i].l3_type;
1311		ptype_lut->ptypes[i].l4_type =
1312			ptype_map->ptypes[i].l4_type;
1313	}
1314err:
1315	dma_free_coherent(&priv->pdev->dev, sizeof(*ptype_map), ptype_map,
1316			  ptype_map_bus);
1317	return err;
1318}
1319
1320static int
1321gve_adminq_configure_flow_rule(struct gve_priv *priv,
1322			       struct gve_adminq_configure_flow_rule *flow_rule_cmd)
1323{
1324	int err = gve_adminq_execute_extended_cmd(priv,
1325			GVE_ADMINQ_CONFIGURE_FLOW_RULE,
1326			sizeof(struct gve_adminq_configure_flow_rule),
1327			flow_rule_cmd);
1328
1329	if (err == -ETIME) {
1330		dev_err(&priv->pdev->dev, "Timeout to configure the flow rule, trigger reset");
1331		gve_reset(priv, true);
1332	} else if (!err) {
1333		priv->flow_rules_cache.rules_cache_synced = false;
1334	}
1335
1336	return err;
1337}
1338
1339int gve_adminq_add_flow_rule(struct gve_priv *priv, struct gve_adminq_flow_rule *rule, u32 loc)
1340{
1341	struct gve_adminq_configure_flow_rule flow_rule_cmd = {
1342		.opcode = cpu_to_be16(GVE_FLOW_RULE_CFG_ADD),
1343		.location = cpu_to_be32(loc),
1344		.rule = *rule,
1345	};
1346
1347	return gve_adminq_configure_flow_rule(priv, &flow_rule_cmd);
1348}
1349
1350int gve_adminq_del_flow_rule(struct gve_priv *priv, u32 loc)
1351{
1352	struct gve_adminq_configure_flow_rule flow_rule_cmd = {
1353		.opcode = cpu_to_be16(GVE_FLOW_RULE_CFG_DEL),
1354		.location = cpu_to_be32(loc),
1355	};
1356
1357	return gve_adminq_configure_flow_rule(priv, &flow_rule_cmd);
1358}
1359
1360int gve_adminq_reset_flow_rules(struct gve_priv *priv)
1361{
1362	struct gve_adminq_configure_flow_rule flow_rule_cmd = {
1363		.opcode = cpu_to_be16(GVE_FLOW_RULE_CFG_RESET),
1364	};
1365
1366	return gve_adminq_configure_flow_rule(priv, &flow_rule_cmd);
1367}
1368
1369int gve_adminq_configure_rss(struct gve_priv *priv, struct ethtool_rxfh_param *rxfh)
1370{
1371	const u32 *hash_lut_to_config = NULL;
1372	const u8 *hash_key_to_config = NULL;
1373	dma_addr_t lut_bus = 0, key_bus = 0;
1374	union gve_adminq_command cmd;
1375	__be32 *lut = NULL;
1376	u8 hash_alg = 0;
1377	u8 *key = NULL;
1378	int err = 0;
1379	u16 i;
1380
1381	switch (rxfh->hfunc) {
1382	case ETH_RSS_HASH_NO_CHANGE:
1383		fallthrough;
1384	case ETH_RSS_HASH_TOP:
1385		hash_alg = ETH_RSS_HASH_TOP;
1386		break;
1387	default:
1388		return -EOPNOTSUPP;
1389	}
1390
1391	if (rxfh->indir) {
1392		if (rxfh->indir_size != priv->rss_lut_size)
1393			return -EINVAL;
1394
1395		hash_lut_to_config = rxfh->indir;
1396	} else if (priv->cache_rss_config) {
1397		hash_lut_to_config = priv->rss_config.hash_lut;
1398	}
1399
1400	if (hash_lut_to_config) {
1401		lut = dma_alloc_coherent(&priv->pdev->dev,
1402					 priv->rss_lut_size * sizeof(*lut),
1403					 &lut_bus, GFP_KERNEL);
1404		if (!lut)
1405			return -ENOMEM;
1406
1407		for (i = 0; i < priv->rss_lut_size; i++)
1408			lut[i] = cpu_to_be32(hash_lut_to_config[i]);
1409	}
1410
1411	if (rxfh->key) {
1412		if (rxfh->key_size != priv->rss_key_size) {
1413			err = -EINVAL;
1414			goto out;
1415		}
1416
1417		hash_key_to_config = rxfh->key;
1418	} else if (priv->cache_rss_config) {
1419		hash_key_to_config = priv->rss_config.hash_key;
1420	}
1421
1422	if (hash_key_to_config) {
1423		key = dma_alloc_coherent(&priv->pdev->dev,
1424					 priv->rss_key_size,
1425					 &key_bus, GFP_KERNEL);
1426		if (!key) {
1427			err = -ENOMEM;
1428			goto out;
1429		}
1430
1431		memcpy(key, hash_key_to_config, priv->rss_key_size);
1432	}
1433
1434	/* Zero-valued fields in the cmd.configure_rss instruct the device to
1435	 * not update those fields.
1436	 */
1437	memset(&cmd, 0, sizeof(cmd));
1438	cmd.opcode = cpu_to_be32(GVE_ADMINQ_CONFIGURE_RSS);
1439	cmd.configure_rss = (struct gve_adminq_configure_rss) {
1440		.hash_types = cpu_to_be16(BIT(GVE_RSS_HASH_TCPV4) |
1441					  BIT(GVE_RSS_HASH_UDPV4) |
1442					  BIT(GVE_RSS_HASH_TCPV6) |
1443					  BIT(GVE_RSS_HASH_UDPV6)),
1444		.hash_alg = hash_alg,
1445		.hash_key_size =
1446			cpu_to_be16((key_bus) ? priv->rss_key_size : 0),
1447		.hash_lut_size =
1448			cpu_to_be16((lut_bus) ? priv->rss_lut_size : 0),
1449		.hash_key_addr = cpu_to_be64(key_bus),
1450		.hash_lut_addr = cpu_to_be64(lut_bus),
1451	};
1452
1453	err = gve_adminq_execute_cmd(priv, &cmd);
1454
1455out:
1456	if (lut)
1457		dma_free_coherent(&priv->pdev->dev,
1458				  priv->rss_lut_size * sizeof(*lut),
1459				  lut, lut_bus);
1460	if (key)
1461		dma_free_coherent(&priv->pdev->dev,
1462				  priv->rss_key_size, key, key_bus);
1463	return err;
1464}
1465
1466/* In the dma memory that the driver allocated for the device to query the flow rules, the device
1467 * will first write it with a struct of gve_query_flow_rules_descriptor. Next to it, the device
1468 * will write an array of rules or rule ids with the count that specified in the descriptor.
1469 * For GVE_FLOW_RULE_QUERY_STATS, the device will only write the descriptor.
1470 */
1471static int gve_adminq_process_flow_rules_query(struct gve_priv *priv, u16 query_opcode,
1472					       struct gve_query_flow_rules_descriptor *descriptor)
1473{
1474	struct gve_flow_rules_cache *flow_rules_cache = &priv->flow_rules_cache;
1475	u32 num_queried_rules, total_memory_len, rule_info_len;
1476	void *rule_info;
1477
1478	total_memory_len = be32_to_cpu(descriptor->total_length);
1479	num_queried_rules = be32_to_cpu(descriptor->num_queried_rules);
1480	rule_info = (void *)(descriptor + 1);
1481
1482	switch (query_opcode) {
1483	case GVE_FLOW_RULE_QUERY_RULES:
1484		rule_info_len = num_queried_rules * sizeof(*flow_rules_cache->rules_cache);
1485		if (sizeof(*descriptor) + rule_info_len != total_memory_len) {
1486			dev_err(&priv->dev->dev, "flow rules query is out of memory.\n");
1487			return -ENOMEM;
1488		}
1489
1490		memcpy(flow_rules_cache->rules_cache, rule_info, rule_info_len);
1491		flow_rules_cache->rules_cache_num = num_queried_rules;
1492		break;
1493	case GVE_FLOW_RULE_QUERY_IDS:
1494		rule_info_len = num_queried_rules * sizeof(*flow_rules_cache->rule_ids_cache);
1495		if (sizeof(*descriptor) + rule_info_len != total_memory_len) {
1496			dev_err(&priv->dev->dev, "flow rule ids query is out of memory.\n");
1497			return -ENOMEM;
1498		}
1499
1500		memcpy(flow_rules_cache->rule_ids_cache, rule_info, rule_info_len);
1501		flow_rules_cache->rule_ids_cache_num = num_queried_rules;
1502		break;
1503	case GVE_FLOW_RULE_QUERY_STATS:
1504		priv->num_flow_rules = be32_to_cpu(descriptor->num_flow_rules);
1505		priv->max_flow_rules = be32_to_cpu(descriptor->max_flow_rules);
1506		return 0;
1507	default:
1508		return -EINVAL;
1509	}
1510
1511	return  0;
1512}
1513
1514int gve_adminq_query_flow_rules(struct gve_priv *priv, u16 query_opcode, u32 starting_loc)
1515{
1516	struct gve_query_flow_rules_descriptor *descriptor;
1517	union gve_adminq_command cmd;
1518	dma_addr_t descriptor_bus;
1519	int err = 0;
1520
1521	memset(&cmd, 0, sizeof(cmd));
1522	descriptor = dma_pool_alloc(priv->adminq_pool, GFP_KERNEL, &descriptor_bus);
1523	if (!descriptor)
1524		return -ENOMEM;
1525
1526	cmd.opcode = cpu_to_be32(GVE_ADMINQ_QUERY_FLOW_RULES);
1527	cmd.query_flow_rules = (struct gve_adminq_query_flow_rules) {
1528		.opcode = cpu_to_be16(query_opcode),
1529		.starting_rule_id = cpu_to_be32(starting_loc),
1530		.available_length = cpu_to_be64(GVE_ADMINQ_BUFFER_SIZE),
1531		.rule_descriptor_addr = cpu_to_be64(descriptor_bus),
1532	};
1533	err = gve_adminq_execute_cmd(priv, &cmd);
1534	if (err)
1535		goto out;
1536
1537	err = gve_adminq_process_flow_rules_query(priv, query_opcode, descriptor);
1538
1539out:
1540	dma_pool_free(priv->adminq_pool, descriptor, descriptor_bus);
1541	return err;
1542}
1543
1544static int gve_adminq_process_rss_query(struct gve_priv *priv,
1545					struct gve_query_rss_descriptor *descriptor,
1546					struct ethtool_rxfh_param *rxfh)
1547{
1548	u32 total_memory_length;
1549	u16 hash_lut_length;
1550	void *rss_info_addr;
1551	__be32 *lut;
1552	u16 i;
1553
1554	total_memory_length = be32_to_cpu(descriptor->total_length);
1555	hash_lut_length = priv->rss_lut_size * sizeof(*rxfh->indir);
1556
1557	if (sizeof(*descriptor) + priv->rss_key_size + hash_lut_length != total_memory_length) {
1558		dev_err(&priv->dev->dev,
1559			"rss query desc from device has invalid length parameter.\n");
1560		return -EINVAL;
1561	}
1562
1563	rxfh->hfunc = descriptor->hash_alg;
1564
1565	rss_info_addr = (void *)(descriptor + 1);
1566	if (rxfh->key) {
1567		rxfh->key_size = priv->rss_key_size;
1568		memcpy(rxfh->key, rss_info_addr, priv->rss_key_size);
1569	}
1570
1571	rss_info_addr += priv->rss_key_size;
1572	lut = (__be32 *)rss_info_addr;
1573	if (rxfh->indir) {
1574		rxfh->indir_size = priv->rss_lut_size;
1575		for (i = 0; i < priv->rss_lut_size; i++)
1576			rxfh->indir[i] = be32_to_cpu(lut[i]);
1577	}
1578
1579	return 0;
1580}
1581
1582int gve_adminq_query_rss_config(struct gve_priv *priv, struct ethtool_rxfh_param *rxfh)
1583{
1584	struct gve_query_rss_descriptor *descriptor;
1585	union gve_adminq_command cmd;
1586	dma_addr_t descriptor_bus;
1587	int err = 0;
1588
1589	descriptor = dma_pool_alloc(priv->adminq_pool, GFP_KERNEL, &descriptor_bus);
1590	if (!descriptor)
1591		return -ENOMEM;
1592
1593	memset(&cmd, 0, sizeof(cmd));
1594	cmd.opcode = cpu_to_be32(GVE_ADMINQ_QUERY_RSS);
1595	cmd.query_rss = (struct gve_adminq_query_rss) {
1596		.available_length = cpu_to_be64(GVE_ADMINQ_BUFFER_SIZE),
1597		.rss_descriptor_addr = cpu_to_be64(descriptor_bus),
1598	};
1599	err = gve_adminq_execute_cmd(priv, &cmd);
1600	if (err)
1601		goto out;
1602
1603	err = gve_adminq_process_rss_query(priv, descriptor, rxfh);
1604
1605out:
1606	dma_pool_free(priv->adminq_pool, descriptor, descriptor_bus);
1607	return err;
1608}