idpf_lib.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/* Copyright (C) 2023 Intel Corporation */
   3
   4#include "idpf.h"
   5#include "idpf_virtchnl.h"
   6#include "idpf_ptp.h"
   7#include "xdp.h"
   8#include "xsk.h"
   9
  10static const struct net_device_ops idpf_netdev_ops;
  11
  12/**
  13 * idpf_init_vector_stack - Fill the MSIX vector stack with vector index
  14 * @adapter: private data struct
  15 *
  16 * Return 0 on success, error on failure
  17 */
  18static int idpf_init_vector_stack(struct idpf_adapter *adapter)
  19{
  20	struct idpf_vector_lifo *stack;
  21	u16 min_vec;
  22	u32 i;
  23
  24	mutex_lock(&adapter->vector_lock);
  25	min_vec = adapter->num_msix_entries - adapter->num_avail_msix;
  26	stack = &adapter->vector_stack;
  27	stack->size = adapter->num_msix_entries;
  28	/* set the base and top to point at start of the 'free pool' to
  29	 * distribute the unused vectors on-demand basis
  30	 */
  31	stack->base = min_vec;
  32	stack->top = min_vec;
  33
  34	stack->vec_idx = kcalloc(stack->size, sizeof(u16), GFP_KERNEL);
  35	if (!stack->vec_idx) {
  36		mutex_unlock(&adapter->vector_lock);
  37
  38		return -ENOMEM;
  39	}
  40
  41	for (i = 0; i < stack->size; i++)
  42		stack->vec_idx[i] = i;
  43
  44	mutex_unlock(&adapter->vector_lock);
  45
  46	return 0;
  47}
  48
  49/**
  50 * idpf_deinit_vector_stack - zero out the MSIX vector stack
  51 * @adapter: private data struct
  52 */
  53static void idpf_deinit_vector_stack(struct idpf_adapter *adapter)
  54{
  55	struct idpf_vector_lifo *stack;
  56
  57	mutex_lock(&adapter->vector_lock);
  58	stack = &adapter->vector_stack;
  59	kfree(stack->vec_idx);
  60	stack->vec_idx = NULL;
  61	mutex_unlock(&adapter->vector_lock);
  62}
  63
  64/**
  65 * idpf_mb_intr_rel_irq - Free the IRQ association with the OS
  66 * @adapter: adapter structure
  67 *
  68 * This will also disable interrupt mode and queue up mailbox task. Mailbox
  69 * task will reschedule itself if not in interrupt mode.
  70 */
  71static void idpf_mb_intr_rel_irq(struct idpf_adapter *adapter)
  72{
  73	clear_bit(IDPF_MB_INTR_MODE, adapter->flags);
  74	kfree(free_irq(adapter->msix_entries[0].vector, adapter));
  75	queue_delayed_work(adapter->mbx_wq, &adapter->mbx_task, 0);
  76}
  77
  78/**
  79 * idpf_intr_rel - Release interrupt capabilities and free memory
  80 * @adapter: adapter to disable interrupts on
  81 */
  82void idpf_intr_rel(struct idpf_adapter *adapter)
  83{
  84	if (!adapter->msix_entries)
  85		return;
  86
  87	idpf_mb_intr_rel_irq(adapter);
  88	pci_free_irq_vectors(adapter->pdev);
  89	idpf_send_dealloc_vectors_msg(adapter);
  90	idpf_deinit_vector_stack(adapter);
  91	kfree(adapter->msix_entries);
  92	adapter->msix_entries = NULL;
  93	kfree(adapter->rdma_msix_entries);
  94	adapter->rdma_msix_entries = NULL;
  95}
  96
  97/**
  98 * idpf_mb_intr_clean - Interrupt handler for the mailbox
  99 * @irq: interrupt number
 100 * @data: pointer to the adapter structure
 101 */
 102static irqreturn_t idpf_mb_intr_clean(int __always_unused irq, void *data)
 103{
 104	struct idpf_adapter *adapter = (struct idpf_adapter *)data;
 105
 106	queue_delayed_work(adapter->mbx_wq, &adapter->mbx_task, 0);
 107
 108	return IRQ_HANDLED;
 109}
 110
 111/**
 112 * idpf_mb_irq_enable - Enable MSIX interrupt for the mailbox
 113 * @adapter: adapter to get the hardware address for register write
 114 */
 115static void idpf_mb_irq_enable(struct idpf_adapter *adapter)
 116{
 117	struct idpf_intr_reg *intr = &adapter->mb_vector.intr_reg;
 118	u32 val;
 119
 120	val = intr->dyn_ctl_intena_m | intr->dyn_ctl_itridx_m;
 121	writel(val, intr->dyn_ctl);
 122	writel(intr->icr_ena_ctlq_m, intr->icr_ena);
 123}
 124
 125/**
 126 * idpf_mb_intr_req_irq - Request irq for the mailbox interrupt
 127 * @adapter: adapter structure to pass to the mailbox irq handler
 128 */
 129static int idpf_mb_intr_req_irq(struct idpf_adapter *adapter)
 130{
 131	int irq_num, mb_vidx = 0, err;
 132	char *name;
 133
 134	irq_num = adapter->msix_entries[mb_vidx].vector;
 135	name = kasprintf(GFP_KERNEL, "%s-%s-%d",
 136			 dev_driver_string(&adapter->pdev->dev),
 137			 "Mailbox", mb_vidx);
 138	err = request_irq(irq_num, adapter->irq_mb_handler, 0, name, adapter);
 139	if (err) {
 140		dev_err(&adapter->pdev->dev,
 141			"IRQ request for mailbox failed, error: %d\n", err);
 142
 143		return err;
 144	}
 145
 146	set_bit(IDPF_MB_INTR_MODE, adapter->flags);
 147
 148	return 0;
 149}
 150
 151/**
 152 * idpf_mb_intr_init - Initialize the mailbox interrupt
 153 * @adapter: adapter structure to store the mailbox vector
 154 */
 155static int idpf_mb_intr_init(struct idpf_adapter *adapter)
 156{
 157	adapter->dev_ops.reg_ops.mb_intr_reg_init(adapter);
 158	adapter->irq_mb_handler = idpf_mb_intr_clean;
 159
 160	return idpf_mb_intr_req_irq(adapter);
 161}
 162
 163/**
 164 * idpf_vector_lifo_push - push MSIX vector index onto stack
 165 * @adapter: private data struct
 166 * @vec_idx: vector index to store
 167 */
 168static int idpf_vector_lifo_push(struct idpf_adapter *adapter, u16 vec_idx)
 169{
 170	struct idpf_vector_lifo *stack = &adapter->vector_stack;
 171
 172	lockdep_assert_held(&adapter->vector_lock);
 173
 174	if (stack->top == stack->base) {
 175		dev_err(&adapter->pdev->dev, "Exceeded the vector stack limit: %d\n",
 176			stack->top);
 177		return -EINVAL;
 178	}
 179
 180	stack->vec_idx[--stack->top] = vec_idx;
 181
 182	return 0;
 183}
 184
 185/**
 186 * idpf_vector_lifo_pop - pop MSIX vector index from stack
 187 * @adapter: private data struct
 188 */
 189static int idpf_vector_lifo_pop(struct idpf_adapter *adapter)
 190{
 191	struct idpf_vector_lifo *stack = &adapter->vector_stack;
 192
 193	lockdep_assert_held(&adapter->vector_lock);
 194
 195	if (stack->top == stack->size) {
 196		dev_err(&adapter->pdev->dev, "No interrupt vectors are available to distribute!\n");
 197
 198		return -EINVAL;
 199	}
 200
 201	return stack->vec_idx[stack->top++];
 202}
 203
 204/**
 205 * idpf_vector_stash - Store the vector indexes onto the stack
 206 * @adapter: private data struct
 207 * @q_vector_idxs: vector index array
 208 * @vec_info: info related to the number of vectors
 209 *
 210 * This function is a no-op if there are no vectors indexes to be stashed
 211 */
 212static void idpf_vector_stash(struct idpf_adapter *adapter, u16 *q_vector_idxs,
 213			      struct idpf_vector_info *vec_info)
 214{
 215	int i, base = 0;
 216	u16 vec_idx;
 217
 218	lockdep_assert_held(&adapter->vector_lock);
 219
 220	if (!vec_info->num_curr_vecs)
 221		return;
 222
 223	/* For default vports, no need to stash vector allocated from the
 224	 * default pool onto the stack
 225	 */
 226	if (vec_info->default_vport)
 227		base = IDPF_MIN_Q_VEC;
 228
 229	for (i = vec_info->num_curr_vecs - 1; i >= base ; i--) {
 230		vec_idx = q_vector_idxs[i];
 231		idpf_vector_lifo_push(adapter, vec_idx);
 232		adapter->num_avail_msix++;
 233	}
 234}
 235
 236/**
 237 * idpf_req_rel_vector_indexes - Request or release MSIX vector indexes
 238 * @adapter: driver specific private structure
 239 * @q_vector_idxs: vector index array
 240 * @vec_info: info related to the number of vectors
 241 *
 242 * This is the core function to distribute the MSIX vectors acquired from the
 243 * OS. It expects the caller to pass the number of vectors required and
 244 * also previously allocated. First, it stashes previously allocated vector
 245 * indexes on to the stack and then figures out if it can allocate requested
 246 * vectors. It can wait on acquiring the mutex lock. If the caller passes 0 as
 247 * requested vectors, then this function just stashes the already allocated
 248 * vectors and returns 0.
 249 *
 250 * Returns actual number of vectors allocated on success, error value on failure
 251 * If 0 is returned, implies the stack has no vectors to allocate which is also
 252 * a failure case for the caller
 253 */
 254int idpf_req_rel_vector_indexes(struct idpf_adapter *adapter,
 255				u16 *q_vector_idxs,
 256				struct idpf_vector_info *vec_info)
 257{
 258	u16 num_req_vecs, num_alloc_vecs = 0, max_vecs;
 259	struct idpf_vector_lifo *stack;
 260	int i, j, vecid;
 261
 262	mutex_lock(&adapter->vector_lock);
 263	stack = &adapter->vector_stack;
 264	num_req_vecs = vec_info->num_req_vecs;
 265
 266	/* Stash interrupt vector indexes onto the stack if required */
 267	idpf_vector_stash(adapter, q_vector_idxs, vec_info);
 268
 269	if (!num_req_vecs)
 270		goto rel_lock;
 271
 272	if (vec_info->default_vport) {
 273		/* As IDPF_MIN_Q_VEC per default vport is put aside in the
 274		 * default pool of the stack, use them for default vports
 275		 */
 276		j = vec_info->index * IDPF_MIN_Q_VEC + IDPF_MBX_Q_VEC;
 277		for (i = 0; i < IDPF_MIN_Q_VEC; i++) {
 278			q_vector_idxs[num_alloc_vecs++] = stack->vec_idx[j++];
 279			num_req_vecs--;
 280		}
 281	}
 282
 283	/* Find if stack has enough vector to allocate */
 284	max_vecs = min(adapter->num_avail_msix, num_req_vecs);
 285
 286	for (j = 0; j < max_vecs; j++) {
 287		vecid = idpf_vector_lifo_pop(adapter);
 288		q_vector_idxs[num_alloc_vecs++] = vecid;
 289	}
 290	adapter->num_avail_msix -= max_vecs;
 291
 292rel_lock:
 293	mutex_unlock(&adapter->vector_lock);
 294
 295	return num_alloc_vecs;
 296}
 297
 298/**
 299 * idpf_intr_req - Request interrupt capabilities
 300 * @adapter: adapter to enable interrupts on
 301 *
 302 * Returns 0 on success, negative on failure
 303 */
 304int idpf_intr_req(struct idpf_adapter *adapter)
 305{
 306	u16 num_lan_vecs, min_lan_vecs, num_rdma_vecs = 0, min_rdma_vecs = 0;
 307	u16 default_vports = idpf_get_default_vports(adapter);
 308	int num_q_vecs, total_vecs, num_vec_ids;
 309	int min_vectors, actual_vecs, err;
 310	unsigned int vector;
 311	u16 *vecids;
 312	int i;
 313
 314	total_vecs = idpf_get_reserved_vecs(adapter);
 315	num_lan_vecs = total_vecs;
 316	if (idpf_is_rdma_cap_ena(adapter)) {
 317		num_rdma_vecs = idpf_get_reserved_rdma_vecs(adapter);
 318		min_rdma_vecs = IDPF_MIN_RDMA_VEC;
 319
 320		if (!num_rdma_vecs) {
 321			/* If idpf_get_reserved_rdma_vecs is 0, vectors are
 322			 * pulled from the LAN pool.
 323			 */
 324			num_rdma_vecs = min_rdma_vecs;
 325		} else if (num_rdma_vecs < min_rdma_vecs) {
 326			dev_err(&adapter->pdev->dev,
 327				"Not enough vectors reserved for RDMA (min: %u, current: %u)\n",
 328				min_rdma_vecs, num_rdma_vecs);
 329			return -EINVAL;
 330		}
 331	}
 332
 333	num_q_vecs = total_vecs - IDPF_MBX_Q_VEC;
 334
 335	err = idpf_send_alloc_vectors_msg(adapter, num_q_vecs);
 336	if (err) {
 337		dev_err(&adapter->pdev->dev,
 338			"Failed to allocate %d vectors: %d\n", num_q_vecs, err);
 339
 340		return -EAGAIN;
 341	}
 342
 343	min_lan_vecs = IDPF_MBX_Q_VEC + IDPF_MIN_Q_VEC * default_vports;
 344	min_vectors = min_lan_vecs + min_rdma_vecs;
 345	actual_vecs = pci_alloc_irq_vectors(adapter->pdev, min_vectors,
 346					    total_vecs, PCI_IRQ_MSIX);
 347	if (actual_vecs < 0) {
 348		dev_err(&adapter->pdev->dev, "Failed to allocate minimum MSIX vectors required: %d\n",
 349			min_vectors);
 350		err = actual_vecs;
 351		goto send_dealloc_vecs;
 352	}
 353
 354	if (idpf_is_rdma_cap_ena(adapter)) {
 355		if (actual_vecs < total_vecs) {
 356			dev_warn(&adapter->pdev->dev,
 357				 "Warning: %d vectors requested, only %d available. Defaulting to minimum (%d) for RDMA and remaining for LAN.\n",
 358				 total_vecs, actual_vecs, IDPF_MIN_RDMA_VEC);
 359			num_rdma_vecs = IDPF_MIN_RDMA_VEC;
 360		}
 361
 362		adapter->rdma_msix_entries = kcalloc(num_rdma_vecs,
 363						     sizeof(struct msix_entry),
 364						     GFP_KERNEL);
 365		if (!adapter->rdma_msix_entries) {
 366			err = -ENOMEM;
 367			goto free_irq;
 368		}
 369	}
 370
 371	num_lan_vecs = actual_vecs - num_rdma_vecs;
 372	adapter->msix_entries = kcalloc(num_lan_vecs, sizeof(struct msix_entry),
 373					GFP_KERNEL);
 374	if (!adapter->msix_entries) {
 375		err = -ENOMEM;
 376		goto free_rdma_msix;
 377	}
 378
 379	adapter->mb_vector.v_idx = le16_to_cpu(adapter->caps.mailbox_vector_id);
 380
 381	vecids = kcalloc(actual_vecs, sizeof(u16), GFP_KERNEL);
 382	if (!vecids) {
 383		err = -ENOMEM;
 384		goto free_msix;
 385	}
 386
 387	num_vec_ids = idpf_get_vec_ids(adapter, vecids, actual_vecs,
 388				       &adapter->req_vec_chunks->vchunks);
 389	if (num_vec_ids < actual_vecs) {
 390		err = -EINVAL;
 391		goto free_vecids;
 392	}
 393
 394	for (vector = 0; vector < num_lan_vecs; vector++) {
 395		adapter->msix_entries[vector].entry = vecids[vector];
 396		adapter->msix_entries[vector].vector =
 397			pci_irq_vector(adapter->pdev, vector);
 398	}
 399	for (i = 0; i < num_rdma_vecs; vector++, i++) {
 400		adapter->rdma_msix_entries[i].entry = vecids[vector];
 401		adapter->rdma_msix_entries[i].vector =
 402			pci_irq_vector(adapter->pdev, vector);
 403	}
 404
 405	/* 'num_avail_msix' is used to distribute excess vectors to the vports
 406	 * after considering the minimum vectors required per each default
 407	 * vport
 408	 */
 409	adapter->num_avail_msix = num_lan_vecs - min_lan_vecs;
 410	adapter->num_msix_entries = num_lan_vecs;
 411	if (idpf_is_rdma_cap_ena(adapter))
 412		adapter->num_rdma_msix_entries = num_rdma_vecs;
 413
 414	/* Fill MSIX vector lifo stack with vector indexes */
 415	err = idpf_init_vector_stack(adapter);
 416	if (err)
 417		goto free_vecids;
 418
 419	err = idpf_mb_intr_init(adapter);
 420	if (err)
 421		goto deinit_vec_stack;
 422	idpf_mb_irq_enable(adapter);
 423	kfree(vecids);
 424
 425	return 0;
 426
 427deinit_vec_stack:
 428	idpf_deinit_vector_stack(adapter);
 429free_vecids:
 430	kfree(vecids);
 431free_msix:
 432	kfree(adapter->msix_entries);
 433	adapter->msix_entries = NULL;
 434free_rdma_msix:
 435	kfree(adapter->rdma_msix_entries);
 436	adapter->rdma_msix_entries = NULL;
 437free_irq:
 438	pci_free_irq_vectors(adapter->pdev);
 439send_dealloc_vecs:
 440	idpf_send_dealloc_vectors_msg(adapter);
 441
 442	return err;
 443}
 444
 445/**
 446 * idpf_del_all_flow_steer_filters - Delete all flow steer filters in list
 447 * @vport: main vport struct
 448 *
 449 * Takes flow_steer_list_lock spinlock.  Deletes all filters
 450 */
 451static void idpf_del_all_flow_steer_filters(struct idpf_vport *vport)
 452{
 453	struct idpf_vport_config *vport_config;
 454	struct idpf_fsteer_fltr *f, *ftmp;
 455
 456	vport_config = vport->adapter->vport_config[vport->idx];
 457
 458	spin_lock_bh(&vport_config->flow_steer_list_lock);
 459	list_for_each_entry_safe(f, ftmp, &vport_config->user_config.flow_steer_list,
 460				 list) {
 461		list_del(&f->list);
 462		kfree(f);
 463	}
 464	vport_config->user_config.num_fsteer_fltrs = 0;
 465	spin_unlock_bh(&vport_config->flow_steer_list_lock);
 466}
 467
 468/**
 469 * idpf_find_mac_filter - Search filter list for specific mac filter
 470 * @vconfig: Vport config structure
 471 * @macaddr: The MAC address
 472 *
 473 * Returns ptr to the filter object or NULL. Must be called while holding the
 474 * mac_filter_list_lock.
 475 **/
 476static struct idpf_mac_filter *idpf_find_mac_filter(struct idpf_vport_config *vconfig,
 477						    const u8 *macaddr)
 478{
 479	struct idpf_mac_filter *f;
 480
 481	if (!macaddr)
 482		return NULL;
 483
 484	list_for_each_entry(f, &vconfig->user_config.mac_filter_list, list) {
 485		if (ether_addr_equal(macaddr, f->macaddr))
 486			return f;
 487	}
 488
 489	return NULL;
 490}
 491
 492/**
 493 * __idpf_del_mac_filter - Delete a MAC filter from the filter list
 494 * @vport_config: Vport config structure
 495 * @macaddr: The MAC address
 496 *
 497 * Returns 0 on success, error value on failure
 498 **/
 499static int __idpf_del_mac_filter(struct idpf_vport_config *vport_config,
 500				 const u8 *macaddr)
 501{
 502	struct idpf_mac_filter *f;
 503
 504	spin_lock_bh(&vport_config->mac_filter_list_lock);
 505	f = idpf_find_mac_filter(vport_config, macaddr);
 506	if (f) {
 507		list_del(&f->list);
 508		kfree(f);
 509	}
 510	spin_unlock_bh(&vport_config->mac_filter_list_lock);
 511
 512	return 0;
 513}
 514
 515/**
 516 * idpf_del_mac_filter - Delete a MAC filter from the filter list
 517 * @vport: Main vport structure
 518 * @np: Netdev private structure
 519 * @macaddr: The MAC address
 520 * @async: Don't wait for return message
 521 *
 522 * Removes filter from list and if interface is up, tells hardware about the
 523 * removed filter.
 524 **/
 525static int idpf_del_mac_filter(struct idpf_vport *vport,
 526			       struct idpf_netdev_priv *np,
 527			       const u8 *macaddr, bool async)
 528{
 529	struct idpf_vport_config *vport_config;
 530	struct idpf_mac_filter *f;
 531
 532	vport_config = np->adapter->vport_config[np->vport_idx];
 533
 534	spin_lock_bh(&vport_config->mac_filter_list_lock);
 535	f = idpf_find_mac_filter(vport_config, macaddr);
 536	if (f) {
 537		f->remove = true;
 538	} else {
 539		spin_unlock_bh(&vport_config->mac_filter_list_lock);
 540
 541		return -EINVAL;
 542	}
 543	spin_unlock_bh(&vport_config->mac_filter_list_lock);
 544
 545	if (test_bit(IDPF_VPORT_UP, np->state)) {
 546		int err;
 547
 548		err = idpf_add_del_mac_filters(vport, np, false, async);
 549		if (err)
 550			return err;
 551	}
 552
 553	return  __idpf_del_mac_filter(vport_config, macaddr);
 554}
 555
 556/**
 557 * __idpf_add_mac_filter - Add mac filter helper function
 558 * @vport_config: Vport config structure
 559 * @macaddr: Address to add
 560 *
 561 * Takes mac_filter_list_lock spinlock to add new filter to list.
 562 */
 563static int __idpf_add_mac_filter(struct idpf_vport_config *vport_config,
 564				 const u8 *macaddr)
 565{
 566	struct idpf_mac_filter *f;
 567
 568	spin_lock_bh(&vport_config->mac_filter_list_lock);
 569
 570	f = idpf_find_mac_filter(vport_config, macaddr);
 571	if (f) {
 572		f->remove = false;
 573		spin_unlock_bh(&vport_config->mac_filter_list_lock);
 574
 575		return 0;
 576	}
 577
 578	f = kzalloc(sizeof(*f), GFP_ATOMIC);
 579	if (!f) {
 580		spin_unlock_bh(&vport_config->mac_filter_list_lock);
 581
 582		return -ENOMEM;
 583	}
 584
 585	ether_addr_copy(f->macaddr, macaddr);
 586	list_add_tail(&f->list, &vport_config->user_config.mac_filter_list);
 587	f->add = true;
 588
 589	spin_unlock_bh(&vport_config->mac_filter_list_lock);
 590
 591	return 0;
 592}
 593
 594/**
 595 * idpf_add_mac_filter - Add a mac filter to the filter list
 596 * @vport: Main vport structure
 597 * @np: Netdev private structure
 598 * @macaddr: The MAC address
 599 * @async: Don't wait for return message
 600 *
 601 * Returns 0 on success or error on failure. If interface is up, we'll also
 602 * send the virtchnl message to tell hardware about the filter.
 603 **/
 604static int idpf_add_mac_filter(struct idpf_vport *vport,
 605			       struct idpf_netdev_priv *np,
 606			       const u8 *macaddr, bool async)
 607{
 608	struct idpf_vport_config *vport_config;
 609	int err;
 610
 611	vport_config = np->adapter->vport_config[np->vport_idx];
 612	err = __idpf_add_mac_filter(vport_config, macaddr);
 613	if (err)
 614		return err;
 615
 616	if (test_bit(IDPF_VPORT_UP, np->state))
 617		err = idpf_add_del_mac_filters(vport, np, true, async);
 618
 619	return err;
 620}
 621
 622/**
 623 * idpf_del_all_mac_filters - Delete all MAC filters in list
 624 * @vport: main vport struct
 625 *
 626 * Takes mac_filter_list_lock spinlock.  Deletes all filters
 627 */
 628static void idpf_del_all_mac_filters(struct idpf_vport *vport)
 629{
 630	struct idpf_vport_config *vport_config;
 631	struct idpf_mac_filter *f, *ftmp;
 632
 633	vport_config = vport->adapter->vport_config[vport->idx];
 634	spin_lock_bh(&vport_config->mac_filter_list_lock);
 635
 636	list_for_each_entry_safe(f, ftmp, &vport_config->user_config.mac_filter_list,
 637				 list) {
 638		list_del(&f->list);
 639		kfree(f);
 640	}
 641
 642	spin_unlock_bh(&vport_config->mac_filter_list_lock);
 643}
 644
 645/**
 646 * idpf_restore_mac_filters - Re-add all MAC filters in list
 647 * @vport: main vport struct
 648 *
 649 * Takes mac_filter_list_lock spinlock.  Sets add field to true for filters to
 650 * resync filters back to HW.
 651 */
 652static void idpf_restore_mac_filters(struct idpf_vport *vport)
 653{
 654	struct idpf_vport_config *vport_config;
 655	struct idpf_mac_filter *f;
 656
 657	vport_config = vport->adapter->vport_config[vport->idx];
 658	spin_lock_bh(&vport_config->mac_filter_list_lock);
 659
 660	list_for_each_entry(f, &vport_config->user_config.mac_filter_list, list)
 661		f->add = true;
 662
 663	spin_unlock_bh(&vport_config->mac_filter_list_lock);
 664
 665	idpf_add_del_mac_filters(vport, netdev_priv(vport->netdev),
 666				 true, false);
 667}
 668
 669/**
 670 * idpf_remove_mac_filters - Remove all MAC filters in list
 671 * @vport: main vport struct
 672 *
 673 * Takes mac_filter_list_lock spinlock. Sets remove field to true for filters
 674 * to remove filters in HW.
 675 */
 676static void idpf_remove_mac_filters(struct idpf_vport *vport)
 677{
 678	struct idpf_vport_config *vport_config;
 679	struct idpf_mac_filter *f;
 680
 681	vport_config = vport->adapter->vport_config[vport->idx];
 682	spin_lock_bh(&vport_config->mac_filter_list_lock);
 683
 684	list_for_each_entry(f, &vport_config->user_config.mac_filter_list, list)
 685		f->remove = true;
 686
 687	spin_unlock_bh(&vport_config->mac_filter_list_lock);
 688
 689	idpf_add_del_mac_filters(vport, netdev_priv(vport->netdev),
 690				 false, false);
 691}
 692
 693/**
 694 * idpf_deinit_mac_addr - deinitialize mac address for vport
 695 * @vport: main vport structure
 696 */
 697static void idpf_deinit_mac_addr(struct idpf_vport *vport)
 698{
 699	struct idpf_vport_config *vport_config;
 700	struct idpf_mac_filter *f;
 701
 702	vport_config = vport->adapter->vport_config[vport->idx];
 703
 704	spin_lock_bh(&vport_config->mac_filter_list_lock);
 705
 706	f = idpf_find_mac_filter(vport_config, vport->default_mac_addr);
 707	if (f) {
 708		list_del(&f->list);
 709		kfree(f);
 710	}
 711
 712	spin_unlock_bh(&vport_config->mac_filter_list_lock);
 713}
 714
 715/**
 716 * idpf_init_mac_addr - initialize mac address for vport
 717 * @vport: main vport structure
 718 * @netdev: pointer to netdev struct associated with this vport
 719 */
 720static int idpf_init_mac_addr(struct idpf_vport *vport,
 721			      struct net_device *netdev)
 722{
 723	struct idpf_netdev_priv *np = netdev_priv(netdev);
 724	struct idpf_adapter *adapter = vport->adapter;
 725	int err;
 726
 727	if (is_valid_ether_addr(vport->default_mac_addr)) {
 728		eth_hw_addr_set(netdev, vport->default_mac_addr);
 729		ether_addr_copy(netdev->perm_addr, vport->default_mac_addr);
 730
 731		return idpf_add_mac_filter(vport, np, vport->default_mac_addr,
 732					   false);
 733	}
 734
 735	if (!idpf_is_cap_ena(adapter, IDPF_OTHER_CAPS,
 736			     VIRTCHNL2_CAP_MACFILTER)) {
 737		dev_err(&adapter->pdev->dev,
 738			"MAC address is not provided and capability is not set\n");
 739
 740		return -EINVAL;
 741	}
 742
 743	eth_hw_addr_random(netdev);
 744	err = idpf_add_mac_filter(vport, np, netdev->dev_addr, false);
 745	if (err)
 746		return err;
 747
 748	dev_info(&adapter->pdev->dev, "Invalid MAC address %pM, using random %pM\n",
 749		 vport->default_mac_addr, netdev->dev_addr);
 750	ether_addr_copy(vport->default_mac_addr, netdev->dev_addr);
 751
 752	return 0;
 753}
 754
 755static void idpf_detach_and_close(struct idpf_adapter *adapter)
 756{
 757	int max_vports = adapter->max_vports;
 758
 759	for (int i = 0; i < max_vports; i++) {
 760		struct net_device *netdev = adapter->netdevs[i];
 761
 762		/* If the interface is in detached state, that means the
 763		 * previous reset was not handled successfully for this
 764		 * vport.
 765		 */
 766		if (!netif_device_present(netdev))
 767			continue;
 768
 769		/* Hold RTNL to protect racing with callbacks */
 770		rtnl_lock();
 771		netif_device_detach(netdev);
 772		if (netif_running(netdev)) {
 773			set_bit(IDPF_VPORT_UP_REQUESTED,
 774				adapter->vport_config[i]->flags);
 775			dev_close(netdev);
 776		}
 777		rtnl_unlock();
 778	}
 779}
 780
 781static void idpf_attach_and_open(struct idpf_adapter *adapter)
 782{
 783	int max_vports = adapter->max_vports;
 784
 785	for (int i = 0; i < max_vports; i++) {
 786		struct idpf_vport *vport = adapter->vports[i];
 787		struct idpf_vport_config *vport_config;
 788		struct net_device *netdev;
 789
 790		/* In case of a critical error in the init task, the vport
 791		 * will be freed. Only continue to restore the netdevs
 792		 * if the vport is allocated.
 793		 */
 794		if (!vport)
 795			continue;
 796
 797		/* No need for RTNL on attach as this function is called
 798		 * following detach and dev_close(). We do take RTNL for
 799		 * dev_open() below as it can race with external callbacks
 800		 * following the call to netif_device_attach().
 801		 */
 802		netdev = adapter->netdevs[i];
 803		netif_device_attach(netdev);
 804		vport_config = adapter->vport_config[vport->idx];
 805		if (test_and_clear_bit(IDPF_VPORT_UP_REQUESTED,
 806				       vport_config->flags)) {
 807			rtnl_lock();
 808			dev_open(netdev, NULL);
 809			rtnl_unlock();
 810		}
 811	}
 812}
 813
 814/**
 815 * idpf_cfg_netdev - Allocate, configure and register a netdev
 816 * @vport: main vport structure
 817 *
 818 * Returns 0 on success, negative value on failure.
 819 */
 820static int idpf_cfg_netdev(struct idpf_vport *vport)
 821{
 822	struct idpf_adapter *adapter = vport->adapter;
 823	struct idpf_vport_config *vport_config;
 824	netdev_features_t other_offloads = 0;
 825	netdev_features_t csum_offloads = 0;
 826	netdev_features_t tso_offloads = 0;
 827	netdev_features_t dflt_features;
 828	struct idpf_netdev_priv *np;
 829	struct net_device *netdev;
 830	u16 idx = vport->idx;
 831	int err;
 832
 833	vport_config = adapter->vport_config[idx];
 834
 835	/* It's possible we already have a netdev allocated and registered for
 836	 * this vport
 837	 */
 838	if (test_bit(IDPF_VPORT_REG_NETDEV, vport_config->flags)) {
 839		netdev = adapter->netdevs[idx];
 840		np = netdev_priv(netdev);
 841		np->vport = vport;
 842		np->vport_idx = vport->idx;
 843		np->vport_id = vport->vport_id;
 844		np->max_tx_hdr_size = idpf_get_max_tx_hdr_size(adapter);
 845		vport->netdev = netdev;
 846
 847		return idpf_init_mac_addr(vport, netdev);
 848	}
 849
 850	netdev = alloc_etherdev_mqs(sizeof(struct idpf_netdev_priv),
 851				    vport_config->max_q.max_txq,
 852				    vport_config->max_q.max_rxq);
 853	if (!netdev)
 854		return -ENOMEM;
 855
 856	vport->netdev = netdev;
 857	np = netdev_priv(netdev);
 858	np->vport = vport;
 859	np->adapter = adapter;
 860	np->vport_idx = vport->idx;
 861	np->vport_id = vport->vport_id;
 862	np->max_tx_hdr_size = idpf_get_max_tx_hdr_size(adapter);
 863	np->tx_max_bufs = idpf_get_max_tx_bufs(adapter);
 864
 865	spin_lock_init(&np->stats_lock);
 866
 867	err = idpf_init_mac_addr(vport, netdev);
 868	if (err) {
 869		free_netdev(vport->netdev);
 870		vport->netdev = NULL;
 871
 872		return err;
 873	}
 874
 875	/* assign netdev_ops */
 876	netdev->netdev_ops = &idpf_netdev_ops;
 877
 878	/* setup watchdog timeout value to be 5 second */
 879	netdev->watchdog_timeo = 5 * HZ;
 880
 881	netdev->dev_port = idx;
 882
 883	/* configure default MTU size */
 884	netdev->min_mtu = ETH_MIN_MTU;
 885	netdev->max_mtu = vport->max_mtu;
 886
 887	dflt_features = NETIF_F_SG	|
 888			NETIF_F_HIGHDMA;
 889
 890	if (idpf_is_cap_ena_all(adapter, IDPF_RSS_CAPS, IDPF_CAP_RSS))
 891		dflt_features |= NETIF_F_RXHASH;
 892	if (idpf_is_cap_ena(adapter, IDPF_OTHER_CAPS,
 893			    VIRTCHNL2_CAP_FLOW_STEER) &&
 894	    idpf_vport_is_cap_ena(vport, VIRTCHNL2_VPORT_SIDEBAND_FLOW_STEER))
 895		dflt_features |= NETIF_F_NTUPLE;
 896	if (idpf_is_cap_ena_all(adapter, IDPF_CSUM_CAPS, IDPF_CAP_TX_CSUM_L4V4))
 897		csum_offloads |= NETIF_F_IP_CSUM;
 898	if (idpf_is_cap_ena_all(adapter, IDPF_CSUM_CAPS, IDPF_CAP_TX_CSUM_L4V6))
 899		csum_offloads |= NETIF_F_IPV6_CSUM;
 900	if (idpf_is_cap_ena(adapter, IDPF_CSUM_CAPS, IDPF_CAP_RX_CSUM))
 901		csum_offloads |= NETIF_F_RXCSUM;
 902	if (idpf_is_cap_ena_all(adapter, IDPF_CSUM_CAPS, IDPF_CAP_TX_SCTP_CSUM))
 903		csum_offloads |= NETIF_F_SCTP_CRC;
 904
 905	if (idpf_is_cap_ena(adapter, IDPF_SEG_CAPS, VIRTCHNL2_CAP_SEG_IPV4_TCP))
 906		tso_offloads |= NETIF_F_TSO;
 907	if (idpf_is_cap_ena(adapter, IDPF_SEG_CAPS, VIRTCHNL2_CAP_SEG_IPV6_TCP))
 908		tso_offloads |= NETIF_F_TSO6;
 909	if (idpf_is_cap_ena_all(adapter, IDPF_SEG_CAPS,
 910				VIRTCHNL2_CAP_SEG_IPV4_UDP |
 911				VIRTCHNL2_CAP_SEG_IPV6_UDP))
 912		tso_offloads |= NETIF_F_GSO_UDP_L4;
 913	if (idpf_is_cap_ena_all(adapter, IDPF_RSC_CAPS, IDPF_CAP_RSC))
 914		other_offloads |= NETIF_F_GRO_HW;
 915	if (idpf_is_cap_ena(adapter, IDPF_OTHER_CAPS, VIRTCHNL2_CAP_LOOPBACK))
 916		other_offloads |= NETIF_F_LOOPBACK;
 917
 918	netdev->features |= dflt_features | csum_offloads | tso_offloads;
 919	netdev->hw_features |=  netdev->features | other_offloads;
 920	netdev->vlan_features |= netdev->features | other_offloads;
 921	netdev->hw_enc_features |= dflt_features | other_offloads;
 922	idpf_xdp_set_features(vport);
 923
 924	idpf_set_ethtool_ops(netdev);
 925	netif_set_affinity_auto(netdev);
 926	SET_NETDEV_DEV(netdev, &adapter->pdev->dev);
 927
 928	/* carrier off on init to avoid Tx hangs */
 929	netif_carrier_off(netdev);
 930
 931	/* make sure transmit queues start off as stopped */
 932	netif_tx_stop_all_queues(netdev);
 933
 934	/* The vport can be arbitrarily released so we need to also track
 935	 * netdevs in the adapter struct
 936	 */
 937	adapter->netdevs[idx] = netdev;
 938
 939	return 0;
 940}
 941
 942/**
 943 * idpf_get_free_slot - get the next non-NULL location index in array
 944 * @adapter: adapter in which to look for a free vport slot
 945 */
 946static int idpf_get_free_slot(struct idpf_adapter *adapter)
 947{
 948	unsigned int i;
 949
 950	for (i = 0; i < adapter->max_vports; i++) {
 951		if (!adapter->vports[i])
 952			return i;
 953	}
 954
 955	return IDPF_NO_FREE_SLOT;
 956}
 957
 958/**
 959 * idpf_remove_features - Turn off feature configs
 960 * @vport: virtual port structure
 961 */
 962static void idpf_remove_features(struct idpf_vport *vport)
 963{
 964	struct idpf_adapter *adapter = vport->adapter;
 965
 966	if (idpf_is_cap_ena(adapter, IDPF_OTHER_CAPS, VIRTCHNL2_CAP_MACFILTER))
 967		idpf_remove_mac_filters(vport);
 968}
 969
 970/**
 971 * idpf_vport_stop - Disable a vport
 972 * @vport: vport to disable
 973 * @rtnl: whether to take RTNL lock
 974 */
 975static void idpf_vport_stop(struct idpf_vport *vport, bool rtnl)
 976{
 977	struct idpf_netdev_priv *np = netdev_priv(vport->netdev);
 978
 979	if (!test_bit(IDPF_VPORT_UP, np->state))
 980		return;
 981
 982	if (rtnl)
 983		rtnl_lock();
 984
 985	netif_carrier_off(vport->netdev);
 986	netif_tx_disable(vport->netdev);
 987
 988	idpf_send_disable_vport_msg(vport);
 989	idpf_send_disable_queues_msg(vport);
 990	idpf_send_map_unmap_queue_vector_msg(vport, false);
 991	/* Normally we ask for queues in create_vport, but if the number of
 992	 * initially requested queues have changed, for example via ethtool
 993	 * set channels, we do delete queues and then add the queues back
 994	 * instead of deleting and reallocating the vport.
 995	 */
 996	if (test_and_clear_bit(IDPF_VPORT_DEL_QUEUES, vport->flags))
 997		idpf_send_delete_queues_msg(vport);
 998
 999	idpf_remove_features(vport);
1000
1001	vport->link_up = false;
1002	idpf_vport_intr_deinit(vport);
1003	idpf_xdp_rxq_info_deinit_all(vport);
1004	idpf_vport_queues_rel(vport);
1005	idpf_vport_intr_rel(vport);
1006	clear_bit(IDPF_VPORT_UP, np->state);
1007
1008	if (rtnl)
1009		rtnl_unlock();
1010}
1011
1012/**
1013 * idpf_stop - Disables a network interface
1014 * @netdev: network interface device structure
1015 *
1016 * The stop entry point is called when an interface is de-activated by the OS,
1017 * and the netdevice enters the DOWN state.  The hardware is still under the
1018 * driver's control, but the netdev interface is disabled.
1019 *
1020 * Returns success only - not allowed to fail
1021 */
1022static int idpf_stop(struct net_device *netdev)
1023{
1024	struct idpf_netdev_priv *np = netdev_priv(netdev);
1025	struct idpf_vport *vport;
1026
1027	if (test_bit(IDPF_REMOVE_IN_PROG, np->adapter->flags))
1028		return 0;
1029
1030	idpf_vport_ctrl_lock(netdev);
1031	vport = idpf_netdev_to_vport(netdev);
1032
1033	idpf_vport_stop(vport, false);
1034
1035	idpf_vport_ctrl_unlock(netdev);
1036
1037	return 0;
1038}
1039
1040/**
1041 * idpf_decfg_netdev - Unregister the netdev
1042 * @vport: vport for which netdev to be unregistered
1043 */
1044static void idpf_decfg_netdev(struct idpf_vport *vport)
1045{
1046	struct idpf_adapter *adapter = vport->adapter;
1047	u16 idx = vport->idx;
1048
1049	kfree(vport->rx_ptype_lkup);
1050	vport->rx_ptype_lkup = NULL;
1051
1052	if (test_and_clear_bit(IDPF_VPORT_REG_NETDEV,
1053			       adapter->vport_config[idx]->flags)) {
1054		unregister_netdev(vport->netdev);
1055		free_netdev(vport->netdev);
1056	}
1057	vport->netdev = NULL;
1058
1059	adapter->netdevs[idx] = NULL;
1060}
1061
1062/**
1063 * idpf_vport_rel - Delete a vport and free its resources
1064 * @vport: the vport being removed
1065 */
1066static void idpf_vport_rel(struct idpf_vport *vport)
1067{
1068	struct idpf_adapter *adapter = vport->adapter;
1069	struct idpf_vport_config *vport_config;
1070	struct idpf_vector_info vec_info;
1071	struct idpf_rss_data *rss_data;
1072	struct idpf_vport_max_q max_q;
1073	u16 idx = vport->idx;
1074
1075	vport_config = adapter->vport_config[vport->idx];
1076	idpf_deinit_rss_lut(vport);
1077	rss_data = &vport_config->user_config.rss_data;
1078	kfree(rss_data->rss_key);
1079	rss_data->rss_key = NULL;
1080
1081	idpf_send_destroy_vport_msg(vport);
1082
1083	/* Release all max queues allocated to the adapter's pool */
1084	max_q.max_rxq = vport_config->max_q.max_rxq;
1085	max_q.max_txq = vport_config->max_q.max_txq;
1086	max_q.max_bufq = vport_config->max_q.max_bufq;
1087	max_q.max_complq = vport_config->max_q.max_complq;
1088	idpf_vport_dealloc_max_qs(adapter, &max_q);
1089
1090	/* Release all the allocated vectors on the stack */
1091	vec_info.num_req_vecs = 0;
1092	vec_info.num_curr_vecs = vport->num_q_vectors;
1093	vec_info.default_vport = vport->default_vport;
1094
1095	idpf_req_rel_vector_indexes(adapter, vport->q_vector_idxs, &vec_info);
1096
1097	kfree(vport->q_vector_idxs);
1098	vport->q_vector_idxs = NULL;
1099
1100	kfree(adapter->vport_params_recvd[idx]);
1101	adapter->vport_params_recvd[idx] = NULL;
1102	kfree(adapter->vport_params_reqd[idx]);
1103	adapter->vport_params_reqd[idx] = NULL;
1104	if (adapter->vport_config[idx]) {
1105		kfree(adapter->vport_config[idx]->req_qs_chunks);
1106		adapter->vport_config[idx]->req_qs_chunks = NULL;
1107	}
1108	kfree(vport->rx_ptype_lkup);
1109	vport->rx_ptype_lkup = NULL;
1110	kfree(vport);
1111	adapter->num_alloc_vports--;
1112}
1113
1114/**
1115 * idpf_vport_dealloc - cleanup and release a given vport
1116 * @vport: pointer to idpf vport structure
1117 *
1118 * returns nothing
1119 */
1120static void idpf_vport_dealloc(struct idpf_vport *vport)
1121{
1122	struct idpf_adapter *adapter = vport->adapter;
1123	unsigned int i = vport->idx;
1124
1125	idpf_idc_deinit_vport_aux_device(vport->vdev_info);
1126
1127	idpf_deinit_mac_addr(vport);
1128
1129	if (!test_bit(IDPF_HR_RESET_IN_PROG, adapter->flags)) {
1130		idpf_vport_stop(vport, true);
1131		idpf_decfg_netdev(vport);
1132	}
1133	if (test_bit(IDPF_REMOVE_IN_PROG, adapter->flags)) {
1134		idpf_del_all_mac_filters(vport);
1135		idpf_del_all_flow_steer_filters(vport);
1136	}
1137
1138	if (adapter->netdevs[i]) {
1139		struct idpf_netdev_priv *np = netdev_priv(adapter->netdevs[i]);
1140
1141		np->vport = NULL;
1142	}
1143
1144	idpf_vport_rel(vport);
1145
1146	adapter->vports[i] = NULL;
1147	adapter->next_vport = idpf_get_free_slot(adapter);
1148}
1149
1150/**
1151 * idpf_is_hsplit_supported - check whether the header split is supported
1152 * @vport: virtual port to check the capability for
1153 *
1154 * Return: true if it's supported by the HW/FW, false if not.
1155 */
1156static bool idpf_is_hsplit_supported(const struct idpf_vport *vport)
1157{
1158	return idpf_is_queue_model_split(vport->rxq_model) &&
1159	       idpf_is_cap_ena_all(vport->adapter, IDPF_HSPLIT_CAPS,
1160				   IDPF_CAP_HSPLIT);
1161}
1162
1163/**
1164 * idpf_vport_get_hsplit - get the current header split feature state
1165 * @vport: virtual port to query the state for
1166 *
1167 * Return: ``ETHTOOL_TCP_DATA_SPLIT_UNKNOWN`` if not supported,
1168 *         ``ETHTOOL_TCP_DATA_SPLIT_DISABLED`` if disabled,
1169 *         ``ETHTOOL_TCP_DATA_SPLIT_ENABLED`` if active.
1170 */
1171u8 idpf_vport_get_hsplit(const struct idpf_vport *vport)
1172{
1173	const struct idpf_vport_user_config_data *config;
1174
1175	if (!idpf_is_hsplit_supported(vport))
1176		return ETHTOOL_TCP_DATA_SPLIT_UNKNOWN;
1177
1178	config = &vport->adapter->vport_config[vport->idx]->user_config;
1179
1180	return test_bit(__IDPF_USER_FLAG_HSPLIT, config->user_flags) ?
1181	       ETHTOOL_TCP_DATA_SPLIT_ENABLED :
1182	       ETHTOOL_TCP_DATA_SPLIT_DISABLED;
1183}
1184
1185/**
1186 * idpf_vport_set_hsplit - enable or disable header split on a given vport
1187 * @vport: virtual port to configure
1188 * @val: Ethtool flag controlling the header split state
1189 *
1190 * Return: true on success, false if not supported by the HW.
1191 */
1192bool idpf_vport_set_hsplit(const struct idpf_vport *vport, u8 val)
1193{
1194	struct idpf_vport_user_config_data *config;
1195
1196	if (!idpf_is_hsplit_supported(vport))
1197		return val == ETHTOOL_TCP_DATA_SPLIT_UNKNOWN;
1198
1199	config = &vport->adapter->vport_config[vport->idx]->user_config;
1200
1201	switch (val) {
1202	case ETHTOOL_TCP_DATA_SPLIT_UNKNOWN:
1203		/* Default is to enable */
1204	case ETHTOOL_TCP_DATA_SPLIT_ENABLED:
1205		__set_bit(__IDPF_USER_FLAG_HSPLIT, config->user_flags);
1206		return true;
1207	case ETHTOOL_TCP_DATA_SPLIT_DISABLED:
1208		__clear_bit(__IDPF_USER_FLAG_HSPLIT, config->user_flags);
1209		return true;
1210	default:
1211		return false;
1212	}
1213}
1214
1215/**
1216 * idpf_vport_alloc - Allocates the next available struct vport in the adapter
1217 * @adapter: board private structure
1218 * @max_q: vport max queue info
1219 *
1220 * returns a pointer to a vport on success, NULL on failure.
1221 */
1222static struct idpf_vport *idpf_vport_alloc(struct idpf_adapter *adapter,
1223					   struct idpf_vport_max_q *max_q)
1224{
1225	struct idpf_rss_data *rss_data;
1226	u16 idx = adapter->next_vport;
1227	struct idpf_vport *vport;
1228	u16 num_max_q;
1229	int err;
1230
1231	if (idx == IDPF_NO_FREE_SLOT)
1232		return NULL;
1233
1234	vport = kzalloc(sizeof(*vport), GFP_KERNEL);
1235	if (!vport)
1236		return vport;
1237
1238	num_max_q = max(max_q->max_txq, max_q->max_rxq) + IDPF_RESERVED_VECS;
1239	if (!adapter->vport_config[idx]) {
1240		struct idpf_vport_config *vport_config;
1241		struct idpf_q_coalesce *q_coal;
1242
1243		vport_config = kzalloc(sizeof(*vport_config), GFP_KERNEL);
1244		if (!vport_config) {
1245			kfree(vport);
1246
1247			return NULL;
1248		}
1249
1250		q_coal = kcalloc(num_max_q, sizeof(*q_coal), GFP_KERNEL);
1251		if (!q_coal) {
1252			kfree(vport_config);
1253			kfree(vport);
1254
1255			return NULL;
1256		}
1257		for (int i = 0; i < num_max_q; i++) {
1258			q_coal[i].tx_intr_mode = IDPF_ITR_DYNAMIC;
1259			q_coal[i].tx_coalesce_usecs = IDPF_ITR_TX_DEF;
1260			q_coal[i].rx_intr_mode = IDPF_ITR_DYNAMIC;
1261			q_coal[i].rx_coalesce_usecs = IDPF_ITR_RX_DEF;
1262		}
1263		vport_config->user_config.q_coalesce = q_coal;
1264
1265		adapter->vport_config[idx] = vport_config;
1266	}
1267
1268	vport->idx = idx;
1269	vport->adapter = adapter;
1270	vport->compln_clean_budget = IDPF_TX_COMPLQ_CLEAN_BUDGET;
1271	vport->default_vport = adapter->num_alloc_vports <
1272			       idpf_get_default_vports(adapter);
1273
1274	vport->q_vector_idxs = kcalloc(num_max_q, sizeof(u16), GFP_KERNEL);
1275	if (!vport->q_vector_idxs)
1276		goto free_vport;
1277
1278	idpf_vport_init(vport, max_q);
1279
1280	/* LUT and key are both initialized here. Key is not strictly dependent
1281	 * on how many queues we have. If we change number of queues and soft
1282	 * reset is initiated, LUT will be freed and a new LUT will be allocated
1283	 * as per the updated number of queues during vport bringup. However,
1284	 * the key remains the same for as long as the vport exists.
1285	 */
1286	rss_data = &adapter->vport_config[idx]->user_config.rss_data;
1287	rss_data->rss_key = kzalloc(rss_data->rss_key_size, GFP_KERNEL);
1288	if (!rss_data->rss_key)
1289		goto free_vector_idxs;
1290
1291	/* Initialize default rss key */
1292	netdev_rss_key_fill((void *)rss_data->rss_key, rss_data->rss_key_size);
1293
1294	/* Initialize default rss LUT */
1295	err = idpf_init_rss_lut(vport);
1296	if (err)
1297		goto free_rss_key;
1298
1299	/* fill vport slot in the adapter struct */
1300	adapter->vports[idx] = vport;
1301	adapter->vport_ids[idx] = idpf_get_vport_id(vport);
1302
1303	adapter->num_alloc_vports++;
1304	/* prepare adapter->next_vport for next use */
1305	adapter->next_vport = idpf_get_free_slot(adapter);
1306
1307	return vport;
1308
1309free_rss_key:
1310	kfree(rss_data->rss_key);
1311free_vector_idxs:
1312	kfree(vport->q_vector_idxs);
1313free_vport:
1314	kfree(vport);
1315
1316	return NULL;
1317}
1318
1319/**
1320 * idpf_get_stats64 - get statistics for network device structure
1321 * @netdev: network interface device structure
1322 * @stats: main device statistics structure
1323 */
1324static void idpf_get_stats64(struct net_device *netdev,
1325			     struct rtnl_link_stats64 *stats)
1326{
1327	struct idpf_netdev_priv *np = netdev_priv(netdev);
1328
1329	spin_lock_bh(&np->stats_lock);
1330	*stats = np->netstats;
1331	spin_unlock_bh(&np->stats_lock);
1332}
1333
1334/**
1335 * idpf_statistics_task - Delayed task to get statistics over mailbox
1336 * @work: work_struct handle to our data
1337 */
1338void idpf_statistics_task(struct work_struct *work)
1339{
1340	struct idpf_adapter *adapter;
1341	int i;
1342
1343	adapter = container_of(work, struct idpf_adapter, stats_task.work);
1344
1345	for (i = 0; i < adapter->max_vports; i++) {
1346		struct idpf_vport *vport = adapter->vports[i];
1347
1348		if (vport && !test_bit(IDPF_HR_RESET_IN_PROG, adapter->flags))
1349			idpf_send_get_stats_msg(vport);
1350	}
1351
1352	queue_delayed_work(adapter->stats_wq, &adapter->stats_task,
1353			   msecs_to_jiffies(10000));
1354}
1355
1356/**
1357 * idpf_mbx_task - Delayed task to handle mailbox responses
1358 * @work: work_struct handle
1359 */
1360void idpf_mbx_task(struct work_struct *work)
1361{
1362	struct idpf_adapter *adapter;
1363
1364	adapter = container_of(work, struct idpf_adapter, mbx_task.work);
1365
1366	if (test_bit(IDPF_MB_INTR_MODE, adapter->flags))
1367		idpf_mb_irq_enable(adapter);
1368	else
1369		queue_delayed_work(adapter->mbx_wq, &adapter->mbx_task,
1370				   usecs_to_jiffies(300));
1371
1372	idpf_recv_mb_msg(adapter);
1373}
1374
1375/**
1376 * idpf_service_task - Delayed task for handling mailbox responses
1377 * @work: work_struct handle to our data
1378 *
1379 */
1380void idpf_service_task(struct work_struct *work)
1381{
1382	struct idpf_adapter *adapter;
1383
1384	adapter = container_of(work, struct idpf_adapter, serv_task.work);
1385
1386	if (idpf_is_reset_detected(adapter) &&
1387	    !idpf_is_reset_in_prog(adapter) &&
1388	    !test_bit(IDPF_REMOVE_IN_PROG, adapter->flags)) {
1389		dev_info(&adapter->pdev->dev, "HW reset detected\n");
1390		set_bit(IDPF_HR_FUNC_RESET, adapter->flags);
1391		queue_delayed_work(adapter->vc_event_wq,
1392				   &adapter->vc_event_task,
1393				   msecs_to_jiffies(10));
1394	}
1395
1396	queue_delayed_work(adapter->serv_wq, &adapter->serv_task,
1397			   msecs_to_jiffies(300));
1398}
1399
1400/**
1401 * idpf_restore_features - Restore feature configs
1402 * @vport: virtual port structure
1403 */
1404static void idpf_restore_features(struct idpf_vport *vport)
1405{
1406	struct idpf_adapter *adapter = vport->adapter;
1407
1408	if (idpf_is_cap_ena(adapter, IDPF_OTHER_CAPS, VIRTCHNL2_CAP_MACFILTER))
1409		idpf_restore_mac_filters(vport);
1410}
1411
1412/**
1413 * idpf_set_real_num_queues - set number of queues for netdev
1414 * @vport: virtual port structure
1415 *
1416 * Returns 0 on success, negative on failure.
1417 */
1418static int idpf_set_real_num_queues(struct idpf_vport *vport)
1419{
1420	int err, txq = vport->num_txq - vport->num_xdp_txq;
1421
1422	err = netif_set_real_num_rx_queues(vport->netdev, vport->num_rxq);
1423	if (err)
1424		return err;
1425
1426	return netif_set_real_num_tx_queues(vport->netdev, txq);
1427}
1428
1429/**
1430 * idpf_up_complete - Complete interface up sequence
1431 * @vport: virtual port structure
1432 *
1433 * Returns 0 on success, negative on failure.
1434 */
1435static int idpf_up_complete(struct idpf_vport *vport)
1436{
1437	struct idpf_netdev_priv *np = netdev_priv(vport->netdev);
1438
1439	if (vport->link_up && !netif_carrier_ok(vport->netdev)) {
1440		netif_carrier_on(vport->netdev);
1441		netif_tx_start_all_queues(vport->netdev);
1442	}
1443
1444	set_bit(IDPF_VPORT_UP, np->state);
1445
1446	return 0;
1447}
1448
1449/**
1450 * idpf_rx_init_buf_tail - Write initial buffer ring tail value
1451 * @vport: virtual port struct
1452 */
1453static void idpf_rx_init_buf_tail(struct idpf_vport *vport)
1454{
1455	int i, j;
1456
1457	for (i = 0; i < vport->num_rxq_grp; i++) {
1458		struct idpf_rxq_group *grp = &vport->rxq_grps[i];
1459
1460		if (idpf_is_queue_model_split(vport->rxq_model)) {
1461			for (j = 0; j < vport->num_bufqs_per_qgrp; j++) {
1462				const struct idpf_buf_queue *q =
1463					&grp->splitq.bufq_sets[j].bufq;
1464
1465				writel(q->next_to_alloc, q->tail);
1466			}
1467		} else {
1468			for (j = 0; j < grp->singleq.num_rxq; j++) {
1469				const struct idpf_rx_queue *q =
1470					grp->singleq.rxqs[j];
1471
1472				writel(q->next_to_alloc, q->tail);
1473			}
1474		}
1475	}
1476}
1477
1478/**
1479 * idpf_vport_open - Bring up a vport
1480 * @vport: vport to bring up
1481 * @rtnl: whether to take RTNL lock
1482 */
1483static int idpf_vport_open(struct idpf_vport *vport, bool rtnl)
1484{
1485	struct idpf_netdev_priv *np = netdev_priv(vport->netdev);
1486	struct idpf_adapter *adapter = vport->adapter;
1487	int err;
1488
1489	if (test_bit(IDPF_VPORT_UP, np->state))
1490		return -EBUSY;
1491
1492	if (rtnl)
1493		rtnl_lock();
1494
1495	/* we do not allow interface up just yet */
1496	netif_carrier_off(vport->netdev);
1497
1498	err = idpf_vport_intr_alloc(vport);
1499	if (err) {
1500		dev_err(&adapter->pdev->dev, "Failed to allocate interrupts for vport %u: %d\n",
1501			vport->vport_id, err);
1502		goto err_rtnl_unlock;
1503	}
1504
1505	err = idpf_vport_queues_alloc(vport);
1506	if (err)
1507		goto intr_rel;
1508
1509	err = idpf_vport_queue_ids_init(vport);
1510	if (err) {
1511		dev_err(&adapter->pdev->dev, "Failed to initialize queue ids for vport %u: %d\n",
1512			vport->vport_id, err);
1513		goto queues_rel;
1514	}
1515
1516	err = idpf_vport_intr_init(vport);
1517	if (err) {
1518		dev_err(&adapter->pdev->dev, "Failed to initialize interrupts for vport %u: %d\n",
1519			vport->vport_id, err);
1520		goto queues_rel;
1521	}
1522
1523	err = idpf_queue_reg_init(vport);
1524	if (err) {
1525		dev_err(&adapter->pdev->dev, "Failed to initialize queue registers for vport %u: %d\n",
1526			vport->vport_id, err);
1527		goto intr_deinit;
1528	}
1529
1530	err = idpf_rx_bufs_init_all(vport);
1531	if (err) {
1532		dev_err(&adapter->pdev->dev, "Failed to initialize RX buffers for vport %u: %d\n",
1533			vport->vport_id, err);
1534		goto intr_deinit;
1535	}
1536
1537	idpf_rx_init_buf_tail(vport);
1538
1539	err = idpf_xdp_rxq_info_init_all(vport);
1540	if (err) {
1541		netdev_err(vport->netdev,
1542			   "Failed to initialize XDP RxQ info for vport %u: %pe\n",
1543			   vport->vport_id, ERR_PTR(err));
1544		goto intr_deinit;
1545	}
1546
1547	idpf_vport_intr_ena(vport);
1548
1549	err = idpf_send_config_queues_msg(vport);
1550	if (err) {
1551		dev_err(&adapter->pdev->dev, "Failed to configure queues for vport %u, %d\n",
1552			vport->vport_id, err);
1553		goto rxq_deinit;
1554	}
1555
1556	err = idpf_send_map_unmap_queue_vector_msg(vport, true);
1557	if (err) {
1558		dev_err(&adapter->pdev->dev, "Failed to map queue vectors for vport %u: %d\n",
1559			vport->vport_id, err);
1560		goto rxq_deinit;
1561	}
1562
1563	err = idpf_send_enable_queues_msg(vport);
1564	if (err) {
1565		dev_err(&adapter->pdev->dev, "Failed to enable queues for vport %u: %d\n",
1566			vport->vport_id, err);
1567		goto unmap_queue_vectors;
1568	}
1569
1570	err = idpf_send_enable_vport_msg(vport);
1571	if (err) {
1572		dev_err(&adapter->pdev->dev, "Failed to enable vport %u: %d\n",
1573			vport->vport_id, err);
1574		err = -EAGAIN;
1575		goto disable_queues;
1576	}
1577
1578	idpf_restore_features(vport);
1579
1580	err = idpf_config_rss(vport);
1581	if (err) {
1582		dev_err(&adapter->pdev->dev, "Failed to configure RSS for vport %u: %d\n",
1583			vport->vport_id, err);
1584		goto disable_vport;
1585	}
1586
1587	err = idpf_up_complete(vport);
1588	if (err) {
1589		dev_err(&adapter->pdev->dev, "Failed to complete interface up for vport %u: %d\n",
1590			vport->vport_id, err);
1591		goto disable_vport;
1592	}
1593
1594	if (rtnl)
1595		rtnl_unlock();
1596
1597	return 0;
1598
1599disable_vport:
1600	idpf_send_disable_vport_msg(vport);
1601disable_queues:
1602	idpf_send_disable_queues_msg(vport);
1603unmap_queue_vectors:
1604	idpf_send_map_unmap_queue_vector_msg(vport, false);
1605rxq_deinit:
1606	idpf_xdp_rxq_info_deinit_all(vport);
1607intr_deinit:
1608	idpf_vport_intr_deinit(vport);
1609queues_rel:
1610	idpf_vport_queues_rel(vport);
1611intr_rel:
1612	idpf_vport_intr_rel(vport);
1613
1614err_rtnl_unlock:
1615	if (rtnl)
1616		rtnl_unlock();
1617
1618	return err;
1619}
1620
1621/**
1622 * idpf_init_task - Delayed initialization task
1623 * @work: work_struct handle to our data
1624 *
1625 * Init task finishes up pending work started in probe. Due to the asynchronous
1626 * nature in which the device communicates with hardware, we may have to wait
1627 * several milliseconds to get a response.  Instead of busy polling in probe,
1628 * pulling it out into a delayed work task prevents us from bogging down the
1629 * whole system waiting for a response from hardware.
1630 */
1631void idpf_init_task(struct work_struct *work)
1632{
1633	struct idpf_vport_config *vport_config;
1634	struct idpf_vport_max_q max_q;
1635	struct idpf_adapter *adapter;
1636	struct idpf_vport *vport;
1637	u16 num_default_vports;
1638	struct pci_dev *pdev;
1639	bool default_vport;
1640	int index, err;
1641
1642	adapter = container_of(work, struct idpf_adapter, init_task.work);
1643
1644	num_default_vports = idpf_get_default_vports(adapter);
1645	if (adapter->num_alloc_vports < num_default_vports)
1646		default_vport = true;
1647	else
1648		default_vport = false;
1649
1650	err = idpf_vport_alloc_max_qs(adapter, &max_q);
1651	if (err)
1652		goto unwind_vports;
1653
1654	err = idpf_send_create_vport_msg(adapter, &max_q);
1655	if (err) {
1656		idpf_vport_dealloc_max_qs(adapter, &max_q);
1657		goto unwind_vports;
1658	}
1659
1660	pdev = adapter->pdev;
1661	vport = idpf_vport_alloc(adapter, &max_q);
1662	if (!vport) {
1663		err = -EFAULT;
1664		dev_err(&pdev->dev, "failed to allocate vport: %d\n",
1665			err);
1666		idpf_vport_dealloc_max_qs(adapter, &max_q);
1667		goto unwind_vports;
1668	}
1669
1670	err = idpf_send_get_rx_ptype_msg(vport);
1671	if (err)
1672		goto unwind_vports;
1673
1674	index = vport->idx;
1675	vport_config = adapter->vport_config[index];
1676
1677	spin_lock_init(&vport_config->mac_filter_list_lock);
1678	spin_lock_init(&vport_config->flow_steer_list_lock);
1679
1680	INIT_LIST_HEAD(&vport_config->user_config.mac_filter_list);
1681	INIT_LIST_HEAD(&vport_config->user_config.flow_steer_list);
1682
1683	err = idpf_check_supported_desc_ids(vport);
1684	if (err) {
1685		dev_err(&pdev->dev, "failed to get required descriptor ids\n");
1686		goto unwind_vports;
1687	}
1688
1689	if (idpf_cfg_netdev(vport))
1690		goto unwind_vports;
1691
1692	/* Spawn and return 'idpf_init_task' work queue until all the
1693	 * default vports are created
1694	 */
1695	if (adapter->num_alloc_vports < num_default_vports) {
1696		queue_delayed_work(adapter->init_wq, &adapter->init_task,
1697				   msecs_to_jiffies(5 * (adapter->pdev->devfn & 0x07)));
1698
1699		return;
1700	}
1701
1702	for (index = 0; index < adapter->max_vports; index++) {
1703		struct net_device *netdev = adapter->netdevs[index];
1704		struct idpf_vport_config *vport_config;
1705
1706		vport_config = adapter->vport_config[index];
1707
1708		if (!netdev ||
1709		    test_bit(IDPF_VPORT_REG_NETDEV, vport_config->flags))
1710			continue;
1711
1712		err = register_netdev(netdev);
1713		if (err) {
1714			dev_err(&pdev->dev, "failed to register netdev for vport %d: %pe\n",
1715				index, ERR_PTR(err));
1716			continue;
1717		}
1718		set_bit(IDPF_VPORT_REG_NETDEV, vport_config->flags);
1719	}
1720
1721	/* Clear the reset and load bits as all vports are created */
1722	clear_bit(IDPF_HR_RESET_IN_PROG, adapter->flags);
1723	clear_bit(IDPF_HR_DRV_LOAD, adapter->flags);
1724	/* Start the statistics task now */
1725	queue_delayed_work(adapter->stats_wq, &adapter->stats_task,
1726			   msecs_to_jiffies(10 * (pdev->devfn & 0x07)));
1727
1728	return;
1729
1730unwind_vports:
1731	if (default_vport) {
1732		for (index = 0; index < adapter->max_vports; index++) {
1733			if (adapter->vports[index])
1734				idpf_vport_dealloc(adapter->vports[index]);
1735		}
1736	}
1737	/* Cleanup after vc_core_init, which has no way of knowing the
1738	 * init task failed on driver load.
1739	 */
1740	if (test_and_clear_bit(IDPF_HR_DRV_LOAD, adapter->flags)) {
1741		cancel_delayed_work_sync(&adapter->serv_task);
1742		cancel_delayed_work_sync(&adapter->mbx_task);
1743	}
1744	idpf_ptp_release(adapter);
1745
1746	clear_bit(IDPF_HR_RESET_IN_PROG, adapter->flags);
1747}
1748
1749/**
1750 * idpf_sriov_ena - Enable or change number of VFs
1751 * @adapter: private data struct
1752 * @num_vfs: number of VFs to allocate
1753 */
1754static int idpf_sriov_ena(struct idpf_adapter *adapter, int num_vfs)
1755{
1756	struct device *dev = &adapter->pdev->dev;
1757	int err;
1758
1759	err = idpf_send_set_sriov_vfs_msg(adapter, num_vfs);
1760	if (err) {
1761		dev_err(dev, "Failed to allocate VFs: %d\n", err);
1762
1763		return err;
1764	}
1765
1766	err = pci_enable_sriov(adapter->pdev, num_vfs);
1767	if (err) {
1768		idpf_send_set_sriov_vfs_msg(adapter, 0);
1769		dev_err(dev, "Failed to enable SR-IOV: %d\n", err);
1770
1771		return err;
1772	}
1773
1774	adapter->num_vfs = num_vfs;
1775
1776	return num_vfs;
1777}
1778
1779/**
1780 * idpf_sriov_configure - Configure the requested VFs
1781 * @pdev: pointer to a pci_dev structure
1782 * @num_vfs: number of vfs to allocate
1783 *
1784 * Enable or change the number of VFs. Called when the user updates the number
1785 * of VFs in sysfs.
1786 **/
1787int idpf_sriov_configure(struct pci_dev *pdev, int num_vfs)
1788{
1789	struct idpf_adapter *adapter = pci_get_drvdata(pdev);
1790
1791	if (!idpf_is_cap_ena(adapter, IDPF_OTHER_CAPS, VIRTCHNL2_CAP_SRIOV)) {
1792		dev_info(&pdev->dev, "SR-IOV is not supported on this device\n");
1793
1794		return -EOPNOTSUPP;
1795	}
1796
1797	if (num_vfs)
1798		return idpf_sriov_ena(adapter, num_vfs);
1799
1800	if (pci_vfs_assigned(pdev)) {
1801		dev_warn(&pdev->dev, "Unable to free VFs because some are assigned to VMs\n");
1802
1803		return -EBUSY;
1804	}
1805
1806	pci_disable_sriov(adapter->pdev);
1807	idpf_send_set_sriov_vfs_msg(adapter, 0);
1808	adapter->num_vfs = 0;
1809
1810	return 0;
1811}
1812
1813/**
1814 * idpf_deinit_task - Device deinit routine
1815 * @adapter: Driver specific private structure
1816 *
1817 * Extended remove logic which will be used for
1818 * hard reset as well
1819 */
1820void idpf_deinit_task(struct idpf_adapter *adapter)
1821{
1822	unsigned int i;
1823
1824	/* Wait until the init_task is done else this thread might release
1825	 * the resources first and the other thread might end up in a bad state
1826	 */
1827	cancel_delayed_work_sync(&adapter->init_task);
1828
1829	if (!adapter->vports)
1830		return;
1831
1832	cancel_delayed_work_sync(&adapter->stats_task);
1833
1834	for (i = 0; i < adapter->max_vports; i++) {
1835		if (adapter->vports[i])
1836			idpf_vport_dealloc(adapter->vports[i]);
1837	}
1838}
1839
1840/**
1841 * idpf_check_reset_complete - check that reset is complete
1842 * @hw: pointer to hw struct
1843 * @reset_reg: struct with reset registers
1844 *
1845 * Returns 0 if device is ready to use, or -EBUSY if it's in reset.
1846 **/
1847static int idpf_check_reset_complete(struct idpf_hw *hw,
1848				     struct idpf_reset_reg *reset_reg)
1849{
1850	struct idpf_adapter *adapter = hw->back;
1851	int i;
1852
1853	for (i = 0; i < 2000; i++) {
1854		u32 reg_val = readl(reset_reg->rstat);
1855
1856		/* 0xFFFFFFFF might be read if other side hasn't cleared the
1857		 * register for us yet and 0xFFFFFFFF is not a valid value for
1858		 * the register, so treat that as invalid.
1859		 */
1860		if (reg_val != 0xFFFFFFFF && (reg_val & reset_reg->rstat_m))
1861			return 0;
1862
1863		usleep_range(5000, 10000);
1864	}
1865
1866	dev_warn(&adapter->pdev->dev, "Device reset timeout!\n");
1867	/* Clear the reset flag unconditionally here since the reset
1868	 * technically isn't in progress anymore from the driver's perspective
1869	 */
1870	clear_bit(IDPF_HR_RESET_IN_PROG, adapter->flags);
1871
1872	return -EBUSY;
1873}
1874
1875/**
1876 * idpf_init_hard_reset - Initiate a hardware reset
1877 * @adapter: Driver specific private structure
1878 *
1879 * Deallocate the vports and all the resources associated with them and
1880 * reallocate. Also reinitialize the mailbox. Return 0 on success,
1881 * negative on failure.
1882 */
1883static void idpf_init_hard_reset(struct idpf_adapter *adapter)
1884{
1885	struct idpf_reg_ops *reg_ops = &adapter->dev_ops.reg_ops;
1886	struct device *dev = &adapter->pdev->dev;
1887	int err;
1888
1889	idpf_detach_and_close(adapter);
1890	mutex_lock(&adapter->vport_ctrl_lock);
1891
1892	dev_info(dev, "Device HW Reset initiated\n");
1893
1894	/* Prepare for reset */
1895	if (test_bit(IDPF_HR_DRV_LOAD, adapter->flags)) {
1896		reg_ops->trigger_reset(adapter, IDPF_HR_DRV_LOAD);
1897	} else if (test_and_clear_bit(IDPF_HR_FUNC_RESET, adapter->flags)) {
1898		bool is_reset = idpf_is_reset_detected(adapter);
1899
1900		idpf_idc_issue_reset_event(adapter->cdev_info);
1901
1902		idpf_vc_core_deinit(adapter);
1903		if (!is_reset)
1904			reg_ops->trigger_reset(adapter, IDPF_HR_FUNC_RESET);
1905		idpf_deinit_dflt_mbx(adapter);
1906	} else {
1907		dev_err(dev, "Unhandled hard reset cause\n");
1908		err = -EBADRQC;
1909		goto unlock_mutex;
1910	}
1911
1912	/* Wait for reset to complete */
1913	err = idpf_check_reset_complete(&adapter->hw, &adapter->reset_reg);
1914	if (err) {
1915		dev_err(dev, "The driver was unable to contact the device's firmware. Check that the FW is running. Driver state= 0x%x\n",
1916			adapter->state);
1917		goto unlock_mutex;
1918	}
1919
1920	/* Reset is complete and so start building the driver resources again */
1921	err = idpf_init_dflt_mbx(adapter);
1922	if (err) {
1923		dev_err(dev, "Failed to initialize default mailbox: %d\n", err);
1924		goto unlock_mutex;
1925	}
1926
1927	queue_delayed_work(adapter->mbx_wq, &adapter->mbx_task, 0);
1928
1929	/* Initialize the state machine, also allocate memory and request
1930	 * resources
1931	 */
1932	err = idpf_vc_core_init(adapter);
1933	if (err) {
1934		cancel_delayed_work_sync(&adapter->mbx_task);
1935		idpf_deinit_dflt_mbx(adapter);
1936		goto unlock_mutex;
1937	}
1938
1939	/* Wait till all the vports are initialized to release the reset lock,
1940	 * else user space callbacks may access uninitialized vports
1941	 */
1942	while (test_bit(IDPF_HR_RESET_IN_PROG, adapter->flags))
1943		msleep(100);
1944
1945unlock_mutex:
1946	mutex_unlock(&adapter->vport_ctrl_lock);
1947
1948	/* Attempt to restore netdevs and initialize RDMA CORE AUX device,
1949	 * provided vc_core_init succeeded. It is still possible that
1950	 * vports are not allocated at this point if the init task failed.
1951	 */
1952	if (!err) {
1953		idpf_attach_and_open(adapter);
1954		idpf_idc_init(adapter);
1955	}
1956}
1957
1958/**
1959 * idpf_vc_event_task - Handle virtchannel event logic
1960 * @work: work queue struct
1961 */
1962void idpf_vc_event_task(struct work_struct *work)
1963{
1964	struct idpf_adapter *adapter;
1965
1966	adapter = container_of(work, struct idpf_adapter, vc_event_task.work);
1967
1968	if (test_bit(IDPF_REMOVE_IN_PROG, adapter->flags))
1969		return;
1970
1971	if (test_bit(IDPF_HR_FUNC_RESET, adapter->flags))
1972		goto func_reset;
1973
1974	if (test_bit(IDPF_HR_DRV_LOAD, adapter->flags))
1975		goto drv_load;
1976
1977	return;
1978
1979func_reset:
1980	idpf_vc_xn_shutdown(adapter->vcxn_mngr);
1981drv_load:
1982	set_bit(IDPF_HR_RESET_IN_PROG, adapter->flags);
1983	idpf_init_hard_reset(adapter);
1984}
1985
1986/**
1987 * idpf_initiate_soft_reset - Initiate a software reset
1988 * @vport: virtual port data struct
1989 * @reset_cause: reason for the soft reset
1990 *
1991 * Soft reset only reallocs vport queue resources. Returns 0 on success,
1992 * negative on failure.
1993 */
1994int idpf_initiate_soft_reset(struct idpf_vport *vport,
1995			     enum idpf_vport_reset_cause reset_cause)
1996{
1997	struct idpf_netdev_priv *np = netdev_priv(vport->netdev);
1998	bool vport_is_up = test_bit(IDPF_VPORT_UP, np->state);
1999	struct idpf_adapter *adapter = vport->adapter;
2000	struct idpf_vport *new_vport;
2001	int err;
2002
2003	/* If the system is low on memory, we can end up in bad state if we
2004	 * free all the memory for queue resources and try to allocate them
2005	 * again. Instead, we can pre-allocate the new resources before doing
2006	 * anything and bailing if the alloc fails.
2007	 *
2008	 * Make a clone of the existing vport to mimic its current
2009	 * configuration, then modify the new structure with any requested
2010	 * changes. Once the allocation of the new resources is done, stop the
2011	 * existing vport and copy the configuration to the main vport. If an
2012	 * error occurred, the existing vport will be untouched.
2013	 *
2014	 */
2015	new_vport = kzalloc(sizeof(*vport), GFP_KERNEL);
2016	if (!new_vport)
2017		return -ENOMEM;
2018
2019	/* This purposely avoids copying the end of the struct because it
2020	 * contains wait_queues and mutexes and other stuff we don't want to
2021	 * mess with. Nothing below should use those variables from new_vport
2022	 * and should instead always refer to them in vport if they need to.
2023	 */
2024	memcpy(new_vport, vport, offsetof(struct idpf_vport, link_up));
2025
2026	/* Adjust resource parameters prior to reallocating resources */
2027	switch (reset_cause) {
2028	case IDPF_SR_Q_CHANGE:
2029		err = idpf_vport_adjust_qs(new_vport);
2030		if (err)
2031			goto free_vport;
2032		break;
2033	case IDPF_SR_Q_DESC_CHANGE:
2034		/* Update queue parameters before allocating resources */
2035		idpf_vport_calc_num_q_desc(new_vport);
2036		break;
2037	case IDPF_SR_MTU_CHANGE:
2038		idpf_idc_vdev_mtu_event(vport->vdev_info,
2039					IIDC_RDMA_EVENT_BEFORE_MTU_CHANGE);
2040		break;
2041	case IDPF_SR_RSC_CHANGE:
2042		break;
2043	default:
2044		dev_err(&adapter->pdev->dev, "Unhandled soft reset cause\n");
2045		err = -EINVAL;
2046		goto free_vport;
2047	}
2048
2049	if (!vport_is_up) {
2050		idpf_send_delete_queues_msg(vport);
2051	} else {
2052		set_bit(IDPF_VPORT_DEL_QUEUES, vport->flags);
2053		idpf_vport_stop(vport, false);
2054	}
2055
2056	/* We're passing in vport here because we need its wait_queue
2057	 * to send a message and it should be getting all the vport
2058	 * config data out of the adapter but we need to be careful not
2059	 * to add code to add_queues to change the vport config within
2060	 * vport itself as it will be wiped with a memcpy later.
2061	 */
2062	err = idpf_send_add_queues_msg(vport, new_vport->num_txq,
2063				       new_vport->num_complq,
2064				       new_vport->num_rxq,
2065				       new_vport->num_bufq);
2066	if (err)
2067		goto err_reset;
2068
2069	/* Same comment as above regarding avoiding copying the wait_queues and
2070	 * mutexes applies here. We do not want to mess with those if possible.
2071	 */
2072	memcpy(vport, new_vport, offsetof(struct idpf_vport, link_up));
2073
2074	if (reset_cause == IDPF_SR_Q_CHANGE)
2075		idpf_vport_alloc_vec_indexes(vport);
2076
2077	err = idpf_set_real_num_queues(vport);
2078	if (err)
2079		goto err_open;
2080
2081	if (reset_cause == IDPF_SR_Q_CHANGE &&
2082	    !netif_is_rxfh_configured(vport->netdev))
2083		idpf_fill_dflt_rss_lut(vport);
2084
2085	if (vport_is_up)
2086		err = idpf_vport_open(vport, false);
2087
2088	goto free_vport;
2089
2090err_reset:
2091	idpf_send_add_queues_msg(vport, vport->num_txq, vport->num_complq,
2092				 vport->num_rxq, vport->num_bufq);
2093
2094err_open:
2095	if (vport_is_up)
2096		idpf_vport_open(vport, false);
2097
2098free_vport:
2099	kfree(new_vport);
2100
2101	if (reset_cause == IDPF_SR_MTU_CHANGE)
2102		idpf_idc_vdev_mtu_event(vport->vdev_info,
2103					IIDC_RDMA_EVENT_AFTER_MTU_CHANGE);
2104
2105	return err;
2106}
2107
2108/**
2109 * idpf_addr_sync - Callback for dev_(mc|uc)_sync to add address
2110 * @netdev: the netdevice
2111 * @addr: address to add
2112 *
2113 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
2114 * __dev_(uc|mc)_sync from .set_rx_mode. Kernel takes addr_list_lock spinlock
2115 * meaning we cannot sleep in this context. Due to this, we have to add the
2116 * filter and send the virtchnl message asynchronously without waiting for the
2117 * response from the other side. We won't know whether or not the operation
2118 * actually succeeded until we get the message back.  Returns 0 on success,
2119 * negative on failure.
2120 */
2121static int idpf_addr_sync(struct net_device *netdev, const u8 *addr)
2122{
2123	struct idpf_netdev_priv *np = netdev_priv(netdev);
2124
2125	return idpf_add_mac_filter(np->vport, np, addr, true);
2126}
2127
2128/**
2129 * idpf_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
2130 * @netdev: the netdevice
2131 * @addr: address to add
2132 *
2133 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
2134 * __dev_(uc|mc)_sync from .set_rx_mode. Kernel takes addr_list_lock spinlock
2135 * meaning we cannot sleep in this context. Due to this we have to delete the
2136 * filter and send the virtchnl message asynchronously without waiting for the
2137 * return from the other side.  We won't know whether or not the operation
2138 * actually succeeded until we get the message back. Returns 0 on success,
2139 * negative on failure.
2140 */
2141static int idpf_addr_unsync(struct net_device *netdev, const u8 *addr)
2142{
2143	struct idpf_netdev_priv *np = netdev_priv(netdev);
2144
2145	/* Under some circumstances, we might receive a request to delete
2146	 * our own device address from our uc list. Because we store the
2147	 * device address in the VSI's MAC filter list, we need to ignore
2148	 * such requests and not delete our device address from this list.
2149	 */
2150	if (ether_addr_equal(addr, netdev->dev_addr))
2151		return 0;
2152
2153	idpf_del_mac_filter(np->vport, np, addr, true);
2154
2155	return 0;
2156}
2157
2158/**
2159 * idpf_set_rx_mode - NDO callback to set the netdev filters
2160 * @netdev: network interface device structure
2161 *
2162 * Stack takes addr_list_lock spinlock before calling our .set_rx_mode.  We
2163 * cannot sleep in this context.
2164 */
2165static void idpf_set_rx_mode(struct net_device *netdev)
2166{
2167	struct idpf_netdev_priv *np = netdev_priv(netdev);
2168	struct idpf_vport_user_config_data *config_data;
2169	struct idpf_adapter *adapter;
2170	bool changed = false;
2171	struct device *dev;
2172	int err;
2173
2174	adapter = np->adapter;
2175	dev = &adapter->pdev->dev;
2176
2177	if (idpf_is_cap_ena(adapter, IDPF_OTHER_CAPS, VIRTCHNL2_CAP_MACFILTER)) {
2178		__dev_uc_sync(netdev, idpf_addr_sync, idpf_addr_unsync);
2179		__dev_mc_sync(netdev, idpf_addr_sync, idpf_addr_unsync);
2180	}
2181
2182	if (!idpf_is_cap_ena(adapter, IDPF_OTHER_CAPS, VIRTCHNL2_CAP_PROMISC))
2183		return;
2184
2185	config_data = &adapter->vport_config[np->vport_idx]->user_config;
2186	/* IFF_PROMISC enables both unicast and multicast promiscuous,
2187	 * while IFF_ALLMULTI only enables multicast such that:
2188	 *
2189	 * promisc  + allmulti		= unicast | multicast
2190	 * promisc  + !allmulti		= unicast | multicast
2191	 * !promisc + allmulti		= multicast
2192	 */
2193	if ((netdev->flags & IFF_PROMISC) &&
2194	    !test_and_set_bit(__IDPF_PROMISC_UC, config_data->user_flags)) {
2195		changed = true;
2196		dev_info(&adapter->pdev->dev, "Entering promiscuous mode\n");
2197		if (!test_and_set_bit(__IDPF_PROMISC_MC, adapter->flags))
2198			dev_info(dev, "Entering multicast promiscuous mode\n");
2199	}
2200
2201	if (!(netdev->flags & IFF_PROMISC) &&
2202	    test_and_clear_bit(__IDPF_PROMISC_UC, config_data->user_flags)) {
2203		changed = true;
2204		dev_info(dev, "Leaving promiscuous mode\n");
2205	}
2206
2207	if (netdev->flags & IFF_ALLMULTI &&
2208	    !test_and_set_bit(__IDPF_PROMISC_MC, config_data->user_flags)) {
2209		changed = true;
2210		dev_info(dev, "Entering multicast promiscuous mode\n");
2211	}
2212
2213	if (!(netdev->flags & (IFF_ALLMULTI | IFF_PROMISC)) &&
2214	    test_and_clear_bit(__IDPF_PROMISC_MC, config_data->user_flags)) {
2215		changed = true;
2216		dev_info(dev, "Leaving multicast promiscuous mode\n");
2217	}
2218
2219	if (!changed)
2220		return;
2221
2222	err = idpf_set_promiscuous(adapter, config_data, np->vport_id);
2223	if (err)
2224		dev_err(dev, "Failed to set promiscuous mode: %d\n", err);
2225}
2226
2227/**
2228 * idpf_set_features - set the netdev feature flags
2229 * @netdev: ptr to the netdev being adjusted
2230 * @features: the feature set that the stack is suggesting
2231 */
2232static int idpf_set_features(struct net_device *netdev,
2233			     netdev_features_t features)
2234{
2235	netdev_features_t changed = netdev->features ^ features;
2236	struct idpf_adapter *adapter;
2237	struct idpf_vport *vport;
2238	int err = 0;
2239
2240	idpf_vport_ctrl_lock(netdev);
2241	vport = idpf_netdev_to_vport(netdev);
2242
2243	adapter = vport->adapter;
2244
2245	if (idpf_is_reset_in_prog(adapter)) {
2246		dev_err(&adapter->pdev->dev, "Device is resetting, changing netdev features temporarily unavailable.\n");
2247		err = -EBUSY;
2248		goto unlock_mutex;
2249	}
2250
2251	if (changed & NETIF_F_RXHASH) {
2252		struct idpf_netdev_priv *np = netdev_priv(netdev);
2253
2254		netdev->features ^= NETIF_F_RXHASH;
2255
2256		/* If the interface is not up when changing the rxhash, update
2257		 * to the HW is skipped. The updated LUT will be committed to
2258		 * the HW when the interface is brought up.
2259		 */
2260		if (test_bit(IDPF_VPORT_UP, np->state)) {
2261			err = idpf_config_rss(vport);
2262			if (err)
2263				goto unlock_mutex;
2264		}
2265	}
2266
2267	if (changed & NETIF_F_GRO_HW) {
2268		netdev->features ^= NETIF_F_GRO_HW;
2269		err = idpf_initiate_soft_reset(vport, IDPF_SR_RSC_CHANGE);
2270		if (err)
2271			goto unlock_mutex;
2272	}
2273
2274	if (changed & NETIF_F_LOOPBACK) {
2275		netdev->features ^= NETIF_F_LOOPBACK;
2276		err = idpf_send_ena_dis_loopback_msg(vport);
2277	}
2278
2279unlock_mutex:
2280	idpf_vport_ctrl_unlock(netdev);
2281
2282	return err;
2283}
2284
2285/**
2286 * idpf_open - Called when a network interface becomes active
2287 * @netdev: network interface device structure
2288 *
2289 * The open entry point is called when a network interface is made
2290 * active by the system (IFF_UP).  At this point all resources needed
2291 * for transmit and receive operations are allocated, the interrupt
2292 * handler is registered with the OS, the netdev watchdog is enabled,
2293 * and the stack is notified that the interface is ready.
2294 *
2295 * Returns 0 on success, negative value on failure
2296 */
2297static int idpf_open(struct net_device *netdev)
2298{
2299	struct idpf_vport *vport;
2300	int err;
2301
2302	idpf_vport_ctrl_lock(netdev);
2303	vport = idpf_netdev_to_vport(netdev);
2304
2305	err = idpf_set_real_num_queues(vport);
2306	if (err)
2307		goto unlock;
2308
2309	err = idpf_vport_open(vport, false);
2310
2311unlock:
2312	idpf_vport_ctrl_unlock(netdev);
2313
2314	return err;
2315}
2316
2317/**
2318 * idpf_change_mtu - NDO callback to change the MTU
2319 * @netdev: network interface device structure
2320 * @new_mtu: new value for maximum frame size
2321 *
2322 * Returns 0 on success, negative on failure
2323 */
2324static int idpf_change_mtu(struct net_device *netdev, int new_mtu)
2325{
2326	struct idpf_vport *vport;
2327	int err;
2328
2329	idpf_vport_ctrl_lock(netdev);
2330	vport = idpf_netdev_to_vport(netdev);
2331
2332	WRITE_ONCE(netdev->mtu, new_mtu);
2333
2334	err = idpf_initiate_soft_reset(vport, IDPF_SR_MTU_CHANGE);
2335
2336	idpf_vport_ctrl_unlock(netdev);
2337
2338	return err;
2339}
2340
2341/**
2342 * idpf_chk_tso_segment - Check skb is not using too many buffers
2343 * @skb: send buffer
2344 * @max_bufs: maximum number of buffers
2345 *
2346 * For TSO we need to count the TSO header and segment payload separately.  As
2347 * such we need to check cases where we have max_bufs-1 fragments or more as we
2348 * can potentially require max_bufs+1 DMA transactions, 1 for the TSO header, 1
2349 * for the segment payload in the first descriptor, and another max_buf-1 for
2350 * the fragments.
2351 *
2352 * Returns true if the packet needs to be software segmented by core stack.
2353 */
2354static bool idpf_chk_tso_segment(const struct sk_buff *skb,
2355				 unsigned int max_bufs)
2356{
2357	const struct skb_shared_info *shinfo = skb_shinfo(skb);
2358	const skb_frag_t *frag, *stale;
2359	int nr_frags, sum;
2360
2361	/* no need to check if number of frags is less than max_bufs - 1 */
2362	nr_frags = shinfo->nr_frags;
2363	if (nr_frags < (max_bufs - 1))
2364		return false;
2365
2366	/* We need to walk through the list and validate that each group
2367	 * of max_bufs-2 fragments totals at least gso_size.
2368	 */
2369	nr_frags -= max_bufs - 2;
2370	frag = &shinfo->frags[0];
2371
2372	/* Initialize size to the negative value of gso_size minus 1.  We use
2373	 * this as the worst case scenario in which the frag ahead of us only
2374	 * provides one byte which is why we are limited to max_bufs-2
2375	 * descriptors for a single transmit as the header and previous
2376	 * fragment are already consuming 2 descriptors.
2377	 */
2378	sum = 1 - shinfo->gso_size;
2379
2380	/* Add size of frags 0 through 4 to create our initial sum */
2381	sum += skb_frag_size(frag++);
2382	sum += skb_frag_size(frag++);
2383	sum += skb_frag_size(frag++);
2384	sum += skb_frag_size(frag++);
2385	sum += skb_frag_size(frag++);
2386
2387	/* Walk through fragments adding latest fragment, testing it, and
2388	 * then removing stale fragments from the sum.
2389	 */
2390	for (stale = &shinfo->frags[0];; stale++) {
2391		int stale_size = skb_frag_size(stale);
2392
2393		sum += skb_frag_size(frag++);
2394
2395		/* The stale fragment may present us with a smaller
2396		 * descriptor than the actual fragment size. To account
2397		 * for that we need to remove all the data on the front and
2398		 * figure out what the remainder would be in the last
2399		 * descriptor associated with the fragment.
2400		 */
2401		if (stale_size > IDPF_TX_MAX_DESC_DATA) {
2402			int align_pad = -(skb_frag_off(stale)) &
2403					(IDPF_TX_MAX_READ_REQ_SIZE - 1);
2404
2405			sum -= align_pad;
2406			stale_size -= align_pad;
2407
2408			do {
2409				sum -= IDPF_TX_MAX_DESC_DATA_ALIGNED;
2410				stale_size -= IDPF_TX_MAX_DESC_DATA_ALIGNED;
2411			} while (stale_size > IDPF_TX_MAX_DESC_DATA);
2412		}
2413
2414		/* if sum is negative we failed to make sufficient progress */
2415		if (sum < 0)
2416			return true;
2417
2418		if (!nr_frags--)
2419			break;
2420
2421		sum -= stale_size;
2422	}
2423
2424	return false;
2425}
2426
2427/**
2428 * idpf_features_check - Validate packet conforms to limits
2429 * @skb: skb buffer
2430 * @netdev: This port's netdev
2431 * @features: Offload features that the stack believes apply
2432 */
2433static netdev_features_t idpf_features_check(struct sk_buff *skb,
2434					     struct net_device *netdev,
2435					     netdev_features_t features)
2436{
2437	struct idpf_netdev_priv *np = netdev_priv(netdev);
2438	u16 max_tx_hdr_size = np->max_tx_hdr_size;
2439	size_t len;
2440
2441	/* No point in doing any of this if neither checksum nor GSO are
2442	 * being requested for this frame.  We can rule out both by just
2443	 * checking for CHECKSUM_PARTIAL
2444	 */
2445	if (skb->ip_summed != CHECKSUM_PARTIAL)
2446		return features;
2447
2448	if (skb_is_gso(skb)) {
2449		/* We cannot support GSO if the MSS is going to be less than
2450		 * 88 bytes. If it is then we need to drop support for GSO.
2451		 */
2452		if (skb_shinfo(skb)->gso_size < IDPF_TX_TSO_MIN_MSS)
2453			features &= ~NETIF_F_GSO_MASK;
2454		else if (idpf_chk_tso_segment(skb, np->tx_max_bufs))
2455			features &= ~NETIF_F_GSO_MASK;
2456	}
2457
2458	/* Ensure MACLEN is <= 126 bytes (63 words) and not an odd size */
2459	len = skb_network_offset(skb);
2460	if (unlikely(len & ~(126)))
2461		goto unsupported;
2462
2463	len = skb_network_header_len(skb);
2464	if (unlikely(len > max_tx_hdr_size))
2465		goto unsupported;
2466
2467	if (!skb->encapsulation)
2468		return features;
2469
2470	/* L4TUNLEN can support 127 words */
2471	len = skb_inner_network_header(skb) - skb_transport_header(skb);
2472	if (unlikely(len & ~(127 * 2)))
2473		goto unsupported;
2474
2475	/* IPLEN can support at most 127 dwords */
2476	len = skb_inner_network_header_len(skb);
2477	if (unlikely(len > max_tx_hdr_size))
2478		goto unsupported;
2479
2480	/* No need to validate L4LEN as TCP is the only protocol with a
2481	 * a flexible value and we support all possible values supported
2482	 * by TCP, which is at most 15 dwords
2483	 */
2484
2485	return features;
2486
2487unsupported:
2488	return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
2489}
2490
2491/**
2492 * idpf_set_mac - NDO callback to set port mac address
2493 * @netdev: network interface device structure
2494 * @p: pointer to an address structure
2495 *
2496 * Returns 0 on success, negative on failure
2497 **/
2498static int idpf_set_mac(struct net_device *netdev, void *p)
2499{
2500	struct idpf_netdev_priv *np = netdev_priv(netdev);
2501	struct idpf_vport_config *vport_config;
2502	struct sockaddr *addr = p;
2503	u8 old_mac_addr[ETH_ALEN];
2504	struct idpf_vport *vport;
2505	int err = 0;
2506
2507	idpf_vport_ctrl_lock(netdev);
2508	vport = idpf_netdev_to_vport(netdev);
2509
2510	if (!idpf_is_cap_ena(vport->adapter, IDPF_OTHER_CAPS,
2511			     VIRTCHNL2_CAP_MACFILTER)) {
2512		dev_info(&vport->adapter->pdev->dev, "Setting MAC address is not supported\n");
2513		err = -EOPNOTSUPP;
2514		goto unlock_mutex;
2515	}
2516
2517	if (!is_valid_ether_addr(addr->sa_data)) {
2518		dev_info(&vport->adapter->pdev->dev, "Invalid MAC address: %pM\n",
2519			 addr->sa_data);
2520		err = -EADDRNOTAVAIL;
2521		goto unlock_mutex;
2522	}
2523
2524	if (ether_addr_equal(netdev->dev_addr, addr->sa_data))
2525		goto unlock_mutex;
2526
2527	ether_addr_copy(old_mac_addr, vport->default_mac_addr);
2528	ether_addr_copy(vport->default_mac_addr, addr->sa_data);
2529	vport_config = vport->adapter->vport_config[vport->idx];
2530	err = idpf_add_mac_filter(vport, np, addr->sa_data, false);
2531	if (err) {
2532		__idpf_del_mac_filter(vport_config, addr->sa_data);
2533		ether_addr_copy(vport->default_mac_addr, netdev->dev_addr);
2534		goto unlock_mutex;
2535	}
2536
2537	if (is_valid_ether_addr(old_mac_addr))
2538		__idpf_del_mac_filter(vport_config, old_mac_addr);
2539
2540	eth_hw_addr_set(netdev, addr->sa_data);
2541
2542unlock_mutex:
2543	idpf_vport_ctrl_unlock(netdev);
2544
2545	return err;
2546}
2547
2548/**
2549 * idpf_alloc_dma_mem - Allocate dma memory
2550 * @hw: pointer to hw struct
2551 * @mem: pointer to dma_mem struct
2552 * @size: size of the memory to allocate
2553 */
2554void *idpf_alloc_dma_mem(struct idpf_hw *hw, struct idpf_dma_mem *mem, u64 size)
2555{
2556	struct idpf_adapter *adapter = hw->back;
2557	size_t sz = ALIGN(size, 4096);
2558
2559	/* The control queue resources are freed under a spinlock, contiguous
2560	 * pages will avoid IOMMU remapping and the use vmap (and vunmap in
2561	 * dma_free_*() path.
2562	 */
2563	mem->va = dma_alloc_attrs(&adapter->pdev->dev, sz, &mem->pa,
2564				  GFP_KERNEL, DMA_ATTR_FORCE_CONTIGUOUS);
2565	mem->size = sz;
2566
2567	return mem->va;
2568}
2569
2570/**
2571 * idpf_free_dma_mem - Free the allocated dma memory
2572 * @hw: pointer to hw struct
2573 * @mem: pointer to dma_mem struct
2574 */
2575void idpf_free_dma_mem(struct idpf_hw *hw, struct idpf_dma_mem *mem)
2576{
2577	struct idpf_adapter *adapter = hw->back;
2578
2579	dma_free_attrs(&adapter->pdev->dev, mem->size,
2580		       mem->va, mem->pa, DMA_ATTR_FORCE_CONTIGUOUS);
2581	mem->size = 0;
2582	mem->va = NULL;
2583	mem->pa = 0;
2584}
2585
2586static int idpf_hwtstamp_set(struct net_device *netdev,
2587			     struct kernel_hwtstamp_config *config,
2588			     struct netlink_ext_ack *extack)
2589{
2590	struct idpf_vport *vport;
2591	int err;
2592
2593	idpf_vport_ctrl_lock(netdev);
2594	vport = idpf_netdev_to_vport(netdev);
2595
2596	if (!vport->link_up) {
2597		idpf_vport_ctrl_unlock(netdev);
2598		return -EPERM;
2599	}
2600
2601	if (!idpf_ptp_is_vport_tx_tstamp_ena(vport) &&
2602	    !idpf_ptp_is_vport_rx_tstamp_ena(vport)) {
2603		idpf_vport_ctrl_unlock(netdev);
2604		return -EOPNOTSUPP;
2605	}
2606
2607	err = idpf_ptp_set_timestamp_mode(vport, config);
2608
2609	idpf_vport_ctrl_unlock(netdev);
2610
2611	return err;
2612}
2613
2614static int idpf_hwtstamp_get(struct net_device *netdev,
2615			     struct kernel_hwtstamp_config *config)
2616{
2617	struct idpf_vport *vport;
2618
2619	idpf_vport_ctrl_lock(netdev);
2620	vport = idpf_netdev_to_vport(netdev);
2621
2622	if (!vport->link_up) {
2623		idpf_vport_ctrl_unlock(netdev);
2624		return -EPERM;
2625	}
2626
2627	if (!idpf_ptp_is_vport_tx_tstamp_ena(vport) &&
2628	    !idpf_ptp_is_vport_rx_tstamp_ena(vport)) {
2629		idpf_vport_ctrl_unlock(netdev);
2630		return 0;
2631	}
2632
2633	*config = vport->tstamp_config;
2634
2635	idpf_vport_ctrl_unlock(netdev);
2636
2637	return 0;
2638}
2639
2640static const struct net_device_ops idpf_netdev_ops = {
2641	.ndo_open = idpf_open,
2642	.ndo_stop = idpf_stop,
2643	.ndo_start_xmit = idpf_tx_start,
2644	.ndo_features_check = idpf_features_check,
2645	.ndo_set_rx_mode = idpf_set_rx_mode,
2646	.ndo_validate_addr = eth_validate_addr,
2647	.ndo_set_mac_address = idpf_set_mac,
2648	.ndo_change_mtu = idpf_change_mtu,
2649	.ndo_get_stats64 = idpf_get_stats64,
2650	.ndo_set_features = idpf_set_features,
2651	.ndo_tx_timeout = idpf_tx_timeout,
2652	.ndo_hwtstamp_get = idpf_hwtstamp_get,
2653	.ndo_hwtstamp_set = idpf_hwtstamp_set,
2654	.ndo_bpf = idpf_xdp,
2655	.ndo_xdp_xmit = idpf_xdp_xmit,
2656	.ndo_xsk_wakeup = idpf_xsk_wakeup,
2657};