cmd.c

   1/*
   2 * Copyright (c) 2013-2016, Mellanox Technologies. All rights reserved.
   3 *
   4 * This software is available to you under a choice of one of two
   5 * licenses.  You may choose to be licensed under the terms of the GNU
   6 * General Public License (GPL) Version 2, available from the file
   7 * COPYING in the main directory of this source tree, or the
   8 * OpenIB.org BSD license below:
   9 *
  10 *     Redistribution and use in source and binary forms, with or
  11 *     without modification, are permitted provided that the following
  12 *     conditions are met:
  13 *
  14 *      - Redistributions of source code must retain the above
  15 *        copyright notice, this list of conditions and the following
  16 *        disclaimer.
  17 *
  18 *      - Redistributions in binary form must reproduce the above
  19 *        copyright notice, this list of conditions and the following
  20 *        disclaimer in the documentation and/or other materials
  21 *        provided with the distribution.
  22 *
  23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30 * SOFTWARE.
  31 */
  32
  33#include <linux/highmem.h>
  34#include <linux/errno.h>
  35#include <linux/pci.h>
  36#include <linux/dma-mapping.h>
  37#include <linux/slab.h>
  38#include <linux/delay.h>
  39#include <linux/random.h>
  40#include <linux/mlx5/driver.h>
  41#include <linux/mlx5/eq.h>
  42#include <linux/debugfs.h>
  43
  44#include "mlx5_core.h"
  45#include "lib/eq.h"
  46#include "lib/tout.h"
  47#define CREATE_TRACE_POINTS
  48#include "diag/cmd_tracepoint.h"
  49
  50struct mlx5_ifc_mbox_out_bits {
  51	u8         status[0x8];
  52	u8         reserved_at_8[0x18];
  53
  54	u8         syndrome[0x20];
  55
  56	u8         reserved_at_40[0x40];
  57};
  58
  59struct mlx5_ifc_mbox_in_bits {
  60	u8         opcode[0x10];
  61	u8         uid[0x10];
  62
  63	u8         reserved_at_20[0x10];
  64	u8         op_mod[0x10];
  65
  66	u8         reserved_at_40[0x40];
  67};
  68
  69enum {
  70	CMD_IF_REV = 5,
  71};
  72
  73enum {
  74	CMD_MODE_POLLING,
  75	CMD_MODE_EVENTS
  76};
  77
  78enum {
  79	MLX5_CMD_DELIVERY_STAT_OK			= 0x0,
  80	MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR		= 0x1,
  81	MLX5_CMD_DELIVERY_STAT_TOK_ERR			= 0x2,
  82	MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR		= 0x3,
  83	MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR	= 0x4,
  84	MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR		= 0x5,
  85	MLX5_CMD_DELIVERY_STAT_FW_ERR			= 0x6,
  86	MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR		= 0x7,
  87	MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR		= 0x8,
  88	MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR	= 0x9,
  89	MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR		= 0x10,
  90};
  91
  92static u16 in_to_opcode(void *in)
  93{
  94	return MLX5_GET(mbox_in, in, opcode);
  95}
  96
  97static u16 in_to_uid(void *in)
  98{
  99	return MLX5_GET(mbox_in, in, uid);
 100}
 101
 102/* Returns true for opcodes that might be triggered very frequently and throttle
 103 * the command interface. Limit their command slots usage.
 104 */
 105static bool mlx5_cmd_is_throttle_opcode(u16 op)
 106{
 107	switch (op) {
 108	case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
 109	case MLX5_CMD_OP_DESTROY_GENERAL_OBJECT:
 110	case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
 111	case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
 112	case MLX5_CMD_OP_SYNC_CRYPTO:
 113		return true;
 114	}
 115	return false;
 116}
 117
 118static struct mlx5_cmd_work_ent *
 119cmd_alloc_ent(struct mlx5_cmd *cmd, struct mlx5_cmd_msg *in,
 120	      struct mlx5_cmd_msg *out, void *uout, int uout_size,
 121	      mlx5_cmd_cbk_t cbk, void *context, int page_queue)
 122{
 123	gfp_t alloc_flags = cbk ? GFP_ATOMIC : GFP_KERNEL;
 124	struct mlx5_cmd_work_ent *ent;
 125
 126	ent = kzalloc(sizeof(*ent), alloc_flags);
 127	if (!ent)
 128		return ERR_PTR(-ENOMEM);
 129
 130	ent->idx	= -EINVAL;
 131	ent->in		= in;
 132	ent->out	= out;
 133	ent->uout	= uout;
 134	ent->uout_size	= uout_size;
 135	ent->callback	= cbk;
 136	ent->context	= context;
 137	ent->cmd	= cmd;
 138	ent->page_queue = page_queue;
 139	ent->op         = in_to_opcode(in->first.data);
 140	refcount_set(&ent->refcnt, 1);
 141
 142	return ent;
 143}
 144
 145static void cmd_free_ent(struct mlx5_cmd_work_ent *ent)
 146{
 147	kfree(ent);
 148}
 149
 150static u8 alloc_token(struct mlx5_cmd *cmd)
 151{
 152	u8 token;
 153
 154	spin_lock(&cmd->token_lock);
 155	cmd->token++;
 156	if (cmd->token == 0)
 157		cmd->token++;
 158	token = cmd->token;
 159	spin_unlock(&cmd->token_lock);
 160
 161	return token;
 162}
 163
 164static int cmd_alloc_index(struct mlx5_cmd *cmd, struct mlx5_cmd_work_ent *ent)
 165{
 166	unsigned long flags;
 167	int ret;
 168
 169	spin_lock_irqsave(&cmd->alloc_lock, flags);
 170	ret = find_first_bit(&cmd->vars.bitmask, cmd->vars.max_reg_cmds);
 171	if (ret < cmd->vars.max_reg_cmds) {
 172		clear_bit(ret, &cmd->vars.bitmask);
 173		ent->idx = ret;
 174		cmd->ent_arr[ent->idx] = ent;
 175	}
 176	spin_unlock_irqrestore(&cmd->alloc_lock, flags);
 177
 178	return ret < cmd->vars.max_reg_cmds ? ret : -ENOMEM;
 179}
 180
 181static void cmd_free_index(struct mlx5_cmd *cmd, int idx)
 182{
 183	lockdep_assert_held(&cmd->alloc_lock);
 184	cmd->ent_arr[idx] = NULL;
 185	set_bit(idx, &cmd->vars.bitmask);
 186}
 187
 188static void cmd_ent_get(struct mlx5_cmd_work_ent *ent)
 189{
 190	refcount_inc(&ent->refcnt);
 191}
 192
 193static void cmd_ent_put(struct mlx5_cmd_work_ent *ent)
 194{
 195	struct mlx5_cmd *cmd = ent->cmd;
 196	unsigned long flags;
 197
 198	spin_lock_irqsave(&cmd->alloc_lock, flags);
 199	if (!refcount_dec_and_test(&ent->refcnt))
 200		goto out;
 201
 202	if (ent->idx >= 0) {
 203		cmd_free_index(cmd, ent->idx);
 204		up(ent->page_queue ? &cmd->vars.pages_sem : &cmd->vars.sem);
 205	}
 206
 207	cmd_free_ent(ent);
 208out:
 209	spin_unlock_irqrestore(&cmd->alloc_lock, flags);
 210}
 211
 212static struct mlx5_cmd_layout *get_inst(struct mlx5_cmd *cmd, int idx)
 213{
 214	return cmd->cmd_buf + (idx << cmd->vars.log_stride);
 215}
 216
 217static int mlx5_calc_cmd_blocks(struct mlx5_cmd_msg *msg)
 218{
 219	int size = msg->len;
 220	int blen = size - min_t(int, sizeof(msg->first.data), size);
 221
 222	return DIV_ROUND_UP(blen, MLX5_CMD_DATA_BLOCK_SIZE);
 223}
 224
 225static u8 xor8_buf(void *buf, size_t offset, int len)
 226{
 227	u8 *ptr = buf;
 228	u8 sum = 0;
 229	int i;
 230	int end = len + offset;
 231
 232	for (i = offset; i < end; i++)
 233		sum ^= ptr[i];
 234
 235	return sum;
 236}
 237
 238static int verify_block_sig(struct mlx5_cmd_prot_block *block)
 239{
 240	size_t rsvd0_off = offsetof(struct mlx5_cmd_prot_block, rsvd0);
 241	int xor_len = sizeof(*block) - sizeof(block->data) - 1;
 242
 243	if (xor8_buf(block, rsvd0_off, xor_len) != 0xff)
 244		return -EHWPOISON;
 245
 246	if (xor8_buf(block, 0, sizeof(*block)) != 0xff)
 247		return -EHWPOISON;
 248
 249	return 0;
 250}
 251
 252static void calc_block_sig(struct mlx5_cmd_prot_block *block)
 253{
 254	int ctrl_xor_len = sizeof(*block) - sizeof(block->data) - 2;
 255	size_t rsvd0_off = offsetof(struct mlx5_cmd_prot_block, rsvd0);
 256
 257	block->ctrl_sig = ~xor8_buf(block, rsvd0_off, ctrl_xor_len);
 258	block->sig = ~xor8_buf(block, 0, sizeof(*block) - 1);
 259}
 260
 261static void calc_chain_sig(struct mlx5_cmd_msg *msg)
 262{
 263	struct mlx5_cmd_mailbox *next = msg->next;
 264	int n = mlx5_calc_cmd_blocks(msg);
 265	int i = 0;
 266
 267	for (i = 0; i < n && next; i++)  {
 268		calc_block_sig(next->buf);
 269		next = next->next;
 270	}
 271}
 272
 273static void set_signature(struct mlx5_cmd_work_ent *ent, int csum)
 274{
 275	ent->lay->sig = ~xor8_buf(ent->lay, 0,  sizeof(*ent->lay));
 276	if (csum) {
 277		calc_chain_sig(ent->in);
 278		calc_chain_sig(ent->out);
 279	}
 280}
 281
 282static void poll_timeout(struct mlx5_cmd_work_ent *ent)
 283{
 284	struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev, cmd);
 285	u64 cmd_to_ms = mlx5_tout_ms(dev, CMD);
 286	unsigned long poll_end;
 287	u8 own;
 288
 289	poll_end = jiffies + msecs_to_jiffies(cmd_to_ms + 1000);
 290
 291	do {
 292		own = READ_ONCE(ent->lay->status_own);
 293		if (!(own & CMD_OWNER_HW)) {
 294			ent->ret = 0;
 295			return;
 296		}
 297		cond_resched();
 298		if (mlx5_cmd_is_down(dev)) {
 299			ent->ret = -ENXIO;
 300			return;
 301		}
 302	} while (time_before(jiffies, poll_end));
 303
 304	ent->ret = -ETIMEDOUT;
 305}
 306
 307static int verify_signature(struct mlx5_cmd_work_ent *ent)
 308{
 309	struct mlx5_cmd_mailbox *next = ent->out->next;
 310	int n = mlx5_calc_cmd_blocks(ent->out);
 311	int err;
 312	u8 sig;
 313	int i = 0;
 314
 315	sig = xor8_buf(ent->lay, 0, sizeof(*ent->lay));
 316	if (sig != 0xff)
 317		return -EHWPOISON;
 318
 319	for (i = 0; i < n && next; i++) {
 320		err = verify_block_sig(next->buf);
 321		if (err)
 322			return -EHWPOISON;
 323
 324		next = next->next;
 325	}
 326
 327	return 0;
 328}
 329
 330static void dump_buf(void *buf, int size, int data_only, int offset, int idx)
 331{
 332	__be32 *p = buf;
 333	int i;
 334
 335	for (i = 0; i < size; i += 16) {
 336		pr_debug("cmd[%d]: %03x: %08x %08x %08x %08x\n", idx, offset,
 337			 be32_to_cpu(p[0]), be32_to_cpu(p[1]),
 338			 be32_to_cpu(p[2]), be32_to_cpu(p[3]));
 339		p += 4;
 340		offset += 16;
 341	}
 342	if (!data_only)
 343		pr_debug("\n");
 344}
 345
 346static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op,
 347				       u32 *synd, u8 *status)
 348{
 349	*synd = 0;
 350	*status = 0;
 351
 352	switch (op) {
 353	case MLX5_CMD_OP_TEARDOWN_HCA:
 354	case MLX5_CMD_OP_DISABLE_HCA:
 355	case MLX5_CMD_OP_MANAGE_PAGES:
 356	case MLX5_CMD_OP_DESTROY_MKEY:
 357	case MLX5_CMD_OP_DESTROY_EQ:
 358	case MLX5_CMD_OP_DESTROY_CQ:
 359	case MLX5_CMD_OP_DESTROY_QP:
 360	case MLX5_CMD_OP_DESTROY_PSV:
 361	case MLX5_CMD_OP_DESTROY_SRQ:
 362	case MLX5_CMD_OP_DESTROY_XRC_SRQ:
 363	case MLX5_CMD_OP_DESTROY_XRQ:
 364	case MLX5_CMD_OP_DESTROY_DCT:
 365	case MLX5_CMD_OP_DEALLOC_Q_COUNTER:
 366	case MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT:
 367	case MLX5_CMD_OP_DESTROY_QOS_PARA_VPORT:
 368	case MLX5_CMD_OP_DEALLOC_PD:
 369	case MLX5_CMD_OP_DEALLOC_UAR:
 370	case MLX5_CMD_OP_DETACH_FROM_MCG:
 371	case MLX5_CMD_OP_DEALLOC_XRCD:
 372	case MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN:
 373	case MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT:
 374	case MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY:
 375	case MLX5_CMD_OP_DESTROY_LAG:
 376	case MLX5_CMD_OP_DESTROY_VPORT_LAG:
 377	case MLX5_CMD_OP_DESTROY_TIR:
 378	case MLX5_CMD_OP_DESTROY_SQ:
 379	case MLX5_CMD_OP_DESTROY_RQ:
 380	case MLX5_CMD_OP_DESTROY_RMP:
 381	case MLX5_CMD_OP_DESTROY_TIS:
 382	case MLX5_CMD_OP_DESTROY_RQT:
 383	case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
 384	case MLX5_CMD_OP_DESTROY_FLOW_GROUP:
 385	case MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY:
 386	case MLX5_CMD_OP_DEALLOC_FLOW_COUNTER:
 387	case MLX5_CMD_OP_2ERR_QP:
 388	case MLX5_CMD_OP_2RST_QP:
 389	case MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT:
 390	case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
 391	case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 392	case MLX5_CMD_OP_SET_FLOW_TABLE_ROOT:
 393	case MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT:
 394	case MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT:
 395	case MLX5_CMD_OP_FPGA_DESTROY_QP:
 396	case MLX5_CMD_OP_DESTROY_GENERAL_OBJECT:
 397	case MLX5_CMD_OP_DEALLOC_MEMIC:
 398	case MLX5_CMD_OP_PAGE_FAULT_RESUME:
 399	case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS:
 400	case MLX5_CMD_OP_DEALLOC_SF:
 401	case MLX5_CMD_OP_DESTROY_UCTX:
 402	case MLX5_CMD_OP_DESTROY_UMEM:
 403	case MLX5_CMD_OP_MODIFY_RQT:
 404		return MLX5_CMD_STAT_OK;
 405
 406	case MLX5_CMD_OP_QUERY_HCA_CAP:
 407	case MLX5_CMD_OP_QUERY_ADAPTER:
 408	case MLX5_CMD_OP_INIT_HCA:
 409	case MLX5_CMD_OP_ENABLE_HCA:
 410	case MLX5_CMD_OP_QUERY_PAGES:
 411	case MLX5_CMD_OP_SET_HCA_CAP:
 412	case MLX5_CMD_OP_QUERY_ISSI:
 413	case MLX5_CMD_OP_SET_ISSI:
 414	case MLX5_CMD_OP_CREATE_MKEY:
 415	case MLX5_CMD_OP_QUERY_MKEY:
 416	case MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS:
 417	case MLX5_CMD_OP_CREATE_EQ:
 418	case MLX5_CMD_OP_QUERY_EQ:
 419	case MLX5_CMD_OP_GEN_EQE:
 420	case MLX5_CMD_OP_CREATE_CQ:
 421	case MLX5_CMD_OP_QUERY_CQ:
 422	case MLX5_CMD_OP_MODIFY_CQ:
 423	case MLX5_CMD_OP_CREATE_QP:
 424	case MLX5_CMD_OP_RST2INIT_QP:
 425	case MLX5_CMD_OP_INIT2RTR_QP:
 426	case MLX5_CMD_OP_RTR2RTS_QP:
 427	case MLX5_CMD_OP_RTS2RTS_QP:
 428	case MLX5_CMD_OP_SQERR2RTS_QP:
 429	case MLX5_CMD_OP_QUERY_QP:
 430	case MLX5_CMD_OP_SQD_RTS_QP:
 431	case MLX5_CMD_OP_INIT2INIT_QP:
 432	case MLX5_CMD_OP_CREATE_PSV:
 433	case MLX5_CMD_OP_CREATE_SRQ:
 434	case MLX5_CMD_OP_QUERY_SRQ:
 435	case MLX5_CMD_OP_ARM_RQ:
 436	case MLX5_CMD_OP_CREATE_XRC_SRQ:
 437	case MLX5_CMD_OP_QUERY_XRC_SRQ:
 438	case MLX5_CMD_OP_ARM_XRC_SRQ:
 439	case MLX5_CMD_OP_CREATE_XRQ:
 440	case MLX5_CMD_OP_QUERY_XRQ:
 441	case MLX5_CMD_OP_ARM_XRQ:
 442	case MLX5_CMD_OP_CREATE_DCT:
 443	case MLX5_CMD_OP_DRAIN_DCT:
 444	case MLX5_CMD_OP_QUERY_DCT:
 445	case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
 446	case MLX5_CMD_OP_QUERY_VPORT_STATE:
 447	case MLX5_CMD_OP_MODIFY_VPORT_STATE:
 448	case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
 449	case MLX5_CMD_OP_MODIFY_ESW_VPORT_CONTEXT:
 450	case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT:
 451	case MLX5_CMD_OP_QUERY_ROCE_ADDRESS:
 452	case MLX5_CMD_OP_SET_ROCE_ADDRESS:
 453	case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
 454	case MLX5_CMD_OP_MODIFY_HCA_VPORT_CONTEXT:
 455	case MLX5_CMD_OP_QUERY_HCA_VPORT_GID:
 456	case MLX5_CMD_OP_QUERY_HCA_VPORT_PKEY:
 457	case MLX5_CMD_OP_QUERY_VNIC_ENV:
 458	case MLX5_CMD_OP_QUERY_VPORT_COUNTER:
 459	case MLX5_CMD_OP_ALLOC_Q_COUNTER:
 460	case MLX5_CMD_OP_QUERY_Q_COUNTER:
 461	case MLX5_CMD_OP_SET_MONITOR_COUNTER:
 462	case MLX5_CMD_OP_ARM_MONITOR_COUNTER:
 463	case MLX5_CMD_OP_SET_PP_RATE_LIMIT:
 464	case MLX5_CMD_OP_QUERY_RATE_LIMIT:
 465	case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
 466	case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
 467	case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
 468	case MLX5_CMD_OP_CREATE_QOS_PARA_VPORT:
 469	case MLX5_CMD_OP_ALLOC_PD:
 470	case MLX5_CMD_OP_ALLOC_UAR:
 471	case MLX5_CMD_OP_CONFIG_INT_MODERATION:
 472	case MLX5_CMD_OP_ACCESS_REG:
 473	case MLX5_CMD_OP_ATTACH_TO_MCG:
 474	case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG:
 475	case MLX5_CMD_OP_MAD_IFC:
 476	case MLX5_CMD_OP_QUERY_MAD_DEMUX:
 477	case MLX5_CMD_OP_SET_MAD_DEMUX:
 478	case MLX5_CMD_OP_NOP:
 479	case MLX5_CMD_OP_ALLOC_XRCD:
 480	case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
 481	case MLX5_CMD_OP_QUERY_CONG_STATUS:
 482	case MLX5_CMD_OP_MODIFY_CONG_STATUS:
 483	case MLX5_CMD_OP_QUERY_CONG_PARAMS:
 484	case MLX5_CMD_OP_MODIFY_CONG_PARAMS:
 485	case MLX5_CMD_OP_QUERY_CONG_STATISTICS:
 486	case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 487	case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 488	case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
 489	case MLX5_CMD_OP_CREATE_LAG:
 490	case MLX5_CMD_OP_MODIFY_LAG:
 491	case MLX5_CMD_OP_QUERY_LAG:
 492	case MLX5_CMD_OP_CREATE_VPORT_LAG:
 493	case MLX5_CMD_OP_CREATE_TIR:
 494	case MLX5_CMD_OP_MODIFY_TIR:
 495	case MLX5_CMD_OP_QUERY_TIR:
 496	case MLX5_CMD_OP_CREATE_SQ:
 497	case MLX5_CMD_OP_MODIFY_SQ:
 498	case MLX5_CMD_OP_QUERY_SQ:
 499	case MLX5_CMD_OP_CREATE_RQ:
 500	case MLX5_CMD_OP_MODIFY_RQ:
 501	case MLX5_CMD_OP_QUERY_RQ:
 502	case MLX5_CMD_OP_CREATE_RMP:
 503	case MLX5_CMD_OP_MODIFY_RMP:
 504	case MLX5_CMD_OP_QUERY_RMP:
 505	case MLX5_CMD_OP_CREATE_TIS:
 506	case MLX5_CMD_OP_MODIFY_TIS:
 507	case MLX5_CMD_OP_QUERY_TIS:
 508	case MLX5_CMD_OP_CREATE_RQT:
 509	case MLX5_CMD_OP_QUERY_RQT:
 510
 511	case MLX5_CMD_OP_CREATE_FLOW_TABLE:
 512	case MLX5_CMD_OP_QUERY_FLOW_TABLE:
 513	case MLX5_CMD_OP_CREATE_FLOW_GROUP:
 514	case MLX5_CMD_OP_QUERY_FLOW_GROUP:
 515	case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
 516	case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
 517	case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
 518	case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
 519	case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
 520	case MLX5_CMD_OP_FPGA_CREATE_QP:
 521	case MLX5_CMD_OP_FPGA_MODIFY_QP:
 522	case MLX5_CMD_OP_FPGA_QUERY_QP:
 523	case MLX5_CMD_OP_FPGA_QUERY_QP_COUNTERS:
 524	case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
 525	case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
 526	case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
 527	case MLX5_CMD_OP_CREATE_UCTX:
 528	case MLX5_CMD_OP_CREATE_UMEM:
 529	case MLX5_CMD_OP_ALLOC_MEMIC:
 530	case MLX5_CMD_OP_MODIFY_XRQ:
 531	case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
 532	case MLX5_CMD_OP_QUERY_VHCA_STATE:
 533	case MLX5_CMD_OP_MODIFY_VHCA_STATE:
 534	case MLX5_CMD_OP_ALLOC_SF:
 535	case MLX5_CMD_OP_SUSPEND_VHCA:
 536	case MLX5_CMD_OP_RESUME_VHCA:
 537	case MLX5_CMD_OP_QUERY_VHCA_MIGRATION_STATE:
 538	case MLX5_CMD_OP_SAVE_VHCA_STATE:
 539	case MLX5_CMD_OP_LOAD_VHCA_STATE:
 540	case MLX5_CMD_OP_SYNC_CRYPTO:
 541	case MLX5_CMD_OP_ALLOW_OTHER_VHCA_ACCESS:
 542		*status = MLX5_DRIVER_STATUS_ABORTED;
 543		*synd = MLX5_DRIVER_SYND;
 544		return -ENOLINK;
 545	default:
 546		mlx5_core_err(dev, "Unknown FW command (%d)\n", op);
 547		return -EINVAL;
 548	}
 549}
 550
 551const char *mlx5_command_str(int command)
 552{
 553#define MLX5_COMMAND_STR_CASE(__cmd) case MLX5_CMD_OP_ ## __cmd: return #__cmd
 554
 555	switch (command) {
 556	MLX5_COMMAND_STR_CASE(QUERY_HCA_CAP);
 557	MLX5_COMMAND_STR_CASE(QUERY_ADAPTER);
 558	MLX5_COMMAND_STR_CASE(INIT_HCA);
 559	MLX5_COMMAND_STR_CASE(TEARDOWN_HCA);
 560	MLX5_COMMAND_STR_CASE(ENABLE_HCA);
 561	MLX5_COMMAND_STR_CASE(DISABLE_HCA);
 562	MLX5_COMMAND_STR_CASE(QUERY_PAGES);
 563	MLX5_COMMAND_STR_CASE(MANAGE_PAGES);
 564	MLX5_COMMAND_STR_CASE(SET_HCA_CAP);
 565	MLX5_COMMAND_STR_CASE(QUERY_ISSI);
 566	MLX5_COMMAND_STR_CASE(SET_ISSI);
 567	MLX5_COMMAND_STR_CASE(SET_DRIVER_VERSION);
 568	MLX5_COMMAND_STR_CASE(CREATE_MKEY);
 569	MLX5_COMMAND_STR_CASE(QUERY_MKEY);
 570	MLX5_COMMAND_STR_CASE(DESTROY_MKEY);
 571	MLX5_COMMAND_STR_CASE(QUERY_SPECIAL_CONTEXTS);
 572	MLX5_COMMAND_STR_CASE(PAGE_FAULT_RESUME);
 573	MLX5_COMMAND_STR_CASE(CREATE_EQ);
 574	MLX5_COMMAND_STR_CASE(DESTROY_EQ);
 575	MLX5_COMMAND_STR_CASE(QUERY_EQ);
 576	MLX5_COMMAND_STR_CASE(GEN_EQE);
 577	MLX5_COMMAND_STR_CASE(CREATE_CQ);
 578	MLX5_COMMAND_STR_CASE(DESTROY_CQ);
 579	MLX5_COMMAND_STR_CASE(QUERY_CQ);
 580	MLX5_COMMAND_STR_CASE(MODIFY_CQ);
 581	MLX5_COMMAND_STR_CASE(CREATE_QP);
 582	MLX5_COMMAND_STR_CASE(DESTROY_QP);
 583	MLX5_COMMAND_STR_CASE(RST2INIT_QP);
 584	MLX5_COMMAND_STR_CASE(INIT2RTR_QP);
 585	MLX5_COMMAND_STR_CASE(RTR2RTS_QP);
 586	MLX5_COMMAND_STR_CASE(RTS2RTS_QP);
 587	MLX5_COMMAND_STR_CASE(SQERR2RTS_QP);
 588	MLX5_COMMAND_STR_CASE(2ERR_QP);
 589	MLX5_COMMAND_STR_CASE(2RST_QP);
 590	MLX5_COMMAND_STR_CASE(QUERY_QP);
 591	MLX5_COMMAND_STR_CASE(SQD_RTS_QP);
 592	MLX5_COMMAND_STR_CASE(INIT2INIT_QP);
 593	MLX5_COMMAND_STR_CASE(CREATE_PSV);
 594	MLX5_COMMAND_STR_CASE(DESTROY_PSV);
 595	MLX5_COMMAND_STR_CASE(CREATE_SRQ);
 596	MLX5_COMMAND_STR_CASE(DESTROY_SRQ);
 597	MLX5_COMMAND_STR_CASE(QUERY_SRQ);
 598	MLX5_COMMAND_STR_CASE(ARM_RQ);
 599	MLX5_COMMAND_STR_CASE(CREATE_XRC_SRQ);
 600	MLX5_COMMAND_STR_CASE(DESTROY_XRC_SRQ);
 601	MLX5_COMMAND_STR_CASE(QUERY_XRC_SRQ);
 602	MLX5_COMMAND_STR_CASE(ARM_XRC_SRQ);
 603	MLX5_COMMAND_STR_CASE(CREATE_DCT);
 604	MLX5_COMMAND_STR_CASE(DESTROY_DCT);
 605	MLX5_COMMAND_STR_CASE(DRAIN_DCT);
 606	MLX5_COMMAND_STR_CASE(QUERY_DCT);
 607	MLX5_COMMAND_STR_CASE(ARM_DCT_FOR_KEY_VIOLATION);
 608	MLX5_COMMAND_STR_CASE(QUERY_VPORT_STATE);
 609	MLX5_COMMAND_STR_CASE(MODIFY_VPORT_STATE);
 610	MLX5_COMMAND_STR_CASE(QUERY_ESW_VPORT_CONTEXT);
 611	MLX5_COMMAND_STR_CASE(MODIFY_ESW_VPORT_CONTEXT);
 612	MLX5_COMMAND_STR_CASE(QUERY_NIC_VPORT_CONTEXT);
 613	MLX5_COMMAND_STR_CASE(MODIFY_NIC_VPORT_CONTEXT);
 614	MLX5_COMMAND_STR_CASE(QUERY_ROCE_ADDRESS);
 615	MLX5_COMMAND_STR_CASE(SET_ROCE_ADDRESS);
 616	MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_CONTEXT);
 617	MLX5_COMMAND_STR_CASE(MODIFY_HCA_VPORT_CONTEXT);
 618	MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_GID);
 619	MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_PKEY);
 620	MLX5_COMMAND_STR_CASE(QUERY_VNIC_ENV);
 621	MLX5_COMMAND_STR_CASE(QUERY_VPORT_COUNTER);
 622	MLX5_COMMAND_STR_CASE(ALLOC_Q_COUNTER);
 623	MLX5_COMMAND_STR_CASE(DEALLOC_Q_COUNTER);
 624	MLX5_COMMAND_STR_CASE(QUERY_Q_COUNTER);
 625	MLX5_COMMAND_STR_CASE(SET_MONITOR_COUNTER);
 626	MLX5_COMMAND_STR_CASE(ARM_MONITOR_COUNTER);
 627	MLX5_COMMAND_STR_CASE(SET_PP_RATE_LIMIT);
 628	MLX5_COMMAND_STR_CASE(QUERY_RATE_LIMIT);
 629	MLX5_COMMAND_STR_CASE(CREATE_SCHEDULING_ELEMENT);
 630	MLX5_COMMAND_STR_CASE(DESTROY_SCHEDULING_ELEMENT);
 631	MLX5_COMMAND_STR_CASE(QUERY_SCHEDULING_ELEMENT);
 632	MLX5_COMMAND_STR_CASE(MODIFY_SCHEDULING_ELEMENT);
 633	MLX5_COMMAND_STR_CASE(CREATE_QOS_PARA_VPORT);
 634	MLX5_COMMAND_STR_CASE(DESTROY_QOS_PARA_VPORT);
 635	MLX5_COMMAND_STR_CASE(ALLOC_PD);
 636	MLX5_COMMAND_STR_CASE(DEALLOC_PD);
 637	MLX5_COMMAND_STR_CASE(ALLOC_UAR);
 638	MLX5_COMMAND_STR_CASE(DEALLOC_UAR);
 639	MLX5_COMMAND_STR_CASE(CONFIG_INT_MODERATION);
 640	MLX5_COMMAND_STR_CASE(ACCESS_REG);
 641	MLX5_COMMAND_STR_CASE(ATTACH_TO_MCG);
 642	MLX5_COMMAND_STR_CASE(DETACH_FROM_MCG);
 643	MLX5_COMMAND_STR_CASE(GET_DROPPED_PACKET_LOG);
 644	MLX5_COMMAND_STR_CASE(MAD_IFC);
 645	MLX5_COMMAND_STR_CASE(QUERY_MAD_DEMUX);
 646	MLX5_COMMAND_STR_CASE(SET_MAD_DEMUX);
 647	MLX5_COMMAND_STR_CASE(NOP);
 648	MLX5_COMMAND_STR_CASE(ALLOC_XRCD);
 649	MLX5_COMMAND_STR_CASE(DEALLOC_XRCD);
 650	MLX5_COMMAND_STR_CASE(ALLOC_TRANSPORT_DOMAIN);
 651	MLX5_COMMAND_STR_CASE(DEALLOC_TRANSPORT_DOMAIN);
 652	MLX5_COMMAND_STR_CASE(QUERY_CONG_STATUS);
 653	MLX5_COMMAND_STR_CASE(MODIFY_CONG_STATUS);
 654	MLX5_COMMAND_STR_CASE(QUERY_CONG_PARAMS);
 655	MLX5_COMMAND_STR_CASE(MODIFY_CONG_PARAMS);
 656	MLX5_COMMAND_STR_CASE(QUERY_CONG_STATISTICS);
 657	MLX5_COMMAND_STR_CASE(ADD_VXLAN_UDP_DPORT);
 658	MLX5_COMMAND_STR_CASE(DELETE_VXLAN_UDP_DPORT);
 659	MLX5_COMMAND_STR_CASE(SET_L2_TABLE_ENTRY);
 660	MLX5_COMMAND_STR_CASE(QUERY_L2_TABLE_ENTRY);
 661	MLX5_COMMAND_STR_CASE(DELETE_L2_TABLE_ENTRY);
 662	MLX5_COMMAND_STR_CASE(SET_WOL_ROL);
 663	MLX5_COMMAND_STR_CASE(QUERY_WOL_ROL);
 664	MLX5_COMMAND_STR_CASE(CREATE_LAG);
 665	MLX5_COMMAND_STR_CASE(MODIFY_LAG);
 666	MLX5_COMMAND_STR_CASE(QUERY_LAG);
 667	MLX5_COMMAND_STR_CASE(DESTROY_LAG);
 668	MLX5_COMMAND_STR_CASE(CREATE_VPORT_LAG);
 669	MLX5_COMMAND_STR_CASE(DESTROY_VPORT_LAG);
 670	MLX5_COMMAND_STR_CASE(CREATE_TIR);
 671	MLX5_COMMAND_STR_CASE(MODIFY_TIR);
 672	MLX5_COMMAND_STR_CASE(DESTROY_TIR);
 673	MLX5_COMMAND_STR_CASE(QUERY_TIR);
 674	MLX5_COMMAND_STR_CASE(CREATE_SQ);
 675	MLX5_COMMAND_STR_CASE(MODIFY_SQ);
 676	MLX5_COMMAND_STR_CASE(DESTROY_SQ);
 677	MLX5_COMMAND_STR_CASE(QUERY_SQ);
 678	MLX5_COMMAND_STR_CASE(CREATE_RQ);
 679	MLX5_COMMAND_STR_CASE(MODIFY_RQ);
 680	MLX5_COMMAND_STR_CASE(DESTROY_RQ);
 681	MLX5_COMMAND_STR_CASE(QUERY_RQ);
 682	MLX5_COMMAND_STR_CASE(CREATE_RMP);
 683	MLX5_COMMAND_STR_CASE(MODIFY_RMP);
 684	MLX5_COMMAND_STR_CASE(DESTROY_RMP);
 685	MLX5_COMMAND_STR_CASE(QUERY_RMP);
 686	MLX5_COMMAND_STR_CASE(CREATE_TIS);
 687	MLX5_COMMAND_STR_CASE(MODIFY_TIS);
 688	MLX5_COMMAND_STR_CASE(DESTROY_TIS);
 689	MLX5_COMMAND_STR_CASE(QUERY_TIS);
 690	MLX5_COMMAND_STR_CASE(CREATE_RQT);
 691	MLX5_COMMAND_STR_CASE(MODIFY_RQT);
 692	MLX5_COMMAND_STR_CASE(DESTROY_RQT);
 693	MLX5_COMMAND_STR_CASE(QUERY_RQT);
 694	MLX5_COMMAND_STR_CASE(SET_FLOW_TABLE_ROOT);
 695	MLX5_COMMAND_STR_CASE(CREATE_FLOW_TABLE);
 696	MLX5_COMMAND_STR_CASE(DESTROY_FLOW_TABLE);
 697	MLX5_COMMAND_STR_CASE(QUERY_FLOW_TABLE);
 698	MLX5_COMMAND_STR_CASE(CREATE_FLOW_GROUP);
 699	MLX5_COMMAND_STR_CASE(DESTROY_FLOW_GROUP);
 700	MLX5_COMMAND_STR_CASE(QUERY_FLOW_GROUP);
 701	MLX5_COMMAND_STR_CASE(SET_FLOW_TABLE_ENTRY);
 702	MLX5_COMMAND_STR_CASE(QUERY_FLOW_TABLE_ENTRY);
 703	MLX5_COMMAND_STR_CASE(DELETE_FLOW_TABLE_ENTRY);
 704	MLX5_COMMAND_STR_CASE(ALLOC_FLOW_COUNTER);
 705	MLX5_COMMAND_STR_CASE(DEALLOC_FLOW_COUNTER);
 706	MLX5_COMMAND_STR_CASE(QUERY_FLOW_COUNTER);
 707	MLX5_COMMAND_STR_CASE(MODIFY_FLOW_TABLE);
 708	MLX5_COMMAND_STR_CASE(ALLOC_PACKET_REFORMAT_CONTEXT);
 709	MLX5_COMMAND_STR_CASE(DEALLOC_PACKET_REFORMAT_CONTEXT);
 710	MLX5_COMMAND_STR_CASE(ALLOC_MODIFY_HEADER_CONTEXT);
 711	MLX5_COMMAND_STR_CASE(DEALLOC_MODIFY_HEADER_CONTEXT);
 712	MLX5_COMMAND_STR_CASE(FPGA_CREATE_QP);
 713	MLX5_COMMAND_STR_CASE(FPGA_MODIFY_QP);
 714	MLX5_COMMAND_STR_CASE(FPGA_QUERY_QP);
 715	MLX5_COMMAND_STR_CASE(FPGA_QUERY_QP_COUNTERS);
 716	MLX5_COMMAND_STR_CASE(FPGA_DESTROY_QP);
 717	MLX5_COMMAND_STR_CASE(CREATE_XRQ);
 718	MLX5_COMMAND_STR_CASE(DESTROY_XRQ);
 719	MLX5_COMMAND_STR_CASE(QUERY_XRQ);
 720	MLX5_COMMAND_STR_CASE(ARM_XRQ);
 721	MLX5_COMMAND_STR_CASE(CREATE_GENERAL_OBJECT);
 722	MLX5_COMMAND_STR_CASE(DESTROY_GENERAL_OBJECT);
 723	MLX5_COMMAND_STR_CASE(MODIFY_GENERAL_OBJECT);
 724	MLX5_COMMAND_STR_CASE(QUERY_GENERAL_OBJECT);
 725	MLX5_COMMAND_STR_CASE(QUERY_MODIFY_HEADER_CONTEXT);
 726	MLX5_COMMAND_STR_CASE(ALLOC_MEMIC);
 727	MLX5_COMMAND_STR_CASE(DEALLOC_MEMIC);
 728	MLX5_COMMAND_STR_CASE(QUERY_ESW_FUNCTIONS);
 729	MLX5_COMMAND_STR_CASE(CREATE_UCTX);
 730	MLX5_COMMAND_STR_CASE(DESTROY_UCTX);
 731	MLX5_COMMAND_STR_CASE(CREATE_UMEM);
 732	MLX5_COMMAND_STR_CASE(DESTROY_UMEM);
 733	MLX5_COMMAND_STR_CASE(RELEASE_XRQ_ERROR);
 734	MLX5_COMMAND_STR_CASE(MODIFY_XRQ);
 735	MLX5_COMMAND_STR_CASE(QUERY_VHCA_STATE);
 736	MLX5_COMMAND_STR_CASE(MODIFY_VHCA_STATE);
 737	MLX5_COMMAND_STR_CASE(ALLOC_SF);
 738	MLX5_COMMAND_STR_CASE(DEALLOC_SF);
 739	MLX5_COMMAND_STR_CASE(SUSPEND_VHCA);
 740	MLX5_COMMAND_STR_CASE(RESUME_VHCA);
 741	MLX5_COMMAND_STR_CASE(QUERY_VHCA_MIGRATION_STATE);
 742	MLX5_COMMAND_STR_CASE(SAVE_VHCA_STATE);
 743	MLX5_COMMAND_STR_CASE(LOAD_VHCA_STATE);
 744	MLX5_COMMAND_STR_CASE(SYNC_CRYPTO);
 745	MLX5_COMMAND_STR_CASE(ALLOW_OTHER_VHCA_ACCESS);
 746	default: return "unknown command opcode";
 747	}
 748}
 749
 750static const char *cmd_status_str(u8 status)
 751{
 752	switch (status) {
 753	case MLX5_CMD_STAT_OK:
 754		return "OK";
 755	case MLX5_CMD_STAT_INT_ERR:
 756		return "internal error";
 757	case MLX5_CMD_STAT_BAD_OP_ERR:
 758		return "bad operation";
 759	case MLX5_CMD_STAT_BAD_PARAM_ERR:
 760		return "bad parameter";
 761	case MLX5_CMD_STAT_BAD_SYS_STATE_ERR:
 762		return "bad system state";
 763	case MLX5_CMD_STAT_BAD_RES_ERR:
 764		return "bad resource";
 765	case MLX5_CMD_STAT_RES_BUSY:
 766		return "resource busy";
 767	case MLX5_CMD_STAT_NOT_READY:
 768		return "FW not ready";
 769	case MLX5_CMD_STAT_LIM_ERR:
 770		return "limits exceeded";
 771	case MLX5_CMD_STAT_BAD_RES_STATE_ERR:
 772		return "bad resource state";
 773	case MLX5_CMD_STAT_IX_ERR:
 774		return "bad index";
 775	case MLX5_CMD_STAT_NO_RES_ERR:
 776		return "no resources";
 777	case MLX5_CMD_STAT_BAD_INP_LEN_ERR:
 778		return "bad input length";
 779	case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR:
 780		return "bad output length";
 781	case MLX5_CMD_STAT_BAD_QP_STATE_ERR:
 782		return "bad QP state";
 783	case MLX5_CMD_STAT_BAD_PKT_ERR:
 784		return "bad packet (discarded)";
 785	case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR:
 786		return "bad size too many outstanding CQEs";
 787	default:
 788		return "unknown status";
 789	}
 790}
 791
 792static int cmd_status_to_err(u8 status)
 793{
 794	switch (status) {
 795	case MLX5_CMD_STAT_OK:				return 0;
 796	case MLX5_CMD_STAT_INT_ERR:			return -EIO;
 797	case MLX5_CMD_STAT_BAD_OP_ERR:			return -EINVAL;
 798	case MLX5_CMD_STAT_BAD_PARAM_ERR:		return -EINVAL;
 799	case MLX5_CMD_STAT_BAD_SYS_STATE_ERR:		return -EIO;
 800	case MLX5_CMD_STAT_BAD_RES_ERR:			return -EINVAL;
 801	case MLX5_CMD_STAT_RES_BUSY:			return -EBUSY;
 802	case MLX5_CMD_STAT_NOT_READY:			return -EAGAIN;
 803	case MLX5_CMD_STAT_LIM_ERR:			return -ENOMEM;
 804	case MLX5_CMD_STAT_BAD_RES_STATE_ERR:		return -EINVAL;
 805	case MLX5_CMD_STAT_IX_ERR:			return -EINVAL;
 806	case MLX5_CMD_STAT_NO_RES_ERR:			return -EAGAIN;
 807	case MLX5_CMD_STAT_BAD_INP_LEN_ERR:		return -EIO;
 808	case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR:		return -EIO;
 809	case MLX5_CMD_STAT_BAD_QP_STATE_ERR:		return -EINVAL;
 810	case MLX5_CMD_STAT_BAD_PKT_ERR:			return -EINVAL;
 811	case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR:	return -EINVAL;
 812	default:					return -EIO;
 813	}
 814}
 815
 816void mlx5_cmd_out_err(struct mlx5_core_dev *dev, u16 opcode, u16 op_mod, void *out)
 817{
 818	u32 syndrome = MLX5_GET(mbox_out, out, syndrome);
 819	u8 status = MLX5_GET(mbox_out, out, status);
 820
 821	mlx5_core_err_rl(dev,
 822			 "%s(0x%x) op_mod(0x%x) failed, status %s(0x%x), syndrome (0x%x), err(%d)\n",
 823			 mlx5_command_str(opcode), opcode, op_mod,
 824			 cmd_status_str(status), status, syndrome, cmd_status_to_err(status));
 825}
 826EXPORT_SYMBOL(mlx5_cmd_out_err);
 827
 828static void cmd_status_print(struct mlx5_core_dev *dev, void *in, void *out)
 829{
 830	u16 opcode, op_mod;
 831	u8 status;
 832	u16 uid;
 833
 834	opcode = in_to_opcode(in);
 835	op_mod = MLX5_GET(mbox_in, in, op_mod);
 836	uid    = in_to_uid(in);
 837	status = MLX5_GET(mbox_out, out, status);
 838
 839	if (!uid && opcode != MLX5_CMD_OP_DESTROY_MKEY &&
 840	    opcode != MLX5_CMD_OP_CREATE_UCTX && status != MLX5_CMD_STAT_NOT_READY)
 841		mlx5_cmd_out_err(dev, opcode, op_mod, out);
 842}
 843
 844int mlx5_cmd_check(struct mlx5_core_dev *dev, int err, void *in, void *out)
 845{
 846	/* aborted due to PCI error or via reset flow mlx5_cmd_trigger_completions() */
 847	if (err == -ENXIO) {
 848		u16 opcode = in_to_opcode(in);
 849		u32 syndrome;
 850		u8 status;
 851
 852		/* PCI Error, emulate command return status, for smooth reset */
 853		err = mlx5_internal_err_ret_value(dev, opcode, &syndrome, &status);
 854		MLX5_SET(mbox_out, out, status, status);
 855		MLX5_SET(mbox_out, out, syndrome, syndrome);
 856		if (!err)
 857			return 0;
 858	}
 859
 860	/* driver or FW delivery error */
 861	if (err != -EREMOTEIO && err)
 862		return err;
 863
 864	/* check outbox status */
 865	err = cmd_status_to_err(MLX5_GET(mbox_out, out, status));
 866	if (err)
 867		cmd_status_print(dev, in, out);
 868
 869	return err;
 870}
 871EXPORT_SYMBOL(mlx5_cmd_check);
 872
 873static void dump_command(struct mlx5_core_dev *dev,
 874			 struct mlx5_cmd_work_ent *ent, int input)
 875{
 876	struct mlx5_cmd_msg *msg = input ? ent->in : ent->out;
 877	struct mlx5_cmd_mailbox *next = msg->next;
 878	int n = mlx5_calc_cmd_blocks(msg);
 879	u16 op = ent->op;
 880	int data_only;
 881	u32 offset = 0;
 882	int dump_len;
 883	int i;
 884
 885	mlx5_core_dbg(dev, "cmd[%d]: start dump\n", ent->idx);
 886	data_only = !!(mlx5_core_debug_mask & (1 << MLX5_CMD_DATA));
 887
 888	if (data_only)
 889		mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_DATA,
 890				   "cmd[%d]: dump command data %s(0x%x) %s\n",
 891				   ent->idx, mlx5_command_str(op), op,
 892				   input ? "INPUT" : "OUTPUT");
 893	else
 894		mlx5_core_dbg(dev, "cmd[%d]: dump command %s(0x%x) %s\n",
 895			      ent->idx, mlx5_command_str(op), op,
 896			      input ? "INPUT" : "OUTPUT");
 897
 898	if (data_only) {
 899		if (input) {
 900			dump_buf(ent->lay->in, sizeof(ent->lay->in), 1, offset, ent->idx);
 901			offset += sizeof(ent->lay->in);
 902		} else {
 903			dump_buf(ent->lay->out, sizeof(ent->lay->out), 1, offset, ent->idx);
 904			offset += sizeof(ent->lay->out);
 905		}
 906	} else {
 907		dump_buf(ent->lay, sizeof(*ent->lay), 0, offset, ent->idx);
 908		offset += sizeof(*ent->lay);
 909	}
 910
 911	for (i = 0; i < n && next; i++)  {
 912		if (data_only) {
 913			dump_len = min_t(int, MLX5_CMD_DATA_BLOCK_SIZE, msg->len - offset);
 914			dump_buf(next->buf, dump_len, 1, offset, ent->idx);
 915			offset += MLX5_CMD_DATA_BLOCK_SIZE;
 916		} else {
 917			mlx5_core_dbg(dev, "cmd[%d]: command block:\n", ent->idx);
 918			dump_buf(next->buf, sizeof(struct mlx5_cmd_prot_block), 0, offset,
 919				 ent->idx);
 920			offset += sizeof(struct mlx5_cmd_prot_block);
 921		}
 922		next = next->next;
 923	}
 924
 925	if (data_only)
 926		pr_debug("\n");
 927
 928	mlx5_core_dbg(dev, "cmd[%d]: end dump\n", ent->idx);
 929}
 930
 931static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool forced);
 932
 933static void cb_timeout_handler(struct work_struct *work)
 934{
 935	struct delayed_work *dwork = to_delayed_work(work);
 936	struct mlx5_cmd_work_ent *ent = container_of(dwork,
 937						     struct mlx5_cmd_work_ent,
 938						     cb_timeout_work);
 939	struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev,
 940						 cmd);
 941
 942	mlx5_cmd_eq_recover(dev);
 943
 944	/* Maybe got handled by eq recover ? */
 945	if (!test_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state)) {
 946		mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, recovered after timeout\n", ent->idx,
 947			       mlx5_command_str(ent->op), ent->op);
 948		goto out; /* phew, already handled */
 949	}
 950
 951	ent->ret = -ETIMEDOUT;
 952	mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, timeout. Will cause a leak of a command resource\n",
 953		       ent->idx, mlx5_command_str(ent->op), ent->op);
 954	mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true);
 955
 956out:
 957	cmd_ent_put(ent); /* for the cmd_ent_get() took on schedule delayed work */
 958}
 959
 960static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg);
 961static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev,
 962			      struct mlx5_cmd_msg *msg);
 963
 964static bool opcode_allowed(struct mlx5_cmd *cmd, u16 opcode)
 965{
 966	if (cmd->allowed_opcode == CMD_ALLOWED_OPCODE_ALL)
 967		return true;
 968
 969	return cmd->allowed_opcode == opcode;
 970}
 971
 972bool mlx5_cmd_is_down(struct mlx5_core_dev *dev)
 973{
 974	return pci_channel_offline(dev->pdev) ||
 975	       dev->cmd.state != MLX5_CMDIF_STATE_UP ||
 976	       dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR;
 977}
 978
 979static void cmd_work_handler(struct work_struct *work)
 980{
 981	struct mlx5_cmd_work_ent *ent = container_of(work, struct mlx5_cmd_work_ent, work);
 982	struct mlx5_cmd *cmd = ent->cmd;
 983	bool poll_cmd = ent->polling;
 984	struct mlx5_cmd_layout *lay;
 985	struct mlx5_core_dev *dev;
 986	unsigned long timeout;
 987	unsigned long flags;
 988	int alloc_ret;
 989	int cmd_mode;
 990
 991	complete(&ent->handling);
 992
 993	dev = container_of(cmd, struct mlx5_core_dev, cmd);
 994	timeout = msecs_to_jiffies(mlx5_tout_ms(dev, CMD));
 995
 996	if (!ent->page_queue) {
 997		if (down_timeout(&cmd->vars.sem, timeout)) {
 998			mlx5_core_warn(dev, "%s(0x%x) timed out while waiting for a slot.\n",
 999				       mlx5_command_str(ent->op), ent->op);
1000			if (ent->callback) {
1001				ent->callback(-EBUSY, ent->context);
1002				mlx5_free_cmd_msg(dev, ent->out);
1003				free_msg(dev, ent->in);
1004				cmd_ent_put(ent);
1005			} else {
1006				ent->ret = -EBUSY;
1007				complete(&ent->done);
1008			}
1009			complete(&ent->slotted);
1010			return;
1011		}
1012		alloc_ret = cmd_alloc_index(cmd, ent);
1013		if (alloc_ret < 0) {
1014			mlx5_core_err_rl(dev, "failed to allocate command entry\n");
1015			if (ent->callback) {
1016				ent->callback(-EAGAIN, ent->context);
1017				mlx5_free_cmd_msg(dev, ent->out);
1018				free_msg(dev, ent->in);
1019				cmd_ent_put(ent);
1020			} else {
1021				ent->ret = -EAGAIN;
1022				complete(&ent->done);
1023			}
1024			up(&cmd->vars.sem);
1025			complete(&ent->slotted);
1026			return;
1027		}
1028	} else {
1029		down(&cmd->vars.pages_sem);
1030		ent->idx = cmd->vars.max_reg_cmds;
1031		spin_lock_irqsave(&cmd->alloc_lock, flags);
1032		clear_bit(ent->idx, &cmd->vars.bitmask);
1033		cmd->ent_arr[ent->idx] = ent;
1034		spin_unlock_irqrestore(&cmd->alloc_lock, flags);
1035	}
1036
1037	complete(&ent->slotted);
1038
1039	lay = get_inst(cmd, ent->idx);
1040	ent->lay = lay;
1041	memset(lay, 0, sizeof(*lay));
1042	memcpy(lay->in, ent->in->first.data, sizeof(lay->in));
1043	if (ent->in->next)
1044		lay->in_ptr = cpu_to_be64(ent->in->next->dma);
1045	lay->inlen = cpu_to_be32(ent->in->len);
1046	if (ent->out->next)
1047		lay->out_ptr = cpu_to_be64(ent->out->next->dma);
1048	lay->outlen = cpu_to_be32(ent->out->len);
1049	lay->type = MLX5_PCI_CMD_XPORT;
1050	lay->token = ent->token;
1051	lay->status_own = CMD_OWNER_HW;
1052	set_signature(ent, !cmd->checksum_disabled);
1053	dump_command(dev, ent, 1);
1054	ent->ts1 = ktime_get_ns();
1055	cmd_mode = cmd->mode;
1056
1057	if (ent->callback && schedule_delayed_work(&ent->cb_timeout_work, timeout))
1058		cmd_ent_get(ent);
1059	set_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state);
1060
1061	cmd_ent_get(ent); /* for the _real_ FW event on completion */
1062	/* Skip sending command to fw if internal error */
1063	if (mlx5_cmd_is_down(dev) || !opcode_allowed(&dev->cmd, ent->op)) {
1064		ent->ret = -ENXIO;
1065		mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true);
1066		return;
1067	}
1068
1069	/* ring doorbell after the descriptor is valid */
1070	mlx5_core_dbg(dev, "writing 0x%x to command doorbell\n", 1 << ent->idx);
1071	wmb();
1072	iowrite32be(1 << ent->idx, &dev->iseg->cmd_dbell);
1073	/* if not in polling don't use ent after this point */
1074	if (cmd_mode == CMD_MODE_POLLING || poll_cmd) {
1075		poll_timeout(ent);
1076		/* make sure we read the descriptor after ownership is SW */
1077		rmb();
1078		mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, !!ent->ret);
1079	}
1080}
1081
1082static int deliv_status_to_err(u8 status)
1083{
1084	switch (status) {
1085	case MLX5_CMD_DELIVERY_STAT_OK:
1086	case MLX5_DRIVER_STATUS_ABORTED:
1087		return 0;
1088	case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR:
1089	case MLX5_CMD_DELIVERY_STAT_TOK_ERR:
1090		return -EBADR;
1091	case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR:
1092	case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR:
1093	case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR:
1094		return -EFAULT; /* Bad address */
1095	case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR:
1096	case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR:
1097	case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR:
1098	case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR:
1099		return -ENOMSG;
1100	case MLX5_CMD_DELIVERY_STAT_FW_ERR:
1101		return -EIO;
1102	default:
1103		return -EINVAL;
1104	}
1105}
1106
1107static const char *deliv_status_to_str(u8 status)
1108{
1109	switch (status) {
1110	case MLX5_CMD_DELIVERY_STAT_OK:
1111		return "no errors";
1112	case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR:
1113		return "signature error";
1114	case MLX5_CMD_DELIVERY_STAT_TOK_ERR:
1115		return "token error";
1116	case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR:
1117		return "bad block number";
1118	case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR:
1119		return "output pointer not aligned to block size";
1120	case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR:
1121		return "input pointer not aligned to block size";
1122	case MLX5_CMD_DELIVERY_STAT_FW_ERR:
1123		return "firmware internal error";
1124	case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR:
1125		return "command input length error";
1126	case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR:
1127		return "command output length error";
1128	case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR:
1129		return "reserved fields not cleared";
1130	case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR:
1131		return "bad command descriptor type";
1132	default:
1133		return "unknown status code";
1134	}
1135}
1136
1137enum {
1138	MLX5_CMD_TIMEOUT_RECOVER_MSEC   = 5 * 1000,
1139};
1140
1141static void wait_func_handle_exec_timeout(struct mlx5_core_dev *dev,
1142					  struct mlx5_cmd_work_ent *ent)
1143{
1144	unsigned long timeout = msecs_to_jiffies(MLX5_CMD_TIMEOUT_RECOVER_MSEC);
1145
1146	mlx5_cmd_eq_recover(dev);
1147
1148	/* Re-wait on the ent->done after executing the recovery flow. If the
1149	 * recovery flow (or any other recovery flow running simultaneously)
1150	 * has recovered an EQE, it should cause the entry to be completed by
1151	 * the command interface.
1152	 */
1153	if (wait_for_completion_timeout(&ent->done, timeout)) {
1154		mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) recovered after timeout\n", ent->idx,
1155			       mlx5_command_str(ent->op), ent->op);
1156		return;
1157	}
1158
1159	mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) No done completion\n", ent->idx,
1160		       mlx5_command_str(ent->op), ent->op);
1161
1162	ent->ret = -ETIMEDOUT;
1163	mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true);
1164}
1165
1166static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent)
1167{
1168	unsigned long timeout = msecs_to_jiffies(mlx5_tout_ms(dev, CMD));
1169	struct mlx5_cmd *cmd = &dev->cmd;
1170	int err;
1171
1172	if (!wait_for_completion_timeout(&ent->handling, timeout) &&
1173	    cancel_work_sync(&ent->work)) {
1174		ent->ret = -ECANCELED;
1175		goto out_err;
1176	}
1177
1178	wait_for_completion(&ent->slotted);
1179
1180	if (cmd->mode == CMD_MODE_POLLING || ent->polling)
1181		wait_for_completion(&ent->done);
1182	else if (!wait_for_completion_timeout(&ent->done, timeout))
1183		wait_func_handle_exec_timeout(dev, ent);
1184
1185out_err:
1186	err = ent->ret;
1187
1188	if (err == -ETIMEDOUT) {
1189		mlx5_core_warn(dev, "%s(0x%x) timeout. Will cause a leak of a command resource\n",
1190			       mlx5_command_str(ent->op), ent->op);
1191	} else if (err == -ECANCELED) {
1192		mlx5_core_warn(dev, "%s(0x%x) canceled on out of queue timeout.\n",
1193			       mlx5_command_str(ent->op), ent->op);
1194	} else if (err == -EBUSY) {
1195		mlx5_core_warn(dev, "%s(0x%x) timeout while waiting for command semaphore.\n",
1196			       mlx5_command_str(ent->op), ent->op);
1197	}
1198	mlx5_core_dbg(dev, "err %d, delivery status %s(%d)\n",
1199		      err, deliv_status_to_str(ent->status), ent->status);
1200
1201	return err;
1202}
1203
1204/* Check if all command slots are stalled (timed out and not recovered).
1205 * returns true if all slots timed out on a recent command and have not been
1206 * completed by FW yet. (stalled state)
1207 * false otherwise (at least one slot is not stalled).
1208 *
1209 * In such odd situation "all_stalled", this serves as a protection mechanism
1210 * to avoid blocking the kernel for long periods of time in case FW is not
1211 * responding to commands.
1212 */
1213static bool mlx5_cmd_all_stalled(struct mlx5_core_dev *dev)
1214{
1215	struct mlx5_cmd *cmd = &dev->cmd;
1216	bool all_stalled = true;
1217	unsigned long flags;
1218	int i;
1219
1220	spin_lock_irqsave(&cmd->alloc_lock, flags);
1221
1222	/* at least one command slot is free */
1223	if (bitmap_weight(&cmd->vars.bitmask, cmd->vars.max_reg_cmds) > 0) {
1224		all_stalled = false;
1225		goto out;
1226	}
1227
1228	for_each_clear_bit(i, &cmd->vars.bitmask, cmd->vars.max_reg_cmds) {
1229		struct mlx5_cmd_work_ent *ent = dev->cmd.ent_arr[i];
1230
1231		if (!test_bit(MLX5_CMD_ENT_STATE_TIMEDOUT, &ent->state)) {
1232			all_stalled = false;
1233			break;
1234		}
1235	}
1236out:
1237	spin_unlock_irqrestore(&cmd->alloc_lock, flags);
1238
1239	return all_stalled;
1240}
1241
1242/*  Notes:
1243 *    1. Callback functions may not sleep
1244 *    2. page queue commands do not support asynchrous completion
1245 *
1246 * return value in case (!callback):
1247 *	ret < 0 : Command execution couldn't be submitted by driver
1248 *	ret > 0 : Command execution couldn't be performed by firmware
1249 *	ret == 0: Command was executed by FW, Caller must check FW outbox status.
1250 *
1251 * return value in case (callback):
1252 *	ret < 0 : Command execution couldn't be submitted by driver
1253 *	ret == 0: Command will be submitted to FW for execution
1254 *		  and the callback will be called for further status updates
1255 */
1256static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in,
1257			   struct mlx5_cmd_msg *out, void *uout, int uout_size,
1258			   mlx5_cmd_cbk_t callback,
1259			   void *context, int page_queue,
1260			   u8 token, bool force_polling)
1261{
1262	struct mlx5_cmd *cmd = &dev->cmd;
1263	struct mlx5_cmd_work_ent *ent;
1264	struct mlx5_cmd_stats *stats;
1265	u8 status = 0;
1266	int err = 0;
1267	s64 ds;
1268
1269	if (callback && page_queue)
1270		return -EINVAL;
1271
1272	if (!page_queue && mlx5_cmd_all_stalled(dev)) {
1273		mlx5_core_err_rl(dev,
1274				 "All CMD slots are stalled, aborting command\n");
1275		/* there's no reason to wait and block the whole kernel if FW
1276		 * isn't currently responding to all slots, fail immediately
1277		 */
1278		return -EAGAIN;
1279	}
1280
1281	ent = cmd_alloc_ent(cmd, in, out, uout, uout_size,
1282			    callback, context, page_queue);
1283	if (IS_ERR(ent))
1284		return PTR_ERR(ent);
1285
1286	/* put for this ent is when consumed, depending on the use case
1287	 * 1) (!callback) blocking flow: by caller after wait_func completes
1288	 * 2) (callback) flow: by mlx5_cmd_comp_handler() when ent is handled
1289	 */
1290
1291	ent->token = token;
1292	ent->polling = force_polling;
1293
1294	init_completion(&ent->handling);
1295	init_completion(&ent->slotted);
1296	if (!callback)
1297		init_completion(&ent->done);
1298
1299	INIT_DELAYED_WORK(&ent->cb_timeout_work, cb_timeout_handler);
1300	INIT_WORK(&ent->work, cmd_work_handler);
1301	if (page_queue) {
1302		cmd_work_handler(&ent->work);
1303	} else if (!queue_work(cmd->wq, &ent->work)) {
1304		mlx5_core_warn(dev, "failed to queue work\n");
1305		err = -EALREADY;
1306		goto out_free;
1307	}
1308
1309	if (callback)
1310		return 0; /* mlx5_cmd_comp_handler() will put(ent) */
1311
1312	err = wait_func(dev, ent);
1313	if (err == -ETIMEDOUT || err == -ECANCELED || err == -EBUSY)
1314		goto out_free;
1315
1316	ds = ent->ts2 - ent->ts1;
1317	stats = xa_load(&cmd->stats, ent->op);
1318	if (stats) {
1319		spin_lock_irq(&stats->lock);
1320		stats->sum += ds;
1321		++stats->n;
1322		spin_unlock_irq(&stats->lock);
1323	}
1324	mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_TIME,
1325			   "fw exec time for %s is %lld nsec\n",
1326			   mlx5_command_str(ent->op), ds);
1327
1328out_free:
1329	status = ent->status;
1330	cmd_ent_put(ent);
1331	return err ? : status;
1332}
1333
1334static ssize_t dbg_write(struct file *filp, const char __user *buf,
1335			 size_t count, loff_t *pos)
1336{
1337	struct mlx5_core_dev *dev = filp->private_data;
1338	struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1339	char lbuf[3];
1340	int err;
1341
1342	if (!dbg->in_msg || !dbg->out_msg)
1343		return -ENOMEM;
1344
1345	if (count < sizeof(lbuf) - 1)
1346		return -EINVAL;
1347
1348	if (copy_from_user(lbuf, buf, sizeof(lbuf) - 1))
1349		return -EFAULT;
1350
1351	lbuf[sizeof(lbuf) - 1] = 0;
1352
1353	if (strcmp(lbuf, "go"))
1354		return -EINVAL;
1355
1356	err = mlx5_cmd_exec(dev, dbg->in_msg, dbg->inlen, dbg->out_msg, dbg->outlen);
1357
1358	return err ? err : count;
1359}
1360
1361static const struct file_operations fops = {
1362	.owner	= THIS_MODULE,
1363	.open	= simple_open,
1364	.write	= dbg_write,
1365};
1366
1367static int mlx5_copy_to_msg(struct mlx5_cmd_msg *to, void *from, int size,
1368			    u8 token)
1369{
1370	struct mlx5_cmd_prot_block *block;
1371	struct mlx5_cmd_mailbox *next;
1372	int copy;
1373
1374	if (!to || !from)
1375		return -ENOMEM;
1376
1377	copy = min_t(int, size, sizeof(to->first.data));
1378	memcpy(to->first.data, from, copy);
1379	size -= copy;
1380	from += copy;
1381
1382	next = to->next;
1383	while (size) {
1384		if (!next) {
1385			/* this is a BUG */
1386			return -ENOMEM;
1387		}
1388
1389		copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE);
1390		block = next->buf;
1391		memcpy(block->data, from, copy);
1392		from += copy;
1393		size -= copy;
1394		block->token = token;
1395		next = next->next;
1396	}
1397
1398	return 0;
1399}
1400
1401static int mlx5_copy_from_msg(void *to, struct mlx5_cmd_msg *from, int size)
1402{
1403	struct mlx5_cmd_prot_block *block;
1404	struct mlx5_cmd_mailbox *next;
1405	int copy;
1406
1407	if (!to || !from)
1408		return -ENOMEM;
1409
1410	copy = min_t(int, size, sizeof(from->first.data));
1411	memcpy(to, from->first.data, copy);
1412	size -= copy;
1413	to += copy;
1414
1415	next = from->next;
1416	while (size) {
1417		if (!next) {
1418			/* this is a BUG */
1419			return -ENOMEM;
1420		}
1421
1422		copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE);
1423		block = next->buf;
1424
1425		memcpy(to, block->data, copy);
1426		to += copy;
1427		size -= copy;
1428		next = next->next;
1429	}
1430
1431	return 0;
1432}
1433
1434static struct mlx5_cmd_mailbox *alloc_cmd_box(struct mlx5_core_dev *dev,
1435					      gfp_t flags)
1436{
1437	struct mlx5_cmd_mailbox *mailbox;
1438
1439	mailbox = kmalloc(sizeof(*mailbox), flags);
1440	if (!mailbox)
1441		return ERR_PTR(-ENOMEM);
1442
1443	mailbox->buf = dma_pool_zalloc(dev->cmd.pool, flags,
1444				       &mailbox->dma);
1445	if (!mailbox->buf) {
1446		mlx5_core_dbg(dev, "failed allocation\n");
1447		kfree(mailbox);
1448		return ERR_PTR(-ENOMEM);
1449	}
1450	mailbox->next = NULL;
1451
1452	return mailbox;
1453}
1454
1455static void free_cmd_box(struct mlx5_core_dev *dev,
1456			 struct mlx5_cmd_mailbox *mailbox)
1457{
1458	dma_pool_free(dev->cmd.pool, mailbox->buf, mailbox->dma);
1459	kfree(mailbox);
1460}
1461
1462static struct mlx5_cmd_msg *mlx5_alloc_cmd_msg(struct mlx5_core_dev *dev,
1463					       gfp_t flags, int size,
1464					       u8 token)
1465{
1466	struct mlx5_cmd_mailbox *tmp, *head = NULL;
1467	struct mlx5_cmd_prot_block *block;
1468	struct mlx5_cmd_msg *msg;
1469	int err;
1470	int n;
1471	int i;
1472
1473	msg = kzalloc(sizeof(*msg), flags);
1474	if (!msg)
1475		return ERR_PTR(-ENOMEM);
1476
1477	msg->len = size;
1478	n = mlx5_calc_cmd_blocks(msg);
1479
1480	for (i = 0; i < n; i++) {
1481		tmp = alloc_cmd_box(dev, flags);
1482		if (IS_ERR(tmp)) {
1483			mlx5_core_warn(dev, "failed allocating block\n");
1484			err = PTR_ERR(tmp);
1485			goto err_alloc;
1486		}
1487
1488		block = tmp->buf;
1489		tmp->next = head;
1490		block->next = cpu_to_be64(tmp->next ? tmp->next->dma : 0);
1491		block->block_num = cpu_to_be32(n - i - 1);
1492		block->token = token;
1493		head = tmp;
1494	}
1495	msg->next = head;
1496	return msg;
1497
1498err_alloc:
1499	while (head) {
1500		tmp = head->next;
1501		free_cmd_box(dev, head);
1502		head = tmp;
1503	}
1504	kfree(msg);
1505
1506	return ERR_PTR(err);
1507}
1508
1509static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev,
1510			      struct mlx5_cmd_msg *msg)
1511{
1512	struct mlx5_cmd_mailbox *head = msg->next;
1513	struct mlx5_cmd_mailbox *next;
1514
1515	while (head) {
1516		next = head->next;
1517		free_cmd_box(dev, head);
1518		head = next;
1519	}
1520	kfree(msg);
1521}
1522
1523static ssize_t data_write(struct file *filp, const char __user *buf,
1524			  size_t count, loff_t *pos)
1525{
1526	struct mlx5_core_dev *dev = filp->private_data;
1527	struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1528	void *ptr;
1529
1530	if (*pos != 0)
1531		return -EINVAL;
1532
1533	kfree(dbg->in_msg);
1534	dbg->in_msg = NULL;
1535	dbg->inlen = 0;
1536	ptr = memdup_user(buf, count);
1537	if (IS_ERR(ptr))
1538		return PTR_ERR(ptr);
1539	dbg->in_msg = ptr;
1540	dbg->inlen = count;
1541
1542	*pos = count;
1543
1544	return count;
1545}
1546
1547static ssize_t data_read(struct file *filp, char __user *buf, size_t count,
1548			 loff_t *pos)
1549{
1550	struct mlx5_core_dev *dev = filp->private_data;
1551	struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1552
1553	if (!dbg->out_msg)
1554		return -ENOMEM;
1555
1556	return simple_read_from_buffer(buf, count, pos, dbg->out_msg,
1557				       dbg->outlen);
1558}
1559
1560static const struct file_operations dfops = {
1561	.owner	= THIS_MODULE,
1562	.open	= simple_open,
1563	.write	= data_write,
1564	.read	= data_read,
1565};
1566
1567static ssize_t outlen_read(struct file *filp, char __user *buf, size_t count,
1568			   loff_t *pos)
1569{
1570	struct mlx5_core_dev *dev = filp->private_data;
1571	struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1572	char outlen[8];
1573	int err;
1574
1575	err = snprintf(outlen, sizeof(outlen), "%d", dbg->outlen);
1576	if (err < 0)
1577		return err;
1578
1579	return simple_read_from_buffer(buf, count, pos, outlen, err);
1580}
1581
1582static ssize_t outlen_write(struct file *filp, const char __user *buf,
1583			    size_t count, loff_t *pos)
1584{
1585	struct mlx5_core_dev *dev = filp->private_data;
1586	struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1587	char outlen_str[8] = {0};
1588	int outlen;
1589	void *ptr;
1590	int err;
1591
1592	if (*pos != 0 || count > 6)
1593		return -EINVAL;
1594
1595	kfree(dbg->out_msg);
1596	dbg->out_msg = NULL;
1597	dbg->outlen = 0;
1598
1599	if (copy_from_user(outlen_str, buf, count))
1600		return -EFAULT;
1601
1602	err = sscanf(outlen_str, "%d", &outlen);
1603	if (err != 1)
1604		return -EINVAL;
1605
1606	ptr = kzalloc(outlen, GFP_KERNEL);
1607	if (!ptr)
1608		return -ENOMEM;
1609
1610	dbg->out_msg = ptr;
1611	dbg->outlen = outlen;
1612
1613	*pos = count;
1614
1615	return count;
1616}
1617
1618static const struct file_operations olfops = {
1619	.owner	= THIS_MODULE,
1620	.open	= simple_open,
1621	.write	= outlen_write,
1622	.read	= outlen_read,
1623};
1624
1625static void set_wqname(struct mlx5_core_dev *dev)
1626{
1627	struct mlx5_cmd *cmd = &dev->cmd;
1628
1629	snprintf(cmd->wq_name, sizeof(cmd->wq_name), "mlx5_cmd_%s",
1630		 dev_name(dev->device));
1631}
1632
1633static void clean_debug_files(struct mlx5_core_dev *dev)
1634{
1635	struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1636
1637	if (!mlx5_debugfs_root)
1638		return;
1639
1640	debugfs_remove_recursive(dbg->dbg_root);
1641}
1642
1643static void create_debugfs_files(struct mlx5_core_dev *dev)
1644{
1645	struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1646
1647	dbg->dbg_root = debugfs_create_dir("cmd", mlx5_debugfs_get_dev_root(dev));
1648
1649	debugfs_create_file("in", 0400, dbg->dbg_root, dev, &dfops);
1650	debugfs_create_file("out", 0200, dbg->dbg_root, dev, &dfops);
1651	debugfs_create_file("out_len", 0600, dbg->dbg_root, dev, &olfops);
1652	debugfs_create_u8("status", 0600, dbg->dbg_root, &dbg->status);
1653	debugfs_create_file("run", 0200, dbg->dbg_root, dev, &fops);
1654}
1655
1656void mlx5_cmd_allowed_opcode(struct mlx5_core_dev *dev, u16 opcode)
1657{
1658	struct mlx5_cmd *cmd = &dev->cmd;
1659	int i;
1660
1661	for (i = 0; i < cmd->vars.max_reg_cmds; i++)
1662		down(&cmd->vars.sem);
1663	down(&cmd->vars.pages_sem);
1664
1665	cmd->allowed_opcode = opcode;
1666
1667	up(&cmd->vars.pages_sem);
1668	for (i = 0; i < cmd->vars.max_reg_cmds; i++)
1669		up(&cmd->vars.sem);
1670}
1671
1672static void mlx5_cmd_change_mod(struct mlx5_core_dev *dev, int mode)
1673{
1674	struct mlx5_cmd *cmd = &dev->cmd;
1675	int i;
1676
1677	for (i = 0; i < cmd->vars.max_reg_cmds; i++)
1678		down(&cmd->vars.sem);
1679	down(&cmd->vars.pages_sem);
1680
1681	cmd->mode = mode;
1682
1683	up(&cmd->vars.pages_sem);
1684	for (i = 0; i < cmd->vars.max_reg_cmds; i++)
1685		up(&cmd->vars.sem);
1686}
1687
1688static int cmd_comp_notifier(struct notifier_block *nb,
1689			     unsigned long type, void *data)
1690{
1691	struct mlx5_core_dev *dev;
1692	struct mlx5_cmd *cmd;
1693	struct mlx5_eqe *eqe;
1694
1695	cmd = mlx5_nb_cof(nb, struct mlx5_cmd, nb);
1696	dev = container_of(cmd, struct mlx5_core_dev, cmd);
1697	eqe = data;
1698
1699	if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
1700		return NOTIFY_DONE;
1701
1702	mlx5_cmd_comp_handler(dev, be32_to_cpu(eqe->data.cmd.vector), false);
1703
1704	return NOTIFY_OK;
1705}
1706void mlx5_cmd_use_events(struct mlx5_core_dev *dev)
1707{
1708	MLX5_NB_INIT(&dev->cmd.nb, cmd_comp_notifier, CMD);
1709	mlx5_eq_notifier_register(dev, &dev->cmd.nb);
1710	mlx5_cmd_change_mod(dev, CMD_MODE_EVENTS);
1711}
1712
1713void mlx5_cmd_use_polling(struct mlx5_core_dev *dev)
1714{
1715	mlx5_cmd_change_mod(dev, CMD_MODE_POLLING);
1716	mlx5_eq_notifier_unregister(dev, &dev->cmd.nb);
1717}
1718
1719static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg)
1720{
1721	unsigned long flags;
1722
1723	if (msg->parent) {
1724		spin_lock_irqsave(&msg->parent->lock, flags);
1725		list_add_tail(&msg->list, &msg->parent->head);
1726		spin_unlock_irqrestore(&msg->parent->lock, flags);
1727	} else {
1728		mlx5_free_cmd_msg(dev, msg);
1729	}
1730}
1731
1732static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool forced)
1733{
1734	struct mlx5_cmd *cmd = &dev->cmd;
1735	struct mlx5_cmd_work_ent *ent;
1736	mlx5_cmd_cbk_t callback;
1737	void *context;
1738	int err;
1739	int i;
1740	s64 ds;
1741	struct mlx5_cmd_stats *stats;
1742	unsigned long flags;
1743	unsigned long vector;
1744
1745	/* there can be at most 32 command queues */
1746	vector = vec & 0xffffffff;
1747	for (i = 0; i < (1 << cmd->vars.log_sz); i++) {
1748		if (test_bit(i, &vector)) {
1749			ent = cmd->ent_arr[i];
1750
1751			if (forced && ent->ret == -ETIMEDOUT)
1752				set_bit(MLX5_CMD_ENT_STATE_TIMEDOUT,
1753					&ent->state);
1754			else if (!forced) /* real FW completion */
1755				clear_bit(MLX5_CMD_ENT_STATE_TIMEDOUT,
1756					  &ent->state);
1757
1758			/* if we already completed the command, ignore it */
1759			if (!test_and_clear_bit(MLX5_CMD_ENT_STATE_PENDING_COMP,
1760						&ent->state)) {
1761				/* only real completion can free the cmd slot */
1762				if (!forced) {
1763					mlx5_core_err(dev, "Command completion arrived after timeout (entry idx = %d).\n",
1764						      ent->idx);
1765					cmd_ent_put(ent);
1766				}
1767				continue;
1768			}
1769
1770			if (ent->callback && cancel_delayed_work(&ent->cb_timeout_work))
1771				cmd_ent_put(ent); /* timeout work was canceled */
1772
1773			if (!forced || /* Real FW completion */
1774			     mlx5_cmd_is_down(dev) || /* No real FW completion is expected */
1775			     !opcode_allowed(cmd, ent->op))
1776				cmd_ent_put(ent);
1777
1778			ent->ts2 = ktime_get_ns();
1779			memcpy(ent->out->first.data, ent->lay->out, sizeof(ent->lay->out));
1780			dump_command(dev, ent, 0);
1781
1782			if (vec & MLX5_TRIGGERED_CMD_COMP)
1783				ent->ret = -ENXIO;
1784
1785			if (!ent->ret) { /* Command completed by FW */
1786				if (!cmd->checksum_disabled)
1787					ent->ret = verify_signature(ent);
1788
1789				ent->status = ent->lay->status_own >> 1;
1790
1791				mlx5_core_dbg(dev, "command completed. ret 0x%x, delivery status %s(0x%x)\n",
1792					      ent->ret, deliv_status_to_str(ent->status), ent->status);
1793			}
1794
1795			if (ent->callback) {
1796				ds = ent->ts2 - ent->ts1;
1797				stats = xa_load(&cmd->stats, ent->op);
1798				if (stats) {
1799					spin_lock_irqsave(&stats->lock, flags);
1800					stats->sum += ds;
1801					++stats->n;
1802					spin_unlock_irqrestore(&stats->lock, flags);
1803				}
1804
1805				callback = ent->callback;
1806				context = ent->context;
1807				err = ent->ret ? : ent->status;
1808				if (err > 0) /* Failed in FW, command didn't execute */
1809					err = deliv_status_to_err(err);
1810
1811				if (!err)
1812					err = mlx5_copy_from_msg(ent->uout,
1813								 ent->out,
1814								 ent->uout_size);
1815
1816				mlx5_free_cmd_msg(dev, ent->out);
1817				free_msg(dev, ent->in);
1818
1819				/* final consumer is done, release ent */
1820				cmd_ent_put(ent);
1821				callback(err, context);
1822			} else {
1823				/* release wait_func() so mlx5_cmd_invoke()
1824				 * can make the final ent_put()
1825				 */
1826				complete(&ent->done);
1827			}
1828		}
1829	}
1830}
1831
1832#define MLX5_MAX_MANAGE_PAGES_CMD_ENT 1
1833#define MLX5_CMD_MASK ((1UL << (cmd->vars.max_reg_cmds + \
1834			   MLX5_MAX_MANAGE_PAGES_CMD_ENT)) - 1)
1835
1836static void mlx5_cmd_trigger_completions(struct mlx5_core_dev *dev)
1837{
1838	struct mlx5_cmd *cmd = &dev->cmd;
1839	unsigned long bitmask;
1840	unsigned long flags;
1841	u64 vector;
1842	int i;
1843
1844	/* wait for pending handlers to complete */
1845	mlx5_eq_synchronize_cmd_irq(dev);
1846	spin_lock_irqsave(&dev->cmd.alloc_lock, flags);
1847	vector = ~dev->cmd.vars.bitmask & MLX5_CMD_MASK;
1848	if (!vector)
1849		goto no_trig;
1850
1851	bitmask = vector;
1852	/* we must increment the allocated entries refcount before triggering the completions
1853	 * to guarantee pending commands will not get freed in the meanwhile.
1854	 * For that reason, it also has to be done inside the alloc_lock.
1855	 */
1856	for_each_set_bit(i, &bitmask, (1 << cmd->vars.log_sz))
1857		cmd_ent_get(cmd->ent_arr[i]);
1858	vector |= MLX5_TRIGGERED_CMD_COMP;
1859	spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags);
1860
1861	mlx5_core_dbg(dev, "vector 0x%llx\n", vector);
1862	mlx5_cmd_comp_handler(dev, vector, true);
1863	for_each_set_bit(i, &bitmask, (1 << cmd->vars.log_sz))
1864		cmd_ent_put(cmd->ent_arr[i]);
1865	return;
1866
1867no_trig:
1868	spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags);
1869}
1870
1871void mlx5_cmd_flush(struct mlx5_core_dev *dev)
1872{
1873	struct mlx5_cmd *cmd = &dev->cmd;
1874	int i;
1875
1876	for (i = 0; i < cmd->vars.max_reg_cmds; i++) {
1877		while (down_trylock(&cmd->vars.sem)) {
1878			mlx5_cmd_trigger_completions(dev);
1879			cond_resched();
1880		}
1881	}
1882
1883	while (down_trylock(&cmd->vars.pages_sem)) {
1884		mlx5_cmd_trigger_completions(dev);
1885		cond_resched();
1886	}
1887
1888	/* Unlock cmdif */
1889	up(&cmd->vars.pages_sem);
1890	for (i = 0; i < cmd->vars.max_reg_cmds; i++)
1891		up(&cmd->vars.sem);
1892}
1893
1894static struct mlx5_cmd_msg *alloc_msg(struct mlx5_core_dev *dev, int in_size,
1895				      gfp_t gfp)
1896{
1897	struct mlx5_cmd_msg *msg = ERR_PTR(-ENOMEM);
1898	struct cmd_msg_cache *ch = NULL;
1899	struct mlx5_cmd *cmd = &dev->cmd;
1900	int i;
1901
1902	if (in_size <= 16)
1903		goto cache_miss;
1904
1905	for (i = 0; i < dev->profile.num_cmd_caches; i++) {
1906		ch = &cmd->cache[i];
1907		if (in_size > ch->max_inbox_size)
1908			continue;
1909		spin_lock_irq(&ch->lock);
1910		if (list_empty(&ch->head)) {
1911			spin_unlock_irq(&ch->lock);
1912			continue;
1913		}
1914		msg = list_entry(ch->head.next, typeof(*msg), list);
1915		/* For cached lists, we must explicitly state what is
1916		 * the real size
1917		 */
1918		msg->len = in_size;
1919		list_del(&msg->list);
1920		spin_unlock_irq(&ch->lock);
1921		break;
1922	}
1923
1924	if (!IS_ERR(msg))
1925		return msg;
1926
1927cache_miss:
1928	msg = mlx5_alloc_cmd_msg(dev, gfp, in_size, 0);
1929	return msg;
1930}
1931
1932static int is_manage_pages(void *in)
1933{
1934	return in_to_opcode(in) == MLX5_CMD_OP_MANAGE_PAGES;
1935}
1936
1937static bool mlx5_has_privileged_uid(struct mlx5_core_dev *dev)
1938{
1939	return !xa_empty(&dev->cmd.vars.privileged_uids);
1940}
1941
1942static bool mlx5_cmd_is_privileged_uid(struct mlx5_core_dev *dev,
1943				       u16 uid)
1944{
1945	return !!xa_load(&dev->cmd.vars.privileged_uids, uid);
1946}
1947
1948/*  Notes:
1949 *    1. Callback functions may not sleep
1950 *    2. Page queue commands do not support asynchrous completion
1951 */
1952static int cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out,
1953		    int out_size, mlx5_cmd_cbk_t callback, void *context,
1954		    bool force_polling)
1955{
1956	struct mlx5_cmd_msg *inb, *outb;
1957	u16 opcode = in_to_opcode(in);
1958	bool throttle_locked = false;
1959	bool unpriv_locked = false;
1960	u16 uid = in_to_uid(in);
1961	int pages_queue;
1962	gfp_t gfp;
1963	u8 token;
1964	int err;
1965
1966	if (mlx5_cmd_is_down(dev) || !opcode_allowed(&dev->cmd, opcode))
1967		return -ENXIO;
1968
1969	if (!callback) {
1970		/* The semaphore is already held for callback commands. It was
1971		 * acquired in mlx5_cmd_exec_cb()
1972		 */
1973		if (uid && mlx5_has_privileged_uid(dev)) {
1974			if (!mlx5_cmd_is_privileged_uid(dev, uid)) {
1975				unpriv_locked = true;
1976				down(&dev->cmd.vars.unprivileged_sem);
1977			}
1978		} else if (mlx5_cmd_is_throttle_opcode(opcode)) {
1979			throttle_locked = true;
1980			down(&dev->cmd.vars.throttle_sem);
1981		}
1982	}
1983
1984	pages_queue = is_manage_pages(in);
1985	gfp = callback ? GFP_ATOMIC : GFP_KERNEL;
1986
1987	inb = alloc_msg(dev, in_size, gfp);
1988	if (IS_ERR(inb)) {
1989		err = PTR_ERR(inb);
1990		goto out_up;
1991	}
1992
1993	token = alloc_token(&dev->cmd);
1994
1995	err = mlx5_copy_to_msg(inb, in, in_size, token);
1996	if (err) {
1997		mlx5_core_warn(dev, "err %d\n", err);
1998		goto out_in;
1999	}
2000
2001	outb = mlx5_alloc_cmd_msg(dev, gfp, out_size, token);
2002	if (IS_ERR(outb)) {
2003		err = PTR_ERR(outb);
2004		goto out_in;
2005	}
2006
2007	err = mlx5_cmd_invoke(dev, inb, outb, out, out_size, callback, context,
2008			      pages_queue, token, force_polling);
2009	if (callback && !err)
2010		return 0;
2011
2012	if (err > 0) /* Failed in FW, command didn't execute */
2013		err = deliv_status_to_err(err);
2014
2015	if (err)
2016		goto out_out;
2017
2018	/* command completed by FW */
2019	err = mlx5_copy_from_msg(out, outb, out_size);
2020out_out:
2021	mlx5_free_cmd_msg(dev, outb);
2022out_in:
2023	free_msg(dev, inb);
2024out_up:
2025	if (throttle_locked)
2026		up(&dev->cmd.vars.throttle_sem);
2027	if (unpriv_locked)
2028		up(&dev->cmd.vars.unprivileged_sem);
2029
2030	return err;
2031}
2032
2033static void mlx5_cmd_err_trace(struct mlx5_core_dev *dev, u16 opcode, u16 op_mod, void *out)
2034{
2035	u32 syndrome = MLX5_GET(mbox_out, out, syndrome);
2036	u8 status = MLX5_GET(mbox_out, out, status);
2037
2038	trace_mlx5_cmd(mlx5_command_str(opcode), opcode, op_mod,
2039		       cmd_status_str(status), status, syndrome,
2040		       cmd_status_to_err(status));
2041}
2042
2043static void cmd_status_log(struct mlx5_core_dev *dev, u16 opcode, u8 status,
2044			   u32 syndrome, int err)
2045{
2046	const char *namep = mlx5_command_str(opcode);
2047	struct mlx5_cmd_stats *stats;
2048	unsigned long flags;
2049
2050	if (!err || !(strcmp(namep, "unknown command opcode")))
2051		return;
2052
2053	stats = xa_load(&dev->cmd.stats, opcode);
2054	if (!stats)
2055		return;
2056	spin_lock_irqsave(&stats->lock, flags);
2057	stats->failed++;
2058	if (err < 0)
2059		stats->last_failed_errno = -err;
2060	if (err == -EREMOTEIO) {
2061		stats->failed_mbox_status++;
2062		stats->last_failed_mbox_status = status;
2063		stats->last_failed_syndrome = syndrome;
2064	}
2065	spin_unlock_irqrestore(&stats->lock, flags);
2066}
2067
2068/* preserve -EREMOTEIO for outbox.status != OK, otherwise return err as is */
2069static int cmd_status_err(struct mlx5_core_dev *dev, int err, u16 opcode, u16 op_mod, void *out)
2070{
2071	u32 syndrome = MLX5_GET(mbox_out, out, syndrome);
2072	u8 status = MLX5_GET(mbox_out, out, status);
2073
2074	if (err == -EREMOTEIO) /* -EREMOTEIO is preserved */
2075		err = -EIO;
2076
2077	if (!err && status != MLX5_CMD_STAT_OK) {
2078		err = -EREMOTEIO;
2079		mlx5_cmd_err_trace(dev, opcode, op_mod, out);
2080	}
2081
2082	cmd_status_log(dev, opcode, status, syndrome, err);
2083	return err;
2084}
2085
2086/**
2087 * mlx5_cmd_do - Executes a fw command, wait for completion.
2088 * Unlike mlx5_cmd_exec, this function will not translate or intercept
2089 * outbox.status and will return -EREMOTEIO when
2090 * outbox.status != MLX5_CMD_STAT_OK
2091 *
2092 * @dev: mlx5 core device
2093 * @in: inbox mlx5_ifc command buffer
2094 * @in_size: inbox buffer size
2095 * @out: outbox mlx5_ifc buffer
2096 * @out_size: outbox size
2097 *
2098 * @return:
2099 * -EREMOTEIO : Command executed by FW, outbox.status != MLX5_CMD_STAT_OK.
2100 *              Caller must check FW outbox status.
2101 *   0 : Command execution successful, outbox.status == MLX5_CMD_STAT_OK.
2102 * < 0 : Command execution couldn't be performed by firmware or driver
2103 */
2104int mlx5_cmd_do(struct mlx5_core_dev *dev, void *in, int in_size, void *out, int out_size)
2105{
2106	int err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, false);
2107	u16 op_mod = MLX5_GET(mbox_in, in, op_mod);
2108	u16 opcode = in_to_opcode(in);
2109
2110	return cmd_status_err(dev, err, opcode, op_mod, out);
2111}
2112EXPORT_SYMBOL(mlx5_cmd_do);
2113
2114/**
2115 * mlx5_cmd_exec - Executes a fw command, wait for completion
2116 *
2117 * @dev: mlx5 core device
2118 * @in: inbox mlx5_ifc command buffer
2119 * @in_size: inbox buffer size
2120 * @out: outbox mlx5_ifc buffer
2121 * @out_size: outbox size
2122 *
2123 * @return: 0 if no error, FW command execution was successful
2124 *          and outbox status is ok.
2125 */
2126int mlx5_cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out,
2127		  int out_size)
2128{
2129	int err = mlx5_cmd_do(dev, in, in_size, out, out_size);
2130
2131	return mlx5_cmd_check(dev, err, in, out);
2132}
2133EXPORT_SYMBOL(mlx5_cmd_exec);
2134
2135/**
2136 * mlx5_cmd_exec_polling - Executes a fw command, poll for completion
2137 *	Needed for driver force teardown, when command completion EQ
2138 *	will not be available to complete the command
2139 *
2140 * @dev: mlx5 core device
2141 * @in: inbox mlx5_ifc command buffer
2142 * @in_size: inbox buffer size
2143 * @out: outbox mlx5_ifc buffer
2144 * @out_size: outbox size
2145 *
2146 * @return: 0 if no error, FW command execution was successful
2147 *          and outbox status is ok.
2148 */
2149int mlx5_cmd_exec_polling(struct mlx5_core_dev *dev, void *in, int in_size,
2150			  void *out, int out_size)
2151{
2152	int err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, true);
2153	u16 op_mod = MLX5_GET(mbox_in, in, op_mod);
2154	u16 opcode = in_to_opcode(in);
2155
2156	err = cmd_status_err(dev, err, opcode, op_mod, out);
2157	return mlx5_cmd_check(dev, err, in, out);
2158}
2159EXPORT_SYMBOL(mlx5_cmd_exec_polling);
2160
2161void mlx5_cmd_init_async_ctx(struct mlx5_core_dev *dev,
2162			     struct mlx5_async_ctx *ctx)
2163{
2164	ctx->dev = dev;
2165	/* Starts at 1 to avoid doing wake_up if we are not cleaning up */
2166	atomic_set(&ctx->num_inflight, 1);
2167	init_completion(&ctx->inflight_done);
2168}
2169EXPORT_SYMBOL(mlx5_cmd_init_async_ctx);
2170
2171/**
2172 * mlx5_cmd_cleanup_async_ctx - Clean up an async_ctx
2173 * @ctx: The ctx to clean
2174 *
2175 * Upon return all callbacks given to mlx5_cmd_exec_cb() have been called. The
2176 * caller must ensure that mlx5_cmd_exec_cb() is not called during or after
2177 * the call mlx5_cleanup_async_ctx().
2178 */
2179void mlx5_cmd_cleanup_async_ctx(struct mlx5_async_ctx *ctx)
2180{
2181	if (!atomic_dec_and_test(&ctx->num_inflight))
2182		wait_for_completion(&ctx->inflight_done);
2183}
2184EXPORT_SYMBOL(mlx5_cmd_cleanup_async_ctx);
2185
2186static void mlx5_cmd_exec_cb_handler(int status, void *_work)
2187{
2188	struct mlx5_async_work *work = _work;
2189	struct mlx5_async_ctx *ctx;
2190	struct mlx5_core_dev *dev;
2191	bool throttle_locked;
2192	bool unpriv_locked;
2193
2194	ctx = work->ctx;
2195	dev = ctx->dev;
2196	throttle_locked = work->throttle_locked;
2197	unpriv_locked = work->unpriv_locked;
2198	status = cmd_status_err(dev, status, work->opcode, work->op_mod, work->out);
2199	work->user_callback(status, work);
2200	/* Can't access "work" from this point on. It could have been freed in
2201	 * the callback.
2202	 */
2203	if (throttle_locked)
2204		up(&dev->cmd.vars.throttle_sem);
2205	if (unpriv_locked)
2206		up(&dev->cmd.vars.unprivileged_sem);
2207	if (atomic_dec_and_test(&ctx->num_inflight))
2208		complete(&ctx->inflight_done);
2209}
2210
2211int mlx5_cmd_exec_cb(struct mlx5_async_ctx *ctx, void *in, int in_size,
2212		     void *out, int out_size, mlx5_async_cbk_t callback,
2213		     struct mlx5_async_work *work)
2214{
2215	struct mlx5_core_dev *dev = ctx->dev;
2216	u16 uid;
2217	int ret;
2218
2219	work->ctx = ctx;
2220	work->user_callback = callback;
2221	work->opcode = in_to_opcode(in);
2222	work->op_mod = MLX5_GET(mbox_in, in, op_mod);
2223	work->out = out;
2224	work->throttle_locked = false;
2225	work->unpriv_locked = false;
2226	uid = in_to_uid(in);
2227
2228	if (WARN_ON(!atomic_inc_not_zero(&ctx->num_inflight)))
2229		return -EIO;
2230
2231	if (uid && mlx5_has_privileged_uid(dev)) {
2232		if (!mlx5_cmd_is_privileged_uid(dev, uid)) {
2233			if (down_trylock(&dev->cmd.vars.unprivileged_sem)) {
2234				ret = -EBUSY;
2235				goto dec_num_inflight;
2236			}
2237			work->unpriv_locked = true;
2238		}
2239	} else if (mlx5_cmd_is_throttle_opcode(in_to_opcode(in))) {
2240		if (down_trylock(&dev->cmd.vars.throttle_sem)) {
2241			ret = -EBUSY;
2242			goto dec_num_inflight;
2243		}
2244		work->throttle_locked = true;
2245	}
2246
2247	ret = cmd_exec(dev, in, in_size, out, out_size,
2248		       mlx5_cmd_exec_cb_handler, work, false);
2249	if (ret)
2250		goto sem_up;
2251
2252	return 0;
2253
2254sem_up:
2255	if (work->throttle_locked)
2256		up(&dev->cmd.vars.throttle_sem);
2257	if (work->unpriv_locked)
2258		up(&dev->cmd.vars.unprivileged_sem);
2259dec_num_inflight:
2260	if (atomic_dec_and_test(&ctx->num_inflight))
2261		complete(&ctx->inflight_done);
2262
2263	return ret;
2264}
2265EXPORT_SYMBOL(mlx5_cmd_exec_cb);
2266
2267int mlx5_cmd_allow_other_vhca_access(struct mlx5_core_dev *dev,
2268				     struct mlx5_cmd_allow_other_vhca_access_attr *attr)
2269{
2270	u32 out[MLX5_ST_SZ_DW(allow_other_vhca_access_out)] = {};
2271	u32 in[MLX5_ST_SZ_DW(allow_other_vhca_access_in)] = {};
2272	void *key;
2273
2274	MLX5_SET(allow_other_vhca_access_in,
2275		 in, opcode, MLX5_CMD_OP_ALLOW_OTHER_VHCA_ACCESS);
2276	MLX5_SET(allow_other_vhca_access_in,
2277		 in, object_type_to_be_accessed, attr->obj_type);
2278	MLX5_SET(allow_other_vhca_access_in,
2279		 in, object_id_to_be_accessed, attr->obj_id);
2280
2281	key = MLX5_ADDR_OF(allow_other_vhca_access_in, in, access_key);
2282	memcpy(key, attr->access_key, sizeof(attr->access_key));
2283
2284	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
2285}
2286
2287int mlx5_cmd_alias_obj_create(struct mlx5_core_dev *dev,
2288			      struct mlx5_cmd_alias_obj_create_attr *alias_attr,
2289			      u32 *obj_id)
2290{
2291	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {};
2292	u32 in[MLX5_ST_SZ_DW(create_alias_obj_in)] = {};
2293	void *param;
2294	void *attr;
2295	void *key;
2296	int ret;
2297
2298	attr = MLX5_ADDR_OF(create_alias_obj_in, in, hdr);
2299	MLX5_SET(general_obj_in_cmd_hdr,
2300		 attr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
2301	MLX5_SET(general_obj_in_cmd_hdr,
2302		 attr, obj_type, alias_attr->obj_type);
2303	param = MLX5_ADDR_OF(general_obj_in_cmd_hdr, in, op_param);
2304	MLX5_SET(general_obj_create_param, param, alias_object, 1);
2305
2306	attr = MLX5_ADDR_OF(create_alias_obj_in, in, alias_ctx);
2307	MLX5_SET(alias_context, attr, vhca_id_to_be_accessed, alias_attr->vhca_id);
2308	MLX5_SET(alias_context, attr, object_id_to_be_accessed, alias_attr->obj_id);
2309
2310	key = MLX5_ADDR_OF(alias_context, attr, access_key);
2311	memcpy(key, alias_attr->access_key, sizeof(alias_attr->access_key));
2312
2313	ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
2314	if (ret)
2315		return ret;
2316
2317	*obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
2318
2319	return 0;
2320}
2321
2322int mlx5_cmd_alias_obj_destroy(struct mlx5_core_dev *dev, u32 obj_id,
2323			       u16 obj_type)
2324{
2325	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {};
2326	u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};
2327
2328	MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
2329	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, obj_type);
2330	MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id);
2331
2332	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
2333}
2334
2335static void destroy_msg_cache(struct mlx5_core_dev *dev)
2336{
2337	struct cmd_msg_cache *ch;
2338	struct mlx5_cmd_msg *msg;
2339	struct mlx5_cmd_msg *n;
2340	int i;
2341
2342	for (i = 0; i < dev->profile.num_cmd_caches; i++) {
2343		ch = &dev->cmd.cache[i];
2344		list_for_each_entry_safe(msg, n, &ch->head, list) {
2345			list_del(&msg->list);
2346			mlx5_free_cmd_msg(dev, msg);
2347		}
2348	}
2349}
2350
2351static unsigned cmd_cache_num_ent[MLX5_NUM_COMMAND_CACHES] = {
2352	512, 32, 16, 8, 2
2353};
2354
2355static unsigned cmd_cache_ent_size[MLX5_NUM_COMMAND_CACHES] = {
2356	16 + MLX5_CMD_DATA_BLOCK_SIZE,
2357	16 + MLX5_CMD_DATA_BLOCK_SIZE * 2,
2358	16 + MLX5_CMD_DATA_BLOCK_SIZE * 16,
2359	16 + MLX5_CMD_DATA_BLOCK_SIZE * 256,
2360	16 + MLX5_CMD_DATA_BLOCK_SIZE * 512,
2361};
2362
2363static void create_msg_cache(struct mlx5_core_dev *dev)
2364{
2365	struct mlx5_cmd *cmd = &dev->cmd;
2366	struct cmd_msg_cache *ch;
2367	struct mlx5_cmd_msg *msg;
2368	int i;
2369	int k;
2370
2371	/* Initialize and fill the caches with initial entries */
2372	for (k = 0; k < dev->profile.num_cmd_caches; k++) {
2373		ch = &cmd->cache[k];
2374		spin_lock_init(&ch->lock);
2375		INIT_LIST_HEAD(&ch->head);
2376		ch->num_ent = cmd_cache_num_ent[k];
2377		ch->max_inbox_size = cmd_cache_ent_size[k];
2378		for (i = 0; i < ch->num_ent; i++) {
2379			msg = mlx5_alloc_cmd_msg(dev, GFP_KERNEL | __GFP_NOWARN,
2380						 ch->max_inbox_size, 0);
2381			if (IS_ERR(msg))
2382				break;
2383			msg->parent = ch;
2384			list_add_tail(&msg->list, &ch->head);
2385		}
2386	}
2387}
2388
2389static int alloc_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd)
2390{
2391	cmd->cmd_alloc_buf = dma_alloc_coherent(mlx5_core_dma_dev(dev), MLX5_ADAPTER_PAGE_SIZE,
2392						&cmd->alloc_dma, GFP_KERNEL);
2393	if (!cmd->cmd_alloc_buf)
2394		return -ENOMEM;
2395
2396	/* make sure it is aligned to 4K */
2397	if (!((uintptr_t)cmd->cmd_alloc_buf & (MLX5_ADAPTER_PAGE_SIZE - 1))) {
2398		cmd->cmd_buf = cmd->cmd_alloc_buf;
2399		cmd->dma = cmd->alloc_dma;
2400		cmd->alloc_size = MLX5_ADAPTER_PAGE_SIZE;
2401		return 0;
2402	}
2403
2404	dma_free_coherent(mlx5_core_dma_dev(dev), MLX5_ADAPTER_PAGE_SIZE, cmd->cmd_alloc_buf,
2405			  cmd->alloc_dma);
2406	cmd->cmd_alloc_buf = dma_alloc_coherent(mlx5_core_dma_dev(dev),
2407						2 * MLX5_ADAPTER_PAGE_SIZE - 1,
2408						&cmd->alloc_dma, GFP_KERNEL);
2409	if (!cmd->cmd_alloc_buf)
2410		return -ENOMEM;
2411
2412	cmd->cmd_buf = PTR_ALIGN(cmd->cmd_alloc_buf, MLX5_ADAPTER_PAGE_SIZE);
2413	cmd->dma = ALIGN(cmd->alloc_dma, MLX5_ADAPTER_PAGE_SIZE);
2414	cmd->alloc_size = 2 * MLX5_ADAPTER_PAGE_SIZE - 1;
2415	return 0;
2416}
2417
2418static void free_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd)
2419{
2420	dma_free_coherent(mlx5_core_dma_dev(dev), cmd->alloc_size, cmd->cmd_alloc_buf,
2421			  cmd->alloc_dma);
2422}
2423
2424static u16 cmdif_rev(struct mlx5_core_dev *dev)
2425{
2426	return ioread32be(&dev->iseg->cmdif_rev_fw_sub) >> 16;
2427}
2428
2429int mlx5_cmd_init(struct mlx5_core_dev *dev)
2430{
2431	struct mlx5_cmd *cmd = &dev->cmd;
2432
2433	cmd->checksum_disabled = 1;
2434
2435	spin_lock_init(&cmd->alloc_lock);
2436	spin_lock_init(&cmd->token_lock);
2437
2438	set_wqname(dev);
2439	cmd->wq = create_singlethread_workqueue(cmd->wq_name);
2440	if (!cmd->wq) {
2441		mlx5_core_err(dev, "failed to create command workqueue\n");
2442		return -ENOMEM;
2443	}
2444
2445	mlx5_cmdif_debugfs_init(dev);
2446
2447	return 0;
2448}
2449
2450void mlx5_cmd_cleanup(struct mlx5_core_dev *dev)
2451{
2452	struct mlx5_cmd *cmd = &dev->cmd;
2453
2454	mlx5_cmdif_debugfs_cleanup(dev);
2455	destroy_workqueue(cmd->wq);
2456}
2457
2458int mlx5_cmd_enable(struct mlx5_core_dev *dev)
2459{
2460	int size = sizeof(struct mlx5_cmd_prot_block);
2461	int align = roundup_pow_of_two(size);
2462	struct mlx5_cmd *cmd = &dev->cmd;
2463	u32 cmd_h, cmd_l;
2464	int err;
2465
2466	memset(&cmd->vars, 0, sizeof(cmd->vars));
2467	cmd->vars.cmdif_rev = cmdif_rev(dev);
2468	if (cmd->vars.cmdif_rev != CMD_IF_REV) {
2469		mlx5_core_err(dev,
2470			      "Driver cmdif rev(%d) differs from firmware's(%d)\n",
2471			      CMD_IF_REV, cmd->vars.cmdif_rev);
2472		return -EINVAL;
2473	}
2474
2475	cmd_l = ioread32be(&dev->iseg->cmdq_addr_l_sz) & 0xff;
2476	cmd->vars.log_sz = cmd_l >> 4 & 0xf;
2477	cmd->vars.log_stride = cmd_l & 0xf;
2478	if (1 << cmd->vars.log_sz > MLX5_MAX_COMMANDS) {
2479		mlx5_core_err(dev, "firmware reports too many outstanding commands %d\n",
2480			      1 << cmd->vars.log_sz);
2481		return -EINVAL;
2482	}
2483
2484	if (cmd->vars.log_sz + cmd->vars.log_stride > MLX5_ADAPTER_PAGE_SHIFT) {
2485		mlx5_core_err(dev, "command queue size overflow\n");
2486		return -EINVAL;
2487	}
2488
2489	cmd->state = MLX5_CMDIF_STATE_DOWN;
2490	cmd->vars.max_reg_cmds = (1 << cmd->vars.log_sz) - 1;
2491	cmd->vars.bitmask = MLX5_CMD_MASK;
2492
2493	sema_init(&cmd->vars.sem, cmd->vars.max_reg_cmds);
2494	sema_init(&cmd->vars.pages_sem, 1);
2495	sema_init(&cmd->vars.throttle_sem, DIV_ROUND_UP(cmd->vars.max_reg_cmds, 2));
2496	sema_init(&cmd->vars.unprivileged_sem,
2497		  DIV_ROUND_UP(cmd->vars.max_reg_cmds, 2));
2498
2499	xa_init(&cmd->vars.privileged_uids);
2500
2501	cmd->pool = dma_pool_create("mlx5_cmd", mlx5_core_dma_dev(dev), size, align, 0);
2502	if (!cmd->pool) {
2503		err = -ENOMEM;
2504		goto err_destroy_xa;
2505	}
2506
2507	err = alloc_cmd_page(dev, cmd);
2508	if (err)
2509		goto err_free_pool;
2510
2511	cmd_h = (u32)((u64)(cmd->dma) >> 32);
2512	cmd_l = (u32)(cmd->dma);
2513	if (cmd_l & 0xfff) {
2514		mlx5_core_err(dev, "invalid command queue address\n");
2515		err = -ENOMEM;
2516		goto err_cmd_page;
2517	}
2518
2519	iowrite32be(cmd_h, &dev->iseg->cmdq_addr_h);
2520	iowrite32be(cmd_l, &dev->iseg->cmdq_addr_l_sz);
2521
2522	/* Make sure firmware sees the complete address before we proceed */
2523	wmb();
2524
2525	mlx5_core_dbg(dev, "descriptor at dma 0x%llx\n", (unsigned long long)(cmd->dma));
2526
2527	cmd->mode = CMD_MODE_POLLING;
2528	cmd->allowed_opcode = CMD_ALLOWED_OPCODE_ALL;
2529
2530	create_msg_cache(dev);
2531	create_debugfs_files(dev);
2532
2533	return 0;
2534
2535err_cmd_page:
2536	free_cmd_page(dev, cmd);
2537err_free_pool:
2538	dma_pool_destroy(cmd->pool);
2539err_destroy_xa:
2540	xa_destroy(&dev->cmd.vars.privileged_uids);
2541	return err;
2542}
2543
2544void mlx5_cmd_disable(struct mlx5_core_dev *dev)
2545{
2546	struct mlx5_cmd *cmd = &dev->cmd;
2547
2548	flush_workqueue(cmd->wq);
2549	clean_debug_files(dev);
2550	destroy_msg_cache(dev);
2551	free_cmd_page(dev, cmd);
2552	dma_pool_destroy(cmd->pool);
2553	xa_destroy(&dev->cmd.vars.privileged_uids);
2554}
2555
2556void mlx5_cmd_set_state(struct mlx5_core_dev *dev,
2557			enum mlx5_cmdif_state cmdif_state)
2558{
2559	dev->cmd.state = cmdif_state;
2560}
2561
2562int mlx5_cmd_add_privileged_uid(struct mlx5_core_dev *dev, u16 uid)
2563{
2564	return xa_insert(&dev->cmd.vars.privileged_uids, uid,
2565			 xa_mk_value(uid), GFP_KERNEL);
2566}
2567EXPORT_SYMBOL(mlx5_cmd_add_privileged_uid);
2568
2569void mlx5_cmd_remove_privileged_uid(struct mlx5_core_dev *dev, u16 uid)
2570{
2571	void *data = xa_erase(&dev->cmd.vars.privileged_uids, uid);
2572
2573	WARN(!data, "Privileged UID %u does not exist\n", uid);
2574}
2575EXPORT_SYMBOL(mlx5_cmd_remove_privileged_uid);