rss.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2
   3#include <net/netdev_lock.h>
   4
   5#include "netlink.h"
   6#include "common.h"
   7
   8struct rss_req_info {
   9	struct ethnl_req_info		base;
  10	u32				rss_context;
  11};
  12
  13struct rss_reply_data {
  14	struct ethnl_reply_data		base;
  15	bool				has_flow_hash;
  16	bool				no_key_fields;
  17	u32				indir_size;
  18	u32				hkey_size;
  19	u32				hfunc;
  20	u32				input_xfrm;
  21	u32				*indir_table;
  22	u8				*hkey;
  23	int				flow_hash[__ETHTOOL_A_FLOW_CNT];
  24};
  25
  26static const u8 ethtool_rxfh_ft_nl2ioctl[] = {
  27	[ETHTOOL_A_FLOW_ETHER]		= ETHER_FLOW,
  28	[ETHTOOL_A_FLOW_IP4]		= IPV4_FLOW,
  29	[ETHTOOL_A_FLOW_IP6]		= IPV6_FLOW,
  30	[ETHTOOL_A_FLOW_TCP4]		= TCP_V4_FLOW,
  31	[ETHTOOL_A_FLOW_UDP4]		= UDP_V4_FLOW,
  32	[ETHTOOL_A_FLOW_SCTP4]		= SCTP_V4_FLOW,
  33	[ETHTOOL_A_FLOW_AH_ESP4]	= AH_ESP_V4_FLOW,
  34	[ETHTOOL_A_FLOW_TCP6]		= TCP_V6_FLOW,
  35	[ETHTOOL_A_FLOW_UDP6]		= UDP_V6_FLOW,
  36	[ETHTOOL_A_FLOW_SCTP6]		= SCTP_V6_FLOW,
  37	[ETHTOOL_A_FLOW_AH_ESP6]	= AH_ESP_V6_FLOW,
  38	[ETHTOOL_A_FLOW_AH4]		= AH_V4_FLOW,
  39	[ETHTOOL_A_FLOW_ESP4]		= ESP_V4_FLOW,
  40	[ETHTOOL_A_FLOW_AH6]		= AH_V6_FLOW,
  41	[ETHTOOL_A_FLOW_ESP6]		= ESP_V6_FLOW,
  42	[ETHTOOL_A_FLOW_GTPU4]		= GTPU_V4_FLOW,
  43	[ETHTOOL_A_FLOW_GTPU6]		= GTPU_V6_FLOW,
  44	[ETHTOOL_A_FLOW_GTPC4]		= GTPC_V4_FLOW,
  45	[ETHTOOL_A_FLOW_GTPC6]		= GTPC_V6_FLOW,
  46	[ETHTOOL_A_FLOW_GTPC_TEID4]	= GTPC_TEID_V4_FLOW,
  47	[ETHTOOL_A_FLOW_GTPC_TEID6]	= GTPC_TEID_V6_FLOW,
  48	[ETHTOOL_A_FLOW_GTPU_EH4]	= GTPU_EH_V4_FLOW,
  49	[ETHTOOL_A_FLOW_GTPU_EH6]	= GTPU_EH_V6_FLOW,
  50	[ETHTOOL_A_FLOW_GTPU_UL4]	= GTPU_UL_V4_FLOW,
  51	[ETHTOOL_A_FLOW_GTPU_UL6]	= GTPU_UL_V6_FLOW,
  52	[ETHTOOL_A_FLOW_GTPU_DL4]	= GTPU_DL_V4_FLOW,
  53	[ETHTOOL_A_FLOW_GTPU_DL6]	= GTPU_DL_V6_FLOW,
  54};
  55
  56#define RSS_REQINFO(__req_base) \
  57	container_of(__req_base, struct rss_req_info, base)
  58
  59#define RSS_REPDATA(__reply_base) \
  60	container_of(__reply_base, struct rss_reply_data, base)
  61
  62const struct nla_policy ethnl_rss_get_policy[] = {
  63	[ETHTOOL_A_RSS_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy),
  64	[ETHTOOL_A_RSS_CONTEXT] = { .type = NLA_U32 },
  65	[ETHTOOL_A_RSS_START_CONTEXT] = { .type = NLA_U32 },
  66};
  67
  68static int
  69rss_parse_request(struct ethnl_req_info *req_info, struct nlattr **tb,
  70		  struct netlink_ext_ack *extack)
  71{
  72	struct rss_req_info *request = RSS_REQINFO(req_info);
  73
  74	if (tb[ETHTOOL_A_RSS_CONTEXT])
  75		request->rss_context = nla_get_u32(tb[ETHTOOL_A_RSS_CONTEXT]);
  76	if (tb[ETHTOOL_A_RSS_START_CONTEXT]) {
  77		NL_SET_BAD_ATTR(extack, tb[ETHTOOL_A_RSS_START_CONTEXT]);
  78		return -EINVAL;
  79	}
  80
  81	return 0;
  82}
  83
  84static void
  85rss_prepare_flow_hash(const struct rss_req_info *req, struct net_device *dev,
  86		      struct rss_reply_data *data, const struct genl_info *info)
  87{
  88	int i;
  89
  90	data->has_flow_hash = false;
  91
  92	if (!dev->ethtool_ops->get_rxfh_fields)
  93		return;
  94	if (req->rss_context && !dev->ethtool_ops->rxfh_per_ctx_fields)
  95		return;
  96
  97	mutex_lock(&dev->ethtool->rss_lock);
  98	for (i = 1; i < __ETHTOOL_A_FLOW_CNT; i++) {
  99		struct ethtool_rxfh_fields fields = {
 100			.flow_type	= ethtool_rxfh_ft_nl2ioctl[i],
 101			.rss_context	= req->rss_context,
 102		};
 103
 104		if (dev->ethtool_ops->get_rxfh_fields(dev, &fields)) {
 105			data->flow_hash[i] = -1; /* Unsupported */
 106			continue;
 107		}
 108
 109		data->flow_hash[i] = fields.data;
 110		data->has_flow_hash = true;
 111	}
 112	mutex_unlock(&dev->ethtool->rss_lock);
 113}
 114
 115static int
 116rss_get_data_alloc(struct net_device *dev, struct rss_reply_data *data)
 117{
 118	const struct ethtool_ops *ops = dev->ethtool_ops;
 119	u32 total_size, indir_bytes;
 120	u8 *rss_config;
 121
 122	data->indir_size = 0;
 123	data->hkey_size = 0;
 124	if (ops->get_rxfh_indir_size)
 125		data->indir_size = ops->get_rxfh_indir_size(dev);
 126	if (ops->get_rxfh_key_size)
 127		data->hkey_size = ops->get_rxfh_key_size(dev);
 128
 129	indir_bytes = data->indir_size * sizeof(u32);
 130	total_size = indir_bytes + data->hkey_size;
 131	rss_config = kzalloc(total_size, GFP_KERNEL);
 132	if (!rss_config)
 133		return -ENOMEM;
 134
 135	if (data->indir_size)
 136		data->indir_table = (u32 *)rss_config;
 137	if (data->hkey_size)
 138		data->hkey = rss_config + indir_bytes;
 139
 140	return 0;
 141}
 142
 143static void rss_get_data_free(const struct rss_reply_data *data)
 144{
 145	kfree(data->indir_table);
 146}
 147
 148static int
 149rss_prepare_get(const struct rss_req_info *request, struct net_device *dev,
 150		struct rss_reply_data *data, const struct genl_info *info)
 151{
 152	const struct ethtool_ops *ops = dev->ethtool_ops;
 153	struct ethtool_rxfh_param rxfh = {};
 154	int ret;
 155
 156	ret = ethnl_ops_begin(dev);
 157	if (ret < 0)
 158		return ret;
 159	mutex_lock(&dev->ethtool->rss_lock);
 160
 161	ret = rss_get_data_alloc(dev, data);
 162	if (ret)
 163		goto out_unlock;
 164
 165	rxfh.indir_size = data->indir_size;
 166	rxfh.indir = data->indir_table;
 167	rxfh.key_size = data->hkey_size;
 168	rxfh.key = data->hkey;
 169
 170	ret = ops->get_rxfh(dev, &rxfh);
 171	if (ret)
 172		goto out_unlock;
 173
 174	data->hfunc = rxfh.hfunc;
 175	data->input_xfrm = rxfh.input_xfrm;
 176out_unlock:
 177	mutex_unlock(&dev->ethtool->rss_lock);
 178	ethnl_ops_complete(dev);
 179	return ret;
 180}
 181
 182static void
 183__rss_prepare_ctx(struct net_device *dev, struct rss_reply_data *data,
 184		  struct ethtool_rxfh_context *ctx)
 185{
 186	if (WARN_ON_ONCE(data->indir_size != ctx->indir_size ||
 187			 data->hkey_size != ctx->key_size))
 188		return;
 189
 190	data->no_key_fields = !dev->ethtool_ops->rxfh_per_ctx_key;
 191
 192	data->hfunc = ctx->hfunc;
 193	data->input_xfrm = ctx->input_xfrm;
 194	memcpy(data->indir_table, ethtool_rxfh_context_indir(ctx),
 195	       data->indir_size * sizeof(u32));
 196	if (data->hkey_size)
 197		memcpy(data->hkey, ethtool_rxfh_context_key(ctx),
 198		       data->hkey_size);
 199}
 200
 201static int
 202rss_prepare_ctx(const struct rss_req_info *request, struct net_device *dev,
 203		struct rss_reply_data *data, const struct genl_info *info)
 204{
 205	struct ethtool_rxfh_context *ctx;
 206	u32 total_size, indir_bytes;
 207	u8 *rss_config;
 208	int ret;
 209
 210	mutex_lock(&dev->ethtool->rss_lock);
 211	ctx = xa_load(&dev->ethtool->rss_ctx, request->rss_context);
 212	if (!ctx) {
 213		ret = -ENOENT;
 214		goto out_unlock;
 215	}
 216
 217	data->indir_size = ctx->indir_size;
 218	data->hkey_size = ctx->key_size;
 219
 220	indir_bytes = data->indir_size * sizeof(u32);
 221	total_size = indir_bytes + data->hkey_size;
 222	rss_config = kzalloc(total_size, GFP_KERNEL);
 223	if (!rss_config) {
 224		ret = -ENOMEM;
 225		goto out_unlock;
 226	}
 227
 228	data->indir_table = (u32 *)rss_config;
 229	if (data->hkey_size)
 230		data->hkey = rss_config + indir_bytes;
 231
 232	__rss_prepare_ctx(dev, data, ctx);
 233
 234	ret = 0;
 235out_unlock:
 236	mutex_unlock(&dev->ethtool->rss_lock);
 237	return ret;
 238}
 239
 240static int
 241rss_prepare(const struct rss_req_info *request, struct net_device *dev,
 242	    struct rss_reply_data *data, const struct genl_info *info)
 243{
 244	rss_prepare_flow_hash(request, dev, data, info);
 245
 246	/* Coming from RSS_SET, driver may only have flow_hash_fields ops */
 247	if (!dev->ethtool_ops->get_rxfh)
 248		return 0;
 249
 250	if (request->rss_context)
 251		return rss_prepare_ctx(request, dev, data, info);
 252	return rss_prepare_get(request, dev, data, info);
 253}
 254
 255static int
 256rss_prepare_data(const struct ethnl_req_info *req_base,
 257		 struct ethnl_reply_data *reply_base,
 258		 const struct genl_info *info)
 259{
 260	struct rss_reply_data *data = RSS_REPDATA(reply_base);
 261	struct rss_req_info *request = RSS_REQINFO(req_base);
 262	struct net_device *dev = reply_base->dev;
 263	const struct ethtool_ops *ops;
 264
 265	ops = dev->ethtool_ops;
 266	if (!ops->get_rxfh)
 267		return -EOPNOTSUPP;
 268
 269	/* Some drivers don't handle rss_context */
 270	if (request->rss_context && !ops->create_rxfh_context)
 271		return -EOPNOTSUPP;
 272
 273	return rss_prepare(request, dev, data, info);
 274}
 275
 276static int
 277rss_reply_size(const struct ethnl_req_info *req_base,
 278	       const struct ethnl_reply_data *reply_base)
 279{
 280	const struct rss_reply_data *data = RSS_REPDATA(reply_base);
 281	int len;
 282
 283	len = nla_total_size(sizeof(u32)) +	/* _RSS_CONTEXT */
 284	      nla_total_size(sizeof(u32)) +	/* _RSS_HFUNC */
 285	      nla_total_size(sizeof(u32)) +	/* _RSS_INPUT_XFRM */
 286	      nla_total_size(sizeof(u32) * data->indir_size) + /* _RSS_INDIR */
 287	      nla_total_size(data->hkey_size) + /* _RSS_HKEY */
 288	      nla_total_size(0) +		/* _RSS_FLOW_HASH */
 289		nla_total_size(sizeof(u32)) * ETHTOOL_A_FLOW_MAX +
 290	      0;
 291
 292	return len;
 293}
 294
 295static int
 296rss_fill_reply(struct sk_buff *skb, const struct ethnl_req_info *req_base,
 297	       const struct ethnl_reply_data *reply_base)
 298{
 299	const struct rss_reply_data *data = RSS_REPDATA(reply_base);
 300	struct rss_req_info *request = RSS_REQINFO(req_base);
 301
 302	if (request->rss_context &&
 303	    nla_put_u32(skb, ETHTOOL_A_RSS_CONTEXT, request->rss_context))
 304		return -EMSGSIZE;
 305
 306	if ((data->indir_size &&
 307	     nla_put(skb, ETHTOOL_A_RSS_INDIR,
 308		     sizeof(u32) * data->indir_size, data->indir_table)))
 309		return -EMSGSIZE;
 310
 311	if (!data->no_key_fields &&
 312	    ((data->hfunc &&
 313	      nla_put_u32(skb, ETHTOOL_A_RSS_HFUNC, data->hfunc)) ||
 314	     (data->input_xfrm &&
 315	      nla_put_u32(skb, ETHTOOL_A_RSS_INPUT_XFRM, data->input_xfrm)) ||
 316	     (data->hkey_size &&
 317	      nla_put(skb, ETHTOOL_A_RSS_HKEY, data->hkey_size, data->hkey))))
 318		return -EMSGSIZE;
 319
 320	if (data->has_flow_hash) {
 321		struct nlattr *nest;
 322		int i;
 323
 324		nest = nla_nest_start(skb, ETHTOOL_A_RSS_FLOW_HASH);
 325		if (!nest)
 326			return -EMSGSIZE;
 327
 328		for (i = 1; i < __ETHTOOL_A_FLOW_CNT; i++) {
 329			if (data->flow_hash[i] >= 0 &&
 330			    nla_put_uint(skb, i, data->flow_hash[i])) {
 331				nla_nest_cancel(skb, nest);
 332				return -EMSGSIZE;
 333			}
 334		}
 335
 336		nla_nest_end(skb, nest);
 337	}
 338
 339	return 0;
 340}
 341
 342static void rss_cleanup_data(struct ethnl_reply_data *reply_base)
 343{
 344	const struct rss_reply_data *data = RSS_REPDATA(reply_base);
 345
 346	rss_get_data_free(data);
 347}
 348
 349struct rss_nl_dump_ctx {
 350	unsigned long		ifindex;
 351	unsigned long		ctx_idx;
 352
 353	/* User wants to only dump contexts from given ifindex */
 354	unsigned int		match_ifindex;
 355	unsigned int		start_ctx;
 356};
 357
 358static struct rss_nl_dump_ctx *rss_dump_ctx(struct netlink_callback *cb)
 359{
 360	NL_ASSERT_CTX_FITS(struct rss_nl_dump_ctx);
 361
 362	return (struct rss_nl_dump_ctx *)cb->ctx;
 363}
 364
 365int ethnl_rss_dump_start(struct netlink_callback *cb)
 366{
 367	const struct genl_info *info = genl_info_dump(cb);
 368	struct rss_nl_dump_ctx *ctx = rss_dump_ctx(cb);
 369	struct ethnl_req_info req_info = {};
 370	struct nlattr **tb = info->attrs;
 371	int ret;
 372
 373	/* Filtering by context not supported */
 374	if (tb[ETHTOOL_A_RSS_CONTEXT]) {
 375		NL_SET_BAD_ATTR(info->extack, tb[ETHTOOL_A_RSS_CONTEXT]);
 376		return -EINVAL;
 377	}
 378	if (tb[ETHTOOL_A_RSS_START_CONTEXT]) {
 379		ctx->start_ctx = nla_get_u32(tb[ETHTOOL_A_RSS_START_CONTEXT]);
 380		ctx->ctx_idx = ctx->start_ctx;
 381	}
 382
 383	ret = ethnl_parse_header_dev_get(&req_info,
 384					 tb[ETHTOOL_A_RSS_HEADER],
 385					 sock_net(cb->skb->sk), cb->extack,
 386					 false);
 387	if (req_info.dev) {
 388		ctx->match_ifindex = req_info.dev->ifindex;
 389		ctx->ifindex = ctx->match_ifindex;
 390		ethnl_parse_header_dev_put(&req_info);
 391		req_info.dev = NULL;
 392	}
 393
 394	return ret;
 395}
 396
 397static int
 398rss_dump_one_ctx(struct sk_buff *skb, struct netlink_callback *cb,
 399		 struct net_device *dev, u32 rss_context)
 400{
 401	const struct genl_info *info = genl_info_dump(cb);
 402	struct rss_reply_data data = {};
 403	struct rss_req_info req = {};
 404	void *ehdr;
 405	int ret;
 406
 407	req.rss_context = rss_context;
 408
 409	ehdr = ethnl_dump_put(skb, cb, ETHTOOL_MSG_RSS_GET_REPLY);
 410	if (!ehdr)
 411		return -EMSGSIZE;
 412
 413	ret = ethnl_fill_reply_header(skb, dev, ETHTOOL_A_RSS_HEADER);
 414	if (ret < 0)
 415		goto err_cancel;
 416
 417	ret = rss_prepare(&req, dev, &data, info);
 418	if (ret)
 419		goto err_cancel;
 420
 421	ret = rss_fill_reply(skb, &req.base, &data.base);
 422	if (ret)
 423		goto err_cleanup;
 424	genlmsg_end(skb, ehdr);
 425
 426	rss_cleanup_data(&data.base);
 427	return 0;
 428
 429err_cleanup:
 430	rss_cleanup_data(&data.base);
 431err_cancel:
 432	genlmsg_cancel(skb, ehdr);
 433	return ret;
 434}
 435
 436static int
 437rss_dump_one_dev(struct sk_buff *skb, struct netlink_callback *cb,
 438		 struct net_device *dev)
 439{
 440	struct rss_nl_dump_ctx *ctx = rss_dump_ctx(cb);
 441	int ret;
 442
 443	if (!dev->ethtool_ops->get_rxfh)
 444		return 0;
 445
 446	if (!ctx->ctx_idx) {
 447		ret = rss_dump_one_ctx(skb, cb, dev, 0);
 448		if (ret)
 449			return ret;
 450		ctx->ctx_idx++;
 451	}
 452
 453	for (; xa_find(&dev->ethtool->rss_ctx, &ctx->ctx_idx,
 454		       ULONG_MAX, XA_PRESENT); ctx->ctx_idx++) {
 455		ret = rss_dump_one_ctx(skb, cb, dev, ctx->ctx_idx);
 456		if (ret)
 457			return ret;
 458	}
 459	ctx->ctx_idx = ctx->start_ctx;
 460
 461	return 0;
 462}
 463
 464int ethnl_rss_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
 465{
 466	struct rss_nl_dump_ctx *ctx = rss_dump_ctx(cb);
 467	struct net *net = sock_net(skb->sk);
 468	struct net_device *dev;
 469	int ret = 0;
 470
 471	rtnl_lock();
 472	for_each_netdev_dump(net, dev, ctx->ifindex) {
 473		if (ctx->match_ifindex && ctx->match_ifindex != ctx->ifindex)
 474			break;
 475
 476		netdev_lock_ops(dev);
 477		ret = rss_dump_one_dev(skb, cb, dev);
 478		netdev_unlock_ops(dev);
 479		if (ret)
 480			break;
 481	}
 482	rtnl_unlock();
 483
 484	return ret;
 485}
 486
 487/* RSS_NTF */
 488
 489static void ethnl_rss_delete_notify(struct net_device *dev, u32 rss_context)
 490{
 491	struct sk_buff *ntf;
 492	size_t ntf_size;
 493	void *hdr;
 494
 495	ntf_size = ethnl_reply_header_size() +
 496		nla_total_size(sizeof(u32));	/* _RSS_CONTEXT */
 497
 498	ntf = genlmsg_new(ntf_size, GFP_KERNEL);
 499	if (!ntf)
 500		goto out_warn;
 501
 502	hdr = ethnl_bcastmsg_put(ntf, ETHTOOL_MSG_RSS_DELETE_NTF);
 503	if (!hdr)
 504		goto out_free_ntf;
 505
 506	if (ethnl_fill_reply_header(ntf, dev, ETHTOOL_A_RSS_HEADER) ||
 507	    nla_put_u32(ntf, ETHTOOL_A_RSS_CONTEXT, rss_context))
 508		goto out_free_ntf;
 509
 510	genlmsg_end(ntf, hdr);
 511	if (ethnl_multicast(ntf, dev))
 512		goto out_warn;
 513
 514	return;
 515
 516out_free_ntf:
 517	nlmsg_free(ntf);
 518out_warn:
 519	pr_warn_once("Failed to send a RSS delete notification");
 520}
 521
 522void ethtool_rss_notify(struct net_device *dev, u32 type, u32 rss_context)
 523{
 524	struct rss_req_info req_info = {
 525		.rss_context = rss_context,
 526	};
 527
 528	if (type == ETHTOOL_MSG_RSS_DELETE_NTF)
 529		ethnl_rss_delete_notify(dev, rss_context);
 530	else
 531		ethnl_notify(dev, type, &req_info.base);
 532}
 533
 534/* RSS_SET */
 535
 536#define RFH_MASK (RXH_L2DA | RXH_VLAN | RXH_IP_SRC | RXH_IP_DST | \
 537		  RXH_L3_PROTO | RXH_L4_B_0_1 | RXH_L4_B_2_3 |	  \
 538		  RXH_GTP_TEID | RXH_DISCARD)
 539#define RFH_MASKv6 (RFH_MASK | RXH_IP6_FL)
 540
 541static const struct nla_policy ethnl_rss_flows_policy[] = {
 542	[ETHTOOL_A_FLOW_ETHER]		= NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
 543	[ETHTOOL_A_FLOW_IP4]		= NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
 544	[ETHTOOL_A_FLOW_IP6]		= NLA_POLICY_MASK(NLA_UINT, RFH_MASKv6),
 545	[ETHTOOL_A_FLOW_TCP4]		= NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
 546	[ETHTOOL_A_FLOW_UDP4]		= NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
 547	[ETHTOOL_A_FLOW_SCTP4]		= NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
 548	[ETHTOOL_A_FLOW_AH_ESP4]	= NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
 549	[ETHTOOL_A_FLOW_TCP6]		= NLA_POLICY_MASK(NLA_UINT, RFH_MASKv6),
 550	[ETHTOOL_A_FLOW_UDP6]		= NLA_POLICY_MASK(NLA_UINT, RFH_MASKv6),
 551	[ETHTOOL_A_FLOW_SCTP6]		= NLA_POLICY_MASK(NLA_UINT, RFH_MASKv6),
 552	[ETHTOOL_A_FLOW_AH_ESP6]	= NLA_POLICY_MASK(NLA_UINT, RFH_MASKv6),
 553	[ETHTOOL_A_FLOW_AH4]		= NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
 554	[ETHTOOL_A_FLOW_ESP4]		= NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
 555	[ETHTOOL_A_FLOW_AH6]		= NLA_POLICY_MASK(NLA_UINT, RFH_MASKv6),
 556	[ETHTOOL_A_FLOW_ESP6]		= NLA_POLICY_MASK(NLA_UINT, RFH_MASKv6),
 557	[ETHTOOL_A_FLOW_GTPU4]		= NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
 558	[ETHTOOL_A_FLOW_GTPU6]		= NLA_POLICY_MASK(NLA_UINT, RFH_MASKv6),
 559	[ETHTOOL_A_FLOW_GTPC4]		= NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
 560	[ETHTOOL_A_FLOW_GTPC6]		= NLA_POLICY_MASK(NLA_UINT, RFH_MASKv6),
 561	[ETHTOOL_A_FLOW_GTPC_TEID4]	= NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
 562	[ETHTOOL_A_FLOW_GTPC_TEID6]	= NLA_POLICY_MASK(NLA_UINT, RFH_MASKv6),
 563	[ETHTOOL_A_FLOW_GTPU_EH4]	= NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
 564	[ETHTOOL_A_FLOW_GTPU_EH6]	= NLA_POLICY_MASK(NLA_UINT, RFH_MASKv6),
 565	[ETHTOOL_A_FLOW_GTPU_UL4]	= NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
 566	[ETHTOOL_A_FLOW_GTPU_UL6]	= NLA_POLICY_MASK(NLA_UINT, RFH_MASKv6),
 567	[ETHTOOL_A_FLOW_GTPU_DL4]	= NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
 568	[ETHTOOL_A_FLOW_GTPU_DL6]	= NLA_POLICY_MASK(NLA_UINT, RFH_MASKv6),
 569};
 570
 571const struct nla_policy ethnl_rss_set_policy[ETHTOOL_A_RSS_FLOW_HASH + 1] = {
 572	[ETHTOOL_A_RSS_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy),
 573	[ETHTOOL_A_RSS_CONTEXT] = { .type = NLA_U32, },
 574	[ETHTOOL_A_RSS_HFUNC] = NLA_POLICY_MIN(NLA_U32, 1),
 575	[ETHTOOL_A_RSS_INDIR] = { .type = NLA_BINARY, },
 576	[ETHTOOL_A_RSS_HKEY] = NLA_POLICY_MIN(NLA_BINARY, 1),
 577	[ETHTOOL_A_RSS_INPUT_XFRM] =
 578		NLA_POLICY_MAX(NLA_U32, RXH_XFRM_SYM_OR_XOR),
 579	[ETHTOOL_A_RSS_FLOW_HASH] = NLA_POLICY_NESTED(ethnl_rss_flows_policy),
 580};
 581
 582static int
 583ethnl_rss_set_validate(struct ethnl_req_info *req_info, struct genl_info *info)
 584{
 585	const struct ethtool_ops *ops = req_info->dev->ethtool_ops;
 586	struct rss_req_info *request = RSS_REQINFO(req_info);
 587	struct nlattr **tb = info->attrs;
 588	struct nlattr *bad_attr = NULL;
 589	u32 input_xfrm;
 590
 591	if (request->rss_context && !ops->create_rxfh_context)
 592		bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_CONTEXT];
 593
 594	if (request->rss_context && !ops->rxfh_per_ctx_key) {
 595		bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_HFUNC];
 596		bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_HKEY];
 597		bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_INPUT_XFRM];
 598	}
 599
 600	input_xfrm = nla_get_u32_default(tb[ETHTOOL_A_RSS_INPUT_XFRM], 0);
 601	if (input_xfrm & ~ops->supported_input_xfrm)
 602		bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_INPUT_XFRM];
 603
 604	if (tb[ETHTOOL_A_RSS_FLOW_HASH] && !ops->set_rxfh_fields)
 605		bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_FLOW_HASH];
 606	if (request->rss_context &&
 607	    tb[ETHTOOL_A_RSS_FLOW_HASH] && !ops->rxfh_per_ctx_fields)
 608		bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_FLOW_HASH];
 609
 610	if (bad_attr) {
 611		NL_SET_BAD_ATTR(info->extack, bad_attr);
 612		return -EOPNOTSUPP;
 613	}
 614
 615	return 1;
 616}
 617
 618static int
 619rss_set_prep_indir(struct net_device *dev, struct genl_info *info,
 620		   struct rss_reply_data *data, struct ethtool_rxfh_param *rxfh,
 621		   bool *reset, bool *mod)
 622{
 623	struct netlink_ext_ack *extack = info->extack;
 624	struct nlattr **tb = info->attrs;
 625	size_t alloc_size;
 626	int num_rx_rings;
 627	u32 user_size;
 628	int i, err;
 629
 630	if (!tb[ETHTOOL_A_RSS_INDIR])
 631		return 0;
 632	if (!data->indir_size)
 633		return -EOPNOTSUPP;
 634
 635	err = ethtool_get_rx_ring_count(dev);
 636	if (err < 0)
 637		return err;
 638	num_rx_rings = err;
 639
 640	if (nla_len(tb[ETHTOOL_A_RSS_INDIR]) % 4) {
 641		NL_SET_BAD_ATTR(info->extack, tb[ETHTOOL_A_RSS_INDIR]);
 642		return -EINVAL;
 643	}
 644	user_size = nla_len(tb[ETHTOOL_A_RSS_INDIR]) / 4;
 645	if (!user_size) {
 646		if (rxfh->rss_context) {
 647			NL_SET_ERR_MSG_ATTR(extack, tb[ETHTOOL_A_RSS_INDIR],
 648					    "can't reset table for a context");
 649			return -EINVAL;
 650		}
 651		*reset = true;
 652	} else if (data->indir_size % user_size) {
 653		NL_SET_ERR_MSG_ATTR_FMT(extack, tb[ETHTOOL_A_RSS_INDIR],
 654					"size (%d) mismatch with device indir table (%d)",
 655					user_size, data->indir_size);
 656		return -EINVAL;
 657	}
 658
 659	rxfh->indir_size = data->indir_size;
 660	alloc_size = array_size(data->indir_size, sizeof(rxfh->indir[0]));
 661	rxfh->indir = kzalloc(alloc_size, GFP_KERNEL);
 662	if (!rxfh->indir)
 663		return -ENOMEM;
 664
 665	nla_memcpy(rxfh->indir, tb[ETHTOOL_A_RSS_INDIR], alloc_size);
 666	for (i = 0; i < user_size; i++) {
 667		if (rxfh->indir[i] < num_rx_rings)
 668			continue;
 669
 670		NL_SET_ERR_MSG_ATTR_FMT(extack, tb[ETHTOOL_A_RSS_INDIR],
 671					"entry %d: queue out of range (%d)",
 672					i, rxfh->indir[i]);
 673		err = -EINVAL;
 674		goto err_free;
 675	}
 676
 677	if (user_size) {
 678		/* Replicate the user-provided table to fill the device table */
 679		for (i = user_size; i < data->indir_size; i++)
 680			rxfh->indir[i] = rxfh->indir[i % user_size];
 681	} else {
 682		for (i = 0; i < data->indir_size; i++)
 683			rxfh->indir[i] =
 684				ethtool_rxfh_indir_default(i, num_rx_rings);
 685	}
 686
 687	*mod |= memcmp(rxfh->indir, data->indir_table, data->indir_size);
 688
 689	return 0;
 690
 691err_free:
 692	kfree(rxfh->indir);
 693	rxfh->indir = NULL;
 694	return err;
 695}
 696
 697static int
 698rss_set_prep_hkey(struct net_device *dev, struct genl_info *info,
 699		  struct rss_reply_data *data, struct ethtool_rxfh_param *rxfh,
 700		  bool *mod)
 701{
 702	struct nlattr **tb = info->attrs;
 703
 704	if (!tb[ETHTOOL_A_RSS_HKEY])
 705		return 0;
 706
 707	if (nla_len(tb[ETHTOOL_A_RSS_HKEY]) != data->hkey_size) {
 708		NL_SET_BAD_ATTR(info->extack, tb[ETHTOOL_A_RSS_HKEY]);
 709		return -EINVAL;
 710	}
 711
 712	rxfh->key_size = data->hkey_size;
 713	rxfh->key = kmemdup(data->hkey, data->hkey_size, GFP_KERNEL);
 714	if (!rxfh->key)
 715		return -ENOMEM;
 716
 717	ethnl_update_binary(rxfh->key, rxfh->key_size, tb[ETHTOOL_A_RSS_HKEY],
 718			    mod);
 719	return 0;
 720}
 721
 722static int
 723rss_check_rxfh_fields_sym(struct net_device *dev, struct genl_info *info,
 724			  struct rss_reply_data *data, bool xfrm_sym)
 725{
 726	struct nlattr **tb = info->attrs;
 727	int i;
 728
 729	if (!xfrm_sym)
 730		return 0;
 731	if (!data->has_flow_hash) {
 732		NL_SET_ERR_MSG_ATTR(info->extack, tb[ETHTOOL_A_RSS_INPUT_XFRM],
 733				    "hash field config not reported");
 734		return -EINVAL;
 735	}
 736
 737	for (i = 1; i < __ETHTOOL_A_FLOW_CNT; i++)
 738		if (data->flow_hash[i] >= 0 &&
 739		    !ethtool_rxfh_config_is_sym(data->flow_hash[i])) {
 740			NL_SET_ERR_MSG_ATTR(info->extack,
 741					    tb[ETHTOOL_A_RSS_INPUT_XFRM],
 742					    "hash field config is not symmetric");
 743			return -EINVAL;
 744		}
 745
 746	return 0;
 747}
 748
 749static int
 750ethnl_set_rss_fields(struct net_device *dev, struct genl_info *info,
 751		     u32 rss_context, struct rss_reply_data *data,
 752		     bool xfrm_sym, bool *mod)
 753{
 754	struct nlattr *flow_nest = info->attrs[ETHTOOL_A_RSS_FLOW_HASH];
 755	struct nlattr *flows[ETHTOOL_A_FLOW_MAX + 1];
 756	const struct ethtool_ops *ops;
 757	int i, ret;
 758
 759	ops = dev->ethtool_ops;
 760
 761	ret = rss_check_rxfh_fields_sym(dev, info, data, xfrm_sym);
 762	if (ret)
 763		return ret;
 764
 765	if (!flow_nest)
 766		return 0;
 767
 768	ret = nla_parse_nested(flows, ARRAY_SIZE(ethnl_rss_flows_policy) - 1,
 769			       flow_nest, ethnl_rss_flows_policy, info->extack);
 770	if (ret < 0)
 771		return ret;
 772
 773	for (i = 1; i < __ETHTOOL_A_FLOW_CNT; i++) {
 774		struct ethtool_rxfh_fields fields = {
 775			.flow_type	= ethtool_rxfh_ft_nl2ioctl[i],
 776			.rss_context	= rss_context,
 777		};
 778
 779		if (!flows[i])
 780			continue;
 781
 782		fields.data = nla_get_u32(flows[i]);
 783		if (data->has_flow_hash && data->flow_hash[i] == fields.data)
 784			continue;
 785
 786		if (xfrm_sym && !ethtool_rxfh_config_is_sym(fields.data)) {
 787			NL_SET_ERR_MSG_ATTR(info->extack, flows[i],
 788					    "conflict with xfrm-input");
 789			return -EINVAL;
 790		}
 791
 792		ret = ops->set_rxfh_fields(dev, &fields, info->extack);
 793		if (ret)
 794			return ret;
 795
 796		*mod = true;
 797	}
 798
 799	return 0;
 800}
 801
 802static void
 803rss_set_ctx_update(struct ethtool_rxfh_context *ctx, struct nlattr **tb,
 804		   struct rss_reply_data *data, struct ethtool_rxfh_param *rxfh)
 805{
 806	int i;
 807
 808	if (rxfh->indir) {
 809		for (i = 0; i < data->indir_size; i++)
 810			ethtool_rxfh_context_indir(ctx)[i] = rxfh->indir[i];
 811		ctx->indir_configured = !!nla_len(tb[ETHTOOL_A_RSS_INDIR]);
 812	}
 813	if (rxfh->key) {
 814		memcpy(ethtool_rxfh_context_key(ctx), rxfh->key,
 815		       data->hkey_size);
 816		ctx->key_configured = !!rxfh->key_size;
 817	}
 818	if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE)
 819		ctx->hfunc = rxfh->hfunc;
 820	if (rxfh->input_xfrm != RXH_XFRM_NO_CHANGE)
 821		ctx->input_xfrm = rxfh->input_xfrm;
 822}
 823
 824static int
 825ethnl_rss_set(struct ethnl_req_info *req_info, struct genl_info *info)
 826{
 827	struct rss_req_info *request = RSS_REQINFO(req_info);
 828	bool indir_reset = false, indir_mod, xfrm_sym;
 829	struct ethtool_rxfh_context *ctx = NULL;
 830	struct net_device *dev = req_info->dev;
 831	bool mod = false, fields_mod = false;
 832	struct ethtool_rxfh_param rxfh = {};
 833	struct nlattr **tb = info->attrs;
 834	struct rss_reply_data data = {};
 835	const struct ethtool_ops *ops;
 836	int ret;
 837
 838	ops = dev->ethtool_ops;
 839	data.base.dev = dev;
 840
 841	ret = rss_prepare(request, dev, &data, info);
 842	if (ret)
 843		return ret;
 844
 845	rxfh.rss_context = request->rss_context;
 846
 847	ret = rss_set_prep_indir(dev, info, &data, &rxfh, &indir_reset, &mod);
 848	if (ret)
 849		goto exit_clean_data;
 850	indir_mod = !!tb[ETHTOOL_A_RSS_INDIR];
 851
 852	rxfh.hfunc = data.hfunc;
 853	ethnl_update_u8(&rxfh.hfunc, tb[ETHTOOL_A_RSS_HFUNC], &mod);
 854	if (rxfh.hfunc == data.hfunc)
 855		rxfh.hfunc = ETH_RSS_HASH_NO_CHANGE;
 856
 857	ret = rss_set_prep_hkey(dev, info, &data, &rxfh, &mod);
 858	if (ret)
 859		goto exit_free_indir;
 860
 861	rxfh.input_xfrm = data.input_xfrm;
 862	ethnl_update_u8(&rxfh.input_xfrm, tb[ETHTOOL_A_RSS_INPUT_XFRM], &mod);
 863	xfrm_sym = rxfh.input_xfrm || data.input_xfrm;
 864	if (rxfh.input_xfrm == data.input_xfrm)
 865		rxfh.input_xfrm = RXH_XFRM_NO_CHANGE;
 866
 867	mutex_lock(&dev->ethtool->rss_lock);
 868	if (request->rss_context) {
 869		ctx = xa_load(&dev->ethtool->rss_ctx, request->rss_context);
 870		if (!ctx) {
 871			ret = -ENOENT;
 872			goto exit_unlock;
 873		}
 874	}
 875
 876	ret = ethnl_set_rss_fields(dev, info, request->rss_context,
 877				   &data, xfrm_sym, &fields_mod);
 878	if (ret)
 879		goto exit_unlock;
 880
 881	if (!mod)
 882		ret = 0; /* nothing to tell the driver */
 883	else if (!ops->set_rxfh)
 884		ret = -EOPNOTSUPP;
 885	else if (!rxfh.rss_context)
 886		ret = ops->set_rxfh(dev, &rxfh, info->extack);
 887	else
 888		ret = ops->modify_rxfh_context(dev, ctx, &rxfh, info->extack);
 889	if (ret)
 890		goto exit_unlock;
 891
 892	if (ctx)
 893		rss_set_ctx_update(ctx, tb, &data, &rxfh);
 894	else if (indir_reset)
 895		dev->priv_flags &= ~IFF_RXFH_CONFIGURED;
 896	else if (indir_mod)
 897		dev->priv_flags |= IFF_RXFH_CONFIGURED;
 898
 899exit_unlock:
 900	mutex_unlock(&dev->ethtool->rss_lock);
 901	kfree(rxfh.key);
 902exit_free_indir:
 903	kfree(rxfh.indir);
 904exit_clean_data:
 905	rss_cleanup_data(&data.base);
 906
 907	return ret ?: mod || fields_mod;
 908}
 909
 910const struct ethnl_request_ops ethnl_rss_request_ops = {
 911	.request_cmd		= ETHTOOL_MSG_RSS_GET,
 912	.reply_cmd		= ETHTOOL_MSG_RSS_GET_REPLY,
 913	.hdr_attr		= ETHTOOL_A_RSS_HEADER,
 914	.req_info_size		= sizeof(struct rss_req_info),
 915	.reply_data_size	= sizeof(struct rss_reply_data),
 916
 917	.parse_request		= rss_parse_request,
 918	.prepare_data		= rss_prepare_data,
 919	.reply_size		= rss_reply_size,
 920	.fill_reply		= rss_fill_reply,
 921	.cleanup_data		= rss_cleanup_data,
 922
 923	.set_validate		= ethnl_rss_set_validate,
 924	.set			= ethnl_rss_set,
 925	.set_ntf_cmd		= ETHTOOL_MSG_RSS_NTF,
 926};
 927
 928/* RSS_CREATE */
 929
 930const struct nla_policy ethnl_rss_create_policy[ETHTOOL_A_RSS_INPUT_XFRM + 1] = {
 931	[ETHTOOL_A_RSS_HEADER]	= NLA_POLICY_NESTED(ethnl_header_policy),
 932	[ETHTOOL_A_RSS_CONTEXT]	= NLA_POLICY_MIN(NLA_U32, 1),
 933	[ETHTOOL_A_RSS_HFUNC]	= NLA_POLICY_MIN(NLA_U32, 1),
 934	[ETHTOOL_A_RSS_INDIR]	= NLA_POLICY_MIN(NLA_BINARY, 1),
 935	[ETHTOOL_A_RSS_HKEY]	= NLA_POLICY_MIN(NLA_BINARY, 1),
 936	[ETHTOOL_A_RSS_INPUT_XFRM] =
 937		NLA_POLICY_MAX(NLA_U32, RXH_XFRM_SYM_OR_XOR),
 938};
 939
 940static int
 941ethnl_rss_create_validate(struct net_device *dev, struct genl_info *info)
 942{
 943	const struct ethtool_ops *ops = dev->ethtool_ops;
 944	struct nlattr **tb = info->attrs;
 945	struct nlattr *bad_attr = NULL;
 946	u32 rss_context, input_xfrm;
 947
 948	if (!ops->create_rxfh_context)
 949		return -EOPNOTSUPP;
 950
 951	rss_context = nla_get_u32_default(tb[ETHTOOL_A_RSS_CONTEXT], 0);
 952	if (ops->rxfh_max_num_contexts &&
 953	    ops->rxfh_max_num_contexts <= rss_context) {
 954		NL_SET_BAD_ATTR(info->extack, tb[ETHTOOL_A_RSS_CONTEXT]);
 955		return -ERANGE;
 956	}
 957
 958	if (!ops->rxfh_per_ctx_key) {
 959		bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_HFUNC];
 960		bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_HKEY];
 961		bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_INPUT_XFRM];
 962	}
 963
 964	input_xfrm = nla_get_u32_default(tb[ETHTOOL_A_RSS_INPUT_XFRM], 0);
 965	if (input_xfrm & ~ops->supported_input_xfrm)
 966		bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_INPUT_XFRM];
 967
 968	if (bad_attr) {
 969		NL_SET_BAD_ATTR(info->extack, bad_attr);
 970		return -EOPNOTSUPP;
 971	}
 972
 973	return 0;
 974}
 975
 976static void
 977ethnl_rss_create_send_ntf(struct sk_buff *rsp, struct net_device *dev)
 978{
 979	struct nlmsghdr *nlh = (void *)rsp->data;
 980	struct genlmsghdr *genl_hdr;
 981
 982	/* Convert the reply into a notification */
 983	nlh->nlmsg_pid = 0;
 984	nlh->nlmsg_seq = ethnl_bcast_seq_next();
 985
 986	genl_hdr = nlmsg_data(nlh);
 987	genl_hdr->cmd =	ETHTOOL_MSG_RSS_CREATE_NTF;
 988
 989	ethnl_multicast(rsp, dev);
 990}
 991
 992int ethnl_rss_create_doit(struct sk_buff *skb, struct genl_info *info)
 993{
 994	bool indir_dflt = false, mod = false, ntf_fail = false;
 995	struct ethtool_rxfh_param rxfh = {};
 996	struct ethtool_rxfh_context *ctx;
 997	struct nlattr **tb = info->attrs;
 998	struct rss_reply_data data = {};
 999	const struct ethtool_ops *ops;
1000	struct rss_req_info req = {};
1001	struct net_device *dev;
1002	struct sk_buff *rsp;
1003	void *hdr;
1004	u32 limit;
1005	int ret;
1006
1007	rsp = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
1008	if (!rsp)
1009		return -ENOMEM;
1010
1011	ret = ethnl_parse_header_dev_get(&req.base, tb[ETHTOOL_A_RSS_HEADER],
1012					 genl_info_net(info), info->extack,
1013					 true);
1014	if (ret < 0)
1015		goto exit_free_rsp;
1016
1017	dev = req.base.dev;
1018	ops = dev->ethtool_ops;
1019
1020	req.rss_context = nla_get_u32_default(tb[ETHTOOL_A_RSS_CONTEXT], 0);
1021
1022	ret = ethnl_rss_create_validate(dev, info);
1023	if (ret)
1024		goto exit_free_dev;
1025
1026	rtnl_lock();
1027	netdev_lock_ops(dev);
1028
1029	ret = ethnl_ops_begin(dev);
1030	if (ret < 0)
1031		goto exit_dev_unlock;
1032
1033	ret = rss_get_data_alloc(dev, &data);
1034	if (ret)
1035		goto exit_ops;
1036
1037	ret = rss_set_prep_indir(dev, info, &data, &rxfh, &indir_dflt, &mod);
1038	if (ret)
1039		goto exit_clean_data;
1040
1041	ethnl_update_u8(&rxfh.hfunc, tb[ETHTOOL_A_RSS_HFUNC], &mod);
1042
1043	ret = rss_set_prep_hkey(dev, info, &data, &rxfh, &mod);
1044	if (ret)
1045		goto exit_free_indir;
1046
1047	rxfh.input_xfrm = RXH_XFRM_NO_CHANGE;
1048	ethnl_update_u8(&rxfh.input_xfrm, tb[ETHTOOL_A_RSS_INPUT_XFRM], &mod);
1049
1050	ctx = ethtool_rxfh_ctx_alloc(ops, data.indir_size, data.hkey_size);
1051	if (!ctx) {
1052		ret = -ENOMEM;
1053		goto exit_free_hkey;
1054	}
1055
1056	mutex_lock(&dev->ethtool->rss_lock);
1057	if (!req.rss_context) {
1058		limit = ops->rxfh_max_num_contexts ?: U32_MAX;
1059		ret = xa_alloc(&dev->ethtool->rss_ctx, &req.rss_context, ctx,
1060			       XA_LIMIT(1, limit - 1), GFP_KERNEL_ACCOUNT);
1061	} else {
1062		ret = xa_insert(&dev->ethtool->rss_ctx,
1063				req.rss_context, ctx, GFP_KERNEL_ACCOUNT);
1064	}
1065	if (ret < 0) {
1066		NL_SET_ERR_MSG_ATTR(info->extack, tb[ETHTOOL_A_RSS_CONTEXT],
1067				    "error allocating context ID");
1068		goto err_unlock_free_ctx;
1069	}
1070	rxfh.rss_context = req.rss_context;
1071
1072	ret = ops->create_rxfh_context(dev, ctx, &rxfh, info->extack);
1073	if (ret)
1074		goto err_ctx_id_free;
1075
1076	/* Make sure driver populates defaults */
1077	WARN_ON_ONCE(!rxfh.key && ops->rxfh_per_ctx_key &&
1078		     !memchr_inv(ethtool_rxfh_context_key(ctx), 0,
1079				 ctx->key_size));
1080
1081	/* Store the config from rxfh to Xarray.. */
1082	rss_set_ctx_update(ctx, tb, &data, &rxfh);
1083	/* .. copy from Xarray to data. */
1084	__rss_prepare_ctx(dev, &data, ctx);
1085
1086	hdr = ethnl_unicast_put(rsp, info->snd_portid, info->snd_seq,
1087				ETHTOOL_MSG_RSS_CREATE_ACT_REPLY);
1088	ntf_fail = ethnl_fill_reply_header(rsp, dev, ETHTOOL_A_RSS_HEADER);
1089	ntf_fail |= rss_fill_reply(rsp, &req.base, &data.base);
1090	if (WARN_ON(!hdr || ntf_fail)) {
1091		ret = -EMSGSIZE;
1092		goto exit_unlock;
1093	}
1094
1095	genlmsg_end(rsp, hdr);
1096
1097	/* Use the same skb for the response and the notification,
1098	 * genlmsg_reply() will copy the skb if it has elevated user count.
1099	 */
1100	skb_get(rsp);
1101	ret = genlmsg_reply(rsp, info);
1102	ethnl_rss_create_send_ntf(rsp, dev);
1103	rsp = NULL;
1104
1105exit_unlock:
1106	mutex_unlock(&dev->ethtool->rss_lock);
1107exit_free_hkey:
1108	kfree(rxfh.key);
1109exit_free_indir:
1110	kfree(rxfh.indir);
1111exit_clean_data:
1112	rss_get_data_free(&data);
1113exit_ops:
1114	ethnl_ops_complete(dev);
1115exit_dev_unlock:
1116	netdev_unlock_ops(dev);
1117	rtnl_unlock();
1118exit_free_dev:
1119	ethnl_parse_header_dev_put(&req.base);
1120exit_free_rsp:
1121	nlmsg_free(rsp);
1122	return ret;
1123
1124err_ctx_id_free:
1125	xa_erase(&dev->ethtool->rss_ctx, req.rss_context);
1126err_unlock_free_ctx:
1127	kfree(ctx);
1128	goto exit_unlock;
1129}
1130
1131/* RSS_DELETE */
1132
1133const struct nla_policy ethnl_rss_delete_policy[ETHTOOL_A_RSS_CONTEXT + 1] = {
1134	[ETHTOOL_A_RSS_HEADER]	= NLA_POLICY_NESTED(ethnl_header_policy),
1135	[ETHTOOL_A_RSS_CONTEXT]	= NLA_POLICY_MIN(NLA_U32, 1),
1136};
1137
1138int ethnl_rss_delete_doit(struct sk_buff *skb, struct genl_info *info)
1139{
1140	struct ethtool_rxfh_context *ctx;
1141	struct nlattr **tb = info->attrs;
1142	struct ethnl_req_info req = {};
1143	const struct ethtool_ops *ops;
1144	struct net_device *dev;
1145	u32 rss_context;
1146	int ret;
1147
1148	if (GENL_REQ_ATTR_CHECK(info, ETHTOOL_A_RSS_CONTEXT))
1149		return -EINVAL;
1150	rss_context = nla_get_u32(tb[ETHTOOL_A_RSS_CONTEXT]);
1151
1152	ret = ethnl_parse_header_dev_get(&req, tb[ETHTOOL_A_RSS_HEADER],
1153					 genl_info_net(info), info->extack,
1154					 true);
1155	if (ret < 0)
1156		return ret;
1157
1158	dev = req.dev;
1159	ops = dev->ethtool_ops;
1160
1161	if (!ops->create_rxfh_context)
1162		goto exit_free_dev;
1163
1164	rtnl_lock();
1165	netdev_lock_ops(dev);
1166
1167	ret = ethnl_ops_begin(dev);
1168	if (ret < 0)
1169		goto exit_dev_unlock;
1170
1171	mutex_lock(&dev->ethtool->rss_lock);
1172	ret = ethtool_check_rss_ctx_busy(dev, rss_context);
1173	if (ret)
1174		goto exit_unlock;
1175
1176	ctx = xa_load(&dev->ethtool->rss_ctx, rss_context);
1177	if (!ctx) {
1178		ret = -ENOENT;
1179		goto exit_unlock;
1180	}
1181
1182	ret = ops->remove_rxfh_context(dev, ctx, rss_context, info->extack);
1183	if (ret)
1184		goto exit_unlock;
1185
1186	WARN_ON(xa_erase(&dev->ethtool->rss_ctx, rss_context) != ctx);
1187	kfree(ctx);
1188
1189	ethnl_rss_delete_notify(dev, rss_context);
1190
1191exit_unlock:
1192	mutex_unlock(&dev->ethtool->rss_lock);
1193	ethnl_ops_complete(dev);
1194exit_dev_unlock:
1195	netdev_unlock_ops(dev);
1196	rtnl_unlock();
1197exit_free_dev:
1198	ethnl_parse_header_dev_put(&req);
1199	return ret;
1200}