spectrum_dpipe.c

   1// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
   2/* Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved */
   3
   4#include <linux/kernel.h>
   5#include <linux/mutex.h>
   6#include <net/devlink.h>
   7
   8#include "spectrum.h"
   9#include "spectrum_dpipe.h"
  10#include "spectrum_router.h"
  11
  12enum mlxsw_sp_field_metadata_id {
  13	MLXSW_SP_DPIPE_FIELD_METADATA_ERIF_PORT,
  14	MLXSW_SP_DPIPE_FIELD_METADATA_L3_FORWARD,
  15	MLXSW_SP_DPIPE_FIELD_METADATA_L3_DROP,
  16	MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_INDEX,
  17	MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_SIZE,
  18	MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_HASH_INDEX,
  19};
  20
  21static struct devlink_dpipe_field mlxsw_sp_dpipe_fields_metadata[] = {
  22	{
  23		.name = "erif_port",
  24		.id = MLXSW_SP_DPIPE_FIELD_METADATA_ERIF_PORT,
  25		.bitwidth = 32,
  26		.mapping_type = DEVLINK_DPIPE_FIELD_MAPPING_TYPE_IFINDEX,
  27	},
  28	{
  29		.name = "l3_forward",
  30		.id = MLXSW_SP_DPIPE_FIELD_METADATA_L3_FORWARD,
  31		.bitwidth = 1,
  32	},
  33	{
  34		.name = "l3_drop",
  35		.id = MLXSW_SP_DPIPE_FIELD_METADATA_L3_DROP,
  36		.bitwidth = 1,
  37	},
  38	{
  39		.name = "adj_index",
  40		.id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_INDEX,
  41		.bitwidth = 32,
  42	},
  43	{
  44		.name = "adj_size",
  45		.id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_SIZE,
  46		.bitwidth = 32,
  47	},
  48	{
  49		.name = "adj_hash_index",
  50		.id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_HASH_INDEX,
  51		.bitwidth = 32,
  52	},
  53};
  54
  55enum mlxsw_sp_dpipe_header_id {
  56	MLXSW_SP_DPIPE_HEADER_METADATA,
  57};
  58
  59static struct devlink_dpipe_header mlxsw_sp_dpipe_header_metadata = {
  60	.name = "mlxsw_meta",
  61	.id = MLXSW_SP_DPIPE_HEADER_METADATA,
  62	.fields = mlxsw_sp_dpipe_fields_metadata,
  63	.fields_count = ARRAY_SIZE(mlxsw_sp_dpipe_fields_metadata),
  64};
  65
  66static struct devlink_dpipe_header *mlxsw_dpipe_headers[] = {
  67	&mlxsw_sp_dpipe_header_metadata,
  68	&devlink_dpipe_header_ethernet,
  69	&devlink_dpipe_header_ipv4,
  70	&devlink_dpipe_header_ipv6,
  71};
  72
  73static struct devlink_dpipe_headers mlxsw_sp_dpipe_headers = {
  74	.headers = mlxsw_dpipe_headers,
  75	.headers_count = ARRAY_SIZE(mlxsw_dpipe_headers),
  76};
  77
  78static int mlxsw_sp_dpipe_table_erif_actions_dump(void *priv,
  79						  struct sk_buff *skb)
  80{
  81	struct devlink_dpipe_action action = {0};
  82	int err;
  83
  84	action.type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
  85	action.header = &mlxsw_sp_dpipe_header_metadata;
  86	action.field_id = MLXSW_SP_DPIPE_FIELD_METADATA_L3_FORWARD;
  87
  88	err = devlink_dpipe_action_put(skb, &action);
  89	if (err)
  90		return err;
  91
  92	action.type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
  93	action.header = &mlxsw_sp_dpipe_header_metadata;
  94	action.field_id = MLXSW_SP_DPIPE_FIELD_METADATA_L3_DROP;
  95
  96	return devlink_dpipe_action_put(skb, &action);
  97}
  98
  99static int mlxsw_sp_dpipe_table_erif_matches_dump(void *priv,
 100						  struct sk_buff *skb)
 101{
 102	struct devlink_dpipe_match match = {0};
 103
 104	match.type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
 105	match.header = &mlxsw_sp_dpipe_header_metadata;
 106	match.field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ERIF_PORT;
 107
 108	return devlink_dpipe_match_put(skb, &match);
 109}
 110
 111static void
 112mlxsw_sp_erif_match_action_prepare(struct devlink_dpipe_match *match,
 113				   struct devlink_dpipe_action *action)
 114{
 115	action->type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
 116	action->header = &mlxsw_sp_dpipe_header_metadata;
 117	action->field_id = MLXSW_SP_DPIPE_FIELD_METADATA_L3_FORWARD;
 118
 119	match->type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
 120	match->header = &mlxsw_sp_dpipe_header_metadata;
 121	match->field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ERIF_PORT;
 122}
 123
 124static int mlxsw_sp_erif_entry_prepare(struct devlink_dpipe_entry *entry,
 125				       struct devlink_dpipe_value *match_value,
 126				       struct devlink_dpipe_match *match,
 127				       struct devlink_dpipe_value *action_value,
 128				       struct devlink_dpipe_action *action)
 129{
 130	entry->match_values = match_value;
 131	entry->match_values_count = 1;
 132
 133	entry->action_values = action_value;
 134	entry->action_values_count = 1;
 135
 136	match_value->match = match;
 137	match_value->value_size = sizeof(u32);
 138	match_value->value = kmalloc(match_value->value_size, GFP_KERNEL);
 139	if (!match_value->value)
 140		return -ENOMEM;
 141
 142	action_value->action = action;
 143	action_value->value_size = sizeof(u32);
 144	action_value->value = kmalloc(action_value->value_size, GFP_KERNEL);
 145	if (!action_value->value)
 146		goto err_action_alloc;
 147	return 0;
 148
 149err_action_alloc:
 150	kfree(match_value->value);
 151	return -ENOMEM;
 152}
 153
 154static int mlxsw_sp_erif_entry_get(struct mlxsw_sp *mlxsw_sp,
 155				   struct devlink_dpipe_entry *entry,
 156				   struct mlxsw_sp_rif *rif,
 157				   bool counters_enabled)
 158{
 159	u32 *action_value;
 160	u32 *rif_value;
 161	u64 cnt;
 162	int err;
 163
 164	/* Set Match RIF index */
 165	rif_value = entry->match_values->value;
 166	*rif_value = mlxsw_sp_rif_index(rif);
 167	entry->match_values->mapping_value = mlxsw_sp_rif_dev_ifindex(rif);
 168	entry->match_values->mapping_valid = true;
 169
 170	/* Set Action Forwarding */
 171	action_value = entry->action_values->value;
 172	*action_value = 1;
 173
 174	entry->counter_valid = false;
 175	entry->counter = 0;
 176	entry->index = mlxsw_sp_rif_index(rif);
 177
 178	if (!counters_enabled)
 179		return 0;
 180
 181	err = mlxsw_sp_rif_counter_value_get(mlxsw_sp, rif,
 182					     MLXSW_SP_RIF_COUNTER_EGRESS,
 183					     &cnt);
 184	if (!err) {
 185		entry->counter = cnt;
 186		entry->counter_valid = true;
 187	}
 188	return 0;
 189}
 190
 191static int
 192mlxsw_sp_dpipe_table_erif_entries_dump(void *priv, bool counters_enabled,
 193				       struct devlink_dpipe_dump_ctx *dump_ctx)
 194{
 195	struct devlink_dpipe_value match_value, action_value;
 196	struct devlink_dpipe_action action = {0};
 197	struct devlink_dpipe_match match = {0};
 198	struct devlink_dpipe_entry entry = {0};
 199	struct mlxsw_sp *mlxsw_sp = priv;
 200	unsigned int rif_count;
 201	int i, j;
 202	int err;
 203
 204	memset(&match_value, 0, sizeof(match_value));
 205	memset(&action_value, 0, sizeof(action_value));
 206
 207	mlxsw_sp_erif_match_action_prepare(&match, &action);
 208	err = mlxsw_sp_erif_entry_prepare(&entry, &match_value, &match,
 209					  &action_value, &action);
 210	if (err)
 211		return err;
 212
 213	rif_count = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
 214	mutex_lock(&mlxsw_sp->router->lock);
 215	i = 0;
 216start_again:
 217	err = devlink_dpipe_entry_ctx_prepare(dump_ctx);
 218	if (err)
 219		goto err_ctx_prepare;
 220	j = 0;
 221	for (; i < rif_count; i++) {
 222		struct mlxsw_sp_rif *rif = mlxsw_sp_rif_by_index(mlxsw_sp, i);
 223
 224		if (!rif || !mlxsw_sp_rif_has_dev(rif))
 225			continue;
 226		err = mlxsw_sp_erif_entry_get(mlxsw_sp, &entry, rif,
 227					      counters_enabled);
 228		if (err)
 229			goto err_entry_get;
 230		err = devlink_dpipe_entry_ctx_append(dump_ctx, &entry);
 231		if (err) {
 232			if (err == -EMSGSIZE) {
 233				if (!j)
 234					goto err_entry_append;
 235				break;
 236			}
 237			goto err_entry_append;
 238		}
 239		j++;
 240	}
 241
 242	devlink_dpipe_entry_ctx_close(dump_ctx);
 243	if (i != rif_count)
 244		goto start_again;
 245	mutex_unlock(&mlxsw_sp->router->lock);
 246
 247	devlink_dpipe_entry_clear(&entry);
 248	return 0;
 249err_entry_append:
 250err_entry_get:
 251err_ctx_prepare:
 252	mutex_unlock(&mlxsw_sp->router->lock);
 253	devlink_dpipe_entry_clear(&entry);
 254	return err;
 255}
 256
 257static int mlxsw_sp_dpipe_table_erif_counters_update(void *priv, bool enable)
 258{
 259	struct mlxsw_sp *mlxsw_sp = priv;
 260	int i;
 261
 262	mutex_lock(&mlxsw_sp->router->lock);
 263	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
 264		struct mlxsw_sp_rif *rif = mlxsw_sp_rif_by_index(mlxsw_sp, i);
 265
 266		if (!rif)
 267			continue;
 268		if (enable)
 269			mlxsw_sp_rif_counter_alloc(rif,
 270						   MLXSW_SP_RIF_COUNTER_EGRESS);
 271		else
 272			mlxsw_sp_rif_counter_free(rif,
 273						  MLXSW_SP_RIF_COUNTER_EGRESS);
 274	}
 275	mutex_unlock(&mlxsw_sp->router->lock);
 276	return 0;
 277}
 278
 279static u64 mlxsw_sp_dpipe_table_erif_size_get(void *priv)
 280{
 281	struct mlxsw_sp *mlxsw_sp = priv;
 282
 283	return MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
 284}
 285
 286static const struct devlink_dpipe_table_ops mlxsw_sp_erif_ops = {
 287	.matches_dump = mlxsw_sp_dpipe_table_erif_matches_dump,
 288	.actions_dump = mlxsw_sp_dpipe_table_erif_actions_dump,
 289	.entries_dump = mlxsw_sp_dpipe_table_erif_entries_dump,
 290	.counters_set_update = mlxsw_sp_dpipe_table_erif_counters_update,
 291	.size_get = mlxsw_sp_dpipe_table_erif_size_get,
 292};
 293
 294static int mlxsw_sp_dpipe_erif_table_init(struct mlxsw_sp *mlxsw_sp)
 295{
 296	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
 297
 298	return devl_dpipe_table_register(devlink,
 299					 MLXSW_SP_DPIPE_TABLE_NAME_ERIF,
 300					 &mlxsw_sp_erif_ops,
 301					 mlxsw_sp, false);
 302}
 303
 304static void mlxsw_sp_dpipe_erif_table_fini(struct mlxsw_sp *mlxsw_sp)
 305{
 306	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
 307
 308	devl_dpipe_table_unregister(devlink, MLXSW_SP_DPIPE_TABLE_NAME_ERIF);
 309}
 310
 311static int mlxsw_sp_dpipe_table_host_matches_dump(struct sk_buff *skb, int type)
 312{
 313	struct devlink_dpipe_match match = {0};
 314	int err;
 315
 316	match.type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
 317	match.header = &mlxsw_sp_dpipe_header_metadata;
 318	match.field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ERIF_PORT;
 319
 320	err = devlink_dpipe_match_put(skb, &match);
 321	if (err)
 322		return err;
 323
 324	switch (type) {
 325	case AF_INET:
 326		match.type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
 327		match.header = &devlink_dpipe_header_ipv4;
 328		match.field_id = DEVLINK_DPIPE_FIELD_IPV4_DST_IP;
 329		break;
 330	case AF_INET6:
 331		match.type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
 332		match.header = &devlink_dpipe_header_ipv6;
 333		match.field_id = DEVLINK_DPIPE_FIELD_IPV6_DST_IP;
 334		break;
 335	default:
 336		WARN_ON(1);
 337		return -EINVAL;
 338	}
 339
 340	return devlink_dpipe_match_put(skb, &match);
 341}
 342
 343static int
 344mlxsw_sp_dpipe_table_host4_matches_dump(void *priv, struct sk_buff *skb)
 345{
 346	return mlxsw_sp_dpipe_table_host_matches_dump(skb, AF_INET);
 347}
 348
 349static int
 350mlxsw_sp_dpipe_table_host_actions_dump(void *priv, struct sk_buff *skb)
 351{
 352	struct devlink_dpipe_action action = {0};
 353
 354	action.type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
 355	action.header = &devlink_dpipe_header_ethernet;
 356	action.field_id = DEVLINK_DPIPE_FIELD_ETHERNET_DST_MAC;
 357
 358	return devlink_dpipe_action_put(skb, &action);
 359}
 360
 361enum mlxsw_sp_dpipe_table_host_match {
 362	MLXSW_SP_DPIPE_TABLE_HOST_MATCH_RIF,
 363	MLXSW_SP_DPIPE_TABLE_HOST_MATCH_DIP,
 364	MLXSW_SP_DPIPE_TABLE_HOST_MATCH_COUNT,
 365};
 366
 367static void
 368mlxsw_sp_dpipe_table_host_match_action_prepare(struct devlink_dpipe_match *matches,
 369					       struct devlink_dpipe_action *action,
 370					       int type)
 371{
 372	struct devlink_dpipe_match *match;
 373
 374	match = &matches[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_RIF];
 375	match->type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
 376	match->header = &mlxsw_sp_dpipe_header_metadata;
 377	match->field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ERIF_PORT;
 378
 379	match = &matches[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_DIP];
 380	match->type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
 381	switch (type) {
 382	case AF_INET:
 383		match->header = &devlink_dpipe_header_ipv4;
 384		match->field_id = DEVLINK_DPIPE_FIELD_IPV4_DST_IP;
 385		break;
 386	case AF_INET6:
 387		match->header = &devlink_dpipe_header_ipv6;
 388		match->field_id = DEVLINK_DPIPE_FIELD_IPV6_DST_IP;
 389		break;
 390	default:
 391		WARN_ON(1);
 392		return;
 393	}
 394
 395	action->type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
 396	action->header = &devlink_dpipe_header_ethernet;
 397	action->field_id = DEVLINK_DPIPE_FIELD_ETHERNET_DST_MAC;
 398}
 399
 400static int
 401mlxsw_sp_dpipe_table_host_entry_prepare(struct devlink_dpipe_entry *entry,
 402					struct devlink_dpipe_value *match_values,
 403					struct devlink_dpipe_match *matches,
 404					struct devlink_dpipe_value *action_value,
 405					struct devlink_dpipe_action *action,
 406					int type)
 407{
 408	struct devlink_dpipe_value *match_value;
 409	struct devlink_dpipe_match *match;
 410
 411	entry->match_values = match_values;
 412	entry->match_values_count = MLXSW_SP_DPIPE_TABLE_HOST_MATCH_COUNT;
 413
 414	entry->action_values = action_value;
 415	entry->action_values_count = 1;
 416
 417	match = &matches[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_RIF];
 418	match_value = &match_values[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_RIF];
 419
 420	match_value->match = match;
 421	match_value->value_size = sizeof(u32);
 422	match_value->value = kmalloc(match_value->value_size, GFP_KERNEL);
 423	if (!match_value->value)
 424		return -ENOMEM;
 425
 426	match = &matches[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_DIP];
 427	match_value = &match_values[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_DIP];
 428
 429	match_value->match = match;
 430	switch (type) {
 431	case AF_INET:
 432		match_value->value_size = sizeof(u32);
 433		break;
 434	case AF_INET6:
 435		match_value->value_size = sizeof(struct in6_addr);
 436		break;
 437	default:
 438		WARN_ON(1);
 439		return -EINVAL;
 440	}
 441
 442	match_value->value = kmalloc(match_value->value_size, GFP_KERNEL);
 443	if (!match_value->value)
 444		return -ENOMEM;
 445
 446	action_value->action = action;
 447	action_value->value_size = sizeof(u64);
 448	action_value->value = kmalloc(action_value->value_size, GFP_KERNEL);
 449	if (!action_value->value)
 450		return -ENOMEM;
 451
 452	return 0;
 453}
 454
 455static void
 456__mlxsw_sp_dpipe_table_host_entry_fill(struct devlink_dpipe_entry *entry,
 457				       struct mlxsw_sp_rif *rif,
 458				       unsigned char *ha, void *dip)
 459{
 460	struct devlink_dpipe_value *value;
 461	u32 *rif_value;
 462	u8 *ha_value;
 463
 464	/* Set Match RIF index */
 465	value = &entry->match_values[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_RIF];
 466
 467	rif_value = value->value;
 468	*rif_value = mlxsw_sp_rif_index(rif);
 469	value->mapping_value = mlxsw_sp_rif_dev_ifindex(rif);
 470	value->mapping_valid = true;
 471
 472	/* Set Match DIP */
 473	value = &entry->match_values[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_DIP];
 474	memcpy(value->value, dip, value->value_size);
 475
 476	/* Set Action DMAC */
 477	value = entry->action_values;
 478	ha_value = value->value;
 479	ether_addr_copy(ha_value, ha);
 480}
 481
 482static void
 483mlxsw_sp_dpipe_table_host4_entry_fill(struct devlink_dpipe_entry *entry,
 484				      struct mlxsw_sp_neigh_entry *neigh_entry,
 485				      struct mlxsw_sp_rif *rif)
 486{
 487	unsigned char *ha;
 488	u32 dip;
 489
 490	ha = mlxsw_sp_neigh_entry_ha(neigh_entry);
 491	dip = mlxsw_sp_neigh4_entry_dip(neigh_entry);
 492	__mlxsw_sp_dpipe_table_host_entry_fill(entry, rif, ha, &dip);
 493}
 494
 495static void
 496mlxsw_sp_dpipe_table_host6_entry_fill(struct devlink_dpipe_entry *entry,
 497				      struct mlxsw_sp_neigh_entry *neigh_entry,
 498				      struct mlxsw_sp_rif *rif)
 499{
 500	struct in6_addr *dip;
 501	unsigned char *ha;
 502
 503	ha = mlxsw_sp_neigh_entry_ha(neigh_entry);
 504	dip = mlxsw_sp_neigh6_entry_dip(neigh_entry);
 505
 506	__mlxsw_sp_dpipe_table_host_entry_fill(entry, rif, ha, dip);
 507}
 508
 509static void
 510mlxsw_sp_dpipe_table_host_entry_fill(struct mlxsw_sp *mlxsw_sp,
 511				     struct devlink_dpipe_entry *entry,
 512				     struct mlxsw_sp_neigh_entry *neigh_entry,
 513				     struct mlxsw_sp_rif *rif,
 514				     int type)
 515{
 516	int err;
 517
 518	switch (type) {
 519	case AF_INET:
 520		mlxsw_sp_dpipe_table_host4_entry_fill(entry, neigh_entry, rif);
 521		break;
 522	case AF_INET6:
 523		mlxsw_sp_dpipe_table_host6_entry_fill(entry, neigh_entry, rif);
 524		break;
 525	default:
 526		WARN_ON(1);
 527		return;
 528	}
 529
 530	err = mlxsw_sp_neigh_counter_get(mlxsw_sp, neigh_entry,
 531					 &entry->counter);
 532	if (!err)
 533		entry->counter_valid = true;
 534}
 535
 536static int
 537mlxsw_sp_dpipe_table_host_entries_get(struct mlxsw_sp *mlxsw_sp,
 538				      struct devlink_dpipe_entry *entry,
 539				      bool counters_enabled,
 540				      struct devlink_dpipe_dump_ctx *dump_ctx,
 541				      int type)
 542{
 543	int rif_neigh_count = 0;
 544	int rif_neigh_skip = 0;
 545	int neigh_count = 0;
 546	int rif_count;
 547	int i, j;
 548	int err;
 549
 550	mutex_lock(&mlxsw_sp->router->lock);
 551	i = 0;
 552	rif_count = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
 553start_again:
 554	err = devlink_dpipe_entry_ctx_prepare(dump_ctx);
 555	if (err)
 556		goto err_ctx_prepare;
 557	j = 0;
 558	rif_neigh_skip = rif_neigh_count;
 559	for (; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
 560		struct mlxsw_sp_rif *rif = mlxsw_sp_rif_by_index(mlxsw_sp, i);
 561		struct mlxsw_sp_neigh_entry *neigh_entry;
 562
 563		if (!rif)
 564			continue;
 565
 566		rif_neigh_count = 0;
 567		mlxsw_sp_rif_neigh_for_each(neigh_entry, rif) {
 568			int neigh_type = mlxsw_sp_neigh_entry_type(neigh_entry);
 569
 570			if (neigh_type != type)
 571				continue;
 572
 573			if (neigh_type == AF_INET6 &&
 574			    mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
 575				continue;
 576
 577			if (rif_neigh_count < rif_neigh_skip)
 578				goto skip;
 579
 580			mlxsw_sp_dpipe_table_host_entry_fill(mlxsw_sp, entry,
 581							     neigh_entry, rif,
 582							     type);
 583			entry->index = neigh_count;
 584			err = devlink_dpipe_entry_ctx_append(dump_ctx, entry);
 585			if (err) {
 586				if (err == -EMSGSIZE) {
 587					if (!j)
 588						goto err_entry_append;
 589					else
 590						goto out;
 591				}
 592				goto err_entry_append;
 593			}
 594			neigh_count++;
 595			j++;
 596skip:
 597			rif_neigh_count++;
 598		}
 599		rif_neigh_skip = 0;
 600	}
 601out:
 602	devlink_dpipe_entry_ctx_close(dump_ctx);
 603	if (i != rif_count)
 604		goto start_again;
 605
 606	mutex_unlock(&mlxsw_sp->router->lock);
 607	return 0;
 608
 609err_ctx_prepare:
 610err_entry_append:
 611	mutex_unlock(&mlxsw_sp->router->lock);
 612	return err;
 613}
 614
 615static int
 616mlxsw_sp_dpipe_table_host_entries_dump(struct mlxsw_sp *mlxsw_sp,
 617				       bool counters_enabled,
 618				       struct devlink_dpipe_dump_ctx *dump_ctx,
 619				       int type)
 620{
 621	struct devlink_dpipe_value match_values[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_COUNT];
 622	struct devlink_dpipe_match matches[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_COUNT];
 623	struct devlink_dpipe_value action_value;
 624	struct devlink_dpipe_action action = {0};
 625	struct devlink_dpipe_entry entry = {0};
 626	int err;
 627
 628	memset(matches, 0, MLXSW_SP_DPIPE_TABLE_HOST_MATCH_COUNT *
 629			   sizeof(matches[0]));
 630	memset(match_values, 0, MLXSW_SP_DPIPE_TABLE_HOST_MATCH_COUNT *
 631				sizeof(match_values[0]));
 632	memset(&action_value, 0, sizeof(action_value));
 633
 634	mlxsw_sp_dpipe_table_host_match_action_prepare(matches, &action, type);
 635	err = mlxsw_sp_dpipe_table_host_entry_prepare(&entry, match_values,
 636						      matches, &action_value,
 637						      &action, type);
 638	if (err)
 639		goto out;
 640
 641	err = mlxsw_sp_dpipe_table_host_entries_get(mlxsw_sp, &entry,
 642						    counters_enabled, dump_ctx,
 643						    type);
 644out:
 645	devlink_dpipe_entry_clear(&entry);
 646	return err;
 647}
 648
 649static int
 650mlxsw_sp_dpipe_table_host4_entries_dump(void *priv, bool counters_enabled,
 651					struct devlink_dpipe_dump_ctx *dump_ctx)
 652{
 653	struct mlxsw_sp *mlxsw_sp = priv;
 654
 655	return mlxsw_sp_dpipe_table_host_entries_dump(mlxsw_sp,
 656						      counters_enabled,
 657						      dump_ctx, AF_INET);
 658}
 659
 660static void
 661mlxsw_sp_dpipe_table_host_counters_update(struct mlxsw_sp *mlxsw_sp,
 662					  bool enable, int type)
 663{
 664	int i;
 665
 666	mutex_lock(&mlxsw_sp->router->lock);
 667	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
 668		struct mlxsw_sp_rif *rif = mlxsw_sp_rif_by_index(mlxsw_sp, i);
 669		struct mlxsw_sp_neigh_entry *neigh_entry;
 670
 671		if (!rif)
 672			continue;
 673		mlxsw_sp_rif_neigh_for_each(neigh_entry, rif) {
 674			int neigh_type = mlxsw_sp_neigh_entry_type(neigh_entry);
 675
 676			if (neigh_type != type)
 677				continue;
 678
 679			if (neigh_type == AF_INET6 &&
 680			    mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
 681				continue;
 682
 683			mlxsw_sp_neigh_entry_counter_update(mlxsw_sp,
 684							    neigh_entry,
 685							    enable);
 686		}
 687	}
 688	mutex_unlock(&mlxsw_sp->router->lock);
 689}
 690
 691static int mlxsw_sp_dpipe_table_host4_counters_update(void *priv, bool enable)
 692{
 693	struct mlxsw_sp *mlxsw_sp = priv;
 694
 695	mlxsw_sp_dpipe_table_host_counters_update(mlxsw_sp, enable, AF_INET);
 696	return 0;
 697}
 698
 699static u64
 700mlxsw_sp_dpipe_table_host_size_get(struct mlxsw_sp *mlxsw_sp, int type)
 701{
 702	u64 size = 0;
 703	int i;
 704
 705	mutex_lock(&mlxsw_sp->router->lock);
 706	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
 707		struct mlxsw_sp_rif *rif = mlxsw_sp_rif_by_index(mlxsw_sp, i);
 708		struct mlxsw_sp_neigh_entry *neigh_entry;
 709
 710		if (!rif)
 711			continue;
 712		mlxsw_sp_rif_neigh_for_each(neigh_entry, rif) {
 713			int neigh_type = mlxsw_sp_neigh_entry_type(neigh_entry);
 714
 715			if (neigh_type != type)
 716				continue;
 717
 718			if (neigh_type == AF_INET6 &&
 719			    mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
 720				continue;
 721
 722			size++;
 723		}
 724	}
 725	mutex_unlock(&mlxsw_sp->router->lock);
 726
 727	return size;
 728}
 729
 730static u64 mlxsw_sp_dpipe_table_host4_size_get(void *priv)
 731{
 732	struct mlxsw_sp *mlxsw_sp = priv;
 733
 734	return mlxsw_sp_dpipe_table_host_size_get(mlxsw_sp, AF_INET);
 735}
 736
 737static const struct devlink_dpipe_table_ops mlxsw_sp_host4_ops = {
 738	.matches_dump = mlxsw_sp_dpipe_table_host4_matches_dump,
 739	.actions_dump = mlxsw_sp_dpipe_table_host_actions_dump,
 740	.entries_dump = mlxsw_sp_dpipe_table_host4_entries_dump,
 741	.counters_set_update = mlxsw_sp_dpipe_table_host4_counters_update,
 742	.size_get = mlxsw_sp_dpipe_table_host4_size_get,
 743};
 744
 745#define MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_HOST4 1
 746
 747static int mlxsw_sp_dpipe_host4_table_init(struct mlxsw_sp *mlxsw_sp)
 748{
 749	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
 750	int err;
 751
 752	err = devl_dpipe_table_register(devlink,
 753					MLXSW_SP_DPIPE_TABLE_NAME_HOST4,
 754					&mlxsw_sp_host4_ops,
 755					mlxsw_sp, false);
 756	if (err)
 757		return err;
 758
 759	err = devl_dpipe_table_resource_set(devlink,
 760					    MLXSW_SP_DPIPE_TABLE_NAME_HOST4,
 761					    MLXSW_SP_RESOURCE_KVD_HASH_SINGLE,
 762					    MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_HOST4);
 763	if (err)
 764		goto err_resource_set;
 765
 766	return 0;
 767
 768err_resource_set:
 769	devl_dpipe_table_unregister(devlink,
 770				    MLXSW_SP_DPIPE_TABLE_NAME_HOST4);
 771	return err;
 772}
 773
 774static void mlxsw_sp_dpipe_host4_table_fini(struct mlxsw_sp *mlxsw_sp)
 775{
 776	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
 777
 778	devl_dpipe_table_unregister(devlink,
 779				    MLXSW_SP_DPIPE_TABLE_NAME_HOST4);
 780}
 781
 782static int
 783mlxsw_sp_dpipe_table_host6_matches_dump(void *priv, struct sk_buff *skb)
 784{
 785	return mlxsw_sp_dpipe_table_host_matches_dump(skb, AF_INET6);
 786}
 787
 788static int
 789mlxsw_sp_dpipe_table_host6_entries_dump(void *priv, bool counters_enabled,
 790					struct devlink_dpipe_dump_ctx *dump_ctx)
 791{
 792	struct mlxsw_sp *mlxsw_sp = priv;
 793
 794	return mlxsw_sp_dpipe_table_host_entries_dump(mlxsw_sp,
 795						      counters_enabled,
 796						      dump_ctx, AF_INET6);
 797}
 798
 799static int mlxsw_sp_dpipe_table_host6_counters_update(void *priv, bool enable)
 800{
 801	struct mlxsw_sp *mlxsw_sp = priv;
 802
 803	mlxsw_sp_dpipe_table_host_counters_update(mlxsw_sp, enable, AF_INET6);
 804	return 0;
 805}
 806
 807static u64 mlxsw_sp_dpipe_table_host6_size_get(void *priv)
 808{
 809	struct mlxsw_sp *mlxsw_sp = priv;
 810
 811	return mlxsw_sp_dpipe_table_host_size_get(mlxsw_sp, AF_INET6);
 812}
 813
 814static const struct devlink_dpipe_table_ops mlxsw_sp_host6_ops = {
 815	.matches_dump = mlxsw_sp_dpipe_table_host6_matches_dump,
 816	.actions_dump = mlxsw_sp_dpipe_table_host_actions_dump,
 817	.entries_dump = mlxsw_sp_dpipe_table_host6_entries_dump,
 818	.counters_set_update = mlxsw_sp_dpipe_table_host6_counters_update,
 819	.size_get = mlxsw_sp_dpipe_table_host6_size_get,
 820};
 821
 822#define MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_HOST6 2
 823
 824static int mlxsw_sp_dpipe_host6_table_init(struct mlxsw_sp *mlxsw_sp)
 825{
 826	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
 827	int err;
 828
 829	err = devl_dpipe_table_register(devlink,
 830					MLXSW_SP_DPIPE_TABLE_NAME_HOST6,
 831					&mlxsw_sp_host6_ops,
 832					mlxsw_sp, false);
 833	if (err)
 834		return err;
 835
 836	err = devl_dpipe_table_resource_set(devlink,
 837					    MLXSW_SP_DPIPE_TABLE_NAME_HOST6,
 838					    MLXSW_SP_RESOURCE_KVD_HASH_DOUBLE,
 839					    MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_HOST6);
 840	if (err)
 841		goto err_resource_set;
 842
 843	return 0;
 844
 845err_resource_set:
 846	devl_dpipe_table_unregister(devlink,
 847				    MLXSW_SP_DPIPE_TABLE_NAME_HOST6);
 848	return err;
 849}
 850
 851static void mlxsw_sp_dpipe_host6_table_fini(struct mlxsw_sp *mlxsw_sp)
 852{
 853	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
 854
 855	devl_dpipe_table_unregister(devlink,
 856				    MLXSW_SP_DPIPE_TABLE_NAME_HOST6);
 857}
 858
 859static int mlxsw_sp_dpipe_table_adj_matches_dump(void *priv,
 860						 struct sk_buff *skb)
 861{
 862	struct devlink_dpipe_match match = {0};
 863	int err;
 864
 865	match.type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
 866	match.header = &mlxsw_sp_dpipe_header_metadata;
 867	match.field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_INDEX;
 868
 869	err = devlink_dpipe_match_put(skb, &match);
 870	if (err)
 871		return err;
 872
 873	match.type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
 874	match.header = &mlxsw_sp_dpipe_header_metadata;
 875	match.field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_SIZE;
 876
 877	err = devlink_dpipe_match_put(skb, &match);
 878	if (err)
 879		return err;
 880
 881	match.type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
 882	match.header = &mlxsw_sp_dpipe_header_metadata;
 883	match.field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_HASH_INDEX;
 884
 885	return devlink_dpipe_match_put(skb, &match);
 886}
 887
 888static int mlxsw_sp_dpipe_table_adj_actions_dump(void *priv,
 889						 struct sk_buff *skb)
 890{
 891	struct devlink_dpipe_action action = {0};
 892	int err;
 893
 894	action.type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
 895	action.header = &devlink_dpipe_header_ethernet;
 896	action.field_id = DEVLINK_DPIPE_FIELD_ETHERNET_DST_MAC;
 897
 898	err = devlink_dpipe_action_put(skb, &action);
 899	if (err)
 900		return err;
 901
 902	action.type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
 903	action.header = &mlxsw_sp_dpipe_header_metadata;
 904	action.field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ERIF_PORT;
 905
 906	return devlink_dpipe_action_put(skb, &action);
 907}
 908
 909static u64 mlxsw_sp_dpipe_table_adj_size(struct mlxsw_sp *mlxsw_sp)
 910{
 911	struct mlxsw_sp_nexthop *nh;
 912	u64 size = 0;
 913
 914	mlxsw_sp_nexthop_for_each(nh, mlxsw_sp->router)
 915		if (mlxsw_sp_nexthop_is_forward(nh) &&
 916		    !mlxsw_sp_nexthop_group_has_ipip(nh))
 917			size++;
 918	return size;
 919}
 920
 921enum mlxsw_sp_dpipe_table_adj_match {
 922	MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_INDEX,
 923	MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_SIZE,
 924	MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_HASH_INDEX,
 925	MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_COUNT,
 926};
 927
 928enum mlxsw_sp_dpipe_table_adj_action {
 929	MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_DST_MAC,
 930	MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_ERIF_PORT,
 931	MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_COUNT,
 932};
 933
 934static void
 935mlxsw_sp_dpipe_table_adj_match_action_prepare(struct devlink_dpipe_match *matches,
 936					      struct devlink_dpipe_action *actions)
 937{
 938	struct devlink_dpipe_action *action;
 939	struct devlink_dpipe_match *match;
 940
 941	match = &matches[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_INDEX];
 942	match->type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
 943	match->header = &mlxsw_sp_dpipe_header_metadata;
 944	match->field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_INDEX;
 945
 946	match = &matches[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_SIZE];
 947	match->type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
 948	match->header = &mlxsw_sp_dpipe_header_metadata;
 949	match->field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_SIZE;
 950
 951	match = &matches[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_HASH_INDEX];
 952	match->type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
 953	match->header = &mlxsw_sp_dpipe_header_metadata;
 954	match->field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_HASH_INDEX;
 955
 956	action = &actions[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_DST_MAC];
 957	action->type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
 958	action->header = &devlink_dpipe_header_ethernet;
 959	action->field_id = DEVLINK_DPIPE_FIELD_ETHERNET_DST_MAC;
 960
 961	action = &actions[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_ERIF_PORT];
 962	action->type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
 963	action->header = &mlxsw_sp_dpipe_header_metadata;
 964	action->field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ERIF_PORT;
 965}
 966
 967static int
 968mlxsw_sp_dpipe_table_adj_entry_prepare(struct devlink_dpipe_entry *entry,
 969				       struct devlink_dpipe_value *match_values,
 970				       struct devlink_dpipe_match *matches,
 971				       struct devlink_dpipe_value *action_values,
 972				       struct devlink_dpipe_action *actions)
 973{	struct devlink_dpipe_value *action_value;
 974	struct devlink_dpipe_value *match_value;
 975	struct devlink_dpipe_action *action;
 976	struct devlink_dpipe_match *match;
 977
 978	entry->match_values = match_values;
 979	entry->match_values_count = MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_COUNT;
 980
 981	entry->action_values = action_values;
 982	entry->action_values_count = MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_COUNT;
 983
 984	match = &matches[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_INDEX];
 985	match_value = &match_values[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_INDEX];
 986
 987	match_value->match = match;
 988	match_value->value_size = sizeof(u32);
 989	match_value->value = kmalloc(match_value->value_size, GFP_KERNEL);
 990	if (!match_value->value)
 991		return -ENOMEM;
 992
 993	match = &matches[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_SIZE];
 994	match_value = &match_values[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_SIZE];
 995
 996	match_value->match = match;
 997	match_value->value_size = sizeof(u32);
 998	match_value->value = kmalloc(match_value->value_size, GFP_KERNEL);
 999	if (!match_value->value)
1000		return -ENOMEM;
1001
1002	match = &matches[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_HASH_INDEX];
1003	match_value = &match_values[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_HASH_INDEX];
1004
1005	match_value->match = match;
1006	match_value->value_size = sizeof(u32);
1007	match_value->value = kmalloc(match_value->value_size, GFP_KERNEL);
1008	if (!match_value->value)
1009		return -ENOMEM;
1010
1011	action = &actions[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_DST_MAC];
1012	action_value = &action_values[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_DST_MAC];
1013
1014	action_value->action = action;
1015	action_value->value_size = sizeof(u64);
1016	action_value->value = kmalloc(action_value->value_size, GFP_KERNEL);
1017	if (!action_value->value)
1018		return -ENOMEM;
1019
1020	action = &actions[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_ERIF_PORT];
1021	action_value = &action_values[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_ERIF_PORT];
1022
1023	action_value->action = action;
1024	action_value->value_size = sizeof(u32);
1025	action_value->value = kmalloc(action_value->value_size, GFP_KERNEL);
1026	if (!action_value->value)
1027		return -ENOMEM;
1028
1029	return 0;
1030}
1031
1032static void
1033__mlxsw_sp_dpipe_table_adj_entry_fill(struct devlink_dpipe_entry *entry,
1034				      u32 adj_index, u32 adj_size,
1035				      u32 adj_hash_index, unsigned char *ha,
1036				      struct mlxsw_sp_rif *rif)
1037{
1038	struct devlink_dpipe_value *value;
1039	u32 *p_rif_value;
1040	u32 *p_index;
1041
1042	value = &entry->match_values[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_INDEX];
1043	p_index = value->value;
1044	*p_index = adj_index;
1045
1046	value = &entry->match_values[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_SIZE];
1047	p_index = value->value;
1048	*p_index = adj_size;
1049
1050	value = &entry->match_values[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_HASH_INDEX];
1051	p_index = value->value;
1052	*p_index = adj_hash_index;
1053
1054	value = &entry->action_values[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_DST_MAC];
1055	ether_addr_copy(value->value, ha);
1056
1057	value = &entry->action_values[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_ERIF_PORT];
1058	p_rif_value = value->value;
1059	*p_rif_value = mlxsw_sp_rif_index(rif);
1060	value->mapping_value = mlxsw_sp_rif_dev_ifindex(rif);
1061	value->mapping_valid = true;
1062}
1063
1064static void mlxsw_sp_dpipe_table_adj_entry_fill(struct mlxsw_sp *mlxsw_sp,
1065						struct mlxsw_sp_nexthop *nh,
1066						struct devlink_dpipe_entry *entry)
1067{
1068	struct mlxsw_sp_rif *rif = mlxsw_sp_nexthop_rif(nh);
1069	unsigned char *ha = mlxsw_sp_nexthop_ha(nh);
1070	u32 adj_hash_index = 0;
1071	u32 adj_index = 0;
1072	u32 adj_size = 0;
1073	int err;
1074
1075	mlxsw_sp_nexthop_indexes(nh, &adj_index, &adj_size, &adj_hash_index);
1076	__mlxsw_sp_dpipe_table_adj_entry_fill(entry, adj_index, adj_size,
1077					      adj_hash_index, ha, rif);
1078	err = mlxsw_sp_nexthop_counter_get(mlxsw_sp, nh, &entry->counter);
1079	if (!err)
1080		entry->counter_valid = true;
1081}
1082
1083static int
1084mlxsw_sp_dpipe_table_adj_entries_get(struct mlxsw_sp *mlxsw_sp,
1085				     struct devlink_dpipe_entry *entry,
1086				     bool counters_enabled,
1087				     struct devlink_dpipe_dump_ctx *dump_ctx)
1088{
1089	struct mlxsw_sp_nexthop *nh;
1090	int entry_index = 0;
1091	int nh_count_max;
1092	int nh_count = 0;
1093	int nh_skip;
1094	int j;
1095	int err;
1096
1097	mutex_lock(&mlxsw_sp->router->lock);
1098	nh_count_max = mlxsw_sp_dpipe_table_adj_size(mlxsw_sp);
1099start_again:
1100	err = devlink_dpipe_entry_ctx_prepare(dump_ctx);
1101	if (err)
1102		goto err_ctx_prepare;
1103	j = 0;
1104	nh_skip = nh_count;
1105	nh_count = 0;
1106	mlxsw_sp_nexthop_for_each(nh, mlxsw_sp->router) {
1107		if (!mlxsw_sp_nexthop_is_forward(nh) ||
1108		    mlxsw_sp_nexthop_group_has_ipip(nh))
1109			continue;
1110
1111		if (nh_count < nh_skip)
1112			goto skip;
1113
1114		mlxsw_sp_dpipe_table_adj_entry_fill(mlxsw_sp, nh, entry);
1115		entry->index = entry_index;
1116		err = devlink_dpipe_entry_ctx_append(dump_ctx, entry);
1117		if (err) {
1118			if (err == -EMSGSIZE) {
1119				if (!j)
1120					goto err_entry_append;
1121				break;
1122			}
1123			goto err_entry_append;
1124		}
1125		entry_index++;
1126		j++;
1127skip:
1128		nh_count++;
1129	}
1130
1131	devlink_dpipe_entry_ctx_close(dump_ctx);
1132	if (nh_count != nh_count_max)
1133		goto start_again;
1134	mutex_unlock(&mlxsw_sp->router->lock);
1135
1136	return 0;
1137
1138err_ctx_prepare:
1139err_entry_append:
1140	mutex_unlock(&mlxsw_sp->router->lock);
1141	return err;
1142}
1143
1144static int
1145mlxsw_sp_dpipe_table_adj_entries_dump(void *priv, bool counters_enabled,
1146				      struct devlink_dpipe_dump_ctx *dump_ctx)
1147{
1148	struct devlink_dpipe_value action_values[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_COUNT];
1149	struct devlink_dpipe_value match_values[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_COUNT];
1150	struct devlink_dpipe_action actions[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_COUNT];
1151	struct devlink_dpipe_match matches[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_COUNT];
1152	struct devlink_dpipe_entry entry = {0};
1153	struct mlxsw_sp *mlxsw_sp = priv;
1154	int err;
1155
1156	memset(matches, 0, MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_COUNT *
1157			   sizeof(matches[0]));
1158	memset(match_values, 0, MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_COUNT *
1159				sizeof(match_values[0]));
1160	memset(actions, 0, MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_COUNT *
1161			   sizeof(actions[0]));
1162	memset(action_values, 0, MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_COUNT *
1163				 sizeof(action_values[0]));
1164
1165	mlxsw_sp_dpipe_table_adj_match_action_prepare(matches, actions);
1166	err = mlxsw_sp_dpipe_table_adj_entry_prepare(&entry,
1167						     match_values, matches,
1168						     action_values, actions);
1169	if (err)
1170		goto out;
1171
1172	err = mlxsw_sp_dpipe_table_adj_entries_get(mlxsw_sp, &entry,
1173						   counters_enabled, dump_ctx);
1174out:
1175	devlink_dpipe_entry_clear(&entry);
1176	return err;
1177}
1178
1179static int mlxsw_sp_dpipe_table_adj_counters_update(void *priv, bool enable)
1180{
1181	char ratr_pl[MLXSW_REG_RATR_LEN];
1182	struct mlxsw_sp *mlxsw_sp = priv;
1183	struct mlxsw_sp_nexthop *nh;
1184	unsigned int n_done = 0;
1185	u32 adj_hash_index = 0;
1186	u32 adj_index = 0;
1187	u32 adj_size = 0;
1188	int err;
1189
1190	mlxsw_sp_nexthop_for_each(nh, mlxsw_sp->router) {
1191		if (!mlxsw_sp_nexthop_is_forward(nh) ||
1192		    mlxsw_sp_nexthop_group_has_ipip(nh))
1193			continue;
1194
1195		mlxsw_sp_nexthop_indexes(nh, &adj_index, &adj_size,
1196					 &adj_hash_index);
1197		if (enable) {
1198			err = mlxsw_sp_nexthop_counter_enable(mlxsw_sp, nh);
1199			if (err)
1200				goto err_counter_enable;
1201		} else {
1202			mlxsw_sp_nexthop_counter_disable(mlxsw_sp, nh);
1203		}
1204		mlxsw_sp_nexthop_eth_update(mlxsw_sp,
1205					    adj_index + adj_hash_index, nh,
1206					    true, ratr_pl);
1207		n_done++;
1208	}
1209	return 0;
1210
1211err_counter_enable:
1212	mlxsw_sp_nexthop_for_each(nh, mlxsw_sp->router) {
1213		if (!n_done--)
1214			break;
1215		mlxsw_sp_nexthop_counter_disable(mlxsw_sp, nh);
1216	}
1217	return err;
1218}
1219
1220static u64
1221mlxsw_sp_dpipe_table_adj_size_get(void *priv)
1222{
1223	struct mlxsw_sp *mlxsw_sp = priv;
1224	u64 size;
1225
1226	mutex_lock(&mlxsw_sp->router->lock);
1227	size = mlxsw_sp_dpipe_table_adj_size(mlxsw_sp);
1228	mutex_unlock(&mlxsw_sp->router->lock);
1229
1230	return size;
1231}
1232
1233static const struct devlink_dpipe_table_ops mlxsw_sp_dpipe_table_adj_ops = {
1234	.matches_dump = mlxsw_sp_dpipe_table_adj_matches_dump,
1235	.actions_dump = mlxsw_sp_dpipe_table_adj_actions_dump,
1236	.entries_dump = mlxsw_sp_dpipe_table_adj_entries_dump,
1237	.counters_set_update = mlxsw_sp_dpipe_table_adj_counters_update,
1238	.size_get = mlxsw_sp_dpipe_table_adj_size_get,
1239};
1240
1241#define MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_ADJ 1
1242
1243static int mlxsw_sp_dpipe_adj_table_init(struct mlxsw_sp *mlxsw_sp)
1244{
1245	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
1246	int err;
1247
1248	err = devl_dpipe_table_register(devlink,
1249					MLXSW_SP_DPIPE_TABLE_NAME_ADJ,
1250					&mlxsw_sp_dpipe_table_adj_ops,
1251					mlxsw_sp, false);
1252	if (err)
1253		return err;
1254
1255	err = devl_dpipe_table_resource_set(devlink,
1256					    MLXSW_SP_DPIPE_TABLE_NAME_ADJ,
1257					    MLXSW_SP_RESOURCE_KVD_LINEAR,
1258					    MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_ADJ);
1259	if (err)
1260		goto err_resource_set;
1261
1262	return 0;
1263
1264err_resource_set:
1265	devl_dpipe_table_unregister(devlink,
1266				    MLXSW_SP_DPIPE_TABLE_NAME_ADJ);
1267	return err;
1268}
1269
1270static void mlxsw_sp_dpipe_adj_table_fini(struct mlxsw_sp *mlxsw_sp)
1271{
1272	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
1273
1274	devl_dpipe_table_unregister(devlink,
1275				    MLXSW_SP_DPIPE_TABLE_NAME_ADJ);
1276}
1277
1278int mlxsw_sp_dpipe_init(struct mlxsw_sp *mlxsw_sp)
1279{
1280	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
1281	int err;
1282
1283	devl_dpipe_headers_register(devlink, &mlxsw_sp_dpipe_headers);
1284
1285	err = mlxsw_sp_dpipe_erif_table_init(mlxsw_sp);
1286	if (err)
1287		goto err_erif_table_init;
1288
1289	err = mlxsw_sp_dpipe_host4_table_init(mlxsw_sp);
1290	if (err)
1291		goto err_host4_table_init;
1292
1293	err = mlxsw_sp_dpipe_host6_table_init(mlxsw_sp);
1294	if (err)
1295		goto err_host6_table_init;
1296
1297	err = mlxsw_sp_dpipe_adj_table_init(mlxsw_sp);
1298	if (err)
1299		goto err_adj_table_init;
1300
1301	return 0;
1302err_adj_table_init:
1303	mlxsw_sp_dpipe_host6_table_fini(mlxsw_sp);
1304err_host6_table_init:
1305	mlxsw_sp_dpipe_host4_table_fini(mlxsw_sp);
1306err_host4_table_init:
1307	mlxsw_sp_dpipe_erif_table_fini(mlxsw_sp);
1308err_erif_table_init:
1309	devl_dpipe_headers_unregister(priv_to_devlink(mlxsw_sp->core));
1310	return err;
1311}
1312
1313void mlxsw_sp_dpipe_fini(struct mlxsw_sp *mlxsw_sp)
1314{
1315	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
1316
1317	mlxsw_sp_dpipe_adj_table_fini(mlxsw_sp);
1318	mlxsw_sp_dpipe_host6_table_fini(mlxsw_sp);
1319	mlxsw_sp_dpipe_host4_table_fini(mlxsw_sp);
1320	mlxsw_sp_dpipe_erif_table_fini(mlxsw_sp);
1321	devl_dpipe_headers_unregister(devlink);
1322}