nfp_target.c

  1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
  2/* Copyright (C) 2015-2018 Netronome Systems, Inc. */
  3
  4/*
  5 * nfp_target.c
  6 * CPP Access Width Decoder
  7 * Authors: Jakub Kicinski <jakub.kicinski@netronome.com>
  8 *          Jason McMullan <jason.mcmullan@netronome.com>
  9 *          Francois H. Theron <francois.theron@netronome.com>
 10 */
 11
 12#define pr_fmt(fmt)       "NFP target: " fmt
 13
 14#include <linux/bitops.h>
 15#include <linux/kernel.h>
 16#include <linux/printk.h>
 17
 18#include "nfp_cpp.h"
 19
 20#include "nfp6000/nfp6000.h"
 21
 22#define P32 1
 23#define P64 2
 24
 25/* This structure ONLY includes items that can be done with a read or write of
 26 * 32-bit or 64-bit words. All others are not listed.
 27 */
 28
 29#define AT(_action, _token, _pull, _push)				\
 30	case NFP_CPP_ID(0, (_action), (_token)):			\
 31		return PUSHPULL((_pull), (_push))
 32
 33static int target_rw(u32 cpp_id, int pp, int start, int len)
 34{
 35	switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
 36	AT(0, 0,  0, pp);
 37	AT(1, 0, pp,  0);
 38	AT(NFP_CPP_ACTION_RW, 0, pp, pp);
 39	default:
 40		return -EINVAL;
 41	}
 42}
 43
 44static int nfp6000_nbi_dma(u32 cpp_id)
 45{
 46	switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
 47	AT(0, 0,   0, P64);	/* ReadNbiDma */
 48	AT(1, 0,   P64, 0);	/* WriteNbiDma */
 49	AT(NFP_CPP_ACTION_RW, 0, P64, P64);
 50	default:
 51		return -EINVAL;
 52	}
 53}
 54
 55static int nfp6000_nbi_stats(u32 cpp_id)
 56{
 57	switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
 58	AT(0, 0,   0, P32);	/* ReadNbiStats */
 59	AT(1, 0,   P32, 0);	/* WriteNbiStats */
 60	AT(NFP_CPP_ACTION_RW, 0, P32, P32);
 61	default:
 62		return -EINVAL;
 63	}
 64}
 65
 66static int nfp6000_nbi_tm(u32 cpp_id)
 67{
 68	switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
 69	AT(0, 0,   0, P64);	/* ReadNbiTM */
 70	AT(1, 0,   P64, 0);	/* WriteNbiTM */
 71	AT(NFP_CPP_ACTION_RW, 0, P64, P64);
 72	default:
 73		return -EINVAL;
 74	}
 75}
 76
 77static int nfp6000_nbi_ppc(u32 cpp_id)
 78{
 79	switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
 80	AT(0, 0,   0, P64);	/* ReadNbiPreclassifier */
 81	AT(1, 0,   P64, 0);	/* WriteNbiPreclassifier */
 82	AT(NFP_CPP_ACTION_RW, 0, P64, P64);
 83	default:
 84		return -EINVAL;
 85	}
 86}
 87
 88static int nfp6000_nbi(u32 cpp_id, u64 address)
 89{
 90	u64 rel_addr = address & 0x3fFFFF;
 91
 92	if (rel_addr < (1 << 20))
 93		return nfp6000_nbi_dma(cpp_id);
 94	if (rel_addr < (2 << 20))
 95		return nfp6000_nbi_stats(cpp_id);
 96	if (rel_addr < (3 << 20))
 97		return nfp6000_nbi_tm(cpp_id);
 98	return nfp6000_nbi_ppc(cpp_id);
 99}
100
101/* This structure ONLY includes items that can be done with a read or write of
102 * 32-bit or 64-bit words. All others are not listed.
103 */
104static int nfp6000_mu_common(u32 cpp_id)
105{
106	switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
107	AT(NFP_CPP_ACTION_RW, 0, P64, P64);	/* read_be/write_be */
108	AT(NFP_CPP_ACTION_RW, 1, P64, P64);	/* read_le/write_le */
109	AT(NFP_CPP_ACTION_RW, 2, P64, P64);	/* read_swap_be/write_swap_be */
110	AT(NFP_CPP_ACTION_RW, 3, P64, P64);	/* read_swap_le/write_swap_le */
111	AT(0, 0,   0, P64);	/* read_be */
112	AT(0, 1,   0, P64);	/* read_le */
113	AT(0, 2,   0, P64);	/* read_swap_be */
114	AT(0, 3,   0, P64);	/* read_swap_le */
115	AT(1, 0, P64,   0);	/* write_be */
116	AT(1, 1, P64,   0);	/* write_le */
117	AT(1, 2, P64,   0);	/* write_swap_be */
118	AT(1, 3, P64,   0);	/* write_swap_le */
119	AT(3, 0,   0, P32);	/* atomic_read */
120	AT(3, 2, P32,   0);	/* mask_compare_write */
121	AT(4, 0, P32,   0);	/* atomic_write */
122	AT(4, 2,   0,   0);	/* atomic_write_imm */
123	AT(4, 3,   0, P32);	/* swap_imm */
124	AT(5, 0, P32,   0);	/* set */
125	AT(5, 3,   0, P32);	/* test_set_imm */
126	AT(6, 0, P32,   0);	/* clr */
127	AT(6, 3,   0, P32);	/* test_clr_imm */
128	AT(7, 0, P32,   0);	/* add */
129	AT(7, 3,   0, P32);	/* test_add_imm */
130	AT(8, 0, P32,   0);	/* addsat */
131	AT(8, 3,   0, P32);	/* test_subsat_imm */
132	AT(9, 0, P32,   0);	/* sub */
133	AT(9, 3,   0, P32);	/* test_sub_imm */
134	AT(10, 0, P32,   0);	/* subsat */
135	AT(10, 3,   0, P32);	/* test_subsat_imm */
136	AT(13, 0,   0, P32);	/* microq128_get */
137	AT(13, 1,   0, P32);	/* microq128_pop */
138	AT(13, 2, P32,   0);	/* microq128_put */
139	AT(15, 0, P32,   0);	/* xor */
140	AT(15, 3,   0, P32);	/* test_xor_imm */
141	AT(28, 0,   0, P32);	/* read32_be */
142	AT(28, 1,   0, P32);	/* read32_le */
143	AT(28, 2,   0, P32);	/* read32_swap_be */
144	AT(28, 3,   0, P32);	/* read32_swap_le */
145	AT(31, 0, P32,   0);	/* write32_be */
146	AT(31, 1, P32,   0);	/* write32_le */
147	AT(31, 2, P32,   0);	/* write32_swap_be */
148	AT(31, 3, P32,   0);	/* write32_swap_le */
149	default:
150		return -EINVAL;
151	}
152}
153
154static int nfp6000_mu_ctm(u32 cpp_id)
155{
156	switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
157	AT(16, 1,   0, P32);	/* packet_read_packet_status */
158	AT(17, 1,   0, P32);	/* packet_credit_get */
159	AT(17, 3,   0, P64);	/* packet_add_thread */
160	AT(18, 2,   0, P64);	/* packet_free_and_return_pointer */
161	AT(18, 3,   0, P64);	/* packet_return_pointer */
162	AT(21, 0,   0, P64);	/* pe_dma_to_memory_indirect */
163	AT(21, 1,   0, P64);	/* pe_dma_to_memory_indirect_swap */
164	AT(21, 2,   0, P64);	/* pe_dma_to_memory_indirect_free */
165	AT(21, 3,   0, P64);	/* pe_dma_to_memory_indirect_free_swap */
166	default:
167		return nfp6000_mu_common(cpp_id);
168	}
169}
170
171static int nfp6000_mu_emu(u32 cpp_id)
172{
173	switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
174	AT(18, 0,   0, P32);	/* read_queue */
175	AT(18, 1,   0, P32);	/* read_queue_ring */
176	AT(18, 2, P32,   0);	/* write_queue */
177	AT(18, 3, P32,   0);	/* write_queue_ring */
178	AT(20, 2, P32,   0);	/* journal */
179	AT(21, 0,   0, P32);	/* get */
180	AT(21, 1,   0, P32);	/* get_eop */
181	AT(21, 2,   0, P32);	/* get_freely */
182	AT(22, 0,   0, P32);	/* pop */
183	AT(22, 1,   0, P32);	/* pop_eop */
184	AT(22, 2,   0, P32);	/* pop_freely */
185	default:
186		return nfp6000_mu_common(cpp_id);
187	}
188}
189
190static int nfp6000_mu_imu(u32 cpp_id)
191{
192	return nfp6000_mu_common(cpp_id);
193}
194
195static int nfp6000_mu(u32 cpp_id, u64 address)
196{
197	int pp;
198
199	if (address < 0x2000000000ULL)
200		pp = nfp6000_mu_ctm(cpp_id);
201	else if (address < 0x8000000000ULL)
202		pp = nfp6000_mu_emu(cpp_id);
203	else if (address < 0x9800000000ULL)
204		pp = nfp6000_mu_ctm(cpp_id);
205	else if (address < 0x9C00000000ULL)
206		pp = nfp6000_mu_emu(cpp_id);
207	else if (address < 0xA000000000ULL)
208		pp = nfp6000_mu_imu(cpp_id);
209	else
210		pp = nfp6000_mu_ctm(cpp_id);
211
212	return pp;
213}
214
215static int nfp6000_ila(u32 cpp_id)
216{
217	switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
218	AT(0, 1,   0, P32);	/* read_check_error */
219	AT(2, 0,   0, P32);	/* read_int */
220	AT(3, 0, P32,   0);	/* write_int */
221	default:
222		return target_rw(cpp_id, P32, 48, 4);
223	}
224}
225
226static int nfp6000_pci(u32 cpp_id)
227{
228	switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
229	AT(2, 0,   0, P32);
230	AT(3, 0, P32,   0);
231	default:
232		return target_rw(cpp_id, P32, 4, 4);
233	}
234}
235
236static int nfp6000_crypto(u32 cpp_id)
237{
238	switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
239	AT(2, 0, P64,   0);
240	default:
241		return target_rw(cpp_id, P64, 12, 4);
242	}
243}
244
245static int nfp6000_cap_xpb(u32 cpp_id)
246{
247	switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
248	AT(0, 1,   0, P32); /* RingGet */
249	AT(0, 2, P32,   0); /* Interthread Signal */
250	AT(1, 1, P32,   0); /* RingPut */
251	AT(1, 2, P32,   0); /* CTNNWr */
252	AT(2, 0,   0, P32); /* ReflectRd, signal none */
253	AT(2, 1,   0, P32); /* ReflectRd, signal self */
254	AT(2, 2,   0, P32); /* ReflectRd, signal remote */
255	AT(2, 3,   0, P32); /* ReflectRd, signal both */
256	AT(3, 0, P32,   0); /* ReflectWr, signal none */
257	AT(3, 1, P32,   0); /* ReflectWr, signal self */
258	AT(3, 2, P32,   0); /* ReflectWr, signal remote */
259	AT(3, 3, P32,   0); /* ReflectWr, signal both */
260	AT(NFP_CPP_ACTION_RW, 1, P32, P32);
261	default:
262		return target_rw(cpp_id, P32, 1, 63);
263	}
264}
265
266static int nfp6000_cls(u32 cpp_id)
267{
268	switch (cpp_id & NFP_CPP_ID(0, ~0, ~0)) {
269	AT(0, 3, P32,  0); /* xor */
270	AT(2, 0, P32,  0); /* set */
271	AT(2, 1, P32,  0); /* clr */
272	AT(4, 0, P32,  0); /* add */
273	AT(4, 1, P32,  0); /* add64 */
274	AT(6, 0, P32,  0); /* sub */
275	AT(6, 1, P32,  0); /* sub64 */
276	AT(6, 2, P32,  0); /* subsat */
277	AT(8, 2, P32,  0); /* hash_mask */
278	AT(8, 3, P32,  0); /* hash_clear */
279	AT(9, 0,  0, P32); /* ring_get */
280	AT(9, 1,  0, P32); /* ring_pop */
281	AT(9, 2,  0, P32); /* ring_get_freely */
282	AT(9, 3,  0, P32); /* ring_pop_freely */
283	AT(10, 0, P32,  0); /* ring_put */
284	AT(10, 2, P32,  0); /* ring_journal */
285	AT(14, 0,  P32, 0); /* reflect_write_sig_local */
286	AT(15, 1,  0, P32); /* reflect_read_sig_local */
287	AT(17, 2, P32,  0); /* statisic */
288	AT(24, 0,  0, P32); /* ring_read */
289	AT(24, 1, P32,  0); /* ring_write */
290	AT(25, 0,  0, P32); /* ring_workq_add_thread */
291	AT(25, 1, P32,  0); /* ring_workq_add_work */
292	default:
293		return target_rw(cpp_id, P32, 0, 64);
294	}
295}
296
297int nfp_target_pushpull(u32 cpp_id, u64 address)
298{
299	switch (NFP_CPP_ID_TARGET_of(cpp_id)) {
300	case NFP_CPP_TARGET_NBI:
301		return nfp6000_nbi(cpp_id, address);
302	case NFP_CPP_TARGET_QDR:
303		return target_rw(cpp_id, P32, 24, 4);
304	case NFP_CPP_TARGET_ILA:
305		return nfp6000_ila(cpp_id);
306	case NFP_CPP_TARGET_MU:
307		return nfp6000_mu(cpp_id, address);
308	case NFP_CPP_TARGET_PCIE:
309		return nfp6000_pci(cpp_id);
310	case NFP_CPP_TARGET_ARM:
311		if (address < 0x10000)
312			return target_rw(cpp_id, P64, 1, 1);
313		else
314			return target_rw(cpp_id, P32, 1, 1);
315	case NFP_CPP_TARGET_CRYPTO:
316		return nfp6000_crypto(cpp_id);
317	case NFP_CPP_TARGET_CT_XPB:
318		return nfp6000_cap_xpb(cpp_id);
319	case NFP_CPP_TARGET_CLS:
320		return nfp6000_cls(cpp_id);
321	case 0:
322		return target_rw(cpp_id, P32, 4, 4);
323	default:
324		return -EINVAL;
325	}
326}
327
328#undef AT
329#undef P32
330#undef P64
331
332/* All magic NFP-6xxx IMB 'mode' numbers here are from:
333 * Databook (1 August 2013)
334 * - System Overview and Connectivity
335 * -- Internal Connectivity
336 * --- Distributed Switch Fabric - Command Push/Pull (DSF-CPP) Bus
337 * ---- CPP addressing
338 * ----- Table 3.6. CPP Address Translation Mode Commands
339 */
340
341#define _NIC_NFP6000_MU_LOCALITY_DIRECT     2
342
343static int nfp_decode_basic(u64 addr, int *dest_island, int cpp_tgt,
344			    int mode, bool addr40, int isld1, int isld0)
345{
346	int iid_lsb, idx_lsb;
347
348	/* This function doesn't handle MU or CTXBP */
349	if (cpp_tgt == NFP_CPP_TARGET_MU || cpp_tgt == NFP_CPP_TARGET_CT_XPB)
350		return -EINVAL;
351
352	switch (mode) {
353	case 0:
354		/* For VQDR, in this mode for 32-bit addressing
355		 * it would be islands 0, 16, 32 and 48 depending on channel
356		 * and upper address bits.
357		 * Since those are not all valid islands, most decode
358		 * cases would result in bad island IDs, but we do them
359		 * anyway since this is decoding an address that is already
360		 * assumed to be used as-is to get to sram.
361		 */
362		iid_lsb = addr40 ? 34 : 26;
363		*dest_island = (addr >> iid_lsb) & 0x3F;
364		return 0;
365	case 1:
366		/* For VQDR 32-bit, this would decode as:
367		 * Channel 0: island#0
368		 * Channel 1: island#0
369		 * Channel 2: island#1
370		 * Channel 3: island#1
371		 * That would be valid as long as both islands
372		 * have VQDR. Let's allow this.
373		 */
374		idx_lsb = addr40 ? 39 : 31;
375		if (addr & BIT_ULL(idx_lsb))
376			*dest_island = isld1;
377		else
378			*dest_island = isld0;
379
380		return 0;
381	case 2:
382		/* For VQDR 32-bit:
383		 * Channel 0: (island#0 | 0)
384		 * Channel 1: (island#0 | 1)
385		 * Channel 2: (island#1 | 0)
386		 * Channel 3: (island#1 | 1)
387		 *
388		 * Make sure we compare against isldN values
389		 * by clearing the LSB.
390		 * This is what the silicon does.
391		 */
392		isld0 &= ~1;
393		isld1 &= ~1;
394
395		idx_lsb = addr40 ? 39 : 31;
396		iid_lsb = idx_lsb - 1;
397
398		if (addr & BIT_ULL(idx_lsb))
399			*dest_island = isld1 | (int)((addr >> iid_lsb) & 1);
400		else
401			*dest_island = isld0 | (int)((addr >> iid_lsb) & 1);
402
403		return 0;
404	case 3:
405		/* In this mode the data address starts to affect the island ID
406		 * so rather not allow it. In some really specific case
407		 * one could use this to send the upper half of the
408		 * VQDR channel to another MU, but this is getting very
409		 * specific.
410		 * However, as above for mode 0, this is the decoder
411		 * and the caller should validate the resulting IID.
412		 * This blindly does what the silicon would do.
413		 */
414		isld0 &= ~3;
415		isld1 &= ~3;
416
417		idx_lsb = addr40 ? 39 : 31;
418		iid_lsb = idx_lsb - 2;
419
420		if (addr & BIT_ULL(idx_lsb))
421			*dest_island = isld1 | (int)((addr >> iid_lsb) & 3);
422		else
423			*dest_island = isld0 | (int)((addr >> iid_lsb) & 3);
424
425		return 0;
426	default:
427		return -EINVAL;
428	}
429}
430
431static int nfp_encode_basic_qdr(u64 addr, int dest_island, int cpp_tgt,
432				int mode, bool addr40, int isld1, int isld0)
433{
434	int v, ret;
435
436	/* Full Island ID and channel bits overlap? */
437	ret = nfp_decode_basic(addr, &v, cpp_tgt, mode, addr40, isld1, isld0);
438	if (ret)
439		return ret;
440
441	/* The current address won't go where expected? */
442	if (dest_island != -1 && dest_island != v)
443		return -EINVAL;
444
445	/* If dest_island was -1, we don't care where it goes. */
446	return 0;
447}
448
449/* Try each option, take first one that fits.
450 * Not sure if we would want to do some smarter
451 * searching and prefer 0 or non-0 island IDs.
452 */
453static int nfp_encode_basic_search(u64 *addr, int dest_island, int *isld,
454				   int iid_lsb, int idx_lsb, int v_max)
455{
456	int i, v;
457
458	for (i = 0; i < 2; i++)
459		for (v = 0; v < v_max; v++) {
460			if (dest_island != (isld[i] | v))
461				continue;
462
463			*addr &= ~GENMASK_ULL(idx_lsb, iid_lsb);
464			*addr |= ((u64)i << idx_lsb);
465			*addr |= ((u64)v << iid_lsb);
466			return 0;
467		}
468
469	return -ENODEV;
470}
471
472/* For VQDR, we may not modify the Channel bits, which might overlap
473 *  with the Index bit. When it does, we need to ensure that isld0 == isld1.
474 */
475static int nfp_encode_basic(u64 *addr, int dest_island, int cpp_tgt,
476			    int mode, bool addr40, int isld1, int isld0)
477{
478	int iid_lsb, idx_lsb;
479	int isld[2];
480	u64 v64;
481
482	isld[0] = isld0;
483	isld[1] = isld1;
484
485	/* This function doesn't handle MU or CTXBP */
486	if (cpp_tgt == NFP_CPP_TARGET_MU || cpp_tgt == NFP_CPP_TARGET_CT_XPB)
487		return -EINVAL;
488
489	switch (mode) {
490	case 0:
491		if (cpp_tgt == NFP_CPP_TARGET_QDR && !addr40)
492			/* In this specific mode we'd rather not modify
493			 * the address but we can verify if the existing
494			 * contents will point to a valid island.
495			 */
496			return nfp_encode_basic_qdr(*addr, cpp_tgt, dest_island,
497						    mode, addr40, isld1, isld0);
498
499		iid_lsb = addr40 ? 34 : 26;
500		/* <39:34> or <31:26> */
501		v64 = GENMASK_ULL(iid_lsb + 5, iid_lsb);
502		*addr &= ~v64;
503		*addr |= ((u64)dest_island << iid_lsb) & v64;
504		return 0;
505	case 1:
506		if (cpp_tgt == NFP_CPP_TARGET_QDR && !addr40)
507			return nfp_encode_basic_qdr(*addr, cpp_tgt, dest_island,
508						    mode, addr40, isld1, isld0);
509
510		idx_lsb = addr40 ? 39 : 31;
511		if (dest_island == isld0) {
512			/* Only need to clear the Index bit */
513			*addr &= ~BIT_ULL(idx_lsb);
514			return 0;
515		}
516
517		if (dest_island == isld1) {
518			/* Only need to set the Index bit */
519			*addr |= BIT_ULL(idx_lsb);
520			return 0;
521		}
522
523		return -ENODEV;
524	case 2:
525		/* iid<0> = addr<30> = channel<0>
526		 * channel<1> = addr<31> = Index
527		 */
528		if (cpp_tgt == NFP_CPP_TARGET_QDR && !addr40)
529			/* Special case where we allow channel bits to
530			 * be set before hand and with them select an island.
531			 * So we need to confirm that it's at least plausible.
532			 */
533			return nfp_encode_basic_qdr(*addr, cpp_tgt, dest_island,
534						    mode, addr40, isld1, isld0);
535
536		/* Make sure we compare against isldN values
537		 * by clearing the LSB.
538		 * This is what the silicon does.
539		 */
540		isld[0] &= ~1;
541		isld[1] &= ~1;
542
543		idx_lsb = addr40 ? 39 : 31;
544		iid_lsb = idx_lsb - 1;
545
546		return nfp_encode_basic_search(addr, dest_island, isld,
547					       iid_lsb, idx_lsb, 2);
548	case 3:
549		if (cpp_tgt == NFP_CPP_TARGET_QDR && !addr40)
550			/* iid<0> = addr<29> = data
551			 * iid<1> = addr<30> = channel<0>
552			 * channel<1> = addr<31> = Index
553			 */
554			return nfp_encode_basic_qdr(*addr, cpp_tgt, dest_island,
555						    mode, addr40, isld1, isld0);
556
557		isld[0] &= ~3;
558		isld[1] &= ~3;
559
560		idx_lsb = addr40 ? 39 : 31;
561		iid_lsb = idx_lsb - 2;
562
563		return nfp_encode_basic_search(addr, dest_island, isld,
564					       iid_lsb, idx_lsb, 4);
565	default:
566		return -EINVAL;
567	}
568}
569
570static int nfp_encode_mu(u64 *addr, int dest_island, int mode,
571			 bool addr40, int isld1, int isld0)
572{
573	int iid_lsb, idx_lsb, locality_lsb;
574	int isld[2];
575	u64 v64;
576	int da;
577
578	isld[0] = isld0;
579	isld[1] = isld1;
580	locality_lsb = nfp_cppat_mu_locality_lsb(mode, addr40);
581
582	if (((*addr >> locality_lsb) & 3) == _NIC_NFP6000_MU_LOCALITY_DIRECT)
583		da = 1;
584	else
585		da = 0;
586
587	switch (mode) {
588	case 0:
589		iid_lsb = addr40 ? 32 : 24;
590		v64 = GENMASK_ULL(iid_lsb + 5, iid_lsb);
591		*addr &= ~v64;
592		*addr |= (((u64)dest_island) << iid_lsb) & v64;
593		return 0;
594	case 1:
595		if (da) {
596			iid_lsb = addr40 ? 32 : 24;
597			v64 = GENMASK_ULL(iid_lsb + 5, iid_lsb);
598			*addr &= ~v64;
599			*addr |= (((u64)dest_island) << iid_lsb) & v64;
600			return 0;
601		}
602
603		idx_lsb = addr40 ? 37 : 29;
604		if (dest_island == isld0) {
605			*addr &= ~BIT_ULL(idx_lsb);
606			return 0;
607		}
608
609		if (dest_island == isld1) {
610			*addr |= BIT_ULL(idx_lsb);
611			return 0;
612		}
613
614		return -ENODEV;
615	case 2:
616		if (da) {
617			iid_lsb = addr40 ? 32 : 24;
618			v64 = GENMASK_ULL(iid_lsb + 5, iid_lsb);
619			*addr &= ~v64;
620			*addr |= (((u64)dest_island) << iid_lsb) & v64;
621			return 0;
622		}
623
624		/* Make sure we compare against isldN values
625		 * by clearing the LSB.
626		 * This is what the silicon does.
627		 */
628		isld[0] &= ~1;
629		isld[1] &= ~1;
630
631		idx_lsb = addr40 ? 37 : 29;
632		iid_lsb = idx_lsb - 1;
633
634		return nfp_encode_basic_search(addr, dest_island, isld,
635					       iid_lsb, idx_lsb, 2);
636	case 3:
637		/* Only the EMU will use 40 bit addressing. Silently
638		 * set the direct locality bit for everyone else.
639		 * The SDK toolchain uses dest_island <= 0 to test
640		 * for atypical address encodings to support access
641		 * to local-island CTM with a 32-but address (high-locality
642		 * is effewctively ignored and just used for
643		 * routing to island #0).
644		 */
645		if (dest_island > 0 && (dest_island < 24 || dest_island > 26)) {
646			*addr |= ((u64)_NIC_NFP6000_MU_LOCALITY_DIRECT)
647							<< locality_lsb;
648			da = 1;
649		}
650
651		if (da) {
652			iid_lsb = addr40 ? 32 : 24;
653			v64 = GENMASK_ULL(iid_lsb + 5, iid_lsb);
654			*addr &= ~v64;
655			*addr |= (((u64)dest_island) << iid_lsb) & v64;
656			return 0;
657		}
658
659		isld[0] &= ~3;
660		isld[1] &= ~3;
661
662		idx_lsb = addr40 ? 37 : 29;
663		iid_lsb = idx_lsb - 2;
664
665		return nfp_encode_basic_search(addr, dest_island, isld,
666					       iid_lsb, idx_lsb, 4);
667	default:
668		return -EINVAL;
669	}
670}
671
672static int nfp_cppat_addr_encode(u64 *addr, int dest_island, int cpp_tgt,
673				 int mode, bool addr40, int isld1, int isld0)
674{
675	switch (cpp_tgt) {
676	case NFP_CPP_TARGET_NBI:
677	case NFP_CPP_TARGET_QDR:
678	case NFP_CPP_TARGET_ILA:
679	case NFP_CPP_TARGET_PCIE:
680	case NFP_CPP_TARGET_ARM:
681	case NFP_CPP_TARGET_CRYPTO:
682	case NFP_CPP_TARGET_CLS:
683		return nfp_encode_basic(addr, dest_island, cpp_tgt, mode,
684					addr40, isld1, isld0);
685
686	case NFP_CPP_TARGET_MU:
687		return nfp_encode_mu(addr, dest_island, mode,
688				     addr40, isld1, isld0);
689
690	case NFP_CPP_TARGET_CT_XPB:
691		if (mode != 1 || addr40)
692			return -EINVAL;
693		*addr &= ~GENMASK_ULL(29, 24);
694		*addr |= ((u64)dest_island << 24) & GENMASK_ULL(29, 24);
695		return 0;
696	default:
697		return -EINVAL;
698	}
699}
700
701int nfp_target_cpp(u32 cpp_island_id, u64 cpp_island_address,
702		   u32 *cpp_target_id, u64 *cpp_target_address,
703		   const u32 *imb_table)
704{
705	const int island = NFP_CPP_ID_ISLAND_of(cpp_island_id);
706	const int target = NFP_CPP_ID_TARGET_of(cpp_island_id);
707	u32 imb;
708	int err;
709
710	if (target < 0 || target >= 16) {
711		pr_err("Invalid CPP target: %d\n", target);
712		return -EINVAL;
713	}
714
715	if (island == 0) {
716		/* Already translated */
717		*cpp_target_id = cpp_island_id;
718		*cpp_target_address = cpp_island_address;
719		return 0;
720	}
721
722	/* CPP + Island only allowed on systems with IMB tables */
723	if (!imb_table)
724		return -EINVAL;
725
726	imb = imb_table[target];
727
728	*cpp_target_address = cpp_island_address;
729	err = nfp_cppat_addr_encode(cpp_target_address, island, target,
730				    ((imb >> 13) & 7), ((imb >> 12) & 1),
731				    ((imb >> 6)  & 0x3f), ((imb >> 0)  & 0x3f));
732	if (err) {
733		pr_err("Can't encode CPP address: %d\n", err);
734		return err;
735	}
736
737	*cpp_target_id = NFP_CPP_ID(target,
738				    NFP_CPP_ID_ACTION_of(cpp_island_id),
739				    NFP_CPP_ID_TOKEN_of(cpp_island_id));
740
741	return 0;
742}