FILE
pio.c
File preview with syntax highlighting.
linux-6.19.14/drivers/i3c/master/mipi-i3c-hci/pio.c
1// SPDX-License-Identifier: BSD-3-Clause 2/* 3 * Copyright (c) 2020, MIPI Alliance, Inc. 4 * 5 * Author: Nicolas Pitre <npitre@baylibre.com> 6 */ 7 8#include <linux/bitfield.h> 9#include <linux/device.h> 10#include <linux/errno.h> 11#include <linux/i3c/master.h> 12#include <linux/io.h> 13 14#include "hci.h" 15#include "cmd.h" 16#include "ibi.h" 17 18 19/* 20 * PIO Access Area 21 */ 22 23#define pio_reg_read(r) readl(hci->PIO_regs + (PIO_##r)) 24#define pio_reg_write(r, v) writel(v, hci->PIO_regs + (PIO_##r)) 25 26#define PIO_COMMAND_QUEUE_PORT 0x00 27#define PIO_RESPONSE_QUEUE_PORT 0x04 28#define PIO_XFER_DATA_PORT 0x08 29#define PIO_IBI_PORT 0x0c 30 31#define PIO_QUEUE_THLD_CTRL 0x10 32#define QUEUE_IBI_STATUS_THLD GENMASK(31, 24) 33#define QUEUE_IBI_DATA_THLD GENMASK(23, 16) 34#define QUEUE_RESP_BUF_THLD GENMASK(15, 8) 35#define QUEUE_CMD_EMPTY_BUF_THLD GENMASK(7, 0) 36 37#define PIO_DATA_BUFFER_THLD_CTRL 0x14 38#define DATA_RX_START_THLD GENMASK(26, 24) 39#define DATA_TX_START_THLD GENMASK(18, 16) 40#define DATA_RX_BUF_THLD GENMASK(10, 8) 41#define DATA_TX_BUF_THLD GENMASK(2, 0) 42 43#define PIO_QUEUE_SIZE 0x18 44#define TX_DATA_BUFFER_SIZE GENMASK(31, 24) 45#define RX_DATA_BUFFER_SIZE GENMASK(23, 16) 46#define IBI_STATUS_SIZE GENMASK(15, 8) 47#define CR_QUEUE_SIZE GENMASK(7, 0) 48 49#define PIO_INTR_STATUS 0x20 50#define PIO_INTR_STATUS_ENABLE 0x24 51#define PIO_INTR_SIGNAL_ENABLE 0x28 52#define PIO_INTR_FORCE 0x2c 53#define STAT_TRANSFER_BLOCKED BIT(25) 54#define STAT_PERR_RESP_UFLOW BIT(24) 55#define STAT_PERR_CMD_OFLOW BIT(23) 56#define STAT_PERR_IBI_UFLOW BIT(22) 57#define STAT_PERR_RX_UFLOW BIT(21) 58#define STAT_PERR_TX_OFLOW BIT(20) 59#define STAT_ERR_RESP_QUEUE_FULL BIT(19) 60#define STAT_WARN_RESP_QUEUE_FULL BIT(18) 61#define STAT_ERR_IBI_QUEUE_FULL BIT(17) 62#define STAT_WARN_IBI_QUEUE_FULL BIT(16) 63#define STAT_ERR_RX_DATA_FULL BIT(15) 64#define STAT_WARN_RX_DATA_FULL BIT(14) 65#define STAT_ERR_TX_DATA_EMPTY BIT(13) 66#define STAT_WARN_TX_DATA_EMPTY BIT(12) 67#define STAT_TRANSFER_ERR BIT(9) 68#define STAT_WARN_INS_STOP_MODE BIT(7) 69#define STAT_TRANSFER_ABORT BIT(5) 70#define STAT_RESP_READY BIT(4) 71#define STAT_CMD_QUEUE_READY BIT(3) 72#define STAT_IBI_STATUS_THLD BIT(2) 73#define STAT_RX_THLD BIT(1) 74#define STAT_TX_THLD BIT(0) 75 76#define PIO_QUEUE_CUR_STATUS 0x38 77#define CUR_IBI_Q_LEVEL GENMASK(28, 20) 78#define CUR_RESP_Q_LEVEL GENMASK(18, 10) 79#define CUR_CMD_Q_EMPTY_LEVEL GENMASK(8, 0) 80 81#define PIO_DATA_BUFFER_CUR_STATUS 0x3c 82#define CUR_RX_BUF_LVL GENMASK(26, 16) 83#define CUR_TX_BUF_LVL GENMASK(10, 0) 84 85/* 86 * Handy status bit combinations 87 */ 88 89#define STAT_LATENCY_WARNINGS (STAT_WARN_RESP_QUEUE_FULL | \ 90 STAT_WARN_IBI_QUEUE_FULL | \ 91 STAT_WARN_RX_DATA_FULL | \ 92 STAT_WARN_TX_DATA_EMPTY | \ 93 STAT_WARN_INS_STOP_MODE) 94 95#define STAT_LATENCY_ERRORS (STAT_ERR_RESP_QUEUE_FULL | \ 96 STAT_ERR_IBI_QUEUE_FULL | \ 97 STAT_ERR_RX_DATA_FULL | \ 98 STAT_ERR_TX_DATA_EMPTY) 99 100#define STAT_PROG_ERRORS (STAT_TRANSFER_BLOCKED | \ 101 STAT_PERR_RESP_UFLOW | \ 102 STAT_PERR_CMD_OFLOW | \ 103 STAT_PERR_IBI_UFLOW | \ 104 STAT_PERR_RX_UFLOW | \ 105 STAT_PERR_TX_OFLOW) 106 107#define STAT_ALL_ERRORS (STAT_TRANSFER_ABORT | \ 108 STAT_TRANSFER_ERR | \ 109 STAT_LATENCY_ERRORS | \ 110 STAT_PROG_ERRORS) 111 112struct hci_pio_dev_ibi_data { 113 struct i3c_generic_ibi_pool *pool; 114 unsigned int max_len; 115}; 116 117struct hci_pio_ibi_data { 118 struct i3c_ibi_slot *slot; 119 void *data_ptr; 120 unsigned int addr; 121 unsigned int seg_len, seg_cnt; 122 unsigned int max_len; 123 bool last_seg; 124}; 125 126struct hci_pio_data { 127 struct hci_xfer *curr_xfer, *xfer_queue; 128 struct hci_xfer *curr_rx, *rx_queue; 129 struct hci_xfer *curr_tx, *tx_queue; 130 struct hci_xfer *curr_resp, *resp_queue; 131 struct hci_pio_ibi_data ibi; 132 unsigned int rx_thresh_size, tx_thresh_size; 133 unsigned int max_ibi_thresh; 134 u32 reg_queue_thresh; 135 u32 enabled_irqs; 136}; 137 138static int hci_pio_init(struct i3c_hci *hci) 139{ 140 struct hci_pio_data *pio; 141 u32 val, size_val, rx_thresh, tx_thresh, ibi_val; 142 143 pio = kzalloc(sizeof(*pio), GFP_KERNEL); 144 if (!pio) 145 return -ENOMEM; 146 147 hci->io_data = pio; 148 149 size_val = pio_reg_read(QUEUE_SIZE); 150 dev_info(&hci->master.dev, "CMD/RESP FIFO = %ld entries\n", 151 FIELD_GET(CR_QUEUE_SIZE, size_val)); 152 dev_info(&hci->master.dev, "IBI FIFO = %ld bytes\n", 153 4 * FIELD_GET(IBI_STATUS_SIZE, size_val)); 154 dev_info(&hci->master.dev, "RX data FIFO = %d bytes\n", 155 4 * (2 << FIELD_GET(RX_DATA_BUFFER_SIZE, size_val))); 156 dev_info(&hci->master.dev, "TX data FIFO = %d bytes\n", 157 4 * (2 << FIELD_GET(TX_DATA_BUFFER_SIZE, size_val))); 158 159 /* 160 * Let's initialize data thresholds to half of the actual FIFO size. 161 * The start thresholds aren't used (set to 0) as the FIFO is always 162 * serviced before the corresponding command is queued. 163 */ 164 rx_thresh = FIELD_GET(RX_DATA_BUFFER_SIZE, size_val); 165 tx_thresh = FIELD_GET(TX_DATA_BUFFER_SIZE, size_val); 166 if (hci->version_major == 1) { 167 /* those are expressed as 2^[n+1), so just sub 1 if not 0 */ 168 if (rx_thresh) 169 rx_thresh -= 1; 170 if (tx_thresh) 171 tx_thresh -= 1; 172 pio->rx_thresh_size = 2 << rx_thresh; 173 pio->tx_thresh_size = 2 << tx_thresh; 174 } else { 175 /* size is 2^(n+1) and threshold is 2^n i.e. already halved */ 176 pio->rx_thresh_size = 1 << rx_thresh; 177 pio->tx_thresh_size = 1 << tx_thresh; 178 } 179 val = FIELD_PREP(DATA_RX_BUF_THLD, rx_thresh) | 180 FIELD_PREP(DATA_TX_BUF_THLD, tx_thresh); 181 pio_reg_write(DATA_BUFFER_THLD_CTRL, val); 182 183 /* 184 * Let's raise an interrupt as soon as there is one free cmd slot 185 * or one available response or IBI. For IBI data let's use half the 186 * IBI queue size within allowed bounds. 187 */ 188 ibi_val = FIELD_GET(IBI_STATUS_SIZE, size_val); 189 pio->max_ibi_thresh = clamp_val(ibi_val/2, 1, 63); 190 val = FIELD_PREP(QUEUE_IBI_STATUS_THLD, 1) | 191 FIELD_PREP(QUEUE_IBI_DATA_THLD, pio->max_ibi_thresh) | 192 FIELD_PREP(QUEUE_RESP_BUF_THLD, 1) | 193 FIELD_PREP(QUEUE_CMD_EMPTY_BUF_THLD, 1); 194 pio_reg_write(QUEUE_THLD_CTRL, val); 195 pio->reg_queue_thresh = val; 196 197 /* Disable all IRQs but allow all status bits */ 198 pio_reg_write(INTR_SIGNAL_ENABLE, 0x0); 199 pio_reg_write(INTR_STATUS_ENABLE, 0xffffffff); 200 201 /* Always accept error interrupts (will be activated on first xfer) */ 202 pio->enabled_irqs = STAT_ALL_ERRORS; 203 204 return 0; 205} 206 207static void hci_pio_cleanup(struct i3c_hci *hci) 208{ 209 struct hci_pio_data *pio = hci->io_data; 210 211 pio_reg_write(INTR_SIGNAL_ENABLE, 0x0); 212 213 if (pio) { 214 dev_dbg(&hci->master.dev, "status = %#x/%#x", 215 pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE)); 216 BUG_ON(pio->curr_xfer); 217 BUG_ON(pio->curr_rx); 218 BUG_ON(pio->curr_tx); 219 BUG_ON(pio->curr_resp); 220 kfree(pio); 221 hci->io_data = NULL; 222 } 223} 224 225static void hci_pio_write_cmd(struct i3c_hci *hci, struct hci_xfer *xfer) 226{ 227 dev_dbg(&hci->master.dev, "cmd_desc[%d] = 0x%08x", 228 0, xfer->cmd_desc[0]); 229 dev_dbg(&hci->master.dev, "cmd_desc[%d] = 0x%08x", 230 1, xfer->cmd_desc[1]); 231 pio_reg_write(COMMAND_QUEUE_PORT, xfer->cmd_desc[0]); 232 pio_reg_write(COMMAND_QUEUE_PORT, xfer->cmd_desc[1]); 233 if (hci->cmd == &mipi_i3c_hci_cmd_v2) { 234 dev_dbg(&hci->master.dev, "cmd_desc[%d] = 0x%08x", 235 2, xfer->cmd_desc[2]); 236 dev_dbg(&hci->master.dev, "cmd_desc[%d] = 0x%08x", 237 3, xfer->cmd_desc[3]); 238 pio_reg_write(COMMAND_QUEUE_PORT, xfer->cmd_desc[2]); 239 pio_reg_write(COMMAND_QUEUE_PORT, xfer->cmd_desc[3]); 240 } 241} 242 243static bool hci_pio_do_rx(struct i3c_hci *hci, struct hci_pio_data *pio) 244{ 245 struct hci_xfer *xfer = pio->curr_rx; 246 unsigned int nr_words; 247 u32 *p; 248 249 p = xfer->data; 250 p += (xfer->data_len - xfer->data_left) / 4; 251 252 while (xfer->data_left >= 4) { 253 /* bail out if FIFO hasn't reached the threshold value yet */ 254 if (!(pio_reg_read(INTR_STATUS) & STAT_RX_THLD)) 255 return false; 256 nr_words = min(xfer->data_left / 4, pio->rx_thresh_size); 257 /* extract data from FIFO */ 258 xfer->data_left -= nr_words * 4; 259 dev_dbg(&hci->master.dev, "now %d left %d", 260 nr_words * 4, xfer->data_left); 261 while (nr_words--) 262 *p++ = pio_reg_read(XFER_DATA_PORT); 263 } 264 265 /* trailing data is retrieved upon response reception */ 266 return !xfer->data_left; 267} 268 269static void hci_pio_do_trailing_rx(struct i3c_hci *hci, 270 struct hci_pio_data *pio, unsigned int count) 271{ 272 struct hci_xfer *xfer = pio->curr_rx; 273 u32 *p; 274 275 dev_dbg(&hci->master.dev, "%d remaining", count); 276 277 p = xfer->data; 278 p += (xfer->data_len - xfer->data_left) / 4; 279 280 if (count >= 4) { 281 unsigned int nr_words = count / 4; 282 /* extract data from FIFO */ 283 xfer->data_left -= nr_words * 4; 284 dev_dbg(&hci->master.dev, "now %d left %d", 285 nr_words * 4, xfer->data_left); 286 while (nr_words--) 287 *p++ = pio_reg_read(XFER_DATA_PORT); 288 } 289 290 count &= 3; 291 if (count) { 292 /* 293 * There are trailing bytes in the last word. 294 * Fetch it and extract bytes in an endian independent way. 295 * Unlike the TX case, we must not write memory past the 296 * end of the destination buffer. 297 */ 298 u8 *p_byte = (u8 *)p; 299 u32 data = pio_reg_read(XFER_DATA_PORT); 300 301 xfer->data_word_before_partial = data; 302 xfer->data_left -= count; 303 data = (__force u32) cpu_to_le32(data); 304 while (count--) { 305 *p_byte++ = data; 306 data >>= 8; 307 } 308 } 309} 310 311static bool hci_pio_do_tx(struct i3c_hci *hci, struct hci_pio_data *pio) 312{ 313 struct hci_xfer *xfer = pio->curr_tx; 314 unsigned int nr_words; 315 u32 *p; 316 317 p = xfer->data; 318 p += (xfer->data_len - xfer->data_left) / 4; 319 320 while (xfer->data_left >= 4) { 321 /* bail out if FIFO free space is below set threshold */ 322 if (!(pio_reg_read(INTR_STATUS) & STAT_TX_THLD)) 323 return false; 324 /* we can fill up to that TX threshold */ 325 nr_words = min(xfer->data_left / 4, pio->tx_thresh_size); 326 /* push data into the FIFO */ 327 xfer->data_left -= nr_words * 4; 328 dev_dbg(&hci->master.dev, "now %d left %d", 329 nr_words * 4, xfer->data_left); 330 while (nr_words--) 331 pio_reg_write(XFER_DATA_PORT, *p++); 332 } 333 334 if (xfer->data_left) { 335 /* 336 * There are trailing bytes to send. We can simply load 337 * them from memory as a word which will keep those bytes 338 * in their proper place even on a BE system. This will 339 * also get some bytes past the actual buffer but no one 340 * should care as they won't be sent out. 341 */ 342 if (!(pio_reg_read(INTR_STATUS) & STAT_TX_THLD)) 343 return false; 344 dev_dbg(&hci->master.dev, "trailing %d", xfer->data_left); 345 pio_reg_write(XFER_DATA_PORT, *p); 346 xfer->data_left = 0; 347 } 348 349 return true; 350} 351 352static bool hci_pio_process_rx(struct i3c_hci *hci, struct hci_pio_data *pio) 353{ 354 while (pio->curr_rx && hci_pio_do_rx(hci, pio)) 355 pio->curr_rx = pio->curr_rx->next_data; 356 return !pio->curr_rx; 357} 358 359static bool hci_pio_process_tx(struct i3c_hci *hci, struct hci_pio_data *pio) 360{ 361 while (pio->curr_tx && hci_pio_do_tx(hci, pio)) 362 pio->curr_tx = pio->curr_tx->next_data; 363 return !pio->curr_tx; 364} 365 366static void hci_pio_queue_data(struct i3c_hci *hci, struct hci_pio_data *pio) 367{ 368 struct hci_xfer *xfer = pio->curr_xfer; 369 struct hci_xfer *prev_queue_tail; 370 371 if (!xfer->data) { 372 xfer->data_len = xfer->data_left = 0; 373 return; 374 } 375 376 if (xfer->rnw) { 377 prev_queue_tail = pio->rx_queue; 378 pio->rx_queue = xfer; 379 if (pio->curr_rx) { 380 prev_queue_tail->next_data = xfer; 381 } else { 382 pio->curr_rx = xfer; 383 if (!hci_pio_process_rx(hci, pio)) 384 pio->enabled_irqs |= STAT_RX_THLD; 385 } 386 } else { 387 prev_queue_tail = pio->tx_queue; 388 pio->tx_queue = xfer; 389 if (pio->curr_tx) { 390 prev_queue_tail->next_data = xfer; 391 } else { 392 pio->curr_tx = xfer; 393 if (!hci_pio_process_tx(hci, pio)) 394 pio->enabled_irqs |= STAT_TX_THLD; 395 } 396 } 397} 398 399static void hci_pio_push_to_next_rx(struct i3c_hci *hci, struct hci_xfer *xfer, 400 unsigned int words_to_keep) 401{ 402 u32 *from = xfer->data; 403 u32 from_last; 404 unsigned int received, count; 405 406 received = (xfer->data_len - xfer->data_left) / 4; 407 if ((xfer->data_len - xfer->data_left) & 3) { 408 from_last = xfer->data_word_before_partial; 409 received += 1; 410 } else { 411 from_last = from[received]; 412 } 413 from += words_to_keep; 414 count = received - words_to_keep; 415 416 while (count) { 417 unsigned int room, left, chunk, bytes_to_move; 418 u32 last_word; 419 420 xfer = xfer->next_data; 421 if (!xfer) { 422 dev_err(&hci->master.dev, "pushing RX data to unexistent xfer\n"); 423 return; 424 } 425 426 room = DIV_ROUND_UP(xfer->data_len, 4); 427 left = DIV_ROUND_UP(xfer->data_left, 4); 428 chunk = min(count, room); 429 if (chunk > left) { 430 hci_pio_push_to_next_rx(hci, xfer, chunk - left); 431 left = chunk; 432 xfer->data_left = left * 4; 433 } 434 435 bytes_to_move = xfer->data_len - xfer->data_left; 436 if (bytes_to_move & 3) { 437 /* preserve word to become partial */ 438 u32 *p = xfer->data; 439 440 xfer->data_word_before_partial = p[bytes_to_move / 4]; 441 } 442 memmove(xfer->data + chunk, xfer->data, bytes_to_move); 443 444 /* treat last word specially because of partial word issues */ 445 chunk -= 1; 446 447 memcpy(xfer->data, from, chunk * 4); 448 xfer->data_left -= chunk * 4; 449 from += chunk; 450 count -= chunk; 451 452 last_word = (count == 1) ? from_last : *from++; 453 if (xfer->data_left < 4) { 454 /* 455 * Like in hci_pio_do_trailing_rx(), preserve original 456 * word to be stored partially then store bytes it 457 * in an endian independent way. 458 */ 459 u8 *p_byte = xfer->data; 460 461 p_byte += chunk * 4; 462 xfer->data_word_before_partial = last_word; 463 last_word = (__force u32) cpu_to_le32(last_word); 464 while (xfer->data_left--) { 465 *p_byte++ = last_word; 466 last_word >>= 8; 467 } 468 } else { 469 u32 *p = xfer->data; 470 471 p[chunk] = last_word; 472 xfer->data_left -= 4; 473 } 474 count--; 475 } 476} 477 478static void hci_pio_err(struct i3c_hci *hci, struct hci_pio_data *pio, 479 u32 status); 480 481static bool hci_pio_process_resp(struct i3c_hci *hci, struct hci_pio_data *pio) 482{ 483 while (pio->curr_resp && 484 (pio_reg_read(INTR_STATUS) & STAT_RESP_READY)) { 485 struct hci_xfer *xfer = pio->curr_resp; 486 u32 resp = pio_reg_read(RESPONSE_QUEUE_PORT); 487 unsigned int tid = RESP_TID(resp); 488 489 dev_dbg(&hci->master.dev, "resp = 0x%08x", resp); 490 if (tid != xfer->cmd_tid) { 491 dev_err(&hci->master.dev, 492 "response tid=%d when expecting %d\n", 493 tid, xfer->cmd_tid); 494 /* let's pretend it is a prog error... any of them */ 495 hci_pio_err(hci, pio, STAT_PROG_ERRORS); 496 return false; 497 } 498 xfer->response = resp; 499 500 if (pio->curr_rx == xfer) { 501 /* 502 * Response availability implies RX completion. 503 * Retrieve trailing RX data if any. 504 * Note that short reads are possible. 505 */ 506 unsigned int received, expected, to_keep; 507 508 received = xfer->data_len - xfer->data_left; 509 expected = RESP_DATA_LENGTH(xfer->response); 510 if (expected > received) { 511 hci_pio_do_trailing_rx(hci, pio, 512 expected - received); 513 } else if (received > expected) { 514 /* we consumed data meant for next xfer */ 515 to_keep = DIV_ROUND_UP(expected, 4); 516 hci_pio_push_to_next_rx(hci, xfer, to_keep); 517 } 518 519 /* then process the RX list pointer */ 520 if (hci_pio_process_rx(hci, pio)) 521 pio->enabled_irqs &= ~STAT_RX_THLD; 522 } 523 524 /* 525 * We're about to give back ownership of the xfer structure 526 * to the waiting instance. Make sure no reference to it 527 * still exists. 528 */ 529 if (pio->curr_rx == xfer) { 530 dev_dbg(&hci->master.dev, "short RX ?"); 531 pio->curr_rx = pio->curr_rx->next_data; 532 } else if (pio->curr_tx == xfer) { 533 dev_dbg(&hci->master.dev, "short TX ?"); 534 pio->curr_tx = pio->curr_tx->next_data; 535 } else if (xfer->data_left) { 536 dev_dbg(&hci->master.dev, 537 "PIO xfer count = %d after response", 538 xfer->data_left); 539 } 540 541 pio->curr_resp = xfer->next_resp; 542 if (xfer->completion) 543 complete(xfer->completion); 544 } 545 return !pio->curr_resp; 546} 547 548static void hci_pio_queue_resp(struct i3c_hci *hci, struct hci_pio_data *pio) 549{ 550 struct hci_xfer *xfer = pio->curr_xfer; 551 struct hci_xfer *prev_queue_tail; 552 553 if (!(xfer->cmd_desc[0] & CMD_0_ROC)) 554 return; 555 556 prev_queue_tail = pio->resp_queue; 557 pio->resp_queue = xfer; 558 if (pio->curr_resp) { 559 prev_queue_tail->next_resp = xfer; 560 } else { 561 pio->curr_resp = xfer; 562 if (!hci_pio_process_resp(hci, pio)) 563 pio->enabled_irqs |= STAT_RESP_READY; 564 } 565} 566 567static bool hci_pio_process_cmd(struct i3c_hci *hci, struct hci_pio_data *pio) 568{ 569 while (pio->curr_xfer && 570 (pio_reg_read(INTR_STATUS) & STAT_CMD_QUEUE_READY)) { 571 /* 572 * Always process the data FIFO before sending the command 573 * so needed TX data or RX space is available upfront. 574 */ 575 hci_pio_queue_data(hci, pio); 576 /* 577 * Then queue our response request. This will also process 578 * the response FIFO in case it got suddenly filled up 579 * with results from previous commands. 580 */ 581 hci_pio_queue_resp(hci, pio); 582 /* 583 * Finally send the command. 584 */ 585 hci_pio_write_cmd(hci, pio->curr_xfer); 586 /* 587 * And move on. 588 */ 589 pio->curr_xfer = pio->curr_xfer->next_xfer; 590 } 591 return !pio->curr_xfer; 592} 593 594static int hci_pio_queue_xfer(struct i3c_hci *hci, struct hci_xfer *xfer, int n) 595{ 596 struct hci_pio_data *pio = hci->io_data; 597 struct hci_xfer *prev_queue_tail; 598 int i; 599 600 dev_dbg(&hci->master.dev, "n = %d", n); 601 602 /* link xfer instances together and initialize data count */ 603 for (i = 0; i < n; i++) { 604 xfer[i].next_xfer = (i + 1 < n) ? &xfer[i + 1] : NULL; 605 xfer[i].next_data = NULL; 606 xfer[i].next_resp = NULL; 607 xfer[i].data_left = xfer[i].data_len; 608 } 609 610 spin_lock_irq(&hci->lock); 611 prev_queue_tail = pio->xfer_queue; 612 pio->xfer_queue = &xfer[n - 1]; 613 if (pio->curr_xfer) { 614 prev_queue_tail->next_xfer = xfer; 615 } else { 616 pio->curr_xfer = xfer; 617 if (!hci_pio_process_cmd(hci, pio)) 618 pio->enabled_irqs |= STAT_CMD_QUEUE_READY; 619 pio_reg_write(INTR_SIGNAL_ENABLE, pio->enabled_irqs); 620 dev_dbg(&hci->master.dev, "status = %#x/%#x", 621 pio_reg_read(INTR_STATUS), 622 pio_reg_read(INTR_SIGNAL_ENABLE)); 623 } 624 spin_unlock_irq(&hci->lock); 625 return 0; 626} 627 628static bool hci_pio_dequeue_xfer_common(struct i3c_hci *hci, 629 struct hci_pio_data *pio, 630 struct hci_xfer *xfer, int n) 631{ 632 struct hci_xfer *p, **p_prev_next; 633 int i; 634 635 /* 636 * To safely dequeue a transfer request, it must be either entirely 637 * processed, or not yet processed at all. If our request tail is 638 * reachable from either the data or resp list that means the command 639 * was submitted and not yet completed. 640 */ 641 for (p = pio->curr_resp; p; p = p->next_resp) 642 for (i = 0; i < n; i++) 643 if (p == &xfer[i]) 644 goto pio_screwed; 645 for (p = pio->curr_rx; p; p = p->next_data) 646 for (i = 0; i < n; i++) 647 if (p == &xfer[i]) 648 goto pio_screwed; 649 for (p = pio->curr_tx; p; p = p->next_data) 650 for (i = 0; i < n; i++) 651 if (p == &xfer[i]) 652 goto pio_screwed; 653 654 /* 655 * The command was completed, or wasn't yet submitted. 656 * Unlink it from the que if the later. 657 */ 658 p_prev_next = &pio->curr_xfer; 659 for (p = pio->curr_xfer; p; p = p->next_xfer) { 660 if (p == &xfer[0]) { 661 *p_prev_next = xfer[n - 1].next_xfer; 662 break; 663 } 664 p_prev_next = &p->next_xfer; 665 } 666 667 /* return true if we actually unqueued something */ 668 return !!p; 669 670pio_screwed: 671 /* 672 * Life is tough. We must invalidate the hardware state and 673 * discard everything that is still queued. 674 */ 675 for (p = pio->curr_resp; p; p = p->next_resp) { 676 p->response = FIELD_PREP(RESP_ERR_FIELD, RESP_ERR_HC_TERMINATED); 677 if (p->completion) 678 complete(p->completion); 679 } 680 for (p = pio->curr_xfer; p; p = p->next_xfer) { 681 p->response = FIELD_PREP(RESP_ERR_FIELD, RESP_ERR_HC_TERMINATED); 682 if (p->completion) 683 complete(p->completion); 684 } 685 pio->curr_xfer = pio->curr_rx = pio->curr_tx = pio->curr_resp = NULL; 686 687 return true; 688} 689 690static bool hci_pio_dequeue_xfer(struct i3c_hci *hci, struct hci_xfer *xfer, int n) 691{ 692 struct hci_pio_data *pio = hci->io_data; 693 int ret; 694 695 spin_lock_irq(&hci->lock); 696 dev_dbg(&hci->master.dev, "n=%d status=%#x/%#x", n, 697 pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE)); 698 dev_dbg(&hci->master.dev, "main_status = %#x/%#x", 699 readl(hci->base_regs + 0x20), readl(hci->base_regs + 0x28)); 700 701 ret = hci_pio_dequeue_xfer_common(hci, pio, xfer, n); 702 spin_unlock_irq(&hci->lock); 703 return ret; 704} 705 706static void hci_pio_err(struct i3c_hci *hci, struct hci_pio_data *pio, 707 u32 status) 708{ 709 /* TODO: this ought to be more sophisticated eventually */ 710 711 if (pio_reg_read(INTR_STATUS) & STAT_RESP_READY) { 712 /* this may happen when an error is signaled with ROC unset */ 713 u32 resp = pio_reg_read(RESPONSE_QUEUE_PORT); 714 715 dev_err(&hci->master.dev, 716 "orphan response (%#x) on error\n", resp); 717 } 718 719 /* dump states on programming errors */ 720 if (status & STAT_PROG_ERRORS) { 721 u32 queue = pio_reg_read(QUEUE_CUR_STATUS); 722 u32 data = pio_reg_read(DATA_BUFFER_CUR_STATUS); 723 724 dev_err(&hci->master.dev, 725 "prog error %#lx (C/R/I = %ld/%ld/%ld, TX/RX = %ld/%ld)\n", 726 status & STAT_PROG_ERRORS, 727 FIELD_GET(CUR_CMD_Q_EMPTY_LEVEL, queue), 728 FIELD_GET(CUR_RESP_Q_LEVEL, queue), 729 FIELD_GET(CUR_IBI_Q_LEVEL, queue), 730 FIELD_GET(CUR_TX_BUF_LVL, data), 731 FIELD_GET(CUR_RX_BUF_LVL, data)); 732 } 733 734 /* just bust out everything with pending responses for now */ 735 hci_pio_dequeue_xfer_common(hci, pio, pio->curr_resp, 1); 736 /* ... and half-way TX transfers if any */ 737 if (pio->curr_tx && pio->curr_tx->data_left != pio->curr_tx->data_len) 738 hci_pio_dequeue_xfer_common(hci, pio, pio->curr_tx, 1); 739 /* then reset the hardware */ 740 mipi_i3c_hci_pio_reset(hci); 741 mipi_i3c_hci_resume(hci); 742 743 dev_dbg(&hci->master.dev, "status=%#x/%#x", 744 pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE)); 745} 746 747static void hci_pio_set_ibi_thresh(struct i3c_hci *hci, 748 struct hci_pio_data *pio, 749 unsigned int thresh_val) 750{ 751 u32 regval = pio->reg_queue_thresh; 752 753 regval &= ~QUEUE_IBI_STATUS_THLD; 754 regval |= FIELD_PREP(QUEUE_IBI_STATUS_THLD, thresh_val); 755 /* write the threshold reg only if it changes */ 756 if (regval != pio->reg_queue_thresh) { 757 pio_reg_write(QUEUE_THLD_CTRL, regval); 758 pio->reg_queue_thresh = regval; 759 dev_dbg(&hci->master.dev, "%d", thresh_val); 760 } 761} 762 763static bool hci_pio_get_ibi_segment(struct i3c_hci *hci, 764 struct hci_pio_data *pio) 765{ 766 struct hci_pio_ibi_data *ibi = &pio->ibi; 767 unsigned int nr_words, thresh_val; 768 u32 *p; 769 770 p = ibi->data_ptr; 771 p += (ibi->seg_len - ibi->seg_cnt) / 4; 772 773 while ((nr_words = ibi->seg_cnt/4)) { 774 /* determine our IBI queue threshold value */ 775 thresh_val = min(nr_words, pio->max_ibi_thresh); 776 hci_pio_set_ibi_thresh(hci, pio, thresh_val); 777 /* bail out if we don't have that amount of data ready */ 778 if (!(pio_reg_read(INTR_STATUS) & STAT_IBI_STATUS_THLD)) 779 return false; 780 /* extract the data from the IBI port */ 781 nr_words = thresh_val; 782 ibi->seg_cnt -= nr_words * 4; 783 dev_dbg(&hci->master.dev, "now %d left %d", 784 nr_words * 4, ibi->seg_cnt); 785 while (nr_words--) 786 *p++ = pio_reg_read(IBI_PORT); 787 } 788 789 if (ibi->seg_cnt) { 790 /* 791 * There are trailing bytes in the last word. 792 * Fetch it and extract bytes in an endian independent way. 793 * Unlike the TX case, we must not write past the end of 794 * the destination buffer. 795 */ 796 u32 data; 797 u8 *p_byte = (u8 *)p; 798 799 hci_pio_set_ibi_thresh(hci, pio, 1); 800 if (!(pio_reg_read(INTR_STATUS) & STAT_IBI_STATUS_THLD)) 801 return false; 802 dev_dbg(&hci->master.dev, "trailing %d", ibi->seg_cnt); 803 data = pio_reg_read(IBI_PORT); 804 data = (__force u32) cpu_to_le32(data); 805 while (ibi->seg_cnt--) { 806 *p_byte++ = data; 807 data >>= 8; 808 } 809 } 810 811 return true; 812} 813 814static bool hci_pio_prep_new_ibi(struct i3c_hci *hci, struct hci_pio_data *pio) 815{ 816 struct hci_pio_ibi_data *ibi = &pio->ibi; 817 struct i3c_dev_desc *dev; 818 struct i3c_hci_dev_data *dev_data; 819 struct hci_pio_dev_ibi_data *dev_ibi; 820 u32 ibi_status; 821 822 /* 823 * We have a new IBI. Try to set up its payload retrieval. 824 * When returning true, the IBI data has to be consumed whether 825 * or not we are set up to capture it. If we return true with 826 * ibi->slot == NULL that means the data payload has to be 827 * drained out of the IBI port and dropped. 828 */ 829 830 ibi_status = pio_reg_read(IBI_PORT); 831 dev_dbg(&hci->master.dev, "status = %#x", ibi_status); 832 ibi->addr = FIELD_GET(IBI_TARGET_ADDR, ibi_status); 833 if (ibi_status & IBI_ERROR) { 834 dev_err(&hci->master.dev, "IBI error from %#x\n", ibi->addr); 835 return false; 836 } 837 838 ibi->last_seg = ibi_status & IBI_LAST_STATUS; 839 ibi->seg_len = FIELD_GET(IBI_DATA_LENGTH, ibi_status); 840 ibi->seg_cnt = ibi->seg_len; 841 842 dev = i3c_hci_addr_to_dev(hci, ibi->addr); 843 if (!dev) { 844 dev_err(&hci->master.dev, 845 "IBI for unknown device %#x\n", ibi->addr); 846 return true; 847 } 848 849 dev_data = i3c_dev_get_master_data(dev); 850 dev_ibi = dev_data->ibi_data; 851 ibi->max_len = dev_ibi->max_len; 852 853 if (ibi->seg_len > ibi->max_len) { 854 dev_err(&hci->master.dev, "IBI payload too big (%d > %d)\n", 855 ibi->seg_len, ibi->max_len); 856 return true; 857 } 858 859 ibi->slot = i3c_generic_ibi_get_free_slot(dev_ibi->pool); 860 if (!ibi->slot) { 861 dev_err(&hci->master.dev, "no free slot for IBI\n"); 862 } else { 863 ibi->slot->len = 0; 864 ibi->data_ptr = ibi->slot->data; 865 } 866 return true; 867} 868 869static void hci_pio_free_ibi_slot(struct i3c_hci *hci, struct hci_pio_data *pio) 870{ 871 struct hci_pio_ibi_data *ibi = &pio->ibi; 872 struct hci_pio_dev_ibi_data *dev_ibi; 873 874 if (ibi->slot) { 875 dev_ibi = ibi->slot->dev->common.master_priv; 876 i3c_generic_ibi_recycle_slot(dev_ibi->pool, ibi->slot); 877 ibi->slot = NULL; 878 } 879} 880 881static bool hci_pio_process_ibi(struct i3c_hci *hci, struct hci_pio_data *pio) 882{ 883 struct hci_pio_ibi_data *ibi = &pio->ibi; 884 885 if (!ibi->slot && !ibi->seg_cnt && ibi->last_seg) 886 if (!hci_pio_prep_new_ibi(hci, pio)) 887 return false; 888 889 for (;;) { 890 u32 ibi_status; 891 unsigned int ibi_addr; 892 893 if (ibi->slot) { 894 if (!hci_pio_get_ibi_segment(hci, pio)) 895 return false; 896 ibi->slot->len += ibi->seg_len; 897 ibi->data_ptr += ibi->seg_len; 898 if (ibi->last_seg) { 899 /* was the last segment: submit it and leave */ 900 i3c_master_queue_ibi(ibi->slot->dev, ibi->slot); 901 ibi->slot = NULL; 902 hci_pio_set_ibi_thresh(hci, pio, 1); 903 return true; 904 } 905 } else if (ibi->seg_cnt) { 906 /* 907 * No slot but a non-zero count. This is the result 908 * of some error and the payload must be drained. 909 * This normally does not happen therefore no need 910 * to be extra optimized here. 911 */ 912 hci_pio_set_ibi_thresh(hci, pio, 1); 913 do { 914 if (!(pio_reg_read(INTR_STATUS) & STAT_IBI_STATUS_THLD)) 915 return false; 916 pio_reg_read(IBI_PORT); 917 } while (--ibi->seg_cnt); 918 if (ibi->last_seg) 919 return true; 920 } 921 922 /* try to move to the next segment right away */ 923 hci_pio_set_ibi_thresh(hci, pio, 1); 924 if (!(pio_reg_read(INTR_STATUS) & STAT_IBI_STATUS_THLD)) 925 return false; 926 ibi_status = pio_reg_read(IBI_PORT); 927 ibi_addr = FIELD_GET(IBI_TARGET_ADDR, ibi_status); 928 if (ibi->addr != ibi_addr) { 929 /* target address changed before last segment */ 930 dev_err(&hci->master.dev, 931 "unexp IBI address changed from %d to %d\n", 932 ibi->addr, ibi_addr); 933 hci_pio_free_ibi_slot(hci, pio); 934 } 935 ibi->last_seg = ibi_status & IBI_LAST_STATUS; 936 ibi->seg_len = FIELD_GET(IBI_DATA_LENGTH, ibi_status); 937 ibi->seg_cnt = ibi->seg_len; 938 if (ibi->slot && ibi->slot->len + ibi->seg_len > ibi->max_len) { 939 dev_err(&hci->master.dev, 940 "IBI payload too big (%d > %d)\n", 941 ibi->slot->len + ibi->seg_len, ibi->max_len); 942 hci_pio_free_ibi_slot(hci, pio); 943 } 944 } 945 946 return false; 947} 948 949static int hci_pio_request_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev, 950 const struct i3c_ibi_setup *req) 951{ 952 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); 953 struct i3c_generic_ibi_pool *pool; 954 struct hci_pio_dev_ibi_data *dev_ibi; 955 956 dev_ibi = kmalloc(sizeof(*dev_ibi), GFP_KERNEL); 957 if (!dev_ibi) 958 return -ENOMEM; 959 pool = i3c_generic_ibi_alloc_pool(dev, req); 960 if (IS_ERR(pool)) { 961 kfree(dev_ibi); 962 return PTR_ERR(pool); 963 } 964 dev_ibi->pool = pool; 965 dev_ibi->max_len = req->max_payload_len; 966 dev_data->ibi_data = dev_ibi; 967 return 0; 968} 969 970static void hci_pio_free_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev) 971{ 972 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); 973 struct hci_pio_dev_ibi_data *dev_ibi = dev_data->ibi_data; 974 975 dev_data->ibi_data = NULL; 976 i3c_generic_ibi_free_pool(dev_ibi->pool); 977 kfree(dev_ibi); 978} 979 980static void hci_pio_recycle_ibi_slot(struct i3c_hci *hci, 981 struct i3c_dev_desc *dev, 982 struct i3c_ibi_slot *slot) 983{ 984 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); 985 struct hci_pio_dev_ibi_data *dev_ibi = dev_data->ibi_data; 986 987 i3c_generic_ibi_recycle_slot(dev_ibi->pool, slot); 988} 989 990static bool hci_pio_irq_handler(struct i3c_hci *hci) 991{ 992 struct hci_pio_data *pio = hci->io_data; 993 u32 status; 994 995 spin_lock(&hci->lock); 996 status = pio_reg_read(INTR_STATUS); 997 dev_dbg(&hci->master.dev, "PIO_INTR_STATUS %#x/%#x", 998 status, pio->enabled_irqs); 999 status &= pio->enabled_irqs | STAT_LATENCY_WARNINGS; 1000 if (!status) { 1001 spin_unlock(&hci->lock); 1002 return false; 1003 } 1004 1005 if (status & STAT_IBI_STATUS_THLD) 1006 hci_pio_process_ibi(hci, pio); 1007 1008 if (status & STAT_RX_THLD) 1009 if (hci_pio_process_rx(hci, pio)) 1010 pio->enabled_irqs &= ~STAT_RX_THLD; 1011 if (status & STAT_TX_THLD) 1012 if (hci_pio_process_tx(hci, pio)) 1013 pio->enabled_irqs &= ~STAT_TX_THLD; 1014 if (status & STAT_RESP_READY) 1015 if (hci_pio_process_resp(hci, pio)) 1016 pio->enabled_irqs &= ~STAT_RESP_READY; 1017 1018 if (unlikely(status & STAT_LATENCY_WARNINGS)) { 1019 pio_reg_write(INTR_STATUS, status & STAT_LATENCY_WARNINGS); 1020 dev_warn_ratelimited(&hci->master.dev, 1021 "encountered warning condition %#lx\n", 1022 status & STAT_LATENCY_WARNINGS); 1023 } 1024 1025 if (unlikely(status & STAT_ALL_ERRORS)) { 1026 pio_reg_write(INTR_STATUS, status & STAT_ALL_ERRORS); 1027 hci_pio_err(hci, pio, status & STAT_ALL_ERRORS); 1028 } 1029 1030 if (status & STAT_CMD_QUEUE_READY) 1031 if (hci_pio_process_cmd(hci, pio)) 1032 pio->enabled_irqs &= ~STAT_CMD_QUEUE_READY; 1033 1034 pio_reg_write(INTR_SIGNAL_ENABLE, pio->enabled_irqs); 1035 dev_dbg(&hci->master.dev, "PIO_INTR_STATUS %#x/%#x", 1036 pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE)); 1037 spin_unlock(&hci->lock); 1038 return true; 1039} 1040 1041const struct hci_io_ops mipi_i3c_hci_pio = { 1042 .init = hci_pio_init, 1043 .cleanup = hci_pio_cleanup, 1044 .queue_xfer = hci_pio_queue_xfer, 1045 .dequeue_xfer = hci_pio_dequeue_xfer, 1046 .irq_handler = hci_pio_irq_handler, 1047 .request_ibi = hci_pio_request_ibi, 1048 .free_ibi = hci_pio_free_ibi, 1049 .recycle_ibi_slot = hci_pio_recycle_ibi_slot, 1050};
~/
Read-only file preview.