igc_ptp.c

   1// SPDX-License-Identifier: GPL-2.0
   2/* Copyright (c)  2019 Intel Corporation */
   3
   4#include "igc.h"
   5
   6#include <linux/module.h>
   7#include <linux/device.h>
   8#include <linux/pci.h>
   9#include <linux/ptp_classify.h>
  10#include <linux/clocksource.h>
  11#include <linux/ktime.h>
  12#include <linux/delay.h>
  13#include <linux/iopoll.h>
  14#include <net/xdp_sock_drv.h>
  15
  16#define INCVALUE_MASK		0x7fffffff
  17#define ISGN			0x80000000
  18
  19#define IGC_PTP_TX_TIMEOUT		(HZ * 15)
  20
  21#define IGC_PTM_STAT_SLEEP		2
  22#define IGC_PTM_STAT_TIMEOUT		100
  23
  24/* SYSTIM read access for I225 */
  25void igc_ptp_read(struct igc_adapter *adapter, struct timespec64 *ts)
  26{
  27	struct igc_hw *hw = &adapter->hw;
  28	u32 sec, nsec;
  29
  30	/* The timestamp is latched when SYSTIML is read. */
  31	nsec = rd32(IGC_SYSTIML);
  32	sec = rd32(IGC_SYSTIMH);
  33
  34	ts->tv_sec = sec;
  35	ts->tv_nsec = nsec;
  36}
  37
  38static void igc_ptp_write_i225(struct igc_adapter *adapter,
  39			       const struct timespec64 *ts)
  40{
  41	struct igc_hw *hw = &adapter->hw;
  42
  43	wr32(IGC_SYSTIML, ts->tv_nsec);
  44	wr32(IGC_SYSTIMH, ts->tv_sec);
  45}
  46
  47static int igc_ptp_adjfine_i225(struct ptp_clock_info *ptp, long scaled_ppm)
  48{
  49	struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
  50					       ptp_caps);
  51	struct igc_hw *hw = &igc->hw;
  52	int neg_adj = 0;
  53	u64 rate;
  54	u32 inca;
  55
  56	if (scaled_ppm < 0) {
  57		neg_adj = 1;
  58		scaled_ppm = -scaled_ppm;
  59	}
  60	rate = scaled_ppm;
  61	rate <<= 14;
  62	rate = div_u64(rate, 78125);
  63
  64	inca = rate & INCVALUE_MASK;
  65	if (neg_adj)
  66		inca |= ISGN;
  67
  68	wr32(IGC_TIMINCA, inca);
  69
  70	return 0;
  71}
  72
  73static int igc_ptp_adjtime_i225(struct ptp_clock_info *ptp, s64 delta)
  74{
  75	struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
  76					       ptp_caps);
  77	struct timespec64 now, then = ns_to_timespec64(delta);
  78	unsigned long flags;
  79
  80	spin_lock_irqsave(&igc->tmreg_lock, flags);
  81
  82	igc_ptp_read(igc, &now);
  83	now = timespec64_add(now, then);
  84	igc_ptp_write_i225(igc, (const struct timespec64 *)&now);
  85
  86	spin_unlock_irqrestore(&igc->tmreg_lock, flags);
  87
  88	return 0;
  89}
  90
  91static int igc_ptp_gettimex64_i225(struct ptp_clock_info *ptp,
  92				   struct timespec64 *ts,
  93				   struct ptp_system_timestamp *sts)
  94{
  95	struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
  96					       ptp_caps);
  97	struct igc_hw *hw = &igc->hw;
  98	unsigned long flags;
  99
 100	spin_lock_irqsave(&igc->tmreg_lock, flags);
 101
 102	ptp_read_system_prets(sts);
 103	ts->tv_nsec = rd32(IGC_SYSTIML);
 104	ts->tv_sec = rd32(IGC_SYSTIMH);
 105	ptp_read_system_postts(sts);
 106
 107	spin_unlock_irqrestore(&igc->tmreg_lock, flags);
 108
 109	return 0;
 110}
 111
 112static int igc_ptp_settime_i225(struct ptp_clock_info *ptp,
 113				const struct timespec64 *ts)
 114{
 115	struct igc_adapter *igc = container_of(ptp, struct igc_adapter,
 116					       ptp_caps);
 117	unsigned long flags;
 118
 119	spin_lock_irqsave(&igc->tmreg_lock, flags);
 120
 121	igc_ptp_write_i225(igc, ts);
 122
 123	spin_unlock_irqrestore(&igc->tmreg_lock, flags);
 124
 125	return 0;
 126}
 127
 128static void igc_pin_direction(int pin, int input, u32 *ctrl, u32 *ctrl_ext)
 129{
 130	u32 *ptr = pin < 2 ? ctrl : ctrl_ext;
 131	static const u32 mask[IGC_N_SDP] = {
 132		IGC_CTRL_SDP0_DIR,
 133		IGC_CTRL_SDP1_DIR,
 134		IGC_CTRL_EXT_SDP2_DIR,
 135		IGC_CTRL_EXT_SDP3_DIR,
 136	};
 137
 138	if (input)
 139		*ptr &= ~mask[pin];
 140	else
 141		*ptr |= mask[pin];
 142}
 143
 144static void igc_pin_perout(struct igc_adapter *igc, int chan, int pin, int freq)
 145{
 146	static const u32 igc_aux0_sel_sdp[IGC_N_SDP] = {
 147		IGC_AUX0_SEL_SDP0, IGC_AUX0_SEL_SDP1, IGC_AUX0_SEL_SDP2, IGC_AUX0_SEL_SDP3,
 148	};
 149	static const u32 igc_aux1_sel_sdp[IGC_N_SDP] = {
 150		IGC_AUX1_SEL_SDP0, IGC_AUX1_SEL_SDP1, IGC_AUX1_SEL_SDP2, IGC_AUX1_SEL_SDP3,
 151	};
 152	static const u32 igc_ts_sdp_en[IGC_N_SDP] = {
 153		IGC_TS_SDP0_EN, IGC_TS_SDP1_EN, IGC_TS_SDP2_EN, IGC_TS_SDP3_EN,
 154	};
 155	static const u32 igc_ts_sdp_sel_tt0[IGC_N_SDP] = {
 156		IGC_TS_SDP0_SEL_TT0, IGC_TS_SDP1_SEL_TT0,
 157		IGC_TS_SDP2_SEL_TT0, IGC_TS_SDP3_SEL_TT0,
 158	};
 159	static const u32 igc_ts_sdp_sel_tt1[IGC_N_SDP] = {
 160		IGC_TS_SDP0_SEL_TT1, IGC_TS_SDP1_SEL_TT1,
 161		IGC_TS_SDP2_SEL_TT1, IGC_TS_SDP3_SEL_TT1,
 162	};
 163	static const u32 igc_ts_sdp_sel_fc0[IGC_N_SDP] = {
 164		IGC_TS_SDP0_SEL_FC0, IGC_TS_SDP1_SEL_FC0,
 165		IGC_TS_SDP2_SEL_FC0, IGC_TS_SDP3_SEL_FC0,
 166	};
 167	static const u32 igc_ts_sdp_sel_fc1[IGC_N_SDP] = {
 168		IGC_TS_SDP0_SEL_FC1, IGC_TS_SDP1_SEL_FC1,
 169		IGC_TS_SDP2_SEL_FC1, IGC_TS_SDP3_SEL_FC1,
 170	};
 171	static const u32 igc_ts_sdp_sel_clr[IGC_N_SDP] = {
 172		IGC_TS_SDP0_SEL_FC1, IGC_TS_SDP1_SEL_FC1,
 173		IGC_TS_SDP2_SEL_FC1, IGC_TS_SDP3_SEL_FC1,
 174	};
 175	struct igc_hw *hw = &igc->hw;
 176	u32 ctrl, ctrl_ext, tssdp = 0;
 177
 178	ctrl = rd32(IGC_CTRL);
 179	ctrl_ext = rd32(IGC_CTRL_EXT);
 180	tssdp = rd32(IGC_TSSDP);
 181
 182	igc_pin_direction(pin, 0, &ctrl, &ctrl_ext);
 183
 184	/* Make sure this pin is not enabled as an input. */
 185	if ((tssdp & IGC_AUX0_SEL_SDP3) == igc_aux0_sel_sdp[pin])
 186		tssdp &= ~IGC_AUX0_TS_SDP_EN;
 187
 188	if ((tssdp & IGC_AUX1_SEL_SDP3) == igc_aux1_sel_sdp[pin])
 189		tssdp &= ~IGC_AUX1_TS_SDP_EN;
 190
 191	tssdp &= ~igc_ts_sdp_sel_clr[pin];
 192	if (freq) {
 193		if (chan == 1)
 194			tssdp |= igc_ts_sdp_sel_fc1[pin];
 195		else
 196			tssdp |= igc_ts_sdp_sel_fc0[pin];
 197	} else {
 198		if (chan == 1)
 199			tssdp |= igc_ts_sdp_sel_tt1[pin];
 200		else
 201			tssdp |= igc_ts_sdp_sel_tt0[pin];
 202	}
 203	tssdp |= igc_ts_sdp_en[pin];
 204
 205	wr32(IGC_TSSDP, tssdp);
 206	wr32(IGC_CTRL, ctrl);
 207	wr32(IGC_CTRL_EXT, ctrl_ext);
 208}
 209
 210static void igc_pin_extts(struct igc_adapter *igc, int chan, int pin)
 211{
 212	static const u32 igc_aux0_sel_sdp[IGC_N_SDP] = {
 213		IGC_AUX0_SEL_SDP0, IGC_AUX0_SEL_SDP1, IGC_AUX0_SEL_SDP2, IGC_AUX0_SEL_SDP3,
 214	};
 215	static const u32 igc_aux1_sel_sdp[IGC_N_SDP] = {
 216		IGC_AUX1_SEL_SDP0, IGC_AUX1_SEL_SDP1, IGC_AUX1_SEL_SDP2, IGC_AUX1_SEL_SDP3,
 217	};
 218	static const u32 igc_ts_sdp_en[IGC_N_SDP] = {
 219		IGC_TS_SDP0_EN, IGC_TS_SDP1_EN, IGC_TS_SDP2_EN, IGC_TS_SDP3_EN,
 220	};
 221	struct igc_hw *hw = &igc->hw;
 222	u32 ctrl, ctrl_ext, tssdp = 0;
 223
 224	ctrl = rd32(IGC_CTRL);
 225	ctrl_ext = rd32(IGC_CTRL_EXT);
 226	tssdp = rd32(IGC_TSSDP);
 227
 228	igc_pin_direction(pin, 1, &ctrl, &ctrl_ext);
 229
 230	/* Make sure this pin is not enabled as an output. */
 231	tssdp &= ~igc_ts_sdp_en[pin];
 232
 233	if (chan == 1) {
 234		tssdp &= ~IGC_AUX1_SEL_SDP3;
 235		tssdp |= igc_aux1_sel_sdp[pin] | IGC_AUX1_TS_SDP_EN;
 236	} else {
 237		tssdp &= ~IGC_AUX0_SEL_SDP3;
 238		tssdp |= igc_aux0_sel_sdp[pin] | IGC_AUX0_TS_SDP_EN;
 239	}
 240
 241	wr32(IGC_TSSDP, tssdp);
 242	wr32(IGC_CTRL, ctrl);
 243	wr32(IGC_CTRL_EXT, ctrl_ext);
 244}
 245
 246static int igc_ptp_feature_enable_i225(struct ptp_clock_info *ptp,
 247				       struct ptp_clock_request *rq, int on)
 248{
 249	struct igc_adapter *igc =
 250		container_of(ptp, struct igc_adapter, ptp_caps);
 251	struct igc_hw *hw = &igc->hw;
 252	unsigned long flags;
 253	struct timespec64 ts;
 254	int use_freq = 0, pin = -1;
 255	u32 tsim, tsauxc, tsauxc_mask, tsim_mask, trgttiml, trgttimh, freqout;
 256	s64 ns;
 257
 258	switch (rq->type) {
 259	case PTP_CLK_REQ_EXTTS:
 260		/* Reject requests failing to enable both edges. */
 261		if ((rq->extts.flags & PTP_STRICT_FLAGS) &&
 262		    (rq->extts.flags & PTP_ENABLE_FEATURE) &&
 263		    (rq->extts.flags & PTP_EXTTS_EDGES) != PTP_EXTTS_EDGES)
 264			return -EOPNOTSUPP;
 265
 266		if (on) {
 267			pin = ptp_find_pin(igc->ptp_clock, PTP_PF_EXTTS,
 268					   rq->extts.index);
 269			if (pin < 0)
 270				return -EBUSY;
 271		}
 272		if (rq->extts.index == 1) {
 273			tsauxc_mask = IGC_TSAUXC_EN_TS1;
 274			tsim_mask = IGC_TSICR_AUTT1;
 275		} else {
 276			tsauxc_mask = IGC_TSAUXC_EN_TS0;
 277			tsim_mask = IGC_TSICR_AUTT0;
 278		}
 279		spin_lock_irqsave(&igc->tmreg_lock, flags);
 280		tsauxc = rd32(IGC_TSAUXC);
 281		tsim = rd32(IGC_TSIM);
 282		if (on) {
 283			igc_pin_extts(igc, rq->extts.index, pin);
 284			tsauxc |= tsauxc_mask;
 285			tsim |= tsim_mask;
 286		} else {
 287			tsauxc &= ~tsauxc_mask;
 288			tsim &= ~tsim_mask;
 289		}
 290		wr32(IGC_TSAUXC, tsauxc);
 291		wr32(IGC_TSIM, tsim);
 292		spin_unlock_irqrestore(&igc->tmreg_lock, flags);
 293		return 0;
 294
 295	case PTP_CLK_REQ_PEROUT:
 296		if (on) {
 297			pin = ptp_find_pin(igc->ptp_clock, PTP_PF_PEROUT,
 298					   rq->perout.index);
 299			if (pin < 0)
 300				return -EBUSY;
 301		}
 302		ts.tv_sec = rq->perout.period.sec;
 303		ts.tv_nsec = rq->perout.period.nsec;
 304		ns = timespec64_to_ns(&ts);
 305		ns = ns >> 1;
 306		if (on && (ns <= 70000000LL || ns == 125000000LL ||
 307			   ns == 250000000LL || ns == 500000000LL)) {
 308			if (ns < 8LL)
 309				return -EINVAL;
 310			use_freq = 1;
 311		}
 312		ts = ns_to_timespec64(ns);
 313		if (rq->perout.index == 1) {
 314			if (use_freq) {
 315				tsauxc_mask = IGC_TSAUXC_EN_CLK1 | IGC_TSAUXC_ST1;
 316				tsim_mask = 0;
 317			} else {
 318				tsauxc_mask = IGC_TSAUXC_EN_TT1;
 319				tsim_mask = IGC_TSICR_TT1;
 320			}
 321			trgttiml = IGC_TRGTTIML1;
 322			trgttimh = IGC_TRGTTIMH1;
 323			freqout = IGC_FREQOUT1;
 324		} else {
 325			if (use_freq) {
 326				tsauxc_mask = IGC_TSAUXC_EN_CLK0 | IGC_TSAUXC_ST0;
 327				tsim_mask = 0;
 328			} else {
 329				tsauxc_mask = IGC_TSAUXC_EN_TT0;
 330				tsim_mask = IGC_TSICR_TT0;
 331			}
 332			trgttiml = IGC_TRGTTIML0;
 333			trgttimh = IGC_TRGTTIMH0;
 334			freqout = IGC_FREQOUT0;
 335		}
 336		spin_lock_irqsave(&igc->tmreg_lock, flags);
 337		tsauxc = rd32(IGC_TSAUXC);
 338		tsim = rd32(IGC_TSIM);
 339		if (rq->perout.index == 1) {
 340			tsauxc &= ~(IGC_TSAUXC_EN_TT1 | IGC_TSAUXC_EN_CLK1 |
 341				    IGC_TSAUXC_ST1);
 342			tsim &= ~IGC_TSICR_TT1;
 343		} else {
 344			tsauxc &= ~(IGC_TSAUXC_EN_TT0 | IGC_TSAUXC_EN_CLK0 |
 345				    IGC_TSAUXC_ST0);
 346			tsim &= ~IGC_TSICR_TT0;
 347		}
 348		if (on) {
 349			struct timespec64 safe_start;
 350			int i = rq->perout.index;
 351
 352			igc_pin_perout(igc, i, pin, use_freq);
 353			igc_ptp_read(igc, &safe_start);
 354
 355			/* PPS output start time is triggered by Target time(TT)
 356			 * register. Programming any past time value into TT
 357			 * register will cause PPS to never start. Need to make
 358			 * sure we program the TT register a time ahead in
 359			 * future. There isn't a stringent need to fire PPS out
 360			 * right away. Adding +2 seconds should take care of
 361			 * corner cases. Let's say if the SYSTIML is close to
 362			 * wrap up and the timer keeps ticking as we program the
 363			 * register, adding +2seconds is safe bet.
 364			 */
 365			safe_start.tv_sec += 2;
 366
 367			if (rq->perout.start.sec < safe_start.tv_sec)
 368				igc->perout[i].start.tv_sec = safe_start.tv_sec;
 369			else
 370				igc->perout[i].start.tv_sec = rq->perout.start.sec;
 371			igc->perout[i].start.tv_nsec = rq->perout.start.nsec;
 372			igc->perout[i].period.tv_sec = ts.tv_sec;
 373			igc->perout[i].period.tv_nsec = ts.tv_nsec;
 374			wr32(trgttimh, (u32)igc->perout[i].start.tv_sec);
 375			/* For now, always select timer 0 as source. */
 376			wr32(trgttiml, (u32)(igc->perout[i].start.tv_nsec |
 377					     IGC_TT_IO_TIMER_SEL_SYSTIM0));
 378			if (use_freq)
 379				wr32(freqout, ns);
 380			tsauxc |= tsauxc_mask;
 381			tsim |= tsim_mask;
 382		}
 383		wr32(IGC_TSAUXC, tsauxc);
 384		wr32(IGC_TSIM, tsim);
 385		spin_unlock_irqrestore(&igc->tmreg_lock, flags);
 386		return 0;
 387
 388	case PTP_CLK_REQ_PPS:
 389		spin_lock_irqsave(&igc->tmreg_lock, flags);
 390		tsim = rd32(IGC_TSIM);
 391		if (on)
 392			tsim |= IGC_TSICR_SYS_WRAP;
 393		else
 394			tsim &= ~IGC_TSICR_SYS_WRAP;
 395		igc->pps_sys_wrap_on = on;
 396		wr32(IGC_TSIM, tsim);
 397		spin_unlock_irqrestore(&igc->tmreg_lock, flags);
 398		return 0;
 399
 400	default:
 401		break;
 402	}
 403
 404	return -EOPNOTSUPP;
 405}
 406
 407static int igc_ptp_verify_pin(struct ptp_clock_info *ptp, unsigned int pin,
 408			      enum ptp_pin_function func, unsigned int chan)
 409{
 410	switch (func) {
 411	case PTP_PF_NONE:
 412	case PTP_PF_EXTTS:
 413	case PTP_PF_PEROUT:
 414		break;
 415	case PTP_PF_PHYSYNC:
 416		return -1;
 417	}
 418	return 0;
 419}
 420
 421/**
 422 * igc_ptp_systim_to_hwtstamp - convert system time value to HW timestamp
 423 * @adapter: board private structure
 424 * @hwtstamps: timestamp structure to update
 425 * @systim: unsigned 64bit system time value
 426 *
 427 * We need to convert the system time value stored in the RX/TXSTMP registers
 428 * into a hwtstamp which can be used by the upper level timestamping functions.
 429 *
 430 * Returns 0 on success.
 431 **/
 432static int igc_ptp_systim_to_hwtstamp(struct igc_adapter *adapter,
 433				      struct skb_shared_hwtstamps *hwtstamps,
 434				      u64 systim)
 435{
 436	switch (adapter->hw.mac.type) {
 437	case igc_i225:
 438		memset(hwtstamps, 0, sizeof(*hwtstamps));
 439		/* Upper 32 bits contain s, lower 32 bits contain ns. */
 440		hwtstamps->hwtstamp = ktime_set(systim >> 32,
 441						systim & 0xFFFFFFFF);
 442		break;
 443	default:
 444		return -EINVAL;
 445	}
 446	return 0;
 447}
 448
 449/**
 450 * igc_ptp_rx_pktstamp - Retrieve timestamp from Rx packet buffer
 451 * @adapter: Pointer to adapter the packet buffer belongs to
 452 * @buf: Pointer to start of timestamp in HW format (2 32-bit words)
 453 *
 454 * This function retrieves and converts the timestamp stored at @buf
 455 * to ktime_t, adjusting for hardware latencies.
 456 *
 457 * Returns timestamp value.
 458 */
 459ktime_t igc_ptp_rx_pktstamp(struct igc_adapter *adapter, __le32 *buf)
 460{
 461	ktime_t timestamp;
 462	u32 secs, nsecs;
 463	int adjust;
 464
 465	nsecs = le32_to_cpu(buf[0]);
 466	secs = le32_to_cpu(buf[1]);
 467
 468	timestamp = ktime_set(secs, nsecs);
 469
 470	/* Adjust timestamp for the RX latency based on link speed */
 471	switch (adapter->link_speed) {
 472	case SPEED_10:
 473		adjust = IGC_I225_RX_LATENCY_10;
 474		break;
 475	case SPEED_100:
 476		adjust = IGC_I225_RX_LATENCY_100;
 477		break;
 478	case SPEED_1000:
 479		adjust = IGC_I225_RX_LATENCY_1000;
 480		break;
 481	case SPEED_2500:
 482		adjust = IGC_I225_RX_LATENCY_2500;
 483		break;
 484	default:
 485		adjust = 0;
 486		netdev_warn_once(adapter->netdev, "Imprecise timestamp\n");
 487		break;
 488	}
 489
 490	return ktime_sub_ns(timestamp, adjust);
 491}
 492
 493static void igc_ptp_disable_rx_timestamp(struct igc_adapter *adapter)
 494{
 495	struct igc_hw *hw = &adapter->hw;
 496	u32 val;
 497	int i;
 498
 499	wr32(IGC_TSYNCRXCTL, 0);
 500
 501	for (i = 0; i < adapter->num_rx_queues; i++) {
 502		val = rd32(IGC_SRRCTL(i));
 503		val &= ~IGC_SRRCTL_TIMESTAMP;
 504		wr32(IGC_SRRCTL(i), val);
 505	}
 506
 507	val = rd32(IGC_RXPBS);
 508	val &= ~IGC_RXPBS_CFG_TS_EN;
 509	wr32(IGC_RXPBS, val);
 510}
 511
 512static void igc_ptp_enable_rx_timestamp(struct igc_adapter *adapter)
 513{
 514	struct igc_hw *hw = &adapter->hw;
 515	u32 val;
 516	int i;
 517
 518	val = rd32(IGC_RXPBS);
 519	val |= IGC_RXPBS_CFG_TS_EN;
 520	wr32(IGC_RXPBS, val);
 521
 522	for (i = 0; i < adapter->num_rx_queues; i++) {
 523		val = rd32(IGC_SRRCTL(i));
 524		/* Enable retrieving timestamps from timer 0, the
 525		 * "adjustable clock" and timer 1 the "free running
 526		 * clock".
 527		 */
 528		val |= IGC_SRRCTL_TIMER1SEL(1) | IGC_SRRCTL_TIMER0SEL(0) |
 529		       IGC_SRRCTL_TIMESTAMP;
 530		wr32(IGC_SRRCTL(i), val);
 531	}
 532
 533	val = IGC_TSYNCRXCTL_ENABLED | IGC_TSYNCRXCTL_TYPE_ALL |
 534	      IGC_TSYNCRXCTL_RXSYNSIG;
 535	wr32(IGC_TSYNCRXCTL, val);
 536}
 537
 538static void igc_ptp_free_tx_buffer(struct igc_adapter *adapter,
 539				   struct igc_tx_timestamp_request *tstamp)
 540{
 541	if (tstamp->buffer_type == IGC_TX_BUFFER_TYPE_XSK) {
 542		/* Release the transmit completion */
 543		tstamp->xsk_tx_buffer->xsk_pending_ts = false;
 544
 545		/* Note: tstamp->skb and tstamp->xsk_tx_buffer are in union.
 546		 * By setting tstamp->xsk_tx_buffer to NULL, tstamp->skb will
 547		 * become NULL as well.
 548		 */
 549		tstamp->xsk_tx_buffer = NULL;
 550		tstamp->buffer_type = 0;
 551
 552		/* Trigger txrx interrupt for transmit completion */
 553		igc_xsk_wakeup(adapter->netdev, tstamp->xsk_queue_index,
 554			       XDP_WAKEUP_TX);
 555
 556		return;
 557	}
 558
 559	dev_kfree_skb_any(tstamp->skb);
 560	tstamp->skb = NULL;
 561}
 562
 563static void igc_ptp_clear_tx_tstamp(struct igc_adapter *adapter)
 564{
 565	unsigned long flags;
 566	int i;
 567
 568	spin_lock_irqsave(&adapter->ptp_tx_lock, flags);
 569
 570	for (i = 0; i < IGC_MAX_TX_TSTAMP_REGS; i++) {
 571		struct igc_tx_timestamp_request *tstamp = &adapter->tx_tstamp[i];
 572
 573		if (tstamp->skb)
 574			igc_ptp_free_tx_buffer(adapter, tstamp);
 575	}
 576
 577	spin_unlock_irqrestore(&adapter->ptp_tx_lock, flags);
 578}
 579
 580/**
 581 * igc_ptp_clear_xsk_tx_tstamp_queue - Clear pending XSK TX timestamps for a queue
 582 * @adapter: Board private structure
 583 * @queue_id: TX queue index to clear timestamps for
 584 *
 585 * Iterates over all TX timestamp registers and releases any pending
 586 * timestamp requests associated with the given TX queue. This is
 587 * called when an XDP pool is being disabled to ensure no stale
 588 * timestamp references remain.
 589 */
 590void igc_ptp_clear_xsk_tx_tstamp_queue(struct igc_adapter *adapter, u16 queue_id)
 591{
 592	unsigned long flags;
 593	int i;
 594
 595	spin_lock_irqsave(&adapter->ptp_tx_lock, flags);
 596
 597	for (i = 0; i < IGC_MAX_TX_TSTAMP_REGS; i++) {
 598		struct igc_tx_timestamp_request *tstamp = &adapter->tx_tstamp[i];
 599
 600		if (tstamp->buffer_type != IGC_TX_BUFFER_TYPE_XSK)
 601			continue;
 602		if (tstamp->xsk_queue_index != queue_id)
 603			continue;
 604		if (!tstamp->xsk_tx_buffer)
 605			continue;
 606
 607		igc_ptp_free_tx_buffer(adapter, tstamp);
 608	}
 609
 610	spin_unlock_irqrestore(&adapter->ptp_tx_lock, flags);
 611}
 612
 613static void igc_ptp_disable_tx_timestamp(struct igc_adapter *adapter)
 614{
 615	struct igc_hw *hw = &adapter->hw;
 616	int i;
 617
 618	/* Clear the flags first to avoid new packets to be enqueued
 619	 * for TX timestamping.
 620	 */
 621	for (i = 0; i < adapter->num_tx_queues; i++) {
 622		struct igc_ring *tx_ring = adapter->tx_ring[i];
 623
 624		clear_bit(IGC_RING_FLAG_TX_HWTSTAMP, &tx_ring->flags);
 625	}
 626
 627	/* Now we can clean the pending TX timestamp requests. */
 628	igc_ptp_clear_tx_tstamp(adapter);
 629
 630	wr32(IGC_TSYNCTXCTL, 0);
 631}
 632
 633static void igc_ptp_enable_tx_timestamp(struct igc_adapter *adapter)
 634{
 635	struct igc_hw *hw = &adapter->hw;
 636	int i;
 637
 638	wr32(IGC_TSYNCTXCTL, IGC_TSYNCTXCTL_ENABLED | IGC_TSYNCTXCTL_TXSYNSIG);
 639
 640	/* Read TXSTMP registers to discard any timestamp previously stored. */
 641	rd32(IGC_TXSTMPL);
 642	rd32(IGC_TXSTMPH);
 643
 644	/* The hardware is ready to accept TX timestamp requests,
 645	 * notify the transmit path.
 646	 */
 647	for (i = 0; i < adapter->num_tx_queues; i++) {
 648		struct igc_ring *tx_ring = adapter->tx_ring[i];
 649
 650		set_bit(IGC_RING_FLAG_TX_HWTSTAMP, &tx_ring->flags);
 651	}
 652
 653}
 654
 655/**
 656 * igc_ptp_set_timestamp_mode - setup hardware for timestamping
 657 * @adapter: networking device structure
 658 * @config: hwtstamp configuration
 659 *
 660 * Return: 0 in case of success, negative errno code otherwise.
 661 */
 662static int igc_ptp_set_timestamp_mode(struct igc_adapter *adapter,
 663				      struct kernel_hwtstamp_config *config)
 664{
 665	switch (config->tx_type) {
 666	case HWTSTAMP_TX_OFF:
 667		igc_ptp_disable_tx_timestamp(adapter);
 668		break;
 669	case HWTSTAMP_TX_ON:
 670		igc_ptp_enable_tx_timestamp(adapter);
 671		break;
 672	default:
 673		return -ERANGE;
 674	}
 675
 676	switch (config->rx_filter) {
 677	case HWTSTAMP_FILTER_NONE:
 678		igc_ptp_disable_rx_timestamp(adapter);
 679		break;
 680	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
 681	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
 682	case HWTSTAMP_FILTER_PTP_V2_EVENT:
 683	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
 684	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
 685	case HWTSTAMP_FILTER_PTP_V2_SYNC:
 686	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
 687	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
 688	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
 689	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
 690	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
 691	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
 692	case HWTSTAMP_FILTER_NTP_ALL:
 693	case HWTSTAMP_FILTER_ALL:
 694		igc_ptp_enable_rx_timestamp(adapter);
 695		config->rx_filter = HWTSTAMP_FILTER_ALL;
 696		break;
 697	default:
 698		return -ERANGE;
 699	}
 700
 701	return 0;
 702}
 703
 704/* Requires adapter->ptp_tx_lock held by caller. */
 705static void igc_ptp_tx_timeout(struct igc_adapter *adapter,
 706			       struct igc_tx_timestamp_request *tstamp)
 707{
 708	if (tstamp->skb)
 709		igc_ptp_free_tx_buffer(adapter, tstamp);
 710
 711	adapter->tx_hwtstamp_timeouts++;
 712
 713	netdev_warn(adapter->netdev, "Tx timestamp timeout\n");
 714}
 715
 716void igc_ptp_tx_hang(struct igc_adapter *adapter)
 717{
 718	struct igc_tx_timestamp_request *tstamp;
 719	struct igc_hw *hw = &adapter->hw;
 720	unsigned long flags;
 721	bool found = false;
 722	int i;
 723
 724	spin_lock_irqsave(&adapter->ptp_tx_lock, flags);
 725
 726	for (i = 0; i < IGC_MAX_TX_TSTAMP_REGS; i++) {
 727		tstamp = &adapter->tx_tstamp[i];
 728
 729		if (!tstamp->skb)
 730			continue;
 731
 732		if (time_is_after_jiffies(tstamp->start + IGC_PTP_TX_TIMEOUT))
 733			continue;
 734
 735		igc_ptp_tx_timeout(adapter, tstamp);
 736		found = true;
 737	}
 738
 739	if (found) {
 740		/* Reading the high register of the first set of timestamp registers
 741		 * clears all the equivalent bits in the TSYNCTXCTL register.
 742		 */
 743		rd32(IGC_TXSTMPH_0);
 744	}
 745
 746	spin_unlock_irqrestore(&adapter->ptp_tx_lock, flags);
 747}
 748
 749static void igc_ptp_tx_reg_to_stamp(struct igc_adapter *adapter,
 750				    struct igc_tx_timestamp_request *tstamp, u64 regval)
 751{
 752	struct skb_shared_hwtstamps shhwtstamps;
 753	struct sk_buff *skb;
 754	int adjust = 0;
 755
 756	skb = tstamp->skb;
 757	if (!skb)
 758		return;
 759
 760	if (igc_ptp_systim_to_hwtstamp(adapter, &shhwtstamps, regval))
 761		return;
 762
 763	switch (adapter->link_speed) {
 764	case SPEED_10:
 765		adjust = IGC_I225_TX_LATENCY_10;
 766		break;
 767	case SPEED_100:
 768		adjust = IGC_I225_TX_LATENCY_100;
 769		break;
 770	case SPEED_1000:
 771		adjust = IGC_I225_TX_LATENCY_1000;
 772		break;
 773	case SPEED_2500:
 774		adjust = IGC_I225_TX_LATENCY_2500;
 775		break;
 776	}
 777
 778	shhwtstamps.hwtstamp =
 779		ktime_add_ns(shhwtstamps.hwtstamp, adjust);
 780
 781	/* Copy the tx hardware timestamp into xdp metadata or skb */
 782	if (tstamp->buffer_type == IGC_TX_BUFFER_TYPE_XSK) {
 783		struct xsk_buff_pool *xsk_pool;
 784
 785		xsk_pool = adapter->tx_ring[tstamp->xsk_queue_index]->xsk_pool;
 786		if (xsk_pool && xp_tx_metadata_enabled(xsk_pool)) {
 787			xsk_tx_metadata_complete(&tstamp->xsk_meta,
 788						 &igc_xsk_tx_metadata_ops,
 789						 &shhwtstamps.hwtstamp);
 790		}
 791	} else {
 792		skb_tstamp_tx(skb, &shhwtstamps);
 793	}
 794
 795	igc_ptp_free_tx_buffer(adapter, tstamp);
 796}
 797
 798/**
 799 * igc_ptp_tx_hwtstamp - utility function which checks for TX time stamp
 800 * @adapter: Board private structure
 801 *
 802 * Check against the ready mask for which of the timestamp register
 803 * sets are ready to be retrieved, then retrieve that and notify the
 804 * rest of the stack.
 805 *
 806 * Context: Expects adapter->ptp_tx_lock to be held by caller.
 807 */
 808static void igc_ptp_tx_hwtstamp(struct igc_adapter *adapter)
 809{
 810	struct igc_hw *hw = &adapter->hw;
 811	u32 txstmpl_old;
 812	u64 regval;
 813	u32 mask;
 814	int i;
 815
 816	/* Establish baseline of TXSTMPL_0 before checking TXTT_0.
 817	 * This baseline is used to detect if a new timestamp arrives in
 818	 * register 0 during the hardware bug workaround below.
 819	 */
 820	txstmpl_old = rd32(IGC_TXSTMPL);
 821
 822	mask = rd32(IGC_TSYNCTXCTL) & IGC_TSYNCTXCTL_TXTT_ANY;
 823	if (mask & IGC_TSYNCTXCTL_TXTT_0) {
 824		regval = rd32(IGC_TXSTMPL);
 825		regval |= (u64)rd32(IGC_TXSTMPH) << 32;
 826	} else {
 827		/* TXTT_0 not set - register 0 has no new timestamp initially.
 828		 *
 829		 * Hardware bug: Future timestamp interrupts won't fire unless
 830		 * TXSTMPH_0 is read, even if the timestamp was captured in
 831		 * registers 1-3.
 832		 *
 833		 * Workaround: Read TXSTMPH_0 here to enable future interrupts.
 834		 * However, this read clears TXTT_0. If a timestamp arrives in
 835		 * register 0 after checking TXTT_0 but before this read, it
 836		 * would be lost.
 837		 *
 838		 * To detect this race: We saved a baseline read of TXSTMPL_0
 839		 * before TXTT_0 check. After performing the workaround read of
 840		 * TXSTMPH_0, we read TXSTMPL_0 again. Since consecutive
 841		 * timestamps never share the same nanosecond value, a change
 842		 * between the baseline and new TXSTMPL_0 indicates a timestamp
 843		 * arrived during the race window. If so, read the complete
 844		 * timestamp.
 845		 */
 846		u32 txstmpl_new;
 847
 848		rd32(IGC_TXSTMPH);
 849		txstmpl_new = rd32(IGC_TXSTMPL);
 850
 851		if (txstmpl_old == txstmpl_new)
 852			goto done;
 853
 854		regval = txstmpl_new;
 855		regval |= (u64)rd32(IGC_TXSTMPH) << 32;
 856	}
 857
 858	igc_ptp_tx_reg_to_stamp(adapter, &adapter->tx_tstamp[0], regval);
 859
 860done:
 861	/* Now that the problematic first register was handled, we can
 862	 * retrieve the timestamps from the other registers
 863	 * (starting from '1') with less complications.
 864	 */
 865	for (i = 1; i < IGC_MAX_TX_TSTAMP_REGS; i++) {
 866		struct igc_tx_timestamp_request *tstamp = &adapter->tx_tstamp[i];
 867
 868		if (!(tstamp->mask & mask))
 869			continue;
 870
 871		regval = rd32(tstamp->regl);
 872		regval |= (u64)rd32(tstamp->regh) << 32;
 873
 874		igc_ptp_tx_reg_to_stamp(adapter, tstamp, regval);
 875	}
 876}
 877
 878/**
 879 * igc_ptp_tx_tstamp_event
 880 * @adapter: board private structure
 881 *
 882 * Called when a TX timestamp interrupt happens to retrieve the
 883 * timestamp and send it up to the socket.
 884 */
 885void igc_ptp_tx_tstamp_event(struct igc_adapter *adapter)
 886{
 887	unsigned long flags;
 888
 889	spin_lock_irqsave(&adapter->ptp_tx_lock, flags);
 890
 891	igc_ptp_tx_hwtstamp(adapter);
 892
 893	spin_unlock_irqrestore(&adapter->ptp_tx_lock, flags);
 894}
 895
 896/**
 897 * igc_ptp_hwtstamp_set - set hardware time stamping config
 898 * @netdev: network interface device structure
 899 * @config: timestamping configuration structure
 900 * @extack: netlink extended ack structure for error reporting
 901 *
 902 **/
 903int igc_ptp_hwtstamp_set(struct net_device *netdev,
 904			 struct kernel_hwtstamp_config *config,
 905			 struct netlink_ext_ack *extack)
 906{
 907	struct igc_adapter *adapter = netdev_priv(netdev);
 908	int err;
 909
 910	err = igc_ptp_set_timestamp_mode(adapter, config);
 911	if (err)
 912		return err;
 913
 914	/* save these settings for future reference */
 915	adapter->tstamp_config = *config;
 916
 917	return 0;
 918}
 919
 920/**
 921 * igc_ptp_hwtstamp_get - get hardware time stamping config
 922 * @netdev: network interface device structure
 923 * @config: timestamping configuration structure
 924 *
 925 * Get the hwtstamp_config settings to return to the user. Rather than attempt
 926 * to deconstruct the settings from the registers, just return a shadow copy
 927 * of the last known settings.
 928 **/
 929int igc_ptp_hwtstamp_get(struct net_device *netdev,
 930			 struct kernel_hwtstamp_config *config)
 931{
 932	struct igc_adapter *adapter = netdev_priv(netdev);
 933
 934	*config = adapter->tstamp_config;
 935
 936	return 0;
 937}
 938
 939/* The two conditions below must be met for cross timestamping via
 940 * PCIe PTM:
 941 *
 942 * 1. We have an way to convert the timestamps in the PTM messages
 943 *    to something related to the system clocks (right now, only
 944 *    X86 systems with support for the Always Running Timer allow that);
 945 *
 946 * 2. We have PTM enabled in the path from the device to the PCIe root port.
 947 */
 948static bool igc_is_crosststamp_supported(struct igc_adapter *adapter)
 949{
 950	if (!IS_ENABLED(CONFIG_X86_TSC))
 951		return false;
 952
 953	/* FIXME: it was noticed that enabling support for PCIe PTM in
 954	 * some i225-V models could cause lockups when bringing the
 955	 * interface up/down. There should be no downsides to
 956	 * disabling crosstimestamping support for i225-V, as it
 957	 * doesn't have any PTP support. That way we gain some time
 958	 * while root causing the issue.
 959	 */
 960	if (adapter->pdev->device == IGC_DEV_ID_I225_V)
 961		return false;
 962
 963	return pcie_ptm_enabled(adapter->pdev);
 964}
 965
 966static struct system_counterval_t igc_device_tstamp_to_system(u64 tstamp)
 967{
 968#if IS_ENABLED(CONFIG_X86_TSC) && !defined(CONFIG_UML)
 969	return (struct system_counterval_t) {
 970		.cs_id		= CSID_X86_ART,
 971		.cycles		= tstamp,
 972		.use_nsecs	= true,
 973	};
 974#else
 975	return (struct system_counterval_t) { };
 976#endif
 977}
 978
 979static void igc_ptm_log_error(struct igc_adapter *adapter, u32 ptm_stat)
 980{
 981	struct net_device *netdev = adapter->netdev;
 982
 983	switch (ptm_stat) {
 984	case IGC_PTM_STAT_RET_ERR:
 985		netdev_err(netdev, "PTM Error: Root port timeout\n");
 986		break;
 987	case IGC_PTM_STAT_BAD_PTM_RES:
 988		netdev_err(netdev, "PTM Error: Bad response, PTM Response Data expected\n");
 989		break;
 990	case IGC_PTM_STAT_T4M1_OVFL:
 991		netdev_err(netdev, "PTM Error: T4 minus T1 overflow\n");
 992		break;
 993	case IGC_PTM_STAT_ADJUST_1ST:
 994		netdev_err(netdev, "PTM Error: 1588 timer adjusted during first PTM cycle\n");
 995		break;
 996	case IGC_PTM_STAT_ADJUST_CYC:
 997		netdev_err(netdev, "PTM Error: 1588 timer adjusted during non-first PTM cycle\n");
 998		break;
 999	default:
1000		netdev_err(netdev, "PTM Error: Unknown error (%#x)\n", ptm_stat);
1001		break;
1002	}
1003}
1004
1005/* The PTM lock: adapter->ptm_lock must be held when calling igc_ptm_trigger() */
1006static void igc_ptm_trigger(struct igc_hw *hw)
1007{
1008	u32 ctrl;
1009
1010	/* To "manually" start the PTM cycle we need to set the
1011	 * trigger (TRIG) bit
1012	 */
1013	ctrl = rd32(IGC_PTM_CTRL);
1014	ctrl |= IGC_PTM_CTRL_TRIG;
1015	wr32(IGC_PTM_CTRL, ctrl);
1016	/* Perform flush after write to CTRL register otherwise
1017	 * transaction may not start
1018	 */
1019	wrfl();
1020}
1021
1022/* The PTM lock: adapter->ptm_lock must be held when calling igc_ptm_reset() */
1023static void igc_ptm_reset(struct igc_hw *hw)
1024{
1025	u32 ctrl;
1026
1027	ctrl = rd32(IGC_PTM_CTRL);
1028	ctrl &= ~IGC_PTM_CTRL_TRIG;
1029	wr32(IGC_PTM_CTRL, ctrl);
1030	/* Write to clear all status */
1031	wr32(IGC_PTM_STAT, IGC_PTM_STAT_ALL);
1032}
1033
1034static int igc_phc_get_syncdevicetime(ktime_t *device,
1035				      struct system_counterval_t *system,
1036				      void *ctx)
1037{
1038	struct igc_adapter *adapter = ctx;
1039	struct igc_hw *hw = &adapter->hw;
1040	u32 stat, t2_curr_h, t2_curr_l;
1041	int err, count = 100;
1042	ktime_t t1, t2_curr;
1043
1044	/* Doing this in a loop because in the event of a
1045	 * badly timed (ha!) system clock adjustment, we may
1046	 * get PTM errors from the PCI root, but these errors
1047	 * are transitory. Repeating the process returns valid
1048	 * data eventually.
1049	 */
1050	do {
1051		/* Get a snapshot of system clocks to use as historic value. */
1052		ktime_get_snapshot(&adapter->snapshot);
1053
1054		igc_ptm_trigger(hw);
1055
1056		err = readx_poll_timeout(rd32, IGC_PTM_STAT, stat,
1057					 stat, IGC_PTM_STAT_SLEEP,
1058					 IGC_PTM_STAT_TIMEOUT);
1059		igc_ptm_reset(hw);
1060
1061		if (err < 0) {
1062			netdev_err(adapter->netdev, "Timeout reading IGC_PTM_STAT register\n");
1063			return err;
1064		}
1065
1066		if ((stat & IGC_PTM_STAT_VALID) == IGC_PTM_STAT_VALID)
1067			break;
1068
1069		igc_ptm_log_error(adapter, stat);
1070	} while (--count);
1071
1072	if (!count) {
1073		netdev_err(adapter->netdev, "Exceeded number of tries for PTM cycle\n");
1074		return -ETIMEDOUT;
1075	}
1076
1077	t1 = ktime_set(rd32(IGC_PTM_T1_TIM0_H), rd32(IGC_PTM_T1_TIM0_L));
1078
1079	t2_curr_l = rd32(IGC_PTM_CURR_T2_L);
1080	t2_curr_h = rd32(IGC_PTM_CURR_T2_H);
1081
1082	/* FIXME: When the register that tells the endianness of the
1083	 * PTM registers are implemented, check them here and add the
1084	 * appropriate conversion.
1085	 */
1086	t2_curr_h = swab32(t2_curr_h);
1087
1088	t2_curr = ((s64)t2_curr_h << 32 | t2_curr_l);
1089
1090	*device = t1;
1091	*system = igc_device_tstamp_to_system(t2_curr);
1092
1093	return 0;
1094}
1095
1096static int igc_ptp_getcrosststamp(struct ptp_clock_info *ptp,
1097				  struct system_device_crosststamp *cts)
1098{
1099	struct igc_adapter *adapter = container_of(ptp, struct igc_adapter,
1100						   ptp_caps);
1101	int ret;
1102
1103	/* This blocks until any in progress PTM transactions complete */
1104	mutex_lock(&adapter->ptm_lock);
1105
1106	ret = get_device_system_crosststamp(igc_phc_get_syncdevicetime,
1107					    adapter, &adapter->snapshot, cts);
1108	mutex_unlock(&adapter->ptm_lock);
1109
1110	return ret;
1111}
1112
1113static int igc_ptp_getcyclesx64(struct ptp_clock_info *ptp,
1114				struct timespec64 *ts,
1115				struct ptp_system_timestamp *sts)
1116{
1117	struct igc_adapter *igc = container_of(ptp, struct igc_adapter, ptp_caps);
1118	struct igc_hw *hw = &igc->hw;
1119	unsigned long flags;
1120
1121	spin_lock_irqsave(&igc->free_timer_lock, flags);
1122
1123	ptp_read_system_prets(sts);
1124	ts->tv_nsec = rd32(IGC_SYSTIML_1);
1125	ts->tv_sec = rd32(IGC_SYSTIMH_1);
1126	ptp_read_system_postts(sts);
1127
1128	spin_unlock_irqrestore(&igc->free_timer_lock, flags);
1129
1130	return 0;
1131}
1132
1133/**
1134 * igc_ptp_init - Initialize PTP functionality
1135 * @adapter: Board private structure
1136 *
1137 * This function is called at device probe to initialize the PTP
1138 * functionality.
1139 */
1140void igc_ptp_init(struct igc_adapter *adapter)
1141{
1142	struct net_device *netdev = adapter->netdev;
1143	struct igc_tx_timestamp_request *tstamp;
1144	struct igc_hw *hw = &adapter->hw;
1145	int i;
1146
1147	tstamp = &adapter->tx_tstamp[0];
1148	tstamp->mask = IGC_TSYNCTXCTL_TXTT_0;
1149	tstamp->regl = IGC_TXSTMPL_0;
1150	tstamp->regh = IGC_TXSTMPH_0;
1151	tstamp->flags = 0;
1152
1153	tstamp = &adapter->tx_tstamp[1];
1154	tstamp->mask = IGC_TSYNCTXCTL_TXTT_1;
1155	tstamp->regl = IGC_TXSTMPL_1;
1156	tstamp->regh = IGC_TXSTMPH_1;
1157	tstamp->flags = IGC_TX_FLAGS_TSTAMP_1;
1158
1159	tstamp = &adapter->tx_tstamp[2];
1160	tstamp->mask = IGC_TSYNCTXCTL_TXTT_2;
1161	tstamp->regl = IGC_TXSTMPL_2;
1162	tstamp->regh = IGC_TXSTMPH_2;
1163	tstamp->flags = IGC_TX_FLAGS_TSTAMP_2;
1164
1165	tstamp = &adapter->tx_tstamp[3];
1166	tstamp->mask = IGC_TSYNCTXCTL_TXTT_3;
1167	tstamp->regl = IGC_TXSTMPL_3;
1168	tstamp->regh = IGC_TXSTMPH_3;
1169	tstamp->flags = IGC_TX_FLAGS_TSTAMP_3;
1170
1171	switch (hw->mac.type) {
1172	case igc_i225:
1173		for (i = 0; i < IGC_N_SDP; i++) {
1174			struct ptp_pin_desc *ppd = &adapter->sdp_config[i];
1175
1176			snprintf(ppd->name, sizeof(ppd->name), "SDP%d", i);
1177			ppd->index = i;
1178			ppd->func = PTP_PF_NONE;
1179		}
1180		snprintf(adapter->ptp_caps.name, 16, "%pm", netdev->dev_addr);
1181		adapter->ptp_caps.owner = THIS_MODULE;
1182		adapter->ptp_caps.max_adj = 62499999;
1183		adapter->ptp_caps.adjfine = igc_ptp_adjfine_i225;
1184		adapter->ptp_caps.adjtime = igc_ptp_adjtime_i225;
1185		adapter->ptp_caps.gettimex64 = igc_ptp_gettimex64_i225;
1186		adapter->ptp_caps.getcyclesx64 = igc_ptp_getcyclesx64;
1187		adapter->ptp_caps.settime64 = igc_ptp_settime_i225;
1188		adapter->ptp_caps.enable = igc_ptp_feature_enable_i225;
1189		adapter->ptp_caps.pps = 1;
1190		adapter->ptp_caps.pin_config = adapter->sdp_config;
1191		adapter->ptp_caps.n_ext_ts = IGC_N_EXTTS;
1192		adapter->ptp_caps.n_per_out = IGC_N_PEROUT;
1193		adapter->ptp_caps.supported_extts_flags = PTP_RISING_EDGE |
1194							  PTP_FALLING_EDGE |
1195							  PTP_STRICT_FLAGS;
1196		adapter->ptp_caps.n_pins = IGC_N_SDP;
1197		adapter->ptp_caps.verify = igc_ptp_verify_pin;
1198
1199		if (!igc_is_crosststamp_supported(adapter))
1200			break;
1201
1202		adapter->ptp_caps.getcrosststamp = igc_ptp_getcrosststamp;
1203		break;
1204	default:
1205		adapter->ptp_clock = NULL;
1206		return;
1207	}
1208
1209	spin_lock_init(&adapter->ptp_tx_lock);
1210	spin_lock_init(&adapter->free_timer_lock);
1211	spin_lock_init(&adapter->tmreg_lock);
1212	mutex_init(&adapter->ptm_lock);
1213
1214	adapter->tstamp_config.rx_filter = HWTSTAMP_FILTER_NONE;
1215	adapter->tstamp_config.tx_type = HWTSTAMP_TX_OFF;
1216
1217	adapter->prev_ptp_time = ktime_to_timespec64(ktime_get_real());
1218	adapter->ptp_reset_start = ktime_get();
1219
1220	adapter->ptp_clock = ptp_clock_register(&adapter->ptp_caps,
1221						&adapter->pdev->dev);
1222	if (IS_ERR(adapter->ptp_clock)) {
1223		adapter->ptp_clock = NULL;
1224		netdev_err(netdev, "ptp_clock_register failed\n");
1225		mutex_destroy(&adapter->ptm_lock);
1226	} else if (adapter->ptp_clock) {
1227		netdev_info(netdev, "PHC added\n");
1228		adapter->ptp_flags |= IGC_PTP_ENABLED;
1229	}
1230}
1231
1232static void igc_ptp_time_save(struct igc_adapter *adapter)
1233{
1234	igc_ptp_read(adapter, &adapter->prev_ptp_time);
1235	adapter->ptp_reset_start = ktime_get();
1236}
1237
1238static void igc_ptp_time_restore(struct igc_adapter *adapter)
1239{
1240	struct timespec64 ts = adapter->prev_ptp_time;
1241	ktime_t delta;
1242
1243	delta = ktime_sub(ktime_get(), adapter->ptp_reset_start);
1244
1245	timespec64_add_ns(&ts, ktime_to_ns(delta));
1246
1247	igc_ptp_write_i225(adapter, &ts);
1248}
1249
1250static void igc_ptm_stop(struct igc_adapter *adapter)
1251{
1252	struct igc_hw *hw = &adapter->hw;
1253	u32 ctrl;
1254
1255	mutex_lock(&adapter->ptm_lock);
1256	ctrl = rd32(IGC_PTM_CTRL);
1257	ctrl &= ~IGC_PTM_CTRL_EN;
1258
1259	wr32(IGC_PTM_CTRL, ctrl);
1260	mutex_unlock(&adapter->ptm_lock);
1261}
1262
1263/**
1264 * igc_ptp_suspend - Disable PTP work items and prepare for suspend
1265 * @adapter: Board private structure
1266 *
1267 * This function stops the overflow check work and PTP Tx timestamp work, and
1268 * will prepare the device for OS suspend.
1269 */
1270void igc_ptp_suspend(struct igc_adapter *adapter)
1271{
1272	if (!(adapter->ptp_flags & IGC_PTP_ENABLED))
1273		return;
1274
1275	igc_ptp_clear_tx_tstamp(adapter);
1276
1277	if (pci_device_is_present(adapter->pdev)) {
1278		igc_ptp_time_save(adapter);
1279		igc_ptm_stop(adapter);
1280	}
1281}
1282
1283/**
1284 * igc_ptp_stop - Disable PTP device and stop the overflow check.
1285 * @adapter: Board private structure.
1286 *
1287 * This function stops the PTP support and cancels the delayed work.
1288 **/
1289void igc_ptp_stop(struct igc_adapter *adapter)
1290{
1291	if (!(adapter->ptp_flags & IGC_PTP_ENABLED))
1292		return;
1293
1294	igc_ptp_suspend(adapter);
1295
1296	adapter->ptp_flags &= ~IGC_PTP_ENABLED;
1297	if (adapter->ptp_clock) {
1298		ptp_clock_unregister(adapter->ptp_clock);
1299		netdev_info(adapter->netdev, "PHC removed\n");
1300		adapter->ptp_flags &= ~IGC_PTP_ENABLED;
1301	}
1302	mutex_destroy(&adapter->ptm_lock);
1303}
1304
1305/**
1306 * igc_ptp_reset - Re-enable the adapter for PTP following a reset.
1307 * @adapter: Board private structure.
1308 *
1309 * This function handles the reset work required to re-enable the PTP device.
1310 **/
1311void igc_ptp_reset(struct igc_adapter *adapter)
1312{
1313	struct igc_hw *hw = &adapter->hw;
1314	u32 cycle_ctrl, ctrl, stat;
1315	unsigned long flags;
1316	u32 timadj;
1317
1318	if (!(adapter->ptp_flags & IGC_PTP_ENABLED))
1319		return;
1320
1321	/* reset the tstamp_config */
1322	igc_ptp_set_timestamp_mode(adapter, &adapter->tstamp_config);
1323
1324	mutex_lock(&adapter->ptm_lock);
1325
1326	spin_lock_irqsave(&adapter->tmreg_lock, flags);
1327
1328	switch (adapter->hw.mac.type) {
1329	case igc_i225:
1330		timadj = rd32(IGC_TIMADJ);
1331		timadj |= IGC_TIMADJ_ADJUST_METH;
1332		wr32(IGC_TIMADJ, timadj);
1333
1334		wr32(IGC_TSAUXC, 0x0);
1335		wr32(IGC_TSSDP, 0x0);
1336		wr32(IGC_TSIM,
1337		     IGC_TSICR_INTERRUPTS |
1338		     (adapter->pps_sys_wrap_on ? IGC_TSICR_SYS_WRAP : 0));
1339		wr32(IGC_IMS, IGC_IMS_TS);
1340
1341		if (!igc_is_crosststamp_supported(adapter))
1342			break;
1343
1344		wr32(IGC_PCIE_DIG_DELAY, IGC_PCIE_DIG_DELAY_DEFAULT);
1345		wr32(IGC_PCIE_PHY_DELAY, IGC_PCIE_PHY_DELAY_DEFAULT);
1346
1347		cycle_ctrl = IGC_PTM_CYCLE_CTRL_CYC_TIME(IGC_PTM_CYC_TIME_DEFAULT);
1348
1349		wr32(IGC_PTM_CYCLE_CTRL, cycle_ctrl);
1350
1351		ctrl = IGC_PTM_CTRL_EN |
1352			IGC_PTM_CTRL_START_NOW |
1353			IGC_PTM_CTRL_SHRT_CYC(IGC_PTM_SHORT_CYC_DEFAULT) |
1354			IGC_PTM_CTRL_PTM_TO(IGC_PTM_TIMEOUT_DEFAULT);
1355
1356		wr32(IGC_PTM_CTRL, ctrl);
1357
1358		/* Force the first cycle to run. */
1359		igc_ptm_trigger(hw);
1360
1361		if (readx_poll_timeout_atomic(rd32, IGC_PTM_STAT, stat,
1362					      stat, IGC_PTM_STAT_SLEEP,
1363					      IGC_PTM_STAT_TIMEOUT))
1364			netdev_err(adapter->netdev, "Timeout reading IGC_PTM_STAT register\n");
1365
1366		igc_ptm_reset(hw);
1367		break;
1368	default:
1369		/* No work to do. */
1370		goto out;
1371	}
1372
1373	/* Re-initialize the timer. */
1374	if (hw->mac.type == igc_i225) {
1375		igc_ptp_time_restore(adapter);
1376	} else {
1377		timecounter_init(&adapter->tc, &adapter->cc,
1378				 ktime_to_ns(ktime_get_real()));
1379	}
1380out:
1381	spin_unlock_irqrestore(&adapter->tmreg_lock, flags);
1382
1383	mutex_unlock(&adapter->ptm_lock);
1384
1385	wrfl();
1386}