igb_ptp.c

   1// SPDX-License-Identifier: GPL-2.0+
   2/* Copyright (C) 2011 Richard Cochran <richardcochran@gmail.com> */
   3
   4#include <linux/module.h>
   5#include <linux/device.h>
   6#include <linux/pci.h>
   7#include <linux/ptp_classify.h>
   8
   9#include "igb.h"
  10
  11#define INCVALUE_MASK		0x7fffffff
  12#define ISGN			0x80000000
  13
  14/* The 82580 timesync updates the system timer every 8ns by 8ns,
  15 * and this update value cannot be reprogrammed.
  16 *
  17 * Neither the 82576 nor the 82580 offer registers wide enough to hold
  18 * nanoseconds time values for very long. For the 82580, SYSTIM always
  19 * counts nanoseconds, but the upper 24 bits are not available. The
  20 * frequency is adjusted by changing the 32 bit fractional nanoseconds
  21 * register, TIMINCA.
  22 *
  23 * For the 82576, the SYSTIM register time unit is affect by the
  24 * choice of the 24 bit TININCA:IV (incvalue) field. Five bits of this
  25 * field are needed to provide the nominal 16 nanosecond period,
  26 * leaving 19 bits for fractional nanoseconds.
  27 *
  28 * We scale the NIC clock cycle by a large factor so that relatively
  29 * small clock corrections can be added or subtracted at each clock
  30 * tick. The drawbacks of a large factor are a) that the clock
  31 * register overflows more quickly (not such a big deal) and b) that
  32 * the increment per tick has to fit into 24 bits.  As a result we
  33 * need to use a shift of 19 so we can fit a value of 16 into the
  34 * TIMINCA register.
  35 *
  36 *
  37 *             SYSTIMH            SYSTIML
  38 *        +--------------+   +---+---+------+
  39 *  82576 |      32      |   | 8 | 5 |  19  |
  40 *        +--------------+   +---+---+------+
  41 *         \________ 45 bits _______/  fract
  42 *
  43 *        +----------+---+   +--------------+
  44 *  82580 |    24    | 8 |   |      32      |
  45 *        +----------+---+   +--------------+
  46 *          reserved  \______ 40 bits _____/
  47 *
  48 *
  49 * The 45 bit 82576 SYSTIM overflows every
  50 *   2^45 * 10^-9 / 3600 = 9.77 hours.
  51 *
  52 * The 40 bit 82580 SYSTIM overflows every
  53 *   2^40 * 10^-9 /  60  = 18.3 minutes.
  54 *
  55 * SYSTIM is converted to real time using a timecounter. As
  56 * timecounter_cyc2time() allows old timestamps, the timecounter needs
  57 * to be updated at least once per half of the SYSTIM interval.
  58 * Scheduling of delayed work is not very accurate, and also the NIC
  59 * clock can be adjusted to run up to 6% faster and the system clock
  60 * up to 10% slower, so we aim for 6 minutes to be sure the actual
  61 * interval in the NIC time is shorter than 9.16 minutes.
  62 */
  63
  64#define IGB_SYSTIM_OVERFLOW_PERIOD	(HZ * 60 * 6)
  65#define IGB_PTP_TX_TIMEOUT		(HZ * 15)
  66#define INCPERIOD_82576			BIT(E1000_TIMINCA_16NS_SHIFT)
  67#define INCVALUE_82576_MASK		GENMASK(E1000_TIMINCA_16NS_SHIFT - 1, 0)
  68#define INCVALUE_82576			(16u << IGB_82576_TSYNC_SHIFT)
  69#define IGB_NBITS_82580			40
  70#define IGB_82580_BASE_PERIOD		0x800000000
  71
  72static void igb_ptp_tx_hwtstamp(struct igb_adapter *adapter);
  73static void igb_ptp_sdp_init(struct igb_adapter *adapter);
  74
  75/* SYSTIM read access for the 82576 */
  76static u64 igb_ptp_read_82576(struct cyclecounter *cc)
  77{
  78	struct igb_adapter *igb = container_of(cc, struct igb_adapter, cc);
  79	struct e1000_hw *hw = &igb->hw;
  80	u64 val;
  81	u32 lo, hi;
  82
  83	lo = rd32(E1000_SYSTIML);
  84	hi = rd32(E1000_SYSTIMH);
  85
  86	val = ((u64) hi) << 32;
  87	val |= lo;
  88
  89	return val;
  90}
  91
  92/* SYSTIM read access for the 82580 */
  93static u64 igb_ptp_read_82580(struct cyclecounter *cc)
  94{
  95	struct igb_adapter *igb = container_of(cc, struct igb_adapter, cc);
  96	struct e1000_hw *hw = &igb->hw;
  97	u32 lo, hi;
  98	u64 val;
  99
 100	/* The timestamp latches on lowest register read. For the 82580
 101	 * the lowest register is SYSTIMR instead of SYSTIML.  However we only
 102	 * need to provide nanosecond resolution, so we just ignore it.
 103	 */
 104	rd32(E1000_SYSTIMR);
 105	lo = rd32(E1000_SYSTIML);
 106	hi = rd32(E1000_SYSTIMH);
 107
 108	val = ((u64) hi) << 32;
 109	val |= lo;
 110
 111	return val;
 112}
 113
 114/* SYSTIM read access for I210/I211 */
 115static void igb_ptp_read_i210(struct igb_adapter *adapter,
 116			      struct timespec64 *ts)
 117{
 118	struct e1000_hw *hw = &adapter->hw;
 119	u32 sec, nsec;
 120
 121	/* The timestamp latches on lowest register read. For I210/I211, the
 122	 * lowest register is SYSTIMR. Since we only need to provide nanosecond
 123	 * resolution, we can ignore it.
 124	 */
 125	rd32(E1000_SYSTIMR);
 126	nsec = rd32(E1000_SYSTIML);
 127	sec = rd32(E1000_SYSTIMH);
 128
 129	ts->tv_sec = sec;
 130	ts->tv_nsec = nsec;
 131}
 132
 133static void igb_ptp_write_i210(struct igb_adapter *adapter,
 134			       const struct timespec64 *ts)
 135{
 136	struct e1000_hw *hw = &adapter->hw;
 137
 138	/* Writing the SYSTIMR register is not necessary as it only provides
 139	 * sub-nanosecond resolution.
 140	 */
 141	wr32(E1000_SYSTIML, ts->tv_nsec);
 142	wr32(E1000_SYSTIMH, (u32)ts->tv_sec);
 143}
 144
 145/**
 146 * igb_ptp_systim_to_hwtstamp - convert system time value to hw timestamp
 147 * @adapter: board private structure
 148 * @hwtstamps: timestamp structure to update
 149 * @systim: unsigned 64bit system time value.
 150 *
 151 * We need to convert the system time value stored in the RX/TXSTMP registers
 152 * into a hwtstamp which can be used by the upper level timestamping functions.
 153 *
 154 * The 'tmreg_lock' spinlock is used to protect the consistency of the
 155 * system time value. This is needed because reading the 64 bit time
 156 * value involves reading two (or three) 32 bit registers. The first
 157 * read latches the value. Ditto for writing.
 158 *
 159 * In addition, here have extended the system time with an overflow
 160 * counter in software.
 161 **/
 162static void igb_ptp_systim_to_hwtstamp(struct igb_adapter *adapter,
 163				       struct skb_shared_hwtstamps *hwtstamps,
 164				       u64 systim)
 165{
 166	unsigned long flags;
 167	u64 ns;
 168
 169	memset(hwtstamps, 0, sizeof(*hwtstamps));
 170
 171	switch (adapter->hw.mac.type) {
 172	case e1000_82576:
 173	case e1000_82580:
 174	case e1000_i354:
 175	case e1000_i350:
 176		spin_lock_irqsave(&adapter->tmreg_lock, flags);
 177		ns = timecounter_cyc2time(&adapter->tc, systim);
 178		spin_unlock_irqrestore(&adapter->tmreg_lock, flags);
 179
 180		hwtstamps->hwtstamp = ns_to_ktime(ns);
 181		break;
 182	case e1000_i210:
 183	case e1000_i211:
 184		/* Upper 32 bits contain s, lower 32 bits contain ns. */
 185		hwtstamps->hwtstamp = ktime_set(systim >> 32,
 186						systim & 0xFFFFFFFF);
 187		break;
 188	default:
 189		break;
 190	}
 191}
 192
 193/* PTP clock operations */
 194static int igb_ptp_adjfine_82576(struct ptp_clock_info *ptp, long scaled_ppm)
 195{
 196	struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
 197					       ptp_caps);
 198	struct e1000_hw *hw = &igb->hw;
 199	u64 incvalue;
 200
 201	incvalue = adjust_by_scaled_ppm(INCVALUE_82576, scaled_ppm);
 202
 203	wr32(E1000_TIMINCA, INCPERIOD_82576 | (incvalue & INCVALUE_82576_MASK));
 204
 205	return 0;
 206}
 207
 208static int igb_ptp_adjfine_82580(struct ptp_clock_info *ptp, long scaled_ppm)
 209{
 210	struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
 211					       ptp_caps);
 212	struct e1000_hw *hw = &igb->hw;
 213	bool neg_adj;
 214	u64 rate;
 215	u32 inca;
 216
 217	neg_adj = diff_by_scaled_ppm(IGB_82580_BASE_PERIOD, scaled_ppm, &rate);
 218
 219	inca = rate & INCVALUE_MASK;
 220	if (neg_adj)
 221		inca |= ISGN;
 222
 223	wr32(E1000_TIMINCA, inca);
 224
 225	return 0;
 226}
 227
 228static int igb_ptp_adjtime_82576(struct ptp_clock_info *ptp, s64 delta)
 229{
 230	struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
 231					       ptp_caps);
 232	unsigned long flags;
 233
 234	spin_lock_irqsave(&igb->tmreg_lock, flags);
 235	timecounter_adjtime(&igb->tc, delta);
 236	spin_unlock_irqrestore(&igb->tmreg_lock, flags);
 237
 238	return 0;
 239}
 240
 241static int igb_ptp_adjtime_i210(struct ptp_clock_info *ptp, s64 delta)
 242{
 243	struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
 244					       ptp_caps);
 245	unsigned long flags;
 246	struct timespec64 now, then = ns_to_timespec64(delta);
 247
 248	spin_lock_irqsave(&igb->tmreg_lock, flags);
 249
 250	igb_ptp_read_i210(igb, &now);
 251	now = timespec64_add(now, then);
 252	igb_ptp_write_i210(igb, (const struct timespec64 *)&now);
 253
 254	spin_unlock_irqrestore(&igb->tmreg_lock, flags);
 255
 256	return 0;
 257}
 258
 259static int igb_ptp_gettimex_82576(struct ptp_clock_info *ptp,
 260				  struct timespec64 *ts,
 261				  struct ptp_system_timestamp *sts)
 262{
 263	struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
 264					       ptp_caps);
 265	struct e1000_hw *hw = &igb->hw;
 266	unsigned long flags;
 267	u32 lo, hi;
 268	u64 ns;
 269
 270	spin_lock_irqsave(&igb->tmreg_lock, flags);
 271
 272	ptp_read_system_prets(sts);
 273	lo = rd32(E1000_SYSTIML);
 274	ptp_read_system_postts(sts);
 275	hi = rd32(E1000_SYSTIMH);
 276
 277	ns = timecounter_cyc2time(&igb->tc, ((u64)hi << 32) | lo);
 278
 279	spin_unlock_irqrestore(&igb->tmreg_lock, flags);
 280
 281	*ts = ns_to_timespec64(ns);
 282
 283	return 0;
 284}
 285
 286static int igb_ptp_gettimex_82580(struct ptp_clock_info *ptp,
 287				  struct timespec64 *ts,
 288				  struct ptp_system_timestamp *sts)
 289{
 290	struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
 291					       ptp_caps);
 292	struct e1000_hw *hw = &igb->hw;
 293	unsigned long flags;
 294	u32 lo, hi;
 295	u64 ns;
 296
 297	spin_lock_irqsave(&igb->tmreg_lock, flags);
 298
 299	ptp_read_system_prets(sts);
 300	rd32(E1000_SYSTIMR);
 301	ptp_read_system_postts(sts);
 302	lo = rd32(E1000_SYSTIML);
 303	hi = rd32(E1000_SYSTIMH);
 304
 305	ns = timecounter_cyc2time(&igb->tc, ((u64)hi << 32) | lo);
 306
 307	spin_unlock_irqrestore(&igb->tmreg_lock, flags);
 308
 309	*ts = ns_to_timespec64(ns);
 310
 311	return 0;
 312}
 313
 314static int igb_ptp_gettimex_i210(struct ptp_clock_info *ptp,
 315				 struct timespec64 *ts,
 316				 struct ptp_system_timestamp *sts)
 317{
 318	struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
 319					       ptp_caps);
 320	struct e1000_hw *hw = &igb->hw;
 321	unsigned long flags;
 322
 323	spin_lock_irqsave(&igb->tmreg_lock, flags);
 324
 325	ptp_read_system_prets(sts);
 326	rd32(E1000_SYSTIMR);
 327	ptp_read_system_postts(sts);
 328	ts->tv_nsec = rd32(E1000_SYSTIML);
 329	ts->tv_sec = rd32(E1000_SYSTIMH);
 330
 331	spin_unlock_irqrestore(&igb->tmreg_lock, flags);
 332
 333	return 0;
 334}
 335
 336static int igb_ptp_settime_82576(struct ptp_clock_info *ptp,
 337				 const struct timespec64 *ts)
 338{
 339	struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
 340					       ptp_caps);
 341	unsigned long flags;
 342	u64 ns;
 343
 344	ns = timespec64_to_ns(ts);
 345
 346	spin_lock_irqsave(&igb->tmreg_lock, flags);
 347
 348	timecounter_init(&igb->tc, &igb->cc, ns);
 349
 350	spin_unlock_irqrestore(&igb->tmreg_lock, flags);
 351
 352	return 0;
 353}
 354
 355static int igb_ptp_settime_i210(struct ptp_clock_info *ptp,
 356				const struct timespec64 *ts)
 357{
 358	struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
 359					       ptp_caps);
 360	unsigned long flags;
 361
 362	spin_lock_irqsave(&igb->tmreg_lock, flags);
 363
 364	igb_ptp_write_i210(igb, ts);
 365
 366	spin_unlock_irqrestore(&igb->tmreg_lock, flags);
 367
 368	return 0;
 369}
 370
 371static void igb_pin_direction(int pin, int input, u32 *ctrl, u32 *ctrl_ext)
 372{
 373	u32 *ptr = pin < 2 ? ctrl : ctrl_ext;
 374	static const u32 mask[IGB_N_SDP] = {
 375		E1000_CTRL_SDP0_DIR,
 376		E1000_CTRL_SDP1_DIR,
 377		E1000_CTRL_EXT_SDP2_DIR,
 378		E1000_CTRL_EXT_SDP3_DIR,
 379	};
 380
 381	if (input)
 382		*ptr &= ~mask[pin];
 383	else
 384		*ptr |= mask[pin];
 385}
 386
 387static void igb_pin_extts(struct igb_adapter *igb, int chan, int pin)
 388{
 389	static const u32 aux0_sel_sdp[IGB_N_SDP] = {
 390		AUX0_SEL_SDP0, AUX0_SEL_SDP1, AUX0_SEL_SDP2, AUX0_SEL_SDP3,
 391	};
 392	static const u32 aux1_sel_sdp[IGB_N_SDP] = {
 393		AUX1_SEL_SDP0, AUX1_SEL_SDP1, AUX1_SEL_SDP2, AUX1_SEL_SDP3,
 394	};
 395	static const u32 ts_sdp_en[IGB_N_SDP] = {
 396		TS_SDP0_EN, TS_SDP1_EN, TS_SDP2_EN, TS_SDP3_EN,
 397	};
 398	struct e1000_hw *hw = &igb->hw;
 399	u32 ctrl, ctrl_ext, tssdp = 0;
 400
 401	ctrl = rd32(E1000_CTRL);
 402	ctrl_ext = rd32(E1000_CTRL_EXT);
 403	tssdp = rd32(E1000_TSSDP);
 404
 405	igb_pin_direction(pin, 1, &ctrl, &ctrl_ext);
 406
 407	/* Make sure this pin is not enabled as an output. */
 408	tssdp &= ~ts_sdp_en[pin];
 409
 410	if (chan == 1) {
 411		tssdp &= ~AUX1_SEL_SDP3;
 412		tssdp |= aux1_sel_sdp[pin] | AUX1_TS_SDP_EN;
 413	} else {
 414		tssdp &= ~AUX0_SEL_SDP3;
 415		tssdp |= aux0_sel_sdp[pin] | AUX0_TS_SDP_EN;
 416	}
 417
 418	wr32(E1000_TSSDP, tssdp);
 419	wr32(E1000_CTRL, ctrl);
 420	wr32(E1000_CTRL_EXT, ctrl_ext);
 421}
 422
 423static void igb_pin_perout(struct igb_adapter *igb, int chan, int pin, int freq)
 424{
 425	static const u32 aux0_sel_sdp[IGB_N_SDP] = {
 426		AUX0_SEL_SDP0, AUX0_SEL_SDP1, AUX0_SEL_SDP2, AUX0_SEL_SDP3,
 427	};
 428	static const u32 aux1_sel_sdp[IGB_N_SDP] = {
 429		AUX1_SEL_SDP0, AUX1_SEL_SDP1, AUX1_SEL_SDP2, AUX1_SEL_SDP3,
 430	};
 431	static const u32 ts_sdp_en[IGB_N_SDP] = {
 432		TS_SDP0_EN, TS_SDP1_EN, TS_SDP2_EN, TS_SDP3_EN,
 433	};
 434	static const u32 ts_sdp_sel_tt0[IGB_N_SDP] = {
 435		TS_SDP0_SEL_TT0, TS_SDP1_SEL_TT0,
 436		TS_SDP2_SEL_TT0, TS_SDP3_SEL_TT0,
 437	};
 438	static const u32 ts_sdp_sel_tt1[IGB_N_SDP] = {
 439		TS_SDP0_SEL_TT1, TS_SDP1_SEL_TT1,
 440		TS_SDP2_SEL_TT1, TS_SDP3_SEL_TT1,
 441	};
 442	static const u32 ts_sdp_sel_fc0[IGB_N_SDP] = {
 443		TS_SDP0_SEL_FC0, TS_SDP1_SEL_FC0,
 444		TS_SDP2_SEL_FC0, TS_SDP3_SEL_FC0,
 445	};
 446	static const u32 ts_sdp_sel_fc1[IGB_N_SDP] = {
 447		TS_SDP0_SEL_FC1, TS_SDP1_SEL_FC1,
 448		TS_SDP2_SEL_FC1, TS_SDP3_SEL_FC1,
 449	};
 450	static const u32 ts_sdp_sel_clr[IGB_N_SDP] = {
 451		TS_SDP0_SEL_FC1, TS_SDP1_SEL_FC1,
 452		TS_SDP2_SEL_FC1, TS_SDP3_SEL_FC1,
 453	};
 454	struct e1000_hw *hw = &igb->hw;
 455	u32 ctrl, ctrl_ext, tssdp = 0;
 456
 457	ctrl = rd32(E1000_CTRL);
 458	ctrl_ext = rd32(E1000_CTRL_EXT);
 459	tssdp = rd32(E1000_TSSDP);
 460
 461	igb_pin_direction(pin, 0, &ctrl, &ctrl_ext);
 462
 463	/* Make sure this pin is not enabled as an input. */
 464	if ((tssdp & AUX0_SEL_SDP3) == aux0_sel_sdp[pin])
 465		tssdp &= ~AUX0_TS_SDP_EN;
 466
 467	if ((tssdp & AUX1_SEL_SDP3) == aux1_sel_sdp[pin])
 468		tssdp &= ~AUX1_TS_SDP_EN;
 469
 470	tssdp &= ~ts_sdp_sel_clr[pin];
 471	if (freq) {
 472		if (chan == 1)
 473			tssdp |= ts_sdp_sel_fc1[pin];
 474		else
 475			tssdp |= ts_sdp_sel_fc0[pin];
 476	} else {
 477		if (chan == 1)
 478			tssdp |= ts_sdp_sel_tt1[pin];
 479		else
 480			tssdp |= ts_sdp_sel_tt0[pin];
 481	}
 482	tssdp |= ts_sdp_en[pin];
 483
 484	wr32(E1000_TSSDP, tssdp);
 485	wr32(E1000_CTRL, ctrl);
 486	wr32(E1000_CTRL_EXT, ctrl_ext);
 487}
 488
 489static int igb_ptp_feature_enable_82580(struct ptp_clock_info *ptp,
 490					struct ptp_clock_request *rq, int on)
 491{
 492	struct igb_adapter *igb =
 493		container_of(ptp, struct igb_adapter, ptp_caps);
 494	u32 tsauxc, tsim, tsauxc_mask, tsim_mask, trgttiml, trgttimh, systiml,
 495		systimh, level_mask, level, rem;
 496	struct e1000_hw *hw = &igb->hw;
 497	struct timespec64 ts, start;
 498	unsigned long flags;
 499	u64 systim, now;
 500	int pin = -1;
 501	s64 ns;
 502
 503	switch (rq->type) {
 504	case PTP_CLK_REQ_EXTTS:
 505		/* Both the rising and falling edge are timestamped */
 506		if (rq->extts.flags & PTP_STRICT_FLAGS &&
 507		    (rq->extts.flags & PTP_ENABLE_FEATURE) &&
 508		    (rq->extts.flags & PTP_EXTTS_EDGES) != PTP_EXTTS_EDGES)
 509			return -EOPNOTSUPP;
 510
 511		if (on) {
 512			pin = ptp_find_pin(igb->ptp_clock, PTP_PF_EXTTS,
 513					   rq->extts.index);
 514			if (pin < 0)
 515				return -EBUSY;
 516		}
 517		if (rq->extts.index == 1) {
 518			tsauxc_mask = TSAUXC_EN_TS1;
 519			tsim_mask = TSINTR_AUTT1;
 520		} else {
 521			tsauxc_mask = TSAUXC_EN_TS0;
 522			tsim_mask = TSINTR_AUTT0;
 523		}
 524		spin_lock_irqsave(&igb->tmreg_lock, flags);
 525		tsauxc = rd32(E1000_TSAUXC);
 526		tsim = rd32(E1000_TSIM);
 527		if (on) {
 528			igb_pin_extts(igb, rq->extts.index, pin);
 529			tsauxc |= tsauxc_mask;
 530			tsim |= tsim_mask;
 531		} else {
 532			tsauxc &= ~tsauxc_mask;
 533			tsim &= ~tsim_mask;
 534		}
 535		wr32(E1000_TSAUXC, tsauxc);
 536		wr32(E1000_TSIM, tsim);
 537		spin_unlock_irqrestore(&igb->tmreg_lock, flags);
 538		return 0;
 539
 540	case PTP_CLK_REQ_PEROUT:
 541		/* Reject requests with unsupported flags */
 542		if (rq->perout.flags)
 543			return -EOPNOTSUPP;
 544
 545		if (on) {
 546			pin = ptp_find_pin(igb->ptp_clock, PTP_PF_PEROUT,
 547					   rq->perout.index);
 548			if (pin < 0)
 549				return -EBUSY;
 550		}
 551		ts.tv_sec = rq->perout.period.sec;
 552		ts.tv_nsec = rq->perout.period.nsec;
 553		ns = timespec64_to_ns(&ts);
 554		ns = ns >> 1;
 555		if (on && ns < 8LL)
 556			return -EINVAL;
 557		ts = ns_to_timespec64(ns);
 558		if (rq->perout.index == 1) {
 559			tsauxc_mask = TSAUXC_EN_TT1;
 560			tsim_mask = TSINTR_TT1;
 561			trgttiml = E1000_TRGTTIML1;
 562			trgttimh = E1000_TRGTTIMH1;
 563		} else {
 564			tsauxc_mask = TSAUXC_EN_TT0;
 565			tsim_mask = TSINTR_TT0;
 566			trgttiml = E1000_TRGTTIML0;
 567			trgttimh = E1000_TRGTTIMH0;
 568		}
 569		spin_lock_irqsave(&igb->tmreg_lock, flags);
 570		tsauxc = rd32(E1000_TSAUXC);
 571		tsim = rd32(E1000_TSIM);
 572		if (rq->perout.index == 1) {
 573			tsauxc &= ~(TSAUXC_EN_TT1 | TSAUXC_EN_CLK1 | TSAUXC_ST1);
 574			tsim &= ~TSINTR_TT1;
 575		} else {
 576			tsauxc &= ~(TSAUXC_EN_TT0 | TSAUXC_EN_CLK0 | TSAUXC_ST0);
 577			tsim &= ~TSINTR_TT0;
 578		}
 579		if (on) {
 580			int i = rq->perout.index;
 581
 582			/* read systim registers in sequence */
 583			rd32(E1000_SYSTIMR);
 584			systiml = rd32(E1000_SYSTIML);
 585			systimh = rd32(E1000_SYSTIMH);
 586			systim = (((u64)(systimh & 0xFF)) << 32) | ((u64)systiml);
 587			now = timecounter_cyc2time(&igb->tc, systim);
 588
 589			if (pin < 2) {
 590				level_mask = (i == 1) ? 0x80000 : 0x40000;
 591				level = (rd32(E1000_CTRL) & level_mask) ? 1 : 0;
 592			} else {
 593				level_mask = (i == 1) ? 0x80 : 0x40;
 594				level = (rd32(E1000_CTRL_EXT) & level_mask) ? 1 : 0;
 595			}
 596
 597			div_u64_rem(now, ns, &rem);
 598			systim = systim + (ns - rem);
 599
 600			/* synchronize pin level with rising/falling edges */
 601			div_u64_rem(now, ns << 1, &rem);
 602			if (rem < ns) {
 603				/* first half of period */
 604				if (level == 0) {
 605					/* output is already low, skip this period */
 606					systim += ns;
 607				}
 608			} else {
 609				/* second half of period */
 610				if (level == 1) {
 611					/* output is already high, skip this period */
 612					systim += ns;
 613				}
 614			}
 615
 616			start = ns_to_timespec64(systim + (ns - rem));
 617			igb_pin_perout(igb, i, pin, 0);
 618			igb->perout[i].start.tv_sec = start.tv_sec;
 619			igb->perout[i].start.tv_nsec = start.tv_nsec;
 620			igb->perout[i].period.tv_sec = ts.tv_sec;
 621			igb->perout[i].period.tv_nsec = ts.tv_nsec;
 622
 623			wr32(trgttiml, (u32)systim);
 624			wr32(trgttimh, ((u32)(systim >> 32)) & 0xFF);
 625			tsauxc |= tsauxc_mask;
 626			tsim |= tsim_mask;
 627		}
 628		wr32(E1000_TSAUXC, tsauxc);
 629		wr32(E1000_TSIM, tsim);
 630		spin_unlock_irqrestore(&igb->tmreg_lock, flags);
 631		return 0;
 632
 633	case PTP_CLK_REQ_PPS:
 634		return -EOPNOTSUPP;
 635	}
 636
 637	return -EOPNOTSUPP;
 638}
 639
 640static int igb_ptp_feature_enable_i210(struct ptp_clock_info *ptp,
 641				       struct ptp_clock_request *rq, int on)
 642{
 643	struct igb_adapter *igb =
 644		container_of(ptp, struct igb_adapter, ptp_caps);
 645	struct e1000_hw *hw = &igb->hw;
 646	u32 tsauxc, tsim, tsauxc_mask, tsim_mask, trgttiml, trgttimh, freqout;
 647	unsigned long flags;
 648	struct timespec64 ts;
 649	int use_freq = 0, pin = -1;
 650	s64 ns;
 651
 652	switch (rq->type) {
 653	case PTP_CLK_REQ_EXTTS:
 654		/* Reject requests failing to enable both edges. */
 655		if ((rq->extts.flags & PTP_STRICT_FLAGS) &&
 656		    (rq->extts.flags & PTP_ENABLE_FEATURE) &&
 657		    (rq->extts.flags & PTP_EXTTS_EDGES) != PTP_EXTTS_EDGES)
 658			return -EOPNOTSUPP;
 659
 660		if (on) {
 661			pin = ptp_find_pin(igb->ptp_clock, PTP_PF_EXTTS,
 662					   rq->extts.index);
 663			if (pin < 0)
 664				return -EBUSY;
 665		}
 666		if (rq->extts.index == 1) {
 667			tsauxc_mask = TSAUXC_EN_TS1;
 668			tsim_mask = TSINTR_AUTT1;
 669		} else {
 670			tsauxc_mask = TSAUXC_EN_TS0;
 671			tsim_mask = TSINTR_AUTT0;
 672		}
 673		spin_lock_irqsave(&igb->tmreg_lock, flags);
 674		tsauxc = rd32(E1000_TSAUXC);
 675		tsim = rd32(E1000_TSIM);
 676		if (on) {
 677			igb_pin_extts(igb, rq->extts.index, pin);
 678			tsauxc |= tsauxc_mask;
 679			tsim |= tsim_mask;
 680		} else {
 681			tsauxc &= ~tsauxc_mask;
 682			tsim &= ~tsim_mask;
 683		}
 684		wr32(E1000_TSAUXC, tsauxc);
 685		wr32(E1000_TSIM, tsim);
 686		spin_unlock_irqrestore(&igb->tmreg_lock, flags);
 687		return 0;
 688
 689	case PTP_CLK_REQ_PEROUT:
 690		/* Reject requests with unsupported flags */
 691		if (rq->perout.flags)
 692			return -EOPNOTSUPP;
 693
 694		if (on) {
 695			pin = ptp_find_pin(igb->ptp_clock, PTP_PF_PEROUT,
 696					   rq->perout.index);
 697			if (pin < 0)
 698				return -EBUSY;
 699		}
 700		ts.tv_sec = rq->perout.period.sec;
 701		ts.tv_nsec = rq->perout.period.nsec;
 702		ns = timespec64_to_ns(&ts);
 703		ns = ns >> 1;
 704		if (on && ((ns <= 70000000LL) || (ns == 125000000LL) ||
 705			   (ns == 250000000LL) || (ns == 500000000LL))) {
 706			if (ns < 8LL)
 707				return -EINVAL;
 708			use_freq = 1;
 709		}
 710		ts = ns_to_timespec64(ns);
 711		if (rq->perout.index == 1) {
 712			if (use_freq) {
 713				tsauxc_mask = TSAUXC_EN_CLK1 | TSAUXC_ST1;
 714				tsim_mask = 0;
 715			} else {
 716				tsauxc_mask = TSAUXC_EN_TT1;
 717				tsim_mask = TSINTR_TT1;
 718			}
 719			trgttiml = E1000_TRGTTIML1;
 720			trgttimh = E1000_TRGTTIMH1;
 721			freqout = E1000_FREQOUT1;
 722		} else {
 723			if (use_freq) {
 724				tsauxc_mask = TSAUXC_EN_CLK0 | TSAUXC_ST0;
 725				tsim_mask = 0;
 726			} else {
 727				tsauxc_mask = TSAUXC_EN_TT0;
 728				tsim_mask = TSINTR_TT0;
 729			}
 730			trgttiml = E1000_TRGTTIML0;
 731			trgttimh = E1000_TRGTTIMH0;
 732			freqout = E1000_FREQOUT0;
 733		}
 734		spin_lock_irqsave(&igb->tmreg_lock, flags);
 735		tsauxc = rd32(E1000_TSAUXC);
 736		tsim = rd32(E1000_TSIM);
 737		if (rq->perout.index == 1) {
 738			tsauxc &= ~(TSAUXC_EN_TT1 | TSAUXC_EN_CLK1 | TSAUXC_ST1);
 739			tsim &= ~TSINTR_TT1;
 740		} else {
 741			tsauxc &= ~(TSAUXC_EN_TT0 | TSAUXC_EN_CLK0 | TSAUXC_ST0);
 742			tsim &= ~TSINTR_TT0;
 743		}
 744		if (on) {
 745			int i = rq->perout.index;
 746			igb_pin_perout(igb, i, pin, use_freq);
 747			igb->perout[i].start.tv_sec = rq->perout.start.sec;
 748			igb->perout[i].start.tv_nsec = rq->perout.start.nsec;
 749			igb->perout[i].period.tv_sec = ts.tv_sec;
 750			igb->perout[i].period.tv_nsec = ts.tv_nsec;
 751			wr32(trgttimh, rq->perout.start.sec);
 752			wr32(trgttiml, rq->perout.start.nsec);
 753			if (use_freq)
 754				wr32(freqout, ns);
 755			tsauxc |= tsauxc_mask;
 756			tsim |= tsim_mask;
 757		}
 758		wr32(E1000_TSAUXC, tsauxc);
 759		wr32(E1000_TSIM, tsim);
 760		spin_unlock_irqrestore(&igb->tmreg_lock, flags);
 761		return 0;
 762
 763	case PTP_CLK_REQ_PPS:
 764		spin_lock_irqsave(&igb->tmreg_lock, flags);
 765		tsim = rd32(E1000_TSIM);
 766		if (on)
 767			tsim |= TSINTR_SYS_WRAP;
 768		else
 769			tsim &= ~TSINTR_SYS_WRAP;
 770		igb->pps_sys_wrap_on = !!on;
 771		wr32(E1000_TSIM, tsim);
 772		spin_unlock_irqrestore(&igb->tmreg_lock, flags);
 773		return 0;
 774	}
 775
 776	return -EOPNOTSUPP;
 777}
 778
 779static int igb_ptp_feature_enable(struct ptp_clock_info *ptp,
 780				  struct ptp_clock_request *rq, int on)
 781{
 782	return -EOPNOTSUPP;
 783}
 784
 785static int igb_ptp_verify_pin(struct ptp_clock_info *ptp, unsigned int pin,
 786			      enum ptp_pin_function func, unsigned int chan)
 787{
 788	switch (func) {
 789	case PTP_PF_NONE:
 790	case PTP_PF_EXTTS:
 791	case PTP_PF_PEROUT:
 792		break;
 793	case PTP_PF_PHYSYNC:
 794		return -1;
 795	}
 796	return 0;
 797}
 798
 799/**
 800 * igb_ptp_tx_work
 801 * @work: pointer to work struct
 802 *
 803 * This work function polls the TSYNCTXCTL valid bit to determine when a
 804 * timestamp has been taken for the current stored skb.
 805 **/
 806static void igb_ptp_tx_work(struct work_struct *work)
 807{
 808	struct igb_adapter *adapter = container_of(work, struct igb_adapter,
 809						   ptp_tx_work);
 810	struct e1000_hw *hw = &adapter->hw;
 811	u32 tsynctxctl;
 812
 813	if (!adapter->ptp_tx_skb)
 814		return;
 815
 816	if (time_is_before_jiffies(adapter->ptp_tx_start +
 817				   IGB_PTP_TX_TIMEOUT)) {
 818		dev_kfree_skb_any(adapter->ptp_tx_skb);
 819		adapter->ptp_tx_skb = NULL;
 820		clear_bit_unlock(__IGB_PTP_TX_IN_PROGRESS, &adapter->state);
 821		adapter->tx_hwtstamp_timeouts++;
 822		/* Clear the tx valid bit in TSYNCTXCTL register to enable
 823		 * interrupt
 824		 */
 825		rd32(E1000_TXSTMPH);
 826		dev_warn(&adapter->pdev->dev, "clearing Tx timestamp hang\n");
 827		return;
 828	}
 829
 830	tsynctxctl = rd32(E1000_TSYNCTXCTL);
 831	if (tsynctxctl & E1000_TSYNCTXCTL_VALID)
 832		igb_ptp_tx_hwtstamp(adapter);
 833	else
 834		/* reschedule to check later */
 835		schedule_work(&adapter->ptp_tx_work);
 836}
 837
 838static void igb_ptp_overflow_check(struct work_struct *work)
 839{
 840	struct igb_adapter *igb =
 841		container_of(work, struct igb_adapter, ptp_overflow_work.work);
 842	struct timespec64 ts;
 843
 844	/* Update the timecounter */
 845	ts = ns_to_timespec64(timecounter_read(&igb->tc));
 846
 847	pr_debug("igb overflow check at %ptSp\n", &ts);
 848
 849	schedule_delayed_work(&igb->ptp_overflow_work,
 850			      IGB_SYSTIM_OVERFLOW_PERIOD);
 851}
 852
 853/**
 854 * igb_ptp_rx_hang - detect error case when Rx timestamp registers latched
 855 * @adapter: private network adapter structure
 856 *
 857 * This watchdog task is scheduled to detect error case where hardware has
 858 * dropped an Rx packet that was timestamped when the ring is full. The
 859 * particular error is rare but leaves the device in a state unable to timestamp
 860 * any future packets.
 861 **/
 862void igb_ptp_rx_hang(struct igb_adapter *adapter)
 863{
 864	struct e1000_hw *hw = &adapter->hw;
 865	u32 tsyncrxctl = rd32(E1000_TSYNCRXCTL);
 866	unsigned long rx_event;
 867
 868	/* Other hardware uses per-packet timestamps */
 869	if (hw->mac.type != e1000_82576)
 870		return;
 871
 872	/* If we don't have a valid timestamp in the registers, just update the
 873	 * timeout counter and exit
 874	 */
 875	if (!(tsyncrxctl & E1000_TSYNCRXCTL_VALID)) {
 876		adapter->last_rx_ptp_check = jiffies;
 877		return;
 878	}
 879
 880	/* Determine the most recent watchdog or rx_timestamp event */
 881	rx_event = adapter->last_rx_ptp_check;
 882	if (time_after(adapter->last_rx_timestamp, rx_event))
 883		rx_event = adapter->last_rx_timestamp;
 884
 885	/* Only need to read the high RXSTMP register to clear the lock */
 886	if (time_is_before_jiffies(rx_event + 5 * HZ)) {
 887		rd32(E1000_RXSTMPH);
 888		adapter->last_rx_ptp_check = jiffies;
 889		adapter->rx_hwtstamp_cleared++;
 890		dev_warn(&adapter->pdev->dev, "clearing Rx timestamp hang\n");
 891	}
 892}
 893
 894/**
 895 * igb_ptp_tx_hang - detect error case where Tx timestamp never finishes
 896 * @adapter: private network adapter structure
 897 */
 898void igb_ptp_tx_hang(struct igb_adapter *adapter)
 899{
 900	struct e1000_hw *hw = &adapter->hw;
 901	bool timeout = time_is_before_jiffies(adapter->ptp_tx_start +
 902					      IGB_PTP_TX_TIMEOUT);
 903
 904	if (!adapter->ptp_tx_skb)
 905		return;
 906
 907	if (!test_bit(__IGB_PTP_TX_IN_PROGRESS, &adapter->state))
 908		return;
 909
 910	/* If we haven't received a timestamp within the timeout, it is
 911	 * reasonable to assume that it will never occur, so we can unlock the
 912	 * timestamp bit when this occurs.
 913	 */
 914	if (timeout) {
 915		cancel_work_sync(&adapter->ptp_tx_work);
 916		dev_kfree_skb_any(adapter->ptp_tx_skb);
 917		adapter->ptp_tx_skb = NULL;
 918		clear_bit_unlock(__IGB_PTP_TX_IN_PROGRESS, &adapter->state);
 919		adapter->tx_hwtstamp_timeouts++;
 920		/* Clear the tx valid bit in TSYNCTXCTL register to enable
 921		 * interrupt
 922		 */
 923		rd32(E1000_TXSTMPH);
 924		dev_warn(&adapter->pdev->dev, "clearing Tx timestamp hang\n");
 925	}
 926}
 927
 928/**
 929 * igb_ptp_tx_hwtstamp - utility function which checks for TX time stamp
 930 * @adapter: Board private structure.
 931 *
 932 * If we were asked to do hardware stamping and such a time stamp is
 933 * available, then it must have been for this skb here because we only
 934 * allow only one such packet into the queue.
 935 **/
 936static void igb_ptp_tx_hwtstamp(struct igb_adapter *adapter)
 937{
 938	struct sk_buff *skb = adapter->ptp_tx_skb;
 939	struct e1000_hw *hw = &adapter->hw;
 940	struct skb_shared_hwtstamps shhwtstamps;
 941	u64 regval;
 942	int adjust = 0;
 943
 944	regval = rd32(E1000_TXSTMPL);
 945	regval |= (u64)rd32(E1000_TXSTMPH) << 32;
 946
 947	igb_ptp_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
 948	/* adjust timestamp for the TX latency based on link speed */
 949	if (hw->mac.type == e1000_i210 || hw->mac.type == e1000_i211) {
 950		switch (adapter->link_speed) {
 951		case SPEED_10:
 952			adjust = IGB_I210_TX_LATENCY_10;
 953			break;
 954		case SPEED_100:
 955			adjust = IGB_I210_TX_LATENCY_100;
 956			break;
 957		case SPEED_1000:
 958			adjust = IGB_I210_TX_LATENCY_1000;
 959			break;
 960		}
 961	}
 962
 963	shhwtstamps.hwtstamp =
 964		ktime_add_ns(shhwtstamps.hwtstamp, adjust);
 965
 966	/* Clear the lock early before calling skb_tstamp_tx so that
 967	 * applications are not woken up before the lock bit is clear. We use
 968	 * a copy of the skb pointer to ensure other threads can't change it
 969	 * while we're notifying the stack.
 970	 */
 971	adapter->ptp_tx_skb = NULL;
 972	clear_bit_unlock(__IGB_PTP_TX_IN_PROGRESS, &adapter->state);
 973
 974	/* Notify the stack and free the skb after we've unlocked */
 975	skb_tstamp_tx(skb, &shhwtstamps);
 976	dev_kfree_skb_any(skb);
 977}
 978
 979/**
 980 * igb_ptp_rx_pktstamp - retrieve Rx per packet timestamp
 981 * @q_vector: Pointer to interrupt specific structure
 982 * @va: Pointer to address containing Rx buffer
 983 * @timestamp: Pointer where timestamp will be stored
 984 *
 985 * This function is meant to retrieve a timestamp from the first buffer of an
 986 * incoming frame.  The value is stored in little endian format starting on
 987 * byte 8
 988 *
 989 * Returns: The timestamp header length or 0 if not available
 990 **/
 991int igb_ptp_rx_pktstamp(struct igb_q_vector *q_vector, void *va,
 992			ktime_t *timestamp)
 993{
 994	struct igb_adapter *adapter = q_vector->adapter;
 995	struct e1000_hw *hw = &adapter->hw;
 996	struct skb_shared_hwtstamps ts;
 997	__le64 *regval = (__le64 *)va;
 998	int adjust = 0;
 999
1000	if (!(adapter->ptp_flags & IGB_PTP_ENABLED))
1001		return 0;
1002
1003	/* The timestamp is recorded in little endian format.
1004	 * DWORD: 0        1        2        3
1005	 * Field: Reserved Reserved SYSTIML  SYSTIMH
1006	 */
1007
1008	/* check reserved dwords are zero, be/le doesn't matter for zero */
1009	if (regval[0])
1010		return 0;
1011
1012	igb_ptp_systim_to_hwtstamp(adapter, &ts, le64_to_cpu(regval[1]));
1013
1014	/* adjust timestamp for the RX latency based on link speed */
1015	if (hw->mac.type == e1000_i210 || hw->mac.type == e1000_i211) {
1016		switch (adapter->link_speed) {
1017		case SPEED_10:
1018			adjust = IGB_I210_RX_LATENCY_10;
1019			break;
1020		case SPEED_100:
1021			adjust = IGB_I210_RX_LATENCY_100;
1022			break;
1023		case SPEED_1000:
1024			adjust = IGB_I210_RX_LATENCY_1000;
1025			break;
1026		}
1027	}
1028
1029	*timestamp = ktime_sub_ns(ts.hwtstamp, adjust);
1030
1031	return IGB_TS_HDR_LEN;
1032}
1033
1034/**
1035 * igb_ptp_rx_rgtstamp - retrieve Rx timestamp stored in register
1036 * @q_vector: Pointer to interrupt specific structure
1037 * @skb: Buffer containing timestamp and packet
1038 *
1039 * This function is meant to retrieve a timestamp from the internal registers
1040 * of the adapter and store it in the skb.
1041 **/
1042void igb_ptp_rx_rgtstamp(struct igb_q_vector *q_vector, struct sk_buff *skb)
1043{
1044	struct igb_adapter *adapter = q_vector->adapter;
1045	struct e1000_hw *hw = &adapter->hw;
1046	int adjust = 0;
1047	u64 regval;
1048
1049	if (!(adapter->ptp_flags & IGB_PTP_ENABLED))
1050		return;
1051
1052	/* If this bit is set, then the RX registers contain the time stamp. No
1053	 * other packet will be time stamped until we read these registers, so
1054	 * read the registers to make them available again. Because only one
1055	 * packet can be time stamped at a time, we know that the register
1056	 * values must belong to this one here and therefore we don't need to
1057	 * compare any of the additional attributes stored for it.
1058	 *
1059	 * If nothing went wrong, then it should have a shared tx_flags that we
1060	 * can turn into a skb_shared_hwtstamps.
1061	 */
1062	if (!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
1063		return;
1064
1065	regval = rd32(E1000_RXSTMPL);
1066	regval |= (u64)rd32(E1000_RXSTMPH) << 32;
1067
1068	igb_ptp_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
1069
1070	/* adjust timestamp for the RX latency based on link speed */
1071	if (adapter->hw.mac.type == e1000_i210) {
1072		switch (adapter->link_speed) {
1073		case SPEED_10:
1074			adjust = IGB_I210_RX_LATENCY_10;
1075			break;
1076		case SPEED_100:
1077			adjust = IGB_I210_RX_LATENCY_100;
1078			break;
1079		case SPEED_1000:
1080			adjust = IGB_I210_RX_LATENCY_1000;
1081			break;
1082		}
1083	}
1084	skb_hwtstamps(skb)->hwtstamp =
1085		ktime_sub_ns(skb_hwtstamps(skb)->hwtstamp, adjust);
1086
1087	/* Update the last_rx_timestamp timer in order to enable watchdog check
1088	 * for error case of latched timestamp on a dropped packet.
1089	 */
1090	adapter->last_rx_timestamp = jiffies;
1091}
1092
1093/**
1094 * igb_ptp_hwtstamp_get - get hardware time stamping config
1095 * @netdev: netdev struct
1096 * @config: timestamping configuration structure
1097 *
1098 * Get the hwtstamp_config settings to return to the user. Rather than attempt
1099 * to deconstruct the settings from the registers, just return a shadow copy
1100 * of the last known settings.
1101 **/
1102int igb_ptp_hwtstamp_get(struct net_device *netdev,
1103			 struct kernel_hwtstamp_config *config)
1104{
1105	struct igb_adapter *adapter = netdev_priv(netdev);
1106
1107	*config = adapter->tstamp_config;
1108
1109	return 0;
1110}
1111
1112/**
1113 * igb_ptp_set_timestamp_mode - setup hardware for timestamping
1114 * @adapter: networking device structure
1115 * @config: hwtstamp configuration
1116 *
1117 * Outgoing time stamping can be enabled and disabled. Play nice and
1118 * disable it when requested, although it shouldn't case any overhead
1119 * when no packet needs it. At most one packet in the queue may be
1120 * marked for time stamping, otherwise it would be impossible to tell
1121 * for sure to which packet the hardware time stamp belongs.
1122 *
1123 * Incoming time stamping has to be configured via the hardware
1124 * filters. Not all combinations are supported, in particular event
1125 * type has to be specified. Matching the kind of event packet is
1126 * not supported, with the exception of "all V2 events regardless of
1127 * level 2 or 4".
1128 */
1129static int igb_ptp_set_timestamp_mode(struct igb_adapter *adapter,
1130				      struct kernel_hwtstamp_config *config)
1131{
1132	struct e1000_hw *hw = &adapter->hw;
1133	u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
1134	u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
1135	u32 tsync_rx_cfg = 0;
1136	bool is_l4 = false;
1137	bool is_l2 = false;
1138	u32 regval;
1139
1140	switch (config->tx_type) {
1141	case HWTSTAMP_TX_OFF:
1142		tsync_tx_ctl = 0;
1143		break;
1144	case HWTSTAMP_TX_ON:
1145		break;
1146	default:
1147		return -ERANGE;
1148	}
1149
1150	switch (config->rx_filter) {
1151	case HWTSTAMP_FILTER_NONE:
1152		tsync_rx_ctl = 0;
1153		break;
1154	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
1155		tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
1156		tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
1157		is_l4 = true;
1158		break;
1159	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
1160		tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
1161		tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
1162		is_l4 = true;
1163		break;
1164	case HWTSTAMP_FILTER_PTP_V2_EVENT:
1165	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1166	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1167	case HWTSTAMP_FILTER_PTP_V2_SYNC:
1168	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1169	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1170	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1171	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1172	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1173		tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
1174		config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
1175		is_l2 = true;
1176		is_l4 = true;
1177		break;
1178	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
1179	case HWTSTAMP_FILTER_NTP_ALL:
1180	case HWTSTAMP_FILTER_ALL:
1181		/* 82576 cannot timestamp all packets, which it needs to do to
1182		 * support both V1 Sync and Delay_Req messages
1183		 */
1184		if (hw->mac.type != e1000_82576) {
1185			tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
1186			config->rx_filter = HWTSTAMP_FILTER_ALL;
1187			break;
1188		}
1189		fallthrough;
1190	default:
1191		config->rx_filter = HWTSTAMP_FILTER_NONE;
1192		return -ERANGE;
1193	}
1194
1195	if (hw->mac.type == e1000_82575) {
1196		if (tsync_rx_ctl | tsync_tx_ctl)
1197			return -EINVAL;
1198		return 0;
1199	}
1200
1201	/* Per-packet timestamping only works if all packets are
1202	 * timestamped, so enable timestamping in all packets as
1203	 * long as one Rx filter was configured.
1204	 */
1205	if ((hw->mac.type >= e1000_82580) && tsync_rx_ctl) {
1206		tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
1207		tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
1208		config->rx_filter = HWTSTAMP_FILTER_ALL;
1209		is_l2 = true;
1210		is_l4 = true;
1211
1212		if ((hw->mac.type == e1000_i210) ||
1213		    (hw->mac.type == e1000_i211)) {
1214			regval = rd32(E1000_RXPBS);
1215			regval |= E1000_RXPBS_CFG_TS_EN;
1216			wr32(E1000_RXPBS, regval);
1217		}
1218	}
1219
1220	/* enable/disable TX */
1221	regval = rd32(E1000_TSYNCTXCTL);
1222	regval &= ~E1000_TSYNCTXCTL_ENABLED;
1223	regval |= tsync_tx_ctl;
1224	wr32(E1000_TSYNCTXCTL, regval);
1225
1226	/* enable/disable RX */
1227	regval = rd32(E1000_TSYNCRXCTL);
1228	regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
1229	regval |= tsync_rx_ctl;
1230	wr32(E1000_TSYNCRXCTL, regval);
1231
1232	/* define which PTP packets are time stamped */
1233	wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
1234
1235	/* define ethertype filter for timestamped packets */
1236	if (is_l2)
1237		wr32(E1000_ETQF(IGB_ETQF_FILTER_1588),
1238		     (E1000_ETQF_FILTER_ENABLE | /* enable filter */
1239		      E1000_ETQF_1588 | /* enable timestamping */
1240		      ETH_P_1588));     /* 1588 eth protocol type */
1241	else
1242		wr32(E1000_ETQF(IGB_ETQF_FILTER_1588), 0);
1243
1244	/* L4 Queue Filter[3]: filter by destination port and protocol */
1245	if (is_l4) {
1246		u32 ftqf = (IPPROTO_UDP /* UDP */
1247			| E1000_FTQF_VF_BP /* VF not compared */
1248			| E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
1249			| E1000_FTQF_MASK); /* mask all inputs */
1250		ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
1251
1252		wr32(E1000_IMIR(3), (__force unsigned int)htons(PTP_EV_PORT));
1253		wr32(E1000_IMIREXT(3),
1254		     (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
1255		if (hw->mac.type == e1000_82576) {
1256			/* enable source port check */
1257			wr32(E1000_SPQF(3), (__force unsigned int)htons(PTP_EV_PORT));
1258			ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
1259		}
1260		wr32(E1000_FTQF(3), ftqf);
1261	} else {
1262		wr32(E1000_FTQF(3), E1000_FTQF_MASK);
1263	}
1264	wrfl();
1265
1266	/* clear TX/RX time stamp registers, just to be sure */
1267	regval = rd32(E1000_TXSTMPL);
1268	regval = rd32(E1000_TXSTMPH);
1269	regval = rd32(E1000_RXSTMPL);
1270	regval = rd32(E1000_RXSTMPH);
1271
1272	return 0;
1273}
1274
1275/**
1276 * igb_ptp_hwtstamp_set - set hardware time stamping config
1277 * @netdev: netdev struct
1278 * @config: timestamping configuration structure
1279 * @extack: netlink extended ack structure for error reporting
1280 **/
1281int igb_ptp_hwtstamp_set(struct net_device *netdev,
1282			 struct kernel_hwtstamp_config *config,
1283			 struct netlink_ext_ack *extack)
1284{
1285	struct igb_adapter *adapter = netdev_priv(netdev);
1286	int err;
1287
1288	err = igb_ptp_set_timestamp_mode(adapter, config);
1289	if (err)
1290		return err;
1291
1292	/* save these settings for future reference */
1293	adapter->tstamp_config = *config;
1294
1295	return 0;
1296}
1297
1298/**
1299 * igb_ptp_init - Initialize PTP functionality
1300 * @adapter: Board private structure
1301 *
1302 * This function is called at device probe to initialize the PTP
1303 * functionality.
1304 */
1305void igb_ptp_init(struct igb_adapter *adapter)
1306{
1307	struct e1000_hw *hw = &adapter->hw;
1308	struct net_device *netdev = adapter->netdev;
1309
1310	switch (hw->mac.type) {
1311	case e1000_82576:
1312		snprintf(adapter->ptp_caps.name, 16, "%pm", netdev->dev_addr);
1313		adapter->ptp_caps.owner = THIS_MODULE;
1314		adapter->ptp_caps.max_adj = 999999881;
1315		adapter->ptp_caps.n_ext_ts = 0;
1316		adapter->ptp_caps.pps = 0;
1317		adapter->ptp_caps.adjfine = igb_ptp_adjfine_82576;
1318		adapter->ptp_caps.adjtime = igb_ptp_adjtime_82576;
1319		adapter->ptp_caps.gettimex64 = igb_ptp_gettimex_82576;
1320		adapter->ptp_caps.settime64 = igb_ptp_settime_82576;
1321		adapter->ptp_caps.enable = igb_ptp_feature_enable;
1322		adapter->cc.read = igb_ptp_read_82576;
1323		adapter->cc.mask = CYCLECOUNTER_MASK(64);
1324		adapter->cc.mult = 1;
1325		adapter->cc.shift = IGB_82576_TSYNC_SHIFT;
1326		adapter->ptp_flags |= IGB_PTP_OVERFLOW_CHECK;
1327		break;
1328	case e1000_82580:
1329	case e1000_i354:
1330	case e1000_i350:
1331		igb_ptp_sdp_init(adapter);
1332		snprintf(adapter->ptp_caps.name, 16, "%pm", netdev->dev_addr);
1333		adapter->ptp_caps.owner = THIS_MODULE;
1334		adapter->ptp_caps.max_adj = 62499999;
1335		adapter->ptp_caps.n_ext_ts = IGB_N_EXTTS;
1336		adapter->ptp_caps.n_per_out = IGB_N_PEROUT;
1337		adapter->ptp_caps.n_pins = IGB_N_SDP;
1338		adapter->ptp_caps.pps = 0;
1339		adapter->ptp_caps.supported_extts_flags = PTP_RISING_EDGE |
1340							  PTP_FALLING_EDGE |
1341							  PTP_STRICT_FLAGS;
1342		adapter->ptp_caps.pin_config = adapter->sdp_config;
1343		adapter->ptp_caps.adjfine = igb_ptp_adjfine_82580;
1344		adapter->ptp_caps.adjtime = igb_ptp_adjtime_82576;
1345		adapter->ptp_caps.gettimex64 = igb_ptp_gettimex_82580;
1346		adapter->ptp_caps.settime64 = igb_ptp_settime_82576;
1347		adapter->ptp_caps.enable = igb_ptp_feature_enable_82580;
1348		adapter->ptp_caps.verify = igb_ptp_verify_pin;
1349		adapter->cc.read = igb_ptp_read_82580;
1350		adapter->cc.mask = CYCLECOUNTER_MASK(IGB_NBITS_82580);
1351		adapter->cc.mult = 1;
1352		adapter->cc.shift = 0;
1353		adapter->ptp_flags |= IGB_PTP_OVERFLOW_CHECK;
1354		break;
1355	case e1000_i210:
1356	case e1000_i211:
1357		igb_ptp_sdp_init(adapter);
1358		snprintf(adapter->ptp_caps.name, 16, "%pm", netdev->dev_addr);
1359		adapter->ptp_caps.owner = THIS_MODULE;
1360		adapter->ptp_caps.max_adj = 62499999;
1361		adapter->ptp_caps.n_ext_ts = IGB_N_EXTTS;
1362		adapter->ptp_caps.n_per_out = IGB_N_PEROUT;
1363		adapter->ptp_caps.n_pins = IGB_N_SDP;
1364		adapter->ptp_caps.supported_extts_flags = PTP_RISING_EDGE |
1365							  PTP_FALLING_EDGE |
1366							  PTP_STRICT_FLAGS;
1367		adapter->ptp_caps.pps = 1;
1368		adapter->ptp_caps.pin_config = adapter->sdp_config;
1369		adapter->ptp_caps.adjfine = igb_ptp_adjfine_82580;
1370		adapter->ptp_caps.adjtime = igb_ptp_adjtime_i210;
1371		adapter->ptp_caps.gettimex64 = igb_ptp_gettimex_i210;
1372		adapter->ptp_caps.settime64 = igb_ptp_settime_i210;
1373		adapter->ptp_caps.enable = igb_ptp_feature_enable_i210;
1374		adapter->ptp_caps.verify = igb_ptp_verify_pin;
1375		break;
1376	default:
1377		adapter->ptp_clock = NULL;
1378		return;
1379	}
1380
1381	adapter->ptp_clock = ptp_clock_register(&adapter->ptp_caps,
1382						&adapter->pdev->dev);
1383	if (IS_ERR(adapter->ptp_clock)) {
1384		adapter->ptp_clock = NULL;
1385		dev_err(&adapter->pdev->dev, "ptp_clock_register failed\n");
1386	} else if (adapter->ptp_clock) {
1387		dev_info(&adapter->pdev->dev, "added PHC on %s\n",
1388			 adapter->netdev->name);
1389		adapter->ptp_flags |= IGB_PTP_ENABLED;
1390
1391		spin_lock_init(&adapter->tmreg_lock);
1392		INIT_WORK(&adapter->ptp_tx_work, igb_ptp_tx_work);
1393
1394		if (adapter->ptp_flags & IGB_PTP_OVERFLOW_CHECK)
1395			INIT_DELAYED_WORK(&adapter->ptp_overflow_work,
1396					  igb_ptp_overflow_check);
1397
1398		adapter->tstamp_config.rx_filter = HWTSTAMP_FILTER_NONE;
1399		adapter->tstamp_config.tx_type = HWTSTAMP_TX_OFF;
1400
1401		igb_ptp_reset(adapter);
1402	}
1403}
1404
1405/**
1406 * igb_ptp_sdp_init - utility function which inits the SDP config structs
1407 * @adapter: Board private structure.
1408 **/
1409void igb_ptp_sdp_init(struct igb_adapter *adapter)
1410{
1411	int i;
1412
1413	for (i = 0; i < IGB_N_SDP; i++) {
1414		struct ptp_pin_desc *ppd = &adapter->sdp_config[i];
1415
1416		snprintf(ppd->name, sizeof(ppd->name), "SDP%d", i);
1417		ppd->index = i;
1418		ppd->func = PTP_PF_NONE;
1419	}
1420}
1421
1422/**
1423 * igb_ptp_suspend - Disable PTP work items and prepare for suspend
1424 * @adapter: Board private structure
1425 *
1426 * This function stops the overflow check work and PTP Tx timestamp work, and
1427 * will prepare the device for OS suspend.
1428 */
1429void igb_ptp_suspend(struct igb_adapter *adapter)
1430{
1431	if (!(adapter->ptp_flags & IGB_PTP_ENABLED))
1432		return;
1433
1434	if (adapter->ptp_flags & IGB_PTP_OVERFLOW_CHECK)
1435		cancel_delayed_work_sync(&adapter->ptp_overflow_work);
1436
1437	cancel_work_sync(&adapter->ptp_tx_work);
1438	if (adapter->ptp_tx_skb) {
1439		dev_kfree_skb_any(adapter->ptp_tx_skb);
1440		adapter->ptp_tx_skb = NULL;
1441		clear_bit_unlock(__IGB_PTP_TX_IN_PROGRESS, &adapter->state);
1442	}
1443}
1444
1445/**
1446 * igb_ptp_stop - Disable PTP device and stop the overflow check.
1447 * @adapter: Board private structure.
1448 *
1449 * This function stops the PTP support and cancels the delayed work.
1450 **/
1451void igb_ptp_stop(struct igb_adapter *adapter)
1452{
1453	igb_ptp_suspend(adapter);
1454
1455	if (adapter->ptp_clock) {
1456		ptp_clock_unregister(adapter->ptp_clock);
1457		dev_info(&adapter->pdev->dev, "removed PHC on %s\n",
1458			 adapter->netdev->name);
1459		adapter->ptp_flags &= ~IGB_PTP_ENABLED;
1460	}
1461}
1462
1463/**
1464 * igb_ptp_reset - Re-enable the adapter for PTP following a reset.
1465 * @adapter: Board private structure.
1466 *
1467 * This function handles the reset work required to re-enable the PTP device.
1468 **/
1469void igb_ptp_reset(struct igb_adapter *adapter)
1470{
1471	struct e1000_hw *hw = &adapter->hw;
1472	unsigned long flags;
1473
1474	/* reset the tstamp_config */
1475	igb_ptp_set_timestamp_mode(adapter, &adapter->tstamp_config);
1476
1477	spin_lock_irqsave(&adapter->tmreg_lock, flags);
1478
1479	switch (adapter->hw.mac.type) {
1480	case e1000_82576:
1481		/* Dial the nominal frequency. */
1482		wr32(E1000_TIMINCA, INCPERIOD_82576 | INCVALUE_82576);
1483		break;
1484	case e1000_82580:
1485	case e1000_i354:
1486	case e1000_i350:
1487	case e1000_i210:
1488	case e1000_i211:
1489		wr32(E1000_TSAUXC, 0x0);
1490		wr32(E1000_TSSDP, 0x0);
1491		wr32(E1000_TSIM,
1492		     TSYNC_INTERRUPTS |
1493		     (adapter->pps_sys_wrap_on ? TSINTR_SYS_WRAP : 0));
1494		wr32(E1000_IMS, E1000_IMS_TS);
1495		break;
1496	default:
1497		/* No work to do. */
1498		goto out;
1499	}
1500
1501	/* Re-initialize the timer. */
1502	if ((hw->mac.type == e1000_i210) || (hw->mac.type == e1000_i211)) {
1503		struct timespec64 ts = ktime_to_timespec64(ktime_get_real());
1504
1505		igb_ptp_write_i210(adapter, &ts);
1506	} else {
1507		timecounter_init(&adapter->tc, &adapter->cc,
1508				 ktime_to_ns(ktime_get_real()));
1509	}
1510out:
1511	spin_unlock_irqrestore(&adapter->tmreg_lock, flags);
1512
1513	wrfl();
1514
1515	if (adapter->ptp_flags & IGB_PTP_OVERFLOW_CHECK)
1516		schedule_delayed_work(&adapter->ptp_overflow_work,
1517				      IGB_SYSTIM_OVERFLOW_PERIOD);
1518}