idpf_ptp.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/* Copyright (C) 2024 Intel Corporation */
   3
   4#include "idpf.h"
   5#include "idpf_ptp.h"
   6
   7/**
   8 * idpf_ptp_get_access - Determine the access type of the PTP features
   9 * @adapter: Driver specific private structure
  10 * @direct: Capability that indicates the direct access
  11 * @mailbox: Capability that indicates the mailbox access
  12 *
  13 * Return: the type of supported access for the PTP feature.
  14 */
  15static enum idpf_ptp_access
  16idpf_ptp_get_access(const struct idpf_adapter *adapter, u32 direct, u32 mailbox)
  17{
  18	if (adapter->ptp->caps & direct)
  19		return IDPF_PTP_DIRECT;
  20	else if (adapter->ptp->caps & mailbox)
  21		return IDPF_PTP_MAILBOX;
  22	else
  23		return IDPF_PTP_NONE;
  24}
  25
  26/**
  27 * idpf_ptp_get_features_access - Determine the access type of PTP features
  28 * @adapter: Driver specific private structure
  29 *
  30 * Fulfill the adapter structure with type of the supported PTP features
  31 * access.
  32 */
  33void idpf_ptp_get_features_access(const struct idpf_adapter *adapter)
  34{
  35	struct idpf_ptp *ptp = adapter->ptp;
  36	u32 direct, mailbox;
  37
  38	/* Get the device clock time */
  39	direct = VIRTCHNL2_CAP_PTP_GET_DEVICE_CLK_TIME;
  40	mailbox = VIRTCHNL2_CAP_PTP_GET_DEVICE_CLK_TIME_MB;
  41	ptp->get_dev_clk_time_access = idpf_ptp_get_access(adapter,
  42							   direct,
  43							   mailbox);
  44
  45	/* Get the cross timestamp */
  46	direct = VIRTCHNL2_CAP_PTP_GET_CROSS_TIME;
  47	mailbox = VIRTCHNL2_CAP_PTP_GET_CROSS_TIME_MB;
  48	ptp->get_cross_tstamp_access = idpf_ptp_get_access(adapter,
  49							   direct,
  50							   mailbox);
  51
  52	/* Set the device clock time */
  53	direct = VIRTCHNL2_CAP_PTP_SET_DEVICE_CLK_TIME;
  54	mailbox = VIRTCHNL2_CAP_PTP_SET_DEVICE_CLK_TIME;
  55	ptp->set_dev_clk_time_access = idpf_ptp_get_access(adapter,
  56							   direct,
  57							   mailbox);
  58
  59	/* Adjust the device clock time */
  60	direct = VIRTCHNL2_CAP_PTP_ADJ_DEVICE_CLK;
  61	mailbox = VIRTCHNL2_CAP_PTP_ADJ_DEVICE_CLK_MB;
  62	ptp->adj_dev_clk_time_access = idpf_ptp_get_access(adapter,
  63							   direct,
  64							   mailbox);
  65
  66	/* Tx timestamping */
  67	direct = VIRTCHNL2_CAP_PTP_TX_TSTAMPS;
  68	mailbox = VIRTCHNL2_CAP_PTP_TX_TSTAMPS_MB;
  69	ptp->tx_tstamp_access = idpf_ptp_get_access(adapter,
  70						    direct,
  71						    mailbox);
  72}
  73
  74/**
  75 * idpf_ptp_enable_shtime - Enable shadow time and execute a command
  76 * @adapter: Driver specific private structure
  77 */
  78static void idpf_ptp_enable_shtime(struct idpf_adapter *adapter)
  79{
  80	u32 shtime_enable, exec_cmd;
  81
  82	/* Get offsets */
  83	shtime_enable = adapter->ptp->cmd.shtime_enable_mask;
  84	exec_cmd = adapter->ptp->cmd.exec_cmd_mask;
  85
  86	/* Set the shtime en and the sync field */
  87	writel(shtime_enable, adapter->ptp->dev_clk_regs.cmd_sync);
  88	writel(exec_cmd | shtime_enable, adapter->ptp->dev_clk_regs.cmd_sync);
  89}
  90
  91/**
  92 * idpf_ptp_read_src_clk_reg_direct - Read directly the main timer value
  93 * @adapter: Driver specific private structure
  94 * @sts: Optional parameter for holding a pair of system timestamps from
  95 *	 the system clock. Will be ignored when NULL is given.
  96 *
  97 * Return: the device clock time.
  98 */
  99static u64 idpf_ptp_read_src_clk_reg_direct(struct idpf_adapter *adapter,
 100					    struct ptp_system_timestamp *sts)
 101{
 102	struct idpf_ptp *ptp = adapter->ptp;
 103	u32 hi, lo;
 104
 105	spin_lock(&ptp->read_dev_clk_lock);
 106
 107	/* Read the system timestamp pre PHC read */
 108	ptp_read_system_prets(sts);
 109
 110	idpf_ptp_enable_shtime(adapter);
 111	lo = readl(ptp->dev_clk_regs.dev_clk_ns_l);
 112
 113	/* Read the system timestamp post PHC read */
 114	ptp_read_system_postts(sts);
 115
 116	hi = readl(ptp->dev_clk_regs.dev_clk_ns_h);
 117
 118	spin_unlock(&ptp->read_dev_clk_lock);
 119
 120	return ((u64)hi << 32) | lo;
 121}
 122
 123/**
 124 * idpf_ptp_read_src_clk_reg_mailbox - Read the main timer value through mailbox
 125 * @adapter: Driver specific private structure
 126 * @sts: Optional parameter for holding a pair of system timestamps from
 127 *	 the system clock. Will be ignored when NULL is given.
 128 * @src_clk: Returned main timer value in nanoseconds unit
 129 *
 130 * Return: 0 on success, -errno otherwise.
 131 */
 132static int idpf_ptp_read_src_clk_reg_mailbox(struct idpf_adapter *adapter,
 133					     struct ptp_system_timestamp *sts,
 134					     u64 *src_clk)
 135{
 136	struct idpf_ptp_dev_timers clk_time;
 137	int err;
 138
 139	/* Read the system timestamp pre PHC read */
 140	ptp_read_system_prets(sts);
 141
 142	err = idpf_ptp_get_dev_clk_time(adapter, &clk_time);
 143	if (err)
 144		return err;
 145
 146	/* Read the system timestamp post PHC read */
 147	ptp_read_system_postts(sts);
 148
 149	*src_clk = clk_time.dev_clk_time_ns;
 150
 151	return 0;
 152}
 153
 154/**
 155 * idpf_ptp_read_src_clk_reg - Read the main timer value
 156 * @adapter: Driver specific private structure
 157 * @src_clk: Returned main timer value in nanoseconds unit
 158 * @sts: Optional parameter for holding a pair of system timestamps from
 159 *	 the system clock. Will be ignored if NULL is given.
 160 *
 161 * Return: the device clock time on success, -errno otherwise.
 162 */
 163static int idpf_ptp_read_src_clk_reg(struct idpf_adapter *adapter, u64 *src_clk,
 164				     struct ptp_system_timestamp *sts)
 165{
 166	switch (adapter->ptp->get_dev_clk_time_access) {
 167	case IDPF_PTP_NONE:
 168		return -EOPNOTSUPP;
 169	case IDPF_PTP_MAILBOX:
 170		return idpf_ptp_read_src_clk_reg_mailbox(adapter, sts, src_clk);
 171	case IDPF_PTP_DIRECT:
 172		*src_clk = idpf_ptp_read_src_clk_reg_direct(adapter, sts);
 173		break;
 174	default:
 175		return -EOPNOTSUPP;
 176	}
 177
 178	return 0;
 179}
 180
 181#if IS_ENABLED(CONFIG_ARM_ARCH_TIMER) || IS_ENABLED(CONFIG_X86)
 182/**
 183 * idpf_ptp_get_sync_device_time_direct - Get the cross time stamp values
 184 *					  directly
 185 * @adapter: Driver specific private structure
 186 * @dev_time: 64bit main timer value
 187 * @sys_time: 64bit system time value
 188 */
 189static void idpf_ptp_get_sync_device_time_direct(struct idpf_adapter *adapter,
 190						 u64 *dev_time, u64 *sys_time)
 191{
 192	u32 dev_time_lo, dev_time_hi, sys_time_lo, sys_time_hi;
 193	struct idpf_ptp *ptp = adapter->ptp;
 194
 195	spin_lock(&ptp->read_dev_clk_lock);
 196
 197	idpf_ptp_enable_shtime(adapter);
 198
 199	dev_time_lo = readl(ptp->dev_clk_regs.dev_clk_ns_l);
 200	dev_time_hi = readl(ptp->dev_clk_regs.dev_clk_ns_h);
 201
 202	sys_time_lo = readl(ptp->dev_clk_regs.sys_time_ns_l);
 203	sys_time_hi = readl(ptp->dev_clk_regs.sys_time_ns_h);
 204
 205	spin_unlock(&ptp->read_dev_clk_lock);
 206
 207	*dev_time = (u64)dev_time_hi << 32 | dev_time_lo;
 208	*sys_time = (u64)sys_time_hi << 32 | sys_time_lo;
 209}
 210
 211/**
 212 * idpf_ptp_get_sync_device_time_mailbox - Get the cross time stamp values
 213 *					   through mailbox
 214 * @adapter: Driver specific private structure
 215 * @dev_time: 64bit main timer value expressed in nanoseconds
 216 * @sys_time: 64bit system time value expressed in nanoseconds
 217 *
 218 * Return: 0 on success, -errno otherwise.
 219 */
 220static int idpf_ptp_get_sync_device_time_mailbox(struct idpf_adapter *adapter,
 221						 u64 *dev_time, u64 *sys_time)
 222{
 223	struct idpf_ptp_dev_timers cross_time;
 224	int err;
 225
 226	err = idpf_ptp_get_cross_time(adapter, &cross_time);
 227	if (err)
 228		return err;
 229
 230	*dev_time = cross_time.dev_clk_time_ns;
 231	*sys_time = cross_time.sys_time_ns;
 232
 233	return err;
 234}
 235
 236/**
 237 * idpf_ptp_get_sync_device_time - Get the cross time stamp info
 238 * @device: Current device time
 239 * @system: System counter value read synchronously with device time
 240 * @ctx: Context provided by timekeeping code
 241 *
 242 * The device and the system clocks time read simultaneously.
 243 *
 244 * Return: 0 on success, -errno otherwise.
 245 */
 246static int idpf_ptp_get_sync_device_time(ktime_t *device,
 247					 struct system_counterval_t *system,
 248					 void *ctx)
 249{
 250	struct idpf_adapter *adapter = ctx;
 251	u64 ns_time_dev, ns_time_sys;
 252	int err;
 253
 254	switch (adapter->ptp->get_cross_tstamp_access) {
 255	case IDPF_PTP_NONE:
 256		return -EOPNOTSUPP;
 257	case IDPF_PTP_DIRECT:
 258		idpf_ptp_get_sync_device_time_direct(adapter, &ns_time_dev,
 259						     &ns_time_sys);
 260		break;
 261	case IDPF_PTP_MAILBOX:
 262		err = idpf_ptp_get_sync_device_time_mailbox(adapter,
 263							    &ns_time_dev,
 264							    &ns_time_sys);
 265		if (err)
 266			return err;
 267		break;
 268	default:
 269		return -EOPNOTSUPP;
 270	}
 271
 272	*device = ns_to_ktime(ns_time_dev);
 273
 274	system->cs_id = IS_ENABLED(CONFIG_X86) ? CSID_X86_ART
 275					       : CSID_ARM_ARCH_COUNTER;
 276	system->cycles = ns_time_sys;
 277	system->use_nsecs = true;
 278
 279	return 0;
 280}
 281
 282/**
 283 * idpf_ptp_get_crosststamp - Capture a device cross timestamp
 284 * @info: the driver's PTP info structure
 285 * @cts: The memory to fill the cross timestamp info
 286 *
 287 * Capture a cross timestamp between the system time and the device PTP hardware
 288 * clock.
 289 *
 290 * Return: cross timestamp value on success, -errno on failure.
 291 */
 292static int idpf_ptp_get_crosststamp(struct ptp_clock_info *info,
 293				    struct system_device_crosststamp *cts)
 294{
 295	struct idpf_adapter *adapter = idpf_ptp_info_to_adapter(info);
 296
 297	return get_device_system_crosststamp(idpf_ptp_get_sync_device_time,
 298					     adapter, NULL, cts);
 299}
 300#endif /* CONFIG_ARM_ARCH_TIMER || CONFIG_X86 */
 301
 302/**
 303 * idpf_ptp_gettimex64 - Get the time of the clock
 304 * @info: the driver's PTP info structure
 305 * @ts: timespec64 structure to hold the current time value
 306 * @sts: Optional parameter for holding a pair of system timestamps from
 307 *	 the system clock. Will be ignored if NULL is given.
 308 *
 309 * Return: the device clock value in ns, after converting it into a timespec
 310 * struct on success, -errno otherwise.
 311 */
 312static int idpf_ptp_gettimex64(struct ptp_clock_info *info,
 313			       struct timespec64 *ts,
 314			       struct ptp_system_timestamp *sts)
 315{
 316	struct idpf_adapter *adapter = idpf_ptp_info_to_adapter(info);
 317	u64 time_ns;
 318	int err;
 319
 320	err = idpf_ptp_read_src_clk_reg(adapter, &time_ns, sts);
 321	if (err)
 322		return -EACCES;
 323
 324	*ts = ns_to_timespec64(time_ns);
 325
 326	return 0;
 327}
 328
 329/**
 330 * idpf_ptp_update_phctime_rxq_grp - Update the cached PHC time for a given Rx
 331 *				     queue group.
 332 * @grp: receive queue group in which Rx timestamp is enabled
 333 * @split: Indicates whether the queue model is split or single queue
 334 * @systime: Cached system time
 335 */
 336static void
 337idpf_ptp_update_phctime_rxq_grp(const struct idpf_rxq_group *grp, bool split,
 338				u64 systime)
 339{
 340	struct idpf_rx_queue *rxq;
 341	u16 i;
 342
 343	if (!split) {
 344		for (i = 0; i < grp->singleq.num_rxq; i++) {
 345			rxq = grp->singleq.rxqs[i];
 346			if (rxq)
 347				WRITE_ONCE(rxq->cached_phc_time, systime);
 348		}
 349	} else {
 350		for (i = 0; i < grp->splitq.num_rxq_sets; i++) {
 351			rxq = &grp->splitq.rxq_sets[i]->rxq;
 352			if (rxq)
 353				WRITE_ONCE(rxq->cached_phc_time, systime);
 354		}
 355	}
 356}
 357
 358/**
 359 * idpf_ptp_update_cached_phctime - Update the cached PHC time values
 360 * @adapter: Driver specific private structure
 361 *
 362 * This function updates the system time values which are cached in the adapter
 363 * structure and the Rx queues.
 364 *
 365 * This function must be called periodically to ensure that the cached value
 366 * is never more than 2 seconds old.
 367 *
 368 * Return: 0 on success, -errno otherwise.
 369 */
 370static int idpf_ptp_update_cached_phctime(struct idpf_adapter *adapter)
 371{
 372	u64 systime;
 373	int err;
 374
 375	err = idpf_ptp_read_src_clk_reg(adapter, &systime, NULL);
 376	if (err)
 377		return -EACCES;
 378
 379	/* Update the cached PHC time stored in the adapter structure.
 380	 * These values are used to extend Tx timestamp values to 64 bit
 381	 * expected by the stack.
 382	 */
 383	WRITE_ONCE(adapter->ptp->cached_phc_time, systime);
 384	WRITE_ONCE(adapter->ptp->cached_phc_jiffies, jiffies);
 385
 386	idpf_for_each_vport(adapter, vport) {
 387		bool split;
 388
 389		if (!vport || !vport->rxq_grps)
 390			continue;
 391
 392		split = idpf_is_queue_model_split(vport->rxq_model);
 393
 394		for (u16 i = 0; i < vport->num_rxq_grp; i++) {
 395			struct idpf_rxq_group *grp = &vport->rxq_grps[i];
 396
 397			idpf_ptp_update_phctime_rxq_grp(grp, split, systime);
 398		}
 399	}
 400
 401	return 0;
 402}
 403
 404/**
 405 * idpf_ptp_settime64 - Set the time of the clock
 406 * @info: the driver's PTP info structure
 407 * @ts: timespec64 structure that holds the new time value
 408 *
 409 * Set the device clock to the user input value. The conversion from timespec
 410 * to ns happens in the write function.
 411 *
 412 * Return: 0 on success, -errno otherwise.
 413 */
 414static int idpf_ptp_settime64(struct ptp_clock_info *info,
 415			      const struct timespec64 *ts)
 416{
 417	struct idpf_adapter *adapter = idpf_ptp_info_to_adapter(info);
 418	enum idpf_ptp_access access;
 419	int err;
 420	u64 ns;
 421
 422	access = adapter->ptp->set_dev_clk_time_access;
 423	if (access != IDPF_PTP_MAILBOX)
 424		return -EOPNOTSUPP;
 425
 426	ns = timespec64_to_ns(ts);
 427
 428	err = idpf_ptp_set_dev_clk_time(adapter, ns);
 429	if (err) {
 430		pci_err(adapter->pdev, "Failed to set the time, err: %pe\n",
 431			ERR_PTR(err));
 432		return err;
 433	}
 434
 435	err = idpf_ptp_update_cached_phctime(adapter);
 436	if (err)
 437		pci_warn(adapter->pdev,
 438			 "Unable to immediately update cached PHC time\n");
 439
 440	return 0;
 441}
 442
 443/**
 444 * idpf_ptp_adjtime_nonatomic - Do a non-atomic clock adjustment
 445 * @info: the driver's PTP info structure
 446 * @delta: Offset in nanoseconds to adjust the time by
 447 *
 448 * Return: 0 on success, -errno otherwise.
 449 */
 450static int idpf_ptp_adjtime_nonatomic(struct ptp_clock_info *info, s64 delta)
 451{
 452	struct timespec64 now, then;
 453	int err;
 454
 455	err = idpf_ptp_gettimex64(info, &now, NULL);
 456	if (err)
 457		return err;
 458
 459	then = ns_to_timespec64(delta);
 460	now = timespec64_add(now, then);
 461
 462	return idpf_ptp_settime64(info, &now);
 463}
 464
 465/**
 466 * idpf_ptp_adjtime - Adjust the time of the clock by the indicated delta
 467 * @info: the driver's PTP info structure
 468 * @delta: Offset in nanoseconds to adjust the time by
 469 *
 470 * Return: 0 on success, -errno otherwise.
 471 */
 472static int idpf_ptp_adjtime(struct ptp_clock_info *info, s64 delta)
 473{
 474	struct idpf_adapter *adapter = idpf_ptp_info_to_adapter(info);
 475	enum idpf_ptp_access access;
 476	int err;
 477
 478	access = adapter->ptp->adj_dev_clk_time_access;
 479	if (access != IDPF_PTP_MAILBOX)
 480		return -EOPNOTSUPP;
 481
 482	/* Hardware only supports atomic adjustments using signed 32-bit
 483	 * integers. For any adjustment outside this range, perform
 484	 * a non-atomic get->adjust->set flow.
 485	 */
 486	if (delta > S32_MAX || delta < S32_MIN)
 487		return idpf_ptp_adjtime_nonatomic(info, delta);
 488
 489	err = idpf_ptp_adj_dev_clk_time(adapter, delta);
 490	if (err) {
 491		pci_err(adapter->pdev, "Failed to adjust the clock with delta %lld err: %pe\n",
 492			delta, ERR_PTR(err));
 493		return err;
 494	}
 495
 496	err = idpf_ptp_update_cached_phctime(adapter);
 497	if (err)
 498		pci_warn(adapter->pdev,
 499			 "Unable to immediately update cached PHC time\n");
 500
 501	return 0;
 502}
 503
 504/**
 505 * idpf_ptp_adjfine - Adjust clock increment rate
 506 * @info: the driver's PTP info structure
 507 * @scaled_ppm: Parts per million with 16-bit fractional field
 508 *
 509 * Adjust the frequency of the clock by the indicated scaled ppm from the
 510 * base frequency.
 511 *
 512 * Return: 0 on success, -errno otherwise.
 513 */
 514static int idpf_ptp_adjfine(struct ptp_clock_info *info, long scaled_ppm)
 515{
 516	struct idpf_adapter *adapter = idpf_ptp_info_to_adapter(info);
 517	enum idpf_ptp_access access;
 518	u64 incval, diff;
 519	int err;
 520
 521	access = adapter->ptp->adj_dev_clk_time_access;
 522	if (access != IDPF_PTP_MAILBOX)
 523		return -EOPNOTSUPP;
 524
 525	incval = adapter->ptp->base_incval;
 526
 527	diff = adjust_by_scaled_ppm(incval, scaled_ppm);
 528	err = idpf_ptp_adj_dev_clk_fine(adapter, diff);
 529	if (err)
 530		pci_err(adapter->pdev, "Failed to adjust clock increment rate for scaled ppm %ld %pe\n",
 531			scaled_ppm, ERR_PTR(err));
 532
 533	return 0;
 534}
 535
 536/**
 537 * idpf_ptp_verify_pin - Verify if pin supports requested pin function
 538 * @info: the driver's PTP info structure
 539 * @pin: Pin index
 540 * @func: Assigned function
 541 * @chan: Assigned channel
 542 *
 543 * Return: EOPNOTSUPP as not supported yet.
 544 */
 545static int idpf_ptp_verify_pin(struct ptp_clock_info *info, unsigned int pin,
 546			       enum ptp_pin_function func, unsigned int chan)
 547{
 548	return -EOPNOTSUPP;
 549}
 550
 551/**
 552 * idpf_ptp_gpio_enable - Enable/disable ancillary features of PHC
 553 * @info: the driver's PTP info structure
 554 * @rq: The requested feature to change
 555 * @on: Enable/disable flag
 556 *
 557 * Return: EOPNOTSUPP as not supported yet.
 558 */
 559static int idpf_ptp_gpio_enable(struct ptp_clock_info *info,
 560				struct ptp_clock_request *rq, int on)
 561{
 562	return -EOPNOTSUPP;
 563}
 564
 565/**
 566 * idpf_ptp_tstamp_extend_32b_to_64b - Convert a 32b nanoseconds Tx or Rx
 567 *				       timestamp value to 64b.
 568 * @cached_phc_time: recently cached copy of PHC time
 569 * @in_timestamp: Ingress/egress 32b nanoseconds timestamp value
 570 *
 571 * Hardware captures timestamps which contain only 32 bits of nominal
 572 * nanoseconds, as opposed to the 64bit timestamps that the stack expects.
 573 *
 574 * Return: Tx timestamp value extended to 64 bits based on cached PHC time.
 575 */
 576u64 idpf_ptp_tstamp_extend_32b_to_64b(u64 cached_phc_time, u32 in_timestamp)
 577{
 578	u32 delta, phc_time_lo;
 579	u64 ns;
 580
 581	/* Extract the lower 32 bits of the PHC time */
 582	phc_time_lo = (u32)cached_phc_time;
 583
 584	/* Calculate the delta between the lower 32bits of the cached PHC
 585	 * time and the in_timestamp value.
 586	 */
 587	delta = in_timestamp - phc_time_lo;
 588
 589	if (delta > U32_MAX / 2) {
 590		/* Reverse the delta calculation here */
 591		delta = phc_time_lo - in_timestamp;
 592		ns = cached_phc_time - delta;
 593	} else {
 594		ns = cached_phc_time + delta;
 595	}
 596
 597	return ns;
 598}
 599
 600/**
 601 * idpf_ptp_extend_ts - Convert a 40b timestamp to 64b nanoseconds
 602 * @vport: Virtual port structure
 603 * @in_tstamp: Ingress/egress timestamp value
 604 *
 605 * It is assumed that the caller verifies the timestamp is valid prior to
 606 * calling this function.
 607 *
 608 * Extract the 32bit nominal nanoseconds and extend them. Use the cached PHC
 609 * time stored in the device private PTP structure as the basis for timestamp
 610 * extension.
 611 *
 612 * Return: Tx timestamp value extended to 64 bits.
 613 */
 614u64 idpf_ptp_extend_ts(struct idpf_vport *vport, u64 in_tstamp)
 615{
 616	struct idpf_ptp *ptp = vport->adapter->ptp;
 617	unsigned long discard_time;
 618
 619	discard_time = ptp->cached_phc_jiffies + 2 * HZ;
 620
 621	if (time_is_before_jiffies(discard_time)) {
 622		u64_stats_update_begin(&vport->tstamp_stats.stats_sync);
 623		u64_stats_inc(&vport->tstamp_stats.discarded);
 624		u64_stats_update_end(&vport->tstamp_stats.stats_sync);
 625
 626		return 0;
 627	}
 628
 629	return idpf_ptp_tstamp_extend_32b_to_64b(ptp->cached_phc_time,
 630						 lower_32_bits(in_tstamp));
 631}
 632
 633/**
 634 * idpf_ptp_request_ts - Request an available Tx timestamp index
 635 * @tx_q: Transmit queue on which the Tx timestamp is requested
 636 * @skb: The SKB to associate with this timestamp request
 637 * @idx: Index of the Tx timestamp latch
 638 *
 639 * Request tx timestamp index negotiated during PTP init that will be set into
 640 * Tx descriptor.
 641 *
 642 * Return: 0 and the index that can be provided to Tx descriptor on success,
 643 * -errno otherwise.
 644 */
 645int idpf_ptp_request_ts(struct idpf_tx_queue *tx_q, struct sk_buff *skb,
 646			u32 *idx)
 647{
 648	struct idpf_ptp_tx_tstamp *ptp_tx_tstamp;
 649	struct list_head *head;
 650
 651	/* Get the index from the free latches list */
 652	spin_lock(&tx_q->cached_tstamp_caps->latches_lock);
 653
 654	head = &tx_q->cached_tstamp_caps->latches_free;
 655	if (list_empty(head)) {
 656		spin_unlock(&tx_q->cached_tstamp_caps->latches_lock);
 657		return -ENOBUFS;
 658	}
 659
 660	ptp_tx_tstamp = list_first_entry(head, struct idpf_ptp_tx_tstamp,
 661					 list_member);
 662	list_del(&ptp_tx_tstamp->list_member);
 663
 664	ptp_tx_tstamp->skb = skb_get(skb);
 665	skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
 666
 667	/* Move the element to the used latches list */
 668	list_add(&ptp_tx_tstamp->list_member,
 669		 &tx_q->cached_tstamp_caps->latches_in_use);
 670	spin_unlock(&tx_q->cached_tstamp_caps->latches_lock);
 671
 672	*idx = ptp_tx_tstamp->idx;
 673
 674	return 0;
 675}
 676
 677/**
 678 * idpf_ptp_set_rx_tstamp - Enable or disable Rx timestamping
 679 * @vport: Virtual port structure
 680 * @rx_filter: Receive timestamp filter
 681 */
 682static void idpf_ptp_set_rx_tstamp(struct idpf_vport *vport, int rx_filter)
 683{
 684	bool enable = true, splitq;
 685
 686	splitq = idpf_is_queue_model_split(vport->rxq_model);
 687
 688	if (rx_filter == HWTSTAMP_FILTER_NONE) {
 689		enable = false;
 690		vport->tstamp_config.rx_filter = HWTSTAMP_FILTER_NONE;
 691	} else {
 692		vport->tstamp_config.rx_filter = HWTSTAMP_FILTER_ALL;
 693	}
 694
 695	for (u16 i = 0; i < vport->num_rxq_grp; i++) {
 696		struct idpf_rxq_group *grp = &vport->rxq_grps[i];
 697		struct idpf_rx_queue *rx_queue;
 698		u16 j, num_rxq;
 699
 700		if (splitq)
 701			num_rxq = grp->splitq.num_rxq_sets;
 702		else
 703			num_rxq = grp->singleq.num_rxq;
 704
 705		for (j = 0; j < num_rxq; j++) {
 706			if (splitq)
 707				rx_queue = &grp->splitq.rxq_sets[j]->rxq;
 708			else
 709				rx_queue = grp->singleq.rxqs[j];
 710
 711			if (enable)
 712				idpf_queue_set(PTP, rx_queue);
 713			else
 714				idpf_queue_clear(PTP, rx_queue);
 715		}
 716	}
 717}
 718
 719/**
 720 * idpf_ptp_set_timestamp_mode - Setup driver for requested timestamp mode
 721 * @vport: Virtual port structure
 722 * @config: Hwtstamp settings requested or saved
 723 *
 724 * Return: 0 on success, -errno otherwise.
 725 */
 726int idpf_ptp_set_timestamp_mode(struct idpf_vport *vport,
 727				struct kernel_hwtstamp_config *config)
 728{
 729	switch (config->tx_type) {
 730	case HWTSTAMP_TX_OFF:
 731		break;
 732	case HWTSTAMP_TX_ON:
 733		if (!idpf_ptp_is_vport_tx_tstamp_ena(vport))
 734			return -EINVAL;
 735		break;
 736	default:
 737		return -EINVAL;
 738	}
 739
 740	vport->tstamp_config.tx_type = config->tx_type;
 741	idpf_ptp_set_rx_tstamp(vport, config->rx_filter);
 742	*config = vport->tstamp_config;
 743
 744	return 0;
 745}
 746
 747/**
 748 * idpf_tstamp_task - Delayed task to handle Tx tstamps
 749 * @work: work_struct handle
 750 */
 751void idpf_tstamp_task(struct work_struct *work)
 752{
 753	struct idpf_vport *vport;
 754
 755	vport = container_of(work, struct idpf_vport, tstamp_task);
 756
 757	idpf_ptp_get_tx_tstamp(vport);
 758}
 759
 760/**
 761 * idpf_ptp_do_aux_work - Do PTP periodic work
 762 * @info: Driver's PTP info structure
 763 *
 764 * Return: Number of jiffies to periodic work.
 765 */
 766static long idpf_ptp_do_aux_work(struct ptp_clock_info *info)
 767{
 768	struct idpf_adapter *adapter = idpf_ptp_info_to_adapter(info);
 769
 770	idpf_ptp_update_cached_phctime(adapter);
 771
 772	return msecs_to_jiffies(500);
 773}
 774
 775/**
 776 * idpf_ptp_set_caps - Set PTP capabilities
 777 * @adapter: Driver specific private structure
 778 *
 779 * This function sets the PTP functions.
 780 */
 781static void idpf_ptp_set_caps(const struct idpf_adapter *adapter)
 782{
 783	struct ptp_clock_info *info = &adapter->ptp->info;
 784
 785	snprintf(info->name, sizeof(info->name), "%s-%s-clk",
 786		 KBUILD_MODNAME, pci_name(adapter->pdev));
 787
 788	info->owner = THIS_MODULE;
 789	info->max_adj = adapter->ptp->max_adj;
 790	info->gettimex64 = idpf_ptp_gettimex64;
 791	info->settime64 = idpf_ptp_settime64;
 792	info->adjfine = idpf_ptp_adjfine;
 793	info->adjtime = idpf_ptp_adjtime;
 794	info->verify = idpf_ptp_verify_pin;
 795	info->enable = idpf_ptp_gpio_enable;
 796	info->do_aux_work = idpf_ptp_do_aux_work;
 797#if IS_ENABLED(CONFIG_ARM_ARCH_TIMER)
 798	info->getcrosststamp = idpf_ptp_get_crosststamp;
 799#elif IS_ENABLED(CONFIG_X86)
 800	if (pcie_ptm_enabled(adapter->pdev) &&
 801	    boot_cpu_has(X86_FEATURE_ART) &&
 802	    boot_cpu_has(X86_FEATURE_TSC_KNOWN_FREQ))
 803		info->getcrosststamp = idpf_ptp_get_crosststamp;
 804#endif /* CONFIG_ARM_ARCH_TIMER */
 805}
 806
 807/**
 808 * idpf_ptp_create_clock - Create PTP clock device for userspace
 809 * @adapter: Driver specific private structure
 810 *
 811 * This function creates a new PTP clock device.
 812 *
 813 * Return: 0 on success, -errno otherwise.
 814 */
 815static int idpf_ptp_create_clock(const struct idpf_adapter *adapter)
 816{
 817	struct ptp_clock *clock;
 818
 819	idpf_ptp_set_caps(adapter);
 820
 821	/* Attempt to register the clock before enabling the hardware. */
 822	clock = ptp_clock_register(&adapter->ptp->info,
 823				   &adapter->pdev->dev);
 824	if (IS_ERR(clock)) {
 825		pci_err(adapter->pdev, "PTP clock creation failed: %pe\n",
 826			clock);
 827		return PTR_ERR(clock);
 828	}
 829
 830	adapter->ptp->clock = clock;
 831
 832	return 0;
 833}
 834
 835/**
 836 * idpf_ptp_release_vport_tstamp - Release the Tx timestamps trakcers for a
 837 *				   given vport.
 838 * @vport: Virtual port structure
 839 *
 840 * Remove the queues and delete lists that tracks Tx timestamp entries for a
 841 * given vport.
 842 */
 843static void idpf_ptp_release_vport_tstamp(struct idpf_vport *vport)
 844{
 845	struct idpf_ptp_tx_tstamp *ptp_tx_tstamp, *tmp;
 846	struct list_head *head;
 847
 848	cancel_work_sync(&vport->tstamp_task);
 849
 850	/* Remove list with free latches */
 851	spin_lock_bh(&vport->tx_tstamp_caps->latches_lock);
 852
 853	head = &vport->tx_tstamp_caps->latches_free;
 854	list_for_each_entry_safe(ptp_tx_tstamp, tmp, head, list_member) {
 855		list_del(&ptp_tx_tstamp->list_member);
 856		kfree(ptp_tx_tstamp);
 857	}
 858
 859	/* Remove list with latches in use */
 860	head = &vport->tx_tstamp_caps->latches_in_use;
 861	u64_stats_update_begin(&vport->tstamp_stats.stats_sync);
 862	list_for_each_entry_safe(ptp_tx_tstamp, tmp, head, list_member) {
 863		u64_stats_inc(&vport->tstamp_stats.flushed);
 864
 865		list_del(&ptp_tx_tstamp->list_member);
 866		if (ptp_tx_tstamp->skb)
 867			consume_skb(ptp_tx_tstamp->skb);
 868
 869		kfree(ptp_tx_tstamp);
 870	}
 871	u64_stats_update_end(&vport->tstamp_stats.stats_sync);
 872
 873	spin_unlock_bh(&vport->tx_tstamp_caps->latches_lock);
 874
 875	kfree(vport->tx_tstamp_caps);
 876	vport->tx_tstamp_caps = NULL;
 877}
 878
 879/**
 880 * idpf_ptp_release_tstamp - Release the Tx timestamps trackers
 881 * @adapter: Driver specific private structure
 882 *
 883 * Remove the queues and delete lists that tracks Tx timestamp entries.
 884 */
 885static void idpf_ptp_release_tstamp(struct idpf_adapter *adapter)
 886{
 887	idpf_for_each_vport(adapter, vport) {
 888		if (!idpf_ptp_is_vport_tx_tstamp_ena(vport))
 889			continue;
 890
 891		idpf_ptp_release_vport_tstamp(vport);
 892	}
 893}
 894
 895/**
 896 * idpf_ptp_get_txq_tstamp_capability - Verify the timestamping capability
 897 *					for a given tx queue.
 898 * @txq: Transmit queue
 899 *
 900 * Since performing timestamp flows requires reading the device clock value and
 901 * the support in the Control Plane, the function checks both factors and
 902 * summarizes the support for the timestamping.
 903 *
 904 * Return: true if the timestamping is supported, false otherwise.
 905 */
 906bool idpf_ptp_get_txq_tstamp_capability(struct idpf_tx_queue *txq)
 907{
 908	if (!txq || !txq->cached_tstamp_caps)
 909		return false;
 910	else if (txq->cached_tstamp_caps->access)
 911		return true;
 912	else
 913		return false;
 914}
 915
 916/**
 917 * idpf_ptp_init - Initialize PTP hardware clock support
 918 * @adapter: Driver specific private structure
 919 *
 920 * Set up the device for interacting with the PTP hardware clock for all
 921 * functions. Function will allocate and register a ptp_clock with the
 922 * PTP_1588_CLOCK infrastructure.
 923 *
 924 * Return: 0 on success, -errno otherwise.
 925 */
 926int idpf_ptp_init(struct idpf_adapter *adapter)
 927{
 928	struct timespec64 ts;
 929	int err;
 930
 931	if (!idpf_is_cap_ena(adapter, IDPF_OTHER_CAPS, VIRTCHNL2_CAP_PTP)) {
 932		pci_dbg(adapter->pdev, "PTP capability is not detected\n");
 933		return -EOPNOTSUPP;
 934	}
 935
 936	adapter->ptp = kzalloc(sizeof(*adapter->ptp), GFP_KERNEL);
 937	if (!adapter->ptp)
 938		return -ENOMEM;
 939
 940	/* add a back pointer to adapter */
 941	adapter->ptp->adapter = adapter;
 942
 943	if (adapter->dev_ops.reg_ops.ptp_reg_init)
 944		adapter->dev_ops.reg_ops.ptp_reg_init(adapter);
 945
 946	err = idpf_ptp_get_caps(adapter);
 947	if (err) {
 948		pci_err(adapter->pdev, "Failed to get PTP caps err %d\n", err);
 949		goto free_ptp;
 950	}
 951
 952	err = idpf_ptp_create_clock(adapter);
 953	if (err)
 954		goto free_ptp;
 955
 956	if (adapter->ptp->get_dev_clk_time_access != IDPF_PTP_NONE)
 957		ptp_schedule_worker(adapter->ptp->clock, 0);
 958
 959	/* Write the default increment time value if the clock adjustments
 960	 * are enabled.
 961	 */
 962	if (adapter->ptp->adj_dev_clk_time_access != IDPF_PTP_NONE) {
 963		err = idpf_ptp_adj_dev_clk_fine(adapter,
 964						adapter->ptp->base_incval);
 965		if (err)
 966			goto remove_clock;
 967	}
 968
 969	/* Write the initial time value if the set time operation is enabled */
 970	if (adapter->ptp->set_dev_clk_time_access != IDPF_PTP_NONE) {
 971		ts = ktime_to_timespec64(ktime_get_real());
 972		err = idpf_ptp_settime64(&adapter->ptp->info, &ts);
 973		if (err)
 974			goto remove_clock;
 975	}
 976
 977	spin_lock_init(&adapter->ptp->read_dev_clk_lock);
 978
 979	pci_dbg(adapter->pdev, "PTP init successful\n");
 980
 981	return 0;
 982
 983remove_clock:
 984	if (adapter->ptp->get_dev_clk_time_access != IDPF_PTP_NONE)
 985		ptp_cancel_worker_sync(adapter->ptp->clock);
 986
 987	ptp_clock_unregister(adapter->ptp->clock);
 988	adapter->ptp->clock = NULL;
 989
 990free_ptp:
 991	kfree(adapter->ptp);
 992	adapter->ptp = NULL;
 993
 994	return err;
 995}
 996
 997/**
 998 * idpf_ptp_release - Clear PTP hardware clock support
 999 * @adapter: Driver specific private structure
1000 */
1001void idpf_ptp_release(struct idpf_adapter *adapter)
1002{
1003	struct idpf_ptp *ptp = adapter->ptp;
1004
1005	if (!ptp)
1006		return;
1007
1008	if (ptp->tx_tstamp_access != IDPF_PTP_NONE &&
1009	    ptp->get_dev_clk_time_access != IDPF_PTP_NONE)
1010		idpf_ptp_release_tstamp(adapter);
1011
1012	if (ptp->clock) {
1013		if (adapter->ptp->get_dev_clk_time_access != IDPF_PTP_NONE)
1014			ptp_cancel_worker_sync(adapter->ptp->clock);
1015
1016		ptp_clock_unregister(ptp->clock);
1017	}
1018
1019	kfree(ptp);
1020	adapter->ptp = NULL;
1021}