iavf_ptp.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* Copyright(c) 2024 Intel Corporation. */
  3
  4#include "iavf.h"
  5#include "iavf_ptp.h"
  6
  7#define iavf_clock_to_adapter(info)				\
  8	container_of_const(info, struct iavf_adapter, ptp.info)
  9
 10/**
 11 * iavf_ptp_disable_rx_tstamp - Disable timestamping in Rx rings
 12 * @adapter: private adapter structure
 13 *
 14 * Disable timestamp reporting for all Rx rings.
 15 */
 16static void iavf_ptp_disable_rx_tstamp(struct iavf_adapter *adapter)
 17{
 18	for (u32 i = 0; i < adapter->num_active_queues; i++)
 19		adapter->rx_rings[i].flags &= ~IAVF_TXRX_FLAGS_HW_TSTAMP;
 20}
 21
 22/**
 23 * iavf_ptp_enable_rx_tstamp - Enable timestamping in Rx rings
 24 * @adapter: private adapter structure
 25 *
 26 * Enable timestamp reporting for all Rx rings.
 27 */
 28static void iavf_ptp_enable_rx_tstamp(struct iavf_adapter *adapter)
 29{
 30	for (u32 i = 0; i < adapter->num_active_queues; i++)
 31		adapter->rx_rings[i].flags |= IAVF_TXRX_FLAGS_HW_TSTAMP;
 32}
 33
 34/**
 35 * iavf_ptp_set_timestamp_mode - Set device timestamping mode
 36 * @adapter: private adapter structure
 37 * @config: pointer to kernel_hwtstamp_config
 38 *
 39 * Set the timestamping mode requested from the userspace.
 40 *
 41 * Note: this function always translates Rx timestamp requests for any packet
 42 * category into HWTSTAMP_FILTER_ALL.
 43 *
 44 * Return: 0 on success, negative error code otherwise.
 45 */
 46static int iavf_ptp_set_timestamp_mode(struct iavf_adapter *adapter,
 47				       struct kernel_hwtstamp_config *config)
 48{
 49	/* Reserved for future extensions. */
 50	if (config->flags)
 51		return -EINVAL;
 52
 53	switch (config->tx_type) {
 54	case HWTSTAMP_TX_OFF:
 55		break;
 56	case HWTSTAMP_TX_ON:
 57		return -EOPNOTSUPP;
 58	default:
 59		return -ERANGE;
 60	}
 61
 62	if (config->rx_filter == HWTSTAMP_FILTER_NONE) {
 63		iavf_ptp_disable_rx_tstamp(adapter);
 64		return 0;
 65	} else if (config->rx_filter > HWTSTAMP_FILTER_NTP_ALL) {
 66		return -ERANGE;
 67	} else if (!(iavf_ptp_cap_supported(adapter,
 68					    VIRTCHNL_1588_PTP_CAP_RX_TSTAMP))) {
 69		return -EOPNOTSUPP;
 70	}
 71
 72	config->rx_filter = HWTSTAMP_FILTER_ALL;
 73	iavf_ptp_enable_rx_tstamp(adapter);
 74
 75	return 0;
 76}
 77
 78/**
 79 * iavf_ptp_set_ts_config - Set timestamping configuration
 80 * @adapter: private adapter structure
 81 * @config: pointer to kernel_hwtstamp_config structure
 82 * @extack: pointer to netlink_ext_ack structure
 83 *
 84 * Program the requested timestamping configuration to the device.
 85 *
 86 * Return: 0 on success, negative error code otherwise.
 87 */
 88int iavf_ptp_set_ts_config(struct iavf_adapter *adapter,
 89			   struct kernel_hwtstamp_config *config,
 90			   struct netlink_ext_ack *extack)
 91{
 92	int err;
 93
 94	err = iavf_ptp_set_timestamp_mode(adapter, config);
 95	if (err)
 96		return err;
 97
 98	/* Save successful settings for future reference */
 99	adapter->ptp.hwtstamp_config = *config;
100
101	return 0;
102}
103
104/**
105 * iavf_ptp_cap_supported - Check if a PTP capability is supported
106 * @adapter: private adapter structure
107 * @cap: the capability bitmask to check
108 *
109 * Return: true if every capability set in cap is also set in the enabled
110 *         capabilities reported by the PF, false otherwise.
111 */
112bool iavf_ptp_cap_supported(const struct iavf_adapter *adapter, u32 cap)
113{
114	if (!IAVF_PTP_ALLOWED(adapter))
115		return false;
116
117	/* Only return true if every bit in cap is set in hw_caps.caps */
118	return (adapter->ptp.hw_caps.caps & cap) == cap;
119}
120
121/**
122 * iavf_allocate_ptp_cmd - Allocate a PTP command message structure
123 * @v_opcode: the virtchnl opcode
124 * @msglen: length in bytes of the associated virtchnl structure
125 *
126 * Allocates a PTP command message and pre-fills it with the provided message
127 * length and opcode.
128 *
129 * Return: allocated PTP command.
130 */
131static struct iavf_ptp_aq_cmd *iavf_allocate_ptp_cmd(enum virtchnl_ops v_opcode,
132						     u16 msglen)
133{
134	struct iavf_ptp_aq_cmd *cmd;
135
136	cmd = kzalloc(struct_size(cmd, msg, msglen), GFP_KERNEL);
137	if (!cmd)
138		return NULL;
139
140	cmd->v_opcode = v_opcode;
141	cmd->msglen = msglen;
142
143	return cmd;
144}
145
146/**
147 * iavf_queue_ptp_cmd - Queue PTP command for sending over virtchnl
148 * @adapter: private adapter structure
149 * @cmd: the command structure to send
150 *
151 * Queue the given command structure into the PTP virtchnl command queue tos
152 * end to the PF.
153 */
154static void iavf_queue_ptp_cmd(struct iavf_adapter *adapter,
155			       struct iavf_ptp_aq_cmd *cmd)
156{
157	mutex_lock(&adapter->ptp.aq_cmd_lock);
158	list_add_tail(&cmd->list, &adapter->ptp.aq_cmds);
159	mutex_unlock(&adapter->ptp.aq_cmd_lock);
160
161	adapter->aq_required |= IAVF_FLAG_AQ_SEND_PTP_CMD;
162	mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0);
163}
164
165/**
166 * iavf_send_phc_read - Send request to read PHC time
167 * @adapter: private adapter structure
168 *
169 * Send a request to obtain the PTP hardware clock time. This allocates the
170 * VIRTCHNL_OP_1588_PTP_GET_TIME message and queues it up to send to
171 * indirectly read the PHC time.
172 *
173 * This function does not wait for the reply from the PF.
174 *
175 * Return: 0 if success, error code otherwise.
176 */
177static int iavf_send_phc_read(struct iavf_adapter *adapter)
178{
179	struct iavf_ptp_aq_cmd *cmd;
180
181	if (!adapter->ptp.clock)
182		return -EOPNOTSUPP;
183
184	cmd = iavf_allocate_ptp_cmd(VIRTCHNL_OP_1588_PTP_GET_TIME,
185				    sizeof(struct virtchnl_phc_time));
186	if (!cmd)
187		return -ENOMEM;
188
189	iavf_queue_ptp_cmd(adapter, cmd);
190
191	return 0;
192}
193
194/**
195 * iavf_read_phc_indirect - Indirectly read the PHC time via virtchnl
196 * @adapter: private adapter structure
197 * @ts: storage for the timestamp value
198 * @sts: system timestamp values before and after the read
199 *
200 * Used when the device does not have direct register access to the PHC time.
201 * Indirectly reads the time via the VIRTCHNL_OP_1588_PTP_GET_TIME, and waits
202 * for the reply from the PF.
203 *
204 * Based on some simple measurements using ftrace and phc2sys, this clock
205 * access method has about a ~110 usec latency even when the system is not
206 * under load. In order to achieve acceptable results when using phc2sys with
207 * the indirect clock access method, it is recommended to use more
208 * conservative proportional and integration constants with the P/I servo.
209 *
210 * Return: 0 if success, error code otherwise.
211 */
212static int iavf_read_phc_indirect(struct iavf_adapter *adapter,
213				  struct timespec64 *ts,
214				  struct ptp_system_timestamp *sts)
215{
216	long ret;
217	int err;
218
219	adapter->ptp.phc_time_ready = false;
220
221	ptp_read_system_prets(sts);
222
223	err = iavf_send_phc_read(adapter);
224	if (err)
225		return err;
226
227	ret = wait_event_interruptible_timeout(adapter->ptp.phc_time_waitqueue,
228					       adapter->ptp.phc_time_ready,
229					       HZ);
230
231	ptp_read_system_postts(sts);
232
233	if (ret < 0)
234		return ret;
235	else if (!ret)
236		return -EBUSY;
237
238	*ts = ns_to_timespec64(adapter->ptp.cached_phc_time);
239
240	return 0;
241}
242
243static int iavf_ptp_gettimex64(struct ptp_clock_info *info,
244			       struct timespec64 *ts,
245			       struct ptp_system_timestamp *sts)
246{
247	struct iavf_adapter *adapter = iavf_clock_to_adapter(info);
248
249	if (!adapter->ptp.clock)
250		return -EOPNOTSUPP;
251
252	return iavf_read_phc_indirect(adapter, ts, sts);
253}
254
255static int iavf_ptp_settime64(struct ptp_clock_info *info,
256			      const struct timespec64 *ts)
257{
258	return -EOPNOTSUPP;
259}
260
261/**
262 * iavf_ptp_cache_phc_time - Cache PHC time for performing timestamp extension
263 * @adapter: private adapter structure
264 *
265 * Periodically cache the PHC time in order to allow for timestamp extension.
266 * This is required because the Tx and Rx timestamps only contain 32bits of
267 * nanoseconds. Timestamp extension allows calculating the corrected 64bit
268 * timestamp. This algorithm relies on the cached time being within ~1 second
269 * of the timestamp.
270 */
271static void iavf_ptp_cache_phc_time(struct iavf_adapter *adapter)
272{
273	if (!time_is_before_jiffies(adapter->ptp.cached_phc_updated + HZ))
274		return;
275
276	/* The response from virtchnl will store the time into
277	 * cached_phc_time.
278	 */
279	iavf_send_phc_read(adapter);
280}
281
282/**
283 * iavf_ptp_do_aux_work - Perform periodic work required for PTP support
284 * @info: PTP clock info structure
285 *
286 * Handler to take care of periodic work required for PTP operation. This
287 * includes the following tasks:
288 *
289 *   1) updating cached_phc_time
290 *
291 *      cached_phc_time is used by the Tx and Rx timestamp flows in order to
292 *      perform timestamp extension, by carefully comparing the timestamp
293 *      32bit nanosecond timestamps and determining the corrected 64bit
294 *      timestamp value to report to userspace. This algorithm only works if
295 *      the cached_phc_time is within ~1 second of the Tx or Rx timestamp
296 *      event. This task periodically reads the PHC time and stores it, to
297 *      ensure that timestamp extension operates correctly.
298 *
299 * Returns: time in jiffies until the periodic task should be re-scheduled.
300 */
301static long iavf_ptp_do_aux_work(struct ptp_clock_info *info)
302{
303	struct iavf_adapter *adapter = iavf_clock_to_adapter(info);
304
305	iavf_ptp_cache_phc_time(adapter);
306
307	/* Check work about twice a second */
308	return msecs_to_jiffies(500);
309}
310
311/**
312 * iavf_ptp_register_clock - Register a new PTP for userspace
313 * @adapter: private adapter structure
314 *
315 * Allocate and register a new PTP clock device if necessary.
316 *
317 * Return: 0 if success, error otherwise.
318 */
319static int iavf_ptp_register_clock(struct iavf_adapter *adapter)
320{
321	struct ptp_clock_info *ptp_info = &adapter->ptp.info;
322	struct device *dev = &adapter->pdev->dev;
323	struct ptp_clock *clock;
324
325	snprintf(ptp_info->name, sizeof(ptp_info->name), "%s-%s-clk",
326		 KBUILD_MODNAME, dev_name(dev));
327	ptp_info->owner = THIS_MODULE;
328	ptp_info->gettimex64 = iavf_ptp_gettimex64;
329	ptp_info->settime64 = iavf_ptp_settime64;
330	ptp_info->do_aux_work = iavf_ptp_do_aux_work;
331
332	clock = ptp_clock_register(ptp_info, dev);
333	if (IS_ERR(clock))
334		return PTR_ERR(clock);
335
336	adapter->ptp.clock = clock;
337
338	dev_dbg(&adapter->pdev->dev, "PTP clock %s registered\n",
339		adapter->ptp.info.name);
340
341	return 0;
342}
343
344/**
345 * iavf_ptp_init - Initialize PTP support if capability was negotiated
346 * @adapter: private adapter structure
347 *
348 * Initialize PTP functionality, based on the capabilities that the PF has
349 * enabled for this VF.
350 */
351void iavf_ptp_init(struct iavf_adapter *adapter)
352{
353	int err;
354
355	if (!iavf_ptp_cap_supported(adapter, VIRTCHNL_1588_PTP_CAP_READ_PHC)) {
356		pci_notice(adapter->pdev,
357			   "Device does not have PTP clock support\n");
358		return;
359	}
360
361	err = iavf_ptp_register_clock(adapter);
362	if (err) {
363		pci_err(adapter->pdev,
364			"Failed to register PTP clock device (%p)\n",
365			ERR_PTR(err));
366		return;
367	}
368
369	for (int i = 0; i < adapter->num_active_queues; i++) {
370		struct iavf_ring *rx_ring = &adapter->rx_rings[i];
371
372		rx_ring->ptp = &adapter->ptp;
373	}
374
375	ptp_schedule_worker(adapter->ptp.clock, 0);
376}
377
378/**
379 * iavf_ptp_release - Disable PTP support
380 * @adapter: private adapter structure
381 *
382 * Release all PTP resources that were previously initialized.
383 */
384void iavf_ptp_release(struct iavf_adapter *adapter)
385{
386	struct iavf_ptp_aq_cmd *cmd, *tmp;
387
388	if (!adapter->ptp.clock)
389		return;
390
391	pci_dbg(adapter->pdev, "removing PTP clock %s\n",
392		adapter->ptp.info.name);
393	ptp_clock_unregister(adapter->ptp.clock);
394	adapter->ptp.clock = NULL;
395
396	/* Cancel any remaining uncompleted PTP clock commands */
397	mutex_lock(&adapter->ptp.aq_cmd_lock);
398	list_for_each_entry_safe(cmd, tmp, &adapter->ptp.aq_cmds, list) {
399		list_del(&cmd->list);
400		kfree(cmd);
401	}
402	adapter->aq_required &= ~IAVF_FLAG_AQ_SEND_PTP_CMD;
403	mutex_unlock(&adapter->ptp.aq_cmd_lock);
404
405	adapter->ptp.hwtstamp_config.rx_filter = HWTSTAMP_FILTER_NONE;
406	iavf_ptp_disable_rx_tstamp(adapter);
407}
408
409/**
410 * iavf_ptp_process_caps - Handle change in PTP capabilities
411 * @adapter: private adapter structure
412 *
413 * Handle any state changes necessary due to change in PTP capabilities, such
414 * as after a device reset or change in configuration from the PF.
415 */
416void iavf_ptp_process_caps(struct iavf_adapter *adapter)
417{
418	bool phc = iavf_ptp_cap_supported(adapter, VIRTCHNL_1588_PTP_CAP_READ_PHC);
419
420	/* Check if the device gained or lost necessary access to support the
421	 * PTP hardware clock. If so, driver must respond appropriately by
422	 * creating or destroying the PTP clock device.
423	 */
424	if (adapter->ptp.clock && !phc)
425		iavf_ptp_release(adapter);
426	else if (!adapter->ptp.clock && phc)
427		iavf_ptp_init(adapter);
428
429	/* Check if the device lost access to Rx timestamp incoming packets */
430	if (!iavf_ptp_cap_supported(adapter, VIRTCHNL_1588_PTP_CAP_RX_TSTAMP)) {
431		adapter->ptp.hwtstamp_config.rx_filter = HWTSTAMP_FILTER_NONE;
432		iavf_ptp_disable_rx_tstamp(adapter);
433	}
434}
435
436/**
437 * iavf_ptp_extend_32b_timestamp - Convert a 32b nanoseconds timestamp to 64b
438 * nanoseconds
439 * @cached_phc_time: recently cached copy of PHC time
440 * @in_tstamp: Ingress/egress 32b nanoseconds timestamp value
441 *
442 * Hardware captures timestamps which contain only 32 bits of nominal
443 * nanoseconds, as opposed to the 64bit timestamps that the stack expects.
444 *
445 * Extend the 32bit nanosecond timestamp using the following algorithm and
446 * assumptions:
447 *
448 * 1) have a recently cached copy of the PHC time
449 * 2) assume that the in_tstamp was captured 2^31 nanoseconds (~2.1
450 *    seconds) before or after the PHC time was captured.
451 * 3) calculate the delta between the cached time and the timestamp
452 * 4) if the delta is smaller than 2^31 nanoseconds, then the timestamp was
453 *    captured after the PHC time. In this case, the full timestamp is just
454 *    the cached PHC time plus the delta.
455 * 5) otherwise, if the delta is larger than 2^31 nanoseconds, then the
456 *    timestamp was captured *before* the PHC time, i.e. because the PHC
457 *    cache was updated after the timestamp was captured by hardware. In this
458 *    case, the full timestamp is the cached time minus the inverse delta.
459 *
460 * This algorithm works even if the PHC time was updated after a Tx timestamp
461 * was requested, but before the Tx timestamp event was reported from
462 * hardware.
463 *
464 * This calculation primarily relies on keeping the cached PHC time up to
465 * date. If the timestamp was captured more than 2^31 nanoseconds after the
466 * PHC time, it is possible that the lower 32bits of PHC time have
467 * overflowed more than once, and we might generate an incorrect timestamp.
468 *
469 * This is prevented by (a) periodically updating the cached PHC time once
470 * a second, and (b) discarding any Tx timestamp packet if it has waited for
471 * a timestamp for more than one second.
472 *
473 * Return: extended timestamp (to 64b).
474 */
475u64 iavf_ptp_extend_32b_timestamp(u64 cached_phc_time, u32 in_tstamp)
476{
477	u32 low = lower_32_bits(cached_phc_time);
478	u32 delta = in_tstamp - low;
479	u64 ns;
480
481	/* Do not assume that the in_tstamp is always more recent than the
482	 * cached PHC time. If the delta is large, it indicates that the
483	 * in_tstamp was taken in the past, and should be converted
484	 * forward.
485	 */
486	if (delta > S32_MAX)
487		ns = cached_phc_time - (low - in_tstamp);
488	else
489		ns = cached_phc_time + delta;
490
491	return ns;
492}