@@ -536,22 +536,58 @@ static void igc_ptp_enable_rx_timestamp(struct igc_adapter *adapter)
536536 wr32 (IGC_TSYNCRXCTL , val );
537537}
538538
539+ static void igc_ptp_clear_tx_tstamp (struct igc_adapter * adapter )
540+ {
541+ unsigned long flags ;
542+
543+ spin_lock_irqsave (& adapter -> ptp_tx_lock , flags );
544+
545+ dev_kfree_skb_any (adapter -> ptp_tx_skb );
546+ adapter -> ptp_tx_skb = NULL ;
547+
548+ spin_unlock_irqrestore (& adapter -> ptp_tx_lock , flags );
549+ }
550+
539551static void igc_ptp_disable_tx_timestamp (struct igc_adapter * adapter )
540552{
541553 struct igc_hw * hw = & adapter -> hw ;
554+ int i ;
555+
556+ /* Clear the flags first to avoid new packets to be enqueued
557+ * for TX timestamping.
558+ */
559+ for (i = 0 ; i < adapter -> num_tx_queues ; i ++ ) {
560+ struct igc_ring * tx_ring = adapter -> tx_ring [i ];
561+
562+ clear_bit (IGC_RING_FLAG_TX_HWTSTAMP , & tx_ring -> flags );
563+ }
564+
565+ /* Now we can clean the pending TX timestamp requests. */
566+ igc_ptp_clear_tx_tstamp (adapter );
542567
543568 wr32 (IGC_TSYNCTXCTL , 0 );
544569}
545570
546571static void igc_ptp_enable_tx_timestamp (struct igc_adapter * adapter )
547572{
548573 struct igc_hw * hw = & adapter -> hw ;
574+ int i ;
549575
550576 wr32 (IGC_TSYNCTXCTL , IGC_TSYNCTXCTL_ENABLED | IGC_TSYNCTXCTL_TXSYNSIG );
551577
552578 /* Read TXSTMP registers to discard any timestamp previously stored. */
553579 rd32 (IGC_TXSTMPL );
554580 rd32 (IGC_TXSTMPH );
581+
582+ /* The hardware is ready to accept TX timestamp requests,
583+ * notify the transmit path.
584+ */
585+ for (i = 0 ; i < adapter -> num_tx_queues ; i ++ ) {
586+ struct igc_ring * tx_ring = adapter -> tx_ring [i ];
587+
588+ set_bit (IGC_RING_FLAG_TX_HWTSTAMP , & tx_ring -> flags );
589+ }
590+
555591}
556592
557593/**
@@ -603,35 +639,35 @@ static int igc_ptp_set_timestamp_mode(struct igc_adapter *adapter,
603639 return 0 ;
604640}
605641
642+ /* Requires adapter->ptp_tx_lock held by caller. */
606643static void igc_ptp_tx_timeout (struct igc_adapter * adapter )
607644{
608645 struct igc_hw * hw = & adapter -> hw ;
609646
610647 dev_kfree_skb_any (adapter -> ptp_tx_skb );
611648 adapter -> ptp_tx_skb = NULL ;
612649 adapter -> tx_hwtstamp_timeouts ++ ;
613- clear_bit_unlock (__IGC_PTP_TX_IN_PROGRESS , & adapter -> state );
614650 /* Clear the tx valid bit in TSYNCTXCTL register to enable interrupt. */
615651 rd32 (IGC_TXSTMPH );
616652 netdev_warn (adapter -> netdev , "Tx timestamp timeout\n" );
617653}
618654
619655void igc_ptp_tx_hang (struct igc_adapter * adapter )
620656{
621- bool timeout = time_is_before_jiffies (adapter -> ptp_tx_start +
622- IGC_PTP_TX_TIMEOUT );
657+ unsigned long flags ;
623658
624- if (!test_bit (__IGC_PTP_TX_IN_PROGRESS , & adapter -> state ))
625- return ;
659+ spin_lock_irqsave (& adapter -> ptp_tx_lock , flags );
626660
627- /* If we haven't received a timestamp within the timeout, it is
628- * reasonable to assume that it will never occur, so we can unlock the
629- * timestamp bit when this occurs.
630- */
631- if (timeout ) {
632- cancel_work_sync (& adapter -> ptp_tx_work );
633- igc_ptp_tx_timeout (adapter );
634- }
661+ if (!adapter -> ptp_tx_skb )
662+ goto unlock ;
663+
664+ if (time_is_after_jiffies (adapter -> ptp_tx_start + IGC_PTP_TX_TIMEOUT ))
665+ goto unlock ;
666+
667+ igc_ptp_tx_timeout (adapter );
668+
669+ unlock :
670+ spin_unlock_irqrestore (& adapter -> ptp_tx_lock , flags );
635671}
636672
637673/**
@@ -641,20 +677,57 @@ void igc_ptp_tx_hang(struct igc_adapter *adapter)
641677 * If we were asked to do hardware stamping and such a time stamp is
642678 * available, then it must have been for this skb here because we only
643679 * allow only one such packet into the queue.
680+ *
681+ * Context: Expects adapter->ptp_tx_lock to be held by caller.
644682 */
645683static void igc_ptp_tx_hwtstamp (struct igc_adapter * adapter )
646684{
647685 struct sk_buff * skb = adapter -> ptp_tx_skb ;
648686 struct skb_shared_hwtstamps shhwtstamps ;
649687 struct igc_hw * hw = & adapter -> hw ;
688+ u32 tsynctxctl ;
650689 int adjust = 0 ;
651690 u64 regval ;
652691
653692 if (WARN_ON_ONCE (!skb ))
654693 return ;
655694
656- regval = rd32 (IGC_TXSTMPL );
657- regval |= (u64 )rd32 (IGC_TXSTMPH ) << 32 ;
695+ tsynctxctl = rd32 (IGC_TSYNCTXCTL );
696+ tsynctxctl &= IGC_TSYNCTXCTL_TXTT_0 ;
697+ if (tsynctxctl ) {
698+ regval = rd32 (IGC_TXSTMPL );
699+ regval |= (u64 )rd32 (IGC_TXSTMPH ) << 32 ;
700+ } else {
701+ /* There's a bug in the hardware that could cause
702+ * missing interrupts for TX timestamping. The issue
703+ * is that for new interrupts to be triggered, the
704+ * IGC_TXSTMPH_0 register must be read.
705+ *
706+ * To avoid discarding a valid timestamp that just
707+ * happened at the "wrong" time, we need to confirm
708+ * that there was no timestamp captured, we do that by
709+ * assuming that no two timestamps in sequence have
710+ * the same nanosecond value.
711+ *
712+ * So, we read the "low" register, read the "high"
713+ * register (to latch a new timestamp) and read the
714+ * "low" register again, if "old" and "new" versions
715+ * of the "low" register are different, a valid
716+ * timestamp was captured, we can read the "high"
717+ * register again.
718+ */
719+ u32 txstmpl_old , txstmpl_new ;
720+
721+ txstmpl_old = rd32 (IGC_TXSTMPL );
722+ rd32 (IGC_TXSTMPH );
723+ txstmpl_new = rd32 (IGC_TXSTMPL );
724+
725+ if (txstmpl_old == txstmpl_new )
726+ return ;
727+
728+ regval = txstmpl_new ;
729+ regval |= (u64 )rd32 (IGC_TXSTMPH ) << 32 ;
730+ }
658731 if (igc_ptp_systim_to_hwtstamp (adapter , & shhwtstamps , regval ))
659732 return ;
660733
@@ -676,41 +749,33 @@ static void igc_ptp_tx_hwtstamp(struct igc_adapter *adapter)
676749 shhwtstamps .hwtstamp =
677750 ktime_add_ns (shhwtstamps .hwtstamp , adjust );
678751
679- /* Clear the lock early before calling skb_tstamp_tx so that
680- * applications are not woken up before the lock bit is clear. We use
681- * a copy of the skb pointer to ensure other threads can't change it
682- * while we're notifying the stack.
683- */
684752 adapter -> ptp_tx_skb = NULL ;
685- clear_bit_unlock (__IGC_PTP_TX_IN_PROGRESS , & adapter -> state );
686753
687754 /* Notify the stack and free the skb after we've unlocked */
688755 skb_tstamp_tx (skb , & shhwtstamps );
689756 dev_kfree_skb_any (skb );
690757}
691758
692759/**
693- * igc_ptp_tx_work
694- * @work: pointer to work struct
760+ * igc_ptp_tx_tstamp_event
761+ * @adapter: board private structure
695762 *
696- * This work function polls the TSYNCTXCTL valid bit to determine when a
697- * timestamp has been taken for the current stored skb .
763+ * Called when a TX timestamp interrupt happens to retrieve the
764+ * timestamp and send it up to the socket .
698765 */
699- static void igc_ptp_tx_work (struct work_struct * work )
766+ void igc_ptp_tx_tstamp_event (struct igc_adapter * adapter )
700767{
701- struct igc_adapter * adapter = container_of (work , struct igc_adapter ,
702- ptp_tx_work );
703- struct igc_hw * hw = & adapter -> hw ;
704- u32 tsynctxctl ;
768+ unsigned long flags ;
705769
706- if (!test_bit (__IGC_PTP_TX_IN_PROGRESS , & adapter -> state ))
707- return ;
770+ spin_lock_irqsave (& adapter -> ptp_tx_lock , flags );
708771
709- tsynctxctl = rd32 (IGC_TSYNCTXCTL );
710- if (WARN_ON_ONCE (!(tsynctxctl & IGC_TSYNCTXCTL_TXTT_0 )))
711- return ;
772+ if (!adapter -> ptp_tx_skb )
773+ goto unlock ;
712774
713775 igc_ptp_tx_hwtstamp (adapter );
776+
777+ unlock :
778+ spin_unlock_irqrestore (& adapter -> ptp_tx_lock , flags );
714779}
715780
716781/**
@@ -959,8 +1024,8 @@ void igc_ptp_init(struct igc_adapter *adapter)
9591024 return ;
9601025 }
9611026
1027+ spin_lock_init (& adapter -> ptp_tx_lock );
9621028 spin_lock_init (& adapter -> tmreg_lock );
963- INIT_WORK (& adapter -> ptp_tx_work , igc_ptp_tx_work );
9641029
9651030 adapter -> tstamp_config .rx_filter = HWTSTAMP_FILTER_NONE ;
9661031 adapter -> tstamp_config .tx_type = HWTSTAMP_TX_OFF ;
@@ -1020,10 +1085,7 @@ void igc_ptp_suspend(struct igc_adapter *adapter)
10201085 if (!(adapter -> ptp_flags & IGC_PTP_ENABLED ))
10211086 return ;
10221087
1023- cancel_work_sync (& adapter -> ptp_tx_work );
1024- dev_kfree_skb_any (adapter -> ptp_tx_skb );
1025- adapter -> ptp_tx_skb = NULL ;
1026- clear_bit_unlock (__IGC_PTP_TX_IN_PROGRESS , & adapter -> state );
1088+ igc_ptp_clear_tx_tstamp (adapter );
10271089
10281090 if (pci_device_is_present (adapter -> pdev )) {
10291091 igc_ptp_time_save (adapter );
0 commit comments