From e1175ce012416191b5b82a59d54ca8daa844f35f Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Tue, 24 Jun 2025 12:49:22 +0200 Subject: [PATCH 01/39] ixgbevf: do not share pages between packets Again, same as in the related iavf commit 920d86f3c552 ("iavf: drop page splitting and recycling"), as an intermediate step, drop the page sharing and recycling logic in a preparation to offload it to page_pool. Instead of the previous sharing and recycling, just allocate a new page every time. Suggested-by: Alexander Lobakin Reviewed-by: Aleksandr Loktionov Reviewed-by: Alexander Lobakin Signed-off-by: Larysa Zaremba --- drivers/net/ethernet/intel/ixgbevf/ixgbevf.h | 44 +--- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 239 ++---------------- 2 files changed, 28 insertions(+), 255 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h index ae2763fea2be2e..2d7ca3f86868bc 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h @@ -45,12 +45,7 @@ struct ixgbevf_tx_buffer { struct ixgbevf_rx_buffer { dma_addr_t dma; struct page *page; -#if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536) __u32 page_offset; -#else - __u16 page_offset; -#endif - __u16 pagecnt_bias; }; struct ixgbevf_stats { @@ -72,7 +67,6 @@ struct ixgbevf_rx_queue_stats { }; enum ixgbevf_ring_state_t { - __IXGBEVF_RX_3K_BUFFER, __IXGBEVF_TX_DETECT_HANG, __IXGBEVF_HANG_CHECK_ARMED, __IXGBEVF_TX_XDP_RING, @@ -143,8 +137,7 @@ struct ixgbevf_ring { #define IXGBEVF_MIN_RXD 64 /* Supported Rx Buffer Sizes */ -#define IXGBEVF_RXBUFFER_256 256 /* Used for packet split */ -#define IXGBEVF_RXBUFFER_2048 2048 +#define IXGBEVF_RXBUFFER_256 256 #define IXGBEVF_RXBUFFER_3072 3072 #define IXGBEVF_RX_HDR_SIZE IXGBEVF_RXBUFFER_256 @@ -152,12 +145,6 @@ struct ixgbevf_ring { #define MAXIMUM_ETHERNET_VLAN_SIZE (VLAN_ETH_FRAME_LEN + ETH_FCS_LEN) #define IXGBEVF_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN) -#if (PAGE_SIZE < 8192) -#define IXGBEVF_MAX_FRAME_BUILD_SKB \ - (SKB_WITH_OVERHEAD(IXGBEVF_RXBUFFER_2048) - IXGBEVF_SKB_PAD) -#else -#define IXGBEVF_MAX_FRAME_BUILD_SKB IXGBEVF_RXBUFFER_2048 -#endif #define IXGBE_TX_FLAGS_CSUM BIT(0) #define IXGBE_TX_FLAGS_VLAN BIT(1) @@ -168,35 +155,6 @@ struct ixgbevf_ring { #define IXGBE_TX_FLAGS_VLAN_PRIO_MASK 0x0000e000 #define IXGBE_TX_FLAGS_VLAN_SHIFT 16 -#define ring_uses_large_buffer(ring) \ - test_bit(__IXGBEVF_RX_3K_BUFFER, &(ring)->state) -#define set_ring_uses_large_buffer(ring) \ - set_bit(__IXGBEVF_RX_3K_BUFFER, &(ring)->state) -#define clear_ring_uses_large_buffer(ring) \ - clear_bit(__IXGBEVF_RX_3K_BUFFER, &(ring)->state) - -static inline unsigned int ixgbevf_rx_bufsz(struct ixgbevf_ring *ring) -{ -#if (PAGE_SIZE < 8192) - if (ring_uses_large_buffer(ring)) - return IXGBEVF_RXBUFFER_3072; - - return IXGBEVF_MAX_FRAME_BUILD_SKB; -#endif - return IXGBEVF_RXBUFFER_2048; -} - -static inline unsigned int ixgbevf_rx_pg_order(struct ixgbevf_ring *ring) -{ -#if (PAGE_SIZE < 8192) - if (ring_uses_large_buffer(ring)) - return 1; -#endif - return 0; -} - -#define ixgbevf_rx_pg_size(_ring) (PAGE_SIZE << ixgbevf_rx_pg_order(_ring)) - #define check_for_tx_hang(ring) \ test_bit(__IXGBEVF_TX_DETECT_HANG, &(ring)->state) #define set_check_for_tx_hang(ring) \ diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index fc48c89c7bb857..05baf28823c869 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -112,9 +112,6 @@ static void ixgbevf_service_event_complete(struct ixgbevf_adapter *adapter) static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter); static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector); static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter); -static bool ixgbevf_can_reuse_rx_page(struct ixgbevf_rx_buffer *rx_buffer); -static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring, - struct ixgbevf_rx_buffer *old_buff); static void ixgbevf_remove_adapter(struct ixgbe_hw *hw) { @@ -537,40 +534,20 @@ struct ixgbevf_rx_buffer *ixgbevf_get_rx_buffer(struct ixgbevf_ring *rx_ring, rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; prefetchw(rx_buffer->page); - /* we are reusing so sync this buffer for CPU use */ dma_sync_single_range_for_cpu(rx_ring->dev, rx_buffer->dma, rx_buffer->page_offset, size, DMA_FROM_DEVICE); - rx_buffer->pagecnt_bias--; - return rx_buffer; } static void ixgbevf_put_rx_buffer(struct ixgbevf_ring *rx_ring, - struct ixgbevf_rx_buffer *rx_buffer, - struct sk_buff *skb) + struct ixgbevf_rx_buffer *rx_buffer) { - if (ixgbevf_can_reuse_rx_page(rx_buffer)) { - /* hand second half of page back to the ring */ - ixgbevf_reuse_rx_page(rx_ring, rx_buffer); - } else { - if (IS_ERR(skb)) - /* We are not reusing the buffer so unmap it and free - * any references we are holding to it - */ - dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, - ixgbevf_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, - IXGBEVF_RX_DMA_ATTR); - __page_frag_cache_drain(rx_buffer->page, - rx_buffer->pagecnt_bias); - } - - /* clear contents of rx_buffer */ - rx_buffer->page = NULL; + dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, PAGE_SIZE, + DMA_FROM_DEVICE, IXGBEVF_RX_DMA_ATTR); } /** @@ -600,38 +577,28 @@ static bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring, return true; } -static inline unsigned int ixgbevf_rx_offset(struct ixgbevf_ring *rx_ring) -{ - return IXGBEVF_SKB_PAD; -} - static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring, struct ixgbevf_rx_buffer *bi) { - struct page *page = bi->page; + struct page *page; dma_addr_t dma; - /* since we are recycling buffers we should seldom need to alloc */ - if (likely(page)) - return true; - /* alloc new page for storage */ - page = dev_alloc_pages(ixgbevf_rx_pg_order(rx_ring)); + page = dev_alloc_page(); if (unlikely(!page)) { rx_ring->rx_stats.alloc_rx_page_failed++; return false; } /* map page for use */ - dma = dma_map_page_attrs(rx_ring->dev, page, 0, - ixgbevf_rx_pg_size(rx_ring), + dma = dma_map_page_attrs(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE, IXGBEVF_RX_DMA_ATTR); /* if mapping failed free memory back to system since * there isn't much point in holding memory we can't use */ if (dma_mapping_error(rx_ring->dev, dma)) { - __free_pages(page, ixgbevf_rx_pg_order(rx_ring)); + __free_page(page); rx_ring->rx_stats.alloc_rx_page_failed++; return false; @@ -639,8 +606,7 @@ static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring, bi->dma = dma; bi->page = page; - bi->page_offset = ixgbevf_rx_offset(rx_ring); - bi->pagecnt_bias = 1; + bi->page_offset = IXGBEVF_SKB_PAD; rx_ring->rx_stats.alloc_rx_page++; return true; @@ -673,7 +639,7 @@ static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, /* sync the buffer for use by the device */ dma_sync_single_range_for_device(rx_ring->dev, bi->dma, bi->page_offset, - ixgbevf_rx_bufsz(rx_ring), + IXGBEVF_RXBUFFER_3072, DMA_FROM_DEVICE); /* Refresh the desc even if pkt_addr didn't change @@ -755,66 +721,6 @@ static bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring, return false; } -/** - * ixgbevf_reuse_rx_page - page flip buffer and store it back on the ring - * @rx_ring: rx descriptor ring to store buffers on - * @old_buff: donor buffer to have page reused - * - * Synchronizes page for reuse by the adapter - **/ -static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring, - struct ixgbevf_rx_buffer *old_buff) -{ - struct ixgbevf_rx_buffer *new_buff; - u16 nta = rx_ring->next_to_alloc; - - new_buff = &rx_ring->rx_buffer_info[nta]; - - /* update, and store next to alloc */ - nta++; - rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; - - /* transfer page from old buffer to new buffer */ - new_buff->page = old_buff->page; - new_buff->dma = old_buff->dma; - new_buff->page_offset = old_buff->page_offset; - new_buff->pagecnt_bias = old_buff->pagecnt_bias; -} - -static bool ixgbevf_can_reuse_rx_page(struct ixgbevf_rx_buffer *rx_buffer) -{ - unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; - struct page *page = rx_buffer->page; - - /* avoid re-using remote and pfmemalloc pages */ - if (!dev_page_is_reusable(page)) - return false; - -#if (PAGE_SIZE < 8192) - /* if we are only owner of page we can reuse it */ - if (unlikely((page_ref_count(page) - pagecnt_bias) > 1)) - return false; -#else -#define IXGBEVF_LAST_OFFSET \ - (SKB_WITH_OVERHEAD(PAGE_SIZE) - IXGBEVF_RXBUFFER_2048) - - if (rx_buffer->page_offset > IXGBEVF_LAST_OFFSET) - return false; - -#endif - - /* If we have drained the page fragment pool we need to update - * the pagecnt_bias and page count so that we fully restock the - * number of references the driver holds. - */ - if (unlikely(!pagecnt_bias)) { - page_ref_add(page, USHRT_MAX); - rx_buffer->pagecnt_bias = USHRT_MAX; - } - - return true; -} - /** * ixgbevf_add_rx_frag - Add contents of Rx buffer to sk_buff * @rx_ring: rx descriptor ring to transact packets on @@ -829,18 +735,10 @@ static void ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring, struct sk_buff *skb, unsigned int size) { -#if (PAGE_SIZE < 8192) - unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2; -#else unsigned int truesize = SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size); -#endif + skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, rx_buffer->page_offset, size, truesize); -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif } static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, @@ -857,13 +755,9 @@ static struct sk_buff *ixgbevf_build_skb(struct ixgbevf_ring *rx_ring, union ixgbe_adv_rx_desc *rx_desc) { unsigned int metasize = xdp->data - xdp->data_meta; -#if (PAGE_SIZE < 8192) - unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2; -#else unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + SKB_DATA_ALIGN(xdp->data_end - xdp->data_hard_start); -#endif struct sk_buff *skb; /* Prefetch first cache line of first page. If xdp->data_meta @@ -884,13 +778,6 @@ static struct sk_buff *ixgbevf_build_skb(struct ixgbevf_ring *rx_ring, if (metasize) skb_metadata_set(skb, metasize); - /* update buffer offset */ -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif - return skb; } @@ -1014,38 +901,11 @@ static int ixgbevf_run_xdp(struct ixgbevf_adapter *adapter, return result; } -static unsigned int ixgbevf_rx_frame_truesize(struct ixgbevf_ring *rx_ring, - unsigned int size) -{ - unsigned int truesize; - -#if (PAGE_SIZE < 8192) - truesize = ixgbevf_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */ -#else - truesize = SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size) + - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); -#endif - return truesize; -} - -static void ixgbevf_rx_buffer_flip(struct ixgbevf_ring *rx_ring, - struct ixgbevf_rx_buffer *rx_buffer, - unsigned int size) -{ - unsigned int truesize = ixgbevf_rx_frame_truesize(rx_ring, size); - -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif -} - static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, struct ixgbevf_ring *rx_ring, int budget) { - unsigned int total_rx_bytes = 0, total_rx_packets = 0, frame_sz = 0; + unsigned int total_rx_bytes = 0, total_rx_packets = 0; struct ixgbevf_adapter *adapter = q_vector->adapter; u16 cleaned_count = ixgbevf_desc_unused(rx_ring); struct sk_buff *skb = rx_ring->skb; @@ -1054,10 +914,7 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, int xdp_res = 0; /* Frame size depend on rx_ring setup when PAGE_SIZE=4K */ -#if (PAGE_SIZE < 8192) - frame_sz = ixgbevf_rx_frame_truesize(rx_ring, 0); -#endif - xdp_init_buff(&xdp, frame_sz, &rx_ring->xdp_rxq); + xdp_init_buff(&xdp, IXGBEVF_RXBUFFER_3072, &rx_ring->xdp_rxq); while (likely(total_rx_packets < budget)) { struct ixgbevf_rx_buffer *rx_buffer; @@ -1081,31 +938,24 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, */ rmb(); - rx_buffer = ixgbevf_get_rx_buffer(rx_ring, size); + rx_buffer = + ixgbevf_get_rx_buffer(rx_ring, IXGBEVF_RXBUFFER_3072); /* retrieve a buffer from the ring */ if (!skb) { - unsigned int offset = ixgbevf_rx_offset(rx_ring); + unsigned int offset = rx_buffer->page_offset; unsigned char *hard_start; hard_start = page_address(rx_buffer->page) + rx_buffer->page_offset - offset; xdp_prepare_buff(&xdp, hard_start, offset, size, true); -#if (PAGE_SIZE > 4096) - /* At larger PAGE_SIZE, frame_sz depend on len size */ - xdp.frame_sz = ixgbevf_rx_frame_truesize(rx_ring, size); -#endif xdp_res = ixgbevf_run_xdp(adapter, rx_ring, &xdp); } if (xdp_res) { - if (xdp_res == IXGBEVF_XDP_TX) { + if (xdp_res == IXGBEVF_XDP_TX) xdp_xmit = true; - ixgbevf_rx_buffer_flip(rx_ring, rx_buffer, - size); - } else { - rx_buffer->pagecnt_bias++; - } + total_rx_packets++; total_rx_bytes += size; } else if (skb) { @@ -1118,11 +968,13 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, /* exit if we failed to retrieve a buffer */ if (!xdp_res && !skb) { rx_ring->rx_stats.alloc_rx_buff_failed++; - rx_buffer->pagecnt_bias++; break; } - ixgbevf_put_rx_buffer(rx_ring, rx_buffer, skb); + ixgbevf_put_rx_buffer(rx_ring, rx_buffer); + if (xdp_res == IXGBEVF_XDP_CONSUMED) + __free_page(rx_buffer->page); + rx_buffer->page = NULL; cleaned_count++; /* fetch next buffer in frame if non-eop */ @@ -1699,10 +1551,7 @@ static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, srrctl = IXGBE_SRRCTL_DROP_EN; srrctl |= IXGBEVF_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT; - if (ring_uses_large_buffer(ring)) - srrctl |= IXGBEVF_RXBUFFER_3072 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; - else - srrctl |= IXGBEVF_RXBUFFER_2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; + srrctl |= IXGBEVF_RXBUFFER_3072 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl); @@ -1880,13 +1729,6 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, if (adapter->hw.mac.type != ixgbe_mac_82599_vf) { rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | IXGBE_RXDCTL_RLPML_EN); - -#if (PAGE_SIZE < 8192) - /* Limit the maximum frame size so we don't overrun the skb */ - if (!ring_uses_large_buffer(ring)) - rxdctl |= IXGBEVF_MAX_FRAME_BUILD_SKB | - IXGBE_RXDCTL_RLPML_EN; -#endif } rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; @@ -1896,24 +1738,6 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, ixgbevf_alloc_rx_buffers(ring, ixgbevf_desc_unused(ring)); } -static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *rx_ring) -{ - struct net_device *netdev = adapter->netdev; - unsigned int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; - - /* set buffer size flags */ - clear_ring_uses_large_buffer(rx_ring); - - if (PAGE_SIZE < 8192) - /* 82599 can't rely on RXDCTL.RLPML to restrict - * the size of the frame - */ - if (max_frame > IXGBEVF_MAX_FRAME_BUILD_SKB || - adapter->hw.mac.type == ixgbe_mac_82599_vf) - set_ring_uses_large_buffer(rx_ring); -} - /** * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset * @adapter: board private structure @@ -1944,7 +1768,6 @@ static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter) for (i = 0; i < adapter->num_rx_queues; i++) { struct ixgbevf_ring *rx_ring = adapter->rx_ring[i]; - ixgbevf_set_rx_buffer_len(adapter, rx_ring); ixgbevf_configure_rx_ring(adapter, rx_ring); } } @@ -2323,19 +2146,13 @@ static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) dma_sync_single_range_for_cpu(rx_ring->dev, rx_buffer->dma, rx_buffer->page_offset, - ixgbevf_rx_bufsz(rx_ring), + IXGBEVF_RXBUFFER_3072, DMA_FROM_DEVICE); /* free resources associated with mapping */ - dma_unmap_page_attrs(rx_ring->dev, - rx_buffer->dma, - ixgbevf_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, - IXGBEVF_RX_DMA_ATTR); - - __page_frag_cache_drain(rx_buffer->page, - rx_buffer->pagecnt_bias); - + ixgbevf_put_rx_buffer(rx_ring, rx_buffer); + __free_page(rx_buffer->page); + rx_buffer->page = NULL; i++; if (i == rx_ring->count) i = 0; @@ -4394,9 +4211,7 @@ static int ixgbevf_xdp_setup(struct net_device *dev, struct bpf_prog *prog) /* verify ixgbevf ring attributes are sufficient for XDP */ for (i = 0; i < adapter->num_rx_queues; i++) { - struct ixgbevf_ring *ring = adapter->rx_ring[i]; - - if (frame_size > ixgbevf_rx_bufsz(ring)) + if (frame_size > IXGBEVF_RXBUFFER_3072) return -EINVAL; } From db3a8a35a7650f5e5b3d19df2bcf4e9344be1ca1 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Wed, 9 Jul 2025 09:28:02 +0200 Subject: [PATCH 02/39] ixgbevf: use libeth in Rx processing Use page_pool buffers by the means of libeth in the Rx queues, this significantly reduces code complexity of the driver itself. Suggested-by: Alexander Lobakin Reviewed-by: Alexander Lobakin Reviewed-by: Aleksandr Loktionov Signed-off-by: Larysa Zaremba --- drivers/net/ethernet/intel/Kconfig | 1 + drivers/net/ethernet/intel/ixgbevf/defines.h | 2 +- drivers/net/ethernet/intel/ixgbevf/ixgbevf.h | 21 +- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 356 +++++++----------- 4 files changed, 141 insertions(+), 239 deletions(-) diff --git a/drivers/net/ethernet/intel/Kconfig b/drivers/net/ethernet/intel/Kconfig index 288fa8ce53af0a..b513baf3cbb296 100644 --- a/drivers/net/ethernet/intel/Kconfig +++ b/drivers/net/ethernet/intel/Kconfig @@ -203,6 +203,7 @@ config IXGBE_IPSEC config IXGBEVF tristate "Intel(R) 10GbE PCI Express Virtual Function Ethernet support" depends on PCI_MSI + select LIBETH_XDP help This driver supports Intel(R) PCI Express virtual functions for the Intel(R) ixgbe driver. For more information on how to identify your diff --git a/drivers/net/ethernet/intel/ixgbevf/defines.h b/drivers/net/ethernet/intel/ixgbevf/defines.h index e177d1d58696aa..afc927dd14381b 100644 --- a/drivers/net/ethernet/intel/ixgbevf/defines.h +++ b/drivers/net/ethernet/intel/ixgbevf/defines.h @@ -71,7 +71,7 @@ typedef u32 ixgbe_link_speed; #define IXGBE_PSRTYPE_L2HDR 0x00001000 /* SRRCTL bit definitions */ -#define IXGBE_SRRCTL_BSIZEPKT_SHIFT 10 /* so many KBs */ +#define IXGBE_SRRCTL_BSIZEPKT_STEP 1024 #define IXGBE_SRRCTL_RDMTS_SHIFT 22 #define IXGBE_SRRCTL_RDMTS_MASK 0x01C00000 #define IXGBE_SRRCTL_DROP_EN 0x10000000 diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h index 2d7ca3f86868bc..ebf771f0caa4b3 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h @@ -42,12 +42,6 @@ struct ixgbevf_tx_buffer { u32 tx_flags; }; -struct ixgbevf_rx_buffer { - dma_addr_t dma; - struct page *page; - __u32 page_offset; -}; - struct ixgbevf_stats { u64 packets; u64 bytes; @@ -84,19 +78,22 @@ struct ixgbevf_ring { struct ixgbevf_ring *next; struct ixgbevf_q_vector *q_vector; /* backpointer to q_vector */ struct net_device *netdev; - struct bpf_prog *xdp_prog; - struct device *dev; + struct bpf_prog __rcu *xdp_prog; + union { + struct page_pool *pp; /* Rx ring */ + struct device *dev; /* Tx ring */ + }; void *desc; /* descriptor ring memory */ dma_addr_t dma; /* phys. address of descriptor ring */ unsigned int size; /* length in bytes */ + u32 truesize; /* Rx buffer full size */ u16 count; /* amount of descriptors */ u16 next_to_use; u16 next_to_clean; - u16 next_to_alloc; union { + struct libeth_fqe *rx_fqes; struct ixgbevf_tx_buffer *tx_buffer_info; - struct ixgbevf_rx_buffer *rx_buffer_info; }; unsigned long state; struct ixgbevf_stats stats; @@ -115,6 +112,7 @@ struct ixgbevf_ring { */ u16 reg_idx; int queue_index; /* needed for multiqueue queue management */ + u32 rx_buf_len; } ____cacheline_internodealigned_in_smp; /* How many Rx Buffers do we bundle into one write to the hardware ? */ @@ -144,7 +142,8 @@ struct ixgbevf_ring { #define MAXIMUM_ETHERNET_VLAN_SIZE (VLAN_ETH_FRAME_LEN + ETH_FCS_LEN) -#define IXGBEVF_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN) +#define IXGBEVF_RX_PAGE_LEN(hr) (ALIGN_DOWN(LIBETH_RX_PAGE_LEN(hr), \ + IXGBE_SRRCTL_BSIZEPKT_STEP)) #define IXGBE_TX_FLAGS_CSUM BIT(0) #define IXGBE_TX_FLAGS_VLAN BIT(1) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index 05baf28823c869..5d32fab4d1a33a 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include "ixgbevf.h" @@ -82,6 +83,7 @@ static const struct pci_device_id ixgbevf_pci_tbl[] = { MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl); MODULE_DESCRIPTION("Intel(R) 10 Gigabit Virtual Function Network Driver"); +MODULE_IMPORT_NS("LIBETH"); MODULE_LICENSE("GPL v2"); #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) @@ -304,7 +306,7 @@ static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, /* free the skb */ if (ring_is_xdp(tx_ring)) - page_frag_free(tx_buffer->data); + libeth_xdp_return_va(tx_buffer->data, true); else napi_consume_skb(tx_buffer->skb, napi_budget); @@ -521,33 +523,6 @@ static void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring, if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_STAT_SECP)) ixgbevf_ipsec_rx(rx_ring, rx_desc, skb); - - skb->protocol = eth_type_trans(skb, rx_ring->netdev); -} - -static -struct ixgbevf_rx_buffer *ixgbevf_get_rx_buffer(struct ixgbevf_ring *rx_ring, - const unsigned int size) -{ - struct ixgbevf_rx_buffer *rx_buffer; - - rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; - prefetchw(rx_buffer->page); - - dma_sync_single_range_for_cpu(rx_ring->dev, - rx_buffer->dma, - rx_buffer->page_offset, - size, - DMA_FROM_DEVICE); - - return rx_buffer; -} - -static void ixgbevf_put_rx_buffer(struct ixgbevf_ring *rx_ring, - struct ixgbevf_rx_buffer *rx_buffer) -{ - dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, PAGE_SIZE, - DMA_FROM_DEVICE, IXGBEVF_RX_DMA_ATTR); } /** @@ -577,41 +552,6 @@ static bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring, return true; } -static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring, - struct ixgbevf_rx_buffer *bi) -{ - struct page *page; - dma_addr_t dma; - - /* alloc new page for storage */ - page = dev_alloc_page(); - if (unlikely(!page)) { - rx_ring->rx_stats.alloc_rx_page_failed++; - return false; - } - - /* map page for use */ - dma = dma_map_page_attrs(rx_ring->dev, page, 0, PAGE_SIZE, - DMA_FROM_DEVICE, IXGBEVF_RX_DMA_ATTR); - - /* if mapping failed free memory back to system since - * there isn't much point in holding memory we can't use - */ - if (dma_mapping_error(rx_ring->dev, dma)) { - __free_page(page); - - rx_ring->rx_stats.alloc_rx_page_failed++; - return false; - } - - bi->dma = dma; - bi->page = page; - bi->page_offset = IXGBEVF_SKB_PAD; - rx_ring->rx_stats.alloc_rx_page++; - - return true; -} - /** * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split * @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on @@ -621,39 +561,34 @@ static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, u16 cleaned_count) { union ixgbe_adv_rx_desc *rx_desc; - struct ixgbevf_rx_buffer *bi; - unsigned int i = rx_ring->next_to_use; + const struct libeth_fq_fp fq = { + .pp = rx_ring->pp, + .fqes = rx_ring->rx_fqes, + .truesize = rx_ring->truesize, + .count = rx_ring->count, + }; + u16 ntu = rx_ring->next_to_use; /* nothing to do or no valid netdev defined */ - if (!cleaned_count || !rx_ring->netdev) + if (unlikely(!cleaned_count || !rx_ring->netdev)) return; - rx_desc = IXGBEVF_RX_DESC(rx_ring, i); - bi = &rx_ring->rx_buffer_info[i]; - i -= rx_ring->count; + rx_desc = IXGBEVF_RX_DESC(rx_ring, ntu); do { - if (!ixgbevf_alloc_mapped_page(rx_ring, bi)) - break; + dma_addr_t addr; - /* sync the buffer for use by the device */ - dma_sync_single_range_for_device(rx_ring->dev, bi->dma, - bi->page_offset, - IXGBEVF_RXBUFFER_3072, - DMA_FROM_DEVICE); + addr = libeth_rx_alloc(&fq, ntu); + if (addr == DMA_MAPPING_ERROR) + break; - /* Refresh the desc even if pkt_addr didn't change - * because each write-back erases this info. - */ - rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); + rx_desc->read.pkt_addr = cpu_to_le64(addr); rx_desc++; - bi++; - i++; - if (unlikely(!i)) { + ntu++; + if (unlikely(ntu == fq.count)) { rx_desc = IXGBEVF_RX_DESC(rx_ring, 0); - bi = rx_ring->rx_buffer_info; - i -= rx_ring->count; + ntu = 0; } /* clear the length for the next_to_use descriptor */ @@ -662,14 +597,9 @@ static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, cleaned_count--; } while (cleaned_count); - i += rx_ring->count; - - if (rx_ring->next_to_use != i) { + if (likely(rx_ring->next_to_use != ntu)) { /* record the next descriptor to use */ - rx_ring->next_to_use = i; - - /* update next to alloc since we have filled the ring */ - rx_ring->next_to_alloc = i; + rx_ring->next_to_use = ntu; /* Force memory writes to complete before letting h/w * know there are new descriptors to fetch. (Only @@ -677,7 +607,7 @@ static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, * such as IA-64). */ wmb(); - ixgbevf_write_tail(rx_ring, i); + ixgbevf_write_tail(rx_ring, ntu); } } @@ -714,10 +644,6 @@ static bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring, } } - /* if eth_skb_pad returns an error the skb was freed */ - if (eth_skb_pad(skb)) - return true; - return false; } @@ -730,15 +656,15 @@ static bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring, * * This function will add the data contained in rx_buffer->page to the skb. **/ -static void ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring, - struct ixgbevf_rx_buffer *rx_buffer, +static void ixgbevf_add_rx_frag(const struct libeth_fqe *rx_buffer, struct sk_buff *skb, unsigned int size) { - unsigned int truesize = SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size); + u32 hr = netmem_get_pp(rx_buffer->netmem)->p.offset; - skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, - rx_buffer->page_offset, size, truesize); + skb_add_rx_frag_netmem(skb, skb_shinfo(skb)->nr_frags, + rx_buffer->netmem, rx_buffer->offset + hr, + size, rx_buffer->truesize); } static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, @@ -749,38 +675,6 @@ static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask); } -static struct sk_buff *ixgbevf_build_skb(struct ixgbevf_ring *rx_ring, - struct ixgbevf_rx_buffer *rx_buffer, - struct xdp_buff *xdp, - union ixgbe_adv_rx_desc *rx_desc) -{ - unsigned int metasize = xdp->data - xdp->data_meta; - unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + - SKB_DATA_ALIGN(xdp->data_end - - xdp->data_hard_start); - struct sk_buff *skb; - - /* Prefetch first cache line of first page. If xdp->data_meta - * is unused, this points to xdp->data, otherwise, we likely - * have a consumer accessing first few bytes of meta data, - * and then actual data. - */ - net_prefetch(xdp->data_meta); - - /* build an skb around the page buffer */ - skb = napi_build_skb(xdp->data_hard_start, truesize); - if (unlikely(!skb)) - return NULL; - - /* update pointers within the skb to store the data */ - skb_reserve(skb, xdp->data - xdp->data_hard_start); - __skb_put(skb, xdp->data_end - xdp->data); - if (metasize) - skb_metadata_set(skb, metasize); - - return skb; -} - #define IXGBEVF_XDP_PASS 0 #define IXGBEVF_XDP_CONSUMED 1 #define IXGBEVF_XDP_TX 2 @@ -864,25 +758,25 @@ static int ixgbevf_xmit_xdp_ring(struct ixgbevf_ring *ring, static int ixgbevf_run_xdp(struct ixgbevf_adapter *adapter, struct ixgbevf_ring *rx_ring, - struct xdp_buff *xdp) + struct libeth_xdp_buff *xdp) { int result = IXGBEVF_XDP_PASS; struct ixgbevf_ring *xdp_ring; struct bpf_prog *xdp_prog; u32 act; - xdp_prog = READ_ONCE(rx_ring->xdp_prog); + xdp_prog = rcu_dereference(rx_ring->xdp_prog); if (!xdp_prog) goto xdp_out; - act = bpf_prog_run_xdp(xdp_prog, xdp); + act = bpf_prog_run_xdp(xdp_prog, &xdp->base); switch (act) { case XDP_PASS: break; case XDP_TX: xdp_ring = adapter->xdp_ring[rx_ring->queue_index]; - result = ixgbevf_xmit_xdp_ring(xdp_ring, xdp); + result = ixgbevf_xmit_xdp_ring(xdp_ring, &xdp->base); if (result == IXGBEVF_XDP_CONSUMED) goto out_failure; break; @@ -895,6 +789,7 @@ static int ixgbevf_run_xdp(struct ixgbevf_adapter *adapter, fallthrough; /* handle aborts by dropping packet */ case XDP_DROP: result = IXGBEVF_XDP_CONSUMED; + libeth_xdp_return_buff(xdp); break; } xdp_out: @@ -909,16 +804,15 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, struct ixgbevf_adapter *adapter = q_vector->adapter; u16 cleaned_count = ixgbevf_desc_unused(rx_ring); struct sk_buff *skb = rx_ring->skb; + LIBETH_XDP_ONSTACK_BUFF(xdp); bool xdp_xmit = false; - struct xdp_buff xdp; int xdp_res = 0; - /* Frame size depend on rx_ring setup when PAGE_SIZE=4K */ - xdp_init_buff(&xdp, IXGBEVF_RXBUFFER_3072, &rx_ring->xdp_rxq); + xdp->base.rxq = &rx_ring->xdp_rxq; while (likely(total_rx_packets < budget)) { - struct ixgbevf_rx_buffer *rx_buffer; union ixgbe_adv_rx_desc *rx_desc; + struct libeth_fqe *rx_buffer; unsigned int size; /* return some buffers to hardware, one at a time is too slow */ @@ -929,7 +823,7 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, rx_desc = IXGBEVF_RX_DESC(rx_ring, rx_ring->next_to_clean); size = le16_to_cpu(rx_desc->wb.upper.length); - if (!size) + if (unlikely(!size)) break; /* This memory barrier is needed to keep us from reading @@ -938,18 +832,14 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, */ rmb(); - rx_buffer = - ixgbevf_get_rx_buffer(rx_ring, IXGBEVF_RXBUFFER_3072); + rx_buffer = &rx_ring->rx_fqes[rx_ring->next_to_clean]; + libeth_rx_sync_for_cpu(rx_buffer, size); /* retrieve a buffer from the ring */ if (!skb) { - unsigned int offset = rx_buffer->page_offset; - unsigned char *hard_start; - - hard_start = page_address(rx_buffer->page) + - rx_buffer->page_offset - offset; - xdp_prepare_buff(&xdp, hard_start, offset, size, true); - xdp_res = ixgbevf_run_xdp(adapter, rx_ring, &xdp); + libeth_xdp_prepare_buff(xdp, rx_buffer, size); + prefetch(xdp->data); + xdp_res = ixgbevf_run_xdp(adapter, rx_ring, xdp); } if (xdp_res) { @@ -959,22 +849,18 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, total_rx_packets++; total_rx_bytes += size; } else if (skb) { - ixgbevf_add_rx_frag(rx_ring, rx_buffer, skb, size); + ixgbevf_add_rx_frag(rx_buffer, skb, size); } else { - skb = ixgbevf_build_skb(rx_ring, rx_buffer, - &xdp, rx_desc); + skb = xdp_build_skb_from_buff(&xdp->base); } /* exit if we failed to retrieve a buffer */ - if (!xdp_res && !skb) { + if (unlikely(!xdp_res && !skb)) { + libeth_xdp_return_buff_slow(xdp); rx_ring->rx_stats.alloc_rx_buff_failed++; break; } - ixgbevf_put_rx_buffer(rx_ring, rx_buffer); - if (xdp_res == IXGBEVF_XDP_CONSUMED) - __free_page(rx_buffer->page); - rx_buffer->page = NULL; cleaned_count++; /* fetch next buffer in frame if non-eop */ @@ -982,21 +868,19 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, continue; /* verify the packet layout is correct */ - if (xdp_res || ixgbevf_cleanup_headers(rx_ring, rx_desc, skb)) { + if (xdp_res || + unlikely(ixgbevf_cleanup_headers(rx_ring, rx_desc, skb))) { skb = NULL; continue; } - /* probably a little skewed due to removing CRC */ - total_rx_bytes += skb->len; - /* Workaround hardware that can't do proper VEPA multicast * source pruning. */ - if ((skb->pkt_type == PACKET_BROADCAST || - skb->pkt_type == PACKET_MULTICAST) && - ether_addr_equal(rx_ring->netdev->dev_addr, - eth_hdr(skb)->h_source)) { + if (unlikely((skb->pkt_type == PACKET_BROADCAST || + skb->pkt_type == PACKET_MULTICAST) && + ether_addr_equal(rx_ring->netdev->dev_addr, + eth_hdr(skb)->h_source))) { dev_kfree_skb_irq(skb); continue; } @@ -1004,13 +888,14 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, /* populate checksum, VLAN, and protocol */ ixgbevf_process_skb_fields(rx_ring, rx_desc, skb); + /* probably a little skewed due to removing CRC */ + total_rx_bytes += skb->len; + total_rx_packets++; + ixgbevf_rx_skb(q_vector, skb); /* reset skb pointer */ skb = NULL; - - /* update budget accounting */ - total_rx_packets++; } /* place incomplete frames back on ring for completion */ @@ -1543,7 +1428,8 @@ static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter) #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *ring, int index) + struct ixgbevf_ring *ring, int index, + bool rlpml_valid) { struct ixgbe_hw *hw = &adapter->hw; u32 srrctl; @@ -1551,7 +1437,11 @@ static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, srrctl = IXGBE_SRRCTL_DROP_EN; srrctl |= IXGBEVF_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT; - srrctl |= IXGBEVF_RXBUFFER_3072 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; + if (rlpml_valid) + srrctl |= DIV_ROUND_UP(ring->rx_buf_len, + IXGBE_SRRCTL_BSIZEPKT_STEP); + else + srrctl |= ring->rx_buf_len / IXGBE_SRRCTL_BSIZEPKT_STEP; srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl); @@ -1682,9 +1572,10 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, { struct ixgbe_hw *hw = &adapter->hw; union ixgbe_adv_rx_desc *rx_desc; + u8 reg_idx = ring->reg_idx; + bool rlpml_valid = false; u64 rdba = ring->dma; u32 rxdctl; - u8 reg_idx = ring->reg_idx; /* disable queue to avoid issues while updating state */ rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); @@ -1710,10 +1601,6 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, IXGBE_WRITE_REG(hw, IXGBE_VFRDT(reg_idx), 0); ring->tail = adapter->io_addr + IXGBE_VFRDT(reg_idx); - /* initialize rx_buffer_info */ - memset(ring->rx_buffer_info, 0, - sizeof(struct ixgbevf_rx_buffer) * ring->count); - /* initialize Rx descriptor 0 */ rx_desc = IXGBEVF_RX_DESC(ring, 0); rx_desc->wb.upper.length = 0; @@ -1721,16 +1608,21 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, /* reset ntu and ntc to place SW in sync with hardwdare */ ring->next_to_clean = 0; ring->next_to_use = 0; - ring->next_to_alloc = 0; - - ixgbevf_configure_srrctl(adapter, ring, reg_idx); /* RXDCTL.RLPML does not work on 82599 */ if (adapter->hw.mac.type != ixgbe_mac_82599_vf) { - rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | - IXGBE_RXDCTL_RLPML_EN); + u32 pkt_len = + READ_ONCE(adapter->netdev->mtu) + LIBETH_RX_LL_LEN; + + rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | IXGBE_RXDCTL_RLPML_EN); + if (pkt_len <= IXGBE_RXDCTL_RLPMLMASK) { + rxdctl |= pkt_len | IXGBE_RXDCTL_RLPML_EN; + rlpml_valid = true; + } } + ixgbevf_configure_srrctl(adapter, ring, reg_idx, rlpml_valid); + rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl); @@ -2126,8 +2018,6 @@ void ixgbevf_up(struct ixgbevf_adapter *adapter) **/ static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) { - u16 i = rx_ring->next_to_clean; - /* Free Rx ring sk_buff */ if (rx_ring->skb) { dev_kfree_skb(rx_ring->skb); @@ -2135,30 +2025,14 @@ static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) } /* Free all the Rx ring pages */ - while (i != rx_ring->next_to_alloc) { - struct ixgbevf_rx_buffer *rx_buffer; - - rx_buffer = &rx_ring->rx_buffer_info[i]; + for (u32 i = rx_ring->next_to_clean; i != rx_ring->next_to_use; ) { + const struct libeth_fqe *rx_fqe = &rx_ring->rx_fqes[i]; - /* Invalidate cache lines that may have been written to by - * device so that we avoid corrupting memory. - */ - dma_sync_single_range_for_cpu(rx_ring->dev, - rx_buffer->dma, - rx_buffer->page_offset, - IXGBEVF_RXBUFFER_3072, - DMA_FROM_DEVICE); - - /* free resources associated with mapping */ - ixgbevf_put_rx_buffer(rx_ring, rx_buffer); - __free_page(rx_buffer->page); - rx_buffer->page = NULL; - i++; - if (i == rx_ring->count) + libeth_rx_recycle_slow(rx_fqe->netmem); + if (unlikely(++i == rx_ring->count)) i = 0; } - rx_ring->next_to_alloc = 0; rx_ring->next_to_clean = 0; rx_ring->next_to_use = 0; } @@ -2177,7 +2051,7 @@ static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring) /* Free all the Tx ring sk_buffs */ if (ring_is_xdp(tx_ring)) - page_frag_free(tx_buffer->data); + libeth_xdp_return_va(tx_buffer->data, false); else dev_kfree_skb_any(tx_buffer->skb); @@ -2575,7 +2449,6 @@ static int ixgbevf_alloc_q_vector(struct ixgbevf_adapter *adapter, int v_idx, while (rxr_count) { /* assign generic ring traits */ - ring->dev = &adapter->pdev->dev; ring->netdev = adapter->netdev; /* configure backlink on ring */ @@ -3259,12 +3132,25 @@ static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter) int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, struct ixgbevf_ring *rx_ring) { - int size; + struct libeth_fq fq = { + .count = rx_ring->count, + .nid = NUMA_NO_NODE, + .type = LIBETH_FQE_MTU, + .xdp = !!rx_ring->xdp_prog, + .buf_len = IXGBEVF_RX_PAGE_LEN(rx_ring->xdp_prog ? + LIBETH_XDP_HEADROOM : + LIBETH_SKB_HEADROOM), + }; + int ret; - size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; - rx_ring->rx_buffer_info = vmalloc(size); - if (!rx_ring->rx_buffer_info) - goto err; + ret = libeth_rx_fq_create(&fq, &rx_ring->q_vector->napi); + if (ret) + return ret; + + rx_ring->pp = fq.pp; + rx_ring->rx_fqes = fq.fqes; + rx_ring->truesize = fq.truesize; + rx_ring->rx_buf_len = fq.buf_len; u64_stats_init(&rx_ring->syncp); @@ -3272,25 +3158,32 @@ int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); rx_ring->size = ALIGN(rx_ring->size, 4096); - rx_ring->desc = dma_alloc_coherent(rx_ring->dev, rx_ring->size, + rx_ring->desc = dma_alloc_coherent(fq.pp->p.dev, rx_ring->size, &rx_ring->dma, GFP_KERNEL); - if (!rx_ring->desc) + if (!rx_ring->desc) { + ret = -ENOMEM; + dev_err(&adapter->pdev->dev, + "Unable to allocate memory for the Rx descriptor ring\n"); goto err; + } /* XDP RX-queue info */ - if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, - rx_ring->queue_index, 0) < 0) + ret = __xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, + rx_ring->queue_index, 0, rx_ring->truesize); + if (ret) goto err; - rx_ring->xdp_prog = adapter->xdp_prog; + xdp_rxq_info_attach_page_pool(&rx_ring->xdp_rxq, fq.pp); + + rcu_assign_pointer(rx_ring->xdp_prog, adapter->xdp_prog); return 0; err: - vfree(rx_ring->rx_buffer_info); - rx_ring->rx_buffer_info = NULL; - dev_err(rx_ring->dev, "Unable to allocate memory for the Rx descriptor ring\n"); - return -ENOMEM; + libeth_rx_fq_destroy(&fq); + rx_ring->rx_fqes = NULL; + rx_ring->pp = NULL; + return ret; } /** @@ -3331,17 +3224,24 @@ static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter) **/ void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring) { + struct libeth_fq fq = { + .fqes = rx_ring->rx_fqes, + .pp = rx_ring->pp, + }; + ixgbevf_clean_rx_ring(rx_ring); - rx_ring->xdp_prog = NULL; + rcu_assign_pointer(rx_ring->xdp_prog, NULL); + xdp_rxq_info_detach_mem_model(&rx_ring->xdp_rxq); xdp_rxq_info_unreg(&rx_ring->xdp_rxq); - vfree(rx_ring->rx_buffer_info); - rx_ring->rx_buffer_info = NULL; - dma_free_coherent(rx_ring->dev, rx_ring->size, rx_ring->desc, + dma_free_coherent(fq.pp->p.dev, rx_ring->size, rx_ring->desc, rx_ring->dma); - rx_ring->desc = NULL; + + libeth_rx_fq_destroy(&fq); + rx_ring->rx_fqes = NULL; + rx_ring->pp = NULL; } /** @@ -4233,7 +4133,9 @@ static int ixgbevf_xdp_setup(struct net_device *dev, struct bpf_prog *prog) ixgbevf_open(dev); } else { for (i = 0; i < adapter->num_rx_queues; i++) - xchg(&adapter->rx_ring[i]->xdp_prog, adapter->xdp_prog); + rcu_assign_pointer(adapter->rx_ring[i]->xdp_prog, + adapter->xdp_prog); + synchronize_net(); } if (old_prog) From b77a4e6188397fb39032a58af91fc408794b9c49 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Tue, 2 Sep 2025 16:31:51 +0200 Subject: [PATCH 03/39] ixgbevf: support XDP multi-buffer on Rx path Implement XDP support for received fragmented packets, this requires using some helpers from libeth_xdp. Reviewed-by: Aleksandr Loktionov Signed-off-by: Larysa Zaremba --- drivers/net/ethernet/intel/ixgbevf/ixgbevf.h | 3 +- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 98 +++++++------------ 2 files changed, 36 insertions(+), 65 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h index ebf771f0caa4b3..2626af0393614b 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include "vf.h" @@ -105,7 +106,6 @@ struct ixgbevf_ring { struct xdp_rxq_info xdp_rxq; u64 hw_csum_rx_error; u8 __iomem *tail; - struct sk_buff *skb; /* holds the special value that gets the hardware register offset * associated with this ring, which is different for DCB and RSS modes @@ -113,6 +113,7 @@ struct ixgbevf_ring { u16 reg_idx; int queue_index; /* needed for multiqueue queue management */ u32 rx_buf_len; + struct libeth_xdp_buff_stash xdp_stash; } ____cacheline_internodealigned_in_smp; /* How many Rx Buffers do we bundle into one write to the hardware ? */ diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index 5d32fab4d1a33a..884f93afa13b5a 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -84,6 +84,7 @@ MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl); MODULE_DESCRIPTION("Intel(R) 10 Gigabit Virtual Function Network Driver"); MODULE_IMPORT_NS("LIBETH"); +MODULE_IMPORT_NS("LIBETH_XDP"); MODULE_LICENSE("GPL v2"); #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) @@ -647,26 +648,6 @@ static bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring, return false; } -/** - * ixgbevf_add_rx_frag - Add contents of Rx buffer to sk_buff - * @rx_ring: rx descriptor ring to transact packets on - * @rx_buffer: buffer containing page to add - * @skb: sk_buff to place the data into - * @size: size of buffer to be added - * - * This function will add the data contained in rx_buffer->page to the skb. - **/ -static void ixgbevf_add_rx_frag(const struct libeth_fqe *rx_buffer, - struct sk_buff *skb, - unsigned int size) -{ - u32 hr = netmem_get_pp(rx_buffer->netmem)->p.offset; - - skb_add_rx_frag_netmem(skb, skb_shinfo(skb)->nr_frags, - rx_buffer->netmem, rx_buffer->offset + hr, - size, rx_buffer->truesize); -} - static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, u32 qmask) { @@ -803,16 +784,16 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, unsigned int total_rx_bytes = 0, total_rx_packets = 0; struct ixgbevf_adapter *adapter = q_vector->adapter; u16 cleaned_count = ixgbevf_desc_unused(rx_ring); - struct sk_buff *skb = rx_ring->skb; LIBETH_XDP_ONSTACK_BUFF(xdp); bool xdp_xmit = false; int xdp_res = 0; - xdp->base.rxq = &rx_ring->xdp_rxq; + libeth_xdp_init_buff(xdp, &rx_ring->xdp_stash, &rx_ring->xdp_rxq); while (likely(total_rx_packets < budget)) { union ixgbe_adv_rx_desc *rx_desc; struct libeth_fqe *rx_buffer; + struct sk_buff *skb; unsigned int size; /* return some buffers to hardware, one at a time is too slow */ @@ -833,43 +814,39 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, rmb(); rx_buffer = &rx_ring->rx_fqes[rx_ring->next_to_clean]; - libeth_rx_sync_for_cpu(rx_buffer, size); + libeth_xdp_process_buff(xdp, rx_buffer, size); - /* retrieve a buffer from the ring */ - if (!skb) { - libeth_xdp_prepare_buff(xdp, rx_buffer, size); - prefetch(xdp->data); - xdp_res = ixgbevf_run_xdp(adapter, rx_ring, xdp); - } + cleaned_count++; + /* fetch next buffer in frame if non-eop */ + if (ixgbevf_is_non_eop(rx_ring, rx_desc) || + unlikely(!xdp->data)) + continue; + + total_rx_packets++; + total_rx_bytes += xdp_get_buff_len(&xdp->base); + xdp_res = ixgbevf_run_xdp(adapter, rx_ring, xdp); if (xdp_res) { if (xdp_res == IXGBEVF_XDP_TX) xdp_xmit = true; - total_rx_packets++; - total_rx_bytes += size; - } else if (skb) { - ixgbevf_add_rx_frag(rx_buffer, skb, size); - } else { - skb = xdp_build_skb_from_buff(&xdp->base); + xdp->data = NULL; + continue; } + skb = xdp_build_skb_from_buff(&xdp->base); + /* exit if we failed to retrieve a buffer */ - if (unlikely(!xdp_res && !skb)) { + if (unlikely(!skb)) { libeth_xdp_return_buff_slow(xdp); rx_ring->rx_stats.alloc_rx_buff_failed++; break; } - cleaned_count++; - - /* fetch next buffer in frame if non-eop */ - if (ixgbevf_is_non_eop(rx_ring, rx_desc)) - continue; + xdp->data = NULL; /* verify the packet layout is correct */ - if (xdp_res || - unlikely(ixgbevf_cleanup_headers(rx_ring, rx_desc, skb))) { + if (unlikely(ixgbevf_cleanup_headers(rx_ring, rx_desc, skb))) { skb = NULL; continue; } @@ -888,18 +865,11 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, /* populate checksum, VLAN, and protocol */ ixgbevf_process_skb_fields(rx_ring, rx_desc, skb); - /* probably a little skewed due to removing CRC */ - total_rx_bytes += skb->len; - total_rx_packets++; - ixgbevf_rx_skb(q_vector, skb); - - /* reset skb pointer */ - skb = NULL; } /* place incomplete frames back on ring for completion */ - rx_ring->skb = skb; + libeth_xdp_save_buff(&rx_ring->xdp_stash, xdp); if (xdp_xmit) { struct ixgbevf_ring *xdp_ring = @@ -2019,10 +1989,7 @@ void ixgbevf_up(struct ixgbevf_adapter *adapter) static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) { /* Free Rx ring sk_buff */ - if (rx_ring->skb) { - dev_kfree_skb(rx_ring->skb); - rx_ring->skb = NULL; - } + libeth_xdp_return_stash(&rx_ring->xdp_stash); /* Free all the Rx ring pages */ for (u32 i = rx_ring->next_to_clean; i != rx_ring->next_to_use; ) { @@ -4103,16 +4070,19 @@ ixgbevf_features_check(struct sk_buff *skb, struct net_device *dev, return features; } -static int ixgbevf_xdp_setup(struct net_device *dev, struct bpf_prog *prog) +static int ixgbevf_xdp_setup(struct net_device *dev, struct bpf_prog *prog, + struct netlink_ext_ack *extack) { - int i, frame_size = dev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; + u32 frame_size = READ_ONCE(dev->mtu) + LIBETH_RX_LL_LEN; struct ixgbevf_adapter *adapter = netdev_priv(dev); struct bpf_prog *old_prog; + bool requires_mbuf; - /* verify ixgbevf ring attributes are sufficient for XDP */ - for (i = 0; i < adapter->num_rx_queues; i++) { - if (frame_size > IXGBEVF_RXBUFFER_3072) - return -EINVAL; + requires_mbuf = frame_size > IXGBEVF_RX_PAGE_LEN(LIBETH_XDP_HEADROOM); + if (prog && !prog->aux->xdp_has_frags && requires_mbuf) { + NL_SET_ERR_MSG_MOD(extack, + "Configured MTU requires non-linear frames and XDP prog does not support frags"); + return -EOPNOTSUPP; } old_prog = xchg(&adapter->xdp_prog, prog); @@ -4132,7 +4102,7 @@ static int ixgbevf_xdp_setup(struct net_device *dev, struct bpf_prog *prog) if (netif_running(dev)) ixgbevf_open(dev); } else { - for (i = 0; i < adapter->num_rx_queues; i++) + for (int i = 0; i < adapter->num_rx_queues; i++) rcu_assign_pointer(adapter->rx_ring[i]->xdp_prog, adapter->xdp_prog); synchronize_net(); @@ -4148,7 +4118,7 @@ static int ixgbevf_xdp(struct net_device *dev, struct netdev_bpf *xdp) { switch (xdp->command) { case XDP_SETUP_PROG: - return ixgbevf_xdp_setup(dev, xdp->prog); + return ixgbevf_xdp_setup(dev, xdp->prog, xdp->extack); default: return -EINVAL; } @@ -4300,7 +4270,7 @@ static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) NETIF_F_HW_VLAN_CTAG_TX; netdev->priv_flags |= IFF_UNICAST_FLT; - netdev->xdp_features = NETDEV_XDP_ACT_BASIC; + netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_RX_SG; /* MTU range: 68 - 1504 or 9710 */ netdev->min_mtu = ETH_MIN_MTU; From c0b1547ae6243e40f4170365d66620532c5b0971 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Tue, 9 Sep 2025 13:46:44 +0200 Subject: [PATCH 04/39] ixgbevf: XDP_TX in multi-buffer through libeth Use libeth to support XDP_TX action for segmented packets. Reviewed-by: Alexander Lobakin Signed-off-by: Larysa Zaremba --- drivers/net/ethernet/intel/ixgbevf/ixgbevf.h | 14 +- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 217 +++++++++--------- .../ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h | 112 +++++++++ 3 files changed, 224 insertions(+), 119 deletions(-) create mode 100644 drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h index 2626af0393614b..a27081ee764b40 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h @@ -81,20 +81,22 @@ struct ixgbevf_ring { struct net_device *netdev; struct bpf_prog __rcu *xdp_prog; union { - struct page_pool *pp; /* Rx ring */ + struct page_pool *pp; /* Rx and XDP rings */ struct device *dev; /* Tx ring */ }; void *desc; /* descriptor ring memory */ - dma_addr_t dma; /* phys. address of descriptor ring */ - unsigned int size; /* length in bytes */ - u32 truesize; /* Rx buffer full size */ + union { + u32 truesize; /* Rx buffer full size */ + u32 pending; /* Sent-not-completed descriptors */ + }; u16 count; /* amount of descriptors */ - u16 next_to_use; u16 next_to_clean; + u32 next_to_use; union { struct libeth_fqe *rx_fqes; struct ixgbevf_tx_buffer *tx_buffer_info; + struct libeth_sqe *xdp_sqes; }; unsigned long state; struct ixgbevf_stats stats; @@ -114,6 +116,8 @@ struct ixgbevf_ring { int queue_index; /* needed for multiqueue queue management */ u32 rx_buf_len; struct libeth_xdp_buff_stash xdp_stash; + unsigned int dma_size; /* length in bytes */ + dma_addr_t dma; /* phys. address of descriptor ring */ } ____cacheline_internodealigned_in_smp; /* How many Rx Buffers do we bundle into one write to the hardware ? */ diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index 884f93afa13b5a..350ed84147e668 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -33,7 +33,7 @@ #include #include -#include "ixgbevf.h" +#include "ixgbevf_xdp_lib.h" const char ixgbevf_driver_name[] = "ixgbevf"; static const char ixgbevf_driver_string[] = @@ -306,10 +306,7 @@ static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, total_ipsec++; /* free the skb */ - if (ring_is_xdp(tx_ring)) - libeth_xdp_return_va(tx_buffer->data, true); - else - napi_consume_skb(tx_buffer->skb, napi_budget); + napi_consume_skb(tx_buffer->skb, napi_budget); /* unmap skb header data */ dma_unmap_single(tx_ring->dev, @@ -392,9 +389,8 @@ static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, eop_desc, (eop_desc ? eop_desc->wb.status : 0), tx_ring->tx_buffer_info[i].time_stamp, jiffies); - if (!ring_is_xdp(tx_ring)) - netif_stop_subqueue(tx_ring->netdev, - tx_ring->queue_index); + netif_stop_subqueue(tx_ring->netdev, + tx_ring->queue_index); /* schedule immediate reset if we believe we hung */ ixgbevf_tx_timeout_reset(adapter); @@ -402,9 +398,6 @@ static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, return true; } - if (ring_is_xdp(tx_ring)) - return !!budget; - #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && (ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) { @@ -660,94 +653,85 @@ static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, #define IXGBEVF_XDP_CONSUMED 1 #define IXGBEVF_XDP_TX 2 -static int ixgbevf_xmit_xdp_ring(struct ixgbevf_ring *ring, - struct xdp_buff *xdp) +static void ixgbevf_clean_xdp_ring(struct ixgbevf_ring *xdp_ring) { - struct ixgbevf_tx_buffer *tx_buffer; - union ixgbe_adv_tx_desc *tx_desc; - u32 len, cmd_type; - dma_addr_t dma; - u16 i; + ixgbevf_clean_xdp_num(xdp_ring, false, xdp_ring->pending); +} - len = xdp->data_end - xdp->data; +static void ixgbevf_xdp_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i, + const struct libeth_xdpsq *sq, + u64 priv) +{ + union ixgbe_adv_tx_desc *tx_desc = + &((union ixgbe_adv_tx_desc *)sq->descs)[i]; - if (unlikely(!ixgbevf_desc_unused(ring))) - return IXGBEVF_XDP_CONSUMED; + u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA | + IXGBE_ADVTXD_DCMD_DEXT | + IXGBE_ADVTXD_DCMD_IFCS | + desc.len; - dma = dma_map_single(ring->dev, xdp->data, len, DMA_TO_DEVICE); - if (dma_mapping_error(ring->dev, dma)) - return IXGBEVF_XDP_CONSUMED; + if (desc.flags & LIBETH_XDP_TX_LAST) + cmd_type |= IXGBE_TXD_CMD_EOP; - /* record the location of the first descriptor for this packet */ - i = ring->next_to_use; - tx_buffer = &ring->tx_buffer_info[i]; - - dma_unmap_len_set(tx_buffer, len, len); - dma_unmap_addr_set(tx_buffer, dma, dma); - tx_buffer->data = xdp->data; - tx_buffer->bytecount = len; - tx_buffer->gso_segs = 1; - tx_buffer->protocol = 0; - - /* Populate minimal context descriptor that will provide for the - * fact that we are expected to process Ethernet frames. - */ - if (!test_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state)) { - struct ixgbe_adv_tx_context_desc *context_desc; + if (desc.flags & LIBETH_XDP_TX_FIRST) { + struct skb_shared_info *sinfo = sq->sqes[i].sinfo; + u16 full_len = desc.len; - set_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state); + if (desc.flags & LIBETH_XDP_TX_MULTI) + full_len += sinfo->xdp_frags_size; - context_desc = IXGBEVF_TX_CTXTDESC(ring, 0); - context_desc->vlan_macip_lens = - cpu_to_le32(ETH_HLEN << IXGBE_ADVTXD_MACLEN_SHIFT); - context_desc->fceof_saidx = 0; - context_desc->type_tucmd_mlhl = - cpu_to_le32(IXGBE_TXD_CMD_DEXT | - IXGBE_ADVTXD_DTYP_CTXT); - context_desc->mss_l4len_idx = 0; - - i = 1; + tx_desc->read.olinfo_status = + cpu_to_le32((full_len << IXGBE_ADVTXD_PAYLEN_SHIFT) | + IXGBE_ADVTXD_CC); } - /* put descriptor type bits */ - cmd_type = IXGBE_ADVTXD_DTYP_DATA | - IXGBE_ADVTXD_DCMD_DEXT | - IXGBE_ADVTXD_DCMD_IFCS; - cmd_type |= len | IXGBE_TXD_CMD; + tx_desc->read.buffer_addr = cpu_to_le64(desc.addr); + tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); +} - tx_desc = IXGBEVF_TX_DESC(ring, i); - tx_desc->read.buffer_addr = cpu_to_le64(dma); +LIBETH_XDP_DEFINE_START(); +LIBETH_XDP_DEFINE_FLUSH_TX(static ixgbevf_xdp_flush_tx, ixgbevf_prep_xdp_sq, + ixgbevf_xdp_xmit_desc); +LIBETH_XDP_DEFINE_END(); - tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); - tx_desc->read.olinfo_status = - cpu_to_le32((len << IXGBE_ADVTXD_PAYLEN_SHIFT) | - IXGBE_ADVTXD_CC); +static void ixgbevf_xdp_set_rs(struct ixgbevf_ring *xdp_ring, u32 cached_ntu) +{ + u32 ltu = (xdp_ring->next_to_use ? : xdp_ring->count) - 1; + union ixgbe_adv_tx_desc *desc; - /* Avoid any potential race with cleanup */ - smp_wmb(); + desc = IXGBEVF_TX_DESC(xdp_ring, ltu); + xdp_ring->xdp_sqes[cached_ntu].rs_idx = ltu + 1; + desc->read.cmd_type_len |= cpu_to_le32(IXGBE_TXD_CMD); +} - /* set next_to_watch value indicating a packet is present */ - i++; - if (i == ring->count) - i = 0; +static void ixgbevf_rx_finalize_xdp(struct libeth_xdp_tx_bulk *tx_bulk, + bool xdp_xmit, u32 cached_ntu) +{ + struct ixgbevf_ring *xdp_ring = tx_bulk->xdpsq; + + if (!xdp_xmit) + goto unlock; + + if (tx_bulk->count) + ixgbevf_xdp_flush_tx(tx_bulk, LIBETH_XDP_TX_DROP); - tx_buffer->next_to_watch = tx_desc; - ring->next_to_use = i; + ixgbevf_xdp_set_rs(xdp_ring, cached_ntu); - return IXGBEVF_XDP_TX; + /* Finish descriptor writes before bumping tail */ + wmb(); + ixgbevf_write_tail(xdp_ring, xdp_ring->next_to_use); +unlock: + rcu_read_unlock(); } -static int ixgbevf_run_xdp(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *rx_ring, +static int ixgbevf_run_xdp(struct libeth_xdp_tx_bulk *tx_bulk, struct libeth_xdp_buff *xdp) { int result = IXGBEVF_XDP_PASS; - struct ixgbevf_ring *xdp_ring; - struct bpf_prog *xdp_prog; + const struct bpf_prog *xdp_prog; u32 act; - xdp_prog = rcu_dereference(rx_ring->xdp_prog); - + xdp_prog = tx_bulk->prog; if (!xdp_prog) goto xdp_out; @@ -756,17 +740,16 @@ static int ixgbevf_run_xdp(struct ixgbevf_adapter *adapter, case XDP_PASS: break; case XDP_TX: - xdp_ring = adapter->xdp_ring[rx_ring->queue_index]; - result = ixgbevf_xmit_xdp_ring(xdp_ring, &xdp->base); - if (result == IXGBEVF_XDP_CONSUMED) - goto out_failure; + result = IXGBEVF_XDP_TX; + if (!libeth_xdp_tx_queue_bulk(tx_bulk, xdp, + ixgbevf_xdp_flush_tx)) + result = IXGBEVF_XDP_CONSUMED; break; default: - bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); + bpf_warn_invalid_xdp_action(tx_bulk->dev, xdp_prog, act); fallthrough; case XDP_ABORTED: -out_failure: - trace_xdp_exception(rx_ring->netdev, xdp_prog, act); + trace_xdp_exception(tx_bulk->dev, xdp_prog, act); fallthrough; /* handle aborts by dropping packet */ case XDP_DROP: result = IXGBEVF_XDP_CONSUMED; @@ -784,11 +767,19 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, unsigned int total_rx_bytes = 0, total_rx_packets = 0; struct ixgbevf_adapter *adapter = q_vector->adapter; u16 cleaned_count = ixgbevf_desc_unused(rx_ring); + LIBETH_XDP_ONSTACK_BULK(xdp_tx_bulk); LIBETH_XDP_ONSTACK_BUFF(xdp); + u32 cached_ntu; bool xdp_xmit = false; int xdp_res = 0; libeth_xdp_init_buff(xdp, &rx_ring->xdp_stash, &rx_ring->xdp_rxq); + libeth_xdp_tx_init_bulk(&xdp_tx_bulk, rx_ring->xdp_prog, + adapter->netdev, adapter->xdp_ring, + adapter->num_xdp_queues); + if (xdp_tx_bulk.prog) + cached_ntu = + ((struct ixgbevf_ring *)xdp_tx_bulk.xdpsq)->next_to_use; while (likely(total_rx_packets < budget)) { union ixgbe_adv_rx_desc *rx_desc; @@ -825,7 +816,7 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, total_rx_packets++; total_rx_bytes += xdp_get_buff_len(&xdp->base); - xdp_res = ixgbevf_run_xdp(adapter, rx_ring, xdp); + xdp_res = ixgbevf_run_xdp(&xdp_tx_bulk, xdp); if (xdp_res) { if (xdp_res == IXGBEVF_XDP_TX) xdp_xmit = true; @@ -871,16 +862,7 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, /* place incomplete frames back on ring for completion */ libeth_xdp_save_buff(&rx_ring->xdp_stash, xdp); - if (xdp_xmit) { - struct ixgbevf_ring *xdp_ring = - adapter->xdp_ring[rx_ring->queue_index]; - - /* Force memory writes to complete before letting h/w - * know there are new descriptors to fetch. - */ - wmb(); - ixgbevf_write_tail(xdp_ring, xdp_ring->next_to_use); - } + ixgbevf_rx_finalize_xdp(&xdp_tx_bulk, xdp_xmit, cached_ntu); u64_stats_update_begin(&rx_ring->syncp); rx_ring->stats.packets += total_rx_packets; @@ -910,6 +892,8 @@ static int ixgbevf_poll(struct napi_struct *napi, int budget) bool clean_complete = true; ixgbevf_for_each_ring(ring, q_vector->tx) { + if (ring_is_xdp(ring)) + continue; if (!ixgbevf_clean_tx_irq(q_vector, ring, budget)) clean_complete = false; } @@ -1349,6 +1333,7 @@ static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, /* reset ntu and ntc to place SW in sync with hardwdare */ ring->next_to_clean = 0; ring->next_to_use = 0; + ring->pending = 0; /* In order to avoid issues WTHRESH + PTHRESH should always be equal * to or less than the number of on chip descriptors, which is @@ -1361,8 +1346,12 @@ static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, 32; /* PTHRESH = 32 */ /* reinitialize tx_buffer_info */ - memset(ring->tx_buffer_info, 0, - sizeof(struct ixgbevf_tx_buffer) * ring->count); + if (!ring_is_xdp(ring)) + memset(ring->tx_buffer_info, 0, + sizeof(struct ixgbevf_tx_buffer) * ring->count); + else + memset(ring->xdp_sqes, 0, + sizeof(struct libeth_sqe) * ring->count); clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &ring->state); clear_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state); @@ -2017,10 +2006,7 @@ static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring) union ixgbe_adv_tx_desc *eop_desc, *tx_desc; /* Free all the Tx ring sk_buffs */ - if (ring_is_xdp(tx_ring)) - libeth_xdp_return_va(tx_buffer->data, false); - else - dev_kfree_skb_any(tx_buffer->skb); + dev_kfree_skb_any(tx_buffer->skb); /* unmap skb header data */ dma_unmap_single(tx_ring->dev, @@ -2089,7 +2075,7 @@ static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter) for (i = 0; i < adapter->num_tx_queues; i++) ixgbevf_clean_tx_ring(adapter->tx_ring[i]); for (i = 0; i < adapter->num_xdp_queues; i++) - ixgbevf_clean_tx_ring(adapter->xdp_ring[i]); + ixgbevf_clean_xdp_ring(adapter->xdp_ring[i]); } void ixgbevf_down(struct ixgbevf_adapter *adapter) @@ -2834,8 +2820,6 @@ static void ixgbevf_check_hang_subtask(struct ixgbevf_adapter *adapter) if (netif_carrier_ok(adapter->netdev)) { for (i = 0; i < adapter->num_tx_queues; i++) set_check_for_tx_hang(adapter->tx_ring[i]); - for (i = 0; i < adapter->num_xdp_queues; i++) - set_check_for_tx_hang(adapter->xdp_ring[i]); } /* get one bit for every active Tx/Rx interrupt vector */ @@ -2979,7 +2963,10 @@ static void ixgbevf_service_task(struct work_struct *work) **/ void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring) { - ixgbevf_clean_tx_ring(tx_ring); + if (!ring_is_xdp(tx_ring)) + ixgbevf_clean_tx_ring(tx_ring); + else + ixgbevf_clean_xdp_ring(tx_ring); vfree(tx_ring->tx_buffer_info); tx_ring->tx_buffer_info = NULL; @@ -2988,7 +2975,7 @@ void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring) if (!tx_ring->desc) return; - dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc, + dma_free_coherent(tx_ring->dev, tx_ring->dma_size, tx_ring->desc, tx_ring->dma); tx_ring->desc = NULL; @@ -3023,7 +3010,9 @@ int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring) struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev); int size; - size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; + size = (!ring_is_xdp(tx_ring) ? sizeof(struct ixgbevf_tx_buffer) : + sizeof(struct libeth_sqe)) * tx_ring->count; + tx_ring->tx_buffer_info = vmalloc(size); if (!tx_ring->tx_buffer_info) goto err; @@ -3031,10 +3020,10 @@ int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring) u64_stats_init(&tx_ring->syncp); /* round up to nearest 4K */ - tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); - tx_ring->size = ALIGN(tx_ring->size, 4096); + tx_ring->dma_size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); + tx_ring->dma_size = ALIGN(tx_ring->dma_size, 4096); - tx_ring->desc = dma_alloc_coherent(tx_ring->dev, tx_ring->size, + tx_ring->desc = dma_alloc_coherent(tx_ring->dev, tx_ring->dma_size, &tx_ring->dma, GFP_KERNEL); if (!tx_ring->desc) goto err; @@ -3122,10 +3111,10 @@ int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, u64_stats_init(&rx_ring->syncp); /* Round up to nearest 4K */ - rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); - rx_ring->size = ALIGN(rx_ring->size, 4096); + rx_ring->dma_size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); + rx_ring->dma_size = ALIGN(rx_ring->dma_size, 4096); - rx_ring->desc = dma_alloc_coherent(fq.pp->p.dev, rx_ring->size, + rx_ring->desc = dma_alloc_coherent(fq.pp->p.dev, rx_ring->dma_size, &rx_ring->dma, GFP_KERNEL); if (!rx_ring->desc) { @@ -3202,7 +3191,7 @@ void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring) xdp_rxq_info_detach_mem_model(&rx_ring->xdp_rxq); xdp_rxq_info_unreg(&rx_ring->xdp_rxq); - dma_free_coherent(fq.pp->p.dev, rx_ring->size, rx_ring->desc, + dma_free_coherent(fq.pp->p.dev, rx_ring->dma_size, rx_ring->desc, rx_ring->dma); rx_ring->desc = NULL; diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h new file mode 100644 index 00000000000000..629a5943eb3e27 --- /dev/null +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2010-2026 Intel Corporation */ + +#ifndef _IXGBEVF_XDP_LIB_H_ +#define _IXGBEVF_XDP_LIB_H_ + +#include + +#include "ixgbevf.h" + +static inline u16 ixgbevf_tx_get_num_sent(struct ixgbevf_ring *xdp_ring) +{ + u16 ntc = xdp_ring->next_to_clean; + u16 to_clean = 0; + + while (likely(to_clean < xdp_ring->pending)) { + u32 idx = xdp_ring->xdp_sqes[ntc].rs_idx; + union ixgbe_adv_tx_desc *rs_desc; + + if (!idx--) + break; + + rs_desc = IXGBEVF_TX_DESC(xdp_ring, idx); + + if (!(rs_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD))) + break; + + xdp_ring->xdp_sqes[ntc].rs_idx = 0; + + to_clean += + (idx >= ntc ? idx : idx + xdp_ring->count) - ntc + 1; + + ntc = (idx + 1 == xdp_ring->count) ? 0 : idx + 1; + } + + return to_clean; +} + +static inline void ixgbevf_clean_xdp_num(struct ixgbevf_ring *xdp_ring, + bool in_napi, u16 to_clean) +{ + struct libeth_xdpsq_napi_stats stats = { }; + u32 ntc = xdp_ring->next_to_clean; + struct xdp_frame_bulk cbulk; + struct libeth_cq_pp cp = { + .bq = &cbulk, + .dev = xdp_ring->dev, + .xss = &stats, + .napi = in_napi, + }; + + xdp_frame_bulk_init(&cbulk); + xdp_ring->pending -= to_clean; + + while (likely(to_clean--)) { + libeth_xdp_complete_tx(&xdp_ring->xdp_sqes[ntc], &cp); + ntc++; + ntc = unlikely(ntc == xdp_ring->count) ? 0 : ntc; + } + + xdp_ring->next_to_clean = ntc; + xdp_flush_frame_bulk(&cbulk); +} + +static inline u32 ixgbevf_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq) +{ + struct ixgbevf_ring *xdp_ring = xdpsq; + + if (unlikely(ixgbevf_desc_unused(xdp_ring) < LIBETH_XDP_TX_BULK)) { + u16 to_clean = ixgbevf_tx_get_num_sent(xdp_ring); + + if (likely(to_clean)) + ixgbevf_clean_xdp_num(xdp_ring, true, to_clean); + } + + if (unlikely(!test_bit(__IXGBEVF_TX_XDP_RING_PRIMED, + &xdp_ring->state))) { + struct ixgbe_adv_tx_context_desc *context_desc; + + set_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &xdp_ring->state); + + context_desc = IXGBEVF_TX_CTXTDESC(xdp_ring, 0); + context_desc->vlan_macip_lens = + cpu_to_le32(ETH_HLEN << IXGBE_ADVTXD_MACLEN_SHIFT); + context_desc->fceof_saidx = 0; + context_desc->type_tucmd_mlhl = + cpu_to_le32(IXGBE_TXD_CMD_DEXT | + IXGBE_ADVTXD_DTYP_CTXT); + context_desc->mss_l4len_idx = 0; + + xdp_ring->next_to_use = 1; + xdp_ring->pending = 1; + + /* Finish descriptor writes before bumping tail */ + wmb(); + ixgbevf_write_tail(xdp_ring, 1); + } + + *sq = (struct libeth_xdpsq) { + .count = xdp_ring->count, + .descs = xdp_ring->desc, + .lock = NULL, + .ntu = &xdp_ring->next_to_use, + .pending = &xdp_ring->pending, + .pool = NULL, + .sqes = xdp_ring->xdp_sqes, + }; + + return ixgbevf_desc_unused(xdp_ring); +} + +#endif /* _IXGBEVF_XDP_LIB_H_ */ From e6d7803b50b4a1049fa0169474ef135cb649b5d4 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Mon, 22 Sep 2025 07:14:24 +0200 Subject: [PATCH 05/39] ixgbevf: support XDP_REDIRECT and .ndo_xdp_xmit To fully support XDP_REDIRECT, utilize more libeth helpers in XDP Rx path, hence save cached_ntu in the ring structure instead of stack. ixgbevf-supported VFs usually have few queues, so use libeth_xdpsq_lock functionality for XDP queue sharing. Adjust filling-in of XDP Tx descriptors to use data from xdp frame. Otherwise, simply use libeth helpers to implement .ndo_xdp_xmit(). While at it, fix a typo in libeth docs. Reviewed-by: Aleksandr Loktionov Signed-off-by: Larysa Zaremba --- drivers/net/ethernet/intel/ixgbevf/ixgbevf.h | 2 + .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 135 ++++++------------ .../ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h | 30 +++- include/net/libeth/xdp.h | 2 +- 4 files changed, 78 insertions(+), 91 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h index a27081ee764b40..ea86679e4f81d0 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h @@ -98,6 +98,8 @@ struct ixgbevf_ring { struct ixgbevf_tx_buffer *tx_buffer_info; struct libeth_sqe *xdp_sqes; }; + struct libeth_xdpsq_lock xdpq_lock; + u32 cached_ntu; unsigned long state; struct ixgbevf_stats stats; struct u64_stats_sync syncp; diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index 350ed84147e668..d8c60546e4b2d5 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -649,13 +649,10 @@ static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask); } -#define IXGBEVF_XDP_PASS 0 -#define IXGBEVF_XDP_CONSUMED 1 -#define IXGBEVF_XDP_TX 2 - static void ixgbevf_clean_xdp_ring(struct ixgbevf_ring *xdp_ring) { ixgbevf_clean_xdp_num(xdp_ring, false, xdp_ring->pending); + libeth_xdpsq_put(&xdp_ring->xdpq_lock, xdp_ring->netdev); } static void ixgbevf_xdp_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i, @@ -674,11 +671,16 @@ static void ixgbevf_xdp_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i, cmd_type |= IXGBE_TXD_CMD_EOP; if (desc.flags & LIBETH_XDP_TX_FIRST) { - struct skb_shared_info *sinfo = sq->sqes[i].sinfo; + struct libeth_sqe *sqe = &sq->sqes[i]; + struct skb_shared_info *sinfo; u16 full_len = desc.len; - if (desc.flags & LIBETH_XDP_TX_MULTI) + if (desc.flags & LIBETH_XDP_TX_MULTI) { + sinfo = sqe->type == LIBETH_SQE_XDP_TX ? + sqe->sinfo : + xdp_get_shared_info_from_frame(sqe->xdpf); full_len += sinfo->xdp_frags_size; + } tx_desc->read.olinfo_status = cpu_to_le32((full_len << IXGBE_ADVTXD_PAYLEN_SHIFT) | @@ -692,74 +694,13 @@ static void ixgbevf_xdp_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i, LIBETH_XDP_DEFINE_START(); LIBETH_XDP_DEFINE_FLUSH_TX(static ixgbevf_xdp_flush_tx, ixgbevf_prep_xdp_sq, ixgbevf_xdp_xmit_desc); +LIBETH_XDP_DEFINE_FLUSH_XMIT(static ixgbevf_xdp_flush_xmit, ixgbevf_prep_xdp_sq, + ixgbevf_xdp_xmit_desc); +LIBETH_XDP_DEFINE_RUN_PROG(static ixgbevf_xdp_run_prog, ixgbevf_xdp_flush_tx); +LIBETH_XDP_DEFINE_FINALIZE(static ixgbevf_xdp_finalize_xdp_napi, + ixgbevf_xdp_flush_tx, ixgbevf_xdp_rs_and_bump); LIBETH_XDP_DEFINE_END(); -static void ixgbevf_xdp_set_rs(struct ixgbevf_ring *xdp_ring, u32 cached_ntu) -{ - u32 ltu = (xdp_ring->next_to_use ? : xdp_ring->count) - 1; - union ixgbe_adv_tx_desc *desc; - - desc = IXGBEVF_TX_DESC(xdp_ring, ltu); - xdp_ring->xdp_sqes[cached_ntu].rs_idx = ltu + 1; - desc->read.cmd_type_len |= cpu_to_le32(IXGBE_TXD_CMD); -} - -static void ixgbevf_rx_finalize_xdp(struct libeth_xdp_tx_bulk *tx_bulk, - bool xdp_xmit, u32 cached_ntu) -{ - struct ixgbevf_ring *xdp_ring = tx_bulk->xdpsq; - - if (!xdp_xmit) - goto unlock; - - if (tx_bulk->count) - ixgbevf_xdp_flush_tx(tx_bulk, LIBETH_XDP_TX_DROP); - - ixgbevf_xdp_set_rs(xdp_ring, cached_ntu); - - /* Finish descriptor writes before bumping tail */ - wmb(); - ixgbevf_write_tail(xdp_ring, xdp_ring->next_to_use); -unlock: - rcu_read_unlock(); -} - -static int ixgbevf_run_xdp(struct libeth_xdp_tx_bulk *tx_bulk, - struct libeth_xdp_buff *xdp) -{ - int result = IXGBEVF_XDP_PASS; - const struct bpf_prog *xdp_prog; - u32 act; - - xdp_prog = tx_bulk->prog; - if (!xdp_prog) - goto xdp_out; - - act = bpf_prog_run_xdp(xdp_prog, &xdp->base); - switch (act) { - case XDP_PASS: - break; - case XDP_TX: - result = IXGBEVF_XDP_TX; - if (!libeth_xdp_tx_queue_bulk(tx_bulk, xdp, - ixgbevf_xdp_flush_tx)) - result = IXGBEVF_XDP_CONSUMED; - break; - default: - bpf_warn_invalid_xdp_action(tx_bulk->dev, xdp_prog, act); - fallthrough; - case XDP_ABORTED: - trace_xdp_exception(tx_bulk->dev, xdp_prog, act); - fallthrough; /* handle aborts by dropping packet */ - case XDP_DROP: - result = IXGBEVF_XDP_CONSUMED; - libeth_xdp_return_buff(xdp); - break; - } -xdp_out: - return result; -} - static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, struct ixgbevf_ring *rx_ring, int budget) @@ -769,17 +710,11 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, u16 cleaned_count = ixgbevf_desc_unused(rx_ring); LIBETH_XDP_ONSTACK_BULK(xdp_tx_bulk); LIBETH_XDP_ONSTACK_BUFF(xdp); - u32 cached_ntu; - bool xdp_xmit = false; - int xdp_res = 0; libeth_xdp_init_buff(xdp, &rx_ring->xdp_stash, &rx_ring->xdp_rxq); libeth_xdp_tx_init_bulk(&xdp_tx_bulk, rx_ring->xdp_prog, adapter->netdev, adapter->xdp_ring, adapter->num_xdp_queues); - if (xdp_tx_bulk.prog) - cached_ntu = - ((struct ixgbevf_ring *)xdp_tx_bulk.xdpsq)->next_to_use; while (likely(total_rx_packets < budget)) { union ixgbe_adv_rx_desc *rx_desc; @@ -816,14 +751,9 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, total_rx_packets++; total_rx_bytes += xdp_get_buff_len(&xdp->base); - xdp_res = ixgbevf_run_xdp(&xdp_tx_bulk, xdp); - if (xdp_res) { - if (xdp_res == IXGBEVF_XDP_TX) - xdp_xmit = true; - - xdp->data = NULL; + if (xdp_tx_bulk.prog && + !ixgbevf_xdp_run_prog(xdp, &xdp_tx_bulk)) continue; - } skb = xdp_build_skb_from_buff(&xdp->base); @@ -862,7 +792,7 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, /* place incomplete frames back on ring for completion */ libeth_xdp_save_buff(&rx_ring->xdp_stash, xdp); - ixgbevf_rx_finalize_xdp(&xdp_tx_bulk, xdp_xmit, cached_ntu); + ixgbevf_xdp_finalize_xdp_napi(&xdp_tx_bulk); u64_stats_update_begin(&rx_ring->syncp); rx_ring->stats.packets += total_rx_packets; @@ -874,6 +804,23 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, return total_rx_packets; } +static int ixgbevf_xdp_xmit(struct net_device *dev, int n, + struct xdp_frame **frames, u32 flags) +{ + struct ixgbevf_adapter *adapter = netdev_priv(dev); + + if (unlikely(test_bit(__IXGBEVF_DOWN, &adapter->state))) + return -ENETDOWN; + + if (unlikely(!adapter->num_xdp_queues)) + return -ENXIO; + + return libeth_xdp_xmit_do_bulk(dev, n, frames, flags, adapter->xdp_ring, + adapter->num_xdp_queues, + ixgbevf_xdp_flush_xmit, + ixgbevf_xdp_rs_and_bump); +} + /** * ixgbevf_poll - NAPI polling calback * @napi: napi struct with our devices info in it @@ -1334,6 +1281,7 @@ static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, ring->next_to_clean = 0; ring->next_to_use = 0; ring->pending = 0; + ring->cached_ntu = 0; /* In order to avoid issues WTHRESH + PTHRESH should always be equal * to or less than the number of on chip descriptors, which is @@ -1346,12 +1294,15 @@ static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, 32; /* PTHRESH = 32 */ /* reinitialize tx_buffer_info */ - if (!ring_is_xdp(ring)) + if (!ring_is_xdp(ring)) { memset(ring->tx_buffer_info, 0, sizeof(struct ixgbevf_tx_buffer) * ring->count); - else + } else { memset(ring->xdp_sqes, 0, sizeof(struct libeth_sqe) * ring->count); + libeth_xdpsq_get(&ring->xdpq_lock, ring->netdev, + num_possible_cpus() > adapter->num_xdp_queues); + } clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &ring->state); clear_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state); @@ -4078,6 +4029,8 @@ static int ixgbevf_xdp_setup(struct net_device *dev, struct bpf_prog *prog, /* If transitioning XDP modes reconfigure rings */ if (!!prog != !!old_prog) { + xdp_features_clear_redirect_target(dev); + /* Hardware has to reinitialize queues and interrupts to * match packet buffer alignment. Unfortunately, the * hardware is not flexible enough to do this dynamically. @@ -4097,6 +4050,9 @@ static int ixgbevf_xdp_setup(struct net_device *dev, struct bpf_prog *prog, synchronize_net(); } + if (prog) + xdp_features_set_redirect_target(dev, true); + if (old_prog) bpf_prog_put(old_prog); @@ -4127,6 +4083,7 @@ static const struct net_device_ops ixgbevf_netdev_ops = { .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid, .ndo_features_check = ixgbevf_features_check, .ndo_bpf = ixgbevf_xdp, + .ndo_xdp_xmit = ixgbevf_xdp_xmit, }; static void ixgbevf_assign_netdev_ops(struct net_device *dev) @@ -4259,7 +4216,7 @@ static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) NETIF_F_HW_VLAN_CTAG_TX; netdev->priv_flags |= IFF_UNICAST_FLT; - netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_RX_SG; + libeth_xdp_set_features_noredir(netdev, NULL, 0, NULL); /* MTU range: 68 - 1504 or 9710 */ netdev->min_mtu = ETH_MIN_MTU; diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h index 629a5943eb3e27..da4f397944ed88 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h @@ -66,6 +66,7 @@ static inline u32 ixgbevf_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq) { struct ixgbevf_ring *xdp_ring = xdpsq; + libeth_xdpsq_lock(&xdp_ring->xdpq_lock); if (unlikely(ixgbevf_desc_unused(xdp_ring) < LIBETH_XDP_TX_BULK)) { u16 to_clean = ixgbevf_tx_get_num_sent(xdp_ring); @@ -99,7 +100,7 @@ static inline u32 ixgbevf_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq) *sq = (struct libeth_xdpsq) { .count = xdp_ring->count, .descs = xdp_ring->desc, - .lock = NULL, + .lock = &xdp_ring->xdpq_lock, .ntu = &xdp_ring->next_to_use, .pending = &xdp_ring->pending, .pool = NULL, @@ -109,4 +110,31 @@ static inline u32 ixgbevf_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq) return ixgbevf_desc_unused(xdp_ring); } +static inline void ixgbevf_xdp_rs_and_bump(void *xdpsq, bool sent, bool flush) +{ + struct ixgbevf_ring *xdp_ring = xdpsq; + union ixgbe_adv_tx_desc *desc; + u32 ltu; + + libeth_xdpsq_lock(&xdp_ring->xdpq_lock); + + if ((!flush && xdp_ring->pending < xdp_ring->count - 1) || + xdp_ring->cached_ntu == xdp_ring->next_to_use) + goto unlock; + + ltu = (xdp_ring->next_to_use ? : xdp_ring->count) - 1; + desc = IXGBEVF_TX_DESC(xdp_ring, ltu); + desc->read.cmd_type_len |= cpu_to_le32(IXGBE_TXD_CMD); + + xdp_ring->xdp_sqes[xdp_ring->cached_ntu].rs_idx = ltu + 1; + xdp_ring->cached_ntu = xdp_ring->next_to_use; + + /* Finish descriptor writes before bumping tail */ + wmb(); + ixgbevf_write_tail(xdp_ring, xdp_ring->next_to_use); + +unlock: + libeth_xdpsq_unlock(&xdp_ring->xdpq_lock); +} + #endif /* _IXGBEVF_XDP_LIB_H_ */ diff --git a/include/net/libeth/xdp.h b/include/net/libeth/xdp.h index 898723ab62e812..2e2154ccecae61 100644 --- a/include/net/libeth/xdp.h +++ b/include/net/libeth/xdp.h @@ -1094,7 +1094,7 @@ __libeth_xdp_xmit_do_bulk(struct libeth_xdp_tx_bulk *bq, * @xqs: array of XDPSQs driver structs * @nqs: number of active XDPSQs, the above array length * @fl: driver callback to flush an XDP xmit bulk - * @fin: driver cabback to finalize the queue + * @fin: driver callback to finalize the queue * * If the driver has active XDPSQs, perform common checks and send the frames. * Finalize the queue, if requested. From 8ef9f693aaa83d3a9b900ad47102bb5491e042bf Mon Sep 17 00:00:00 2001 From: Natalia Wochtman Date: Fri, 19 Sep 2025 15:30:15 +0200 Subject: [PATCH 06/39] ixgbevf: add pseudo header split Introduce pseudo header split support in the ixgbevf driver, specifically targeting ixgbe_mac_82599_vf. On older hardware (e.g. ixgbe_mac_82599_vf), RX DMA write size can only be limited in 1K increments. This causes issues when attempting to fit multiple packets per page, as a DMA write may overwrite the headroom of the next packet. To address this, introduce pseudo header split support, where the hardware copies the full L2 header into a dedicated header buffer. This avoids the need for HR/TR alignment and allows safe skb construction from the header buffer without risking overwrites. Given that once packet is too big to fit into a single page, the behaviour is the same for all supported HW, use pseudo header split only for smaller packets. Signed-off-by: Natalia Wochtman Reviewed-by: Aleksandr Loktionov Co-developed-by: Larysa Zaremba Signed-off-by: Larysa Zaremba --- drivers/net/ethernet/intel/ixgbevf/ixgbevf.h | 7 + .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 181 +++++++++++++++--- 2 files changed, 164 insertions(+), 24 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h index ea86679e4f81d0..17958cfb4ee65b 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h @@ -89,6 +89,7 @@ struct ixgbevf_ring { u32 truesize; /* Rx buffer full size */ u32 pending; /* Sent-not-completed descriptors */ }; + u32 hdr_truesize; /* Rx header buffer full size */ u16 count; /* amount of descriptors */ u16 next_to_clean; u32 next_to_use; @@ -107,6 +108,8 @@ struct ixgbevf_ring { struct ixgbevf_tx_queue_stats tx_stats; struct ixgbevf_rx_queue_stats rx_stats; }; + struct libeth_fqe *hdr_fqes; + struct page_pool *hdr_pp; struct xdp_rxq_info xdp_rxq; u64 hw_csum_rx_error; u8 __iomem *tail; @@ -151,6 +154,8 @@ struct ixgbevf_ring { #define IXGBEVF_RX_PAGE_LEN(hr) (ALIGN_DOWN(LIBETH_RX_PAGE_LEN(hr), \ IXGBE_SRRCTL_BSIZEPKT_STEP)) +#define IXGBEVF_RX_SRRCTL_BUF_SIZE(mtu) (ALIGN((mtu) + LIBETH_RX_LL_LEN, \ + IXGBE_SRRCTL_BSIZEPKT_STEP)) #define IXGBE_TX_FLAGS_CSUM BIT(0) #define IXGBE_TX_FLAGS_VLAN BIT(1) @@ -349,6 +354,8 @@ enum ixbgevf_state_t { __IXGBEVF_QUEUE_RESET_REQUESTED, }; +#define IXGBEVF_FLAG_HSPLIT BIT(0) + enum ixgbevf_boards { board_82599_vf, board_82599_vf_hv, diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index d8c60546e4b2d5..7cbd3d92728aff 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -561,6 +561,12 @@ static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, .truesize = rx_ring->truesize, .count = rx_ring->count, }; + const struct libeth_fq_fp hdr_fq = { + .pp = rx_ring->hdr_pp, + .fqes = rx_ring->hdr_fqes, + .truesize = rx_ring->hdr_truesize, + .count = rx_ring->count, + }; u16 ntu = rx_ring->next_to_use; /* nothing to do or no valid netdev defined */ @@ -578,6 +584,14 @@ static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, rx_desc->read.pkt_addr = cpu_to_le64(addr); + if (hdr_fq.pp) { + addr = libeth_rx_alloc(&hdr_fq, ntu); + if (addr == DMA_MAPPING_ERROR) { + libeth_rx_recycle_slow(fq.fqes[ntu].netmem); + break; + } + } + rx_desc++; ntu++; if (unlikely(ntu == fq.count)) { @@ -701,6 +715,32 @@ LIBETH_XDP_DEFINE_FINALIZE(static ixgbevf_xdp_finalize_xdp_napi, ixgbevf_xdp_flush_tx, ixgbevf_xdp_rs_and_bump); LIBETH_XDP_DEFINE_END(); +static u32 ixgbevf_rx_hsplit_wa(const struct libeth_fqe *hdr, + struct libeth_fqe *buf, u32 data_len) +{ + u32 copy = data_len <= L1_CACHE_BYTES ? data_len : ETH_HLEN; + struct page *hdr_page, *buf_page; + const void *src; + void *dst; + + if (unlikely(netmem_is_net_iov(buf->netmem)) || + !libeth_rx_sync_for_cpu(buf, copy)) + return 0; + + hdr_page = __netmem_to_page(hdr->netmem); + buf_page = __netmem_to_page(buf->netmem); + + dst = page_address(hdr_page) + hdr->offset + + pp_page_to_nmdesc(hdr_page)->pp->p.offset; + src = page_address(buf_page) + buf->offset + + pp_page_to_nmdesc(buf_page)->pp->p.offset; + + memcpy(dst, src, LARGEST_ALIGN(copy)); + buf->offset += copy; + + return copy; +} + static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, struct ixgbevf_ring *rx_ring, int budget) @@ -740,6 +780,23 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, rmb(); rx_buffer = &rx_ring->rx_fqes[rx_ring->next_to_clean]; + + if (unlikely(rx_ring->hdr_pp)) { + struct libeth_fqe *hdr_buff; + unsigned int hdr_size = 0; + + hdr_buff = &rx_ring->hdr_fqes[rx_ring->next_to_clean]; + + if (!xdp->data) { + hdr_size = ixgbevf_rx_hsplit_wa(hdr_buff, + rx_buffer, + size); + size -= hdr_size ? : size; + } + + libeth_xdp_process_buff(xdp, hdr_buff, hdr_size); + } + libeth_xdp_process_buff(xdp, rx_buffer, size); cleaned_count++; @@ -1477,6 +1534,87 @@ static void ixgbevf_setup_vfmrqc(struct ixgbevf_adapter *adapter) IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, vfmrqc); } +static void ixgbevf_rx_destroy_pp(struct ixgbevf_ring *rx_ring) +{ + struct libeth_fq fq = { + .pp = rx_ring->pp, + .fqes = rx_ring->rx_fqes, + }; + + libeth_rx_fq_destroy(&fq); + rx_ring->rx_fqes = NULL; + rx_ring->pp = NULL; + + if (!rx_ring->hdr_pp) + return; + + fq = (struct libeth_fq) { + .pp = rx_ring->hdr_pp, + .fqes = rx_ring->hdr_fqes, + }; + + libeth_rx_fq_destroy(&fq); + rx_ring->hdr_fqes = NULL; + rx_ring->hdr_pp = NULL; +} + +static int ixgbevf_rx_create_pp(struct ixgbevf_ring *rx_ring) +{ + u32 adapter_flags = rx_ring->q_vector->adapter->flags; + struct libeth_fq fq = { + .count = rx_ring->count, + .nid = NUMA_NO_NODE, + .type = LIBETH_FQE_MTU, + .xdp = !!rx_ring->xdp_prog, + .buf_len = IXGBEVF_RX_PAGE_LEN(rx_ring->xdp_prog ? + LIBETH_XDP_HEADROOM : + LIBETH_SKB_HEADROOM), + }; + u32 frame_size; + int ret; + + /* Some HW requires DMA write sizes to be aligned to 1K, + * which warrants fake header split usage, but this is + * not an issue if the frame size is at its maximum of 3K + */ + frame_size = + IXGBEVF_RX_SRRCTL_BUF_SIZE(READ_ONCE(rx_ring->netdev->mtu)); + fq.hsplit = (adapter_flags & IXGBEVF_FLAG_HSPLIT) && + frame_size < fq.buf_len; + ret = libeth_rx_fq_create(&fq, &rx_ring->q_vector->napi); + if (ret) + return ret; + + rx_ring->pp = fq.pp; + rx_ring->rx_fqes = fq.fqes; + rx_ring->truesize = fq.truesize; + rx_ring->rx_buf_len = fq.buf_len; + + if (!fq.hsplit) + return 0; + + fq = (struct libeth_fq) { + .count = rx_ring->count, + .nid = NUMA_NO_NODE, + .type = LIBETH_FQE_HDR, + .xdp = !!rx_ring->xdp_prog, + }; + + ret = libeth_rx_fq_create(&fq, &rx_ring->q_vector->napi); + if (ret) + goto err; + + rx_ring->hdr_pp = fq.pp; + rx_ring->hdr_fqes = fq.fqes; + rx_ring->hdr_truesize = fq.truesize; + + return 0; + +err: + ixgbevf_rx_destroy_pp(rx_ring); + return ret; +} + static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, struct ixgbevf_ring *ring) { @@ -1934,8 +2072,13 @@ static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) /* Free all the Rx ring pages */ for (u32 i = rx_ring->next_to_clean; i != rx_ring->next_to_use; ) { const struct libeth_fqe *rx_fqe = &rx_ring->rx_fqes[i]; + const struct libeth_fqe *hdr_fqe = rx_ring->hdr_fqes ? + &rx_ring->hdr_fqes[i] : + NULL; libeth_rx_recycle_slow(rx_fqe->netmem); + if (hdr_fqe) + libeth_rx_recycle_slow(hdr_fqe->netmem); if (unlikely(++i == rx_ring->count)) i = 0; } @@ -2596,6 +2739,9 @@ static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter) goto out; } + if (adapter->hw.mac.type == ixgbe_mac_82599_vf) + adapter->flags |= IXGBEVF_FLAG_HSPLIT; + /* assume legacy case in which PF would only give VF 2 queues */ hw->mac.max_tx_queues = 2; hw->mac.max_rx_queues = 2; @@ -3030,42 +3176,29 @@ static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter) } /** - * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors) + * ixgbevf_setup_rx_resources - allocate Rx resources * @adapter: board private structure * @rx_ring: Rx descriptor ring (for a specific queue) to setup * - * Returns 0 on success, negative on failure + * Returns: 0 on success, negative on failure. **/ int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, struct ixgbevf_ring *rx_ring) { - struct libeth_fq fq = { - .count = rx_ring->count, - .nid = NUMA_NO_NODE, - .type = LIBETH_FQE_MTU, - .xdp = !!rx_ring->xdp_prog, - .buf_len = IXGBEVF_RX_PAGE_LEN(rx_ring->xdp_prog ? - LIBETH_XDP_HEADROOM : - LIBETH_SKB_HEADROOM), - }; int ret; - ret = libeth_rx_fq_create(&fq, &rx_ring->q_vector->napi); + ret = ixgbevf_rx_create_pp(rx_ring); if (ret) return ret; - rx_ring->pp = fq.pp; - rx_ring->rx_fqes = fq.fqes; - rx_ring->truesize = fq.truesize; - rx_ring->rx_buf_len = fq.buf_len; - u64_stats_init(&rx_ring->syncp); /* Round up to nearest 4K */ rx_ring->dma_size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); rx_ring->dma_size = ALIGN(rx_ring->dma_size, 4096); - rx_ring->desc = dma_alloc_coherent(fq.pp->p.dev, rx_ring->dma_size, + rx_ring->desc = dma_alloc_coherent(rx_ring->pp->p.dev, + rx_ring->dma_size, &rx_ring->dma, GFP_KERNEL); if (!rx_ring->desc) { @@ -3081,15 +3214,14 @@ int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, if (ret) goto err; - xdp_rxq_info_attach_page_pool(&rx_ring->xdp_rxq, fq.pp); + xdp_rxq_info_attach_page_pool(&rx_ring->xdp_rxq, rx_ring->pp); rcu_assign_pointer(rx_ring->xdp_prog, adapter->xdp_prog); return 0; err: - libeth_rx_fq_destroy(&fq); - rx_ring->rx_fqes = NULL; - rx_ring->pp = NULL; + ixgbevf_rx_destroy_pp(rx_ring); + return ret; } @@ -4018,10 +4150,11 @@ static int ixgbevf_xdp_setup(struct net_device *dev, struct bpf_prog *prog, struct bpf_prog *old_prog; bool requires_mbuf; - requires_mbuf = frame_size > IXGBEVF_RX_PAGE_LEN(LIBETH_XDP_HEADROOM); + requires_mbuf = frame_size > IXGBEVF_RX_PAGE_LEN(LIBETH_XDP_HEADROOM) || + adapter->flags & IXGBEVF_FLAG_HSPLIT; if (prog && !prog->aux->xdp_has_frags && requires_mbuf) { NL_SET_ERR_MSG_MOD(extack, - "Configured MTU requires non-linear frames and XDP prog does not support frags"); + "Configured MTU or HW limitations require non-linear frames and XDP prog does not support frags"); return -EOPNOTSUPP; } From 37da1356f9c3dadb03702705a97983975ec748ae Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Tue, 25 Nov 2025 16:31:14 +0100 Subject: [PATCH 07/39] ixgbevf: reconfigure page pool when reallocating buffers Currently, when MTU is changed, page pool is not reconfigured, which leads to usage of suboptimal buffer sizes. Always destroy page pool when cleaning the ring up and create it anew when we first allocate Rx buffers. Reviewed-by: Aleksandr Loktionov Signed-off-by: Larysa Zaremba --- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 77 ++++++++++--------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index 7cbd3d92728aff..cc6f9ba23be9fa 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -569,8 +569,8 @@ static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, }; u16 ntu = rx_ring->next_to_use; - /* nothing to do or no valid netdev defined */ - if (unlikely(!cleaned_count || !rx_ring->netdev)) + /* nothing to do or page pool is not present */ + if (unlikely(!cleaned_count || !fq.pp)) return; rx_desc = IXGBEVF_RX_DESC(rx_ring, ntu); @@ -1541,6 +1541,14 @@ static void ixgbevf_rx_destroy_pp(struct ixgbevf_ring *rx_ring) .fqes = rx_ring->rx_fqes, }; + if (!fq.pp) + return; + + if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq)) { + xdp_rxq_info_detach_mem_model(&rx_ring->xdp_rxq); + xdp_rxq_info_unreg(&rx_ring->xdp_rxq); + } + libeth_rx_fq_destroy(&fq); rx_ring->rx_fqes = NULL; rx_ring->pp = NULL; @@ -1590,6 +1598,14 @@ static int ixgbevf_rx_create_pp(struct ixgbevf_ring *rx_ring) rx_ring->truesize = fq.truesize; rx_ring->rx_buf_len = fq.buf_len; + /* XDP RX-queue info */ + ret = __xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, + rx_ring->queue_index, 0, rx_ring->truesize); + if (ret) + goto err; + + xdp_rxq_info_attach_page_pool(&rx_ring->xdp_rxq, rx_ring->pp); + if (!fq.hsplit) return 0; @@ -1624,6 +1640,7 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, bool rlpml_valid = false; u64 rdba = ring->dma; u32 rxdctl; + int err; /* disable queue to avoid issues while updating state */ rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); @@ -1657,6 +1674,14 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, ring->next_to_clean = 0; ring->next_to_use = 0; + err = ixgbevf_rx_create_pp(ring); + if (err) { + netdev_err(ring->netdev, + "Failed to create Page Pool for buffer allocation: (%pe), RxQ %d is disabled, driver reload may be needed\n", + ERR_PTR(err), ring->queue_index); + return; + } + /* RXDCTL.RLPML does not work on 82599 */ if (adapter->hw.mac.type != ixgbe_mac_82599_vf) { u32 pkt_len = @@ -2154,8 +2179,10 @@ static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter) { int i; - for (i = 0; i < adapter->num_rx_queues; i++) + for (i = 0; i < adapter->num_rx_queues; i++) { ixgbevf_clean_rx_ring(adapter->rx_ring[i]); + ixgbevf_rx_destroy_pp(adapter->rx_ring[i]); + } } /** @@ -3175,6 +3202,11 @@ static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter) return err; } +static struct device *ixgbevf_dma_dev_from_ring(struct ixgbevf_ring *ring) +{ + return &ring->q_vector->adapter->pdev->dev; +} + /** * ixgbevf_setup_rx_resources - allocate Rx resources * @adapter: board private structure @@ -3185,44 +3217,25 @@ static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter) int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, struct ixgbevf_ring *rx_ring) { - int ret; - - ret = ixgbevf_rx_create_pp(rx_ring); - if (ret) - return ret; - u64_stats_init(&rx_ring->syncp); /* Round up to nearest 4K */ rx_ring->dma_size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); rx_ring->dma_size = ALIGN(rx_ring->dma_size, 4096); - rx_ring->desc = dma_alloc_coherent(rx_ring->pp->p.dev, + rx_ring->desc = dma_alloc_coherent(ixgbevf_dma_dev_from_ring(rx_ring), rx_ring->dma_size, &rx_ring->dma, GFP_KERNEL); if (!rx_ring->desc) { - ret = -ENOMEM; dev_err(&adapter->pdev->dev, "Unable to allocate memory for the Rx descriptor ring\n"); - goto err; + return -ENOMEM; } - /* XDP RX-queue info */ - ret = __xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, - rx_ring->queue_index, 0, rx_ring->truesize); - if (ret) - goto err; - - xdp_rxq_info_attach_page_pool(&rx_ring->xdp_rxq, rx_ring->pp); - rcu_assign_pointer(rx_ring->xdp_prog, adapter->xdp_prog); return 0; -err: - ixgbevf_rx_destroy_pp(rx_ring); - - return ret; } /** @@ -3263,24 +3276,14 @@ static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter) **/ void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring) { - struct libeth_fq fq = { - .fqes = rx_ring->rx_fqes, - .pp = rx_ring->pp, - }; - ixgbevf_clean_rx_ring(rx_ring); - + ixgbevf_rx_destroy_pp(rx_ring); rcu_assign_pointer(rx_ring->xdp_prog, NULL); - xdp_rxq_info_detach_mem_model(&rx_ring->xdp_rxq); - xdp_rxq_info_unreg(&rx_ring->xdp_rxq); - dma_free_coherent(fq.pp->p.dev, rx_ring->dma_size, rx_ring->desc, + dma_free_coherent(ixgbevf_dma_dev_from_ring(rx_ring), + rx_ring->dma_size, rx_ring->desc, rx_ring->dma); rx_ring->desc = NULL; - - libeth_rx_fq_destroy(&fq); - rx_ring->rx_fqes = NULL; - rx_ring->pp = NULL; } /** From 49cc75b6658ae95fcc1f026d38e6caa7e1aa4232 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Thu, 8 Jan 2026 13:44:03 +0100 Subject: [PATCH 08/39] ixgbevf: allow changing MTU when XDP program is attached xskxceiver attempts to change MTU after attaching XDP program, ixgbevf rejects the request leading to test being failed. Support MTU change operation even when XDP program is already attached, perform the same frame size check as when attaching a program. Reviewed-by: Aleksandr Loktionov Signed-off-by: Larysa Zaremba --- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index cc6f9ba23be9fa..7a6b73a57504d6 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -3969,6 +3969,18 @@ static int ixgbevf_set_mac(struct net_device *netdev, void *p) return 0; } +static bool ixgbevf_xdp_mtu_ok(const struct ixgbevf_adapter *adapter, + const struct bpf_prog *prog, unsigned int mtu) +{ + u32 frame_size = mtu + LIBETH_RX_LL_LEN; + bool requires_mbuf; + + requires_mbuf = frame_size > IXGBEVF_RX_PAGE_LEN(LIBETH_XDP_HEADROOM) || + adapter->flags & IXGBEVF_FLAG_HSPLIT; + + return prog->aux->xdp_has_frags || !requires_mbuf; +} + /** * ixgbevf_change_mtu - Change the Maximum Transfer Unit * @netdev: network interface device structure @@ -3984,8 +3996,10 @@ static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu) int ret; /* prevent MTU being changed to a size unsupported by XDP */ - if (adapter->xdp_prog) { - dev_warn(&adapter->pdev->dev, "MTU cannot be changed while XDP program is loaded\n"); + if (adapter->xdp_prog && + !ixgbevf_xdp_mtu_ok(adapter, adapter->xdp_prog, new_mtu)) { + netdev_warn(netdev, + "MTU value provided cannot be set while current XDP program is attached\n"); return -EPERM; } @@ -4148,14 +4162,10 @@ ixgbevf_features_check(struct sk_buff *skb, struct net_device *dev, static int ixgbevf_xdp_setup(struct net_device *dev, struct bpf_prog *prog, struct netlink_ext_ack *extack) { - u32 frame_size = READ_ONCE(dev->mtu) + LIBETH_RX_LL_LEN; struct ixgbevf_adapter *adapter = netdev_priv(dev); struct bpf_prog *old_prog; - bool requires_mbuf; - requires_mbuf = frame_size > IXGBEVF_RX_PAGE_LEN(LIBETH_XDP_HEADROOM) || - adapter->flags & IXGBEVF_FLAG_HSPLIT; - if (prog && !prog->aux->xdp_has_frags && requires_mbuf) { + if (prog && !ixgbevf_xdp_mtu_ok(adapter, prog, READ_ONCE(dev->mtu))) { NL_SET_ERR_MSG_MOD(extack, "Configured MTU or HW limitations require non-linear frames and XDP prog does not support frags"); return -EOPNOTSUPP; From dff91d22175189301a6aafbcffacbfe091c5d782 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Fri, 24 Oct 2025 12:19:51 +0200 Subject: [PATCH 09/39] ixgbevf: move skb-filling code to a header AF_XDP ZC Rx path is also required to implement skb creation. Move all common functions to a header file as inlines. Signed-off-by: Larysa Zaremba --- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 167 +----------------- .../ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h | 162 +++++++++++++++++ 2 files changed, 164 insertions(+), 165 deletions(-) create mode 100644 drivers/net/ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index 7a6b73a57504d6..28c18c41642241 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -33,6 +33,7 @@ #include #include +#include "ixgbevf_txrx_lib.h" #include "ixgbevf_xdp_lib.h" const char ixgbevf_driver_name[] = "ixgbevf"; @@ -418,134 +419,6 @@ static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, return !!budget; } -/** - * ixgbevf_rx_skb - Helper function to determine proper Rx method - * @q_vector: structure containing interrupt and ring information - * @skb: packet to send up - **/ -static void ixgbevf_rx_skb(struct ixgbevf_q_vector *q_vector, - struct sk_buff *skb) -{ - napi_gro_receive(&q_vector->napi, skb); -} - -#define IXGBE_RSS_L4_TYPES_MASK \ - ((1ul << IXGBE_RXDADV_RSSTYPE_IPV4_TCP) | \ - (1ul << IXGBE_RXDADV_RSSTYPE_IPV4_UDP) | \ - (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_TCP) | \ - (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_UDP)) - -static inline void ixgbevf_rx_hash(struct ixgbevf_ring *ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) -{ - u16 rss_type; - - if (!(ring->netdev->features & NETIF_F_RXHASH)) - return; - - rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) & - IXGBE_RXDADV_RSSTYPE_MASK; - - if (!rss_type) - return; - - skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), - (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ? - PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); -} - -/** - * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum - * @ring: structure containig ring specific data - * @rx_desc: current Rx descriptor being processed - * @skb: skb currently being received and modified - **/ -static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) -{ - skb_checksum_none_assert(skb); - - /* Rx csum disabled */ - if (!(ring->netdev->features & NETIF_F_RXCSUM)) - return; - - /* if IP and error */ - if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) && - ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { - ring->rx_stats.csum_err++; - return; - } - - if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS)) - return; - - if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) { - ring->rx_stats.csum_err++; - return; - } - - /* It must be a TCP or UDP packet with a valid checksum */ - skb->ip_summed = CHECKSUM_UNNECESSARY; -} - -/** - * ixgbevf_process_skb_fields - Populate skb header fields from Rx descriptor - * @rx_ring: rx descriptor ring packet is being transacted on - * @rx_desc: pointer to the EOP Rx descriptor - * @skb: pointer to current skb being populated - * - * This function checks the ring, descriptor, and packet information in - * order to populate the checksum, VLAN, protocol, and other fields within - * the skb. - **/ -static void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) -{ - ixgbevf_rx_hash(rx_ring, rx_desc, skb); - ixgbevf_rx_checksum(rx_ring, rx_desc, skb); - - if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { - u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan); - unsigned long *active_vlans = netdev_priv(rx_ring->netdev); - - if (test_bit(vid & VLAN_VID_MASK, active_vlans)) - __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); - } - - if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_STAT_SECP)) - ixgbevf_ipsec_rx(rx_ring, rx_desc, skb); -} - -/** - * ixgbevf_is_non_eop - process handling of non-EOP buffers - * @rx_ring: Rx ring being processed - * @rx_desc: Rx descriptor for current buffer - * - * This function updates next to clean. If the buffer is an EOP buffer - * this function exits returning false, otherwise it will place the - * sk_buff in the next buffer to be chained and return true indicating - * that this is in fact a non-EOP buffer. - **/ -static bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring, - union ixgbe_adv_rx_desc *rx_desc) -{ - u32 ntc = rx_ring->next_to_clean + 1; - - /* fetch, update, and store next to clean */ - ntc = (ntc < rx_ring->count) ? ntc : 0; - rx_ring->next_to_clean = ntc; - - prefetch(IXGBEVF_RX_DESC(rx_ring, ntc)); - - if (likely(ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) - return false; - - return true; -} - /** * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split * @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on @@ -619,42 +492,6 @@ static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, } } -/** - * ixgbevf_cleanup_headers - Correct corrupted or empty headers - * @rx_ring: rx descriptor ring packet is being transacted on - * @rx_desc: pointer to the EOP Rx descriptor - * @skb: pointer to current skb being fixed - * - * Check for corrupted packet headers caused by senders on the local L2 - * embedded NIC switch not setting up their Tx Descriptors right. These - * should be very rare. - * - * Also address the case where we are pulling data in on pages only - * and as such no data is present in the skb header. - * - * In addition if skb is not at least 60 bytes we need to pad it so that - * it is large enough to qualify as a valid Ethernet frame. - * - * Returns true if an error was encountered and skb was freed. - **/ -static bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) -{ - /* verify that the packet does not have any known errors */ - if (unlikely(ixgbevf_test_staterr(rx_desc, - IXGBE_RXDADV_ERR_FRAME_ERR_MASK))) { - struct net_device *netdev = rx_ring->netdev; - - if (!(netdev->features & NETIF_F_RXALL)) { - dev_kfree_skb_any(skb); - return true; - } - } - - return false; -} - static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, u32 qmask) { @@ -843,7 +680,7 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, /* populate checksum, VLAN, and protocol */ ixgbevf_process_skb_fields(rx_ring, rx_desc, skb); - ixgbevf_rx_skb(q_vector, skb); + napi_gro_receive(&q_vector->napi, skb); } /* place incomplete frames back on ring for completion */ diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h new file mode 100644 index 00000000000000..851d1c054d9810 --- /dev/null +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h @@ -0,0 +1,162 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2010-2026 Intel Corporation */ + +#ifndef _IXGBEVF_TXRX_LIB_H_ +#define _IXGBEVF_TXRX_LIB_H_ + +#include "ixgbevf.h" + +/** + * ixgbevf_is_non_eop - process handling of non-EOP buffers + * @rx_ring: Rx ring being processed + * @rx_desc: Rx descriptor for current buffer + * + * This function updates next to clean. If the buffer is an EOP buffer + * this function exits returning false, otherwise it will place the + * sk_buff in the next buffer to be chained and return true indicating + * that this is in fact a non-EOP buffer. + **/ +static inline bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring, + union ixgbe_adv_rx_desc *rx_desc) +{ + u32 ntc = rx_ring->next_to_clean + 1; + + /* fetch, update, and store next to clean */ + ntc = (ntc < rx_ring->count) ? ntc : 0; + rx_ring->next_to_clean = ntc; + + prefetch(IXGBEVF_RX_DESC(rx_ring, ntc)); + + if (likely(ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) + return false; + + return true; +} + +/** + * ixgbevf_cleanup_headers - Correct corrupted or empty headers + * @rx_ring: rx descriptor ring packet is being transacted on + * @rx_desc: pointer to the EOP Rx descriptor + * @skb: pointer to current skb being fixed + * + * Check for corrupted packet headers caused by senders on the local L2 + * embedded NIC switch not setting up their Tx Descriptors right. These + * should be very rare. + * + * Also address the case where we are pulling data in on pages only + * and as such no data is present in the skb header. + * + * In addition if skb is not at least 60 bytes we need to pad it so that + * it is large enough to qualify as a valid Ethernet frame. + * + * Returns true if an error was encountered and skb was freed. + **/ +static inline bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring, + union ixgbe_adv_rx_desc *rx_desc, + struct sk_buff *skb) +{ + /* verify that the packet does not have any known errors */ + if (unlikely(ixgbevf_test_staterr(rx_desc, + IXGBE_RXDADV_ERR_FRAME_ERR_MASK))) { + struct net_device *netdev = rx_ring->netdev; + + if (!(netdev->features & NETIF_F_RXALL)) { + dev_kfree_skb_any(skb); + return true; + } + } + + return false; +} + +#define IXGBE_RSS_L4_TYPES_MASK \ + ((1ul << IXGBE_RXDADV_RSSTYPE_IPV4_TCP) | \ + (1ul << IXGBE_RXDADV_RSSTYPE_IPV4_UDP) | \ + (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_TCP) | \ + (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_UDP)) + +static inline void ixgbevf_rx_hash(struct ixgbevf_ring *ring, + union ixgbe_adv_rx_desc *rx_desc, + struct sk_buff *skb) +{ + u16 rss_type; + + if (!(ring->netdev->features & NETIF_F_RXHASH)) + return; + + rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) & + IXGBE_RXDADV_RSSTYPE_MASK; + + if (!rss_type) + return; + + skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), + (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ? + PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); +} + +/** + * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum + * @ring: structure containing ring specific data + * @rx_desc: current Rx descriptor being processed + * @skb: skb currently being received and modified + **/ +static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring, + union ixgbe_adv_rx_desc *rx_desc, + struct sk_buff *skb) +{ + skb_checksum_none_assert(skb); + + /* Rx csum disabled */ + if (!(ring->netdev->features & NETIF_F_RXCSUM)) + return; + + /* if IP and error */ + if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) && + ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { + ring->rx_stats.csum_err++; + return; + } + + if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS)) + return; + + if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) { + ring->rx_stats.csum_err++; + return; + } + + /* It must be a TCP or UDP packet with a valid checksum */ + skb->ip_summed = CHECKSUM_UNNECESSARY; +} + +/** + * ixgbevf_process_skb_fields - Populate skb header fields from Rx descriptor + * @rx_ring: rx descriptor ring packet is being transacted on + * @rx_desc: pointer to the EOP Rx descriptor + * @skb: pointer to current skb being populated + * + * This function checks the ring, descriptor, and packet information in + * order to populate the checksum, VLAN, protocol, and other fields within + * the skb. + **/ +static inline void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring, + union ixgbe_adv_rx_desc *rx_desc, + struct sk_buff *skb) +{ + ixgbevf_rx_hash(rx_ring, rx_desc, skb); + ixgbevf_rx_checksum(rx_ring, rx_desc, skb); + + if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { + u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan); + unsigned long *active_vlans = netdev_priv(rx_ring->netdev); + + if (test_bit(vid & VLAN_VID_MASK, active_vlans)) + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); + } + + if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_STAT_SECP)) + ixgbevf_ipsec_rx(rx_ring, rx_desc, skb); +} + +#endif /* _IXGBEVF_TXRX_LIB_H_ */ From 2a6c73887e530f70a6c70b60636338a69c686c40 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Mon, 27 Oct 2025 13:32:49 +0100 Subject: [PATCH 10/39] ixgbevf: implement AF_XDP ZC initialization Implement xsk_buff_pool configuration and supporting functionality, such as a single queue pair reconfiguration. Also, properly initialize Rx buffers. Signed-off-by: Larysa Zaremba --- drivers/net/ethernet/intel/ixgbevf/Makefile | 2 +- drivers/net/ethernet/intel/ixgbevf/ixgbevf.h | 32 +++- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 155 ++++++++++++++---- .../net/ethernet/intel/ixgbevf/ixgbevf_xsk.c | 133 +++++++++++++++ .../net/ethernet/intel/ixgbevf/ixgbevf_xsk.h | 12 ++ 5 files changed, 299 insertions(+), 35 deletions(-) create mode 100644 drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c create mode 100644 drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h diff --git a/drivers/net/ethernet/intel/ixgbevf/Makefile b/drivers/net/ethernet/intel/ixgbevf/Makefile index 01d3e892f3fa7b..cdae62f25fd926 100644 --- a/drivers/net/ethernet/intel/ixgbevf/Makefile +++ b/drivers/net/ethernet/intel/ixgbevf/Makefile @@ -6,5 +6,5 @@ obj-$(CONFIG_IXGBEVF) += ixgbevf.o -ixgbevf-y := vf.o mbx.o ethtool.o ixgbevf_main.o +ixgbevf-y := vf.o mbx.o ethtool.o ixgbevf_main.o ixgbevf_xsk.o ixgbevf-$(CONFIG_IXGBEVF_IPSEC) += ipsec.o diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h index 17958cfb4ee65b..d8f841515ca62a 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h @@ -66,6 +66,7 @@ enum ixgbevf_ring_state_t { __IXGBEVF_HANG_CHECK_ARMED, __IXGBEVF_TX_XDP_RING, __IXGBEVF_TX_XDP_RING_PRIMED, + __IXGBEVF_RXTX_XSK_RING, }; #define ring_is_xdp(ring) \ @@ -75,6 +76,13 @@ enum ixgbevf_ring_state_t { #define clear_ring_xdp(ring) \ clear_bit(__IXGBEVF_TX_XDP_RING, &(ring)->state) +#define ring_is_xsk(ring) \ + test_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state) +#define set_ring_xsk(ring) \ + set_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state) +#define clear_ring_xsk(ring) \ + clear_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state) + struct ixgbevf_ring { struct ixgbevf_ring *next; struct ixgbevf_q_vector *q_vector; /* backpointer to q_vector */ @@ -85,22 +93,22 @@ struct ixgbevf_ring { struct device *dev; /* Tx ring */ }; void *desc; /* descriptor ring memory */ - union { - u32 truesize; /* Rx buffer full size */ - u32 pending; /* Sent-not-completed descriptors */ - }; + u32 truesize; /* Rx buffer full size */ u32 hdr_truesize; /* Rx header buffer full size */ u16 count; /* amount of descriptors */ u16 next_to_clean; u32 next_to_use; + u32 pending; /* Sent-not-completed descriptors */ union { struct libeth_fqe *rx_fqes; + struct libeth_xdp_buff **xsk_fqes; struct ixgbevf_tx_buffer *tx_buffer_info; struct libeth_sqe *xdp_sqes; }; struct libeth_xdpsq_lock xdpq_lock; u32 cached_ntu; + u32 thresh; unsigned long state; struct ixgbevf_stats stats; struct u64_stats_sync syncp; @@ -121,8 +129,10 @@ struct ixgbevf_ring { int queue_index; /* needed for multiqueue queue management */ u32 rx_buf_len; struct libeth_xdp_buff_stash xdp_stash; + struct libeth_xdp_buff *xsk_xdp_head; unsigned int dma_size; /* length in bytes */ dma_addr_t dma; /* phys. address of descriptor ring */ + struct xsk_buff_pool *xsk_pool; /* AF_XDP ZC rings */ } ____cacheline_internodealigned_in_smp; /* How many Rx Buffers do we bundle into one write to the hardware ? */ @@ -399,14 +409,28 @@ int ixgbevf_open(struct net_device *netdev); int ixgbevf_close(struct net_device *netdev); void ixgbevf_up(struct ixgbevf_adapter *adapter); void ixgbevf_down(struct ixgbevf_adapter *adapter); +void ixgbevf_flush_tx_queue(struct ixgbevf_ring *ring); +void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring); +void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring); void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter); void ixgbevf_reset(struct ixgbevf_adapter *adapter); void ixgbevf_set_ethtool_ops(struct net_device *netdev); int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, struct ixgbevf_ring *rx_ring); +void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter); +void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring); int ixgbevf_setup_tx_resources(struct ixgbevf_ring *); +void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring); void ixgbevf_free_rx_resources(struct ixgbevf_ring *); +void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring); +void ixgbevf_rx_destroy_pp(struct ixgbevf_ring *rx_ring); void ixgbevf_free_tx_resources(struct ixgbevf_ring *); +void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring); +void ixgbevf_clean_xdp_ring(struct ixgbevf_ring *xdp_ring); void ixgbevf_update_stats(struct ixgbevf_adapter *adapter); int ethtool_ioctl(struct ifreq *ifr); diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index 28c18c41642241..d10d850cec4029 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -30,11 +30,12 @@ #include #include #include -#include +#include #include #include "ixgbevf_txrx_lib.h" #include "ixgbevf_xdp_lib.h" +#include "ixgbevf_xsk.h" const char ixgbevf_driver_name[] = "ixgbevf"; static const char ixgbevf_driver_string[] = @@ -500,7 +501,7 @@ static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask); } -static void ixgbevf_clean_xdp_ring(struct ixgbevf_ring *xdp_ring) +void ixgbevf_clean_xdp_ring(struct ixgbevf_ring *xdp_ring) { ixgbevf_clean_xdp_num(xdp_ring, false, xdp_ring->pending); libeth_xdpsq_put(&xdp_ring->xdpq_lock, xdp_ring->netdev); @@ -1123,7 +1124,7 @@ static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter) * ixgbevf_irq_enable - Enable default interrupt generation settings * @adapter: board private structure **/ -static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter) +void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; @@ -1132,6 +1133,24 @@ static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter) IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask); } +/** + * ixgbevf_xsk_pool_from_q - get ZC XSK buffer pool bound to a queue ID + * @ring: Rx or Tx ring + * + * Return: A pointer to xsk_buff_pool structure if there is a buffer pool + * attached, configured as zero-copy, and usable by this queue, NULL otherwise. + */ +static struct xsk_buff_pool *ixgbevf_xsk_pool_from_q(struct ixgbevf_ring *ring) +{ + struct xsk_buff_pool *pool = + xsk_get_pool_from_qid(ring->netdev, ring->queue_index); + + if (!rcu_access_pointer(ring->xdp_prog) && !ring_is_xdp(ring)) + return NULL; + + return (pool && pool->dev) ? pool : NULL; +} + /** * ixgbevf_configure_tx_ring - Configure 82599 VF Tx ring after Reset * @adapter: board private structure @@ -1139,8 +1158,8 @@ static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter) * * Configure the Tx descriptor ring after a reset. **/ -static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *ring) +void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; u64 tdba = ring->dma; @@ -1198,6 +1217,12 @@ static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, num_possible_cpus() > adapter->num_xdp_queues); } + ring->xsk_pool = ixgbevf_xsk_pool_from_q(ring); + if (ring_is_xdp(ring) && ring->xsk_pool) + set_ring_xsk(ring); + else + clear_ring_xsk(ring); + clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &ring->state); clear_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state); @@ -1267,8 +1292,8 @@ static void ixgbevf_setup_psrtype(struct ixgbevf_adapter *adapter) } #define IXGBEVF_MAX_RX_DESC_POLL 10 -static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *ring) +void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; int wait_loop = IXGBEVF_MAX_RX_DESC_POLL; @@ -1292,10 +1317,15 @@ static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter, if (!wait_loop) pr_err("RXDCTL.ENABLE queue %d not cleared while polling\n", reg_idx); + + /* Specification calls for 100 usec of delay after + * RXDCTL.ENABLE is cleared + */ + usleep_range(100, 200); } -static void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *ring) +void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; int wait_loop = IXGBEVF_MAX_RX_DESC_POLL; @@ -1371,14 +1401,14 @@ static void ixgbevf_setup_vfmrqc(struct ixgbevf_adapter *adapter) IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, vfmrqc); } -static void ixgbevf_rx_destroy_pp(struct ixgbevf_ring *rx_ring) +void ixgbevf_rx_destroy_pp(struct ixgbevf_ring *rx_ring) { struct libeth_fq fq = { .pp = rx_ring->pp, .fqes = rx_ring->rx_fqes, }; - if (!fq.pp) + if (!fq.pp && !rx_ring->xsk_fqes) return; if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq)) { @@ -1386,6 +1416,21 @@ static void ixgbevf_rx_destroy_pp(struct ixgbevf_ring *rx_ring) xdp_rxq_info_unreg(&rx_ring->xdp_rxq); } + if (test_and_clear_bit(__IXGBEVF_RXTX_XSK_RING, &rx_ring->state)) { + struct libeth_xskfq xskfq = { + .fqes = rx_ring->xsk_fqes, + }; + + libeth_xskfq_destroy(&xskfq); + rx_ring->xsk_fqes = NULL; + rx_ring->pending = xskfq.pending; + rx_ring->thresh = xskfq.thresh; + rx_ring->rx_buf_len = xskfq.buf_len; + rx_ring->xsk_pool = NULL; + + return; + } + libeth_rx_fq_destroy(&fq); rx_ring->rx_fqes = NULL; rx_ring->pp = NULL; @@ -1415,9 +1460,44 @@ static int ixgbevf_rx_create_pp(struct ixgbevf_ring *rx_ring) LIBETH_XDP_HEADROOM : LIBETH_SKB_HEADROOM), }; + struct xsk_buff_pool *pool; u32 frame_size; int ret; + pool = ixgbevf_xsk_pool_from_q(rx_ring); + if (pool) { + u32 frag_sz = xsk_pool_get_rx_frag_step(pool); + struct libeth_xskfq xskfq = { + .nid = numa_node_id(), + .count = rx_ring->count, + .pool = pool, + }; + + ret = libeth_xskfq_create(&xskfq); + if (ret) + return ret; + + rx_ring->xsk_pool = xskfq.pool; + rx_ring->xsk_fqes = xskfq.fqes; + rx_ring->pending = xskfq.count - 1; + rx_ring->thresh = xskfq.thresh; + rx_ring->rx_buf_len = xskfq.buf_len; + set_ring_xsk(rx_ring); + + ret = __xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, + rx_ring->queue_index, 0, frag_sz); + if (ret) + goto err; + + ret = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq, + MEM_TYPE_XSK_BUFF_POOL, + rx_ring->xsk_pool); + if (ret) + goto err; + + return 0; + } + /* Some HW requires DMA write sizes to be aligned to 1K, * which warrants fake header split usage, but this is * not an issue if the frame size is at its maximum of 3K @@ -1468,8 +1548,8 @@ static int ixgbevf_rx_create_pp(struct ixgbevf_ring *rx_ring) return ret; } -static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *ring) +void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; union ixgbe_adv_rx_desc *rx_desc; @@ -1510,6 +1590,7 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, /* reset ntu and ntc to place SW in sync with hardwdare */ ring->next_to_clean = 0; ring->next_to_use = 0; + ring->pending = ixgbevf_desc_unused(ring); err = ixgbevf_rx_create_pp(ring); if (err) { @@ -1527,7 +1608,8 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | IXGBE_RXDCTL_RLPML_EN); if (pkt_len <= IXGBE_RXDCTL_RLPMLMASK) { rxdctl |= pkt_len | IXGBE_RXDCTL_RLPML_EN; - rlpml_valid = true; + if (pkt_len <= ring->rx_buf_len) + rlpml_valid = true; } } @@ -1537,7 +1619,11 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl); ixgbevf_rx_desc_queue_enable(adapter, ring); - ixgbevf_alloc_rx_buffers(ring, ixgbevf_desc_unused(ring)); + + if (ring_is_xsk(ring)) + ixgbevf_xsk_alloc_rx_bufs(ring, ring->pending); + else + ixgbevf_alloc_rx_buffers(ring, ring->pending); } /** @@ -1926,8 +2012,13 @@ void ixgbevf_up(struct ixgbevf_adapter *adapter) * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue * @rx_ring: ring to free buffers from **/ -static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) +void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) { + if (ring_is_xsk(rx_ring)) { + ixgbevf_rx_xsk_ring_free_buffs(rx_ring); + goto reset; + } + /* Free Rx ring sk_buff */ libeth_xdp_return_stash(&rx_ring->xdp_stash); @@ -1945,15 +2036,17 @@ static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) i = 0; } +reset: rx_ring->next_to_clean = 0; rx_ring->next_to_use = 0; + rx_ring->pending = 0; } /** * ixgbevf_clean_tx_ring - Free Tx Buffers * @tx_ring: ring to be cleaned **/ -static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring) +void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring) { u16 i = tx_ring->next_to_clean; struct ixgbevf_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; @@ -2036,10 +2129,17 @@ static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter) ixgbevf_clean_xdp_ring(adapter->xdp_ring[i]); } +void ixgbevf_flush_tx_queue(struct ixgbevf_ring *ring) +{ + u8 reg_idx = ring->reg_idx; + + IXGBE_WRITE_REG(&ring->q_vector->adapter->hw, IXGBE_VFTXDCTL(reg_idx), + IXGBE_TXDCTL_SWFLSH); +} + void ixgbevf_down(struct ixgbevf_adapter *adapter) { struct net_device *netdev = adapter->netdev; - struct ixgbe_hw *hw = &adapter->hw; int i; /* signal that we are down to the interrupt handler */ @@ -2065,19 +2165,11 @@ void ixgbevf_down(struct ixgbevf_adapter *adapter) timer_delete_sync(&adapter->service_timer); /* disable transmits in the hardware now that interrupts are off */ - for (i = 0; i < adapter->num_tx_queues; i++) { - u8 reg_idx = adapter->tx_ring[i]->reg_idx; - - IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), - IXGBE_TXDCTL_SWFLSH); - } - - for (i = 0; i < adapter->num_xdp_queues; i++) { - u8 reg_idx = adapter->xdp_ring[i]->reg_idx; + for (i = 0; i < adapter->num_tx_queues; i++) + ixgbevf_flush_tx_queue(adapter->tx_ring[i]); - IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), - IXGBE_TXDCTL_SWFLSH); - } + for (i = 0; i < adapter->num_xdp_queues; i++) + ixgbevf_flush_tx_queue(adapter->xdp_ring[i]); if (!pci_channel_offline(adapter->pdev)) ixgbevf_reset(adapter); @@ -4047,6 +4139,9 @@ static int ixgbevf_xdp(struct net_device *dev, struct netdev_bpf *xdp) switch (xdp->command) { case XDP_SETUP_PROG: return ixgbevf_xdp_setup(dev, xdp->prog, xdp->extack); + case XDP_SETUP_XSK_POOL: + return ixgbevf_setup_xsk_pool(netdev_priv(dev), xdp->xsk.pool, + xdp->xsk.queue_id); default: return -EINVAL; } diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c new file mode 100644 index 00000000000000..21b1cbd25bc9ed --- /dev/null +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2026 Intel Corporation */ + +#include + +#include "ixgbevf.h" +#include "ixgbevf_xsk.h" + +/** + * ixgbevf_single_irq_disable - Mask off interrupt generation on a single vector + * @adapter: board private structure + * @vidx: vector id + **/ +static void ixgbevf_single_irq_disable(struct ixgbevf_adapter *adapter, + u16 vidx) +{ + struct ixgbe_hw *hw = &adapter->hw; + + IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, + adapter->eims_enable_mask & ~BIT(vidx)); + IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, BIT(vidx)); + IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, + adapter->eims_enable_mask & ~BIT(vidx)); + + IXGBE_WRITE_FLUSH(hw); + + synchronize_irq(adapter->msix_entries[vidx].vector); +} + +static void ixgbevf_qp_dis(struct ixgbevf_adapter *adapter, u16 qid) +{ + struct ixgbevf_ring *tx_ring, *rx_ring = adapter->rx_ring[qid]; + struct ixgbevf_q_vector *q_vector = rx_ring->q_vector; + + netif_stop_subqueue(adapter->netdev, qid); + ixgbevf_single_irq_disable(adapter, q_vector->v_idx); + napi_disable(&q_vector->napi); + + synchronize_net(); + + ixgbevf_disable_rx_queue(adapter, adapter->rx_ring[qid]); + ixgbevf_clean_rx_ring(rx_ring); + ixgbevf_rx_destroy_pp(rx_ring); + + /* Clean both XDP and normal Tx queue */ + ixgbevf_for_each_ring(tx_ring, q_vector->tx) { + ixgbevf_flush_tx_queue(tx_ring); + if (ring_is_xdp(tx_ring)) + ixgbevf_clean_xdp_ring(tx_ring); + else + ixgbevf_clean_tx_ring(tx_ring); + } +} + +static void ixgbevf_qp_ena(struct ixgbevf_adapter *adapter, u16 qid) +{ + struct ixgbevf_ring *tx_ring, *rx_ring = adapter->rx_ring[qid]; + struct ixgbevf_q_vector *q_vector = rx_ring->q_vector; + + ixgbevf_configure_rx_ring(adapter, rx_ring); + ixgbevf_for_each_ring(tx_ring, q_vector->tx) + ixgbevf_configure_tx_ring(adapter, tx_ring); + + napi_enable(&q_vector->napi); + ixgbevf_irq_enable(adapter); + netif_start_subqueue(adapter->netdev, qid); +} + +int ixgbevf_setup_xsk_pool(struct ixgbevf_adapter *adapter, + struct xsk_buff_pool *pool, u16 qid) +{ + bool running = !test_bit(__IXGBEVF_DOWN, &adapter->state) && + adapter->xdp_prog; + int err; + + if (running) + ixgbevf_qp_dis(adapter, qid); + + err = libeth_xsk_setup_pool(adapter->netdev, qid, !!pool); + + if (running) + ixgbevf_qp_ena(adapter, qid); + + return err; +} + +static void ixgbevf_fill_rx_xsk_desc(const struct libeth_xskfq_fp *fq, u32 i) +{ + union ixgbe_adv_rx_desc *rx_desc = + &((union ixgbe_adv_rx_desc *)fq->descs)[i]; + + rx_desc->read.pkt_addr = + cpu_to_le64(libeth_xsk_buff_xdp_get_dma(fq->fqes[i])); + rx_desc->wb.upper.length = 0; +} + +void ixgbevf_xsk_alloc_rx_bufs(struct ixgbevf_ring *rx_ring, u32 num) +{ + struct libeth_xskfq_fp fq = { + .count = rx_ring->count, + .descs = rx_ring->desc, + .fqes = rx_ring->xsk_fqes, + .ntu = rx_ring->next_to_use, + .pool = rx_ring->xsk_pool, + }; + u32 done; + + done = libeth_xskfqe_alloc(&fq, num, ixgbevf_fill_rx_xsk_desc); + if (likely(done)) { + /* Finish descriptor writes before bumping tail */ + wmb(); + ixgbevf_write_tail(rx_ring, fq.ntu); + } + + rx_ring->next_to_use = fq.ntu; + rx_ring->pending -= done; +} + +void ixgbevf_rx_xsk_ring_free_buffs(struct ixgbevf_ring *rx_ring) +{ + u32 ntc = rx_ring->next_to_clean; + + if (rx_ring->xsk_xdp_head) + xsk_buff_free(&rx_ring->xsk_xdp_head->base); + + rx_ring->xsk_xdp_head = NULL; + + while (ntc != rx_ring->next_to_use) { + xsk_buff_free(&rx_ring->xsk_fqes[ntc]->base); + ntc++; + ntc = ntc == rx_ring->count ? 0 : ntc; + } +} diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h new file mode 100644 index 00000000000000..1cbcea803509b6 --- /dev/null +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2026 Intel Corporation */ + +#ifndef _IXGBEVF_XSK_H_ +#define _IXGBEVF_XSK_H_ + +int ixgbevf_setup_xsk_pool(struct ixgbevf_adapter *adapter, + struct xsk_buff_pool *pool, u16 qid); +void ixgbevf_xsk_alloc_rx_bufs(struct ixgbevf_ring *rx_ring, u32 num); +void ixgbevf_rx_xsk_ring_free_buffs(struct ixgbevf_ring *rx_ring); + +#endif /* _IXGBEVF_XSK_H_ */ From d822cedf23539a78d3defc08538b6b9396ea1755 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Mon, 27 Oct 2025 14:21:11 +0100 Subject: [PATCH 11/39] ixgbevf: implement AF_XDP zero-copy Tx Add code that handles Tx ZC queues inside of napi_poll(), utilize libeth. As NIC's multiple buffer conventions do not play nicely with AF_XDP's, leave handling of segments for later. Signed-off-by: Larysa Zaremba --- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 14 +++++--- .../ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h | 20 ++++++++--- .../net/ethernet/intel/ixgbevf/ixgbevf_xsk.c | 34 ++++++++++++++++++- .../net/ethernet/intel/ixgbevf/ixgbevf_xsk.h | 5 +++ 4 files changed, 64 insertions(+), 9 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index d10d850cec4029..930c275a37730c 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -734,10 +734,13 @@ static int ixgbevf_poll(struct napi_struct *napi, int budget) bool clean_complete = true; ixgbevf_for_each_ring(ring, q_vector->tx) { - if (ring_is_xdp(ring)) - continue; - if (!ixgbevf_clean_tx_irq(q_vector, ring, budget)) - clean_complete = false; + if (ring_is_xsk(ring)) + clean_complete &= + ixgbevf_clean_xsk_tx_irq(q_vector, ring, + budget); + else if (!ring_is_xdp(ring)) + clean_complete &= + ixgbevf_clean_tx_irq(q_vector, ring, budget); } if (budget <= 0) @@ -1223,6 +1226,9 @@ void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, else clear_ring_xsk(ring); + ring->thresh = ring_is_xsk(ring) ? IXGBEVF_XSK_TX_CLEAN_THRESH(ring) : + XDP_BULK_QUEUE_SIZE; + clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &ring->state); clear_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state); diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h index da4f397944ed88..08b0c427d1a3b5 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h @@ -4,7 +4,7 @@ #ifndef _IXGBEVF_XDP_LIB_H_ #define _IXGBEVF_XDP_LIB_H_ -#include +#include #include "ixgbevf.h" @@ -40,6 +40,7 @@ static inline void ixgbevf_clean_xdp_num(struct ixgbevf_ring *xdp_ring, bool in_napi, u16 to_clean) { struct libeth_xdpsq_napi_stats stats = { }; + bool xsk_ring = ring_is_xsk(xdp_ring); u32 ntc = xdp_ring->next_to_clean; struct xdp_frame_bulk cbulk; struct libeth_cq_pp cp = { @@ -48,11 +49,14 @@ static inline void ixgbevf_clean_xdp_num(struct ixgbevf_ring *xdp_ring, .xss = &stats, .napi = in_napi, }; + u32 xsk_frames = 0; xdp_frame_bulk_init(&cbulk); xdp_ring->pending -= to_clean; while (likely(to_clean--)) { + xsk_frames += xsk_ring && + likely(!xdp_ring->xdp_sqes[ntc].type) ? 1 : 0; libeth_xdp_complete_tx(&xdp_ring->xdp_sqes[ntc], &cp); ntc++; ntc = unlikely(ntc == xdp_ring->count) ? 0 : ntc; @@ -60,6 +64,8 @@ static inline void ixgbevf_clean_xdp_num(struct ixgbevf_ring *xdp_ring, xdp_ring->next_to_clean = ntc; xdp_flush_frame_bulk(&cbulk); + if (xsk_frames) + xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames); } static inline u32 ixgbevf_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq) @@ -67,8 +73,8 @@ static inline u32 ixgbevf_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq) struct ixgbevf_ring *xdp_ring = xdpsq; libeth_xdpsq_lock(&xdp_ring->xdpq_lock); - if (unlikely(ixgbevf_desc_unused(xdp_ring) < LIBETH_XDP_TX_BULK)) { - u16 to_clean = ixgbevf_tx_get_num_sent(xdp_ring); + if (unlikely(ixgbevf_desc_unused(xdp_ring) < xdp_ring->thresh)) { + u16 to_clean = ixgbevf_tx_get_num_sent(xdpsq); if (likely(to_clean)) ixgbevf_clean_xdp_num(xdp_ring, true, to_clean); @@ -91,6 +97,7 @@ static inline u32 ixgbevf_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq) xdp_ring->next_to_use = 1; xdp_ring->pending = 1; + xdp_ring->xdp_sqes[0].type = LIBETH_SQE_CTX; /* Finish descriptor writes before bumping tail */ wmb(); @@ -103,7 +110,7 @@ static inline u32 ixgbevf_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq) .lock = &xdp_ring->xdpq_lock, .ntu = &xdp_ring->next_to_use, .pending = &xdp_ring->pending, - .pool = NULL, + .pool = xdp_ring->xsk_pool, .sqes = xdp_ring->xdp_sqes, }; @@ -123,6 +130,11 @@ static inline void ixgbevf_xdp_rs_and_bump(void *xdpsq, bool sent, bool flush) goto unlock; ltu = (xdp_ring->next_to_use ? : xdp_ring->count) - 1; + + /* We will not get DD on a context descriptor */ + if (unlikely(xdp_ring->xdp_sqes[ltu].type == LIBETH_SQE_CTX)) + goto unlock; + desc = IXGBEVF_TX_DESC(xdp_ring, ltu); desc->read.cmd_type_len |= cpu_to_le32(IXGBE_TXD_CMD); diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c index 21b1cbd25bc9ed..3566857cebc723 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c @@ -3,7 +3,8 @@ #include -#include "ixgbevf.h" +#include "ixgbevf_txrx_lib.h" +#include "ixgbevf_xdp_lib.h" #include "ixgbevf_xsk.h" /** @@ -131,3 +132,34 @@ void ixgbevf_rx_xsk_ring_free_buffs(struct ixgbevf_ring *rx_ring) ntc = ntc == rx_ring->count ? 0 : ntc; } } + +static void ixgbevf_xsk_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i, + const struct libeth_xdpsq *sq, u64 priv) +{ + union ixgbe_adv_tx_desc *tx_desc = + &((union ixgbe_adv_tx_desc *)sq->descs)[i]; + + u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA | + IXGBE_ADVTXD_DCMD_DEXT | + IXGBE_ADVTXD_DCMD_IFCS | + IXGBE_TXD_CMD_EOP | + desc.len; + + tx_desc->read.olinfo_status = + cpu_to_le32((desc.len << IXGBE_ADVTXD_PAYLEN_SHIFT) | + IXGBE_ADVTXD_CC); + + tx_desc->read.buffer_addr = cpu_to_le64(desc.addr); + tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); +} + +bool ixgbevf_clean_xsk_tx_irq(struct ixgbevf_q_vector *q_vector, + struct ixgbevf_ring *tx_ring, int napi_budget) +{ + u32 budget = min_t(u32, napi_budget, tx_ring->thresh); + + return libeth_xsk_xmit_do_bulk(tx_ring->xsk_pool, tx_ring, budget, + NULL, ixgbevf_prep_xdp_sq, + ixgbevf_xsk_xmit_desc, + ixgbevf_xdp_rs_and_bump); +} diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h index 1cbcea803509b6..eda3e9b9554763 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h @@ -4,9 +4,14 @@ #ifndef _IXGBEVF_XSK_H_ #define _IXGBEVF_XSK_H_ +/* Process completions as soon as possible */ +#define IXGBEVF_XSK_TX_CLEAN_THRESH(r) ((r)->count - 1) + int ixgbevf_setup_xsk_pool(struct ixgbevf_adapter *adapter, struct xsk_buff_pool *pool, u16 qid); void ixgbevf_xsk_alloc_rx_bufs(struct ixgbevf_ring *rx_ring, u32 num); void ixgbevf_rx_xsk_ring_free_buffs(struct ixgbevf_ring *rx_ring); +bool ixgbevf_clean_xsk_tx_irq(struct ixgbevf_q_vector *q_vector, + struct ixgbevf_ring *tx_ring, int napi_budget); #endif /* _IXGBEVF_XSK_H_ */ From dbf79e180bf1bb8e69455d60b8f84dcdd4771206 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Mon, 27 Oct 2025 14:24:57 +0100 Subject: [PATCH 12/39] ixgbevf: implement AF_XDP zero-copy Rx Add code that handles AF_XDP ZC Rx queues inside of napi_poll(), utilize libeth helpers. Signed-off-by: Larysa Zaremba --- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 5 +- .../ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h | 1 + .../net/ethernet/intel/ixgbevf/ixgbevf_xsk.c | 121 +++++++++++++++++- .../net/ethernet/intel/ixgbevf/ixgbevf_xsk.h | 4 +- 4 files changed, 128 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index 930c275a37730c..2bd9c6e7d1dd7d 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -755,7 +755,10 @@ static int ixgbevf_poll(struct napi_struct *napi, int budget) per_ring_budget = budget; ixgbevf_for_each_ring(ring, q_vector->rx) { - int cleaned = ixgbevf_clean_rx_irq(q_vector, ring, + int cleaned = ring_is_xsk(ring) ? + ixgbevf_clean_xsk_rx_irq(q_vector, ring, + per_ring_budget) : + ixgbevf_clean_rx_irq(q_vector, ring, per_ring_budget); work_done += cleaned; if (cleaned >= per_ring_budget) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h index 851d1c054d9810..ac36fac9f79717 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h @@ -24,6 +24,7 @@ static inline bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring, /* fetch, update, and store next to clean */ ntc = (ntc < rx_ring->count) ? ntc : 0; rx_ring->next_to_clean = ntc; + rx_ring->pending++; prefetch(IXGBEVF_RX_DESC(rx_ring, ntc)); diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c index 3566857cebc723..30ab99dcf49176 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c @@ -65,6 +65,7 @@ static void ixgbevf_qp_ena(struct ixgbevf_adapter *adapter, u16 qid) napi_enable(&q_vector->napi); ixgbevf_irq_enable(adapter); netif_start_subqueue(adapter->netdev, qid); + napi_schedule(&q_vector->napi); } int ixgbevf_setup_xsk_pool(struct ixgbevf_adapter *adapter, @@ -95,7 +96,7 @@ static void ixgbevf_fill_rx_xsk_desc(const struct libeth_xskfq_fp *fq, u32 i) rx_desc->wb.upper.length = 0; } -void ixgbevf_xsk_alloc_rx_bufs(struct ixgbevf_ring *rx_ring, u32 num) +bool ixgbevf_xsk_alloc_rx_bufs(struct ixgbevf_ring *rx_ring, u32 num) { struct libeth_xskfq_fp fq = { .count = rx_ring->count, @@ -113,8 +114,13 @@ void ixgbevf_xsk_alloc_rx_bufs(struct ixgbevf_ring *rx_ring, u32 num) ixgbevf_write_tail(rx_ring, fq.ntu); } + /* Prevent ntc circling past ntu, e.g. in case of starvation */ + IXGBEVF_RX_DESC(rx_ring, fq.ntu)->wb.upper.length = 0; + rx_ring->next_to_use = fq.ntu; rx_ring->pending -= done; + + return done == num; } void ixgbevf_rx_xsk_ring_free_buffs(struct ixgbevf_ring *rx_ring) @@ -153,6 +159,119 @@ static void ixgbevf_xsk_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i, tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); } +LIBETH_XDP_DEFINE_START(); +LIBETH_XSK_DEFINE_FLUSH_TX(static ixgbevf_xsk_flush_tx, ixgbevf_prep_xdp_sq, + ixgbevf_xsk_xmit_desc); +LIBETH_XSK_DEFINE_RUN_PROG(static ixgbevf_xsk_run_prog, ixgbevf_xsk_flush_tx); +LIBETH_XSK_DEFINE_FINALIZE(static ixgbevf_xsk_finalize_xdp_napi, + ixgbevf_xsk_flush_tx, ixgbevf_xdp_rs_and_bump); +LIBETH_XDP_DEFINE_END(); + +u32 ixgbevf_clean_xsk_rx_irq(struct ixgbevf_q_vector *q_vector, + struct ixgbevf_ring *rx_ring, int budget) +{ + struct ixgbevf_adapter *adapter = q_vector->adapter; + u32 total_rx_bytes = 0, total_rx_packets = 0; + LIBETH_XDP_ONSTACK_BULK(xdp_tx_bulk); + struct libeth_xdp_buff *head_xdp; + bool failure = false, wake; + struct sk_buff *skb; + + wake = xsk_uses_need_wakeup(rx_ring->xsk_pool); + if (wake) + xsk_clear_rx_need_wakeup(rx_ring->xsk_pool); + + head_xdp = rx_ring->xsk_xdp_head; + libeth_xsk_tx_init_bulk(&xdp_tx_bulk, rx_ring->xdp_prog, + adapter->netdev, adapter->xdp_ring, + adapter->num_xdp_queues); + + while (likely(total_rx_packets < budget)) { + union ixgbe_adv_rx_desc *rx_desc; + struct libeth_xdp_buff *rx_buffer; + unsigned int size; + u32 xdp_result; + + rx_desc = IXGBEVF_RX_DESC(rx_ring, rx_ring->next_to_clean); + size = le16_to_cpu(rx_desc->wb.upper.length); + if (unlikely(!size)) + break; + + /* Avoid reading other descriptor fields before checking size */ + rmb(); + + rx_buffer = rx_ring->xsk_fqes[rx_ring->next_to_clean]; + head_xdp = libeth_xsk_process_buff(head_xdp, rx_buffer, size); + if (ixgbevf_is_non_eop(rx_ring, rx_desc) || unlikely(!head_xdp)) + continue; + + total_rx_packets++; + total_rx_bytes += xdp_get_buff_len(&head_xdp->base); + + xdp_result = ixgbevf_xsk_run_prog(head_xdp, &xdp_tx_bulk); + if (xdp_result) { + head_xdp = NULL; + if (likely(xdp_result != LIBETH_XDP_ABORTED)) + continue; + failure = true; + break; + } + + skb = xdp_build_skb_from_zc(&head_xdp->base); + + if (unlikely(!skb)) { + libeth_xdp_return_buff_slow(head_xdp); + head_xdp = NULL; + rx_ring->rx_stats.alloc_rx_buff_failed++; + break; + } + + head_xdp = NULL; + + if (unlikely(ixgbevf_cleanup_headers(rx_ring, rx_desc, skb))) { + skb = NULL; + continue; + } + + if (unlikely((skb->pkt_type == PACKET_BROADCAST || + skb->pkt_type == PACKET_MULTICAST) && + ether_addr_equal(rx_ring->netdev->dev_addr, + eth_hdr(skb)->h_source))) { + dev_kfree_skb_irq(skb); + continue; + } + + /* populate checksum, VLAN, and protocol */ + ixgbevf_process_skb_fields(rx_ring, rx_desc, skb); + + napi_gro_receive(&q_vector->napi, skb); + } + + if (rx_ring->pending >= rx_ring->thresh) + failure |= !ixgbevf_xsk_alloc_rx_bufs(rx_ring, + rx_ring->pending); + + /* place incomplete frames back on ring for completion */ + rx_ring->xsk_xdp_head = head_xdp; + + ixgbevf_xsk_finalize_xdp_napi(&xdp_tx_bulk); + + u64_stats_update_begin(&rx_ring->syncp); + rx_ring->stats.packets += total_rx_packets; + rx_ring->stats.bytes += total_rx_bytes; + u64_stats_update_end(&rx_ring->syncp); + q_vector->rx.total_packets += total_rx_packets; + q_vector->rx.total_bytes += total_rx_bytes; + + if (likely(!failure)) + return total_rx_packets; + + if (wake) + xsk_set_rx_need_wakeup(rx_ring->xsk_pool); + + return budget; +} + bool ixgbevf_clean_xsk_tx_irq(struct ixgbevf_q_vector *q_vector, struct ixgbevf_ring *tx_ring, int napi_budget) { diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h index eda3e9b9554763..042a90cfa9131c 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h @@ -9,8 +9,10 @@ int ixgbevf_setup_xsk_pool(struct ixgbevf_adapter *adapter, struct xsk_buff_pool *pool, u16 qid); -void ixgbevf_xsk_alloc_rx_bufs(struct ixgbevf_ring *rx_ring, u32 num); +bool ixgbevf_xsk_alloc_rx_bufs(struct ixgbevf_ring *rx_ring, u32 num); void ixgbevf_rx_xsk_ring_free_buffs(struct ixgbevf_ring *rx_ring); +u32 ixgbevf_clean_xsk_rx_irq(struct ixgbevf_q_vector *q_vector, + struct ixgbevf_ring *rx_ring, int budget); bool ixgbevf_clean_xsk_tx_irq(struct ixgbevf_q_vector *q_vector, struct ixgbevf_ring *tx_ring, int napi_budget); From 9ee2c99c997abb1cd71ca9dcd3c8f28826123467 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Mon, 27 Oct 2025 14:26:46 +0100 Subject: [PATCH 13/39] ixgbevf: implement .ndo_xsk_wakeup() and set features To finalize basic AF_XDP implementation, set features and add .ndo_xsk_wakeup() handler. Signed-off-by: Larysa Zaremba --- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 3 ++- .../net/ethernet/intel/ixgbevf/ixgbevf_xsk.c | 24 +++++++++++++++++++ .../net/ethernet/intel/ixgbevf/ixgbevf_xsk.h | 1 + 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index 2bd9c6e7d1dd7d..e9cbe523177966 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -4171,6 +4171,7 @@ static const struct net_device_ops ixgbevf_netdev_ops = { .ndo_features_check = ixgbevf_features_check, .ndo_bpf = ixgbevf_xdp, .ndo_xdp_xmit = ixgbevf_xdp_xmit, + .ndo_xsk_wakeup = ixgbevf_xsk_wakeup, }; static void ixgbevf_assign_netdev_ops(struct net_device *dev) @@ -4303,7 +4304,7 @@ static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) NETIF_F_HW_VLAN_CTAG_TX; netdev->priv_flags |= IFF_UNICAST_FLT; - libeth_xdp_set_features_noredir(netdev, NULL, 0, NULL); + libeth_xdp_set_features_noredir(netdev, NULL, 1, NULL); /* MTU range: 68 - 1504 or 9710 */ netdev->min_mtu = ETH_MIN_MTU; diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c index 30ab99dcf49176..6f76696a8eb9d0 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c @@ -282,3 +282,27 @@ bool ixgbevf_clean_xsk_tx_irq(struct ixgbevf_q_vector *q_vector, ixgbevf_xsk_xmit_desc, ixgbevf_xdp_rs_and_bump); } + +int ixgbevf_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) +{ + struct ixgbevf_adapter *adapter = netdev_priv(dev); + struct ixgbevf_q_vector *q_vector; + struct ixgbevf_ring *rx_ring; + + if (unlikely(test_bit(__IXGBEVF_DOWN, &adapter->state))) + return -ENETDOWN; + + if (unlikely(queue_id >= adapter->num_xdp_queues)) + return -EINVAL; + + rx_ring = adapter->rx_ring[queue_id]; + if (unlikely(!ring_is_xsk(rx_ring))) + return -EINVAL; + + q_vector = rx_ring->q_vector; + if (!napi_if_scheduled_mark_missed(&q_vector->napi)) + IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS, + BIT(q_vector->v_idx)); + + return 0; +} diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h index 042a90cfa9131c..7af14c78ead978 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h @@ -15,5 +15,6 @@ u32 ixgbevf_clean_xsk_rx_irq(struct ixgbevf_q_vector *q_vector, struct ixgbevf_ring *rx_ring, int budget); bool ixgbevf_clean_xsk_tx_irq(struct ixgbevf_q_vector *q_vector, struct ixgbevf_ring *tx_ring, int napi_budget); +int ixgbevf_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags); #endif /* _IXGBEVF_XSK_H_ */ From 0a7a647e06bd61ef3f8ba33dc3d90a2dc861fbc9 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Fri, 19 Dec 2025 14:48:57 +0100 Subject: [PATCH 14/39] ixgbevf: multi-buffer AF_XDP Tx Transmitting multi-buffer AF_XDP packets is not very straightforward given HW limitations in ixgbevf, namely that the first data descriptor must contain the length of the whole packet. Use private data of an sqe to store the length of an unfinished packet so far and the first descriptor index. Once EoP zero-copy descriptor is processed, write the accumulated length into the saved first descriptor. Signed-off-by: Larysa Zaremba --- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 2 +- .../ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h | 3 + .../net/ethernet/intel/ixgbevf/ixgbevf_xsk.c | 62 ++++++++++++++++--- .../net/ethernet/intel/ixgbevf/ixgbevf_xsk.h | 1 + 4 files changed, 59 insertions(+), 9 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index e9cbe523177966..759592618549fb 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -4304,7 +4304,7 @@ static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) NETIF_F_HW_VLAN_CTAG_TX; netdev->priv_flags |= IFF_UNICAST_FLT; - libeth_xdp_set_features_noredir(netdev, NULL, 1, NULL); + libeth_xdp_set_features_noredir(netdev, NULL, IXGBEVF_XSK_MAX_ZC_FRAGS); /* MTU range: 68 - 1504 or 9710 */ netdev->min_mtu = ETH_MIN_MTU; diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h index 08b0c427d1a3b5..ad1e09ae6aa1c3 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h @@ -141,6 +141,9 @@ static inline void ixgbevf_xdp_rs_and_bump(void *xdpsq, bool sent, bool flush) xdp_ring->xdp_sqes[xdp_ring->cached_ntu].rs_idx = ltu + 1; xdp_ring->cached_ntu = xdp_ring->next_to_use; + /* In case the packet was interrupted, discard it */ + xdp_ring->xdp_sqes[ltu].priv = 0; + /* Finish descriptor writes before bumping tail */ wmb(); ixgbevf_write_tail(xdp_ring, xdp_ring->next_to_use); diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c index 6f76696a8eb9d0..0ef43b62ed932e 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c @@ -139,24 +139,70 @@ void ixgbevf_rx_xsk_ring_free_buffs(struct ixgbevf_ring *rx_ring) } } +struct ixgbevf_zc_sqe_priv { + u16 first_desc; + u16 len; +}; + +LIBETH_SQE_CHECK_PRIV(struct ixgbevf_zc_sqe_priv); + static void ixgbevf_xsk_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i, const struct libeth_xdpsq *sq, u64 priv) { - union ixgbe_adv_tx_desc *tx_desc = - &((union ixgbe_adv_tx_desc *)sq->descs)[i]; + union ixgbe_adv_tx_desc *descs = sq->descs, *tx_desc = &descs[i]; + u32 ltu = (i ? : sq->count) - 1; u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA | IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DCMD_IFCS | - IXGBE_TXD_CMD_EOP | desc.len; - tx_desc->read.olinfo_status = - cpu_to_le32((desc.len << IXGBE_ADVTXD_PAYLEN_SHIFT) | - IXGBE_ADVTXD_CC); - tx_desc->read.buffer_addr = cpu_to_le64(desc.addr); - tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); + + if (likely((desc.flags & LIBETH_XDP_TX_LAST) && !sq->sqes[ltu].priv)) { + tx_desc->read.olinfo_status = + cpu_to_le32((desc.len << IXGBE_ADVTXD_PAYLEN_SHIFT) | + IXGBE_ADVTXD_CC); + tx_desc->read.cmd_type_len = + cpu_to_le32(cmd_type | IXGBE_TXD_CMD_EOP); + return; + } + + /* No previous packet */ + if (!sq->sqes[ltu].priv) { + struct ixgbevf_zc_sqe_priv *sqe_priv = + (void *)&sq->sqes[i].priv; + + sqe_priv->first_desc = i; + sqe_priv->len = desc.len; + + tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); + + return; + } + + if (sq->sqes[ltu].priv) { + struct ixgbevf_zc_sqe_priv *sqe_priv = + (void *)&sq->sqes[i].priv; + + sq->sqes[i].priv = sq->sqes[ltu].priv; + sq->sqes[ltu].priv = 0; + sqe_priv->len += desc.len; + + if (desc.flags & LIBETH_XDP_TX_LAST) { + union ixgbe_adv_tx_desc *first_desc = + &descs[sqe_priv->first_desc]; + + first_desc->read.olinfo_status = + cpu_to_le32((sqe_priv->len << + IXGBE_ADVTXD_PAYLEN_SHIFT) | + IXGBE_ADVTXD_CC); + cmd_type |= IXGBE_TXD_CMD_EOP; + sq->sqes[i].priv = 0; + } + + tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); + } } LIBETH_XDP_DEFINE_START(); diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h index 7af14c78ead978..2bb39735b10efd 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h @@ -6,6 +6,7 @@ /* Process completions as soon as possible */ #define IXGBEVF_XSK_TX_CLEAN_THRESH(r) ((r)->count - 1) +#define IXGBEVF_XSK_MAX_ZC_FRAGS min(18, MAX_SKB_FRAGS) int ixgbevf_setup_xsk_pool(struct ixgbevf_adapter *adapter, struct xsk_buff_pool *pool, u16 qid); From dae76dede9a838243f537150b5cc10fbce113f55 Mon Sep 17 00:00:00 2001 From: Magdalena Pytel Date: Mon, 13 Apr 2026 12:12:32 +0200 Subject: [PATCH 15/39] ixgbe: remove legacy rx Similarly, like for virtual function in ixgbevf, remove ixgbe_construct_skb(), the legacy-rx private flag, and all ring_uses_build_skb() conditionals. build_skb is now the only RX code path. This is a preparation for the conversion to libeth and page_pool based Rx buffer management. Signed-off-by: Magdalena Pytel --- drivers/net/ethernet/intel/ixgbe/ixgbe.h | 24 +-- .../net/ethernet/intel/ixgbe/ixgbe_ethtool.c | 13 +- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 176 ++---------------- 3 files changed, 27 insertions(+), 186 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index dce4936708eb44..0abc460b55765d 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -129,14 +129,7 @@ static inline int ixgbe_skb_pad(void) #define IXGBE_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN) #endif -/* - * NOTE: netdev_alloc_skb reserves up to 64 bytes, NET_IP_ALIGN means we - * reserve 64 more, and skb_shared_info adds an additional 320 bytes more, - * this adds up to 448 bytes of extra data. - * - * Since netdev_alloc_skb now allocates a page fragment we can use a value - * of 256 and the resultant skb will have a truesize of 960 or less. - */ +/* Default HW header buffer size */ #define IXGBE_RX_HDR_SIZE IXGBE_RXBUFFER_256 /* How many Rx Buffers do we bundle into one write to the hardware ? */ @@ -312,7 +305,6 @@ struct ixgbe_rx_queue_stats { enum ixgbe_ring_state_t { __IXGBE_RX_3K_BUFFER, - __IXGBE_RX_BUILD_SKB_ENABLED, __IXGBE_RX_RSC_ENABLED, __IXGBE_RX_CSUM_UDP_ZERO_ERR, __IXGBE_RX_FCOE, @@ -324,9 +316,6 @@ enum ixgbe_ring_state_t { __IXGBE_TX_DISABLED, }; -#define ring_uses_build_skb(ring) \ - test_bit(__IXGBE_RX_BUILD_SKB_ENABLED, &(ring)->state) - struct ixgbe_fwd_adapter { unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)]; struct net_device *netdev; @@ -397,7 +386,6 @@ struct ixgbe_ring { struct ixgbe_tx_queue_stats tx_stats; struct ixgbe_rx_queue_stats rx_stats; }; - u16 rx_offset; struct xdp_rxq_info xdp_rxq; spinlock_t tx_lock; /* used in XDP mode */ struct xsk_buff_pool *xsk_pool; @@ -456,8 +444,7 @@ static inline unsigned int ixgbe_rx_bufsz(struct ixgbe_ring *ring) if (test_bit(__IXGBE_RX_3K_BUFFER, &ring->state)) return IXGBE_RXBUFFER_3K; #if (PAGE_SIZE < 8192) - if (ring_uses_build_skb(ring)) - return IXGBE_MAX_2K_FRAME_BUILD_SKB; + return IXGBE_MAX_2K_FRAME_BUILD_SKB; #endif return IXGBE_RXBUFFER_2K; } @@ -472,6 +459,11 @@ static inline unsigned int ixgbe_rx_pg_order(struct ixgbe_ring *ring) } #define ixgbe_rx_pg_size(_ring) (PAGE_SIZE << ixgbe_rx_pg_order(_ring)) +static inline unsigned int ixgbe_rx_offset(struct ixgbe_ring *rx_ring) +{ + return IXGBE_SKB_PAD; +} + #define IXGBE_ITR_ADAPTIVE_MIN_INC 2 #define IXGBE_ITR_ADAPTIVE_MIN_USECS 10 #define IXGBE_ITR_ADAPTIVE_MAX_USECS 126 @@ -672,7 +664,7 @@ struct ixgbe_adapter { #define IXGBE_FLAG2_VLAN_PROMISC BIT(13) #define IXGBE_FLAG2_EEE_CAPABLE BIT(14) #define IXGBE_FLAG2_EEE_ENABLED BIT(15) -#define IXGBE_FLAG2_RX_LEGACY BIT(16) +/* BIT16 used to be reserved for legacy RX flag */ #define IXGBE_FLAG2_IPSEC_ENABLED BIT(17) #define IXGBE_FLAG2_VF_IPSEC_ENABLED BIT(18) #define IXGBE_FLAG2_AUTO_DISABLE_VF BIT(19) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c index 2ad81f687a844a..eaf50ce47fe5e1 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c @@ -132,11 +132,9 @@ static const char ixgbe_gstrings_test[][ETH_GSTRING_LEN] = { #define IXGBE_TEST_LEN sizeof(ixgbe_gstrings_test) / ETH_GSTRING_LEN static const char ixgbe_priv_flags_strings[][ETH_GSTRING_LEN] = { -#define IXGBE_PRIV_FLAGS_LEGACY_RX BIT(0) - "legacy-rx", -#define IXGBE_PRIV_FLAGS_VF_IPSEC_EN BIT(1) +#define IXGBE_PRIV_FLAGS_VF_IPSEC_EN BIT(0) "vf-ipsec", -#define IXGBE_PRIV_FLAGS_AUTO_DISABLE_VF BIT(2) +#define IXGBE_PRIV_FLAGS_AUTO_DISABLE_VF BIT(1) "mdd-disable-vf", }; @@ -3667,9 +3665,6 @@ static u32 ixgbe_get_priv_flags(struct net_device *netdev) struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); u32 priv_flags = 0; - if (adapter->flags2 & IXGBE_FLAG2_RX_LEGACY) - priv_flags |= IXGBE_PRIV_FLAGS_LEGACY_RX; - if (adapter->flags2 & IXGBE_FLAG2_VF_IPSEC_ENABLED) priv_flags |= IXGBE_PRIV_FLAGS_VF_IPSEC_EN; @@ -3685,10 +3680,6 @@ static int ixgbe_set_priv_flags(struct net_device *netdev, u32 priv_flags) unsigned int flags2 = adapter->flags2; unsigned int i; - flags2 &= ~IXGBE_FLAG2_RX_LEGACY; - if (priv_flags & IXGBE_PRIV_FLAGS_LEGACY_RX) - flags2 |= IXGBE_FLAG2_RX_LEGACY; - flags2 &= ~IXGBE_FLAG2_VF_IPSEC_ENABLED; if (priv_flags & IXGBE_PRIV_FLAGS_VF_IPSEC_EN) flags2 |= IXGBE_FLAG2_VF_IPSEC_ENABLED; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index c58051e4350be2..0533fbb92339eb 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -1730,11 +1730,6 @@ static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring, } } -static unsigned int ixgbe_rx_offset(struct ixgbe_ring *rx_ring) -{ - return ring_uses_build_skb(rx_ring) ? IXGBE_SKB_PAD : 0; -} - static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring, struct ixgbe_rx_buffer *bi) { @@ -1771,7 +1766,7 @@ static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring, bi->dma = dma; bi->page = page; - bi->page_offset = rx_ring->rx_offset; + bi->page_offset = ixgbe_rx_offset(rx_ring); page_ref_add(page, USHRT_MAX - 1); bi->pagecnt_bias = USHRT_MAX; rx_ring->rx_stats.alloc_rx_page++; @@ -1979,48 +1974,6 @@ static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring, return true; } -/** - * ixgbe_pull_tail - ixgbe specific version of skb_pull_tail - * @rx_ring: rx descriptor ring packet is being transacted on - * @skb: pointer to current skb being adjusted - * - * This function is an ixgbe specific version of __pskb_pull_tail. The - * main difference between this version and the original function is that - * this function can make several assumptions about the state of things - * that allow for significant optimizations versus the standard function. - * As a result we can do things like drop a frag and maintain an accurate - * truesize for the skb. - */ -static void ixgbe_pull_tail(struct ixgbe_ring *rx_ring, - struct sk_buff *skb) -{ - skb_frag_t *frag = &skb_shinfo(skb)->frags[0]; - unsigned char *va; - unsigned int pull_len; - - /* - * it is valid to use page_address instead of kmap since we are - * working with pages allocated out of the lomem pool per - * alloc_page(GFP_ATOMIC) - */ - va = skb_frag_address(frag); - - /* - * we need the header to contain the greater of either ETH_HLEN or - * 60 bytes if the skb->len is less than 60 for skb_pad. - */ - pull_len = eth_get_headlen(skb->dev, va, IXGBE_RX_HDR_SIZE); - - /* align pull length to size of long to optimize memcpy performance */ - skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long))); - - /* update all of the pointers */ - skb_frag_size_sub(frag, pull_len); - skb_frag_off_add(frag, pull_len); - skb->data_len -= pull_len; - skb->tail += pull_len; -} - /** * ixgbe_dma_sync_frag - perform DMA sync for first frag of SKB * @rx_ring: rx descriptor ring packet is being transacted on @@ -2034,24 +1987,12 @@ static void ixgbe_pull_tail(struct ixgbe_ring *rx_ring, static void ixgbe_dma_sync_frag(struct ixgbe_ring *rx_ring, struct sk_buff *skb) { - if (ring_uses_build_skb(rx_ring)) { - unsigned long mask = (unsigned long)ixgbe_rx_pg_size(rx_ring) - 1; - unsigned long offset = (unsigned long)(skb->data) & mask; - - dma_sync_single_range_for_cpu(rx_ring->dev, - IXGBE_CB(skb)->dma, - offset, - skb_headlen(skb), - DMA_FROM_DEVICE); - } else { - skb_frag_t *frag = &skb_shinfo(skb)->frags[0]; + unsigned long mask = (unsigned long)ixgbe_rx_pg_size(rx_ring) - 1; + unsigned long offset = (unsigned long)(skb->data) & mask; - dma_sync_single_range_for_cpu(rx_ring->dev, - IXGBE_CB(skb)->dma, - skb_frag_off(frag), - skb_frag_size(frag), - DMA_FROM_DEVICE); - } + dma_sync_single_range_for_cpu(rx_ring->dev, IXGBE_CB(skb)->dma, + offset, skb_headlen(skb), + DMA_FROM_DEVICE); /* If the page was released, just unmap it. */ if (unlikely(IXGBE_CB(skb)->page_released)) { @@ -2101,10 +2042,6 @@ bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring, return true; } - /* place header in linear portion of buffer */ - if (!skb_headlen(skb)) - ixgbe_pull_tail(rx_ring, skb); - #ifdef IXGBE_FCOE /* do not attempt to pad FCoE Frames as this will disrupt DDP */ if (ixgbe_rx_is_fcoe(rx_ring, rx_desc)) @@ -2192,13 +2129,9 @@ static bool ixgbe_can_reuse_rx_page(struct ixgbe_rx_buffer *rx_buffer, * @skb: sk_buff to place the data into * @size: size of data in rx_buffer * - * This function will add the data contained in rx_buffer->page to the skb. - * This is done either through a direct copy if the data in the buffer is - * less than the skb header size, otherwise it will just attach the page as - * a frag to the skb. - * - * The function will then update the page offset if necessary and return - * true if the buffer can be reused by the adapter. + * This function attaches the page contained in rx_buffer to the skb as + * a fragment. It then updates the page offset based on the configured + * PAGE_SIZE to prepare the buffer for the next packet. **/ static void ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring, struct ixgbe_rx_buffer *rx_buffer, @@ -2208,9 +2141,7 @@ static void ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring, #if (PAGE_SIZE < 8192) unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2; #else - unsigned int truesize = rx_ring->rx_offset ? - SKB_DATA_ALIGN(rx_ring->rx_offset + size) : - SKB_DATA_ALIGN(size); + unsigned int truesize = SKB_DATA_ALIGN(IXGBE_SKB_PAD + size); #endif skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, rx_buffer->page_offset, size, truesize); @@ -2291,65 +2222,6 @@ static void ixgbe_put_rx_buffer(struct ixgbe_ring *rx_ring, rx_buffer->skb = NULL; } -static struct sk_buff *ixgbe_construct_skb(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *rx_buffer, - struct xdp_buff *xdp, - union ixgbe_adv_rx_desc *rx_desc) -{ - unsigned int size = xdp->data_end - xdp->data; -#if (PAGE_SIZE < 8192) - unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2; -#else - unsigned int truesize = SKB_DATA_ALIGN(xdp->data_end - - xdp->data_hard_start); -#endif - struct sk_buff *skb; - - /* prefetch first cache line of first page */ - net_prefetch(xdp->data); - - /* Note, we get here by enabling legacy-rx via: - * - * ethtool --set-priv-flags legacy-rx on - * - * In this mode, we currently get 0 extra XDP headroom as - * opposed to having legacy-rx off, where we process XDP - * packets going to stack via ixgbe_build_skb(). The latter - * provides us currently with 192 bytes of headroom. - * - * For ixgbe_construct_skb() mode it means that the - * xdp->data_meta will always point to xdp->data, since - * the helper cannot expand the head. Should this ever - * change in future for legacy-rx mode on, then lets also - * add xdp->data_meta handling here. - */ - - /* allocate a skb to store the frags */ - skb = napi_alloc_skb(&rx_ring->q_vector->napi, IXGBE_RX_HDR_SIZE); - if (unlikely(!skb)) - return NULL; - - if (size > IXGBE_RX_HDR_SIZE) { - if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)) - IXGBE_CB(skb)->dma = rx_buffer->dma; - - skb_add_rx_frag(skb, 0, rx_buffer->page, - xdp->data - page_address(rx_buffer->page), - size, truesize); -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif - } else { - memcpy(__skb_put(skb, size), - xdp->data, ALIGN(size, sizeof(long))); - rx_buffer->pagecnt_bias++; - } - - return skb; -} - static struct sk_buff *ixgbe_build_skb(struct ixgbe_ring *rx_ring, struct ixgbe_rx_buffer *rx_buffer, struct xdp_buff *xdp, @@ -2460,10 +2332,8 @@ static unsigned int ixgbe_rx_frame_truesize(struct ixgbe_ring *rx_ring, #if (PAGE_SIZE < 8192) truesize = ixgbe_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */ #else - truesize = rx_ring->rx_offset ? - SKB_DATA_ALIGN(rx_ring->rx_offset + size) + - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) : - SKB_DATA_ALIGN(size); + truesize = SKB_DATA_ALIGN(IXGBE_SKB_PAD + size) + + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); #endif return truesize; } @@ -2504,7 +2374,6 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, unsigned int mss = 0; #endif /* IXGBE_FCOE */ u16 cleaned_count = ixgbe_desc_unused(rx_ring); - unsigned int offset = rx_ring->rx_offset; unsigned int xdp_xmit = 0; struct xdp_buff xdp; int xdp_res = 0; @@ -2543,6 +2412,7 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, /* retrieve a buffer from the ring */ if (!skb) { + unsigned int offset = ixgbe_rx_offset(rx_ring); unsigned char *hard_start; hard_start = page_address(rx_buffer->page) + @@ -2567,12 +2437,9 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, total_rx_bytes += size; } else if (skb) { ixgbe_add_rx_frag(rx_ring, rx_buffer, skb, size); - } else if (ring_uses_build_skb(rx_ring)) { + } else { skb = ixgbe_build_skb(rx_ring, rx_buffer, &xdp, rx_desc); - } else { - skb = ixgbe_construct_skb(rx_ring, rx_buffer, - &xdp, rx_desc); } /* exit if we failed to retrieve a buffer */ @@ -4557,15 +4424,12 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, * This can happen in SRIOV mode when the MTU of the VF is * higher than the MTU of the PF. */ - if (ring_uses_build_skb(ring) && - !test_bit(__IXGBE_RX_3K_BUFFER, &ring->state)) + if (!test_bit(__IXGBE_RX_3K_BUFFER, &ring->state)) rxdctl |= IXGBE_MAX_2K_FRAME_BUILD_SKB | IXGBE_RXDCTL_RLPML_EN; #endif } - ring->rx_offset = ixgbe_rx_offset(ring); - if (ring->xsk_pool && hw->mac.type != ixgbe_mac_82599EB) { u32 xsk_buf_len = xsk_pool_get_rx_frame_size(ring->xsk_pool); @@ -4734,7 +4598,6 @@ static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter) clear_ring_rsc_enabled(rx_ring); clear_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); - clear_bit(__IXGBE_RX_BUILD_SKB_ENABLED, &rx_ring->state); if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) set_ring_rsc_enabled(rx_ring); @@ -4742,11 +4605,6 @@ static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter) if (test_bit(__IXGBE_RX_FCOE, &rx_ring->state)) set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); - if (adapter->flags2 & IXGBE_FLAG2_RX_LEGACY) - continue; - - set_bit(__IXGBE_RX_BUILD_SKB_ENABLED, &rx_ring->state); - #if (PAGE_SIZE < 8192) if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); @@ -7199,7 +7057,7 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, rx_ring->queue_index, ixgbe_rx_napi_id(rx_ring)) < 0) goto err; - WRITE_ONCE(rx_ring->xdp_prog, adapter->xdp_prog); + rx_ring->xdp_prog = adapter->xdp_prog; return 0; err: @@ -7335,7 +7193,7 @@ static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter) */ static int ixgbe_max_xdp_frame_size(struct ixgbe_adapter *adapter) { - if (PAGE_SIZE >= 8192 || adapter->flags2 & IXGBE_FLAG2_RX_LEGACY) + if (PAGE_SIZE >= 8192) return IXGBE_RXBUFFER_2K; else return IXGBE_RXBUFFER_3K; From 42f9bf5f2cbb4861bdf243647de05523087a6b8b Mon Sep 17 00:00:00 2001 From: Magdalena Pytel Date: Thu, 7 May 2026 23:33:54 +0200 Subject: [PATCH 16/39] ixgbe: do not share pages between packets Stop sharing pages between multiple Rx buffers. Convert ixgbe Rx path from page-sharing/recycling to a simple one-page-per-packet model. Remove logic that refers to page-sharing buffer management. This is a preparatory step before converting the Rx path to page_pool and libeth. Fixed XDP_DROP/ABORTED: reaches ixgbe_put_rx_buffer() but page is never freed. Signed-off-by: Magdalena Pytel --- drivers/net/ethernet/intel/ixgbe/ixgbe.h | 91 +--- .../net/ethernet/intel/ixgbe/ixgbe_ethtool.c | 8 +- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 413 ++++-------------- 3 files changed, 86 insertions(+), 426 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index 0abc460b55765d..ef57e6029e492f 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -69,65 +69,13 @@ /* Supported Rx Buffer Sizes */ #define IXGBE_RXBUFFER_256 256 /* Used for skb receive header */ -#define IXGBE_RXBUFFER_1536 1536 -#define IXGBE_RXBUFFER_2K 2048 #define IXGBE_RXBUFFER_3K 3072 #define IXGBE_RXBUFFER_4K 4096 #define IXGBE_MAX_RXBUFFER 16384 /* largest size for a single descriptor */ #define IXGBE_PKT_HDR_PAD (ETH_HLEN + ETH_FCS_LEN + (VLAN_HLEN * 2)) -/* Attempt to maximize the headroom available for incoming frames. We - * use a 2K buffer for receives and need 1536/1534 to store the data for - * the frame. This leaves us with 512 bytes of room. From that we need - * to deduct the space needed for the shared info and the padding needed - * to IP align the frame. - * - * Note: For cache line sizes 256 or larger this value is going to end - * up negative. In these cases we should fall back to the 3K - * buffers. - */ -#if (PAGE_SIZE < 8192) -#define IXGBE_MAX_2K_FRAME_BUILD_SKB (IXGBE_RXBUFFER_1536 - NET_IP_ALIGN) -#define IXGBE_2K_TOO_SMALL_WITH_PADDING \ -((NET_SKB_PAD + IXGBE_RXBUFFER_1536) > SKB_WITH_OVERHEAD(IXGBE_RXBUFFER_2K)) - -static inline int ixgbe_compute_pad(int rx_buf_len) -{ - int page_size, pad_size; - - page_size = ALIGN(rx_buf_len, PAGE_SIZE / 2); - pad_size = SKB_WITH_OVERHEAD(page_size) - rx_buf_len; - - return pad_size; -} - -static inline int ixgbe_skb_pad(void) -{ - int rx_buf_len; - - /* If a 2K buffer cannot handle a standard Ethernet frame then - * optimize padding for a 3K buffer instead of a 1.5K buffer. - * - * For a 3K buffer we need to add enough padding to allow for - * tailroom due to NET_IP_ALIGN possibly shifting us out of - * cache-line alignment. - */ - if (IXGBE_2K_TOO_SMALL_WITH_PADDING) - rx_buf_len = IXGBE_RXBUFFER_3K + SKB_DATA_ALIGN(NET_IP_ALIGN); - else - rx_buf_len = IXGBE_RXBUFFER_1536; - - /* if needed make room for NET_IP_ALIGN */ - rx_buf_len -= NET_IP_ALIGN; - - return ixgbe_compute_pad(rx_buf_len); -} - -#define IXGBE_SKB_PAD ixgbe_skb_pad() -#else #define IXGBE_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN) -#endif /* Default HW header buffer size */ #define IXGBE_RX_HDR_SIZE IXGBE_RXBUFFER_256 @@ -267,11 +215,9 @@ struct ixgbe_tx_buffer { struct ixgbe_rx_buffer { union { struct { - struct sk_buff *skb; dma_addr_t dma; struct page *page; __u32 page_offset; - __u16 pagecnt_bias; }; struct { bool discard; @@ -304,7 +250,6 @@ struct ixgbe_rx_queue_stats { #define IXGBE_TS_HDR_LEN 8 enum ixgbe_ring_state_t { - __IXGBE_RX_3K_BUFFER, __IXGBE_RX_RSC_ENABLED, __IXGBE_RX_CSUM_UDP_ZERO_ERR, __IXGBE_RX_FCOE, @@ -371,12 +316,9 @@ struct ixgbe_ring { unsigned long last_rx_timestamp; - union { - u16 next_to_alloc; - struct { - u8 atr_sample_rate; - u8 atr_count; - }; + struct { + u8 atr_sample_rate; + u8 atr_count; }; u8 dcb_tc; @@ -386,6 +328,7 @@ struct ixgbe_ring { struct ixgbe_tx_queue_stats tx_stats; struct ixgbe_rx_queue_stats rx_stats; }; + struct sk_buff *skb; /* partial Rx skb across non-EOP */ struct xdp_rxq_info xdp_rxq; spinlock_t tx_lock; /* used in XDP mode */ struct xsk_buff_pool *xsk_pool; @@ -434,31 +377,6 @@ struct ixgbe_ring_feature { #define IXGBE_82599_VMDQ_4Q_MASK 0x7C #define IXGBE_82599_VMDQ_2Q_MASK 0x7E -/* - * FCoE requires that all Rx buffers be over 2200 bytes in length. Since - * this is twice the size of a half page we need to double the page order - * for FCoE enabled Rx queues. - */ -static inline unsigned int ixgbe_rx_bufsz(struct ixgbe_ring *ring) -{ - if (test_bit(__IXGBE_RX_3K_BUFFER, &ring->state)) - return IXGBE_RXBUFFER_3K; -#if (PAGE_SIZE < 8192) - return IXGBE_MAX_2K_FRAME_BUILD_SKB; -#endif - return IXGBE_RXBUFFER_2K; -} - -static inline unsigned int ixgbe_rx_pg_order(struct ixgbe_ring *ring) -{ -#if (PAGE_SIZE < 8192) - if (test_bit(__IXGBE_RX_3K_BUFFER, &ring->state)) - return 1; -#endif - return 0; -} -#define ixgbe_rx_pg_size(_ring) (PAGE_SIZE << ixgbe_rx_pg_order(_ring)) - static inline unsigned int ixgbe_rx_offset(struct ixgbe_ring *rx_ring) { return IXGBE_SKB_PAD; @@ -911,7 +829,6 @@ struct ixgbe_cb { }; dma_addr_t dma; u16 append_cnt; - bool page_released; }; #define IXGBE_CB(skb) ((struct ixgbe_cb *)(skb)->cb) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c index eaf50ce47fe5e1..b9db6ee9668e93 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c @@ -1961,6 +1961,10 @@ static int ixgbe_setup_desc_rings(struct ixgbe_adapter *adapter) hw->mac.ops.disable_rx(hw); + clear_ring_rsc_enabled(rx_ring); + if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) + set_ring_rsc_enabled(rx_ring); + ixgbe_configure_rx_ring(adapter, rx_ring); rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_RXCTRL); @@ -2119,7 +2123,7 @@ static u16 ixgbe_clean_test_rings(struct ixgbe_ring *rx_ring, /* sync Rx buffer for CPU read */ dma_sync_single_for_cpu(rx_ring->dev, rx_buffer->dma, - ixgbe_rx_bufsz(rx_ring), + IXGBE_RXBUFFER_3K, DMA_FROM_DEVICE); /* verify contents of skb */ @@ -2131,7 +2135,7 @@ static u16 ixgbe_clean_test_rings(struct ixgbe_ring *rx_ring, /* sync Rx buffer for device write */ dma_sync_single_for_device(rx_ring->dev, rx_buffer->dma, - ixgbe_rx_bufsz(rx_ring), + IXGBE_RXBUFFER_3K, DMA_FROM_DEVICE); /* increment Rx next to clean counter */ diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 0533fbb92339eb..73e46aea5c671d 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -592,7 +592,6 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) struct my_u0 { u64 a; u64 b; } *u0; struct ixgbe_ring *rx_ring; union ixgbe_adv_rx_desc *rx_desc; - struct ixgbe_rx_buffer *rx_buffer_info; int i = 0; if (!netif_msg_hw(adapter)) @@ -790,14 +789,12 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) pr_info("------------------------------------\n"); pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index); pr_info("------------------------------------\n"); - pr_info("%s%s%s\n", + pr_info("%s%s\n", "R [desc] [ PktBuf A0] ", - "[ HeadBuf DD] [bi->dma ] [bi->skb ] ", - "<-- Adv Rx Read format"); - pr_info("%s%s%s\n", + "[ HeadBuf DD] <-- Adv Rx Read format"); + pr_info("%s%s\n", "RWB[desc] [PcsmIpSHl PtRs] ", - "[vl er S cks ln] ---------------- [bi->skb ] ", - "<-- Adv Rx Write-Back format"); + "[vl er S cks ln] <-- Adv Rx Write-Back format"); for (i = 0; i < rx_ring->count; i++) { const char *ring_desc; @@ -809,34 +806,30 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) else ring_desc = ""; - rx_buffer_info = &rx_ring->rx_buffer_info[i]; rx_desc = IXGBE_RX_DESC(rx_ring, i); u0 = (struct my_u0 *)rx_desc; if (rx_desc->wb.upper.length) { - /* Descriptor Done */ - pr_info("RWB[0x%03X] %016llX %016llX ---------------- %p%s\n", + pr_info("RWB[0x%03X] %016llX %016llX%s\n", i, le64_to_cpu((__force __le64)u0->a), le64_to_cpu((__force __le64)u0->b), - rx_buffer_info->skb, ring_desc); } else { - pr_info("R [0x%03X] %016llX %016llX %016llX %p%s\n", + pr_info("R [0x%03X] %016llX %016llX %016llX %s\n", i, le64_to_cpu((__force __le64)u0->a), le64_to_cpu((__force __le64)u0->b), - (u64)rx_buffer_info->dma, - rx_buffer_info->skb, + (u64)rx_ring->rx_buffer_info[i].dma, ring_desc); + } - if (netif_msg_pktdata(adapter) && - rx_buffer_info->dma) { - print_hex_dump(KERN_INFO, "", - DUMP_PREFIX_ADDRESS, 16, 1, - page_address(rx_buffer_info->page) + - rx_buffer_info->page_offset, - ixgbe_rx_bufsz(rx_ring), true); - } + if (netif_msg_pktdata(adapter) && + rx_ring->rx_buffer_info[i].dma) { + print_hex_dump(KERN_INFO, "", + DUMP_PREFIX_ADDRESS, 16, 1, + page_address(rx_ring->rx_buffer_info[i].page) + + rx_ring->rx_buffer_info[i].page_offset, + IXGBE_RXBUFFER_3K, true); } } } @@ -1736,29 +1729,19 @@ static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring, struct page *page = bi->page; dma_addr_t dma; - /* since we are recycling buffers we should seldom need to alloc */ - if (likely(page)) - return true; - /* alloc new page for storage */ - page = dev_alloc_pages(ixgbe_rx_pg_order(rx_ring)); + page = dev_alloc_page(); if (unlikely(!page)) { rx_ring->rx_stats.alloc_rx_page_failed++; return false; } /* map page for use */ - dma = dma_map_page_attrs(rx_ring->dev, page, 0, - ixgbe_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, - IXGBE_RX_DMA_ATTR); + dma = dma_map_page_attrs(rx_ring->dev, page, 0, PAGE_SIZE, + DMA_FROM_DEVICE, IXGBE_RX_DMA_ATTR); - /* - * if mapping failed free memory back to system since - * there isn't much point in holding memory we can't use - */ if (dma_mapping_error(rx_ring->dev, dma)) { - __free_pages(page, ixgbe_rx_pg_order(rx_ring)); + __free_page(page); rx_ring->rx_stats.alloc_rx_page_failed++; return false; @@ -1766,9 +1749,8 @@ static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring, bi->dma = dma; bi->page = page; - bi->page_offset = ixgbe_rx_offset(rx_ring); - page_ref_add(page, USHRT_MAX - 1); - bi->pagecnt_bias = USHRT_MAX; + bi->page_offset = rx_ring->xdp_prog ? XDP_PACKET_HEADROOM : + IXGBE_SKB_PAD; rx_ring->rx_stats.alloc_rx_page++; return true; @@ -1784,7 +1766,6 @@ void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) union ixgbe_adv_rx_desc *rx_desc; struct ixgbe_rx_buffer *bi; u16 i = rx_ring->next_to_use; - u16 bufsz; /* nothing to do */ if (!cleaned_count) @@ -1794,15 +1775,14 @@ void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) bi = &rx_ring->rx_buffer_info[i]; i -= rx_ring->count; - bufsz = ixgbe_rx_bufsz(rx_ring); - do { if (!ixgbe_alloc_mapped_page(rx_ring, bi)) break; /* sync the buffer for use by the device */ dma_sync_single_range_for_device(rx_ring->dev, bi->dma, - bi->page_offset, bufsz, + bi->page_offset, + IXGBE_RXBUFFER_3K, DMA_FROM_DEVICE); /* @@ -1831,9 +1811,6 @@ void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) if (rx_ring->next_to_use != i) { rx_ring->next_to_use = i; - /* update next to alloc since we have filled the ring */ - rx_ring->next_to_alloc = i; - /* Force memory writes to complete before letting h/w * know there are new descriptors to fetch. (Only * applicable for weak-ordered memory model archs, @@ -1967,41 +1944,11 @@ static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring, if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) return false; - /* place skb in next buffer to be received */ - rx_ring->rx_buffer_info[ntc].skb = skb; rx_ring->rx_stats.non_eop_descs++; return true; } -/** - * ixgbe_dma_sync_frag - perform DMA sync for first frag of SKB - * @rx_ring: rx descriptor ring packet is being transacted on - * @skb: pointer to current skb being updated - * - * This function provides a basic DMA sync up for the first fragment of an - * skb. The reason for doing this is that the first fragment cannot be - * unmapped until we have reached the end of packet descriptor for a buffer - * chain. - */ -static void ixgbe_dma_sync_frag(struct ixgbe_ring *rx_ring, - struct sk_buff *skb) -{ - unsigned long mask = (unsigned long)ixgbe_rx_pg_size(rx_ring) - 1; - unsigned long offset = (unsigned long)(skb->data) & mask; - - dma_sync_single_range_for_cpu(rx_ring->dev, IXGBE_CB(skb)->dma, - offset, skb_headlen(skb), - DMA_FROM_DEVICE); - - /* If the page was released, just unmap it. */ - if (unlikely(IXGBE_CB(skb)->page_released)) { - dma_unmap_page_attrs(rx_ring->dev, IXGBE_CB(skb)->dma, - ixgbe_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, - IXGBE_RX_DMA_ATTR); - } -} /** * ixgbe_cleanup_headers - Correct corrupted or empty headers @@ -2048,80 +1995,10 @@ bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring, return false; #endif - /* if eth_skb_pad returns an error the skb was freed */ - if (eth_skb_pad(skb)) - return true; return false; } -/** - * ixgbe_reuse_rx_page - page flip buffer and store it back on the ring - * @rx_ring: rx descriptor ring to store buffers on - * @old_buff: donor buffer to have page reused - * - * Synchronizes page for reuse by the adapter - **/ -static void ixgbe_reuse_rx_page(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *old_buff) -{ - struct ixgbe_rx_buffer *new_buff; - u16 nta = rx_ring->next_to_alloc; - - new_buff = &rx_ring->rx_buffer_info[nta]; - - /* update, and store next to alloc */ - nta++; - rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; - - /* Transfer page from old buffer to new buffer. - * Move each member individually to avoid possible store - * forwarding stalls and unnecessary copy of skb. - */ - new_buff->dma = old_buff->dma; - new_buff->page = old_buff->page; - new_buff->page_offset = old_buff->page_offset; - new_buff->pagecnt_bias = old_buff->pagecnt_bias; -} - -static bool ixgbe_can_reuse_rx_page(struct ixgbe_rx_buffer *rx_buffer, - int rx_buffer_pgcnt) -{ - unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; - struct page *page = rx_buffer->page; - - /* avoid re-using remote and pfmemalloc pages */ - if (!dev_page_is_reusable(page)) - return false; - -#if (PAGE_SIZE < 8192) - /* if we are only owner of page we can reuse it */ - if (unlikely((rx_buffer_pgcnt - pagecnt_bias) > 1)) - return false; -#else - /* The last offset is a bit aggressive in that we assume the - * worst case of FCoE being enabled and using a 3K buffer. - * However this should have minimal impact as the 1K extra is - * still less than one buffer in size. - */ -#define IXGBE_LAST_OFFSET \ - (SKB_WITH_OVERHEAD(PAGE_SIZE) - IXGBE_RXBUFFER_3K) - if (rx_buffer->page_offset > IXGBE_LAST_OFFSET) - return false; -#endif - - /* If we have drained the page fragment pool we need to update - * the pagecnt_bias and page count so that we fully restock the - * number of references the driver holds. - */ - if (unlikely(pagecnt_bias == 1)) { - page_ref_add(page, USHRT_MAX - 1); - rx_buffer->pagecnt_bias = USHRT_MAX; - } - - return true; -} - /** * ixgbe_add_rx_frag - Add contents of Rx buffer to sk_buff * @rx_ring: rx descriptor ring to transact packets on @@ -2129,97 +2006,42 @@ static bool ixgbe_can_reuse_rx_page(struct ixgbe_rx_buffer *rx_buffer, * @skb: sk_buff to place the data into * @size: size of data in rx_buffer * - * This function attaches the page contained in rx_buffer to the skb as - * a fragment. It then updates the page offset based on the configured - * PAGE_SIZE to prepare the buffer for the next packet. + * This function will add the data contained in rx_buffer->page to the skb. **/ static void ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring, struct ixgbe_rx_buffer *rx_buffer, struct sk_buff *skb, unsigned int size) { -#if (PAGE_SIZE < 8192) - unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2; -#else unsigned int truesize = SKB_DATA_ALIGN(IXGBE_SKB_PAD + size); -#endif + skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, rx_buffer->page_offset, size, truesize); -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif } static struct ixgbe_rx_buffer *ixgbe_get_rx_buffer(struct ixgbe_ring *rx_ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff **skb, - const unsigned int size, - int *rx_buffer_pgcnt) + const unsigned int size) { struct ixgbe_rx_buffer *rx_buffer; rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; - *rx_buffer_pgcnt = -#if (PAGE_SIZE < 8192) - page_count(rx_buffer->page); -#else - 0; -#endif prefetchw(rx_buffer->page); - *skb = rx_buffer->skb; - - /* Delay unmapping of the first packet. It carries the header - * information, HW may still access the header after the writeback. - * Only unmap it when EOP is reached - */ - if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)) { - if (!*skb) - goto skip_sync; - } else { - if (*skb) - ixgbe_dma_sync_frag(rx_ring, *skb); - } - /* we are reusing so sync this buffer for CPU use */ + /* sync this buffer for CPU use */ dma_sync_single_range_for_cpu(rx_ring->dev, rx_buffer->dma, rx_buffer->page_offset, size, DMA_FROM_DEVICE); -skip_sync: - rx_buffer->pagecnt_bias--; return rx_buffer; } static void ixgbe_put_rx_buffer(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *rx_buffer, - struct sk_buff *skb, - int rx_buffer_pgcnt) + struct ixgbe_rx_buffer *rx_buffer) { - if (ixgbe_can_reuse_rx_page(rx_buffer, rx_buffer_pgcnt)) { - /* hand second half of page back to the ring */ - ixgbe_reuse_rx_page(rx_ring, rx_buffer); - } else { - if (skb && IXGBE_CB(skb)->dma == rx_buffer->dma) { - /* the page has been released from the ring */ - IXGBE_CB(skb)->page_released = true; - } else { - /* we are not reusing the buffer so unmap it */ - dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, - ixgbe_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, - IXGBE_RX_DMA_ATTR); - } - __page_frag_cache_drain(rx_buffer->page, - rx_buffer->pagecnt_bias); - } - - /* clear contents of rx_buffer */ - rx_buffer->page = NULL; - rx_buffer->skb = NULL; + dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, PAGE_SIZE, + DMA_FROM_DEVICE, IXGBE_RX_DMA_ATTR); } static struct sk_buff *ixgbe_build_skb(struct ixgbe_ring *rx_ring, @@ -2228,13 +2050,9 @@ static struct sk_buff *ixgbe_build_skb(struct ixgbe_ring *rx_ring, union ixgbe_adv_rx_desc *rx_desc) { unsigned int metasize = xdp->data - xdp->data_meta; -#if (PAGE_SIZE < 8192) - unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2; -#else unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + SKB_DATA_ALIGN(xdp->data_end - xdp->data_hard_start); -#endif struct sk_buff *skb; /* Prefetch first cache line of first page. If xdp->data_meta @@ -2259,13 +2077,6 @@ static struct sk_buff *ixgbe_build_skb(struct ixgbe_ring *rx_ring, if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)) IXGBE_CB(skb)->dma = rx_buffer->dma; - /* update buffer offset */ -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif - return skb; } @@ -2324,31 +2135,6 @@ static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, return result; } -static unsigned int ixgbe_rx_frame_truesize(struct ixgbe_ring *rx_ring, - unsigned int size) -{ - unsigned int truesize; - -#if (PAGE_SIZE < 8192) - truesize = ixgbe_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */ -#else - truesize = SKB_DATA_ALIGN(IXGBE_SKB_PAD + size) + - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); -#endif - return truesize; -} - -static void ixgbe_rx_buffer_flip(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *rx_buffer, - unsigned int size) -{ - unsigned int truesize = ixgbe_rx_frame_truesize(rx_ring, size); -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif -} /** * ixgbe_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf @@ -2367,7 +2153,7 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, struct ixgbe_ring *rx_ring, const int budget) { - unsigned int total_rx_bytes = 0, total_rx_packets = 0, frame_sz = 0; + unsigned int total_rx_bytes = 0, total_rx_packets = 0; struct ixgbe_adapter *adapter = q_vector->adapter; #ifdef IXGBE_FCOE int ddp_bytes; @@ -2376,19 +2162,14 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, u16 cleaned_count = ixgbe_desc_unused(rx_ring); unsigned int xdp_xmit = 0; struct xdp_buff xdp; + struct sk_buff *skb = rx_ring->skb; int xdp_res = 0; - /* Frame size depend on rx_ring setup when PAGE_SIZE=4K */ -#if (PAGE_SIZE < 8192) - frame_sz = ixgbe_rx_frame_truesize(rx_ring, 0); -#endif - xdp_init_buff(&xdp, frame_sz, &rx_ring->xdp_rxq); + xdp_init_buff(&xdp, IXGBE_RXBUFFER_3K, &rx_ring->xdp_rxq); while (likely(total_rx_packets < budget)) { union ixgbe_adv_rx_desc *rx_desc; struct ixgbe_rx_buffer *rx_buffer; - struct sk_buff *skb; - int rx_buffer_pgcnt; unsigned int size; /* return some buffers to hardware, one at a time is too slow */ @@ -2408,7 +2189,7 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, */ dma_rmb(); - rx_buffer = ixgbe_get_rx_buffer(rx_ring, rx_desc, &skb, size, &rx_buffer_pgcnt); + rx_buffer = ixgbe_get_rx_buffer(rx_ring, size); /* retrieve a buffer from the ring */ if (!skb) { @@ -2419,20 +2200,13 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, rx_buffer->page_offset - offset; xdp_prepare_buff(&xdp, hard_start, offset, size, true); xdp_buff_clear_frags_flag(&xdp); -#if (PAGE_SIZE > 4096) - /* At larger PAGE_SIZE, frame_sz depend on len size */ - xdp.frame_sz = ixgbe_rx_frame_truesize(rx_ring, size); -#endif xdp_res = ixgbe_run_xdp(adapter, rx_ring, &xdp); } if (xdp_res) { - if (xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR)) { + if (xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR)) xdp_xmit |= xdp_res; - ixgbe_rx_buffer_flip(rx_ring, rx_buffer, size); - } else { - rx_buffer->pagecnt_bias++; - } + total_rx_packets++; total_rx_bytes += size; } else if (skb) { @@ -2445,11 +2219,13 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, /* exit if we failed to retrieve a buffer */ if (!xdp_res && !skb) { rx_ring->rx_stats.alloc_rx_buff_failed++; - rx_buffer->pagecnt_bias++; + cleaned_count++; break; } - ixgbe_put_rx_buffer(rx_ring, rx_buffer, skb, rx_buffer_pgcnt); + ixgbe_put_rx_buffer(rx_ring, rx_buffer); + if (xdp_res == IXGBE_XDP_CONSUMED) + __free_page(rx_buffer->page); cleaned_count++; /* place incomplete frames back on ring for completion */ @@ -2497,6 +2273,8 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, total_rx_packets++; } + rx_ring->skb = skb; + if (xdp_xmit & IXGBE_XDP_REDIR) xdp_do_flush(); @@ -4040,10 +3818,8 @@ static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter, srrctl |= PAGE_SIZE >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; else srrctl |= xsk_buf_len >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; - } else if (test_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state)) { - srrctl |= IXGBE_RXBUFFER_3K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; } else { - srrctl |= IXGBE_RXBUFFER_2K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; + srrctl |= IXGBE_RXBUFFER_3K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; } /* configure descriptor type */ @@ -4419,17 +4195,8 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, } else if (hw->mac.type != ixgbe_mac_82599EB) { rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | IXGBE_RXDCTL_RLPML_EN); - - /* Limit the maximum frame size so we don't overrun the skb. - * This can happen in SRIOV mode when the MTU of the VF is - * higher than the MTU of the PF. - */ - if (!test_bit(__IXGBE_RX_3K_BUFFER, &ring->state)) - rxdctl |= IXGBE_MAX_2K_FRAME_BUILD_SKB | - IXGBE_RXDCTL_RLPML_EN; -#endif } - +#endif if (ring->xsk_pool && hw->mac.type != ixgbe_mac_82599EB) { u32 xsk_buf_len = xsk_pool_get_rx_frame_size(ring->xsk_pool); @@ -4439,11 +4206,9 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, ring->rx_buf_len = xsk_buf_len; } - /* initialize rx_buffer_info */ memset(ring->rx_buffer_info, 0, - sizeof(struct ixgbe_rx_buffer) * ring->count); - + sizeof(struct ixgbe_rx_buffer) * ring->count); /* initialize Rx descriptor 0 */ rx_desc = IXGBE_RX_DESC(ring, 0); rx_desc->wb.upper.length = 0; @@ -4560,8 +4325,6 @@ static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter) struct ixgbe_hw *hw = &adapter->hw; struct net_device *netdev = adapter->netdev; int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; - struct ixgbe_ring *rx_ring; - int i; u32 mhadd, hlreg0; #ifdef IXGBE_FCOE @@ -4569,7 +4332,6 @@ static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter) if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) && (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE)) max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE; - #endif /* IXGBE_FCOE */ /* adjust max frame to be at least the size of a standard frame */ @@ -4585,35 +4347,11 @@ static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter) } hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); - /* set jumbo enable since MHADD.MFS is keeping size locked at max_frame */ + /* set jumbo enable since MHADD.MFS is keeping size locked at + * max_frame + */ hlreg0 |= IXGBE_HLREG0_JUMBOEN; IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0); - - /* - * Setup the HW Rx Head and Tail Descriptor Pointers and - * the Base and Length of the Rx Descriptor Ring - */ - for (i = 0; i < adapter->num_rx_queues; i++) { - rx_ring = adapter->rx_ring[i]; - - clear_ring_rsc_enabled(rx_ring); - clear_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); - - if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) - set_ring_rsc_enabled(rx_ring); - - if (test_bit(__IXGBE_RX_FCOE, &rx_ring->state)) - set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); - -#if (PAGE_SIZE < 8192) - if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) - set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); - - if (IXGBE_2K_TOO_SMALL_WITH_PADDING || - (max_frame > (ETH_FRAME_LEN + ETH_FCS_LEN))) - set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); -#endif - } } static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter) @@ -4698,8 +4436,15 @@ static void ixgbe_configure_rx(struct ixgbe_adapter *adapter) * Setup the HW Rx Head and Tail Descriptor Pointers and * the Base and Length of the Rx Descriptor Ring */ - for (i = 0; i < adapter->num_rx_queues; i++) - ixgbe_configure_rx_ring(adapter, adapter->rx_ring[i]); + for (i = 0; i < adapter->num_rx_queues; i++) { + struct ixgbe_ring *rx_ring = adapter->rx_ring[i]; + + clear_ring_rsc_enabled(rx_ring); + if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) + set_ring_rsc_enabled(rx_ring); + + ixgbe_configure_rx_ring(adapter, rx_ring); + } rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); /* disable drop enable for 82598 parts */ @@ -5605,35 +5350,28 @@ static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) goto skip_free; } - /* Free all the Rx ring sk_buffs */ - while (i != rx_ring->next_to_alloc) { - if (rx_buffer->skb) { - struct sk_buff *skb = rx_buffer->skb; - if (IXGBE_CB(skb)->page_released) - dma_unmap_page_attrs(rx_ring->dev, - IXGBE_CB(skb)->dma, - ixgbe_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, - IXGBE_RX_DMA_ATTR); - dev_kfree_skb(skb); - } + /* Free Rx ring sk_buff */ + if (rx_ring->skb) { + dev_kfree_skb(rx_ring->skb); + rx_ring->skb = NULL; + } + /* Free all the Rx ring pages */ + while (i != rx_ring->next_to_use) { /* Invalidate cache lines that may have been written to by * device so that we avoid corrupting memory. */ dma_sync_single_range_for_cpu(rx_ring->dev, rx_buffer->dma, rx_buffer->page_offset, - ixgbe_rx_bufsz(rx_ring), + IXGBE_RXBUFFER_3K, DMA_FROM_DEVICE); /* free resources associated with mapping */ dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, - ixgbe_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, + PAGE_SIZE, DMA_FROM_DEVICE, IXGBE_RX_DMA_ATTR); - __page_frag_cache_drain(rx_buffer->page, - rx_buffer->pagecnt_bias); + __free_page(rx_buffer->page); i++; rx_buffer++; @@ -5644,7 +5382,6 @@ static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) } skip_free: - rx_ring->next_to_alloc = 0; rx_ring->next_to_clean = 0; rx_ring->next_to_use = 0; } @@ -7193,10 +6930,7 @@ static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter) */ static int ixgbe_max_xdp_frame_size(struct ixgbe_adapter *adapter) { - if (PAGE_SIZE >= 8192) - return IXGBE_RXBUFFER_2K; - else - return IXGBE_RXBUFFER_3K; + return IXGBE_RXBUFFER_3K; } /** @@ -10761,11 +10495,11 @@ static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog) if (ring_is_rsc_enabled(ring)) return -EINVAL; - - if (frame_size > ixgbe_rx_bufsz(ring)) - return -EINVAL; } + if (frame_size > IXGBE_RXBUFFER_3K) + return -EINVAL; + /* if the number of cpus is much larger than the maximum of queues, * we should stop it and then return with ENOMEM like before. */ @@ -11131,6 +10865,11 @@ void ixgbe_txrx_ring_enable(struct ixgbe_adapter *adapter, int ring) ixgbe_configure_tx_ring(adapter, tx_ring); if (xdp_ring) ixgbe_configure_tx_ring(adapter, xdp_ring); + + clear_ring_rsc_enabled(rx_ring); + if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) + set_ring_rsc_enabled(rx_ring); + ixgbe_configure_rx_ring(adapter, rx_ring); clear_bit(__IXGBE_TX_DISABLED, &tx_ring->state); From 31040ff5982bd4ac7b2812210b46e165e768c5a5 Mon Sep 17 00:00:00 2001 From: Magdalena Pytel Date: Thu, 7 May 2026 23:34:48 +0200 Subject: [PATCH 17/39] ixgbe: use libeth in Rx processing Convert ixgbe Rx path from manual page-per-packet DMA management to libeth/page_pool infrastructure, and integrates libeth XDP helpers. Each packet gets its own buffer instead of sharing pages between packets. Signed-off-by: Magdalena Pytel --- drivers/net/ethernet/intel/Kconfig | 1 + drivers/net/ethernet/intel/ixgbe/ixgbe.h | 39 +- .../net/ethernet/intel/ixgbe/ixgbe_ethtool.c | 23 +- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 1149 ++++++++--------- drivers/net/ethernet/intel/ixgbe/ixgbe_type.h | 2 +- drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c | 19 +- 6 files changed, 554 insertions(+), 679 deletions(-) diff --git a/drivers/net/ethernet/intel/Kconfig b/drivers/net/ethernet/intel/Kconfig index b513baf3cbb296..072133b4a4416d 100644 --- a/drivers/net/ethernet/intel/Kconfig +++ b/drivers/net/ethernet/intel/Kconfig @@ -146,6 +146,7 @@ config IXGBE tristate "Intel(R) 10GbE PCI Express adapters support" depends on PCI depends on PTP_1588_CLOCK_OPTIONAL + select LIBETH_XDP select LIBIE_FWLOG if DEBUG_FS select MDIO select NET_DEVLINK diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index ef57e6029e492f..f5842f35d498bd 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -33,6 +33,7 @@ #include "ixgbe_ipsec.h" #include +#include /* common prefix used by pr_<> macros */ #undef pr_fmt @@ -73,9 +74,8 @@ #define IXGBE_RXBUFFER_4K 4096 #define IXGBE_MAX_RXBUFFER 16384 /* largest size for a single descriptor */ -#define IXGBE_PKT_HDR_PAD (ETH_HLEN + ETH_FCS_LEN + (VLAN_HLEN * 2)) - -#define IXGBE_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN) +#define IXGBE_RX_PAGE_LEN(hr) (ALIGN_DOWN(LIBETH_RX_PAGE_LEN(hr), \ + IXGBE_SRRCTL_BSIZEPKT_STEP)) /* Default HW header buffer size */ #define IXGBE_RX_HDR_SIZE IXGBE_RXBUFFER_256 @@ -212,18 +212,9 @@ struct ixgbe_tx_buffer { u32 tx_flags; }; -struct ixgbe_rx_buffer { - union { - struct { - dma_addr_t dma; - struct page *page; - __u32 page_offset; - }; - struct { - bool discard; - struct xdp_buff *xdp; - }; - }; +struct ixgbe_xsk_rx_buffer { + bool discard; + struct xdp_buff *xdp; }; struct ixgbe_queue_stats { @@ -291,17 +282,22 @@ struct ixgbe_ring { struct ixgbe_ring *next; /* pointer to next ring in q_vector */ struct ixgbe_q_vector *q_vector; /* backpointer to host q_vector */ struct net_device *netdev; /* netdev ring belongs to */ - struct bpf_prog *xdp_prog; - struct device *dev; /* device for DMA mapping */ + struct bpf_prog __rcu *xdp_prog; + union { + struct page_pool *pp; /* Rx ring */ + struct device *dev; /* Tx ring */ + }; void *desc; /* descriptor ring memory */ union { + struct libeth_fqe *rx_fqes; + struct ixgbe_xsk_rx_buffer *rx_xsk_buffer_info; struct ixgbe_tx_buffer *tx_buffer_info; - struct ixgbe_rx_buffer *rx_buffer_info; }; unsigned long state; u8 __iomem *tail; dma_addr_t dma; /* phys. address of descriptor ring */ unsigned int size; /* length in bytes */ + u32 truesize; u16 count; /* amount of descriptors */ @@ -333,7 +329,7 @@ struct ixgbe_ring { spinlock_t tx_lock; /* used in XDP mode */ struct xsk_buff_pool *xsk_pool; u16 ring_idx; /* {rx,tx,xdp}_ring back reference idx */ - u16 rx_buf_len; + u32 rx_buf_len; } ____cacheline_internodealigned_in_smp; enum ixgbe_ring_f_enum { @@ -377,11 +373,6 @@ struct ixgbe_ring_feature { #define IXGBE_82599_VMDQ_4Q_MASK 0x7C #define IXGBE_82599_VMDQ_2Q_MASK 0x7E -static inline unsigned int ixgbe_rx_offset(struct ixgbe_ring *rx_ring) -{ - return IXGBE_SKB_PAD; -} - #define IXGBE_ITR_ADAPTIVE_MIN_INC 2 #define IXGBE_ITR_ADAPTIVE_MIN_USECS 10 #define IXGBE_ITR_ADAPTIVE_MAX_USECS 126 diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c index b9db6ee9668e93..a2cc9b39eb634c 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c @@ -1949,7 +1949,6 @@ static int ixgbe_setup_desc_rings(struct ixgbe_adapter *adapter) /* Setup Rx Descriptor ring and Rx buffers */ rx_ring->count = IXGBE_DEFAULT_RXD; rx_ring->queue_index = 0; - rx_ring->dev = &adapter->pdev->dev; rx_ring->netdev = adapter->netdev; rx_ring->reg_idx = adapter->rx_ring[0]->reg_idx; @@ -2060,14 +2059,16 @@ static void ixgbe_create_lbtest_frame(struct sk_buff *skb, skb->data[frame_size + 12] = 0xAF; } -static bool ixgbe_check_lbtest_frame(struct ixgbe_rx_buffer *rx_buffer, +static bool ixgbe_check_lbtest_frame(const struct libeth_fqe *rx_buffer, unsigned int frame_size) { + u32 hr = netmem_get_pp(rx_buffer->netmem)->p.offset; unsigned char *data; frame_size >>= 1; - data = page_address(rx_buffer->page) + rx_buffer->page_offset; + data = page_address(__netmem_to_page(rx_buffer->netmem)) + + rx_buffer->offset + hr; return data[3] == 0xFF && data[frame_size + 10] == 0xBE && data[frame_size + 12] == 0xAF; @@ -2115,16 +2116,13 @@ static u16 ixgbe_clean_test_rings(struct ixgbe_ring *rx_ring, } while (rx_desc->wb.upper.length) { - struct ixgbe_rx_buffer *rx_buffer; + struct libeth_fqe *rx_buffer; /* check Rx buffer */ - rx_buffer = &rx_ring->rx_buffer_info[rx_ntc]; + rx_buffer = &rx_ring->rx_fqes[rx_ntc]; /* sync Rx buffer for CPU read */ - dma_sync_single_for_cpu(rx_ring->dev, - rx_buffer->dma, - IXGBE_RXBUFFER_3K, - DMA_FROM_DEVICE); + libeth_rx_sync_for_cpu(rx_buffer, rx_ring->rx_buf_len); /* verify contents of skb */ if (ixgbe_check_lbtest_frame(rx_buffer, size)) @@ -2132,11 +2130,8 @@ static u16 ixgbe_clean_test_rings(struct ixgbe_ring *rx_ring, else break; - /* sync Rx buffer for device write */ - dma_sync_single_for_device(rx_ring->dev, - rx_buffer->dma, - IXGBE_RXBUFFER_3K, - DMA_FROM_DEVICE); + /* recycle the page back to the pool */ + libeth_rx_recycle_slow(rx_buffer->netmem); /* increment Rx next to clean counter */ rx_ntc++; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 73e46aea5c671d..c50a5a1e73eb19 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -54,16 +55,16 @@ char ixgbe_driver_name[] = "ixgbe"; static const char ixgbe_driver_string[] = - "Intel(R) 10 Gigabit PCI Express Network Driver"; + "Intel(R) 10 Gigabit PCI Express Network Driver"; #ifdef IXGBE_FCOE char ixgbe_default_device_descr[] = - "Intel(R) 10 Gigabit Network Connection"; + "Intel(R) 10 Gigabit Network Connection"; #else static char ixgbe_default_device_descr[] = - "Intel(R) 10 Gigabit Network Connection"; + "Intel(R) 10 Gigabit Network Connection"; #endif static const char ixgbe_copyright[] = - "Copyright (c) 1999-2016 Intel Corporation."; + "Copyright (c) 1999-2016 Intel Corporation."; static const char ixgbe_overheat_msg[] = "Network adapter has been stopped because it has over heated. Restart the computer. If the problem persists, power off the system and replace the adapter"; @@ -147,7 +148,7 @@ MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl); #ifdef CONFIG_IXGBE_DCA static int ixgbe_notify_dca(struct notifier_block *, unsigned long event, - void *p); + void *p); static struct notifier_block dca_notifier = { .notifier_call = ixgbe_notify_dca, .next = NULL, @@ -174,6 +175,8 @@ MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); MODULE_IMPORT_NS("LIBIE_FWLOG"); MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver"); +MODULE_IMPORT_NS("LIBETH"); +MODULE_IMPORT_NS("LIBETH_XDP"); MODULE_LICENSE("GPL v2"); DEFINE_STATIC_KEY_FALSE(ixgbe_xdp_locking_key); @@ -212,7 +215,7 @@ static int ixgbe_read_pci_cfg_word_parent(struct ixgbe_adapter *adapter, pcie_capability_read_word(parent_dev, reg, value); if (*value == IXGBE_FAILED_READ_CFG_WORD && - ixgbe_check_cfg_remove(&adapter->hw, parent_dev)) + ixgbe_check_cfg_remove(&adapter->hw, parent_dev)) return -1; return 0; } @@ -264,7 +267,7 @@ static bool ixgbe_pcie_from_parent(struct ixgbe_hw *hw) } static void ixgbe_check_minimum_link(struct ixgbe_adapter *adapter, - int expected_gts) + int expected_gts) { struct ixgbe_hw *hw = &adapter->hw; struct pci_dev *pdev; @@ -288,8 +291,8 @@ static void ixgbe_check_minimum_link(struct ixgbe_adapter *adapter, static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter) { if (!test_bit(__IXGBE_DOWN, &adapter->state) && - !test_bit(__IXGBE_REMOVING, &adapter->state) && - !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state)) + !test_bit(__IXGBE_REMOVING, &adapter->state) && + !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state)) queue_work(ixgbe_wq, &adapter->service_task); } @@ -354,7 +357,7 @@ u32 ixgbe_read_reg(struct ixgbe_hw *hw, u32 reg) if (ixgbe_removed(reg_addr)) return IXGBE_FAILED_READ_REG; if (unlikely(hw->phy.nw_mng_if_sel & - IXGBE_NW_MNG_IF_SEL_SGMII_ENABLE)) { + IXGBE_NW_MNG_IF_SEL_SGMII_ENABLE)) { struct ixgbe_adapter *adapter; int i; @@ -401,7 +404,7 @@ u16 ixgbe_read_pci_cfg_word(struct ixgbe_hw *hw, u32 reg) return IXGBE_FAILED_READ_CFG_WORD; pci_read_config_word(adapter->pdev, reg, &value); if (value == IXGBE_FAILED_READ_CFG_WORD && - ixgbe_check_cfg_remove(hw, adapter->pdev)) + ixgbe_check_cfg_remove(hw, adapter->pdev)) return IXGBE_FAILED_READ_CFG_WORD; return value; } @@ -416,7 +419,7 @@ static u32 ixgbe_read_pci_cfg_dword(struct ixgbe_hw *hw, u32 reg) return IXGBE_FAILED_READ_CFG_DWORD; pci_read_config_dword(adapter->pdev, reg, &value); if (value == IXGBE_FAILED_READ_CFG_DWORD && - ixgbe_check_cfg_remove(hw, adapter->pdev)) + ixgbe_check_cfg_remove(hw, adapter->pdev)) return IXGBE_FAILED_READ_CFG_DWORD; return value; } @@ -612,7 +615,7 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) dev_info(&adapter->pdev->dev, "Register Dump\n"); pr_info(" Register Name Value\n"); for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl; - reginfo->name; reginfo++) { + reginfo->name; reginfo++) { ixgbe_regdump(hw, reginfo); } @@ -693,7 +696,7 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) const char *ring_desc; if (i == ring->next_to_use && - i == ring->next_to_clean) + i == ring->next_to_clean) ring_desc = " NTC/U"; else if (i == ring->next_to_use) ring_desc = " NTU"; @@ -713,7 +716,7 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) ring_desc); if (netif_msg_pktdata(adapter) && - tx_buffer->skb) + tx_buffer->skb) print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS, 16, 1, tx_buffer->skb->data, @@ -819,17 +822,19 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) i, le64_to_cpu((__force __le64)u0->a), le64_to_cpu((__force __le64)u0->b), - (u64)rx_ring->rx_buffer_info[i].dma, + (u64)page_pool_get_dma_addr_netmem(rx_ring->rx_fqes[i].netmem), ring_desc); } if (netif_msg_pktdata(adapter) && - rx_ring->rx_buffer_info[i].dma) { + rx_ring->rx_fqes[i].netmem) { + struct libeth_fqe *fqe = &rx_ring->rx_fqes[i]; + print_hex_dump(KERN_INFO, "", - DUMP_PREFIX_ADDRESS, 16, 1, - page_address(rx_ring->rx_buffer_info[i].page) + - rx_ring->rx_buffer_info[i].page_offset, - IXGBE_RXBUFFER_3K, true); + DUMP_PREFIX_ADDRESS, 16, 1, + page_address(__netmem_to_page(fqe->netmem)) + + fqe->offset, + IXGBE_RXBUFFER_3K, true); } } } @@ -910,7 +915,7 @@ static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction, } void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter, - u64 qmask) + u64 qmask) { u32 mask; @@ -943,7 +948,7 @@ static void ixgbe_update_xoff_rx_lfc(struct ixgbe_adapter *adapter) u32 data; if ((hw->fc.current_mode != ixgbe_fc_full) && - (hw->fc.current_mode != ixgbe_fc_rx_pause)) + (hw->fc.current_mode != ixgbe_fc_rx_pause)) return; switch (hw->mac.type) { @@ -1127,7 +1132,7 @@ static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter) * @maxrate: desired maximum transmit bitrate **/ static int ixgbe_tx_maxrate(struct net_device *netdev, - int queue_index, u32 maxrate) + int queue_index, u32 maxrate) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); struct ixgbe_hw *hw = &adapter->hw; @@ -1199,7 +1204,7 @@ void ixgbe_update_rx_ring_stats(struct ixgbe_ring *rx_ring, * Prints a message containing details about the tx hang. */ static void ixgbe_pf_handle_tx_hang(struct ixgbe_ring *tx_ring, - unsigned int next) + unsigned int next) { struct ixgbe_adapter *adapter = netdev_priv(tx_ring->netdev); struct ixgbe_hw *hw = &adapter->hw; @@ -1212,14 +1217,14 @@ static void ixgbe_pf_handle_tx_hang(struct ixgbe_ring *tx_ring, "tx_buffer_info[next_to_clean]\n" " time_stamp <%lx>\n" " jiffies <%lx>\n", - tx_ring->queue_index, - IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)), - IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)), - tx_ring->next_to_use, next, - tx_ring->tx_buffer_info[next].time_stamp, jiffies); + tx_ring->queue_index, + IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)), + IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)), + tx_ring->next_to_use, next, + tx_ring->tx_buffer_info[next].time_stamp, jiffies); netif_stop_subqueue(tx_ring->netdev, - tx_ring->queue_index); + tx_ring->queue_index); } /** @@ -1238,8 +1243,8 @@ static void ixgbe_vf_handle_tx_hang(struct ixgbe_adapter *adapter, u16 vf) return; e_warn(drv, - "Malicious Driver Detection tx hang detected on PF %d VF %d MAC: %pM", - hw->bus.func, vf, adapter->vfinfo[vf].vf_mac_addresses); + "Malicious Driver Detection tx hang detected on PF %d VF %d MAC: %pM", + hw->bus.func, vf, adapter->vfinfo[vf].vf_mac_addresses); adapter->tx_hang_count[vf]++; if (adapter->tx_hang_count[vf] == IXGBE_MAX_TX_VF_HANGS) { @@ -1266,7 +1271,7 @@ static u32 ixgbe_poll_tx_icache(struct ixgbe_hw *hw, u16 queue, u16 idx) * Returns: true if queue contain illegal packet. */ static bool ixgbe_check_illegal_queue(struct ixgbe_adapter *adapter, - u16 queue) + u16 queue) { u32 hdr_len_reg, mss_len_reg, type_reg; struct ixgbe_hw *hw = &adapter->hw; @@ -1289,13 +1294,13 @@ static bool ixgbe_check_illegal_queue(struct ixgbe_adapter *adapter, /* check if Advanced Context Descriptor */ if (FIELD_GET(IXGBE_ADVTXD_DTYP_MASK, type_reg) != - IXGBE_ADVTXD_DTYP_CTXT) + IXGBE_ADVTXD_DTYP_CTXT) continue; /* check for illegal MSS and Header length */ mss_len = FIELD_GET(IXGBE_ADVTXD_MSS_MASK, mss_len_reg); header_len = FIELD_GET(IXGBE_ADVTXD_HEADER_LEN_MASK, - hdr_len_reg); + hdr_len_reg); if ((mss_len + header_len) > SZ_16K) { e_warn(probe, "mss len + header len too long\n"); return true; @@ -1326,7 +1331,7 @@ static void ixgbe_handle_mdd_event(struct ixgbe_adapter *adapter, /* malicious vf not detected */ for (q = 0; q < IXGBE_MAX_TX_QUEUES; q++) { if (ixgbe_check_illegal_queue(adapter, q) && - !ixgbe_get_vf_idx(adapter, q, &vf)) + !ixgbe_get_vf_idx(adapter, q, &vf)) /* illegal queue detected */ ixgbe_vf_handle_tx_hang(adapter, vf); } @@ -1340,7 +1345,7 @@ static void ixgbe_handle_mdd_event(struct ixgbe_adapter *adapter, * @napi_budget: Used to determine if we are in netpoll **/ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, - struct ixgbe_ring *tx_ring, int napi_budget) + struct ixgbe_ring *tx_ring, int napi_budget) { struct ixgbe_adapter *adapter = q_vector->adapter; struct ixgbe_tx_buffer *tx_buffer; @@ -1409,9 +1414,9 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, /* unmap any remaining paged data */ if (dma_unmap_len(tx_buffer, len)) { dma_unmap_page(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); dma_unmap_len_set(tx_buffer, len, 0); } } @@ -1449,7 +1454,7 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, ixgbe_pf_handle_tx_hang(tx_ring, i); e_info(probe, - "tx hang %d detected on queue %d, resetting adapter\n", + "tx hang %d detected on queue %d, resetting adapter\n", adapter->tx_timeout_count + 1, tx_ring->queue_index); /* schedule immediate reset if we believe we hung */ @@ -1518,7 +1523,7 @@ static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter, u8 reg_idx = rx_ring->reg_idx; if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) - rxctrl = dca3_get_tag(rx_ring->dev, cpu); + rxctrl = dca3_get_tag(&adapter->pdev->dev, cpu); switch (hw->mac.type) { case ixgbe_mac_82599EB: @@ -1636,8 +1641,8 @@ static inline void ixgbe_rx_hash(struct ixgbe_ring *ring, return; skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), - (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ? - PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); + (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ? + PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); } #ifdef IXGBE_FCOE @@ -1649,14 +1654,14 @@ static inline void ixgbe_rx_hash(struct ixgbe_ring *ring, * Returns : true if it is FCoE pkt */ static inline bool ixgbe_rx_is_fcoe(struct ixgbe_ring *ring, - union ixgbe_adv_rx_desc *rx_desc) + union ixgbe_adv_rx_desc *rx_desc) { __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info; return test_bit(__IXGBE_RX_FCOE, &ring->state) && - ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) == + ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) == (cpu_to_le16(IXGBE_ETQF_FILTER_FCOE << - IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT))); + IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT))); } #endif /* IXGBE_FCOE */ @@ -1667,8 +1672,8 @@ static inline bool ixgbe_rx_is_fcoe(struct ixgbe_ring *ring, * @skb: skb currently being received and modified **/ static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) + union ixgbe_adv_rx_desc *rx_desc, + struct sk_buff *skb) { __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info; bool encap_pkt = false; @@ -1687,7 +1692,7 @@ static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring, /* if IP and error */ if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) && - ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { + ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { ring->rx_stats.csum_err++; return; } @@ -1701,7 +1706,7 @@ static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring, * checksum errors. */ if ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_UDP)) && - test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state)) + test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state)) return; ring->rx_stats.csum_err++; @@ -1723,39 +1728,6 @@ static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring, } } -static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *bi) -{ - struct page *page = bi->page; - dma_addr_t dma; - - /* alloc new page for storage */ - page = dev_alloc_page(); - if (unlikely(!page)) { - rx_ring->rx_stats.alloc_rx_page_failed++; - return false; - } - - /* map page for use */ - dma = dma_map_page_attrs(rx_ring->dev, page, 0, PAGE_SIZE, - DMA_FROM_DEVICE, IXGBE_RX_DMA_ATTR); - - if (dma_mapping_error(rx_ring->dev, dma)) { - __free_page(page); - - rx_ring->rx_stats.alloc_rx_page_failed++; - return false; - } - - bi->dma = dma; - bi->page = page; - bi->page_offset = rx_ring->xdp_prog ? XDP_PACKET_HEADROOM : - IXGBE_SKB_PAD; - rx_ring->rx_stats.alloc_rx_page++; - - return true; -} - /** * ixgbe_alloc_rx_buffers - Replace used receive buffers * @rx_ring: ring to place buffers on @@ -1764,40 +1736,34 @@ static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring, void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) { union ixgbe_adv_rx_desc *rx_desc; - struct ixgbe_rx_buffer *bi; - u16 i = rx_ring->next_to_use; + const struct libeth_fq_fp fq = { + .pp = rx_ring->pp, + .fqes = rx_ring->rx_fqes, + .truesize = rx_ring->truesize, + .count = rx_ring->count, + }; + u16 ntu = rx_ring->next_to_use; /* nothing to do */ if (!cleaned_count) return; - - rx_desc = IXGBE_RX_DESC(rx_ring, i); - bi = &rx_ring->rx_buffer_info[i]; - i -= rx_ring->count; + rx_desc = IXGBE_RX_DESC(rx_ring, ntu); do { - if (!ixgbe_alloc_mapped_page(rx_ring, bi)) - break; + dma_addr_t addr; - /* sync the buffer for use by the device */ - dma_sync_single_range_for_device(rx_ring->dev, bi->dma, - bi->page_offset, - IXGBE_RXBUFFER_3K, - DMA_FROM_DEVICE); + addr = libeth_rx_alloc(&fq, ntu); + if (addr == DMA_MAPPING_ERROR) + break; - /* - * Refresh the desc even if buffer_addrs didn't change - * because each write-back erases this info. - */ - rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); + rx_desc->read.pkt_addr = cpu_to_le64(addr); rx_desc++; - bi++; - i++; - if (unlikely(!i)) { + ntu++; + + if (unlikely(ntu == rx_ring->count)) { rx_desc = IXGBE_RX_DESC(rx_ring, 0); - bi = rx_ring->rx_buffer_info; - i -= rx_ring->count; + ntu = 0; } /* clear the length for the next_to_use descriptor */ @@ -1806,18 +1772,16 @@ void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) cleaned_count--; } while (cleaned_count); - i += rx_ring->count; - - if (rx_ring->next_to_use != i) { - rx_ring->next_to_use = i; - + if (likely(rx_ring->next_to_use != ntu)) { + /* record the next descriptor to use */ + rx_ring->next_to_use = ntu; /* Force memory writes to complete before letting h/w * know there are new descriptors to fetch. (Only * applicable for weak-ordered memory model archs, * such as IA-64). */ wmb(); - writel(i, rx_ring->tail); + writel(ntu, rx_ring->tail); } } @@ -1859,8 +1823,8 @@ static void ixgbe_update_rsc_stats(struct ixgbe_ring *rx_ring, * other fields within the skb. **/ void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) + union ixgbe_adv_rx_desc *rx_desc, + struct sk_buff *skb) { struct net_device *dev = rx_ring->netdev; u32 flags = rx_ring->q_vector->adapter->flags; @@ -1875,7 +1839,7 @@ void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring, ixgbe_ptp_rx_hwtstamp(rx_ring, rx_desc, skb); if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) && - ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { + ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan); __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); } @@ -1889,8 +1853,6 @@ void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring, else macvlan_count_rx(netdev_priv(dev), skb->len + ETH_HLEN, true, false); - - skb->protocol = eth_type_trans(skb, dev); } void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector, @@ -1911,8 +1873,8 @@ void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector, * that this is in fact a non-EOP buffer. **/ static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) + union ixgbe_adv_rx_desc *rx_desc, + struct sk_buff *skb) { u32 ntc = rx_ring->next_to_clean + 1; @@ -1925,7 +1887,7 @@ static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring, /* update RSC append count if present */ if (ring_is_rsc_enabled(rx_ring)) { __le32 rsc_enabled = rx_desc->wb.lower.lo_dword.data & - cpu_to_le32(IXGBE_RXDADV_RSCCNT_MASK); + cpu_to_le32(IXGBE_RXDADV_RSCCNT_MASK); if (unlikely(rsc_enabled)) { u32 rsc_cnt = le32_to_cpu(rsc_enabled); @@ -1982,9 +1944,9 @@ bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring, * errors that would be unacceptable to the netdev. */ if (!netdev || - (unlikely(ixgbe_test_staterr(rx_desc, + (unlikely(ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_FRAME_ERR_MASK) && - !(netdev->features & NETIF_F_RXALL)))) { + !(netdev->features & NETIF_F_RXALL)))) { dev_kfree_skb_any(skb); return true; } @@ -2007,82 +1969,28 @@ bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring, * @size: size of data in rx_buffer * * This function will add the data contained in rx_buffer->page to the skb. + * This is done either through a direct copy if the data in the buffer is + * less than the skb header size, otherwise it will just attach the page as + * a frag to the skb. + * + * The function will then update the page offset if necessary and return + * true if the buffer can be reused by the adapter. **/ -static void ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *rx_buffer, - struct sk_buff *skb, - unsigned int size) +static void ixgbe_add_rx_frag(const struct libeth_fqe *rx_buffer, + struct sk_buff *skb, + unsigned int size) { - unsigned int truesize = SKB_DATA_ALIGN(IXGBE_SKB_PAD + size); + u32 hr = netmem_get_pp(rx_buffer->netmem)->p.offset; - skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, - rx_buffer->page_offset, size, truesize); + skb_add_rx_frag_netmem(skb, skb_shinfo(skb)->nr_frags, + rx_buffer->netmem, rx_buffer->offset + hr, + size, rx_buffer->truesize); } -static struct ixgbe_rx_buffer *ixgbe_get_rx_buffer(struct ixgbe_ring *rx_ring, - const unsigned int size) -{ - struct ixgbe_rx_buffer *rx_buffer; - - rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; - prefetchw(rx_buffer->page); - - /* sync this buffer for CPU use */ - dma_sync_single_range_for_cpu(rx_ring->dev, - rx_buffer->dma, - rx_buffer->page_offset, - size, - DMA_FROM_DEVICE); - - return rx_buffer; -} - -static void ixgbe_put_rx_buffer(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *rx_buffer) -{ - dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, PAGE_SIZE, - DMA_FROM_DEVICE, IXGBE_RX_DMA_ATTR); -} - -static struct sk_buff *ixgbe_build_skb(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *rx_buffer, - struct xdp_buff *xdp, - union ixgbe_adv_rx_desc *rx_desc) -{ - unsigned int metasize = xdp->data - xdp->data_meta; - unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + - SKB_DATA_ALIGN(xdp->data_end - - xdp->data_hard_start); - struct sk_buff *skb; - - /* Prefetch first cache line of first page. If xdp->data_meta - * is unused, this points exactly as xdp->data, otherwise we - * likely have a consumer accessing first few bytes of meta - * data, and then actual data. - */ - net_prefetch(xdp->data_meta); - - /* build an skb to around the page buffer */ - skb = napi_build_skb(xdp->data_hard_start, truesize); - if (unlikely(!skb)) - return NULL; - - /* update pointers within the skb to store the data */ - skb_reserve(skb, xdp->data - xdp->data_hard_start); - __skb_put(skb, xdp->data_end - xdp->data); - if (metasize) - skb_metadata_set(skb, metasize); - - /* record DMA address if this is the start of a chain of buffers */ - if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)) - IXGBE_CB(skb)->dma = rx_buffer->dma; - - return skb; -} static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, struct ixgbe_ring *rx_ring, - struct xdp_buff *xdp) + struct libeth_xdp_buff *xdp) { int err, result = IXGBE_XDP_PASS; struct bpf_prog *xdp_prog; @@ -2095,14 +2003,14 @@ static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, if (!xdp_prog) goto xdp_out; - prefetchw(xdp->data_hard_start); /* xdp_frame write */ + prefetchw(xdp->base.data_hard_start); /* xdp_frame write */ - act = bpf_prog_run_xdp(xdp_prog, xdp); + act = bpf_prog_run_xdp(xdp_prog, &xdp->base); switch (act) { case XDP_PASS: break; case XDP_TX: - xdpf = xdp_convert_buff_to_frame(xdp); + xdpf = xdp_convert_buff_to_frame(&xdp->base); if (unlikely(!xdpf)) goto out_failure; ring = ixgbe_determine_xdp_ring(adapter); @@ -2115,7 +2023,7 @@ static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, goto out_failure; break; case XDP_REDIRECT: - err = xdp_do_redirect(adapter->netdev, xdp, xdp_prog); + err = xdp_do_redirect(adapter->netdev, &xdp->base, xdp_prog); if (err) goto out_failure; result = IXGBE_XDP_REDIR; @@ -2129,13 +2037,13 @@ static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, fallthrough; /* handle aborts by dropping packet */ case XDP_DROP: result = IXGBE_XDP_CONSUMED; + libeth_xdp_return_buff(xdp); break; } xdp_out: return result; } - /** * ixgbe_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf * @q_vector: structure containing interrupt and ring information @@ -2150,8 +2058,8 @@ static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, * Returns amount of work completed **/ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, - struct ixgbe_ring *rx_ring, - const int budget) + struct ixgbe_ring *rx_ring, + const int budget) { unsigned int total_rx_bytes = 0, total_rx_packets = 0; struct ixgbe_adapter *adapter = q_vector->adapter; @@ -2160,16 +2068,17 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, unsigned int mss = 0; #endif /* IXGBE_FCOE */ u16 cleaned_count = ixgbe_desc_unused(rx_ring); - unsigned int xdp_xmit = 0; - struct xdp_buff xdp; + LIBETH_XDP_ONSTACK_BUFF(xdp); struct sk_buff *skb = rx_ring->skb; + unsigned int xdp_xmit = 0; int xdp_res = 0; - xdp_init_buff(&xdp, IXGBE_RXBUFFER_3K, &rx_ring->xdp_rxq); + xdp->data = NULL; + xdp->base.rxq = &rx_ring->xdp_rxq; while (likely(total_rx_packets < budget)) { union ixgbe_adv_rx_desc *rx_desc; - struct ixgbe_rx_buffer *rx_buffer; + struct libeth_fqe *rx_buffer; unsigned int size; /* return some buffers to hardware, one at a time is too slow */ @@ -2180,7 +2089,7 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, rx_desc = IXGBE_RX_DESC(rx_ring, rx_ring->next_to_clean); size = le16_to_cpu(rx_desc->wb.upper.length); - if (!size) + if (unlikely(!size)) break; /* This memory barrier is needed to keep us from reading @@ -2189,18 +2098,14 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, */ dma_rmb(); - rx_buffer = ixgbe_get_rx_buffer(rx_ring, size); + rx_buffer = &rx_ring->rx_fqes[rx_ring->next_to_clean]; /* retrieve a buffer from the ring */ if (!skb) { - unsigned int offset = ixgbe_rx_offset(rx_ring); - unsigned char *hard_start; - - hard_start = page_address(rx_buffer->page) + - rx_buffer->page_offset - offset; - xdp_prepare_buff(&xdp, hard_start, offset, size, true); - xdp_buff_clear_frags_flag(&xdp); - xdp_res = ixgbe_run_xdp(adapter, rx_ring, &xdp); + libeth_xdp_process_buff(xdp, rx_buffer, size); + xdp_res = ixgbe_run_xdp(adapter, rx_ring, xdp); + } else { + libeth_rx_sync_for_cpu(rx_buffer, size); } if (xdp_res) { @@ -2210,22 +2115,19 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, total_rx_packets++; total_rx_bytes += size; } else if (skb) { - ixgbe_add_rx_frag(rx_ring, rx_buffer, skb, size); + ixgbe_add_rx_frag(rx_buffer, skb, size); } else { - skb = ixgbe_build_skb(rx_ring, rx_buffer, - &xdp, rx_desc); + skb = xdp_build_skb_from_buff(&xdp->base); } /* exit if we failed to retrieve a buffer */ - if (!xdp_res && !skb) { + if (unlikely(!xdp_res && !skb)) { + libeth_xdp_return_buff_slow(xdp); rx_ring->rx_stats.alloc_rx_buff_failed++; cleaned_count++; break; } - ixgbe_put_rx_buffer(rx_ring, rx_buffer); - if (xdp_res == IXGBE_XDP_CONSUMED) - __free_page(rx_buffer->page); cleaned_count++; /* place incomplete frames back on ring for completion */ @@ -2233,8 +2135,10 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, continue; /* verify the packet layout is correct */ - if (xdp_res || ixgbe_cleanup_headers(rx_ring, rx_desc, skb)) + if (xdp_res || unlikely(ixgbe_cleanup_headers(rx_ring, rx_desc, skb))) { + skb = NULL; continue; + } /* probably a little skewed due to removing CRC */ total_rx_bytes += skb->len; @@ -2262,12 +2166,14 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, } if (!ddp_bytes) { dev_kfree_skb_any(skb); + skb = NULL; continue; } } #endif /* IXGBE_FCOE */ ixgbe_rx_skb(q_vector, skb); + skb = NULL; /* update budget accounting */ total_rx_packets++; @@ -2329,7 +2235,7 @@ static void ixgbe_configure_msix(struct ixgbe_adapter *adapter) switch (adapter->hw.mac.type) { case ixgbe_mac_82598EB: ixgbe_set_ivar(adapter, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX, - v_idx); + v_idx); break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: @@ -2370,7 +2276,7 @@ static void ixgbe_configure_msix(struct ixgbe_adapter *adapter) * while increasing bulk throughput. **/ static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector, - struct ixgbe_ring_container *ring_container) + struct ixgbe_ring_container *ring_container) { unsigned int itr = IXGBE_ITR_ADAPTIVE_MIN_USECS | IXGBE_ITR_ADAPTIVE_LATENCY; @@ -2519,8 +2425,8 @@ static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector, case IXGBE_LINK_SPEED_100_FULL: default: itr += DIV_ROUND_UP(avg_wire_size, - IXGBE_ITR_ADAPTIVE_MIN_INC * 256) * - IXGBE_ITR_ADAPTIVE_MIN_INC; + IXGBE_ITR_ADAPTIVE_MIN_INC * 256) * + IXGBE_ITR_ADAPTIVE_MIN_INC; break; case IXGBE_LINK_SPEED_2_5GB_FULL: case IXGBE_LINK_SPEED_1GB_FULL: @@ -2528,8 +2434,8 @@ static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector, if (avg_wire_size > 8064) avg_wire_size = 8064; itr += DIV_ROUND_UP(avg_wire_size, - IXGBE_ITR_ADAPTIVE_MIN_INC * 64) * - IXGBE_ITR_ADAPTIVE_MIN_INC; + IXGBE_ITR_ADAPTIVE_MIN_INC * 64) * + IXGBE_ITR_ADAPTIVE_MIN_INC; break; } @@ -2631,7 +2537,7 @@ static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter) * check if we got a LSC */ if (!(eicr & IXGBE_EICR_GPI_SDP0_8259X) && - !(eicr & IXGBE_EICR_LSC)) + !(eicr & IXGBE_EICR_LSC)) return; if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) { @@ -2671,7 +2577,7 @@ static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr) struct ixgbe_hw *hw = &adapter->hw; if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) && - (eicr & IXGBE_EICR_GPI_SDP1(hw))) { + (eicr & IXGBE_EICR_GPI_SDP1(hw))) { e_crit(probe, "Fan has stopped, replace the adapter\n"); /* write to clear the interrupt */ IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1(hw)); @@ -2692,8 +2598,8 @@ static void ixgbe_check_overtemp_event(struct ixgbe_adapter *adapter, u32 eicr) * on service task */ if (((eicr & IXGBE_EICR_GPI_SDP0(hw)) || - (eicr & IXGBE_EICR_LSC)) && - (!test_bit(__IXGBE_DOWN, &adapter->state))) { + (eicr & IXGBE_EICR_LSC)) && + (!test_bit(__IXGBE_DOWN, &adapter->state))) { adapter->interrupt_event = eicr; adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT; ixgbe_service_event_schedule(adapter); @@ -2768,7 +2674,7 @@ static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr) } if (adapter->hw.mac.type == ixgbe_mac_82599EB && - (eicr & IXGBE_EICR_GPI_SDP1(hw))) { + (eicr & IXGBE_EICR_GPI_SDP1(hw))) { /* Clear the interrupt */ IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1(hw)); if (!test_bit(__IXGBE_DOWN, &adapter->state)) { @@ -2800,7 +2706,7 @@ static void ixgbe_check_lsc(struct ixgbe_adapter *adapter) * Check if external PHY FW load failed and print an error message if it did. */ static void ixgbe_check_phy_fw_load(struct ixgbe_adapter *adapter, - u8 link_cfg_err) + u8 link_cfg_err) { if (!(link_cfg_err & IXGBE_ACI_LINK_EXTERNAL_PHY_LOAD_FAILURE)) { adapter->flags2 &= ~IXGBE_FLAG2_PHY_FW_LOAD_FAILED; @@ -2825,11 +2731,11 @@ static void ixgbe_check_phy_fw_load(struct ixgbe_adapter *adapter, * and print error messages if module power level is not supported. */ static void ixgbe_check_module_power(struct ixgbe_adapter *adapter, - u8 link_cfg_err) + u8 link_cfg_err) { /* If module power level is supported, clear the flag. */ if (!(link_cfg_err & (IXGBE_ACI_LINK_INVAL_MAX_POWER_LIMIT | - IXGBE_ACI_LINK_MODULE_POWER_UNSUPPORTED))) { + IXGBE_ACI_LINK_MODULE_POWER_UNSUPPORTED))) { adapter->flags2 &= ~IXGBE_FLAG2_MOD_POWER_UNSUPPORTED; return; } @@ -2858,7 +2764,7 @@ static void ixgbe_check_module_power(struct ixgbe_adapter *adapter, * link_cfg_err parameter in the link info structure. */ static void ixgbe_check_link_cfg_err(struct ixgbe_adapter *adapter, - u8 link_cfg_err) + u8 link_cfg_err) { ixgbe_check_module_power(adapter, link_cfg_err); ixgbe_check_phy_fw_load(adapter, link_cfg_err); @@ -2897,12 +2803,12 @@ ixgbe_process_link_status_event(struct ixgbe_adapter *adapter, bool link_up, /* Turn off PHY if media was removed. */ if (!(adapter->flags2 & IXGBE_FLAG2_NO_MEDIA) && - !(hw->link.link_info.link_info & IXGBE_ACI_MEDIA_AVAILABLE)) + !(hw->link.link_info.link_info & IXGBE_ACI_MEDIA_AVAILABLE)) adapter->flags2 |= IXGBE_FLAG2_NO_MEDIA; if (link_up == adapter->link_up && - link_up == netif_carrier_ok(adapter->netdev) && - link_speed == adapter->link_speed) + link_up == netif_carrier_ok(adapter->netdev) && + link_speed == adapter->link_speed) return 0; adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; @@ -2924,7 +2830,7 @@ ixgbe_process_link_status_event(struct ixgbe_adapter *adapter, bool link_up, */ static void ixgbe_handle_link_status_event(struct ixgbe_adapter *adapter, - struct ixgbe_aci_event *e) + struct ixgbe_aci_event *e) { struct ixgbe_aci_cmd_get_link_status_data *link_data; u16 link_speed; @@ -2949,8 +2855,8 @@ ixgbe_handle_link_status_event(struct ixgbe_adapter *adapter, static void ixgbe_schedule_fw_event(struct ixgbe_adapter *adapter) { if (!test_bit(__IXGBE_DOWN, &adapter->state) && - !test_bit(__IXGBE_REMOVING, &adapter->state) && - !test_bit(__IXGBE_RESETTING, &adapter->state)) { + !test_bit(__IXGBE_REMOVING, &adapter->state) && + !test_bit(__IXGBE_RESETTING, &adapter->state)) { adapter->flags2 |= IXGBE_FLAG2_FW_ASYNC_EVENT; ixgbe_service_event_schedule(adapter); } @@ -3003,7 +2909,7 @@ static void ixgbe_handle_fw_event(struct ixgbe_adapter *adapter) break; case libie_aqc_opc_fw_logs_event: libie_get_fwlog_data(&hw->fwlog, event.msg_buf, - le16_to_cpu(event.desc.datalen)); + le16_to_cpu(event.desc.datalen)); break; default: e_warn(hw, "unknown FW async event captured\n"); @@ -3049,7 +2955,7 @@ static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter, * @flush: flush register write **/ static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues, - bool flush) + bool flush) { struct ixgbe_hw *hw = &adapter->hw; u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); @@ -3087,8 +2993,8 @@ static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues, fallthrough; case ixgbe_mac_x550em_a: if (adapter->hw.device_id == IXGBE_DEV_ID_X550EM_X_SFP || - adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP || - adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP_N) + adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP || + adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP_N) mask |= IXGBE_EIMS_GPI_SDP0(&adapter->hw); if (adapter->hw.phy.type == ixgbe_phy_x550em_ext_t) mask |= IXGBE_EICR_GPI_SDP0_X540; @@ -3100,7 +3006,7 @@ static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues, } if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) && - !(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT)) + !(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT)) mask |= IXGBE_EIMS_FLOW_DIR; IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask); @@ -3152,7 +3058,7 @@ static irqreturn_t ixgbe_msix_other(int irq, void *data) case ixgbe_mac_x550em_a: case ixgbe_mac_e610: if (hw->phy.type == ixgbe_phy_x550em_ext_t && - (eicr & IXGBE_EICR_GPI_SDP0_X540)) { + (eicr & IXGBE_EICR_GPI_SDP0_X540)) { adapter->flags2 |= IXGBE_FLAG2_PHY_INTERRUPT; ixgbe_service_event_schedule(adapter); IXGBE_WRITE_REG(hw, IXGBE_EICR, @@ -3171,7 +3077,7 @@ static irqreturn_t ixgbe_msix_other(int irq, void *data) for (i = 0; i < adapter->num_tx_queues; i++) { struct ixgbe_ring *ring = adapter->tx_ring[i]; if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE, - &ring->state)) + &ring->state)) reinit_count++; } if (reinit_count) { @@ -3255,9 +3161,9 @@ int ixgbe_poll(struct napi_struct *napi, int budget) ixgbe_for_each_ring(ring, q_vector->rx) { int cleaned = ring->xsk_pool ? - ixgbe_clean_rx_irq_zc(q_vector, ring, - per_ring_budget) : - ixgbe_clean_rx_irq(q_vector, ring, + ixgbe_clean_rx_irq_zc(q_vector, ring, + per_ring_budget) : + ixgbe_clean_rx_irq(q_vector, ring, per_ring_budget); work_done += cleaned; @@ -3316,7 +3222,7 @@ static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter) q_vector->name, q_vector); if (err) { e_err(probe, "request_irq failed for MSIX interrupt " - "Error: %d\n", err); + "Error: %d\n", err); goto free_queue_irqs; } /* If Flow Director is enabled, set interrupt affinity */ @@ -3546,7 +3452,7 @@ static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter) * Configure the Tx descriptor ring after a reset. **/ void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter, - struct ixgbe_ring *ring) + struct ixgbe_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; u64 tdba = ring->dma; @@ -3608,22 +3514,22 @@ void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter, if (q_vector) netif_set_xps_queue(ring->netdev, - &q_vector->affinity_mask, - ring->queue_index); + &q_vector->affinity_mask, + ring->queue_index); } clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state); /* reinitialize tx_buffer_info */ memset(ring->tx_buffer_info, 0, - sizeof(struct ixgbe_tx_buffer) * ring->count); + sizeof(struct ixgbe_tx_buffer) * ring->count); /* enable queue */ IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl); /* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */ if (hw->mac.type == ixgbe_mac_82598EB && - !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) + !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) return; /* poll to verify queue is enabled */ @@ -3768,7 +3674,7 @@ static void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter) * and performance reasons. */ if (adapter->num_vfs || (adapter->num_rx_queues > 1 && - !(adapter->hw.fc.current_mode & ixgbe_fc_tx_pause) && !pfc_en)) { + !(adapter->hw.fc.current_mode & ixgbe_fc_tx_pause) && !pfc_en)) { for (i = 0; i < adapter->num_rx_queues; i++) ixgbe_enable_rx_drop(adapter, adapter->rx_ring[i]); } else { @@ -3783,11 +3689,11 @@ static void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter) #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter, - struct ixgbe_ring *rx_ring) + struct ixgbe_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; u32 srrctl; - u8 reg_idx = rx_ring->reg_idx; + u8 reg_idx = ring->reg_idx; if (hw->mac.type == ixgbe_mac_82598EB) { u16 mask = adapter->ring_feature[RING_F_RSS].mask; @@ -3802,25 +3708,12 @@ static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter, /* configure header buffer length, needed for RSC */ srrctl = IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT; - /* configure the packet buffer length */ - if (rx_ring->xsk_pool) { - u32 xsk_buf_len = xsk_pool_get_rx_frame_size(rx_ring->xsk_pool); - - /* If the MAC support setting RXDCTL.RLPML, the - * SRRCTL[n].BSIZEPKT is set to PAGE_SIZE and - * RXDCTL.RLPML is set to the actual UMEM buffer - * size. If not, then we are stuck with a 1k buffer - * size resolution. In this case frames larger than - * the UMEM buffer size viewed in a 1k resolution will - * be dropped. - */ - if (hw->mac.type != ixgbe_mac_82599EB) - srrctl |= PAGE_SIZE >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; - else - srrctl |= xsk_buf_len >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; - } else { - srrctl |= IXGBE_RXBUFFER_3K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; - } + if (ring->xsk_pool) + srrctl |= DIV_ROUND_UP(xsk_pool_get_rx_frame_size(ring->xsk_pool), + IXGBE_SRRCTL_BSIZEPKT_STEP); + else + srrctl |= DIV_ROUND_UP(ring->rx_buf_len, + IXGBE_SRRCTL_BSIZEPKT_STEP); /* configure descriptor type */ srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; @@ -4051,9 +3944,9 @@ static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter) /* Perform hash on these packet types */ rss_field |= IXGBE_MRQC_RSS_FIELD_IPV4 | - IXGBE_MRQC_RSS_FIELD_IPV4_TCP | - IXGBE_MRQC_RSS_FIELD_IPV6 | - IXGBE_MRQC_RSS_FIELD_IPV6_TCP; + IXGBE_MRQC_RSS_FIELD_IPV4_TCP | + IXGBE_MRQC_RSS_FIELD_IPV6 | + IXGBE_MRQC_RSS_FIELD_IPV6_TCP; if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV4_UDP) rss_field |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; @@ -4061,7 +3954,7 @@ static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter) rss_field |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; if ((hw->mac.type >= ixgbe_mac_X550) && - (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) { + (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) { u16 pool = adapter->num_rx_pools; /* Enable VF RSS mode */ @@ -4112,7 +4005,7 @@ static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter, #define IXGBE_MAX_RX_DESC_POLL 10 static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter, - struct ixgbe_ring *ring) + struct ixgbe_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; int wait_loop = IXGBE_MAX_RX_DESC_POLL; @@ -4123,7 +4016,7 @@ static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter, return; /* RXDCTL.EN will return 0 on 82598 if link is down, so skip it */ if (hw->mac.type == ixgbe_mac_82598EB && - !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) + !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) return; do { @@ -4133,12 +4026,12 @@ static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter, if (!wait_loop) { e_err(drv, "RXDCTL.ENABLE on Rx queue %d not set within " - "the polling period\n", reg_idx); + "the polling period\n", reg_idx); } } void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, - struct ixgbe_ring *ring) + struct ixgbe_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; union ixgbe_adv_rx_desc *rx_desc; @@ -4154,8 +4047,7 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, NULL)); xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq); } else { - WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, - MEM_TYPE_PAGE_SHARED, NULL)); + xdp_rxq_info_attach_page_pool(&ring->xdp_rxq, ring->pp); } /* disable queue to avoid use of these values while updating state */ @@ -4177,7 +4069,6 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, IXGBE_WRITE_REG(hw, IXGBE_RDT(reg_idx), 0); ring->tail = adapter->io_addr + IXGBE_RDT(reg_idx); - ixgbe_configure_srrctl(adapter, ring); ixgbe_configure_rscctl(adapter, ring); if (hw->mac.type == ixgbe_mac_82598EB) { @@ -4194,21 +4085,19 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, /* RXDCTL.RLPML does not work on 82599 */ } else if (hw->mac.type != ixgbe_mac_82599EB) { rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | - IXGBE_RXDCTL_RLPML_EN); + IXGBE_RXDCTL_RLPML_EN); } #endif if (ring->xsk_pool && hw->mac.type != ixgbe_mac_82599EB) { - u32 xsk_buf_len = xsk_pool_get_rx_frame_size(ring->xsk_pool); + u32 pkt_len = + READ_ONCE(adapter->netdev->mtu) + LIBETH_RX_LL_LEN; - rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | - IXGBE_RXDCTL_RLPML_EN); - rxdctl |= xsk_buf_len | IXGBE_RXDCTL_RLPML_EN; - - ring->rx_buf_len = xsk_buf_len; + rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | IXGBE_RXDCTL_RLPML_EN); + if (pkt_len <= IXGBE_RXDCTL_RLPMLMASK) + rxdctl |= pkt_len | IXGBE_RXDCTL_RLPML_EN; } - /* initialize rx_buffer_info */ - memset(ring->rx_buffer_info, 0, - sizeof(struct ixgbe_rx_buffer) * ring->count); + + ixgbe_configure_srrctl(adapter, ring); /* initialize Rx descriptor 0 */ rx_desc = IXGBE_RX_DESC(ring, 0); rx_desc->wb.upper.length = 0; @@ -4232,10 +4121,10 @@ static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter) /* PSRTYPE must be initialized in non 82598 adapters */ u32 psrtype = IXGBE_PSRTYPE_TCPHDR | - IXGBE_PSRTYPE_UDPHDR | - IXGBE_PSRTYPE_IPV4HDR | - IXGBE_PSRTYPE_L2HDR | - IXGBE_PSRTYPE_IPV6HDR; + IXGBE_PSRTYPE_UDPHDR | + IXGBE_PSRTYPE_IPV4HDR | + IXGBE_PSRTYPE_L2HDR | + IXGBE_PSRTYPE_IPV6HDR; if (hw->mac.type == ixgbe_mac_82598EB) return; @@ -4320,40 +4209,6 @@ static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter) } } -static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter) -{ - struct ixgbe_hw *hw = &adapter->hw; - struct net_device *netdev = adapter->netdev; - int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; - u32 mhadd, hlreg0; - -#ifdef IXGBE_FCOE - /* adjust max frame to be able to do baby jumbo for FCoE */ - if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) && - (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE)) - max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE; -#endif /* IXGBE_FCOE */ - - /* adjust max frame to be at least the size of a standard frame */ - if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) - max_frame = (ETH_FRAME_LEN + ETH_FCS_LEN); - - mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); - if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) { - mhadd &= ~IXGBE_MHADD_MFS_MASK; - mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT; - - IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); - } - - hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); - /* set jumbo enable since MHADD.MFS is keeping size locked at - * max_frame - */ - hlreg0 |= IXGBE_HLREG0_JUMBOEN; - IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0); -} - static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; @@ -4407,8 +4262,12 @@ static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter) static void ixgbe_configure_rx(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; + struct net_device *netdev = adapter->netdev; + int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; + struct ixgbe_ring *rx_ring; int i; u32 rxctrl, rfctl; + u32 mhadd, hlreg0; /* disable receives while setting up the descriptors */ hw->mac.ops.disable_rx(hw); @@ -4429,15 +4288,38 @@ static void ixgbe_configure_rx(struct ixgbe_adapter *adapter) /* Program registers for the distribution of queues */ ixgbe_setup_mrqc(adapter); - /* set_rx_buffer_len must be called before ring initialization */ - ixgbe_set_rx_buffer_len(adapter); +#ifdef IXGBE_FCOE + /* adjust max frame to be able to do baby jumbo for FCoE */ + if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) && + (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE)) + max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE; +#endif /* IXGBE_FCOE */ + + /* adjust max frame to be at least the size of a standard frame */ + if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) + max_frame = (ETH_FRAME_LEN + ETH_FCS_LEN); + + mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); + if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) { + mhadd &= ~IXGBE_MHADD_MFS_MASK; + mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT; + + IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); + } + + /* set jumbo enable since MHADD.MFS is keeping size locked at + * max_frame + */ + hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); + hlreg0 |= IXGBE_HLREG0_JUMBOEN; + IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0); /* * Setup the HW Rx Head and Tail Descriptor Pointers and * the Base and Length of the Rx Descriptor Ring */ for (i = 0; i < adapter->num_rx_queues; i++) { - struct ixgbe_ring *rx_ring = adapter->rx_ring[i]; + rx_ring = adapter->rx_ring[i]; clear_ring_rsc_enabled(rx_ring); if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) @@ -4708,7 +4590,7 @@ static void ixgbe_vlan_promisc_disable(struct ixgbe_adapter *adapter) IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); if (!(adapter->flags & IXGBE_FLAG_VMDQ_ENABLED) || - hw->mac.type == ixgbe_mac_82598EB) + hw->mac.type == ixgbe_mac_82598EB) return; /* We are not in VLAN promisc, nothing to do */ @@ -4773,9 +4655,9 @@ void ixgbe_full_sync_mac_table(struct ixgbe_adapter *adapter) if (mac_table->state & IXGBE_MAC_STATE_IN_USE) hw->mac.ops.set_rar(hw, i, - mac_table->addr, - mac_table->pool, - IXGBE_RAH_AV); + mac_table->addr, + mac_table->pool, + IXGBE_RAH_AV); else hw->mac.ops.clear_rar(hw, i); } @@ -4796,9 +4678,9 @@ static void ixgbe_sync_mac_table(struct ixgbe_adapter *adapter) if (mac_table->state & IXGBE_MAC_STATE_IN_USE) hw->mac.ops.set_rar(hw, i, - mac_table->addr, - mac_table->pool, - IXGBE_RAH_AV); + mac_table->addr, + mac_table->pool, + IXGBE_RAH_AV); else hw->mac.ops.clear_rar(hw, i); } @@ -4853,7 +4735,7 @@ static void ixgbe_mac_set_default_filter(struct ixgbe_adapter *adapter) mac_table->state = IXGBE_MAC_STATE_DEFAULT | IXGBE_MAC_STATE_IN_USE; hw->mac.ops.set_rar(hw, 0, mac_table->addr, mac_table->pool, - IXGBE_RAH_AV); + IXGBE_RAH_AV); } int ixgbe_add_mac_filter(struct ixgbe_adapter *adapter, @@ -4874,7 +4756,7 @@ int ixgbe_add_mac_filter(struct ixgbe_adapter *adapter, mac_table->pool = pool; mac_table->state |= IXGBE_MAC_STATE_MODIFIED | - IXGBE_MAC_STATE_IN_USE; + IXGBE_MAC_STATE_IN_USE; ixgbe_sync_mac_table(adapter); @@ -5165,7 +5047,7 @@ static int ixgbe_hpbthresh(struct ixgbe_adapter *adapter, int pb) #ifdef IXGBE_FCOE /* FCoE traffic class uses FCOE jumbo frames */ if (dev->fcoe_mtu && tc < IXGBE_FCOE_JUMBO_FRAME_SIZE && - (pb == ixgbe_fcoe_get_tc(adapter))) + (pb == ixgbe_fcoe_get_tc(adapter))) tc = IXGBE_FCOE_JUMBO_FRAME_SIZE; #endif @@ -5199,8 +5081,8 @@ static int ixgbe_hpbthresh(struct ixgbe_adapter *adapter, int pb) */ if (marker < 0) { e_warn(drv, "Packet Buffer(%i) can not provide enough" - "headroom to support flow control." - "Decrease MTU or number of traffic classes\n", pb); + "headroom to support flow control." + "Decrease MTU or number of traffic classes\n", pb); marker = tc + 1; } @@ -5226,7 +5108,7 @@ static int ixgbe_lpbthresh(struct ixgbe_adapter *adapter, int pb) #ifdef IXGBE_FCOE /* FCoE traffic class uses FCOE jumbo frames */ if (dev->fcoe_mtu && tc < IXGBE_FCOE_JUMBO_FRAME_SIZE && - (pb == netdev_get_prio_tc_map(dev, adapter->fcoe.up))) + (pb == netdev_get_prio_tc_map(dev, adapter->fcoe.up))) tc = IXGBE_FCOE_JUMBO_FRAME_SIZE; #endif @@ -5280,7 +5162,7 @@ static void ixgbe_configure_pb(struct ixgbe_adapter *adapter) u8 tc = adapter->hw_tcs; if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE || - adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) + adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) hdrm = 32 << adapter->fdir_pballoc; else hdrm = 0; @@ -5311,13 +5193,13 @@ static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter) if (!vf && (ring >= adapter->num_rx_queues)) { e_err(drv, "FDIR restore failed without VF, ring: %u\n", - ring); + ring); continue; } else if (vf && ((vf > adapter->num_vfs) || - ring >= adapter->num_rx_queues_per_pool)) { + ring >= adapter->num_rx_queues_per_pool)) { e_err(drv, "FDIR restore failed with VF, vf: %hhu, ring: %u\n", - vf, ring); + vf, ring); continue; } @@ -5342,9 +5224,6 @@ static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter) **/ static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) { - u16 i = rx_ring->next_to_clean; - struct ixgbe_rx_buffer *rx_buffer = &rx_ring->rx_buffer_info[i]; - if (rx_ring->xsk_pool) { ixgbe_xsk_clean_rx_ring(rx_ring); goto skip_free; @@ -5356,29 +5235,12 @@ static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) rx_ring->skb = NULL; } - /* Free all the Rx ring pages */ - while (i != rx_ring->next_to_use) { - /* Invalidate cache lines that may have been written to by - * device so that we avoid corrupting memory. - */ - dma_sync_single_range_for_cpu(rx_ring->dev, - rx_buffer->dma, - rx_buffer->page_offset, - IXGBE_RXBUFFER_3K, - DMA_FROM_DEVICE); - - /* free resources associated with mapping */ - dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, - PAGE_SIZE, DMA_FROM_DEVICE, - IXGBE_RX_DMA_ATTR); - __free_page(rx_buffer->page); + for (u32 i = rx_ring->next_to_clean; i != rx_ring->next_to_use; ) { + const struct libeth_fqe *rx_fqe = &rx_ring->rx_fqes[i]; - i++; - rx_buffer++; - if (i == rx_ring->count) { + libeth_rx_recycle_slow(rx_fqe->netmem); + if (unlikely(++i == rx_ring->count)) i = 0; - rx_buffer = rx_ring->rx_buffer_info; - } } skip_free: @@ -5387,7 +5249,7 @@ static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) } static int ixgbe_fwd_ring_up(struct ixgbe_adapter *adapter, - struct ixgbe_fwd_adapter *accel) + struct ixgbe_fwd_adapter *accel) { u16 rss_i = adapter->ring_feature[RING_F_RSS].indices; int num_tc = netdev_get_num_tc(adapter->netdev); @@ -5405,7 +5267,7 @@ static int ixgbe_fwd_ring_up(struct ixgbe_adapter *adapter, /* record configuration for macvlan interface in vdev */ for (i = 0; i < num_tc; i++) netdev_bind_sb_channel_queue(adapter->netdev, vdev, - i, rss_i, baseq + (rss_i * i)); + i, rss_i, baseq + (rss_i * i)); for (i = 0; i < adapter->num_rx_queues_per_pool; i++) adapter->rx_ring[baseq + i]->netdev = vdev; @@ -5442,7 +5304,7 @@ static int ixgbe_fwd_ring_up(struct ixgbe_adapter *adapter, } static int ixgbe_macvlan_up(struct net_device *vdev, - struct netdev_nested_priv *priv) + struct netdev_nested_priv *priv) { struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)priv->data; struct ixgbe_fwd_adapter *accel; @@ -5466,7 +5328,7 @@ static void ixgbe_configure_dfwd(struct ixgbe_adapter *adapter) }; netdev_walk_all_upper_dev_rcu(adapter->netdev, - ixgbe_macvlan_up, &priv); + ixgbe_macvlan_up, &priv); } static void ixgbe_configure(struct ixgbe_adapter *adapter) @@ -5501,7 +5363,7 @@ static void ixgbe_configure(struct ixgbe_adapter *adapter) adapter->fdir_pballoc); } else if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) { ixgbe_init_fdir_perfect_82599(&adapter->hw, - adapter->fdir_pballoc); + adapter->fdir_pballoc); ixgbe_fdir_filter_restore(adapter); } @@ -5602,7 +5464,7 @@ static void ixgbe_sfp_link_config(struct ixgbe_adapter *adapter) static int ixgbe_non_sfp_link_config(struct ixgbe_hw *hw) { struct ixgbe_adapter *adapter = container_of(hw, struct ixgbe_adapter, - hw); + hw); u16 mask = ~((u16)(IXGBE_ACI_LINK_EVENT_UPDOWN | IXGBE_ACI_LINK_EVENT_MEDIA_NA | IXGBE_ACI_LINK_EVENT_MODULE_QUAL_FAIL | @@ -5726,7 +5588,7 @@ static void ixgbe_setup_gpie(struct ixgbe_adapter *adapter) if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) { gpie = IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT | - IXGBE_GPIE_OCD; + IXGBE_GPIE_OCD; gpie |= IXGBE_GPIE_EIAME; /* * use EIAM to auto-mask when MSI-X interrupt is asserted @@ -5960,7 +5822,7 @@ void ixgbe_disable_rx(struct ixgbe_adapter *adapter) /* RXDCTL.EN may not change on 82598 if link is down, so skip it */ if (hw->mac.type == ixgbe_mac_82598EB && - !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) + !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) return; /* Determine our minimum delay interval. We will increase this value @@ -6001,7 +5863,7 @@ void ixgbe_disable_rx(struct ixgbe_adapter *adapter) } e_err(drv, - "RXDCTL.ENABLE for one or more queues not cleared within the polling period\n"); + "RXDCTL.ENABLE for one or more queues not cleared within the polling period\n"); } void ixgbe_disable_tx(struct ixgbe_adapter *adapter) @@ -6082,7 +5944,7 @@ void ixgbe_disable_tx(struct ixgbe_adapter *adapter) } e_err(drv, - "TXDCTL.ENABLE for one or more queues not cleared within the polling period\n"); + "TXDCTL.ENABLE for one or more queues not cleared within the polling period\n"); dma_engine_disable: /* Disable the Tx DMA engine on 82599 and later MAC */ @@ -6116,7 +5978,7 @@ void ixgbe_reset(struct ixgbe_adapter *adapter) /* clear all SFP and link config related flags while holding SFP_INIT */ adapter->flags2 &= ~(IXGBE_FLAG2_SEARCH_FOR_SFP | - IXGBE_FLAG2_SFP_NEEDS_RESET); + IXGBE_FLAG2_SFP_NEEDS_RESET); adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG; err = hw->mac.ops.init_hw(hw); @@ -6212,9 +6074,9 @@ static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring) /* unmap any remaining paged data */ if (dma_unmap_len(tx_buffer, len)) dma_unmap_page(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); } /* move us one more past the eop_desc for start of next pkt */ @@ -6428,7 +6290,7 @@ static void ixgbe_init_dcb(struct ixgbe_adapter *adapter) if (adapter->flags & IXGBE_FLAG_DCB_CAPABLE) adapter->dcbx_cap = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_CEE; memcpy(&adapter->temp_dcb_cfg, &adapter->dcb_cfg, - sizeof(adapter->temp_dcb_cfg)); + sizeof(adapter->temp_dcb_cfg)); } #endif @@ -6500,8 +6362,8 @@ static int ixgbe_sw_init(struct ixgbe_adapter *adapter, adapter->jump_tables[i] = NULL; adapter->mac_table = kcalloc(hw->mac.num_rar_entries, - sizeof(struct ixgbe_mac_addr), - GFP_KERNEL); + sizeof(struct ixgbe_mac_addr), + GFP_KERNEL); if (!adapter->mac_table) return -ENOMEM; @@ -6737,13 +6599,6 @@ static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter) return err; } -static int ixgbe_rx_napi_id(struct ixgbe_ring *rx_ring) -{ - struct ixgbe_q_vector *q_vector = rx_ring->q_vector; - - return q_vector ? q_vector->napi.napi_id : 0; -} - /** * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors) * @adapter: pointer to ixgbe_adapter @@ -6752,56 +6607,77 @@ static int ixgbe_rx_napi_id(struct ixgbe_ring *rx_ring) * Returns 0 on success, negative on failure **/ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, - struct ixgbe_ring *rx_ring) + struct ixgbe_ring *rx_ring) { - struct device *dev = rx_ring->dev; + struct libeth_fq fq = { + .count = rx_ring->count, + .nid = NUMA_NO_NODE, + .type = LIBETH_FQE_MTU, + .xdp = !!rx_ring->xdp_prog, + .buf_len = IXGBE_RX_PAGE_LEN(rx_ring->xdp_prog ? + LIBETH_XDP_HEADROOM : + LIBETH_SKB_HEADROOM), + }; + struct device *dev = &adapter->pdev->dev; int orig_node = dev_to_node(dev); int ring_node = NUMA_NO_NODE; - int size; - - size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count; + int ret; - if (rx_ring->q_vector) + if (rx_ring->q_vector) { + fq.nid = rx_ring->q_vector->numa_node; ring_node = rx_ring->q_vector->numa_node; + } - rx_ring->rx_buffer_info = vmalloc_node(size, ring_node); - if (!rx_ring->rx_buffer_info) - rx_ring->rx_buffer_info = vmalloc(size); - if (!rx_ring->rx_buffer_info) - goto err; + ret = libeth_rx_fq_create(&fq, &rx_ring->q_vector->napi); + if (ret) + return ret; + + rx_ring->pp = fq.pp; + rx_ring->rx_fqes = fq.fqes; + rx_ring->truesize = fq.truesize; + rx_ring->rx_buf_len = fq.buf_len; /* Round up to nearest 4K */ rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); rx_ring->size = ALIGN(rx_ring->size, 4096); set_dev_node(dev, ring_node); - rx_ring->desc = dma_alloc_coherent(dev, - rx_ring->size, - &rx_ring->dma, - GFP_KERNEL); + rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, + &rx_ring->dma, GFP_KERNEL); set_dev_node(dev, orig_node); if (!rx_ring->desc) rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, - &rx_ring->dma, GFP_KERNEL); - if (!rx_ring->desc) - goto err; + &rx_ring->dma, + GFP_KERNEL); + if (!rx_ring->desc) { + ret = -ENOMEM; + goto err_destroy_fq; + } rx_ring->next_to_clean = 0; rx_ring->next_to_use = 0; /* XDP RX-queue info */ - if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, - rx_ring->queue_index, ixgbe_rx_napi_id(rx_ring)) < 0) - goto err; + ret = __xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, + rx_ring->queue_index, 0, rx_ring->truesize); + if (ret < 0) + goto err_free_desc; - rx_ring->xdp_prog = adapter->xdp_prog; + xdp_rxq_info_attach_page_pool(&rx_ring->xdp_rxq, fq.pp); + rcu_assign_pointer(rx_ring->xdp_prog, adapter->xdp_prog); return 0; -err: - vfree(rx_ring->rx_buffer_info); - rx_ring->rx_buffer_info = NULL; + +err_free_desc: + dma_free_coherent(dev, rx_ring->size, + rx_ring->desc, rx_ring->dma); + rx_ring->desc = NULL; +err_destroy_fq: dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n"); - return -ENOMEM; + libeth_rx_fq_destroy(&fq); + rx_ring->rx_fqes = NULL; + rx_ring->pp = NULL; + return ret; } /** @@ -6888,21 +6764,32 @@ static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter) **/ void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring) { + struct libeth_fq fq = { + .fqes = rx_ring->rx_fqes, + .pp = rx_ring->pp, + }; + ixgbe_clean_rx_ring(rx_ring); - rx_ring->xdp_prog = NULL; + rcu_assign_pointer(rx_ring->xdp_prog, NULL); + xdp_rxq_info_detach_mem_model(&rx_ring->xdp_rxq); xdp_rxq_info_unreg(&rx_ring->xdp_rxq); - vfree(rx_ring->rx_buffer_info); - rx_ring->rx_buffer_info = NULL; /* if not set, then don't free */ if (!rx_ring->desc) return; - dma_free_coherent(rx_ring->dev, rx_ring->size, - rx_ring->desc, rx_ring->dma); + dma_free_coherent(fq.pp->p.dev, rx_ring->size, rx_ring->desc, + rx_ring->dma); rx_ring->desc = NULL; + + kvfree(rx_ring->rx_xsk_buffer_info); + rx_ring->rx_xsk_buffer_info = NULL; + + libeth_rx_fq_destroy(&fq); + rx_ring->rx_fqes = NULL; + rx_ring->pp = NULL; } /** @@ -6945,7 +6832,7 @@ static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu) struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); if (ixgbe_enabled_xdp_adapter(adapter)) { - int new_frame_size = new_mtu + IXGBE_PKT_HDR_PAD; + int new_frame_size = new_mtu + LIBETH_RX_LL_LEN; if (new_frame_size > ixgbe_max_xdp_frame_size(adapter)) { e_warn(probe, "Requested MTU size is not supported with XDP\n"); @@ -6959,8 +6846,8 @@ static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu) * warning that legacy VFs will be disabled. */ if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && - (adapter->hw.mac.type == ixgbe_mac_82599EB) && - (new_mtu > ETH_DATA_LEN)) + (adapter->hw.mac.type == ixgbe_mac_82599EB) && + (new_mtu > ETH_DATA_LEN)) e_warn(probe, "Setting MTU > 1500 will disable legacy VFs\n"); netdev_dbg(netdev, "changing MTU from %d to %d\n", @@ -7258,7 +7145,7 @@ void ixgbe_update_stats(struct ixgbe_adapter *adapter) u64 bytes = 0, packets = 0, hw_csum_rx_error = 0; if (test_bit(__IXGBE_DOWN, &adapter->state) || - test_bit(__IXGBE_RESETTING, &adapter->state)) + test_bit(__IXGBE_RESETTING, &adapter->state)) return; if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) { @@ -7359,11 +7246,11 @@ void ixgbe_update_stats(struct ixgbe_adapter *adapter) hwstats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); hwstats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); if (hw->mac.type == ixgbe_mac_82599EB || - hw->mac.type == ixgbe_mac_X540 || - hw->mac.type == ixgbe_mac_X550 || - hw->mac.type == ixgbe_mac_X550EM_x || - hw->mac.type == ixgbe_mac_x550em_a || - hw->mac.type == ixgbe_mac_e610) { + hw->mac.type == ixgbe_mac_X540 || + hw->mac.type == ixgbe_mac_X550 || + hw->mac.type == ixgbe_mac_X550EM_x || + hw->mac.type == ixgbe_mac_x550em_a || + hw->mac.type == ixgbe_mac_e610) { hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i)); IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)); /* to clear */ hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i)); @@ -7399,7 +7286,7 @@ void ixgbe_update_stats(struct ixgbe_adapter *adapter) case ixgbe_mac_82599EB: for (i = 0; i < 16; i++) adapter->hw_rx_no_dma_resources += - IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); + IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL); IXGBE_READ_REG(hw, IXGBE_GORCH); /* to clear */ hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL); @@ -7546,7 +7433,7 @@ static void ixgbe_fdir_reinit_subtask(struct ixgbe_adapter *adapter) IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR); } else { e_err(probe, "failed to finish FDIR re-initialization, " - "ignored adding FDIR ATR filters\n"); + "ignored adding FDIR ATR filters\n"); } } @@ -7567,8 +7454,8 @@ static void ixgbe_check_hang_subtask(struct ixgbe_adapter *adapter) /* If we're down, removing or resetting, just bail */ if (test_bit(__IXGBE_DOWN, &adapter->state) || - test_bit(__IXGBE_REMOVING, &adapter->state) || - test_bit(__IXGBE_RESETTING, &adapter->state)) + test_bit(__IXGBE_REMOVING, &adapter->state) || + test_bit(__IXGBE_RESETTING, &adapter->state)) return; /* Force detection of hung controller */ @@ -7628,7 +7515,7 @@ static void ixgbe_watchdog_update_link(struct ixgbe_adapter *adapter) } if (link_up || - time_after(jiffies, (adapter->link_check_timeout + + time_after(jiffies, (adapter->link_check_timeout + IXGBE_TRY_LINK_TIMEOUT))) { adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE; IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMC_LSC); @@ -7644,9 +7531,9 @@ static void ixgbe_update_default_up(struct ixgbe_adapter *adapter) #ifdef CONFIG_IXGBE_DCB struct net_device *netdev = adapter->netdev; struct dcb_app app = { - .selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE, - .protocol = 0, - }; + .selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE, + .protocol = 0, + }; u8 up = 0; if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_IEEE) @@ -7730,9 +7617,9 @@ static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter) break; } e_info(drv, "NIC Link is Up %s, Flow Control: %s\n", speed_str, - ((flow_rx && flow_tx) ? "RX/TX" : - (flow_rx ? "RX" : - (flow_tx ? "TX" : "None")))); + ((flow_rx && flow_tx) ? "RX/TX" : + (flow_rx ? "RX" : + (flow_tx ? "TX" : "None")))); netif_carrier_on(netdev); ixgbe_check_vf_rate_limit(adapter); @@ -7835,7 +7722,7 @@ static void ixgbe_watchdog_flush_tx(struct ixgbe_adapter *adapter) { if (!netif_carrier_ok(adapter->netdev)) { if (ixgbe_ring_tx_pending(adapter) || - ixgbe_vf_tx_pending(adapter)) { + ixgbe_vf_tx_pending(adapter)) { /* We've lost link, so the controller stops DMA, * but we've got queued Tx work that's never going * to get done, so reset controller to flush Tx. @@ -7853,18 +7740,18 @@ static void ixgbe_bad_vf_abort(struct ixgbe_adapter *adapter, u32 vf) struct ixgbe_hw *hw = &adapter->hw; if (adapter->hw.mac.type == ixgbe_mac_82599EB && - adapter->flags2 & IXGBE_FLAG2_AUTO_DISABLE_VF) { + adapter->flags2 & IXGBE_FLAG2_AUTO_DISABLE_VF) { adapter->vfinfo[vf].primary_abort_count++; if (adapter->vfinfo[vf].primary_abort_count == - IXGBE_PRIMARY_ABORT_LIMIT) { + IXGBE_PRIMARY_ABORT_LIMIT) { ixgbe_set_vf_link_state(adapter, vf, IFLA_VF_LINK_STATE_DISABLE); adapter->vfinfo[vf].primary_abort_count = 0; e_info(drv, - "Malicious Driver Detection event detected on PF %d VF %d MAC: %pM mdd-disable-vf=on", - hw->bus.func, vf, - adapter->vfinfo[vf].vf_mac_addresses); + "Malicious Driver Detection event detected on PF %d VF %d MAC: %pM mdd-disable-vf=on", + hw->bus.func, vf, + adapter->vfinfo[vf].vf_mac_addresses); } } } @@ -7900,7 +7787,7 @@ static void ixgbe_check_for_bad_vf(struct ixgbe_adapter *adapter) continue; pci_read_config_word(vfdev, PCI_STATUS, &status_reg); if (status_reg != IXGBE_FAILED_READ_CFG_WORD && - status_reg & PCI_STATUS_REC_MASTER_ABORT) { + status_reg & PCI_STATUS_REC_MASTER_ABORT) { ixgbe_bad_vf_abort(adapter, vf); pcie_flr(vfdev); } @@ -7913,7 +7800,7 @@ static void ixgbe_spoof_check(struct ixgbe_adapter *adapter) /* Do not perform spoof check for 82598 or if not in IOV mode */ if (adapter->hw.mac.type == ixgbe_mac_82598EB || - adapter->num_vfs == 0) + adapter->num_vfs == 0) return; ssvpc = IXGBE_READ_REG(&adapter->hw, IXGBE_SSVPC); @@ -7947,8 +7834,8 @@ static void ixgbe_watchdog_subtask(struct ixgbe_adapter *adapter) { /* if interface is down, removing or resetting, do nothing */ if (test_bit(__IXGBE_DOWN, &adapter->state) || - test_bit(__IXGBE_REMOVING, &adapter->state) || - test_bit(__IXGBE_RESETTING, &adapter->state)) + test_bit(__IXGBE_REMOVING, &adapter->state) || + test_bit(__IXGBE_RESETTING, &adapter->state)) return; ixgbe_watchdog_update_link(adapter); @@ -7976,11 +7863,11 @@ static void ixgbe_sfp_detection_subtask(struct ixgbe_adapter *adapter) /* not searching for SFP so there is nothing to do here */ if (!(adapter->flags2 & IXGBE_FLAG2_SEARCH_FOR_SFP) && - !(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET)) + !(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET)) return; if (adapter->sfp_poll_time && - time_after(adapter->sfp_poll_time, jiffies)) + time_after(adapter->sfp_poll_time, jiffies)) return; /* If not yet time to poll for SFP */ /* someone else is in init, wait until next service event */ @@ -8029,7 +7916,7 @@ static void ixgbe_sfp_detection_subtask(struct ixgbe_adapter *adapter) clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state); if (err == -EOPNOTSUPP && - adapter->netdev->reg_state == NETREG_REGISTERED) { + adapter->netdev->reg_state == NETREG_REGISTERED) { e_dev_err("failed to initialize because an unsupported " "SFP+ module type was detected.\n"); e_dev_err("Reload the driver after installing a " @@ -8065,7 +7952,7 @@ static void ixgbe_sfp_link_config_subtask(struct ixgbe_adapter *adapter) speed = IXGBE_LINK_SPEED_10GB_FULL; else speed = cap_speed & (IXGBE_LINK_SPEED_10GB_FULL | - IXGBE_LINK_SPEED_1GB_FULL); + IXGBE_LINK_SPEED_1GB_FULL); if (hw->mac.ops.setup_link) hw->mac.ops.setup_link(hw, speed, true); @@ -8123,8 +8010,8 @@ static void ixgbe_reset_subtask(struct ixgbe_adapter *adapter) rtnl_lock(); /* If we're already down, removing or resetting, just bail */ if (test_bit(__IXGBE_DOWN, &adapter->state) || - test_bit(__IXGBE_REMOVING, &adapter->state) || - test_bit(__IXGBE_RESETTING, &adapter->state)) { + test_bit(__IXGBE_REMOVING, &adapter->state) || + test_bit(__IXGBE_RESETTING, &adapter->state)) { rtnl_unlock(); return; } @@ -8182,7 +8069,7 @@ static bool ixgbe_check_fw_error(struct ixgbe_adapter *adapter) /* skip if E610's FW is reloading, warning in that case may be misleading */ if (fwsm & IXGBE_FWSM_EXT_ERR_IND_MASK || - (!(fwsm & IXGBE_FWSM_FW_VAL_BIT) && !(hw->mac.type == ixgbe_mac_e610))) + (!(fwsm & IXGBE_FWSM_FW_VAL_BIT) && !(hw->mac.type == ixgbe_mac_e610))) e_dev_warn("Warning firmware error detected FWSM: 0x%08X\n", fwsm); @@ -8208,7 +8095,7 @@ static bool ixgbe_check_fw_error(struct ixgbe_adapter *adapter) goto no_version; if (hw->mac.ops.get_nvm_ver && - hw->mac.ops.get_nvm_ver(hw, nvm_info)) + hw->mac.ops.get_nvm_ver(hw, nvm_info)) goto no_version; snprintf(ver_buff, sizeof(ver_buff), @@ -8228,8 +8115,8 @@ static bool ixgbe_check_fw_error(struct ixgbe_adapter *adapter) static void ixgbe_recovery_service_task(struct work_struct *work) { struct ixgbe_adapter *adapter = container_of(work, - struct ixgbe_adapter, - service_task); + struct ixgbe_adapter, + service_task); ixgbe_handle_fw_event(adapter); ixgbe_service_event_complete(adapter); @@ -8244,8 +8131,8 @@ static void ixgbe_recovery_service_task(struct work_struct *work) static void ixgbe_service_task(struct work_struct *work) { struct ixgbe_adapter *adapter = container_of(work, - struct ixgbe_adapter, - service_task); + struct ixgbe_adapter, + service_task); if (ixgbe_removed(adapter->hw.hw_addr)) { if (!test_bit(__IXGBE_DOWN, &adapter->state)) { rtnl_lock(); @@ -8291,9 +8178,9 @@ static void ixgbe_service_task(struct work_struct *work) } static int ixgbe_tso(struct ixgbe_ring *tx_ring, - struct ixgbe_tx_buffer *first, - u8 *hdr_len, - struct ixgbe_ipsec_tx_data *itd) + struct ixgbe_tx_buffer *first, + u8 *hdr_len, + struct ixgbe_ipsec_tx_data *itd) { u32 vlan_macip_lens, type_tucmd, mss_l4len_idx; struct sk_buff *skb = first->skb; @@ -8329,7 +8216,7 @@ static int ixgbe_tso(struct ixgbe_ring *tx_ring, /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ type_tucmd = (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) ? - IXGBE_ADVTXD_TUCMD_L4T_UDP : IXGBE_ADVTXD_TUCMD_L4T_TCP; + IXGBE_ADVTXD_TUCMD_L4T_UDP : IXGBE_ADVTXD_TUCMD_L4T_TCP; /* initialize outer IP header fields */ if (ip.v4->version == 4) { @@ -8366,12 +8253,12 @@ static int ixgbe_tso(struct ixgbe_ring *tx_ring, /* compute length of segmentation header */ *hdr_len = (l4.tcp->doff * 4) + l4_offset; csum_replace_by_diff(&l4.tcp->check, - (__force __wsum)htonl(paylen)); + (__force __wsum)htonl(paylen)); } else { /* compute length of segmentation header */ *hdr_len = sizeof(*l4.udp) + l4_offset; csum_replace_by_diff(&l4.udp->check, - (__force __wsum)htonl(paylen)); + (__force __wsum)htonl(paylen)); } /* update gso size and bytecount with header size */ @@ -8455,8 +8342,8 @@ static u32 ixgbe_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) { /* set type for advanced descriptor with frame checksum insertion */ u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA | - IXGBE_ADVTXD_DCMD_DEXT | - IXGBE_ADVTXD_DCMD_IFCS; + IXGBE_ADVTXD_DCMD_DEXT | + IXGBE_ADVTXD_DCMD_IFCS; /* set HW vlan bit if vlan is present */ cmd_type |= IXGBE_SET_FLAG(tx_flags, IXGBE_TX_FLAGS_HW_VLAN, @@ -8510,7 +8397,7 @@ static void ixgbe_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc, static int __ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size) { if (!netif_subqueue_try_stop(tx_ring->netdev, tx_ring->queue_index, - ixgbe_desc_unused(tx_ring), size)) + ixgbe_desc_unused(tx_ring), size)) return -EBUSY; ++tx_ring->tx_stats.restart_queue; @@ -8610,7 +8497,7 @@ static int ixgbe_tx_map(struct ixgbe_ring *tx_ring, data_len -= size; dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, - DMA_TO_DEVICE); + DMA_TO_DEVICE); tx_buffer = &tx_ring->tx_buffer_info[i]; } @@ -8660,9 +8547,9 @@ static int ixgbe_tx_map(struct ixgbe_ring *tx_ring, tx_buffer = &tx_ring->tx_buffer_info[i]; if (dma_unmap_len(tx_buffer, len)) dma_unmap_page(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); dma_unmap_len_set(tx_buffer, len, 0); if (tx_buffer == first) break; @@ -8680,7 +8567,7 @@ static int ixgbe_tx_map(struct ixgbe_ring *tx_ring, } static void ixgbe_atr(struct ixgbe_ring *ring, - struct ixgbe_tx_buffer *first) + struct ixgbe_tx_buffer *first) { struct ixgbe_q_vector *q_vector = ring->q_vector; union ixgbe_atr_hash_dword input = { .dword = 0 }; @@ -8708,7 +8595,7 @@ static void ixgbe_atr(struct ixgbe_ring *ring, /* currently only IPv4/IPv6 with TCP is supported */ if ((first->protocol != htons(ETH_P_IP)) && - (first->protocol != htons(ETH_P_IPV6))) + (first->protocol != htons(ETH_P_IPV6))) return; /* snag network header to get L4 type and address */ @@ -8717,21 +8604,21 @@ static void ixgbe_atr(struct ixgbe_ring *ring, if (unlikely(hdr.network <= skb->data)) return; if (skb->encapsulation && - first->protocol == htons(ETH_P_IP) && - hdr.ipv4->protocol == IPPROTO_UDP) { + first->protocol == htons(ETH_P_IP) && + hdr.ipv4->protocol == IPPROTO_UDP) { struct ixgbe_adapter *adapter = q_vector->adapter; if (unlikely(skb_tail_pointer(skb) < hdr.network + - vxlan_headroom(0))) + vxlan_headroom(0))) return; /* verify the port is recognized as VXLAN */ if (adapter->vxlan_port && - udp_hdr(skb)->dest == adapter->vxlan_port) + udp_hdr(skb)->dest == adapter->vxlan_port) hdr.network = skb_inner_network_header(skb); if (adapter->geneve_port && - udp_hdr(skb)->dest == adapter->geneve_port) + udp_hdr(skb)->dest == adapter->geneve_port) hdr.network = skb_inner_network_header(skb); } @@ -8761,7 +8648,7 @@ static void ixgbe_atr(struct ixgbe_ring *ring, return; if (unlikely(skb_tail_pointer(skb) < hdr.network + - hlen + sizeof(struct tcphdr))) + hlen + sizeof(struct tcphdr))) return; th = (struct tcphdr *)(hdr.network + hlen); @@ -8806,13 +8693,13 @@ static void ixgbe_atr(struct ixgbe_ring *ring, case 6: input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV6; common.ip ^= hdr.ipv6->saddr.s6_addr32[0] ^ - hdr.ipv6->saddr.s6_addr32[1] ^ - hdr.ipv6->saddr.s6_addr32[2] ^ - hdr.ipv6->saddr.s6_addr32[3] ^ - hdr.ipv6->daddr.s6_addr32[0] ^ - hdr.ipv6->daddr.s6_addr32[1] ^ - hdr.ipv6->daddr.s6_addr32[2] ^ - hdr.ipv6->daddr.s6_addr32[3]; + hdr.ipv6->saddr.s6_addr32[1] ^ + hdr.ipv6->saddr.s6_addr32[2] ^ + hdr.ipv6->saddr.s6_addr32[3] ^ + hdr.ipv6->daddr.s6_addr32[0] ^ + hdr.ipv6->daddr.s6_addr32[1] ^ + hdr.ipv6->daddr.s6_addr32[2] ^ + hdr.ipv6->daddr.s6_addr32[3]; break; default: break; @@ -8823,12 +8710,12 @@ static void ixgbe_atr(struct ixgbe_ring *ring, /* This assumes the Rx queue and Tx queue are bound to the same CPU */ ixgbe_fdir_add_signature_filter_82599(&q_vector->adapter->hw, - input, common, ring->queue_index); + input, common, ring->queue_index); } #ifdef IXGBE_FCOE static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb, - struct net_device *sb_dev) + struct net_device *sb_dev) { struct ixgbe_adapter *adapter; struct ixgbe_ring_feature *f; @@ -8941,8 +8828,8 @@ int ixgbe_xmit_xdp_ring(struct ixgbe_ring *ring, tx_buff = &ring->tx_buffer_info[index]; if (dma_unmap_len(tx_buff, len)) dma_unmap_page(ring->dev, dma_unmap_addr(tx_buff, dma), - dma_unmap_len(tx_buff, len), - DMA_TO_DEVICE); + dma_unmap_len(tx_buff, len), + DMA_TO_DEVICE); dma_unmap_len_set(tx_buff, len, 0); if (tx_buff == tx_head) break; @@ -9008,9 +8895,9 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, protocol = vlan_get_protocol(skb); if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && - adapter->ptp_clock) { + adapter->ptp_clock) { if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON && - !test_and_set_bit_lock(__IXGBE_PTP_TX_IN_PROGRESS, + !test_and_set_bit_lock(__IXGBE_PTP_TX_IN_PROGRESS, &adapter->state)) { skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; tx_flags |= IXGBE_TX_FLAGS_TSTAMP; @@ -9035,8 +8922,8 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, #endif /* DCB maps skb priorities 0-7 onto 3 bit PCP of VLAN tag. */ if ((adapter->flags & IXGBE_FLAG_DCB_ENABLED) && - ((tx_flags & (IXGBE_TX_FLAGS_HW_VLAN | IXGBE_TX_FLAGS_SW_VLAN)) || - (skb->priority != TC_PRIO_CONTROL))) { + ((tx_flags & (IXGBE_TX_FLAGS_HW_VLAN | IXGBE_TX_FLAGS_SW_VLAN)) || + (skb->priority != TC_PRIO_CONTROL))) { tx_flags &= ~IXGBE_TX_FLAGS_VLAN_PRIO_MASK; tx_flags |= (skb->priority & 0x7) << IXGBE_TX_FLAGS_VLAN_PRIO_SHIFT; @@ -9060,7 +8947,7 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, #ifdef IXGBE_FCOE /* setup tx offload for FCoE */ if ((protocol == htons(ETH_P_FCOE)) && - (tx_ring->netdev->features & (NETIF_F_FSO | NETIF_F_FCOE_CRC))) { + (tx_ring->netdev->features & (NETIF_F_FSO | NETIF_F_FCOE_CRC))) { tso = ixgbe_fso(tx_ring, first, &hdr_len); if (tso < 0) goto out_drop; @@ -9072,7 +8959,7 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, #ifdef CONFIG_IXGBE_IPSEC if (xfrm_offload(skb) && - !ixgbe_ipsec_tx(tx_ring, first, &ipsec_tx)) + !ixgbe_ipsec_tx(tx_ring, first, &ipsec_tx)) goto out_drop; #endif tso = ixgbe_tso(tx_ring, first, &hdr_len, &ipsec_tx); @@ -9108,8 +8995,8 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, } static netdev_tx_t __ixgbe_xmit_frame(struct sk_buff *skb, - struct net_device *netdev, - struct ixgbe_ring *ring) + struct net_device *netdev, + struct ixgbe_ring *ring) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); struct ixgbe_ring *tx_ring; @@ -9129,7 +9016,7 @@ static netdev_tx_t __ixgbe_xmit_frame(struct sk_buff *skb, } static netdev_tx_t ixgbe_xmit_frame(struct sk_buff *skb, - struct net_device *netdev) + struct net_device *netdev) { return __ixgbe_xmit_frame(skb, netdev, NULL); } @@ -9185,7 +9072,7 @@ ixgbe_mdio_read(struct net_device *netdev, int prtad, int devad, u16 addr) } static int ixgbe_mdio_write(struct net_device *netdev, int prtad, int devad, - u16 addr, u16 value) + u16 addr, u16 value) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); struct ixgbe_hw *hw = &adapter->hw; @@ -9282,7 +9169,7 @@ static void ixgbe_get_ring_stats64(struct rtnl_link_stats64 *stats, } static void ixgbe_get_stats64(struct net_device *netdev, - struct rtnl_link_stats64 *stats) + struct rtnl_link_stats64 *stats) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); int i; @@ -9406,7 +9293,7 @@ static void ixgbe_set_prio_tc_map(struct ixgbe_adapter *adapter) #endif /* CONFIG_IXGBE_DCB */ static int ixgbe_reassign_macvlan_pool(struct net_device *vdev, - struct netdev_nested_priv *priv) + struct netdev_nested_priv *priv) { struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)priv->data; struct ixgbe_fwd_adapter *accel; @@ -9454,7 +9341,7 @@ static void ixgbe_defrag_macvlan_pools(struct net_device *dev) /* walk through upper devices reassigning pools */ netdev_walk_all_upper_dev_rcu(dev, ixgbe_reassign_macvlan_pool, - &priv); + &priv); } /** @@ -9534,7 +9421,7 @@ int ixgbe_setup_tc(struct net_device *dev, u8 tc) } static int ixgbe_delete_clsu32(struct ixgbe_adapter *adapter, - struct tc_cls_u32_offload *cls) + struct tc_cls_u32_offload *cls) { u32 hdl = cls->knode.handle; u32 uhtid = TC_U32_USERHTID(cls->knode.handle); @@ -9570,8 +9457,8 @@ static int ixgbe_delete_clsu32(struct ixgbe_adapter *adapter, continue; spin_lock(&adapter->fdir_perfect_lock); err = ixgbe_update_ethtool_fdir_entry(adapter, - NULL, - j + 1); + NULL, + j + 1); spin_unlock(&adapter->fdir_perfect_lock); clear_bit(j, jump->child_loc_map); } @@ -9591,7 +9478,7 @@ static int ixgbe_delete_clsu32(struct ixgbe_adapter *adapter, } static int ixgbe_configure_clsu32_add_hnode(struct ixgbe_adapter *adapter, - struct tc_cls_u32_offload *cls) + struct tc_cls_u32_offload *cls) { u32 uhtid = TC_U32_USERHTID(cls->hnode.handle); @@ -9609,7 +9496,7 @@ static int ixgbe_configure_clsu32_add_hnode(struct ixgbe_adapter *adapter, } static int ixgbe_configure_clsu32_del_hnode(struct ixgbe_adapter *adapter, - struct tc_cls_u32_offload *cls) + struct tc_cls_u32_offload *cls) { u32 uhtid = TC_U32_USERHTID(cls->hnode.handle); @@ -9629,7 +9516,7 @@ struct upper_walk_data { }; static int get_macvlan_queue(struct net_device *upper, - struct netdev_nested_priv *priv) + struct netdev_nested_priv *priv) { if (netif_is_macvlan(upper)) { struct ixgbe_fwd_adapter *vadapter = macvlan_accel_priv(upper); @@ -9688,7 +9575,7 @@ static int handle_redirect_action(struct ixgbe_adapter *adapter, int ifindex, } static int parse_tc_actions(struct ixgbe_adapter *adapter, - struct tcf_exts *exts, u64 *action, u8 *queue) + struct tcf_exts *exts, u64 *action, u8 *queue) { const struct tc_action *a; int i; @@ -9711,7 +9598,7 @@ static int parse_tc_actions(struct ixgbe_adapter *adapter, if (!dev) return -EINVAL; return handle_redirect_action(adapter, dev->ifindex, - queue, action); + queue, action); } return -EINVAL; @@ -9721,17 +9608,17 @@ static int parse_tc_actions(struct ixgbe_adapter *adapter, } #else static int parse_tc_actions(struct ixgbe_adapter *adapter, - struct tcf_exts *exts, u64 *action, u8 *queue) + struct tcf_exts *exts, u64 *action, u8 *queue) { return -EINVAL; } #endif /* CONFIG_NET_CLS_ACT */ static int ixgbe_clsu32_build_input(struct ixgbe_fdir_filter *input, - union ixgbe_atr_input *mask, - struct tc_cls_u32_offload *cls, - struct ixgbe_mat_field *field_ptr, - struct ixgbe_nexthdr *nexthdr) + union ixgbe_atr_input *mask, + struct tc_cls_u32_offload *cls, + struct ixgbe_mat_field *field_ptr, + struct ixgbe_nexthdr *nexthdr) { int i, j, off; __be32 val, m; @@ -9754,10 +9641,10 @@ static int ixgbe_clsu32_build_input(struct ixgbe_fdir_filter *input, } if (nexthdr) { if (nexthdr->off == cls->knode.sel->keys[i].off && - nexthdr->val == - (__force u32)cls->knode.sel->keys[i].val && - nexthdr->mask == - (__force u32)cls->knode.sel->keys[i].mask) + nexthdr->val == + (__force u32)cls->knode.sel->keys[i].val && + nexthdr->mask == + (__force u32)cls->knode.sel->keys[i].mask) found_jump_field = true; else continue; @@ -9771,7 +9658,7 @@ static int ixgbe_clsu32_build_input(struct ixgbe_fdir_filter *input, return 0; mask->formatted.flow_type = IXGBE_ATR_L4TYPE_IPV6_MASK | - IXGBE_ATR_L4TYPE_MASK; + IXGBE_ATR_L4TYPE_MASK; if (input->filter.formatted.flow_type == IXGBE_ATR_FLOW_TYPE_IPV4) mask->formatted.flow_type &= IXGBE_ATR_L4TYPE_IPV6_MASK; @@ -9852,17 +9739,17 @@ static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter, * and link to it. */ if (adapter->jump_tables[link_uhtid] && - (adapter->jump_tables[link_uhtid])->link_hdl) { + (adapter->jump_tables[link_uhtid])->link_hdl) { e_err(drv, "Link filter exists for link: %x\n", - link_uhtid); + link_uhtid); return err; } for (i = 0; nexthdr[i].jump; i++) { if (nexthdr[i].o != cls->knode.sel->offoff || - nexthdr[i].s != cls->knode.sel->offshift || - nexthdr[i].m != - (__force u32)cls->knode.sel->offmask) + nexthdr[i].s != cls->knode.sel->offshift || + nexthdr[i].m != + (__force u32)cls->knode.sel->offmask) return err; jump = kzalloc(sizeof(*jump), GFP_KERNEL); @@ -9883,7 +9770,7 @@ static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter, jump->link_hdl = cls->knode.handle; err = ixgbe_clsu32_build_input(input, mask, cls, - field_ptr, &nexthdr[i]); + field_ptr, &nexthdr[i]); if (!err) { jump->mat = nexthdr[i].jump; adapter->jump_tables[link_uhtid] = jump; @@ -9909,10 +9796,10 @@ static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter, if ((uhtid != 0x800) && (adapter->jump_tables[uhtid])) { if ((adapter->jump_tables[uhtid])->input) memcpy(input, (adapter->jump_tables[uhtid])->input, - sizeof(*input)); + sizeof(*input)); if ((adapter->jump_tables[uhtid])->mask) memcpy(mask, (adapter->jump_tables[uhtid])->mask, - sizeof(*mask)); + sizeof(*mask)); /* Lookup in all child hash tables if this location is already * filled with a filter @@ -9922,7 +9809,7 @@ static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter, if (link && (test_bit(loc - 1, link->child_loc_map))) { e_err(drv, "Filter exists in location: %x\n", - loc); + loc); err = -EINVAL; goto err_out; } @@ -9933,7 +9820,7 @@ static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter, goto err_out; err = parse_tc_actions(adapter, cls->knode.exts, &input->action, - &queue); + &queue); if (err < 0) goto err_out; @@ -9953,7 +9840,7 @@ static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter, ixgbe_atr_compute_perfect_hash_82599(&input->filter, mask); err = ixgbe_fdir_write_perfect_filter_82599(hw, &input->filter, - input->sw_idx, queue); + input->sw_idx, queue); if (err) goto err_out_w_lock; @@ -10021,7 +9908,7 @@ static int ixgbe_setup_tc_mqprio(struct net_device *dev, static LIST_HEAD(ixgbe_block_cb_list); static int __ixgbe_setup_tc(struct net_device *dev, enum tc_setup_type type, - void *type_data) + void *type_data) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(dev); @@ -10060,7 +9947,7 @@ void ixgbe_do_reset(struct net_device *netdev) } static netdev_features_t ixgbe_fix_features(struct net_device *netdev, - netdev_features_t features) + netdev_features_t features) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); @@ -10088,7 +9975,7 @@ static void ixgbe_reset_l2fw_offload(struct ixgbe_adapter *adapter) /* go back to full RSS if we're not running SR-IOV */ if (!adapter->ring_feature[RING_F_VMDQ].offset) adapter->flags &= ~(IXGBE_FLAG_VMDQ_ENABLED | - IXGBE_FLAG_SRIOV_ENABLED); + IXGBE_FLAG_SRIOV_ENABLED); adapter->ring_feature[RING_F_RSS].limit = rss; adapter->ring_feature[RING_F_VMDQ].limit = 1; @@ -10097,7 +9984,7 @@ static void ixgbe_reset_l2fw_offload(struct ixgbe_adapter *adapter) } static int ixgbe_set_features(struct net_device *netdev, - netdev_features_t features) + netdev_features_t features) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); netdev_features_t changed = netdev->features ^ features; @@ -10111,12 +9998,12 @@ static int ixgbe_set_features(struct net_device *netdev, } else if ((adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE) && !(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)) { if (adapter->rx_itr_setting == 1 || - adapter->rx_itr_setting > IXGBE_MIN_RSC_ITR) { + adapter->rx_itr_setting > IXGBE_MIN_RSC_ITR) { adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED; need_reset = true; } else if ((changed ^ features) & NETIF_F_LRO) { e_info(probe, "rx-usecs set too low, " - "disabling RSC\n"); + "disabling RSC\n"); } } @@ -10140,12 +10027,12 @@ static int ixgbe_set_features(struct net_device *netdev, /* We cannot enable ATR if SR-IOV is enabled */ if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED || - /* We cannot enable ATR if we have 2 or more tcs */ - (adapter->hw_tcs > 1) || - /* We cannot enable ATR if RSS is disabled */ - (adapter->ring_feature[RING_F_RSS].limit <= 1) || - /* A sample rate of 0 indicates ATR disabled */ - (!adapter->atr_sample_rate)) + /* We cannot enable ATR if we have 2 or more tcs */ + (adapter->hw_tcs > 1) || + /* We cannot enable ATR if RSS is disabled */ + (adapter->ring_feature[RING_F_RSS].limit <= 1) || + /* A sample rate of 0 indicates ATR disabled */ + (!adapter->atr_sample_rate)) ; /* do nothing not supported */ else /* otherwise supported and set the flag */ adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE; @@ -10161,17 +10048,17 @@ static int ixgbe_set_features(struct net_device *netdev, else if (need_reset) ixgbe_do_reset(netdev); else if (changed & (NETIF_F_HW_VLAN_CTAG_RX | - NETIF_F_HW_VLAN_CTAG_FILTER)) + NETIF_F_HW_VLAN_CTAG_FILTER)) ixgbe_set_rx_mode(netdev); return 1; } static int ixgbe_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], - struct net_device *dev, - const unsigned char *addr, u16 vid, - u16 flags, bool *notified, - struct netlink_ext_ack *extack) + struct net_device *dev, + const unsigned char *addr, u16 vid, + u16 flags, bool *notified, + struct netlink_ext_ack *extack) { /* guarantee we can provide a unique filter for the unicast address */ if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) { @@ -10193,7 +10080,7 @@ static int ixgbe_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], * Configure some settings require for various bridge modes. **/ static int ixgbe_configure_bridge_mode(struct ixgbe_adapter *adapter, - __u16 mode) + __u16 mode) { struct ixgbe_hw *hw = &adapter->hw; unsigned int p, num_pools; @@ -10219,8 +10106,8 @@ static int ixgbe_configure_bridge_mode(struct ixgbe_adapter *adapter, for (p = 0; p < num_pools; p++) { if (hw->mac.ops.set_source_address_pruning) hw->mac.ops.set_source_address_pruning(hw, - true, - p); + true, + p); } break; case BRIDGE_MODE_VEB: @@ -10243,8 +10130,8 @@ static int ixgbe_configure_bridge_mode(struct ixgbe_adapter *adapter, for (p = 0; p < num_pools; p++) { if (hw->mac.ops.set_source_address_pruning) hw->mac.ops.set_source_address_pruning(hw, - false, - p); + false, + p); } break; default: @@ -10254,14 +10141,14 @@ static int ixgbe_configure_bridge_mode(struct ixgbe_adapter *adapter, adapter->bridge_mode = mode; e_info(drv, "enabling bridge mode: %s\n", - mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); + mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); return 0; } static int ixgbe_ndo_bridge_setlink(struct net_device *dev, - struct nlmsghdr *nlh, u16 flags, - struct netlink_ext_ack *extack) + struct nlmsghdr *nlh, u16 flags, + struct netlink_ext_ack *extack) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(dev); struct nlattr *attr, *br_spec; @@ -10288,8 +10175,8 @@ static int ixgbe_ndo_bridge_setlink(struct net_device *dev, } static int ixgbe_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, - struct net_device *dev, - u32 filter_mask, int nlflags) + struct net_device *dev, + u32 filter_mask, int nlflags) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(dev); @@ -10297,8 +10184,8 @@ static int ixgbe_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, return 0; return ndo_dflt_bridge_getlink(skb, pid, seq, dev, - adapter->bridge_mode, 0, 0, nlflags, - filter_mask, NULL); + adapter->bridge_mode, 0, 0, nlflags, + filter_mask, NULL); } static void *ixgbe_fwd_add(struct net_device *pdev, struct net_device *vdev) @@ -10333,8 +10220,8 @@ static void *ixgbe_fwd_add(struct net_device *pdev, struct net_device *vdev) u16 reserved_pools; if (((adapter->flags & IXGBE_FLAG_DCB_ENABLED) && - adapter->num_rx_pools >= (MAX_TX_QUEUES / tcs)) || - adapter->num_rx_pools > IXGBE_MAX_MACVLANS) + adapter->num_rx_pools >= (MAX_TX_QUEUES / tcs)) || + adapter->num_rx_pools > IXGBE_MAX_MACVLANS) return ERR_PTR(-EBUSY); /* Hardware has a limited number of available pools. Each VF, @@ -10354,12 +10241,12 @@ static void *ixgbe_fwd_add(struct net_device *pdev, struct net_device *vdev) */ if (used_pools < 32 && adapter->num_rx_pools < 16) reserved_pools = min_t(u16, - 32 - used_pools, - 16 - adapter->num_rx_pools); + 32 - used_pools, + 16 - adapter->num_rx_pools); else if (adapter->num_rx_pools < 32) reserved_pools = min_t(u16, - 64 - used_pools, - 32 - adapter->num_rx_pools); + 64 - used_pools, + 32 - adapter->num_rx_pools); else reserved_pools = 64 - used_pools; @@ -10406,7 +10293,7 @@ static void ixgbe_fwd_del(struct net_device *pdev, void *priv) /* delete unicast filter associated with offloaded interface */ ixgbe_del_mac_filter(adapter, accel->netdev->dev_addr, - VMDQ_P(accel->pool)); + VMDQ_P(accel->pool)); /* Allow remaining Rx packets to get flushed out of the * Rx FIFO before we drop the netdev for the ring. @@ -10438,7 +10325,7 @@ static void ixgbe_fwd_del(struct net_device *pdev, void *priv) static netdev_features_t ixgbe_features_check(struct sk_buff *skb, struct net_device *dev, - netdev_features_t features) + netdev_features_t features) { unsigned int network_hdr_len, mac_hdr_len; @@ -10446,19 +10333,19 @@ ixgbe_features_check(struct sk_buff *skb, struct net_device *dev, mac_hdr_len = skb_network_offset(skb); if (unlikely(mac_hdr_len > IXGBE_MAX_MAC_HDR_LEN)) return features & ~(NETIF_F_HW_CSUM | - NETIF_F_SCTP_CRC | - NETIF_F_GSO_UDP_L4 | - NETIF_F_HW_VLAN_CTAG_TX | - NETIF_F_TSO | - NETIF_F_TSO6); + NETIF_F_SCTP_CRC | + NETIF_F_GSO_UDP_L4 | + NETIF_F_HW_VLAN_CTAG_TX | + NETIF_F_TSO | + NETIF_F_TSO6); network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); if (unlikely(network_hdr_len > IXGBE_MAX_NETWORK_HDR_LEN)) return features & ~(NETIF_F_HW_CSUM | - NETIF_F_SCTP_CRC | - NETIF_F_GSO_UDP_L4 | - NETIF_F_TSO | - NETIF_F_TSO6); + NETIF_F_SCTP_CRC | + NETIF_F_GSO_UDP_L4 | + NETIF_F_TSO | + NETIF_F_TSO6); /* We can only support IPV4 TSO in tunnels if we can mangle the * inner IP ID field, so strip TSO if MANGLEID is not supported. @@ -10477,7 +10364,7 @@ ixgbe_features_check(struct sk_buff *skb, struct net_device *dev, static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog) { - int i, frame_size = dev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; + int i, frame_size = READ_ONCE(dev->mtu) + LIBETH_RX_LL_LEN; struct ixgbe_adapter *adapter = ixgbe_from_netdev(dev); struct bpf_prog *old_prog; bool need_reset; @@ -10541,7 +10428,7 @@ static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog) for (i = 0; i < num_queues; i++) if (adapter->xdp_ring[i]->xsk_pool) (void)ixgbe_xsk_wakeup(adapter->netdev, i, - XDP_WAKEUP_RX); + XDP_WAKEUP_RX); xdp_features_set_redirect_target(dev, true); } @@ -10557,7 +10444,7 @@ static int ixgbe_xdp(struct net_device *dev, struct netdev_bpf *xdp) return ixgbe_xdp_setup(dev, xdp->prog); case XDP_SETUP_XSK_POOL: return ixgbe_xsk_pool_setup(adapter, xdp->xsk.pool, - xdp->xsk.queue_id); + xdp->xsk.queue_id); default: return -EINVAL; @@ -10594,7 +10481,7 @@ static int ixgbe_xdp_xmit(struct net_device *dev, int n, return -ENETDOWN; if (!netif_carrier_ok(adapter->netdev) || - !netif_running(adapter->netdev)) + !netif_running(adapter->netdev)) return -ENETDOWN; if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) @@ -10711,7 +10598,7 @@ static void ixgbe_disable_txr_hw(struct ixgbe_adapter *adapter, } static void ixgbe_disable_txr(struct ixgbe_adapter *adapter, - struct ixgbe_ring *tx_ring) + struct ixgbe_ring *tx_ring) { set_bit(__IXGBE_TX_DISABLED, &tx_ring->state); ixgbe_disable_txr_hw(adapter, tx_ring); @@ -10735,7 +10622,7 @@ static void ixgbe_disable_rxr_hw(struct ixgbe_adapter *adapter, /* RXDCTL.EN may not change on 82598 if link is down, so skip it */ if (hw->mac.type == ixgbe_mac_82598EB && - !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) + !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) return; /* delay mechanism from ixgbe_disable_rx */ @@ -10915,7 +10802,7 @@ static inline int ixgbe_enumerate_functions(struct ixgbe_adapter *adapter) * this case, simply return -1 to indicate this. */ if ((entry->vendor != pdev->vendor) || - (entry->device != pdev->device)) + (entry->device != pdev->device)) return -1; physfns++; @@ -10947,8 +10834,8 @@ bool ixgbe_wol_supported(struct ixgbe_adapter *adapter, u16 device_id, /* check eeprom to see if WOL is enabled for X540 and newer */ if (hw->mac.type >= ixgbe_mac_X540) { if ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0_1) || - ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0) && - (hw->bus.func == 0))) + ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0) && + (hw->bus.func == 0))) return true; } @@ -11135,7 +11022,7 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) */ if (pdev->is_virtfn) { WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n", - pci_name(pdev), pdev->vendor, pdev->device); + pci_name(pdev), pdev->vendor, pdev->device); return -EINVAL; } @@ -11195,7 +11082,7 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); hw->hw_addr = ioremap(pci_resource_start(pdev, 0), - pci_resource_len(pdev, 0)); + pci_resource_len(pdev, 0)); adapter->io_addr = hw->hw_addr; if (!hw->hw_addr) { err = -EIO; @@ -11338,15 +11225,15 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) NETIF_F_HW_CSUM; #define IXGBE_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ - NETIF_F_GSO_GRE_CSUM | \ - NETIF_F_GSO_IPXIP4 | \ - NETIF_F_GSO_IPXIP6 | \ - NETIF_F_GSO_UDP_TUNNEL | \ - NETIF_F_GSO_UDP_TUNNEL_CSUM) + NETIF_F_GSO_GRE_CSUM | \ + NETIF_F_GSO_IPXIP4 | \ + NETIF_F_GSO_IPXIP6 | \ + NETIF_F_GSO_UDP_TUNNEL | \ + NETIF_F_GSO_UDP_TUNNEL_CSUM) netdev->gso_partial_features = IXGBE_GSO_PARTIAL_FEATURES; netdev->features |= NETIF_F_GSO_PARTIAL | - IXGBE_GSO_PARTIAL_FEATURES; + IXGBE_GSO_PARTIAL_FEATURES; if (hw->mac.type >= ixgbe_mac_82599EB) netdev->features |= NETIF_F_SCTP_CRC | NETIF_F_GSO_UDP_L4; @@ -11361,15 +11248,15 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) #endif /* copy netdev features into list of user selectable features */ netdev->hw_features |= netdev->features | - NETIF_F_HW_VLAN_CTAG_FILTER | - NETIF_F_HW_VLAN_CTAG_RX | - NETIF_F_HW_VLAN_CTAG_TX | - NETIF_F_RXALL | - NETIF_F_HW_L2FW_DOFFLOAD; + NETIF_F_HW_VLAN_CTAG_FILTER | + NETIF_F_HW_VLAN_CTAG_RX | + NETIF_F_HW_VLAN_CTAG_TX | + NETIF_F_RXALL | + NETIF_F_HW_L2FW_DOFFLOAD; if (hw->mac.type >= ixgbe_mac_82599EB) netdev->hw_features |= NETIF_F_NTUPLE | - NETIF_F_HW_TC; + NETIF_F_HW_TC; netdev->features |= NETIF_F_HIGHDMA; @@ -11383,14 +11270,14 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* set this bit last since it cannot be part of vlan_features */ netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER | - NETIF_F_HW_VLAN_CTAG_RX | - NETIF_F_HW_VLAN_CTAG_TX; + NETIF_F_HW_VLAN_CTAG_RX | + NETIF_F_HW_VLAN_CTAG_TX; netdev->priv_flags |= IFF_UNICAST_FLT; netdev->priv_flags |= IFF_SUPP_NOFCS; netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | - NETDEV_XDP_ACT_XSK_ZEROCOPY; + NETDEV_XDP_ACT_XSK_ZEROCOPY; /* MTU range: 68 - 9710 */ netdev->min_mtu = ETH_MIN_MTU; @@ -11416,7 +11303,7 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adapter->ring_feature[RING_F_FCOE].limit = fcoe_l; netdev->features |= NETIF_F_FSO | - NETIF_F_FCOE_CRC; + NETIF_F_FCOE_CRC; netdev->vlan_features |= NETIF_F_FSO | NETIF_F_FCOE_CRC; @@ -11435,7 +11322,7 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } eth_platform_get_mac_address(&adapter->pdev->dev, - adapter->hw.mac.perm_addr); + adapter->hw.mac.perm_addr); eth_hw_addr_set(netdev, hw->mac.perm_addr); @@ -11753,7 +11640,7 @@ static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev, u16 req_id, pf_func; if (adapter->hw.mac.type == ixgbe_mac_82598EB || - adapter->num_vfs == 0) + adapter->num_vfs == 0) goto skip_bad_vf_detection; bdev = pdev->bus->self; @@ -11818,7 +11705,7 @@ static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev, if (vfdev->devfn == (req_id & 0xFF)) break; vfdev = pci_get_device(PCI_VENDOR_ID_INTEL, - device_id, vfdev); + device_id, vfdev); } /* * There's a slim chance the VF could have been hot plugged, @@ -12005,7 +11892,7 @@ static void __exit ixgbe_exit_module(void) #ifdef CONFIG_IXGBE_DCA static int ixgbe_notify_dca(struct notifier_block *nb, unsigned long event, - void *p) + void *p) { int ret_val; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h index b1bfeb21537acc..825cf17fa1878d 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h @@ -2452,7 +2452,7 @@ enum { #define IXGBE_PSRTYPE_L2HDR 0x00001000 /* SRRCTL bit definitions */ -#define IXGBE_SRRCTL_BSIZEPKT_SHIFT 10 /* so many KBs */ +#define IXGBE_SRRCTL_BSIZEPKT_STEP 1024 #define IXGBE_SRRCTL_RDMTS_SHIFT 22 #define IXGBE_SRRCTL_RDMTS_MASK 0x01C00000 #define IXGBE_SRRCTL_DROP_EN 0x10000000 diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c index 7b941505a9d024..15425d3c367872 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c @@ -151,7 +151,7 @@ static int ixgbe_run_xdp_zc(struct ixgbe_adapter *adapter, bool ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 count) { union ixgbe_adv_rx_desc *rx_desc; - struct ixgbe_rx_buffer *bi; + struct ixgbe_xsk_rx_buffer *bi; u16 i = rx_ring->next_to_use; dma_addr_t dma; bool ok = true; @@ -161,7 +161,7 @@ bool ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 count) return true; rx_desc = IXGBE_RX_DESC(rx_ring, i); - bi = &rx_ring->rx_buffer_info[i]; + bi = &rx_ring->rx_xsk_buffer_info[i]; i -= rx_ring->count; do { @@ -183,7 +183,7 @@ bool ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 count) i++; if (unlikely(!i)) { rx_desc = IXGBE_RX_DESC(rx_ring, 0); - bi = rx_ring->rx_buffer_info; + bi = rx_ring->rx_xsk_buffer_info; i -= rx_ring->count; } @@ -257,7 +257,7 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, while (likely(total_rx_packets < budget)) { union ixgbe_adv_rx_desc *rx_desc; - struct ixgbe_rx_buffer *bi; + struct ixgbe_xsk_rx_buffer *bi; unsigned int size; /* return some buffers to hardware, one at a time is too slow */ @@ -279,17 +279,17 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, */ dma_rmb(); - bi = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; + bi = &rx_ring->rx_xsk_buffer_info[rx_ring->next_to_clean]; if (unlikely(!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) { - struct ixgbe_rx_buffer *next_bi; + struct ixgbe_xsk_rx_buffer *next_bi; xsk_buff_free(bi->xdp); bi->xdp = NULL; ixgbe_inc_ntc(rx_ring); next_bi = - &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; + &rx_ring->rx_xsk_buffer_info[rx_ring->next_to_clean]; next_bi->discard = true; continue; } @@ -345,6 +345,7 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, total_rx_bytes += skb->len; total_rx_packets++; + skb->protocol = eth_type_trans(skb, rx_ring->netdev); ixgbe_process_skb_fields(rx_ring, rx_desc, skb); ixgbe_rx_skb(q_vector, skb); } @@ -374,11 +375,11 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, void ixgbe_xsk_clean_rx_ring(struct ixgbe_ring *rx_ring) { - struct ixgbe_rx_buffer *bi; + struct ixgbe_xsk_rx_buffer *bi; u16 i; for (i = 0; i < rx_ring->count; i++) { - bi = &rx_ring->rx_buffer_info[i]; + bi = &rx_ring->rx_xsk_buffer_info[i]; if (!bi->xdp) continue; From aab8eaf3365f2558acdd54c48d778a4f996e1f42 Mon Sep 17 00:00:00 2001 From: Magdalena Pytel Date: Thu, 28 May 2026 15:03:24 +0200 Subject: [PATCH 18/39] ixgbe: branch prediction and cleanup * Added branch prediction hints to the ixgbe Rx path and clean up the packet accounting. * Moved total_rx_bytes and total_rx_packets accounting together. Signed-off-by: Magdalena Pytel --- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index c50a5a1e73eb19..776fef0b4764cc 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -1745,7 +1745,7 @@ void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) u16 ntu = rx_ring->next_to_use; /* nothing to do */ - if (!cleaned_count) + if (unlikely(!cleaned_count)) return; rx_desc = IXGBE_RX_DESC(rx_ring, ntu); @@ -1761,7 +1761,7 @@ void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) rx_desc++; ntu++; - if (unlikely(ntu == rx_ring->count)) { + if (unlikely(ntu == fq.count)) { rx_desc = IXGBE_RX_DESC(rx_ring, 0); ntu = 0; } @@ -2122,7 +2122,6 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, /* exit if we failed to retrieve a buffer */ if (unlikely(!xdp_res && !skb)) { - libeth_xdp_return_buff_slow(xdp); rx_ring->rx_stats.alloc_rx_buff_failed++; cleaned_count++; break; @@ -2135,14 +2134,12 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, continue; /* verify the packet layout is correct */ - if (xdp_res || unlikely(ixgbe_cleanup_headers(rx_ring, rx_desc, skb))) { + if (xdp_res || + unlikely(ixgbe_cleanup_headers(rx_ring, rx_desc, skb))) { skb = NULL; continue; } - /* probably a little skewed due to removing CRC */ - total_rx_bytes += skb->len; - /* populate checksum, timestamp, VLAN, and protocol */ ixgbe_process_skb_fields(rx_ring, rx_desc, skb); @@ -2172,11 +2169,12 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, } #endif /* IXGBE_FCOE */ + /* probably a little skewed due to removing CRC */ + total_rx_bytes += skb->len; + total_rx_packets++; + ixgbe_rx_skb(q_vector, skb); skb = NULL; - - /* update budget accounting */ - total_rx_packets++; } rx_ring->skb = skb; @@ -6620,12 +6618,16 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, }; struct device *dev = &adapter->pdev->dev; int orig_node = dev_to_node(dev); + struct napi_struct *napi_dev; int ring_node = NUMA_NO_NODE; int ret; if (rx_ring->q_vector) { fq.nid = rx_ring->q_vector->numa_node; ring_node = rx_ring->q_vector->numa_node; + napi_dev = &rx_ring->q_vector->napi; + } else { + napi_dev = NULL; } ret = libeth_rx_fq_create(&fq, &rx_ring->q_vector->napi); From f0e1ead1e433802c445aad83802deacb41706722 Mon Sep 17 00:00:00 2001 From: Magdalena Pytel Date: Mon, 1 Jun 2026 09:51:36 +0200 Subject: [PATCH 19/39] ixgbe: support XDP multi-buffer on Rx path Implement XDP multi-buffer support for Rx fragmented packets using libeth_xdp. * Added MODULE_IMPORT_NS("LIBETH_XDP") and #include * Replaced struct sk_buff *skb in ixgbe_ring with struct libeth_xdp_buff_stash and ixgbe_clean_rx_ring skb cleanup with libeth_xdp_return_stash * Removed ixgbe_add_rx_frag() * Reorganized ixgbe_clean_rx_irq() and ixgbe_is_non_eop() * Updated ixgbe_xdp_setup() to allow multi-buffer XDP programs * Added NETDEV_XDP_ACT_RX_SG in xdp_features Signed-off-by: Magdalena Pytel --- drivers/net/ethernet/intel/ixgbe/ixgbe.h | 3 +- drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c | 1 - drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 122 +++++++----------- 3 files changed, 52 insertions(+), 74 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index f5842f35d498bd..af801eacc45c65 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -34,6 +34,7 @@ #include #include +#include /* common prefix used by pr_<> macros */ #undef pr_fmt @@ -324,12 +325,12 @@ struct ixgbe_ring { struct ixgbe_tx_queue_stats tx_stats; struct ixgbe_rx_queue_stats rx_stats; }; - struct sk_buff *skb; /* partial Rx skb across non-EOP */ struct xdp_rxq_info xdp_rxq; spinlock_t tx_lock; /* used in XDP mode */ struct xsk_buff_pool *xsk_pool; u16 ring_idx; /* {rx,tx,xdp}_ring back reference idx */ u32 rx_buf_len; + struct libeth_xdp_buff_stash xdp_stash; } ____cacheline_internodealigned_in_smp; enum ixgbe_ring_f_enum { diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c index a1d04914fbbca8..2d139d2217fac4 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c @@ -965,7 +965,6 @@ static int ixgbe_alloc_q_vector(struct ixgbe_adapter *adapter, while (rxr_count) { /* assign generic ring traits */ - ring->dev = &adapter->pdev->dev; ring->netdev = adapter->netdev; /* configure backlink on ring */ diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 776fef0b4764cc..a59939ba094b2a 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -1865,7 +1865,7 @@ void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector, * ixgbe_is_non_eop - process handling of non-EOP buffers * @rx_ring: Rx ring being processed * @rx_desc: Rx descriptor for current buffer - * @skb: Current socket buffer containing buffer in progress + * @rsc_append_cnt: pointer to RSC append count accumulator * * This function updates next to clean. If the buffer is an EOP buffer * this function exits returning false, otherwise it will place the @@ -1873,8 +1873,8 @@ void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector, * that this is in fact a non-EOP buffer. **/ static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) + union ixgbe_adv_rx_desc *rx_desc, + u32 *rsc_append_cnt) { u32 ntc = rx_ring->next_to_clean + 1; @@ -1893,7 +1893,7 @@ static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring, u32 rsc_cnt = le32_to_cpu(rsc_enabled); rsc_cnt >>= IXGBE_RXDADV_RSCCNT_SHIFT; - IXGBE_CB(skb)->append_cnt += rsc_cnt - 1; + *rsc_append_cnt += rsc_cnt - 1; /* update ntc based on RSC value */ ntc = le32_to_cpu(rx_desc->wb.upper.status_error); @@ -1961,33 +1961,6 @@ bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring, return false; } -/** - * ixgbe_add_rx_frag - Add contents of Rx buffer to sk_buff - * @rx_ring: rx descriptor ring to transact packets on - * @rx_buffer: buffer containing page to add - * @skb: sk_buff to place the data into - * @size: size of data in rx_buffer - * - * This function will add the data contained in rx_buffer->page to the skb. - * This is done either through a direct copy if the data in the buffer is - * less than the skb header size, otherwise it will just attach the page as - * a frag to the skb. - * - * The function will then update the page offset if necessary and return - * true if the buffer can be reused by the adapter. - **/ -static void ixgbe_add_rx_frag(const struct libeth_fqe *rx_buffer, - struct sk_buff *skb, - unsigned int size) -{ - u32 hr = netmem_get_pp(rx_buffer->netmem)->p.offset; - - skb_add_rx_frag_netmem(skb, skb_shinfo(skb)->nr_frags, - rx_buffer->netmem, rx_buffer->offset + hr, - size, rx_buffer->truesize); -} - - static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, struct ixgbe_ring *rx_ring, struct libeth_xdp_buff *xdp) @@ -2069,16 +2042,16 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, #endif /* IXGBE_FCOE */ u16 cleaned_count = ixgbe_desc_unused(rx_ring); LIBETH_XDP_ONSTACK_BUFF(xdp); - struct sk_buff *skb = rx_ring->skb; unsigned int xdp_xmit = 0; + u32 rsc_append_cnt = 0; int xdp_res = 0; - xdp->data = NULL; - xdp->base.rxq = &rx_ring->xdp_rxq; + libeth_xdp_init_buff(xdp, &rx_ring->xdp_stash, &rx_ring->xdp_rxq); while (likely(total_rx_packets < budget)) { union ixgbe_adv_rx_desc *rx_desc; struct libeth_fqe *rx_buffer; + struct sk_buff *skb; unsigned int size; /* return some buffers to hardware, one at a time is too slow */ @@ -2099,43 +2072,43 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, dma_rmb(); rx_buffer = &rx_ring->rx_fqes[rx_ring->next_to_clean]; + libeth_xdp_process_buff(xdp, rx_buffer, size); - /* retrieve a buffer from the ring */ - if (!skb) { - libeth_xdp_process_buff(xdp, rx_buffer, size); - xdp_res = ixgbe_run_xdp(adapter, rx_ring, xdp); - } else { - libeth_rx_sync_for_cpu(rx_buffer, size); - } + cleaned_count++; + /* fetch next buffer in frame if non-eop */ + if (ixgbe_is_non_eop(rx_ring, rx_desc, &rsc_append_cnt) || + unlikely(!xdp->data)) + continue; + xdp_res = ixgbe_run_xdp(adapter, rx_ring, xdp); if (xdp_res) { if (xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR)) xdp_xmit |= xdp_res; total_rx_packets++; - total_rx_bytes += size; - } else if (skb) { - ixgbe_add_rx_frag(rx_buffer, skb, size); - } else { - skb = xdp_build_skb_from_buff(&xdp->base); + total_rx_bytes += xdp_get_buff_len(&xdp->base); + rsc_append_cnt = 0; + xdp->data = NULL; + continue; } + skb = xdp_build_skb_from_buff(&xdp->base); + /* exit if we failed to retrieve a buffer */ - if (unlikely(!xdp_res && !skb)) { + if (unlikely(!skb)) { + rsc_append_cnt = 0; + libeth_xdp_return_buff_slow(xdp); rx_ring->rx_stats.alloc_rx_buff_failed++; - cleaned_count++; break; } + xdp->data = NULL; - cleaned_count++; - - /* place incomplete frames back on ring for completion */ - if (ixgbe_is_non_eop(rx_ring, rx_desc, skb)) - continue; + /* apply RSC append count to skb if accumulated */ + IXGBE_CB(skb)->append_cnt = rsc_append_cnt; + rsc_append_cnt = 0; /* verify the packet layout is correct */ - if (xdp_res || - unlikely(ixgbe_cleanup_headers(rx_ring, rx_desc, skb))) { + if (unlikely(ixgbe_cleanup_headers(rx_ring, rx_desc, skb))) { skb = NULL; continue; } @@ -2174,10 +2147,10 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, total_rx_packets++; ixgbe_rx_skb(q_vector, skb); - skb = NULL; } - rx_ring->skb = skb; + /* place incomplete frames back on ring for completion */ + libeth_xdp_save_buff(&rx_ring->xdp_stash, xdp); if (xdp_xmit & IXGBE_XDP_REDIR) xdp_do_flush(); @@ -5227,11 +5200,8 @@ static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) goto skip_free; } - /* Free Rx ring sk_buff */ - if (rx_ring->skb) { - dev_kfree_skb(rx_ring->skb); - rx_ring->skb = NULL; - } + /* Free Rx ring xdp stash */ + libeth_xdp_return_stash(&rx_ring->xdp_stash); for (u32 i = rx_ring->next_to_clean; i != rx_ring->next_to_use; ) { const struct libeth_fqe *rx_fqe = &rx_ring->rx_fqes[i]; @@ -6653,6 +6623,8 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, GFP_KERNEL); if (!rx_ring->desc) { ret = -ENOMEM; + dev_err(&adapter->pdev->dev, + "Unable to allocate memory for the Rx descriptor ring\n"); goto err_destroy_fq; } @@ -6675,7 +6647,6 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, rx_ring->desc, rx_ring->dma); rx_ring->desc = NULL; err_destroy_fq: - dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n"); libeth_rx_fq_destroy(&fq); rx_ring->rx_fqes = NULL; rx_ring->pp = NULL; @@ -10364,13 +10335,15 @@ ixgbe_features_check(struct sk_buff *skb, struct net_device *dev, return features; } -static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog) +static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog, + struct netlink_ext_ack *extack) { - int i, frame_size = READ_ONCE(dev->mtu) + LIBETH_RX_LL_LEN; + u32 frame_size = READ_ONCE(dev->mtu) + LIBETH_RX_LL_LEN; struct ixgbe_adapter *adapter = ixgbe_from_netdev(dev); struct bpf_prog *old_prog; bool need_reset; int num_queues; + bool requires_mbuf; if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) return -EINVAL; @@ -10379,15 +10352,19 @@ static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog) return -EINVAL; /* verify ixgbe ring attributes are sufficient for XDP */ - for (i = 0; i < adapter->num_rx_queues; i++) { + for (int i = 0; i < adapter->num_rx_queues; i++) { struct ixgbe_ring *ring = adapter->rx_ring[i]; if (ring_is_rsc_enabled(ring)) return -EINVAL; } - if (frame_size > IXGBE_RXBUFFER_3K) - return -EINVAL; + requires_mbuf = frame_size > IXGBE_RX_PAGE_LEN(LIBETH_XDP_HEADROOM); + if (prog && !prog->aux->xdp_has_frags && requires_mbuf) { + NL_SET_ERR_MSG_MOD(extack, + "Configured MTU requires non-linear frames and XDP prog does not support frags"); + return -EOPNOTSUPP; + } /* if the number of cpus is much larger than the maximum of queues, * we should stop it and then return with ENOMEM like before. @@ -10412,7 +10389,7 @@ static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog) if (!prog) xdp_features_clear_redirect_target(dev); } else { - for (i = 0; i < adapter->num_rx_queues; i++) { + for (int i = 0; i < adapter->num_rx_queues; i++) { WRITE_ONCE(adapter->rx_ring[i]->xdp_prog, adapter->xdp_prog); } @@ -10427,7 +10404,7 @@ static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog) if (need_reset && prog) { num_queues = min_t(int, adapter->num_rx_queues, adapter->num_xdp_queues); - for (i = 0; i < num_queues; i++) + for (int i = 0; i < num_queues; i++) if (adapter->xdp_ring[i]->xsk_pool) (void)ixgbe_xsk_wakeup(adapter->netdev, i, XDP_WAKEUP_RX); @@ -10443,7 +10420,7 @@ static int ixgbe_xdp(struct net_device *dev, struct netdev_bpf *xdp) switch (xdp->command) { case XDP_SETUP_PROG: - return ixgbe_xdp_setup(dev, xdp->prog); + return ixgbe_xdp_setup(dev, xdp->prog, xdp->extack); case XDP_SETUP_XSK_POOL: return ixgbe_xsk_pool_setup(adapter, xdp->xsk.pool, xdp->xsk.queue_id); @@ -11279,7 +11256,8 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) netdev->priv_flags |= IFF_SUPP_NOFCS; netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | - NETDEV_XDP_ACT_XSK_ZEROCOPY; + NETDEV_XDP_ACT_XSK_ZEROCOPY | + NETDEV_XDP_ACT_RX_SG; /* MTU range: 68 - 9710 */ netdev->min_mtu = ETH_MIN_MTU; From a66f091496171a00c16874e5098083785f16aa23 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Wed, 12 Aug 2026 12:25:15 +0200 Subject: [PATCH 20/39] [SPLIT] Move descriptor definitions to a separate file + introduce and use libie_xg_ring --- drivers/net/ethernet/intel/ixgbevf/defines.h | 191 ------------------ include/linux/net/intel/ixgbe_desc.h | 200 +++++++++++++++++++ 2 files changed, 200 insertions(+), 191 deletions(-) create mode 100644 include/linux/net/intel/ixgbe_desc.h diff --git a/drivers/net/ethernet/intel/ixgbevf/defines.h b/drivers/net/ethernet/intel/ixgbevf/defines.h index afc927dd14381b..a67d1ecd6012a6 100644 --- a/drivers/net/ethernet/intel/ixgbevf/defines.h +++ b/drivers/net/ethernet/intel/ixgbevf/defines.h @@ -84,197 +84,6 @@ typedef u32 ixgbe_link_speed; #define IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS 0x0A000000 #define IXGBE_SRRCTL_DESCTYPE_MASK 0x0E000000 -/* Receive Descriptor bit definitions */ -#define IXGBE_RXD_STAT_DD 0x01 /* Descriptor Done */ -#define IXGBE_RXD_STAT_EOP 0x02 /* End of Packet */ -#define IXGBE_RXD_STAT_FLM 0x04 /* FDir Match */ -#define IXGBE_RXD_STAT_VP 0x08 /* IEEE VLAN Packet */ -#define IXGBE_RXDADV_NEXTP_MASK 0x000FFFF0 /* Next Descriptor Index */ -#define IXGBE_RXDADV_NEXTP_SHIFT 0x00000004 -#define IXGBE_RXD_STAT_UDPCS 0x10 /* UDP xsum calculated */ -#define IXGBE_RXD_STAT_L4CS 0x20 /* L4 xsum calculated */ -#define IXGBE_RXD_STAT_IPCS 0x40 /* IP xsum calculated */ -#define IXGBE_RXD_STAT_PIF 0x80 /* passed in-exact filter */ -#define IXGBE_RXD_STAT_CRCV 0x100 /* Speculative CRC Valid */ -#define IXGBE_RXD_STAT_VEXT 0x200 /* 1st VLAN found */ -#define IXGBE_RXD_STAT_UDPV 0x400 /* Valid UDP checksum */ -#define IXGBE_RXD_STAT_DYNINT 0x800 /* Pkt caused INT via DYNINT */ -#define IXGBE_RXD_STAT_TS 0x10000 /* Time Stamp */ -#define IXGBE_RXD_STAT_SECP 0x20000 /* Security Processing */ -#define IXGBE_RXD_STAT_LB 0x40000 /* Loopback Status */ -#define IXGBE_RXD_STAT_ACK 0x8000 /* ACK Packet indication */ -#define IXGBE_RXD_ERR_CE 0x01 /* CRC Error */ -#define IXGBE_RXD_ERR_LE 0x02 /* Length Error */ -#define IXGBE_RXD_ERR_PE 0x08 /* Packet Error */ -#define IXGBE_RXD_ERR_OSE 0x10 /* Oversize Error */ -#define IXGBE_RXD_ERR_USE 0x20 /* Undersize Error */ -#define IXGBE_RXD_ERR_TCPE 0x40 /* TCP/UDP Checksum Error */ -#define IXGBE_RXD_ERR_IPE 0x80 /* IP Checksum Error */ -#define IXGBE_RXDADV_ERR_MASK 0xFFF00000 /* RDESC.ERRORS mask */ -#define IXGBE_RXDADV_ERR_SHIFT 20 /* RDESC.ERRORS shift */ -#define IXGBE_RXDADV_ERR_HBO 0x00800000 /*Header Buffer Overflow */ -#define IXGBE_RXDADV_ERR_CE 0x01000000 /* CRC Error */ -#define IXGBE_RXDADV_ERR_LE 0x02000000 /* Length Error */ -#define IXGBE_RXDADV_ERR_PE 0x08000000 /* Packet Error */ -#define IXGBE_RXDADV_ERR_OSE 0x10000000 /* Oversize Error */ -#define IXGBE_RXDADV_ERR_USE 0x20000000 /* Undersize Error */ -#define IXGBE_RXDADV_ERR_TCPE 0x40000000 /* TCP/UDP Checksum Error */ -#define IXGBE_RXDADV_ERR_IPE 0x80000000 /* IP Checksum Error */ -#define IXGBE_RXD_VLAN_ID_MASK 0x0FFF /* VLAN ID is in lower 12 bits */ -#define IXGBE_RXD_PRI_MASK 0xE000 /* Priority is in upper 3 bits */ -#define IXGBE_RXD_PRI_SHIFT 13 -#define IXGBE_RXD_CFI_MASK 0x1000 /* CFI is bit 12 */ -#define IXGBE_RXD_CFI_SHIFT 12 - -#define IXGBE_RXDADV_STAT_DD IXGBE_RXD_STAT_DD /* Done */ -#define IXGBE_RXDADV_STAT_EOP IXGBE_RXD_STAT_EOP /* End of Packet */ -#define IXGBE_RXDADV_STAT_FLM IXGBE_RXD_STAT_FLM /* FDir Match */ -#define IXGBE_RXDADV_STAT_VP IXGBE_RXD_STAT_VP /* IEEE VLAN Pkt */ -#define IXGBE_RXDADV_STAT_MASK 0x000FFFFF /* Stat/NEXTP: bit 0-19 */ -#define IXGBE_RXDADV_STAT_FCEOFS 0x00000040 /* FCoE EOF/SOF Stat */ -#define IXGBE_RXDADV_STAT_FCSTAT 0x00000030 /* FCoE Pkt Stat */ -#define IXGBE_RXDADV_STAT_FCSTAT_NOMTCH 0x00000000 /* 00: No Ctxt Match */ -#define IXGBE_RXDADV_STAT_FCSTAT_NODDP 0x00000010 /* 01: Ctxt w/o DDP */ -#define IXGBE_RXDADV_STAT_FCSTAT_FCPRSP 0x00000020 /* 10: Recv. FCP_RSP */ -#define IXGBE_RXDADV_STAT_FCSTAT_DDP 0x00000030 /* 11: Ctxt w/ DDP */ -#define IXGBE_RXDADV_STAT_SECP 0x00020000 /* IPsec/MACsec pkt found */ - -#define IXGBE_RXDADV_RSSTYPE_MASK 0x0000000F -#define IXGBE_RXDADV_PKTTYPE_MASK 0x0000FFF0 -#define IXGBE_RXDADV_PKTTYPE_IPV4 0x00000010 /* IPv4 hdr present */ -#define IXGBE_RXDADV_PKTTYPE_IPV6 0x00000040 /* IPv6 hdr present */ -#define IXGBE_RXDADV_PKTTYPE_IPSEC_ESP 0x00001000 /* IPSec ESP */ -#define IXGBE_RXDADV_PKTTYPE_IPSEC_AH 0x00002000 /* IPSec AH */ -#define IXGBE_RXDADV_PKTTYPE_MASK_EX 0x0001FFF0 -#define IXGBE_RXDADV_HDRBUFLEN_MASK 0x00007FE0 -#define IXGBE_RXDADV_RSCCNT_MASK 0x001E0000 -#define IXGBE_RXDADV_RSCCNT_SHIFT 17 -#define IXGBE_RXDADV_HDRBUFLEN_SHIFT 5 -#define IXGBE_RXDADV_SPLITHEADER_EN 0x00001000 -#define IXGBE_RXDADV_SPH 0x8000 - -/* RSS Hash results */ -#define IXGBE_RXDADV_RSSTYPE_NONE 0x00000000 -#define IXGBE_RXDADV_RSSTYPE_IPV4_TCP 0x00000001 -#define IXGBE_RXDADV_RSSTYPE_IPV4 0x00000002 -#define IXGBE_RXDADV_RSSTYPE_IPV6_TCP 0x00000003 -#define IXGBE_RXDADV_RSSTYPE_IPV6_EX 0x00000004 -#define IXGBE_RXDADV_RSSTYPE_IPV6 0x00000005 -#define IXGBE_RXDADV_RSSTYPE_IPV6_TCP_EX 0x00000006 -#define IXGBE_RXDADV_RSSTYPE_IPV4_UDP 0x00000007 -#define IXGBE_RXDADV_RSSTYPE_IPV6_UDP 0x00000008 -#define IXGBE_RXDADV_RSSTYPE_IPV6_UDP_EX 0x00000009 - -#define IXGBE_RXD_ERR_FRAME_ERR_MASK ( \ - IXGBE_RXD_ERR_CE | \ - IXGBE_RXD_ERR_LE | \ - IXGBE_RXD_ERR_PE | \ - IXGBE_RXD_ERR_OSE | \ - IXGBE_RXD_ERR_USE) - -#define IXGBE_RXDADV_ERR_FRAME_ERR_MASK ( \ - IXGBE_RXDADV_ERR_CE | \ - IXGBE_RXDADV_ERR_LE | \ - IXGBE_RXDADV_ERR_PE | \ - IXGBE_RXDADV_ERR_OSE | \ - IXGBE_RXDADV_ERR_USE) - -#define IXGBE_TXD_POPTS_IXSM 0x01 /* Insert IP checksum */ -#define IXGBE_TXD_POPTS_TXSM 0x02 /* Insert TCP/UDP checksum */ -#define IXGBE_TXD_CMD_EOP 0x01000000 /* End of Packet */ -#define IXGBE_TXD_CMD_IFCS 0x02000000 /* Insert FCS (Ethernet CRC) */ -#define IXGBE_TXD_CMD_IC 0x04000000 /* Insert Checksum */ -#define IXGBE_TXD_CMD_RS 0x08000000 /* Report Status */ -#define IXGBE_TXD_CMD_DEXT 0x20000000 /* Descriptor ext (0 = legacy) */ -#define IXGBE_TXD_CMD_VLE 0x40000000 /* Add VLAN tag */ -#define IXGBE_TXD_STAT_DD 0x00000001 /* Descriptor Done */ -#define IXGBE_TXD_CMD (IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS) - -/* Transmit Descriptor - Advanced */ -union ixgbe_adv_tx_desc { - struct { - __le64 buffer_addr; /* Address of descriptor's data buf */ - __le32 cmd_type_len; - __le32 olinfo_status; - } read; - struct { - __le64 rsvd; /* Reserved */ - __le32 nxtseq_seed; - __le32 status; - } wb; -}; - -/* Receive Descriptor - Advanced */ -union ixgbe_adv_rx_desc { - struct { - __le64 pkt_addr; /* Packet buffer address */ - __le64 hdr_addr; /* Header buffer address */ - } read; - struct { - struct { - union { - __le32 data; - struct { - __le16 pkt_info; /* RSS, Pkt type */ - __le16 hdr_info; /* Splithdr, hdrlen */ - } hs_rss; - } lo_dword; - union { - __le32 rss; /* RSS Hash */ - struct { - __le16 ip_id; /* IP id */ - __le16 csum; /* Packet Checksum */ - } csum_ip; - } hi_dword; - } lower; - struct { - __le32 status_error; /* ext status/error */ - __le16 length; /* Packet length */ - __le16 vlan; /* VLAN tag */ - } upper; - } wb; /* writeback */ -}; - -/* Context descriptors */ -struct ixgbe_adv_tx_context_desc { - __le32 vlan_macip_lens; - __le32 fceof_saidx; - __le32 type_tucmd_mlhl; - __le32 mss_l4len_idx; -}; - -/* Adv Transmit Descriptor Config Masks */ -#define IXGBE_ADVTXD_DTYP_MASK 0x00F00000 /* DTYP mask */ -#define IXGBE_ADVTXD_DTYP_CTXT 0x00200000 /* Advanced Context Desc */ -#define IXGBE_ADVTXD_DTYP_DATA 0x00300000 /* Advanced Data Descriptor */ -#define IXGBE_ADVTXD_DCMD_EOP IXGBE_TXD_CMD_EOP /* End of Packet */ -#define IXGBE_ADVTXD_DCMD_IFCS IXGBE_TXD_CMD_IFCS /* Insert FCS */ -#define IXGBE_ADVTXD_DCMD_RS IXGBE_TXD_CMD_RS /* Report Status */ -#define IXGBE_ADVTXD_DCMD_DEXT IXGBE_TXD_CMD_DEXT /* Desc ext (1=Adv) */ -#define IXGBE_ADVTXD_DCMD_VLE IXGBE_TXD_CMD_VLE /* VLAN pkt enable */ -#define IXGBE_ADVTXD_DCMD_TSE 0x80000000 /* TCP Seg enable */ -#define IXGBE_ADVTXD_STAT_DD IXGBE_TXD_STAT_DD /* Descriptor Done */ -#define IXGBE_ADVTXD_TUCMD_IPV4 0x00000400 /* IP Packet Type: 1=IPv4 */ -#define IXGBE_ADVTXD_TUCMD_IPV6 0x00000000 /* IP Packet Type: 0=IPv6 */ -#define IXGBE_ADVTXD_TUCMD_L4T_UDP 0x00000000 /* L4 Packet TYPE of UDP */ -#define IXGBE_ADVTXD_TUCMD_L4T_TCP 0x00000800 /* L4 Packet TYPE of TCP */ -#define IXGBE_ADVTXD_TUCMD_L4T_SCTP 0x00001000 /* L4 Packet TYPE of SCTP */ -#define IXGBE_ADVTXD_TUCMD_IPSEC_TYPE_ESP 0x00002000 /* IPSec Type ESP */ -#define IXGBE_ADVTXD_TUCMD_IPSEC_ENCRYPT_EN 0x00004000 /* ESP Encrypt Enable */ -#define IXGBE_ADVTXD_IDX_SHIFT 4 /* Adv desc Index shift */ -#define IXGBE_ADVTXD_CC 0x00000080 /* Check Context */ -#define IXGBE_ADVTXD_POPTS_SHIFT 8 /* Adv desc POPTS shift */ -#define IXGBE_ADVTXD_POPTS_IPSEC 0x00000400 /* IPSec offload request */ -#define IXGBE_ADVTXD_POPTS_IXSM (IXGBE_TXD_POPTS_IXSM << \ - IXGBE_ADVTXD_POPTS_SHIFT) -#define IXGBE_ADVTXD_POPTS_TXSM (IXGBE_TXD_POPTS_TXSM << \ - IXGBE_ADVTXD_POPTS_SHIFT) -#define IXGBE_ADVTXD_PAYLEN_SHIFT 14 /* Adv desc PAYLEN shift */ -#define IXGBE_ADVTXD_MACLEN_SHIFT 9 /* Adv ctxt desc mac len shift */ -#define IXGBE_ADVTXD_VLAN_SHIFT 16 /* Adv ctxt vlan tag shift */ -#define IXGBE_ADVTXD_L4LEN_SHIFT 8 /* Adv ctxt L4LEN shift */ -#define IXGBE_ADVTXD_MSS_SHIFT 16 /* Adv ctxt MSS shift */ - /* Interrupt register bitmasks */ #define IXGBE_EITR_CNT_WDIS 0x80000000 diff --git a/include/linux/net/intel/ixgbe_desc.h b/include/linux/net/intel/ixgbe_desc.h new file mode 100644 index 00000000000000..fdf8a35044f539 --- /dev/null +++ b/include/linux/net/intel/ixgbe_desc.h @@ -0,0 +1,200 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2026 Intel Corporation */ + +#ifndef __IXGBE_DESC_H +#define __IXGBE_DESC_H + +#include + +/* Receive Descriptor bit definitions */ +#define IXGBE_RXD_STAT_DD 0x01 /* Descriptor Done */ +#define IXGBE_RXD_STAT_EOP 0x02 /* End of Packet */ +#define IXGBE_RXD_STAT_FLM 0x04 /* FDir Match */ +#define IXGBE_RXD_STAT_VP 0x08 /* IEEE VLAN Packet */ +#define IXGBE_RXDADV_NEXTP_MASK 0x000FFFF0 /* Next Descriptor Index */ +#define IXGBE_RXDADV_NEXTP_SHIFT 0x00000004 +#define IXGBE_RXD_STAT_UDPCS 0x10 /* UDP xsum calculated */ +#define IXGBE_RXD_STAT_L4CS 0x20 /* L4 xsum calculated */ +#define IXGBE_RXD_STAT_IPCS 0x40 /* IP xsum calculated */ +#define IXGBE_RXD_STAT_PIF 0x80 /* passed in-exact filter */ +#define IXGBE_RXD_STAT_CRCV 0x100 /* Speculative CRC Valid */ +#define IXGBE_RXD_STAT_VEXT 0x200 /* 1st VLAN found */ +#define IXGBE_RXD_STAT_UDPV 0x400 /* Valid UDP checksum */ +#define IXGBE_RXD_STAT_DYNINT 0x800 /* Pkt caused INT via DYNINT */ +#define IXGBE_RXD_STAT_TS 0x10000 /* Time Stamp */ +#define IXGBE_RXD_STAT_SECP 0x20000 /* Security Processing */ +#define IXGBE_RXD_STAT_LB 0x40000 /* Loopback Status */ +#define IXGBE_RXD_STAT_ACK 0x8000 /* ACK Packet indication */ +#define IXGBE_RXD_ERR_CE 0x01 /* CRC Error */ +#define IXGBE_RXD_ERR_LE 0x02 /* Length Error */ +#define IXGBE_RXD_ERR_PE 0x08 /* Packet Error */ +#define IXGBE_RXD_ERR_OSE 0x10 /* Oversize Error */ +#define IXGBE_RXD_ERR_USE 0x20 /* Undersize Error */ +#define IXGBE_RXD_ERR_TCPE 0x40 /* TCP/UDP Checksum Error */ +#define IXGBE_RXD_ERR_IPE 0x80 /* IP Checksum Error */ +#define IXGBE_RXDADV_ERR_MASK 0xFFF00000 /* RDESC.ERRORS mask */ +#define IXGBE_RXDADV_ERR_SHIFT 20 /* RDESC.ERRORS shift */ +#define IXGBE_RXDADV_ERR_HBO 0x00800000 /*Header Buffer Overflow */ +#define IXGBE_RXDADV_ERR_CE 0x01000000 /* CRC Error */ +#define IXGBE_RXDADV_ERR_LE 0x02000000 /* Length Error */ +#define IXGBE_RXDADV_ERR_PE 0x08000000 /* Packet Error */ +#define IXGBE_RXDADV_ERR_OSE 0x10000000 /* Oversize Error */ +#define IXGBE_RXDADV_ERR_USE 0x20000000 /* Undersize Error */ +#define IXGBE_RXDADV_ERR_TCPE 0x40000000 /* TCP/UDP Checksum Error */ +#define IXGBE_RXDADV_ERR_IPE 0x80000000 /* IP Checksum Error */ +#define IXGBE_RXD_VLAN_ID_MASK 0x0FFF /* VLAN ID is in lower 12 bits */ +#define IXGBE_RXD_PRI_MASK 0xE000 /* Priority is in upper 3 bits */ +#define IXGBE_RXD_PRI_SHIFT 13 +#define IXGBE_RXD_CFI_MASK 0x1000 /* CFI is bit 12 */ +#define IXGBE_RXD_CFI_SHIFT 12 + +#define IXGBE_RXDADV_STAT_DD IXGBE_RXD_STAT_DD /* Done */ +#define IXGBE_RXDADV_STAT_EOP IXGBE_RXD_STAT_EOP /* End of Packet */ +#define IXGBE_RXDADV_STAT_FLM IXGBE_RXD_STAT_FLM /* FDir Match */ +#define IXGBE_RXDADV_STAT_VP IXGBE_RXD_STAT_VP /* IEEE VLAN Pkt */ +#define IXGBE_RXDADV_STAT_MASK 0x000FFFFF /* Stat/NEXTP: bit 0-19 */ +#define IXGBE_RXDADV_STAT_FCEOFS 0x00000040 /* FCoE EOF/SOF Stat */ +#define IXGBE_RXDADV_STAT_FCSTAT 0x00000030 /* FCoE Pkt Stat */ +#define IXGBE_RXDADV_STAT_FCSTAT_NOMTCH 0x00000000 /* 00: No Ctxt Match */ +#define IXGBE_RXDADV_STAT_FCSTAT_NODDP 0x00000010 /* 01: Ctxt w/o DDP */ +#define IXGBE_RXDADV_STAT_FCSTAT_FCPRSP 0x00000020 /* 10: Recv. FCP_RSP */ +#define IXGBE_RXDADV_STAT_FCSTAT_DDP 0x00000030 /* 11: Ctxt w/ DDP */ +#define IXGBE_RXDADV_STAT_SECP 0x00020000 /* IPsec/MACsec pkt found */ + +#define IXGBE_RXDADV_RSSTYPE_MASK 0x0000000F +#define IXGBE_RXDADV_PKTTYPE_MASK 0x0000FFF0 +#define IXGBE_RXDADV_PKTTYPE_IPV4 0x00000010 /* IPv4 hdr present */ +#define IXGBE_RXDADV_PKTTYPE_IPV6 0x00000040 /* IPv6 hdr present */ +#define IXGBE_RXDADV_PKTTYPE_IPSEC_ESP 0x00001000 /* IPSec ESP */ +#define IXGBE_RXDADV_PKTTYPE_IPSEC_AH 0x00002000 /* IPSec AH */ +#define IXGBE_RXDADV_PKTTYPE_MASK_EX 0x0001FFF0 +#define IXGBE_RXDADV_HDRBUFLEN_MASK 0x00007FE0 +#define IXGBE_RXDADV_RSCCNT_MASK 0x001E0000 +#define IXGBE_RXDADV_RSCCNT_SHIFT 17 +#define IXGBE_RXDADV_HDRBUFLEN_SHIFT 5 +#define IXGBE_RXDADV_SPLITHEADER_EN 0x00001000 +#define IXGBE_RXDADV_SPH 0x8000 + +/* RSS Hash results */ +#define IXGBE_RXDADV_RSSTYPE_NONE 0x00000000 +#define IXGBE_RXDADV_RSSTYPE_IPV4_TCP 0x00000001 +#define IXGBE_RXDADV_RSSTYPE_IPV4 0x00000002 +#define IXGBE_RXDADV_RSSTYPE_IPV6_TCP 0x00000003 +#define IXGBE_RXDADV_RSSTYPE_IPV6_EX 0x00000004 +#define IXGBE_RXDADV_RSSTYPE_IPV6 0x00000005 +#define IXGBE_RXDADV_RSSTYPE_IPV6_TCP_EX 0x00000006 +#define IXGBE_RXDADV_RSSTYPE_IPV4_UDP 0x00000007 +#define IXGBE_RXDADV_RSSTYPE_IPV6_UDP 0x00000008 +#define IXGBE_RXDADV_RSSTYPE_IPV6_UDP_EX 0x00000009 + +#define IXGBE_RXD_ERR_FRAME_ERR_MASK ( \ + IXGBE_RXD_ERR_CE | \ + IXGBE_RXD_ERR_LE | \ + IXGBE_RXD_ERR_PE | \ + IXGBE_RXD_ERR_OSE | \ + IXGBE_RXD_ERR_USE) + +#define IXGBE_RXDADV_ERR_FRAME_ERR_MASK ( \ + IXGBE_RXDADV_ERR_CE | \ + IXGBE_RXDADV_ERR_LE | \ + IXGBE_RXDADV_ERR_PE | \ + IXGBE_RXDADV_ERR_OSE | \ + IXGBE_RXDADV_ERR_USE) + +#define IXGBE_TXD_POPTS_IXSM 0x01 /* Insert IP checksum */ +#define IXGBE_TXD_POPTS_TXSM 0x02 /* Insert TCP/UDP checksum */ +#define IXGBE_TXD_CMD_EOP 0x01000000 /* End of Packet */ +#define IXGBE_TXD_CMD_IFCS 0x02000000 /* Insert FCS (Ethernet CRC) */ +#define IXGBE_TXD_CMD_IC 0x04000000 /* Insert Checksum */ +#define IXGBE_TXD_CMD_RS 0x08000000 /* Report Status */ +#define IXGBE_TXD_CMD_DEXT 0x20000000 /* Descriptor ext (0 = legacy) */ +#define IXGBE_TXD_CMD_VLE 0x40000000 /* Add VLAN tag */ +#define IXGBE_TXD_STAT_DD 0x00000001 /* Descriptor Done */ +#define IXGBE_TXD_CMD (IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS) + +/* Transmit Descriptor - Advanced */ +union ixgbe_adv_tx_desc { + struct { + __le64 buffer_addr; /* Address of descriptor's data buf */ + __le32 cmd_type_len; + __le32 olinfo_status; + } read; + struct { + __le64 rsvd; /* Reserved */ + __le32 nxtseq_seed; + __le32 status; + } wb; +}; + +/* Receive Descriptor - Advanced */ +union ixgbe_adv_rx_desc { + struct { + __le64 pkt_addr; /* Packet buffer address */ + __le64 hdr_addr; /* Header buffer address */ + } read; + struct { + struct { + union { + __le32 data; + struct { + __le16 pkt_info; /* RSS, Pkt type */ + __le16 hdr_info; /* Splithdr, hdrlen */ + } hs_rss; + } lo_dword; + union { + __le32 rss; /* RSS Hash */ + struct { + __le16 ip_id; /* IP id */ + __le16 csum; /* Packet Checksum */ + } csum_ip; + } hi_dword; + } lower; + struct { + __le32 status_error; /* ext status/error */ + __le16 length; /* Packet length */ + __le16 vlan; /* VLAN tag */ + } upper; + } wb; /* writeback */ +}; + +/* Context descriptors */ +struct ixgbe_adv_tx_context_desc { + __le32 vlan_macip_lens; + __le32 fceof_saidx; + __le32 type_tucmd_mlhl; + __le32 mss_l4len_idx; +}; + +/* Adv Transmit Descriptor Config Masks */ +#define IXGBE_ADVTXD_DTYP_MASK 0x00F00000 /* DTYP mask */ +#define IXGBE_ADVTXD_DTYP_CTXT 0x00200000 /* Advanced Context Desc */ +#define IXGBE_ADVTXD_DTYP_DATA 0x00300000 /* Advanced Data Descriptor */ +#define IXGBE_ADVTXD_DCMD_EOP IXGBE_TXD_CMD_EOP /* End of Packet */ +#define IXGBE_ADVTXD_DCMD_IFCS IXGBE_TXD_CMD_IFCS /* Insert FCS */ +#define IXGBE_ADVTXD_DCMD_RS IXGBE_TXD_CMD_RS /* Report Status */ +#define IXGBE_ADVTXD_DCMD_DEXT IXGBE_TXD_CMD_DEXT /* Desc ext (1=Adv) */ +#define IXGBE_ADVTXD_DCMD_VLE IXGBE_TXD_CMD_VLE /* VLAN pkt enable */ +#define IXGBE_ADVTXD_DCMD_TSE 0x80000000 /* TCP Seg enable */ +#define IXGBE_ADVTXD_STAT_DD IXGBE_TXD_STAT_DD /* Descriptor Done */ +#define IXGBE_ADVTXD_TUCMD_IPV4 0x00000400 /* IP Packet Type: 1=IPv4 */ +#define IXGBE_ADVTXD_TUCMD_IPV6 0x00000000 /* IP Packet Type: 0=IPv6 */ +#define IXGBE_ADVTXD_TUCMD_L4T_UDP 0x00000000 /* L4 Packet TYPE of UDP */ +#define IXGBE_ADVTXD_TUCMD_L4T_TCP 0x00000800 /* L4 Packet TYPE of TCP */ +#define IXGBE_ADVTXD_TUCMD_L4T_SCTP 0x00001000 /* L4 Packet TYPE of SCTP */ +#define IXGBE_ADVTXD_TUCMD_IPSEC_TYPE_ESP 0x00002000 /* IPSec Type ESP */ +#define IXGBE_ADVTXD_TUCMD_IPSEC_ENCRYPT_EN 0x00004000 /* ESP Encrypt Enable */ +#define IXGBE_ADVTXD_IDX_SHIFT 4 /* Adv desc Index shift */ +#define IXGBE_ADVTXD_CC 0x00000080 /* Check Context */ +#define IXGBE_ADVTXD_POPTS_SHIFT 8 /* Adv desc POPTS shift */ +#define IXGBE_ADVTXD_POPTS_IPSEC 0x00000400 /* IPSec offload request */ +#define IXGBE_ADVTXD_POPTS_IXSM (IXGBE_TXD_POPTS_IXSM << \ + IXGBE_ADVTXD_POPTS_SHIFT) +#define IXGBE_ADVTXD_POPTS_TXSM (IXGBE_TXD_POPTS_TXSM << \ + IXGBE_ADVTXD_POPTS_SHIFT) +#define IXGBE_ADVTXD_PAYLEN_SHIFT 14 /* Adv desc PAYLEN shift */ +#define IXGBE_ADVTXD_MACLEN_SHIFT 9 /* Adv ctxt desc mac len shift */ +#define IXGBE_ADVTXD_VLAN_SHIFT 16 /* Adv ctxt vlan tag shift */ +#define IXGBE_ADVTXD_L4LEN_SHIFT 8 /* Adv ctxt L4LEN shift */ +#define IXGBE_ADVTXD_MSS_SHIFT 16 /* Adv ctxt MSS shift */ + +#endif From e2db87a2780f4282bc96c74b11a781b2b757cb0c Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Tue, 18 Aug 2026 11:26:41 +0200 Subject: [PATCH 21/39] libie_xg_ring fixup --- include/linux/net/intel/libie/xg_ring.h | 82 +++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 include/linux/net/intel/libie/xg_ring.h diff --git a/include/linux/net/intel/libie/xg_ring.h b/include/linux/net/intel/libie/xg_ring.h new file mode 100644 index 00000000000000..768861c5e0688c --- /dev/null +++ b/include/linux/net/intel/libie/xg_ring.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2026 Intel Corporation */ + +#ifndef __LIBIE_XG_RING_H +#define __LIBIE_XG_RING_H + +#include +#include +#include + +#include + +struct libie_xg_queue_stats { + u64 packets; + u64 bytes; +}; + +struct libie_xg_tx_queue_stats { + u64 restart_queue; + u64 tx_busy; + u64 tx_done_old; +}; + +struct libie_xg_rx_queue_stats { + /* u64 rsc_count; <- was in ixgbe + u64 rsc_flush; */ + u64 non_eop_descs; + u64 alloc_rx_page; + u64 alloc_rx_page_failed; + u64 alloc_rx_buff_failed; + u64 csum_err; +}; + +struct libie_xg_tx_buffer { + union ixgbe_adv_tx_desc *next_to_watch; + unsigned long time_stamp; + union { + struct sk_buff *skb; + /* XDP uses address ptr on irq_clean */ + void *data; + }; + unsigned int bytecount; + unsigned short gso_segs; + __be16 protocol; + DEFINE_DMA_UNMAP_ADDR(dma); + DEFINE_DMA_UNMAP_LEN(len); + u32 tx_flags; +}; + +struct libie_xg_ring { + struct device *dev; /* Tx ring */ + union { + struct page_pool *pp; /* Rx and XDP rings */ + struct xsk_buff_pool *xsk_pool; /* AF_XDP ZC rings */ + }; + void *desc; /* descriptor ring memory */ + u16 count; /* amount of descriptors */ + u16 next_to_clean; + u32 next_to_use; + u32 pending; /* Sent-not-completed descriptors */ + u32 cached_ntu; + + union { + struct libeth_fqe *rx_fqes; + struct libeth_xdp_buff **xsk_fqes; + struct libie_xg_tx_buffer *tx_buffer_info; + struct libeth_sqe *xdp_sqes; + }; + u8 __iomem *tail; + + struct libeth_xdp_buff *xsk_xdp_head; + struct libeth_xdpsq_lock xdpq_lock; +}; + +#define LIBIE_XG_RX_DESC(R, i) \ + (&(((union ixgbe_adv_rx_desc *)((R)->desc))[i])) +#define LIBIE_XG_TX_DESC(R, i) \ + (&(((union ixgbe_adv_tx_desc *)((R)->desc))[i])) +#define LIBIE_XG_TX_CTXTDESC(R, i) \ + (&(((struct ixgbe_adv_tx_context_desc *)((R)->desc))[i])) + +#endif \ No newline at end of file From cc5fca500b6065ca2b160ef23a721f20587f2263 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Tue, 18 Aug 2026 11:04:40 +0200 Subject: [PATCH 22/39] ixgbevf, libie: separate ixgbevf_ring base and move to libie_xg In order to share some generation-specific non-trivial XDP code between ixgbevf and ixgbe, identify required ring fields and separate them into a base ring structure. Use the structure in a somewhat struct_group fashion, exposing a named .base structure (for shared code to use) as well as direct field access in a union (to not disturb the ancient code). But given that 2 different structures will need to include the base, instead of struct_group, utilize a relatively recently enabled -fms-extensions anonymous member structures feature. Signed-off-by: Larysa Zaremba --- drivers/net/ethernet/intel/ixgbevf/ipsec.c | 2 +- drivers/net/ethernet/intel/ixgbevf/ixgbevf.h | 49 +++---------------- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 18 +++---- 3 files changed, 17 insertions(+), 52 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbevf/ipsec.c b/drivers/net/ethernet/intel/ixgbevf/ipsec.c index fce35924ff8b51..e40abea3eea4f5 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ipsec.c +++ b/drivers/net/ethernet/intel/ixgbevf/ipsec.c @@ -451,7 +451,7 @@ static const struct xfrmdev_ops ixgbevf_xfrmdev_ops = { * @itd: ipsec Tx data for later use in building context descriptor **/ int ixgbevf_ipsec_tx(struct ixgbevf_ring *tx_ring, - struct ixgbevf_tx_buffer *first, + struct libie_xg_tx_buffer *first, struct ixgbevf_ipsec_tx_data *itd) { struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev); diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h index d8f841515ca62a..e1ef7b44ceecea 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -24,25 +25,6 @@ #define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD) #define DESC_NEEDED (MAX_SKB_FRAGS + 4) -/* wrapper around a pointer to a socket buffer, - * so a DMA handle can be stored along with the buffer - */ -struct ixgbevf_tx_buffer { - union ixgbe_adv_tx_desc *next_to_watch; - unsigned long time_stamp; - union { - struct sk_buff *skb; - /* XDP uses address ptr on irq_clean */ - void *data; - }; - unsigned int bytecount; - unsigned short gso_segs; - __be16 protocol; - DEFINE_DMA_UNMAP_ADDR(dma); - DEFINE_DMA_UNMAP_LEN(len); - u32 tx_flags; -}; - struct ixgbevf_stats { u64 packets; u64 bytes; @@ -84,30 +66,16 @@ enum ixgbevf_ring_state_t { clear_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state) struct ixgbevf_ring { + union { + struct libie_xg_ring base; + struct libie_xg_ring; + }; struct ixgbevf_ring *next; struct ixgbevf_q_vector *q_vector; /* backpointer to q_vector */ struct net_device *netdev; struct bpf_prog __rcu *xdp_prog; - union { - struct page_pool *pp; /* Rx and XDP rings */ - struct device *dev; /* Tx ring */ - }; - void *desc; /* descriptor ring memory */ u32 truesize; /* Rx buffer full size */ u32 hdr_truesize; /* Rx header buffer full size */ - u16 count; /* amount of descriptors */ - u16 next_to_clean; - u32 next_to_use; - u32 pending; /* Sent-not-completed descriptors */ - - union { - struct libeth_fqe *rx_fqes; - struct libeth_xdp_buff **xsk_fqes; - struct ixgbevf_tx_buffer *tx_buffer_info; - struct libeth_sqe *xdp_sqes; - }; - struct libeth_xdpsq_lock xdpq_lock; - u32 cached_ntu; u32 thresh; unsigned long state; struct ixgbevf_stats stats; @@ -120,7 +88,6 @@ struct ixgbevf_ring { struct page_pool *hdr_pp; struct xdp_rxq_info xdp_rxq; u64 hw_csum_rx_error; - u8 __iomem *tail; /* holds the special value that gets the hardware register offset * associated with this ring, which is different for DCB and RSS modes @@ -129,10 +96,8 @@ struct ixgbevf_ring { int queue_index; /* needed for multiqueue queue management */ u32 rx_buf_len; struct libeth_xdp_buff_stash xdp_stash; - struct libeth_xdp_buff *xsk_xdp_head; unsigned int dma_size; /* length in bytes */ dma_addr_t dma; /* phys. address of descriptor ring */ - struct xsk_buff_pool *xsk_pool; /* AF_XDP ZC rings */ } ____cacheline_internodealigned_in_smp; /* How many Rx Buffers do we bundle into one write to the hardware ? */ @@ -444,7 +409,7 @@ void ixgbevf_ipsec_rx(struct ixgbevf_ring *rx_ring, union ixgbe_adv_rx_desc *rx_desc, struct sk_buff *skb); int ixgbevf_ipsec_tx(struct ixgbevf_ring *tx_ring, - struct ixgbevf_tx_buffer *first, + struct libie_xg_tx_buffer *first, struct ixgbevf_ipsec_tx_data *itd); #else static inline void ixgbevf_init_ipsec_offload(struct ixgbevf_adapter *adapter) @@ -456,7 +421,7 @@ static inline void ixgbevf_ipsec_rx(struct ixgbevf_ring *rx_ring, union ixgbe_adv_rx_desc *rx_desc, struct sk_buff *skb) { } static inline int ixgbevf_ipsec_tx(struct ixgbevf_ring *tx_ring, - struct ixgbevf_tx_buffer *first, + struct libie_xg_tx_buffer *first, struct ixgbevf_ipsec_tx_data *itd) { return 0; } #endif /* CONFIG_IXGBEVF_IPSEC */ diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index 759592618549fb..c5fbde9b65d766 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -271,7 +271,7 @@ static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, struct ixgbevf_ring *tx_ring, int napi_budget) { struct ixgbevf_adapter *adapter = q_vector->adapter; - struct ixgbevf_tx_buffer *tx_buffer; + struct libie_xg_tx_buffer *tx_buffer; union ixgbe_adv_tx_desc *tx_desc; unsigned int total_bytes = 0, total_packets = 0, total_ipsec = 0; unsigned int budget = tx_ring->count / 2; @@ -1215,7 +1215,7 @@ void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, /* reinitialize tx_buffer_info */ if (!ring_is_xdp(ring)) { memset(ring->tx_buffer_info, 0, - sizeof(struct ixgbevf_tx_buffer) * ring->count); + sizeof(struct libie_xg_tx_buffer) * ring->count); } else { memset(ring->xdp_sqes, 0, sizeof(struct libeth_sqe) * ring->count); @@ -2058,7 +2058,7 @@ void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring) { u16 i = tx_ring->next_to_clean; - struct ixgbevf_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; + struct libie_xg_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; while (i != tx_ring->next_to_use) { union ixgbe_adv_tx_desc *eop_desc, *tx_desc; @@ -3072,7 +3072,7 @@ int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring) struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev); int size; - size = (!ring_is_xdp(tx_ring) ? sizeof(struct ixgbevf_tx_buffer) : + size = (!ring_is_xdp(tx_ring) ? sizeof(struct libie_xg_tx_buffer) : sizeof(struct libeth_sqe)) * tx_ring->count; tx_ring->tx_buffer_info = vmalloc(size); @@ -3415,7 +3415,7 @@ static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring, } static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, - struct ixgbevf_tx_buffer *first, + struct libie_xg_tx_buffer *first, u8 *hdr_len, struct ixgbevf_ipsec_tx_data *itd) { @@ -3512,7 +3512,7 @@ static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, } static void ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring, - struct ixgbevf_tx_buffer *first, + struct libie_xg_tx_buffer *first, struct ixgbevf_ipsec_tx_data *itd) { struct sk_buff *skb = first->skb; @@ -3608,11 +3608,11 @@ static void ixgbevf_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc, } static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring, - struct ixgbevf_tx_buffer *first, + struct libie_xg_tx_buffer *first, const u8 hdr_len) { struct sk_buff *skb = first->skb; - struct ixgbevf_tx_buffer *tx_buffer; + struct libie_xg_tx_buffer *tx_buffer; union ixgbe_adv_tx_desc *tx_desc; skb_frag_t *frag; dma_addr_t dma; @@ -3776,7 +3776,7 @@ static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) static int ixgbevf_xmit_frame_ring(struct sk_buff *skb, struct ixgbevf_ring *tx_ring) { - struct ixgbevf_tx_buffer *first; + struct libie_xg_tx_buffer *first; int tso; u32 tx_flags = 0; u16 count = TXD_USE_COUNT(skb_headlen(skb)); From de55c97e28bf0a7245065ec54b8f3cd2de615d5c Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Wed, 12 Aug 2026 12:26:06 +0200 Subject: [PATCH 23/39] ixgbevf, libie_xg: move selected XDP code to a libie_xg header Some functions used in ixgbevf XDP_TX and AF_XDP ZC implementation are at the same time non-trivial and trivially shareable with ixgbe. Move those functions to a new libie_xg header and rename them accordingly. Inline sharing was chosen, since a big chunk of those function are used in datapath, and a hypothetical small module would introduce more memory overhead than plain code duplication. Signed-off-by: Larysa Zaremba --- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 47 +--- .../ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h | 102 +------ .../net/ethernet/intel/ixgbevf/ixgbevf_xsk.c | 121 +------- include/linux/net/intel/libie/xg_xdp.h | 261 ++++++++++++++++++ 4 files changed, 277 insertions(+), 254 deletions(-) create mode 100644 include/linux/net/intel/libie/xg_xdp.h diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index c5fbde9b65d766..e5bf50a82268a8 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -503,54 +503,19 @@ static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, void ixgbevf_clean_xdp_ring(struct ixgbevf_ring *xdp_ring) { - ixgbevf_clean_xdp_num(xdp_ring, false, xdp_ring->pending); + libie_xg_clean_xdp_num(&xdp_ring->base, false, xdp_ring->pending, + ring_is_xsk(xdp_ring)); libeth_xdpsq_put(&xdp_ring->xdpq_lock, xdp_ring->netdev); } -static void ixgbevf_xdp_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i, - const struct libeth_xdpsq *sq, - u64 priv) -{ - union ixgbe_adv_tx_desc *tx_desc = - &((union ixgbe_adv_tx_desc *)sq->descs)[i]; - - u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA | - IXGBE_ADVTXD_DCMD_DEXT | - IXGBE_ADVTXD_DCMD_IFCS | - desc.len; - - if (desc.flags & LIBETH_XDP_TX_LAST) - cmd_type |= IXGBE_TXD_CMD_EOP; - - if (desc.flags & LIBETH_XDP_TX_FIRST) { - struct libeth_sqe *sqe = &sq->sqes[i]; - struct skb_shared_info *sinfo; - u16 full_len = desc.len; - - if (desc.flags & LIBETH_XDP_TX_MULTI) { - sinfo = sqe->type == LIBETH_SQE_XDP_TX ? - sqe->sinfo : - xdp_get_shared_info_from_frame(sqe->xdpf); - full_len += sinfo->xdp_frags_size; - } - - tx_desc->read.olinfo_status = - cpu_to_le32((full_len << IXGBE_ADVTXD_PAYLEN_SHIFT) | - IXGBE_ADVTXD_CC); - } - - tx_desc->read.buffer_addr = cpu_to_le64(desc.addr); - tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); -} - LIBETH_XDP_DEFINE_START(); LIBETH_XDP_DEFINE_FLUSH_TX(static ixgbevf_xdp_flush_tx, ixgbevf_prep_xdp_sq, - ixgbevf_xdp_xmit_desc); + libie_xg_xdp_xmit_desc); LIBETH_XDP_DEFINE_FLUSH_XMIT(static ixgbevf_xdp_flush_xmit, ixgbevf_prep_xdp_sq, - ixgbevf_xdp_xmit_desc); + libie_xg_xdp_xmit_desc); LIBETH_XDP_DEFINE_RUN_PROG(static ixgbevf_xdp_run_prog, ixgbevf_xdp_flush_tx); LIBETH_XDP_DEFINE_FINALIZE(static ixgbevf_xdp_finalize_xdp_napi, - ixgbevf_xdp_flush_tx, ixgbevf_xdp_rs_and_bump); + ixgbevf_xdp_flush_tx, libie_xg_xdp_rs_and_bump); LIBETH_XDP_DEFINE_END(); static u32 ixgbevf_rx_hsplit_wa(const struct libeth_fqe *hdr, @@ -713,7 +678,7 @@ static int ixgbevf_xdp_xmit(struct net_device *dev, int n, return libeth_xdp_xmit_do_bulk(dev, n, frames, flags, adapter->xdp_ring, adapter->num_xdp_queues, ixgbevf_xdp_flush_xmit, - ixgbevf_xdp_rs_and_bump); + libie_xg_xdp_rs_and_bump); } /** diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h index ad1e09ae6aa1c3..13d2be98bd7fa8 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h @@ -4,80 +4,21 @@ #ifndef _IXGBEVF_XDP_LIB_H_ #define _IXGBEVF_XDP_LIB_H_ -#include +#include #include "ixgbevf.h" -static inline u16 ixgbevf_tx_get_num_sent(struct ixgbevf_ring *xdp_ring) -{ - u16 ntc = xdp_ring->next_to_clean; - u16 to_clean = 0; - - while (likely(to_clean < xdp_ring->pending)) { - u32 idx = xdp_ring->xdp_sqes[ntc].rs_idx; - union ixgbe_adv_tx_desc *rs_desc; - - if (!idx--) - break; - - rs_desc = IXGBEVF_TX_DESC(xdp_ring, idx); - - if (!(rs_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD))) - break; - - xdp_ring->xdp_sqes[ntc].rs_idx = 0; - - to_clean += - (idx >= ntc ? idx : idx + xdp_ring->count) - ntc + 1; - - ntc = (idx + 1 == xdp_ring->count) ? 0 : idx + 1; - } - - return to_clean; -} - -static inline void ixgbevf_clean_xdp_num(struct ixgbevf_ring *xdp_ring, - bool in_napi, u16 to_clean) -{ - struct libeth_xdpsq_napi_stats stats = { }; - bool xsk_ring = ring_is_xsk(xdp_ring); - u32 ntc = xdp_ring->next_to_clean; - struct xdp_frame_bulk cbulk; - struct libeth_cq_pp cp = { - .bq = &cbulk, - .dev = xdp_ring->dev, - .xss = &stats, - .napi = in_napi, - }; - u32 xsk_frames = 0; - - xdp_frame_bulk_init(&cbulk); - xdp_ring->pending -= to_clean; - - while (likely(to_clean--)) { - xsk_frames += xsk_ring && - likely(!xdp_ring->xdp_sqes[ntc].type) ? 1 : 0; - libeth_xdp_complete_tx(&xdp_ring->xdp_sqes[ntc], &cp); - ntc++; - ntc = unlikely(ntc == xdp_ring->count) ? 0 : ntc; - } - - xdp_ring->next_to_clean = ntc; - xdp_flush_frame_bulk(&cbulk); - if (xsk_frames) - xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames); -} - static inline u32 ixgbevf_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq) { struct ixgbevf_ring *xdp_ring = xdpsq; libeth_xdpsq_lock(&xdp_ring->xdpq_lock); if (unlikely(ixgbevf_desc_unused(xdp_ring) < xdp_ring->thresh)) { - u16 to_clean = ixgbevf_tx_get_num_sent(xdpsq); + u16 to_clean = libie_xg_tx_get_num_sent(xdpsq); if (likely(to_clean)) - ixgbevf_clean_xdp_num(xdp_ring, true, to_clean); + libie_xg_clean_xdp_num(&xdp_ring->base, true, to_clean, + ring_is_xsk(xdp_ring)); } if (unlikely(!test_bit(__IXGBEVF_TX_XDP_RING_PRIMED, @@ -117,39 +58,4 @@ static inline u32 ixgbevf_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq) return ixgbevf_desc_unused(xdp_ring); } -static inline void ixgbevf_xdp_rs_and_bump(void *xdpsq, bool sent, bool flush) -{ - struct ixgbevf_ring *xdp_ring = xdpsq; - union ixgbe_adv_tx_desc *desc; - u32 ltu; - - libeth_xdpsq_lock(&xdp_ring->xdpq_lock); - - if ((!flush && xdp_ring->pending < xdp_ring->count - 1) || - xdp_ring->cached_ntu == xdp_ring->next_to_use) - goto unlock; - - ltu = (xdp_ring->next_to_use ? : xdp_ring->count) - 1; - - /* We will not get DD on a context descriptor */ - if (unlikely(xdp_ring->xdp_sqes[ltu].type == LIBETH_SQE_CTX)) - goto unlock; - - desc = IXGBEVF_TX_DESC(xdp_ring, ltu); - desc->read.cmd_type_len |= cpu_to_le32(IXGBE_TXD_CMD); - - xdp_ring->xdp_sqes[xdp_ring->cached_ntu].rs_idx = ltu + 1; - xdp_ring->cached_ntu = xdp_ring->next_to_use; - - /* In case the packet was interrupted, discard it */ - xdp_ring->xdp_sqes[ltu].priv = 0; - - /* Finish descriptor writes before bumping tail */ - wmb(); - ixgbevf_write_tail(xdp_ring, xdp_ring->next_to_use); - -unlock: - libeth_xdpsq_unlock(&xdp_ring->xdpq_lock); -} - #endif /* _IXGBEVF_XDP_LIB_H_ */ diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c index 0ef43b62ed932e..f5d372127d1049 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c @@ -86,131 +86,22 @@ int ixgbevf_setup_xsk_pool(struct ixgbevf_adapter *adapter, return err; } -static void ixgbevf_fill_rx_xsk_desc(const struct libeth_xskfq_fp *fq, u32 i) -{ - union ixgbe_adv_rx_desc *rx_desc = - &((union ixgbe_adv_rx_desc *)fq->descs)[i]; - - rx_desc->read.pkt_addr = - cpu_to_le64(libeth_xsk_buff_xdp_get_dma(fq->fqes[i])); - rx_desc->wb.upper.length = 0; -} - bool ixgbevf_xsk_alloc_rx_bufs(struct ixgbevf_ring *rx_ring, u32 num) { - struct libeth_xskfq_fp fq = { - .count = rx_ring->count, - .descs = rx_ring->desc, - .fqes = rx_ring->xsk_fqes, - .ntu = rx_ring->next_to_use, - .pool = rx_ring->xsk_pool, - }; - u32 done; - - done = libeth_xskfqe_alloc(&fq, num, ixgbevf_fill_rx_xsk_desc); - if (likely(done)) { - /* Finish descriptor writes before bumping tail */ - wmb(); - ixgbevf_write_tail(rx_ring, fq.ntu); - } - - /* Prevent ntc circling past ntu, e.g. in case of starvation */ - IXGBEVF_RX_DESC(rx_ring, fq.ntu)->wb.upper.length = 0; - - rx_ring->next_to_use = fq.ntu; - rx_ring->pending -= done; - - return done == num; + return libie_xg_xsk_alloc_rx_bufs(&rx_ring->base, num); } void ixgbevf_rx_xsk_ring_free_buffs(struct ixgbevf_ring *rx_ring) { - u32 ntc = rx_ring->next_to_clean; - - if (rx_ring->xsk_xdp_head) - xsk_buff_free(&rx_ring->xsk_xdp_head->base); - - rx_ring->xsk_xdp_head = NULL; - - while (ntc != rx_ring->next_to_use) { - xsk_buff_free(&rx_ring->xsk_fqes[ntc]->base); - ntc++; - ntc = ntc == rx_ring->count ? 0 : ntc; - } -} - -struct ixgbevf_zc_sqe_priv { - u16 first_desc; - u16 len; -}; - -LIBETH_SQE_CHECK_PRIV(struct ixgbevf_zc_sqe_priv); - -static void ixgbevf_xsk_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i, - const struct libeth_xdpsq *sq, u64 priv) -{ - union ixgbe_adv_tx_desc *descs = sq->descs, *tx_desc = &descs[i]; - u32 ltu = (i ? : sq->count) - 1; - - u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA | - IXGBE_ADVTXD_DCMD_DEXT | - IXGBE_ADVTXD_DCMD_IFCS | - desc.len; - - tx_desc->read.buffer_addr = cpu_to_le64(desc.addr); - - if (likely((desc.flags & LIBETH_XDP_TX_LAST) && !sq->sqes[ltu].priv)) { - tx_desc->read.olinfo_status = - cpu_to_le32((desc.len << IXGBE_ADVTXD_PAYLEN_SHIFT) | - IXGBE_ADVTXD_CC); - tx_desc->read.cmd_type_len = - cpu_to_le32(cmd_type | IXGBE_TXD_CMD_EOP); - return; - } - - /* No previous packet */ - if (!sq->sqes[ltu].priv) { - struct ixgbevf_zc_sqe_priv *sqe_priv = - (void *)&sq->sqes[i].priv; - - sqe_priv->first_desc = i; - sqe_priv->len = desc.len; - - tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); - - return; - } - - if (sq->sqes[ltu].priv) { - struct ixgbevf_zc_sqe_priv *sqe_priv = - (void *)&sq->sqes[i].priv; - - sq->sqes[i].priv = sq->sqes[ltu].priv; - sq->sqes[ltu].priv = 0; - sqe_priv->len += desc.len; - - if (desc.flags & LIBETH_XDP_TX_LAST) { - union ixgbe_adv_tx_desc *first_desc = - &descs[sqe_priv->first_desc]; - - first_desc->read.olinfo_status = - cpu_to_le32((sqe_priv->len << - IXGBE_ADVTXD_PAYLEN_SHIFT) | - IXGBE_ADVTXD_CC); - cmd_type |= IXGBE_TXD_CMD_EOP; - sq->sqes[i].priv = 0; - } - - tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); - } + return libie_xg_rx_xsk_ring_free_buffs(&rx_ring->base); } LIBETH_XDP_DEFINE_START(); LIBETH_XSK_DEFINE_FLUSH_TX(static ixgbevf_xsk_flush_tx, ixgbevf_prep_xdp_sq, - ixgbevf_xsk_xmit_desc); + libie_xg_xsk_xmit_desc); LIBETH_XSK_DEFINE_RUN_PROG(static ixgbevf_xsk_run_prog, ixgbevf_xsk_flush_tx); LIBETH_XSK_DEFINE_FINALIZE(static ixgbevf_xsk_finalize_xdp_napi, - ixgbevf_xsk_flush_tx, ixgbevf_xdp_rs_and_bump); + ixgbevf_xsk_flush_tx, libie_xg_xdp_rs_and_bump); LIBETH_XDP_DEFINE_END(); u32 ixgbevf_clean_xsk_rx_irq(struct ixgbevf_q_vector *q_vector, @@ -325,8 +216,8 @@ bool ixgbevf_clean_xsk_tx_irq(struct ixgbevf_q_vector *q_vector, return libeth_xsk_xmit_do_bulk(tx_ring->xsk_pool, tx_ring, budget, NULL, ixgbevf_prep_xdp_sq, - ixgbevf_xsk_xmit_desc, - ixgbevf_xdp_rs_and_bump); + libie_xg_xsk_xmit_desc, + libie_xg_xdp_rs_and_bump); } int ixgbevf_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) diff --git a/include/linux/net/intel/libie/xg_xdp.h b/include/linux/net/intel/libie/xg_xdp.h new file mode 100644 index 00000000000000..749934ac01c51e --- /dev/null +++ b/include/linux/net/intel/libie/xg_xdp.h @@ -0,0 +1,261 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2026 Intel Corporation */ + +#ifndef __LIBIE_XG_XDP_H +#define __LIBIE_XG_XDP_H + +#include +#include + +static inline void libie_xg_fill_rx_xsk_desc(const struct libeth_xskfq_fp *fq, u32 i) +{ + union ixgbe_adv_rx_desc *rx_desc = + &((union ixgbe_adv_rx_desc *)fq->descs)[i]; + + rx_desc->read.pkt_addr = + cpu_to_le64(libeth_xsk_buff_xdp_get_dma(fq->fqes[i])); + rx_desc->wb.upper.length = 0; +} + +static inline bool libie_xg_xsk_alloc_rx_bufs(struct libie_xg_ring *rx_ring, + u32 num) +{ + struct libeth_xskfq_fp fq = { + .count = rx_ring->count, + .descs = rx_ring->desc, + .fqes = rx_ring->xsk_fqes, + .ntu = rx_ring->next_to_use, + .pool = rx_ring->xsk_pool, + }; + u32 done; + + done = libeth_xskfqe_alloc(&fq, num, libie_xg_fill_rx_xsk_desc); + if (likely(done)) { + /* Finish descriptor writes before bumping tail */ + wmb(); + writel(fq.ntu, rx_ring->tail); + } + + /* Prevent ntc circling past ntu, e.g. in case of starvation */ + LIBIE_XG_RX_DESC(rx_ring, fq.ntu)->wb.upper.length = 0; + + rx_ring->next_to_use = fq.ntu; + rx_ring->pending -= done; + + return done == num; +} + +static inline void libie_xg_rx_xsk_ring_free_buffs(struct libie_xg_ring *rx_ring) +{ + u32 ntc = rx_ring->next_to_clean; + + if (rx_ring->xsk_xdp_head) + xsk_buff_free(&rx_ring->xsk_xdp_head->base); + + rx_ring->xsk_xdp_head = NULL; + + while (ntc != rx_ring->next_to_use) { + xsk_buff_free(&rx_ring->xsk_fqes[ntc]->base); + ntc++; + ntc = ntc == rx_ring->count ? 0 : ntc; + } +} + +struct libie_xg_zc_sqe_priv { + u16 first_desc; + u16 len; +}; + +LIBETH_SQE_CHECK_PRIV(struct libie_xg_zc_sqe_priv); + +static inline void libie_xg_xsk_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i, + const struct libeth_xdpsq *sq, + u64 priv) +{ + union ixgbe_adv_tx_desc *descs = sq->descs, *tx_desc = &descs[i]; + u32 ltu = (i ? : sq->count) - 1; + + u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA | + IXGBE_ADVTXD_DCMD_DEXT | + IXGBE_ADVTXD_DCMD_IFCS | + desc.len; + + tx_desc->read.buffer_addr = cpu_to_le64(desc.addr); + + if (likely((desc.flags & LIBETH_XDP_TX_LAST) && !sq->sqes[ltu].priv)) { + tx_desc->read.olinfo_status = + cpu_to_le32((desc.len << IXGBE_ADVTXD_PAYLEN_SHIFT) | + IXGBE_ADVTXD_CC); + tx_desc->read.cmd_type_len = + cpu_to_le32(cmd_type | IXGBE_TXD_CMD_EOP); + return; + } + + /* No previous packet */ + if (!sq->sqes[ltu].priv) { + struct libie_xg_zc_sqe_priv *sqe_priv = + (void *)&sq->sqes[i].priv; + + sqe_priv->first_desc = i; + sqe_priv->len = desc.len; + + tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); + + return; + } + + if (sq->sqes[ltu].priv) { + struct libie_xg_zc_sqe_priv *sqe_priv = + (void *)&sq->sqes[i].priv; + + sq->sqes[i].priv = sq->sqes[ltu].priv; + sq->sqes[ltu].priv = 0; + sqe_priv->len += desc.len; + + if (desc.flags & LIBETH_XDP_TX_LAST) { + union ixgbe_adv_tx_desc *first_desc = + &descs[sqe_priv->first_desc]; + + first_desc->read.olinfo_status = + cpu_to_le32((sqe_priv->len << + IXGBE_ADVTXD_PAYLEN_SHIFT) | + IXGBE_ADVTXD_CC); + cmd_type |= IXGBE_TXD_CMD_EOP; + sq->sqes[i].priv = 0; + } + + tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); + } +} + +static inline void libie_xg_xdp_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i, + const struct libeth_xdpsq *sq, + u64 priv) +{ + union ixgbe_adv_tx_desc *tx_desc = + &((union ixgbe_adv_tx_desc *)sq->descs)[i]; + + u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA | + IXGBE_ADVTXD_DCMD_DEXT | + IXGBE_ADVTXD_DCMD_IFCS | + desc.len; + + if (desc.flags & LIBETH_XDP_TX_LAST) + cmd_type |= IXGBE_TXD_CMD_EOP; + + if (desc.flags & LIBETH_XDP_TX_FIRST) { + struct libeth_sqe *sqe = &sq->sqes[i]; + struct skb_shared_info *sinfo; + u16 full_len = desc.len; + + if (desc.flags & LIBETH_XDP_TX_MULTI) { + sinfo = sqe->type == LIBETH_SQE_XDP_TX ? + sqe->sinfo : + xdp_get_shared_info_from_frame(sqe->xdpf); + full_len += sinfo->xdp_frags_size; + } + + tx_desc->read.olinfo_status = + cpu_to_le32((full_len << IXGBE_ADVTXD_PAYLEN_SHIFT) | + IXGBE_ADVTXD_CC); + } + + tx_desc->read.buffer_addr = cpu_to_le64(desc.addr); + tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); +} + +static inline u16 libie_xg_tx_get_num_sent(struct libie_xg_ring *xdp_ring) +{ + u16 ntc = xdp_ring->next_to_clean; + u16 to_clean = 0; + + while (likely(to_clean < xdp_ring->pending)) { + u32 idx = xdp_ring->xdp_sqes[ntc].rs_idx; + union ixgbe_adv_tx_desc *rs_desc; + + if (!idx--) + break; + + rs_desc = LIBIE_XG_TX_DESC(xdp_ring, idx); + + if (!(rs_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD))) + break; + + xdp_ring->xdp_sqes[ntc].rs_idx = 0; + + to_clean += + (idx >= ntc ? idx : idx + xdp_ring->count) - ntc + 1; + + ntc = (idx + 1 == xdp_ring->count) ? 0 : idx + 1; + } + + return to_clean; +} + +static inline void libie_xg_clean_xdp_num(struct libie_xg_ring *xdp_ring, + bool in_napi, u16 to_clean, bool xsk_ring) +{ + struct libeth_xdpsq_napi_stats stats = { }; + u32 ntc = xdp_ring->next_to_clean; + struct xdp_frame_bulk cbulk; + struct libeth_cq_pp cp = { + .bq = &cbulk, + .dev = xdp_ring->dev, + .xss = &stats, + .napi = in_napi, + }; + u32 xsk_frames = 0; + + xdp_frame_bulk_init(&cbulk); + xdp_ring->pending -= to_clean; + + while (likely(to_clean--)) { + xsk_frames += xsk_ring && + likely(!xdp_ring->xdp_sqes[ntc].type) ? 1 : 0; + libeth_xdp_complete_tx(&xdp_ring->xdp_sqes[ntc], &cp); + ntc++; + ntc = unlikely(ntc == xdp_ring->count) ? 0 : ntc; + } + + xdp_ring->next_to_clean = ntc; + xdp_flush_frame_bulk(&cbulk); + if (xsk_frames) + xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames); +} + +static inline void libie_xg_xdp_rs_and_bump(void *xdpsq, bool sent, bool flush) +{ + struct libie_xg_ring *xdp_ring = xdpsq; + union ixgbe_adv_tx_desc *desc; + u32 ltu; + + libeth_xdpsq_lock(&xdp_ring->xdpq_lock); + + if ((!flush && xdp_ring->pending < xdp_ring->count - 1) || + xdp_ring->cached_ntu == xdp_ring->next_to_use) + goto unlock; + + ltu = (xdp_ring->next_to_use ? : xdp_ring->count) - 1; + + /* We will not get DD on a context descriptor */ + if (unlikely(xdp_ring->xdp_sqes[ltu].type == LIBETH_SQE_CTX)) + goto unlock; + + desc = LIBIE_XG_TX_DESC(xdp_ring, ltu); + desc->read.cmd_type_len |= cpu_to_le32(IXGBE_TXD_CMD); + + xdp_ring->xdp_sqes[xdp_ring->cached_ntu].rs_idx = ltu + 1; + xdp_ring->cached_ntu = xdp_ring->next_to_use; + + /* In case the packet was interrupted, discard it */ + xdp_ring->xdp_sqes[ltu].priv = 0; + + /* Finish descriptor writes before bumping tail */ + wmb(); + writel(xdp_ring->next_to_use, xdp_ring->tail); + +unlock: + libeth_xdpsq_unlock(&xdp_ring->xdpq_lock); +} + +#endif \ No newline at end of file From b2dc0067e55f1d2dc44b68205d0d2b40a2b30ae7 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Tue, 18 Aug 2026 12:00:47 +0200 Subject: [PATCH 24/39] Revert "[SPLIT] Move descriptor definitions to a separate file + introduce and use libie_xg_ring" This reverts commit a66f091496171a00c16874e5098083785f16aa23. --- drivers/net/ethernet/intel/ixgbevf/defines.h | 191 ++++++++++++++++++ include/linux/net/intel/ixgbe_desc.h | 200 ------------------- 2 files changed, 191 insertions(+), 200 deletions(-) delete mode 100644 include/linux/net/intel/ixgbe_desc.h diff --git a/drivers/net/ethernet/intel/ixgbevf/defines.h b/drivers/net/ethernet/intel/ixgbevf/defines.h index a67d1ecd6012a6..afc927dd14381b 100644 --- a/drivers/net/ethernet/intel/ixgbevf/defines.h +++ b/drivers/net/ethernet/intel/ixgbevf/defines.h @@ -84,6 +84,197 @@ typedef u32 ixgbe_link_speed; #define IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS 0x0A000000 #define IXGBE_SRRCTL_DESCTYPE_MASK 0x0E000000 +/* Receive Descriptor bit definitions */ +#define IXGBE_RXD_STAT_DD 0x01 /* Descriptor Done */ +#define IXGBE_RXD_STAT_EOP 0x02 /* End of Packet */ +#define IXGBE_RXD_STAT_FLM 0x04 /* FDir Match */ +#define IXGBE_RXD_STAT_VP 0x08 /* IEEE VLAN Packet */ +#define IXGBE_RXDADV_NEXTP_MASK 0x000FFFF0 /* Next Descriptor Index */ +#define IXGBE_RXDADV_NEXTP_SHIFT 0x00000004 +#define IXGBE_RXD_STAT_UDPCS 0x10 /* UDP xsum calculated */ +#define IXGBE_RXD_STAT_L4CS 0x20 /* L4 xsum calculated */ +#define IXGBE_RXD_STAT_IPCS 0x40 /* IP xsum calculated */ +#define IXGBE_RXD_STAT_PIF 0x80 /* passed in-exact filter */ +#define IXGBE_RXD_STAT_CRCV 0x100 /* Speculative CRC Valid */ +#define IXGBE_RXD_STAT_VEXT 0x200 /* 1st VLAN found */ +#define IXGBE_RXD_STAT_UDPV 0x400 /* Valid UDP checksum */ +#define IXGBE_RXD_STAT_DYNINT 0x800 /* Pkt caused INT via DYNINT */ +#define IXGBE_RXD_STAT_TS 0x10000 /* Time Stamp */ +#define IXGBE_RXD_STAT_SECP 0x20000 /* Security Processing */ +#define IXGBE_RXD_STAT_LB 0x40000 /* Loopback Status */ +#define IXGBE_RXD_STAT_ACK 0x8000 /* ACK Packet indication */ +#define IXGBE_RXD_ERR_CE 0x01 /* CRC Error */ +#define IXGBE_RXD_ERR_LE 0x02 /* Length Error */ +#define IXGBE_RXD_ERR_PE 0x08 /* Packet Error */ +#define IXGBE_RXD_ERR_OSE 0x10 /* Oversize Error */ +#define IXGBE_RXD_ERR_USE 0x20 /* Undersize Error */ +#define IXGBE_RXD_ERR_TCPE 0x40 /* TCP/UDP Checksum Error */ +#define IXGBE_RXD_ERR_IPE 0x80 /* IP Checksum Error */ +#define IXGBE_RXDADV_ERR_MASK 0xFFF00000 /* RDESC.ERRORS mask */ +#define IXGBE_RXDADV_ERR_SHIFT 20 /* RDESC.ERRORS shift */ +#define IXGBE_RXDADV_ERR_HBO 0x00800000 /*Header Buffer Overflow */ +#define IXGBE_RXDADV_ERR_CE 0x01000000 /* CRC Error */ +#define IXGBE_RXDADV_ERR_LE 0x02000000 /* Length Error */ +#define IXGBE_RXDADV_ERR_PE 0x08000000 /* Packet Error */ +#define IXGBE_RXDADV_ERR_OSE 0x10000000 /* Oversize Error */ +#define IXGBE_RXDADV_ERR_USE 0x20000000 /* Undersize Error */ +#define IXGBE_RXDADV_ERR_TCPE 0x40000000 /* TCP/UDP Checksum Error */ +#define IXGBE_RXDADV_ERR_IPE 0x80000000 /* IP Checksum Error */ +#define IXGBE_RXD_VLAN_ID_MASK 0x0FFF /* VLAN ID is in lower 12 bits */ +#define IXGBE_RXD_PRI_MASK 0xE000 /* Priority is in upper 3 bits */ +#define IXGBE_RXD_PRI_SHIFT 13 +#define IXGBE_RXD_CFI_MASK 0x1000 /* CFI is bit 12 */ +#define IXGBE_RXD_CFI_SHIFT 12 + +#define IXGBE_RXDADV_STAT_DD IXGBE_RXD_STAT_DD /* Done */ +#define IXGBE_RXDADV_STAT_EOP IXGBE_RXD_STAT_EOP /* End of Packet */ +#define IXGBE_RXDADV_STAT_FLM IXGBE_RXD_STAT_FLM /* FDir Match */ +#define IXGBE_RXDADV_STAT_VP IXGBE_RXD_STAT_VP /* IEEE VLAN Pkt */ +#define IXGBE_RXDADV_STAT_MASK 0x000FFFFF /* Stat/NEXTP: bit 0-19 */ +#define IXGBE_RXDADV_STAT_FCEOFS 0x00000040 /* FCoE EOF/SOF Stat */ +#define IXGBE_RXDADV_STAT_FCSTAT 0x00000030 /* FCoE Pkt Stat */ +#define IXGBE_RXDADV_STAT_FCSTAT_NOMTCH 0x00000000 /* 00: No Ctxt Match */ +#define IXGBE_RXDADV_STAT_FCSTAT_NODDP 0x00000010 /* 01: Ctxt w/o DDP */ +#define IXGBE_RXDADV_STAT_FCSTAT_FCPRSP 0x00000020 /* 10: Recv. FCP_RSP */ +#define IXGBE_RXDADV_STAT_FCSTAT_DDP 0x00000030 /* 11: Ctxt w/ DDP */ +#define IXGBE_RXDADV_STAT_SECP 0x00020000 /* IPsec/MACsec pkt found */ + +#define IXGBE_RXDADV_RSSTYPE_MASK 0x0000000F +#define IXGBE_RXDADV_PKTTYPE_MASK 0x0000FFF0 +#define IXGBE_RXDADV_PKTTYPE_IPV4 0x00000010 /* IPv4 hdr present */ +#define IXGBE_RXDADV_PKTTYPE_IPV6 0x00000040 /* IPv6 hdr present */ +#define IXGBE_RXDADV_PKTTYPE_IPSEC_ESP 0x00001000 /* IPSec ESP */ +#define IXGBE_RXDADV_PKTTYPE_IPSEC_AH 0x00002000 /* IPSec AH */ +#define IXGBE_RXDADV_PKTTYPE_MASK_EX 0x0001FFF0 +#define IXGBE_RXDADV_HDRBUFLEN_MASK 0x00007FE0 +#define IXGBE_RXDADV_RSCCNT_MASK 0x001E0000 +#define IXGBE_RXDADV_RSCCNT_SHIFT 17 +#define IXGBE_RXDADV_HDRBUFLEN_SHIFT 5 +#define IXGBE_RXDADV_SPLITHEADER_EN 0x00001000 +#define IXGBE_RXDADV_SPH 0x8000 + +/* RSS Hash results */ +#define IXGBE_RXDADV_RSSTYPE_NONE 0x00000000 +#define IXGBE_RXDADV_RSSTYPE_IPV4_TCP 0x00000001 +#define IXGBE_RXDADV_RSSTYPE_IPV4 0x00000002 +#define IXGBE_RXDADV_RSSTYPE_IPV6_TCP 0x00000003 +#define IXGBE_RXDADV_RSSTYPE_IPV6_EX 0x00000004 +#define IXGBE_RXDADV_RSSTYPE_IPV6 0x00000005 +#define IXGBE_RXDADV_RSSTYPE_IPV6_TCP_EX 0x00000006 +#define IXGBE_RXDADV_RSSTYPE_IPV4_UDP 0x00000007 +#define IXGBE_RXDADV_RSSTYPE_IPV6_UDP 0x00000008 +#define IXGBE_RXDADV_RSSTYPE_IPV6_UDP_EX 0x00000009 + +#define IXGBE_RXD_ERR_FRAME_ERR_MASK ( \ + IXGBE_RXD_ERR_CE | \ + IXGBE_RXD_ERR_LE | \ + IXGBE_RXD_ERR_PE | \ + IXGBE_RXD_ERR_OSE | \ + IXGBE_RXD_ERR_USE) + +#define IXGBE_RXDADV_ERR_FRAME_ERR_MASK ( \ + IXGBE_RXDADV_ERR_CE | \ + IXGBE_RXDADV_ERR_LE | \ + IXGBE_RXDADV_ERR_PE | \ + IXGBE_RXDADV_ERR_OSE | \ + IXGBE_RXDADV_ERR_USE) + +#define IXGBE_TXD_POPTS_IXSM 0x01 /* Insert IP checksum */ +#define IXGBE_TXD_POPTS_TXSM 0x02 /* Insert TCP/UDP checksum */ +#define IXGBE_TXD_CMD_EOP 0x01000000 /* End of Packet */ +#define IXGBE_TXD_CMD_IFCS 0x02000000 /* Insert FCS (Ethernet CRC) */ +#define IXGBE_TXD_CMD_IC 0x04000000 /* Insert Checksum */ +#define IXGBE_TXD_CMD_RS 0x08000000 /* Report Status */ +#define IXGBE_TXD_CMD_DEXT 0x20000000 /* Descriptor ext (0 = legacy) */ +#define IXGBE_TXD_CMD_VLE 0x40000000 /* Add VLAN tag */ +#define IXGBE_TXD_STAT_DD 0x00000001 /* Descriptor Done */ +#define IXGBE_TXD_CMD (IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS) + +/* Transmit Descriptor - Advanced */ +union ixgbe_adv_tx_desc { + struct { + __le64 buffer_addr; /* Address of descriptor's data buf */ + __le32 cmd_type_len; + __le32 olinfo_status; + } read; + struct { + __le64 rsvd; /* Reserved */ + __le32 nxtseq_seed; + __le32 status; + } wb; +}; + +/* Receive Descriptor - Advanced */ +union ixgbe_adv_rx_desc { + struct { + __le64 pkt_addr; /* Packet buffer address */ + __le64 hdr_addr; /* Header buffer address */ + } read; + struct { + struct { + union { + __le32 data; + struct { + __le16 pkt_info; /* RSS, Pkt type */ + __le16 hdr_info; /* Splithdr, hdrlen */ + } hs_rss; + } lo_dword; + union { + __le32 rss; /* RSS Hash */ + struct { + __le16 ip_id; /* IP id */ + __le16 csum; /* Packet Checksum */ + } csum_ip; + } hi_dword; + } lower; + struct { + __le32 status_error; /* ext status/error */ + __le16 length; /* Packet length */ + __le16 vlan; /* VLAN tag */ + } upper; + } wb; /* writeback */ +}; + +/* Context descriptors */ +struct ixgbe_adv_tx_context_desc { + __le32 vlan_macip_lens; + __le32 fceof_saidx; + __le32 type_tucmd_mlhl; + __le32 mss_l4len_idx; +}; + +/* Adv Transmit Descriptor Config Masks */ +#define IXGBE_ADVTXD_DTYP_MASK 0x00F00000 /* DTYP mask */ +#define IXGBE_ADVTXD_DTYP_CTXT 0x00200000 /* Advanced Context Desc */ +#define IXGBE_ADVTXD_DTYP_DATA 0x00300000 /* Advanced Data Descriptor */ +#define IXGBE_ADVTXD_DCMD_EOP IXGBE_TXD_CMD_EOP /* End of Packet */ +#define IXGBE_ADVTXD_DCMD_IFCS IXGBE_TXD_CMD_IFCS /* Insert FCS */ +#define IXGBE_ADVTXD_DCMD_RS IXGBE_TXD_CMD_RS /* Report Status */ +#define IXGBE_ADVTXD_DCMD_DEXT IXGBE_TXD_CMD_DEXT /* Desc ext (1=Adv) */ +#define IXGBE_ADVTXD_DCMD_VLE IXGBE_TXD_CMD_VLE /* VLAN pkt enable */ +#define IXGBE_ADVTXD_DCMD_TSE 0x80000000 /* TCP Seg enable */ +#define IXGBE_ADVTXD_STAT_DD IXGBE_TXD_STAT_DD /* Descriptor Done */ +#define IXGBE_ADVTXD_TUCMD_IPV4 0x00000400 /* IP Packet Type: 1=IPv4 */ +#define IXGBE_ADVTXD_TUCMD_IPV6 0x00000000 /* IP Packet Type: 0=IPv6 */ +#define IXGBE_ADVTXD_TUCMD_L4T_UDP 0x00000000 /* L4 Packet TYPE of UDP */ +#define IXGBE_ADVTXD_TUCMD_L4T_TCP 0x00000800 /* L4 Packet TYPE of TCP */ +#define IXGBE_ADVTXD_TUCMD_L4T_SCTP 0x00001000 /* L4 Packet TYPE of SCTP */ +#define IXGBE_ADVTXD_TUCMD_IPSEC_TYPE_ESP 0x00002000 /* IPSec Type ESP */ +#define IXGBE_ADVTXD_TUCMD_IPSEC_ENCRYPT_EN 0x00004000 /* ESP Encrypt Enable */ +#define IXGBE_ADVTXD_IDX_SHIFT 4 /* Adv desc Index shift */ +#define IXGBE_ADVTXD_CC 0x00000080 /* Check Context */ +#define IXGBE_ADVTXD_POPTS_SHIFT 8 /* Adv desc POPTS shift */ +#define IXGBE_ADVTXD_POPTS_IPSEC 0x00000400 /* IPSec offload request */ +#define IXGBE_ADVTXD_POPTS_IXSM (IXGBE_TXD_POPTS_IXSM << \ + IXGBE_ADVTXD_POPTS_SHIFT) +#define IXGBE_ADVTXD_POPTS_TXSM (IXGBE_TXD_POPTS_TXSM << \ + IXGBE_ADVTXD_POPTS_SHIFT) +#define IXGBE_ADVTXD_PAYLEN_SHIFT 14 /* Adv desc PAYLEN shift */ +#define IXGBE_ADVTXD_MACLEN_SHIFT 9 /* Adv ctxt desc mac len shift */ +#define IXGBE_ADVTXD_VLAN_SHIFT 16 /* Adv ctxt vlan tag shift */ +#define IXGBE_ADVTXD_L4LEN_SHIFT 8 /* Adv ctxt L4LEN shift */ +#define IXGBE_ADVTXD_MSS_SHIFT 16 /* Adv ctxt MSS shift */ + /* Interrupt register bitmasks */ #define IXGBE_EITR_CNT_WDIS 0x80000000 diff --git a/include/linux/net/intel/ixgbe_desc.h b/include/linux/net/intel/ixgbe_desc.h deleted file mode 100644 index fdf8a35044f539..00000000000000 --- a/include/linux/net/intel/ixgbe_desc.h +++ /dev/null @@ -1,200 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright (C) 2026 Intel Corporation */ - -#ifndef __IXGBE_DESC_H -#define __IXGBE_DESC_H - -#include - -/* Receive Descriptor bit definitions */ -#define IXGBE_RXD_STAT_DD 0x01 /* Descriptor Done */ -#define IXGBE_RXD_STAT_EOP 0x02 /* End of Packet */ -#define IXGBE_RXD_STAT_FLM 0x04 /* FDir Match */ -#define IXGBE_RXD_STAT_VP 0x08 /* IEEE VLAN Packet */ -#define IXGBE_RXDADV_NEXTP_MASK 0x000FFFF0 /* Next Descriptor Index */ -#define IXGBE_RXDADV_NEXTP_SHIFT 0x00000004 -#define IXGBE_RXD_STAT_UDPCS 0x10 /* UDP xsum calculated */ -#define IXGBE_RXD_STAT_L4CS 0x20 /* L4 xsum calculated */ -#define IXGBE_RXD_STAT_IPCS 0x40 /* IP xsum calculated */ -#define IXGBE_RXD_STAT_PIF 0x80 /* passed in-exact filter */ -#define IXGBE_RXD_STAT_CRCV 0x100 /* Speculative CRC Valid */ -#define IXGBE_RXD_STAT_VEXT 0x200 /* 1st VLAN found */ -#define IXGBE_RXD_STAT_UDPV 0x400 /* Valid UDP checksum */ -#define IXGBE_RXD_STAT_DYNINT 0x800 /* Pkt caused INT via DYNINT */ -#define IXGBE_RXD_STAT_TS 0x10000 /* Time Stamp */ -#define IXGBE_RXD_STAT_SECP 0x20000 /* Security Processing */ -#define IXGBE_RXD_STAT_LB 0x40000 /* Loopback Status */ -#define IXGBE_RXD_STAT_ACK 0x8000 /* ACK Packet indication */ -#define IXGBE_RXD_ERR_CE 0x01 /* CRC Error */ -#define IXGBE_RXD_ERR_LE 0x02 /* Length Error */ -#define IXGBE_RXD_ERR_PE 0x08 /* Packet Error */ -#define IXGBE_RXD_ERR_OSE 0x10 /* Oversize Error */ -#define IXGBE_RXD_ERR_USE 0x20 /* Undersize Error */ -#define IXGBE_RXD_ERR_TCPE 0x40 /* TCP/UDP Checksum Error */ -#define IXGBE_RXD_ERR_IPE 0x80 /* IP Checksum Error */ -#define IXGBE_RXDADV_ERR_MASK 0xFFF00000 /* RDESC.ERRORS mask */ -#define IXGBE_RXDADV_ERR_SHIFT 20 /* RDESC.ERRORS shift */ -#define IXGBE_RXDADV_ERR_HBO 0x00800000 /*Header Buffer Overflow */ -#define IXGBE_RXDADV_ERR_CE 0x01000000 /* CRC Error */ -#define IXGBE_RXDADV_ERR_LE 0x02000000 /* Length Error */ -#define IXGBE_RXDADV_ERR_PE 0x08000000 /* Packet Error */ -#define IXGBE_RXDADV_ERR_OSE 0x10000000 /* Oversize Error */ -#define IXGBE_RXDADV_ERR_USE 0x20000000 /* Undersize Error */ -#define IXGBE_RXDADV_ERR_TCPE 0x40000000 /* TCP/UDP Checksum Error */ -#define IXGBE_RXDADV_ERR_IPE 0x80000000 /* IP Checksum Error */ -#define IXGBE_RXD_VLAN_ID_MASK 0x0FFF /* VLAN ID is in lower 12 bits */ -#define IXGBE_RXD_PRI_MASK 0xE000 /* Priority is in upper 3 bits */ -#define IXGBE_RXD_PRI_SHIFT 13 -#define IXGBE_RXD_CFI_MASK 0x1000 /* CFI is bit 12 */ -#define IXGBE_RXD_CFI_SHIFT 12 - -#define IXGBE_RXDADV_STAT_DD IXGBE_RXD_STAT_DD /* Done */ -#define IXGBE_RXDADV_STAT_EOP IXGBE_RXD_STAT_EOP /* End of Packet */ -#define IXGBE_RXDADV_STAT_FLM IXGBE_RXD_STAT_FLM /* FDir Match */ -#define IXGBE_RXDADV_STAT_VP IXGBE_RXD_STAT_VP /* IEEE VLAN Pkt */ -#define IXGBE_RXDADV_STAT_MASK 0x000FFFFF /* Stat/NEXTP: bit 0-19 */ -#define IXGBE_RXDADV_STAT_FCEOFS 0x00000040 /* FCoE EOF/SOF Stat */ -#define IXGBE_RXDADV_STAT_FCSTAT 0x00000030 /* FCoE Pkt Stat */ -#define IXGBE_RXDADV_STAT_FCSTAT_NOMTCH 0x00000000 /* 00: No Ctxt Match */ -#define IXGBE_RXDADV_STAT_FCSTAT_NODDP 0x00000010 /* 01: Ctxt w/o DDP */ -#define IXGBE_RXDADV_STAT_FCSTAT_FCPRSP 0x00000020 /* 10: Recv. FCP_RSP */ -#define IXGBE_RXDADV_STAT_FCSTAT_DDP 0x00000030 /* 11: Ctxt w/ DDP */ -#define IXGBE_RXDADV_STAT_SECP 0x00020000 /* IPsec/MACsec pkt found */ - -#define IXGBE_RXDADV_RSSTYPE_MASK 0x0000000F -#define IXGBE_RXDADV_PKTTYPE_MASK 0x0000FFF0 -#define IXGBE_RXDADV_PKTTYPE_IPV4 0x00000010 /* IPv4 hdr present */ -#define IXGBE_RXDADV_PKTTYPE_IPV6 0x00000040 /* IPv6 hdr present */ -#define IXGBE_RXDADV_PKTTYPE_IPSEC_ESP 0x00001000 /* IPSec ESP */ -#define IXGBE_RXDADV_PKTTYPE_IPSEC_AH 0x00002000 /* IPSec AH */ -#define IXGBE_RXDADV_PKTTYPE_MASK_EX 0x0001FFF0 -#define IXGBE_RXDADV_HDRBUFLEN_MASK 0x00007FE0 -#define IXGBE_RXDADV_RSCCNT_MASK 0x001E0000 -#define IXGBE_RXDADV_RSCCNT_SHIFT 17 -#define IXGBE_RXDADV_HDRBUFLEN_SHIFT 5 -#define IXGBE_RXDADV_SPLITHEADER_EN 0x00001000 -#define IXGBE_RXDADV_SPH 0x8000 - -/* RSS Hash results */ -#define IXGBE_RXDADV_RSSTYPE_NONE 0x00000000 -#define IXGBE_RXDADV_RSSTYPE_IPV4_TCP 0x00000001 -#define IXGBE_RXDADV_RSSTYPE_IPV4 0x00000002 -#define IXGBE_RXDADV_RSSTYPE_IPV6_TCP 0x00000003 -#define IXGBE_RXDADV_RSSTYPE_IPV6_EX 0x00000004 -#define IXGBE_RXDADV_RSSTYPE_IPV6 0x00000005 -#define IXGBE_RXDADV_RSSTYPE_IPV6_TCP_EX 0x00000006 -#define IXGBE_RXDADV_RSSTYPE_IPV4_UDP 0x00000007 -#define IXGBE_RXDADV_RSSTYPE_IPV6_UDP 0x00000008 -#define IXGBE_RXDADV_RSSTYPE_IPV6_UDP_EX 0x00000009 - -#define IXGBE_RXD_ERR_FRAME_ERR_MASK ( \ - IXGBE_RXD_ERR_CE | \ - IXGBE_RXD_ERR_LE | \ - IXGBE_RXD_ERR_PE | \ - IXGBE_RXD_ERR_OSE | \ - IXGBE_RXD_ERR_USE) - -#define IXGBE_RXDADV_ERR_FRAME_ERR_MASK ( \ - IXGBE_RXDADV_ERR_CE | \ - IXGBE_RXDADV_ERR_LE | \ - IXGBE_RXDADV_ERR_PE | \ - IXGBE_RXDADV_ERR_OSE | \ - IXGBE_RXDADV_ERR_USE) - -#define IXGBE_TXD_POPTS_IXSM 0x01 /* Insert IP checksum */ -#define IXGBE_TXD_POPTS_TXSM 0x02 /* Insert TCP/UDP checksum */ -#define IXGBE_TXD_CMD_EOP 0x01000000 /* End of Packet */ -#define IXGBE_TXD_CMD_IFCS 0x02000000 /* Insert FCS (Ethernet CRC) */ -#define IXGBE_TXD_CMD_IC 0x04000000 /* Insert Checksum */ -#define IXGBE_TXD_CMD_RS 0x08000000 /* Report Status */ -#define IXGBE_TXD_CMD_DEXT 0x20000000 /* Descriptor ext (0 = legacy) */ -#define IXGBE_TXD_CMD_VLE 0x40000000 /* Add VLAN tag */ -#define IXGBE_TXD_STAT_DD 0x00000001 /* Descriptor Done */ -#define IXGBE_TXD_CMD (IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS) - -/* Transmit Descriptor - Advanced */ -union ixgbe_adv_tx_desc { - struct { - __le64 buffer_addr; /* Address of descriptor's data buf */ - __le32 cmd_type_len; - __le32 olinfo_status; - } read; - struct { - __le64 rsvd; /* Reserved */ - __le32 nxtseq_seed; - __le32 status; - } wb; -}; - -/* Receive Descriptor - Advanced */ -union ixgbe_adv_rx_desc { - struct { - __le64 pkt_addr; /* Packet buffer address */ - __le64 hdr_addr; /* Header buffer address */ - } read; - struct { - struct { - union { - __le32 data; - struct { - __le16 pkt_info; /* RSS, Pkt type */ - __le16 hdr_info; /* Splithdr, hdrlen */ - } hs_rss; - } lo_dword; - union { - __le32 rss; /* RSS Hash */ - struct { - __le16 ip_id; /* IP id */ - __le16 csum; /* Packet Checksum */ - } csum_ip; - } hi_dword; - } lower; - struct { - __le32 status_error; /* ext status/error */ - __le16 length; /* Packet length */ - __le16 vlan; /* VLAN tag */ - } upper; - } wb; /* writeback */ -}; - -/* Context descriptors */ -struct ixgbe_adv_tx_context_desc { - __le32 vlan_macip_lens; - __le32 fceof_saidx; - __le32 type_tucmd_mlhl; - __le32 mss_l4len_idx; -}; - -/* Adv Transmit Descriptor Config Masks */ -#define IXGBE_ADVTXD_DTYP_MASK 0x00F00000 /* DTYP mask */ -#define IXGBE_ADVTXD_DTYP_CTXT 0x00200000 /* Advanced Context Desc */ -#define IXGBE_ADVTXD_DTYP_DATA 0x00300000 /* Advanced Data Descriptor */ -#define IXGBE_ADVTXD_DCMD_EOP IXGBE_TXD_CMD_EOP /* End of Packet */ -#define IXGBE_ADVTXD_DCMD_IFCS IXGBE_TXD_CMD_IFCS /* Insert FCS */ -#define IXGBE_ADVTXD_DCMD_RS IXGBE_TXD_CMD_RS /* Report Status */ -#define IXGBE_ADVTXD_DCMD_DEXT IXGBE_TXD_CMD_DEXT /* Desc ext (1=Adv) */ -#define IXGBE_ADVTXD_DCMD_VLE IXGBE_TXD_CMD_VLE /* VLAN pkt enable */ -#define IXGBE_ADVTXD_DCMD_TSE 0x80000000 /* TCP Seg enable */ -#define IXGBE_ADVTXD_STAT_DD IXGBE_TXD_STAT_DD /* Descriptor Done */ -#define IXGBE_ADVTXD_TUCMD_IPV4 0x00000400 /* IP Packet Type: 1=IPv4 */ -#define IXGBE_ADVTXD_TUCMD_IPV6 0x00000000 /* IP Packet Type: 0=IPv6 */ -#define IXGBE_ADVTXD_TUCMD_L4T_UDP 0x00000000 /* L4 Packet TYPE of UDP */ -#define IXGBE_ADVTXD_TUCMD_L4T_TCP 0x00000800 /* L4 Packet TYPE of TCP */ -#define IXGBE_ADVTXD_TUCMD_L4T_SCTP 0x00001000 /* L4 Packet TYPE of SCTP */ -#define IXGBE_ADVTXD_TUCMD_IPSEC_TYPE_ESP 0x00002000 /* IPSec Type ESP */ -#define IXGBE_ADVTXD_TUCMD_IPSEC_ENCRYPT_EN 0x00004000 /* ESP Encrypt Enable */ -#define IXGBE_ADVTXD_IDX_SHIFT 4 /* Adv desc Index shift */ -#define IXGBE_ADVTXD_CC 0x00000080 /* Check Context */ -#define IXGBE_ADVTXD_POPTS_SHIFT 8 /* Adv desc POPTS shift */ -#define IXGBE_ADVTXD_POPTS_IPSEC 0x00000400 /* IPSec offload request */ -#define IXGBE_ADVTXD_POPTS_IXSM (IXGBE_TXD_POPTS_IXSM << \ - IXGBE_ADVTXD_POPTS_SHIFT) -#define IXGBE_ADVTXD_POPTS_TXSM (IXGBE_TXD_POPTS_TXSM << \ - IXGBE_ADVTXD_POPTS_SHIFT) -#define IXGBE_ADVTXD_PAYLEN_SHIFT 14 /* Adv desc PAYLEN shift */ -#define IXGBE_ADVTXD_MACLEN_SHIFT 9 /* Adv ctxt desc mac len shift */ -#define IXGBE_ADVTXD_VLAN_SHIFT 16 /* Adv ctxt vlan tag shift */ -#define IXGBE_ADVTXD_L4LEN_SHIFT 8 /* Adv ctxt L4LEN shift */ -#define IXGBE_ADVTXD_MSS_SHIFT 16 /* Adv ctxt MSS shift */ - -#endif From 69b5631df814033e8c3e56824d11b7e819c71849 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Tue, 18 Aug 2026 13:15:10 +0200 Subject: [PATCH 25/39] fixup! Revert "[SPLIT] Move descriptor definitions to a separate file + introduce and use libie_xg_ring" --- .../ethernet/intel/ixgbe/ixgbe_txrx_common.h | 3 - drivers/net/ethernet/intel/ixgbe/ixgbe_type.h | 56 +---------- drivers/net/ethernet/intel/ixgbevf/defines.h | 58 +---------- include/linux/net/intel/ixgbe_desc_common.h | 99 +++++++++++++++++++ 4 files changed, 102 insertions(+), 114 deletions(-) create mode 100644 include/linux/net/intel/ixgbe_desc_common.h diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h index 78deea5ec5369c..f681f95377980c 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h @@ -10,9 +10,6 @@ #define IXGBE_XDP_REDIR BIT(2) #define IXGBE_XDP_EXIT BIT(3) -#define IXGBE_TXD_CMD (IXGBE_TXD_CMD_EOP | \ - IXGBE_TXD_CMD_RS) - int ixgbe_xmit_xdp_ring(struct ixgbe_ring *ring, struct xdp_frame *xdpf); bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring, diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h index 825cf17fa1878d..914f10ce4b3367 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include "ixgbe_type_e610.h" @@ -2358,13 +2359,8 @@ enum { #define IXGBE_TXD_POPTS_IXSM 0x01 /* Insert IP checksum */ #define IXGBE_TXD_POPTS_TXSM 0x02 /* Insert TCP/UDP checksum */ -#define IXGBE_TXD_CMD_EOP 0x01000000 /* End of Packet */ -#define IXGBE_TXD_CMD_IFCS 0x02000000 /* Insert FCS (Ethernet CRC) */ #define IXGBE_TXD_CMD_IC 0x04000000 /* Insert Checksum */ -#define IXGBE_TXD_CMD_RS 0x08000000 /* Report Status */ -#define IXGBE_TXD_CMD_DEXT 0x20000000 /* Descriptor extension (0 = legacy) */ #define IXGBE_TXD_CMD_VLE 0x40000000 /* Add VLAN tag */ -#define IXGBE_TXD_STAT_DD 0x00000001 /* Descriptor Done */ /* Multiple Transmit Queue Command Register */ #define IXGBE_MTQC_RT_ENA 0x1 /* DCB Enable */ @@ -2884,51 +2880,6 @@ struct ixgbe_hic_phy_activity_resp { __be32 data[FW_PHY_ACT_DATA_COUNT]; }; -/* Transmit Descriptor - Advanced */ -union ixgbe_adv_tx_desc { - struct { - __le64 buffer_addr; /* Address of descriptor's data buf */ - __le32 cmd_type_len; - __le32 olinfo_status; - } read; - struct { - __le64 rsvd; /* Reserved */ - __le32 nxtseq_seed; - __le32 status; - } wb; -}; - -/* Receive Descriptor - Advanced */ -union ixgbe_adv_rx_desc { - struct { - __le64 pkt_addr; /* Packet buffer address */ - __le64 hdr_addr; /* Header buffer address */ - } read; - struct { - struct { - union { - __le32 data; - struct { - __le16 pkt_info; /* RSS, Pkt type */ - __le16 hdr_info; /* Splithdr, hdrlen */ - } hs_rss; - } lo_dword; - union { - __le32 rss; /* RSS Hash */ - struct { - __le16 ip_id; /* IP id */ - __le16 csum; /* Packet Checksum */ - } csum_ip; - } hi_dword; - } lower; - struct { - __le32 status_error; /* ext status/error */ - __le16 length; /* Packet length */ - __le16 vlan; /* VLAN tag */ - } upper; - } wb; /* writeback */ -}; - /* Context descriptors */ struct ixgbe_adv_tx_context_desc { __le32 vlan_macip_lens; @@ -2952,19 +2903,15 @@ enum { #define IXGBE_ADVTXD_IPSEC_ESP_LEN_MASK 0x000001FF /* IPSec ESP length */ #define IXGBE_ADVTXD_DTYP_MASK 0x00F00000 /* DTYP mask */ #define IXGBE_ADVTXD_DTYP_CTXT 0x2 /* Advanced Context Desc */ -#define IXGBE_ADVTXD_DTYP_DATA 0x00300000 /* Advanced Data Descriptor */ #define IXGBE_ADVTXD_DCMD_EOP IXGBE_TXD_CMD_EOP /* End of Packet */ -#define IXGBE_ADVTXD_DCMD_IFCS IXGBE_TXD_CMD_IFCS /* Insert FCS */ #define IXGBE_ADVTXD_DCMD_RS IXGBE_TXD_CMD_RS /* Report Status */ #define IXGBE_ADVTXD_DCMD_DDTYP_ISCSI 0x10000000 /* DDP hdr type or iSCSI */ -#define IXGBE_ADVTXD_DCMD_DEXT IXGBE_TXD_CMD_DEXT /* Desc ext (1=Adv) */ #define IXGBE_ADVTXD_DCMD_VLE IXGBE_TXD_CMD_VLE /* VLAN pkt enable */ #define IXGBE_ADVTXD_DCMD_TSE 0x80000000 /* TCP Seg enable */ #define IXGBE_ADVTXD_STAT_DD IXGBE_TXD_STAT_DD /* Descriptor Done */ #define IXGBE_ADVTXD_STAT_SN_CRC 0x00000002 /* NXTSEQ/SEED pres in WB */ #define IXGBE_ADVTXD_STAT_RSV 0x0000000C /* STA Reserved */ #define IXGBE_ADVTXD_IDX_SHIFT 4 /* Adv desc Index shift */ -#define IXGBE_ADVTXD_CC 0x00000080 /* Check Context */ #define IXGBE_ADVTXD_POPTS_SHIFT 8 /* Adv desc POPTS shift */ #define IXGBE_ADVTXD_POPTS_IXSM (IXGBE_TXD_POPTS_IXSM << \ IXGBE_ADVTXD_POPTS_SHIFT) @@ -2976,7 +2923,6 @@ enum { #define IXGBE_ADVTXD_POPTS_ISCO_LAST 0x00001000 /* Last TSO of iSCSI PDU */ #define IXGBE_ADVTXD_POPTS_ISCO_FULL 0x00001800 /* 1st&Last TSO-full iSCSI PDU */ #define IXGBE_ADVTXD_POPTS_RSV 0x00002000 /* POPTS Reserved */ -#define IXGBE_ADVTXD_PAYLEN_SHIFT 14 /* Adv desc PAYLEN shift */ #define IXGBE_ADVTXD_MACLEN_SHIFT 9 /* Adv ctxt desc mac len shift */ #define IXGBE_ADVTXD_VLAN_SHIFT 16 /* Adv ctxt vlan tag shift */ #define IXGBE_ADVTXD_TUCMD_IPV4 0x00000400 /* IP Packet Type: 1=IPv4 */ diff --git a/drivers/net/ethernet/intel/ixgbevf/defines.h b/drivers/net/ethernet/intel/ixgbevf/defines.h index afc927dd14381b..b3554b75c591e8 100644 --- a/drivers/net/ethernet/intel/ixgbevf/defines.h +++ b/drivers/net/ethernet/intel/ixgbevf/defines.h @@ -4,6 +4,8 @@ #ifndef _IXGBEVF_DEFINES_H_ #define _IXGBEVF_DEFINES_H_ +#include + /* Device IDs */ #define IXGBE_DEV_ID_82599_VF 0x10ED #define IXGBE_DEV_ID_X540_VF 0x1515 @@ -181,59 +183,8 @@ typedef u32 ixgbe_link_speed; #define IXGBE_TXD_POPTS_IXSM 0x01 /* Insert IP checksum */ #define IXGBE_TXD_POPTS_TXSM 0x02 /* Insert TCP/UDP checksum */ -#define IXGBE_TXD_CMD_EOP 0x01000000 /* End of Packet */ -#define IXGBE_TXD_CMD_IFCS 0x02000000 /* Insert FCS (Ethernet CRC) */ #define IXGBE_TXD_CMD_IC 0x04000000 /* Insert Checksum */ -#define IXGBE_TXD_CMD_RS 0x08000000 /* Report Status */ -#define IXGBE_TXD_CMD_DEXT 0x20000000 /* Descriptor ext (0 = legacy) */ #define IXGBE_TXD_CMD_VLE 0x40000000 /* Add VLAN tag */ -#define IXGBE_TXD_STAT_DD 0x00000001 /* Descriptor Done */ -#define IXGBE_TXD_CMD (IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS) - -/* Transmit Descriptor - Advanced */ -union ixgbe_adv_tx_desc { - struct { - __le64 buffer_addr; /* Address of descriptor's data buf */ - __le32 cmd_type_len; - __le32 olinfo_status; - } read; - struct { - __le64 rsvd; /* Reserved */ - __le32 nxtseq_seed; - __le32 status; - } wb; -}; - -/* Receive Descriptor - Advanced */ -union ixgbe_adv_rx_desc { - struct { - __le64 pkt_addr; /* Packet buffer address */ - __le64 hdr_addr; /* Header buffer address */ - } read; - struct { - struct { - union { - __le32 data; - struct { - __le16 pkt_info; /* RSS, Pkt type */ - __le16 hdr_info; /* Splithdr, hdrlen */ - } hs_rss; - } lo_dword; - union { - __le32 rss; /* RSS Hash */ - struct { - __le16 ip_id; /* IP id */ - __le16 csum; /* Packet Checksum */ - } csum_ip; - } hi_dword; - } lower; - struct { - __le32 status_error; /* ext status/error */ - __le16 length; /* Packet length */ - __le16 vlan; /* VLAN tag */ - } upper; - } wb; /* writeback */ -}; /* Context descriptors */ struct ixgbe_adv_tx_context_desc { @@ -246,11 +197,8 @@ struct ixgbe_adv_tx_context_desc { /* Adv Transmit Descriptor Config Masks */ #define IXGBE_ADVTXD_DTYP_MASK 0x00F00000 /* DTYP mask */ #define IXGBE_ADVTXD_DTYP_CTXT 0x00200000 /* Advanced Context Desc */ -#define IXGBE_ADVTXD_DTYP_DATA 0x00300000 /* Advanced Data Descriptor */ #define IXGBE_ADVTXD_DCMD_EOP IXGBE_TXD_CMD_EOP /* End of Packet */ -#define IXGBE_ADVTXD_DCMD_IFCS IXGBE_TXD_CMD_IFCS /* Insert FCS */ #define IXGBE_ADVTXD_DCMD_RS IXGBE_TXD_CMD_RS /* Report Status */ -#define IXGBE_ADVTXD_DCMD_DEXT IXGBE_TXD_CMD_DEXT /* Desc ext (1=Adv) */ #define IXGBE_ADVTXD_DCMD_VLE IXGBE_TXD_CMD_VLE /* VLAN pkt enable */ #define IXGBE_ADVTXD_DCMD_TSE 0x80000000 /* TCP Seg enable */ #define IXGBE_ADVTXD_STAT_DD IXGBE_TXD_STAT_DD /* Descriptor Done */ @@ -262,14 +210,12 @@ struct ixgbe_adv_tx_context_desc { #define IXGBE_ADVTXD_TUCMD_IPSEC_TYPE_ESP 0x00002000 /* IPSec Type ESP */ #define IXGBE_ADVTXD_TUCMD_IPSEC_ENCRYPT_EN 0x00004000 /* ESP Encrypt Enable */ #define IXGBE_ADVTXD_IDX_SHIFT 4 /* Adv desc Index shift */ -#define IXGBE_ADVTXD_CC 0x00000080 /* Check Context */ #define IXGBE_ADVTXD_POPTS_SHIFT 8 /* Adv desc POPTS shift */ #define IXGBE_ADVTXD_POPTS_IPSEC 0x00000400 /* IPSec offload request */ #define IXGBE_ADVTXD_POPTS_IXSM (IXGBE_TXD_POPTS_IXSM << \ IXGBE_ADVTXD_POPTS_SHIFT) #define IXGBE_ADVTXD_POPTS_TXSM (IXGBE_TXD_POPTS_TXSM << \ IXGBE_ADVTXD_POPTS_SHIFT) -#define IXGBE_ADVTXD_PAYLEN_SHIFT 14 /* Adv desc PAYLEN shift */ #define IXGBE_ADVTXD_MACLEN_SHIFT 9 /* Adv ctxt desc mac len shift */ #define IXGBE_ADVTXD_VLAN_SHIFT 16 /* Adv ctxt vlan tag shift */ #define IXGBE_ADVTXD_L4LEN_SHIFT 8 /* Adv ctxt L4LEN shift */ diff --git a/include/linux/net/intel/ixgbe_desc_common.h b/include/linux/net/intel/ixgbe_desc_common.h new file mode 100644 index 00000000000000..144d5333d798c3 --- /dev/null +++ b/include/linux/net/intel/ixgbe_desc_common.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2026 Intel Corporation */ + +#ifndef __IXGBE_DESC_COMMON_H +#define __IXGBE_DESC_COMMON_H + +#include + +/** + * union ixgbe_adv_rx_desc - Receive Descriptor - Advanced + * @read: read descriptor + * @read.pkt_addr: packet buffer address + * @read.hdr_addr: header buffer address + * @wb: writeback descriptor + * @wb.lower: lower 8 bytes + * @wb.lower.lo_dword: lowest 4 bytes + * @wb.lower.lo_dword.data: lower 4 bytes + * @wb.lower.lo_dword.hs_rss: RSS packet type and header split info + * @wb.lower.lo_dword.hs_rss.pkt_info: RSS Packet Type + * @wb.lower.lo_dword.hs_rss.hdr_info: header split info + * @wb.lower.hi_dword: higher 4 bytes of lower 8 bytes + * @wb.lower.hi_dword.rss: RSS hash + * @wb.lower.hi_dword.csum_ip: IP ID and packet checksum + * @wb.lower.hi_dword.csum_ip.ip_id: IP ID + * @wb.lower.hi_dword.csum_ip.csum: packet checksum + * @wb.upper: upper 8 bytes + * @wb.upper.status_error: reported status and errors + * @wb.upper.length: packet length + * @wb.upper.vlan: VLAN tag + */ +union ixgbe_adv_rx_desc { + struct { + __le64 pkt_addr; + __le64 hdr_addr; + } read; + struct { + struct { + union { + __le32 data; + struct { + __le16 pkt_info; + __le16 hdr_info; + } hs_rss; + } lo_dword; + union { + __le32 rss; + struct { + __le16 ip_id; + __le16 csum; + } csum_ip; + } hi_dword; + } lower; + struct { + __le32 status_error; + __le16 length; + __le16 vlan; + } upper; + } wb; +}; + +/** + * union ixgbe_adv_tx_desc - Transmit Descriptor - Advanced + * @read: read descriptor + * @read.buffer_addr: address of the descriptor's data buffer + * @read.cmd_type_len: length and basic frame flags (EoP, RS, etc.) + * @read.olinfo_status: per-packet offload information + * @wb: writeback descriptor + * @wb.rsvd1: reserved + * @wb.rsvd2: reserved + * @wb.status: completion status + */ +union ixgbe_adv_tx_desc { + struct { + __le64 buffer_addr; + __le32 cmd_type_len; + __le32 olinfo_status; + } read; + struct { + __le64 rsvd1; + __le32 rsvd2; + __le32 status; + } wb; +}; + +#define IXGBE_TXD_CMD_EOP 0x01000000 /* End of Packet */ +#define IXGBE_TXD_CMD_IFCS 0x02000000 /* Insert FCS (Ethernet CRC) */ +#define IXGBE_TXD_CMD_RS 0x08000000 /* Report Status */ +#define IXGBE_TXD_CMD_DEXT 0x20000000 /* Descriptor ext (0 = legacy) */ +#define IXGBE_TXD_STAT_DD 0x00000001 /* Descriptor Done */ +#define IXGBE_TXD_CMD (IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS) + +#define IXGBE_ADVTXD_DTYP_DATA 0x00300000 /* Advanced Data Descriptor */ +#define IXGBE_ADVTXD_DCMD_IFCS IXGBE_TXD_CMD_IFCS /* Insert FCS */ +#define IXGBE_ADVTXD_DCMD_DEXT IXGBE_TXD_CMD_DEXT /* Desc ext (1=Adv) */ +#define IXGBE_ADVTXD_CC 0x00000080 /* Check Context */ + +#define IXGBE_ADVTXD_PAYLEN_SHIFT 14 /* Adv desc PAYLEN shift */ + +#endif /* __IXGBE_DESC_COMMON_H */ From a5cf86e2045ed9d8723b840e524900b587afa8ac Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Tue, 18 Aug 2026 13:18:16 +0200 Subject: [PATCH 26/39] fixup! ixgbevf, libie: separate ixgbevf_ring base and move to libie_xg --- include/linux/net/intel/libie/xg_ring.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/net/intel/libie/xg_ring.h b/include/linux/net/intel/libie/xg_ring.h index 768861c5e0688c..671780c85824e8 100644 --- a/include/linux/net/intel/libie/xg_ring.h +++ b/include/linux/net/intel/libie/xg_ring.h @@ -8,7 +8,7 @@ #include #include -#include +#include struct libie_xg_queue_stats { u64 packets; From 0a893243c0465563272f2e29757afd6c6f053578 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Tue, 18 Aug 2026 15:06:49 +0200 Subject: [PATCH 27/39] fixup! ixgbevf, libie: separate ixgbevf_ring base and move to libie_xg --- include/linux/net/intel/libie/xg_ring.h | 29 ++----------------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/include/linux/net/intel/libie/xg_ring.h b/include/linux/net/intel/libie/xg_ring.h index 671780c85824e8..04c3814e531158 100644 --- a/include/linux/net/intel/libie/xg_ring.h +++ b/include/linux/net/intel/libie/xg_ring.h @@ -4,33 +4,10 @@ #ifndef __LIBIE_XG_RING_H #define __LIBIE_XG_RING_H -#include +#include #include #include -#include - -struct libie_xg_queue_stats { - u64 packets; - u64 bytes; -}; - -struct libie_xg_tx_queue_stats { - u64 restart_queue; - u64 tx_busy; - u64 tx_done_old; -}; - -struct libie_xg_rx_queue_stats { - /* u64 rsc_count; <- was in ixgbe - u64 rsc_flush; */ - u64 non_eop_descs; - u64 alloc_rx_page; - u64 alloc_rx_page_failed; - u64 alloc_rx_buff_failed; - u64 csum_err; -}; - struct libie_xg_tx_buffer { union ixgbe_adv_tx_desc *next_to_watch; unsigned long time_stamp; @@ -76,7 +53,5 @@ struct libie_xg_ring { (&(((union ixgbe_adv_rx_desc *)((R)->desc))[i])) #define LIBIE_XG_TX_DESC(R, i) \ (&(((union ixgbe_adv_tx_desc *)((R)->desc))[i])) -#define LIBIE_XG_TX_CTXTDESC(R, i) \ - (&(((struct ixgbe_adv_tx_context_desc *)((R)->desc))[i])) -#endif \ No newline at end of file +#endif /* __LIBIE_XG_RING_H */ From 2c742865da5263eaf346396d15eb21f54441b21e Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Tue, 18 Aug 2026 15:18:08 +0200 Subject: [PATCH 28/39] ixgbe, libie: use libie_xg_ring in ixgbe Replace overlapping fields in ixgbe_ring with libie_xg_ring, included both as anonymous and named structure. For now, add xdp_frame to the generic Tx buffer, but it will be rendered unnecessary by further commits. Signed-off-by: Larysa Zaremba --- drivers/net/ethernet/intel/ixgbe/ixgbe.h | 45 ++++--------------- .../net/ethernet/intel/ixgbe/ixgbe_ethtool.c | 2 +- drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c | 2 +- .../net/ethernet/intel/ixgbe/ixgbe_ipsec.c | 2 +- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 28 ++++++------ drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c | 8 ++-- include/linux/net/intel/libie/xg_ring.h | 2 +- 7 files changed, 31 insertions(+), 58 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index af801eacc45c65..ea69a056b3f5ae 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -32,6 +32,7 @@ #endif #include "ixgbe_ipsec.h" +#include #include #include #include @@ -196,23 +197,6 @@ struct vf_macvlans { #define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD) #define DESC_NEEDED (MAX_SKB_FRAGS + 4) -/* wrapper around a pointer to a socket buffer, - * so a DMA handle can be stored along with the buffer */ -struct ixgbe_tx_buffer { - union ixgbe_adv_tx_desc *next_to_watch; - unsigned long time_stamp; - union { - struct sk_buff *skb; - struct xdp_frame *xdpf; - }; - unsigned int bytecount; - unsigned short gso_segs; - __be16 protocol; - DEFINE_DMA_UNMAP_ADDR(dma); - DEFINE_DMA_UNMAP_LEN(len); - u32 tx_flags; -}; - struct ixgbe_xsk_rx_buffer { bool discard; struct xdp_buff *xdp; @@ -280,36 +264,25 @@ struct ixgbe_fwd_adapter { #define clear_ring_xdp(ring) \ clear_bit(__IXGBE_TX_XDP_RING, &(ring)->state) struct ixgbe_ring { + union { + struct libie_xg_ring base; + struct libie_xg_ring; + }; struct ixgbe_ring *next; /* pointer to next ring in q_vector */ struct ixgbe_q_vector *q_vector; /* backpointer to host q_vector */ struct net_device *netdev; /* netdev ring belongs to */ struct bpf_prog __rcu *xdp_prog; - union { - struct page_pool *pp; /* Rx ring */ - struct device *dev; /* Tx ring */ - }; - void *desc; /* descriptor ring memory */ - union { - struct libeth_fqe *rx_fqes; - struct ixgbe_xsk_rx_buffer *rx_xsk_buffer_info; - struct ixgbe_tx_buffer *tx_buffer_info; - }; unsigned long state; - u8 __iomem *tail; dma_addr_t dma; /* phys. address of descriptor ring */ unsigned int size; /* length in bytes */ u32 truesize; - u16 count; /* amount of descriptors */ - u8 queue_index; /* needed for multiqueue queue management */ u8 reg_idx; /* holds the special value that gets * the hardware register offset * associated with this ring, which is * different for DCB and RSS modes */ - u16 next_to_use; - u16 next_to_clean; unsigned long last_rx_timestamp; @@ -327,10 +300,10 @@ struct ixgbe_ring { }; struct xdp_rxq_info xdp_rxq; spinlock_t tx_lock; /* used in XDP mode */ - struct xsk_buff_pool *xsk_pool; u16 ring_idx; /* {rx,tx,xdp}_ring back reference idx */ u32 rx_buf_len; struct libeth_xdp_buff_stash xdp_stash; + struct ixgbe_xsk_rx_buffer *rx_xsk_buffer_info; } ____cacheline_internodealigned_in_smp; enum ixgbe_ring_f_enum { @@ -923,7 +896,7 @@ int ixgbe_sysfs_init(struct ixgbe_adapter *adapter); #endif /* CONFIG_IXGBE_HWMON */ #ifdef IXGBE_FCOE void ixgbe_configure_fcoe(struct ixgbe_adapter *adapter); -int ixgbe_fso(struct ixgbe_ring *tx_ring, struct ixgbe_tx_buffer *first, +int ixgbe_fso(struct ixgbe_ring *tx_ring, struct libie_xg_tx_buffer *first, u8 *hdr_len); int ixgbe_fcoe_ddp(struct ixgbe_adapter *adapter, union ixgbe_adv_rx_desc *rx_desc, struct sk_buff *skb); @@ -1012,7 +985,7 @@ void ixgbe_ipsec_restore(struct ixgbe_adapter *adapter); void ixgbe_ipsec_rx(struct ixgbe_ring *rx_ring, union ixgbe_adv_rx_desc *rx_desc, struct sk_buff *skb); -int ixgbe_ipsec_tx(struct ixgbe_ring *tx_ring, struct ixgbe_tx_buffer *first, +int ixgbe_ipsec_tx(struct ixgbe_ring *tx_ring, struct libie_xg_tx_buffer *first, struct ixgbe_ipsec_tx_data *itd); void ixgbe_ipsec_vf_clear(struct ixgbe_adapter *adapter, u32 vf); int ixgbe_ipsec_vf_add_sa(struct ixgbe_adapter *adapter, u32 *mbuf, u32 vf); @@ -1025,7 +998,7 @@ static inline void ixgbe_ipsec_rx(struct ixgbe_ring *rx_ring, union ixgbe_adv_rx_desc *rx_desc, struct sk_buff *skb) { } static inline int ixgbe_ipsec_tx(struct ixgbe_ring *tx_ring, - struct ixgbe_tx_buffer *first, + struct libie_xg_tx_buffer *first, struct ixgbe_ipsec_tx_data *itd) { return 0; } static inline void ixgbe_ipsec_vf_clear(struct ixgbe_adapter *adapter, u32 vf) { } diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c index a2cc9b39eb634c..903053c892a203 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c @@ -2087,8 +2087,8 @@ static u16 ixgbe_clean_test_rings(struct ixgbe_ring *rx_ring, rx_desc = IXGBE_RX_DESC(rx_ring, rx_ntc); while (tx_ntc != tx_ring->next_to_use) { + struct libie_xg_tx_buffer *tx_buffer; union ixgbe_adv_tx_desc *tx_desc; - struct ixgbe_tx_buffer *tx_buffer; tx_desc = IXGBE_TX_DESC(tx_ring, tx_ntc); diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c index 011fda9c61939c..4660c62bb3e7e8 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c @@ -486,7 +486,7 @@ int ixgbe_fcoe_ddp(struct ixgbe_adapter *adapter, * Returns : 0 indicates success, < 0 for error */ int ixgbe_fso(struct ixgbe_ring *tx_ring, - struct ixgbe_tx_buffer *first, + struct libie_xg_tx_buffer *first, u8 *hdr_len) { struct sk_buff *skb = first->skb; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c index d1f4073b36f906..8e2c01daf7adb1 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c @@ -1054,7 +1054,7 @@ int ixgbe_ipsec_vf_del_sa(struct ixgbe_adapter *adapter, u32 *msgbuf, u32 vf) * @itd: ipsec Tx data for later use in building context descriptor **/ int ixgbe_ipsec_tx(struct ixgbe_ring *tx_ring, - struct ixgbe_tx_buffer *first, + struct libie_xg_tx_buffer *first, struct ixgbe_ipsec_tx_data *itd) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(tx_ring->netdev); diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index a59939ba094b2a..9ccff868a24040 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -569,7 +569,7 @@ static void ixgbe_regdump(struct ixgbe_hw *hw, struct ixgbe_reg_info *reginfo) static void ixgbe_print_buffer(struct ixgbe_ring *ring, int n) { - struct ixgbe_tx_buffer *tx_buffer; + struct libie_xg_tx_buffer *tx_buffer; tx_buffer = &ring->tx_buffer_info[ring->next_to_clean]; pr_info(" %5d %5X %5X %016llX %08X %p %016llX\n", @@ -590,7 +590,7 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) struct ixgbe_reg_info *reginfo; int n = 0; struct ixgbe_ring *ring; - struct ixgbe_tx_buffer *tx_buffer; + struct libie_xg_tx_buffer *tx_buffer; union ixgbe_adv_tx_desc *tx_desc; struct my_u0 { u64 a; u64 b; } *u0; struct ixgbe_ring *rx_ring; @@ -1348,7 +1348,7 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, struct ixgbe_ring *tx_ring, int napi_budget) { struct ixgbe_adapter *adapter = q_vector->adapter; - struct ixgbe_tx_buffer *tx_buffer; + struct libie_xg_tx_buffer *tx_buffer; union ixgbe_adv_tx_desc *tx_desc; unsigned int total_bytes = 0, total_packets = 0, total_ipsec = 0; unsigned int budget = q_vector->tx.work_limit; @@ -3493,7 +3493,7 @@ void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter, /* reinitialize tx_buffer_info */ memset(ring->tx_buffer_info, 0, - sizeof(struct ixgbe_tx_buffer) * ring->count); + sizeof(struct libie_xg_tx_buffer) * ring->count); /* enable queue */ IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl); @@ -6002,7 +6002,7 @@ void ixgbe_reset(struct ixgbe_adapter *adapter) static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring) { u16 i = tx_ring->next_to_clean; - struct ixgbe_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; + struct libie_xg_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; if (tx_ring->xsk_pool) { ixgbe_xsk_clean_tx_ring(tx_ring); @@ -6488,7 +6488,7 @@ int ixgbe_setup_tx_resources(struct ixgbe_ring *tx_ring) int ring_node = NUMA_NO_NODE; int size; - size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count; + size = sizeof(struct libie_xg_tx_buffer) * tx_ring->count; if (tx_ring->q_vector) ring_node = tx_ring->q_vector->numa_node; @@ -8151,7 +8151,7 @@ static void ixgbe_service_task(struct work_struct *work) } static int ixgbe_tso(struct ixgbe_ring *tx_ring, - struct ixgbe_tx_buffer *first, + struct libie_xg_tx_buffer *first, u8 *hdr_len, struct ixgbe_ipsec_tx_data *itd) { @@ -8257,7 +8257,7 @@ static int ixgbe_tso(struct ixgbe_ring *tx_ring, } static void ixgbe_tx_csum(struct ixgbe_ring *tx_ring, - struct ixgbe_tx_buffer *first, + struct libie_xg_tx_buffer *first, struct ixgbe_ipsec_tx_data *itd) { struct sk_buff *skb = first->skb; @@ -8386,11 +8386,11 @@ static inline int ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size) } static int ixgbe_tx_map(struct ixgbe_ring *tx_ring, - struct ixgbe_tx_buffer *first, + struct libie_xg_tx_buffer *first, const u8 hdr_len) { struct sk_buff *skb = first->skb; - struct ixgbe_tx_buffer *tx_buffer; + struct libie_xg_tx_buffer *tx_buffer; union ixgbe_adv_tx_desc *tx_desc; skb_frag_t *frag; dma_addr_t dma; @@ -8540,7 +8540,7 @@ static int ixgbe_tx_map(struct ixgbe_ring *tx_ring, } static void ixgbe_atr(struct ixgbe_ring *ring, - struct ixgbe_tx_buffer *first) + struct libie_xg_tx_buffer *first) { struct ixgbe_q_vector *q_vector = ring->q_vector; union ixgbe_atr_hash_dword input = { .dword = 0 }; @@ -8739,8 +8739,8 @@ int ixgbe_xmit_xdp_ring(struct ixgbe_ring *ring, struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf); u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0; u16 i = 0, index = ring->next_to_use; - struct ixgbe_tx_buffer *tx_head = &ring->tx_buffer_info[index]; - struct ixgbe_tx_buffer *tx_buff = tx_head; + struct libie_xg_tx_buffer *tx_head = &ring->tx_buffer_info[index]; + struct libie_xg_tx_buffer *tx_buff = tx_head; union ixgbe_adv_tx_desc *tx_desc = IXGBE_TX_DESC(ring, index); u32 cmd_type, len = xdpf->len; void *data = xdpf->data; @@ -8819,7 +8819,7 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, struct ixgbe_adapter *adapter, struct ixgbe_ring *tx_ring) { - struct ixgbe_tx_buffer *first; + struct libie_xg_tx_buffer *first; int tso; u32 tx_flags = 0; unsigned short f; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c index 15425d3c367872..c84753e2bf904c 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c @@ -393,7 +393,7 @@ static bool ixgbe_xmit_zc(struct ixgbe_ring *xdp_ring, unsigned int budget) { struct xsk_buff_pool *pool = xdp_ring->xsk_pool; union ixgbe_adv_tx_desc *tx_desc = NULL; - struct ixgbe_tx_buffer *tx_bi; + struct libie_xg_tx_buffer *tx_bi; bool work_done = true; struct xdp_desc desc; dma_addr_t dma; @@ -447,7 +447,7 @@ static bool ixgbe_xmit_zc(struct ixgbe_ring *xdp_ring, unsigned int budget) } static void ixgbe_clean_xdp_tx_buffer(struct ixgbe_ring *tx_ring, - struct ixgbe_tx_buffer *tx_bi) + struct libie_xg_tx_buffer *tx_bi) { xdp_return_frame(tx_bi->xdpf); dma_unmap_single(tx_ring->dev, @@ -463,7 +463,7 @@ bool ixgbe_clean_xdp_tx_irq(struct ixgbe_q_vector *q_vector, unsigned int total_packets = 0, total_bytes = 0; struct xsk_buff_pool *pool = tx_ring->xsk_pool; union ixgbe_adv_tx_desc *tx_desc; - struct ixgbe_tx_buffer *tx_bi; + struct libie_xg_tx_buffer *tx_bi; u32 xsk_frames = 0; tx_bi = &tx_ring->tx_buffer_info[ntc]; @@ -544,7 +544,7 @@ void ixgbe_xsk_clean_tx_ring(struct ixgbe_ring *tx_ring) { u16 ntc = tx_ring->next_to_clean, ntu = tx_ring->next_to_use; struct xsk_buff_pool *pool = tx_ring->xsk_pool; - struct ixgbe_tx_buffer *tx_bi; + struct libie_xg_tx_buffer *tx_bi; u32 xsk_frames = 0; while (ntc != ntu) { diff --git a/include/linux/net/intel/libie/xg_ring.h b/include/linux/net/intel/libie/xg_ring.h index 04c3814e531158..3c9fde6b8d739c 100644 --- a/include/linux/net/intel/libie/xg_ring.h +++ b/include/linux/net/intel/libie/xg_ring.h @@ -13,8 +13,8 @@ struct libie_xg_tx_buffer { unsigned long time_stamp; union { struct sk_buff *skb; - /* XDP uses address ptr on irq_clean */ void *data; + struct xdp_frame *xdpf; }; unsigned int bytecount; unsigned short gso_segs; From eaa0d9ceb83d01f4cea53c5c8bd932b486e204f5 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Wed, 19 Aug 2026 10:13:53 +0200 Subject: [PATCH 29/39] ixgbe: simplify xsk wakeup Wakeup is not necessarily tied to Tx processing, so Tx not being active is not a reason to skip on Rx buffer refill. Also, some checks are redundant. Rely on Rx rings to access the interrupt, and clear up the redundant code. Signed-off-by: Larysa Zaremba --- drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c index c84753e2bf904c..cae45570d3e36a 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c @@ -517,18 +517,12 @@ int ixgbe_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) if (test_bit(__IXGBE_DOWN, &adapter->state)) return -ENETDOWN; - if (!READ_ONCE(adapter->xdp_prog)) - return -EINVAL; - if (qid >= adapter->num_xdp_queues) return -EINVAL; - ring = adapter->xdp_ring[qid]; - - if (test_bit(__IXGBE_TX_DISABLED, &ring->state)) - return -ENETDOWN; + ring = adapter->rx_ring[qid]; - if (!ring->xsk_pool) + if (unlikely(!ring->xsk_pool)) return -EINVAL; if (!napi_if_scheduled_mark_missed(&ring->q_vector->napi)) { From 3fd633cddfc23eebff0aebe9c6d24baefcfe233d Mon Sep 17 00:00:00 2001 From: Magdalena Pytel Date: Mon, 1 Jun 2026 15:34:38 +0200 Subject: [PATCH 30/39] ixgbe: add pseudo header split Introduce pseudo header split support in the ixgbe driver. * Added IXGBE_RX_SRRCTL_BUF_SIZE macro and IXGBE_FLAG2_HSPLIT flag * Added hdr_truesize, hdr_fqes, hdr_pp, hdr_buf_len to ixgbe_ring * Added ixgbe_rx_hsplit_wa() to copy header from data to header buffer * Added ixgbe_rx_create_pp()/ixgbe_rx_destroy_pp() helpers * Updated ixgbe_alloc_rx_buffers to allocate header buffers * Updated ixgbe_clean_rx_irq to process header split * Refactored ixgbe_setup_rx_resources to use ixgbe_rx_create_pp * Refactored ixgbe_free_rx_resources to use ixgbe_rx_destroy_pp * Set IXGBE_FLAG2_HSPLIT for 82599 in ixgbe_sw_init * Updated ixgbe_xdp_setup to require mbuf frags with hsplit Signed-off-by: Magdalena Pytel --- drivers/net/ethernet/intel/ixgbe/ixgbe.h | 7 + drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 187 ++++++++++++++---- 2 files changed, 157 insertions(+), 37 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index ea69a056b3f5ae..991c846dcd778c 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -78,6 +78,8 @@ #define IXGBE_RX_PAGE_LEN(hr) (ALIGN_DOWN(LIBETH_RX_PAGE_LEN(hr), \ IXGBE_SRRCTL_BSIZEPKT_STEP)) +#define IXGBE_RX_SRRCTL_BUF_SIZE(mtu) (ALIGN((mtu) + LIBETH_RX_LL_LEN, \ + IXGBE_SRRCTL_BSIZEPKT_STEP)) /* Default HW header buffer size */ #define IXGBE_RX_HDR_SIZE IXGBE_RXBUFFER_256 @@ -276,6 +278,7 @@ struct ixgbe_ring { dma_addr_t dma; /* phys. address of descriptor ring */ unsigned int size; /* length in bytes */ u32 truesize; + u32 hdr_truesize; /* Rx header buffer full size */ u8 queue_index; /* needed for multiqueue queue management */ u8 reg_idx; /* holds the special value that gets @@ -298,9 +301,12 @@ struct ixgbe_ring { struct ixgbe_tx_queue_stats tx_stats; struct ixgbe_rx_queue_stats rx_stats; }; + struct libeth_fqe *hdr_fqes; + struct page_pool *hdr_pp; struct xdp_rxq_info xdp_rxq; spinlock_t tx_lock; /* used in XDP mode */ u16 ring_idx; /* {rx,tx,xdp}_ring back reference idx */ + u32 hdr_buf_len; u32 rx_buf_len; struct libeth_xdp_buff_stash xdp_stash; struct ixgbe_xsk_rx_buffer *rx_xsk_buffer_info; @@ -556,6 +562,7 @@ struct ixgbe_adapter { #define IXGBE_FLAG2_MOD_POWER_UNSUPPORTED BIT(22) #define IXGBE_FLAG2_API_MISMATCH BIT(23) #define IXGBE_FLAG2_FW_ROLLBACK BIT(24) +#define IXGBE_FLAG2_HSPLIT BIT(25) /* Tx fast path data */ int num_tx_queues; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 9ccff868a24040..40d3248297a098 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -1742,6 +1742,12 @@ void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) .truesize = rx_ring->truesize, .count = rx_ring->count, }; + const struct libeth_fq_fp hdr_fq = { + .pp = rx_ring->hdr_pp, + .fqes = rx_ring->hdr_fqes, + .truesize = rx_ring->hdr_truesize, + .count = rx_ring->count, + }; u16 ntu = rx_ring->next_to_use; /* nothing to do */ @@ -1758,6 +1764,14 @@ void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) rx_desc->read.pkt_addr = cpu_to_le64(addr); + if (hdr_fq.pp) { + addr = libeth_rx_alloc(&hdr_fq, ntu); + if (addr == DMA_MAPPING_ERROR) { + libeth_rx_recycle_slow(fq.fqes[ntu].netmem); + break; + } + } + rx_desc++; ntu++; @@ -2017,6 +2031,32 @@ static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, return result; } +static u32 ixgbe_rx_hsplit_wa(const struct libeth_fqe *hdr, + struct libeth_fqe *buf, u32 data_len) +{ + u32 copy = data_len <= L1_CACHE_BYTES ? data_len : ETH_HLEN; + struct page *hdr_page, *buf_page; + const void *src; + void *dst; + + if (unlikely(netmem_is_net_iov(buf->netmem)) || + !libeth_rx_sync_for_cpu(buf, copy)) + return 0; + + hdr_page = __netmem_to_page(hdr->netmem); + buf_page = __netmem_to_page(buf->netmem); + + dst = page_address(hdr_page) + hdr->offset + + pp_page_to_nmdesc(hdr_page)->pp->p.offset; + src = page_address(buf_page) + buf->offset + + pp_page_to_nmdesc(buf_page)->pp->p.offset; + + memcpy(dst, src, LARGEST_ALIGN(copy)); + buf->offset += copy; + + return copy; +} + /** * ixgbe_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf * @q_vector: structure containing interrupt and ring information @@ -2072,6 +2112,23 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, dma_rmb(); rx_buffer = &rx_ring->rx_fqes[rx_ring->next_to_clean]; + + if (unlikely(rx_ring->hdr_pp)) { + struct libeth_fqe *hdr_buff; + unsigned int hdr_size = 0; + + hdr_buff = &rx_ring->hdr_fqes[rx_ring->next_to_clean]; + + if (!xdp->data) { + hdr_size = ixgbe_rx_hsplit_wa(hdr_buff, + rx_buffer, + size); + size -= hdr_size ? : size; + } + + libeth_xdp_process_buff(xdp, hdr_buff, hdr_size); + } + libeth_xdp_process_buff(xdp, rx_buffer, size); cleaned_count++; @@ -6365,6 +6422,7 @@ static int ixgbe_sw_init(struct ixgbe_adapter *adapter, case ixgbe_mac_82599EB: if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE; + adapter->flags2 |= IXGBE_FLAG2_HSPLIT; break; case ixgbe_mac_X540: fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM(hw)); @@ -6567,38 +6625,53 @@ static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter) return err; } -/** - * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors) - * @adapter: pointer to ixgbe_adapter - * @rx_ring: rx descriptor ring (for a specific queue) to setup - * - * Returns 0 on success, negative on failure - **/ -int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, - struct ixgbe_ring *rx_ring) +static void ixgbe_rx_destroy_pp(struct ixgbe_ring *rx_ring) +{ + struct libeth_fq fq = { + .pp = rx_ring->pp, + .fqes = rx_ring->rx_fqes, + }; + + libeth_rx_fq_destroy(&fq); + rx_ring->rx_fqes = NULL; + rx_ring->pp = NULL; + + if (!rx_ring->hdr_pp) + return; + + fq = (struct libeth_fq) { + .pp = rx_ring->hdr_pp, + .fqes = rx_ring->hdr_fqes, + }; + + libeth_rx_fq_destroy(&fq); + rx_ring->hdr_fqes = NULL; + rx_ring->hdr_pp = NULL; +} + +static int ixgbe_rx_create_pp(struct ixgbe_ring *rx_ring) { + struct ixgbe_adapter *adapter = netdev_priv(rx_ring->netdev); struct libeth_fq fq = { .count = rx_ring->count, .nid = NUMA_NO_NODE, .type = LIBETH_FQE_MTU, .xdp = !!rx_ring->xdp_prog, .buf_len = IXGBE_RX_PAGE_LEN(rx_ring->xdp_prog ? - LIBETH_XDP_HEADROOM : - LIBETH_SKB_HEADROOM), + LIBETH_XDP_HEADROOM : + LIBETH_SKB_HEADROOM), }; - struct device *dev = &adapter->pdev->dev; - int orig_node = dev_to_node(dev); - struct napi_struct *napi_dev; - int ring_node = NUMA_NO_NODE; + u32 frame_size; int ret; - if (rx_ring->q_vector) { - fq.nid = rx_ring->q_vector->numa_node; - ring_node = rx_ring->q_vector->numa_node; - napi_dev = &rx_ring->q_vector->napi; - } else { - napi_dev = NULL; - } + /* Some HW requires DMA write sizes to be aligned to 1K, + * which warrants fake header split usage, but this is + * not an issue if the frame size is at its maximum of 3K + */ + frame_size = + IXGBE_RX_SRRCTL_BUF_SIZE(READ_ONCE(rx_ring->netdev->mtu)); + fq.hsplit = (adapter->flags2 & IXGBE_FLAG2_HSPLIT) && + frame_size < fq.buf_len; ret = libeth_rx_fq_create(&fq, &rx_ring->q_vector->napi); if (ret) @@ -6609,6 +6682,53 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, rx_ring->truesize = fq.truesize; rx_ring->rx_buf_len = fq.buf_len; + if (!fq.hsplit) + return 0; + + fq = (struct libeth_fq) { + .count = rx_ring->count, + .nid = NUMA_NO_NODE, + .type = LIBETH_FQE_HDR, + .xdp = !!rx_ring->xdp_prog, + }; + + ret = libeth_rx_fq_create(&fq, &rx_ring->q_vector->napi); + if (ret) + goto err; + + rx_ring->hdr_pp = fq.pp; + rx_ring->hdr_fqes = fq.fqes; + rx_ring->hdr_truesize = fq.truesize; + + return 0; + +err: + ixgbe_rx_destroy_pp(rx_ring); + return ret; +} + +/** + * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors) + * @adapter: pointer to ixgbe_adapter + * @rx_ring: rx descriptor ring (for a specific queue) to setup + * + * Returns 0 on success, negative on failure + **/ +int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, + struct ixgbe_ring *rx_ring) +{ + struct device *dev = &adapter->pdev->dev; + int orig_node = dev_to_node(dev); + int ring_node = NUMA_NO_NODE; + int ret; + + if (rx_ring->q_vector) + ring_node = rx_ring->q_vector->numa_node; + + ret = ixgbe_rx_create_pp(rx_ring); + if (ret) + return ret; + /* Round up to nearest 4K */ rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); rx_ring->size = ALIGN(rx_ring->size, 4096); @@ -6637,7 +6757,7 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, if (ret < 0) goto err_free_desc; - xdp_rxq_info_attach_page_pool(&rx_ring->xdp_rxq, fq.pp); + xdp_rxq_info_attach_page_pool(&rx_ring->xdp_rxq, rx_ring->pp); rcu_assign_pointer(rx_ring->xdp_prog, adapter->xdp_prog); return 0; @@ -6647,9 +6767,8 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, rx_ring->desc, rx_ring->dma); rx_ring->desc = NULL; err_destroy_fq: - libeth_rx_fq_destroy(&fq); - rx_ring->rx_fqes = NULL; - rx_ring->pp = NULL; + dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n"); + ixgbe_rx_destroy_pp(rx_ring); return ret; } @@ -6737,11 +6856,6 @@ static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter) **/ void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring) { - struct libeth_fq fq = { - .fqes = rx_ring->rx_fqes, - .pp = rx_ring->pp, - }; - ixgbe_clean_rx_ring(rx_ring); rcu_assign_pointer(rx_ring->xdp_prog, NULL); @@ -6752,7 +6866,7 @@ void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring) if (!rx_ring->desc) return; - dma_free_coherent(fq.pp->p.dev, rx_ring->size, rx_ring->desc, + dma_free_coherent(rx_ring->pp->p.dev, rx_ring->size, rx_ring->desc, rx_ring->dma); rx_ring->desc = NULL; @@ -6760,9 +6874,7 @@ void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring) kvfree(rx_ring->rx_xsk_buffer_info); rx_ring->rx_xsk_buffer_info = NULL; - libeth_rx_fq_destroy(&fq); - rx_ring->rx_fqes = NULL; - rx_ring->pp = NULL; + ixgbe_rx_destroy_pp(rx_ring); } /** @@ -10359,10 +10471,11 @@ static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog, return -EINVAL; } - requires_mbuf = frame_size > IXGBE_RX_PAGE_LEN(LIBETH_XDP_HEADROOM); + requires_mbuf = frame_size > IXGBE_RX_PAGE_LEN(LIBETH_XDP_HEADROOM) || + adapter->flags2 & IXGBE_FLAG2_HSPLIT; if (prog && !prog->aux->xdp_has_frags && requires_mbuf) { NL_SET_ERR_MSG_MOD(extack, - "Configured MTU requires non-linear frames and XDP prog does not support frags"); + "HW limitations or configured MTU requires non-linear frames and XDP prog does not support frags"); return -EOPNOTSUPP; } From 48623077507d3934c31169ea5b271ab91d67c458 Mon Sep 17 00:00:00 2001 From: Magdalena Pytel Date: Mon, 1 Jun 2026 16:06:54 +0200 Subject: [PATCH 31/39] ixgbe: reconfigure page pool when reallocating buffers * Moved page pool creation from ixgbe_setup_rx_resources() to ixgbe_configure_rx_ring() * Moved page pool destruction to ixgbe_clean_all_rx_rings() and ixgbe_free_rx_resources() * Added ixgbe_dma_dev_from_ring() helper to obtain DMA device without relying on page pool pointer. * Handled XSK path in ixgbe_configure_rx_ring() Signed-off-by: Magdalena Pytel --- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 87 +++++++++++-------- 1 file changed, 51 insertions(+), 36 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 40d3248297a098..9941c48ae89a1a 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -4031,6 +4031,10 @@ static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter, IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl); } +static int ixgbe_rx_create_pp(struct ixgbe_ring *rx_ring); +static void ixgbe_rx_destroy_pp(struct ixgbe_ring *rx_ring); +static int ixgbe_rx_napi_id(struct ixgbe_ring *rx_ring); + #define IXGBE_MAX_RX_DESC_POLL 10 static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter, struct ixgbe_ring *ring) @@ -4067,15 +4071,18 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, u32 rxdctl; u8 reg_idx = ring->reg_idx; - xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq); + ixgbe_rx_destroy_pp(ring); ring->xsk_pool = ixgbe_xsk_pool(adapter, ring); if (ring->xsk_pool) { + __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, + ring->queue_index, + ixgbe_rx_napi_id(ring), 0); WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, MEM_TYPE_XSK_BUFF_POOL, NULL)); xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq); } else { - xdp_rxq_info_attach_page_pool(&ring->xdp_rxq, ring->pp); + ixgbe_rx_create_pp(ring); } /* disable queue to avoid use of these values while updating state */ @@ -6131,8 +6138,10 @@ static void ixgbe_clean_all_rx_rings(struct ixgbe_adapter *adapter) { int i; - for (i = 0; i < adapter->num_rx_queues; i++) + for (i = 0; i < adapter->num_rx_queues; i++) { ixgbe_clean_rx_ring(adapter->rx_ring[i]); + ixgbe_rx_destroy_pp(adapter->rx_ring[i]); + } } /** @@ -6625,6 +6634,13 @@ static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter) return err; } +static int ixgbe_rx_napi_id(struct ixgbe_ring *rx_ring) +{ + struct ixgbe_q_vector *q_vector = rx_ring->q_vector; + + return q_vector ? q_vector->napi.napi_id : 0; +} + static void ixgbe_rx_destroy_pp(struct ixgbe_ring *rx_ring) { struct libeth_fq fq = { @@ -6632,6 +6648,14 @@ static void ixgbe_rx_destroy_pp(struct ixgbe_ring *rx_ring) .fqes = rx_ring->rx_fqes, }; + if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq)) { + xdp_rxq_info_detach_mem_model(&rx_ring->xdp_rxq); + xdp_rxq_info_unreg(&rx_ring->xdp_rxq); + } + + if (!fq.pp) + return; + libeth_rx_fq_destroy(&fq); rx_ring->rx_fqes = NULL; rx_ring->pp = NULL; @@ -6682,6 +6706,16 @@ static int ixgbe_rx_create_pp(struct ixgbe_ring *rx_ring) rx_ring->truesize = fq.truesize; rx_ring->rx_buf_len = fq.buf_len; + /* XDP RX-queue info */ + ret = __xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, + rx_ring->queue_index, + ixgbe_rx_napi_id(rx_ring), + rx_ring->truesize); + if (ret) + goto err; + + xdp_rxq_info_attach_page_pool(&rx_ring->xdp_rxq, rx_ring->pp); + if (!fq.hsplit) return 0; @@ -6707,6 +6741,13 @@ static int ixgbe_rx_create_pp(struct ixgbe_ring *rx_ring) return ret; } +static struct device *ixgbe_dma_dev_from_ring(struct ixgbe_ring *ring) +{ + struct ixgbe_adapter *adapter = netdev_priv(ring->netdev); + + return &adapter->pdev->dev; +} + /** * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors) * @adapter: pointer to ixgbe_adapter @@ -6720,15 +6761,10 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, struct device *dev = &adapter->pdev->dev; int orig_node = dev_to_node(dev); int ring_node = NUMA_NO_NODE; - int ret; if (rx_ring->q_vector) ring_node = rx_ring->q_vector->numa_node; - ret = ixgbe_rx_create_pp(rx_ring); - if (ret) - return ret; - /* Round up to nearest 4K */ rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); rx_ring->size = ALIGN(rx_ring->size, 4096); @@ -6742,34 +6778,16 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, &rx_ring->dma, GFP_KERNEL); if (!rx_ring->desc) { - ret = -ENOMEM; - dev_err(&adapter->pdev->dev, - "Unable to allocate memory for the Rx descriptor ring\n"); - goto err_destroy_fq; + dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n"); + return -ENOMEM; } rx_ring->next_to_clean = 0; rx_ring->next_to_use = 0; - /* XDP RX-queue info */ - ret = __xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, - rx_ring->queue_index, 0, rx_ring->truesize); - if (ret < 0) - goto err_free_desc; - - xdp_rxq_info_attach_page_pool(&rx_ring->xdp_rxq, rx_ring->pp); - rcu_assign_pointer(rx_ring->xdp_prog, adapter->xdp_prog); + WRITE_ONCE(rx_ring->xdp_prog, adapter->xdp_prog); return 0; - -err_free_desc: - dma_free_coherent(dev, rx_ring->size, - rx_ring->desc, rx_ring->dma); - rx_ring->desc = NULL; -err_destroy_fq: - dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n"); - ixgbe_rx_destroy_pp(rx_ring); - return ret; } /** @@ -6857,24 +6875,21 @@ static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter) void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring) { ixgbe_clean_rx_ring(rx_ring); + ixgbe_rx_destroy_pp(rx_ring); - rcu_assign_pointer(rx_ring->xdp_prog, NULL); - xdp_rxq_info_detach_mem_model(&rx_ring->xdp_rxq); - xdp_rxq_info_unreg(&rx_ring->xdp_rxq); + rx_ring->xdp_prog = NULL; /* if not set, then don't free */ if (!rx_ring->desc) return; - dma_free_coherent(rx_ring->pp->p.dev, rx_ring->size, rx_ring->desc, - rx_ring->dma); + dma_free_coherent(ixgbe_dma_dev_from_ring(rx_ring), rx_ring->size, + rx_ring->desc, rx_ring->dma); rx_ring->desc = NULL; kvfree(rx_ring->rx_xsk_buffer_info); rx_ring->rx_xsk_buffer_info = NULL; - - ixgbe_rx_destroy_pp(rx_ring); } /** From d7e23e943649c039f373330576990c8a6a0a60d7 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Wed, 19 Aug 2026 14:06:43 +0200 Subject: [PATCH 32/39] fixup! ixgbe, libie: use libie_xg_ring in ixgbe --- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 9941c48ae89a1a..956182d25cd6a7 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -4072,8 +4072,9 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, u8 reg_idx = ring->reg_idx; ixgbe_rx_destroy_pp(ring); - ring->xsk_pool = ixgbe_xsk_pool(adapter, ring); - if (ring->xsk_pool) { + if (ixgbe_xsk_pool(adapter, ring)) { + ring->xsk_pool = ixgbe_xsk_pool(adapter, ring); + set_ring_xsk(ring); __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, ring->queue_index, ixgbe_rx_napi_id(ring), 0); From abad4ae9c5320477f16ddd0ed9a0943bfdac3d04 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Wed, 19 Aug 2026 14:20:40 +0200 Subject: [PATCH 33/39] fixup! ixgbe, libie: use libie_xg_ring in ixgbe --- drivers/net/ethernet/intel/ixgbe/ixgbe.h | 7 +++++ drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 29 ++++++++++--------- drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c | 2 +- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index 991c846dcd778c..d9f56f0d6fbe34 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -237,6 +237,7 @@ enum ixgbe_ring_state_t { __IXGBE_HANG_CHECK_ARMED, __IXGBE_TX_XDP_RING, __IXGBE_TX_DISABLED, + __IXGBEVF_RXTX_XSK_RING, }; struct ixgbe_fwd_adapter { @@ -265,6 +266,12 @@ struct ixgbe_fwd_adapter { set_bit(__IXGBE_TX_XDP_RING, &(ring)->state) #define clear_ring_xdp(ring) \ clear_bit(__IXGBE_TX_XDP_RING, &(ring)->state) +#define ring_is_xsk(ring) \ + test_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state) +#define set_ring_xsk(ring) \ + set_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state) +#define clear_ring_xsk(ring) \ + clear_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state) struct ixgbe_ring { union { struct libie_xg_ring base; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 956182d25cd6a7..336b669a3caa50 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -3168,7 +3168,7 @@ int ixgbe_poll(struct napi_struct *napi, int budget) #endif ixgbe_for_each_ring(ring, q_vector->tx) { - bool wd = ring->xsk_pool ? + bool wd = ring_is_xsk(ring) ? ixgbe_clean_xdp_tx_irq(q_vector, ring, budget) : ixgbe_clean_tx_irq(q_vector, ring, budget); @@ -3188,7 +3188,7 @@ int ixgbe_poll(struct napi_struct *napi, int budget) per_ring_budget = budget; ixgbe_for_each_ring(ring, q_vector->rx) { - int cleaned = ring->xsk_pool ? + int cleaned = ring_is_xsk(ring) ? ixgbe_clean_rx_irq_zc(q_vector, ring, per_ring_budget) : ixgbe_clean_rx_irq(q_vector, ring, @@ -3482,15 +3482,18 @@ static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter) void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter, struct ixgbe_ring *ring) { + struct xsk_buff_pool *xsk_pool = NULL; struct ixgbe_hw *hw = &adapter->hw; - u64 tdba = ring->dma; - int wait_loop = 10; u32 txdctl = IXGBE_TXDCTL_ENABLE; u8 reg_idx = ring->reg_idx; + u64 tdba = ring->dma; + int wait_loop = 10; - ring->xsk_pool = NULL; - if (ring_is_xdp(ring)) - ring->xsk_pool = ixgbe_xsk_pool(adapter, ring); + xsk_pool = ixgbe_xsk_pool(adapter, ring); + if (xsk_pool && ring_is_xdp(ring)) { + ring->xsk_pool = xsk_pool; + set_ring_xsk(ring); + } /* disable queue to avoid issues while updating state */ IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), 0); @@ -3736,7 +3739,7 @@ static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter, /* configure header buffer length, needed for RSC */ srrctl = IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT; - if (ring->xsk_pool) + if (ring_is_xsk(ring)) srrctl |= DIV_ROUND_UP(xsk_pool_get_rx_frame_size(ring->xsk_pool), IXGBE_SRRCTL_BSIZEPKT_STEP); else @@ -4124,7 +4127,7 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, IXGBE_RXDCTL_RLPML_EN); } #endif - if (ring->xsk_pool && hw->mac.type != ixgbe_mac_82599EB) { + if (ring_is_xsk(ring) && hw->mac.type != ixgbe_mac_82599EB) { u32 pkt_len = READ_ONCE(adapter->netdev->mtu) + LIBETH_RX_LL_LEN; @@ -4143,7 +4146,7 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl); ixgbe_rx_desc_queue_enable(adapter, ring); - if (ring->xsk_pool) + if (ring_is_xsk(ring)) ixgbe_alloc_rx_buffers_zc(ring, ixgbe_desc_unused(ring)); else ixgbe_alloc_rx_buffers(ring, ixgbe_desc_unused(ring)); @@ -5260,7 +5263,7 @@ static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter) **/ static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) { - if (rx_ring->xsk_pool) { + if (ring_is_xsk(rx_ring)) { ixgbe_xsk_clean_rx_ring(rx_ring); goto skip_free; } @@ -6069,7 +6072,7 @@ static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring) u16 i = tx_ring->next_to_clean; struct libie_xg_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; - if (tx_ring->xsk_pool) { + if (ring_is_xsk(tx_ring)) { ixgbe_xsk_clean_tx_ring(tx_ring); goto out; } @@ -10534,7 +10537,7 @@ static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog, num_queues = min_t(int, adapter->num_rx_queues, adapter->num_xdp_queues); for (int i = 0; i < num_queues; i++) - if (adapter->xdp_ring[i]->xsk_pool) + if (ring_is_xsk(adapter->xdp_ring[i])) (void)ixgbe_xsk_wakeup(adapter->netdev, i, XDP_WAKEUP_RX); xdp_features_set_redirect_target(dev, true); diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c index cae45570d3e36a..bc19470c0adaaa 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c @@ -522,7 +522,7 @@ int ixgbe_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) ring = adapter->rx_ring[qid]; - if (unlikely(!ring->xsk_pool)) + if (unlikely(!ring_is_xsk(ring))) return -EINVAL; if (!napi_if_scheduled_mark_missed(&ring->q_vector->napi)) { From d3cadbb8f6dad875d279ccf76cc315cb94649e54 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Wed, 19 Aug 2026 14:24:12 +0200 Subject: [PATCH 34/39] fixup! ixgbe: reconfigure page pool when reallocating buffers --- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 59 ++++++++++++------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 336b669a3caa50..f3d16810be3a8a 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -1751,7 +1751,7 @@ void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) u16 ntu = rx_ring->next_to_use; /* nothing to do */ - if (unlikely(!cleaned_count)) + if (unlikely(!cleaned_count) || unlikely(!fq.pp)) return; rx_desc = IXGBE_RX_DESC(rx_ring, ntu); @@ -4034,8 +4034,8 @@ static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter, IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl); } -static int ixgbe_rx_create_pp(struct ixgbe_ring *rx_ring); -static void ixgbe_rx_destroy_pp(struct ixgbe_ring *rx_ring); +static int ixgbe_rx_create_fq(struct ixgbe_ring *rx_ring); +static void ixgbe_rx_destroy_fq(struct ixgbe_ring *rx_ring); static int ixgbe_rx_napi_id(struct ixgbe_ring *rx_ring); #define IXGBE_MAX_RX_DESC_POLL 10 @@ -4073,20 +4073,14 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, u64 rdba = ring->dma; u32 rxdctl; u8 reg_idx = ring->reg_idx; + int err; - ixgbe_rx_destroy_pp(ring); - if (ixgbe_xsk_pool(adapter, ring)) { - ring->xsk_pool = ixgbe_xsk_pool(adapter, ring); - set_ring_xsk(ring); - __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, - ring->queue_index, - ixgbe_rx_napi_id(ring), 0); - WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, - MEM_TYPE_XSK_BUFF_POOL, - NULL)); - xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq); - } else { - ixgbe_rx_create_pp(ring); + err = ixgbe_rx_create_fq(ring); + if (err) { + netdev_err(ring->netdev, + "Failed to create Page Pool for buffer allocation: (%pe), RxQ %d is disabled, driver reload may be needed\n", + ERR_PTR(err), ring->queue_index); + return; } /* disable queue to avoid use of these values while updating state */ @@ -6144,7 +6138,7 @@ static void ixgbe_clean_all_rx_rings(struct ixgbe_adapter *adapter) for (i = 0; i < adapter->num_rx_queues; i++) { ixgbe_clean_rx_ring(adapter->rx_ring[i]); - ixgbe_rx_destroy_pp(adapter->rx_ring[i]); + ixgbe_rx_destroy_fq(adapter->rx_ring[i]); } } @@ -6645,7 +6639,7 @@ static int ixgbe_rx_napi_id(struct ixgbe_ring *rx_ring) return q_vector ? q_vector->napi.napi_id : 0; } -static void ixgbe_rx_destroy_pp(struct ixgbe_ring *rx_ring) +static void ixgbe_rx_destroy_fq(struct ixgbe_ring *rx_ring) { struct libeth_fq fq = { .pp = rx_ring->pp, @@ -6657,6 +6651,9 @@ static void ixgbe_rx_destroy_pp(struct ixgbe_ring *rx_ring) xdp_rxq_info_unreg(&rx_ring->xdp_rxq); } + if (ring_is_xsk(rx_ring)) + return; + if (!fq.pp) return; @@ -6677,9 +6674,10 @@ static void ixgbe_rx_destroy_pp(struct ixgbe_ring *rx_ring) rx_ring->hdr_pp = NULL; } -static int ixgbe_rx_create_pp(struct ixgbe_ring *rx_ring) +static int ixgbe_rx_create_fq(struct ixgbe_ring *rx_ring) { struct ixgbe_adapter *adapter = netdev_priv(rx_ring->netdev); + struct xsk_buff_pool *xsk_pool; struct libeth_fq fq = { .count = rx_ring->count, .nid = NUMA_NO_NODE, @@ -6692,6 +6690,24 @@ static int ixgbe_rx_create_pp(struct ixgbe_ring *rx_ring) u32 frame_size; int ret; + xsk_pool = ixgbe_xsk_pool(adapter, rx_ring); + if (xsk_pool) { + rx_ring->xsk_pool = xsk_pool; + set_ring_xsk(rx_ring); + __xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, + rx_ring->queue_index, + ixgbe_rx_napi_id(rx_ring), 0); + ret = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq, + MEM_TYPE_XSK_BUFF_POOL, + NULL); + if (ret) + return ret; + + xsk_pool_set_rxq_info(rx_ring->xsk_pool, &rx_ring->xdp_rxq); + + return 0; + } + /* Some HW requires DMA write sizes to be aligned to 1K, * which warrants fake header split usage, but this is * not an issue if the frame size is at its maximum of 3K @@ -6741,7 +6757,7 @@ static int ixgbe_rx_create_pp(struct ixgbe_ring *rx_ring) return 0; err: - ixgbe_rx_destroy_pp(rx_ring); + ixgbe_rx_destroy_fq(rx_ring); return ret; } @@ -6879,7 +6895,7 @@ static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter) void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring) { ixgbe_clean_rx_ring(rx_ring); - ixgbe_rx_destroy_pp(rx_ring); + ixgbe_rx_destroy_fq(rx_ring); rx_ring->xdp_prog = NULL; @@ -10837,6 +10853,7 @@ void ixgbe_txrx_ring_disable(struct ixgbe_adapter *adapter, int ring) if (xdp_ring) ixgbe_clean_tx_ring(xdp_ring); ixgbe_clean_rx_ring(rx_ring); + ixgbe_rx_destroy_fq(rx_ring); ixgbe_reset_txr_stats(tx_ring); if (xdp_ring) From dc4e8c4c7e6e1bc4cf0cc26166a4d753c71c171a Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Wed, 19 Aug 2026 14:24:42 +0200 Subject: [PATCH 35/39] ixgbevf TODO --- drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index e5bf50a82268a8..494449e844e71b 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -1422,6 +1422,7 @@ void ixgbevf_rx_destroy_pp(struct ixgbevf_ring *rx_ring) rx_ring->hdr_pp = NULL; } +// TODO : rename to create_fq static int ixgbevf_rx_create_pp(struct ixgbevf_ring *rx_ring) { u32 adapter_flags = rx_ring->q_vector->adapter->flags; From 4a7d55059334e9802f09002f2faae0903e4803b5 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Wed, 19 Aug 2026 14:45:34 +0200 Subject: [PATCH 36/39] fixup! ixgbe, libie: use libie_xg_ring in ixgbe --- drivers/net/ethernet/intel/ixgbe/ixgbe.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index d9f56f0d6fbe34..0ad93645aa1cfc 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -237,7 +237,7 @@ enum ixgbe_ring_state_t { __IXGBE_HANG_CHECK_ARMED, __IXGBE_TX_XDP_RING, __IXGBE_TX_DISABLED, - __IXGBEVF_RXTX_XSK_RING, + __IXGBE_RXTX_XSK_RING, }; struct ixgbe_fwd_adapter { @@ -267,11 +267,11 @@ struct ixgbe_fwd_adapter { #define clear_ring_xdp(ring) \ clear_bit(__IXGBE_TX_XDP_RING, &(ring)->state) #define ring_is_xsk(ring) \ - test_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state) + test_bit(__IXGBE_RXTX_XSK_RING, &(ring)->state) #define set_ring_xsk(ring) \ - set_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state) + set_bit(__IXGBE_RXTX_XSK_RING, &(ring)->state) #define clear_ring_xsk(ring) \ - clear_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state) + clear_bit(__IXGBE_RXTX_XSK_RING, &(ring)->state) struct ixgbe_ring { union { struct libie_xg_ring base; From ddfe684c37300d37b82a69c78053201b2a9390bf Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Wed, 19 Aug 2026 17:32:17 +0200 Subject: [PATCH 37/39] [SQUASH SMWR] enable ZC frags in features --- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index f3d16810be3a8a..094523b90004fa 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -11404,9 +11404,7 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) netdev->priv_flags |= IFF_UNICAST_FLT; netdev->priv_flags |= IFF_SUPP_NOFCS; - netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | - NETDEV_XDP_ACT_XSK_ZEROCOPY | - NETDEV_XDP_ACT_RX_SG; + libeth_xdp_set_features(netdev, NULL, MAX_SKB_FRAGS); /* MTU range: 68 - 9710 */ netdev->min_mtu = ETH_MIN_MTU; From 0189623eaee87956cf2f9b7f2b9d29d553012b2c Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Wed, 19 Aug 2026 17:33:31 +0200 Subject: [PATCH 38/39] ixgbe: support AF_XDP multi-buffer via libeth and libie_xg Utilize libeth and libie_xg libraries to support segments in AF_XDP ZC mode. Since Tx path supports frags, it does not need updating for now. Signed-off-by: Larysa Zaremba --- drivers/net/ethernet/intel/ixgbe/ixgbe.h | 6 +- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 29 ++- drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c | 186 ++++-------------- 3 files changed, 63 insertions(+), 158 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index 0ad93645aa1cfc..7d784662f494e8 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -199,11 +199,6 @@ struct vf_macvlans { #define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD) #define DESC_NEEDED (MAX_SKB_FRAGS + 4) -struct ixgbe_xsk_rx_buffer { - bool discard; - struct xdp_buff *xdp; -}; - struct ixgbe_queue_stats { u64 packets; u64 bytes; @@ -317,6 +312,7 @@ struct ixgbe_ring { u32 rx_buf_len; struct libeth_xdp_buff_stash xdp_stash; struct ixgbe_xsk_rx_buffer *rx_xsk_buffer_info; + u32 thresh; } ____cacheline_internodealigned_in_smp; enum ixgbe_ring_f_enum { diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 094523b90004fa..e977c2218194ca 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -31,7 +31,7 @@ #include #include #include -#include +#include #include #include #include @@ -6651,8 +6651,19 @@ static void ixgbe_rx_destroy_fq(struct ixgbe_ring *rx_ring) xdp_rxq_info_unreg(&rx_ring->xdp_rxq); } - if (ring_is_xsk(rx_ring)) - return; + // TODO : previous code was not clearing xsk bit + if (test_and_clear_bit(__IXGBE_RXTX_XSK_RING, &rx_ring->state)) { + struct libeth_xskfq xskfq = { + .fqes = rx_ring->xsk_fqes, + }; + + libeth_xskfq_destroy(&xskfq); + rx_ring->xsk_fqes = NULL; + rx_ring->pending = xskfq.pending; + rx_ring->thresh = xskfq.thresh; + rx_ring->rx_buf_len = xskfq.buf_len; + rx_ring->xsk_pool = NULL; + } if (!fq.pp) return; @@ -6692,8 +6703,20 @@ static int ixgbe_rx_create_fq(struct ixgbe_ring *rx_ring) xsk_pool = ixgbe_xsk_pool(adapter, rx_ring); if (xsk_pool) { + struct libeth_xskfq xskfq = { + .nid = numa_node_id(), + .count = rx_ring->count, + .pool = xsk_pool, + }; + + libeth_xskfq_create(&xskfq); rx_ring->xsk_pool = xsk_pool; + rx_ring->xsk_fqes = xskfq.fqes; + rx_ring->pending = xskfq.count - 1; + rx_ring->thresh = xskfq.thresh; + rx_ring->rx_buf_len = xskfq.buf_len; set_ring_xsk(rx_ring); + __xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, rx_ring->queue_index, ixgbe_rx_napi_id(rx_ring), 0); diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c index bc19470c0adaaa..526610c76d83d9 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c @@ -2,6 +2,7 @@ /* Copyright(c) 2018 Intel Corporation. */ #include +#include #include #include @@ -150,98 +151,24 @@ static int ixgbe_run_xdp_zc(struct ixgbe_adapter *adapter, bool ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 count) { - union ixgbe_adv_rx_desc *rx_desc; - struct ixgbe_xsk_rx_buffer *bi; - u16 i = rx_ring->next_to_use; - dma_addr_t dma; - bool ok = true; - - /* nothing to do */ - if (!count) - return true; - - rx_desc = IXGBE_RX_DESC(rx_ring, i); - bi = &rx_ring->rx_xsk_buffer_info[i]; - i -= rx_ring->count; - - do { - bi->xdp = xsk_buff_alloc(rx_ring->xsk_pool); - if (!bi->xdp) { - ok = false; - break; - } - - dma = xsk_buff_xdp_get_dma(bi->xdp); - - /* Refresh the desc even if buffer_addrs didn't change - * because each write-back erases this info. - */ - rx_desc->read.pkt_addr = cpu_to_le64(dma); - - rx_desc++; - bi++; - i++; - if (unlikely(!i)) { - rx_desc = IXGBE_RX_DESC(rx_ring, 0); - bi = rx_ring->rx_xsk_buffer_info; - i -= rx_ring->count; - } - - /* clear the length for the next_to_use descriptor */ - rx_desc->wb.upper.length = 0; - - count--; - } while (count); - - i += rx_ring->count; - - if (rx_ring->next_to_use != i) { - rx_ring->next_to_use = i; - - /* Force memory writes to complete before letting h/w - * know there are new descriptors to fetch. (Only - * applicable for weak-ordered memory model archs, - * such as IA-64). - */ - wmb(); - writel(i, rx_ring->tail); - } - - return ok; -} - -static struct sk_buff *ixgbe_construct_skb_zc(struct ixgbe_ring *rx_ring, - const struct xdp_buff *xdp) -{ - unsigned int totalsize = xdp->data_end - xdp->data_meta; - unsigned int metasize = xdp->data - xdp->data_meta; - struct sk_buff *skb; - - net_prefetch(xdp->data_meta); - - /* allocate a skb to store the frags */ - skb = napi_alloc_skb(&rx_ring->q_vector->napi, totalsize); - if (unlikely(!skb)) - return NULL; - - memcpy(__skb_put(skb, totalsize), xdp->data_meta, - ALIGN(totalsize, sizeof(long))); - - if (metasize) { - skb_metadata_set(skb, metasize); - __skb_pull(skb, metasize); - } - - return skb; + return libie_xg_xsk_alloc_rx_bufs(&rx_ring->base, count); } -static void ixgbe_inc_ntc(struct ixgbe_ring *rx_ring) +static bool ixgbe_xsk_is_non_eop(struct ixgbe_ring *rx_ring, + union ixgbe_adv_rx_desc *rx_desc) { u32 ntc = rx_ring->next_to_clean + 1; ntc = (ntc < rx_ring->count) ? ntc : 0; rx_ring->next_to_clean = ntc; + rx_ring->pending++; + prefetch(IXGBE_RX_DESC(rx_ring, ntc)); + + if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) + return false; + + return true; } int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, @@ -250,27 +177,19 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, { unsigned int total_rx_bytes = 0, total_rx_packets = 0; struct ixgbe_adapter *adapter = q_vector->adapter; - u16 cleaned_count = ixgbe_desc_unused(rx_ring); unsigned int xdp_res, xdp_xmit = 0; + struct libeth_xdp_buff *head_xdp; bool failure = false; struct sk_buff *skb; while (likely(total_rx_packets < budget)) { + struct libeth_xdp_buff *rx_buffer; union ixgbe_adv_rx_desc *rx_desc; - struct ixgbe_xsk_rx_buffer *bi; unsigned int size; - /* return some buffers to hardware, one at a time is too slow */ - if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) { - failure = failure || - !ixgbe_alloc_rx_buffers_zc(rx_ring, - cleaned_count); - cleaned_count = 0; - } - rx_desc = IXGBE_RX_DESC(rx_ring, rx_ring->next_to_clean); size = le16_to_cpu(rx_desc->wb.upper.length); - if (!size) + if (unlikely(!size)) break; /* This memory barrier is needed to keep us from reading @@ -279,77 +198,55 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, */ dma_rmb(); - bi = &rx_ring->rx_xsk_buffer_info[rx_ring->next_to_clean]; - - if (unlikely(!ixgbe_test_staterr(rx_desc, - IXGBE_RXD_STAT_EOP))) { - struct ixgbe_xsk_rx_buffer *next_bi; - - xsk_buff_free(bi->xdp); - bi->xdp = NULL; - ixgbe_inc_ntc(rx_ring); - next_bi = - &rx_ring->rx_xsk_buffer_info[rx_ring->next_to_clean]; - next_bi->discard = true; + rx_buffer = rx_ring->xsk_fqes[rx_ring->next_to_clean]; + head_xdp = libeth_xsk_process_buff(head_xdp, rx_buffer, size); + if (ixgbe_xsk_is_non_eop(rx_ring, rx_desc) || + unlikely(!head_xdp)) continue; - } - if (unlikely(bi->discard)) { - xsk_buff_free(bi->xdp); - bi->xdp = NULL; - bi->discard = false; - ixgbe_inc_ntc(rx_ring); - continue; - } - - bi->xdp->data_end = bi->xdp->data + size; - xsk_buff_dma_sync_for_cpu(bi->xdp); - xdp_res = ixgbe_run_xdp_zc(adapter, rx_ring, bi->xdp); + total_rx_packets++; + total_rx_bytes += xdp_get_buff_len(&head_xdp->base); + xdp_res = ixgbe_run_xdp_zc(adapter, rx_ring, &head_xdp->base); if (likely(xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR))) { xdp_xmit |= xdp_res; } else if (xdp_res == IXGBE_XDP_EXIT) { failure = true; break; } else if (xdp_res == IXGBE_XDP_CONSUMED) { - xsk_buff_free(bi->xdp); + libeth_xdp_return_buff(head_xdp); } else if (xdp_res == IXGBE_XDP_PASS) { goto construct_skb; } - bi->xdp = NULL; - total_rx_packets++; - total_rx_bytes += size; + head_xdp = NULL; - cleaned_count++; - ixgbe_inc_ntc(rx_ring); continue; construct_skb: /* XDP_PASS path */ - skb = ixgbe_construct_skb_zc(rx_ring, bi->xdp); - if (!skb) { + skb = xdp_build_skb_from_zc(&head_xdp->base); + if (unlikely(!skb)) { + libeth_xdp_return_buff_slow(head_xdp); + head_xdp = NULL; rx_ring->rx_stats.alloc_rx_buff_failed++; break; } - xsk_buff_free(bi->xdp); - bi->xdp = NULL; - - cleaned_count++; - ixgbe_inc_ntc(rx_ring); - - if (eth_skb_pad(skb)) - continue; - - total_rx_bytes += skb->len; - total_rx_packets++; + head_xdp = NULL; skb->protocol = eth_type_trans(skb, rx_ring->netdev); ixgbe_process_skb_fields(rx_ring, rx_desc, skb); ixgbe_rx_skb(q_vector, skb); } + if (rx_ring->pending >= rx_ring->thresh) + failure |= !ixgbe_alloc_rx_buffers_zc(rx_ring, + rx_ring->pending); + + /* place incomplete frames back on ring for completion */ + rx_ring->xsk_xdp_head = head_xdp; + if (xdp_xmit & IXGBE_XDP_REDIR) xdp_do_flush(); @@ -375,18 +272,7 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, void ixgbe_xsk_clean_rx_ring(struct ixgbe_ring *rx_ring) { - struct ixgbe_xsk_rx_buffer *bi; - u16 i; - - for (i = 0; i < rx_ring->count; i++) { - bi = &rx_ring->rx_xsk_buffer_info[i]; - - if (!bi->xdp) - continue; - - xsk_buff_free(bi->xdp); - bi->xdp = NULL; - } + return libie_xg_rx_xsk_ring_free_buffs(&rx_ring->base); } static bool ixgbe_xmit_zc(struct ixgbe_ring *xdp_ring, unsigned int budget) From 510fcde47637379e3a9c189410967bc3323a7405 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Wed, 19 Aug 2026 11:31:44 +0200 Subject: [PATCH 39/39] [REWORD] ixgbe: refactor .ndo_xdp_xmit and XDP_TX (also ZC Rx) --- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 250 ++++-------------- .../ethernet/intel/ixgbe/ixgbe_txrx_common.h | 5 +- drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c | 245 ++--------------- 3 files changed, 85 insertions(+), 415 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index e977c2218194ca..e64e7c1619ebb8 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -1386,10 +1387,7 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, total_ipsec++; /* free the skb */ - if (ring_is_xdp(tx_ring)) - xdp_return_frame(tx_buffer->xdpf); - else - napi_consume_skb(tx_buffer->skb, napi_budget); + napi_consume_skb(tx_buffer->skb, napi_budget); /* unmap skb header data */ dma_unmap_single(tx_ring->dev, @@ -1444,9 +1442,6 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, total_bytes); adapter->tx_ipsec += total_ipsec; - if (ring_is_xdp(tx_ring)) - return !!budget; - if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) { if (adapter->hw.mac.type == ixgbe_mac_e610) ixgbe_handle_mdd_event(adapter, tx_ring); @@ -1975,60 +1970,30 @@ bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring, return false; } -static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, - struct ixgbe_ring *rx_ring, - struct libeth_xdp_buff *xdp) +u32 ixgbe_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq) { - int err, result = IXGBE_XDP_PASS; - struct bpf_prog *xdp_prog; - struct ixgbe_ring *ring; - struct xdp_frame *xdpf; - u32 act; - - xdp_prog = READ_ONCE(rx_ring->xdp_prog); - - if (!xdp_prog) - goto xdp_out; - - prefetchw(xdp->base.data_hard_start); /* xdp_frame write */ - - act = bpf_prog_run_xdp(xdp_prog, &xdp->base); - switch (act) { - case XDP_PASS: - break; - case XDP_TX: - xdpf = xdp_convert_buff_to_frame(&xdp->base); - if (unlikely(!xdpf)) - goto out_failure; - ring = ixgbe_determine_xdp_ring(adapter); - if (static_branch_unlikely(&ixgbe_xdp_locking_key)) - spin_lock(&ring->tx_lock); - result = ixgbe_xmit_xdp_ring(ring, xdpf); - if (static_branch_unlikely(&ixgbe_xdp_locking_key)) - spin_unlock(&ring->tx_lock); - if (result == IXGBE_XDP_CONSUMED) - goto out_failure; - break; - case XDP_REDIRECT: - err = xdp_do_redirect(adapter->netdev, &xdp->base, xdp_prog); - if (err) - goto out_failure; - result = IXGBE_XDP_REDIR; - break; - default: - bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); - fallthrough; - case XDP_ABORTED: -out_failure: - trace_xdp_exception(rx_ring->netdev, xdp_prog, act); - fallthrough; /* handle aborts by dropping packet */ - case XDP_DROP: - result = IXGBE_XDP_CONSUMED; - libeth_xdp_return_buff(xdp); - break; + struct ixgbe_ring *xdp_ring = xdpsq; + + libeth_xdpsq_lock(&xdp_ring->xdpq_lock); + if (unlikely(ixgbe_desc_unused(xdp_ring) < xdp_ring->thresh)) { + u16 to_clean = libie_xg_tx_get_num_sent(xdpsq); + + if (likely(to_clean)) + libie_xg_clean_xdp_num(&xdp_ring->base, true, to_clean, + false); } -xdp_out: - return result; + + *sq = (struct libeth_xdpsq) { + .count = xdp_ring->count, + .descs = xdp_ring->desc, + .lock = &xdp_ring->xdpq_lock, + .ntu = &xdp_ring->next_to_use, + .pending = &xdp_ring->pending, + .pool = xdp_ring->xsk_pool, + .sqes = xdp_ring->xdp_sqes, + }; + + return ixgbe_desc_unused(xdp_ring); } static u32 ixgbe_rx_hsplit_wa(const struct libeth_fqe *hdr, @@ -2057,6 +2022,16 @@ static u32 ixgbe_rx_hsplit_wa(const struct libeth_fqe *hdr, return copy; } +LIBETH_XDP_DEFINE_START(); +LIBETH_XDP_DEFINE_FLUSH_TX(static ixgbe_xdp_flush_tx, ixgbe_prep_xdp_sq, + libie_xg_xdp_xmit_desc); +LIBETH_XDP_DEFINE_FLUSH_XMIT(static ixgbe_xdp_flush_xmit, ixgbe_prep_xdp_sq, + libie_xg_xdp_xmit_desc); +LIBETH_XDP_DEFINE_RUN_PROG(static ixgbe_xdp_run_prog, ixgbe_xdp_flush_tx); +LIBETH_XDP_DEFINE_FINALIZE(static ixgbe_xdp_finalize_xdp_napi, + ixgbe_xdp_flush_tx, libie_xg_xdp_rs_and_bump); +LIBETH_XDP_DEFINE_END(); + /** * ixgbe_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf * @q_vector: structure containing interrupt and ring information @@ -2081,12 +2056,14 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, unsigned int mss = 0; #endif /* IXGBE_FCOE */ u16 cleaned_count = ixgbe_desc_unused(rx_ring); + LIBETH_XDP_ONSTACK_BULK(xdp_tx_bulk); LIBETH_XDP_ONSTACK_BUFF(xdp); - unsigned int xdp_xmit = 0; u32 rsc_append_cnt = 0; - int xdp_res = 0; libeth_xdp_init_buff(xdp, &rx_ring->xdp_stash, &rx_ring->xdp_rxq); + libeth_xdp_tx_init_bulk(&xdp_tx_bulk, rx_ring->xdp_prog, + rx_ring->netdev, adapter->xdp_ring, + adapter->num_xdp_queues); while (likely(total_rx_packets < budget)) { union ixgbe_adv_rx_desc *rx_desc; @@ -2137,17 +2114,8 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, unlikely(!xdp->data)) continue; - xdp_res = ixgbe_run_xdp(adapter, rx_ring, xdp); - if (xdp_res) { - if (xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR)) - xdp_xmit |= xdp_res; - - total_rx_packets++; - total_rx_bytes += xdp_get_buff_len(&xdp->base); - rsc_append_cnt = 0; - xdp->data = NULL; - continue; - } + if (xdp_tx_bulk.prog && !ixgbe_xdp_run_prog(xdp, &xdp_tx_bulk)) + continue; skb = xdp_build_skb_from_buff(&xdp->base); @@ -2209,14 +2177,7 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, /* place incomplete frames back on ring for completion */ libeth_xdp_save_buff(&rx_ring->xdp_stash, xdp); - if (xdp_xmit & IXGBE_XDP_REDIR) - xdp_do_flush(); - - if (xdp_xmit & IXGBE_XDP_TX) { - struct ixgbe_ring *ring = ixgbe_determine_xdp_ring(adapter); - - ixgbe_xdp_ring_update_tail_locked(ring); - } + ixgbe_xdp_finalize_xdp_napi(&xdp_tx_bulk); ixgbe_update_rx_ring_stats(rx_ring, q_vector, total_rx_packets, total_rx_bytes); @@ -3170,6 +3131,7 @@ int ixgbe_poll(struct napi_struct *napi, int budget) ixgbe_for_each_ring(ring, q_vector->tx) { bool wd = ring_is_xsk(ring) ? ixgbe_clean_xdp_tx_irq(q_vector, ring, budget) : + ring_is_xdp(ring) ? true : ixgbe_clean_tx_irq(q_vector, ring, budget); if (!wd) @@ -6057,6 +6019,13 @@ void ixgbe_reset(struct ixgbe_adapter *adapter) } } +static void ixgbe_clean_xdp_ring(struct ixgbe_ring *xdp_ring) +{ + libie_xg_clean_xdp_num(&xdp_ring->base, false, xdp_ring->pending, + !!xdp_ring->xsk_pool); + libeth_xdpsq_put(&xdp_ring->xdpq_lock, xdp_ring->netdev); +} + /** * ixgbe_clean_tx_ring - Free Tx Buffers * @tx_ring: ring to be cleaned @@ -6066,8 +6035,8 @@ static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring) u16 i = tx_ring->next_to_clean; struct libie_xg_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; - if (ring_is_xsk(tx_ring)) { - ixgbe_xsk_clean_tx_ring(tx_ring); + if (ring_is_xdp(tx_ring)) { + ixgbe_clean_xdp_ring(tx_ring); goto out; } @@ -8903,87 +8872,6 @@ static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb, } #endif -int ixgbe_xmit_xdp_ring(struct ixgbe_ring *ring, - struct xdp_frame *xdpf) -{ - struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf); - u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0; - u16 i = 0, index = ring->next_to_use; - struct libie_xg_tx_buffer *tx_head = &ring->tx_buffer_info[index]; - struct libie_xg_tx_buffer *tx_buff = tx_head; - union ixgbe_adv_tx_desc *tx_desc = IXGBE_TX_DESC(ring, index); - u32 cmd_type, len = xdpf->len; - void *data = xdpf->data; - - if (unlikely(ixgbe_desc_unused(ring) < 1 + nr_frags)) - return IXGBE_XDP_CONSUMED; - - tx_head->bytecount = xdp_get_frame_len(xdpf); - tx_head->gso_segs = 1; - tx_head->xdpf = xdpf; - - tx_desc->read.olinfo_status = - cpu_to_le32(tx_head->bytecount << IXGBE_ADVTXD_PAYLEN_SHIFT); - - for (;;) { - dma_addr_t dma; - - dma = dma_map_single(ring->dev, data, len, DMA_TO_DEVICE); - if (dma_mapping_error(ring->dev, dma)) - goto unmap; - - dma_unmap_len_set(tx_buff, len, len); - dma_unmap_addr_set(tx_buff, dma, dma); - - cmd_type = IXGBE_ADVTXD_DTYP_DATA | IXGBE_ADVTXD_DCMD_DEXT | - IXGBE_ADVTXD_DCMD_IFCS | len; - tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); - tx_desc->read.buffer_addr = cpu_to_le64(dma); - tx_buff->protocol = 0; - - if (++index == ring->count) - index = 0; - - if (i == nr_frags) - break; - - tx_buff = &ring->tx_buffer_info[index]; - tx_desc = IXGBE_TX_DESC(ring, index); - tx_desc->read.olinfo_status = 0; - - data = skb_frag_address(&sinfo->frags[i]); - len = skb_frag_size(&sinfo->frags[i]); - i++; - } - /* put descriptor type bits */ - tx_desc->read.cmd_type_len |= cpu_to_le32(IXGBE_TXD_CMD); - - /* Avoid any potential race with xdp_xmit and cleanup */ - smp_wmb(); - - tx_head->next_to_watch = tx_desc; - ring->next_to_use = index; - - return IXGBE_XDP_TX; - -unmap: - for (;;) { - tx_buff = &ring->tx_buffer_info[index]; - if (dma_unmap_len(tx_buff, len)) - dma_unmap_page(ring->dev, dma_unmap_addr(tx_buff, dma), - dma_unmap_len(tx_buff, len), - DMA_TO_DEVICE); - dma_unmap_len_set(tx_buff, len, 0); - if (tx_buff == tx_head) - break; - - if (!index) - index += ring->count; - index--; - } - - return IXGBE_XDP_CONSUMED; -} netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, struct ixgbe_adapter *adapter, @@ -10623,9 +10511,6 @@ static int ixgbe_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, u32 flags) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(dev); - struct ixgbe_ring *ring; - int nxmit = 0; - int i; if (unlikely(test_bit(__IXGBE_DOWN, &adapter->state))) return -ENETDOWN; @@ -10637,36 +10522,13 @@ static int ixgbe_xdp_xmit(struct net_device *dev, int n, if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) return -EINVAL; - /* During program transitions its possible adapter->xdp_prog is assigned - * but ring has not been configured yet. In this case simply abort xmit. - */ - ring = adapter->xdp_prog ? ixgbe_determine_xdp_ring(adapter) : NULL; - if (unlikely(!ring)) + if (unlikely(!adapter->num_xdp_queues)) return -ENXIO; - if (unlikely(test_bit(__IXGBE_TX_DISABLED, &ring->state))) - return -ENXIO; - - if (static_branch_unlikely(&ixgbe_xdp_locking_key)) - spin_lock(&ring->tx_lock); - - for (i = 0; i < n; i++) { - struct xdp_frame *xdpf = frames[i]; - int err; - - err = ixgbe_xmit_xdp_ring(ring, xdpf); - if (err != IXGBE_XDP_TX) - break; - nxmit++; - } - - if (unlikely(flags & XDP_XMIT_FLUSH)) - ixgbe_xdp_ring_update_tail(ring); - - if (static_branch_unlikely(&ixgbe_xdp_locking_key)) - spin_unlock(&ring->tx_lock); - - return nxmit; + return libeth_xdp_xmit_do_bulk(dev, n, frames, flags, adapter->xdp_ring, + adapter->num_xdp_queues, + ixgbe_xdp_flush_xmit, + libie_xg_xdp_rs_and_bump); } static const struct net_device_ops ixgbe_netdev_ops = { diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h index f681f95377980c..85eb4e442d9482 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h @@ -10,8 +10,6 @@ #define IXGBE_XDP_REDIR BIT(2) #define IXGBE_XDP_EXIT BIT(3) -int ixgbe_xmit_xdp_ring(struct ixgbe_ring *ring, - struct xdp_frame *xdpf); bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring, union ixgbe_adv_rx_desc *rx_desc, struct sk_buff *skb); @@ -41,7 +39,6 @@ void ixgbe_xsk_clean_rx_ring(struct ixgbe_ring *rx_ring); bool ixgbe_clean_xdp_tx_irq(struct ixgbe_q_vector *q_vector, struct ixgbe_ring *tx_ring, int napi_budget); int ixgbe_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags); -void ixgbe_xsk_clean_tx_ring(struct ixgbe_ring *tx_ring); void ixgbe_update_tx_ring_stats(struct ixgbe_ring *tx_ring, struct ixgbe_q_vector *q_vector, u64 pkts, @@ -50,4 +47,6 @@ void ixgbe_update_rx_ring_stats(struct ixgbe_ring *rx_ring, struct ixgbe_q_vector *q_vector, u64 pkts, u64 bytes); +u32 ixgbe_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq); + #endif /* #define _IXGBE_TXRX_COMMON_H_ */ diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c index 526610c76d83d9..902540624e6282 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c @@ -95,60 +95,6 @@ int ixgbe_xsk_pool_setup(struct ixgbe_adapter *adapter, ixgbe_xsk_pool_disable(adapter, qid); } -static int ixgbe_run_xdp_zc(struct ixgbe_adapter *adapter, - struct ixgbe_ring *rx_ring, - struct xdp_buff *xdp) -{ - int err, result = IXGBE_XDP_PASS; - struct bpf_prog *xdp_prog; - struct ixgbe_ring *ring; - struct xdp_frame *xdpf; - u32 act; - - xdp_prog = READ_ONCE(rx_ring->xdp_prog); - act = bpf_prog_run_xdp(xdp_prog, xdp); - - if (likely(act == XDP_REDIRECT)) { - err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); - if (!err) - return IXGBE_XDP_REDIR; - if (xsk_uses_need_wakeup(rx_ring->xsk_pool) && err == -ENOBUFS) - result = IXGBE_XDP_EXIT; - else - result = IXGBE_XDP_CONSUMED; - goto out_failure; - } - - switch (act) { - case XDP_PASS: - break; - case XDP_TX: - xdpf = xdp_convert_buff_to_frame(xdp); - if (unlikely(!xdpf)) - goto out_failure; - ring = ixgbe_determine_xdp_ring(adapter); - if (static_branch_unlikely(&ixgbe_xdp_locking_key)) - spin_lock(&ring->tx_lock); - result = ixgbe_xmit_xdp_ring(ring, xdpf); - if (static_branch_unlikely(&ixgbe_xdp_locking_key)) - spin_unlock(&ring->tx_lock); - if (result == IXGBE_XDP_CONSUMED) - goto out_failure; - break; - case XDP_DROP: - result = IXGBE_XDP_CONSUMED; - break; - default: - bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); - fallthrough; - case XDP_ABORTED: - result = IXGBE_XDP_CONSUMED; -out_failure: - trace_xdp_exception(rx_ring->netdev, xdp_prog, act); - } - return result; -} - bool ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 count) { return libie_xg_xsk_alloc_rx_bufs(&rx_ring->base, count); @@ -171,21 +117,35 @@ static bool ixgbe_xsk_is_non_eop(struct ixgbe_ring *rx_ring, return true; } +LIBETH_XDP_DEFINE_START(); +LIBETH_XSK_DEFINE_FLUSH_TX(static ixgbe_xsk_flush_tx, ixgbe_prep_xdp_sq, + libie_xg_xsk_xmit_desc); +LIBETH_XSK_DEFINE_RUN_PROG(static ixgbe_xsk_run_prog, ixgbe_xsk_flush_tx); +LIBETH_XSK_DEFINE_FINALIZE(static ixgbe_xsk_finalize_xdp_napi, + ixgbe_xsk_flush_tx, libie_xg_xdp_rs_and_bump); +LIBETH_XDP_DEFINE_END(); + int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, struct ixgbe_ring *rx_ring, const int budget) { unsigned int total_rx_bytes = 0, total_rx_packets = 0; struct ixgbe_adapter *adapter = q_vector->adapter; - unsigned int xdp_res, xdp_xmit = 0; + LIBETH_XDP_ONSTACK_BULK(xdp_tx_bulk); struct libeth_xdp_buff *head_xdp; bool failure = false; struct sk_buff *skb; + head_xdp = rx_ring->xsk_xdp_head; + libeth_xsk_tx_init_bulk(&xdp_tx_bulk, rx_ring->xdp_prog, + adapter->netdev, adapter->xdp_ring, + adapter->num_xdp_queues); + while (likely(total_rx_packets < budget)) { struct libeth_xdp_buff *rx_buffer; union ixgbe_adv_rx_desc *rx_desc; unsigned int size; + u32 xdp_result; rx_desc = IXGBE_RX_DESC(rx_ring, rx_ring->next_to_clean); size = le16_to_cpu(rx_desc->wb.upper.length); @@ -207,26 +167,17 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, total_rx_packets++; total_rx_bytes += xdp_get_buff_len(&head_xdp->base); - xdp_res = ixgbe_run_xdp_zc(adapter, rx_ring, &head_xdp->base); - if (likely(xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR))) { - xdp_xmit |= xdp_res; - } else if (xdp_res == IXGBE_XDP_EXIT) { + xdp_result = ixgbe_xsk_run_prog(head_xdp, &xdp_tx_bulk); + if (xdp_result) { + head_xdp = NULL; + if (likely(xdp_result != LIBETH_XDP_ABORTED)) + continue; failure = true; break; - } else if (xdp_res == IXGBE_XDP_CONSUMED) { - libeth_xdp_return_buff(head_xdp); - } else if (xdp_res == IXGBE_XDP_PASS) { - goto construct_skb; } - head_xdp = NULL; - - continue; - -construct_skb: - /* XDP_PASS path */ skb = xdp_build_skb_from_zc(&head_xdp->base); - if (unlikely(!skb)) { + if (!skb) { libeth_xdp_return_buff_slow(head_xdp); head_xdp = NULL; rx_ring->rx_stats.alloc_rx_buff_failed++; @@ -247,14 +198,7 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, /* place incomplete frames back on ring for completion */ rx_ring->xsk_xdp_head = head_xdp; - if (xdp_xmit & IXGBE_XDP_REDIR) - xdp_do_flush(); - - if (xdp_xmit & IXGBE_XDP_TX) { - struct ixgbe_ring *ring = ixgbe_determine_xdp_ring(adapter); - - ixgbe_xdp_ring_update_tail_locked(ring); - } + ixgbe_xsk_finalize_xdp_napi(&xdp_tx_bulk); ixgbe_update_rx_ring_stats(rx_ring, q_vector, total_rx_packets, total_rx_bytes); @@ -275,124 +219,15 @@ void ixgbe_xsk_clean_rx_ring(struct ixgbe_ring *rx_ring) return libie_xg_rx_xsk_ring_free_buffs(&rx_ring->base); } -static bool ixgbe_xmit_zc(struct ixgbe_ring *xdp_ring, unsigned int budget) -{ - struct xsk_buff_pool *pool = xdp_ring->xsk_pool; - union ixgbe_adv_tx_desc *tx_desc = NULL; - struct libie_xg_tx_buffer *tx_bi; - bool work_done = true; - struct xdp_desc desc; - dma_addr_t dma; - u32 cmd_type; - - while (likely(budget)) { - if (unlikely(!ixgbe_desc_unused(xdp_ring))) { - work_done = false; - break; - } - - if (!netif_carrier_ok(xdp_ring->netdev)) - break; - - if (!xsk_tx_peek_desc(pool, &desc)) - break; - - dma = xsk_buff_raw_get_dma(pool, desc.addr); - xsk_buff_raw_dma_sync_for_device(pool, dma, desc.len); - - tx_bi = &xdp_ring->tx_buffer_info[xdp_ring->next_to_use]; - tx_bi->bytecount = desc.len; - tx_bi->xdpf = NULL; - tx_bi->gso_segs = 1; - - tx_desc = IXGBE_TX_DESC(xdp_ring, xdp_ring->next_to_use); - tx_desc->read.buffer_addr = cpu_to_le64(dma); - - /* put descriptor type bits */ - cmd_type = IXGBE_ADVTXD_DTYP_DATA | - IXGBE_ADVTXD_DCMD_DEXT | - IXGBE_ADVTXD_DCMD_IFCS; - cmd_type |= desc.len | IXGBE_TXD_CMD; - tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); - tx_desc->read.olinfo_status = - cpu_to_le32(desc.len << IXGBE_ADVTXD_PAYLEN_SHIFT); - - xdp_ring->next_to_use++; - if (xdp_ring->next_to_use == xdp_ring->count) - xdp_ring->next_to_use = 0; - - budget--; - } - - if (tx_desc) { - ixgbe_xdp_ring_update_tail(xdp_ring); - xsk_tx_release(pool); - } - - return !!budget && work_done; -} - -static void ixgbe_clean_xdp_tx_buffer(struct ixgbe_ring *tx_ring, - struct libie_xg_tx_buffer *tx_bi) -{ - xdp_return_frame(tx_bi->xdpf); - dma_unmap_single(tx_ring->dev, - dma_unmap_addr(tx_bi, dma), - dma_unmap_len(tx_bi, len), DMA_TO_DEVICE); - dma_unmap_len_set(tx_bi, len, 0); -} - bool ixgbe_clean_xdp_tx_irq(struct ixgbe_q_vector *q_vector, struct ixgbe_ring *tx_ring, int napi_budget) { - u16 ntc = tx_ring->next_to_clean, ntu = tx_ring->next_to_use; - unsigned int total_packets = 0, total_bytes = 0; - struct xsk_buff_pool *pool = tx_ring->xsk_pool; - union ixgbe_adv_tx_desc *tx_desc; - struct libie_xg_tx_buffer *tx_bi; - u32 xsk_frames = 0; - - tx_bi = &tx_ring->tx_buffer_info[ntc]; - tx_desc = IXGBE_TX_DESC(tx_ring, ntc); - - while (ntc != ntu) { - if (!(tx_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD))) - break; - - total_bytes += tx_bi->bytecount; - total_packets += tx_bi->gso_segs; - - if (tx_bi->xdpf) - ixgbe_clean_xdp_tx_buffer(tx_ring, tx_bi); - else - xsk_frames++; - - tx_bi->xdpf = NULL; - - tx_bi++; - tx_desc++; - ntc++; - if (unlikely(ntc == tx_ring->count)) { - ntc = 0; - tx_bi = tx_ring->tx_buffer_info; - tx_desc = IXGBE_TX_DESC(tx_ring, 0); - } + u32 budget = min_t(u32, napi_budget, tx_ring->thresh); - /* issue prefetch for next Tx descriptor */ - prefetch(tx_desc); - } - - tx_ring->next_to_clean = ntc; - ixgbe_update_tx_ring_stats(tx_ring, q_vector, total_packets, - total_bytes); - - if (xsk_frames) - xsk_tx_completed(pool, xsk_frames); - - if (xsk_uses_need_wakeup(pool)) - xsk_set_tx_need_wakeup(pool); - - return ixgbe_xmit_zc(tx_ring, q_vector->tx.work_limit); + return libeth_xsk_xmit_do_bulk(tx_ring->xsk_pool, tx_ring, budget, + NULL, ixgbe_prep_xdp_sq, + libie_xg_xsk_xmit_desc, + libie_xg_xdp_rs_and_bump); } int ixgbe_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) @@ -419,29 +254,3 @@ int ixgbe_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) return 0; } - -void ixgbe_xsk_clean_tx_ring(struct ixgbe_ring *tx_ring) -{ - u16 ntc = tx_ring->next_to_clean, ntu = tx_ring->next_to_use; - struct xsk_buff_pool *pool = tx_ring->xsk_pool; - struct libie_xg_tx_buffer *tx_bi; - u32 xsk_frames = 0; - - while (ntc != ntu) { - tx_bi = &tx_ring->tx_buffer_info[ntc]; - - if (tx_bi->xdpf) - ixgbe_clean_xdp_tx_buffer(tx_ring, tx_bi); - else - xsk_frames++; - - tx_bi->xdpf = NULL; - - ntc++; - if (ntc == tx_ring->count) - ntc = 0; - } - - if (xsk_frames) - xsk_tx_completed(pool, xsk_frames); -}