Skip to content

Commit b1d9154

Browse files
kuba-mookuba-moo
committed
Merge branch 'net-mlx5e-shampo-fixes-for-64kb-page-size'
Tariq Toukan says: ==================== net/mlx5e: SHAMPO fixes for 64KB page size This series by Dragos contains fixes for HW-GRO issues found on systems with 64KB page size. ==================== Link: https://patch.msgid.link/1762238915-1027590-1-git-send-email-tariqt@nvidia.com Signed-off-by: Jakub Kicinski <kuba@kernel.org>
2 parents ae4789a + d8a7ed9 commit b1d9154

3 files changed

Lines changed: 61 additions & 38 deletions

File tree

‎drivers/net/ethernet/mellanox/mlx5/core/en.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -634,7 +634,10 @@ struct mlx5e_dma_info {
634634
struct mlx5e_shampo_hd {
635635
struct mlx5e_frag_page *pages;
636636
u32 hd_per_wq;
637+
u32 hd_per_page;
637638
u16 hd_per_wqe;
639+
u8 log_hd_per_page;
640+
u8 log_hd_entry_size;
638641
unsigned long *bitmap;
639642
u16 pi;
640643
u16 ci;

‎drivers/net/ethernet/mellanox/mlx5/core/en_main.c‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -791,8 +791,9 @@ static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev,
791791
int node)
792792
{
793793
void *wqc = MLX5_ADDR_OF(rqc, rqp->rqc, wq);
794+
u8 log_hd_per_page, log_hd_entry_size;
795+
u16 hd_per_wq, hd_per_wqe;
794796
u32 hd_pool_size;
795-
u16 hd_per_wq;
796797
int wq_size;
797798
int err;
798799

@@ -815,11 +816,24 @@ static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev,
815816
if (err)
816817
goto err_umr_mkey;
817818

818-
rq->mpwqe.shampo->hd_per_wqe =
819-
mlx5e_shampo_hd_per_wqe(mdev, params, rqp);
819+
hd_per_wqe = mlx5e_shampo_hd_per_wqe(mdev, params, rqp);
820820
wq_size = BIT(MLX5_GET(wq, wqc, log_wq_sz));
821-
hd_pool_size = (rq->mpwqe.shampo->hd_per_wqe * wq_size) /
822-
MLX5E_SHAMPO_WQ_HEADER_PER_PAGE;
821+
822+
BUILD_BUG_ON(MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE > PAGE_SHIFT);
823+
if (hd_per_wqe >= MLX5E_SHAMPO_WQ_HEADER_PER_PAGE) {
824+
log_hd_per_page = MLX5E_SHAMPO_LOG_WQ_HEADER_PER_PAGE;
825+
log_hd_entry_size = MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE;
826+
} else {
827+
log_hd_per_page = order_base_2(hd_per_wqe);
828+
log_hd_entry_size = order_base_2(PAGE_SIZE / hd_per_wqe);
829+
}
830+
831+
rq->mpwqe.shampo->hd_per_wqe = hd_per_wqe;
832+
rq->mpwqe.shampo->hd_per_page = BIT(log_hd_per_page);
833+
rq->mpwqe.shampo->log_hd_per_page = log_hd_per_page;
834+
rq->mpwqe.shampo->log_hd_entry_size = log_hd_entry_size;
835+
836+
hd_pool_size = (hd_per_wqe * wq_size) >> log_hd_per_page;
823837

824838
if (netif_rxq_has_unreadable_mp(rq->netdev, rq->ix)) {
825839
/* Separate page pool for shampo headers */

‎drivers/net/ethernet/mellanox/mlx5/core/en_rx.c‎

Lines changed: 39 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -648,17 +648,20 @@ static void build_ksm_umr(struct mlx5e_icosq *sq, struct mlx5e_umr_wqe *umr_wqe,
648648
umr_wqe->hdr.uctrl.mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE);
649649
}
650650

651-
static struct mlx5e_frag_page *mlx5e_shampo_hd_to_frag_page(struct mlx5e_rq *rq, int header_index)
651+
static struct mlx5e_frag_page *mlx5e_shampo_hd_to_frag_page(struct mlx5e_rq *rq,
652+
int header_index)
652653
{
653-
BUILD_BUG_ON(MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE > PAGE_SHIFT);
654+
struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo;
654655

655-
return &rq->mpwqe.shampo->pages[header_index >> MLX5E_SHAMPO_LOG_WQ_HEADER_PER_PAGE];
656+
return &shampo->pages[header_index >> shampo->log_hd_per_page];
656657
}
657658

658-
static u64 mlx5e_shampo_hd_offset(int header_index)
659+
static u64 mlx5e_shampo_hd_offset(struct mlx5e_rq *rq, int header_index)
659660
{
660-
return (header_index & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1)) <<
661-
MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE;
661+
struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo;
662+
u32 hd_per_page = shampo->hd_per_page;
663+
664+
return (header_index & (hd_per_page - 1)) << shampo->log_hd_entry_size;
662665
}
663666

664667
static void mlx5e_free_rx_shampo_hd_entry(struct mlx5e_rq *rq, u16 header_index);
@@ -671,33 +674,32 @@ static int mlx5e_build_shampo_hd_umr(struct mlx5e_rq *rq,
671674
u16 pi, header_offset, err, wqe_bbs;
672675
u32 lkey = rq->mdev->mlx5e_res.hw_objs.mkey;
673676
struct mlx5e_umr_wqe *umr_wqe;
674-
int headroom, i = 0;
677+
int headroom, i;
675678

676679
headroom = rq->buff.headroom;
677680
wqe_bbs = MLX5E_KSM_UMR_WQEBBS(ksm_entries);
678681
pi = mlx5e_icosq_get_next_pi(sq, wqe_bbs);
679682
umr_wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
680683
build_ksm_umr(sq, umr_wqe, shampo->mkey_be, index, ksm_entries);
681684

682-
WARN_ON_ONCE(ksm_entries & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1));
683-
while (i < ksm_entries) {
684-
struct mlx5e_frag_page *frag_page = mlx5e_shampo_hd_to_frag_page(rq, index);
685+
for (i = 0; i < ksm_entries; i++, index++) {
686+
struct mlx5e_frag_page *frag_page;
685687
u64 addr;
686688

687-
err = mlx5e_page_alloc_fragmented(rq->hd_page_pool, frag_page);
688-
if (unlikely(err))
689-
goto err_unmap;
689+
frag_page = mlx5e_shampo_hd_to_frag_page(rq, index);
690+
header_offset = mlx5e_shampo_hd_offset(rq, index);
691+
if (!header_offset) {
692+
err = mlx5e_page_alloc_fragmented(rq->hd_page_pool,
693+
frag_page);
694+
if (err)
695+
goto err_unmap;
696+
}
690697

691698
addr = page_pool_get_dma_addr_netmem(frag_page->netmem);
692-
693-
for (int j = 0; j < MLX5E_SHAMPO_WQ_HEADER_PER_PAGE; j++) {
694-
header_offset = mlx5e_shampo_hd_offset(index++);
695-
696-
umr_wqe->inline_ksms[i++] = (struct mlx5_ksm) {
697-
.key = cpu_to_be32(lkey),
698-
.va = cpu_to_be64(addr + header_offset + headroom),
699-
};
700-
}
699+
umr_wqe->inline_ksms[i] = (struct mlx5_ksm) {
700+
.key = cpu_to_be32(lkey),
701+
.va = cpu_to_be64(addr + header_offset + headroom),
702+
};
701703
}
702704

703705
sq->db.wqe_info[pi] = (struct mlx5e_icosq_wqe_info) {
@@ -713,9 +715,9 @@ static int mlx5e_build_shampo_hd_umr(struct mlx5e_rq *rq,
713715
return 0;
714716

715717
err_unmap:
716-
while (--i) {
718+
while (--i >= 0) {
717719
--index;
718-
header_offset = mlx5e_shampo_hd_offset(index);
720+
header_offset = mlx5e_shampo_hd_offset(rq, index);
719721
if (!header_offset) {
720722
struct mlx5e_frag_page *frag_page = mlx5e_shampo_hd_to_frag_page(rq, index);
721723

@@ -735,12 +737,11 @@ static int mlx5e_alloc_rx_hd_mpwqe(struct mlx5e_rq *rq)
735737
struct mlx5e_icosq *sq = rq->icosq;
736738
int i, err, max_ksm_entries, len;
737739

738-
max_ksm_entries = ALIGN_DOWN(MLX5E_MAX_KSM_PER_WQE(rq->mdev),
739-
MLX5E_SHAMPO_WQ_HEADER_PER_PAGE);
740+
max_ksm_entries = MLX5E_MAX_KSM_PER_WQE(rq->mdev);
740741
ksm_entries = bitmap_find_window(shampo->bitmap,
741742
shampo->hd_per_wqe,
742743
shampo->hd_per_wq, shampo->pi);
743-
ksm_entries = ALIGN_DOWN(ksm_entries, MLX5E_SHAMPO_WQ_HEADER_PER_PAGE);
744+
ksm_entries = ALIGN_DOWN(ksm_entries, shampo->hd_per_page);
744745
if (!ksm_entries)
745746
return 0;
746747

@@ -858,7 +859,7 @@ mlx5e_free_rx_shampo_hd_entry(struct mlx5e_rq *rq, u16 header_index)
858859
{
859860
struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo;
860861

861-
if (((header_index + 1) & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1)) == 0) {
862+
if (((header_index + 1) & (shampo->hd_per_page - 1)) == 0) {
862863
struct mlx5e_frag_page *frag_page = mlx5e_shampo_hd_to_frag_page(rq, header_index);
863864

864865
mlx5e_page_release_fragmented(rq->hd_page_pool, frag_page);
@@ -1225,9 +1226,10 @@ static unsigned int mlx5e_lro_update_hdr(struct sk_buff *skb,
12251226
static void *mlx5e_shampo_get_packet_hd(struct mlx5e_rq *rq, u16 header_index)
12261227
{
12271228
struct mlx5e_frag_page *frag_page = mlx5e_shampo_hd_to_frag_page(rq, header_index);
1228-
u16 head_offset = mlx5e_shampo_hd_offset(header_index) + rq->buff.headroom;
1229+
u16 head_offset = mlx5e_shampo_hd_offset(rq, header_index);
1230+
void *addr = netmem_address(frag_page->netmem);
12291231

1230-
return netmem_address(frag_page->netmem) + head_offset;
1232+
return addr + head_offset + rq->buff.headroom;
12311233
}
12321234

12331235
static void mlx5e_shampo_update_ipv4_udp_hdr(struct mlx5e_rq *rq, struct iphdr *ipv4)
@@ -2267,7 +2269,8 @@ mlx5e_skb_from_cqe_shampo(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
22672269
struct mlx5_cqe64 *cqe, u16 header_index)
22682270
{
22692271
struct mlx5e_frag_page *frag_page = mlx5e_shampo_hd_to_frag_page(rq, header_index);
2270-
u16 head_offset = mlx5e_shampo_hd_offset(header_index);
2272+
u16 head_offset = mlx5e_shampo_hd_offset(rq, header_index);
2273+
struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo;
22712274
u16 head_size = cqe->shampo.header_size;
22722275
u16 rx_headroom = rq->buff.headroom;
22732276
struct sk_buff *skb = NULL;
@@ -2283,7 +2286,7 @@ mlx5e_skb_from_cqe_shampo(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
22832286
data = hdr + rx_headroom;
22842287
frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + head_size);
22852288

2286-
if (likely(frag_size <= BIT(MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE))) {
2289+
if (likely(frag_size <= BIT(shampo->log_hd_entry_size))) {
22872290
/* build SKB around header */
22882291
dma_sync_single_range_for_cpu(rq->pdev, dma_addr, 0, frag_size, rq->buff.map_dir);
22892292
net_prefetchw(hdr);
@@ -2356,7 +2359,10 @@ mlx5e_hw_gro_skb_has_enough_space(struct sk_buff *skb, u16 data_bcnt)
23562359
{
23572360
int nr_frags = skb_shinfo(skb)->nr_frags;
23582361

2359-
return PAGE_SIZE * nr_frags + data_bcnt <= GRO_LEGACY_MAX_SIZE;
2362+
if (PAGE_SIZE >= GRO_LEGACY_MAX_SIZE)
2363+
return skb->len + data_bcnt <= GRO_LEGACY_MAX_SIZE;
2364+
else
2365+
return PAGE_SIZE * nr_frags + data_bcnt <= GRO_LEGACY_MAX_SIZE;
23602366
}
23612367

23622368
static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)

0 commit comments

Comments
 (0)