Skip to content

Commit 25ba596

Browse files
sanmanp2111993kuba-moo
sanmanp2111993
authored and
kuba-moo
committed
eth: fbnic: add PCIe hardware statistics
Add PCIe hardware statistics support to the fbnic driver. These stats provide insight into PCIe transaction performance and error conditions. Which includes, read/write and completion TLP counts and DWORD counts and debug counters for tag, completion credit and NP credit exhaustion The stats are exposed via debugfs and can be used to monitor PCIe performance and debug PCIe issues. Signed-off-by: Sanman Pradhan <sanman.p211993@gmail.com> Reviewed-by: Andrew Lunn <andrew@lunn.ch> Link: https://patch.msgid.link/20241115015344.757567-5-kuba@kernel.org Signed-off-by: Jakub Kicinski <kuba@kernel.org>
1 parent 08606cb commit 25ba596

6 files changed

Lines changed: 231 additions & 0 deletions

File tree

‎Documentation/networking/device_drivers/ethernet/meta/fbnic.rst‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,3 +27,33 @@ driver takes over.
2727
devlink dev info provides version information for all three components. In
2828
addition to the version the hg commit hash of the build is included as a
2929
separate entry.
30+
31+
Statistics
32+
----------
33+
34+
PCIe
35+
~~~~
36+
37+
The fbnic driver exposes PCIe hardware performance statistics through debugfs
38+
(``pcie_stats``). These statistics provide insights into PCIe transaction
39+
behavior and potential performance bottlenecks.
40+
41+
1. PCIe Transaction Counters:
42+
43+
These counters track PCIe transaction activity:
44+
- ``pcie_ob_rd_tlp``: Outbound read Transaction Layer Packets count
45+
- ``pcie_ob_rd_dword``: DWORDs transferred in outbound read transactions
46+
- ``pcie_ob_wr_tlp``: Outbound write Transaction Layer Packets count
47+
- ``pcie_ob_wr_dword``: DWORDs transferred in outbound write
48+
transactions
49+
- ``pcie_ob_cpl_tlp``: Outbound completion TLP count
50+
- ``pcie_ob_cpl_dword``: DWORDs transferred in outbound completion TLPs
51+
52+
2. PCIe Resource Monitoring:
53+
54+
These counters indicate PCIe resource exhaustion events:
55+
- ``pcie_ob_rd_no_tag``: Read requests dropped due to tag unavailability
56+
- ``pcie_ob_rd_no_cpl_cred``: Read requests dropped due to completion
57+
credit exhaustion
58+
- ``pcie_ob_rd_no_np_cred``: Read requests dropped due to non-posted
59+
credit exhaustion

‎drivers/net/ethernet/meta/fbnic/fbnic_csr.h‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -918,6 +918,43 @@ enum {
918918
#define FBNIC_MAX_QUEUES 128
919919
#define FBNIC_CSR_END_QUEUE (0x40000 + 0x400 * FBNIC_MAX_QUEUES - 1)
920920

921+
/* PUL User Registers*/
922+
#define FBNIC_PUL_USER_OB_RD_TLP_CNT_31_0 \
923+
0x3106e /* 0xc41b8 */
924+
#define FBNIC_PUL_USER_OB_RD_DWORD_CNT_31_0 \
925+
0x31070 /* 0xc41c0 */
926+
#define FBNIC_PUL_USER_OB_RD_DWORD_CNT_63_32 \
927+
0x31071 /* 0xc41c4 */
928+
#define FBNIC_PUL_USER_OB_WR_TLP_CNT_31_0 \
929+
0x31072 /* 0xc41c8 */
930+
#define FBNIC_PUL_USER_OB_WR_TLP_CNT_63_32 \
931+
0x31073 /* 0xc41cc */
932+
#define FBNIC_PUL_USER_OB_WR_DWORD_CNT_31_0 \
933+
0x31074 /* 0xc41d0 */
934+
#define FBNIC_PUL_USER_OB_WR_DWORD_CNT_63_32 \
935+
0x31075 /* 0xc41d4 */
936+
#define FBNIC_PUL_USER_OB_CPL_TLP_CNT_31_0 \
937+
0x31076 /* 0xc41d8 */
938+
#define FBNIC_PUL_USER_OB_CPL_TLP_CNT_63_32 \
939+
0x31077 /* 0xc41dc */
940+
#define FBNIC_PUL_USER_OB_CPL_DWORD_CNT_31_0 \
941+
0x31078 /* 0xc41e0 */
942+
#define FBNIC_PUL_USER_OB_CPL_DWORD_CNT_63_32 \
943+
0x31079 /* 0xc41e4 */
944+
#define FBNIC_PUL_USER_OB_RD_DBG_CNT_CPL_CRED_31_0 \
945+
0x3107a /* 0xc41e8 */
946+
#define FBNIC_PUL_USER_OB_RD_DBG_CNT_CPL_CRED_63_32 \
947+
0x3107b /* 0xc41ec */
948+
#define FBNIC_PUL_USER_OB_RD_DBG_CNT_TAG_31_0 \
949+
0x3107c /* 0xc41f0 */
950+
#define FBNIC_PUL_USER_OB_RD_DBG_CNT_TAG_63_32 \
951+
0x3107d /* 0xc41f4 */
952+
#define FBNIC_PUL_USER_OB_RD_DBG_CNT_NP_CRED_31_0 \
953+
0x3107e /* 0xc41f8 */
954+
#define FBNIC_PUL_USER_OB_RD_DBG_CNT_NP_CRED_63_32 \
955+
0x3107f /* 0xc41fc */
956+
#define FBNIC_CSR_END_PUL_USER 0x31080 /* CSR section delimiter */
957+
921958
/* BAR 4 CSRs */
922959

923960
/* The IPC mailbox consists of 32 mailboxes, with each mailbox consisting

‎drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,51 @@
33

44
#include <linux/debugfs.h>
55
#include <linux/pci.h>
6+
#include <linux/rtnetlink.h>
7+
#include <linux/seq_file.h>
68

79
#include "fbnic.h"
810

911
static struct dentry *fbnic_dbg_root;
1012

13+
static int fbnic_dbg_pcie_stats_show(struct seq_file *s, void *v)
14+
{
15+
struct fbnic_dev *fbd = s->private;
16+
17+
rtnl_lock();
18+
fbnic_get_hw_stats(fbd);
19+
20+
seq_printf(s, "ob_rd_tlp: %llu\n", fbd->hw_stats.pcie.ob_rd_tlp.value);
21+
seq_printf(s, "ob_rd_dword: %llu\n",
22+
fbd->hw_stats.pcie.ob_rd_dword.value);
23+
seq_printf(s, "ob_wr_tlp: %llu\n", fbd->hw_stats.pcie.ob_wr_tlp.value);
24+
seq_printf(s, "ob_wr_dword: %llu\n",
25+
fbd->hw_stats.pcie.ob_wr_dword.value);
26+
seq_printf(s, "ob_cpl_tlp: %llu\n",
27+
fbd->hw_stats.pcie.ob_cpl_tlp.value);
28+
seq_printf(s, "ob_cpl_dword: %llu\n",
29+
fbd->hw_stats.pcie.ob_cpl_dword.value);
30+
seq_printf(s, "ob_rd_no_tag: %llu\n",
31+
fbd->hw_stats.pcie.ob_rd_no_tag.value);
32+
seq_printf(s, "ob_rd_no_cpl_cred: %llu\n",
33+
fbd->hw_stats.pcie.ob_rd_no_cpl_cred.value);
34+
seq_printf(s, "ob_rd_no_np_cred: %llu\n",
35+
fbd->hw_stats.pcie.ob_rd_no_np_cred.value);
36+
rtnl_unlock();
37+
38+
return 0;
39+
}
40+
41+
DEFINE_SHOW_ATTRIBUTE(fbnic_dbg_pcie_stats);
42+
1143
void fbnic_dbg_fbd_init(struct fbnic_dev *fbd)
1244
{
1345
struct pci_dev *pdev = to_pci_dev(fbd->dev);
1446
const char *name = pci_name(pdev);
1547

1648
fbd->dbg_fbd = debugfs_create_dir(name, fbnic_dbg_root);
49+
debugfs_create_file("pcie_stats", 0400, fbd->dbg_fbd, fbd,
50+
&fbnic_dbg_pcie_stats_fops);
1751
}
1852

1953
void fbnic_dbg_fbd_exit(struct fbnic_dev *fbd)

‎drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.c‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,3 +28,117 @@ u64 fbnic_stat_rd64(struct fbnic_dev *fbd, u32 reg, u32 offset)
2828
*/
2929
return ((u64)upper << 32);
3030
}
31+
32+
static void fbnic_hw_stat_rst64(struct fbnic_dev *fbd, u32 reg, s32 offset,
33+
struct fbnic_stat_counter *stat)
34+
{
35+
/* Record initial counter values and compute deltas from there to ensure
36+
* stats start at 0 after reboot/reset. This avoids exposing absolute
37+
* hardware counter values to userspace.
38+
*/
39+
stat->u.old_reg_value_64 = fbnic_stat_rd64(fbd, reg, offset);
40+
}
41+
42+
static void fbnic_hw_stat_rd64(struct fbnic_dev *fbd, u32 reg, s32 offset,
43+
struct fbnic_stat_counter *stat)
44+
{
45+
u64 new_reg_value;
46+
47+
new_reg_value = fbnic_stat_rd64(fbd, reg, offset);
48+
stat->value += new_reg_value - stat->u.old_reg_value_64;
49+
stat->u.old_reg_value_64 = new_reg_value;
50+
}
51+
52+
static void fbnic_reset_pcie_stats_asic(struct fbnic_dev *fbd,
53+
struct fbnic_pcie_stats *pcie)
54+
{
55+
fbnic_hw_stat_rst64(fbd,
56+
FBNIC_PUL_USER_OB_RD_TLP_CNT_31_0,
57+
1,
58+
&pcie->ob_rd_tlp);
59+
fbnic_hw_stat_rst64(fbd,
60+
FBNIC_PUL_USER_OB_RD_DWORD_CNT_31_0,
61+
1,
62+
&pcie->ob_rd_dword);
63+
fbnic_hw_stat_rst64(fbd,
64+
FBNIC_PUL_USER_OB_CPL_TLP_CNT_31_0,
65+
1,
66+
&pcie->ob_cpl_tlp);
67+
fbnic_hw_stat_rst64(fbd,
68+
FBNIC_PUL_USER_OB_CPL_DWORD_CNT_31_0,
69+
1,
70+
&pcie->ob_cpl_dword);
71+
fbnic_hw_stat_rst64(fbd,
72+
FBNIC_PUL_USER_OB_WR_TLP_CNT_31_0,
73+
1,
74+
&pcie->ob_wr_tlp);
75+
fbnic_hw_stat_rst64(fbd,
76+
FBNIC_PUL_USER_OB_WR_DWORD_CNT_31_0,
77+
1,
78+
&pcie->ob_wr_dword);
79+
80+
fbnic_hw_stat_rst64(fbd,
81+
FBNIC_PUL_USER_OB_RD_DBG_CNT_TAG_31_0,
82+
1,
83+
&pcie->ob_rd_no_tag);
84+
fbnic_hw_stat_rst64(fbd,
85+
FBNIC_PUL_USER_OB_RD_DBG_CNT_CPL_CRED_31_0,
86+
1,
87+
&pcie->ob_rd_no_cpl_cred);
88+
fbnic_hw_stat_rst64(fbd,
89+
FBNIC_PUL_USER_OB_RD_DBG_CNT_NP_CRED_31_0,
90+
1,
91+
&pcie->ob_rd_no_np_cred);
92+
}
93+
94+
static void fbnic_get_pcie_stats_asic64(struct fbnic_dev *fbd,
95+
struct fbnic_pcie_stats *pcie)
96+
{
97+
fbnic_hw_stat_rd64(fbd,
98+
FBNIC_PUL_USER_OB_RD_TLP_CNT_31_0,
99+
1,
100+
&pcie->ob_rd_tlp);
101+
fbnic_hw_stat_rd64(fbd,
102+
FBNIC_PUL_USER_OB_RD_DWORD_CNT_31_0,
103+
1,
104+
&pcie->ob_rd_dword);
105+
fbnic_hw_stat_rd64(fbd,
106+
FBNIC_PUL_USER_OB_WR_TLP_CNT_31_0,
107+
1,
108+
&pcie->ob_wr_tlp);
109+
fbnic_hw_stat_rd64(fbd,
110+
FBNIC_PUL_USER_OB_WR_DWORD_CNT_31_0,
111+
1,
112+
&pcie->ob_wr_dword);
113+
fbnic_hw_stat_rd64(fbd,
114+
FBNIC_PUL_USER_OB_CPL_TLP_CNT_31_0,
115+
1,
116+
&pcie->ob_cpl_tlp);
117+
fbnic_hw_stat_rd64(fbd,
118+
FBNIC_PUL_USER_OB_CPL_DWORD_CNT_31_0,
119+
1,
120+
&pcie->ob_cpl_dword);
121+
122+
fbnic_hw_stat_rd64(fbd,
123+
FBNIC_PUL_USER_OB_RD_DBG_CNT_TAG_31_0,
124+
1,
125+
&pcie->ob_rd_no_tag);
126+
fbnic_hw_stat_rd64(fbd,
127+
FBNIC_PUL_USER_OB_RD_DBG_CNT_CPL_CRED_31_0,
128+
1,
129+
&pcie->ob_rd_no_cpl_cred);
130+
fbnic_hw_stat_rd64(fbd,
131+
FBNIC_PUL_USER_OB_RD_DBG_CNT_NP_CRED_31_0,
132+
1,
133+
&pcie->ob_rd_no_np_cred);
134+
}
135+
136+
void fbnic_reset_hw_stats(struct fbnic_dev *fbd)
137+
{
138+
fbnic_reset_pcie_stats_asic(fbd, &fbd->hw_stats.pcie);
139+
}
140+
141+
void fbnic_get_hw_stats(struct fbnic_dev *fbd)
142+
{
143+
fbnic_get_pcie_stats_asic64(fbd, &fbd->hw_stats.pcie);
144+
}

‎drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.h‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,24 @@ struct fbnic_mac_stats {
3737
struct fbnic_eth_mac_stats eth_mac;
3838
};
3939

40+
struct fbnic_pcie_stats {
41+
struct fbnic_stat_counter ob_rd_tlp, ob_rd_dword;
42+
struct fbnic_stat_counter ob_wr_tlp, ob_wr_dword;
43+
struct fbnic_stat_counter ob_cpl_tlp, ob_cpl_dword;
44+
45+
struct fbnic_stat_counter ob_rd_no_tag;
46+
struct fbnic_stat_counter ob_rd_no_cpl_cred;
47+
struct fbnic_stat_counter ob_rd_no_np_cred;
48+
};
49+
4050
struct fbnic_hw_stats {
4151
struct fbnic_mac_stats mac;
52+
struct fbnic_pcie_stats pcie;
4253
};
4354

4455
u64 fbnic_stat_rd64(struct fbnic_dev *fbd, u32 reg, u32 offset);
4556

57+
void fbnic_reset_hw_stats(struct fbnic_dev *fbd);
4658
void fbnic_get_hw_stats(struct fbnic_dev *fbd);
4759

4860
#endif /* _FBNIC_HW_STATS_H_ */

‎drivers/net/ethernet/meta/fbnic/fbnic_pci.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
#include "fbnic.h"
1111
#include "fbnic_drvinfo.h"
12+
#include "fbnic_hw_stats.h"
1213
#include "fbnic_netdev.h"
1314

1415
char fbnic_driver_name[] = DRV_NAME;
@@ -290,6 +291,9 @@ static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
290291
fbnic_devlink_register(fbd);
291292
fbnic_dbg_fbd_init(fbd);
292293

294+
/* Capture snapshot of hardware stats so netdev can calculate delta */
295+
fbnic_reset_hw_stats(fbd);
296+
293297
fbnic_hwmon_register(fbd);
294298

295299
if (!fbd->dsn) {

0 commit comments

Comments
 (0)