Skip to content

Commit f2c3037

Browse files
shirazsaleemrleon
shirazsaleem
authored and
rleon
committed
RDMA/irdma: Fix data race on CQP completion stats
CQP completion statistics is read lockesly in irdma_wait_event and irdma_check_cqp_progress while it can be updated in the completion thread irdma_sc_ccq_get_cqe_info on another CPU as KCSAN reports. Make completion statistics an atomic variable to reflect coherent updates to it. This will also avoid load/store tearing logic bug potentially possible by compiler optimizations. [77346.170861] BUG: KCSAN: data-race in irdma_handle_cqp_op [irdma] / irdma_sc_ccq_get_cqe_info [irdma] [77346.171383] write to 0xffff8a3250b108e0 of 8 bytes by task 9544 on cpu 4: [77346.171483] irdma_sc_ccq_get_cqe_info+0x27a/0x370 [irdma] [77346.171658] irdma_cqp_ce_handler+0x164/0x270 [irdma] [77346.171835] cqp_compl_worker+0x1b/0x20 [irdma] [77346.172009] process_one_work+0x4d1/0xa40 [77346.172024] worker_thread+0x319/0x700 [77346.172037] kthread+0x180/0x1b0 [77346.172054] ret_from_fork+0x22/0x30 [77346.172136] read to 0xffff8a3250b108e0 of 8 bytes by task 9838 on cpu 2: [77346.172234] irdma_handle_cqp_op+0xf4/0x4b0 [irdma] [77346.172413] irdma_cqp_aeq_cmd+0x75/0xa0 [irdma] [77346.172592] irdma_create_aeq+0x390/0x45a [irdma] [77346.172769] irdma_rt_init_hw.cold+0x212/0x85d [irdma] [77346.172944] irdma_probe+0x54f/0x620 [irdma] [77346.173122] auxiliary_bus_probe+0x66/0xa0 [77346.173137] really_probe+0x140/0x540 [77346.173154] __driver_probe_device+0xc7/0x220 [77346.173173] driver_probe_device+0x5f/0x140 [77346.173190] __driver_attach+0xf0/0x2c0 [77346.173208] bus_for_each_dev+0xa8/0xf0 [77346.173225] driver_attach+0x29/0x30 [77346.173240] bus_add_driver+0x29c/0x2f0 [77346.173255] driver_register+0x10f/0x1a0 [77346.173272] __auxiliary_driver_register+0xbc/0x140 [77346.173287] irdma_init_module+0x55/0x1000 [irdma] [77346.173460] do_one_initcall+0x7d/0x410 [77346.173475] do_init_module+0x81/0x2c0 [77346.173491] load_module+0x1232/0x12c0 [77346.173506] __do_sys_finit_module+0x101/0x180 [77346.173522] __x64_sys_finit_module+0x3c/0x50 [77346.173538] do_syscall_64+0x39/0x90 [77346.173553] entry_SYSCALL_64_after_hwframe+0x63/0xcd [77346.173634] value changed: 0x0000000000000094 -> 0x0000000000000095 Fixes: 915cc7a ("RDMA/irdma: Add miscellaneous utility definitions") Signed-off-by: Shiraz Saleem <shiraz.saleem@intel.com> Link: https://lore.kernel.org/r/20230711175253.1289-3-shiraz.saleem@intel.com Signed-off-by: Leon Romanovsky <leon@kernel.org>
1 parent 4984eb5 commit f2c3037

4 files changed

Lines changed: 36 additions & 36 deletions

File tree

‎drivers/infiniband/hw/irdma/ctrl.c‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2712,13 +2712,13 @@ static int irdma_sc_cq_modify(struct irdma_sc_cq *cq,
27122712
*/
27132713
void irdma_check_cqp_progress(struct irdma_cqp_timeout *timeout, struct irdma_sc_dev *dev)
27142714
{
2715-
if (timeout->compl_cqp_cmds != dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]) {
2716-
timeout->compl_cqp_cmds = dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS];
2715+
u64 completed_ops = atomic64_read(&dev->cqp->completed_ops);
2716+
2717+
if (timeout->compl_cqp_cmds != completed_ops) {
2718+
timeout->compl_cqp_cmds = completed_ops;
27172719
timeout->count = 0;
2718-
} else {
2719-
if (dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS] !=
2720-
timeout->compl_cqp_cmds)
2721-
timeout->count++;
2720+
} else if (timeout->compl_cqp_cmds != dev->cqp->requested_ops) {
2721+
timeout->count++;
27222722
}
27232723
}
27242724

@@ -2761,7 +2761,7 @@ static int irdma_cqp_poll_registers(struct irdma_sc_cqp *cqp, u32 tail,
27612761
if (newtail != tail) {
27622762
/* SUCCESS */
27632763
IRDMA_RING_MOVE_TAIL(cqp->sq_ring);
2764-
cqp->dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]++;
2764+
atomic64_inc(&cqp->completed_ops);
27652765
return 0;
27662766
}
27672767
udelay(cqp->dev->hw_attrs.max_sleep_count);
@@ -3121,8 +3121,8 @@ int irdma_sc_cqp_init(struct irdma_sc_cqp *cqp,
31213121
info->dev->cqp = cqp;
31223122

31233123
IRDMA_RING_INIT(cqp->sq_ring, cqp->sq_size);
3124-
cqp->dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS] = 0;
3125-
cqp->dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS] = 0;
3124+
cqp->requested_ops = 0;
3125+
atomic64_set(&cqp->completed_ops, 0);
31263126
/* for the cqp commands backlog. */
31273127
INIT_LIST_HEAD(&cqp->dev->cqp_cmd_head);
31283128

@@ -3274,7 +3274,7 @@ __le64 *irdma_sc_cqp_get_next_send_wqe_idx(struct irdma_sc_cqp *cqp, u64 scratch
32743274
if (ret_code)
32753275
return NULL;
32763276

3277-
cqp->dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS]++;
3277+
cqp->requested_ops++;
32783278
if (!*wqe_idx)
32793279
cqp->polarity = !cqp->polarity;
32803280
wqe = cqp->sq_base[*wqe_idx].elem;
@@ -3400,7 +3400,7 @@ int irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq,
34003400
dma_wmb(); /* make sure shadow area is updated before moving tail */
34013401

34023402
IRDMA_RING_MOVE_TAIL(cqp->sq_ring);
3403-
ccq->dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]++;
3403+
atomic64_inc(&cqp->completed_ops);
34043404

34053405
return ret_code;
34063406
}

‎drivers/infiniband/hw/irdma/defs.h‎

Lines changed: 22 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -191,32 +191,30 @@ enum irdma_cqp_op_type {
191191
IRDMA_OP_MANAGE_VF_PBLE_BP = 25,
192192
IRDMA_OP_QUERY_FPM_VAL = 26,
193193
IRDMA_OP_COMMIT_FPM_VAL = 27,
194-
IRDMA_OP_REQ_CMDS = 28,
195-
IRDMA_OP_CMPL_CMDS = 29,
196-
IRDMA_OP_AH_CREATE = 30,
197-
IRDMA_OP_AH_MODIFY = 31,
198-
IRDMA_OP_AH_DESTROY = 32,
199-
IRDMA_OP_MC_CREATE = 33,
200-
IRDMA_OP_MC_DESTROY = 34,
201-
IRDMA_OP_MC_MODIFY = 35,
202-
IRDMA_OP_STATS_ALLOCATE = 36,
203-
IRDMA_OP_STATS_FREE = 37,
204-
IRDMA_OP_STATS_GATHER = 38,
205-
IRDMA_OP_WS_ADD_NODE = 39,
206-
IRDMA_OP_WS_MODIFY_NODE = 40,
207-
IRDMA_OP_WS_DELETE_NODE = 41,
208-
IRDMA_OP_WS_FAILOVER_START = 42,
209-
IRDMA_OP_WS_FAILOVER_COMPLETE = 43,
210-
IRDMA_OP_SET_UP_MAP = 44,
211-
IRDMA_OP_GEN_AE = 45,
212-
IRDMA_OP_QUERY_RDMA_FEATURES = 46,
213-
IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY = 47,
214-
IRDMA_OP_ADD_LOCAL_MAC_ENTRY = 48,
215-
IRDMA_OP_DELETE_LOCAL_MAC_ENTRY = 49,
216-
IRDMA_OP_CQ_MODIFY = 50,
194+
IRDMA_OP_AH_CREATE = 28,
195+
IRDMA_OP_AH_MODIFY = 29,
196+
IRDMA_OP_AH_DESTROY = 30,
197+
IRDMA_OP_MC_CREATE = 31,
198+
IRDMA_OP_MC_DESTROY = 32,
199+
IRDMA_OP_MC_MODIFY = 33,
200+
IRDMA_OP_STATS_ALLOCATE = 34,
201+
IRDMA_OP_STATS_FREE = 35,
202+
IRDMA_OP_STATS_GATHER = 36,
203+
IRDMA_OP_WS_ADD_NODE = 37,
204+
IRDMA_OP_WS_MODIFY_NODE = 38,
205+
IRDMA_OP_WS_DELETE_NODE = 39,
206+
IRDMA_OP_WS_FAILOVER_START = 40,
207+
IRDMA_OP_WS_FAILOVER_COMPLETE = 41,
208+
IRDMA_OP_SET_UP_MAP = 42,
209+
IRDMA_OP_GEN_AE = 43,
210+
IRDMA_OP_QUERY_RDMA_FEATURES = 44,
211+
IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY = 45,
212+
IRDMA_OP_ADD_LOCAL_MAC_ENTRY = 46,
213+
IRDMA_OP_DELETE_LOCAL_MAC_ENTRY = 47,
214+
IRDMA_OP_CQ_MODIFY = 48,
217215

218216
/* Must be last entry*/
219-
IRDMA_MAX_CQP_OPS = 51,
217+
IRDMA_MAX_CQP_OPS = 49,
220218
};
221219

222220
/* CQP SQ WQES */

‎drivers/infiniband/hw/irdma/type.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -365,6 +365,8 @@ struct irdma_sc_cqp {
365365
struct irdma_dcqcn_cc_params dcqcn_params;
366366
__le64 *host_ctx;
367367
u64 *scratch_array;
368+
u64 requested_ops;
369+
atomic64_t completed_ops;
368370
u32 cqp_id;
369371
u32 sq_size;
370372
u32 hw_sq_size;

‎drivers/infiniband/hw/irdma/utils.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -567,7 +567,7 @@ static int irdma_wait_event(struct irdma_pci_f *rf,
567567
bool cqp_error = false;
568568
int err_code = 0;
569569

570-
cqp_timeout.compl_cqp_cmds = rf->sc_dev.cqp_cmd_stats[IRDMA_OP_CMPL_CMDS];
570+
cqp_timeout.compl_cqp_cmds = atomic64_read(&rf->sc_dev.cqp->completed_ops);
571571
do {
572572
irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq);
573573
if (wait_event_timeout(cqp_request->waitq,

0 commit comments

Comments
 (0)