efa: Fix work request index double use

[ Upstream commit 050b787 ]

When polling a CQE related to a destroyed QP, we do not update the
current work queue reference, what results in "returning" the work
request index of the CQE to a wrong pool. This can later result in two
different inflight work requests using same index and mistakenly
completing with same work request id.

Fix by zeroing the work queue pointer on error and unify conditions
with unsolicited completion flow.

Fixes: 40ee2e2 ("efa: Introduce create extended CQ support")
Reviewed-by: Daniel Kranzdorf <[email protected]>
Reviewed-by: Nitzan Lavy <[email protected]>
Signed-off-by: Michael Margolin <[email protected]>
Signed-off-by: Nicolas Morey <[email protected]>

Author: mrgolin

providers/efa/verbs.c

@@ -646,7 +646,13 @@ static void efa_process_cqe(struct efa_cq *cq, struct ibv_wc *wc,
 		struct efa_io_rx_cdesc_ex *rcqe =
 			container_of(cqe, struct efa_io_rx_cdesc_ex, base.common);
 
-		cq->cur_wq = &qp->rq.wq;
+		if (EFA_GET(&cqe->flags, EFA_IO_CDESC_COMMON_UNSOLICITED)) {
+			cq->cur_wq = NULL;
+			wc->wr_id = 0;
+		} else {
+			cq->cur_wq = &qp->rq.wq;
+			wc->wr_id = cq->cur_wq->wrid[wrid_idx];
+		}
 
 		wc->byte_len = rcqe->base.length;
 
@@ -666,9 +672,6 @@ static void efa_process_cqe(struct efa_cq *cq, struct ibv_wc *wc,
 			wc->wc_flags |= IBV_WC_WITH_IMM;
 		}
 
-		wc->wr_id = !EFA_GET(&cqe->flags, EFA_IO_CDESC_COMMON_UNSOLICITED) ?
-			cq->cur_wq->wrid[wrid_idx] : 0;
-
 		rdma_tracepoint(rdma_core_efa, process_completion, cq->dev->name, wc->wr_id,
 				wc->status, wc->opcode, wc->src_qp, wc->qp_num, wc->slid,
 				wc->byte_len);
@@ -692,9 +695,14 @@ static void efa_process_ex_cqe(struct efa_cq *cq, struct efa_qp *qp)
 				ibvcqx->status, efa_wc_read_opcode(ibvcqx), cqe->qp_num,
 				UINT32_MAX, UINT16_MAX, efa_wc_read_byte_len(ibvcqx));
 	} else {
-		cq->cur_wq = &qp->rq.wq;
-		ibvcqx->wr_id = !EFA_GET(&cqe->flags, EFA_IO_CDESC_COMMON_UNSOLICITED) ?
-			cq->cur_wq->wrid[wrid_idx] : 0;
+		if (EFA_GET(&cqe->flags, EFA_IO_CDESC_COMMON_UNSOLICITED)) {
+			cq->cur_wq = NULL;
+			ibvcqx->wr_id = 0;
+		} else {
+			cq->cur_wq = &qp->rq.wq;
+			ibvcqx->wr_id = cq->cur_wq->wrid[wrid_idx];
+		}
+
 		ibvcqx->status = to_ibv_status(cqe->status);
 
 		rdma_tracepoint(rdma_core_efa, process_completion, cq->dev->name, ibvcqx->wr_id,
@@ -726,6 +734,7 @@ static inline int efa_poll_sub_cq(struct efa_cq *cq, struct efa_sub_cq *sub_cq,
 		 */
 		*cur_qp = ctx->qp_table[qpn & ctx->qp_table_sz_m1];
 		if (!*cur_qp) {
+			cq->cur_wq = NULL;
 			verbs_err(&ctx->ibvctx,
 				  "QP[%u] does not exist in QP table\n",
 				  qpn);
@@ -737,7 +746,7 @@ static inline int efa_poll_sub_cq(struct efa_cq *cq, struct efa_sub_cq *sub_cq,
 		efa_process_ex_cqe(cq, *cur_qp);
 	} else {
 		efa_process_cqe(cq, wc, *cur_qp);
-		if (!EFA_GET(&cq->cur_cqe->flags, EFA_IO_CDESC_COMMON_UNSOLICITED))
+		if (cq->cur_wq)
 			efa_wq_put_wrid_idx_unlocked(cq->cur_wq, cq->cur_cqe->req_id);
 	}
 
@@ -822,7 +831,7 @@ static int efa_next_poll(struct ibv_cq_ex *ibvcqx)
 	struct efa_cq *cq = to_efa_cq_ex(ibvcqx);
 	int ret;
 
-	if (!EFA_GET(&cq->cur_cqe->flags, EFA_IO_CDESC_COMMON_UNSOLICITED))
+	if (cq->cur_wq)
 		efa_wq_put_wrid_idx_unlocked(cq->cur_wq, cq->cur_cqe->req_id);
 	ret = efa_poll_sub_cqs(cq, NULL, true);
 
@@ -834,7 +843,7 @@ static void efa_end_poll(struct ibv_cq_ex *ibvcqx)
 	struct efa_cq *cq = to_efa_cq_ex(ibvcqx);
 
 	if (cq->cur_cqe) {
-		if (!EFA_GET(&cq->cur_cqe->flags, EFA_IO_CDESC_COMMON_UNSOLICITED))
+		if (cq->cur_wq)
 			efa_wq_put_wrid_idx_unlocked(cq->cur_wq, cq->cur_cqe->req_id);
 		if (cq->db)
 			efa_update_cq_doorbell(cq, false);