efa: Add option to create CQ with external memory

Extend the EFA direct verbs interface to enable creation of CQs on top
of pre-allocated memory buffers. The memory can be passed by supplying
a dmabuf fd and offset.

Reviewed-by: Daniel Kranzdorf <[email protected]>
Reviewed-by: Yonatan Nachum <[email protected]>
Signed-off-by: Michael Margolin <[email protected]>

Author: mrgolin

providers/efa/efa.h

@@ -69,6 +69,7 @@ struct efa_cq {
 	size_t cqe_size;
 	uint8_t *buf;
 	size_t buf_size;
+	bool buf_mmaped;
 	uint32_t *db;
 	uint8_t *db_mmap_addr;
 	uint16_t cc; /* Consumer Counter */

providers/efa/efadv.h

@@ -48,6 +48,7 @@ enum {
 	EFADV_DEVICE_ATTR_CAPS_CQ_WITH_SGID = 1 << 2,
 	EFADV_DEVICE_ATTR_CAPS_RDMA_WRITE = 1 << 3,
 	EFADV_DEVICE_ATTR_CAPS_UNSOLICITED_WRITE_RECV = 1 << 4,
+	EFADV_DEVICE_ATTR_CAPS_CQ_WITH_EXT_MEM_DMABUF = 1 << 5,
 };
 
 struct efadv_device_attr {
@@ -86,9 +87,21 @@ enum {
 	EFADV_WC_EX_WITH_IS_UNSOLICITED = 1 << 1,
 };
 
+enum {
+	EFADV_CQ_INIT_FLAGS_EXT_MEM_DMABUF = 1 << 0,
+};
+
 struct efadv_cq_init_attr {
 	uint64_t comp_mask;
 	uint64_t wc_flags;
+	uint64_t flags;
+	struct {
+		uint8_t *buffer;
+		uint64_t length;
+		uint64_t offset;
+		int32_t fd;
+		uint8_t reserved[4];
+	} ext_mem_dmabuf;
 };
 
 struct ibv_cq_ex *efadv_create_cq(struct ibv_context *ibvctx,

providers/efa/man/efadv_create_cq.3.md

@@ -47,6 +47,14 @@ Compatibility is handled using the comp_mask and inlen fields.
 struct efadv_cq_init_attr {
 	uint64_t comp_mask;
 	uint64_t wc_flags;
+	uint64_t flags;
+	struct {
+		uint8_t *buffer;
+		uint64_t length;
+		uint64_t offset;
+		int32_t fd;
+		uint8_t reserved[4];
+	} ext_mem_dmabuf;
 };
 ```
 
@@ -65,6 +73,28 @@ struct efadv_cq_init_attr {
 	EFADV_WC_EX_WITH_IS_UNSOLICITED:
 		request for an option to check whether a receive WC is unsolicited.
 
+*flags*
+:       A bitwise OR of the various values described below.
+
+	EFADV_CQ_INIT_FLAGS_EXT_MEM_DMABUF:
+		create CQ with external memory provided via dmabuf.
+
+*ext_mem_dmabuf*
+:       Structure containing information about external memory when using
+	EFADV_CQ_INIT_FLAGS_EXT_MEM_DMABUF flag.
+
+	buffer:
+		Pointer to the memory mapped in the process's virtual address space. The field is
+		optional, but if not provided, the use of CQ poll interfaces should be avoided.
+
+	length:
+		Length of the memory region to use.
+
+	fd:
+		File descriptor of the dmabuf.
+
+	offset:
+		Offset within the dmabuf.
 
 # Completion iterator functions
 

providers/efa/man/efadv_query_device.3.md

@@ -85,6 +85,10 @@ struct efadv_device_attr {
 		requests in order to receive RDMA write with immediate and a WC generated for such
 		receive will be marked as unsolicited.
 
+	EFADV_DEVICE_ATTR_CAPS_CQ_WITH_EXT_MEM_DMABUF:
+		Indicates that creating CQs with external memory buffers by passing dmabuf is
+		supported.
+
 *max_rdma_size*
 :	Maximum RDMA transfer size in bytes.
 

providers/efa/verbs.c

@@ -175,6 +175,9 @@ int efadv_query_device(struct ibv_context *ibvctx,
 
 		if (EFA_DEV_CAP(ctx, UNSOLICITED_WRITE_RECV))
 			attr->device_caps |= EFADV_DEVICE_ATTR_CAPS_UNSOLICITED_WRITE_RECV;
+
+		if (EFA_DEV_CAP(ctx, CQ_WITH_EXT_MEM))
+			attr->device_caps |= EFADV_DEVICE_ATTR_CAPS_CQ_WITH_EXT_MEM_DMABUF;
 	}
 
 	if (vext_field_avail(typeof(*attr), max_rdma_size, inlen)) {
@@ -879,9 +882,9 @@ static void efa_cq_fill_pfns(struct efa_cq *cq,
 	if (attr->wc_flags & IBV_WC_EX_WITH_DLID_PATH_BITS)
 		ibvcqx->read_dlid_path_bits = efa_wc_read_dlid_path_bits;
 
-	if (efa_attr && (efa_attr->wc_flags & EFADV_WC_EX_WITH_SGID))
+	if (efa_attr->wc_flags & EFADV_WC_EX_WITH_SGID)
 		cq->dv_cq.wc_read_sgid = efa_wc_read_sgid;
-	if (efa_attr && (efa_attr->wc_flags & EFADV_WC_EX_WITH_IS_UNSOLICITED))
+	if (efa_attr->wc_flags & EFADV_WC_EX_WITH_IS_UNSOLICITED)
 		cq->dv_cq.wc_is_unsolicited = efa_wc_is_unsolicited;
 }
 
@@ -900,9 +903,11 @@ static struct ibv_cq_ex *create_cq(struct ibv_context *ibvctx,
 				   struct efadv_cq_init_attr *efa_attr)
 {
 	struct efa_context *ctx = to_efa_context(ibvctx);
+	struct verbs_create_cq_prov_attr prov_attr = {};
 	uint16_t cqe_size = ctx->ex_cqe_size;
 	struct efa_create_cq_resp resp = {};
 	struct efa_create_cq cmd = {};
+	uint32_t cmd_flags = 0;
 	uint16_t num_sub_cqs;
 	struct efa_cq *cq;
 	int sub_buf_size;
@@ -930,42 +935,58 @@ static struct ibv_cq_ex *create_cq(struct ibv_context *ibvctx,
 	if (!cq)
 		return NULL;
 
-	if (efa_attr && (efa_attr->wc_flags & EFADV_WC_EX_WITH_SGID))
+	if (efa_attr->wc_flags & EFADV_WC_EX_WITH_SGID)
 		cmd.flags |= EFA_CREATE_CQ_WITH_SGID;
 
 	num_sub_cqs = ctx->sub_cqs_per_cq;
 	cmd.num_sub_cqs = num_sub_cqs;
 	cmd.cq_entry_size = cqe_size;
+
+	if (efa_attr->flags & EFADV_CQ_INIT_FLAGS_EXT_MEM_DMABUF) {
+		prov_attr.buffer.length = efa_attr->ext_mem_dmabuf.length;
+		prov_attr.buffer.dmabuf.offset = efa_attr->ext_mem_dmabuf.offset;
+		prov_attr.buffer.dmabuf.fd = efa_attr->ext_mem_dmabuf.fd;
+		cmd_flags = CREATE_CQ_CMD_FLAGS_WITH_MEM_DMABUF;
+	}
+
 	if (attr->channel)
 		cmd.flags |= EFA_CREATE_CQ_WITH_COMPLETION_CHANNEL;
 
 	attr->cqe = roundup_pow_of_two(attr->cqe);
-	err = ibv_cmd_create_cq_ex(ibvctx, attr, NULL, &cq->verbs_cq,
+	err = ibv_cmd_create_cq_ex(ibvctx, attr, &prov_attr, &cq->verbs_cq,
 				   &cmd.ibv_cmd, sizeof(cmd),
-				   &resp.ibv_resp, sizeof(resp), 0);
+				   &resp.ibv_resp, sizeof(resp), cmd_flags);
 	if (err) {
 		errno = err;
 		goto err_free_cq;
 	}
 
 	sub_cq_size = cq->verbs_cq.cq.cqe;
 	cq->cqn = resp.cq_idx;
-	cq->buf_size = resp.q_mmap_size;
 	cq->num_sub_cqs = num_sub_cqs;
 	cq->cqe_size = cqe_size;
 	cq->dev = ibvctx->device;
 
-	cq->buf = mmap(NULL, cq->buf_size, PROT_READ, MAP_SHARED,
-		       ibvctx->cmd_fd, resp.q_mmap_key);
-	if (cq->buf == MAP_FAILED)
-		goto err_destroy_cq;
+	if (efa_attr->flags & EFADV_CQ_INIT_FLAGS_EXT_MEM_DMABUF) {
+		cq->buf_size = efa_attr->ext_mem_dmabuf.length;
+		cq->buf = efa_attr->ext_mem_dmabuf.buffer;
+	} else {
+		cq->buf_size = resp.q_mmap_size;
+		cq->buf = mmap(NULL, cq->buf_size, PROT_READ, MAP_SHARED, ibvctx->cmd_fd,
+			       resp.q_mmap_key);
+		if (cq->buf == MAP_FAILED)
+			goto err_destroy_cq;
+
+		cq->buf_mmaped = true;
+	}
 
-	buf = cq->buf;
-	sub_buf_size = cq->cqe_size * sub_cq_size;
-	for (i = 0; i < num_sub_cqs; i++) {
-		efa_sub_cq_initialize(&cq->sub_cq_arr[i], buf, sub_cq_size,
-				      cq->cqe_size);
-		buf += sub_buf_size;
+	if (cq->buf) {
+		buf = cq->buf;
+		sub_buf_size = cq->cqe_size * sub_cq_size;
+		for (i = 0; i < num_sub_cqs; i++) {
+			efa_sub_cq_initialize(&cq->sub_cq_arr[i], buf, sub_cq_size, cq->cqe_size);
+			buf += sub_buf_size;
+		}
 	}
 
 	if (resp.comp_mask & EFA_CREATE_CQ_RESP_DB_OFF) {
@@ -984,7 +1005,8 @@ static struct ibv_cq_ex *create_cq(struct ibv_context *ibvctx,
 	return &cq->verbs_cq.cq_ex;
 
 err_unmap_cq:
-	munmap(cq->buf, cq->buf_size);
+	if (cq->buf_mmaped)
+		munmap(cq->buf, cq->buf_size);
 err_destroy_cq:
 	ibv_cmd_destroy_cq(&cq->verbs_cq.cq);
 err_free_cq:
@@ -996,29 +1018,33 @@ static struct ibv_cq_ex *create_cq(struct ibv_context *ibvctx,
 struct ibv_cq *efa_create_cq(struct ibv_context *ibvctx, int ncqe,
 			     struct ibv_comp_channel *channel, int vec)
 {
+	struct efadv_cq_init_attr efa_attr = {};
 	struct ibv_cq_init_attr_ex attr_ex = {
 		.cqe = ncqe,
 		.channel = channel,
 		.comp_vector = vec
 	};
 	struct ibv_cq_ex *ibvcqx;
 
-	ibvcqx = create_cq(ibvctx, &attr_ex, NULL);
+	ibvcqx = create_cq(ibvctx, &attr_ex, &efa_attr);
 
 	return ibvcqx ? ibv_cq_ex_to_cq(ibvcqx) : NULL;
 }
 
 struct ibv_cq_ex *efa_create_cq_ex(struct ibv_context *ibvctx,
 				   struct ibv_cq_init_attr_ex *attr_ex)
 {
-	return create_cq(ibvctx, attr_ex, NULL);
+	struct efadv_cq_init_attr efa_attr = {};
+
+	return create_cq(ibvctx, attr_ex, &efa_attr);
 }
 
 struct ibv_cq_ex *efadv_create_cq(struct ibv_context *ibvctx,
 				  struct ibv_cq_init_attr_ex *attr_ex,
 				  struct efadv_cq_init_attr *efa_attr,
 				  uint32_t inlen)
 {
+	struct efadv_cq_init_attr local_efa_attr = {};
 	uint64_t supp_wc_flags = 0;
 	struct efa_context *ctx;
 
@@ -1048,7 +1074,8 @@ struct ibv_cq_ex *efadv_create_cq(struct ibv_context *ibvctx,
 		return NULL;
 	}
 
-	return create_cq(ibvctx, attr_ex, efa_attr);
+	memcpy(&local_efa_attr, efa_attr, min_t(uint32_t, inlen, sizeof(local_efa_attr)));
+	return create_cq(ibvctx, attr_ex, &local_efa_attr);
 }
 
 struct efadv_cq *efadv_cq_from_ibv_cq_ex(struct ibv_cq_ex *ibvcqx)
@@ -1071,7 +1098,8 @@ int efa_destroy_cq(struct ibv_cq *ibvcq)
 	}
 
 	munmap(cq->db_mmap_addr, to_efa_dev(cq->dev)->pg_sz);
-	munmap(cq->buf, cq->buf_size);
+	if (cq->buf_mmaped)
+		munmap(cq->buf, cq->buf_size);
 
 	pthread_spin_destroy(&cq->lock);