Skip to content

Commit f544285

Browse files
rleonrleon
authored
Merge pull request #1608 from amzn/cq-with-ext-mem
Enable creation of CQs with external memory
2 parents d0fcdd9 + b41b472 commit f544285

21 files changed

+418
-63
lines changed

‎CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ set(PACKAGE_VERSION "59.0")
8383
# When this is changed the values in these files need changing too:
8484
# debian/control
8585
# debian/libibverbs1.symbols
86-
set(IBVERBS_PABI_VERSION "57")
86+
set(IBVERBS_PABI_VERSION "59")
8787
set(IBVERBS_PROVIDER_SUFFIX "-rdmav${IBVERBS_PABI_VERSION}.so")
8888

8989
#-------------------------

‎debian/control‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,7 @@ Section: libs
146146
Pre-Depends: ${misc:Pre-Depends}
147147
Depends: adduser, ${misc:Depends}, ${shlibs:Depends}
148148
Recommends: ibverbs-providers
149-
Breaks: ibverbs-providers (<< 57~)
149+
Breaks: ibverbs-providers (<< 59~)
150150
Description: Library for direct userspace use of RDMA (InfiniBand/iWARP)
151151
libibverbs is a library that allows userspace processes to use RDMA
152152
"verbs" as described in the InfiniBand Architecture Specification and

‎debian/ibverbs-providers.symbols‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,13 +170,16 @@ libefa.so.1 ibverbs-providers #MINVER#
170170
EFA_1.1@EFA_1.1 26
171171
EFA_1.2@EFA_1.2 43
172172
EFA_1.3@EFA_1.3 50
173+
EFA_1.4@EFA_1.4 59
173174
efadv_create_driver_qp@EFA_1.0 24
174175
efadv_create_qp_ex@EFA_1.1 26
175176
efadv_query_device@EFA_1.1 26
176177
efadv_query_ah@EFA_1.1 26
177178
efadv_cq_from_ibv_cq_ex@EFA_1.2 43
178179
efadv_create_cq@EFA_1.2 43
179180
efadv_query_mr@EFA_1.3 50
181+
efadv_query_qp_wqs@EFA_1.4 59
182+
efadv_query_cq@EFA_1.4 59
180183
libhns.so.1 ibverbs-providers #MINVER#
181184
* Build-Depends-Package: libibverbs-dev
182185
HNS_1.0@HNS_1.0 51

‎debian/libibverbs1.symbols‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ libibverbs.so.1 libibverbs1 #MINVER#
1313
IBVERBS_1.13@IBVERBS_1.13 35
1414
IBVERBS_1.14@IBVERBS_1.14 36
1515
IBVERBS_1.15@IBVERBS_1.15 37
16-
(symver)IBVERBS_PRIVATE_57 57
16+
(symver)IBVERBS_PRIVATE_59 59
1717
_ibv_query_gid_ex@IBVERBS_1.11 32
1818
_ibv_query_gid_table@IBVERBS_1.11 32
1919
ibv_ack_async_event@IBVERBS_1.0 1.1.6

‎libibverbs/cmd_cq.c‎

Lines changed: 31 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535

3636
static int ibv_icmd_create_cq(struct ibv_context *context, int cqe,
3737
struct ibv_comp_channel *channel, int comp_vector,
38+
struct verbs_create_cq_prov_attr *prov_attr,
3839
uint32_t flags, struct ibv_cq *cq,
3940
struct ibv_command_buffer *link,
4041
uint32_t cmd_flags)
@@ -43,11 +44,18 @@ static int ibv_icmd_create_cq(struct ibv_context *context, int cqe,
4344
struct verbs_ex_private *priv = get_priv(context);
4445
struct ib_uverbs_attr *handle;
4546
struct ib_uverbs_attr *async_fd_attr;
47+
bool prov_atts_expected;
4648
uint32_t resp_cqe;
4749
int ret;
4850

4951
cq->context = context;
5052

53+
prov_atts_expected = cmd_flags & (CREATE_CQ_CMD_FLAGS_WITH_MEM_VA |
54+
CREATE_CQ_CMD_FLAGS_WITH_MEM_DMABUF);
55+
56+
if (prov_atts_expected && !prov_attr)
57+
return EINVAL;
58+
5159
handle = fill_attr_out_obj(cmdb, UVERBS_ATTR_CREATE_CQ_HANDLE);
5260
fill_attr_out_ptr(cmdb, UVERBS_ATTR_CREATE_CQ_RESP_CQE, &resp_cqe);
5361

@@ -63,6 +71,22 @@ static int ibv_icmd_create_cq(struct ibv_context *context, int cqe,
6371
/* Prevent fallback to the 'write' mode if kernel doesn't support it */
6472
attr_optional(async_fd_attr);
6573

74+
if (cmd_flags & CREATE_CQ_CMD_FLAGS_WITH_MEM_VA) {
75+
fill_attr_in_uint64(cmdb, UVERBS_ATTR_CREATE_CQ_BUFFER_VA,
76+
(uintptr_t)prov_attr->buffer.ptr);
77+
fill_attr_in_uint64(cmdb, UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH,
78+
prov_attr->buffer.length);
79+
fallback_require_ioctl(cmdb);
80+
} else if (cmd_flags & CREATE_CQ_CMD_FLAGS_WITH_MEM_DMABUF) {
81+
fill_attr_in_fd(cmdb, UVERBS_ATTR_CREATE_CQ_BUFFER_FD,
82+
prov_attr->buffer.dmabuf.fd);
83+
fill_attr_in_uint64(cmdb, UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET,
84+
prov_attr->buffer.dmabuf.offset);
85+
fill_attr_in_uint64(cmdb, UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH,
86+
prov_attr->buffer.length);
87+
fallback_require_ioctl(cmdb);
88+
}
89+
6690
if (flags) {
6791
if ((flags & ~IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION) ||
6892
(!(cmd_flags & CREATE_CQ_CMD_FLAGS_TS_IGNORED_EX)))
@@ -129,6 +153,7 @@ static int ibv_icmd_create_cq(struct ibv_context *context, int cqe,
129153

130154
static int ibv_icmd_create_cq_ex(struct ibv_context *context,
131155
const struct ibv_cq_init_attr_ex *cq_attr,
156+
struct verbs_create_cq_prov_attr *prov_attr,
132157
struct verbs_cq *cq,
133158
struct ibv_command_buffer *cmdb,
134159
uint32_t cmd_flags)
@@ -149,7 +174,7 @@ static int ibv_icmd_create_cq_ex(struct ibv_context *context,
149174
flags |= IB_UVERBS_CQ_FLAGS_IGNORE_OVERRUN;
150175

151176
return ibv_icmd_create_cq(context, cq_attr->cqe, cq_attr->channel,
152-
cq_attr->comp_vector, flags,
177+
cq_attr->comp_vector, prov_attr, flags,
153178
&cq->cq, cmdb, cmd_flags);
154179
}
155180

@@ -163,12 +188,13 @@ int ibv_cmd_create_cq(struct ibv_context *context, int cqe,
163188
UVERBS_METHOD_CQ_CREATE, cmd, cmd_size, resp,
164189
resp_size);
165190

166-
return ibv_icmd_create_cq(context, cqe, channel, comp_vector, 0, cq,
191+
return ibv_icmd_create_cq(context, cqe, channel, comp_vector, NULL, 0, cq,
167192
cmdb, 0);
168193
}
169194

170195
int ibv_cmd_create_cq_ex(struct ibv_context *context,
171196
const struct ibv_cq_init_attr_ex *cq_attr,
197+
struct verbs_create_cq_prov_attr *prov_attr,
172198
struct verbs_cq *cq,
173199
struct ibv_create_cq_ex *cmd,
174200
size_t cmd_size,
@@ -180,11 +206,12 @@ int ibv_cmd_create_cq_ex(struct ibv_context *context,
180206
UVERBS_METHOD_CQ_CREATE, cmd, cmd_size, resp,
181207
resp_size);
182208

183-
return ibv_icmd_create_cq_ex(context, cq_attr, cq, cmdb, cmd_flags);
209+
return ibv_icmd_create_cq_ex(context, cq_attr, prov_attr, cq, cmdb, cmd_flags);
184210
}
185211

186212
int ibv_cmd_create_cq_ex2(struct ibv_context *context,
187213
const struct ibv_cq_init_attr_ex *cq_attr,
214+
struct verbs_create_cq_prov_attr *prov_attr,
188215
struct verbs_cq *cq,
189216
struct ibv_create_cq_ex *cmd,
190217
size_t cmd_size,
@@ -197,7 +224,7 @@ int ibv_cmd_create_cq_ex2(struct ibv_context *context,
197224
UVERBS_METHOD_CQ_CREATE,
198225
driver, cmd, cmd_size, resp, resp_size);
199226

200-
return ibv_icmd_create_cq_ex(context, cq_attr, cq, cmdb, cmd_flags);
227+
return ibv_icmd_create_cq_ex(context, cq_attr, prov_attr, cq, cmdb, cmd_flags);
201228
}
202229

203230
int ibv_cmd_destroy_cq(struct ibv_cq *cq)

‎libibverbs/driver.h‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,22 @@ enum verbs_xrcd_mask {
106106

107107
enum create_cq_cmd_flags {
108108
CREATE_CQ_CMD_FLAGS_TS_IGNORED_EX = 1 << 0,
109+
CREATE_CQ_CMD_FLAGS_WITH_MEM_VA = 1 << 1,
110+
CREATE_CQ_CMD_FLAGS_WITH_MEM_DMABUF = 1 << 2,
111+
};
112+
113+
/* Must change the PRIVATE IBVERBS_PRIVATE_ symbol if this is changed */
114+
struct verbs_create_cq_prov_attr {
115+
struct {
116+
uint64_t length;
117+
union {
118+
uint8_t *ptr;
119+
struct {
120+
uint64_t offset;
121+
int fd;
122+
} dmabuf;
123+
};
124+
} buffer;
109125
};
110126

111127
struct verbs_xrcd {
@@ -587,6 +603,7 @@ int ibv_cmd_create_cq(struct ibv_context *context, int cqe,
587603
struct ib_uverbs_create_cq_resp *resp, size_t resp_size);
588604
int ibv_cmd_create_cq_ex(struct ibv_context *context,
589605
const struct ibv_cq_init_attr_ex *cq_attr,
606+
struct verbs_create_cq_prov_attr *prov_attr,
590607
struct verbs_cq *cq,
591608
struct ibv_create_cq_ex *cmd,
592609
size_t cmd_size,
@@ -595,6 +612,7 @@ int ibv_cmd_create_cq_ex(struct ibv_context *context,
595612
uint32_t cmd_flags);
596613
int ibv_cmd_create_cq_ex2(struct ibv_context *context,
597614
const struct ibv_cq_init_attr_ex *cq_attr,
615+
struct verbs_create_cq_prov_attr *prov_attr,
598616
struct verbs_cq *cq,
599617
struct ibv_create_cq_ex *cmd,
600618
size_t cmd_size,

‎providers/efa/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ if (ENABLE_LTTNG AND LTTNGUST_FOUND)
33
endif()
44

55
rdma_shared_provider(efa libefa.map
6-
1 1.3.${PACKAGE_VERSION}
6+
1 1.4.${PACKAGE_VERSION}
77
${TRACE_FILE}
88
efa.c
99
verbs.c

‎providers/efa/efa.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ struct efa_cq {
6969
size_t cqe_size;
7070
uint8_t *buf;
7171
size_t buf_size;
72+
bool buf_mmaped;
7273
uint32_t *db;
7374
uint8_t *db_mmap_addr;
7475
uint16_t cc; /* Consumer Counter */

‎providers/efa/efadv.h‎

Lines changed: 62 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
22
/*
3-
* Copyright 2019-2024 Amazon.com, Inc. or its affiliates. All rights reserved.
3+
* Copyright 2019-2025 Amazon.com, Inc. or its affiliates. All rights reserved.
44
*/
55

66
#ifndef __EFADV_H__
@@ -16,38 +16,13 @@
1616
extern "C" {
1717
#endif
1818

19-
enum {
20-
/* Values must match the values in efa-abi.h */
21-
EFADV_QP_DRIVER_TYPE_SRD = 0,
22-
};
23-
24-
struct ibv_qp *efadv_create_driver_qp(struct ibv_pd *ibvpd,
25-
struct ibv_qp_init_attr *attr,
26-
uint32_t driver_qp_type);
27-
28-
enum {
29-
EFADV_QP_FLAGS_UNSOLICITED_WRITE_RECV = 1 << 0,
30-
};
31-
32-
struct efadv_qp_init_attr {
33-
uint64_t comp_mask;
34-
uint32_t driver_qp_type;
35-
uint16_t flags;
36-
uint8_t sl;
37-
uint8_t reserved[1];
38-
};
39-
40-
struct ibv_qp *efadv_create_qp_ex(struct ibv_context *ibvctx,
41-
struct ibv_qp_init_attr_ex *attr_ex,
42-
struct efadv_qp_init_attr *efa_attr,
43-
uint32_t inlen);
44-
4519
enum {
4620
EFADV_DEVICE_ATTR_CAPS_RDMA_READ = 1 << 0,
4721
EFADV_DEVICE_ATTR_CAPS_RNR_RETRY = 1 << 1,
4822
EFADV_DEVICE_ATTR_CAPS_CQ_WITH_SGID = 1 << 2,
4923
EFADV_DEVICE_ATTR_CAPS_RDMA_WRITE = 1 << 3,
5024
EFADV_DEVICE_ATTR_CAPS_UNSOLICITED_WRITE_RECV = 1 << 4,
25+
EFADV_DEVICE_ATTR_CAPS_CQ_WITH_EXT_MEM_DMABUF = 1 << 5,
5126
};
5227

5328
struct efadv_device_attr {
@@ -75,6 +50,45 @@ struct efadv_ah_attr {
7550
int efadv_query_ah(struct ibv_ah *ibvah, struct efadv_ah_attr *attr,
7651
uint32_t inlen);
7752

53+
enum {
54+
/* Values must match the values in efa-abi.h */
55+
EFADV_QP_DRIVER_TYPE_SRD = 0,
56+
};
57+
58+
struct ibv_qp *efadv_create_driver_qp(struct ibv_pd *ibvpd,
59+
struct ibv_qp_init_attr *attr,
60+
uint32_t driver_qp_type);
61+
62+
enum {
63+
EFADV_QP_FLAGS_UNSOLICITED_WRITE_RECV = 1 << 0,
64+
};
65+
66+
struct efadv_qp_init_attr {
67+
uint64_t comp_mask;
68+
uint32_t driver_qp_type;
69+
uint16_t flags;
70+
uint8_t sl;
71+
uint8_t reserved;
72+
};
73+
74+
struct ibv_qp *efadv_create_qp_ex(struct ibv_context *ibvctx,
75+
struct ibv_qp_init_attr_ex *attr_ex,
76+
struct efadv_qp_init_attr *efa_attr,
77+
uint32_t inlen);
78+
79+
struct efadv_wq_attr {
80+
uint64_t comp_mask;
81+
uint8_t *buffer;
82+
uint32_t entry_size;
83+
uint32_t num_entries;
84+
uint32_t *doorbell;
85+
uint32_t max_batch;
86+
uint8_t reserved[4];
87+
};
88+
89+
int efadv_query_qp_wqs(struct ibv_qp *ibvqp, struct efadv_wq_attr *sq_attr,
90+
struct efadv_wq_attr *rq_attr, uint32_t inlen);
91+
7892
struct efadv_cq {
7993
uint64_t comp_mask;
8094
int (*wc_read_sgid)(struct efadv_cq *efadv_cq, union ibv_gid *sgid);
@@ -86,16 +100,37 @@ enum {
86100
EFADV_WC_EX_WITH_IS_UNSOLICITED = 1 << 1,
87101
};
88102

103+
enum {
104+
EFADV_CQ_INIT_FLAGS_EXT_MEM_DMABUF = 1 << 0,
105+
};
106+
89107
struct efadv_cq_init_attr {
90108
uint64_t comp_mask;
91109
uint64_t wc_flags;
110+
uint64_t flags;
111+
struct {
112+
uint8_t *buffer;
113+
uint64_t length;
114+
uint64_t offset;
115+
int32_t fd;
116+
uint8_t reserved[4];
117+
} ext_mem_dmabuf;
92118
};
93119

94120
struct ibv_cq_ex *efadv_create_cq(struct ibv_context *ibvctx,
95121
struct ibv_cq_init_attr_ex *attr_ex,
96122
struct efadv_cq_init_attr *efa_attr,
97123
uint32_t inlen);
98124

125+
struct efadv_cq_attr {
126+
uint64_t comp_mask;
127+
uint8_t *buffer;
128+
uint32_t entry_size;
129+
uint32_t num_entries;
130+
};
131+
132+
int efadv_query_cq(struct ibv_cq *ibvcq, struct efadv_cq_attr *attr, uint32_t inlen);
133+
99134
struct efadv_cq *efadv_cq_from_ibv_cq_ex(struct ibv_cq_ex *ibvcqx);
100135

101136
static inline int efadv_wc_read_sgid(struct efadv_cq *efadv_cq,

‎providers/efa/libefa.map‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,3 +23,9 @@ EFA_1.3 {
2323
global:
2424
efadv_query_mr;
2525
} EFA_1.2;
26+
27+
EFA_1.4 {
28+
global:
29+
efadv_query_qp_wqs;
30+
efadv_query_cq;
31+
} EFA_1.3;

0 commit comments

Comments
 (0)