rping: terminate CM event thread before exiting

The CM event thread processes events in a loop with no explicit
termination. When the last CM event is received, the main thread
proceeds to clean up and destroy the CM event channel. If this occurs
after the CM event thread has processed the last event, but before it
reaches rdma_get_cm_event again, then the subsequent call to
rdma_get_cm_event will fail and cause the process to exit with a failure
code even though the test was actually successful.

This causes flakiness in test scripts that use rping for basic
functional testing.

Fix this by using an eventfd+poll to explicitly signal the CM event
thread for termination.

Tested by running 4096 parallel rping processes.

Fixes: 6f640ff ("r7019: Introduce event channels.")
Signed-off-by: Jacob Moroni <[email protected]>

Author: jakemoroni

librdmacm/examples/rping.c

@@ -36,6 +36,9 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <errno.h>
+#include <unistd.h>
+#include <sys/poll.h>
+#include <sys/eventfd.h>
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <netdb.h>
@@ -151,6 +154,7 @@ struct rping_cb {
 	int validate;			/* validate ping data */
 
 	/* CM stuff */
+	int eventfd;
 	pthread_t cmthread;
 	struct rdma_event_channel *cm_channel;
 	struct rdma_cm_id *cm_id;	/* connection on client side,*/
@@ -666,18 +670,36 @@ static void *cm_thread(void *arg)
 {
 	struct rping_cb *cb = arg;
 	struct rdma_cm_event *event;
+	struct pollfd pfds[2];
 	int ret;
 
+	pfds[0].fd = cb->eventfd;
+	pfds[0].events = POLLIN;
+	pfds[1].fd = cb->cm_channel->fd;
+	pfds[1].events = POLLIN;
+
 	while (1) {
-		ret = rdma_get_cm_event(cb->cm_channel, &event);
-		if (ret) {
-			perror("rdma_get_cm_event");
+		ret = poll(pfds, 2, -1);
+		if (ret == -1 && errno != EINTR) {
+			perror("poll failed");
 			exit(ret);
+		} else if (ret < 1)
+			continue;
+
+		if (pfds[0].revents & POLLIN)
+			return NULL;
+
+		if (pfds[1].revents & POLLIN) {
+			ret = rdma_get_cm_event(cb->cm_channel, &event);
+			if (ret) {
+				perror("rdma_get_cm_event");
+				exit(ret);
+			}
+			ret = rping_cma_event_handler(event->id, event);
+			rdma_ack_cm_event(event);
+			if (ret)
+				exit(ret);
 		}
-		ret = rping_cma_event_handler(event->id, event);
-		rdma_ack_cm_event(event);
-		if (ret)
-			exit(ret);
 	}
 }
 
@@ -1279,6 +1301,7 @@ int main(int argc, char *argv[])
 	int op;
 	int ret = 0;
 	int persistent_server = 0;
+	const uint64_t efdw = 1;
 
 	cb = malloc(sizeof(*cb));
 	if (!cb)
@@ -1366,6 +1389,13 @@ int main(int argc, char *argv[])
 		goto out;
 	}
 
+	cb->eventfd = eventfd(0, EFD_NONBLOCK);
+	if (cb->eventfd == -1) {
+		perror("Could not create event FD");
+		ret = errno;
+		goto out;
+	}
+
 	cb->cm_channel = create_event_channel();
 	if (!cb->cm_channel) {
 		ret = errno;
@@ -1397,6 +1427,10 @@ int main(int argc, char *argv[])
 	DEBUG_LOG("destroy cm_id %p\n", cb->cm_id);
 	rdma_destroy_id(cb->cm_id);
 out2:
+	if (write(cb->eventfd, &efdw, sizeof(efdw)) != sizeof(efdw))
+		fprintf(stderr, "Failed to signal CM thread\n");
+
+	pthread_join(cb->cmthread, NULL);
 	rdma_destroy_event_channel(cb->cm_channel);
 out:
 	free(cb);