Get ready to receive before other side starts to send

Have server side of pingpong get ready to receive before client side
starts sending, to close a race condition.

Signed-off-by: Roland Dreier <rolandd@cisco.com>

3 files changed, 172 insertions, 143 deletions

diff --git a/examples/rc_pingpong.c b/examples/rc_pingpong.c
index 0862d6e..85b4a4a 100644
--- a/examples/rc_pingpong.c
+++ b/examples/rc_pingpong.c
@@ -87,6 +87,56 @@ static uint16_t pp_get_local_lid(struct pingpong_context *ctx, int port)
 	return attr.lid;
 }
 
+static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
+			  struct pingpong_dest *dest)
+{
+	struct ibv_qp_attr attr = {
+		.qp_state		= IBV_QPS_RTR,
+		.path_mtu		= IBV_MTU_1024,
+		.dest_qp_num		= dest->qpn,
+		.rq_psn 		= dest->psn,
+		.max_dest_rd_atomic	= 1,
+		.min_rnr_timer		= 12,
+		.ah_attr		= {
+			.is_global	= 0,
+			.dlid		= dest->lid,
+			.sl		= 0,
+			.src_path_bits	= 0,
+			.port_num	= port
+		}
+	};
+	if (ibv_modify_qp(ctx->qp, &attr,
+			  IBV_QP_STATE              |
+			  IBV_QP_AV                 |
+			  IBV_QP_PATH_MTU           |
+			  IBV_QP_DEST_QPN           |
+			  IBV_QP_RQ_PSN             |
+			  IBV_QP_MAX_DEST_RD_ATOMIC |
+			  IBV_QP_MIN_RNR_TIMER)) {
+		fprintf(stderr, "Failed to modify QP to RTR\n");
+		return 1;
+	}
+
+	attr.qp_state 	    = IBV_QPS_RTS;
+	attr.timeout 	    = 14;
+	attr.retry_cnt 	    = 7;
+	attr.rnr_retry 	    = 7;
+	attr.sq_psn 	    = my_psn;
+	attr.max_rd_atomic  = 1;
+	if (ibv_modify_qp(ctx->qp, &attr,
+			  IBV_QP_STATE              |
+			  IBV_QP_TIMEOUT            |
+			  IBV_QP_RETRY_CNT          |
+			  IBV_QP_RNR_RETRY          |
+			  IBV_QP_SQ_PSN             |
+			  IBV_QP_MAX_QP_RD_ATOMIC)) {
+		fprintf(stderr, "Failed to modify QP to RTS\n");
+		return 1;
+	}
+
+	return 0;
+}
+
 static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port,
 						 const struct pingpong_dest *my_dest)
 {
@@ -151,7 +201,9 @@ out:
 	return rem_dest;
 }
 
-static struct pingpong_dest *pp_server_exch_dest(int port, const struct pingpong_dest *my_dest)
+static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
+						 int ib_port, int port,
+						 const struct pingpong_dest *my_dest)
 {
 	struct addrinfo *res, *t;
 	struct addrinfo hints = {
@@ -215,6 +267,13 @@ static struct pingpong_dest *pp_server_exch_dest(int port, const struct pingpong
 
 	sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn);
 
+	if (pp_connect_ctx(ctx, ib_port, my_dest->psn, rem_dest)) {
+		fprintf(stderr, "Couldn't connect to remote QP\n");
+		free(rem_dest);
+		rem_dest = NULL;
+		goto out;
+	}
+
 	sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn);
 	if (write(connfd, msg, sizeof msg) != sizeof msg) {
 		fprintf(stderr, "Couldn't send local address\n");
@@ -357,56 +416,6 @@ static int pp_post_send(struct pingpong_context *ctx)
 	return ibv_post_send(ctx->qp, &wr, &bad_wr);
 }
 
-static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
-			  struct pingpong_dest *dest)
-{
-	struct ibv_qp_attr attr = {
-		.qp_state		= IBV_QPS_RTR,
-		.path_mtu		= IBV_MTU_1024,
-		.dest_qp_num		= dest->qpn,
-		.rq_psn 		= dest->psn,
-		.max_dest_rd_atomic	= 1,
-		.min_rnr_timer		= 12,
-		.ah_attr		= {
-			.is_global	= 0,
-			.dlid		= dest->lid,
-			.sl		= 0,
-			.src_path_bits	= 0,
-			.port_num	= port
-		}
-	};
-	if (ibv_modify_qp(ctx->qp, &attr,
-			  IBV_QP_STATE              |
-			  IBV_QP_AV                 |
-			  IBV_QP_PATH_MTU           |
-			  IBV_QP_DEST_QPN           |
-			  IBV_QP_RQ_PSN             |
-			  IBV_QP_MAX_DEST_RD_ATOMIC |
-			  IBV_QP_MIN_RNR_TIMER)) {
-		fprintf(stderr, "Failed to modify QP to RTR\n");
-		return 1;
-	}
-
-	attr.qp_state 	    = IBV_QPS_RTS;
-	attr.timeout 	    = 14;
-	attr.retry_cnt 	    = 7;
-	attr.rnr_retry 	    = 7;
-	attr.sq_psn 	    = my_psn;
-	attr.max_rd_atomic  = 1;
-	if (ibv_modify_qp(ctx->qp, &attr,
-			  IBV_QP_STATE              |
-			  IBV_QP_TIMEOUT            |
-			  IBV_QP_RETRY_CNT          |
-			  IBV_QP_RNR_RETRY          |
-			  IBV_QP_SQ_PSN             |
-			  IBV_QP_MAX_QP_RD_ATOMIC)) {
-		fprintf(stderr, "Failed to modify QP to RTS\n");
-		return 1;
-	}
-
-	return 0;
-}
-
 static void usage(const char *argv0)
 {
 	printf("Usage:\n");
@@ -556,7 +565,7 @@ int main(int argc, char *argv[])
 	if (servername)
 		rem_dest = pp_client_exch_dest(servername, port, &my_dest);
 	else
-		rem_dest = pp_server_exch_dest(port, &my_dest);
+		rem_dest = pp_server_exch_dest(ctx, ib_port, port, &my_dest);
 
 	if (!rem_dest)
 		return 1;
@@ -564,8 +573,9 @@ int main(int argc, char *argv[])
 	printf("  remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n",
 	       rem_dest->lid, rem_dest->qpn, rem_dest->psn);
 
-	if (pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest))
-		return 1;
+	if (servername)
+		if (pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest))
+			return 1;
 
 	if (use_event)
 		if (ibv_req_notify_cq(ctx->cq, 0)) {
diff --git a/examples/uc_pingpong.c b/examples/uc_pingpong.c
index 8a9fb33..98617e9 100644
--- a/examples/uc_pingpong.c
+++ b/examples/uc_pingpong.c
@@ -87,6 +87,48 @@ static uint16_t pp_get_local_lid(struct pingpong_context *ctx, int port)
 	return attr.lid;
 }
 
+static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
+			  struct pingpong_dest *dest)
+{
+	struct ibv_qp_attr attr = {
+		.qp_state		= IBV_QPS_RTR,
+		.path_mtu		= IBV_MTU_1024,
+		.dest_qp_num		= dest->qpn,
+		.rq_psn 		= dest->psn,
+		.max_dest_rd_atomic	= 1,
+		.ah_attr		= {
+			.is_global	= 0,
+			.dlid		= dest->lid,
+			.sl		= 0,
+			.src_path_bits	= 0,
+			.port_num	= port
+		}
+	};
+	if (ibv_modify_qp(ctx->qp, &attr,
+			  IBV_QP_STATE              |
+			  IBV_QP_AV                 |
+			  IBV_QP_PATH_MTU           |
+			  IBV_QP_DEST_QPN           |
+			  IBV_QP_RQ_PSN             |
+			  IBV_QP_MAX_DEST_RD_ATOMIC)) {
+		fprintf(stderr, "Failed to modify QP to RTR\n");
+		return 1;
+	}
+
+	attr.qp_state 	    = IBV_QPS_RTS;
+	attr.sq_psn 	    = my_psn;
+	attr.max_rd_atomic  = 1;
+	if (ibv_modify_qp(ctx->qp, &attr,
+			  IBV_QP_STATE              |
+			  IBV_QP_SQ_PSN             |
+			  IBV_QP_MAX_QP_RD_ATOMIC)) {
+		fprintf(stderr, "Failed to modify QP to RTS\n");
+		return 1;
+	}
+
+	return 0;
+}
+
 static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port,
 						 const struct pingpong_dest *my_dest)
 {
@@ -151,7 +193,9 @@ out:
 	return rem_dest;
 }
 
-static struct pingpong_dest *pp_server_exch_dest(int port, const struct pingpong_dest *my_dest)
+static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
+						 int ib_port, int port,
+						 const struct pingpong_dest *my_dest)
 {
 	struct addrinfo *res, *t;
 	struct addrinfo hints = {
@@ -215,6 +259,13 @@ static struct pingpong_dest *pp_server_exch_dest(int port, const struct pingpong
 
 	sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn);
 
+	if (pp_connect_ctx(ctx, ib_port, my_dest->psn, rem_dest)) {
+		fprintf(stderr, "Couldn't connect to remote QP\n");
+		free(rem_dest);
+		rem_dest = NULL;
+		goto out;
+	}
+
 	sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn);
 	if (write(connfd, msg, sizeof msg) != sizeof msg) {
 		fprintf(stderr, "Couldn't send local address\n");
@@ -357,48 +408,6 @@ static int pp_post_send(struct pingpong_context *ctx)
 	return ibv_post_send(ctx->qp, &wr, &bad_wr);
 }
 
-static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
-			  struct pingpong_dest *dest)
-{
-	struct ibv_qp_attr attr = {
-		.qp_state		= IBV_QPS_RTR,
-		.path_mtu		= IBV_MTU_1024,
-		.dest_qp_num		= dest->qpn,
-		.rq_psn 		= dest->psn,
-		.max_dest_rd_atomic	= 1,
-		.ah_attr		= {
-			.is_global	= 0,
-			.dlid		= dest->lid,
-			.sl		= 0,
-			.src_path_bits	= 0,
-			.port_num	= port
-		}
-	};
-	if (ibv_modify_qp(ctx->qp, &attr,
-			  IBV_QP_STATE              |
-			  IBV_QP_AV                 |
-			  IBV_QP_PATH_MTU           |
-			  IBV_QP_DEST_QPN           |
-			  IBV_QP_RQ_PSN             |
-			  IBV_QP_MAX_DEST_RD_ATOMIC)) {
-		fprintf(stderr, "Failed to modify QP to RTR\n");
-		return 1;
-	}
-
-	attr.qp_state 	    = IBV_QPS_RTS;
-	attr.sq_psn 	    = my_psn;
-	attr.max_rd_atomic  = 1;
-	if (ibv_modify_qp(ctx->qp, &attr,
-			  IBV_QP_STATE              |
-			  IBV_QP_SQ_PSN             |
-			  IBV_QP_MAX_QP_RD_ATOMIC)) {
-		fprintf(stderr, "Failed to modify QP to RTS\n");
-		return 1;
-	}
-
-	return 0;
-}
-
 static void usage(const char *argv0)
 {
 	printf("Usage:\n");
@@ -548,7 +557,7 @@ int main(int argc, char *argv[])
 	if (servername)
 		rem_dest = pp_client_exch_dest(servername, port, &my_dest);
 	else
-		rem_dest = pp_server_exch_dest(port, &my_dest);
+		rem_dest = pp_server_exch_dest(ctx, ib_port, port, &my_dest);
 
 	if (!rem_dest)
 		return 1;
@@ -556,8 +565,9 @@ int main(int argc, char *argv[])
 	printf("  remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n",
 	       rem_dest->lid, rem_dest->qpn, rem_dest->psn);
 
-	if (pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest))
-		return 1;
+	if (servername)
+		if (pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest))
+			return 1;
 
 	if (use_event)
 		if (ibv_req_notify_cq(ctx->cq, 0)) {
diff --git a/examples/ud-pingpong.c b/examples/ud-pingpong.c
index 4936d30..0faf429 100644
--- a/examples/ud-pingpong.c
+++ b/examples/ud-pingpong.c
@@ -78,7 +78,6 @@ struct pingpong_dest {
 	int psn;
 };
 
-
 static uint16_t pp_get_local_lid(struct pingpong_context *ctx, int port)
 {
 	struct ibv_port_attr attr;
@@ -89,6 +88,44 @@ static uint16_t pp_get_local_lid(struct pingpong_context *ctx, int port)
 	return attr.lid;
 }
 
+static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
+			  struct pingpong_dest *dest)
+{
+	struct ibv_qp_attr attr;
+	struct ibv_ah_attr ah_attr = {
+		.is_global     = 0,
+		.dlid          = dest->lid,
+		.sl            = 0,
+		.src_path_bits = 0,
+		.port_num      = port
+	};
+
+	attr.qp_state 		= IBV_QPS_RTR;
+
+	if (ibv_modify_qp(ctx->qp, &attr, IBV_QP_STATE)) {
+		fprintf(stderr, "Failed to modify QP to RTR\n");
+		return 1;
+	}
+
+	attr.qp_state 	    = IBV_QPS_RTS;
+	attr.sq_psn 	    = my_psn;
+
+	if (ibv_modify_qp(ctx->qp, &attr,
+			  IBV_QP_STATE              |
+			  IBV_QP_SQ_PSN)) {
+		fprintf(stderr, "Failed to modify QP to RTS\n");
+		return 1;
+	}
+
+	ctx->ah = ibv_create_ah(ctx->pd, &ah_attr);
+	if (!ctx->ah) {
+		fprintf(stderr, "Failed to create AH\n");
+		return 1;
+	}
+
+	return 0;
+}
+
 static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port,
 						 const struct pingpong_dest *my_dest)
 {
@@ -153,7 +190,9 @@ out:
 	return rem_dest;
 }
 
-static struct pingpong_dest *pp_server_exch_dest(int port, const struct pingpong_dest *my_dest)
+static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
+						 int ib_port, int port,
+						 const struct pingpong_dest *my_dest)
 {
 	struct addrinfo *res, *t;
 	struct addrinfo hints = {
@@ -217,6 +256,13 @@ static struct pingpong_dest *pp_server_exch_dest(int port, const struct pingpong
 
 	sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn);
 
+	if (pp_connect_ctx(ctx, ib_port, my_dest->psn, rem_dest)) {
+		fprintf(stderr, "Couldn't connect to remote QP\n");
+		free(rem_dest);
+		rem_dest = NULL;
+		goto out;
+	}
+
 	sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn);
 	if (write(connfd, msg, sizeof msg) != sizeof msg) {
 		fprintf(stderr, "Couldn't send local address\n");
@@ -366,44 +412,6 @@ static int pp_post_send(struct pingpong_context *ctx, uint32_t qpn)
 	return ibv_post_send(ctx->qp, &wr, &bad_wr);
 }
 
-static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
-			  struct pingpong_dest *dest)
-{
-	struct ibv_qp_attr attr;
-	struct ibv_ah_attr ah_attr = {
-		.is_global     = 0,
-		.dlid          = dest->lid,
-		.sl            = 0,
-		.src_path_bits = 0,
-		.port_num      = port
-	};
-
-	attr.qp_state 		= IBV_QPS_RTR;
-
-	if (ibv_modify_qp(ctx->qp, &attr, IBV_QP_STATE)) {
-		fprintf(stderr, "Failed to modify QP to RTR\n");
-		return 1;
-	}
-
-	attr.qp_state 	    = IBV_QPS_RTS;
-	attr.sq_psn 	    = my_psn;
-
-	if (ibv_modify_qp(ctx->qp, &attr,
-			  IBV_QP_STATE              |
-			  IBV_QP_SQ_PSN)) {
-		fprintf(stderr, "Failed to modify QP to RTS\n");
-		return 1;
-	}
-
-	ctx->ah = ibv_create_ah(ctx->pd, &ah_attr);
-	if (!ctx->ah) {
-		fprintf(stderr, "Failed to create AH\n");
-		return 1;
-	}
-
-	return 0;
-}
-
 static void usage(const char *argv0)
 {
 	printf("Usage:\n");
@@ -553,7 +561,7 @@ int main(int argc, char *argv[])
 	if (servername)
 		rem_dest = pp_client_exch_dest(servername, port, &my_dest);
 	else
-		rem_dest = pp_server_exch_dest(port, &my_dest);
+		rem_dest = pp_server_exch_dest(ctx, ib_port, port, &my_dest);
 
 	if (!rem_dest)
 		return 1;
@@ -561,8 +569,9 @@ int main(int argc, char *argv[])
 	printf("  remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n",
 	       rem_dest->lid, rem_dest->qpn, rem_dest->psn);
 
-	if (pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest))
-		return 1;
+	if (servername)
+		if (pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest))
+			return 1;
 
 	if (use_event)
 		if (ibv_req_notify_cq(ctx->cq, 0)) {