Update libibverbs for stale completion event handling

Signed-off-by: Roland Dreier <rolandd@cisco.com>

8 files changed, 63 insertions, 20 deletions

diff --git a/ChangeLog b/ChangeLog
index 10cd30c..41626e9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,8 +1,16 @@
+2005-09-06  Roland Dreier  <roland@cisco.com>
+
+	* include/infiniband/kern-abi.h, include/infiniband/verbs.h,
+	src/cmd.c, src/device.c, src/verbs.c, examples/asyncwatch.c:
+	Update to handle new kernel ABI for avoiding stale completion
+	events.  This is completely analogous to the previous asynchronous
+	event change.
+
 2005-08-31  Roland Dreier  <roland@cisco.com>
 
 	* include/infiniband/kern-abi.h, include/infiniband/verbs.h,
 	src/cmd.c, src/device.c, src/ibverbs.h, src/init.c, src/verbs.c,
-	examples/asyncwatch.h: Update to handle new kernel ABI for
+	examples/asyncwatch.c: Update to handle new kernel ABI for
 	avoiding stale asynchronous events.  When a CQ, QP or SRQ is
 	destroyed, the kernel reports the number of events it has given to
 	userspace, and we wait until we've handled the same number of
diff --git a/examples/asyncwatch.c b/examples/asyncwatch.c
index 42a2219..d4943ca 100644
--- a/examples/asyncwatch.c
+++ b/examples/asyncwatch.c
@@ -86,7 +86,7 @@ int main(int argc, char *argv[])
 		printf("  event_type %d, port %d\n", event.event_type,
 		       event.element.port_num);
 
-		ibv_put_async_event(&event);
+		ibv_ack_async_event(&event);
 	}
 
 	return 0;
diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h
index 980aa05..e89fcde 100644
--- a/include/infiniband/kern-abi.h
+++ b/include/infiniband/kern-abi.h
@@ -335,7 +335,8 @@ struct ibv_destroy_cq {
 };
 
 struct ibv_destroy_cq_resp {
-	__u32 events_reported;
+	__u32 comp_events_reported;
+	__u32 async_events_reported;
 };
 
 struct ibv_create_qp {
diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index 66845be..07f0054 100644
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -502,7 +502,8 @@ struct ibv_cq {
 
 	pthread_mutex_t		mutex;
 	pthread_cond_t		cond;
-	uint32_t		events_completed;
+	uint32_t		comp_events_completed;
+	uint32_t		async_events_completed;
 };
 
 struct ibv_ah {
@@ -608,21 +609,22 @@ extern int ibv_close_device(struct ibv_context *context);
  * ibv_get_async_event - Get next async event
  * @event: Pointer to use to return async event
  *
- * The event returned must eventually be released via ibv_put_async_event().
+ * All async events returned by ibv_get_async_event() must eventually
+ * be acknowledged with ibv_ack_async_event().
  */
 extern int ibv_get_async_event(struct ibv_context *context,
 			       struct ibv_async_event *event);
 
 /**
- * ibv_put_async_event - Free an async event
- * @event: Event to be released.
+ * ibv_ack_async_event - Free an async event
+ * @event: Event to be acknowledged.
  *
- * All events which are returned by ib_get_async_event() must be
- * released.  There should be a one-to-one correspondence between
- * successful gets and puts.
+ * All async events which are returned by ibv_get_async_event() must
+ * be acknowledged.  Destroying an object (CQ, SRQ or QP) will wait
+ * for all affiliated events to be acknowledged, so there should be a
+ * one-to-one correspondence between acks and successful gets.
  */
-extern void ibv_put_async_event(struct ibv_async_event *event);
-
+extern void ibv_ack_async_event(struct ibv_async_event *event);
 
 /**
  * ibv_query_device - Get device properties
@@ -682,10 +684,31 @@ extern int ibv_destroy_cq(struct ibv_cq *cq);
 
 /**
  * ibv_get_cq_event - Read next CQ event
+ * @context: Context to get CQ event for
+ * @comp_num: Index of completion event to check.  Must be >= 0 and
+ *   <= context->num_comp.
+ * @cq: Used to return pointer to CQ.
+ * @cq_context: Used to return consumer-supplied CQ context.
+ *
+ * All completion events returned by ibv_get_cq_event() must
+ * eventually be acknowledged with ibv_ack_cq_events().
  */
 extern int ibv_get_cq_event(struct ibv_context *context, int comp_num,
 			    struct ibv_cq **cq, void **cq_context);
- 
+
+/**
+ * ibv_ack_cq_events - Free an async event
+ * @cq: CQ to acknowledge events for
+ * @nevents: Number of events to acknowledge.
+ *
+ * All completion events which are returned by ibv_get_cq_event() must
+ * be acknowledged.  ibv_destroy_cq() will wait for all completion
+ * events to be acknowledged, so there should be a one-to-one
+ * correspondence between acks and successful gets.  An application
+ * may accumulate multiple completion events and acknowledge them in a
+ * single call by passing the number of events to ack in @nevents.
+ */
+extern void ibv_ack_cq_events(struct ibv_cq *cq, unsigned int nevents);
 
 /**
  * ibv_poll_cq - Poll a CQ for work completions
diff --git a/src/cmd.c b/src/cmd.c
index d7b34cd..92d8ef3 100644
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -303,7 +303,8 @@ int ibv_cmd_destroy_cq(struct ibv_cq *cq)
 		return errno;
 
 	pthread_mutex_lock(&cq->mutex);
-	while (cq->events_completed != resp.events_reported)
+	while (cq->comp_events_completed  != resp.comp_events_reported ||
+	       cq->async_events_completed != resp.async_events_reported)
 		pthread_cond_wait(&cq->cond, &cq->mutex);
 	pthread_mutex_unlock(&cq->mutex);
 
diff --git a/src/device.c b/src/device.c
index 650d7dc..19a8c98 100644
--- a/src/device.c
+++ b/src/device.c
@@ -171,7 +171,7 @@ int ibv_get_async_event(struct ibv_context *context,
 	return 0;
 }
 
-void ibv_put_async_event(struct ibv_async_event *event)
+void ibv_ack_async_event(struct ibv_async_event *event)
 {
 	switch (event->event_type) {
 	case IBV_EVENT_CQ_ERR:
@@ -179,7 +179,7 @@ void ibv_put_async_event(struct ibv_async_event *event)
 		struct ibv_cq *cq = event->element.cq;
 
 		pthread_mutex_lock(&cq->mutex);
-		++cq->events_completed;
+		++cq->async_events_completed;
 		pthread_cond_signal(&cq->cond);
 		pthread_mutex_unlock(&cq->mutex);
 
diff --git a/src/libibverbs.map b/src/libibverbs.map
index 6a445b5..5300a40 100644
--- a/src/libibverbs.map
+++ b/src/libibverbs.map
@@ -6,7 +6,7 @@ IBVERBS_1.0 {
 		ibv_open_device;
 		ibv_close_device;
 		ibv_get_async_event;
-		ibv_put_async_event;
+		ibv_ack_async_event;
 		ibv_query_device;
 		ibv_query_port;
 		ibv_query_gid;
@@ -18,6 +18,7 @@ IBVERBS_1.0 {
 		ibv_create_cq;
 		ibv_destroy_cq;
 		ibv_get_cq_event;
+		ibv_ack_cq_events;
 		ibv_create_srq;
 		ibv_modify_srq;
 		ibv_destroy_srq;
diff --git a/src/verbs.c b/src/verbs.c
index 0d6925e..28fddcc 100644
--- a/src/verbs.c
+++ b/src/verbs.c
@@ -107,9 +107,10 @@ struct ibv_cq *ibv_create_cq(struct ibv_context *context, int cqe,
 	struct ibv_cq *cq = context->ops.create_cq(context, cqe);
 
 	if (cq) {
-		cq->context    	     = context;
-		cq->cq_context 	     = cq_context;
-		cq->events_completed = 0;
+		cq->context    	     	   = context;
+		cq->cq_context 	     	   = cq_context;
+		cq->comp_events_completed  = 0;
+		cq->async_events_completed = 0;
 		pthread_mutex_init(&cq->mutex, NULL);
 		pthread_cond_init(&cq->cond, NULL);
 	}
@@ -143,6 +144,14 @@ int ibv_get_cq_event(struct ibv_context *context, int comp_num,
 	return 0;
 }
 
+void ibv_ack_cq_events(struct ibv_cq *cq, unsigned int nevents)
+{
+	pthread_mutex_lock(&cq->mutex);
+	cq->comp_events_completed += nevents;
+	pthread_cond_signal(&cq->cond);
+	pthread_mutex_unlock(&cq->mutex);
+}
+
 struct ibv_srq *ibv_create_srq(struct ibv_pd *pd,
 			       struct ibv_srq_init_attr *srq_init_attr)
 {