Fixes for man pages

Some fixes and updates to several man pages:
 * Correct formatting in a few places.
 * Add more "SEE ALSO" functions where appropriate.
 * Document byte order of GUID and P_Key fields.
 * Fix example code in ibv_get_cq_event.3
 * Document GRH handling on receive.

Signed-off-by: Dotan Barak <dotanb@dev.mellanox.co.il>
Signed-off-by: Roland Dreier <rolandd@cisco.com>

10 files changed, 33 insertions, 13 deletions

diff --git a/man/ibv_create_cq.3 b/man/ibv_create_cq.3
index c39fe83..bb256d5 100644
--- a/man/ibv_create_cq.3
+++ b/man/ibv_create_cq.3
@@ -51,6 +51,7 @@ fails if any queue pair is still associated with this CQ.
 .SH "SEE ALSO"
 .BR ibv_resize_cq (3),
 .BR ibv_req_notify_cq (3),
+.BR ibv_ack_cq_events (3),
 .BR ibv_create_qp (3)
 .SH "AUTHORS"
 .TP
diff --git a/man/ibv_create_qp.3 b/man/ibv_create_qp.3
index 68a08b0..abd5449 100644
--- a/man/ibv_create_qp.3
+++ b/man/ibv_create_qp.3
@@ -8,9 +8,9 @@ ibv_create_qp, ibv_destroy_qp \- create or destroy a queue pair (QP)
 .B #include <infiniband/verbs.h>
 .sp
 .BI "struct ibv_qp *ibv_create_qp(struct ibv_pd " "*pd" ,
-.BI "                             struct ibv_qp_init_attr " "*qp_init_attr)" ;
+.BI "                             struct ibv_qp_init_attr " "*qp_init_attr" );
 .nl
-.BI "int ibv_destroy_qp(struct ibv_qp " "*qp)" ;
+.BI "int ibv_destroy_qp(struct ibv_qp " "*qp" );
 .fi
 .SH "DESCRIPTION"
 .B ibv_create_qp()
diff --git a/man/ibv_fork_init.3 b/man/ibv_fork_init.3
index d911c3f..b34f71f 100644
--- a/man/ibv_fork_init.3
+++ b/man/ibv_fork_init.3
@@ -49,7 +49,9 @@ regions.  The precise performance impact depends on the workload and
 usually will not be significant.
 .SH "SEE ALSO"
 .BR fork (2),
+.BR wait (2),
 .BR system (3),
+.BR exec (3),
 .BR ibv_get_device_list (3)
 .SH "AUTHORS"
 .TP
diff --git a/man/ibv_get_cq_event.3 b/man/ibv_get_cq_event.3
index 695734b..27dca6f 100644
--- a/man/ibv_get_cq_event.3
+++ b/man/ibv_get_cq_event.3
@@ -18,11 +18,11 @@ ibv_get_cq_event, ibv_ack_cq_events \- get and acknowledge completion queue (CQ)
 .B ibv_get_cq_event()
 waits for the next completion event in the completion event channel
 .I channel\fR.
-The argument
+Fills the arguments
 .I cq
-is used to return the CQ that caused the event and
+with the CQ that got the event and
 .I cq_context
-is used to return the CQ's context.
+with the CQ's context\fR.
 .PP
 .B ibv_ack_cq_events()
 acknowledges
@@ -102,7 +102,7 @@ if (ibv_get_cq_event(channel, &ev_cq, &ev_ctx)) {
 ibv_ack_cq_events(ev_cq, 1);
 .PP
 /* Request notification upon the next completion event */
-if (ibv_req_notify_cq(cq, 0)) {
+if (ibv_req_notify_cq(ev_cq, 0)) {
         fprintf(stderr, "Couldn't request CQ notification\en");
         return 1;
 }
@@ -114,6 +114,10 @@ do {
                 fprintf(stderr, "Failed to poll completions from the CQ\en");
                 return 1;
         }
+
+        /* there may be an extra event with no completion in the CQ */
+        if (ne == 0)
+                continue;
 .PP
         if (wc.status != IBV_WC_SUCCESS) {
                 fprintf(stderr, "Completion with status 0x%x was found\en", wc.status);
diff --git a/man/ibv_modify_qp.3 b/man/ibv_modify_qp.3
index a870744..f045900 100644
--- a/man/ibv_modify_qp.3
+++ b/man/ibv_modify_qp.3
@@ -130,7 +130,7 @@ Next state     Required attributes
 \-\-\-\-\-\-\-\-\-\-     \-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-
 Init \fB          IBV_QP_STATE, IBV_QP_PKEY_INDEX, IBV_QP_PORT, \fR
      \fB          IBV_QP_QKEY \fR
-RTR  \fB          None \fR
+RTR  \fB          IBV_QP_STATE \fR
 RTS  \fB          IBV_QP_STATE, IBV_QP_SQ_PSN \fR
 .fi
 .PP
diff --git a/man/ibv_poll_cq.3 b/man/ibv_poll_cq.3
index b634fc9..75e4d7c 100644
--- a/man/ibv_poll_cq.3
+++ b/man/ibv_poll_cq.3
@@ -32,7 +32,7 @@ uint32_t                vendor_err;     /* Vendor error syndrome */
 uint32_t                byte_len;       /* Number of bytes transferred */
 uint32_t                imm_data;       /* Immediate data (in network byte order) */
 uint32_t                qp_num;         /* Local QP number of completed WR */
-uint32_t                src_qp;         /* Source QP number (remote QP number) of completed WR */
+uint32_t                src_qp;         /* Source QP number (remote QP number) of completed WR (valid only for UD QPs) */
 enum ibv_wc_flags       wc_flags;       /* Flags of the completed WR */
 uint16_t                pkey_index;     /* P_Key index (valid only for GSI QPs) */
 uint16_t                slid;           /* Source LID */
@@ -47,7 +47,7 @@ The attribute wc_flags describes the properties of the work completion.
 It is either 0 or the bitwise OR of one or more of the following flags:
 .PP
 .TP
-.B IBV_WC_GRH \fR      GRH is present
+.B IBV_WC_GRH \fR      GRH is present (valid only for UD QPs)
 .TP
 .B IBV_WC_WITH_IMM \fR Immediate data value is valid
 .PP
diff --git a/man/ibv_post_recv.3 b/man/ibv_post_recv.3
index 478b67a..46a630b 100644
--- a/man/ibv_post_recv.3
+++ b/man/ibv_post_recv.3
@@ -44,7 +44,6 @@ uint32_t                lkey;      /* Key of the local Memory Region */
 .in -8
 };
 .fi
-
 .SH "RETURN VALUE"
 .B ibv_post_recv()
 returns 0 on success, or the value of errno on failure (which indicates the failure reason).
@@ -60,6 +59,13 @@ is associated with a shared receive queue, you must use the function
 and not
 .B ibv_post_recv()\fR,
 since the QP's own receive queue will not be used.
+.PP
+If a WR is being posted to a UD QP, the Global Routing Header (GRH) of
+the incoming message will be placed in the first 40 bytes of the
+buffer(s) in the scatter list.  If no GRH is present in the incoming
+message, then the first bytes will be undefined.  This means that in
+all cases, the actual data of the incoming message will start at an
+offset of 40 bytes into the buffer(s) in the scatter list.
 .SH "SEE ALSO"
 .BR ibv_create_qp (3),
 .BR ibv_post_send (3),
diff --git a/man/ibv_post_srq_recv.3 b/man/ibv_post_srq_recv.3
index c7e2302..65877cd 100644
--- a/man/ibv_post_srq_recv.3
+++ b/man/ibv_post_srq_recv.3
@@ -51,6 +51,13 @@ returns 0 on success, or the value of errno on failure (which indicates the fail
 The buffers used by a WR can only be safely reused after WR the
 request is fully executed and a work completion has been retrieved
 from the corresponding completion queue (CQ).
+.PP
+If a WR is being posted to a UD QP, the Global Routing Header (GRH) of
+the incoming message will be placed in the first 40 bytes of the
+buffer(s) in the scatter list.  If no GRH is present in the incoming
+message, then the first bytes will be undefined.  This means that in
+all cases, the actual data of the incoming message will start at an
+offset of 40 bytes into the buffer(s) in the scatter list.
 .SH "SEE ALSO"
 .BR ibv_create_qp (3),
 .BR ibv_post_send (3),
diff --git a/man/ibv_query_device.3 b/man/ibv_query_device.3
index 344f5b3..f327769 100644
--- a/man/ibv_query_device.3
+++ b/man/ibv_query_device.3
@@ -22,8 +22,8 @@ is a pointer to an ibv_device_attr struct, as defined in <infiniband/verbs.h>.
 struct ibv_device_attr {
 .in +8
 char                    fw_ver[64];             /* FW version */
-uint64_t                node_guid;              /* Node GUID */
-uint64_t                sys_image_guid;         /* System image GUID */
+uint64_t                node_guid;              /* Node GUID (in network byte order) */
+uint64_t                sys_image_guid;         /* System image GUID (in network byte order) */
 uint64_t                max_mr_size;            /* Largest contiguous block that can be registered */
 uint64_t                page_size_cap;          /* Supported memory shift sizes */
 uint32_t                vendor_id;              /* Vendor ID, per IEEE */
diff --git a/man/ibv_query_pkey.3 b/man/ibv_query_pkey.3
index 489f1ac..fcdfe6c 100644
--- a/man/ibv_query_pkey.3
+++ b/man/ibv_query_pkey.3
@@ -12,7 +12,7 @@ ibv_query_pkey \- query an InfiniBand port's P_Key table
 .fi
 .SH "DESCRIPTION"
 .B ibv_query_pkey()
-returns the P_Key value in entry
+returns the P_Key value (in network byte order) in entry
 .I index
 of port
 .I port_num