libibverbs changes for query QP and query SRQ verbs

Signed-off-by: Dotan Barak <dotanb@mellanox.co.il>
Signed-off-by: Roland Dreier <rolandd@cisco.com>

7 files changed, 236 insertions, 0 deletions

diff --git a/ChangeLog b/ChangeLog
index 2966074..b16495c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,20 @@
+2006-02-13  Dotan Barak  <dotanb@mellanox.co.il>
+
+	* src/cmd.c (ibv_cmd_query_qp, ibv_cmd_query_srq),
+	include/infiniband/driver.h: Add driver interface for calling
+	query QP and query SRQ kernel commands.
+
+	* include/infiniband/kern-abi.h: Add kernel ABI for query QP and
+	query SRQ.
+
+	* src/verbs.c (ibv_query_qp, ibv_query_srq),
+	include/infiniband/verbs.h: Add query QP and query SRQ library
+	APIs.  This changes the provider ABI, since new fields are added
+	to struct ibv_context_ops; source compatibility with provider
+	libraries is preserved, but binaries will have to be recompiled.
+	Neither source nor binary compatibility with consumers of
+	libibverbs is affected.
+
 2006-02-01  Roland Dreier  <rdreier@cisco.com>
 
 	* examples/rc_pingpong.c, examples/uc_pingpong.c,
diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h
index 6d04210..0d55f7e 100644
--- a/include/infiniband/driver.h
+++ b/include/infiniband/driver.h
@@ -107,11 +107,18 @@ int ibv_cmd_modify_srq(struct ibv_srq *srq,
 		       struct ibv_srq_attr *srq_attr,
 		       enum ibv_srq_attr_mask srq_attr_mask,
 		       struct ibv_modify_srq *cmd, size_t cmd_size);
+int ibv_cmd_query_srq(struct ibv_srq *srq,
+		      struct ibv_srq_attr *srq_attr,
+		      struct ibv_query_srq *cmd, size_t cmd_size);
 int ibv_cmd_destroy_srq(struct ibv_srq *srq);
 
 int ibv_cmd_create_qp(struct ibv_pd *pd,
 		      struct ibv_qp *qp, struct ibv_qp_init_attr *attr,
 		      struct ibv_create_qp *cmd, size_t cmd_size);
+int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *qp_attr,
+		     enum ibv_qp_attr_mask attr_mask,
+		     struct ibv_qp_init_attr *qp_init_attr,
+		     struct ibv_query_qp *cmd, size_t cmd_size);
 int ibv_cmd_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
 		      enum ibv_qp_attr_mask attr_mask,
 		      struct ibv_modify_qp *cmd, size_t cmd_size);
diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h
index 705d21a..ee93561 100644
--- a/include/infiniband/kern-abi.h
+++ b/include/infiniband/kern-abi.h
@@ -476,6 +476,50 @@ struct ibv_qp_dest {
 	__u8  port_num;
 };
 
+struct ibv_query_qp {
+	__u32 command;
+	__u16 in_words;
+	__u16 out_words;
+	__u64 response;
+	__u32 qp_handle;
+	__u32 attr_mask;
+	__u64 driver_data[0];
+};
+
+struct ibv_query_qp_resp {
+	struct ibv_qp_dest dest;
+	struct ibv_qp_dest alt_dest;
+	__u32 max_send_wr;
+	__u32 max_recv_wr;
+	__u32 max_send_sge;
+	__u32 max_recv_sge;
+	__u32 max_inline_data;
+	__u32 qkey;
+	__u32 rq_psn;
+	__u32 sq_psn;
+	__u32 dest_qp_num;
+	__u32 qp_access_flags;
+	__u16 pkey_index;
+	__u16 alt_pkey_index;
+	__u8  qp_state;
+	__u8  cur_qp_state;
+	__u8  path_mtu;
+	__u8  path_mig_state;
+	__u8  en_sqd_async_notify;
+	__u8  max_rd_atomic;
+	__u8  max_dest_rd_atomic;
+	__u8  min_rnr_timer;
+	__u8  port_num;
+	__u8  timeout;
+	__u8  retry_cnt;
+	__u8  rnr_retry;
+	__u8  alt_port_num;
+	__u8  alt_timeout;
+	__u8  sq_sig_all;
+	__u8  reserved[5];
+	__u64 driver_data[0];
+};
+
 struct ibv_modify_qp {
 	__u32 command;
 	__u16 in_words;
@@ -676,6 +720,23 @@ struct ibv_modify_srq {
 	__u64 driver_data[0];
 };
 
+struct ibv_query_srq {
+	__u32 command;
+	__u16 in_words;
+	__u16 out_words;
+	__u64 response;
+	__u32 srq_handle;
+	__u32 reserved;
+	__u64 driver_data[0];
+};
+
+struct ibv_query_srq_resp {
+	__u32 max_wr;
+	__u32 max_sge;
+	__u32 srq_limit;
+	__u32 reserved;
+};
+
 struct ibv_destroy_srq {
 	__u32 command;
 	__u16 in_words;
diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index 7a4f1ea..d33a880 100644
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -556,11 +556,16 @@ struct ibv_context_ops {
 	int			(*modify_srq)(struct ibv_srq *srq,
 					      struct ibv_srq_attr *srq_attr,
 					      enum ibv_srq_attr_mask srq_attr_mask);
+	int			(*query_srq)(struct ibv_srq *srq,
+					     struct ibv_srq_attr *srq_attr);
 	int			(*destroy_srq)(struct ibv_srq *srq);
 	int			(*post_srq_recv)(struct ibv_srq *srq,
 						 struct ibv_recv_wr *recv_wr,
 						 struct ibv_recv_wr **bad_recv_wr);
 	struct ibv_qp *		(*create_qp)(struct ibv_pd *pd, struct ibv_qp_init_attr *attr);
+	int			(*query_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+					    enum ibv_qp_attr_mask attr_mask,
+					    struct ibv_qp_init_attr *init_attr);
 	int			(*modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr,
 					     enum ibv_qp_attr_mask attr_mask);
 	int			(*destroy_qp)(struct ibv_qp *qp);
@@ -815,6 +820,14 @@ int ibv_modify_srq(struct ibv_srq *srq,
 		   enum ibv_srq_attr_mask srq_attr_mask);
 
 /**
+ * ibv_query_srq - Returns the attribute list and current values for the
+ *   specified SRQ.
+ * @srq: The SRQ to query.
+ * @srq_attr: The attributes of the specified SRQ.
+ */
+int ibv_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr);
+
+/**
  * ibv_destroy_srq - Destroys the specified SRQ.
  * @srq: The SRQ to destroy.
  */
@@ -847,6 +860,21 @@ int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
 		  enum ibv_qp_attr_mask attr_mask);
 
 /**
+ * ibv_query_qp - Returns the attribute list and current values for the
+ *   specified QP.
+ * @qp: The QP to query.
+ * @attr: The attributes of the specified QP.
+ * @attr_mask: A bit-mask used to select specific attributes to query.
+ * @init_attr: Additional attributes of the selected QP.
+ *
+ * The qp_attr_mask may be used to limit the query to gathering only the
+ * selected attributes.
+ */
+int ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+		 enum ibv_qp_attr_mask attr_mask,
+		 struct ibv_qp_init_attr *init_attr);
+
+/**
  * ibv_destroy_qp - Destroy a queue pair.
  */
 int ibv_destroy_qp(struct ibv_qp *qp);
diff --git a/src/cmd.c b/src/cmd.c
index e11c651..5da5671 100644
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -488,6 +488,24 @@ int ibv_cmd_modify_srq(struct ibv_srq *srq,
 	return 0;
 }
 
+int ibv_cmd_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr,
+		      struct ibv_query_srq *cmd, size_t cmd_size)
+{
+	struct ibv_query_srq_resp resp;
+
+	IBV_INIT_CMD_RESP(cmd, cmd_size, QUERY_SRQ, &resp, sizeof resp);
+	cmd->srq_handle = srq->handle;
+
+	if (write(srq->context->cmd_fd, cmd, cmd_size) != cmd_size)
+		return errno;
+
+	srq_attr->max_wr    = resp.max_wr;
+	srq_attr->max_sge   = resp.max_sge;
+	srq_attr->srq_limit = resp.srq_limit;
+
+	return 0;
+}
+
 static int ibv_cmd_destroy_srq_v1(struct ibv_srq *srq)
 {
 	struct ibv_destroy_srq_v1 cmd;
@@ -569,6 +587,86 @@ int ibv_cmd_create_qp(struct ibv_pd *pd,
 	return 0;
 }
 
+int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+		     enum ibv_qp_attr_mask attr_mask,
+		     struct ibv_qp_init_attr *init_attr,
+		     struct ibv_query_qp *cmd, size_t cmd_size)
+{
+	struct ibv_query_qp_resp resp;
+
+	IBV_INIT_CMD_RESP(cmd, cmd_size, QUERY_QP, &resp, sizeof resp);
+	cmd->qp_handle = qp->handle;
+	cmd->attr_mask = attr_mask;
+
+	if (write(qp->context->cmd_fd, cmd, cmd_size) != cmd_size)
+		return errno;
+
+	attr->qkey                          = resp.qkey;
+	attr->rq_psn                        = resp.rq_psn;
+	attr->sq_psn                        = resp.sq_psn;
+	attr->dest_qp_num                   = resp.dest_qp_num;
+	attr->qp_access_flags               = resp.qp_access_flags;
+	attr->pkey_index                    = resp.pkey_index;
+	attr->alt_pkey_index                = resp.alt_pkey_index;
+	attr->qp_state                      = resp.qp_state;
+	attr->cur_qp_state                  = resp.cur_qp_state;
+	attr->path_mtu                      = resp.path_mtu;
+	attr->path_mig_state                = resp.path_mig_state;
+	attr->en_sqd_async_notify           = resp.en_sqd_async_notify;
+	attr->max_rd_atomic                 = resp.max_rd_atomic;
+	attr->max_dest_rd_atomic            = resp.max_dest_rd_atomic;
+	attr->min_rnr_timer                 = resp.min_rnr_timer;
+	attr->port_num                      = resp.port_num;
+	attr->timeout                       = resp.timeout;
+	attr->retry_cnt                     = resp.retry_cnt;
+	attr->rnr_retry                     = resp.rnr_retry;
+	attr->alt_port_num                  = resp.alt_port_num;
+	attr->alt_timeout                   = resp.alt_timeout;
+	attr->cap.max_send_wr               = resp.max_send_wr;
+	attr->cap.max_recv_wr               = resp.max_recv_wr;
+	attr->cap.max_send_sge              = resp.max_send_sge;
+	attr->cap.max_recv_sge              = resp.max_recv_sge;
+	attr->cap.max_inline_data           = resp.max_inline_data;
+
+	memcpy(attr->ah_attr.grh.dgid.raw, resp.dest.dgid, 16);
+	attr->ah_attr.grh.flow_label        = resp.dest.flow_label;
+	attr->ah_attr.dlid                  = resp.dest.dlid;
+	attr->ah_attr.grh.sgid_index        = resp.dest.sgid_index;
+	attr->ah_attr.grh.hop_limit         = resp.dest.hop_limit;
+	attr->ah_attr.grh.traffic_class     = resp.dest.traffic_class;
+	attr->ah_attr.sl                    = resp.dest.sl;
+	attr->ah_attr.src_path_bits         = resp.dest.src_path_bits;
+	attr->ah_attr.static_rate           = resp.dest.static_rate;
+	attr->ah_attr.is_global             = resp.dest.is_global;
+	attr->ah_attr.port_num              = resp.dest.port_num;
+
+	memcpy(attr->alt_ah_attr.grh.dgid.raw, resp.alt_dest.dgid, 16);
+	attr->alt_ah_attr.grh.flow_label    = resp.alt_dest.flow_label;
+	attr->alt_ah_attr.dlid              = resp.alt_dest.dlid;
+	attr->alt_ah_attr.grh.sgid_index    = resp.alt_dest.sgid_index;
+	attr->alt_ah_attr.grh.hop_limit     = resp.alt_dest.hop_limit;
+	attr->alt_ah_attr.grh.traffic_class = resp.alt_dest.traffic_class;
+	attr->alt_ah_attr.sl                = resp.alt_dest.sl;
+	attr->alt_ah_attr.src_path_bits     = resp.alt_dest.src_path_bits;
+	attr->alt_ah_attr.static_rate       = resp.alt_dest.static_rate;
+	attr->alt_ah_attr.is_global         = resp.alt_dest.is_global;
+	attr->alt_ah_attr.port_num          = resp.alt_dest.port_num;
+
+	init_attr->qp_context               = qp->qp_context;
+	init_attr->send_cq                  = qp->send_cq;
+	init_attr->recv_cq                  = qp->recv_cq;
+	init_attr->srq                      = qp->srq;
+	init_attr->qp_type                  = qp->qp_type;
+	init_attr->cap.max_send_wr          = resp.max_send_wr;
+	init_attr->cap.max_recv_wr          = resp.max_recv_wr;
+	init_attr->cap.max_send_sge         = resp.max_send_sge;
+	init_attr->cap.max_recv_sge         = resp.max_recv_sge;
+	init_attr->cap.max_inline_data      = resp.max_inline_data;
+	init_attr->sq_sig_all               = resp.sq_sig_all;
+
+	return 0;
+}
+
 int ibv_cmd_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
 		      enum ibv_qp_attr_mask attr_mask,
 		      struct ibv_modify_qp *cmd, size_t cmd_size)
diff --git a/src/libibverbs.map b/src/libibverbs.map
index f35bcf7..2bebc5d 100644
--- a/src/libibverbs.map
+++ b/src/libibverbs.map
@@ -25,8 +25,10 @@ IBVERBS_1.0 {
 		ibv_ack_cq_events;
 		ibv_create_srq;
 		ibv_modify_srq;
+		ibv_query_srq;
 		ibv_destroy_srq;
 		ibv_create_qp;
+		ibv_query_qp;
 		ibv_modify_qp;
 		ibv_destroy_qp;
 		ibv_create_ah;
@@ -49,8 +51,10 @@ IBVERBS_1.0 {
 		ibv_cmd_destroy_cq;
 		ibv_cmd_create_srq;
 		ibv_cmd_modify_srq;
+		ibv_cmd_query_srq;
 		ibv_cmd_destroy_srq;
 		ibv_cmd_create_qp;
+		ibv_cmd_query_qp;
 		ibv_cmd_modify_qp;
 		ibv_cmd_destroy_qp;
 		ibv_cmd_post_send;
diff --git a/src/verbs.c b/src/verbs.c
index 7d6ef0e..a5c031c 100644
--- a/src/verbs.c
+++ b/src/verbs.c
@@ -280,6 +280,11 @@ int ibv_modify_srq(struct ibv_srq *srq,
 	return srq->context->ops.modify_srq(srq, srq_attr, srq_attr_mask);
 }
 
+int ibv_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr)
+{
+	return srq->context->ops.query_srq(srq, srq_attr);
+}
+
 int ibv_destroy_srq(struct ibv_srq *srq)
 {
 	return srq->context->ops.destroy_srq(srq);
@@ -306,6 +311,22 @@ struct ibv_qp *ibv_create_qp(struct ibv_pd *pd,
 	return qp;
 }
 
+int ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+		 enum ibv_qp_attr_mask attr_mask,
+		 struct ibv_qp_init_attr *init_attr)
+{
+	int ret;
+
+	ret = qp->context->ops.query_qp(qp, attr, attr_mask, init_attr);
+	if (ret)
+		return ret;
+
+	if (attr_mask & IBV_QP_STATE)
+		qp->state = attr->qp_state;
+
+	return 0;
+}
+
 int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
 		  enum ibv_qp_attr_mask attr_mask)
 {