Add SRQ support to libmthca

Signed-off-by: Roland Dreier <rolandd@cisco.com>

9 files changed, 612 insertions, 96 deletions

diff --git a/Makefile.am b/Makefile.am
index 38a8d73..dac8935 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -4,7 +4,7 @@ mthcalibdir = $(libdir)/infiniband
 
 mthcalib_LTLIBRARIES = src/mthca.la
 
-src_mthca_la_CFLAGS = -Wall -D_GNU_SOURCE
+src_mthca_la_CFLAGS = -g -Wall -D_GNU_SOURCE
 
 if HAVE_LD_VERSION_SCRIPT
     mthca_version_script = -Wl,--version-script=$(srcdir)/src/mthca.map
@@ -12,7 +12,8 @@ else
     mthca_version_script =
 endif
 
-src_mthca_la_SOURCES = src/ah.c src/cq.c src/memfree.c src/mthca.c src/qp.c src/verbs.c
+src_mthca_la_SOURCES = src/ah.c src/cq.c src/memfree.c src/mthca.c src/qp.c \
+    src/srq.c src/verbs.c
 src_mthca_la_LDFLAGS = -avoid-version -module \
     $(mthca_version_script)
 
diff --git a/src/cq.c b/src/cq.c
index 9625d40..5aaf0d1 100644
--- a/src/cq.c
+++ b/src/cq.c
@@ -234,6 +234,13 @@ static int handle_error_cqe(struct mthca_cq *cq,
 		break;
 	}
 
+	/*
+	 * Mem-free HCAs always generate one CQE per WQE, even in the
+	 * error case, so we don't have to check the doorbell count, etc.
+	 */
+	if (mthca_is_memfree(cq->ibv_cq.context))
+		return 0;
+
 	err = mthca_free_err_wqe(qp, is_send, wqe_index, &dbd, &new_wqe);
 	if (err)
 		return err;
@@ -242,12 +249,8 @@ static int handle_error_cqe(struct mthca_cq *cq,
 	 * If we're at the end of the WQE chain, or we've used up our
 	 * doorbell count, free the CQE.  Otherwise just update it for
 	 * the next poll operation.
-	 * 
-	 * This does not apply to mem-free HCAs: they don't use the
-	 * doorbell count field, and so we should always free the CQE.
 	 */
-	if (mthca_is_memfree(cq->ibv_cq.context) ||
-	    !(new_wqe & htonl(0x3f)) || (!cqe->db_cnt && dbd))
+	if (!(new_wqe & htonl(0x3f)) || (!cqe->db_cnt && dbd))
 		return 0;
 
 	cqe->db_cnt   = htons(ntohs(cqe->db_cnt) - dbd);
@@ -274,7 +277,9 @@ static inline int mthca_poll_one(struct mthca_cq *cq,
 {
 	struct mthca_wq *wq;
 	struct mthca_cqe *cqe;
+	struct mthca_srq *srq;
 	uint32_t qpn;
+	uint32_t wqe;
 	int wqe_index;
 	int is_error;
 	int is_send;
@@ -319,18 +324,27 @@ static inline int mthca_poll_one(struct mthca_cq *cq,
 		wq = &(*cur_qp)->sq;
 		wqe_index = ((ntohl(cqe->wqe) - (*cur_qp)->send_wqe_offset) >> wq->wqe_shift);
 		wc->wr_id = (*cur_qp)->wrid[wqe_index + (*cur_qp)->rq.max];
+	} else if ((*cur_qp)->ibv_qp.srq) {
+		srq = to_msrq((*cur_qp)->ibv_qp.srq);
+		wqe = htonl(cqe->wqe);
+		wq = NULL;
+		wqe_index = wqe >> srq->wqe_shift;
+		wc->wr_id = srq->wrid[wqe_index];
+		mthca_free_srq_wqe(srq, wqe);
 	} else {
 		wq = &(*cur_qp)->rq;
 		wqe_index = ntohl(cqe->wqe) >> wq->wqe_shift;
 		wc->wr_id = (*cur_qp)->wrid[wqe_index];
 	}
 
-	if (wq->last_comp < wqe_index)
-		wq->tail += wqe_index - wq->last_comp;
-	else
-		wq->tail += wqe_index + wq->max - wq->last_comp;
+	if (wq) {
+		if (wq->last_comp < wqe_index)
+			wq->tail += wqe_index - wq->last_comp;
+		else
+			wq->tail += wqe_index + wq->max - wq->last_comp;
 
-	wq->last_comp = wqe_index;
+		wq->last_comp = wqe_index;
+	}
 
 	if (is_error) {
 		err = handle_error_cqe(cq, *cur_qp, wqe_index, is_send,
diff --git a/src/mthca-abi.h b/src/mthca-abi.h
index fb2d12d..b760616 100644
--- a/src/mthca-abi.h
+++ b/src/mthca-abi.h
@@ -65,6 +65,19 @@ struct mthca_create_cq_resp {
 	__u32				reserved;
 };
 
+struct mthca_create_srq {
+	struct ibv_create_srq		ibv_cmd;
+	__u32				lkey;
+	__u32				db_index;
+	__u64				db_page;
+};
+
+struct mthca_create_srq_resp {
+	struct ibv_create_srq_resp	ibv_resp;
+	__u32				srqn;
+	__u32				reserved;
+};
+
 struct mthca_create_qp {
 	struct ibv_create_qp		ibv_cmd;
 	__u32				lkey;
diff --git a/src/mthca.c b/src/mthca.c
index 10e2715..7d04f66 100644
--- a/src/mthca.c
+++ b/src/mthca.c
@@ -108,6 +108,8 @@ static struct ibv_context_ops mthca_ctx_ops = {
 	.create_cq     = mthca_create_cq,
 	.poll_cq       = mthca_poll_cq,
 	.destroy_cq    = mthca_destroy_cq,
+	.create_srq    = mthca_create_srq,
+	.destroy_srq   = mthca_destroy_srq,
 	.create_qp     = mthca_create_qp,
 	.modify_qp     = mthca_modify_qp,
 	.destroy_qp    = mthca_destroy_qp,
@@ -176,11 +178,13 @@ static struct ibv_context *mthca_alloc_context(struct ibv_device *ibdev,
 		context->ibv_ctx.ops.cq_event      = mthca_arbel_cq_event;
 		context->ibv_ctx.ops.post_send     = mthca_arbel_post_send;
 		context->ibv_ctx.ops.post_recv     = mthca_arbel_post_recv;
+		context->ibv_ctx.ops.post_srq_recv = mthca_arbel_post_srq_recv;
 	} else {
 		context->ibv_ctx.ops.req_notify_cq = mthca_tavor_arm_cq;
 		context->ibv_ctx.ops.cq_event      = NULL;
 		context->ibv_ctx.ops.post_send     = mthca_tavor_post_send;
 		context->ibv_ctx.ops.post_recv     = mthca_tavor_post_recv;
+		context->ibv_ctx.ops.post_srq_recv = mthca_tavor_post_srq_recv;
 	}
 
 	return &context->ibv_ctx;
diff --git a/src/mthca.h b/src/mthca.h
index d4da755..69bbce8 100644
--- a/src/mthca.h
+++ b/src/mthca.h
@@ -142,6 +142,27 @@ struct mthca_cq {
 	int                arm_sn;
 };
 
+struct mthca_srq {
+	struct ibv_srq     ibv_srq;
+	void              *buf;
+	void           	  *last;
+	pthread_spinlock_t lock;
+	struct ibv_mr 	  *mr;
+	uint64_t      	  *wrid;
+	uint32_t       	   srqn;
+	int            	   max;
+	int            	   max_gs;
+	int            	   wqe_shift;
+	int            	   first_free;
+	int            	   last_free;
+	int                buf_size;
+
+	/* Next fields are mem-free only */
+	int           	   db_index;
+	uint32_t      	  *db;
+	uint16_t      	   counter;
+};
+
 struct mthca_wq {
 	pthread_spinlock_t lock;
 	int            	   max;
@@ -233,6 +254,11 @@ static inline struct mthca_cq *to_mcq(struct ibv_cq *ibcq)
 	return to_mxxx(cq, cq);
 }
 
+static inline struct mthca_srq *to_msrq(struct ibv_srq *ibsrq)
+{
+	return to_mxxx(srq, srq);
+}
+
 static inline struct mthca_qp *to_mqp(struct ibv_qp *ibqp)
 {
 	return to_mxxx(qp, qp);
@@ -279,6 +305,22 @@ extern int mthca_arbel_arm_cq(struct ibv_cq *cq, int solicited);
 extern void mthca_arbel_cq_event(struct ibv_cq *cq);
 extern void mthca_init_cq_buf(struct mthca_cq *cq, int nent);
 
+extern struct ibv_srq *mthca_create_srq(struct ibv_pd *pd,
+					struct ibv_srq_init_attr *attr);
+extern int mthca_modify_srq(struct ibv_srq *srq,
+			    struct ibv_srq_attr *attr,
+			    enum ibv_srq_attr_mask mask);
+extern int mthca_destroy_srq(struct ibv_srq *srq);
+extern int mthca_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr,
+			       struct mthca_srq *srq);
+extern void mthca_free_srq_wqe(struct mthca_srq *srq, uint32_t wqe_addr);
+extern int mthca_tavor_post_srq_recv(struct ibv_srq *ibsrq,
+				     struct ibv_recv_wr *wr,
+				     struct ibv_recv_wr **bad_wr);
+extern int mthca_arbel_post_srq_recv(struct ibv_srq *ibsrq,
+				     struct ibv_recv_wr *wr,
+				     struct ibv_recv_wr **bad_wr);
+
 extern struct ibv_qp *mthca_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr);
 extern int mthca_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
 			   enum ibv_qp_attr_mask attr_mask);
diff --git a/src/qp.c b/src/qp.c
index 745d186..49512bf 100644
--- a/src/qp.c
+++ b/src/qp.c
@@ -43,81 +43,7 @@
 
 #include "mthca.h"
 #include "doorbell.h"
-
-enum {
-	MTHCA_SEND_DOORBELL	= 0x10,
-        MTHCA_RECV_DOORBELL	= 0x18
-};
-
-enum {
-	MTHCA_NEXT_DBD       = 1 << 7,
-	MTHCA_NEXT_FENCE     = 1 << 6,
-	MTHCA_NEXT_CQ_UPDATE = 1 << 3,
-	MTHCA_NEXT_EVENT_GEN = 1 << 2,
-	MTHCA_NEXT_SOLICIT   = 1 << 1,
-};
-
-enum {
-	MTHCA_INVAL_LKEY = 0x100
-};
-
-enum {
-	MTHCA_INLINE_SEG = 1 << 31
-};
-
-struct mthca_next_seg {
-	uint32_t	nda_op;	/* [31:6] next WQE [4:0] next opcode */
-	uint32_t	ee_nds;	/* [31:8] next EE  [7] DBD [6] F [5:0] next WQE size */
-	uint32_t	flags;	/* [3] CQ [2] Event [1] Solicit */
-	uint32_t	imm;	/* immediate data */
-};
-
-struct mthca_tavor_ud_seg {
-	uint32_t	reserved1;
-	uint32_t	lkey;
-	uint64_t	av_addr;
-	uint32_t	reserved2[4];
-	uint32_t	dqpn;
-	uint32_t	qkey;
-	uint32_t	reserved3[2];
-};
-
-struct mthca_arbel_ud_seg {
-	uint32_t	av[8];
-	uint32_t	dqpn;
-	uint32_t	qkey;
-	uint32_t	reserved[2];
-};
-
-struct mthca_bind_seg {
-	uint32_t	flags;	/* [31] Atomic [30] rem write [29] rem read */
-	uint32_t	reserved;
-	uint32_t	new_rkey;
-	uint32_t	lkey;
-	uint64_t	addr;
-	uint64_t	length;
-};
-
-struct mthca_raddr_seg {
-	uint64_t	raddr;
-	uint32_t	rkey;
-	uint32_t	reserved;
-};
-
-struct mthca_atomic_seg {
-	uint64_t	swap_add;
-	uint64_t	compare;
-};
-
-struct mthca_data_seg {
-	uint32_t	byte_count;
-	uint32_t	lkey;
-	uint64_t	addr;
-};
-
-struct mthca_inline_seg {
-	uint32_t	byte_count;
-};
+#include "wqe.h"
 
 static const uint8_t mthca_opcode[] = {
 	[IBV_WR_SEND]                 = MTHCA_OPCODE_SEND,
@@ -925,15 +851,21 @@ int mthca_free_err_wqe(struct mthca_qp *qp, int is_send,
 {
 	struct mthca_next_seg *next;
 
+	/*
+	 * For SRQs, all WQEs generate a CQE, so we're always at the
+	 * end of the doorbell chain.
+	 */
+	if (qp->ibv_qp.srq) {
+		*new_wqe = 0;
+		return 0;
+	}
+
 	if (is_send)
 		next = get_send_wqe(qp, index);
 	else
 		next = get_recv_wqe(qp, index);
 
-	if (mthca_is_memfree(qp->ibv_qp.context))
-		*dbd = 1;
-	else
-		*dbd = !!(next->ee_nds & htonl(MTHCA_NEXT_DBD));
+	*dbd = !!(next->ee_nds & htonl(MTHCA_NEXT_DBD));
 	if (next->ee_nds & htonl(0x3f))
 		*new_wqe = (next->nda_op & htonl(~0x3f)) |
 			(next->ee_nds & htonl(0x3f));
diff --git a/src/srq.c b/src/srq.c
new file mode 100644
index 0000000..a626e68
--- /dev/null
+++ b/src/srq.c
@@ -0,0 +1,286 @@
+/*
+ * Copyright (c) 2005 Cisco Systems.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id$
+ */
+
+#if HAVE_CONFIG_H
+#  include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <stdlib.h>
+#include <netinet/in.h>
+#include <pthread.h>
+
+#include "mthca.h"
+#include "doorbell.h"
+#include "wqe.h"
+
+static void *get_wqe(struct mthca_srq *srq, int n)
+{
+	return srq->buf + (n << srq->wqe_shift);
+}
+
+void mthca_free_srq_wqe(struct mthca_srq *srq, uint32_t wqe_addr)
+{
+	int ind;
+
+	ind = wqe_addr >> srq->wqe_shift;
+
+	pthread_spin_lock(&srq->lock);
+
+	if (srq->first_free >= 0)
+		*(int *) get_wqe(srq, srq->last_free) = ind;
+	else
+		srq->first_free = ind;
+
+	*(int *) get_wqe(srq, ind) = -1;
+	srq->last_free = ind;
+
+	pthread_spin_unlock(&srq->lock);
+}
+
+int mthca_tavor_post_srq_recv(struct ibv_srq *ibsrq,
+			      struct ibv_recv_wr *wr,
+			      struct ibv_recv_wr **bad_wr)
+{
+	struct mthca_srq *srq = to_msrq(ibsrq);
+	int err = 0;
+	int first_ind;
+	int ind;
+	int next_ind;
+	int nreq;
+	int i;
+	void *wqe;
+	void *prev_wqe;
+
+	pthread_spin_lock(&srq->lock);
+
+	first_ind = srq->first_free;
+
+	for (nreq = 0; wr; ++nreq, wr = wr->next) {
+		ind = srq->first_free;
+
+		if (ind < 0) {
+			err = -1;
+			*bad_wr = wr;
+			return nreq;
+		}
+
+		wqe       = get_wqe(srq, ind);
+		next_ind  = *(int *) wqe;
+		prev_wqe  = srq->last;
+		srq->last = wqe;
+
+		((struct mthca_next_seg *) wqe)->nda_op = 0;
+		((struct mthca_next_seg *) wqe)->ee_nds = 0;
+		/* flags field will always remain 0 */
+
+		wqe += sizeof (struct mthca_next_seg);
+
+		if (wr->num_sge > srq->max_gs) {
+			err = -1;
+			*bad_wr = wr;
+			srq->last = prev_wqe;
+			return nreq;
+		}
+
+		for (i = 0; i < wr->num_sge; ++i) {
+			((struct mthca_data_seg *) wqe)->byte_count =
+				htonl(wr->sg_list[i].length);
+			((struct mthca_data_seg *) wqe)->lkey =
+				htonl(wr->sg_list[i].lkey);
+			((struct mthca_data_seg *) wqe)->addr =
+				htonll(wr->sg_list[i].addr);
+			wqe += sizeof (struct mthca_data_seg);
+		}
+
+		if (i < srq->max_gs) {
+			((struct mthca_data_seg *) wqe)->byte_count = 0;
+			((struct mthca_data_seg *) wqe)->lkey = htonl(MTHCA_INVAL_LKEY);
+			((struct mthca_data_seg *) wqe)->addr = 0;
+		}
+
+		if (prev_wqe) {
+			((struct mthca_next_seg *) prev_wqe)->nda_op =
+				htonl((ind << srq->wqe_shift) | 1);
+			mb();
+			((struct mthca_next_seg *) prev_wqe)->ee_nds =
+				htonl(MTHCA_NEXT_DBD);
+		}
+
+		srq->wrid[ind]  = wr->wr_id;
+		srq->first_free = next_ind;
+	}
+
+	if (nreq) {
+		uint32_t doorbell[2];
+
+		doorbell[0] = htonl(first_ind << srq->wqe_shift);
+		doorbell[1] = htonl((srq->srqn << 8) | nreq);
+
+		/*
+		 * Make sure that descriptors are written before
+		 * doorbell is rung.
+		 */
+		mb();
+
+		mthca_write64(doorbell, to_mctx(ibsrq->context), MTHCA_RECV_DOORBELL);
+	}
+
+	pthread_spin_unlock(&srq->lock);
+	return err;
+}
+
+int mthca_arbel_post_srq_recv(struct ibv_srq *ibsrq,
+			      struct ibv_recv_wr *wr,
+			      struct ibv_recv_wr **bad_wr)
+{
+	struct mthca_srq *srq = to_msrq(ibsrq);
+	int err = 0;
+	int ind;
+	int next_ind;
+	int nreq;
+	int i;
+	void *wqe;
+
+	pthread_spin_lock(&srq->lock);
+
+	for (nreq = 0; wr; ++nreq, wr = wr->next) {
+		ind = srq->first_free;
+
+		if (ind < 0) {
+			err = -1;
+			*bad_wr = wr;
+			return nreq;
+		}
+
+		wqe       = get_wqe(srq, ind);
+		next_ind  = *(int *) wqe;
+
+		((struct mthca_next_seg *) wqe)->nda_op =
+			htonl((next_ind << srq->wqe_shift) | 1);
+		((struct mthca_next_seg *) wqe)->ee_nds = 0;
+		/* flags field will always remain 0 */
+
+		wqe += sizeof (struct mthca_next_seg);
+
+		if (wr->num_sge > srq->max_gs) {
+			err = -1;
+			*bad_wr = wr;
+			return nreq;
+		}
+
+		for (i = 0; i < wr->num_sge; ++i) {
+			((struct mthca_data_seg *) wqe)->byte_count =
+				htonl(wr->sg_list[i].length);
+			((struct mthca_data_seg *) wqe)->lkey =
+				htonl(wr->sg_list[i].lkey);
+			((struct mthca_data_seg *) wqe)->addr =
+				htonll(wr->sg_list[i].addr);
+			wqe += sizeof (struct mthca_data_seg);
+		}
+
+		if (i < srq->max_gs) {
+			((struct mthca_data_seg *) wqe)->byte_count = 0;
+			((struct mthca_data_seg *) wqe)->lkey = htonl(MTHCA_INVAL_LKEY);
+			((struct mthca_data_seg *) wqe)->addr = 0;
+		}
+
+		srq->wrid[ind]  = wr->wr_id;
+		srq->first_free = next_ind;
+	}
+
+	if (nreq) {
+		srq->counter += nreq;
+
+		/*
+		 * Make sure that descriptors are written before
+		 * we write doorbell record.
+		 */
+		mb();
+		*srq->db = htonl(srq->counter);
+	}
+
+	pthread_spin_unlock(&srq->lock);
+	return err;
+}
+
+int mthca_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr,
+		       struct mthca_srq *srq)
+{
+	struct mthca_data_seg *scatter;
+	void *wqe;
+	int size;
+	int i;
+
+	srq->wrid = malloc(srq->max * sizeof (uint64_t));
+	if (!srq->wrid)
+		return -1;
+
+	size = sizeof (struct mthca_next_seg) +
+		srq->max_gs * sizeof (struct mthca_data_seg);
+
+	for (srq->wqe_shift = 6; 1 << srq->wqe_shift < size; ++srq->wqe_shift)
+		; /* nothing */
+
+	srq->buf_size = srq->max << srq->wqe_shift;
+
+	if (posix_memalign(&srq->buf, to_mdev(pd->context->device)->page_size,
+			   align(srq->buf_size, to_mdev(pd->context->device)->page_size))) {
+		free(srq->wrid);
+		return -1;
+	}
+
+	memset(srq->buf, 0, srq->buf_size);
+
+	/*
+	 * Now initialize the SRQ buffer so that all of the WQEs are
+	 * linked into the list of free WQEs.  In addition, set the
+	 * scatter list L_Keys to the sentry value of 0x100.
+	 */
+
+	for (i = 0; i < srq->max; ++i) {
+		wqe = get_wqe(srq, i);
+
+		*(int *) wqe = i < srq->max - 1 ? i + 1 : -1;
+
+		for (scatter = wqe + sizeof (struct mthca_next_seg);
+		     (void *) scatter < wqe + (1 << srq->wqe_shift);
+		     ++scatter)
+			scatter->lkey = htonl(MTHCA_INVAL_LKEY);
+	}
+
+	srq->first_free = 0;
+	srq->last_free  = srq->max - 1;
+
+	return 0;
+}
diff --git a/src/verbs.c b/src/verbs.c
index 9addccc..4728788 100644
--- a/src/verbs.c
+++ b/src/verbs.c
@@ -265,17 +265,127 @@ int mthca_destroy_cq(struct ibv_cq *cq)
 	return 0;
 }
 
-static int align_qp_size(struct ibv_context *context, int size)
+static int align_queue_size(struct ibv_context *context, int size, int spare)
 {
 	int ret;
 
+	/*
+	 * If someone asks for a 0-sized queue, presumably they're not
+	 * going to use it.  So don't mess with their size.
+	 */
+	if (!size)
+		return 0;
+
 	if (mthca_is_memfree(context)) {
-		for (ret = 1; ret < size; ret <<= 1)
+		for (ret = 1; ret < size + spare; ret <<= 1)
 			; /* nothing */
 
 		return ret;
 	} else
-		return size;
+		return size + spare;
+}
+
+struct ibv_srq *mthca_create_srq(struct ibv_pd *pd,
+				 struct ibv_srq_init_attr *attr)
+{
+	struct mthca_create_srq      cmd;
+	struct mthca_create_srq_resp resp;
+	struct mthca_srq            *srq;
+	int                          ret;
+
+	/* Sanity check SRQ size before proceeding */
+	if (attr->attr.max_wr > 16 << 20 || attr->attr.max_sge > 64)
+		return NULL;
+
+	srq = malloc(sizeof *srq);
+	if (!srq)
+		return NULL;
+
+	if (pthread_spin_init(&srq->lock, PTHREAD_PROCESS_PRIVATE))
+		goto err;
+
+	srq->max     = align_queue_size(pd->context, attr->attr.max_wr, 1);
+	srq->max_gs  = attr->attr.max_sge;
+	srq->last    = NULL;
+	srq->counter = 0;
+
+	if (mthca_alloc_srq_buf(pd, &attr->attr, srq))
+		goto err;
+
+	srq->mr = __mthca_reg_mr(pd, srq->buf, srq->buf_size, 0, 0);
+	if (!srq->mr)
+		goto err_free;
+
+	srq->mr->context = pd->context;
+
+	if (mthca_is_memfree(pd->context)) {
+		srq->db_index = mthca_alloc_db(to_mctx(pd->context)->db_tab,
+					       MTHCA_DB_TYPE_SRQ, &srq->db);
+		if (srq->db_index < 0)
+			goto err_unreg;
+
+		cmd.db_page  = db_align(srq->db);
+		cmd.db_index = srq->db_index;
+	}
+
+	cmd.lkey = srq->mr->lkey;
+
+	ret = ibv_cmd_create_srq(pd, &srq->ibv_srq, attr,
+				 &cmd.ibv_cmd, sizeof cmd,
+				 &resp.ibv_resp, sizeof resp);
+	if (ret)
+		goto err_db;
+
+	srq->srqn = resp.srqn;
+
+	if (mthca_is_memfree(pd->context))
+		mthca_set_db_qn(srq->db, MTHCA_DB_TYPE_SRQ, srq->srqn);
+
+	return &srq->ibv_srq;
+
+err_db:
+	if (mthca_is_memfree(pd->context))
+		mthca_free_db(to_mctx(pd->context)->db_tab, MTHCA_DB_TYPE_SRQ,
+			      srq->db_index);
+
+err_unreg:
+	mthca_dereg_mr(srq->mr);
+
+err_free:
+	free(srq->wrid);
+	free(srq->buf);
+
+err:
+	free(srq);
+
+	return NULL;
+}
+
+int mthca_modify_srq(struct ibv_srq *srq,
+		     struct ibv_srq_attr *attr,
+		     enum ibv_srq_attr_mask mask)
+{
+	return -1;
+}
+
+int mthca_destroy_srq(struct ibv_srq *srq)
+{
+	int ret;
+
+	ret = ibv_cmd_destroy_srq(srq);
+	if (ret)
+		return ret;
+
+	if (mthca_is_memfree(srq->context))
+		mthca_free_db(to_mctx(srq->context)->db_tab, MTHCA_DB_TYPE_SRQ,
+			      to_msrq(srq)->db_index);
+
+	mthca_dereg_mr(to_msrq(srq)->mr);
+
+	free(to_msrq(srq)->buf);
+	free(to_msrq(srq)->wrid);
+
+	return 0;
 }
 
 struct ibv_qp *mthca_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr)
@@ -298,14 +408,14 @@ struct ibv_qp *mthca_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr)
 
 	qp->qpt = attr->qp_type;
 
-	qp->sq.max    	 = align_qp_size(pd->context, attr->cap.max_send_wr);
+	qp->sq.max    	 = align_queue_size(pd->context, attr->cap.max_send_wr, 0);
 	qp->sq.next_ind  = 0;
 	qp->sq.last_comp = qp->sq.max - 1;
 	qp->sq.head    	 = 0;
 	qp->sq.tail    	 = 0;
 	qp->sq.last      = NULL;
 
-	qp->rq.max    	 = align_qp_size(pd->context, attr->cap.max_recv_wr);
+	qp->rq.max       = align_queue_size(pd->context, attr->cap.max_recv_wr, 0);
 	qp->rq.next_ind	 = 0;
 	qp->rq.last_comp = qp->rq.max - 1;
 	qp->rq.head    	 = 0;
diff --git a/src/wqe.h b/src/wqe.h
new file mode 100644
index 0000000..e76b24f
--- /dev/null
+++ b/src/wqe.h
@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Cisco Systems.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id$
+ */
+
+#ifndef WQE_H
+#define WQE_H
+
+enum {
+	MTHCA_SEND_DOORBELL	= 0x10,
+	MTHCA_RECV_DOORBELL	= 0x18
+};
+
+enum {
+	MTHCA_NEXT_DBD       = 1 << 7,
+	MTHCA_NEXT_FENCE     = 1 << 6,
+	MTHCA_NEXT_CQ_UPDATE = 1 << 3,
+	MTHCA_NEXT_EVENT_GEN = 1 << 2,
+	MTHCA_NEXT_SOLICIT   = 1 << 1,
+};
+
+enum {
+	MTHCA_INLINE_SEG = 1 << 31
+};
+
+enum {
+	MTHCA_INVAL_LKEY = 0x100
+};
+
+struct mthca_next_seg {
+	uint32_t	nda_op;	/* [31:6] next WQE [4:0] next opcode */
+	uint32_t	ee_nds;	/* [31:8] next EE  [7] DBD [6] F [5:0] next WQE size */
+	uint32_t	flags;	/* [3] CQ [2] Event [1] Solicit */
+	uint32_t	imm;	/* immediate data */
+};
+
+struct mthca_tavor_ud_seg {
+	uint32_t	reserved1;
+	uint32_t	lkey;
+	uint64_t	av_addr;
+	uint32_t	reserved2[4];
+	uint32_t	dqpn;
+	uint32_t	qkey;
+	uint32_t	reserved3[2];
+};
+
+struct mthca_arbel_ud_seg {
+	uint32_t	av[8];
+	uint32_t	dqpn;
+	uint32_t	qkey;
+	uint32_t	reserved[2];
+};
+
+struct mthca_bind_seg {
+	uint32_t	flags;	/* [31] Atomic [30] rem write [29] rem read */
+	uint32_t	reserved;
+	uint32_t	new_rkey;
+	uint32_t	lkey;
+	uint64_t	addr;
+	uint64_t	length;
+};
+
+struct mthca_raddr_seg {
+	uint64_t	raddr;
+	uint32_t	rkey;
+	uint32_t	reserved;
+};
+
+struct mthca_atomic_seg {
+	uint64_t	swap_add;
+	uint64_t	compare;
+};
+
+struct mthca_data_seg {
+	uint32_t	byte_count;
+	uint32_t	lkey;
+	uint64_t	addr;
+};
+
+struct mthca_inline_seg {
+	uint32_t	byte_count;
+};
+
+#endif /* WQE_H */