Add support for userspace RDMA connection management abstraction (CMA)

Add common user/kernel data structures and copy routines in libibverbs.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>

8 files changed, 473 insertions, 22 deletions

diff --git a/ChangeLog b/ChangeLog
index d6cb76b..b167835 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,17 @@
+2005-11-10  Sean Hefty <sean.hefty@intel.com>
+
+	* include/infiniband/sa-kern-abi.h: New include file to contain
+	definitions of SA structures passed between userspace and kernel.
+
+	* include/infiniband/sa.h: New include file for definitions of 
+	SA structures used by multiple libraries.
+
+	* include/infiniband/marshall.h src/marshall.c: New files to define
+	routines used to exchange data with kernel modules.
+
+	* include/infiniband/kern-abi.h: Added data structures used to exchange
+	QP attribute with kernel modules.
+
 2005-11-09  Michael S. Tsirkin  <mst@mellanox.co.il>
 
 	* src/device.c (ibv_get_devices): Make function reentrant by using
diff --git a/Makefile.am b/Makefile.am
index eafbac1..972d8f7 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -14,7 +14,8 @@ else
     libibverbs_version_script =
 endif
 
-src_libibverbs_la_SOURCES = src/cmd.c src/device.c src/init.c src/memory.c src/verbs.c
+src_libibverbs_la_SOURCES = src/cmd.c src/device.c src/init.c src/marshall.c \
+			    src/memory.c src/verbs.c
 src_libibverbs_la_LDFLAGS = -version-info 1 -export-dynamic \
     $(libibverbs_version_script)
 src_libibverbs_la_DEPENDENCIES = $(srcdir)/src/libibverbs.map
@@ -40,7 +41,8 @@ examples_ibv_asyncwatch_LDADD = $(top_builddir)/src/libibverbs.la
 libibverbsincludedir = $(includedir)/infiniband
 
 libibverbsinclude_HEADERS = include/infiniband/arch.h include/infiniband/driver.h \
-    include/infiniband/kern-abi.h include/infiniband/opcode.h include/infiniband/verbs.h
+    include/infiniband/kern-abi.h include/infiniband/opcode.h include/infiniband/verbs.h \
+    include/infiniband/sa-kern-abi.h include/infiniband/sa.h include/infiniband/marshall.h
 
 man_MANS = man/ibv_asyncwatch.1 man/ibv_devices.1 man/ibv_devinfo.1 \
     man/ibv_rc_pingpong.1 man/ibv_uc_pingpong.1 man/ibv_ud_pingpong.1 \
@@ -53,6 +55,8 @@ DEBIAN = debian/changelog debian/compat debian/control debian/copyright \
 
 EXTRA_DIST = include/infiniband/driver.h include/infiniband/kern-abi.h \
     include/infiniband/opcode.h include/infiniband/verbs.h src/ibverbs.h \
+    include/infiniband/marshall.h include/sa-kern-abi.h \
+    include/infiniband/sa.h \
     src/libibverbs.map libibverbs.spec.in $(man_MANS) $(DEBIAN)
 
 dist-hook: libibverbs.spec
diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h
index 32c3e29..11830e8 100644
--- a/include/infiniband/kern-abi.h
+++ b/include/infiniband/kern-abi.h
@@ -357,6 +357,64 @@ struct ibv_destroy_cq_resp {
 	__u32 async_events_reported;
 };
 
+struct ibv_kern_global_route {
+	__u8  dgid[16];
+	__u32 flow_label;
+	__u8  sgid_index;
+	__u8  hop_limit;
+	__u8  traffic_class;
+	__u8  reserved;
+};
+
+struct ibv_kern_ah_attr {
+	struct ibv_kern_global_route grh;
+	__u16 dlid;
+	__u8  sl;
+	__u8  src_path_bits;
+	__u8  static_rate;
+	__u8  is_global;
+	__u8  port_num;
+	__u8  reserved;
+};
+
+struct ibv_kern_qp_attr {
+	__u32	qp_attr_mask;
+	__u32	qp_state;
+	__u32	cur_qp_state;
+	__u32	path_mtu;
+	__u32	path_mig_state;
+	__u32	qkey;
+	__u32	rq_psn;
+	__u32	sq_psn;
+	__u32	dest_qp_num;
+	__u32	qp_access_flags;
+
+	struct ibv_kern_ah_attr ah_attr;
+	struct ibv_kern_ah_attr alt_ah_attr;
+
+	/* ib_qp_cap */
+	__u32	max_send_wr;
+	__u32	max_recv_wr;
+	__u32	max_send_sge;
+	__u32	max_recv_sge;
+	__u32	max_inline_data;
+
+	__u16	pkey_index;
+	__u16	alt_pkey_index;
+	__u8	en_sqd_async_notify;
+	__u8	sq_draining;
+	__u8	max_rd_atomic;
+	__u8	max_dest_rd_atomic;
+	__u8	min_rnr_timer;
+	__u8	port_num;
+	__u8	timeout;
+	__u8	retry_cnt;
+	__u8	rnr_retry;
+	__u8	alt_port_num;
+	__u8	alt_timeout;
+	__u8	reserved[5];
+};
+
 struct ibv_create_qp {
 	__u32 command;
 	__u16 in_words;
@@ -532,26 +590,6 @@ struct ibv_post_srq_recv_resp {
 	__u32 bad_wr;
 };
 
-struct ibv_kern_global_route {
-	__u8  dgid[16];
-	__u32 flow_label;
-	__u8  sgid_index;
-	__u8  hop_limit;
-	__u8  traffic_class;
-	__u8  reserved;
-};
-
-struct ibv_kern_ah_attr {
-	struct ibv_kern_global_route grh;
-	__u16 dlid;
-	__u8  sl;
-	__u8  src_path_bits;
-	__u8  static_rate;
-	__u8  is_global;
-	__u8  port_num;
-	__u8  reserved;
-};
-
 struct ibv_create_ah {
 	__u32 command;
 	__u16 in_words;
diff --git a/include/infiniband/marshall.h b/include/infiniband/marshall.h
new file mode 100644
index 0000000..d6bd38f
--- /dev/null
+++ b/include/infiniband/marshall.h
@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2005 Intel Corporation.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef INFINIBAND_MARSHALL_H
+#define INFINIBAND_MARSHALL_H
+
+#include <infiniband/verbs.h>
+#include <infiniband/sa.h>
+#include <infiniband/kern-abi.h>
+#include <infiniband/sa-kern-abi.h>
+
+#ifdef __cplusplus
+#  define BEGIN_C_DECLS extern "C" {
+#  define END_C_DECLS   }
+#else /* !__cplusplus */
+#  define BEGIN_C_DECLS
+#  define END_C_DECLS
+#endif /* __cplusplus */
+
+BEGIN_C_DECLS
+
+void ib_copy_qp_attr_from_kern(struct ibv_qp_attr *dst,
+			       struct ibv_kern_qp_attr *src);
+
+void ib_copy_path_rec_from_kern(struct ib_sa_path_rec *dst,
+				struct ib_kern_path_rec *src);
+
+void ib_copy_path_rec_to_kern(struct ib_kern_path_rec *dst,
+			      struct ib_sa_path_rec *src);
+
+END_C_DECLS
+
+#endif /* INFINIBAND_MARSHALL_H */
diff --git a/include/infiniband/sa-kern-abi.h b/include/infiniband/sa-kern-abi.h
new file mode 100644
index 0000000..fe1155e
--- /dev/null
+++ b/include/infiniband/sa-kern-abi.h
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2005 Intel Corporation.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef SA_KERN_ABI_H
+#define SA_KERN_ABI_H
+
+#include <linux/types.h>
+
+struct ib_kern_path_rec {
+	__u8  dgid[16];
+	__u8  sgid[16];
+	__u16 dlid;
+	__u16 slid;
+	__u32 raw_traffic;
+	__u32 flow_label;
+	__u32 reversible;
+	__u32 mtu;
+	__u16 pkey;
+	__u8  hop_limit;
+	__u8  traffic_class;
+	__u8  numb_path;
+	__u8  sl;
+	__u8  mtu_selector;
+	__u8  rate_selector;
+	__u8  rate;
+	__u8  packet_life_time_selector;
+	__u8  packet_life_time;
+	__u8  preference;
+};
+
+#endif /* SA_KERN_ABI_H */
diff --git a/include/infiniband/sa.h b/include/infiniband/sa.h
new file mode 100644
index 0000000..951da37
--- /dev/null
+++ b/include/infiniband/sa.h
@@ -0,0 +1,130 @@
+/*
+ * Copyright (c) 2004 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Voltaire, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id: sa.h 2616 2005-06-15 15:22:39Z halr $
+ */
+
+#ifndef IB_SA_H
+#define IB_SA_H
+
+#include <infiniband/verbs.h>
+
+enum ib_sa_rate {
+	IB_SA_RATE_2_5_GBPS = 2,
+	IB_SA_RATE_5_GBPS   = 5,
+	IB_SA_RATE_10_GBPS  = 3,
+	IB_SA_RATE_20_GBPS  = 6,
+	IB_SA_RATE_30_GBPS  = 4,
+	IB_SA_RATE_40_GBPS  = 7,
+	IB_SA_RATE_60_GBPS  = 8,
+	IB_SA_RATE_80_GBPS  = 9,
+	IB_SA_RATE_120_GBPS = 10
+};
+
+static inline int ib_sa_rate_enum_to_int(enum ib_sa_rate rate)
+{
+	switch (rate) {
+	case IB_SA_RATE_2_5_GBPS: return  1;
+	case IB_SA_RATE_5_GBPS:   return  2;
+	case IB_SA_RATE_10_GBPS:  return  4;
+	case IB_SA_RATE_20_GBPS:  return  8;
+	case IB_SA_RATE_30_GBPS:  return 12;
+	case IB_SA_RATE_40_GBPS:  return 16;
+	case IB_SA_RATE_60_GBPS:  return 24;
+	case IB_SA_RATE_80_GBPS:  return 32;
+	case IB_SA_RATE_120_GBPS: return 48;
+	default: 	          return -1;
+	}
+}
+
+struct ib_sa_path_rec {
+	/* reserved */
+	/* reserved */
+	union ibv_gid dgid;
+	union ibv_gid sgid;
+	uint16_t      dlid;
+	uint16_t      slid;
+	int           raw_traffic;
+	/* reserved */
+	uint32_t      flow_label;
+	uint8_t       hop_limit;
+	uint8_t       traffic_class;
+	int           reversible;
+	uint8_t       numb_path;
+	uint16_t      pkey;
+	/* reserved */
+	uint8_t       sl;
+	uint8_t       mtu_selector;
+	uint8_t	      mtu;
+	uint8_t       rate_selector;
+	uint8_t       rate;
+	uint8_t       packet_life_time_selector;
+	uint8_t       packet_life_time;
+	uint8_t       preference;
+};
+
+struct ib_sa_mcmember_rec {
+	union ibv_gid mgid;
+	union ibv_gid port_gid;
+	uint32_t      qkey;
+	uint16_t      mlid;
+	uint8_t       mtu_selector;
+	uint8_t       mtu;
+	uint8_t       traffic_class;
+	uint16_t      pkey;
+	uint8_t       rate_selector;
+	uint8_t       rate;
+	uint8_t       packet_life_time_selector;
+	uint8_t       packet_life_time;
+	uint8_t       sl;
+	uint32_t      flow_label;
+	uint8_t       hop_limit;
+	uint8_t       scope;
+	uint8_t       join_state;
+	int           proxy_join;
+};
+
+struct ib_sa_service_rec {
+	uint64_t      id;
+	union ibv_gid gid;
+	uint16_t      pkey;
+	/* uint16_t  resv;   */
+	uint32_t      lease;
+	uint8_t       key[16];
+	uint8_t       name[64];
+	uint8_t       data8[16];
+	uint16_t      data16[8];
+	uint32_t      data32[4];
+	uint64_t      data64[2];
+};
+
+#endif /* IB_SA_H */
diff --git a/src/libibverbs.map b/src/libibverbs.map
index def6e7f..56314ed 100644
--- a/src/libibverbs.map
+++ b/src/libibverbs.map
@@ -57,5 +57,8 @@ IBVERBS_1.0 {
 		ibv_cmd_destroy_ah;
 		ibv_cmd_attach_mcast;
 		ibv_cmd_detach_mcast;
+		ib_copy_qp_attr_from_kern;
+		ib_copy_path_rec_from_kern;
+		ib_copy_path_rec_to_kern;
 	local: *;
 };
diff --git a/src/marshall.c b/src/marshall.c
new file mode 100644
index 0000000..6f6c387
--- /dev/null
+++ b/src/marshall.c
@@ -0,0 +1,140 @@
+/*
+ * Copyright (c) 2005 Intel Corporation.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#if HAVE_CONFIG_H
+#  include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <infiniband/marshall.h>
+
+static void ib_copy_ah_attr_from_kern(struct ibv_ah_attr *dst,
+				      struct ibv_kern_ah_attr *src)
+{
+	memcpy(dst->grh.dgid.raw, src->grh.dgid, sizeof dst->grh.dgid);
+	dst->grh.flow_label = src->grh.flow_label;
+	dst->grh.sgid_index = src->grh.sgid_index;
+	dst->grh.hop_limit = src->grh.hop_limit;
+	dst->grh.traffic_class = src->grh.traffic_class;
+
+	dst->dlid = src->dlid;
+	dst->sl = src->sl;
+	dst->src_path_bits = src->src_path_bits;
+	dst->static_rate = src->static_rate;
+	dst->is_global = src->is_global;
+	dst->port_num = src->port_num;
+}
+
+void ib_copy_qp_attr_from_kern(struct ibv_qp_attr *dst,
+			       struct ibv_kern_qp_attr *src)
+{
+	dst->cur_qp_state = src->cur_qp_state;
+	dst->path_mtu = src->path_mtu;
+	dst->path_mig_state = src->path_mig_state;
+	dst->qkey = src->qkey;
+	dst->rq_psn = src->rq_psn;
+	dst->sq_psn = src->sq_psn;
+	dst->dest_qp_num = src->dest_qp_num;
+	dst->qp_access_flags = src->qp_access_flags;
+
+	dst->cap.max_send_wr = src->max_send_wr;
+	dst->cap.max_recv_wr = src->max_recv_wr;
+	dst->cap.max_send_sge = src->max_send_sge;
+	dst->cap.max_recv_sge = src->max_recv_sge;
+	dst->cap.max_inline_data = src->max_inline_data;
+
+	ib_copy_ah_attr_from_kern(&dst->ah_attr, &src->ah_attr);
+	ib_copy_ah_attr_from_kern(&dst->alt_ah_attr, &src->alt_ah_attr);
+
+	dst->pkey_index = src->pkey_index;
+	dst->alt_pkey_index = src->alt_pkey_index;
+	dst->en_sqd_async_notify = src->en_sqd_async_notify;
+	dst->sq_draining = src->sq_draining;
+	dst->max_rd_atomic = src->max_rd_atomic;
+	dst->max_dest_rd_atomic = src->max_dest_rd_atomic;
+	dst->min_rnr_timer = src->min_rnr_timer;
+	dst->port_num = src->port_num;
+	dst->timeout = src->timeout;
+	dst->retry_cnt = src->retry_cnt;
+	dst->rnr_retry = src->rnr_retry;
+	dst->alt_port_num = src->alt_port_num;
+	dst->alt_timeout = src->alt_timeout;
+}
+
+void ib_copy_path_rec_from_kern(struct ib_sa_path_rec *dst,
+				struct ib_kern_path_rec *src)
+{
+	memcpy(dst->dgid.raw, src->dgid, sizeof dst->dgid);
+	memcpy(dst->sgid.raw, src->sgid, sizeof dst->sgid);
+
+	dst->dlid		= src->dlid;
+	dst->slid		= src->slid;
+	dst->raw_traffic	= src->raw_traffic;
+	dst->flow_label		= src->flow_label;
+	dst->hop_limit		= src->hop_limit;
+	dst->traffic_class	= src->traffic_class;
+	dst->reversible		= src->reversible;
+	dst->numb_path		= src->numb_path;
+	dst->pkey		= src->pkey;
+	dst->sl			= src->sl;
+	dst->mtu_selector	= src->mtu_selector;
+	dst->mtu		= src->mtu;
+	dst->rate_selector	= src->rate_selector;
+	dst->rate		= src->rate;
+	dst->packet_life_time	= src->packet_life_time;
+	dst->preference		= src->preference;
+	dst->packet_life_time_selector = src->packet_life_time_selector;
+}
+
+void ib_copy_path_rec_to_kern(struct ib_kern_path_rec *dst,
+			      struct ib_sa_path_rec *src)
+{
+	memcpy(dst->dgid, src->dgid.raw, sizeof src->dgid);
+	memcpy(dst->sgid, src->sgid.raw, sizeof src->sgid);
+
+	dst->dlid		= src->dlid;
+	dst->slid		= src->slid;
+	dst->raw_traffic	= src->raw_traffic;
+	dst->flow_label		= src->flow_label;
+	dst->hop_limit		= src->hop_limit;
+	dst->traffic_class	= src->traffic_class;
+	dst->reversible		= src->reversible;
+	dst->numb_path		= src->numb_path;
+	dst->pkey		= src->pkey;
+	dst->sl			= src->sl;
+	dst->mtu_selector	= src->mtu_selector;
+	dst->mtu		= src->mtu;
+	dst->rate_selector	= src->rate_selector;
+	dst->rate		= src->rate;
+	dst->packet_life_time	= src->packet_life_time;
+	dst->preference		= src->preference;
+	dst->packet_life_time_selector = src->packet_life_time_selector;
+}