Continue implementing verbs

Add support for opening kernel uverbs file, getting context and event
FDs, and reading async events.

9 files changed, 460 insertions, 20 deletions

diff --git a/Makefile.am b/Makefile.am
index 7bafe38..01a3f6e 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -17,9 +17,13 @@ src_libibverbs_la_LDFLAGS = -version-info 1 -export-dynamic \
     $(libibverbs_version_script)
 src_libibverbs_la_DEPENDENCIES = $(srcdir)/src/libibverbs.map
 
-bin_PROGRAMS = examples/ib_devices
+bin_PROGRAMS = examples/ib_devices examples/asyncwatch examples/pingpong
 examples_ib_devices_SOURCES = examples/device_list.c
 examples_ib_devices_LDADD = $(top_builddir)/src/libibverbs.la
+examples_pingpong_SOURCES = examples/pingpong.c
+examples_pingpong_LDADD = $(top_builddir)/src/libibverbs.la
+examples_asyncwatch_SOURCES = examples/asyncwatch.c
+examples_asyncwatch_LDADD = $(top_builddir)/src/libibverbs.la
 
 libibverbsincludedir = $(includedir)/infiniband
 
diff --git a/examples/asyncwatch.c b/examples/asyncwatch.c
new file mode 100644
index 0000000..add05f3
--- /dev/null
+++ b/examples/asyncwatch.c
@@ -0,0 +1,87 @@
+/*
+ * Copyright (c) 2005 Topspin Communications.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id: device_list.c 1393 2004-12-28 02:15:24Z roland $
+ */
+
+#if HAVE_CONFIG_H
+#  include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <stdio.h>
+#include <endian.h>
+#include <byteswap.h>
+
+#include <infiniband/verbs.h>
+
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+static inline uint64_t be64_to_cpu(uint64_t x) { return bswap_64(x); }
+#elif __BYTE_ORDER == __BIG_ENDIAN
+static inline uint64_t be64_to_cpu(uint64_t x) { return x; }
+#endif
+
+int main(int argc, char *argv[])
+{
+	struct dlist *dev_list;
+	struct ibv_device *ib_dev;
+	struct ibv_context *context;
+	struct ibv_async_event event;
+
+	dev_list = ibv_get_devices();
+
+	dlist_start(dev_list);
+	ib_dev = dlist_next(dev_list);
+
+	if (!ib_dev) {
+		fprintf(stderr, "No IB devices found\n");
+		return 1;
+	}
+
+	context = ibv_open_device(ib_dev);
+	if (!context) {
+		fprintf(stderr, "Couldn't get context for %s\n",
+			ibv_get_device_name(ib_dev));
+		return 1;
+	}
+
+	printf("%s: async event FD %d\n",
+	       ibv_get_device_name(ib_dev), context->async_fd);
+
+	while (1) {
+		if (ibv_get_async_event(context, &event))
+			return 1;
+
+		printf("  event_type %d, port %d\n", event.event_type,
+		       event.element.port_num);
+	}
+
+	return 0;
+}
diff --git a/examples/pingpong.c b/examples/pingpong.c
new file mode 100644
index 0000000..c8e6502
--- /dev/null
+++ b/examples/pingpong.c
@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2005 Topspin Communications.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id: device_list.c 1393 2004-12-28 02:15:24Z roland $
+ */
+
+#if HAVE_CONFIG_H
+#  include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <stdio.h>
+#include <endian.h>
+#include <byteswap.h>
+
+#include <infiniband/verbs.h>
+
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+static inline uint64_t be64_to_cpu(uint64_t x) { return bswap_64(x); }
+#elif __BYTE_ORDER == __BIG_ENDIAN
+static inline uint64_t be64_to_cpu(uint64_t x) { return x; }
+#endif
+
+int main(int argc, char *argv[])
+{
+	struct dlist *dev_list;
+	struct ibv_device *ib_dev;
+	struct ibv_context *context;
+
+	dev_list = ibv_get_devices();
+
+	dlist_start(dev_list);
+	ib_dev = dlist_next(dev_list);
+
+	if (!ib_dev) {
+		fprintf(stderr, "No IB devices found\n");
+		return 1;
+	}
+
+	context = ibv_open_device(ib_dev);
+	if (!context) {
+		fprintf(stderr, "Couldn't get context for %s\n",
+			ibv_get_device_name(ib_dev));
+		return 1;
+	}
+
+	return 0;
+}
diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index d0ac245..f5fa340 100644
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -49,6 +49,32 @@
 
 BEGIN_C_DECLS
 
+enum ib_event_type {
+	IBV_EVENT_CQ_ERR,
+	IBV_EVENT_QP_FATAL,
+	IBV_EVENT_QP_REQ_ERR,
+	IBV_EVENT_QP_ACCESS_ERR,
+	IBV_EVENT_COMM_EST,
+	IBV_EVENT_SQ_DRAINED,
+	IBV_EVENT_PATH_MIG,
+	IBV_EVENT_PATH_MIG_ERR,
+	IBV_EVENT_DEVICE_FATAL,
+	IBV_EVENT_PORT_ACTIVE,
+	IBV_EVENT_PORT_ERR,
+	IBV_EVENT_LID_CHANGE,
+	IBV_EVENT_PKEY_CHANGE,
+	IBV_EVENT_SM_CHANGE
+};
+
+struct ibv_async_event {
+	union {
+		struct ibv_cq *cq;
+		struct ibv_qp *qp;
+		int            port_num;
+	}                  element;
+	enum ib_event_type event_type;
+};
+
 enum ibv_access_flags {
 	IBV_ACCESS_LOCAL_WRITE		= 1,
 	IBV_ACCESS_REMOTE_WRITE		= (1<<1),
@@ -79,12 +105,17 @@ struct ibv_device_ops {
 
 struct ibv_device {
 	struct sysfs_class_device *dev;
+	struct sysfs_class_device *ibdev;
 	struct ibv_driver         *driver;
 	struct ibv_device_ops      ops;
 };
 
 struct ibv_context {
 	struct ibv_device         *device;
+	int                        cmd_fd;
+	int                        async_fd;
+	int                        num_comp;
+	int                        cq_fd[1];
 };
 
 /**
@@ -113,6 +144,12 @@ extern struct ibv_context *ibv_open_device(struct ibv_device *device);
 extern int ibv_close_device(struct ibv_context *context);
 
 /**
+ * ibv_get_async_event - Get next async event
+ */
+extern int ibv_get_async_event(struct ibv_context *context,
+			       struct ibv_async_event *event);
+
+/**
  * ibv_alloc_pd - Allocate a protection domain
  */
 extern struct ibv_pd *ibv_alloc_pd(struct ibv_context *context);
diff --git a/src/device.c b/src/device.c
index f3a3073..cf32602 100644
--- a/src/device.c
+++ b/src/device.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -38,6 +38,11 @@
 
 #include <stdio.h>
 #include <netinet/in.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <alloca.h>
 
 #include "ibverbs.h"
 
@@ -48,7 +53,7 @@ struct dlist *ibv_get_devices(void)
 
 const char *ibv_get_device_name(struct ibv_device *device)
 {
-	return device->dev->name;
+	return device->ibdev->name;
 }
 
 uint64_t ibv_get_device_guid(struct ibv_device *device)
@@ -57,7 +62,7 @@ uint64_t ibv_get_device_guid(struct ibv_device *device)
 	uint16_t guid[4];
 	int i;
 
-	attr = sysfs_get_classdev_attr(device->dev, "node_guid");
+	attr = sysfs_get_classdev_attr(device->ibdev, "node_guid");
 	if (!attr)
 		return 0;
 
@@ -73,7 +78,13 @@ uint64_t ibv_get_device_guid(struct ibv_device *device)
 
 struct ibv_context *ibv_open_device(struct ibv_device *device)
 {
-	struct ibv_context *context;
+	struct ibv_context *context, *tmp;
+	char *devpath;
+	struct ibv_get_context context_cmd;
+	struct ibv_get_context_resp context_resp;
+	struct ibv_get_event_fds event_fds_cmd;
+	struct ibv_get_event_fds_resp *event_fds_resp;
+	int i;
 
 	context = malloc(sizeof *context);
 	if (!context)
@@ -81,12 +92,81 @@ struct ibv_context *ibv_open_device(struct ibv_device *device)
 
 	context->device = device;
 
+	asprintf(&devpath, "/dev/infiniband/%s", device->dev->name);
+	context->cmd_fd = open(devpath, O_WRONLY);
+
+	if (context->cmd_fd < 0)
+		goto err;
+
+	context_cmd.command   = IB_USER_VERBS_CMD_GET_CONTEXT;
+	context_cmd.in_words  = sizeof context_cmd / 4;
+	context_cmd.out_words = sizeof context_resp / 4;
+	context_cmd.response  = (unsigned long) &context_resp;
+
+	if (write(context->cmd_fd, &context_cmd, sizeof context_cmd) != sizeof context_cmd)
+		goto err_close;
+
+	context->num_comp = context_resp.num_cq_events;
+
+	if (context->num_comp > 1) {
+		tmp = realloc(context, sizeof *context + context->num_comp * sizeof (int));
+		if (!tmp)
+			goto err_close;
+		context = tmp;
+	}
+
+	event_fds_resp = alloca(sizeof *event_fds_resp + context->num_comp * 4);
+
+	event_fds_cmd.command   = IB_USER_VERBS_CMD_GET_EVENT_FDS;
+	event_fds_cmd.in_words  = sizeof event_fds_cmd / 4;
+	event_fds_cmd.out_words = sizeof *event_fds_resp / 4 + context->num_comp;
+	event_fds_cmd.response  = (unsigned long) event_fds_resp;
+
+	if (write(context->cmd_fd, &event_fds_cmd, sizeof event_fds_cmd) !=
+	    sizeof event_fds_cmd)
+		goto err_close;
+
+	context->async_fd = event_fds_resp->async_fd;
+	for (i = 0; i < context->num_comp; ++i)
+		context->cq_fd[i] = event_fds_resp->cq_fd[i];
+
 	return context;
+
+err_close:
+	close(context->cmd_fd);
+
+err:
+	free(context);
+	return NULL;
 }
 
 int ibv_close_device(struct ibv_context *context)
 {
+	int i;
+
+	close(context->async_fd);
+	for (i = 0; i < context->num_comp; ++i)
+		close(context->cq_fd[i]);
+	close(context->cmd_fd);
+
 	free(context);
 
 	return 0;
 }
+
+int ibv_get_async_event(struct ibv_context *context,
+			struct ibv_async_event *event)
+{
+	struct ibv_kern_async_event ev;
+
+	int ret = read(context->async_fd, &ev, sizeof ev);
+
+	if (ret != sizeof ev)
+		return -1;
+
+	/* XXX convert CQ/QP handles back to pointers */
+	event->element.port_num = ev.element;
+	event->event_type       = ev.event_type;
+
+	return 0;
+}
diff --git a/src/ibverbs.h b/src/ibverbs.h
index f97e93e..678dec9 100644
--- a/src/ibverbs.h
+++ b/src/ibverbs.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -40,6 +40,8 @@
 #include <infiniband/verbs.h>
 #include <infiniband/driver.h>
 
+#include "kern_abi.h"
+
 #define HIDDEN		__attribute__((visibility ("hidden")))
 
 #define INIT		__attribute__((constructor))
diff --git a/src/init.c b/src/init.c
index 1f19d36..1bcd570 100644
--- a/src/init.c
+++ b/src/init.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -108,16 +108,36 @@ static void find_drivers(char *dir)
 		load_driver(so_glob.gl_pathv[i]);
 }
 
-static void init_drivers(struct sysfs_class_device *ib_dev)
+static void init_drivers(struct sysfs_class_device *verbs_dev)
 {
+	struct sysfs_class_device *ib_dev; 
+	struct sysfs_attribute *attr;
 	struct ibv_driver *driver;
 	struct ibv_device *dev;
+	char ibdev_name[64];
+
+	attr = sysfs_get_classdev_attr(verbs_dev, "ibdev");
+	if (!attr) {
+		fprintf(stderr, PFX "Warning: no ibdev class attr for %s\n",
+			verbs_dev->name);
+		return;
+	}
+
+	sscanf(attr->value, "%63s", ibdev_name);
+
+	ib_dev = sysfs_open_class_device("infiniband", ibdev_name);
+	if (!ib_dev) {
+		fprintf(stderr, PFX "Warning: no infiniband class device %s for %s\n",
+			attr->value, verbs_dev->name);
+		return;
+	}
 
 	dlist_for_each_data(driver_list, driver, struct ibv_driver) {
-		dev = driver->init_func(ib_dev);
+		dev = driver->init_func(verbs_dev);
 		if (dev) {
-			dev->dev    	    = ib_dev;
-			dev->driver 	    = driver;
+			dev->dev    = verbs_dev;
+			dev->ibdev  = ib_dev;
+			dev->driver = driver;
 
 			dlist_push(device_list, dev);
 
@@ -125,16 +145,47 @@ static void init_drivers(struct sysfs_class_device *ib_dev)
 		}
 	}
 
-	printf(PFX "Warning: no driver for %s\n", ib_dev->name);
+	fprintf(stderr, PFX "Warning: no driver for %s\n", verbs_dev->name);
 }
 
+static void check_abi_version(void)
+{
+	char path[256];
+	char val[16];
+	int ver;
+
+	if (sysfs_get_mnt_path(path, sizeof path)) {
+		fprintf(stderr, PFX "Fatal: couldn't find sysfs mount.\n");
+		abort();
+	}
+
+	strncat(path, "/class/infiniband_verbs/abi_version", sizeof path);
+
+	if (sysfs_read_attribute_value(path, val, sizeof val)) {
+		fprintf(stderr, PFX "Fatal: couldn't read uverbs ABI version.\n");
+		abort();
+	}
+
+	ver = strtol(val, NULL, 10);
+
+	if (ver != IB_USER_VERBS_ABI_VERSION) {
+		fprintf(stderr, PFX "Fatal: kernel ABI version %d "
+			"doesn't match library version %d.\n",
+			ver, IB_USER_VERBS_ABI_VERSION);
+		abort();
+	}
+}
+
+
 static void INIT ibverbs_init(void)
 {
 	const char *user_path;
 	char *wr_path, *dir;
 	struct sysfs_class *cls;
-	Dlist *ib_dev_list;
-	struct sysfs_class_device *ib_dev;
+	Dlist *verbs_dev_list;
+	struct sysfs_class_device *verbs_dev;
+
+	check_abi_version();
 
 	if (ibv_init_mem_map())
 		abort();
@@ -153,18 +204,18 @@ static void INIT ibverbs_init(void)
 
 	find_drivers(default_path);
 
-	cls = sysfs_open_class("infiniband");
+	cls = sysfs_open_class("infiniband_verbs");
 	if (!cls) {
 		fprintf(stderr, PFX "Fatal: couldn't open infiniband sysfs class.\n");
 		abort();
 	}
 
-	ib_dev_list = sysfs_get_class_devices(cls);
-	if (!ib_dev_list) {
+	verbs_dev_list = sysfs_get_class_devices(cls);
+	if (!verbs_dev_list) {
 		fprintf(stderr, PFX "Fatal: no infiniband class devices found.\n");
 		abort();
 	}
 
-	dlist_for_each_data(ib_dev_list, ib_dev, struct sysfs_class_device)
-		init_drivers(ib_dev);
+	dlist_for_each_data(verbs_dev_list, verbs_dev, struct sysfs_class_device)
+		init_drivers(verbs_dev);
 }
diff --git a/src/kern_abi.h b/src/kern_abi.h
new file mode 100644
index 0000000..f439294
--- /dev/null
+++ b/src/kern_abi.h
@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) 2005 Topspin Communications.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id$
+ */
+
+#ifndef KERN_ABI_H
+#define KERN_ABI_H
+
+#include <linux/types.h>
+
+/*
+ * Increment this value if any changes that break userspace ABI
+ * compatibility are made.
+ */
+#define IB_USER_VERBS_ABI_VERSION	1
+
+enum {
+	IB_USER_VERBS_CMD_GET_CONTEXT,
+	IB_USER_VERBS_CMD_GET_EVENT_FDS,
+	IB_USER_VERBS_CMD_ALLOC_PD,
+	IB_USER_VERBS_CMD_DEALLOC_PD,
+	IB_USER_VERBS_CMD_REG_MR,
+	IB_USER_VERBS_CMD_DEREG_MR
+};
+
+/*
+ * Make sure that all structs defined in this file remain laid out so
+ * that they pack the same way on 32-bit and 64-bit architectures (to
+ * avoid incompatibility between 32-bit userspace and 64-bit kernels).
+ * In particular do not use pointer types -- pass pointers in __u64
+ * instead.
+ */
+
+struct ibv_kern_async_event {
+	__u32 event_type;
+	__u32 element;
+};
+
+struct ibv_comp_event {
+	__u32 cq_handle;
+};
+
+/*
+ * All commands from userspace should start with a __u32 command field
+ * followed by __u16 in_words and out_words fields (which give the
+ * length of the command block and response buffer if any in 32-bit
+ * words).  The kernel driver will read these fields first and read
+ * the rest of the command struct based on these value.
+ */
+
+struct ibv_get_context {
+	__u32 command;
+	__u16 in_words;
+	__u16 out_words;
+	__u64 response;
+};
+
+struct ibv_get_context_resp {
+	__u32 num_cq_events;
+};
+
+struct ibv_get_event_fds {
+	__u32 command;
+	__u16 in_words;
+	__u16 out_words;
+	__u64 response;
+};
+
+struct ibv_get_event_fds_resp {
+	__u32 async_fd;
+	__u32 cq_fd[1];
+};
+
+#endif /* KERN_ABI_H */
diff --git a/src/libibverbs.map b/src/libibverbs.map
index 81c2d1e..941b2bf 100644
--- a/src/libibverbs.map
+++ b/src/libibverbs.map
@@ -5,6 +5,7 @@ IBVERBS_1.0 {
 		ibv_get_device_guid;
 		ibv_open_device;
 		ibv_close_device;
+		ibv_get_async_event;
 		ibv_alloc_pd;
 		ibv_dealloc_pd;
 		ibv_reg_mr;