Update to use libiverbs extension support / verbs_register_driver()

Have libmlx4 register with libibverbs with verbs_register_driver() to
indicate that libmlx4 supports verbs extensions, supplying its
verbs_init_func() for further initialization.  This will be used to
add XRC support.

Acked-by: Jason Gunthorpe <jgunthorpe@obsidianresearch.com>
Signed-off-by: Yishai Hadas <yishaih@mellanox.com>
Signed-off-by: Tzahi Oved <tzahio@mellanox.com>

[ Update configure script and deb/rpm packaging to check for new enough
  libibverbs.  - Roland ]

Signed-off-by: Roland Dreier <roland@purestorage.com>

5 files changed, 50 insertions, 43 deletions

diff --git a/configure.ac b/configure.ac
index d7cd4c2..ba29195 100644
--- a/configure.ac
+++ b/configure.ac
@@ -46,8 +46,8 @@ AC_C_CONST
 AC_CHECK_SIZEOF(long)
 
 dnl Checks for library functions
-AC_CHECK_FUNC(ibv_register_driver, [],
-    AC_MSG_ERROR([ibv_register_driver() not found.  libmlx4 requires libibverbs >= 1.1.7.]))
+AC_CHECK_FUNC(verbs_register_driver, [],
+    AC_MSG_ERROR([verbs_register_driver() not found.  libmlx4 requires libibverbs >= 1.1.8.]))
 
 AC_CACHE_CHECK([for IBV_QPT_RAW_PACKET],
                [ac_cv_ibv_qpt_raw_packet],
@@ -56,7 +56,7 @@ AC_CACHE_CHECK([for IBV_QPT_RAW_PACKET],
                                    [ac_cv_ibv_qpt_raw_packet=yes],
                                    [ac_cv_ibv_qpt_raw_packet=no])])
 if test $ac_cv_ibv_qpt_raw_packet = no; then
-    AC_MSG_ERROR([IBV_QPT_RAW_PACKET not found.  libmlx4 requires libibverbs >= 1.1.7.])
+    AC_MSG_ERROR([IBV_QPT_RAW_PACKET not found.  libmlx4 requires libibverbs >= 1.1.8.])
 fi
 
 dnl Now check if for libibverbs 1.0 vs 1.1
diff --git a/debian/control b/debian/control
index 75a3eef..9d5532c 100644
--- a/debian/control
+++ b/debian/control
@@ -1,7 +1,7 @@
 Source: libmlx4
 Priority: extra
 Maintainer: Roland Dreier <rbd@debian.org>
-Build-Depends: debhelper (>= 7.0.50~), dpkg-dev (>= 1.13.19), libibverbs-dev (>= 1.1.7)
+Build-Depends: debhelper (>= 7.0.50~), dpkg-dev (>= 1.13.19), libibverbs-dev (>= 1.1.8)
 Standards-Version: 3.9.2
 Section: libs
 Homepage: http://www.openfabrics.org/
@@ -9,7 +9,7 @@ Homepage: http://www.openfabrics.org/
 Package: libmlx4-1
 Section: libs
 Architecture: any
-Depends: ${shlibs:Depends}, ${misc:Depends}, libibverbs1 (>= 1.1.7)
+Depends: ${shlibs:Depends}, ${misc:Depends}, libibverbs1 (>= 1.1.8)
 Description: Userspace driver for Mellanox ConnectX InfiniBand HCAs
  libmlx4 is a device-specific driver for Mellanox ConnectX InfiniBand
  host channel adapters (HCAs) for the libibverbs library.  This allows
diff --git a/libmlx4.spec.in b/libmlx4.spec.in
index 22ecd8c..9b10cee 100644
--- a/libmlx4.spec.in
+++ b/libmlx4.spec.in
@@ -9,7 +9,7 @@ Url: http://openfabrics.org/
 Source: http://openfabrics.org/downloads/mlx4/libmlx4-1.0.5.tar.gz
 BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX)
 
-BuildRequires: libibverbs-devel >= 1.1.7
+BuildRequires: libibverbs-devel >= 1.1.8
 
 %description
 libmlx4 provides a device-specific userspace driver for Mellanox
diff --git a/src/mlx4.c b/src/mlx4.c
index 4120e5c..da9f611 100644
--- a/src/mlx4.c
+++ b/src/mlx4.c
@@ -116,7 +116,8 @@ static struct ibv_context_ops mlx4_ctx_ops = {
 	.detach_mcast  = ibv_cmd_detach_mcast
 };
 
-static struct ibv_context *mlx4_alloc_context(struct ibv_device *ibdev, int cmd_fd)
+static int mlx4_init_context(struct verbs_device *v_device,
+				struct ibv_context *ibv_ctx, int cmd_fd)
 {
 	struct mlx4_context	       *context;
 	struct ibv_get_context		cmd;
@@ -124,26 +125,30 @@ static struct ibv_context *mlx4_alloc_context(struct ibv_device *ibdev, int cmd_
 	int				i;
 	struct mlx4_alloc_ucontext_resp_v3 resp_v3;
 	__u16				bf_reg_size;
-	struct mlx4_device		*dev = to_mdev(ibdev);
+	struct mlx4_device              *dev = to_mdev(&v_device->device);
+	/* verbs_context should be used for new verbs
+	* struct verbs_context *verbs_ctx = verbs_get_ctx(ibv_ctx);
+	*/
 
-	context = calloc(1, sizeof *context);
-	if (!context)
-		return NULL;
 
-	context->ibv_ctx.cmd_fd = cmd_fd;
+	/* memory footprint of mlx4_context and verbs_context share
+	* struct ibv_context.
+	*/
+	context = to_mctx(ibv_ctx);
+	ibv_ctx->cmd_fd = cmd_fd;
 
 	if (dev->abi_version <= MLX4_UVERBS_NO_DEV_CAPS_ABI_VERSION) {
-		if (ibv_cmd_get_context(&context->ibv_ctx, &cmd, sizeof cmd,
+		if (ibv_cmd_get_context(ibv_ctx, &cmd, sizeof cmd,
 					&resp_v3.ibv_resp, sizeof resp_v3))
-			goto err_free;
+			return errno;
 
 		context->num_qps  = resp_v3.qp_tab_size;
 		bf_reg_size	  = resp_v3.bf_reg_size;
 		context->cqe_size = sizeof (struct mlx4_cqe);
 	} else  {
-		if (ibv_cmd_get_context(&context->ibv_ctx, &cmd, sizeof cmd,
+		if (ibv_cmd_get_context(ibv_ctx, &cmd, sizeof cmd,
 					&resp.ibv_resp, sizeof resp))
-			goto err_free;
+			return errno;
 
 		context->num_qps  = resp.qp_tab_size;
 		bf_reg_size	  = resp.bf_reg_size;
@@ -165,15 +170,15 @@ static struct ibv_context *mlx4_alloc_context(struct ibv_device *ibdev, int cmd_
 
 	pthread_mutex_init(&context->db_list_mutex, NULL);
 
-	context->uar = mmap(NULL, to_mdev(ibdev)->page_size, PROT_WRITE,
+	context->uar = mmap(NULL, dev->page_size, PROT_WRITE,
 			    MAP_SHARED, cmd_fd, 0);
 	if (context->uar == MAP_FAILED)
-		goto err_free;
+		return errno;
 
 	if (bf_reg_size) {
-		context->bf_page = mmap(NULL, to_mdev(ibdev)->page_size,
+		context->bf_page = mmap(NULL, dev->page_size,
 					PROT_WRITE, MAP_SHARED, cmd_fd,
-					to_mdev(ibdev)->page_size);
+					dev->page_size);
 		if (context->bf_page == MAP_FAILED) {
 			fprintf(stderr, PFX "Warning: BlueFlame available, "
 				"but failed to mmap() BlueFlame page.\n");
@@ -190,32 +195,25 @@ static struct ibv_context *mlx4_alloc_context(struct ibv_device *ibdev, int cmd_
 	}
 
 	pthread_spin_init(&context->uar_lock, PTHREAD_PROCESS_PRIVATE);
+	ibv_ctx->ops = mlx4_ctx_ops;
+	/* New verbs should be added by using verbs_set_ctx_op */
 
-	context->ibv_ctx.ops = mlx4_ctx_ops;
-
-	return &context->ibv_ctx;
+	return 0;
 
-err_free:
-	free(context);
-	return NULL;
 }
 
-static void mlx4_free_context(struct ibv_context *ibctx)
+static void mlx4_uninit_context(struct verbs_device *v_device,
+					struct ibv_context *ibv_ctx)
 {
-	struct mlx4_context *context = to_mctx(ibctx);
+	struct mlx4_context *context = to_mctx(ibv_ctx);
 
-	munmap(context->uar, to_mdev(ibctx->device)->page_size);
+	munmap(context->uar, to_mdev(&v_device->device)->page_size);
 	if (context->bf_page)
-		munmap(context->bf_page, to_mdev(ibctx->device)->page_size);
-	free(context);
-}
+		munmap(context->bf_page, to_mdev(&v_device->device)->page_size);
 
-static struct ibv_device_ops mlx4_dev_ops = {
-	.alloc_context = mlx4_alloc_context,
-	.free_context  = mlx4_free_context
-};
+}
 
-static struct ibv_device *mlx4_driver_init(const char *uverbs_sys_path, int abi_version)
+static struct verbs_device *mlx4_driver_init(const char *uverbs_sys_path, int abi_version)
 {
 	char			value[8];
 	struct mlx4_device    *dev;
@@ -250,21 +248,27 @@ found:
 		return NULL;
 	}
 
-	dev = malloc(sizeof *dev);
+	dev = calloc(1, sizeof *dev);
 	if (!dev) {
 		fprintf(stderr, PFX "Fatal: couldn't allocate device for %s\n",
 			uverbs_sys_path);
 		return NULL;
 	}
 
-	dev->ibv_dev.ops = mlx4_dev_ops;
 	dev->page_size   = sysconf(_SC_PAGESIZE);
 	dev->abi_version = abi_version;
 
-	return &dev->ibv_dev;
+	dev->verbs_dev.sz = sizeof(*dev);
+	dev->verbs_dev.size_of_context =
+		sizeof(struct mlx4_context) - sizeof(struct ibv_context);
+	/* mlx4_init_context will initialize provider calls */
+	dev->verbs_dev.init_context = mlx4_init_context;
+	dev->verbs_dev.uninit_context = mlx4_uninit_context;
+
+	return &dev->verbs_dev;
 }
 
 static __attribute__((constructor)) void mlx4_register_driver(void)
 {
-	ibv_register_driver("mlx4", mlx4_driver_init);
+	verbs_register_driver("mlx4", mlx4_driver_init);
 }
diff --git a/src/mlx4.h b/src/mlx4.h
index a4e9b32..cfb0b50 100644
--- a/src/mlx4.h
+++ b/src/mlx4.h
@@ -127,7 +127,7 @@ enum {
 };
 
 struct mlx4_device {
-	struct ibv_device		ibv_dev;
+	struct verbs_device		verbs_dev;
 	int				page_size;
 	int				abi_version;
 };
@@ -272,7 +272,10 @@ static inline unsigned long align(unsigned long val, unsigned long align)
 
 static inline struct mlx4_device *to_mdev(struct ibv_device *ibdev)
 {
-	return to_mxxx(dev, device);
+	/* ibv_device is first field of verbs_device
+	 * see try_driver() in libibverbs.
+	 */
+	return container_of(ibdev, struct mlx4_device, verbs_dev);
 }
 
 static inline struct mlx4_context *to_mctx(struct ibv_context *ibctx)