Do not use enum types for bit flags

Arithmetic operations on enum members do not result in the enum type;
C++ is stricter about this than C.  So using flag enums results in
compile errors when they are OR'd together in a C++ application.

To fix this, replace all flag enum objects with int.  int was selected
to preserve the ABI; we checked that enum types are the same size as
int on at least i386, x86-64, ppc32, ppc64, ia64, and mips, and arm
and sparc also appear compatible with this choice.

Signed-off-by: Jason Gunthorpe <jgunthorpe@obsidianresearch.com>
Signed-off-by: Roland Dreier <rolandd@cisco.com>

11 files changed, 45 insertions, 44 deletions

diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h
index e54e0e3..9a81416 100644
--- a/include/infiniband/driver.h
+++ b/include/infiniband/driver.h
@@ -77,7 +77,7 @@ int ibv_cmd_alloc_pd(struct ibv_context *context, struct ibv_pd *pd,
 int ibv_cmd_dealloc_pd(struct ibv_pd *pd);
 #define IBV_CMD_REG_MR_HAS_RESP_PARAMS
 int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
-		   uint64_t hca_va, enum ibv_access_flags access,
+		   uint64_t hca_va, int access,
 		   struct ibv_mr *mr, struct ibv_reg_mr *cmd,
 		   size_t cmd_size,
 		   struct ibv_reg_mr_resp *resp, size_t resp_size);
@@ -101,7 +101,7 @@ int ibv_cmd_create_srq(struct ibv_pd *pd,
 		       struct ibv_create_srq_resp *resp, size_t resp_size);
 int ibv_cmd_modify_srq(struct ibv_srq *srq,
 		       struct ibv_srq_attr *srq_attr,
-		       enum ibv_srq_attr_mask srq_attr_mask,
+		       int srq_attr_mask,
 		       struct ibv_modify_srq *cmd, size_t cmd_size);
 int ibv_cmd_query_srq(struct ibv_srq *srq,
 		      struct ibv_srq_attr *srq_attr,
@@ -113,11 +113,11 @@ int ibv_cmd_create_qp(struct ibv_pd *pd,
 		      struct ibv_create_qp *cmd, size_t cmd_size,
 		      struct ibv_create_qp_resp *resp, size_t resp_size);
 int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *qp_attr,
-		     enum ibv_qp_attr_mask attr_mask,
+		     int attr_mask,
 		     struct ibv_qp_init_attr *qp_init_attr,
 		     struct ibv_query_qp *cmd, size_t cmd_size);
 int ibv_cmd_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
-		      enum ibv_qp_attr_mask attr_mask,
+		      int attr_mask,
 		      struct ibv_modify_qp *cmd, size_t cmd_size);
 int ibv_cmd_destroy_qp(struct ibv_qp *qp);
 int ibv_cmd_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index 226d85e..0f1cb2e 100644
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -269,7 +269,7 @@ struct ibv_wc {
 	uint32_t		imm_data;	/* in network byte order */
 	uint32_t		qp_num;
 	uint32_t		src_qp;
-	enum ibv_wc_flags	wc_flags;
+	int			wc_flags;
 	uint16_t		pkey_index;
 	uint16_t		slid;
 	uint8_t			sl;
@@ -508,7 +508,7 @@ struct ibv_send_wr {
 	struct ibv_sge	       *sg_list;
 	int			num_sge;
 	enum ibv_wr_opcode	opcode;
-	enum ibv_send_flags	send_flags;
+	int			send_flags;
 	uint32_t		imm_data;	/* in network byte order */
 	union {
 		struct {
@@ -541,8 +541,8 @@ struct ibv_mw_bind {
 	struct ibv_mr	       *mr;
 	void		       *addr;
 	size_t			length;
-	enum ibv_send_flags	send_flags;
-	enum ibv_access_flags	mw_access_flags;
+	int			send_flags;
+	int			mw_access_flags;
 };
 
 struct ibv_srq {
@@ -633,12 +633,12 @@ struct ibv_context_ops {
 	struct ibv_pd *		(*alloc_pd)(struct ibv_context *context);
 	int			(*dealloc_pd)(struct ibv_pd *pd);
 	struct ibv_mr *		(*reg_mr)(struct ibv_pd *pd, void *addr, size_t length,
-					  enum ibv_access_flags access);
+					  int access);
 	struct ibv_mr *		(*rereg_mr)(struct ibv_mr *mr,
-					    enum ibv_rereg_mr_flags flags,
+					    int flags,
 					    struct ibv_pd *pd, void *addr,
 					    size_t length,
-					    enum ibv_access_flags access);
+					    int access);
 	int			(*dereg_mr)(struct ibv_mr *mr);
 	struct ibv_mw *		(*alloc_mw)(struct ibv_pd *pd, enum ibv_mw_type type);
 	int			(*bind_mw)(struct ibv_qp *qp, struct ibv_mw *mw,
@@ -656,7 +656,7 @@ struct ibv_context_ops {
 					      struct ibv_srq_init_attr *srq_init_attr);
 	int			(*modify_srq)(struct ibv_srq *srq,
 					      struct ibv_srq_attr *srq_attr,
-					      enum ibv_srq_attr_mask srq_attr_mask);
+					      int srq_attr_mask);
 	int			(*query_srq)(struct ibv_srq *srq,
 					     struct ibv_srq_attr *srq_attr);
 	int			(*destroy_srq)(struct ibv_srq *srq);
@@ -665,10 +665,10 @@ struct ibv_context_ops {
 						 struct ibv_recv_wr **bad_recv_wr);
 	struct ibv_qp *		(*create_qp)(struct ibv_pd *pd, struct ibv_qp_init_attr *attr);
 	int			(*query_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr,
-					    enum ibv_qp_attr_mask attr_mask,
+					    int attr_mask,
 					    struct ibv_qp_init_attr *init_attr);
 	int			(*modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr,
-					     enum ibv_qp_attr_mask attr_mask);
+					     int attr_mask);
 	int			(*destroy_qp)(struct ibv_qp *qp);
 	int			(*post_send)(struct ibv_qp *qp, struct ibv_send_wr *wr,
 					     struct ibv_send_wr **bad_wr);
@@ -793,7 +793,7 @@ int ibv_dealloc_pd(struct ibv_pd *pd);
  * ibv_reg_mr - Register a memory region
  */
 struct ibv_mr *ibv_reg_mr(struct ibv_pd *pd, void *addr,
-			  size_t length, enum ibv_access_flags access);
+			  size_t length, int access);
 
 /**
  * ibv_dereg_mr - Deregister a memory region
@@ -926,7 +926,7 @@ struct ibv_srq *ibv_create_srq(struct ibv_pd *pd,
  */
 int ibv_modify_srq(struct ibv_srq *srq,
 		   struct ibv_srq_attr *srq_attr,
-		   enum ibv_srq_attr_mask srq_attr_mask);
+		   int srq_attr_mask);
 
 /**
  * ibv_query_srq - Returns the attribute list and current values for the
@@ -966,7 +966,7 @@ struct ibv_qp *ibv_create_qp(struct ibv_pd *pd,
  * ibv_modify_qp - Modify a queue pair.
  */
 int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
-		  enum ibv_qp_attr_mask attr_mask);
+		  int attr_mask);
 
 /**
  * ibv_query_qp - Returns the attribute list and current values for the
@@ -980,7 +980,7 @@ int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
  * selected attributes.
  */
 int ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
-		 enum ibv_qp_attr_mask attr_mask,
+		 int attr_mask,
 		 struct ibv_qp_init_attr *init_attr);
 
 /**
diff --git a/man/ibv_modify_qp.3 b/man/ibv_modify_qp.3
index a022dac..9eabcdf 100644
--- a/man/ibv_modify_qp.3
+++ b/man/ibv_modify_qp.3
@@ -8,7 +8,7 @@ ibv_modify_qp \- modify the attributes of a queue pair (QP)
 .B #include <infiniband/verbs.h>
 .sp
 .BI "int ibv_modify_qp(struct ibv_qp " "*qp" ", struct ibv_qp_attr " "*attr" ,
-.BI "                  enum ibv_qp_attr_mask " "attr_mask" );
+.BI "                  int " "attr_mask" );
 .fi
 .SH "DESCRIPTION"
 .B ibv_modify_qp()
diff --git a/man/ibv_modify_srq.3 b/man/ibv_modify_srq.3
index fa23c3a..9cc0497 100644
--- a/man/ibv_modify_srq.3
+++ b/man/ibv_modify_srq.3
@@ -9,7 +9,7 @@ ibv_modify_srq \- modify attributes of a shared receive queue (SRQ)
 .sp
 .BI "int ibv_modify_srq(struct ibv_srq " "*srq" ,
 .BI "                   struct ibv_srq_attr " "*srq_attr" ,
-.BI "                   enum ibv_srq_attr_mask " "srq_attr_mask" );
+.BI "                   int " "srq_attr_mask" );
 .fi
 .SH "DESCRIPTION"
 .B ibv_modify_srq()
diff --git a/man/ibv_poll_cq.3 b/man/ibv_poll_cq.3
index cf70efc..57c6daa 100644
--- a/man/ibv_poll_cq.3
+++ b/man/ibv_poll_cq.3
@@ -33,7 +33,7 @@ uint32_t                byte_len;       /* Number of bytes transferred */
 uint32_t                imm_data;       /* Immediate data (in network byte order) */
 uint32_t                qp_num;         /* Local QP number of completed WR */
 uint32_t                src_qp;         /* Source QP number (remote QP number) of completed WR (valid only for UD QPs) */
-enum ibv_wc_flags       wc_flags;       /* Flags of the completed WR */
+int                     wc_flags;       /* Flags of the completed WR */
 uint16_t                pkey_index;     /* P_Key index (valid only for GSI QPs) */
 uint16_t                slid;           /* Source LID */
 uint8_t                 sl;             /* Service Level */
diff --git a/man/ibv_post_send.3 b/man/ibv_post_send.3
index 51fc1cf..33fbb50 100644
--- a/man/ibv_post_send.3
+++ b/man/ibv_post_send.3
@@ -33,7 +33,7 @@ struct ibv_send_wr     *next;                   /* Pointer to next WR in list, N
 struct ibv_sge         *sg_list;                /* Pointer to the s/g array */
 int                     num_sge;                /* Size of the s/g array */
 enum ibv_wr_opcode      opcode;                 /* Operation type */
-enum ibv_send_flags     send_flags;             /* Flags of the WR properties */
+int                     send_flags;             /* Flags of the WR properties */
 uint32_t                imm_data;               /* Immediate data (in network byte order) */
 union {
 .in +8
diff --git a/man/ibv_query_qp.3 b/man/ibv_query_qp.3
index 7101dc2..3893ec8 100644
--- a/man/ibv_query_qp.3
+++ b/man/ibv_query_qp.3
@@ -8,7 +8,7 @@ ibv_query_qp \- get the attributes of a queue pair (QP)
 .B #include <infiniband/verbs.h>
 .sp
 .BI "int ibv_query_qp(struct ibv_qp " "*qp" ", struct ibv_qp_attr " "*attr" ,
-.BI "                 enum ibv_qp_attr_mask " "attr_mask" ,
+.BI "                 int " "attr_mask" ,
 .BI "                 struct ibv_qp_init_attr " "*init_attr" );
 .fi
 .SH "DESCRIPTION"
@@ -69,7 +69,9 @@ The argument
 .I attr_mask
 is a hint that specifies the minimum list of attributes to retrieve.
 Some RDMA devices may return extra attributes not requested, for
-example if the value can be returned cheaply.
+example if the value can be returned cheaply. This has the same
+form as in
+.B ibv_modify_qp()\fR.
 .PP
 Attribute values are valid if they have been set using
 .B ibv_modify_qp()\fR.
diff --git a/man/ibv_reg_mr.3 b/man/ibv_reg_mr.3
index 7cd3de8..7723771 100644
--- a/man/ibv_reg_mr.3
+++ b/man/ibv_reg_mr.3
@@ -8,8 +8,7 @@ ibv_reg_mr, ibv_dereg_mr \- register or deregister a memory region (MR)
 .B #include <infiniband/verbs.h>
 .sp
 .BI "struct ibv_mr *ibv_reg_mr(struct ibv_pd " "*pd" ", void " "*addr" ,
-.BI "                          size_t " "length" ,
-.BI "                          enum ibv_access_flags " "access" );
+.BI "                          size_t " "length" ", int " "access" );
 .sp
 .BI "int ibv_dereg_mr(struct ibv_mr " "*mr" );
 .fi
diff --git a/src/cmd.c b/src/cmd.c
index c2e76cf..cbd5288 100644
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -231,7 +231,7 @@ int ibv_cmd_dealloc_pd(struct ibv_pd *pd)
 }
 
 int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
-		   uint64_t hca_va, enum ibv_access_flags access,
+		   uint64_t hca_va, int access,
 		   struct ibv_mr *mr, struct ibv_reg_mr *cmd,
 		   size_t cmd_size,
 		   struct ibv_reg_mr_resp *resp, size_t resp_size)
@@ -485,7 +485,7 @@ int ibv_cmd_create_srq(struct ibv_pd *pd,
 
 static int ibv_cmd_modify_srq_v3(struct ibv_srq *srq,
 				 struct ibv_srq_attr *srq_attr,
-				 enum ibv_srq_attr_mask srq_attr_mask,
+				 int srq_attr_mask,
 				 struct ibv_modify_srq *new_cmd,
 				 size_t new_cmd_size)
 {
@@ -513,7 +513,7 @@ static int ibv_cmd_modify_srq_v3(struct ibv_srq *srq,
 
 int ibv_cmd_modify_srq(struct ibv_srq *srq,
 		       struct ibv_srq_attr *srq_attr,
-		       enum ibv_srq_attr_mask srq_attr_mask,
+		       int srq_attr_mask,
 		       struct ibv_modify_srq *cmd, size_t cmd_size)
 {
 	if (abi_ver == 3)
@@ -651,7 +651,7 @@ int ibv_cmd_create_qp(struct ibv_pd *pd,
 }
 
 int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
-		     enum ibv_qp_attr_mask attr_mask,
+		     int attr_mask,
 		     struct ibv_qp_init_attr *init_attr,
 		     struct ibv_query_qp *cmd, size_t cmd_size)
 {
@@ -733,7 +733,7 @@ int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
 }
 
 int ibv_cmd_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
-		      enum ibv_qp_attr_mask attr_mask,
+		      int attr_mask,
 		      struct ibv_modify_qp *cmd, size_t cmd_size)
 {
 	IBV_INIT_CMD(cmd, cmd_size, MODIFY_QP);
diff --git a/src/compat-1_0.c b/src/compat-1_0.c
index 459ade9..3f5ff35 100644
--- a/src/compat-1_0.c
+++ b/src/compat-1_0.c
@@ -88,7 +88,7 @@ struct ibv_send_wr_1_0 {
 	struct ibv_sge	       *sg_list;
 	int			num_sge;
 	enum ibv_wr_opcode	opcode;
-	enum ibv_send_flags	send_flags;
+	int			send_flags;
 	uint32_t		imm_data;	/* in network byte order */
 	union {
 		struct {
@@ -172,7 +172,7 @@ struct ibv_context_ops_1_0 {
 	struct ibv_pd *		(*alloc_pd)(struct ibv_context *context);
 	int			(*dealloc_pd)(struct ibv_pd *pd);
 	struct ibv_mr *		(*reg_mr)(struct ibv_pd *pd, void *addr, size_t length,
-					  enum ibv_access_flags access);
+					  int access);
 	int			(*dereg_mr)(struct ibv_mr *mr);
 	struct ibv_cq *		(*create_cq)(struct ibv_context *context, int cqe,
 					     struct ibv_comp_channel *channel,
@@ -188,7 +188,7 @@ struct ibv_context_ops_1_0 {
 					      struct ibv_srq_init_attr *srq_init_attr);
 	int			(*modify_srq)(struct ibv_srq *srq,
 					      struct ibv_srq_attr *srq_attr,
-					      enum ibv_srq_attr_mask srq_attr_mask);
+					      int srq_attr_mask);
 	int			(*query_srq)(struct ibv_srq *srq,
 					     struct ibv_srq_attr *srq_attr);
 	int			(*destroy_srq)(struct ibv_srq *srq);
@@ -197,10 +197,10 @@ struct ibv_context_ops_1_0 {
 						 struct ibv_recv_wr_1_0 **bad_recv_wr);
 	struct ibv_qp *		(*create_qp)(struct ibv_pd *pd, struct ibv_qp_init_attr *attr);
 	int			(*query_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr,
-					    enum ibv_qp_attr_mask attr_mask,
+					    int attr_mask,
 					    struct ibv_qp_init_attr *init_attr);
 	int			(*modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr,
-					     enum ibv_qp_attr_mask attr_mask);
+					     int attr_mask);
 	int			(*destroy_qp)(struct ibv_qp *qp);
 	int			(*post_send)(struct ibv_qp_1_0 *qp,
 					     struct ibv_send_wr_1_0 *wr,
@@ -596,7 +596,7 @@ int __ibv_dealloc_pd_1_0(struct ibv_pd_1_0 *pd)
 symver(__ibv_dealloc_pd_1_0, ibv_dealloc_pd, IBVERBS_1.0);
 
 struct ibv_mr_1_0 *__ibv_reg_mr_1_0(struct ibv_pd_1_0 *pd, void *addr,
-				    size_t length, enum ibv_access_flags access)
+				    size_t length, int access)
 {
 	struct ibv_mr *real_mr;
 	struct ibv_mr_1_0 *mr;
@@ -736,7 +736,7 @@ symver(__ibv_create_srq_1_0, ibv_create_srq, IBVERBS_1.0);
 
 int __ibv_modify_srq_1_0(struct ibv_srq_1_0 *srq,
 			 struct ibv_srq_attr *srq_attr,
-			 enum ibv_srq_attr_mask srq_attr_mask)
+			 int srq_attr_mask)
 {
 	return ibv_modify_srq(srq->real_srq, srq_attr, srq_attr_mask);
 }
@@ -806,7 +806,7 @@ struct ibv_qp_1_0 *__ibv_create_qp_1_0(struct ibv_pd_1_0 *pd,
 symver(__ibv_create_qp_1_0, ibv_create_qp, IBVERBS_1.0);
 
 int __ibv_query_qp_1_0(struct ibv_qp_1_0 *qp, struct ibv_qp_attr *attr,
-		       enum ibv_qp_attr_mask attr_mask,
+		       int attr_mask,
 		       struct ibv_qp_init_attr_1_0 *init_attr)
 {
 	struct ibv_qp_init_attr real_init_attr;
@@ -829,7 +829,7 @@ int __ibv_query_qp_1_0(struct ibv_qp_1_0 *qp, struct ibv_qp_attr *attr,
 symver(__ibv_query_qp_1_0, ibv_query_qp, IBVERBS_1.0);
 
 int __ibv_modify_qp_1_0(struct ibv_qp_1_0 *qp, struct ibv_qp_attr *attr,
-			enum ibv_qp_attr_mask attr_mask)
+			int attr_mask)
 {
 	return ibv_modify_qp(qp->real_qp, attr, attr_mask);
 }
diff --git a/src/verbs.c b/src/verbs.c
index 477e412..ba3c0a4 100644
--- a/src/verbs.c
+++ b/src/verbs.c
@@ -155,7 +155,7 @@ int __ibv_dealloc_pd(struct ibv_pd *pd)
 default_symver(__ibv_dealloc_pd, ibv_dealloc_pd);
 
 struct ibv_mr *__ibv_reg_mr(struct ibv_pd *pd, void *addr,
-			    size_t length, enum ibv_access_flags access)
+			    size_t length, int access)
 {
 	struct ibv_mr *mr;
 
@@ -377,7 +377,7 @@ default_symver(__ibv_create_srq, ibv_create_srq);
 
 int __ibv_modify_srq(struct ibv_srq *srq,
 		     struct ibv_srq_attr *srq_attr,
-		     enum ibv_srq_attr_mask srq_attr_mask)
+		     int srq_attr_mask)
 {
 	return srq->context->ops.modify_srq(srq, srq_attr, srq_attr_mask);
 }
@@ -419,7 +419,7 @@ struct ibv_qp *__ibv_create_qp(struct ibv_pd *pd,
 default_symver(__ibv_create_qp, ibv_create_qp);
 
 int __ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
-		   enum ibv_qp_attr_mask attr_mask,
+		   int attr_mask,
 		   struct ibv_qp_init_attr *init_attr)
 {
 	int ret;
@@ -436,7 +436,7 @@ int __ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
 default_symver(__ibv_query_qp, ibv_query_qp);
 
 int __ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
-		    enum ibv_qp_attr_mask attr_mask)
+		    int attr_mask)
 {
 	int ret;